package main
import (
"context"
"errors"
"log"
"net"
"net/http"
"time"
"craftdesign.group/quarry/internal/drain"
)
const (
// serveReadHeaderTimeout bounds how long a client may take to send
// its request headers. A real client sends them at once; one that
// trickles them is holding a connection and a goroutine for nothing.
serveReadHeaderTimeout = 10 * time.Second
// serveIdleTimeout bounds how long a kept-alive connection may sit
// between requests. It is longer than the idle timeouts of the
// proxies commonly put in front of a server (nginx 60s, Go's own
// transport 90s, Caddy 2m), so the proxy always closes an idle
// connection first and never reuses one this server has just shut.
serveIdleTimeout = 3 * time.Minute
// serveDrainBound is how long a stop waits for requests already in
// flight. Pushes and saves finish in seconds; anything still running
// after this is cancelled so a deploy is never held by one request.
serveDrainBound = 10 * time.Second
// serveStopBound is how long the whole stop may take, drain and
// teardown together, before the process exits regardless. It sits
// well inside systemd's default 90s stop timeout, so a restart ends
// with our exit rather than a SIGKILL.
serveStopBound = 25 * time.Second
// serveHandlerWait is how long a stop waits, after the drain bound,
// for handlers that ignored their cancelled context. Close drops
// their connections without waiting, and the teardown after
// serveUntil closes the stores they may still be using.
serveHandlerWait = 2 * time.Second
)
// newServer is the HTTP server the forge listens with. It carries no
// ReadTimeout or WriteTimeout: both bound the whole request, which
// would end every live stream and every long upload at the same fixed
// age. Streams set a write deadline per frame instead, which cuts off a
// reader that stopped reading without touching one that is keeping up.
func newServer(h http.Handler) *http.Server {
return &http.Server{
Handler: h,
ReadHeaderTimeout: serveReadHeaderTimeout,
IdleTimeout: serveIdleTimeout,
}
}
// serveUntil serves srv on ln until ctx ends, then stops in order: the
// listener closes, live streams are cancelled at once through drain,
// requests in flight get until bound to finish, and whatever is still
// running then has its context cancelled and its connection closed.
// It returns once the server has stopped; the caller's deferred
// teardown (ingest, leases, events, database) runs after it.
//
// The drain depends on how the process is stopped. systemd's default
// KillMode=control-group sends SIGTERM to every process in the unit at
// once, so the zit children serving in-flight pushes and store writes
// die while this drain is still waiting for them, and those requests
// fail as "the forge was restarting". The unit needs KillMode=mixed:
// SIGTERM to quarry alone, which stops its own children in teardown,
// and SIGKILL to whatever is left only after quarry has exited.
func serveUntil(ctx context.Context, srv *http.Server, ln net.Listener, dr *drain.Drain, bound time.Duration) error {
// Every request context derives from base, so the last step of the
// drain can reach handlers that ignored the gentler ones.
base, cancelAll := context.WithCancel(context.Background())
defer cancelAll()
srv.BaseContext = func(net.Listener) context.Context { return base }
srv.RegisterOnShutdown(dr.StopStreams)
served := make(chan error, 1)
go func() { served <- srv.Serve(ln) }()
select {
case err := <-served:
return err
case <-ctx.Done():
}
log.Printf("stopping: draining requests for up to %v", bound)
began := time.Now()
sctx, cancel := context.WithTimeout(context.Background(), bound)
defer cancel()
if err := srv.Shutdown(sctx); err != nil {
log.Printf("stopping: requests outlived the %v drain; cancelling them", bound)
cancelAll()
_ = srv.Close()
if !dr.Wait(serveHandlerWait) {
log.Printf("stopping: handlers still running after a further %v; tearing down under them", serveHandlerWait)
}
}
if err := <-served; err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
log.Printf("stopping: requests drained in %v", time.Since(began).Round(time.Millisecond))
return nil
}