package main
import (
"context"
"errors"
"log"
"net"
"net/http"
"time"
"craftdesign.group/quarry/internal/children"
"craftdesign.group/quarry/internal/drain"
)
const (
// serveReadHeaderTimeout bounds how long a client may take to send
// its request headers. A real client sends them at once; one that
// trickles them is holding a connection and a goroutine for nothing.
serveReadHeaderTimeout = 10 * time.Second
// serveIdleTimeout bounds how long a kept-alive connection may sit
// between requests. It is longer than the idle timeouts of the
// proxies commonly put in front of a server (nginx 60s, Go's own
// transport 90s, Caddy 2m), so the proxy always closes an idle
// connection first and never reuses one this server has just shut.
serveIdleTimeout = 3 * time.Minute
// serveDrainBound is how long a stop waits for requests already in
// flight. Pushes and saves finish in seconds; anything still running
// after this is cancelled so a deploy is never held by one request.
serveDrainBound = 10 * time.Second
// serveStopBound is how long the whole stop may take, drain and
// teardown together, before the process exits regardless. It sits
// well inside systemd's default 90s stop timeout, so a restart ends
// with our exit rather than a SIGKILL.
serveStopBound = 25 * time.Second
// serveHandlerWait is how long a stop waits, after the drain bound,
// for handlers that ignored their cancelled context. Close drops
// their connections without waiting, and the teardown after
// serveUntil closes the stores they may still be using.
serveHandlerWait = 2 * time.Second
// serveChildWait is how long a stop waits, after teardown, for zit
// children still running, and serveChildGrace how long those that
// outlive it get after SIGTERM. With the drain and the handler wait
// the stop stays inside serveStopBound; anything quarry has not
// reaped when it exits is SIGKILLed by systemd.
serveChildWait = 8 * time.Second
serveChildGrace = 2 * time.Second
)
// newServer is the HTTP server the forge listens with. It carries no
// ReadTimeout or WriteTimeout: both bound the whole request, which
// would end every live stream and every long upload at the same fixed
// age. Streams set a write deadline per frame instead, which cuts off a
// reader that stopped reading without touching one that is keeping up.
func newServer(h http.Handler) *http.Server {
return &http.Server{
Handler: h,
ReadHeaderTimeout: serveReadHeaderTimeout,
IdleTimeout: serveIdleTimeout,
}
}
// serveUntil serves srv on ln until ctx ends, then stops in order: the
// listener closes, live streams are cancelled at once through drain,
// requests in flight get until bound to finish, and whatever is still
// running then has its context cancelled and its connection closed,
// and gets up to handlerWait more to return (serveHandlerWait in
// production; a parameter so a test can hold a handler without racing
// it). It returns once the server has stopped; the caller's deferred
// teardown (ingest, leases, events, database) runs after it.
//
// The drain assumes KillMode=mixed; under control-group every zit
// child is signalled with the process and only pure-Go work drains.
// systemd's control-group mode sends SIGTERM to every process in the
// unit at once, so the children serving in-flight pushes and store
// writes die while this drain is still waiting for them. Under mixed
// only quarry is signalled; it waits for its own children (see
// stopChildren) and systemd SIGKILLs whatever is left once it exits.
func serveUntil(ctx context.Context, srv *http.Server, ln net.Listener, dr *drain.Drain, bound, handlerWait time.Duration) error {
// Every request context derives from base, so the last step of the
// drain can reach handlers that ignored the gentler ones.
base, cancelAll := context.WithCancel(context.Background())
defer cancelAll()
srv.BaseContext = func(net.Listener) context.Context { return base }
srv.RegisterOnShutdown(dr.StopStreams)
served := make(chan error, 1)
go func() { served <- srv.Serve(ln) }()
select {
case err := <-served:
return err
case <-ctx.Done():
}
// From here every zit child that ends is logged, so the stop's log
// shows which ended during the drain and which only in teardown.
children.Default.Stopping()
log.Printf("stopping: draining requests for up to %v", bound)
began := time.Now()
sctx, cancel := context.WithTimeout(context.Background(), bound)
defer cancel()
if err := srv.Shutdown(sctx); err != nil {
log.Printf("stopping: requests outlived the %v drain; cancelling them", bound)
cancelAll()
_ = srv.Close()
if !dr.Wait(handlerWait) {
log.Printf("stopping: handlers still running after a further %v; tearing down under them", handlerWait)
}
}
if err := <-served; err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
log.Printf("stopping: requests drained in %v", time.Since(began).Round(time.Millisecond))
return nil
}
// stopChildren is the last step of a stop, after serve's teardown has
// stopped ingest and the fronter: it waits for every zit child still
// running, and logs each one as it ends.
//
// Wait runs even with nothing live: it also closes the set, so nothing
// the process does between here and exit can start a child that systemd
// would then SIGKILL with nobody waiting for it.
func stopChildren() {
had := children.Default.Live()
if left := children.Default.Wait(serveChildWait, serveChildGrace); left > 0 {
log.Printf("stopping: exiting with %d zit children still running", left)
return
}
if had > 0 {
log.Printf("stopping: every zit child has ended")
}
}