Skip to content
quarry Artifact [52119853a8]
Login

Artifact [52119853a8]

Artifact 52119853a8c48de71007b5db7f41f6b654bd146056526c1cc7e479cf84d4e5d1:


// Package cibridge lets a runner on its owner's machine reach the CI
// engine through the forge (f24d78d9).
//
// dagnats already has an HTTP worker protocol — connect, poll, resolve,
// logs — authenticated by worker tokens scoped to task-type prefixes and
// one worker group (dagnats#627, #695). It listens only on the forge's
// box. This package forwards exactly those four routes, under a
// repository's URL, so a runner needs nothing but the forge's address,
// the repository's name and a token minted in Settings → Runners:
//
//	POST /r/{repo}/ci/bridge/v1/workers/connect
//	POST /r/{repo}/ci/bridge/v1/tasks/poll
//	POST /r/{repo}/ci/bridge/v1/tasks/{id}/resolve
//	POST /r/{repo}/ci/bridge/v1/tasks/{id}/logs
//
// A fifth route is the forge's own, not the engine's (RFC 0019
// §secrets):
//
//	POST /r/{repo}/ci/bridge/v1/tasks/{id}/secrets
//
// and a sixth (RFC 0021 §runs):
//
//	POST /r/{repo}/ci/bridge/v1/tasks/{id}/token
//
// and a seventh, the paths the task's check-in changed (QUARRY_CHANGED):
//
//	POST /r/{repo}/ci/bridge/v1/tasks/{id}/changed
//
// A runner asks for the secrets a job it is running names, and for the
// job's run token, which the forge mints or declines. It proves it
// holds the task by the engine accepting a heartbeat for it with the
// runner's own token: dagnats lets only the token that claimed a task
// act on it (#627). What may be delivered is then the forge's decision
// alone, made from its own records (Deliver).
//
// Two things are decided here rather than trusted to the caller. A poll
// is pinned to the repository's own worker group, so a runner cannot
// ask for another repository's work by naming its group (the token
// would refuse it too; this makes the URL mean what it says). And the
// engine's ADMIN bearer is refused outright: on the box it is a
// convenience, on a public path it would be a master key.
package cibridge

import (
	"bytes"
	"crypto/subtle"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"craftdesign.group/quarry/internal/obs"
)

// maxBody bounds what a runner may send in one request. A log batch is
// the largest legitimate body; a runner sends many small ones.
const maxBody = 4 << 20

// Deliver answers a runner's request for secrets once it has proved it
// holds the task: the values by name, or an error saying why not. It
// must decide from the forge's own records, never from the request.
type Deliver func(repo, taskID, environment string, names []string) (map[string]string, error)

// DeliverToken answers a runner's request for its job's run token once
// it has proved it holds the task: the job's environment additions, or
// none and a note for the job's log saying why. It decides from the
// forge's records alone; the request names nothing.
type DeliverToken func(repo, taskID string) (env map[string]string, note string, err error)

// ReasonHeader names why the bridge refused a request, for the refusals
// whose meaning a status code alone does not carry. A runner decides on
// it, never on the words of the body, and a forge without it (an older
// one) is answered as before. The values are a closed set:
//
//	repo-not-served  the forge does not serve the repository (404)
//	not-held         this runner does not hold the task (403): its run
//	                 was cancelled between the queue and the claim
//	task-gone        the engine has no such task (404 on resolve, logs)
//	task-resolved    the engine says the task is already resolved or its
//	                 run cancelled (409 on resolve, logs)
//	repo-held        the repository is under a takedown (RFC 0023) and
//	                 runs no CI while it is (451, with Retry-After on
//	                 every route): nothing is lost, and the same work is
//	                 offered once the takedown is lifted
//	other-repo       the token is a live worker token, scoped to another
//	                 repository's worker group (403, only when WithRepoOfGroup
//	                 is set); the JSON body names that repository, which is
//	                 the token holder's own. A wrong, revoked or unknown token
//	                 is the engine's plain 401, naming nothing
//
// Only task-gone and task-resolved are final for a report of a task's
// end; repo-not-served is what a forge answers while it starts, so it is
// retried.
const ReasonHeader = "Quarry-Bridge-Reason"

const (
	ReasonRepoNotServed = "repo-not-served"
	ReasonNotHeld       = "not-held"
	ReasonTaskGone      = "task-gone"
	ReasonTaskResolved  = "task-resolved"
	ReasonRepoHeld      = "repo-held"
	ReasonOtherRepo     = "other-repo"
)

// heldRetryAfter is the delay, in seconds, a runner of a held repository
// is told to wait. A lift takes a person; a runner that asked every few
// seconds would only fill both logs.
const heldRetryAfter = "300"

// refuseHeld answers a repository under a takedown, and says whether it
// did. Asked after the repository is known and before anything is
// forwarded, so the engine hears nothing on a held repository's behalf.
func refuseHeld(w http.ResponseWriter, held func(string) bool, repo string) bool {
	if held == nil || !held(repo) {
		return false
	}
	w.Header().Set(ReasonHeader, ReasonRepoHeld)
	w.Header().Set("Retry-After", heldRetryAfter)
	http.Error(w, "this repository is unavailable for legal reasons and runs no CI", http.StatusUnavailableForLegalReasons)
	return true
}

// DeliverChanged answers a runner's request for the paths its task's
// check-in changed, once it has proved it holds the task. The task names
// the check-in in the forge's own run records; the request names nothing.
type DeliverChanged func(repo, taskID string) (ChangedPaths, error)

// ChangedPaths are the lists QUARRY_CHANGED and QUARRY_TOUCHED are made
// of. A nil list is "none to give" (a check-in on the root line has no
// branch change); an empty one is a real answer of no paths.
type ChangedPaths struct {
	// Branch is the branch the run's check-in is on, from the forge's
	// own index, for a branch-only runner; "" when it cannot say.
	Branch  string   `json:"branch,omitempty"`
	Changed []string `json:"changed"`
	Touched []string `json:"touched"`
}

// Option adjusts Handler.
type Option func(*options)

type options struct {
	starting func(repo string) bool
	held     func(repo string) bool
	changed  DeliverChanged
	repoOf   func(group string) (string, bool)
}

// WithRepoOfGroup lets the bridge name the repository a refused token
// belongs to: repoOf answers the repository whose worker group is the
// one given. The engine refuses a live token scoped to another group
// with a 403 that lists its groups; the bridge answers that with
// ReasonOtherRepo and the repository's name. The engine's 401, for a
// token it does not know, passes through untouched.
func WithRepoOfGroup(repoOf func(group string) (string, bool)) Option {
	return func(o *options) { o.repoOf = repoOf }
}

// WithHeld says whether a repository is under a takedown (RFC 0023):
// its runners are refused on every route with ReasonRepoHeld.
func WithHeld(held func(repo string) bool) Option {
	return func(o *options) { o.held = held }
}

// WithChanged adds the changed-paths route, so a job on a private
// repository gets its lists by the worker token that holds its task
// and not by a read credential on the repository's API.
func WithChanged(f DeliverChanged) Option {
	return func(o *options) { o.changed = f }
}

// WithStarting says whether a repository the forge has promised but not
// yet registered is meant: that is answered 503 with Retry-After, the
// truthful answer while the forge starts, not a 404.
func WithStarting(starting func(repo string) bool) Option {
	return func(o *options) { o.starting = starting }
}

// refuseUnknownRepo answers a repository the forge does not serve.
func refuseUnknownRepo(w http.ResponseWriter, starting func(string) bool, repo string) {
	if starting != nil && starting(repo) {
		w.Header().Set("Retry-After", "5")
		http.Error(w, "this repository is still starting", http.StatusServiceUnavailable)
		return
	}
	w.Header().Set(ReasonHeader, ReasonRepoNotServed)
	http.Error(w, "no such repository", http.StatusNotFound)
}

// Handler forwards the worker routes to engine. groupFor names a
// repository's worker group; known says whether the forge serves the
// repository at all; admin is the engine's admin bearer, refused here.
// deliver answers the secrets route and token the token route; nil
// leaves its route out.
func Handler(engine string, admin string, groupFor func(repo string) string, known func(repo string) bool, deliver Deliver, token DeliverToken, opts ...Option) (http.Handler, error) {
	var o options
	for _, opt := range opts {
		opt(&o)
	}
	base, err := url.Parse(engine)
	if err != nil || base.Scheme == "" || base.Host == "" {
		return nil, fmt.Errorf("cibridge: engine URL %q is not an absolute URL", engine)
	}
	proxy := &httputil.ReverseProxy{
		Rewrite: func(pr *httputil.ProxyRequest) {
			pr.Out.URL.Scheme, pr.Out.URL.Host = base.Scheme, base.Host
			pr.Out.Host = base.Host
			// The forwarding headers name the runner's address to the
			// engine, which neither needs nor logs it; drop them.
			pr.Out.Header.Del("X-Forwarded-For")
		},
		// connect is a Server-Sent Events stream: every heartbeat must
		// reach the runner as it happens, or the runner sees silence
		// and the engine sees a dead worker.
		FlushInterval: -1,
		// The engine's own answer for a task it no longer has or has
		// already resolved is final; say so, so a runner need not read
		// the words.
		ModifyResponse: func(resp *http.Response) error {
			// The reason is the bridge's to give: one the engine sent
			// itself is never passed on.
			resp.Header.Del(ReasonHeader)
			p := resp.Request.URL.Path
			if o.repoOf != nil && resp.StatusCode == http.StatusForbidden &&
				(p == "/v1/tasks/poll" || p == "/v1/workers/connect") {
				return nameOtherRepo(resp, o.repoOf)
			}
			if !strings.HasPrefix(p, "/v1/tasks/") || !(strings.HasSuffix(p, "/resolve") || strings.HasSuffix(p, "/logs")) {
				return nil
			}
			switch resp.StatusCode {
			case http.StatusNotFound:
				resp.Header.Set(ReasonHeader, ReasonTaskGone)
			case http.StatusConflict:
				resp.Header.Set(ReasonHeader, ReasonTaskResolved)
			}
			return nil
		},
	}
	mux := http.NewServeMux()
	route := func(pattern, upstream string, pin bool) {
		mux.HandleFunc(pattern, func(w http.ResponseWriter, req *http.Request) {
			// A runner's poll is held open until a job comes: its time is
			// the wait, and the traffic page must not read it as a slow
			// page.
			if upstream == "/v1/tasks/poll" {
				obs.NoteOf(req.Context()).LongPoll()
			}
			repo := req.PathValue("repo")
			if known != nil && !known(repo) {
				refuseUnknownRepo(w, o.starting, repo)
				return
			}
			if refuseHeld(w, o.held, repo) {
				return
			}
			tok, ok := bearer(req)
			if !ok {
				w.Header().Set("WWW-Authenticate", `Bearer realm="quarry runners"`)
				http.Error(w, "a runner authenticates with a worker token from Settings → Runners", http.StatusUnauthorized)
				return
			}
			if admin != "" && subtle.ConstantTimeCompare([]byte(tok), []byte(admin)) == 1 {
				http.Error(w, "the engine's admin token is not accepted here; mint a worker token in Settings → Runners", http.StatusForbidden)
				return
			}
			req.Body = http.MaxBytesReader(w, req.Body, maxBody)
			if pin {
				body, err := io.ReadAll(req.Body)
				if err != nil {
					http.Error(w, "request too large", http.StatusRequestEntityTooLarge)
					return
				}
				pinned, err := pinGroup(body, groupFor(repo))
				if err != nil {
					http.Error(w, err.Error(), http.StatusBadRequest)
					return
				}
				req.Body = io.NopCloser(bytes.NewReader(pinned))
				req.ContentLength = int64(len(pinned))
			}
			path := upstream
			if id := req.PathValue("id"); id != "" {
				path = strings.Replace(upstream, "{id}", url.PathEscape(id), 1)
			}
			req.URL.Path, req.URL.RawPath = path, ""
			proxy.ServeHTTP(w, req)
		})
	}
	route("POST /r/{repo}/ci/bridge/v1/workers/connect", "/v1/workers/connect", true)
	route("POST /r/{repo}/ci/bridge/v1/tasks/poll", "/v1/tasks/poll", true)
	route("POST /r/{repo}/ci/bridge/v1/tasks/{id}/resolve", "/v1/tasks/{id}/resolve", false)
	route("POST /r/{repo}/ci/bridge/v1/tasks/{id}/logs", "/v1/tasks/{id}/logs", false)
	if deliver != nil {
		mux.HandleFunc("POST /r/{repo}/ci/bridge/v1/tasks/{id}/secrets", secretsHandler(base, admin, known, o.starting, o.held, deliver))
	}
	if token != nil {
		mux.HandleFunc("POST /r/{repo}/ci/bridge/v1/tasks/{id}/token", tokenHandler(base, admin, known, o.starting, o.held, token))
	}
	if o.changed != nil {
		mux.HandleFunc("POST /r/{repo}/ci/bridge/v1/tasks/{id}/changed", changedHandler(base, admin, known, o.starting, o.held, o.changed))
	}
	return mux, nil
}

func changedHandler(base *url.URL, admin string, known func(string) bool, starting, held func(string) bool, deliver DeliverChanged) http.HandlerFunc {
	client := &http.Client{Timeout: 15 * time.Second}
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Cache-Control", "no-store")
		repo, id := req.PathValue("repo"), req.PathValue("id")
		tok, ok := runnerToken(w, req, admin, known, starting, held)
		if !ok || !holds(w, req, client, base, id, tok) {
			return
		}
		out, err := deliver(repo, id)
		if err != nil {
			http.Error(w, err.Error(), http.StatusForbidden)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(out)
	}
}

// secretsBody is a runner's request: the job's environment ("" for
// none) and the secret names its steps export.
type secretsBody struct {
	Environment string   `json:"environment"`
	Names       []string `json:"names"`
}

func secretsHandler(base *url.URL, admin string, known func(string) bool, starting, held func(string) bool, deliver Deliver) http.HandlerFunc {
	client := &http.Client{Timeout: 15 * time.Second}
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Cache-Control", "no-store")
		repo, id := req.PathValue("repo"), req.PathValue("id")
		tok, ok := runnerToken(w, req, admin, known, starting, held)
		if !ok {
			return
		}
		var body secretsBody
		if err := json.NewDecoder(http.MaxBytesReader(w, req.Body, 64<<10)).Decode(&body); err != nil || len(body.Names) == 0 || len(body.Names) > 100 {
			http.Error(w, "the body is {\"environment\": \"...\", \"names\": [...]} with 1 to 100 names", http.StatusBadRequest)
			return
		}
		if !holds(w, req, client, base, id, tok) {
			return
		}
		values, err := deliver(repo, id, body.Environment, body.Names)
		if err != nil {
			http.Error(w, err.Error(), http.StatusForbidden)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(map[string]map[string]string{"secrets": values})
	}
}

func tokenHandler(base *url.URL, admin string, known func(string) bool, starting, held func(string) bool, deliver DeliverToken) http.HandlerFunc {
	client := &http.Client{Timeout: 15 * time.Second}
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Cache-Control", "no-store")
		repo, id := req.PathValue("repo"), req.PathValue("id")
		tok, ok := runnerToken(w, req, admin, known, starting, held)
		if !ok || !holds(w, req, client, base, id, tok) {
			return
		}
		env, note, err := deliver(repo, id)
		if err != nil {
			http.Error(w, err.Error(), http.StatusForbidden)
			return
		}
		if env == nil {
			env = map[string]string{}
		}
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(map[string]any{"env": env, "note": note})
	}
}

// runnerToken is a forge-route request's worker token, the request
// answered when it has none or it is the engine's admin bearer.
func runnerToken(w http.ResponseWriter, req *http.Request, admin string, known func(string) bool, starting, held func(string) bool) (string, bool) {
	if known != nil && !known(req.PathValue("repo")) {
		refuseUnknownRepo(w, starting, req.PathValue("repo"))
		return "", false
	}
	if refuseHeld(w, held, req.PathValue("repo")) {
		return "", false
	}
	tok, ok := bearer(req)
	if !ok {
		w.Header().Set("WWW-Authenticate", `Bearer realm="quarry runners"`)
		http.Error(w, "a runner authenticates with a worker token from Settings → Runners", http.StatusUnauthorized)
		return "", false
	}
	if admin != "" && subtle.ConstantTimeCompare([]byte(tok), []byte(admin)) == 1 {
		http.Error(w, "the engine's admin token is not accepted here", http.StatusForbidden)
		return "", false
	}
	return tok, true
}

// holds is the proof of claim: only the token that claimed a task can
// heartbeat it, and a heartbeat changes nothing but its lease. A
// runner that does not hold the task is answered here.
func holds(w http.ResponseWriter, req *http.Request, client *http.Client, base *url.URL, id, tok string) bool {
	hb, _ := http.NewRequestWithContext(req.Context(), "POST",
		base.Scheme+"://"+base.Host+"/v1/tasks/"+url.PathEscape(id)+"/resolve",
		strings.NewReader(`{"action":"heartbeat"}`))
	hb.Header.Set("Authorization", "Bearer "+tok)
	hb.Header.Set("Content-Type", "application/json")
	resp, err := client.Do(hb)
	if err != nil {
		http.Error(w, "the engine could not be asked whether this runner holds the task", http.StatusBadGateway)
		return false
	}
	io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
	resp.Body.Close()
	switch {
	case resp.StatusCode < 300:
		return true
	case resp.StatusCode == http.StatusNotFound, resp.StatusCode == http.StatusConflict, resp.StatusCode == http.StatusForbidden:
		// The engine's heartbeat answers exactly these for a task it has
		// no claim for: 404 not in flight (resolved, reaped, cancelled,
		// or the engine restarted), 409 its run was cancelled, 403 held
		// by another token (bridge/resolve.go, ackmap.go).
		w.Header().Set(ReasonHeader, ReasonNotHeld)
		http.Error(w, "this runner does not hold that task", http.StatusForbidden)
	case resp.StatusCode == http.StatusUnauthorized:
		// The runner's own token is not accepted: a failure to show, not
		// a task it lost.
		http.Error(w, "the engine does not accept this runner's token", http.StatusForbidden)
	default:
		http.Error(w, "the engine could not say whether this runner holds the task", http.StatusBadGateway)
	}
	return false
}

// groupsRefused finds the groups a token is scoped to in the engine's
// refusal of a group it may not drain: `scoped to worker groups ["a" "b"]`.
var groupsRefused = regexp.MustCompile(`scoped to worker groups \[([^\]]*)\]`)
var quotedGroup = regexp.MustCompile(`"((?:[^"\\]|\\.)*)"`)

// nameOtherRepo rewrites the engine's group refusal of a live token into
// ReasonOtherRepo with the repository it is scoped to. Any other 403
// (a task-type refusal, a group no repository owns) is passed on as it
// was.
func nameOtherRepo(resp *http.Response, repoOf func(string) (string, bool)) error {
	raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
	resp.Body.Close()
	if err != nil {
		return err
	}
	resp.Body = io.NopCloser(bytes.NewReader(raw))
	m := groupsRefused.FindSubmatch(raw)
	if m == nil {
		return nil
	}
	for _, q := range quotedGroup.FindAllSubmatch(m[1], -1) {
		g, err := strconv.Unquote(`"` + string(q[1]) + `"`)
		if err != nil {
			continue
		}
		repo, ok := repoOf(g)
		if !ok {
			continue
		}
		body, _ := json.Marshal(map[string]string{"reason": ReasonOtherRepo, "repository": repo})
		resp.Header.Set(ReasonHeader, ReasonOtherRepo)
		resp.Header.Set("Content-Type", "application/json")
		resp.Header.Del("Content-Length")
		resp.Body = io.NopCloser(bytes.NewReader(body))
		resp.ContentLength = int64(len(body))
		return nil
	}
	return nil
}

func bearer(req *http.Request) (string, bool) {
	h := req.Header.Get("Authorization")
	tok, ok := strings.CutPrefix(h, "Bearer ")
	tok = strings.TrimSpace(tok)
	return tok, ok && tok != ""
}

// pinGroup sets a poll's or a connect's worker_group to the repository's own, and
// refuses one that names another. Every other field passes through
// untouched.
func pinGroup(body []byte, own string) ([]byte, error) {
	var poll map[string]json.RawMessage
	if len(bytes.TrimSpace(body)) == 0 {
		poll = map[string]json.RawMessage{}
	} else if err := json.Unmarshal(body, &poll); err != nil {
		return nil, fmt.Errorf("a poll is a JSON object: %v", err)
	}
	if raw, ok := poll["worker_group"]; ok {
		var named string
		if err := json.Unmarshal(raw, &named); err != nil {
			return nil, fmt.Errorf("worker_group must be a string")
		}
		if named != "" && named != own {
			return nil, fmt.Errorf("this repository's runners poll worker group %q, not %q", own, named)
		}
	}
	g, _ := json.Marshal(own)
	poll["worker_group"] = g
	return json.Marshal(poll)
}