// Package runner is `quarry runner`: a generic CI worker a repository's
// owner runs on their own machine (f24d78d9).
//
// It speaks the engine's HTTP worker protocol through the forge
// (internal/cibridge), so all it needs is the repository's URL and a
// worker token from Settings → Runners. For each task it checks out the
// check-in under test, reads ci/runner.json AT that check-in, runs the
// argv the manifest names for the task's type, streams its output to
// the run's log, and reports the verdict.
//
// It runs whatever the check-in's manifest says. That is the point on
// the owner's machine and exactly why it must never be pointed at a
// repository whose writers the machine's owner does not trust.
package runner
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"maps"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"craftdesign.group/quarry/internal/cibridge"
)
// Checkout puts a check-in's tree in dir and returns nothing else: the
// runner reads the manifest from the tree it is about to test.
type Checkout interface {
Checkout(ctx context.Context, dir, checkin string) error
}
// Config is what a runner needs.
type Config struct {
Bridge string // https://forge/r/<repo>/ci/bridge — the four worker routes live under it
Token string // a worker token from Settings → Runners
Tasks []string // task types to poll for
MaxTasks int // tasks run at once
WorkDir string // numbered workspaces per task type under it
// TempRoot holds one temp directory per running task, which a task
// finds as TMPDIR and GOTMPDIR and which goes when the task ends;
// the runner removes its own leftover task dirs as it starts. It
// defaults to a short root under /var/tmp named for WorkDir
// (defaultTempRoot); it must be a real directory the runner owns,
// never a symlink.
TempRoot string
WorkerID string
Version string // printed in the ready line, so a log says which build ran
Checkout Checkout
Client *http.Client
// Heartbeat is how often a running task extends its claim; the
// engine redelivers a task whose claim lapses.
Heartbeat time.Duration
// CheckoutRetry is the wait before each further checkout attempt;
// why the checkout retries here and the step does not is in
// ci/quarry.json's _comment. Every checkout error is retried, so a
// permanent one (a hash the forge does not have) fails after ~65s.
CheckoutRetry []time.Duration
// ResolveRetry is the wait before each further attempt to report a
// task's end. A deploy restarts the forge, and a report sent in that
// window (refused, or 404 while the repository is not yet served)
// was lost: the run stayed running until the claim lapsed and the
// whole task ran again. 409, a cancelled run, is never retried.
ResolveRetry []time.Duration
// InfoRetry is the wait before each further ask of the bridge for a
// task's branch and changed paths.
InfoRetry []time.Duration
Logf func(format string, args ...any)
// Branch, when set, makes this a runner for one branch's check-ins
// only: a deploy runner, holding a production secret in its
// environment, refuses a branch's code and every premerge pair
// whatever the pipeline that sent the task says. BranchOf answers
// which branch a check-in is on; the forge is the authority.
Branch string
BranchOf func(ctx context.Context, checkin string) (string, error)
// Changed, when set, answers the paths a check-in's branch changed
// since its fork point. A task gets them as a file named by
// QUARRY_CHANGED, so it can test only what the branch touched; a
// check-in on a branch with no fork point (trunk), or a failed
// lookup, gets no list and the task runs whole.
Changed func(ctx context.Context, checkin string) ([]string, error)
// Touched, when set, answers the paths a check-in itself changed
// against its first parent: for a landing's merge onto trunk, all
// it landed. A task gets QUARRY_TOUCHED naming a file of the paths
// the change under test touched: the branch's list where there is
// one, else this. Unlike QUARRY_CHANGED it is given on trunk too,
// so a check can tell a change that touched no code at all.
Touched func(ctx context.Context, checkin string) ([]string, error)
// JobUsers, when set, runs every task as one of these users, a pool
// the runner may become through sudo, one per running task (see
// isolate.go). It is what lets a runner receive secrets; without it
// a job asking for one is refused. Self is this program's path, whose
// job-env subcommand starts each command.
JobUsers []string
Self string
// DrainBudget is how long Drain lets running jobs go on before
// ending them and handing their tasks back; it must fit inside the
// service manager's stop timeout (default DefaultDrainBudget, for
// systemd's 90 s).
DrainBudget time.Duration
}
// DefaultDrainBudget leaves 15 s of systemd's default 90 s stop timeout
// for ending the job's processes, the job user's reclaim and the
// hand-back.
const DefaultDrainBudget = 75 * time.Second
// Runner polls, runs and reports until its context ends.
type Runner struct {
cfg Config
slots chan struct{}
users *slots
// places numbers the workspaces, one per task the runner may run at
// once.
places chan int
locks sync.Map // workspace dir -> *sync.Mutex
infos sync.Map // task id -> *taskInfoEntry, for the task's life
// drain closes on the first stop signal: no new task is claimed.
// expired closes DrainBudget later: running jobs are ended and
// their tasks handed back.
drain chan struct{}
expired chan struct{}
drainOnce sync.Once
running sync.WaitGroup // tasks claimed and not yet reported
// refusal is the token refusal last said, so the connect stream and
// every poll loop, all refused alike, print it once.
refusalMu sync.Mutex
refusal string
}
// New checks cfg and fills its defaults.
func New(cfg Config) (*Runner, error) {
if cfg.Bridge == "" || cfg.Token == "" || len(cfg.Tasks) == 0 || cfg.Checkout == nil || cfg.WorkDir == "" {
return nil, errors.New("runner: bridge, token, tasks, workdir and checkout are all required")
}
if cfg.Branch != "" && cfg.BranchOf == nil {
return nil, errors.New("runner: a branch-only runner needs BranchOf")
}
if cfg.InfoRetry == nil {
cfg.InfoRetry = []time.Duration{500 * time.Millisecond, 2 * time.Second}
}
if cfg.MaxTasks < 1 {
cfg.MaxTasks = 1
}
if cfg.TempRoot == "" {
cfg.TempRoot = defaultTempRoot(cfg.WorkDir)
}
if cfg.Client == nil {
cfg.Client = &http.Client{}
}
if cfg.Heartbeat <= 0 {
cfg.Heartbeat = 20 * time.Second
}
if cfg.CheckoutRetry == nil {
cfg.CheckoutRetry = []time.Duration{5 * time.Second, 15 * time.Second, 45 * time.Second}
}
if cfg.ResolveRetry == nil {
cfg.ResolveRetry = []time.Duration{2 * time.Second, 5 * time.Second, 10 * time.Second,
20 * time.Second, 30 * time.Second, 60 * time.Second, 60 * time.Second}
}
if cfg.DrainBudget <= 0 {
cfg.DrainBudget = DefaultDrainBudget
}
if cfg.WorkerID == "" {
h, _ := os.Hostname()
cfg.WorkerID = fmt.Sprintf("quarry-runner-%s-%d", h, os.Getpid())
}
if cfg.Logf == nil {
cfg.Logf = log.Printf
}
users, err := newSlots(cfg.JobUsers, cfg.Self)
if err != nil {
return nil, err
}
places := make(chan int, cfg.MaxTasks)
for i := 0; i < cfg.MaxTasks; i++ {
places <- i
}
return &Runner{cfg: cfg, slots: make(chan struct{}, cfg.MaxTasks), users: users, places: places,
drain: make(chan struct{}), expired: make(chan struct{})}, nil
}
// Drain is the first stop signal: the runner claims no further task, lets
// each running job go on for DrainBudget, then ends it and hands its task
// back to the engine to be redelivered at once. Run returns when the last
// task is reported or handed back. Safe to call more than once.
func (r *Runner) Drain() {
r.drainOnce.Do(func() {
close(r.drain)
r.cfg.Logf("drain: stop requested; claiming nothing new, running jobs have %s", r.cfg.DrainBudget)
time.AfterFunc(r.cfg.DrainBudget, func() { close(r.expired) })
})
}
func (r *Runner) draining() bool {
select {
case <-r.drain:
return true
default:
return false
}
}
// Run connects and polls every task type until ctx ends, or until Drain
// has been called and the running tasks are done. ctx is the hard stop. One poller per
// type, because a poll's answer does not say which of several types a
// task was for, and the type is what picks the manifest entry.
func (r *Runner) Run(ctx context.Context) error {
if err := r.sweepTemp(); err != nil {
return err
}
r.handBackWorkspaces()
go r.stayConnected(ctx)
var wg sync.WaitGroup
for _, t := range r.cfg.Tasks {
wg.Add(1)
go func(taskType string) {
defer wg.Done()
r.pollLoop(ctx, taskType)
}(t)
}
r.cfg.Logf("quarry runner %s ready · %s · tasks %s · max-tasks %d", r.cfg.Version, r.cfg.Bridge, strings.Join(r.cfg.Tasks, ", "), r.cfg.MaxTasks)
wg.Wait()
r.running.Wait()
return ctx.Err()
}
// stayConnected holds the registration stream open, reconnecting with
// backoff, so the engine lists this worker and its capacity.
func (r *Runner) stayConnected(ctx context.Context) {
backoff := time.Second
for ctx.Err() == nil {
body, _ := json.Marshal(map[string]any{"worker_id": r.cfg.WorkerID, "task_types": r.cfg.Tasks, "max_tasks": r.cfg.MaxTasks})
resp, err := r.post(ctx, "/v1/workers/connect", body)
if err == nil {
backoff = time.Second
r.clearRefusal()
io.Copy(io.Discard, resp.Body) // the heartbeat stream, until it ends
resp.Body.Close()
} else if ctx.Err() == nil {
if wait, held := pauseAfter(err); held {
r.cfg.Logf("connect: the repository is under a takedown and runs no CI; asking again in %s", wait)
backoff = wait
} else if msg, refused := r.tokenRefusal(err); refused {
r.noticeRefusal(msg)
backoff = tokenPause
} else {
r.cfg.Logf("connect: %v; retrying in %s", err, backoff)
}
}
select {
case <-ctx.Done():
return
case <-time.After(backoff):
}
if backoff < time.Minute {
backoff *= 2
}
}
}
type polled struct {
TaskID string `json:"task_id"`
RunID string `json:"run_id"`
StepID string `json:"step_id"`
Input json.RawMessage `json:"input"`
// Metadata is the step's own, copied by the engine: a workflow job
// travels here under JobKey.
Metadata map[string]string `json:"metadata,omitempty"`
}
// pollLoop takes a slot, then asks for one task of its type; a slot is
// held before claiming so a claimed task never waits for a slot here.
func (r *Runner) pollLoop(hard context.Context, taskType string) {
// ctx ends on the hard stop or on Drain; only polling stops on
// Drain, a running task goes on under hard.
ctx, stopPolling := context.WithCancel(hard)
defer stopPolling()
go func() {
select {
case <-r.drain:
stopPolling()
case <-ctx.Done():
}
}()
for ctx.Err() == nil {
select {
case r.slots <- struct{}{}:
case <-ctx.Done():
return
}
body, _ := json.Marshal(map[string]any{"task_types": []string{taskType}, "max_tasks": 1, "timeout_ms": 25000})
resp, err := r.post(ctx, "/v1/tasks/poll", body)
if err != nil {
<-r.slots
if ctx.Err() == nil {
wait, held := pauseAfter(err)
if held {
r.cfg.Logf("poll %s: the repository is under a takedown and runs no CI; asking again in %s", taskType, wait)
} else if msg, refused := r.tokenRefusal(err); refused {
r.noticeRefusal(msg)
} else {
r.cfg.Logf("poll %s: %v", taskType, err)
}
select {
case <-ctx.Done():
case <-time.After(wait):
}
}
continue
}
r.clearRefusal()
var tasks []polled
err = json.NewDecoder(resp.Body).Decode(&tasks)
resp.Body.Close()
if err != nil || len(tasks) == 0 {
<-r.slots
continue
}
t := tasks[0]
r.running.Add(1)
go func() {
defer r.running.Done()
defer func() { <-r.slots }()
r.runTask(hard, taskType, t)
}()
}
}
// runTask checks out, reads the manifest at the check-in, runs the
// task's argv, and resolves the task with the verdict.
func (r *Runner) runTask(ctx context.Context, taskType string, t polled) {
in, carried := subjectOf(t.Input)
start := time.Now()
logs := newLogSink(ctx, r, t.TaskID, "out")
errs := newLogSink(ctx, r, t.TaskID, "err")
hbCtx, stopHB := context.WithCancel(ctx)
taskCtx, stopTask := context.WithCancel(ctx)
defer stopTask()
var cancelled atomic.Bool
go r.heartbeat(hbCtx, t.TaskID, func() { cancelled.Store(true); stopTask() })
// A job that outlasts the drain budget is ended through taskCtx
// (its process group is killed); execute's deferred job-user reclaim
// then ends what it left running as that user.
var outlasted atomic.Bool
watchDone := make(chan struct{})
go func() {
select {
case <-r.expired:
outlasted.Store(true)
stopTask()
case <-watchDone:
}
}()
var waited time.Duration
var verdict error
late := r.draining()
if late {
// Claimed in the instant the stop arrived: nothing has run, so
// it goes straight back.
stopTask()
verdict = context.Canceled
} else {
verdict = r.execute(taskCtx, t.TaskID, taskType, in.Checkin, in.Tag, t.Metadata[JobKey], &waited, logs, errs)
}
close(watchDone)
stopHB()
// Interrupted by the drain budget, or by a second signal while
// draining: not a verdict on the check-in.
handBack := verdict != nil && !cancelled.Load() && (late || outlasted.Load() || (r.draining() && ctx.Err() != nil))
if cancelled.Load() {
fmt.Fprintln(errs, "stopped: this task's run was cancelled")
}
logs.Close()
errs.Close()
if errors.Is(verdict, errNotHeld) {
// Cancelled between the queue and the claim: no job ran, so
// there is no verdict to report, failing or otherwise.
r.cfg.Logf("%s %s @ %.12s: skipped: its run was cancelled before this runner could start it", taskType, in.Repo, in.Checkin)
return
}
if handBack {
r.handBackTask(ctx, taskType, in.Repo, in.Checkin, jobName(t, taskType), t.TaskID, start)
return
}
if cancelled.Load() {
// The engine withdrew the task; a resolve would only be refused.
r.cfg.Logf("%s %s @ %.12s: stopped after %s: its run was cancelled", taskType, in.Repo, in.Checkin, time.Since(start).Round(time.Second))
return
}
// The time a task spent waiting for a workspace is not its own:
// said apart, so "ok in 4m" is how long the work took.
took := (time.Since(start) - waited).Round(time.Second)
queue := ""
if waited >= time.Second {
queue = fmt.Sprintf(" (waited %s for a workspace)", waited.Round(time.Second))
}
if r.draining() {
r.cfg.Logf("drain: %s finished in %s: %s", jobName(t, taskType), took, map[bool]string{true: "ok", false: "FAIL"}[verdict == nil])
}
if verdict != nil {
r.cfg.Logf("%s %s @ %.12s: FAIL in %s%s: %v", taskType, in.Repo, in.Checkin, took, queue, verdict)
r.finish(ctx, t.TaskID, map[string]any{"action": "fail", "error": verdict.Error()})
return
}
r.cfg.Logf("%s %s @ %.12s: ok in %s%s", taskType, in.Repo, in.Checkin, took, queue)
r.finish(ctx, t.TaskID, map[string]any{"action": "complete", "output": carried})
}
// jobName names a task in the drain log: its workflow job's name, else
// its task type.
func jobName(t polled, taskType string) string {
if raw := t.Metadata[JobKey]; raw != "" {
var j struct {
Name string `json:"name"`
}
if json.Unmarshal([]byte(raw), &j) == nil && j.Name != "" {
return j.Name
}
}
return taskType
}
// handBackTask returns an interrupted task to the engine: the bridge's
// pause action NAKs the message with a 1 ms delay, so it is redelivered
// to a runner at once instead of after the engine's ack wait, and it
// records no verdict for the check-in. It runs even after a hard stop,
// on a context of its own with a short deadline.
func (r *Runner) handBackTask(ctx context.Context, taskType, repo, checkin, name, taskID string, start time.Time) {
hctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
err := r.resolve(hctx, taskID, map[string]any{"action": "pause", "duration_ms": 1, "checkpoint": json.RawMessage("{}")})
took := time.Since(start).Round(time.Millisecond)
if err != nil {
r.cfg.Logf("drain: %s (%s %s @ %.12s): interrupted after %s; handing it back failed (%v), the engine redelivers it after its ack wait", name, taskType, repo, checkin, took, err)
return
}
r.cfg.Logf("drain: %s (%s %s @ %.12s): interrupted after %s and handed back for redelivery", name, taskType, repo, checkin, took)
}
// finish reports a task's end, retrying with ResolveRetry's waits so a
// forge restarting under a deploy does not lose a finished task's
// verdict. A final refusal (404 task gone, 409 already resolved or
// cancelled) and a stopping runner end it at once; connection errors,
// 5xx and 429 are retried.
func (r *Runner) finish(ctx context.Context, taskID string, body map[string]any) {
for attempt := 0; ; attempt++ {
err := r.resolve(ctx, taskID, body)
if err == nil || finalRefusal(err) || ctx.Err() != nil || attempt >= len(r.cfg.ResolveRetry) {
return
}
select {
case <-time.After(r.cfg.ResolveRetry[attempt]):
case <-ctx.Done():
return
}
}
}
// subject is what a task is about: the repository and the check-in.
type subject struct {
Repo string `json:"repo"`
Checkin string `json:"checkin"`
// Tag is the tag whose adding started the run, for a run the tag
// event started (RFC 0020); its steps see it as QUARRY_TAG.
Tag string `json:"tag,omitempty"`
}
// subjectOf reads a task's subject from its input, and returns the input
// a step after it should receive. The engine hands a step with no
// depends_on the run's input, but a step with one dependency that
// step's output, and a step with several a map of each one's output
// (dagnats dag.ResolveInput). So a task's output is its subject, which
// is how a deploy that waits for the tests still knows what to deploy;
// with several dependencies the subject is taken from their outputs,
// and they must agree.
func subjectOf(raw json.RawMessage) (subject, json.RawMessage) {
var in subject
if json.Unmarshal(raw, &in) == nil && in.Checkin != "" {
return in, raw
}
var fanIn map[string]json.RawMessage
if json.Unmarshal(raw, &fanIn) != nil {
return subject{}, raw
}
var found subject
var from json.RawMessage
for _, out := range fanIn {
var s subject
if json.Unmarshal(out, &s) != nil || s.Checkin == "" {
continue
}
if found.Checkin != "" && s != found {
return subject{}, raw // the dependencies disagree: no subject, so the task fails naming no check-in
}
found, from = s, out
}
if found.Checkin == "" {
return subject{}, raw
}
return found, from
}
// execute runs one task at a check-in: the workflow job it carries, or
// else the command ci/runner.json names for its type there.
func (r *Runner) execute(ctx context.Context, taskID, taskType, checkin, tag, job string, waited *time.Duration, logs, errs io.Writer) error {
defer r.infos.Delete(taskID)
if checkin == "" {
return errors.New("the task names no check-in")
}
if err := r.onBranch(ctx, taskID, checkin); err != nil {
return err
}
queued := time.Now()
// Every task, manifest or job, takes a numbered workspace of its
// task type from one pool of --max-tasks, so tasks of one type run
// side by side (every workflow job has the type <group>.job, and a
// manifest type like quarry.race would otherwise queue behind one
// workspace). A slot is held before a task is claimed and there are
// as many places as slots, so a claimed task always has a place to
// go to and never waits here: the lock below is only a safety. A
// reused place keeps its checkout; the first use of one is a clone.
place := <-r.places
defer func() { r.places <- place }()
dir := fmt.Sprintf("%s-%d", filepath.Join(r.cfg.WorkDir, "ws", safeName(taskType)), place)
mu, _ := r.locks.LoadOrStore(dir, &sync.Mutex{})
mu.(*sync.Mutex).Lock()
defer mu.(*sync.Mutex).Unlock()
if waited != nil {
*waited = time.Since(queued)
}
if err := r.checkout(ctx, dir, checkin, logs); err != nil {
return err
}
// Made before the job user's reclaim defers below, so it is removed
// after that has ended whatever still holds files in it, and removed
// even when that reclaim fails.
tmp, err := r.taskTemp()
if err != nil {
return err
}
defer func() {
if err := removeAll(tmp); err != nil {
r.cfg.Logf("task temp dir %s was not removed: %v", tmp, err)
}
}()
// Written before the temp dir is shared with a job user, so the
// share covers it.
changed := r.changedList(ctx, taskID, checkin, tmp, logs)
start := withTemp(plainStarter, tmp)
if r.users != nil {
slot := <-r.users.free
defer func() {
if err := r.users.reclaim(slot, dir, tmp); err != nil {
r.cfg.Logf("job user %s is out of the pool until the runner restarts: %v", slot, err)
return
}
r.users.free <- slot
}()
if err := share(dir, slot); err != nil {
return fmt.Errorf("sharing the checkout with job user %s: %w", slot, err)
}
if err := share(tmp, slot); err != nil {
return fmt.Errorf("sharing the temp dir with job user %s: %w", slot, err)
}
start = withTemp(r.users.starter(slot, dir), tmp)
}
if job != "" {
j, err := ParseJob(job)
if err != nil {
return err
}
// The job's run token (RFC 0021 §runs), whatever its file
// declared: the forge decides from its own records, and says
// why when it hands none.
// A runner running jobs as itself cannot keep a token to the
// job, so it does not ask for one: the job runs without.
var tokenEnv map[string]string
if r.users == nil {
fmt.Fprintln(logs, "no run token: this runner runs jobs as itself and cannot keep one to the job; start it with --job-users")
} else {
env, note, err := r.fetchRunToken(ctx, taskID)
if err != nil {
return err
}
if note != "" {
fmt.Fprintln(logs, note)
}
tokenEnv = env
}
var secrets map[string]string
if names := j.secrets(); len(names) > 0 {
if r.users == nil {
// Refused rather than run without them: a deploy that
// quietly got no token would fail later and less clearly.
return fmt.Errorf("the job asks for secrets (%s), and this runner runs jobs as itself, so it cannot receive them; start it with --job-users", strings.Join(names, ", "))
}
if secrets, err = r.fetchSecrets(ctx, taskID, j.Environment, names); err != nil {
return err
}
}
masked := maps.Clone(secrets)
if v := tokenEnv["QUARRY_TOKEN"]; v != "" {
if masked == nil {
masked = map[string]string{}
}
masked["QUARRY_TOKEN"] = v
}
if len(masked) > 0 {
ml, me := newMasker(logs, masked), newMasker(errs, masked)
defer ml.Close()
defer me.Close()
logs, errs = ml, me
}
run := maps.Clone(tokenEnv)
if run == nil {
run = map[string]string{}
}
maps.Copy(run, changed)
if tag != "" {
if run == nil {
run = map[string]string{}
}
run["QUARRY_TAG"] = tag
}
return runJob(ctx, j, dir, run, secrets, start, logs, errs)
}
raw, err := os.ReadFile(filepath.Join(dir, ManifestPath))
if err != nil {
return fmt.Errorf("%s at %.12s: %v", ManifestPath, checkin, err)
}
m, err := ParseManifest(raw)
if err != nil {
return err
}
task, ok := m.Tasks[taskType]
if !ok {
return fmt.Errorf("%s at %.12s names no task %q (it names %s)", ManifestPath, checkin, taskType, strings.Join(m.Names(), ", "))
}
fmt.Fprintf(logs, "$ %s\n", strings.Join(task.Argv, " "))
cmd := start(ctx, filepath.Join(dir, filepath.FromSlash(task.Dir)), task.Argv, task.Env, changed)
var caught bytes.Buffer
cmd.Stdout = logs
if task.FailOnOutput {
cmd.Stdout = io.MultiWriter(logs, &caught)
}
cmd.Stderr = errs
if err := cmd.Run(); err != nil {
return err
}
if task.FailOnOutput && strings.TrimSpace(caught.String()) != "" {
n := len(strings.Fields(caught.String()))
fmt.Fprintf(errs, "%d line(s) of output from a task that must print nothing\n", n)
return fmt.Errorf("%d line(s) of output from a task that must print nothing", n)
}
return nil
}
// changedList writes the paths checkin's branch changed into tmp and
// returns QUARRY_CHANGED naming that file. With no list to give (no
// Changed, a check-in on trunk, or a lookup that failed) QUARRY_CHANGED
// is set empty, never left out: a task inherits the runner's
// environment, and a runner started from inside another CI task (as
// this package's own tests are) would otherwise hand that task's list
// to a check-in it does not describe. A premerge pair is asked about
// its branch tip, the branch side of the merge.
func (r *Runner) changedList(ctx context.Context, taskID, checkin, tmp string, logs io.Writer) map[string]string {
env := map[string]string{"QUARRY_CHANGED": "", "QUARRY_TOUCHED": ""}
tip := checkin
if IsPair(checkin) {
tip = strings.Split(checkin, ":")[1]
}
// The forge's bridge answers for the task this runner holds, whoever
// may read the repository's API; a forge without the route, or one
// that fails, leaves the runner's own lookups.
forgeAnswered := false
viaBridge, berr := r.forgeInfo(ctx, taskID)
switch {
case berr == nil:
forgeAnswered = true
case errors.Is(berr, errNoRoute):
// An older forge: the runner's own lookups, which carry no
// credential, answer for a public repository only.
default:
fmt.Fprintf(logs, "changed paths: none to give (the forge's list could not be read: %v)\n", withoutURL(berr))
return env
}
var branch []string
if forgeAnswered && viaBridge.Changed != nil {
if file, ok := writeList(tmp, "quarry-changed", viaBridge.Changed, logs); ok {
fmt.Fprintf(logs, "changed paths: %d on the branch, in QUARRY_CHANGED\n", len(viaBridge.Changed))
env["QUARRY_CHANGED"], branch = file, viaBridge.Changed
}
} else if !forgeAnswered && r.cfg.Changed != nil {
paths, err := r.cfg.Changed(ctx, tip)
if err != nil {
fmt.Fprintf(logs, "changed paths: none to give (%v)\n", err)
} else if file, ok := writeList(tmp, "quarry-changed", paths, logs); ok {
fmt.Fprintf(logs, "changed paths: %d on the branch, in QUARRY_CHANGED\n", len(paths))
env["QUARRY_CHANGED"], branch = file, paths
}
}
touched := branch
if touched == nil && !IsPair(checkin) {
if forgeAnswered {
touched = viaBridge.Touched
} else if r.cfg.Touched != nil {
paths, err := r.cfg.Touched(ctx, checkin)
if err != nil {
fmt.Fprintf(logs, "touched paths: none to give (%v)\n", err)
return env
}
touched = paths
}
}
if touched != nil {
if file, ok := writeList(tmp, "quarry-touched", touched, logs); ok {
env["QUARRY_TOUCHED"] = file
}
}
return env
}
// taskInfo is what the forge's bridge says of a task: its check-in's
// branch and the lists QUARRY_CHANGED and QUARRY_TOUCHED are made of.
type taskInfo struct {
Branch string `json:"branch"`
Changed []string `json:"changed"`
Touched []string `json:"touched"`
}
type taskInfoEntry struct {
once sync.Once
info taskInfo
err error
}
// forgeInfo asks the forge's bridge about the task this runner holds
// (POST .../tasks/{id}/changed), once per task however many things need
// it. The runner's worker token proves it holds the task; the forge
// reads the check-in from its own record of the run. A transient
// failure is retried against the bridge; only a forge without the route
// (errNoRoute) is the caller's cue for another way.
func (r *Runner) forgeInfo(ctx context.Context, taskID string) (taskInfo, error) {
if taskID == "" {
return taskInfo{}, errNoRoute
}
v, _ := r.infos.LoadOrStore(taskID, &taskInfoEntry{})
e := v.(*taskInfoEntry)
e.once.Do(func() {
for attempt := 0; ; attempt++ {
e.err = r.fetchTaskInfo(ctx, taskID, &e.info)
if e.err == nil || errors.Is(e.err, errNoRoute) || attempt >= len(r.cfg.InfoRetry) || ctx.Err() != nil {
return
}
select {
case <-time.After(r.cfg.InfoRetry[attempt]):
case <-ctx.Done():
return
}
}
})
return e.info, e.err
}
func (r *Runner) fetchTaskInfo(ctx context.Context, taskID string, into *taskInfo) error {
resp, err := r.post(ctx, "/v1/tasks/"+url.PathEscape(taskID)+"/changed", []byte(`{}`))
if err != nil {
return err
}
defer resp.Body.Close()
return json.NewDecoder(io.LimitReader(resp.Body, 64<<20)).Decode(into)
}
// writeList writes paths one a line to tmp/name.
func writeList(tmp, name string, paths []string, logs io.Writer) (string, bool) {
file := filepath.Join(tmp, name)
body := strings.Join(paths, "\n")
if body != "" {
body += "\n"
}
if err := os.WriteFile(file, []byte(body), 0o644); err != nil {
fmt.Fprintf(logs, "%s: none to give (%v)\n", name, err)
return "", false
}
return file, true
}
// checkout gets the tree, retrying with CheckoutRetry's waits: the
// only part of a task whose failure says nothing about the check-in.
func (r *Runner) checkout(ctx context.Context, dir, checkin string, logs io.Writer) error {
for attempt := 0; ; attempt++ {
fmt.Fprintf(logs, "$ checkout %s\n", SubjectLabel(checkin))
err := r.cfg.Checkout.Checkout(ctx, dir, checkin)
if err == nil {
return nil
}
err = withoutURL(err)
var conflict *MergeConflict
if errors.As(err, &conflict) {
// A verdict, not a fault: the same pair will not merge on
// the next try either. The paths are the step's log.
fmt.Fprintln(logs, conflict.Error())
return conflict
}
if attempt >= len(r.cfg.CheckoutRetry) || ctx.Err() != nil {
fmt.Fprintf(logs, "checkout failed: %v\n", err)
return fmt.Errorf("checkout: %w", err)
}
wait := r.cfg.CheckoutRetry[attempt]
fmt.Fprintf(logs, "checkout failed: %v; retrying in %s\n", err, wait)
select {
case <-time.After(wait):
case <-ctx.Done():
return fmt.Errorf("checkout: %w", err)
}
}
}
// heartbeat extends the task's claim while it runs.
// heartbeat keeps a task's claim alive, and stops the task when the
// engine answers that its run was cancelled: a closed landing's merge
// run, or one cancelled by hand, ends here rather than running to its
// end and holding the runner (dagnats#737).
func (r *Runner) heartbeat(ctx context.Context, taskID string, stop func()) {
tick := time.NewTicker(r.cfg.Heartbeat)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if errors.Is(r.resolve(ctx, taskID, map[string]any{"action": "heartbeat"}), errRunCancelled) {
stop()
return
}
}
}
}
func (r *Runner) resolve(ctx context.Context, taskID string, body map[string]any) error {
b, _ := json.Marshal(body)
resp, err := r.post(ctx, "/v1/tasks/"+taskID+"/resolve", b)
if err != nil {
if ctx.Err() == nil && !errors.Is(err, errRunCancelled) {
r.cfg.Logf("resolve %s (%v): %v", taskID, body["action"], err)
}
return err
}
resp.Body.Close()
return nil
}
func (r *Runner) post(ctx context.Context, path string, body []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, "POST", strings.TrimSuffix(r.cfg.Bridge, "/")+path, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+r.cfg.Token)
req.Header.Set("Content-Type", "application/json")
resp, err := r.cfg.Client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode >= 300 {
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
resp.Body.Close()
var wait time.Duration
if secs, err := strconv.Atoi(resp.Header.Get("Retry-After")); err == nil && secs > 0 {
wait = time.Duration(secs) * time.Second
}
return nil, &statusError{Code: resp.StatusCode, Path: path, Msg: strings.TrimSpace(string(msg)),
Reason: resp.Header.Get(cibridge.ReasonHeader), RetryAfter: wait}
}
return resp, nil
}
// pollFailPause is the wait after an ordinary failed poll.
const pollFailPause = 5 * time.Second
// heldPause is the wait after a held refusal that names no delay.
const heldPause = 5 * time.Minute
// pauseAfter says how long to wait before asking the bridge again, and
// whether the bridge refused for a takedown (cibridge.ReasonRepoHeld):
// then the delay it named, so a runner of a held repository neither
// spins nor outlasts the lift by much. Anything else is the ordinary
// pause.
func pauseAfter(err error) (time.Duration, bool) {
var se *statusError
if errors.As(err, &se) {
if se.Reason == cibridge.ReasonRepoHeld {
if se.RetryAfter > 0 {
return se.RetryAfter, true
}
return heldPause, true
}
if isTokenRefusal(se) {
return tokenPause, false
}
}
return pollFailPause, false
}
// tokenPause is the wait after the bridge refuses the worker token: only
// a person minting or fixing a token changes the answer.
const tokenPause = time.Minute
// isTokenRefusal says the bridge refused the token itself: a token of
// another repository (cibridge.ReasonOtherRepo) or one the engine does
// not accept (401).
func isTokenRefusal(se *statusError) bool {
return se.Reason == cibridge.ReasonOtherRepo || se.Code == http.StatusUnauthorized
}
// tokenRefusal words a token refusal for the log: which repository a
// token of another repository belongs to, or that the token was not
// accepted at all, which names nothing, as the bridge does not.
func (r *Runner) tokenRefusal(err error) (string, bool) {
var se *statusError
if !errors.As(err, &se) || !isTokenRefusal(se) {
return "", false
}
if se.Reason == cibridge.ReasonOtherRepo {
var body struct {
Repository string `json:"repository"`
}
if json.Unmarshal([]byte(se.Msg), &body) == nil && body.Repository != "" {
return fmt.Sprintf("this worker token belongs to repository %s, not %s", body.Repository, r.ownRepo()), true
}
}
return "the worker token was refused: revoked or not a worker token", true
}
// ownRepo is the repository the bridge address is for, "/r/<repo>/ci/bridge".
func (r *Runner) ownRepo() string {
_, rest, ok := strings.Cut(r.cfg.Bridge, "/r/")
if !ok {
return r.cfg.Bridge
}
repo, _, _ := strings.Cut(rest, "/ci/")
return repo
}
// noticeRefusal logs a refusal unless it is the one last said.
func (r *Runner) noticeRefusal(msg string) {
r.refusalMu.Lock()
same := r.refusal == msg
r.refusal = msg
r.refusalMu.Unlock()
if !same {
r.cfg.Logf("%s; asking again in %s", msg, tokenPause)
}
}
// clearRefusal forgets the last refusal after an accepted request, so a
// later one is said again.
func (r *Runner) clearRefusal() {
r.refusalMu.Lock()
r.refusal = ""
r.refusalMu.Unlock()
}
// statusError is the bridge answering a request with an error status.
// The code is what callers decide on: it is told apart from a transport
// failure or a 5xx by type, never by the message.
type statusError struct {
Code int
Path string
Msg string
// Reason is the bridge's own reason for refusing (cibridge.ReasonHeader),
// empty from a forge that sends none. RetryAfter is the delay the
// bridge asked for, zero when it named none.
Reason string
RetryAfter time.Duration
}
func (e *statusError) Error() string {
s := fmt.Sprintf("%s: %d %s", e.Path, e.Code, e.Msg)
switch e.Code {
case http.StatusConflict:
return fmt.Sprintf("%v: %s", errRunCancelled, s)
case http.StatusNotFound:
return fmt.Sprintf("%v: %s", errNoRoute, s)
}
return s
}
func (e *statusError) Is(target error) bool {
// dagnats#737: every resolve action on a task of a cancelled run,
// heartbeat included, answers 409.
return target == errRunCancelled && e.Code == http.StatusConflict ||
target == errNoRoute && e.Code == http.StatusNotFound
}
// finalRefusal reports whether the bridge answered that the task is
// gone or already resolved: asking again cannot change it, so a report
// of the task's end or log is dropped, not retried. A 404 is final only
// when the bridge says the task is gone: a forge that is starting
// answers 404 for every repository it has not yet served, and a report
// sent then must be retried. A 409 has always been final (the engine's
// answer for a cancelled run, dagnats#737), with or without a reason.
func finalRefusal(err error) bool {
var se *statusError
if !errors.As(err, &se) {
return false
}
switch se.Reason {
case cibridge.ReasonTaskGone, cibridge.ReasonTaskResolved:
return true
}
return se.Code == http.StatusConflict
}
// errNoRoute is a 404: a forge older than the route asked for, or a
// task the engine no longer has.
var errNoRoute = errors.New("not found")
// errNotHeld is the bridge saying (cibridge.ReasonNotHeld) this runner
// no longer holds a claimed task: its run was cancelled between the
// queue and the claim. It is not the job failing. A 403 without that
// reason (a wrong token, a refused delivery) stays a failure.
var errNotHeld = errors.New("this runner does not hold the task")
// notHeld says a refusal the bridge gave as "not held" is that.
func notHeld(err error) error {
var se *statusError
if errors.As(err, &se) && se.Code == http.StatusForbidden && se.Reason == cibridge.ReasonNotHeld {
return fmt.Errorf("%w: %v", errNotHeld, err)
}
return err
}
// errRunCancelled is the engine saying a task's run was cancelled
// (dagnats#737): the task is withdrawn, its work should stop, and
// nothing it would report is recorded.
var errRunCancelled = errors.New("run cancelled")
// logSink batches one stream of a task's output into the run's log: a
// flush every half second or 32 KB, whichever comes first, so a quiet
// step still shows progress and a loud one does not make a request per
// line. The engine names its two streams "out" and "err"; anything else
// is refused (measured: "stdout" was, so the first live run showed no
// output at all).
type logSink struct {
r *Runner
stream string
ctx context.Context
taskID string
mu sync.Mutex
buf bytes.Buffer
warned bool
dead atomic.Bool
done chan struct{}
wg sync.WaitGroup
}
func newLogSink(ctx context.Context, r *Runner, taskID, stream string) *logSink {
s := &logSink{r: r, ctx: ctx, taskID: taskID, stream: stream, done: make(chan struct{})}
s.wg.Add(1)
go func() {
defer s.wg.Done()
tick := time.NewTicker(500 * time.Millisecond)
defer tick.Stop()
for {
select {
case <-s.done:
s.flush()
return
case <-tick.C:
s.flush()
}
}
}()
return s
}
func (s *logSink) Write(p []byte) (int, error) {
s.mu.Lock()
s.buf.Write(p)
big := s.buf.Len() >= 32<<10
s.mu.Unlock()
if big {
s.flush()
}
return len(p), nil
}
func (s *logSink) flush() {
s.mu.Lock()
if s.buf.Len() == 0 {
s.mu.Unlock()
return
}
data := append([]byte(nil), s.buf.Bytes()...)
s.buf.Reset()
s.mu.Unlock()
body, _ := json.Marshal(map[string]any{"chunks": []map[string]string{{"stream": s.stream, "data": base64.StdEncoding.EncodeToString(data)}}})
if s.dead.Load() {
return
}
resp, err := s.r.post(s.ctx, "/v1/tasks/"+s.taskID+"/logs", body)
if err != nil {
if finalRefusal(err) {
// The engine no longer has the task: its run ended or was
// cancelled, or the engine restarted under a deploy (it holds
// claims in memory and redelivers an unfinished task). No
// later batch will be taken either.
s.dead.Store(true)
}
// A lost batch does not fail the task, but it must not be
// silent: a run page that shows no output while the runner
// printed plenty is the confident-nothing shape. Said once per
// task, so a dead log lane does not flood the runner's output.
s.mu.Lock()
first := !s.warned
s.warned = true
s.mu.Unlock()
if first {
if finalRefusal(err) {
s.r.cfg.Logf("logs for %s stop here: the engine no longer has the task (its run ended, or the engine restarted and will hand the task out again): %v", s.taskID, err)
} else {
s.r.cfg.Logf("logs for %s could not be sent (the run page will show none): %v", s.taskID, err)
}
}
return
}
resp.Body.Close()
}
func (s *logSink) Close() {
close(s.done)
s.wg.Wait()
}
func safeName(s string) string {
if s == "" {
return "_"
}
return strings.Map(func(c rune) rune {
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '-', c == '_', c == '.':
return c
}
return '-'
}, s)
}
// onBranch refuses a check-in that is not on the runner's branch, before
// anything of it is fetched or run. A runner with no branch runs all.
func (r *Runner) onBranch(ctx context.Context, taskID, checkin string) error {
if r.cfg.Branch == "" {
return nil
}
if IsPair(checkin) {
return fmt.Errorf("this runner runs %s check-ins only, and a landing's merge is not one", r.cfg.Branch)
}
// The forge's own record of the run names the branch through the
// bridge, for the task this runner holds; an older forge, or one with
// no answer, leaves the runner's own lookup.
info, berr := r.forgeInfo(ctx, taskID)
got := info.Branch
switch {
case errors.Is(berr, errNoRoute):
var err error
if got, err = r.cfg.BranchOf(ctx, checkin); err != nil {
return fmt.Errorf("which branch %.12s is on: %w", checkin, err)
}
case berr != nil:
return fmt.Errorf("which branch %.12s is on: %w", checkin, withoutURL(berr))
case got == "":
return fmt.Errorf("which branch %.12s is on: the forge could not say", checkin)
}
if got != r.cfg.Branch {
return fmt.Errorf("this runner runs %s check-ins only, and %.12s is on %q", r.cfg.Branch, checkin, got)
}
return nil
}