package runner
import (
"context"
"craftdesign.group/quarry/internal/cibridge"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/user"
"path/filepath"
"sync/atomic"
"testing"
"time"
)
func finalRefusalRunner(t *testing.T, url string, waits ...time.Duration) *Runner {
t.Helper()
r, err := New(Config{Bridge: url, Token: "WT", Tasks: []string{"t"}, MaxTasks: 1, WorkDir: t.TempDir(),
Checkout: appCheckout{}, ResolveRetry: waits, Logf: func(string, ...any) {}})
if err != nil {
t.Fatal(err)
}
return r
}
// A REPORT THE BRIDGE REFUSES FOR GOOD IS DROPPED AT ONCE.
//
// A task whose run was cancelled answers 404 or 409 to every report, so
// retrying it only delayed the next poll by minutes. The waits here are
// an hour each: a retry would hang the test, and the deadline says so.
// A transient answer (5xx, 429, a dropped connection) keeps retrying.
func TestAFinalRefusalOfAReportIsNotRetried(t *testing.T) {
for _, c := range []struct {
name string
answers []int // 0 closes the connection
reason string
want int32
}{
{"task gone", []int{404, 200}, cibridge.ReasonTaskGone, 1},
{"already resolved", []int{409, 200}, cibridge.ReasonTaskResolved, 1},
{"409 from an older forge", []int{409, 200}, "", 1},
{"404 from an older forge", []int{404, 200}, "", 2},
{"repository not served", []int{404, 200}, cibridge.ReasonRepoNotServed, 2},
{"unavailable then taken", []int{503, 200}, "", 2},
{"throttled then taken", []int{429, 200}, "", 2},
{"connection dropped then taken", []int{0, 200}, "", 2},
} {
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
n := int(calls.Add(1)) - 1
code := 200
if n < len(c.answers) {
code = c.answers[n]
}
if code == 0 {
conn, _, _ := w.(http.Hijacker).Hijack()
conn.Close()
return
}
if c.reason != "" {
w.Header().Set(cibridge.ReasonHeader, c.reason)
}
w.WriteHeader(code)
}))
wait := time.Hour
if c.want == 2 {
wait = time.Millisecond
}
r := finalRefusalRunner(t, srv.URL, wait, wait)
start := time.Now()
r.finish(context.Background(), "task-1", map[string]any{"action": "complete"})
srv.Close()
if got := calls.Load(); got != c.want {
t.Errorf("FAIL: %s: %d reports sent, want %d", c.name, got, c.want)
}
if took := time.Since(start); took > time.Second {
t.Errorf("FAIL: %s: took %s", c.name, took)
}
}
if !t.Failed() {
t.Log("ok")
}
}
// A LOG BATCH REFUSED FOR GOOD ENDS THE TASK'S LOG UPLOAD.
func TestALogUploadRefusedForGoodSendsNoMore(t *testing.T) {
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
calls.Add(1)
w.Header().Set(cibridge.ReasonHeader, cibridge.ReasonTaskResolved)
w.WriteHeader(http.StatusConflict)
}))
defer srv.Close()
r := finalRefusalRunner(t, srv.URL)
s := newLogSink(context.Background(), r, "task-1", "out")
for i := 0; i < 3; i++ {
s.Write([]byte("line\n"))
s.flush()
}
s.Close()
if got := calls.Load(); got != 1 {
t.Errorf("FAIL: %d log uploads sent after a 409, want 1", got)
}
if !t.Failed() {
t.Log("ok")
}
}
// A TASK WHOSE RUN WAS CANCELLED BEFORE THE CLAIM IS SKIPPED, NOT FAILED.
//
// The forge refuses the run token (403) of a task this runner does not
// hold. No job ran, so nothing is reported for the check-in, and the
// runner goes on to the next task.
func TestATaskCancelledBeforeItsClaimIsSkippedNotFailed(t *testing.T) {
for _, c := range []struct {
name, reason string
skipped bool
}{
{"not held", cibridge.ReasonNotHeld, true},
{"refused for another reason", "", false},
} {
t.Run(c.name, func(t *testing.T) { refusedToken(t, c.reason, c.skipped) })
}
}
func refusedToken(t *testing.T, reason string, skipped bool) {
u, err := user.Current()
if err != nil {
t.Fatal(err)
}
g, err := user.LookupGroupId(u.Gid)
if err != nil {
t.Skipf("no group name for gid %s: %v", u.Gid, err)
}
bin := t.TempDir()
fake := "#!/bin/sh\n[ \"$6\" = job-reclaim ] && exit 0\nshift 4\nexec env QUARRY_TEST_JOBENV=1 \"$@\"\n"
if err := os.WriteFile(filepath.Join(bin, "sudo"), []byte(fake), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
self, err := os.Executable()
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(Job{Steps: []JobStep{{Run: "true"}}})
with := func(id string) polled {
p := task(id, "c1")
p.Metadata = map[string]string{JobKey: string(raw)}
return p
}
b := &fakeBridge{queue: map[string][]polled{"t-job": {with("dead"), with("live")}},
resolved: map[string]map[string]any{}, logs: map[string]string{},
token: func(string) (map[string]string, string) { return nil, "" }}
inner := b.handler()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path == "/v1/tasks/dead/token" {
if reason != "" {
w.Header().Set(cibridge.ReasonHeader, reason)
}
w.WriteHeader(http.StatusForbidden)
return
}
inner.ServeHTTP(w, req)
}))
defer srv.Close()
r, err := New(Config{Bridge: srv.URL, Token: "WT", Tasks: []string{"t-job"}, MaxTasks: 1,
WorkDir: t.TempDir(), Checkout: appCheckout{}, Heartbeat: time.Hour, Logf: func(string, ...any) {},
JobUsers: []string{g.Name}, Self: self})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { r.Run(ctx); close(done) }()
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
b.mu.Lock()
_, ok := b.resolved["live"]
b.mu.Unlock()
if ok {
break
}
}
cancel()
<-done
b.mu.Lock()
defer b.mu.Unlock()
if got := b.resolved["live"]["action"]; got != "complete" {
t.Errorf("FAIL: the task after the dead one resolved %v, want complete", got)
}
if skipped {
for _, a := range b.actions {
if a == "dead fail" || a == "dead complete" {
t.Errorf("FAIL: the runner reported %q for a task it never held", a)
}
}
if _, ok := b.resolved["dead"]; ok {
t.Errorf("FAIL: the dead task has a recorded result %v", b.resolved["dead"])
}
} else if got := b.resolved["dead"]["action"]; got != "fail" {
t.Errorf("FAIL: a token refused with no reason resolved %v, want fail", got)
}
if !t.Failed() {
t.Log("ok")
}
}