Skip to content
quarry Artifact [ad235a77f8]
Login

Artifact [ad235a77f8]

Artifact ad235a77f80eaa5f223e426d8836625ffa0db187470b24d2e2e38248aee66fd6:


package runner

import (
	"context"
	"craftdesign.group/quarry/internal/cibridge"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"os/user"
	"path/filepath"
	"sync/atomic"
	"testing"
	"time"
)

func finalRefusalRunner(t *testing.T, url string, waits ...time.Duration) *Runner {
	t.Helper()
	r, err := New(Config{Bridge: url, Token: "WT", Tasks: []string{"t"}, MaxTasks: 1, WorkDir: t.TempDir(),
		Checkout: appCheckout{}, ResolveRetry: waits, Logf: func(string, ...any) {}})
	if err != nil {
		t.Fatal(err)
	}
	return r
}

// A REPORT THE BRIDGE REFUSES FOR GOOD IS DROPPED AT ONCE.
//
// A task whose run was cancelled answers 404 or 409 to every report, so
// retrying it only delayed the next poll by minutes. The waits here are
// an hour each: a retry would hang the test, and the deadline says so.
// A transient answer (5xx, 429, a dropped connection) keeps retrying.
func TestAFinalRefusalOfAReportIsNotRetried(t *testing.T) {
	for _, c := range []struct {
		name    string
		answers []int // 0 closes the connection
		reason  string
		want    int32
	}{
		{"task gone", []int{404, 200}, cibridge.ReasonTaskGone, 1},
		{"already resolved", []int{409, 200}, cibridge.ReasonTaskResolved, 1},
		{"409 from an older forge", []int{409, 200}, "", 1},
		{"404 from an older forge", []int{404, 200}, "", 2},
		{"repository not served", []int{404, 200}, cibridge.ReasonRepoNotServed, 2},
		{"unavailable then taken", []int{503, 200}, "", 2},
		{"throttled then taken", []int{429, 200}, "", 2},
		{"connection dropped then taken", []int{0, 200}, "", 2},
	} {
		var calls atomic.Int32
		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			n := int(calls.Add(1)) - 1
			code := 200
			if n < len(c.answers) {
				code = c.answers[n]
			}
			if code == 0 {
				conn, _, _ := w.(http.Hijacker).Hijack()
				conn.Close()
				return
			}
			if c.reason != "" {
				w.Header().Set(cibridge.ReasonHeader, c.reason)
			}
			w.WriteHeader(code)
		}))
		wait := time.Hour
		if c.want == 2 {
			wait = time.Millisecond
		}
		r := finalRefusalRunner(t, srv.URL, wait, wait)
		start := time.Now()
		r.finish(context.Background(), "task-1", map[string]any{"action": "complete"})
		srv.Close()
		if got := calls.Load(); got != c.want {
			t.Errorf("FAIL: %s: %d reports sent, want %d", c.name, got, c.want)
		}
		if took := time.Since(start); took > time.Second {
			t.Errorf("FAIL: %s: took %s", c.name, took)
		}
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A LOG BATCH REFUSED FOR GOOD ENDS THE TASK'S LOG UPLOAD.
func TestALogUploadRefusedForGoodSendsNoMore(t *testing.T) {
	var calls atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		calls.Add(1)
		w.Header().Set(cibridge.ReasonHeader, cibridge.ReasonTaskResolved)
		w.WriteHeader(http.StatusConflict)
	}))
	defer srv.Close()
	r := finalRefusalRunner(t, srv.URL)
	s := newLogSink(context.Background(), r, "task-1", "out")
	for i := 0; i < 3; i++ {
		s.Write([]byte("line\n"))
		s.flush()
	}
	s.Close()
	if got := calls.Load(); got != 1 {
		t.Errorf("FAIL: %d log uploads sent after a 409, want 1", got)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A TASK WHOSE RUN WAS CANCELLED BEFORE THE CLAIM IS SKIPPED, NOT FAILED.
//
// The forge refuses the run token (403) of a task this runner does not
// hold. No job ran, so nothing is reported for the check-in, and the
// runner goes on to the next task.
func TestATaskCancelledBeforeItsClaimIsSkippedNotFailed(t *testing.T) {
	for _, c := range []struct {
		name, reason string
		skipped      bool
	}{
		{"not held", cibridge.ReasonNotHeld, true},
		{"refused for another reason", "", false},
	} {
		t.Run(c.name, func(t *testing.T) { refusedToken(t, c.reason, c.skipped) })
	}
}

func refusedToken(t *testing.T, reason string, skipped bool) {
	u, err := user.Current()
	if err != nil {
		t.Fatal(err)
	}
	g, err := user.LookupGroupId(u.Gid)
	if err != nil {
		t.Skipf("no group name for gid %s: %v", u.Gid, err)
	}
	bin := t.TempDir()
	fake := "#!/bin/sh\n[ \"$6\" = job-reclaim ] && exit 0\nshift 4\nexec env QUARRY_TEST_JOBENV=1 \"$@\"\n"
	if err := os.WriteFile(filepath.Join(bin, "sudo"), []byte(fake), 0o755); err != nil {
		t.Fatal(err)
	}
	t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
	self, err := os.Executable()
	if err != nil {
		t.Fatal(err)
	}
	raw, _ := json.Marshal(Job{Steps: []JobStep{{Run: "true"}}})
	with := func(id string) polled {
		p := task(id, "c1")
		p.Metadata = map[string]string{JobKey: string(raw)}
		return p
	}
	b := &fakeBridge{queue: map[string][]polled{"t-job": {with("dead"), with("live")}},
		resolved: map[string]map[string]any{}, logs: map[string]string{},
		token: func(string) (map[string]string, string) { return nil, "" }}
	inner := b.handler()
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/v1/tasks/dead/token" {
			if reason != "" {
				w.Header().Set(cibridge.ReasonHeader, reason)
			}
			w.WriteHeader(http.StatusForbidden)
			return
		}
		inner.ServeHTTP(w, req)
	}))
	defer srv.Close()
	r, err := New(Config{Bridge: srv.URL, Token: "WT", Tasks: []string{"t-job"}, MaxTasks: 1,
		WorkDir: t.TempDir(), Checkout: appCheckout{}, Heartbeat: time.Hour, Logf: func(string, ...any) {},
		JobUsers: []string{g.Name}, Self: self})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() { r.Run(ctx); close(done) }()
	for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
		b.mu.Lock()
		_, ok := b.resolved["live"]
		b.mu.Unlock()
		if ok {
			break
		}
	}
	cancel()
	<-done
	b.mu.Lock()
	defer b.mu.Unlock()
	if got := b.resolved["live"]["action"]; got != "complete" {
		t.Errorf("FAIL: the task after the dead one resolved %v, want complete", got)
	}
	if skipped {
		for _, a := range b.actions {
			if a == "dead fail" || a == "dead complete" {
				t.Errorf("FAIL: the runner reported %q for a task it never held", a)
			}
		}
		if _, ok := b.resolved["dead"]; ok {
			t.Errorf("FAIL: the dead task has a recorded result %v", b.resolved["dead"])
		}
	} else if got := b.resolved["dead"]["action"]; got != "fail" {
		t.Errorf("FAIL: a token refused with no reason resolved %v, want fail", got)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}