Skip to content
quarry Artifact [066f32ffb1]
Login

Artifact [066f32ffb1]

Artifact 066f32ffb10f429cdc5a1b888125f973a9a1f05e485869e4da76efd94ac72766:


package api

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"

	"craftdesign.group/quarry/internal/oltp"
	"craftdesign.group/quarry/internal/ops"
)

// runnerRig is the pickup rig with the start route in front of it, an
// owner who answers for repositories "test" (public) and "hidden"
// (private: an anonymous caller does not see it), and a clock.
type runnerRig struct {
	pickupRig
	st    *SetupStarter
	attn  *fakeAttention
	clock time.Time
	lost  bool
	addr  int
	mail  mailRecorder
}

// mailRecorder counts the notify events a start raised, by workflow.
type mailRecorder struct {
	mu sync.Mutex
	by map[string]int
}

// count waits briefly for the deliveries, which ride goroutines, then
// reads one workflow's count.
func (m *mailRecorder) count(workflow string, want int) int {
	for i := 0; i < 100; i++ {
		m.mu.Lock()
		n := m.by[workflow]
		m.mu.Unlock()
		if n >= want {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	time.Sleep(50 * time.Millisecond)
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.by[workflow]
}

func (m *mailRecorder) Trigger(_ context.Context, e NotifyEvent) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.by == nil {
		m.by = map[string]int{}
	}
	m.by[e.Workflow]++
	return nil
}

func newRunnerRig(t *testing.T) *runnerRig {
	t.Helper()
	g := &runnerRig{attn: newFakeAttention(), clock: time.Now()}
	g.pickupRig = newPickupRigWith(t, func(d *Deps) {
		d.Owners = oneOwner("jva")
		d.Attention = g.attn
		d.Origin = "https://forge.test"
		d.Now = func() time.Time { return g.clock }
		d.Repos = NewRepos(map[string]*Repo{
			"test":   {Index: wikiIndex(t), Code: "code-test", Chat: "code-test"},
			"hidden": {Index: wikiIndex(t), Code: "code-hidden", Chat: "code-hidden"},
		})
		d.CanRead = func(ctx context.Context, repo string) bool { return repo != "hidden" || ops.User(ctx) != "" }
		d.CapsFor = func(who, repo string) CapSet {
			if g.lost {
				return oltp.CapSet("3")
			}
			return oltp.CapSet("a")
		}
	})
	g.deps.Notify = &g.mail
	d := g.deps
	d.SetupStarts = g.db
	g.st = NewSetupStarter(d, nil)
	g.st.now = func() time.Time { return g.clock }
	g.coll = NewPickupCollector(d, g.lim, nil)
	g.coll.now = func() time.Time { return g.clock }
	return g
}

func (g *runnerRig) start(repo, host, pickup, from string) (int, string, http.Header) {
	if from == "" {
		g.addr++
		from = fmt.Sprintf("198.51.100.%d", g.addr)
	}
	return g.startRaw(fmt.Sprintf(`{"kind":"runner","repo":%q,"host":%q,"pickup":%q}`, repo, host, pickup), from)
}

func (g *runnerRig) startRaw(body, from string) (int, string, http.Header) {
	req := httptest.NewRequest("POST", "/api/v1/setup/start", strings.NewReader(body))
	req.RemoteAddr = from + ":5000"
	w := httptest.NewRecorder()
	g.st.ServeHTTP(w, req)
	return w.Code, w.Body.String(), w.Header()
}

func (g *runnerRig) waiting(t *testing.T) []SignOffRequest {
	t.Helper()
	rows, err := g.db.RunnerSignOffsFor("jva", 200)
	if err != nil {
		t.Fatal(err)
	}
	var out []SignOffRequest
	for _, r := range rows {
		if r.State == "pending" {
			out = append(out, SignOffRequest(r))
		}
	}
	return out
}

func (g *runnerRig) begin(t *testing.T, repo, host string) (id, key, code string) {
	t.Helper()
	key, pickup := newPickupKey(t)
	status, body, _ := g.start(repo, host, pickup, "")
	var out setupStartOut
	if err := json.Unmarshal([]byte(body), &out); err != nil || status != 200 || out.Code == "" {
		t.Fatalf("FAIL: a runner start answered %d %s", status, body)
	}
	for _, r := range g.waiting(t) {
		if r.Pickup == pickup {
			return r.ID, key, out.Code
		}
	}
	t.Fatalf("FAIL: the start left no request waiting on its owner")
	return
}

func (g *runnerRig) collectRunner(key string) (int, string) {
	req := httptest.NewRequest("POST", collectPath, nil)
	req.Header.Set("Authorization", "Pickup "+key)
	req.Header.Set("Quarry-Pickup-Kind", "runner")
	req.RemoteAddr = "192.0.2.7:4000"
	w := httptest.NewRecorder()
	g.coll.ServeHTTP(w, req)
	return w.Code, w.Body.String()
}

func (g *runnerRig) listed(t *testing.T) []SignOffRequest {
	t.Helper()
	out, err := g.r.Invoke(asUser("jva"), "signoff.runner.list", signOffListIn{})
	if err != nil {
		t.Fatal(err)
	}
	var waiting []SignOffRequest
	for _, s := range out.(signOffRunnerListOut).Requests {
		if s.State == "pending" {
			waiting = append(waiting, s)
		}
	}
	return waiting
}

// A RUNNER START FOR A REPOSITORY AN ANONYMOUS CALLER SEES FILES ONE
// REQUEST ON ITS OWNER, IN THEIR INBOX, BOUND TO THE START'S PICKUP
// CODE, AND ANSWERS THE SHORT CODE THE INSTALLER SHOWS (RFC 0030, A
// runner without an install code).
func TestARunnerStartFilesOneRequestOnTheOwner(t *testing.T) {
	g := newRunnerRig(t)
	id, _, code := g.begin(t, "test", "build-box")
	if !shortCode.MatchString(code) {
		t.Errorf("FAIL: the short code %q is not XXXX-XXXX", code)
	}
	row, _, _ := g.db.SignOffByID(id)
	if row.Op != "ci.worker.token.new" || row.Repo != "test" || row.Owner != "jva" || !isRunnerAsker(row.Agent) || row.State != "pending" {
		t.Errorf("FAIL: the request is %+v", row)
	}
	if RunnerShortCode(row.Pickup) != code {
		t.Errorf("FAIL: the code answered is not the one the page derives from the pickup code")
	}
	var seen bool
	for _, e := range g.attn.sent["jva"] {
		if e.Kind == "signoff" && e.Repo == "test" && e.Link == "/me/requests/"+id && isRunnerAsker(e.From) {
			seen = true
		}
	}
	if !seen {
		t.Errorf("FAIL: no inbox event for the owner, linking the request")
	}
	if n := len(g.waiting(t)); n != 1 {
		t.Errorf("FAIL: %d requests waiting, want 1", n)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A START FOR A REPOSITORY THE CALLER CANNOT SEE ANSWERS EXACTLY AS ONE
// FOR A REPOSITORY THAT DOES NOT EXIST, AND FILES NOTHING.
func TestARunnerStartForAnUnseeableRepositoryAnswersLikeAMissingOne(t *testing.T) {
	g := newRunnerRig(t)
	_, pickup := newPickupKey(t)
	hs, hb, _ := g.start("hidden", "build-box", pickup, "")
	ms, mb, _ := g.start("no-such-repo", "build-box", pickup, "")
	if hs != ms || hb != mb || hs != 404 {
		t.Errorf("FAIL: hidden answered %d %q, missing %d %q", hs, hb, ms, mb)
	}
	if n := len(g.waiting(t)); n != 0 {
		t.Errorf("FAIL: %d requests were filed for a repository the caller cannot see", n)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// AT MOST THREE RUNNER REQUESTS WAIT FROM ONE ADDRESS; NO CAP IS PER
// REPOSITORY, SO A STRANGER CANNOT LOCK THE OWNER'S OWN INSTALL OUT, EVEN
// BY CLAIMING THE OWNER'S HOST NAME; A LAPSED ONE DOES NOT COUNT.
func TestAnAddressHoldsAtMostThreeOpenRunnerStartsAndARepositoryHoldsNoCap(t *testing.T) {
	g := newRunnerRig(t)
	for i := 0; i < 3; i++ {
		_, pickup := newPickupKey(t)
		if st, _, _ := g.start("test", "owners-laptop", pickup, "203.0.113.50"); st != 200 {
			t.Fatalf("FAIL: start %d from one address answered %d", i, st)
		}
		g.st.rates = pickupRates{}
	}
	_, pickup := newPickupKey(t)
	st, _, h := g.start("test", "owners-laptop", pickup, "203.0.113.50")
	if st != 429 || h.Get("Retry-After") == "" {
		t.Errorf("FAIL: the fourth start from one address answered %d, want 429 with Retry-After", st)
	}
	for i := 0; i < 6; i++ {
		g.begin(t, "test", "owners-laptop")
	}
	g.clock = g.clock.Add(11 * time.Minute)
	g.st.rates = pickupRates{}
	if st, _, _ := g.start("test", "box-6", pickup, "203.0.113.50"); st != 200 {
		t.Errorf("FAIL: after the three lapsed, a start answered %d, want 200", st)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// THE HOST IS THE INSTALLER'S OWN WORD: THE SUMMARY SAYS SO, AND
// RunnerHost READS IT BACK.
func TestTheRunnerSummaryCarriesTheReportedHost(t *testing.T) {
	if got := RunnerHost(runnerSummary("build-box", "suit")); got != "build-box" {
		t.Errorf("FAIL: host read back as %q", got)
	}
	if RunnerHost("Mint a worker token for suit") != "" {
		t.Errorf("FAIL: a summary that is not a runner's gave a host")
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// THE START IS SHAPED: a repository, a host and a pickup code, and
// nothing else; the per-address budget applies.
func TestARunnerStartIsHeldToItsShapeAndTheAddressBudget(t *testing.T) {
	g := newRunnerRig(t)
	_, pickup := newPickupKey(t)
	for name, body := range map[string]string{
		"no repo":     fmt.Sprintf(`{"kind":"runner","host":"h","pickup":%q}`, pickup),
		"no host":     fmt.Sprintf(`{"kind":"runner","repo":"test","pickup":%q}`, pickup),
		"spaced host": fmt.Sprintf(`{"kind":"runner","repo":"test","host":"a b","pickup":%q}`, pickup),
		"bad pickup":  `{"kind":"runner","repo":"test","host":"h","pickup":"nope"}`,
		"has a name":  fmt.Sprintf(`{"kind":"runner","name":"x","repo":"test","host":"h","pickup":%q}`, pickup),
		"push + repo": fmt.Sprintf(`{"kind":"push","name":"x","repo":"test","pickup":%q}`, pickup),
		"read + host": fmt.Sprintf(`{"kind":"read","name":"x","host":"h","pickup":%q}`, pickup),
	} {
		g.addr++
		if st, _, _ := g.startRaw(body, fmt.Sprintf("203.0.113.%d", 100+g.addr)); st != 400 {
			t.Errorf("FAIL: %s answered %d, want 400", name, st)
		}
	}
	if n := len(g.waiting(t)); n != 0 {
		t.Errorf("FAIL: a malformed start filed %d requests", n)
	}
	// Every call spends its address's budget of five a minute.
	var last int
	for i := 0; i < 6; i++ {
		last, _, _ = g.startRaw(`{"kind":"runner"}`, "203.0.113.2")
	}
	if last != 429 {
		t.Errorf("FAIL: the sixth call from one address answered %d, want 429", last)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// APPROVING IS ONE ACT: IT MINTS THE WORKER TOKEN AS THE APPROVER, THE
// KEY HOLDER COLLECTS IT AS KIND RUNNER, AND THE SETTINGS LOG SAYS WHO
// APPROVED.
func TestApprovingARunnerRequestMintsAndTheInstallerCollects(t *testing.T) {
	g := newRunnerRig(t)
	id, key, _ := g.begin(t, "test", "build-box")
	if st, _ := g.collectRunner(key); st != 202 {
		t.Errorf("FAIL: before approval the collect answered %d, want 202", st)
	}
	g.decide(t, id, true)
	if g.eng.mints != 1 {
		t.Fatalf("FAIL: %d tokens minted, want 1", g.eng.mints)
	}
	st, body := g.collectRunner(key)
	var ans struct {
		Op     string `json:"op"`
		Repo   string `json:"repo"`
		Secret struct {
			Value string `json:"value"`
		} `json:"secret"`
		Result struct {
			Token struct {
				ID string `json:"id"`
			} `json:"token"`
		} `json:"result"`
	}
	_ = json.Unmarshal([]byte(body), &ans)
	if st != 200 || ans.Op != "ci.worker.token.new" || ans.Repo != "test" || ans.Secret.Value == "" || ans.Result.Token.ID == "" {
		t.Errorf("FAIL: collect answered %d with op %q repo %q", st, ans.Op, ans.Repo)
	}
	if st, _ := g.collectRunner(key); st != 410 {
		t.Errorf("FAIL: a second collect answered %d, want 410", st)
	}
	var logged bool
	for _, r := range g.rec.rows {
		if r.field == "worker-token" && strings.Contains(r.now, "by runner request, approved by jva") && strings.Contains(r.now, ans.Result.Token.ID) && strings.Contains(r.now, "build-box") {
			logged = true
		}
		if strings.Contains(r.now, "s3cret-bearer-value") || strings.Contains(r.was, "s3cret-bearer-value") {
			t.Errorf("FAIL: the settings log carries the token value")
		}
	}
	if !logged {
		t.Errorf("FAIL: the settings log has no row naming the request and its approver: %+v", g.rec.rows)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// DENYING, A LAPSE AND AN APPROVER WHO HAS LOST THE LETTER EACH LEAVE
// NO TOKEN; THE INSTALLER IS TOLD THE STATE.
func TestADeniedLapsedOrUnauthorisedRunnerRequestMintsNothing(t *testing.T) {
	g := newRunnerRig(t)
	id, key, _ := g.begin(t, "test", "box-a")
	g.decide(t, id, false)
	if st, body := g.collectRunner(key); st != 410 || !strings.Contains(body, `"denied"`) {
		t.Errorf("FAIL: after a denial collect answered %d %s", st, body)
	}

	id2, key2, _ := g.begin(t, "test", "box-b")
	g.clock = g.clock.Add(11 * time.Minute)
	if st, body := g.collectRunner(key2); st != 410 || !strings.Contains(body, `"expired"`) {
		t.Errorf("FAIL: a lapsed request collected as %d %s", st, body)
	}
	if _, err := g.r.Invoke(asUser("jva"), "signoff.decide", signOffDecideIn{ID: id2, Approve: true}); err == nil {
		t.Errorf("FAIL: a request past its ten minutes was approved")
	}
	if rows := g.listed(t); len(rows) != 0 {
		t.Errorf("FAIL: %d lapsed requests are still listed as waiting", len(rows))
	}

	id3, _, _ := g.begin(t, "test", "box-c")
	g.lost = true
	if _, err := g.r.Invoke(asUser("jva"), "signoff.decide", signOffDecideIn{ID: id3, Approve: true}); err == nil {
		t.Errorf("FAIL: an approver without the admin letter minted a token")
	}
	if g.eng.mints != 0 {
		t.Errorf("FAIL: %d tokens were minted", g.eng.mints)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// RUNNER REQUESTS NEVER DISPLACE OTHER REQUESTS: 300 OPEN ONES AND ONE
// REAL REQUEST LIST THE REAL ONE ALONE, AND THE RUNNER ONES COUNT APART.
func TestRunnerRequestsAreListedAndCountedApartFromRealOnes(t *testing.T) {
	g := newRunnerRig(t)
	for i := 0; i < 300; i++ {
		if _, err := g.db.RequestSignOff(oltpRunner(fmt.Sprintf("10.%d.%d.1", i/250, i%250), i)); err != nil {
			t.Fatal(err)
		}
	}
	real, err := g.db.RequestSignOff(oltpReal())
	if err != nil {
		t.Fatal(err)
	}
	out, err := g.r.Invoke(asUser("jva"), "signoff.list", signOffListIn{})
	if err != nil {
		t.Fatal(err)
	}
	got := out.(signOffListOut).Requests
	if len(got) != 1 || got[0].ID != real.ID || got[0].State != "pending" {
		t.Errorf("FAIL: signoff.list holds %d rows, want the real request alone", len(got))
	}
	ro, err := g.r.Invoke(asUser("jva"), "signoff.runner.list", signOffListIn{})
	if err != nil {
		t.Fatal(err)
	}
	if n := ro.(signOffRunnerListOut).Count; n != 200 && n != 300 {
		t.Errorf("FAIL: the runner section counts %d", n)
	}
	for _, s := range ro.(signOffRunnerListOut).Requests {
		if !isRunnerAsker(s.Agent) {
			t.Errorf("FAIL: the runner list holds a request of %s", s.Agent)
		}
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// AN OWNER HOLDS AT MOST TEN OPEN RUNNER REQUESTS ACROSS THEIR
// REPOSITORIES; THE ELEVENTH START IS THE SAME 429 AS THE ADDRESS CAP.
func TestAnOwnerHoldsAtMostTenOpenRunnerRequests(t *testing.T) {
	g := newRunnerRig(t)
	for i := 0; i < 10; i++ {
		g.begin(t, "test", fmt.Sprintf("box-%d", i))
	}
	_, pickup := newPickupKey(t)
	st, _, h := g.start("test", "box-11", pickup, "")
	if st != 429 || h.Get("Retry-After") == "" {
		t.Errorf("FAIL: the eleventh open request answered %d, want 429 with Retry-After", st)
	}
	if n := len(g.waiting(t)); n != 10 {
		t.Errorf("FAIL: %d waiting, want 10", n)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// ONE EMAIL PER REPOSITORY PER TEN MINUTES, HOWEVER MANY REQUESTS COME;
// EVERY REQUEST STILL GETS ITS INBOX ITEM.
func TestRunnerRequestsSendOneEmailPerRepositoryPerTenMinutes(t *testing.T) {
	g := newRunnerRig(t)
	for i := 0; i < 5; i++ {
		g.begin(t, "test", fmt.Sprintf("box-%d", i))
	}
	if n := g.mail.count("quarry-signoff-email", 1); n != 1 {
		t.Errorf("FAIL: %d emails for 5 requests, want 1", n)
	}
	if n := g.mail.count("quarry-signoff", 5); n != 5 {
		t.Errorf("FAIL: %d inbox notifications, want 5", n)
	}
	inbox := 0
	for _, e := range g.attn.sent["jva"] {
		if e.Kind == "signoff" {
			inbox++
		}
	}
	if inbox != 5 {
		t.Errorf("FAIL: %d attention events, want 5", inbox)
	}
	g.clock = g.clock.Add(11 * time.Minute)
	g.begin(t, "test", "box-late")
	if n := g.mail.count("quarry-signoff-email", 2); n != 2 {
		t.Errorf("FAIL: %d emails after the window passed, want 2", n)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

func oltpRunner(addr string, i int) oltpSignOff {
	return oltpSignOff{Agent: runnerAsker(addr), Owner: "jva", Op: "ci.worker.token.new", Repo: "test",
		Input: fmt.Sprintf(`{"repo":"test","label":"h%d"}`, i), Summary: "s", Pickup: fmt.Sprintf("%064x", i+1)}
}

func oltpReal() oltpSignOff {
	return oltpSignOff{Agent: "jva-bot", Owner: "jva", Op: "repo.delete", Repo: "test", Input: "{}", Summary: "real"}
}

type oltpSignOff = oltp.SignOff