package ci
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
)
// onceEngine registers and compiles anything (a ci.yml's on: is the
// JSON after a first line "# on: ", or null, RFC 0019's default), lists what it started as running with its
// labels so a supersede can be seen, and records, by "checkin
// pipeline", what was started, with which labels, and what cancelled.
type onceEngine struct {
*Engine
mu sync.Mutex
started []string
cancelled []string
labels map[string]map[string]string
specs []string // every ci.yml as the engine was handed it
runs []Run
what map[string]string
}
func newOnceEngine(t *testing.T) *onceEngine {
t.Helper()
o := &onceEngine{labels: map[string]map[string]string{}, what: map[string]string{}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
o.mu.Lock()
defer o.mu.Unlock()
switch {
case r.URL.Path == "/workflows":
w.WriteHeader(http.StatusCreated)
w.Write([]byte(`{"status":"registered"}`))
case r.URL.Path == "/v1/ci/compile" || r.URL.Path == "/v1/ci/validate":
var body struct{ Name, Spec string }
json.NewDecoder(r.Body).Decode(&body)
o.specs = append(o.specs, body.Spec)
on := "null"
if first, _, _ := strings.Cut(body.Spec, "\n"); strings.HasPrefix(first, "# on: ") {
on = strings.TrimPrefix(first, "# on: ")
}
fmt.Fprintf(w, `{"workflow":{"name":%q},"registered":true,"on":%s}`, body.Name, on)
case r.URL.Path == "/runs" && r.Method == http.MethodGet:
json.NewEncoder(w).Encode(o.runs)
case r.URL.Path == "/runs":
var body struct {
Input map[string]any `json:"input"`
Labels map[string]string `json:"labels"`
}
json.NewDecoder(r.Body).Decode(&body)
key := fmt.Sprint(body.Input["checkin"], " ", body.Input["pipeline"])
o.started = append(o.started, key)
o.labels[key] = body.Labels
id := fmt.Sprintf("r-%d", len(o.what)+1)
o.what[id] = key
o.runs = append(o.runs, Run{RunID: id, Status: "running", Labels: body.Labels})
w.WriteHeader(http.StatusCreated)
fmt.Fprintf(w, `{"run_id":%q}`, id)
case strings.HasPrefix(r.URL.Path, "/runs/") && strings.HasSuffix(r.URL.Path, "/cancel"):
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/runs/"), "/cancel")
o.cancelled = append(o.cancelled, o.what[id])
for i := range o.runs {
if o.runs[i].RunID == id {
o.runs[i].Status = "cancelled"
}
}
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
o.Engine = New(srv.URL)
return o
}
func (o *onceEngine) take(l *[]string) []string {
o.mu.Lock()
defer o.mu.Unlock()
out := append([]string(nil), (*l)...)
*l = nil
slices.Sort(out)
return out
}
func (o *onceEngine) Started() []string { return o.take(&o.started) }
func (o *onceEngine) Cancelled() []string { return o.take(&o.cancelled) }
func (o *onceEngine) Labels(key string) map[string]string {
o.mu.Lock()
defer o.mu.Unlock()
return o.labels[key]
}
// runIDOf is the engine's id for a started "checkin pipeline".
func (o *onceEngine) runIDOf(key string) string {
o.mu.Lock()
defer o.mu.Unlock()
for id, k := range o.what {
if k == key {
return id
}
}
return ""
}
const deployYAML = "# on: {\"push\":{}}\non:\n push:\njobs:\n deploy:\n environment: production\n steps:\n - run: ./deploy\n"
func onceRepo() *fakeRepo {
r := landingRepo()
r.files = append(r.files, "ci/deploy.yml")
r.blobs["ci/deploy.yml"] = []byte(deployYAML)
r.tips = map[string]string{"trunk": "t1", "release": "r1"}
r.targets = map[string]string{}
r.merges = map[string]string{}
r.parents = map[string][]string{}
r.same = map[string]bool{}
return r
}
func push(t *testing.T, e *onceEngine, st *fakeStore, repo *fakeRepo, cs ...Checkin) {
t.Helper()
repo.news = cs
if err := Trigger(e.Engine, st, "alpha", repo, "http://x/r/alpha"); err != nil {
t.Fatal(err)
}
}
// A PUSH TO A PROPOSED BRANCH TESTS ITS MERGE; ANY OTHER, ITS TIP.
//
// With a landing open, a push runs on its merge with the landing's
// target, and runs what the landing would as well as what the push
// would. With none open it runs on its bare tip, as it always did: a
// branch nobody proposed (a release line, say) is tested as itself. A
// workflow with an environment: job stays on the tip either way, and a
// push to the default branch is tested as itself.
func TestOnlyAProposedBranchPushTestsItsMerge(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := onceRepo()
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
if got, want := e.Started(), []string{"b1 ci/deploy.yml", "b1 ci/quarry.json"}; !slices.Equal(got, want) {
t.Errorf("FAIL: a push to an unproposed branch started %v; want %v", got, want)
}
repo.landings["feat"], repo.targets["feat"] = true, "release"
push(t, e, st, repo, Checkin{Hash: "b2", Branch: "feat"})
if got, want := e.Started(), []string{"b2 ci/deploy.yml", Pair("b2", "r1") + " ci/quarry.json", Pair("b2", "r1") + " ci/race.json"}; !slices.Equal(got, want) {
t.Errorf("FAIL: a push to a branch proposed into release started %v; want %v", got, want)
}
if l := e.Labels(Pair("b2", "r1") + " ci/quarry.json"); l["subject"] != "push-pair" {
t.Errorf("FAIL: a push's merge ran with subject %q; want push-pair, the push's own", l["subject"])
}
if l := e.Labels("b2 ci/deploy.yml"); l["subject"] != "" {
t.Errorf("FAIL: the environment workflow ran with subject %q; want a plain tip run", l["subject"])
}
push(t, e, st, repo, Checkin{Hash: "t2", Branch: "trunk"})
if got, want := e.Started(), []string{"t2 ci/deploy.yml", "t2 ci/quarry.json", "t2 ci/race.json"}; !slices.Equal(got, want) {
t.Errorf("FAIL: a push to the default branch started %v; want %v", got, want)
}
t.Log("ok: only a proposed branch's push tests its merge; the rest run on the tip")
}
// A PUSH WHOSE MERGE CONFLICTS IS TESTED AS ITS BARE TIP, AND SAYS SO.
// No verdict at all is the one outcome never allowed.
func TestAConflictingPushFallsBackToItsTip(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := onceRepo()
repo.landings["feat"] = true
repo.merges["b1:t1"] = "conflict"
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
if got, want := e.Started(), []string{"b1 ci/deploy.yml", "b1 ci/quarry.json"}; !slices.Equal(got, want) {
t.Fatalf("FAIL: a conflicting push started %v; want %v", got, want)
}
id := e.runIDOf("b1 ci/quarry.json")
if st.notes[id] == "" || !strings.Contains(st.notes[id], "f.txt") {
t.Errorf("FAIL: the fallback run carries note %q; want one naming the conflicting path", st.notes[id])
}
if n := st.notes[e.runIDOf("b1 ci/deploy.yml")]; n != "" {
t.Errorf("FAIL: the environment run, on its tip by design, carries note %q", n)
}
repo.merges["b2:t1"] = "unrelated"
push(t, e, st, repo, Checkin{Hash: "b2", Branch: "feat"})
if got := e.Started(); !slices.Contains(got, "b2 ci/quarry.json") || st.notes[e.runIDOf("b2 ci/quarry.json")] == "" {
t.Errorf("FAIL: an unrelated push started %v with note %q; want its tip, noted", got, st.notes[e.runIDOf("b2 ci/quarry.json")])
}
t.Log("ok: a merge that does not apply is tested as the tip, with the reason on the run")
}
// THE NEXT PUSH SUPERSEDES A BRANCH'S MERGE RUNS; a landing's merge of
// the same tip does not.
func TestTheNextPushSupersedesABranchMerge(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := onceRepo()
repo.landings["feat"] = true
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
e.Started()
if _, err := RunPairAt(e.Engine, st, "alpha", repo, "http://x/r/alpha", "feat", "b1", "t1"); err != nil {
t.Fatal(err)
}
if got := e.Cancelled(); len(got) != 0 {
t.Errorf("FAIL: a landing's merge of the same tip cancelled %v", got)
}
e.Started()
push(t, e, st, repo, Checkin{Hash: "b2", Branch: "feat"})
want := []string{"b1 ci/deploy.yml", Pair("b1", "t1") + " ci/quarry.json", Pair("b1", "t1") + " ci/race.json"}
if got := e.Cancelled(); !slices.Equal(got, want) {
t.Errorf("FAIL: the next push cancelled %v; want %v", got, want)
}
t.Log("ok: a push supersedes the branch's earlier runs, merge runs included")
}
// A LANDING'S MERGE REUSES WHAT A PUSH ALREADY TESTED OF IT.
//
// RunPairAt starts what the same pair has no verdict for, and nothing
// the push's merge already ran; a moved target is a new merge.
func TestALandingReusesThePushedMerge(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := onceRepo()
repo.landings["feat"] = true
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
e.Started()
if _, err := RunPairAt(e.Engine, st, "alpha", repo, "http://x/r/alpha", "feat", "b1", "t1"); err != nil {
t.Fatal(err)
}
if got := e.Started(); len(got) != 0 {
t.Errorf("FAIL: asking for the pushed merge again started %v", got)
}
if _, err := RunPairAt(e.Engine, st, "alpha", repo, "http://x/r/alpha", "feat", "b1", "t2"); err != nil {
t.Fatal(err)
}
if got, want := e.Started(), []string{Pair("b1", "t2") + " ci/quarry.json", Pair("b1", "t2") + " ci/race.json"}; !slices.Equal(got, want) {
t.Errorf("FAIL: a moved target started %v; want %v", got, want)
}
if l := e.Labels(Pair("b1", "t2") + " ci/quarry.json"); l["subject"] != "pair" {
t.Errorf("FAIL: a landing's merge ran with subject %q; want pair, the landing's own", l["subject"])
}
t.Log("ok: an unchanged pair is reused, a moved target runs whole")
}
// heirRepo is a default-branch merge m1 of b1 into t1 whose tree is
// the pair's, with deploy and an opted-out pipeline beside the tests.
func heirRepo() *fakeRepo {
r := onceRepo()
r.files = append(r.files, "ci/fresh.yml")
r.blobs["ci/fresh.yml"] = []byte("inherit: false\non:\n push:\njobs:\n test:\n steps:\n - run: make\n")
r.parents["m1"] = []string{"t1", "b1"}
r.same["m1 "+Pair("b1", "t1")] = true
return r
}
// THE DEFAULT BRANCH'S MERGE INHERITS THE VERDICT ITS PAIR REACHED.
//
// Only a workflow the pair ran to a pass or a fail, on an equal tree;
// never an environment workflow, never one that opted out, and never
// what the pair did not run. The inherited verdict is the check-in's
// own, recorded with the run it came from.
func TestTheDefaultBranchInheritsAnEqualPair(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := heirRepo()
pair := Pair("b1", "t1")
ran := func(id, p, status string) RunRecord {
return RunRecord{RunID: id, Pipeline: p, Status: status, Def: DefHash(repo.blobs[p])}
}
st.subjects = map[string][]RunRecord{pair: {
ran("p-race", "ci/race.json", "failed"),
ran("p-quarry", "ci/quarry.json", "completed"),
ran("p-deploy", "ci/deploy.yml", "completed"),
ran("p-fresh", "ci/fresh.yml", "completed"),
}}
push(t, e, st, repo, Checkin{Hash: "m1", Branch: "trunk"})
if got, want := e.Started(), []string{"m1 ci/deploy.yml", "m1 ci/fresh.yml"}; !slices.Equal(got, want) {
t.Errorf("FAIL: the merge started %v; want %v", got, want)
}
slices.Sort(st.inherited)
if want := []string{"m1 ci/quarry.json completed p-quarry", "m1 ci/race.json failed p-race"}; !slices.Equal(st.inherited, want) {
t.Errorf("FAIL: inherited %v; want %v", st.inherited, want)
}
for _, spec := range e.specs {
if strings.Contains(spec, "inherit") {
t.Errorf("FAIL: the engine was handed quarry's inherit: key:\n%s", spec)
}
}
t.Log("ok: pass and fail inherited from the equal pair; deploy and the opt-out ran")
}
// A DIFFERENT TREE, OR A PAIR WITH NO VERDICT, RUNS.
func TestTheDefaultBranchRunsWhatItCannotInherit(t *testing.T) {
for name, tc := range map[string]struct {
same bool
status string
def string // the definition the pair ran; "" is the merge's own
}{
"a different tree": {false, "completed", ""},
"a pair still going": {true, "running", ""},
"a pair in error": {true, "error", ""},
"a cancelled pair": {true, "cancelled", ""},
// The pair ran an older ci/quarry.json than the merge holds: a
// branch tip's, from before the target changed it.
"another definition": {true, "completed", `{"name":"quarry","version":"0.9","steps":[]}`},
"an unknown definition": {true, "completed", "-"},
} {
t.Run(name, func(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := heirRepo()
repo.same["m1 "+Pair("b1", "t1")] = tc.same
def := func(p string) string {
switch tc.def {
case "":
return DefHash(repo.blobs[p])
case "-":
return "" // a run recorded before definitions were
}
return DefHash([]byte(tc.def))
}
st.subjects = map[string][]RunRecord{Pair("b1", "t1"): {
{RunID: "p-quarry", Pipeline: "ci/quarry.json", Status: tc.status, Def: def("ci/quarry.json")},
{RunID: "p-race", Pipeline: "ci/race.json", Status: tc.status, Def: def("ci/race.json")},
}}
push(t, e, st, repo, Checkin{Hash: "m1", Branch: "trunk"})
if got := e.Started(); !slices.Contains(got, "m1 ci/quarry.json") || !slices.Contains(got, "m1 ci/race.json") {
t.Errorf("FAIL: started %v; want the tests run on m1", got)
}
if len(st.inherited) != 0 {
t.Errorf("FAIL: inherited %v", st.inherited)
}
})
}
}
// AN inherit: THAT IS NOT true OR false IS REFUSED, as quarry_when is:
// guessing either way would be the wrong half of the time.
func TestABadInheritIsRefused(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := heirRepo()
repo.blobs["ci/fresh.yml"] = []byte("inherit: sometimes\njobs: {}\n")
repo.blobs["ci/quarry.json"] = []byte(`{"name":"quarry","version":"1.0","inherit":"no","steps":[]}`)
push(t, e, st, repo, Checkin{Hash: "t2", Branch: "trunk"})
if got := e.Started(); slices.Contains(got, "t2 ci/fresh.yml") || slices.Contains(got, "t2 ci/quarry.json") {
t.Errorf("FAIL: started %v; want both malformed pipelines refused", got)
}
if len(st.refusals) != 2 {
t.Errorf("FAIL: refusals %v; want two", st.refusals)
}
diags, _, err := e.ValidateYAML("fresh", []byte("inherit: sometimes\njobs: {}\n"), "g")
if err != nil || len(diags) != 1 {
t.Errorf("FAIL: validating a bad inherit: answered %v, %v; want one diagnostic", diags, err)
}
t.Log("ok: a malformed inherit is refused at the trigger and by Validate")
}
// A WORKFLOW THAT NAMES ANY SECRET STAYS ON THE TIP AND NEVER INHERITS,
// as one with an environment: does. The forge refuses every secret to
// a merge, a repository's own included, so on a merge it could only
// fail; and a verdict reached without its secrets is no verdict on it.
func TestASecretWorkflowStaysOnTheTip(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := heirRepo()
repo.files = append(repo.files, "ci/publish.yml", "ci/sign.json")
repo.blobs["ci/publish.yml"] = []byte("jobs:\n publish:\n steps:\n - run: ./publish\n env:\n TOKEN: ${{ secrets.NPM_TOKEN }}\n")
repo.blobs["ci/sign.json"] = []byte(`{"name":"sign","version":"1.0","steps":[{"id":"s","metadata":{"ci.job":{"steps":[{"run":"sign"}],"secrets":["KEY"]}}}]}`)
repo.landings["feat"] = true
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
got := e.Started()
for _, want := range []string{"b1 ci/publish.yml", "b1 ci/sign.json", Pair("b1", "t1") + " ci/quarry.json"} {
if !slices.Contains(got, want) {
t.Errorf("FAIL: a push started %v; want %q among them", got, want)
}
}
st.subjects = map[string][]RunRecord{Pair("b1", "t1"): {
{RunID: "p-publish", Pipeline: "ci/publish.yml", Status: "completed", Def: DefHash(repo.blobs["ci/publish.yml"])},
{RunID: "p-sign", Pipeline: "ci/sign.json", Status: "completed", Def: DefHash(repo.blobs["ci/sign.json"])},
}}
push(t, e, st, repo, Checkin{Hash: "m1", Branch: "trunk"})
if got := e.Started(); !slices.Contains(got, "m1 ci/publish.yml") || !slices.Contains(got, "m1 ci/sign.json") {
t.Errorf("FAIL: the merge started %v; want both secret workflows run, not inherited", got)
}
for _, i := range st.inherited {
if strings.Contains(i, "publish") || strings.Contains(i, "sign") {
t.Errorf("FAIL: inherited %q", i)
}
}
t.Log("ok: secret workflows run on the tip and on the default branch, never on a merge")
}
// A MERGE RUNS THE PIPELINES THE MERGED TREE HOLDS.
//
// A pair tests what would land, so its definitions are the merge's,
// not the branch tip's: a pipeline the target added runs, and one the
// target changed runs as changed, and each run records the definition
// it ran. A pair that does not merge has no tree, and its runs read the
// tip's definitions, to fail on the conflict.
func TestAPairReadsItsDefinitionsFromTheMergedTree(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := onceRepo()
repo.landings["feat"] = true
merged := []byte(`{"name":"quarry","version":"2.0","steps":[]}`)
repo.pairDefs = map[string]map[string][]byte{Pair("b1", "t1"): {
"ci/quarry.json": merged,
"ci/added.json": []byte(`{"name":"added","version":"1.0","steps":[]}`),
"ci/deploy.yml": repo.blobs["ci/deploy.yml"],
}}
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
if got, want := e.Started(), []string{"b1 ci/deploy.yml", Pair("b1", "t1") + " ci/added.json", Pair("b1", "t1") + " ci/quarry.json"}; !slices.Equal(got, want) {
t.Errorf("FAIL: the merge started %v; want %v (the merged tree's pipelines)", got, want)
}
for _, r := range st.subjects[Pair("b1", "t1")] {
if r.Pipeline == "ci/quarry.json" && r.Def != DefHash(merged) {
t.Errorf("FAIL: the merge's ci/quarry.json run recorded definition %q; want the merged tree's", r.Def)
}
}
repo.merges["b1:t9"] = "conflict"
if _, err := RunPairAt(e.Engine, st, "alpha", repo, "http://x/r/alpha", "feat", "b1", "t9"); err != nil {
t.Fatal(err)
}
if got := e.Started(); !slices.Contains(got, Pair("b1", "t9")+" ci/quarry.json") {
t.Errorf("FAIL: a conflicting merge started %v; want the tip's pipelines, to fail on the conflict", got)
}
t.Log("ok: a merge runs the merged tree's pipelines and records what it ran")
}
// WHAT NEEDS SECRETS IS READ FROM THE PARSED WORKFLOW, NOT ITS TEXT.
//
// A job's environment: and any secrets reference count however the
// YAML spells them: quoted keys, flow mappings, an environment given
// as a mapping. Text that only looks like one (a comment, a line of a
// block script, an env var named ENVIRONMENT) does not.
func TestReviewedOnlyReadsTheParsedWorkflow(t *testing.T) {
for name, tc := range map[string]struct {
yml string
want bool
}{
"plain": {"jobs:\n d:\n environment: production\n steps:\n - run: x\n", true},
"quoted key": {"jobs:\n d:\n \"environment\": production\n steps:\n - run: x\n", true},
"flow mapping": {"jobs: {d: {environment: production, steps: [{run: x}]}}\n", true},
"as a mapping": {"jobs:\n d:\n environment:\n name: production\n steps:\n - run: x\n", true},
"flow secret": {"jobs: {d: {env: {T: \"${{ secrets.T }}\"}, steps: [{run: x}]}}\n", true},
"workflow env": {"env:\n T: ${{secrets.T}}\njobs:\n d:\n steps:\n - run: x\n", true},
"a comment": {"# environment: production\njobs:\n d:\n steps:\n - run: x\n", false},
"a script line": {"jobs:\n d:\n steps:\n - run: |\n environment: production\n", false},
"an env var": {"jobs:\n d:\n env:\n ENVIRONMENT: production\n steps:\n - run: x\n", false},
"a secrets word": {"jobs:\n d:\n steps:\n - run: echo secrets.txt\n", false},
"does not parse": {"jobs: [\n", true},
"two documents": {"jobs:\n d:\n steps:\n - run: x\n---\njobs:\n e:\n environment: production\n", true},
} {
if got := reviewedOnly([]byte(tc.yml), false); got != tc.want {
t.Errorf("FAIL: %s: reviewedOnly = %v, want %v", name, got, tc.want)
}
}
}
// A JSON PIPELINE THAT DOES NOT PARSE IS KEPT ON THE TIP, as a ci.yml
// that does not is: what it would need cannot be read, so it is never
// run on a merge nor inherited.
func TestReviewedOnlyKeepsBadJSONOnTheTip(t *testing.T) {
for name, tc := range map[string]struct {
json string
want bool
}{
"plain": {`{"name":"q","steps":[]}`, false},
"environment": {`{"name":"q","steps":[{"metadata":{"ci.job":{"environment":"prod"}}}]}`, true},
"does not parse": {`{"name":"q","steps":[`, true},
} {
if got := reviewedOnly([]byte(tc.json), true); got != tc.want {
t.Errorf("FAIL: %s: reviewedOnly = %v, want %v", name, got, tc.want)
}
}
}
// A LANDING'S MERGE AGAINST A MOVED TARGET SUPERSEDES THE PUSH'S MERGE
// of the same tip, whose target is stale now, as it does its own older
// merges.
func TestALandingsNewerMergeSupersedesThePushs(t *testing.T) {
e := newOnceEngine(t)
st := &fakeStore{mark: map[string]string{"alpha": "old"}}
repo := onceRepo()
repo.landings["feat"] = true
push(t, e, st, repo, Checkin{Hash: "b1", Branch: "feat"})
e.Started()
if _, err := RunPairAt(e.Engine, st, "alpha", repo, "http://x/r/alpha", "feat", "b1", "t2"); err != nil {
t.Fatal(err)
}
if got, want := e.Cancelled(), []string{Pair("b1", "t1") + " ci/quarry.json", Pair("b1", "t1") + " ci/race.json"}; !slices.Equal(got, want) {
t.Errorf("FAIL: the newer merge cancelled %v; want the push's stale merge runs %v", got, want)
}
}