Skip to content
quarry Artifact [2ecbc182aa]
Login

Artifact [2ecbc182aa]

Artifact 2ecbc182aacd8a6895cb9207ca860e840b258d7be6a28840c5a186f6442d9506:


package api

import (
	"context"
	"errors"
	"testing"
	"time"
)

// alertMirrors counts failures and answers the streak, like the store.
type alertMirrors struct {
	memMirrors
	streak int
}

func (a *alertMirrors) RecordMirrorRun(_ string, _ time.Duration, _ bool, err error) (int, error) {
	if err != nil {
		a.streak++
	} else {
		a.streak = 0
	}
	return a.streak, nil
}

// An admin is told at the third failure, at the tenth, and when a
// streak that was told about ends — not on every failure, and not on
// a first blip. 29 silent failures were how the wapp outage went
// unnoticed; a ticket sat 14 days after recovery because nothing said
// so (a06c9ad3, 82eb641a).
func TestAFailingMirrorTellsAnAdminAtChosenStreaksAndOnRecovery(t *testing.T) {
	fn := &fakeNotify{}
	store := &alertMirrors{}
	fail := errors.New("dial tcp: no such host")
	d := Deps{Notify: fn, Mirrors: store, Repos: NewRepos(map[string]*Repo{"w": {
		Mirror: func(string) (bool, error) { return false, fail },
	}})}
	m := Mirror{Repo: "w", URL: "https://example.invalid/w"}
	for i := 1; i <= 12; i++ {
		m.Failures = i - 1 // what the scheduler would read before this attempt
		d.PullMirror(m)
	}
	evs := waitNotify(fn, 2)
	if len(evs) != 2 {
		t.Fatalf("FAIL: %d alerts over 12 failures, want 2 (at 3 and 10): %+v", len(evs), evs)
	}
	// fireNotify is asynchronous, so the two may arrive in either order.
	got := map[string]bool{evs[0].TxnID: true, evs[1].TxnID: true}
	if !got["mirror:w:failing:3"] || !got["mirror:w:failing:10"] {
		t.Fatalf("FAIL: alerts %s, %s", evs[0].TxnID, evs[1].TxnID)
	}
	t.Logf("ok: told at 3 and 10 of 12 failures")

	// Then it comes back.
	d.Repos = NewRepos(map[string]*Repo{"w": {Chat: "w", Mirror: func(string) (bool, error) { return true, nil }}})
	m.Failures, m.LastAt = 12, "2026-09-22T00:00:00Z"
	d.PullMirror(m)
	evs = waitNotify(fn, 3)
	recovered := false
	for _, e := range evs {
		recovered = recovered || e.TxnID == "mirror:w:recovered:2026-09-22T00:00:00Z"
	}
	if len(evs) != 3 || !recovered {
		t.Fatalf("FAIL: recovery not told: %+v", evs)
	}
	t.Log("ok: recovery told once")

	// A single blip that recovers before the third failure says nothing.
	fn2 := &fakeNotify{}
	d2 := Deps{Notify: fn2, Mirrors: &alertMirrors{}, Repos: NewRepos(map[string]*Repo{"w": {Chat: "w", Mirror: func(string) (bool, error) { return false, fail }}})}
	d2.PullMirror(Mirror{Repo: "w"})
	d2.Repos = NewRepos(map[string]*Repo{"w": {Chat: "w", Mirror: func(string) (bool, error) { return true, nil }}})
	d2.PullMirror(Mirror{Repo: "w", Failures: 1})
	if got := waitNotify(fn2, 0); len(got) != 0 {
		t.Fatalf("FAIL: a one-failure blip alerted: %+v", got)
	}
	t.Log("ok: a blip is silent")
	_ = context.Background()
}