Skip to content
quarry Artifact [65be4109e3]
Login

Artifact [65be4109e3]

Artifact 65be4109e3e8eb3200f4f29b894741c1cdb890dd8dc2482e1a22d14487fe75d7:


package chat

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"craftdesign.group/quarry/internal/events"
)

// pausePeer is another Fossil server as the bridge meets it when it
// is unhappy, counting every request it is asked. mode picks the
// answer: what Fossil sends for a refused login (login.c: 401, no
// cookie), for an account that signs in but has no chat (chat.c: a 200
// JSON "Missing permissions or not logged in"), or what a front-end
// limiter sends (429 or 503 with a Retry-After), or a plain failure.
type pausePeer struct {
	srv  *httptest.Server
	hits atomic.Int64
	mu   sync.Mutex
	mode string
	// okFor makes the next n poll answers healthy before mode applies
	// again, for the "a success clears the count" case.
	okFor int
	// status is peerBlocked's answer; sent is what /chat-send took and
	// sendHits how many times it was asked.
	status   int
	flips    int
	sent     []string
	sendHits atomic.Int64
}

const (
	peerRefuseLogin = "refuse-login"
	peerNoChat      = "no-chat"
	peerThrottle    = "throttle"
	peerDown        = "down"
	peerOK          = "ok"
	// peerBlocked answers every path with status (0 is a 200 HTML page),
	// no wait named: a front end blocking the forge's address.
	peerBlocked = "blocked"
)

func newPausePeer(t *testing.T, mode string) *pausePeer {
	t.Helper()
	p := &pausePeer{mode: mode}
	notChat := `{"msgs":[{"isError":true,"xfrom":null,` +
		`"xmsg":"Missing permissions or not logged in. Try <a href='/login'>logging in</a>."}]}`
	mux := http.NewServeMux()
	blocked := func(w http.ResponseWriter, r *http.Request) {
		p.mu.Lock()
		st := p.status
		p.mu.Unlock()
		switch st {
		case 0:
			fmt.Fprint(w, "<html>Access denied</html>")
		case 302:
			http.Redirect(w, r, "/blocked", http.StatusFound)
		default:
			http.Error(w, "blocked", st)
		}
	}
	mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
		p.hits.Add(1)
		switch p.get() {
		case peerBlocked:
			blocked(w, r)
		case peerRefuseLogin:
			http.Error(w, "unknown user or incorrect password", http.StatusUnauthorized)
		case peerThrottle:
			w.Header().Set("Retry-After", "3600")
			http.Error(w, "blocked", http.StatusServiceUnavailable)
		case peerDown:
			http.Error(w, "boom", http.StatusInternalServerError)
		default:
			http.SetCookie(w, &http.Cookie{Name: "fossil-0123456789abcdef", Value: "tok", Path: "/"})
			http.Redirect(w, r, "/chat", http.StatusFound)
		}
	})
	serve := func(w http.ResponseWriter, r *http.Request) {
		p.hits.Add(1)
		if r.URL.Path == "/chat-send" {
			p.sendHits.Add(1)
		}
		if p.get() == peerBlocked {
			blocked(w, r)
			return
		}
		if p.get() == peerOK && r.URL.Path == "/chat-send" {
			_ = r.ParseForm()
			p.mu.Lock()
			p.sent = append(p.sent, r.FormValue("msg"))
			p.mu.Unlock()
			return
		}
		p.mu.Lock()
		ok := p.okFor > 0
		if ok {
			p.okFor--
		}
		p.mu.Unlock()
		if ok {
			fmt.Fprint(w, `{"msgs":[]}`)
			return
		}
		switch p.get() {
		case peerThrottle:
			w.Header().Set("Retry-After", "3600")
			http.Error(w, "blocked", http.StatusServiceUnavailable)
		case peerDown:
			http.Error(w, "boom", http.StatusInternalServerError)
		case peerRefuseLogin:
			http.Error(w, "no", http.StatusUnauthorized)
		case "flip":
			p.mu.Lock()
			p.flips++
			even := p.flips%2 == 0
			p.mu.Unlock()
			if even {
				fmt.Fprint(w, `{"msgs":[]}`)
			} else {
				fmt.Fprint(w, notChat)
			}
		case peerOK:
			fmt.Fprint(w, `{"msgs":[]}`)
		default:
			fmt.Fprint(w, notChat)
		}
	}
	mux.HandleFunc("/chat-poll", serve)
	mux.HandleFunc("/chat-send", serve)
	mux.HandleFunc("/json/config/chat-timeline-user", func(w http.ResponseWriter, r *http.Request) {
		p.hits.Add(1)
		if p.get() == peerBlocked {
			blocked(w, r)
			return
		}
		http.NotFound(w, r)
	})
	p.srv = httptest.NewServer(mux)
	t.Cleanup(p.srv.Close)
	return p
}

func (p *pausePeer) get() string  { p.mu.Lock(); defer p.mu.Unlock(); return p.mode }
func (p *pausePeer) set(m string) { p.mu.Lock(); p.mode = m; p.mu.Unlock() }

// simClock is a clock the test advances by the waits the bridge asks
// for: a wait returns at once and moves time forward by what was
// asked, so a day passes in no time. After maxWaits the wait blocks
// until the bridge's context ends, which ends a loop that would
// otherwise spin for ever.
type simClock struct {
	mu       sync.Mutex
	t        time.Time
	waits    []time.Duration
	maxWaits int
}

func (c *simClock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }

func (c *simClock) advance(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }

func (c *simClock) sleep(ctx context.Context, d time.Duration) bool {
	c.mu.Lock()
	c.waits = append(c.waits, d)
	c.t = c.t.Add(d)
	spent := len(c.waits) >= c.maxWaits
	c.mu.Unlock()
	if spent {
		<-ctx.Done()
		return false
	}
	return ctx.Err() == nil
}

func (c *simClock) asked() []time.Duration {
	c.mu.Lock()
	defer c.mu.Unlock()
	return append([]time.Duration(nil), c.waits...)
}

// pauseFixture is a bridge against a pausePeer on a simulated clock.
func pauseFixture(t *testing.T, mode string, maxWaits int) (*Room, *pausePeer, *memBridgeStore, *Bridge, *simClock, *int32) {
	t.Helper()
	ev, err := events.Start(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(ev.Close)
	r := room(t, ev)
	peer := newPausePeer(t, mode)
	store := newMemStore()
	b := NewBridge(r, store, BridgeConfig{
		Repo: "alpha", Code: "codeA", URL: peer.srv.URL, User: "bot", Secret: "pw", AllowPrivate: true,
	})
	b.Client = bridgeClient(5*time.Second, true)
	b.SendClient = bridgeClient(5*time.Second, true)
	clk := &simClock{t: time.Date(2026, 11, 16, 9, 0, 0, 0, time.UTC), maxWaits: maxWaits}
	b.now, b.sleep = clk.now, clk.sleep
	var told int32
	b.Stopped = func(repo, why string) {
		if repo != "alpha" || why == "" {
			t.Errorf("FAIL: Stopped heard (%q, %q), want the repository and a reason", repo, why)
		}
		atomic.AddInt32(&told, 1)
	}
	return r, peer, store, b, clk, &told
}

// runDone runs the bridge and closes the channel when Run returns.
func runDone(b *Bridge, ctx context.Context) <-chan struct{} {
	done := make(chan struct{})
	go func() { defer close(done); b.Run(ctx) }()
	return done
}

func finished(t *testing.T, done <-chan struct{}, what string) {
	t.Helper()
	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatalf("FAIL: %s", what)
	}
}

// A login the other server keeps refusing stops the bridge after the
// stated number of refusals, whichever way the refusal is spelled; the
// stop is stored with its reason and time, the admins are told exactly
// once, and over a simulated day the peer is asked nothing more, in
// either direction.
func TestBridgeStopsAfterRefusedLogin(t *testing.T) {
	for _, mode := range []string{peerRefuseLogin, peerNoChat} {
		t.Run(mode, func(t *testing.T) {
			r, peer, store, b, clk, told := pauseFixture(t, mode, 1000)
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			done := runDone(b, ctx)
			finished(t, done, "the bridge did not stop itself after repeated refusals")

			store.mu.Lock()
			stops, why, at := store.stops, store.stoppedWhy, store.stoppedAt
			store.mu.Unlock()
			if stops != 1 || why == "" || at.IsZero() {
				t.Errorf("FAIL: stored stop = (%d stops, %q, %v), want one stop with a reason and time", stops, why, at)
			}
			if got := atomic.LoadInt32(told); got != 1 {
				t.Errorf("FAIL: admins told %d times, want exactly once", got)
			}

			// A day later, and a message posted meanwhile: nothing reaches the peer.
			seen := peer.hits.Load()
			clk.advance(24 * time.Hour)
			if _, err := r.Post("codeA", Message{Repo: "alpha", Kind: KindMsg, User: "ann", Text: "anyone there"}); err != nil {
				t.Fatal(err)
			}
			clk.advance(24 * time.Hour)
			if got := peer.hits.Load(); got != seen {
				t.Errorf("FAIL: the stopped bridge made %d further requests to the peer", got-seen)
			}
			if !t.Failed() {
				t.Log("ok")
			}
		})
	}
}

// Network errors, timeouts and 5xx back off up to the cap and retry for
// ever: they are the upstream failing, not refusing, and never stop it.
func TestBridgeNetworkErrorsNeverStop(t *testing.T) {
	_, peer, store, b, clk, told := pauseFixture(t, peerDown, 12)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	done := runDone(b, ctx)
	waitFor(t, "twelve back-offs", func() bool { return len(clk.asked()) >= 12 })

	store.mu.Lock()
	stops := store.stops
	store.mu.Unlock()
	if stops != 0 || atomic.LoadInt32(told) != 0 {
		t.Errorf("FAIL: server errors stopped the bridge (%d stops, %d told)", stops, atomic.LoadInt32(told))
	}
	var longest time.Duration
	for _, w := range clk.asked() {
		longest = max(longest, w)
	}
	if longest != backoffMax {
		t.Errorf("FAIL: longest back-off %v, want the cap %v", longest, backoffMax)
	}
	select {
	case <-done:
		t.Errorf("FAIL: Run returned while the peer was only failing")
	default:
	}
	if peer.hits.Load() < 12 {
		t.Errorf("FAIL: the peer saw %d requests, want a retry per back-off", peer.hits.Load())
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A throttle answer's wait is honoured and is not a refusal, however
// many arrive.
func TestBridgeThrottleHonoursRetryAfter(t *testing.T) {
	_, _, store, b, clk, told := pauseFixture(t, peerThrottle, 10)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	runDone(b, ctx)
	waitFor(t, "ten waits", func() bool { return len(clk.asked()) >= 10 })

	for i, w := range clk.asked() {
		if w < time.Hour {
			t.Errorf("FAIL: wait %d was %v, want at least the hour Retry-After asked for", i, w)
		}
	}
	store.mu.Lock()
	stops := store.stops
	store.mu.Unlock()
	if stops != 0 || atomic.LoadInt32(told) != 0 {
		t.Errorf("FAIL: throttling stopped the bridge (%d stops)", stops)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A healthy stretch of ten minutes clears the count, so a bridge refused
// now and then is not stopped; a single healthy answer does not.
func TestBridgeRefusalsMustBeConsecutive(t *testing.T) {
	_, _, store, b, clk, told := pauseFixture(t, peerNoChat, 1)
	refused := refusedError{fmt.Errorf("not logged in")}
	for round := 0; round < 3; round++ {
		for i := 0; i < refusalsToStop-1; i++ {
			if b.note(HaltRefused, refused) {
				t.Fatalf("FAIL: stopped after %d refusals, want %d", i+1, refusalsToStop)
			}
		}
		// A healthy answer straight after does not forgive them; a
		// healthy stretch does.
		b.cleared()
		clk.advance(healthyFor)
		b.cleared()
	}
	store.mu.Lock()
	stops := store.stops
	store.mu.Unlock()
	if stops != 0 {
		t.Errorf("FAIL: refusals separated by a success stopped the bridge")
	}
	for i := 0; i < refusalsToStop; i++ {
		b.note(HaltRefused, refused)
	}
	store.mu.Lock()
	stops = store.stops
	store.mu.Unlock()
	if stops != 1 || atomic.LoadInt32(told) != 1 {
		t.Errorf("FAIL: %d refusals in a row gave %d stops, %d notices; want 1 and 1", refusalsToStop, stops, atomic.LoadInt32(told))
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// Once the peer answers again the stored failure is cleared by the
// first healthy poll, not left to read "failing" until a message comes.
func TestBridgeClearsTheStoredFailureWhenItRecovers(t *testing.T) {
	_, peer, store, b, _, _ := pauseFixture(t, peerDown, 400)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	runDone(b, ctx)
	waitFor(t, "a recorded failure", func() bool { return store.err() != "" })
	peer.set(peerOK)
	waitFor(t, "the failure to clear", func() bool { return store.err() == "" })
	if !t.Failed() {
		t.Log("ok")
	}
}

// A peer that answers but not with chat (a front end blocking the
// forge's address: 403, 404, a redirect, an HTML page, a 503 naming no
// wait) is asked ever less often and then not at all: thirty requests
// across about a day, then none. Run on a simulated clock.
func TestBridgeStopsWhenThePeerNeverServesChat(t *testing.T) {
	for _, status := range []int{403, 404, 302, 0, 503, 429} {
		t.Run(strconv.Itoa(status), func(t *testing.T) {
			r, peer, store, b, clk, told := pauseFixture(t, peerBlocked, 100000)
			peer.mu.Lock()
			peer.status = status
			peer.mu.Unlock()
			start := clk.now()
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			done := runDone(b, ctx)
			finished(t, done, "the bridge never stopped on a peer that does not serve chat")

			elapsed := clk.now().Sub(start)
			if elapsed < 20*time.Hour || elapsed > 30*time.Hour {
				t.Errorf("FAIL: stopped after %v of simulated time, want about a day", elapsed)
			}
			if n := peer.hits.Load(); n > unservedToStop+5 {
				t.Errorf("FAIL: the blocking peer was asked %d times, want about %d", n, unservedToStop)
			}
			for i, w := range clk.asked() {
				if w > unservedCap {
					t.Errorf("FAIL: wait %d was %v, over the cap %v", i, w, unservedCap)
				}
			}
			store.mu.Lock()
			kind, stops := store.haltKind, store.stops
			store.mu.Unlock()
			if stops != 1 || kind != HaltUnserved || atomic.LoadInt32(told) != 1 {
				t.Errorf("FAIL: stored stop (%d, %q), told %d; want one %q stop, told once", stops, kind, atomic.LoadInt32(told), HaltUnserved)
			}
			// A day more: not one request. The channel got one system line.
			seen := peer.hits.Load()
			clk.advance(48 * time.Hour)
			if peer.hits.Load() != seen {
				t.Errorf("FAIL: requests continued after the stop")
			}
			if n := systemLines(t, r, "stopped"); n != 1 {
				t.Errorf("FAIL: %d stop lines in the channel, want one", n)
			}
			if !t.Failed() {
				t.Logf("ok: %d requests, then none, over %v", seen, elapsed.Round(time.Hour))
			}
		})
	}
}

func systemLines(t *testing.T, r *Room, contains string) int {
	t.Helper()
	n := 0
	for _, m := range folded(t, r, "codeA") {
		if m.Kind == KindRobot && strings.Contains(m.Text, contains) {
			n++
		}
	}
	return n
}

// Plain 5xx answers keep the five-minute cap (no stretch) and, with
// network errors, never stop; Retry-After-bearing throttles are
// covered above. Eighty answers over simulated days: still going.
func TestBridgeFiveHundredsKeepTheShortCap(t *testing.T) {
	_, _, store, b, clk, _ := pauseFixture(t, peerDown, 80)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	runDone(b, ctx)
	waitFor(t, "eighty waits", func() bool { return len(clk.asked()) >= 80 })
	for _, w := range clk.asked() {
		if w > backoffMax {
			t.Fatalf("FAIL: a 5xx waited %v, over the %v cap", w, backoffMax)
		}
	}
	store.mu.Lock()
	stops := store.stops
	store.mu.Unlock()
	if stops != 0 {
		t.Errorf("FAIL: 5xx answers stopped the bridge")
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A peer that alternates a refusal with a good answer is still stopped:
// the count clears only after ten healthy minutes, not at one good poll.
func TestBridgeAlternatingPeerIsStillStopped(t *testing.T) {
	_, peer, store, b, _, _ := pauseFixture(t, peerNoChat, 100000)
	peer.mu.Lock()
	peer.mode = "flip"
	peer.mu.Unlock()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	done := runDone(b, ctx)
	finished(t, done, "an alternating peer kept the bridge going")
	store.mu.Lock()
	stops := store.stops
	store.mu.Unlock()
	if stops != 1 || peer.hits.Load() > 30 {
		t.Errorf("FAIL: %d stops after %d requests, want one stop within a few", stops, peer.hits.Load())
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// Retry-After is clamped as a number before it is a duration.
func TestRetryAfterIsClampedBeforeItOverflows(t *testing.T) {
	_, _, _, b, clk, _ := pauseFixture(t, peerOK, 1)
	past := clk.now().Add(-time.Hour).UTC().Format(http.TimeFormat)
	soon := clk.now().Add(90 * time.Second).UTC().Format(http.TimeFormat)
	for _, c := range []struct {
		in   string
		want time.Duration
		ok   bool
	}{
		{"120", 2 * time.Minute, true},
		{"99999999999999999999", maxThrottleWait, true},
		{"9223372036854775807", maxThrottleWait, true},
		{"-5", 0, true},
		{"-99999999999999999999", 0, true},
		{"soon", 0, false},
		{"", 0, false},
		{past, 0, true},
		{soon, 90 * time.Second, true},
	} {
		got, ok := b.retryAfter(c.in)
		if got != c.want || ok != c.ok {
			t.Errorf("FAIL: retryAfter(%q) = %v, %v; want %v, %v", c.in, got, ok, c.want, c.ok)
		}
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A stop leaves one line in the channel that is never relayed, drops
// the message held at that moment (and says so), and does not send it
// when the bridge is resumed.
func TestBridgeStopDropsTheHeldMessageAndSaysSo(t *testing.T) {
	r, peer, store, b, _, _ := pauseFixture(t, peerRefuseLogin, 100000)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	b.halt = cancel
	done := make(chan struct{})
	ready := make(chan struct{})
	close(ready)
	go func() { defer close(done); b.outbound(ctx, ready) }()
	for i := 0; peer.sendHits.Load() == 0; i++ {
		if _, err := r.Post("codeA", Message{Repo: "alpha", Kind: KindMsg, User: "ann", Text: fmt.Sprintf("held-%d", i)}); err != nil {
			t.Fatal(err)
		}
		time.Sleep(5 * time.Millisecond)
	}
	finished(t, done, "the sender did not stop on repeated refusals")
	if systemLines(t, r, "not yet sent") != 1 {
		t.Errorf("FAIL: want one stop line naming the unsent message")
	}
	store.mu.Lock()
	stops := store.stops
	store.mu.Unlock()
	if stops != 1 {
		t.Fatalf("FAIL: %d stops", stops)
	}
	// Resumed: a new bridge on the same channel against a healthy peer.
	peer.set(peerOK)
	b2 := NewBridge(r, newMemStore(), BridgeConfig{Repo: "alpha", Code: "codeA", URL: peer.srv.URL, User: "bot", Secret: "pw", AllowPrivate: true})
	b2.Client, b2.SendClient = bridgeClient(5*time.Second, true), bridgeClient(5*time.Second, true)
	ctx2, cancel2 := context.WithCancel(context.Background())
	defer cancel2()
	go b2.Run(ctx2)
	for i := 0; ; i++ {
		peer.mu.Lock()
		n := len(peer.sent)
		peer.mu.Unlock()
		if n > 0 {
			break
		}
		if i > 400 {
			t.Fatalf("FAIL: the resumed bridge relayed nothing (peer asked %d times, %d sends)", peer.hits.Load(), peer.sendHits.Load())
		}
		_, _ = r.Post("codeA", Message{Repo: "alpha", Kind: KindMsg, User: "ann", Text: fmt.Sprintf("after-resume-%d", i)})
		time.Sleep(10 * time.Millisecond)
	}
	peer.mu.Lock()
	defer peer.mu.Unlock()
	for _, m := range peer.sent {
		if strings.Contains(m, "held-") || strings.Contains(m, "bridge to") {
			t.Errorf("FAIL: the resumed bridge relayed %q", m)
		}
	}
	if !t.Failed() {
		t.Log("ok")
	}
}