package chat
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"craftdesign.group/quarry/internal/events"
)
// pausePeer is another Fossil server as the bridge meets it when it
// is unhappy, counting every request it is asked. mode picks the
// answer: what Fossil sends for a refused login (login.c: 401, no
// cookie), for an account that signs in but has no chat (chat.c: a 200
// JSON "Missing permissions or not logged in"), or what a front-end
// limiter sends (429 or 503 with a Retry-After), or a plain failure.
type pausePeer struct {
srv *httptest.Server
hits atomic.Int64
mu sync.Mutex
mode string
// okFor makes the next n poll answers healthy before mode applies
// again, for the "a success clears the count" case.
okFor int
// status is peerBlocked's answer; sent is what /chat-send took and
// sendHits how many times it was asked.
status int
flips int
sent []string
sendHits atomic.Int64
}
const (
peerRefuseLogin = "refuse-login"
peerNoChat = "no-chat"
peerThrottle = "throttle"
peerDown = "down"
peerOK = "ok"
// peerBlocked answers every path with status (0 is a 200 HTML page),
// no wait named: a front end blocking the forge's address.
peerBlocked = "blocked"
)
func newPausePeer(t *testing.T, mode string) *pausePeer {
t.Helper()
p := &pausePeer{mode: mode}
notChat := `{"msgs":[{"isError":true,"xfrom":null,` +
`"xmsg":"Missing permissions or not logged in. Try <a href='/login'>logging in</a>."}]}`
mux := http.NewServeMux()
blocked := func(w http.ResponseWriter, r *http.Request) {
p.mu.Lock()
st := p.status
p.mu.Unlock()
switch st {
case 0:
fmt.Fprint(w, "<html>Access denied</html>")
case 302:
http.Redirect(w, r, "/blocked", http.StatusFound)
default:
http.Error(w, "blocked", st)
}
}
mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
p.hits.Add(1)
switch p.get() {
case peerBlocked:
blocked(w, r)
case peerRefuseLogin:
http.Error(w, "unknown user or incorrect password", http.StatusUnauthorized)
case peerThrottle:
w.Header().Set("Retry-After", "3600")
http.Error(w, "blocked", http.StatusServiceUnavailable)
case peerDown:
http.Error(w, "boom", http.StatusInternalServerError)
default:
http.SetCookie(w, &http.Cookie{Name: "fossil-0123456789abcdef", Value: "tok", Path: "/"})
http.Redirect(w, r, "/chat", http.StatusFound)
}
})
serve := func(w http.ResponseWriter, r *http.Request) {
p.hits.Add(1)
if r.URL.Path == "/chat-send" {
p.sendHits.Add(1)
}
if p.get() == peerBlocked {
blocked(w, r)
return
}
if p.get() == peerOK && r.URL.Path == "/chat-send" {
_ = r.ParseForm()
p.mu.Lock()
p.sent = append(p.sent, r.FormValue("msg"))
p.mu.Unlock()
return
}
p.mu.Lock()
ok := p.okFor > 0
if ok {
p.okFor--
}
p.mu.Unlock()
if ok {
fmt.Fprint(w, `{"msgs":[]}`)
return
}
switch p.get() {
case peerThrottle:
w.Header().Set("Retry-After", "3600")
http.Error(w, "blocked", http.StatusServiceUnavailable)
case peerDown:
http.Error(w, "boom", http.StatusInternalServerError)
case peerRefuseLogin:
http.Error(w, "no", http.StatusUnauthorized)
case "flip":
p.mu.Lock()
p.flips++
even := p.flips%2 == 0
p.mu.Unlock()
if even {
fmt.Fprint(w, `{"msgs":[]}`)
} else {
fmt.Fprint(w, notChat)
}
case peerOK:
fmt.Fprint(w, `{"msgs":[]}`)
default:
fmt.Fprint(w, notChat)
}
}
mux.HandleFunc("/chat-poll", serve)
mux.HandleFunc("/chat-send", serve)
mux.HandleFunc("/json/config/chat-timeline-user", func(w http.ResponseWriter, r *http.Request) {
p.hits.Add(1)
if p.get() == peerBlocked {
blocked(w, r)
return
}
http.NotFound(w, r)
})
p.srv = httptest.NewServer(mux)
t.Cleanup(p.srv.Close)
return p
}
func (p *pausePeer) get() string { p.mu.Lock(); defer p.mu.Unlock(); return p.mode }
func (p *pausePeer) set(m string) { p.mu.Lock(); p.mode = m; p.mu.Unlock() }
// simClock is a clock the test advances by the waits the bridge asks
// for: a wait returns at once and moves time forward by what was
// asked, so a day passes in no time. After maxWaits the wait blocks
// until the bridge's context ends, which ends a loop that would
// otherwise spin for ever.
type simClock struct {
mu sync.Mutex
t time.Time
waits []time.Duration
maxWaits int
}
func (c *simClock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
func (c *simClock) advance(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }
func (c *simClock) sleep(ctx context.Context, d time.Duration) bool {
c.mu.Lock()
c.waits = append(c.waits, d)
c.t = c.t.Add(d)
spent := len(c.waits) >= c.maxWaits
c.mu.Unlock()
if spent {
<-ctx.Done()
return false
}
return ctx.Err() == nil
}
func (c *simClock) asked() []time.Duration {
c.mu.Lock()
defer c.mu.Unlock()
return append([]time.Duration(nil), c.waits...)
}
// pauseFixture is a bridge against a pausePeer on a simulated clock.
func pauseFixture(t *testing.T, mode string, maxWaits int) (*Room, *pausePeer, *memBridgeStore, *Bridge, *simClock, *int32) {
t.Helper()
ev, err := events.Start(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(ev.Close)
r := room(t, ev)
peer := newPausePeer(t, mode)
store := newMemStore()
b := NewBridge(r, store, BridgeConfig{
Repo: "alpha", Code: "codeA", URL: peer.srv.URL, User: "bot", Secret: "pw", AllowPrivate: true,
})
b.Client = bridgeClient(5*time.Second, true)
b.SendClient = bridgeClient(5*time.Second, true)
clk := &simClock{t: time.Date(2026, 11, 16, 9, 0, 0, 0, time.UTC), maxWaits: maxWaits}
b.now, b.sleep = clk.now, clk.sleep
var told int32
b.Stopped = func(repo, why string) {
if repo != "alpha" || why == "" {
t.Errorf("FAIL: Stopped heard (%q, %q), want the repository and a reason", repo, why)
}
atomic.AddInt32(&told, 1)
}
return r, peer, store, b, clk, &told
}
// runDone runs the bridge and closes the channel when Run returns.
func runDone(b *Bridge, ctx context.Context) <-chan struct{} {
done := make(chan struct{})
go func() { defer close(done); b.Run(ctx) }()
return done
}
func finished(t *testing.T, done <-chan struct{}, what string) {
t.Helper()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatalf("FAIL: %s", what)
}
}
// A login the other server keeps refusing stops the bridge after the
// stated number of refusals, whichever way the refusal is spelled; the
// stop is stored with its reason and time, the admins are told exactly
// once, and over a simulated day the peer is asked nothing more, in
// either direction.
func TestBridgeStopsAfterRefusedLogin(t *testing.T) {
for _, mode := range []string{peerRefuseLogin, peerNoChat} {
t.Run(mode, func(t *testing.T) {
r, peer, store, b, clk, told := pauseFixture(t, mode, 1000)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := runDone(b, ctx)
finished(t, done, "the bridge did not stop itself after repeated refusals")
store.mu.Lock()
stops, why, at := store.stops, store.stoppedWhy, store.stoppedAt
store.mu.Unlock()
if stops != 1 || why == "" || at.IsZero() {
t.Errorf("FAIL: stored stop = (%d stops, %q, %v), want one stop with a reason and time", stops, why, at)
}
if got := atomic.LoadInt32(told); got != 1 {
t.Errorf("FAIL: admins told %d times, want exactly once", got)
}
// A day later, and a message posted meanwhile: nothing reaches the peer.
seen := peer.hits.Load()
clk.advance(24 * time.Hour)
if _, err := r.Post("codeA", Message{Repo: "alpha", Kind: KindMsg, User: "ann", Text: "anyone there"}); err != nil {
t.Fatal(err)
}
clk.advance(24 * time.Hour)
if got := peer.hits.Load(); got != seen {
t.Errorf("FAIL: the stopped bridge made %d further requests to the peer", got-seen)
}
if !t.Failed() {
t.Log("ok")
}
})
}
}
// Network errors, timeouts and 5xx back off up to the cap and retry for
// ever: they are the upstream failing, not refusing, and never stop it.
func TestBridgeNetworkErrorsNeverStop(t *testing.T) {
_, peer, store, b, clk, told := pauseFixture(t, peerDown, 12)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := runDone(b, ctx)
waitFor(t, "twelve back-offs", func() bool { return len(clk.asked()) >= 12 })
store.mu.Lock()
stops := store.stops
store.mu.Unlock()
if stops != 0 || atomic.LoadInt32(told) != 0 {
t.Errorf("FAIL: server errors stopped the bridge (%d stops, %d told)", stops, atomic.LoadInt32(told))
}
var longest time.Duration
for _, w := range clk.asked() {
longest = max(longest, w)
}
if longest != backoffMax {
t.Errorf("FAIL: longest back-off %v, want the cap %v", longest, backoffMax)
}
select {
case <-done:
t.Errorf("FAIL: Run returned while the peer was only failing")
default:
}
if peer.hits.Load() < 12 {
t.Errorf("FAIL: the peer saw %d requests, want a retry per back-off", peer.hits.Load())
}
if !t.Failed() {
t.Log("ok")
}
}
// A throttle answer's wait is honoured and is not a refusal, however
// many arrive.
func TestBridgeThrottleHonoursRetryAfter(t *testing.T) {
_, _, store, b, clk, told := pauseFixture(t, peerThrottle, 10)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runDone(b, ctx)
waitFor(t, "ten waits", func() bool { return len(clk.asked()) >= 10 })
for i, w := range clk.asked() {
if w < time.Hour {
t.Errorf("FAIL: wait %d was %v, want at least the hour Retry-After asked for", i, w)
}
}
store.mu.Lock()
stops := store.stops
store.mu.Unlock()
if stops != 0 || atomic.LoadInt32(told) != 0 {
t.Errorf("FAIL: throttling stopped the bridge (%d stops)", stops)
}
if !t.Failed() {
t.Log("ok")
}
}
// A healthy stretch of ten minutes clears the count, so a bridge refused
// now and then is not stopped; a single healthy answer does not.
func TestBridgeRefusalsMustBeConsecutive(t *testing.T) {
_, _, store, b, clk, told := pauseFixture(t, peerNoChat, 1)
refused := refusedError{fmt.Errorf("not logged in")}
for round := 0; round < 3; round++ {
for i := 0; i < refusalsToStop-1; i++ {
if b.note(HaltRefused, refused) {
t.Fatalf("FAIL: stopped after %d refusals, want %d", i+1, refusalsToStop)
}
}
// A healthy answer straight after does not forgive them; a
// healthy stretch does.
b.cleared()
clk.advance(healthyFor)
b.cleared()
}
store.mu.Lock()
stops := store.stops
store.mu.Unlock()
if stops != 0 {
t.Errorf("FAIL: refusals separated by a success stopped the bridge")
}
for i := 0; i < refusalsToStop; i++ {
b.note(HaltRefused, refused)
}
store.mu.Lock()
stops = store.stops
store.mu.Unlock()
if stops != 1 || atomic.LoadInt32(told) != 1 {
t.Errorf("FAIL: %d refusals in a row gave %d stops, %d notices; want 1 and 1", refusalsToStop, stops, atomic.LoadInt32(told))
}
if !t.Failed() {
t.Log("ok")
}
}
// Once the peer answers again the stored failure is cleared by the
// first healthy poll, not left to read "failing" until a message comes.
func TestBridgeClearsTheStoredFailureWhenItRecovers(t *testing.T) {
_, peer, store, b, _, _ := pauseFixture(t, peerDown, 400)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runDone(b, ctx)
waitFor(t, "a recorded failure", func() bool { return store.err() != "" })
peer.set(peerOK)
waitFor(t, "the failure to clear", func() bool { return store.err() == "" })
if !t.Failed() {
t.Log("ok")
}
}
// A peer that answers but not with chat (a front end blocking the
// forge's address: 403, 404, a redirect, an HTML page, a 503 naming no
// wait) is asked ever less often and then not at all: thirty requests
// across about a day, then none. Run on a simulated clock.
func TestBridgeStopsWhenThePeerNeverServesChat(t *testing.T) {
for _, status := range []int{403, 404, 302, 0, 503, 429} {
t.Run(strconv.Itoa(status), func(t *testing.T) {
r, peer, store, b, clk, told := pauseFixture(t, peerBlocked, 100000)
peer.mu.Lock()
peer.status = status
peer.mu.Unlock()
start := clk.now()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := runDone(b, ctx)
finished(t, done, "the bridge never stopped on a peer that does not serve chat")
elapsed := clk.now().Sub(start)
if elapsed < 20*time.Hour || elapsed > 30*time.Hour {
t.Errorf("FAIL: stopped after %v of simulated time, want about a day", elapsed)
}
if n := peer.hits.Load(); n > unservedToStop+5 {
t.Errorf("FAIL: the blocking peer was asked %d times, want about %d", n, unservedToStop)
}
for i, w := range clk.asked() {
if w > unservedCap {
t.Errorf("FAIL: wait %d was %v, over the cap %v", i, w, unservedCap)
}
}
store.mu.Lock()
kind, stops := store.haltKind, store.stops
store.mu.Unlock()
if stops != 1 || kind != HaltUnserved || atomic.LoadInt32(told) != 1 {
t.Errorf("FAIL: stored stop (%d, %q), told %d; want one %q stop, told once", stops, kind, atomic.LoadInt32(told), HaltUnserved)
}
// A day more: not one request. The channel got one system line.
seen := peer.hits.Load()
clk.advance(48 * time.Hour)
if peer.hits.Load() != seen {
t.Errorf("FAIL: requests continued after the stop")
}
if n := systemLines(t, r, "stopped"); n != 1 {
t.Errorf("FAIL: %d stop lines in the channel, want one", n)
}
if !t.Failed() {
t.Logf("ok: %d requests, then none, over %v", seen, elapsed.Round(time.Hour))
}
})
}
}
func systemLines(t *testing.T, r *Room, contains string) int {
t.Helper()
n := 0
for _, m := range folded(t, r, "codeA") {
if m.Kind == KindRobot && strings.Contains(m.Text, contains) {
n++
}
}
return n
}
// Plain 5xx answers keep the five-minute cap (no stretch) and, with
// network errors, never stop; Retry-After-bearing throttles are
// covered above. Eighty answers over simulated days: still going.
func TestBridgeFiveHundredsKeepTheShortCap(t *testing.T) {
_, _, store, b, clk, _ := pauseFixture(t, peerDown, 80)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runDone(b, ctx)
waitFor(t, "eighty waits", func() bool { return len(clk.asked()) >= 80 })
for _, w := range clk.asked() {
if w > backoffMax {
t.Fatalf("FAIL: a 5xx waited %v, over the %v cap", w, backoffMax)
}
}
store.mu.Lock()
stops := store.stops
store.mu.Unlock()
if stops != 0 {
t.Errorf("FAIL: 5xx answers stopped the bridge")
}
if !t.Failed() {
t.Log("ok")
}
}
// A peer that alternates a refusal with a good answer is still stopped:
// the count clears only after ten healthy minutes, not at one good poll.
func TestBridgeAlternatingPeerIsStillStopped(t *testing.T) {
_, peer, store, b, _, _ := pauseFixture(t, peerNoChat, 100000)
peer.mu.Lock()
peer.mode = "flip"
peer.mu.Unlock()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := runDone(b, ctx)
finished(t, done, "an alternating peer kept the bridge going")
store.mu.Lock()
stops := store.stops
store.mu.Unlock()
if stops != 1 || peer.hits.Load() > 30 {
t.Errorf("FAIL: %d stops after %d requests, want one stop within a few", stops, peer.hits.Load())
}
if !t.Failed() {
t.Log("ok")
}
}
// Retry-After is clamped as a number before it is a duration.
func TestRetryAfterIsClampedBeforeItOverflows(t *testing.T) {
_, _, _, b, clk, _ := pauseFixture(t, peerOK, 1)
past := clk.now().Add(-time.Hour).UTC().Format(http.TimeFormat)
soon := clk.now().Add(90 * time.Second).UTC().Format(http.TimeFormat)
for _, c := range []struct {
in string
want time.Duration
ok bool
}{
{"120", 2 * time.Minute, true},
{"99999999999999999999", maxThrottleWait, true},
{"9223372036854775807", maxThrottleWait, true},
{"-5", 0, true},
{"-99999999999999999999", 0, true},
{"soon", 0, false},
{"", 0, false},
{past, 0, true},
{soon, 90 * time.Second, true},
} {
got, ok := b.retryAfter(c.in)
if got != c.want || ok != c.ok {
t.Errorf("FAIL: retryAfter(%q) = %v, %v; want %v, %v", c.in, got, ok, c.want, c.ok)
}
}
if !t.Failed() {
t.Log("ok")
}
}
// A stop leaves one line in the channel that is never relayed, drops
// the message held at that moment (and says so), and does not send it
// when the bridge is resumed.
func TestBridgeStopDropsTheHeldMessageAndSaysSo(t *testing.T) {
r, peer, store, b, _, _ := pauseFixture(t, peerRefuseLogin, 100000)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
b.halt = cancel
done := make(chan struct{})
ready := make(chan struct{})
close(ready)
go func() { defer close(done); b.outbound(ctx, ready) }()
for i := 0; peer.sendHits.Load() == 0; i++ {
if _, err := r.Post("codeA", Message{Repo: "alpha", Kind: KindMsg, User: "ann", Text: fmt.Sprintf("held-%d", i)}); err != nil {
t.Fatal(err)
}
time.Sleep(5 * time.Millisecond)
}
finished(t, done, "the sender did not stop on repeated refusals")
if systemLines(t, r, "not yet sent") != 1 {
t.Errorf("FAIL: want one stop line naming the unsent message")
}
store.mu.Lock()
stops := store.stops
store.mu.Unlock()
if stops != 1 {
t.Fatalf("FAIL: %d stops", stops)
}
// Resumed: a new bridge on the same channel against a healthy peer.
peer.set(peerOK)
b2 := NewBridge(r, newMemStore(), BridgeConfig{Repo: "alpha", Code: "codeA", URL: peer.srv.URL, User: "bot", Secret: "pw", AllowPrivate: true})
b2.Client, b2.SendClient = bridgeClient(5*time.Second, true), bridgeClient(5*time.Second, true)
ctx2, cancel2 := context.WithCancel(context.Background())
defer cancel2()
go b2.Run(ctx2)
for i := 0; ; i++ {
peer.mu.Lock()
n := len(peer.sent)
peer.mu.Unlock()
if n > 0 {
break
}
if i > 400 {
t.Fatalf("FAIL: the resumed bridge relayed nothing (peer asked %d times, %d sends)", peer.hits.Load(), peer.sendHits.Load())
}
_, _ = r.Post("codeA", Message{Repo: "alpha", Kind: KindMsg, User: "ann", Text: fmt.Sprintf("after-resume-%d", i)})
time.Sleep(10 * time.Millisecond)
}
peer.mu.Lock()
defer peer.mu.Unlock()
for _, m := range peer.sent {
if strings.Contains(m, "held-") || strings.Contains(m, "bridge to") {
t.Errorf("FAIL: the resumed bridge relayed %q", m)
}
}
if !t.Failed() {
t.Log("ok")
}
}