Skip to content
quarry Artifact [d5a42d0f47]
Login

Artifact [d5a42d0f47]

Artifact d5a42d0f47671830a91a6686edfe29401518db8dfb52564d251c30ec1eff91ba:


// bridge.go is RFC 0011 §9: an opt-in connector between one quarry
// channel and one Fossil server's chat.
//
// Fossil's chat is a server-local table, never synced, gated by `C`,
// long-polled through /chat-poll and written through /chat-send. Every
// key below was checked against a real fossil 2.28 (`msgid`, `mtime`,
// `xfrom`, `xmsg`, `fsize`, `fname`, `mdel`) rather than assumed.
//
// Two constraints that a reading of chat.c alone does not surface, and
// that a bridge silently fails without:
//
//  1. HTTP Basic authentication is refused unless the upstream has the
//     `http_authentication_ok` config entry set. Without it every
//     request is anonymous and every answer is a permissions error.
//  2. /chat-poll creates its full-text TEMP triggers on the way in, so
//     fossil's CSRF gate must consider the request safe or the poll
//     dies with "not authorized: CREATE TEMP TRIGGER". A same-origin
//     `Referer` header is what makes it safe, so every request here
//     carries one.
//
// What the bridge does not pretend: fossil messages cannot be edited
// from quarry, two fossil servers are two chats, and a bridged
// channel's history is still quarry's seven-day window.
package chat

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"craftdesign.group/quarry/internal/outbound"
)

const (
	// pollTimeout must exceed fossil's chat-poll-timeout (420 s by
	// default) or every poll is a client-side cancellation that reads
	// nothing and looks, from the log, like an upstream that is down.
	pollTimeout = 9 * time.Minute
	// sendTimeout is an ordinary write; nothing about it hangs.
	sendTimeout = 30 * time.Second
	// backoffMin/backoffMax pace retries after a failed poll. Doubling
	// from half a minute to five keeps a broken upstream from being
	// hammered while a transient one recovers within a minute.
	backoffMin = 30 * time.Second
	backoffMax = 5 * time.Minute
	// refusalsToStop is how many failed attempts in a row the other
	// server may answer "no" (a login it will not take, an account with
	// no chat, a 401) before the bridge stops and waits for a person.
	// Fossil itself has no login limiter (login.c sleeps a second per
	// refused login and logs it, and nothing in chat.c, login.c or
	// main.c answers 429), so the number is set by what a polite client
	// owes a server it cannot see the limits of, and by the front-end
	// limiters that do exist in front of a public one (#407): a wrong
	// password is not fixed by trying it again, a typo or a moment's
	// misconfiguration is fixed by one more, and three failed cycles
	// is a few minutes of backoff, not an hour of them.
	refusalsToStop = 3
	// maxThrottleWait bounds how long a Retry-After may park the bridge.
	maxThrottleWait = 24 * time.Hour
	// unservedCap is the longest back-off for a peer that answers but
	// not with chat; unservedToStop is how many such answers in a row
	// stop the bridge. The ramp (30s doubling to the hour) is about an
	// hour, then one request an hour, so thirty answers is about a day
	// of nothing else: 30 requests in the first 23 hours and none after,
	// where the five-minute cap made 288 a day for ever (#407).
	unservedCap    = time.Hour
	unservedToStop = 30
	// healthyFor is how long the upstream must go without a refusal or
	// an unserved answer before either count clears. A peer that
	// alternates one good answer with a refusal would otherwise reset
	// the count each time and be asked about 5000 times a day; with this
	// it reaches the limit in minutes and stays stopped.
	healthyFor = 10 * time.Minute
	// HaltRefused and HaltUnserved are the two reasons a bridge stops
	// itself, stored beside the stop.
	HaltRefused  = "refused"
	HaltUnserved = "unserved"
)

// unservedError is an answer from the peer that is not chat and not a
// refusal the credential explains: a 403, 404 or 302 from a front end
// that blocks the forge's address, a page where JSON was expected, a
// throttle that names no wait. Not a network failure (no answer) and not
// a 5xx (the peer is down): those only back off.
// errPaused is what a request made under a takedown answers: not a
// failure of the upstream, so it is neither recorded nor backed off.
var errPaused = errors.New("chat: bridge paused by a takedown")

func (b *Bridge) isHeld() bool { return b.Held != nil && b.Held(b.cfg.Repo) }

func (b *Bridge) heldEvery() time.Duration {
	if b.HeldEvery > 0 {
		return b.HeldEvery
	}
	return 5 * time.Second
}

// resume waits while the repository is held and answers false when ctx
// ends first.
func (b *Bridge) resume(ctx context.Context) bool {
	for b.isHeld() {
		if !b.wait(ctx, b.heldEvery()) {
			return false
		}
	}
	return ctx.Err() == nil
}

type unservedError struct{ error }

func (e unservedError) Unwrap() error { return e.error }

// refusedError is an answer that says "not you", as Fossil gives it: a
// 401 (login.c cgi_set_status(401) for a refused login or Basic
// credential), a login page redrawn with no session, or chat's
// "Missing permissions or not logged in" (chat.c, the same words for a
// lapsed session and for an account without the chat capability). Trying
// again with the same credential does not change the answer.
type refusedError struct{ error }

func (e refusedError) Unwrap() error { return e.error }

// throttledError is a server asking for time (429 or 503, with its
// Retry-After when it sent one). It is not a refusal of the credential
// and never counts toward stopping.
type throttledError struct {
	error
	wait time.Duration
}

func (e *throttledError) Unwrap() error { return e.error }

// BridgeConfig is one bridged channel: which repository, which channel
// key, and the upstream credential the poller replays.
type BridgeConfig struct {
	Repo   string
	Code   string
	URL    string
	User   string
	Secret string
	// LastMsgID is where the poller resumes; 0 asks fossil for the
	// channel's recent history rather than for nothing.
	LastMsgID int64
	// AllowPrivate permits an upstream on a loopback, private or
	// link-local address. Off by default: the bridge is a server-side
	// fetcher holding a credential, so an admin who can name a URL
	// could otherwise point it at the metadata service or at anything
	// else inside the network the forge sits in. Operators bridging a
	// fossil on the same host turn it on deliberately
	// (--allow-private-bridges), and the tests set it because their
	// fossil is on 127.0.0.1.
	AllowPrivate bool
}

// BridgeStore is the cursor and the msgid map. Both are OLTP, both are
// window-bounded, and a nil store is a bridge with no memory: it still
// carries messages, it just re-reads the upstream from where it
// started every time the process restarts.
type BridgeStore interface {
	// SetChatBridgeCursor advances how far the poller has read.
	SetChatBridgeCursor(repo string, msgid int64, at time.Time) error
	// RecordChatBridgeError remembers why the last poll failed.
	RecordChatBridgeError(repo string, err error) error
	// MapChatBridgeMsg remembers which sequence an upstream msgid
	// became, so a later mdel can tombstone the right message.
	MapChatBridgeMsg(repo string, msgid int64, seq uint64) error
	// ChatBridgeSeq answers that question back.
	ChatBridgeSeq(repo string, msgid int64) (uint64, bool)
	// HaltChatBridge records that the bridge stopped itself (kind is
	// HaltRefused or HaltUnserved), and says whether this call is the one
	// that stopped it. It is stored apart from the chat-closed stop, so
	// neither hides the other; it survives a restart and only a person
	// clears it.
	HaltChatBridge(repo, kind, why string, at time.Time) (bool, error)
}

// Bridge carries one channel both ways.
type Bridge struct {
	room  *Room
	store BridgeStore
	cfg   BridgeConfig
	// Client, SendClient, BackoffMin and BackoffMax are SEAMS, not
	// configuration: NewBridge fills all four with the values §9 calls
	// for, and nothing in the composition root sets them. They are
	// exported so a test in another package could drive a bridge
	// against a stub, and so this package's own tests can shorten a
	// half-minute backoff to a hundred milliseconds without waiting it
	// out. Changing them in production changes the protocol timings.
	//
	// Client is the poller's; its timeout must exceed the upstream's
	// poll hang. SendClient is the short-timeout one for outbound.
	Client     *http.Client
	SendClient *http.Client
	// RobotOn reports whether this repository's own activity robot is
	// on. When it is, the upstream's timeline announcements are
	// skipped: otherwise every check-in is announced twice, once by
	// each robot, which is the failure that teaches people to mute the
	// channel. nil means "off", the direction this must fail in.
	RobotOn func(repo string) bool
	// Arrived hears each message the bridge imported, after the channel
	// took it: what a post through the forge does for notifications
	// (mentions, "every message" followers) has to happen for a bridged
	// one too, or a bridged channel is silent to everyone following it.
	// nil hears nothing.
	Arrived func(repo, user, text string, seq uint64)
	// Stopped hears, once, that the bridge stopped itself after the
	// upstream kept refusing it, with the reason: the forge uses it to
	// tell the repository's admins. nil hears nothing.
	Stopped func(repo, why string)
	// Held says the repository is under a takedown (RFC 0023). While it
	// is, the bridge asks the upstream nothing, signs in nowhere, sends
	// nothing out and writes nothing into the channel: it is paused, not
	// halted, so nothing is stored and it resumes by itself the moment
	// the takedown is lifted. What was said locally meanwhile is not
	// relayed afterwards. nil means never held.
	Held func(repo string) bool
	// HeldEvery is how often a paused bridge looks again; a SEAM, like
	// BackoffMin.
	HeldEvery time.Duration
	// now and sleep are the clock seams: nil is the real one. A test
	// supplies a clock it advances and a wait that returns at once, so
	// "nothing is asked of the upstream for a day" is shown without a day.
	now   func() time.Time
	sleep func(ctx context.Context, d time.Duration) bool
	// halt ends Run's context; refusals counts failed attempts the
	// upstream answered "no" to since the last success (smu guards it
	// and stopped).
	halt     context.CancelFunc
	refusals int
	unserved int
	lastBad  time.Time
	stopped  bool
	// failed is whether a failure is on the stored row, so the first
	// healthy poll after it clears it: a bridge that has recovered but
	// has heard nothing new reads "connecting", not the old error.
	failed bool
	// timelineUser is the upstream's chat-timeline-user when it could
	// be learned; "" falls back to the shape heuristic below.
	timelineUser string
	// BackoffMin and BackoffMax pace retries in BOTH directions.
	BackoffMin, BackoffMax time.Duration
	// mu guards the cursor alone. The inbound loop advances it while
	// anything else — a test, a future status call — may read it.
	mu sync.Mutex
	// session is the upstream's login cookie ("fossil-…=…"), and
	// loggedIn when it was taken; smu guards both. A fossil honours
	// HTTP Basic auth only with http_authentication_ok on, which is not
	// its default, so the bridge signs in through /login as a browser
	// does (login, below) and sends the cookie instead.
	smu      sync.Mutex
	session  string
	loggedIn time.Time
	loginErr error // why the last sign-in failed; nil once one succeeds
	// post is Room.Post, named so a test can make the channel refuse.
	// "The cursor does not move past a message that was not
	// delivered" is the property this bridge lives or dies by, and it
	// cannot be proved against a broker that always accepts.
	post func(code string, m Message) (uint64, error)
}

func NewBridge(room *Room, store BridgeStore, cfg BridgeConfig) *Bridge {
	b := &Bridge{
		room: room, store: store, cfg: cfg,
		Client:     bridgeClient(pollTimeout, cfg.AllowPrivate),
		SendClient: bridgeClient(sendTimeout, cfg.AllowPrivate),
	}
	b.BackoffMin, b.BackoffMax = backoffMin, backoffMax
	// A row may carry a failure from before this process started; the
	// first healthy answer clears it.
	b.failed = true
	b.post = func(code string, m Message) (uint64, error) { return b.room.Post(code, m) }
	return b
}

// bridgeClient is the only kind of HTTP client this package makes: the
// forge's guarded client (outbound.Policy.NewClient), which refuses
// private destinations at connect time, on the address actually dialled,
// unless the operator allowed them.
//
// Redirects are REFUSED rather than followed. Go re-sends an
// Authorization header on a same-host redirect, and the bridge's
// header is a fossil password: a compromised or merely misconfigured
// upstream that answers 302 could otherwise walk the credential
// somewhere the operator never named. A bridge that needs a redirect
// is a bridge configured with the wrong URL, and saying so is better
// than quietly following one.
func bridgeClient(timeout time.Duration, allowPrivate bool) *http.Client {
	c := outbound.Bridges.Allowing(allowPrivate).NewClient(timeout)
	c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		// A redirect is an answer, and not chat: unserved, not a network
		// failure.
		return unservedError{fmt.Errorf("chat bridge: %s redirected to %s; configure the final URL",
			safeURL(via[len(via)-1].URL.String()), safeURL(req.URL.String()))}
	}
	return c
}

// ResolveHost resolves host once, with no judgement on the answer, by
// the resolver the forge's address rule uses, so a test or an operator
// permitting private addresses (--allow-private-bridges) goes through
// the same lookup. A host that is an address resolves to itself.
func ResolveHost(ctx context.Context, host string) ([]net.IPAddr, error) {
	if ip := net.ParseIP(host); ip != nil {
		return []net.IPAddr{{IP: ip}}, nil
	}
	ips, err := outbound.Bridges.Resolver()(ctx, host)
	if err != nil {
		return nil, fmt.Errorf("the host %s does not resolve", host)
	}
	if len(ips) == 0 {
		return nil, fmt.Errorf("no address for %s", host)
	}
	return ips, nil
}

// PublicAddrs resolves host once and returns its addresses, or an error
// if there are none or ANY is in a class the forge will not call on a
// user's behalf (outbound.Reserved) and the operator has not allowed.
// The one place a Fossil mirror's pinned push takes its addresses from,
// so what was checked is what is dialled.
func PublicAddrs(ctx context.Context, host string) ([]net.IPAddr, error) {
	return outbound.Bridges.Addrs(ctx, host)
}

// CheckPublicHost is PublicAddrs for a caller that wants the answer at
// once and will resolve again itself.
func CheckPublicHost(ctx context.Context, host string) error {
	_, err := PublicAddrs(ctx, host)
	return err
}

// safeURL is a URL as it may appear in an error, a log line or a
// stored status: scheme, host and path, with any userinfo and query
// removed. Credentials and tokens live in exactly those two places,
// and an error string is copied into the OLTP row a status page
// renders.
func safeURL(raw string) string {
	u, err := url.Parse(raw)
	if err != nil || u.Host == "" {
		return "the upstream"
	}
	return u.Scheme + "://" + u.Host + u.Path
}

// fossilMsg is one /chat-poll entry, field for field as fossil emits
// it. Anything not named here is fossil's presentation (uclr, lmtime)
// and has no meaning on this side.
type fossilMsg struct {
	MsgID   int64  `json:"msgid"`
	MTime   string `json:"mtime"`
	XFrom   string `json:"xfrom"`
	XMsg    string `json:"xmsg"`
	FSize   int64  `json:"fsize"`
	FName   string `json:"fname"`
	MDel    int64  `json:"mdel"`
	IsError bool   `json:"isError"`
}

type fossilPoll struct {
	Msgs []fossilMsg `json:"msgs"`
}

// Run carries the channel both ways until ctx is done. Inbound is a
// long poll; outbound is a tail of the channel from NOW — a bridge
// starting up must not replay the window into the upstream, which
// would post a week of quarry's conversation as new fossil messages.
func (b *Bridge) Run(ctx context.Context) {
	if b == nil || b.room == nil || b.cfg.URL == "" {
		return
	}
	// A refused sign-in is not yet a failure: an upstream with
	// http_authentication_ok on still takes Basic auth. It is logged,
	// and named on the bridge's error if a poll or send is then refused
	// as signed out (signedOutErr).
	ctx, b.halt = context.WithCancel(ctx)
	// The outbound loop listens from the first moment, held or not: a
	// message a person posts the instant a takedown is lifted must find
	// the tail already open, and one said during it is discarded as it
	// arrives. It sends nothing until the sign-in below is done (ready).
	// Run returns only when both directions have, so a stopped bridge
	// has no goroutine left that could still ask the upstream anything.
	out := make(chan struct{})
	ready := make(chan struct{})
	go func() { defer close(out); b.outbound(ctx, ready) }()
	defer func() { b.halt(); <-out }()
	if !b.resume(ctx) {
		return
	}
	if err := b.login(ctx); err != nil && !errors.Is(err, errPaused) {
		log.Printf("chat: bridge %s: %v", b.cfg.Repo, err)
		wait, stop := b.after(err, b.BackoffMin)
		if stop {
			return
		}
		var th *throttledError
		if errors.As(err, &th) || isUnserved(err) {
			if !b.wait(ctx, wait) {
				return
			}
		}
	}
	b.timelineUser = b.fetchTimelineUser(ctx)
	close(ready)
	b.inbound(ctx)
}

func (b *Bridge) inbound(ctx context.Context) {
	backoff := b.BackoffMin
	for ctx.Err() == nil {
		if !b.resume(ctx) {
			return
		}
		n, err := b.pollOnce(ctx)
		if err != nil {
			if ctx.Err() != nil {
				return
			}
			if errors.Is(err, errPaused) {
				continue
			}
			log.Printf("chat: bridge %s: poll: %v", b.cfg.Repo, err)
			b.fail(err)
			wait, stop := b.after(err, backoff)
			if stop || !b.wait(ctx, wait) {
				return
			}
			if backoff *= 2; backoff > b.BackoffMax {
				backoff = b.BackoffMax
			}
			continue
		}
		backoff = b.BackoffMin
		if n == 0 {
			// A poll that hung out its timeout and returned nothing is
			// the ordinary quiet case, not a reason to pause — but a
			// server answering instantly with nothing would spin, so
			// the loop yields briefly either way.
			if !b.wait(ctx, time.Second) {
				return
			}
		}
	}
}

func isRefused(err error) bool {
	var r refusedError
	return errors.As(err, &r)
}

// wait sleeps d, or answers false the moment ctx ends.
func (b *Bridge) wait(ctx context.Context, d time.Duration) bool {
	if b.sleep != nil {
		return b.sleep(ctx, d)
	}
	select {
	case <-ctx.Done():
		return false
	case <-time.After(d):
		return true
	}
}

func (b *Bridge) clock() time.Time {
	if b.now != nil {
		return b.now()
	}
	return time.Now()
}

// after reads a failed attempt: how long to wait before the next one,
// and whether the bridge has stopped itself. A throttle answer's wait is
// honoured and counts as nothing; a refusal counts toward a stop; an
// answer that is not chat (unservedError) stretches the back-off to an
// hour and counts toward a stop a day on; anything else (no answer at
// all, a 5xx) only backs off as before and never stops the bridge.
func (b *Bridge) after(err error, backoff time.Duration) (time.Duration, bool) {
	var th *throttledError
	switch {
	case errors.As(err, &th):
		return b.throttleWait(backoff, th), false
	case isRefused(err):
		return backoff, b.note(HaltRefused, err)
	case isUnserved(err):
		return b.unservedWait(), b.note(HaltUnserved, err)
	}
	return backoff, false
}

func isUnserved(err error) bool {
	var u unservedError
	return errors.As(err, &u)
}

func (b *Bridge) throttleWait(backoff time.Duration, th *throttledError) time.Duration {
	return min(max(backoff, th.wait), maxThrottleWait)
}

// unservedWait is the back-off after the count so far: BackoffMin
// doubling per consecutive answer, to unservedCap.
func (b *Bridge) unservedWait() time.Duration {
	b.smu.Lock()
	n := b.unserved
	b.smu.Unlock()
	w := b.BackoffMin
	for i := 1; i < n && w < unservedCap; i++ {
		w *= 2
	}
	return min(w, unservedCap)
}

// note counts one refusal or unserved answer and, at its limit, stops
// the bridge: the stop is stored (a restart keeps it), the admins are
// told once, one line says so in the channel, and Run's context ends so
// neither direction asks the upstream anything more. It reports whether
// the bridge has stopped.
func (b *Bridge) note(kind string, err error) bool {
	b.smu.Lock()
	b.lastBad = b.clock()
	limit := refusalsToStop
	if kind == HaltUnserved {
		b.unserved++
		limit = unservedToStop
	} else {
		b.refusals++
	}
	n := b.refusals
	if kind == HaltUnserved {
		n = b.unserved
	}
	stop := n >= limit && !b.stopped
	if stop {
		b.stopped = true
	}
	done := b.stopped
	b.smu.Unlock()
	if !stop {
		return done
	}
	why := err.Error()
	log.Printf("chat: bridge %s stopped (%s) after %d: %s", b.cfg.Repo, kind, limit, why)
	first := true
	if b.store != nil {
		var serr error
		first, serr = b.store.HaltChatBridge(b.cfg.Repo, kind, why, b.clock())
		if serr != nil {
			log.Printf("chat: bridge %s: recording the stop: %v", b.cfg.Repo, serr)
		}
	}
	if first {
		b.sayStopped(kind)
		if b.Stopped != nil {
			b.Stopped(b.cfg.Repo, why)
		}
	}
	if b.halt != nil {
		b.halt()
	}
	return true
}

// StoppedLine and ResumedLine are the two lines a stop and its undoing
// leave in the channel, as the activity robot's kind of line so nothing
// relays them to the upstream. Exported for the stops the forge makes
// itself (chat opened beyond named people) and for the resume.
func StoppedLine(host, why string) string {
	return "The bridge to " + host + " stopped: " + why + ". It is not relaying in either direction until it is resumed (see the repository's settings); " +
		"messages posted here meanwhile, including any not yet sent, will not reach it."
}

func ResumedLine(host string) string {
	return "The bridge to " + host + " resumed. Messages posted while it was stopped were not sent."
}

// SystemLine posts one such line into the channel.
func (r *Room) SystemLine(repo, code, text string) {
	if _, err := r.Post(code, Message{Repo: repo, Kind: KindRobot, Origin: "robot", User: "robot", Text: text}); err != nil {
		log.Printf("chat: %s: system line: %v", repo, err)
	}
}

func (b *Bridge) sayStopped(kind string) {
	why := "the Fossil server refused its login"
	if kind == HaltUnserved {
		why = "the Fossil server has not served chat to this forge for about a day"
	}
	host := safeURL(b.cfg.URL)
	if u, err := url.Parse(host); err == nil && u.Host != "" {
		host = u.Host
	}
	b.room.SystemLine(b.cfg.Repo, b.cfg.Code, StoppedLine(host, why))
}

// cleared notes a healthy answer. The counts clear only once the
// upstream has gone healthyFor without a refusal or an unserved answer:
// a healthy poll in between does not forgive one. The stored failure is
// cleared at the first healthy answer, which is a different thing.
func (b *Bridge) cleared() {
	b.smu.Lock()
	if b.lastBad.IsZero() || b.clock().Sub(b.lastBad) >= healthyFor {
		b.refusals, b.unserved = 0, 0
	}
	was := b.failed
	b.failed = false
	b.smu.Unlock()
	if was && b.store != nil {
		if err := b.store.RecordChatBridgeError(b.cfg.Repo, nil); err != nil {
			log.Printf("chat: bridge %s: clearing the failure: %v", b.cfg.Repo, err)
		}
	}
}

// pollOnce reads one /chat-poll response into the channel and answers
// how many upstream entries it consumed.
func (b *Bridge) pollOnce(ctx context.Context) (int, error) {
	poll := fmt.Sprintf("%s/chat-poll?name=%d&raw=1", strings.TrimRight(b.cfg.URL, "/"), b.cursor())
	body, err := b.get(ctx, poll)
	if err != nil {
		return 0, err
	}
	if b.isHeld() {
		// The takedown landed while the poll hung: what it brought is
		// not written into the channel, and the cursor stays put.
		return 0, errPaused
	}
	// A poll that hangs out its timeout with nothing new answers an
	// EMPTY BODY, not an empty envelope (verified against fossil 2.28).
	// Reading that as malformed JSON would turn every quiet interval
	// into a recorded failure and a thirty-second backoff — a bridge
	// that works only on a busy channel.
	if len(strings.TrimSpace(string(body))) == 0 {
		b.cleared()
		return 0, nil
	}
	var out fossilPoll
	if err := json.Unmarshal(body, &out); err != nil {
		// A fossil that refuses answers HTML, not JSON. Saying so is
		// more use than "invalid character '<'".
		return 0, unservedError{fmt.Errorf("chat-poll did not answer JSON (upstream refused or is not a fossil server): %w", err)}
	}
	// The cursor advances one entry at a time, and only PAST an entry
	// this bridge is finished with. A batch that fails halfway leaves
	// the cursor on the last entry that landed, so the retry after the
	// backoff re-reads from the message that did not — the alternative
	// drops it for ever, since fossil will never offer it again.
	n := 0
	if len(out.Msgs) == 0 || !out.Msgs[0].IsError {
		b.cleared()
	}
	for _, m := range out.Msgs {
		if m.IsError {
			msg := plainText(m.XMsg)
			if signedOut(msg) {
				if b.relogin(ctx) {
					return n, nil // the next poll goes out with the new session
				}
				return n, b.signedOutErr("chat-poll: " + msg)
			}
			return n, errors.New("chat-poll: " + msg)
		}
		if err := b.consume(m); err != nil {
			return n, err
		}
		b.advance(m.MsgID)
		n++
	}
	return n, nil
}

// consume turns one upstream entry into a channel line.
//
// nil means "this entry is finished with" — carried, or deliberately
// skipped — and only then may the caller advance the cursor past it.
// An error means the entry was NOT delivered: the caller stops the
// batch there, so the retry after the backoff starts at this same
// msgid. Advancing regardless would drop the message permanently,
// because fossil only ever offers what is newer than `name`.
//
// An entry the msgid map already knows was carried by an earlier poll
// — a replay, a cursor reset, or a second node briefly running the
// same bridge — and posting it again would double it in the channel.
func (b *Bridge) consume(m fossilMsg) error {
	if b.alreadyBridged(m.MsgID) {
		return nil
	}
	switch {
	case m.MDel != 0:
		// A deletion names the msgid it removed. Without a mapping the
		// message is either one the bridge never imported or one the
		// window has already dropped, and in both cases there is
		// nothing here to tombstone.
		if b.store == nil {
			return nil
		}
		seq, ok := b.store.ChatBridgeSeq(b.cfg.Repo, m.MDel)
		if !ok {
			return nil
		}
		tomb, err := b.post(b.cfg.Code, Message{
			Repo: b.cfg.Repo, Kind: KindDel, Ref: seq,
			Origin: "fossil", User: bridgeUser(m.XFrom),
		})
		if err != nil {
			return fmt.Errorf("tombstone for #%d: %w", seq, err)
		}
		// The tombstone's OWN msgid is mapped too, so a replayed
		// deletion is recognised as done rather than posted twice.
		b.remember(m.MsgID, tomb)
		return nil
	case b.skip(m):
		return nil
	default:
		text := b.textOf(m)
		if strings.TrimSpace(text) == "" {
			return nil
		}
		seq, err := b.post(b.cfg.Code, Message{
			Repo: b.cfg.Repo, Kind: KindMsg, Origin: "fossil",
			User: bridgeUser(m.XFrom), Text: text,
		})
		if err != nil {
			return fmt.Errorf("importing msgid %d: %w", m.MsgID, err)
		}
		b.remember(m.MsgID, seq)
		if b.Arrived != nil {
			b.Arrived(b.cfg.Repo, bridgeUser(m.XFrom), text, seq)
		}
		return nil
	}
}

// alreadyBridged reports whether this upstream msgid is already in the
// channel. Without a store there is no memory and every poll is taken
// at face value, which is the documented cost of running with none.
func (b *Bridge) alreadyBridged(msgid int64) bool {
	if b.store == nil || msgid == 0 {
		return false
	}
	_, ok := b.store.ChatBridgeSeq(b.cfg.Repo, msgid)
	return ok
}

func (b *Bridge) remember(msgid int64, seq uint64) {
	if b.store == nil {
		return
	}
	if err := b.store.MapChatBridgeMsg(b.cfg.Repo, msgid, seq); err != nil {
		log.Printf("chat: bridge %s: mapping msgid %d: %v", b.cfg.Repo, msgid, err)
	}
}

// cursor and setCursor are the only readers and writers of
// cfg.LastMsgID once Run has started. The poll loop owns it in
// practice, but it is read from outside — so it is taken under the
// lock rather than left as a field two goroutines happen to touch.
func (b *Bridge) cursor() int64 {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.cfg.LastMsgID
}

func (b *Bridge) setCursor(msgid int64) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.cfg.LastMsgID = msgid
}

func (b *Bridge) advance(msgid int64) {
	if msgid <= b.cursor() {
		return
	}
	b.setCursor(msgid)
	if b.store != nil {
		if err := b.store.SetChatBridgeCursor(b.cfg.Repo, msgid, time.Now()); err != nil {
			log.Printf("chat: bridge %s: cursor: %v", b.cfg.Repo, err)
		}
	}
}

// skip is the two reasons an inbound message is not carried.
//
// The loop guard is first: outbound posts arrive at the upstream under
// the bridge's own login, so re-importing them would echo every quarry
// message back into quarry. It skips only what the bridge itself wrote,
// told by send's "**user:** " prefix (echo). The login is often a
// person's own, and what they type on the fossil itself is theirs to see
// here: skipping every message from the login dropped all of it
// (libfossil, 2026-11-02).
//
// The second is the upstream's timeline robot, skipped only while
// quarry's own robot is announcing the same events (§10). Fossil does
// not expose `chat-timeline-user` on any endpoint a bridge login can
// read, so the name is used when /json/config volunteered it and the
// announcement's own SHAPE is used when it did not: fossil builds
// those messages with a trailing "(check-in: <a href=…" clause that no
// hand-typed message carries.
func (b *Bridge) skip(m fossilMsg) bool {
	if m.XFrom == "" || (strings.EqualFold(m.XFrom, b.cfg.User) && echo.MatchString(m.XMsg)) {
		return true
	}
	robotOn := b.RobotOn != nil && b.RobotOn(b.cfg.Repo)
	if !robotOn {
		return false
	}
	if b.timelineUser != "" {
		return strings.EqualFold(m.XFrom, b.timelineUser)
	}
	return timelineShape.MatchString(m.XMsg)
}

// echo is the shape send gives every outbound line: "**user:** text".
var echo = regexp.MustCompile(`^\*\*[^*\n]+:\*\* `)

// timelineShape is fossil's chat_msg_from_event() format: the event's
// own words followed by a parenthesised artifact clause it builds as
// HTML even in a raw message.
var timelineShape = regexp.MustCompile(`\((check-in|artifact|ticket|wiki|forum post|event): <a href=`)

// textOf is the message body as the channel should hold it: fossil's
// markdown verbatim, plus a link to any attachment. Fossil keeps the
// file itself; the bridge links it rather than copying it, because a
// copy in the channel window would be a second, shorter-lived truth.
func (b *Bridge) textOf(m fossilMsg) string {
	text := TidyFossil(m.XMsg, b.cfg.URL)
	if m.FSize <= 0 {
		return text
	}
	name := m.FName
	if name == "" {
		name = "attachment"
	}
	// The name is the upstream's, so it is escaped rather than
	// trusted: a filename holding `](javascript:...)` would otherwise
	// close this link and open one the uploader chose.
	link := fmt.Sprintf("[%s](%s/chat-download?name=%d) (%d bytes)",
		escapeLinkText(name), strings.TrimRight(b.cfg.URL, "/"), m.MsgID, m.FSize)
	if strings.TrimSpace(text) == "" {
		return link
	}
	return text + "\n\n" + link
}

// fossilAnchor is the one piece of HTML fossil writes into chat text
// itself: chat_msg_from_event()'s artifact clause, "(check-in: <a
// href='/r/x/info/abc'>abc</a>, user: <a href=...>u</a>)".
var fossilAnchor = regexp.MustCompile(`<a href=(?:'([^'<>]*)'|"([^"<>]*)")>([^<]*)</a>`)

// fossilFormatTag is the formatting markup fossil's own robots write
// into chat text: system-watchdog's "<h1 class='error'>Shutting down
// httpd ...</h1>", a timeline line's <b> or <span>. Only these names, and
// only as tags, so a person writing "use <div> here" in prose is not
// edited unless it is one.
var fossilFormatTag = regexp.MustCompile(`(?i)</?(?:h[1-6]|p|b|i|u|em|strong|span|div|code|pre|small|big|font|center|br|hr)(?:\s[^<>]*)?/?>`)

// TidyFossil is a fossil chat line as the channel holds it: its anchors
// as markdown links at upstream, its formatting tags as their words.
// With no upstream, an anchor keeps only its text: that is how a line
// stored before the bridge tidied is shown.
func TidyFossil(text, upstream string) string {
	return StripFossilFormatting(anchorsAsMarkdown(text, upstream))
}

// fossilBreakTag is a line break or rule, which separates words.
var fossilBreakTag = regexp.MustCompile(`(?i)<(?:br|hr)\s*/?>`)

// StripFossilFormatting drops fossil's formatting tags and keeps their
// words: the channel shows the sentence, not the markup (a libfossil
// digest quoted "<h1 class='error'>...</h1>" whole). Exported so a
// digest can read a line bridged before this the same way.
func StripFossilFormatting(text string) string {
	if !strings.Contains(text, "<") {
		return text
	}
	// A line break or rule becomes a space, so "a<br/>b" stays two words.
	text = fossilBreakTag.ReplaceAllString(text, " ")
	return strings.TrimSpace(fossilFormatTag.ReplaceAllString(text, ""))
}

// anchorsAsMarkdown turns fossil's anchors into markdown links at the
// upstream, so the channel shows a link and an email a word, not the
// markup (the libfossil digest quoted "<a href='/r/libfossil/info/...'>"
// whole). An href fossil wrote as a path is the upstream's, so it is
// joined to the upstream's origin; anything but http(s) or a path keeps
// only its text.
func anchorsAsMarkdown(text, upstream string) string {
	if !strings.Contains(text, "<a href=") {
		return text
	}
	origin := ""
	if u, err := url.Parse(upstream); err == nil && u.Scheme != "" && u.Host != "" {
		origin = u.Scheme + "://" + u.Host
	}
	return fossilAnchor.ReplaceAllStringFunc(text, func(a string) string {
		m := fossilAnchor.FindStringSubmatch(a)
		href, label := m[1]+m[2], escapeLinkText(html.UnescapeString(m[3]))
		switch {
		case strings.HasPrefix(href, "/") && !strings.HasPrefix(href, "//") && origin != "":
			href = origin + href
		case strings.HasPrefix(href, "https://"), strings.HasPrefix(href, "http://"):
		default:
			return label
		}
		return "[" + label + "](" + strings.NewReplacer("(", "%28", ")", "%29", " ", "%20").Replace(html.UnescapeString(href)) + ")"
	})
}

// bridgeUser is Decision G: a bridged line shows as `fossil:<xfrom>`.
// There is no account mapping — the prefix is the honest statement
// that this name belongs to another server's user table, not to a
// quarry account somebody could be confused for.
func bridgeUser(xfrom string) string {
	if xfrom == "" {
		return "fossil:anonymous"
	}
	return "fossil:" + xfrom
}

// outbound posts quarry's own messages upstream. Only a person's:
// robot lines are quarry's announcements about quarry, and a
// fossil-origin line came FROM there — echoing either back is the loop
// this half has to not be.
func (b *Bridge) outbound(ctx context.Context, ready <-chan struct{}) {
	msgs, cancel := b.room.Tail(b.cfg.Code, 0)
	defer cancel()
	// A send that fails is a bridge that is not connected, so it is
	// recorded where the status line reads from — an upstream
	// accepting polls while refusing sends would otherwise show as
	// healthy while half of it was down.
	//
	// A message is HELD until the upstream takes it. The next Tail
	// message is not read while one is pending, so a refused message
	// is retried rather than skipped, order is preserved, and the rest
	// queue at the broker — Tail never drops, so nothing is lost while
	// this loop waits.
	backoff := b.BackoffMin
	var lastLog time.Time
	var pending *Message
	for {
		if pending == nil {
			select {
			case <-ctx.Done():
				return
			case m, ok := <-msgs:
				if !ok {
					return
				}
				if m.Origin != "" || m.Kind != KindMsg || strings.TrimSpace(m.Text) == "" || b.isHeld() {
					continue // said during a takedown: never relayed, now or later
				}
				held := m
				pending = &held
			}
		}
		select {
		case <-ready:
		case <-ctx.Done():
			return
		}
		err := b.send(ctx, *pending)
		if errors.Is(err, errPaused) {
			pending = nil
			continue
		}
		if err == nil {
			pending = nil
			backoff = b.BackoffMin
			continue
		}
		if ctx.Err() != nil {
			return
		}
		b.fail(err)
		// One line per backoff window. A channel that is busy while
		// the upstream is down would otherwise write a line per
		// attempt, which buries every other bridge in the log.
		if b.clock().Sub(lastLog) >= backoff {
			log.Printf("chat: bridge %s: sending #%d: %v", b.cfg.Repo, pending.Seq, err)
			lastLog = b.clock()
		}
		wait, stop := b.after(err, backoff)
		if stop || !b.wait(ctx, wait) {
			return
		}
		if backoff *= 2; backoff > b.BackoffMax {
			backoff = b.BackoffMax
		}
	}
}

// fail records why the bridge is not working, for the status line.
// Both directions report here: "connected" has to mean both.
func (b *Bridge) fail(err error) {
	if b.store == nil || err == nil {
		return
	}
	b.smu.Lock()
	b.failed = true
	b.smu.Unlock()
	if serr := b.store.RecordChatBridgeError(b.cfg.Repo, err); serr != nil {
		log.Printf("chat: bridge %s: recording the failure: %v", b.cfg.Repo, serr)
	}
}

// send posts one quarry message to /chat-send. Fossil attributes
// everything to the login that sent it, so the author's name is
// carried in the text — the only place the protocol leaves for it.
func (b *Bridge) send(ctx context.Context, m Message) error {
	if b.isHeld() {
		return errPaused
	}
	user := m.User
	if user == "" {
		user = "anonymous"
	}
	form := url.Values{"msg": {"**" + user + ":** " + m.Text}}
	req, err := http.NewRequestWithContext(ctx, "POST",
		strings.TrimRight(b.cfg.URL, "/")+"/chat-send", strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	b.arm(req)
	resp, err := b.SendClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
	if resp.StatusCode != http.StatusOK {
		return b.statusError(resp, "chat-send to "+safeURL(b.cfg.URL))
	}
	// A refusal is a 200 carrying a JSON error object; success is an
	// empty body. Reading the status alone would call every rejected
	// message delivered.
	if e := errorIn(body); e != "" {
		if signedOut(e) {
			if b.relogin(ctx) {
				return b.send(ctx, m)
			}
			return b.signedOutErr("chat-send: " + e)
		}
		return errors.New("chat-send: " + e)
	}
	b.cleared()
	return nil
}

// statusError is a non-200 answer read for what Fossil means by it: a
// 401 is a refusal of the credential; 429 and 503 naming a wait are a
// server asking for time (a front-end's limiter, never Fossil's own),
// honoured; 429 and 503 naming none, and any 3xx or 4xx (a front end
// blocking the address answers 403, 404 or a redirect), are an answer
// that is not chat; any other 5xx is the peer failing and just backs off.
func (b *Bridge) statusError(resp *http.Response, what string) error {
	err := fmt.Errorf("%s: %s", what, resp.Status)
	switch {
	case resp.StatusCode == http.StatusUnauthorized:
		return refusedError{err}
	case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable:
		if wait, ok := b.retryAfter(resp.Header.Get("Retry-After")); ok {
			return &throttledError{error: err, wait: wait}
		}
		return unservedError{err}
	case resp.StatusCode >= 500:
		return err
	}
	return unservedError{err}
}

// retryAfter reads a Retry-After header, seconds or an HTTP date, and
// clamps it to [0, maxThrottleWait]; ok is false when it is absent or
// unreadable. The number is clamped as a number, before it becomes a
// Duration, so a huge one cannot overflow into a negative wait.
func (b *Bridge) retryAfter(v string) (time.Duration, bool) {
	v = strings.TrimSpace(v)
	if v == "" {
		return 0, false
	}
	if secs, err := strconv.ParseInt(v, 10, 64); err == nil || errors.Is(err, strconv.ErrRange) {
		limit := int64(maxThrottleWait / time.Second)
		if err != nil {
			secs = limit
			if strings.HasPrefix(v, "-") {
				secs = 0
			}
		}
		return time.Duration(min(max(secs, 0), limit)) * time.Second, true
	}
	if at, err := http.ParseTime(v); err == nil {
		return min(max(at.Sub(b.clock()), 0), maxThrottleWait), true
	}
	return 0, false
}

// signedOut reports whether an upstream refusal is fossil's "not
// logged in" answer: a session that lapsed, or one never had.
func signedOut(msg string) bool {
	m := strings.ToLower(msg)
	return strings.Contains(m, "not logged in")
}

// signedOutErr is a signed-out refusal, with the reason the bridge's
// last sign-in failed when there is one: "not logged in" alone sends the
// operator looking at the upstream, when the password is what is wrong.
func (b *Bridge) signedOutErr(msg string) error {
	b.smu.Lock()
	why := b.loginErr
	b.smu.Unlock()
	if why != nil {
		return refusedError{fmt.Errorf("%s (%v)", msg, why)}
	}
	return refusedError{errors.New(msg)}
}

// reloginEvery bounds how often a refusal may sign the bridge in again:
// a login the upstream accepts but whose account lacks the chat
// capability would otherwise loop login, refuse, login.
const reloginEvery = time.Minute

// relogin signs in again after a "not logged in" refusal, at most once
// a minute, and reports whether a fresh session is now held.
func (b *Bridge) relogin(ctx context.Context) bool {
	b.smu.Lock()
	recent := b.clock().Sub(b.loggedIn) < reloginEvery
	b.smu.Unlock()
	if recent || b.cfg.User == "" {
		return false
	}
	if err := b.login(ctx); err != nil {
		log.Printf("chat: bridge %s: %v", b.cfg.Repo, err)
		return false
	}
	return true
}

// login signs the bridge in through the upstream's /login form and keeps
// the session cookie it sets. The answer is a redirect, which this one
// request reads rather than follows (CheckRedirect, as bridgeClient
// explains, never follows one), so the password goes to the configured
// /login and nowhere else. No cookie means the upstream refused: a
// wrong password, or a login that does not exist there.
func (b *Bridge) login(ctx context.Context) error {
	if b.cfg.User == "" {
		return nil
	}
	if b.isHeld() {
		return errPaused
	}
	b.smu.Lock()
	b.loggedIn = b.clock()
	b.smu.Unlock()
	base := strings.TrimRight(b.cfg.URL, "/")
	form := url.Values{"u": {b.cfg.User}, "p": {b.cfg.Secret}, "g": {base + "/chat"}}
	req, err := http.NewRequestWithContext(ctx, "POST", base+"/login", strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	// fossil refuses a login whose Referer is not its own login page.
	req.Header.Set("Referer", base+"/login")
	c := *b.SendClient
	c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
	resp, err := c.Do(req)
	if err != nil {
		err = fmt.Errorf("chat bridge: signing in to %s: %w", safeURL(base), err)
		b.smu.Lock()
		b.loginErr = err
		b.smu.Unlock()
		return err
	}
	defer resp.Body.Close()
	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
	for _, ck := range resp.Cookies() {
		if strings.HasPrefix(ck.Name, "fossil-") && ck.Value != "" {
			b.smu.Lock()
			b.session, b.loginErr = ck.Name+"="+ck.Value, nil
			b.smu.Unlock()
			// Not cleared here: an account that signs in but has no chat
			// capability is refused on every poll, and a sign-in that
			// reset the count would keep it from ever stopping.
			return nil
		}
	}
	err = fmt.Errorf("chat bridge: %s refused the login for %s (wrong password, or no such user there)", safeURL(base), b.cfg.User)
	b.smu.Lock()
	b.loginErr = err
	b.smu.Unlock()
	// Only an answer that says "not you" is a refusal: Fossil's 401, or
	// its login form drawn again (200). Anything else without a cookie
	// is read by statusError: a limiter's wait, a front end's 403 or
	// redirect, or the server failing.
	if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized {
		return refusedError{err}
	}
	return b.statusError(resp, "chat bridge: signing in to "+safeURL(base))
}

func (b *Bridge) get(ctx context.Context, u string) ([]byte, error) {
	if b.isHeld() {
		return nil, errPaused
	}
	req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
	if err != nil {
		return nil, err
	}
	b.arm(req)
	resp, err := b.Client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, b.statusError(resp, safeURL(u))
	}
	return body, nil
}

// arm puts the credential and the same-origin Referer on a request.
// The Referer is not decoration: fossil treats a request without one
// as CSRF-unsafe and refuses /chat-poll's own table setup, which
// surfaces as a database error rather than as an auth failure.
func (b *Bridge) arm(req *http.Request) {
	b.smu.Lock()
	session := b.session
	b.smu.Unlock()
	switch {
	case session != "":
		req.Header.Set("Cookie", session)
	case b.cfg.User != "":
		// No session (login refused, or never tried): Basic auth, which
		// a fossil with http_authentication_ok on still accepts.
		req.SetBasicAuth(b.cfg.User, b.cfg.Secret)
	}
	req.Header.Set("Referer", strings.TrimRight(b.cfg.URL, "/")+"/chat")
}

// fetchTimelineUser asks the upstream for its chat-timeline-user once,
// at startup. Most fossil builds ship without the JSON API and answer
// HTML or 404; that is expected, costs one request, and leaves the
// shape heuristic in charge.
func (b *Bridge) fetchTimelineUser(ctx context.Context) string {
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	body, err := b.get(ctx, strings.TrimRight(b.cfg.URL, "/")+"/json/config/chat-timeline-user")
	if err != nil {
		return ""
	}
	var out struct {
		Payload map[string]any `json:"payload"`
	}
	if json.Unmarshal(body, &out) != nil {
		return ""
	}
	name, _ := out.Payload["chat-timeline-user"].(string)
	return name
}

// errorIn answers the message inside a fossil JSON error object, "" if
// the body is not one.
func errorIn(body []byte) string {
	var e struct {
		IsError bool   `json:"isError"`
		XMsg    string `json:"xmsg"`
	}
	if json.Unmarshal(body, &e) != nil || !e.IsError {
		return ""
	}
	return plainText(e.XMsg)
}

// plainText strips the anchor markup fossil puts in its own error
// strings, so a log line reads as a sentence.
func plainText(s string) string {
	var b strings.Builder
	depth := 0
	for _, r := range s {
		switch {
		case r == '<':
			depth++
		case r == '>' && depth > 0:
			depth--
		case depth == 0:
			b.WriteRune(r)
		}
	}
	return strings.TrimSpace(b.String())
}

// escapeLinkText makes an upstream-supplied string safe as the text of
// a markdown link: the brackets that would end it, and the backslash
// that would smuggle one back in, are escaped. Newlines go too — a
// link label cannot span lines, and one that tries ends the link.
func escapeLinkText(s string) string {
	r := strings.NewReplacer(
		`\`, `\\`, `[`, `\[`, `]`, `\]`, "\n", " ", "\r", " ")
	return r.Replace(s)
}