// container.go — the sync protocol's compressed message container and
// the card peeks quarry's edge needs (sync RFC §4.1, §6): a 4-byte
// big-endian pre-compression length followed by a zlib stream. Quarry
// inspects requests only enough to gate pushes; payload semantics stay
// zit's business.
package xfer
import (
"bytes"
"compress/zlib"
"crypto/sha1"
"crypto/subtle"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
)
// maxDecoded caps what one request may inflate to. The claimed length
// comes from the client, and zlib reaches about a thousand to one, so a
// request inside maxRequest could otherwise claim and inflate to 4 GiB.
const maxDecoded = 2 * maxRequest
// errTooLarge is a container that claims, or inflates to, more than the
// caller allows. It is not malformed: treating it as plaintext would
// hand the same bytes to the next reader, which inflates them in full.
var errTooLarge = errors.New("container inflates past the limit")
// containerClaim returns the length a compressed container claims, and
// whether raw is shaped like one at all: a length and a valid zlib
// header (RFC 1950 §2.2, as compress/zlib checks it). A plaintext body
// practically never starts that way: its fifth byte would have to name
// deflate, which no card name has there.
func containerClaim(raw []byte) (uint32, bool) {
if len(raw) < 6 {
return 0, false
}
cmf, flg := raw[4], raw[5]
if cmf&0x0f != 8 || cmf>>4 > 7 || (uint(cmf)<<8|uint(flg))%31 != 0 || flg&0x20 != 0 {
return 0, false
}
return binary.BigEndian.Uint32(raw[:4]), true
}
// openContainer returns the decompressed body of an
// application/x-fossil message, and whether the bytes were a compressed
// container, so a message the front changes can go back on the wire in
// the form it arrived in. Malformed containers return the bytes as-is
// (the RFC: the receiver retains the original buffer and applies
// ordinary card processing to it).
//
// It never inflates past limit. A container claiming more is refused on
// its claim, and one whose stream runs past its claim is cut there and
// refused, so neither is passed on as plaintext. Both return raw with
// errTooLarge.
func openContainer(raw []byte, limit int64) ([]byte, bool, error) {
capacity, ok := containerClaim(raw)
if !ok {
return raw, false, nil
}
if int64(capacity) > limit {
return raw, false, errTooLarge
}
zr, err := zlib.NewReader(bytes.NewReader(raw[4:]))
if err != nil {
return raw, false, nil
}
defer zr.Close()
out, err := io.ReadAll(io.LimitReader(zr, int64(capacity)+1))
if uint64(len(out)) > uint64(capacity) {
return raw, false, errTooLarge
}
if err != nil {
return raw, false, nil
}
return out, true, nil
}
// decodeBudget bounds the decoded request bytes held across the whole
// process. maxDecoded bounds one request, but nothing bounds how many
// arrive at once, and a handful at the limit would exhaust the host.
// It counts inflated bodies only; the compressed bodies they came from
// are bounded per request by maxRequest and not in total.
type decodeBudget struct {
mu sync.Mutex
used, size int64
}
// requestBudget is shared by every Fronter in the process, because the
// memory it protects is.
var requestBudget = &decodeBudget{size: 2 << 30}
// take reserves n bytes, or reports that they are not to be had.
func (b *decodeBudget) take(n int64) bool {
b.mu.Lock()
defer b.mu.Unlock()
if b.used+n > b.size {
return false
}
b.used += n
return true
}
// give returns n bytes taken earlier.
func (b *decodeBudget) give(n int64) {
b.mu.Lock()
b.used -= n
b.mu.Unlock()
}
// encodeContainer wraps a plaintext message for the wire.
func encodeContainer(body []byte) []byte {
var buf bytes.Buffer
var lenb [4]byte
binary.BigEndian.PutUint32(lenb[:], uint32(len(body)))
buf.Write(lenb[:])
zw := zlib.NewWriter(&buf)
zw.Write(body)
zw.Close()
return buf.Bytes()
}
// pushRequest holds what the gate needs from a request: whether it
// mutates, and the login card's parts with the nonce-covered region.
type pushRequest struct {
hasPush bool
// newClone is a clone card opening a NEW clone, not a later round of
// one. The protocol says "first sequence is 1" (sync draft §8), so
// SEQNO 1 is the only round that means somebody started cloning —
// counting every clone card would count ROUNDS, and a big store
// takes many.
newClone bool
// hasPull is a pull card. Unlike newClone this is per ROUND, because
// the protocol gives a fetch no opening marker: "push, pull, and
// sync requests carry push, pull, or both each round".
hasPull bool
// version is the client's self-reported build, from the
// "pragma client-version SEQ DATE TIME" card. Self-reported and
// unverifiable — good enough to answer "may I retire the old
// protocol path", never evidence of anything.
version string
projectCode string // from the push card
user string
nonce string
sig string
// coveredSum is the SHA-1 of the bytes the login signs: after the
// login line's LF, through the end. The digest, not the bytes, so a
// parsed request never keeps the decoded body alive.
coveredSum [sha1.Size]byte
// coveredFrom is where covered starts in the body: a card before it
// is not signed by the login, whatever the login verifies as.
coveredFrom int
// firstWrite is where the first write-bearing card (push, file,
// cfile, config) starts in the body, or -1 for none. One before
// coveredFrom is nobody's, as fossil reads it: its page_xfer runs
// the cards in order, and before the login card it is nobody.
firstWrite int
// uv is the request's unversioned cards (RFC 0020), in order. The
// front answers them itself and the child never sees one.
uv []uvCard
// uvWrite is a uvfile card: a write, gated on y rather than i.
uvWrite bool
// dagWrite is a card the child would store (file, cfile, config),
// which an unversioned-only credential may not carry.
dagWrite bool
// uvBad is fossil's words for an unversioned card the scan could
// not frame; the request fails closed and the front refuses it.
uvBad string
// files is each file card's artifact and where it starts, so the
// front can say who carried what: only a card the login signs is
// the login's.
files []fileCard
}
type fileCard struct {
hash string
start int
}
// isFossilWS matches the sync protocol's card-token separators (§5.2:
// SP, HTAB, VT, FF, CR — LF terminates the line, never separates). The
// gate MUST tokenize exactly as zit's tokenizeCard does: a card the
// backend honors but the gate misses is a full auth bypass. In
// particular leading whitespace is skippable, so a card keyword is not
// anchored at byte 0.
func isFossilWS(r rune) bool {
switch r {
case ' ', '\t', 0x0b, 0x0c, '\r':
return true
}
return false
}
// cardTokens tokenizes one card line the way zit does — runs of Fossil
// whitespace (including a leading run) separate tokens.
func cardTokens(line []byte) []string {
return strings.FieldsFunc(string(line), isFossilWS)
}
// scanRequest reads the decompressed card stream CARD BY CARD, framed
// the way zit's strict request scan frames it (sync.zig, request pass 1):
//
// - a "file" card (3 or 4 tokens, SIZE last) is followed by exactly SIZE
// payload bytes and no separator; the next card starts right after;
// - a "config" card (NAME SIZE) is followed by SIZE bytes and one
// mandatory LF;
// - "cfile" is outside a request's vocabulary, and zit aborts the
// whole request on it;
// - "uvfile NAME MTIME HASH SIZE FLAGS" (six tokens) is followed by
// SIZE payload bytes and no separator, unless SIZE is 0 or FLAGS
// has 0x0001 (deleted) or 0x0004 (content omitted) set, which is
// exactly when fossil's xfer_accept_unversioned_file reads none;
// "uvgimme NAME" and "pragma uv-hash HASH" are one line each. These
// three are RFC 0020's, answered by the front and cut out of what
// the child sees, so each is recorded with the span it occupies.
//
// Payload bytes are never read as cards. Reading them as cards is how a
// pushed Go file holding an SQL column named "login" became a login
// card for user "TEXT", which beat the real x-f-l-c cookie and refused
// every push carrying that file (zit#213). A line in a payload reading
// "clone 3 1" or a "push" card with a project code did the same to the
// other fields.
//
// When the framing cannot be followed (bad arity, a SIZE this parser
// does not read exactly as zit would, a payload running past the end)
// the scan FAILS CLOSED: it marks the request a push, so it needs a
// verified credential, and stops. zit aborts such a request anyway, so
// no valid client is affected, and a parsing difference between this
// gate and zit can only ever refuse, never let a write through
// unauthenticated. (See isFossilWS: a card the backend honours but the
// gate misses is an auth bypass.)
//
// The RFC places the login card first; zit emits it after the
// client-version pragma, so the first login card in any position is
// the one used, and its nonce covers the bytes after its LF.
// Classification is by first token after Fossil-whitespace skipping,
// never a raw byte-prefix, which an indented card would defeat while
// the backend still honoured it.
func scanRequest(body []byte) pushRequest {
pr := pushRequest{firstWrite: -1}
seenLogin := false
failClosed := func() pushRequest {
pr.hasPush, pr.dagWrite = true, true
return pr
}
uvFailClosed := func(why string) pushRequest {
pr.uvBad = why
return failClosed()
}
pos := 0
for pos < len(body) {
start := pos
line := body[pos:]
next := len(body)
if k := bytes.IndexByte(line, '\n'); k >= 0 {
line, next = line[:k], pos+k+1
}
pos = next
f := cardTokens(line)
if len(f) == 0 {
continue
}
switch f[0] {
case "push", "file", "cfile", "config":
if pr.firstWrite < 0 {
pr.firstWrite = start
}
}
switch f[0] {
case "file", "cfile", "config":
pr.dagWrite = true
}
switch f[0] {
case "login":
if !seenLogin {
seenLogin = true
if len(f) == 4 {
pr.user, pr.nonce, pr.sig = defossilize(f[1]), f[2], f[3]
pr.coveredSum, pr.coveredFrom = sha1.Sum(body[next:]), next
}
}
case "push":
pr.hasPush = true
if len(f) == 3 {
pr.projectCode = f[2]
}
case "pull":
pr.hasPull = true
case "pragma":
// pragma client-version SEQ [DATE TIME]. Only the sequence
// is kept: it is the part that orders builds, and the
// date pair adds nothing a version does not already say.
if len(f) >= 3 && f[1] == "client-version" {
pr.version = f[2]
}
if len(f) >= 2 && f[1] == "uv-hash" {
pr.uv = append(pr.uv, uvCard{kind: "uv-hash", f: f, start: start, end: pos})
}
case "clone":
// clone VERSION SEQNO. Arity is checked because a malformed
// card must not be counted as a clone, and SEQNO is compared
// as a string: the spec's own rule is digit-only recognition,
// so "01" is not sequence one and neither is " 1".
if len(f) == 3 && f[2] == "1" {
pr.newClone = true
}
case "file":
size, ok := payloadSize(f, 3, 4)
if !ok || size > len(body)-pos {
return failClosed()
}
pos += size
pr.files = append(pr.files, fileCard{hash: f[1], start: start})
case "config":
size, ok := payloadSize(f, 3, 3)
if !ok || size >= len(body)-pos || body[pos+size] != '\n' {
return failClosed()
}
pos += size + 1
case "cfile":
return failClosed()
case "uvfile":
c, ok := frameUVFile(f, body, start, pos)
if !ok {
return uvFailClosed("malformed uvfile line")
}
pos = c.end
pr.uv = append(pr.uv, c)
pr.uvWrite = true
case "uvgimme":
if len(f) != 2 {
return uvFailClosed("bad command: uvgimme")
}
pr.uv = append(pr.uv, uvCard{kind: "uvgimme", f: f, start: start, end: pos})
}
}
return pr
}
// uvCard is one unversioned card: its kind ("uvfile", "uvgimme" or
// "uv-hash"), its tokens, a uvfile's payload (nil when none was sent),
// and the bytes it occupies in the body, payload included.
type uvCard struct {
kind string
f []string
flags int
payload []byte
start, end int
}
// frameUVFile frames "uvfile NAME MTIME HASH SIZE FLAGS" whose line
// starts at start and ends before pos: SIZE payload bytes follow with no
// separator, unless SIZE is 0 or FLAGS has 0x0001 (deleted) or 0x0004
// (content omitted) set, which is exactly when fossil's
// xfer_accept_unversioned_file reads none. The card's end is the first
// byte after it. ok is false when the line cannot be framed. A request
// and a reply frame the card the same way, so both use this.
func frameUVFile(f []string, body []byte, start, pos int) (uvCard, bool) {
size, ok := payloadSize(f[:min(len(f), 5)], 5, 5)
flags, fok := payloadSize(f[min(len(f), 5):], 1, 1)
if !ok || !fok || len(f) != 6 {
return uvCard{}, false
}
c := uvCard{kind: "uvfile", f: f, flags: flags, start: start}
if size > 0 && flags&0x0005 == 0 {
if size > len(body)-pos {
return uvCard{}, false
}
c.payload = body[pos : pos+size]
pos += size
}
c.end = pos
return c, true
}
// omitted reports a uvfile whose content was left out on purpose (flag
// 0x0004), as opposed to an empty file or a deletion.
func (c uvCard) omitted() bool { return c.flags&0x0004 != 0 && c.f[3] != "-" }
// stripUV is body without its unversioned cards: what the child is
// sent. The spans are in order and do not overlap, since the scan
// produced them walking forward.
func stripUV(body []byte, cards []uvCard) []byte {
out := make([]byte, 0, len(body))
pos := 0
for _, c := range cards {
out = append(out, body[pos:c.start]...)
pos = c.end
}
return append(out, body[pos:]...)
}
// payloadSize reads a payload card's SIZE, its last token, when the
// card has between min and max tokens. Only plain decimal digits are
// accepted: zit parses SIZE as a u32, and anything this does not read
// exactly the same way (a sign, a separator, an overflow) is reported
// as unreadable so the caller fails closed rather than guessing.
func payloadSize(f []string, min, max int) (int, bool) {
if len(f) < min || len(f) > max {
return 0, false
}
s := f[len(f)-1]
if s == "" || len(s) > 10 {
return 0, false
}
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return 0, false
}
}
n, err := strconv.ParseUint(s, 10, 32)
if err != nil {
return 0, false
}
return int(n), true
}
// cookieLogin parses the x-f-l-c cookie (RFC §6.5): the body-login
// card with spaces as + and the USER percent-encoded. Its nonce covers
// the entire decompressed body.
func cookieLogin(req *http.Request, body []byte) (pushRequest, bool) {
c, err := req.Cookie("x-f-l-c")
if err != nil {
return pushRequest{}, false
}
f := strings.Split(c.Value, "+")
if len(f) != 4 || f[0] != "login" {
return pushRequest{}, false
}
user, err := url.QueryUnescape(f[1])
if err != nil {
return pushRequest{}, false
}
return pushRequest{user: defossilize(user), nonce: f[2], sig: f[3], coveredSum: sha1.Sum(body)}, true
}
// verifyLogin checks the RFC §6 chain: NONCE is SHA-1 over the covered
// bytes; SIGNATURE is SHA-1 over nonce-hex || shared-secret-hex.
func verifyLogin(pr pushRequest, secret string) bool {
if pr.user == "" || len(pr.nonce) != 40 || len(pr.sig) != 40 {
return false
}
if hex.EncodeToString(pr.coveredSum[:]) != pr.nonce {
return false
}
sig := sha1.Sum([]byte(pr.nonce + secret))
return subtle.ConstantTimeCompare([]byte(hex.EncodeToString(sig[:])), []byte(pr.sig)) == 1
}
// SharedSecret derives the stored credential (RFC §6.3): SHA-1 of
// "projectcode/user/password". Plaintext never persists.
func SharedSecret(projectCode, user, password string) string {
sum := sha1.Sum([]byte(projectCode + "/" + user + "/" + password))
return hex.EncodeToString(sum[:])
}
// errorReply builds a complete application/x-fossil reply holding one
// error card (message spaces fossilized as \s).
func errorReply(msg string) []byte {
return encodeContainer(errorCard(msg))
}
// errorCard is one error card, uncompressed, for adding to a reply.
func errorCard(msg string) []byte {
return []byte("error " + strings.ReplaceAll(msg, " ", `\s`) + "\n")
}
// defossilize is fossil's src/encode.c defossilize, fossilize's
// inverse: \n \s \t \r \v \f \\ are their characters, any other
// escaped character is itself, a trailing backslash stays, and \0 ends
// the token, as a NUL ends the C string fossil reads it into.
func defossilize(s string) string {
i := strings.IndexByte(s, '\\')
if i < 0 {
return s
}
var b strings.Builder
b.WriteString(s[:i])
for ; i < len(s); i++ {
c := s[i]
if c == '\\' && i+1 < len(s) {
i++
switch c = s[i]; c {
case 'n':
c = '\n'
case 's':
c = ' '
case 't':
c = '\t'
case 'r':
c = '\r'
case 'v':
c = '\v'
case 'f':
c = '\f'
case '0':
return b.String()
}
}
b.WriteByte(c)
}
return b.String()
}
// ErrNoProjectCode reports a handshake that produced no push card.
var ErrNoProjectCode = fmt.Errorf("handshake reply carried no project code")