Artifact [afe72d3f88]

Artifact afe72d3f8875f5e758f9c5a488121c5576c0d029c2938e38a2580ce3e90acd33:

• File internal/xfer/decode.go — part of check-in [cc42ad8829] at 2026-07-23 09:52:23 on branch trunk — sync: store received wire frames verbatim instead of re-encoding (#132)

  * fix: store received wire frames verbatim instead of re-encoding

  Clone reproduced content correctly but not bytes: the receive path decompressed every cfile frame and re-compressed it with blob.Compress's unparameterized zlib writer, so a clone's blob.content never matched its source's — about 0.5% larger on the wapp corpus, purely from re-encoding frames that were already correctly encoded on the wire.

  xfer's cfile decoder now hands back StoredBlob alongside the decompressed Content: the same bytes recast into Fossil's on-disk blob format ([4-byte BE size][zlib data]) without a second zlib pass — verbatim when the wire already sent that format (Fossil's send_compressed_file case), or the received zlib stream with a length prefix bolted on when it didn't (our own encoder's plain-zlib case). storeReceivedFile/storeDeltaContent/ storeResolvedContent and blob.StoreDeltaRaw thread that through via a new blob.EncodeForStorage helper, which writes the verbatim bytes when given and only falls back to Compress when there's nothing to preserve — covering both the clone/pull and server-push receive paths.

  Locally authored content (never received over the wire) always takes the Compress fallback, so #71's deltification policy is untouched. The clone BUGGIFY corruption test drops storedBlob alongside the corrupted content it's exercising, so verbatim storage can't mask the corruption it exists to catch.

  Adds a corpus-level regression test cloning a real multi-revision fossil repo and asserting sum(length(content)) matches between source and clone.

  * test: use RequireFossilBin instead of removed HasFossil helper

  testutil.HasFossil was removed upstream in favor of RequireFossilBin; this test's skip guard needed the same update to compile against current main. (user: [email protected], size: 19329) [annotate] [blame] [check-ins using]

package xfer

import (
	"bufio"
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
	"strconv"
	"strings"

	"github.com/danmestas/go-libfossil/internal/deck"
)

// readPayload reads exactly size bytes from the reader.
func readPayload(r *bufio.Reader, size int) ([]byte, error) {
	if r == nil {
		panic("xfer.readPayload: r must not be nil")
	}
	if size < 0 {
		panic("xfer.readPayload: size must not be negative")
	}
	if size == 0 {
		return []byte{}, nil
	}
	buf := make([]byte, size)
	_, err := io.ReadFull(r, buf)
	if err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}
	return buf, nil
}

// readPayloadWithTrailingNewline reads size bytes then consumes one trailing \n.
func readPayloadWithTrailingNewline(r *bufio.Reader, size int) ([]byte, error) {
	buf, err := readPayload(r, size)
	if err != nil {
		return nil, err
	}
	b, err := r.ReadByte()
	if err == nil && b != '\n' {
		r.UnreadByte()
	}
	return buf, nil
}

// uvFileOmitsContent returns true if the flags indicate no binary payload follows.
func uvFileOmitsContent(flags int) bool {
	return flags&0x0001 != 0 || flags&0x0004 != 0
}

// DecodeCard reads one card from r. It skips comment lines (starting with #)
// and empty lines. Returns io.EOF when the reader is exhausted.
// Unrecognized command words produce an *UnknownCard (not an error).
func DecodeCard(r *bufio.Reader) (Card, error) {
	if r == nil {
		panic("xfer.DecodeCard: r must not be nil")
	}
	for {
		line, err := r.ReadString('\n')
		// Handle final line without trailing newline
		if err != nil && err != io.EOF {
			return nil, err
		}
		line = strings.TrimRight(line, "\n\r")

		// EOF with no data left
		if err == io.EOF && line == "" {
			return nil, io.EOF
		}

		// Skip empty lines and comments
		if line == "" || strings.HasPrefix(line, "#") {
			if err == io.EOF {
				return nil, io.EOF
			}
			continue
		}

		card, parseErr := parseLine(r, line)
		if parseErr != nil {
			return nil, parseErr
		}
		return card, nil
	}
}

// parseLine dispatches a trimmed, non-empty line to the appropriate card parser.
// The reader r is passed through for payload cards that need to read binary data.
func parseLine(r *bufio.Reader, line string) (Card, error) {
	fields := strings.Fields(line)
	if len(fields) == 0 {
		return nil, fmt.Errorf("xfer: empty line after split")
	}
	cmd := fields[0]
	args := fields[1:]

	switch cmd {
	case "igot":
		return parseIGot(args)
	case "gimme":
		return parseGimme(args)
	case "push":
		return parsePush(args)
	case "pull":
		return parsePull(args)
	case "cookie":
		return parseCookie(args)
	case "reqconfig":
		return parseReqConfig(args)
	case "private":
		return parsePrivate(args)
	case "clone":
		return parseClone(args)
	case "clone_seqno":
		return parseCloneSeqNo(args)
	case "uvgimme":
		return parseUVGimme(args)
	case "pragma":
		return parsePragma(args)
	case "login":
		return parseLogin(args)
	case "error":
		return parseError(args)
	case "message":
		return parseMessage(args)
	case "uvigot":
		return parseUVIGot(args)
	case "file":
		return parseFile(r, args)
	case "cfile":
		return parseCFile(r, args)
	case "config":
		return parseConfig(r, args)
	case "uvfile":
		return parseUVFile(r, args)
	case "schema":
		return parseSchema(r, args)
	case "xigot":
		return parseXIGot(args)
	case "xgimme":
		return parseXGimme(args)
	case "xrow":
		return parseXRow(r, args)
	case "xdelete":
		return parseXDelete(r, args)
	default:
		return &UnknownCard{Command: cmd, Args: args}, nil
	}
}

func parseIGot(args []string) (Card, error) {
	if len(args) < 1 || len(args) > 2 {
		return nil, fmt.Errorf("xfer: igot requires 1-2 args, got %d", len(args))
	}
	c := &IGotCard{UUID: args[0]}
	if len(args) == 2 && args[1] == "1" {
		c.IsPrivate = true
	}
	return c, nil
}

func parseGimme(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: gimme requires 1 arg, got %d", len(args))
	}
	return &GimmeCard{UUID: args[0]}, nil
}

// parsePush parses "push <project-code> [<server-code>]".
//
// Fossil C (src/xfer.c) always emits the project code; the server code is
// optional (present when the client has a cached server-code from a prior
// session). A bare "push\n" is rejected by real Fossil servers, so we treat
// 0-arg push as an error to tighten interop — our encoder now panics rather
// than emitting bare "push\n".
//
// Arg order: project-code first, server-code second (matches Fossil C).
func parsePush(args []string) (Card, error) {
	switch len(args) {
	case 0:
		return nil, fmt.Errorf("xfer: push requires at least 1 arg (project-code), got 0")
	case 1:
		return &PushCard{ProjectCode: args[0]}, nil
	case 2:
		return &PushCard{ProjectCode: args[0], ServerCode: args[1]}, nil
	default:
		return nil, fmt.Errorf("xfer: push requires 1-2 args, got %d", len(args))
	}
}

// parsePull parses "pull <project-code> <server-code>".
//
// Fossil C requires both args on pull. A bare "pull\n" or single-arg pull is
// rejected by real Fossil servers.
//
// Arg order: project-code first, server-code second (matches Fossil C).
func parsePull(args []string) (Card, error) {
	switch len(args) {
	case 0, 1:
		return nil, fmt.Errorf("xfer: pull requires 2 args (project-code server-code), got %d", len(args))
	case 2:
		return &PullCard{ProjectCode: args[0], ServerCode: args[1]}, nil
	default:
		return nil, fmt.Errorf("xfer: pull requires 2 args, got %d", len(args))
	}
}

func parseCookie(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: cookie requires 1 arg, got %d", len(args))
	}
	return &CookieCard{Value: args[0]}, nil
}

func parseReqConfig(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: reqconfig requires 1 arg, got %d", len(args))
	}
	return &ReqConfigCard{Name: args[0]}, nil
}

func parsePrivate(args []string) (Card, error) {
	if len(args) != 0 {
		return nil, fmt.Errorf("xfer: private takes 0 args, got %d", len(args))
	}
	return &PrivateCard{}, nil
}

// isDecimal reports whether s matches the spec's `decimal = 1*DIGIT`
// production (Appendix B). Tokens such as "-1" parse as integers but are
// not decimal, and several card rules turn on that distinction rather than
// on the parsed value.
func isDecimal(s string) bool {
	if s == "" {
		return false
	}
	for i := 0; i < len(s); i++ {
		if s[i] < '0' || s[i] > '9' {
			return false
		}
	}
	return true
}

func parseClone(args []string) (Card, error) {
	if len(args) != 0 && len(args) != 2 {
		return nil, fmt.Errorf("xfer: clone requires 0 or 2 args, got %d", len(args))
	}
	if len(args) == 0 {
		return &CloneCard{}, nil
	}
	// §8.1 withholds its fatal from a SEQNO that fails the digit-only rule
	// (isDecimal below) but a token strconv.Atoi cannot parse at all -- e.g.
	// "abc" or "3x" -- is rarer still and outside anything §8.1 discusses.
	// Erroring the whole message for it is harsher than the fatal the spec
	// was careful to withhold, so degrade the same way parseCloneSeqNo does
	// for clone_seqno: carry the raw tokens forward as an UnknownCard rather
	// than failing every later card in the message.
	v, vErr := strconv.Atoi(args[0])
	s, sErr := strconv.Atoi(args[1])
	if vErr != nil || sErr != nil {
		return &UnknownCard{Command: "clone", Args: args}, nil
	}
	return &CloneCard{
		Version: v,
		SeqNo:   s,
		// §8.1 keys its fatal on a digit-only SEQNO and explicitly withholds
		// it from one that fails digit-only recognition, so record whether
		// this token qualifies rather than only what it parsed to.
		SeqNoIsDecimal: isDecimal(args[1]),
	}, nil
}

func parseCloneSeqNo(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: clone_seqno requires 1 arg, got %d", len(args))
	}
	if !isDecimal(args[0]) {
		// §3.2 records NEXT only when it is decimal, and §8.2 makes a visible
		// but non-decimal NEXT a receiver-tolerance exception that leaves the
		// recorded sequence unchanged without stopping later reply cards.
		// Surfacing it as an unknown card keeps the token while denying it any
		// effect on the cursor — a CloneSeqNoCard here would be recorded.
		return &UnknownCard{Command: "clone_seqno", Args: args}, nil
	}
	s, err := strconv.Atoi(args[0])
	if err != nil {
		return nil, fmt.Errorf("xfer: clone_seqno: %w", err)
	}
	return &CloneSeqNoCard{SeqNo: s}, nil
}

func parseUVGimme(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: uvgimme requires 1 arg, got %d", len(args))
	}
	return &UVGimmeCard{Name: args[0]}, nil
}

func parsePragma(args []string) (Card, error) {
	if len(args) < 1 {
		return nil, fmt.Errorf("xfer: pragma requires at least 1 arg, got %d", len(args))
	}
	c := &PragmaCard{Name: args[0]}
	if len(args) > 1 {
		c.Values = args[1:]
	}
	return c, nil
}

func parseLogin(args []string) (Card, error) {
	if len(args) != 3 {
		return nil, fmt.Errorf("xfer: login requires 3 args, got %d", len(args))
	}
	return &LoginCard{
		User:      deck.FossilDecode(args[0]),
		Nonce:     args[1],
		Signature: args[2],
	}, nil
}

func parseError(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: error requires 1 arg, got %d", len(args))
	}
	return &ErrorCard{Message: deck.FossilDecode(args[0])}, nil
}

func parseMessage(args []string) (Card, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("xfer: message requires 1 arg, got %d", len(args))
	}
	return &MessageCard{Message: deck.FossilDecode(args[0])}, nil
}

func parseUVIGot(args []string) (Card, error) {
	if len(args) != 4 {
		return nil, fmt.Errorf("xfer: uvigot requires 4 args, got %d", len(args))
	}
	mtime, err := strconv.ParseInt(args[1], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("xfer: uvigot mtime: %w", err)
	}
	size, err := strconv.Atoi(args[3])
	if err != nil {
		return nil, fmt.Errorf("xfer: uvigot size: %w", err)
	}
	return &UVIGotCard{
		Name:  args[0],
		MTime: mtime,
		Hash:  args[2],
		Size:  size,
	}, nil
}

// --- Payload card parsers ---

// parseFile decodes: file UUID SIZE \n CONTENT  or  file UUID DELTASRC SIZE \n CONTENT
// No trailing newline after CONTENT.
func parseFile(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 2 && len(args) != 3 {
		return nil, fmt.Errorf("xfer: file requires 2-3 args, got %d", len(args))
	}
	c := &FileCard{UUID: args[0]}
	var sizeStr string
	if len(args) == 3 {
		c.DeltaSrc = args[1]
		sizeStr = args[2]
	} else {
		sizeStr = args[1]
	}
	size, err := strconv.Atoi(sizeStr)
	if err != nil {
		return nil, fmt.Errorf("xfer: file size: %w", err)
	}
	content, err := readPayload(r, size)
	if err != nil {
		return nil, fmt.Errorf("xfer: file payload: %w", err)
	}
	c.Content = content
	return c, nil
}

// parseCFile decodes: cfile UUID USIZE CSIZE \n ZCONTENT
// or: cfile UUID DELTASRC USIZE CSIZE \n ZCONTENT
// No trailing newline after ZCONTENT.
func parseCFile(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 3 && len(args) != 4 {
		return nil, fmt.Errorf("xfer: cfile requires 3-4 args, got %d", len(args))
	}
	c := &CFileCard{UUID: args[0]}
	var usizeStr, csizeStr string
	if len(args) == 4 {
		c.DeltaSrc = args[1]
		usizeStr = args[2]
		csizeStr = args[3]
	} else {
		usizeStr = args[1]
		csizeStr = args[2]
	}
	usize, err := strconv.Atoi(usizeStr)
	if err != nil {
		return nil, fmt.Errorf("xfer: cfile usize: %w", err)
	}
	csize, err := strconv.Atoi(csizeStr)
	if err != nil {
		return nil, fmt.Errorf("xfer: cfile csize: %w", err)
	}
	compressed, err := readPayload(r, csize)
	if err != nil {
		return nil, fmt.Errorf("xfer: cfile payload: %w", err)
	}
	// Decompress zlib. Fossil's server may send blobs in its blob format
	// ([4-byte BE size prefix][zlib data]) or plain zlib. Try plain first
	// (matches our own encoder), fall back to skipping a 4-byte prefix
	// (matches Fossil's send_compressed_file which sends raw blob content).
	// decompressCFile also hands back storedBlob: the same bytes recast
	// into Fossil's on-disk blob format, built without a second zlib pass,
	// so a receive path can persist storedBlob verbatim instead of
	// recompressing decompressed content and getting different bytes.
	decompressed, storedBlob, err := decompressCFile(compressed)
	if err != nil {
		return nil, err
	}
	// For non-delta cfiles, decompressed size == usize (full content).
	// For delta cfiles, decompressed size is the delta payload size, which
	// is smaller than usize (the fully expanded content size). Fossil's
	// send_compressed_file sets usize = blob.size (full content) regardless.
	// Only validate for non-delta cards where the sizes must match.
	if c.DeltaSrc == "" && len(decompressed) != usize {
		return nil, fmt.Errorf("xfer: cfile usize mismatch: header says %d, got %d", usize, len(decompressed))
	}
	c.USize = usize
	c.Content = decompressed
	c.StoredBlob = storedBlob
	return c, nil
}

// parseConfig decodes: config NAME SIZE \n CONTENT \n
// Note the trailing \n after CONTENT.
func parseConfig(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 2 {
		return nil, fmt.Errorf("xfer: config requires 2 args, got %d", len(args))
	}
	name := args[0]
	size, err := strconv.Atoi(args[1])
	if err != nil {
		return nil, fmt.Errorf("xfer: config size: %w", err)
	}
	content, err := readPayloadWithTrailingNewline(r, size)
	if err != nil {
		return nil, fmt.Errorf("xfer: config payload: %w", err)
	}
	return &ConfigCard{Name: name, Content: content}, nil
}

// parseUVFile decodes: uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
// FLAGS bits: 0x0001 = deleted, 0x0004 = content omitted. When set, no payload follows.
func parseUVFile(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 5 {
		return nil, fmt.Errorf("xfer: uvfile requires 5 args, got %d", len(args))
	}
	mtime, err := strconv.ParseInt(args[1], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("xfer: uvfile mtime: %w", err)
	}
	size, err := strconv.Atoi(args[3])
	if err != nil {
		return nil, fmt.Errorf("xfer: uvfile size: %w", err)
	}
	flags, err := strconv.Atoi(args[4])
	if err != nil {
		return nil, fmt.Errorf("xfer: uvfile flags: %w", err)
	}
	c := &UVFileCard{
		Name:  args[0],
		MTime: mtime,
		Hash:  args[2],
		Size:  size,
		Flags: flags,
	}
	if !uvFileOmitsContent(flags) {
		content, err := readPayload(r, size)
		if err != nil {
			return nil, fmt.Errorf("xfer: uvfile payload: %w", err)
		}
		c.Content = content
	}
	return c, nil
}

// decompressCFile decompresses a cfile card's wire payload and also returns
// storedBlob: the same bytes recast into Fossil's on-disk blob format
// ([4-byte BE uncompressed-size prefix][zlib data]), built without ever
// invoking zlib as a writer. This lets a caller persist storedBlob directly
// as blob.content -- reproducing whatever bytes the sender's own storage
// held -- instead of decompressing here and recompressing later, which
// would depend on this process's zlib writer parameters rather than the
// sender's.
func decompressCFile(data []byte) (decompressed []byte, storedBlob []byte, err error) {
	// Try plain zlib (matches our own encoder, which sends no prefix).
	if zr, zerr := zlib.NewReader(bytes.NewReader(data)); zerr == nil {
		out, readErr := io.ReadAll(zr)
		zr.Close()
		if readErr == nil {
			// data carries no size prefix. Fossil's on-disk format needs
			// one, so prepend it -- the zlib bytes themselves pass through
			// unchanged, so this is bookkeeping, not recompression.
			var buf bytes.Buffer
			if err := binary.Write(&buf, binary.BigEndian, uint32(len(out))); err != nil {
				return nil, nil, fmt.Errorf("xfer: cfile size prefix: %w", err)
			}
			buf.Write(data)
			return out, buf.Bytes(), nil
		}
	}
	// Try with 4-byte size prefix skip (Fossil blob format). Fossil's
	// send_compressed_file transmits the source's own blob.content bytes
	// unmodified, so data IS already the canonical on-disk form here --
	// nothing needs to be rebuilt.
	if len(data) > 4 {
		if zr, zerr := zlib.NewReader(bytes.NewReader(data[4:])); zerr == nil {
			out, readErr := io.ReadAll(zr)
			zr.Close()
			if readErr == nil {
				return out, data, nil
			}
		}
	}
	return nil, nil, fmt.Errorf("xfer: cfile zlib init: could not decompress payload (%d bytes)", len(data))
}

func parseSchema(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 5 {
		return nil, fmt.Errorf("xfer: schema requires 5 args, got %d", len(args))
	}
	version, err := strconv.Atoi(args[1])
	if err != nil {
		return nil, fmt.Errorf("xfer: schema version: %w", err)
	}
	mtime, err := strconv.ParseInt(args[3], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("xfer: schema mtime: %w", err)
	}
	size, err := strconv.Atoi(args[4])
	if err != nil {
		return nil, fmt.Errorf("xfer: schema size: %w", err)
	}
	const maxSchemaSize = 64 << 20 // 64 MiB — schema payloads should be small JSON.
	if size < 0 || size > maxSchemaSize {
		return nil, fmt.Errorf("xfer: schema size out of bounds: %d", size)
	}
	content, err := readPayloadWithTrailingNewline(r, size)
	if err != nil {
		return nil, fmt.Errorf("xfer: schema payload: %w", err)
	}
	return &SchemaCard{Table: args[0], Version: version, Hash: args[2], MTime: mtime, Content: content}, nil
}

func parseXIGot(args []string) (Card, error) {
	if len(args) != 3 {
		return nil, fmt.Errorf("xfer: xigot requires 3 args, got %d", len(args))
	}
	mtime, err := strconv.ParseInt(args[2], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("xfer: xigot mtime: %w", err)
	}
	return &XIGotCard{Table: args[0], PKHash: args[1], MTime: mtime}, nil
}

func parseXGimme(args []string) (Card, error) {
	if len(args) != 2 {
		return nil, fmt.Errorf("xfer: xgimme requires 2 args, got %d", len(args))
	}
	return &XGimmeCard{Table: args[0], PKHash: args[1]}, nil
}

func parseXDelete(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 4 {
		return nil, fmt.Errorf("xfer: xdelete requires 4 args, got %d", len(args))
	}
	mtime, err := strconv.ParseInt(args[2], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("xfer: xdelete mtime: %w", err)
	}
	size, err := strconv.Atoi(args[3])
	if err != nil {
		return nil, fmt.Errorf("xfer: xdelete size: %w", err)
	}
	const maxXDeleteSize = 64 << 20 // 64 MiB — PK data should be tiny JSON
	if size < 0 || size > maxXDeleteSize {
		return nil, fmt.Errorf("xfer: xdelete size out of range: %d", size)
	}
	pkData, err := readPayloadWithTrailingNewline(r, size)
	if err != nil {
		return nil, fmt.Errorf("xfer: xdelete payload: %w", err)
	}
	return &XDeleteCard{Table: args[0], PKHash: args[1], MTime: mtime, PKData: pkData}, nil
}

func parseXRow(r *bufio.Reader, args []string) (Card, error) {
	if len(args) != 4 {
		return nil, fmt.Errorf("xfer: xrow requires 4 args, got %d", len(args))
	}
	mtime, err := strconv.ParseInt(args[2], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("xfer: xrow mtime: %w", err)
	}
	size, err := strconv.Atoi(args[3])
	if err != nil {
		return nil, fmt.Errorf("xfer: xrow size: %w", err)
	}
	const maxXRowSize = 64 << 20 // 64 MiB — row payloads should be small JSON.
	if size < 0 || size > maxXRowSize {
		return nil, fmt.Errorf("xfer: xrow size out of bounds: %d", size)
	}
	content, err := readPayloadWithTrailingNewline(r, size)
	if err != nil {
		return nil, fmt.Errorf("xfer: xrow payload: %w", err)
	}
	return &XRowCard{Table: args[0], PKHash: args[1], MTime: mtime, Content: content}, nil
}