Artifact [1f211b42b5]

Artifact 1f211b42b57ec3e66fed3e86214f891c331929985e1188daee77edce4c2aacd5:

• File internal/semdiff/semdiff.go — part of check-in [cb17edbefa] at 2026-08-17 20:32:05 on branch trunk — web: report render time in the footer

  Fossil publishes its own ('This page was generated in about 0.003s') and that is what made our timeline regression measurable against it. Same number, same place, every page.

  semdiff gains data-volatile so the disposability harness compares state rather than the stopwatch — without it a wipe-and-reingest fails on '<1ms' vs '4ms', which is true about the clock and false about the index. (user: dmestas, size: 5445) [annotate] [blame] [check-ins using]

// Package semdiff is the comparison core of the disposability test
// (0001 §11): snapshots taken before and after a wipe-and-reingest must be
// *semantically* equal. HTML is compared as canonicalized DOM, JSON with
// sorted keys, timestamps normalized to UTC RFC3339 — so hostage bytes fail
// the build while harmless ordering and formatting drift does not.
package semdiff

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"golang.org/x/net/html"
)

// Kind selects the canonicalizer.
type Kind int

const (
	JSON Kind = iota
	HTML
)

// Diff canonicalizes both inputs and returns "" when semantically equal,
// or a first-divergence report suitable for a test failure message.
func Diff(kind Kind, a, b []byte) (string, error) {
	canon := CanonJSON
	if kind == HTML {
		canon = CanonHTML
	}
	ca, err := canon(a)
	if err != nil {
		return "", fmt.Errorf("semdiff: left: %w", err)
	}
	cb, err := canon(b)
	if err != nil {
		return "", fmt.Errorf("semdiff: right: %w", err)
	}
	if bytes.Equal(ca, cb) {
		return "", nil
	}
	return firstDivergence(ca, cb), nil
}

// firstDivergence reports the first differing line plus a few following
// lines from each side — enough context to see *what* diverged (a dropped
// key's name lives just past the line where the streams split).
func firstDivergence(a, b []byte) string {
	al := strings.Split(string(a), "\n")
	bl := strings.Split(string(b), "\n")
	for i := 0; i < len(al) || i < len(bl); i++ {
		if line(al, i) != line(bl, i) {
			var sb strings.Builder
			fmt.Fprintf(&sb, "line %d:", i+1)
			for j := i; j < i+4 && j < len(al); j++ {
				fmt.Fprintf(&sb, "\n- %s", al[j])
			}
			for j := i; j < i+4 && j < len(bl); j++ {
				fmt.Fprintf(&sb, "\n+ %s", bl[j])
			}
			return sb.String()
		}
	}
	return "inputs differ (no line divergence?)"
}

func line(l []string, i int) string {
	if i < len(l) {
		return l[i]
	}
	return ""
}

// CanonJSON reparses, normalizes timestamp-shaped string leaves to UTC
// RFC3339, and re-marshals with indentation. encoding/json already emits
// map keys sorted, which gives the sorted-keys guarantee.
func CanonJSON(in []byte) ([]byte, error) {
	var v any
	if err := json.Unmarshal(in, &v); err != nil {
		return nil, err
	}
	v = normalizeJSON(v)
	return json.MarshalIndent(v, "", "  ")
}

func normalizeJSON(v any) any {
	switch t := v.(type) {
	case map[string]any:
		for k, e := range t {
			t[k] = normalizeJSON(e)
		}
	case []any:
		for i, e := range t {
			t[i] = normalizeJSON(e)
		}
	case string:
		return normalizeTimestamp(t)
	}
	return v
}

// timestampLayouts are the representations quarry's surfaces may emit.
// Extend as snapshot formats appear; a layout here means "this is a
// timestamp, unify it", never a guess over free text.
var timestampLayouts = []string{
	time.RFC3339Nano,
	time.RFC3339,
	"2006-01-02 15:04:05Z07:00",
	"2006-01-02 15:04:05",
}

// timestampShape gates parsing so ordinary strings ("1.2.3", "2026 was a
// year") are never touched: full-string date-time with optional zone.
var timestampShape = regexp.MustCompile(
	`^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$`)

func normalizeTimestamp(s string) string {
	if !timestampShape.MatchString(s) {
		return s
	}
	for _, layout := range timestampLayouts {
		if ts, err := time.Parse(layout, s); err == nil {
			return ts.UTC().Format(time.RFC3339)
		}
	}
	return s
}

// CanonHTML parses to a DOM and renders it back with attributes sorted,
// inter-tag whitespace collapsed, and timestamp-shaped text normalized.
// <pre>, <script>, <style>, and <textarea> subtrees keep their bytes.
func CanonHTML(in []byte) ([]byte, error) {
	doc, err := html.Parse(bytes.NewReader(in))
	if err != nil {
		return nil, err
	}
	canonNode(doc, false)
	var out bytes.Buffer
	if err := html.Render(&out, doc); err != nil {
		return nil, err
	}
	return out.Bytes(), nil
}

var verbatimTags = map[string]bool{
	"pre": true, "script": true, "style": true, "textarea": true,
}

// volatileAttr marks content that legitimately differs between two runs
// of the same request — render timings, and anything else measured
// rather than derived. Canonicalization empties those elements so a
// comparison sees the state, not the stopwatch. Without this the
// disposability harness fails on "rendered in <1ms" vs "rendered in
// 4ms", which is a true observation about the clock and a false one
// about the index.
const volatileAttr = "data-volatile"

func canonNode(n *html.Node, verbatim bool) {
	if n.Type == html.ElementNode {
		for _, a := range n.Attr {
			if a.Key == volatileAttr {
				for c := n.FirstChild; c != nil; {
					next := c.NextSibling
					n.RemoveChild(c)
					c = next
				}
				return
			}
		}
		sort.Slice(n.Attr, func(i, j int) bool {
			return n.Attr[i].Key < n.Attr[j].Key
		})
		verbatim = verbatim || verbatimTags[n.Data]
	}
	if n.Type == html.TextNode && !verbatim {
		n.Data = collapseSpace(n.Data)
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		canonNode(c, verbatim)
	}
}

// collapseSpace unifies runs of whitespace and normalizes any
// timestamp-shaped token in place (T-form RFC3339 is one token; the
// space-separated layouts only appear in JSON surfaces).
func collapseSpace(s string) string {
	fields := strings.Fields(s)
	if len(fields) == 0 {
		return ""
	}
	for i, f := range fields {
		fields[i] = normalizeTimestamp(f)
	}
	return " " + strings.Join(fields, " ") + " "
}