// Package semdiff is the comparison core of the disposability test
// (0001 §11): snapshots taken before and after a wipe-and-reingest must be
// *semantically* equal. HTML is compared as canonicalized DOM, JSON with
// sorted keys, timestamps normalized to UTC RFC3339 — so hostage bytes fail
// the build while harmless ordering and formatting drift does not.
package semdiff
import (
"bytes"
"encoding/json"
"fmt"
"regexp"
"slices"
"sort"
"strings"
"time"
"golang.org/x/net/html"
)
// Kind selects the canonicalizer.
type Kind int
const (
JSON Kind = iota
HTML
)
// Diff canonicalizes both inputs and returns "" when semantically equal,
// or a first-divergence report suitable for a test failure message.
func Diff(kind Kind, a, b []byte) (string, error) {
canon := CanonJSON
if kind == HTML {
canon = CanonHTML
}
ca, err := canon(a)
if err != nil {
return "", fmt.Errorf("semdiff: left: %w", err)
}
cb, err := canon(b)
if err != nil {
return "", fmt.Errorf("semdiff: right: %w", err)
}
if bytes.Equal(ca, cb) {
return "", nil
}
return firstDivergence(ca, cb), nil
}
// firstDivergence reports the first differing line plus a few following
// lines from each side — enough context to see *what* diverged (a dropped
// key's name lives just past the line where the streams split).
func firstDivergence(a, b []byte) string {
al := strings.Split(string(a), "\n")
bl := strings.Split(string(b), "\n")
for i := 0; i < len(al) || i < len(bl); i++ {
if line(al, i) != line(bl, i) {
var sb strings.Builder
fmt.Fprintf(&sb, "line %d:", i+1)
for j := i; j < i+4 && j < len(al); j++ {
fmt.Fprintf(&sb, "\n- %s", al[j])
}
for j := i; j < i+4 && j < len(bl); j++ {
fmt.Fprintf(&sb, "\n+ %s", bl[j])
}
return sb.String()
}
}
return "inputs differ (no line divergence?)"
}
func line(l []string, i int) string {
if i < len(l) {
return l[i]
}
return ""
}
// CanonJSON reparses, normalizes timestamp-shaped string leaves to UTC
// RFC3339, and re-marshals with indentation. encoding/json already emits
// map keys sorted, which gives the sorted-keys guarantee.
func CanonJSON(in []byte) ([]byte, error) {
var v any
if err := json.Unmarshal(in, &v); err != nil {
return nil, err
}
v = normalizeJSON(v)
return json.MarshalIndent(v, "", " ")
}
func normalizeJSON(v any) any {
switch t := v.(type) {
case map[string]any:
for k, e := range t {
t[k] = normalizeJSON(e)
}
case []any:
for i, e := range t {
t[i] = normalizeJSON(e)
}
case string:
return normalizeTimestamp(t)
}
return v
}
// timestampLayouts are the representations quarry's surfaces may emit.
// Extend as snapshot formats appear; a layout here means "this is a
// timestamp, unify it", never a guess over free text.
var timestampLayouts = []string{
time.RFC3339Nano,
time.RFC3339,
"2006-01-02 15:04:05Z07:00",
"2006-01-02 15:04:05",
}
// timestampShape gates parsing so ordinary strings ("1.2.3", "2026 was a
// year") are never touched: full-string date-time with optional zone.
var timestampShape = regexp.MustCompile(
`^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$`)
func normalizeTimestamp(s string) string {
if !timestampShape.MatchString(s) {
return s
}
for _, layout := range timestampLayouts {
if ts, err := time.Parse(layout, s); err == nil {
return ts.UTC().Format(time.RFC3339)
}
}
return s
}
// CanonHTML parses to a DOM and renders it back with attributes sorted,
// inter-tag whitespace collapsed, and timestamp-shaped text normalized.
// <pre>, <script>, <style>, and <textarea> subtrees keep their bytes.
func CanonHTML(in []byte) ([]byte, error) {
doc, err := html.Parse(bytes.NewReader(in))
if err != nil {
return nil, err
}
canonNode(doc, false)
var out bytes.Buffer
if err := html.Render(&out, doc); err != nil {
return nil, err
}
return out.Bytes(), nil
}
var verbatimTags = map[string]bool{
"pre": true, "script": true, "style": true, "textarea": true,
}
// volatileAttr marks content that legitimately differs between two runs
// of the same request — render timings, and anything else measured
// rather than derived. Canonicalization empties those elements so a
// comparison sees the state, not the stopwatch. Without this the
// disposability harness fails on "rendered in <1ms" vs "rendered in
// 4ms", which is a true observation about the clock and a false one
// about the index.
//
// Bare, it empties the element's children. Given a space-separated list
// of attribute names it blanks those instead, which is what a void
// element like <meta> needs: its payload is an attribute, and an
// og:url built from the host the reader arrived on is volatile in
// exactly this sense — it says nothing about the state being compared.
const volatileAttr = "data-volatile"
func canonNode(n *html.Node, verbatim bool) {
if n.Type == html.ElementNode {
for _, a := range n.Attr {
if a.Key != volatileAttr {
continue
}
if names := strings.Fields(a.Val); len(names) > 0 {
for i := range n.Attr {
if slices.Contains(names, n.Attr[i].Key) {
n.Attr[i].Val = ""
}
}
break
}
for c := n.FirstChild; c != nil; {
next := c.NextSibling
n.RemoveChild(c)
c = next
}
return
}
sort.Slice(n.Attr, func(i, j int) bool {
return n.Attr[i].Key < n.Attr[j].Key
})
verbatim = verbatim || verbatimTags[n.Data]
}
if n.Type == html.TextNode && !verbatim {
n.Data = collapseSpace(n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
canonNode(c, verbatim)
}
}
// collapseSpace unifies runs of whitespace and normalizes any
// timestamp-shaped token in place (T-form RFC3339 is one token; the
// space-separated layouts only appear in JSON surfaces).
func collapseSpace(s string) string {
fields := strings.Fields(s)
if len(fields) == 0 {
return ""
}
for i, f := range fields {
fields[i] = normalizeTimestamp(f)
}
return " " + strings.Join(fields, " ") + " "
}