Artifact ee062f4a988a0f60d53cf46d5a9a0ba9d9ed7b0254406028c02d2161828de0c3:
- File
conformance/.agent-config/trace/d2ad26f4-b70b-48a8-9dec-f86b91c13bcd.jsonl
— part of check-in
[57a558fcbf]
at
2026-10-10 23:52:34
on branch trunk
— test(conformance): harness for an RFC conformance suite (#262) (#264)
Adds the conformance package that the per-draft suites build on. Nothing in the repository has yet checked go-libfossil against the Fossil drafts requirement by requirement.
- A Case names its requirement. A Gap (an issue) or Deviation (a reason) marks a known difference. A marked case still runs: while it fails it is skipped, and once it conforms it fails until the mark is removed, so the list of known failures cannot go stale. - Requirements tables pin the SHA-256 of the drafts and errata they were written against. Those are copied unmodified into conformance/rfcs by scripts/refresh-conformance-rfcs.sh. A changed draft fails the coverage test until its table has been re-read and its pin moved. - Coverage is derived from the cases rather than recorded: - COVERED: every case conforms. - PARTIAL: the table's row says what its cases leave out. - DEVIATION: a case is marked as a deliberate difference. - FAILING: a case is marked with an open gap. - GAP: no case yet.
CONFORMANCE_REPORT=<path> writes the requirement-by-requirement report as Markdown. - fossil(1) is run on every case's input, through test-parse-manifest, so a case's expectation is checked against fossil each time the suite runs. withZ appends the Z card, so a case that edits one card never also breaks the checksum.
The harness's own rules are tested in harness_self_test.go. No suites are registered yet; the artifact-format suite is next (#263). (user: [email protected], size: 9342) [annotate] [blame] [check-ins using]
{"ts":"2026-10-10T23:43:45Z","tool":"Bash","args":{"command":"cd /Users/dmestas/projects/go-libfossil/conformance && python3 - <<'EOF'\np='coverage_test.go'\ns=open(p).read()\ns=s.replace('\\tsort.SliceStable(parts, func(i, j int) bool { return false })\\n','')\ns=s.replace('\\t\"sort\"\\n','')\nopen(p,'w').write(s)\nEOF","description":"Remove no-op sort from coverage test"},"status":"ok","duration_ms":818} {"ts":"2026-10-10T23:43:57Z","tool":"Write","args":{"file_path":"/Users/dmestas/projects/go-libfossil/conformance/fossil_test.go","content":"package conformance\n\nimport (\n\t\"crypto/md5\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/danmestas/go-libfossil/testutil\"\n)\n\n// fossilReference runs fossil(1) on the same input a case gives\n// go-libfossil, so every case's expectation is checked against fossil on\n// every run rather than once, when the case was written.\ntype fossilReference struct {\n\tbin string\n\tdir string\n\tcount int\n}\n\n// newFossilReference finds fossil(1). Without it the cases still run against\n// go-libfossil, but nothing checks their expectations; CI sets\n// REQUIRE_FOSSIL_BIN=1 so a missing binary fails there instead.\nfunc newFossilReference(t *testing.T) *fossilReference {\n\tt.Helper()\n\tbin := testutil.FossilBinary()\n\tif bin == \"\" {\n\t\tif os.Getenv(\"REQUIRE_FOSSIL_BIN\") == \"1\" {\n\t\t\tt.Fatal(\"REQUIRE_FOSSIL_BIN=1 but no fossil binary found\")\n\t\t}\n\t\tt.Log(\"fossil not found: case expectations are not checked against fossil(1)\")\n\t\treturn nil\n\t}\n\treturn &fossilReference{bin: bin, dir: t.TempDir()}\n}\n\n// parseArtifact reports whether fossil's manifest parser accepts data, and\n// fossil's reason when it does not. test-parse-manifest exits 0 either way,\n// so the verdict is read from its output.\nfunc (f *fossilReference) parseArtifact(data []byte) (accepted bool, reason string, err error) {\n\tif f == nil {\n\t\tpanic(\"conformance: parseArtifact without fossil\")\n\t}\n\tf.count++\n\tpath := filepath.Join(f.dir, fmt.Sprintf(\"artifact-%d\", f.count))\n\tif err := os.WriteFile(path, data, 0o644); err != nil {\n\t\treturn false, \"\", err\n\t}\n\toutput, err := exec.Command(f.bin, \"test-parse-manifest\", path).CombinedOutput()\n\tif err != nil {\n\t\treturn false, \"\", fmt.Errorf(\"fossil test-parse-manifest: %v: %s\", err, output)\n\t}\n\ttext := string(output)\n\tif strings.Contains(text, \"manifest_parse() worked\") {\n\t\treturn true, \"\", nil\n\t}\n\tfor _, line := range strings.Split(text, \"\\n\") {\n\t\tif strings.HasPrefix(line, \"ERROR: \") {\n\t\t\treturn false, strings.TrimPrefix(line, \"ERROR: \"), nil\n\t\t}\n\t}\n\treturn false, \"\", fmt.Errorf(\"fossil test-parse-manifest: unrecognized output %q\", text)\n}\n\n// withZ appends the Z card for body: the MD5 of every byte before it. Cases\n// build artifacts without one so that a single edit never also breaks the\n// checksum, which would make any rejection look deserved.\nfunc withZ(body string) []byte {\n\tif body != \"\" && !strings.HasSuffix(body, \"\\n\") {\n\t\tpanic(\"conformance: artifact body must end in a newline\")\n\t}\n\tsum := md5.Sum([]byte(body))\n\treturn []byte(body + \"Z \" + hex.EncodeToString(sum[:]) + \"\\n\")\n}\n"},"status":"ok","duration_ms":8} {"ts":"2026-10-10T23:44:17Z","tool":"Write","args":{"file_path":"/Users/dmestas/projects/go-libfossil/conformance/harness_self_test.go","content":"package conformance\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n)\n\n// The harness decides what a run means, so its own rules are tested here:\n// how a marked case is reported, how tables are read and audited, and how\n// a requirement's status is derived from its cases.\n\nfunc TestJudge(t *testing.T) {\n\tproblem := errors.New(\"rejected\")\n\ttests := []struct {\n\t\tname string\n\t\tc Case\n\t\terr error\n\t\twant outcome\n\t}{\n\t\t{\"conforms\", Case{ID: \"x.1\"}, nil, outcomePass},\n\t\t{\"fails\", Case{ID: \"x.1\"}, problem, outcomeFail},\n\t\t{\"known gap still failing\", Case{ID: \"x.1\", Gap: \"#1\"}, problem, outcomeSkip},\n\t\t{\"known gap now conforming\", Case{ID: \"x.1\", Gap: \"#1\"}, nil, outcomeFail},\n\t\t{\"deviation still present\", Case{ID: \"x.1\", Deviation: \"why\"}, problem, outcomeSkip},\n\t\t{\"deviation gone\", Case{ID: \"x.1\", Deviation: \"why\"}, nil, outcomeFail},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot, message := judge(tt.c, tt.err)\n\t\t\tif got != tt.want {\n\t\t\t\tt.Fatalf(\"judge = %d (%q), want %d\", got, message, tt.want)\n\t\t\t}\n\t\t\tif got != outcomePass && !strings.Contains(message, tt.c.ID) {\n\t\t\t\tt.Fatalf(\"message %q does not name the case\", message)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestJudgeRefusesDoubleMark(t *testing.T) {\n\tdefer func() {\n\t\tif recover() == nil {\n\t\t\tt.Fatal(\"a case marked both Gap and Deviation was accepted\")\n\t\t}\n\t}()\n\tjudge(Case{ID: \"x.1\", Gap: \"#1\", Deviation: \"why\"}, nil)\n}\n\nfunc TestParseTable(t *testing.T) {\n\tdata := \"# a comment\\n# draft\\td.txt\\tabc\\n# erratum\\te.md\\tdef\\n\\n\" +\n\t\t\"R-1\\t3.1\\tNames are hashes.\\n\" +\n\t\t\"R-2\\t3.2\\tLengths are fixed.\\tonly SHA1 checked\\n\"\n\ttbl, err := parseTable([]byte(data))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(tbl.sources) != 2 || tbl.sources[1].Path != \"erratum/e.md\" {\n\t\tt.Fatalf(\"sources = %+v\", tbl.sources)\n\t}\n\tif len(tbl.requirements) != 2 || tbl.requirements[1].Scope != \"only SHA1 checked\" {\n\t\tt.Fatalf(\"requirements = %+v\", tbl.requirements)\n\t}\n\tfor _, bad := range []string{\n\t\t\"R-1\\t3.1\\tNo pin.\\n\",\n\t\t\"# draft\\td.txt\\tabc\\nR-1\\t3.1\\n\",\n\t\t\"# draft\\td.txt\\nR-1\\t3.1\\tx\\n\",\n\t\t\"# draft\\td.txt\\tabc\\nR-1\\t\\tx\\n\",\n\t} {\n\t\tif _, err := parseTable([]byte(bad)); err == nil {\n\t\t\tt.Errorf(\"parseTable accepted %q\", bad)\n\t\t}\n\t}\n}\n\nfunc TestVerifySources(t *testing.T) {\n\tdir := t.TempDir()\n\tif err := os.WriteFile(filepath.Join(dir, \"d.txt\"), []byte(\"draft\\n\"), 0o644); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// SHA-256 of \"draft\\n\".\n\tgood := table{sources: []source{{Path: \"d.txt\",\n\t\tSHA256: \"d4ee4cff1b6e8e8ba9c3a1b0e5a0d3a1e6a0b2c7b1f0f8f4c1e0d5a3b2c1d0e9\"}}}\n\tproblems := verifySources(good, dir)\n\tif len(problems) != 1 || !strings.Contains(problems[0].Error(), \"move the pin\") {\n\t\tt.Fatalf(\"a wrong pin gave %v\", problems)\n\t}\n\tmissing := table{sources: []source{{Path: \"absent.txt\", SHA256: \"00\"}}}\n\tif problems := verifySources(missing, dir); len(problems) != 1 {\n\t\tt.Fatalf(\"a missing draft gave %v\", problems)\n\t}\n}\n\nfunc TestAudit(t *testing.T) {\n\ttbl := table{requirements: []requirement{{ID: \"R-1\"}, {ID: \"R-1\"}}}\n\ts := suite{name: \"x\", cases: []Case{\n\t\t{ID: \"x.a\", Req: \"R-1\"},\n\t\t{ID: \"x.a\", Req: \"R-1\"},\n\t\t{ID: \"y.b\", Req: \"R-1\"},\n\t\t{ID: \"x.c\", Req: \"R-9\"},\n\t\t{ID: \"x.d\", Req: \"R-1\", Gap: \"270\"},\n\t}}\n\tproblems := audit(s, tbl)\n\tfor _, want := range []string{\"R-1 listed twice\", \"x.a listed twice\", `y.b does not start`,\n\t\t`unknown requirement \"R-9\"`, \"must be an issue reference\"} {\n\t\tfound := false\n\t\tfor _, p := range problems {\n\t\t\tfound = found || strings.Contains(p.Error(), want)\n\t\t}\n\t\tif !found {\n\t\t\tt.Errorf(\"audit missed %q in %v\", want, problems)\n\t\t}\n\t}\n}\n\nfunc TestAssess(t *testing.T) {\n\trow := requirement{ID: \"R-1\"}\n\tscoped := requirement{ID: \"R-1\", Scope: \"SHA1 only\"}\n\ttests := []struct {\n\t\tname string\n\t\trow requirement\n\t\tcases []Case\n\t\twant status\n\t}{\n\t\t{\"no cases\", row, nil, statusGap},\n\t\t{\"all conform\", row, []Case{{ID: \"a\"}}, statusCovered},\n\t\t{\"scoped\", scoped, []Case{{ID: \"a\"}}, statusPartial},\n\t\t{\"deviation\", scoped, []Case{{ID: \"a\", Deviation: \"why\"}}, statusDeviation},\n\t\t{\"failing outranks deviation\", row,\n\t\t\t[]Case{{ID: \"a\", Deviation: \"why\"}, {ID: \"b\", Gap: \"#1\"}}, statusFailing},\n\t}\n\tfor _, tt := range tests {\n\t\tif got, _ := assess(tt.row, tt.cases); got != tt.want {\n\t\t\tt.Errorf(\"%s: assess = %s, want %s\", tt.name, got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestWithZ(t *testing.T) {\n\t// The Z card fossil computed for this body (fossil test-parse-manifest\n\t// reports \"expected 74f3096fa00a0995e85bb466be787372\" for it).\n\tgot := string(withZ(\"C x\\nD 2026-01-02T03:04:05\\n\"))\n\tif want := \"C x\\nD 2026-01-02T03:04:05\\nZ 74f3096fa00a0995e85bb466be787372\\n\"; got != want {\n\t\tt.Fatalf(\"withZ = %q, want %q\", got, want)\n\t}\n}\n\nfunc TestFossilReference(t *testing.T) {\n\tref := newFossilReference(t)\n\tif ref == nil {\n\t\tt.Skip(\"fossil not found\")\n\t}\n\taccepted, reason, err := ref.parseArtifact(withZ(\"C x\\nD 2026-01-02T03:04:05\\n\"))\n\tif err != nil || !accepted {\n\t\tt.Fatalf(\"fossil rejected a valid control artifact: %v %q\", err, reason)\n\t}\n\taccepted, reason, err = ref.parseArtifact(withZ(\"D 2026-01-02T03:04:05\\nC x\\n\"))\n\tif err != nil || accepted || reason == \"\" {\n\t\tt.Fatalf(\"fossil verdict on misordered cards: accepted=%v reason=%q err=%v\",\n\t\t\taccepted, reason, err)\n\t}\n}\n"},"status":"ok","duration_ms":12}