package markup
import (
"encoding/json"
"os"
"strings"
"testing"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
)
// --- conformance corpus (third_party/sanitizer-corpus, see ledger) ---
type corpusCase struct {
Name string `json:"name"`
Input string `json:"input"`
Deny []string `json:"deny"`
Allow []string `json:"allow"`
}
type corpusFile struct {
Source string `json:"source"`
Cases []corpusCase `json:"cases"`
}
func TestConformanceCorpus(t *testing.T) {
for _, name := range []string{"gitea", "forgejo"} {
raw, err := os.ReadFile("../../third_party/sanitizer-corpus/" + name + ".json")
if err != nil {
t.Fatal(err)
}
var cf corpusFile
if err := json.Unmarshal(raw, &cf); err != nil {
t.Fatalf("%s: %v", name, err)
}
if len(cf.Cases) < 10 {
t.Fatalf("%s: suspiciously small corpus (%d)", name, len(cf.Cases))
}
for _, c := range cf.Cases {
out := Sanitize(c.Input)
for _, bad := range c.Deny {
if strings.Contains(out, bad) {
t.Errorf("%s/%s: output contains %q:\n%s", name, c.Name, bad, out)
}
}
for _, good := range c.Allow {
if !strings.Contains(out, good) {
t.Errorf("%s/%s: output lost %q:\n%s", name, c.Name, good, out)
}
}
}
}
}
// --- rendering ---
func TestRenderMarkdown(t *testing.T) {
out, err := Render([]byte("# Title\n\nSome *emphasis* and `code`.\n\n- [ ] task\n"), nil)
if err != nil {
t.Fatal(err)
}
s := string(out)
for _, want := range []string{"<h2", "<em>emphasis</em>", "<code>code</code>", `type="checkbox"`} {
if !strings.Contains(s, want) {
t.Errorf("missing %q in:\n%s", want, s)
}
}
}
func TestRenderStripsRawHTMLAttacks(t *testing.T) {
out, err := Render([]byte("hello <script>alert(1)</script> <a href=\"javascript:x\" onclick=\"y\">link</a>\n"), nil)
if err != nil {
t.Fatal(err)
}
s := string(out)
for _, bad := range []string{"<script", "javascript:", "onclick"} {
if strings.Contains(s, bad) {
t.Errorf("attack survived rendering: %q in\n%s", bad, s)
}
}
}
func TestRenderFencedCodeHighlights(t *testing.T) {
out, err := Render([]byte("```go\npackage main\n```\n"), nil)
if err != nil {
t.Fatal(err)
}
s := string(out)
// Chroma emits classed spans; the sanitizer must keep them and the
// keyword must be inside one.
if !strings.Contains(s, `<span class="kn">package</span>`) &&
!strings.Contains(s, `<span class="k">package</span>`) {
t.Fatalf("fenced code not highlighted:\n%s", s)
}
}
func TestRenderFencedCodeUnknownLanguageStaysPlain(t *testing.T) {
out, err := Render([]byte("```zzznotalang\nhello\n```\n"), nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), "hello") {
t.Fatalf("code content lost:\n%s", out)
}
}
func TestRenderTable(t *testing.T) {
out, err := Render([]byte("| a | b |\n|---|---|\n| 1 | 2 |\n"), nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), "<table>") {
t.Fatalf("GFM table lost:\n%s", out)
}
}
// --- resolver seam ---
type fakeResolver struct{}
func (fakeResolver) ResolveHash(prefix string) string {
if strings.HasPrefix(prefix, "abc123") {
return "/checkin/" + prefix
}
return ""
}
func (fakeResolver) ResolveWiki(name string) string {
if name == "Design" {
return "/wiki/Design"
}
return ""
}
func TestResolveWikiRef(t *testing.T) {
out, err := Render([]byte("See [[Design]] for details.\n"), fakeResolver{})
if err != nil {
t.Fatal(err)
}
// The hook, not the byte string: the anchor gained preview
// attributes (RFC 0011 §14) and will gain more.
if !strings.Contains(string(out), `href="/wiki/Design"`) ||
!strings.Contains(string(out), `data-ref="wiki/Design"`) {
t.Fatalf("wiki ref not resolved:\n%s", out)
}
}
func TestResolveUnknownWikiStaysText(t *testing.T) {
out, err := Render([]byte("See [[Nowhere]].\n"), fakeResolver{})
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(out), "<a") {
t.Fatalf("unresolvable ref linked:\n%s", out)
}
if !strings.Contains(string(out), "[[Nowhere]]") {
t.Fatalf("unresolved ref text lost:\n%s", out)
}
}
func TestResolveHashRef(t *testing.T) {
out, err := Render([]byte("Fixed in abc123def456.\n"), fakeResolver{})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), `href="/checkin/abc123def456"`) ||
!strings.Contains(string(out), `data-ref="[abc123def456]"`) {
t.Fatalf("hash ref not resolved:\n%s", out)
}
}
func TestResolveSkipsCodeSpans(t *testing.T) {
out, err := Render([]byte("`abc123def456` and [[Design]] in code: `[[Design]]`\n"), fakeResolver{})
if err != nil {
t.Fatal(err)
}
s := string(out)
if strings.Contains(s, `<code><a`) || strings.Contains(s, "code><a") {
t.Fatalf("resolved inside code span:\n%s", s)
}
if !strings.Contains(s, `href="/wiki/Design"`) {
t.Fatalf("prose ref should still resolve:\n%s", s)
}
}
func TestResolverOutputStillSafe(t *testing.T) {
// A hostile resolver URL must not smuggle scripts — Render output
// re-enters pages as trusted HTML, so the seam is load-bearing.
out, err := Render([]byte("[[Design]]\n"), evilResolver{})
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(out), "<script") {
t.Fatalf("resolver injected markup:\n%s", out)
}
}
type evilResolver struct{}
func (evilResolver) ResolveHash(string) string { return "" }
func (evilResolver) ResolveWiki(string) string {
return `"><script>alert(1)</script>`
}
// --- fuzz ---
func FuzzSanitize(f *testing.F) {
f.Add(`<a href="javascript:alert(1)" onmouseover=x>y</a>`)
f.Add("<scrİpt>hi</scrİpt>")
f.Add(`<img src=x onerror=alert(1)>`)
f.Add(`<code class="language-go ui modal">x</code>`)
f.Add("plain **markdown** text")
f.Fuzz(func(t *testing.T, in string) {
// The oracle parses the output: dangerous constructs must not
// exist as MARKUP. (Substring checks false-positive on inputs
// that are plain text like "onmouseover=0".)
body := &html.Node{Type: html.ElementNode, Data: "body", DataAtom: atom.Body}
nodes, err := html.ParseFragment(strings.NewReader(Sanitize(in)), body)
if err != nil {
t.Fatalf("output unparseable for %q: %v", in, err)
}
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode {
switch n.Data {
case "script", "style", "iframe", "object", "embed", "form":
t.Fatalf("policy leaked <%s> for input %q", n.Data, in)
}
for _, a := range n.Attr {
if strings.HasPrefix(strings.ToLower(a.Key), "on") {
t.Fatalf("policy leaked %s= on <%s> for input %q", a.Key, n.Data, in)
}
if a.Key == "href" || a.Key == "src" {
v := strings.ToLower(strings.TrimSpace(a.Val))
if strings.HasPrefix(v, "javascript:") || strings.HasPrefix(v, "vbscript:") || strings.HasPrefix(v, "data:") {
t.Fatalf("policy leaked %s=%q for input %q", a.Key, a.Val, in)
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
for _, n := range nodes {
walk(n)
}
})
}
func TestHeadingsDemoted(t *testing.T) {
// Ticket 110: the page template owns h1; content headings shift down.
out, err := Render([]byte("# Top\n\n## Sub\n\n###### Deep\n"), nil)
if err != nil {
t.Fatal(err)
}
s := string(out)
if strings.Contains(s, "<h1") {
t.Fatalf("content h1 survived:\n%s", s)
}
for _, want := range []string{"<h2", "<h3", "<h6"} {
if !strings.Contains(s, want) {
t.Fatalf("missing %s (h6 must cap, not vanish):\n%s", want, s)
}
}
}
// GFM completeness: footnotes and heading anchors must survive the
// sanitizer (enabling a goldmark extension is only half the job — the
// policy has to admit the attributes it emits).
func TestFootnotesAndHeadingIDs(t *testing.T) {
out, err := Render([]byte("## A Heading Here\n\nText with a note.[^1]\n\n[^1]: The note body.\n"), nil)
if err != nil {
t.Fatal(err)
}
got := string(out)
if !strings.Contains(got, `id="a-heading-here"`) {
t.Fatalf("heading anchor missing or stripped:\n%s", got)
}
if !strings.Contains(got, "The note body") || !strings.Contains(got, "footnotes") {
t.Fatalf("footnote missing or stripped:\n%s", got)
}
if !strings.Contains(got, "<sup") {
t.Fatalf("footnote reference marker stripped:\n%s", got)
}
}
func (fakeResolver) ResolvePath(string) string { return "" }
func (evilResolver) ResolvePath(string) string { return "" }
// A mailing-list-style post carries its code as an INDENTED block —
// "here is the patch::" plus four spaces — which markdown parses as
// ast.CodeBlock with no language. goldmark's highlighting extension
// only handles FENCED blocks, so these rendered as flat <pre><code>
// while a fenced block two paragraphs away was colored: the reader saw
// the difference and read it as breakage. Content-sniffing covers them
// — a diff is unmistakable — and a block chroma cannot identify stays
// a plain block rather than being guessed wrong.
func TestIndentedCodeBlocksAreHighlighted(t *testing.T) {
src := []byte("a patch::\n\n --- althttpd.c.orig\n +++ althttpd.c\n @@ -318,8 +318,8 @@\n -static int nIn = 0;\n +static long nIn = 0;\n")
out, err := Render(src, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), `class="chroma"`) {
t.Fatalf("an unmistakable diff rendered unhighlighted:\n%s", out)
}
// Fenced blocks with a language keep working through the extension.
fenced, err := Render([]byte("```go\nfunc main() {}\n```\n"), nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(fenced), `class="chroma"`) {
t.Fatalf("fenced block lost its highlighting:\n%s", fenced)
}
// Unidentifiable content stays a plain block — no wrong guesses.
plain, err := Render([]byte(" zxqv 001 vryn\n 003 kwex\n"), nil)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(plain), `class="chroma"`) &&
!strings.Contains(string(plain), "<pre><code>") {
t.Logf("note: analyse guessed a lexer for noise; acceptable if stable:\n%s", plain)
}
}
type describingResolver struct{ fakeResolver }
func (describingResolver) DescribeHash(prefix string) string {
if prefix == "abc1234567" {
return "fix shift key firing twice"
}
return ""
}
// A described hash renders GitHub's way: message as hover title, an
// excerpt beside the hash. Undescribed hashes stay bare links. The
// hooks are asserted, not the drawing: the excerpt is whatever the
// describer said, and its punctuation is the stylesheet's business.
func TestDescribedHashCarriesComment(t *testing.T) {
const desc = "fix shift key firing twice"
gotB, err := Render([]byte("see abc1234567 here"), describingResolver{})
if err != nil {
t.Fatal(err)
}
got := string(gotB)
for _, want := range []string{
`data-ref="[abc1234567]"`,
`data-ref-kind="hash"`,
`title="` + desc + `"`,
`class="cref"`,
} {
if !strings.Contains(got, want) {
t.Fatalf("missing %q in:\n%s", want, got)
}
}
if i := strings.Index(got, `class="cref">`); i < 0 || !strings.Contains(got[i:], desc) {
t.Fatalf("the excerpt does not carry the describer's line:\n%s", got)
}
// A resolver without the describer upgrade renders exactly as before.
plainB, err := Render([]byte("see abc1234567 here"), fakeResolver{})
if err != nil {
t.Fatal(err)
}
plain := string(plainB)
if strings.Contains(plain, "cref") || strings.Contains(plain, "title=") {
t.Fatalf("undescribed link grew chrome:\n%s", plain)
}
if !strings.Contains(plain, `data-ref="[abc1234567]"`) {
t.Fatalf("undescribed link lost its hook:\n%s", plain)
}
}
type userResolver struct{ fakeResolver }
func (userResolver) ResolveUser(name string) (string, int, bool) {
if name == "stephan" {
return "/people/stephan", 123, true
}
return "", 0, false
}
// A known @mention links in the person's hue; an unknown one stays
// plain text — never a dead link. An email's @ is not a mention.
//
// The rendered LABEL drops the sigil: a mention reads as the person's
// mark followed by their name, the way a byline does, so the reader is
// not shown the syntax twice. The source keeps its @, which is what the
// unknown-mention case below asserts survives untouched.
func TestMentionsResolveInAuthorHue(t *testing.T) {
out, err := Render([]byte("ping @stephan and @nobody, mail [email protected]\n"), userResolver{})
if err != nil {
t.Fatal(err)
}
got := string(out)
if !strings.Contains(got, `<a href="/people/stephan" class="uref" style="--uh: 123">stephan</a>`) {
t.Fatalf("mention not resolved:\n%s", got)
}
if strings.Contains(got, ">@stephan<") {
t.Errorf("the sigil reached the rendered label:\n%s", got)
}
if strings.Contains(got, `href="/people/nobody"`) || !strings.Contains(got, "@nobody") {
t.Fatalf("unknown mention mishandled:\n%s", got)
}
if strings.Contains(got, `href="/people/b.com"`) {
t.Fatalf("email linkified as mention:\n%s", got)
}
}
// An artifact with no N card IS fossil wiki — fossil's default, and
// what anything writing through the zit CLI produces. Mentions, wiki
// refs and hash refs are dialect-INDEPENDENT, so they must resolve
// there exactly as they do in markdown; they used to reach only the
// markdown path, and the majority of quarry's own forum posts are
// stored with no N card.
func TestFossilWikiResolvesMentionsToo(t *testing.T) {
out, err := RenderFossilWiki([]byte("ping @stephan and @nobody\n"), userResolver{})
if err != nil {
t.Fatal(err)
}
got := string(out)
if !strings.Contains(got, `<a href="/people/stephan" class="uref" style="--uh: 123">stephan</a>`) {
t.Fatalf("mention not resolved in fossil wiki:\n%s", got)
}
if strings.Contains(got, ">@stephan<") {
t.Errorf("the sigil reached the rendered label:\n%s", got)
}
if strings.Contains(got, `href="/people/nobody"`) || !strings.Contains(got, "@nobody") {
t.Fatalf("unknown mention mishandled:\n%s", got)
}
}
// The reference pass runs on rendered HTML and skips a/code/pre, so it
// must not reach inside a link fossil-wiki's own [target] syntax just
// built, nor inside a <verbatim> block.
func TestFossilWikiRefsSkipLinksAndVerbatim(t *testing.T) {
out, err := RenderFossilWiki(
[]byte("<verbatim>@stephan</verbatim>\n"), userResolver{})
if err != nil {
t.Fatal(err)
}
if got := string(out); strings.Contains(got, `href="/people/stephan"`) {
t.Errorf("verbatim content was linkified:\n%s", got)
}
}
// "[label] (url)" — the space-separated form — is still a link with
// the bracketed text as its label; the strict form is unchanged.
func TestSpacedLink(t *testing.T) {
for src, want := range map[string]string{
"[link] (www.google.com)": `<a href="http://www.google.com" rel="nofollow">link</a>`,
"[docs] (https://x.org/a?b=1)": `<a href="https://x.org/a?b=1" rel="nofollow">docs</a>`,
"[link](https://x.org)": `<a href="https://x.org" rel="nofollow">link</a>`,
"[a] (b c)": `[a] (b c)`,
"see [1] (twice) and [2] (x y)": `[2] (x y)`,
} {
out, err := Render([]byte(src), nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), want) {
t.Errorf("%q: want %q in\n%s", src, want, out)
}
}
}
// WikiRefs must read a body exactly the way the renderer resolves it,
// because a structure page built from a second, nearly-identical rule
// eventually disagrees with the links on the pages it describes.
func TestWikiRefsReadsBothConstructsOnce(t *testing.T) {
body := []byte("See [[Design]] and [Guide|the guide] and [Roadmap].\n" +
"External: [https://example.com|a site] and [[Design]] again.\n")
// Markdown pages have only the double-bracket construct, so the
// fossil-wiki form must stay ordinary text.
md := WikiRefs(body, false)
if len(md) != 1 || md[0].Name != "Design" || !md[0].Explicit {
t.Fatalf("markdown refs = %+v, want [[Design]] alone, marked explicit", md)
}
fw := WikiRefs(body, true)
want := []WikiRef{{"Design", true}, {"Guide", false}, {"Roadmap", false}}
if len(fw) != len(want) {
t.Fatalf("fossil-wiki refs = %+v, want %+v — no duplicates, no URLs, and nothing read out of [[...]]", fw, want)
}
for i, w := range want {
if fw[i] != w {
t.Fatalf("fossil-wiki refs = %+v, want %+v", fw, want)
}
}
}
// The test resolvers draw no identity mark: an empty answer is valid and
// renders a mention as a plain link, which keeps these assertions about
// the constructs they are actually testing.
func (fakeResolver) UserMark(string) string { return "" }
func (evilResolver) UserMark(string) string { return "" }