// Package markup renders wiki/ticket/chat text to safe HTML (0002 §2):
// goldmark for markdown, a CLEANROOM bluemonday policy for
// sanitization — written fresh, copied from neither the Gitea nor the
// Forgejo lineage — and one Resolver seam for zit reference resolution
// (artifact hashes, [[wiki]] links) so the markdown assembly stays free
// of index knowledge.
//
// The policy's conformance gate is the two-lineage vector corpus under
// third_party/sanitizer-corpus (inputs are data, recorded in the
// provenance ledger); markup_test runs it, and FuzzSanitize hammers the
// policy beyond it.
package markup
import (
"bytes"
gohtml "html"
"sync/atomic"
"craftdesign.group/quarry/internal/highlight"
"regexp"
"strings"
chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
"github.com/microcosm-cc/bluemonday"
"github.com/yuin/goldmark"
highlighting "github.com/yuin/goldmark-highlighting/v2"
gast "github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/renderer/html"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
)
// Resolver turns zit references found in text into links. Empty return
// means "not resolvable — leave as plain text". Implementations sit on
// the index; markup never touches it directly.
type Resolver interface {
// ResolveHash resolves a hex token (artifact-name shaped, 10–64
// chars) to a URL — a check-in page, a ticket page, whatever the
// index says the artifact is.
ResolveHash(prefix string) string
// ResolveWiki resolves a [[Wiki Name]] reference to a URL.
ResolveWiki(name string) string
// ResolvePath maps a Fossil site-relative URL found in imported
// content (/doc/trunk/README.md, /info/<hash>, /timeline…) to the
// equivalent quarry path. Empty means "quarry has no equivalent",
// and the caller drops the link rather than leaving a dead one.
ResolvePath(fossilPath string) string
// ResolveAttachment resolves `attach:NAME` — a file attached to the
// SAME object this body belongs to — to a URL, or "" when the object
// carries no live attachment under that name. image asks for the
// inline route rather than the download: the caller owns both URL
// shapes, markup only knows which one it is about to write.
ResolveAttachment(name string, image bool) string
// UserMark is the identity mark for a mentioned account, as inline
// SVG — the same glyph the forge draws beside a byline, so a person
// named in prose looks like the same person named anywhere else.
//
// Through the seam rather than computed here: markup knows nothing
// about accounts, and drawing the mark is not its job. Empty is a
// valid answer and renders the mention as a plain link.
UserMark(name string) string
}
// sanitize is the cleanroom policy. Everything here is written from the
// question "what does markdown rendered by goldmark plus quarry's own
// features actually need", not from either lineage's list.
var sanitize = func() *bluemonday.Policy {
p := bluemonday.NewPolicy()
// Block structure.
p.AllowElements("p", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6",
"blockquote", "pre", "ul", "ol", "li", "dl", "dt", "dd",
"table", "thead", "tbody", "tr", "th", "td",
"details", "summary", "picture", "source")
// Inline structure.
p.AllowElements("em", "strong", "b", "i", "s", "del", "ins", "u",
"code", "kbd", "sub", "sup", "mark", "q", "cite", "abbr")
// Links: safe schemes plus relative URLs; rel=nofollow is forced on
// every absolute link.
p.AllowAttrs("href").OnElements("a")
p.AllowStandardURLs()
// `attach:` is quarry's own reference scheme (RFC 0016) and is
// admitted only so the attachment pass can see it: rewriteAttachments
// turns every one into a route or into text, so no attach: URL ever
// reaches a reader.
p.AllowURLSchemes("attach")
p.RequireNoFollowOnLinks(true)
// Images.
p.AllowAttrs("src", "alt", "title", "width", "height").OnElements("img")
p.AllowAttrs("loading").Matching(regexp.MustCompile(`^lazy$`)).OnElements("img")
p.AllowAttrs("media", "srcset").OnElements("source")
// Syntax-highlight hooks: language classes on code, chroma's short
// token classes on span, and the chroma wrapper class on pre —
// nothing framework-shaped gets through (the class-injection attacks
// in the corpus).
p.AllowAttrs("class").Matching(regexp.MustCompile(`^language-[a-zA-Z0-9_+#.-]+$`)).OnElements("code")
p.AllowAttrs("class").Matching(regexp.MustCompile(`^[a-z][a-z0-9]{0,3}$`)).OnElements("span")
p.AllowAttrs("class").Matching(regexp.MustCompile(`^chroma$`)).OnElements("pre")
// Task lists.
p.AllowAttrs("type").Matching(regexp.MustCompile(`^checkbox$`)).OnElements("input")
p.AllowAttrs("checked", "disabled").Matching(regexp.MustCompile(`^$|^checked$|^disabled$`)).OnElements("input")
p.AllowAttrs("class").Matching(regexp.MustCompile(`^(checked|unchecked|indeterminate)$`)).OnElements("li")
p.AllowAttrs("class").Matching(regexp.MustCompile(`^md-tasks$`)).OnElements("ul")
// Emoji spans and the two color properties, on non-code elements.
p.AllowAttrs("class").Matching(regexp.MustCompile(`^emoji$`)).OnElements("span")
p.AllowAttrs("aria-label", "data-alias").OnElements("span")
p.AllowStyles("color", "background-color").OnElements("span", "p", "th", "td", "div")
// Table alignment emitted by the GFM extension.
p.AllowAttrs("align").Matching(regexp.MustCompile(`^(left|center|right)$`)).OnElements("th", "td")
// Heading anchors (WithAutoHeadingID) and footnote plumbing. The id
// pattern is deliberately narrow — a slug or goldmark's fnref form —
// so an id cannot be shaped to collide with page chrome or a
// framework hook. Footnote back-references need href (already
// allowed on a), role, and the section wrapper.
slug := regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_:.-]{0,127}$`)
p.AllowAttrs("id").Matching(slug).OnElements("h1", "h2", "h3", "h4", "h5", "h6", "li", "sup", "a")
// goldmark wraps footnotes in <div class="footnotes" role="doc-endnotes">
// (not <section>), with role/class on the ref and backref anchors.
p.AllowElements("div")
p.AllowAttrs("class").Matching(regexp.MustCompile(`^footnotes$`)).OnElements("div")
p.AllowAttrs("class").Matching(regexp.MustCompile(`^footnote-(ref|backref)$`)).OnElements("a")
p.AllowAttrs("role").Matching(regexp.MustCompile(`^doc-(endnotes|backlink|noteref)$`)).OnElements("div", "a")
return p
}()
// demoteHeadings shifts every markdown heading one level down (h1→h2,
// capped at h6): the page template owns the single h1, so content
// headings slot beneath it and the document outline stays unambiguous.
type demoteHeadings struct{}
func (demoteHeadings) Transform(doc *gast.Document, _ text.Reader, _ parser.Context) {
gast.Walk(doc, func(n gast.Node, entering bool) (gast.WalkStatus, error) {
if h, ok := n.(*gast.Heading); ok && entering && h.Level < 6 {
h.Level++
}
return gast.WalkContinue, nil
})
}
// codeBlockHighlighter renders INDENTED code blocks (ast.CodeBlock)
// through chroma's content sniffing. The highlighting extension only
// handles fenced blocks, so a mailing-list-style post — "the patch::"
// plus indentation, which is how forum bodies in this corpus carry
// diffs — rendered flat next to colored fenced blocks. A block the
// sniffer cannot identify stays a plain <pre><code>: no guess is
// better than a wrong one.
// looksLikeDiff recognizes unified-diff shape: a hunk header, or the
// ---/+++ file pair. Nothing else writes lines like these at column 0.
func looksLikeDiff(b []byte) bool {
var minus, plus bool
for _, line := range bytes.Split(b, []byte("\n")) {
switch {
case bytes.HasPrefix(line, []byte("@@ -")):
return true
case bytes.HasPrefix(line, []byte("--- ")):
minus = true
case bytes.HasPrefix(line, []byte("+++ ")):
plus = true
}
if minus && plus {
return true
}
}
return false
}
type codeBlockHighlighter struct{}
func (codeBlockHighlighter) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
reg.Register(gast.KindCodeBlock, renderCodeBlock)
}
func renderCodeBlock(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) {
if !entering {
return gast.WalkContinue, nil
}
var body bytes.Buffer
for i := 0; i < node.Lines().Len(); i++ {
line := node.Lines().At(i)
body.Write(line.Value(source))
}
// chroma's Analyse has no analyzer for diffs — the one format these
// blocks nearly always carry — so the unmistakable shape is named
// outright and everything else may still sniff.
lang := ""
if looksLikeDiff(body.Bytes()) {
lang = "diff"
}
if hl, ok := highlight.Code(body.Bytes(), lang, ""); ok {
_, _ = w.WriteString(hl)
return gast.WalkSkipChildren, nil
}
_, _ = w.WriteString("<pre><code>")
_, _ = w.WriteString(gohtml.EscapeString(body.String()))
_, _ = w.WriteString("</code></pre>\n")
return gast.WalkSkipChildren, nil
}
var md = goldmark.New(
goldmark.WithExtensions(
// GFM covers tables, autolinks, strikethrough and task lists;
// footnotes are also GFM but ship as their own extension.
extension.Linkify, extension.Strikethrough, extension.TaskList,
// Table alignment as the align attribute: the sanitizer strips
// the style attribute goldmark would otherwise use.
extension.NewTable(extension.WithTableCellAlignMethod(extension.TableCellAlignAttribute)),
extension.Footnote,
highlighting.NewHighlighting(
highlighting.WithStyle("github"),
highlighting.WithFormatOptions(chromahtml.WithClasses(true)),
),
),
goldmark.WithParserOptions(
// Heading anchors, as GitHub emits them, so deep links into long
// documents work.
parser.WithAutoHeadingID(),
parser.WithASTTransformers(
util.Prioritized(demoteHeadings{}, 100),
util.Prioritized(taskLists{}, 200),
),
parser.WithInlineParsers(util.Prioritized(spacedLink{}, 150)),
),
goldmark.WithRendererOptions(html.WithUnsafe(), // sanitizer is the gate, not the renderer
renderer.WithNodeRenderers(util.Prioritized(codeBlockHighlighter{}, 100))),
)
// imageProxy maps an external image URL to the address this forge serves
// it from; nil leaves images as the author wrote them. Set once at start.
var imageProxy atomic.Pointer[func(src string) (proxied string, w, h int)]
// SetImageProxy routes every external <img> in rendered content through
// f — the forge's /img proxy — so a document's images load from this
// forge, not from wherever the author linked. f returns "" for a target
// it will not proxy, which is left alone. nil turns it off.
func SetImageProxy(f func(src string) (proxied string, w, h int)) {
if f == nil {
imageProxy.Store(nil)
return
}
imageProxy.Store(&f)
}
// externalImg matches a whole sanitized <img> tag, and imgSrc picks the
// source out of it. The whole tag, not just the part before src, because
// an author's own width or loading can sit on either side of it and must
// not be written twice. The sanitizer has already escaped the attribute,
// so the value is unescaped before it is handed on.
var (
externalImg = regexp.MustCompile(`(?is)<img\s([^>]*)>`)
imgSrc = regexp.MustCompile(`(?i)\bsrc\s*=\s*"(https?://[^"]*)"`)
)
// hasAttr answers whether a tag's attributes already carry name, so an
// author's own width, height or loading is never overwritten.
func hasAttr(attrs, name string) bool {
return regexp.MustCompile(`(?i)\b` + name + `\s*=`).MatchString(attrs)
}
// proxyImages rewrites external image sources through the proxy. It runs
// after sanitizing, on every renderer's output, because an image can
// arrive through markdown, fossil wiki or raw HTML alike.
//
// It also writes what the proxy has learned about the image into the tag:
//
// - width and height, once the forge has fetched it once, so the page
// does not reflow as images land. Unknown sizes are simply omitted.
// - loading and decoding on every image but the FIRST. The first one in
// a document is usually on screen at once and lazy-loading it delays
// the very thing the reader came for; the rest are below the fold on
// any README long enough to matter.
func proxyImages(html []byte) []byte {
fp := imageProxy.Load()
if fp == nil {
return html
}
n := 0
return externalImg.ReplaceAllFunc(html, func(m []byte) []byte {
attrs := string(externalImg.FindSubmatch(m)[1])
src := imgSrc.FindStringSubmatch(attrs)
if src == nil {
return m // a relative image, an attachment: not ours
}
proxied, w, h := (*fp)(gohtml.UnescapeString(src[1]))
if proxied == "" {
return m
}
n++
// Self-closing tags keep their slash.
closing := ">"
if strings.HasSuffix(attrs, "/") {
attrs, closing = strings.TrimSuffix(attrs, "/"), "/>"
}
out := `<img ` + imgSrc.ReplaceAllString(attrs, `src="`+escapeAttr(proxied)+`"`)
if w > 0 && h > 0 && !hasAttr(attrs, "width") && !hasAttr(attrs, "height") {
out += ` width="` + itoa(w) + `" height="` + itoa(h) + `"`
}
if n > 1 {
if !hasAttr(attrs, "loading") {
out += ` loading="lazy"`
}
if !hasAttr(attrs, "decoding") {
out += ` decoding="async"`
}
}
return []byte(out + closing)
})
}
// Render converts markdown to sanitized HTML. r may be nil (no
// reference resolution).
func Render(src []byte, r Resolver) ([]byte, error) {
// Pikchr fences are lifted before goldmark sees them and restored
// after sanitizing: the diagram needs the narrow SVG policy, not the
// document policy (which has no business admitting <svg> generally).
src, diagrams, nonce := liftPikchrFences(src)
src = continueQuotes(src)
var buf bytes.Buffer
if err := md.Convert(src, &buf); err != nil {
return nil, err
}
out := proxyImages(sanitize.SanitizeBytes(buf.Bytes()))
// Unconditional: a nil resolver must still strip the scheme.
out = rewriteAttachments(out, r)
if r != nil {
// Order matters: rewrite the author's own hrefs BEFORE
// resolveRefs mints quarry links, so the two passes never see
// each other's output.
out = rewriteFossilHrefs(out, r)
// After the site-relative pass and before refs: the two look at
// disjoint targets (that one at /… , this one at everything
// else), and neither should see the other's output.
out = rewriteDocRelative(out, r)
out = resolveRefs(out, r)
}
return restorePikchrFences(out, diagrams, nonce), nil
}
// Sanitize applies the policy alone — the surface the conformance
// corpus and the fuzz target exercise directly.
func Sanitize(in string) string {
return string(proxyImages([]byte(sanitize.Sanitize(in))))
}
// siteRelHref matches a rendered link whose target is a site-relative
// path — in imported content those are Fossil's own routes, which mean
// nothing under quarry's URL space.
var siteRelHref = regexp.MustCompile(`<a href="(/[^"]*)"([^>]*)>(.*?)</a>`)
// rewriteFossilHrefs maps Fossil site-relative links onto quarry routes,
// and unwraps the ones quarry cannot serve into plain text — a reader
// should never be handed a link that 404s.
func rewriteFossilHrefs(html []byte, r Resolver) []byte {
return siteRelHref.ReplaceAllFunc(html, func(m []byte) []byte {
sub := siteRelHref.FindSubmatch(m)
href, rest, label := string(sub[1]), string(sub[2]), string(sub[3])
// quarry's own routes are already correct; only /verb paths that
// quarry does not serve need mapping.
if strings.HasPrefix(href, "/r/") || strings.HasPrefix(href, "/docs") ||
strings.HasPrefix(href, "#") {
return m
}
if mapped := r.ResolvePath(href); mapped != "" {
return []byte(`<a href="` + mapped + `"` + rest + `>` + label + `</a>`)
}
return []byte(label)
})
}
// docRelHref matches a rendered link or image target in a document from
// the tree. Markdown has no wiki-page ambiguity to worry about — a link
// target is a path — so a BARE relative name counts, not only ./ and ../.
var docRelHref = regexp.MustCompile(`<(a|img)\s([^>]*?)(href|src)="([^"]*)"`)
// rewriteDocRelative re-bases relative targets in a document onto the
// document's own directory.
//
// Fossil never has to: it serves a document at /doc/<version>/<path>, so
// the URL's directory IS the document's directory and the browser
// resolves a sibling link correctly for free. Quarry renders the same
// file on the repository home page and under a tree listing as well,
// where the page's URL sits somewhere else entirely — and there a
// relative href lands outside the repository. SQLite's README links to
// doc/compile-for-unix.md; rendered at /r/sqlite the browser asked for
// /r/doc/compile-for-unix.md and got a 404, while the page itself sat at
// /r/sqlite/doc/trunk/doc/compile-for-unix.md the whole time.
//
// A resolver that is not a DocResolver — a wiki page, a forum post, a
// ticket comment, none of which sit at a path — is left alone.
func rewriteDocRelative(html []byte, r Resolver) []byte {
dr, ok := r.(DocResolver)
if !ok {
return html
}
return docRelHref.ReplaceAllFunc(html, func(m []byte) []byte {
sub := docRelHref.FindSubmatch(m)
tag, mid, attr, target := sub[1], string(sub[2]), sub[3], string(sub[4])
if !isDocRelative(target) {
return m
}
u := dr.DocRelative(target)
if u == "" {
return m
}
return []byte(`<` + string(tag) + ` ` + mid + string(attr) + `="` + escapeAttr(u) + `"`)
})
}
// isDocRelative answers for a target that names a file beside the
// document: not empty, not site-relative, not a fragment or bare query,
// and carrying no scheme (http:, mailto:, data:, attach:).
func isDocRelative(t string) bool {
if t == "" || strings.HasPrefix(t, "/") ||
strings.HasPrefix(t, "#") || strings.HasPrefix(t, "?") {
return false
}
// A colon before the first slash is a scheme, not a directory.
if i := strings.IndexAny(t, ":/"); i >= 0 && t[i] == ':' {
return false
}
return true
}