Artifact 90bb2a1d59762484e13559fcd2bb27375ffe3c68752f28684f6db9c02820c578:
- File
internal/highlight/highlight.go
— part of check-in
[f4e226e5a4]
at
2026-08-31 12:18:02
on branch trunk
— large files arrive: size-gate the highlighter and cache it by content
Highlighting was paid on every request with no cache, and its cost tracks token density: sqlite's btree.c, 407KB of dense C, took 490ms and rendered 2.3MB of HTML. Past 256KB the lexer is skipped and the page says so; below it, identical content is highlighted once. (user: quarry, size: 6820) [annotate] [blame] [check-ins using]
// Package highlight is quarry's thin chroma wrapper (0002 §2 lists the // Forgejo one as [COPY]; theirs is coupled to their settings and caches, // so this is a smaller fresh write). Output uses CSS classes, never // inline styles, so the two-theme story stays in one stylesheet. package highlight import ( "bytes" "crypto/sha256" "encoding/hex" "fmt" "regexp" "strings" "sync" "github.com/alecthomas/chroma/v2" chromahtml "github.com/alecthomas/chroma/v2/formatters/html" "github.com/alecthomas/chroma/v2/lexers" "github.com/alecthomas/chroma/v2/styles" ) var formatter = chromahtml.New( chromahtml.WithClasses(true), chromahtml.PreventSurroundingPre(false), ) // MaxBytes is the size past which a file is served as plain text. // // Highlighting is not free and its cost tracks TOKEN DENSITY, not file // size: across sqlite's tree, src/sqlite.h.in is the largest file at // 559KB and among the fastest at 208ms (it is mostly doc comments), // while src/btree.c is 407KB of dense C and took 490ms, every request, // uncached. Chroma also emits a span per token, which inflated that // file to 2.3MB of HTML — 5.7x its source. // // Past this threshold the lexer is skipped and the caller renders an // escaped <pre>. Colour on a 400KB file is worth less than the page // arriving, and every forge draws this line somewhere. const MaxBytes = 256 << 10 // TooBig reports whether Code will decline this content on size alone, // so a caller can say why rather than silently dropping the colour. func TooBig(code []byte) bool { return len(code) > MaxBytes } // cache memoises highlighted output. A blob is immutable, so its // rendering is too: the key is the content hash plus the lexer hints, // and a hit skips tokenising entirely. // // Bounded by TOTAL BYTES rather than entry count, because entries are // wildly uneven — one btree.c is worth thousands of small files, and a // count-based bound would hold a gigabyte or nothing depending on what // was browsed. On overflow the map is dropped wholesale: a generational // sweep with no bookkeeping, which for a read-through cache of // immutable content costs one recompute rather than correctness. var cache = struct { sync.Mutex m map[string]string bytes int }{m: map[string]string{}} // cacheBudget is how much highlighted HTML to retain. Roughly a dozen // large source files, or thousands of ordinary ones. const cacheBudget = 64 << 20 func cacheKey(code []byte, language, filename string) string { sum := sha256.Sum256(code) return hex.EncodeToString(sum[:16]) + "\x00" + language + "\x00" + filename } func cacheGet(key string) (string, bool) { cache.Lock() defer cache.Unlock() v, ok := cache.m[key] return v, ok } func cachePut(key, html string) { cache.Lock() defer cache.Unlock() if cache.bytes+len(html) > cacheBudget { cache.m = map[string]string{} cache.bytes = 0 } cache.m[key] = html cache.bytes += len(html) } // Code highlights source for the given language hint and/or filename; // either may be empty. ok is false when no lexer matched, or when the // content is past MaxBytes — callers fall back to an escaped <pre>. func Code(code []byte, language, filename string) (string, bool) { if TooBig(code) { return "", false } key := cacheKey(code, language, filename) if html, ok := cacheGet(key); ok { return html, html != "" } html, ok := code_(code, language, filename) // A miss is cached too, as the empty string: "no lexer matched" is // as expensive to discover as a match and just as stable. cachePut(key, html) return html, ok } func code_(code []byte, language, filename string) (string, bool) { var lexer chroma.Lexer if language != "" { lexer = lexers.Get(language) } if lexer == nil && filename != "" { lexer = lexers.Match(filename) } // Content analysis ONLY for a shebang. Chroma's Analyse is eager // enough to read an English README as SQL (is/and/or/in/where are // all keywords) and paint the prose red; a #! line is the one // content signal that names a language on purpose. if lexer == nil && bytes.HasPrefix(code, []byte("#!")) { lexer = lexers.Analyse(string(code)) } if lexer == nil { return "", false } lexer = chroma.Coalesce(lexer) it, err := lexer.Tokenise(nil, string(code)) if err != nil { return "", false } var buf bytes.Buffer if err := formatter.Format(&buf, styles.Get("github"), it); err != nil { return "", false } return buf.String(), true } // CSS returns the chroma class rules for both themes: light rules bare, // dark rules inside a prefers-color-scheme block — matching the page // layout's theming (0002 §4 item 5: both themes). // // A chroma style only emits rules for token classes it colors // explicitly, so a class present in the light output but absent from // the dark output would fall through to its light (near-black) color on // the dark background — invisible text (ticket 030). Every such class // gets an explicit dark override at the base text color. func CSS() string { var light, dark strings.Builder formatter.WriteCSS(&light, styles.Get("github")) formatter.WriteCSS(&dark, styles.Get("github-dark")) darkCSS := dark.String() darkClasses := map[string]bool{} for _, m := range classRule.FindAllStringSubmatch(darkCSS, -1) { darkClasses[m[1]] = true } base := "#e6edf3" if m := baseColor.FindStringSubmatch(darkCSS); m != nil { base = m[1] } var missing strings.Builder for _, m := range classRule.FindAllStringSubmatch(light.String(), -1) { if !darkClasses[m[1]] { fmt.Fprintf(&missing, ".chroma .%s { color: %s }\n", m[1], base) darkClasses[m[1]] = true } } // Three theme states, matching the page: the media query is guarded // against an explicit light choice, and the same rules repeat under // an explicit dark one. Without both, a reader who picks light on a // dark-mode device gets dark code colours on a light page, and // picking dark on a light-mode device does nothing at all. darkRules := darkCSS + missing.String() return light.String() + "\n@media (prefers-color-scheme: dark) {\n" + prefixRules(darkRules, `:root:not([data-theme="light"]) `) + "\n}\n" + "\n" + prefixRules(darkRules, `:root[data-theme="dark"] `) + "\n" } var ( classRule = regexp.MustCompile(`\.chroma \.([a-zA-Z0-9]+) `) baseColor = regexp.MustCompile(`\.chroma \{[^}]*?color: (#[0-9a-fA-F]{3,8})`) ) // prefixRules qualifies every ".chroma" selector with a theme scope, so // one set of dark declarations can be emitted twice — once under the // media query and once under the explicit choice — without writing the // theme out by hand. A plain prefix rather than CSS nesting: same // result, and it asks nothing of the browser. func prefixRules(css, scope string) string { return strings.ReplaceAll(css, ".chroma ", scope+".chroma ") }