// Package uv is fossil's unversioned files as quarry relies on them
// (RFC 0020): the rules for a name, the content hash, which of two
// versions wins, the catalog's uv-hash, and where a file lives in the
// repository's bucket.
//
// It is a leaf, like evt. The ops layer writes the catalog with it and
// the sync front answers fossil's cards with it, and the second must not
// have to import the first to agree with it about a name or a hash.
//
// Everything here is cited from fossil's own source at trunk:
// src/unversioned.c (unversioned_content_hash, unversioned_status, the
// checks `uv add` makes on a name), src/file.c
// (file_is_simple_pathname) and src/xfer.c
// (xfer_accept_unversioned_file).
package uv
import (
"crypto/sha1"
"crypto/sha3"
"encoding/hex"
"errors"
"path"
"sort"
"strings"
"time"
"unicode/utf8"
)
// File is one catalog row: the newest version of one name, or the
// tombstone that removed it. A tombstone keeps its row, as fossil's
// does, because the tombstone is what syncs a removal.
type File struct {
Name string `json:"name"`
// Mtime is fossil's: seconds since 1970, chosen by whoever wrote
// the file, and the only thing that decides which version wins.
Mtime int64 `json:"mtime"`
// Hash is the content's hash under the policy it was written with,
// SHA1 or SHA3-256; empty for a tombstone.
Hash string `json:"hash"`
Size int64 `json:"size"`
Deleted bool `json:"deleted"`
// Who wrote this version: the signed-in uploader, the sync login
// that pushed it, or mirror:<host> for an import.
Who string `json:"who"`
// At is when quarry recorded it, RFC 3339. Not fossil's; fossil
// keeps only a receive id.
At string `json:"at"`
}
// MaxName is fossil's bound on a name, in bytes.
const MaxName = 500
const (
// Prefix is where a file's content lives in the bucket.
Prefix = "unversioned/"
// TombstonePrefix is where a removal lives: an empty object whose
// metadata carries the removal's mtime. Inside Prefix, so one
// listing finds both.
TombstonePrefix = Prefix + tombstoneDir
tombstoneDir = ".tombstones/"
)
// Key is the object holding name's content.
func Key(name string) string { return Prefix + name }
// TombstoneKey is the object recording name's removal.
func TombstoneKey(name string) string { return TombstonePrefix + name }
// NameOfKey reads a key from a bucket listing back to the name it
// holds. ok is false for anything that is not a file this layout wrote,
// so a stray object under the prefix is skipped rather than catalogued.
func NameOfKey(key string) (name string, tombstone, ok bool) {
rest, found := strings.CutPrefix(key, Prefix)
if !found {
return "", false, false
}
if t, isTomb := strings.CutPrefix(rest, tombstoneDir); isTomb {
rest, tombstone = t, true
}
if CheckName(rest) != nil {
return "", false, false
}
return rest, tombstone, true
}
// CheckName refuses a name fossil's `uv add` would refuse, in fossil's
// words, plus one rule of quarry's: a name may not start with the
// directory tombstones are kept under, or its content and another
// file's removal would share a key.
func CheckName(name string) error {
switch {
case name == "":
return errors.New("unversioned filenames may not be empty string")
case name[0] == '/':
return errors.New("unversioned filenames may not be absolute")
case !simplePathname(name):
return errors.New("unversioned filenames may not contain complex paths")
case strings.ContainsAny(name, " \t\n\v\f\r"):
return errors.New("unversioned filenames may not contain whitespace")
case strings.ContainsFunc(name, func(r rune) bool { return r < 0x20 || r == 0x7f }):
// Fossil lets the rest of the control characters through. They
// are refused here because a name is a bucket key, a URL path
// and a line of the uv-hash, and a NUL or an escape is a
// different string in each.
return errors.New("unversioned filenames may not contain control characters")
case len(name) > MaxName:
return errors.New("unversioned filenames may not be more than 500 bytes long")
case strings.HasPrefix(name, tombstoneDir):
return errors.New("unversioned filenames may not start with " + tombstoneDir + ", where this forge keeps removals")
}
return nil
}
// simplePathname is file_is_simple_pathname with strict UTF-8: no
// leading or trailing "/", no "//", no "." or ".." element, no
// backslash, and only the characters fossil accepts (nothing above
// U+FFFF, no private use, no noncharacters).
func simplePathname(z string) bool {
if z == "" || z[0] == '/' || strings.HasSuffix(z, "/") || strings.Contains(z, "//") || strings.ContainsRune(z, '\\') {
return false
}
if !utf8.ValidString(z) {
return false
}
for _, r := range z {
switch {
case r > 0xFFFF,
r >= 0xE000 && r <= 0xF8FF,
r >= 0xFDD0 && r <= 0xFDEF,
r == 0xFFFE, r == 0xFFFF:
return false
}
}
for _, el := range strings.Split(z, "/") {
if el == "." || el == ".." {
return false
}
}
return true
}
// Hash is the content hash quarry writes new content under: SHA3-256,
// fossil's default policy. quarry cannot read a repository's policy
// (zit keeps no hash-policy setting), and a SHA1 hash a client sends is
// kept as sent, so a file that arrived from a SHA1 repository hashes
// the same here as there.
func Hash(content []byte) string {
sum := sha3.Sum256(content)
return hex.EncodeToString(sum[:])
}
// Verify reports whether hash names content under either of fossil's
// policies, told apart by length as hname_verify_hash does.
func Verify(hash string, content []byte) bool {
switch len(hash) {
case 40:
sum := sha1.Sum(content)
return hash == hex.EncodeToString(sum[:])
case 64:
return hash == Hash(content)
}
return false
}
// Replaces reports whether in should replace have (exists false: there
// is no row for the name). It is unversioned_status as
// xfer_accept_unversioned_file applies it: the newer mtime wins; at the
// same mtime the larger hash wins, a tombstone's hash being "-"; the
// same hash with a newer mtime updates the mtime; and a tombstone for a
// name the catalog never held changes nothing, since fossil's UPDATE
// finds no row.
func Replaces(in, have File, exists bool) bool {
if !exists {
return !in.Deleted
}
inHash, haveHash := wireHash(in), wireHash(have)
switch {
case have.Mtime < in.Mtime:
return true
case have.Mtime > in.Mtime:
return false
}
return haveHash < inHash
}
// wireHash is a row's hash as the uvfile card spells it.
func wireHash(f File) string {
if f.Deleted || f.Hash == "" {
return "-"
}
return f.Hash
}
// CatalogHash is fossil's uv-hash, the value a client sends in
// `pragma uv-hash` to ask whether the server holds anything different.
// From unversioned_content_hash in fossil's src/unversioned.c:
//
// SELECT printf('%s %s %s\n',name,datetime(mtime,'unixepoch'),hash)
// FROM unversioned WHERE hash IS NOT NULL ORDER BY name
//
// fed line by line to SHA1. So tombstones are left out, the time is
// UTC "YYYY-MM-DD HH:MM:SS", and the order is SQLite's BINARY collation,
// which is byte order, which is Go's string order. An empty catalog
// hashes to SHA1 of nothing, as fossil's does.
//
// An mtime past 9999-12-31 23:59:59 is outside SQLite's date range, so
// datetime() answers NULL and printf writes it as nothing: the line is
// "NAME HASH", two spaces, and fossil 2.28 hashes it that way (the
// test pins a hash fossil produced with such a row). The line is kept,
// not skipped, because fossil keeps it. Negative mtimes never reach the
// catalog: fossil's receiver refuses them and so does UVWrite.
func CatalogHash(files []File) string {
live := make([]File, 0, len(files))
for _, f := range files {
if !f.Deleted && f.Hash != "" {
live = append(live, f)
}
}
sort.Slice(live, func(i, j int) bool { return live[i].Name < live[j].Name })
h := sha1.New()
for _, f := range live {
h.Write([]byte(f.Name + " " + sqliteDatetime(f.Mtime) + " " + f.Hash + "\n"))
}
return hex.EncodeToString(h.Sum(nil))
}
// maxDatetime is the last second SQLite's datetime() can spell.
const maxDatetime = 253402300799
// sqliteDatetime is datetime(mtime,'unixepoch') for a non-negative
// mtime, with SQLite's NULL as the empty string printf makes of it.
func sqliteDatetime(mtime int64) string {
if mtime < 0 || mtime > maxDatetime {
return ""
}
return time.Unix(mtime, 0).UTC().Format(time.DateTime)
}
// ContentType is the type a download is served with, from the name's
// extension by a fixed table, and whether a browser may show it in
// place. Anything that could run in a reader's browser (HTML, SVG,
// script, XML that may carry either) is a download, which is where this
// parts from fossil: fossil renders /uv/download.html as a page, and
// here publishing HTML is one `y` grant from script on the forge's
// readers (RFC 0020, The web). What the table does not name is bytes.
func ContentType(name string) (ctype string, inline bool) {
if t, ok := types[strings.ToLower(path.Ext(name))]; ok {
return t.ctype, t.inline
}
return "application/octet-stream", false
}
type contentType struct {
ctype string
inline bool
}
var types = map[string]contentType{
// Active: served as downloads, never rendered.
".html": {"text/html; charset=utf-8", false},
".htm": {"text/html; charset=utf-8", false},
".xhtml": {"application/xhtml+xml", false},
".svg": {"image/svg+xml", false},
".js": {"text/javascript; charset=utf-8", false},
".mjs": {"text/javascript; charset=utf-8", false},
".css": {"text/css; charset=utf-8", false},
".xml": {"application/xml", false},
".xsl": {"application/xml", false},
".pdf": {"application/pdf", false},
".wasm": {"application/wasm", false},
// Passive: safe to show in place.
".txt": {"text/plain; charset=utf-8", true},
".md": {"text/plain; charset=utf-8", true},
".json": {"application/json", true},
".png": {"image/png", true},
".jpg": {"image/jpeg", true},
".jpeg": {"image/jpeg", true},
".gif": {"image/gif", true},
".webp": {"image/webp", true},
".mp4": {"video/mp4", true},
".webm": {"video/webm", true},
".mp3": {"audio/mpeg", true},
".ogg": {"audio/ogg", true},
".wav": {"audio/wav", true},
// Archives and packages: named, and downloads.
".gz": {"application/gzip", false},
".tgz": {"application/gzip", false},
".zip": {"application/zip", false},
".tar": {"application/x-tar", false},
".xz": {"application/x-xz", false},
".bz2": {"application/x-bzip2", false},
".zst": {"application/zstd", false},
}