// Package archive caches derived archives on disk.
//
// An archive is a pure function of a check-in: zit guarantees the bytes
// are reproducible, so the same resolved hash yields the same file
// forever. That makes it cacheable without an invalidation story — an
// entry is never stale, only unwanted, so eviction is about disk budget
// rather than correctness.
//
// Building one is not cheap. sqlite at trunk costs about three seconds
// of gzip for thirteen megabytes, on an anonymous route: without a cache
// every reader pays it again, and concurrent readers pay it in parallel.
package archive
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"sync"
)
// Cache is a bounded directory of built archives.
type Cache struct {
dir string
max int64
mu sync.Mutex
build map[string]*sync.WaitGroup // keys currently being built
}
// New opens (creating if needed) a cache directory holding at most max
// bytes of archives.
func New(dir string, max int64) (*Cache, error) {
if err := os.MkdirAll(dir, 0o755); err != nil {
return nil, fmt.Errorf("archive: cache dir: %w", err)
}
return &Cache{dir: dir, max: max, build: map[string]*sync.WaitGroup{}}, nil
}
// path is the on-disk name for a key. The key is hashed so a repository
// or branch name can never escape the cache directory or collide with
// another key that merely shares a prefix.
func (c *Cache) path(key string) string {
sum := sha256.Sum256([]byte(key))
return filepath.Join(c.dir, hex.EncodeToString(sum[:])+".bin")
}
// Get returns the path of the cached entry for key, calling fill to
// build it if absent. fill writes the archive to the path it is given.
//
// Concurrent callers for the same key build once: the rest wait and then
// read the finished file. Without that, ten readers asking for the same
// popular archive would run ten subprocesses to produce ten identical
// results — the exact case a cache exists to prevent.
func (c *Cache) Get(key string, fill func(dst string) error) (string, error) {
final := c.path(key)
for {
if _, err := os.Stat(final); err == nil {
touch(final)
return final, nil
}
c.mu.Lock()
if wg, building := c.build[key]; building {
c.mu.Unlock()
wg.Wait()
// The builder may have failed, in which case the file is
// still absent and this caller takes its turn at building.
if _, err := os.Stat(final); err == nil {
return final, nil
}
continue
}
var wg sync.WaitGroup
wg.Add(1)
c.build[key] = &wg
c.mu.Unlock()
err := c.fill(final, fill)
c.mu.Lock()
delete(c.build, key)
c.mu.Unlock()
wg.Done()
if err != nil {
return "", err
}
c.evict()
return final, nil
}
}
// fill builds into a temporary file and renames it into place, so a
// failed or interrupted build never leaves a truncated archive that a
// later reader would serve as if it were complete.
func (c *Cache) fill(final string, build func(dst string) error) error {
tmp, err := os.CreateTemp(c.dir, "building-*")
if err != nil {
return err
}
path := tmp.Name()
tmp.Close()
if err := build(path); err != nil {
os.Remove(path)
return err
}
if err := os.Rename(path, final); err != nil {
os.Remove(path)
return err
}
return nil
}
// evict drops least-recently-used entries until the directory fits the
// budget. Entries are immutable, so evicting one costs a rebuild and
// never a wrong answer.
func (c *Cache) evict() {
c.mu.Lock()
defer c.mu.Unlock()
entries, err := os.ReadDir(c.dir)
if err != nil {
return
}
type ent struct {
path string
size int64
age int64
}
var files []ent
var total int64
for _, e := range entries {
if e.IsDir() || filepath.Ext(e.Name()) != ".bin" {
continue
}
fi, err := e.Info()
if err != nil {
continue
}
files = append(files, ent{filepath.Join(c.dir, e.Name()), fi.Size(), fi.ModTime().Unix()})
total += fi.Size()
}
if total <= c.max {
return
}
sort.Slice(files, func(i, j int) bool { return files[i].age < files[j].age })
for _, f := range files {
if total <= c.max {
return
}
if os.Remove(f.path) == nil {
total -= f.size
}
}
}
// touch records a read so eviction can tell a popular archive from one
// built once and never wanted again.
func touch(path string) {
now := nowFunc()
_ = os.Chtimes(path, now, now)
}
// ErrTooLarge reports an archive past the size a cache entry may take.
var ErrTooLarge = errors.New("archive exceeds the size limit")