package main
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"craftdesign.group/quarry/internal/api"
"craftdesign.group/quarry/internal/lease"
"craftdesign.group/quarry/internal/zitcli"
)
// A repository that follows a Git upstream (#358). The forge keeps a
// Git work tree per repository, with nothing checked out, and fetches
// the upstream's branches and tags straight into its own refs; zit
// import-git reads that work tree. zit asks for a work tree, not a bare
// clone (it checks rev-parse --is-inside-work-tree), so this is the
// shape it takes rather than a mirror clone.
//
// The first import makes the store; every pull after is a fetch and
// `zit import-git --into`, which adds only what is new and refuses a
// store written to since its last import. Git mirrors therefore never
// accept local writes.
// gitFetchTimeout bounds one fetch, and gitImportTimeout one import: a
// first run on a large history takes minutes, and a stalled one must
// not hold the store's slot or its lease.
const (
gitFetchTimeout = 30 * time.Minute
gitImportTimeout = 60 * time.Minute
)
// gitEnv is the environment every git the forge runs gets: no global
// or system configuration (no hook, helper or url rewrite in the forge
// account's own setup reaches an upstream), no prompting, and only
// https, so an address cannot make git speak another protocol.
func gitEnv(protocols string) []string {
return []string{
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GIT_TERMINAL_PROMPT=0",
"GIT_CONFIG_COUNT=2",
"GIT_CONFIG_KEY_0=protocol.allow", "GIT_CONFIG_VALUE_0=never",
"GIT_CONFIG_KEY_1=protocol." + protocols + ".allow", "GIT_CONFIG_VALUE_1=always",
}
}
// gitImportDir is where the forge keeps name's Git work tree.
func (rg *registrar) gitImportDir(name string) string {
return filepath.Join(rg.dir, "gitimport", name)
}
// gitProtocol is the one protocol git may use: https, or what a test
// serves its upstream over.
func (rg *registrar) gitProtocol() string {
if rg.gitTestProtocol != "" {
return rg.gitTestProtocol
}
return "https"
}
// gitFetch brings the upstream's branches and tags into dir, making the
// work tree the first time, and points HEAD at the upstream's default
// branch so the import names it trunk.
func (rg *registrar) gitFetch(ctx context.Context, dir, upstream string) error {
ctx, cancel := context.WithTimeout(ctx, gitFetchTimeout)
defer cancel()
env := gitEnv(rg.gitProtocol())
if _, err := os.Stat(filepath.Join(dir, ".git")); errors.Is(err, os.ErrNotExist) {
if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil {
return err
}
if _, err := runGit(ctx, env, "", "init", "-q", dir); err != nil {
return err
}
}
if _, err := runGit(ctx, env, dir, "fetch", "-q", "--prune", "--no-tags", "--update-head-ok", "--",
upstream, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"); err != nil {
return err
}
head := ""
if out, err := runGit(ctx, env, dir, "ls-remote", "--symref", "--", upstream, "HEAD"); err == nil {
for _, l := range strings.Split(out, "\n") {
if rest, ok := strings.CutPrefix(l, "ref: "); ok {
head, _, _ = strings.Cut(rest, "\t")
break
}
}
}
if head == "" {
// An upstream that does not say: the conventional names, in
// the order hosts default to them.
for _, b := range []string{"refs/heads/main", "refs/heads/master", "refs/heads/trunk"} {
if _, err := runGit(ctx, env, dir, "rev-parse", "-q", "--verify", b); err == nil {
head = b
break
}
}
}
if head == "" {
return errors.New("the upstream has no default branch to follow")
}
// The name is the upstream's to choose; only a branch is followed,
// and nothing that could read as an option reaches git.
if !strings.HasPrefix(head, "refs/heads/") || strings.ContainsAny(head, " \t\n") {
return fmt.Errorf("the upstream's default branch %q is not a branch", head)
}
_, err := runGit(ctx, env, dir, "symbolic-ref", "HEAD", head)
return err
}
// runGit runs git with env added, in its own process group so a
// deadline ends every child it started, and answers its output.
func runGit(ctx context.Context, env []string, dir string, args ...string) (string, error) {
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Dir = dir
cmd.Env = append(withoutGitEnv(os.Environ()), env...)
zitcli.OwnGroup(cmd)
cmd.WaitDelay = 2 * time.Second
var out bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &out
if err := cmd.Run(); err != nil {
if ctx.Err() != nil {
return "", fmt.Errorf("git %s did not finish within %v", args[0], gitFetchTimeout)
}
return "", fmt.Errorf("git %s: %s", args[0], gitLast(out.String(), err))
}
return out.String(), nil
}
// withoutGitEnv drops the GIT_* variables the forge process inherited:
// one such as GIT_ALLOW_PROTOCOL or GIT_SSH_COMMAND would override what
// gitEnv sets.
func withoutGitEnv(env []string) []string {
out := env[:0:0]
for _, kv := range env {
if !strings.HasPrefix(kv, "GIT_") {
out = append(out, kv)
}
}
return out
}
// forgetGitDirs removes what the forge keeps on disk for a
// repository's Git mirrors, in and out: the work tree it imports from
// and the Git repository it exports into. Both are rebuilt from the
// store and the upstream, so a purge takes them with it.
func (rg *registrar) forgetGitDirs(name string) {
if rg.dir == "" || name == "" {
return
}
for _, d := range []string{rg.gitImportDir(name), filepath.Join(rg.dir, "gitmirror", "exports", name)} {
if err := os.RemoveAll(d); err != nil {
log.Printf("git dirs %s: %v", name, err)
}
}
}
// gitLock is the per-repository lock a Git mirror's fetch and import
// run under: a manual pull and a scheduled one must not fetch into
// the same work tree while the other's import reads it.
func (rg *registrar) gitLock(name string) func() {
v, _ := rg.gitLocks.LoadOrStore(name, &sync.Mutex{})
mu := v.(*sync.Mutex)
mu.Lock()
return mu.Unlock
}
// gitLast is git's last line of output, or the process error.
func gitLast(out string, err error) string {
lines := strings.Split(strings.TrimSpace(out), "\n")
for i := len(lines) - 1; i >= 0; i-- {
if l := strings.TrimSpace(lines[i]); l != "" && !strings.HasPrefix(l, "hint:") {
return l
}
}
return err.Error()
}
// importGitClone makes the store at spec.Path from the Git repository
// at spec.Clone: a fetch, then a fresh import. A failure removes both
// what it fetched and anything it wrote.
func (rg *registrar) importGitClone(ctx context.Context, spec api.RepoSpec) error {
dir := rg.gitImportDir(spec.Name)
os.RemoveAll(dir) // a previous import of this name is not this one's
if err := rg.gitFetch(ctx, dir, spec.Clone); err != nil {
os.RemoveAll(dir)
return err
}
// zit writes a fresh destination only; the empty directory a caller
// may have made for it (Add accepts one) goes first. Remove refuses
// a directory that is not empty, which Add already ruled out.
_ = os.Remove(spec.Path)
ictx, cancel := context.WithTimeout(ctx, gitImportTimeout)
defer cancel()
r, err := zitcli.ImportGit(ictx, dir, spec.Path, gitEnv(rg.gitProtocol()))
if err != nil {
os.RemoveAll(dir)
return err
}
log.Printf("import %s from Git %s: %d check-ins", spec.Name, spec.Clone, r.Added)
return nil
}
// pullGit is one scheduled pull of a Git mirror: fetch, then add what
// is new under the store's lease. A work tree this node does not have
// (a first pull here, or one lost) is fetched afresh: the import map in
// the store, not the work tree, says what the store already holds.
func (rg *registrar) pullGit(name string, cli *zitcli.Client, guard func(func(lease.Token) error) error, upstream string) (bool, error) {
defer rg.gitLock(name)()
dir := rg.gitImportDir(name)
if err := rg.gitFetch(context.Background(), dir, upstream); err != nil {
return false, err
}
var r zitcli.GitImport
err := guard(func(tok lease.Token) error {
ctx, cancel := context.WithTimeout(context.Background(), gitImportTimeout)
defer cancel()
var ierr error
r, ierr = cli.ImportGitInto(tok, ctx, dir, gitEnv(rg.gitProtocol()))
return ierr
})
if err != nil {
return false, err
}
for _, m := range r.Moved {
log.Printf("mirror %s: %s", name, m)
}
return r.Added > 0 || len(r.Moved) > 0, nil
}