package main
import (
"context"
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
"craftdesign.group/quarry/internal/lease"
)
// A repository's mirror to a Git host (RFC 0022 rung 2): after trunk
// moves, the node holding the store's lease runs `zit git export
// --autopush` with the repository's deploy key, and records what
// happened in words.
//
// Triggered from afterFold, which is every way trunk moves (a push, an
// accepted landing, a merge the forge writes), and from a slow tick,
// which retries a failure after its backoff and makes a just-enabled
// mirror's first push. One run per repository at a time: a trigger
// while one runs makes exactly one more.
// gitMirrorTimeout bounds one run. A first export of a large history
// takes minutes; a hung ssh must not hold the repository's slot for
// ever.
const gitMirrorTimeout = 30 * time.Minute
// gitMirrorTick is how often the retry pass looks.
const gitMirrorTick = 30 * time.Second
// gitMirrors is the per-repository run state. The zero value is ready.
type gitMirrors struct {
mu sync.Mutex
running map[string]bool
again map[string]bool
// ssh, when set, replaces the ssh program in GIT_SSH_COMMAND: a
// test's stub that serves the push from a local bare repository.
ssh string
}
// gitMirrorKick runs name's mirror now, or once more after the run in
// flight. Never blocks the caller.
func (rg *registrar) gitMirrorKick(name string) {
if rg.odb == nil || !rg.jobs.store(name) {
return
}
g := &rg.gitMirror
g.mu.Lock()
if g.running == nil {
g.running, g.again = map[string]bool{}, map[string]bool{}
}
if g.running[name] {
g.again[name] = true
g.mu.Unlock()
return
}
g.running[name] = true
g.mu.Unlock()
go func() {
for {
rg.gitMirrorRun(name, time.Now())
g.mu.Lock()
if !g.again[name] {
delete(g.running, name)
g.mu.Unlock()
return
}
delete(g.again, name)
g.mu.Unlock()
}
}()
}
// gitMirrorRetry kicks every mirror this node runs; gitMirrorRun
// decides whether there is anything to do, which costs a row read.
func (rg *registrar) gitMirrorRetry() {
if rg.odb == nil {
return
}
repos, err := rg.odb.GitMirrorRepos()
if err != nil {
log.Printf("git mirror: list: %v", err)
return
}
for _, name := range repos {
rg.gitMirrorKick(name)
}
}
// gitMirrorBackoff is the wait after the nth failure in a row: a
// minute, doubling, at most an hour.
func gitMirrorBackoff(fails int) time.Duration {
d := time.Minute
for i := 1; i < fails && d < time.Hour; i++ {
d *= 2
}
return min(d, time.Hour)
}
// gitMirrorRun is one pass for one repository: nothing when there is
// no mirror, it is paused, trunk is already pushed, or a failure is
// still backing off; otherwise one export and push.
func (rg *registrar) gitMirrorRun(name string, at time.Time) {
m, on, err := rg.odb.GitMirrorOf(name)
if err != nil || !on {
return
}
// Paused, not failed: making the repository public again, or
// lifting the takedown, resumes without anyone touching the
// mirror. The settings page says which (RFC 0022, Private).
if set, _ := rg.privateSet(); set[name] {
return
}
if _, held, err := rg.odb.TakedownOf(name); err != nil || held {
return
}
repo, ok := rg.deps.Repos.Get(name)
if !ok || repo.Zit == nil || repo.Index == nil || repo.Guard == nil {
return
}
tip, ok, err := repo.Index.TipOf("trunk")
if err != nil || !ok || tip.Tip == m.Pushed {
return
}
if m.ErrAt != "" {
if failed, err := time.Parse(time.RFC3339, m.ErrAt); err == nil && at.Before(failed.Add(gitMirrorBackoff(m.Fails))) {
return
}
}
// Through the store's write lease: zit records the mirror in the
// store's config, and a node that loses the lease stops pushing.
var why string
if err := repo.Guard(func(tok lease.Token) error {
why = rg.gitMirrorPush(name, func(ctx context.Context, mirror, remote, branch string, env []string) (string, error) {
return repo.Zit.GitExport(tok, ctx, mirror, remote, branch, env)
}, m.Remote, m.Branch)
return nil
}); err != nil {
// Not this node's to push, or the lease is in doubt: no outcome
// to record.
return
}
if why == "" {
if err := rg.odb.RecordGitMirrorPush(name, m.Remote, m.Branch, tip.Tip); err != nil {
log.Printf("git mirror: %s: record: %v", name, err)
}
return
}
if err := rg.odb.RecordGitMirrorFailure(name, m.Remote, m.Branch, why); err != nil {
log.Printf("git mirror: %s: record: %v", name, err)
}
}
// gitExport is zitcli.Client.GitExport, named so a run can be handed one.
type gitExport func(ctx context.Context, mirror, remote, branch string, env []string) (string, error)
// gitMirrorPush writes the key for the length of one push and runs it.
// The answer is "" on success, else the failure in words.
//
// The key file lives in a fresh 0700 directory, is 0600, and is removed
// when the push ends, whatever happened. ssh reads no configuration but
// what is given here (-F /dev/null), so an alias or identity in the
// forge account's own ssh config cannot reach a mirror's host, and git
// reads no global or system configuration, so no hook or helper there
// runs either. The host key is trusted on first use and kept, per
// forge, beside the mirrors.
func (rg *registrar) gitMirrorPush(name string, export gitExport, remote, branch string) string {
key, err := rg.odb.GitMirrorKey(name)
if err != nil {
log.Printf("git mirror: %s: key: %v", name, err)
return "the deploy key could not be read; stop mirroring and mirror again to make a new one"
}
home := filepath.Join(rg.dir, "gitmirror")
if err := os.MkdirAll(home, 0o700); err != nil {
log.Printf("git mirror: %s: %v", name, err)
return "the forge could not keep the mirror's Git repository"
}
run, err := os.MkdirTemp("", "quarry-gitmirror-")
if err != nil {
log.Printf("git mirror: %s: %v", name, err)
return "the forge could not write the deploy key for the push"
}
defer os.RemoveAll(run)
keyFile := filepath.Join(run, "key")
if err := os.WriteFile(keyFile, []byte(key), 0o600); err != nil {
log.Printf("git mirror: %s: %v", name, err)
return "the forge could not write the deploy key for the push"
}
ssh := rg.gitMirror.ssh
if ssh == "" {
ssh = "ssh"
}
sshCmd := ssh + " -F /dev/null -i " + shellQuote(keyFile) +
" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20" +
" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=" + shellQuote(filepath.Join(home, "known_hosts"))
env := []string{"GIT_SSH_COMMAND=" + sshCmd, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1",
"GIT_TERMINAL_PROMPT=0"}
ctx, cancel := context.WithTimeout(context.Background(), gitMirrorTimeout)
defer cancel()
out, err := export(ctx, filepath.Join(home, name+".git"), remote, branch, env)
if err == nil {
return ""
}
if ctx.Err() != nil {
return "the push did not finish within " + gitMirrorTimeout.String()
}
log.Printf("git mirror: %s: %v: %s", name, err, lastLines(out, 6))
return gitMirrorWords(out, branch)
}
// gitMirrorWords puts a refused push in words a repository admin can
// act on, from what git and ssh said.
func gitMirrorWords(out, branch string) string {
has := func(subs ...string) bool {
for _, s := range subs {
if strings.Contains(out, s) {
return true
}
}
return false
}
switch {
case has("Permission denied", "Key is invalid", "key is not authorized", "Write access to repository not granted",
"The requested URL returned error: 403"):
return "the host refused the deploy key: check it is added with write access"
case has("[rejected]", "non-fast-forward", "fetch first"):
return "the remote's " + branch + " has history this repository does not: it was not pushed"
case has("Host key verification failed", "REMOTE HOST IDENTIFICATION HAS CHANGED"):
return "the host's key is not the one it showed on the first push: it was not pushed"
case has("Repository not found", "does not appear to be a git repository", "repository not found"):
return "the host has no such repository, or the deploy key cannot see it"
case has("Could not resolve hostname", "Connection timed out", "Connection refused", "No route to host",
"Network is unreachable", "Operation timed out", "Connection closed by"):
return "the host could not be reached"
}
if l := lastLines(out, 1); l != "" {
return "the push failed: " + l
}
return "the push failed"
}
// lastLines is the last n non-empty lines of out that are not git's
// hints, joined.
func lastLines(out string, n int) string {
var keep []string
for _, l := range strings.Split(out, "\n") {
l = strings.TrimSpace(l)
if l == "" || strings.HasPrefix(l, "hint:") {
continue
}
keep = append(keep, l)
}
if len(keep) > n {
keep = keep[len(keep)-n:]
}
return strings.Join(keep, " / ")
}
// shellQuote quotes s for the shell git runs GIT_SSH_COMMAND through.
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}