package main
import (
"context"
"crypto/rand"
"encoding/json"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"craftdesign.group/quarry/internal/api"
"craftdesign.group/quarry/internal/ci"
"craftdesign.group/quarry/internal/lease"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/zitcli"
)
// needGitImport skips where this zit cannot keep a Git import current.
func needGitImport(t *testing.T) {
t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git not on PATH")
}
out, _ := exec.Command("zit", "import-git", "--help").CombinedOutput()
if !strings.Contains(string(out), "--into") {
t.Skip("this zit has no import-git --into")
}
}
func gitIn(t *testing.T, dir string, args ...string) {
t.Helper()
args = append([]string{"-C", dir, "-c", "[email protected]", "-c", "user.name=A"}, args...)
if out, err := exec.Command("git", args...).CombinedOutput(); err != nil {
t.Fatalf("git %v: %v %s", args, err, out)
}
}
func timelineCount(t *testing.T, store string) int {
t.Helper()
out, err := exec.Command("zit", "timeline", "-R", store, "-n", "100").CombinedOutput()
if err != nil {
t.Fatalf("zit timeline: %v %s", err, out)
}
return strings.Count(string(out), "] ")
}
// #358: a repository made from a Git upstream, then kept current: a
// pull with nothing new changes nothing, new commits arrive, and a
// force-push upstream is taken in rather than refused.
func TestARepositoryFollowsAGitUpstream(t *testing.T) {
needGitImport(t)
t.Setenv("GIT_AUTHOR_DATE", "2001-01-01T00:00:00Z")
t.Setenv("GIT_COMMITTER_DATE", "2001-01-01T00:00:00Z")
up := t.TempDir()
gitIn(t, up, "init", "-q", "-b", "main")
gitIn(t, up, "commit", "-q", "--allow-empty", "-m", "one")
gitIn(t, up, "commit", "-q", "--allow-empty", "-m", "two")
commitFile := func(content string) {
t.Helper()
if err := os.WriteFile(filepath.Join(up, "value.txt"), []byte(content), 0o600); err != nil {
t.Fatal(err)
}
gitIn(t, up, "add", "value.txt")
gitIn(t, up, "commit", "-q", "-m", content)
}
rg := newTestRegistrar(t, 25)
rg.dir = t.TempDir()
rg.gitTestProtocol = "file"
store := filepath.Join(t.TempDir(), "repos", "follow") // the repos root may not exist yet
spec := api.RepoSpec{Name: "follow", Path: store, Clone: up, Git: true, Mirror: true}
if err := rg.importGitClone(context.Background(), spec); err != nil {
t.Fatalf("FAIL: the first import: %v", err)
}
if n := timelineCount(t, store); n != 2 {
t.Fatalf("FAIL: the import holds %d check-ins, want 2", n)
}
cli := zitcli.New(store)
guard := func(fn func(lease.Token) error) error { return fn(lease.Unguarded("test")) }
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || changed {
t.Errorf("FAIL: a pull with nothing new: changed %v err %v", changed, err)
}
// The imported head is authoritative after a rewrite; Fossil's trunk
// may still resolve to an abandoned, later-dated leaf (zit#252).
assertHead := func(want string) {
t.Helper()
name, hash, err := cli.ImportedHead(context.Background())
if err != nil || name != "heads/main" {
t.Fatalf("FAIL: imported head %q: %v", name, err)
}
out, err := cli.Cat(context.Background(), "value.txt", hash)
if err != nil || string(out) != want {
t.Fatalf("FAIL: imported head content %q, want %q: %v", out, want, err)
}
}
t.Setenv("GIT_AUTHOR_DATE", "2001-01-03T00:00:00Z")
t.Setenv("GIT_COMMITTER_DATE", "2001-01-03T00:00:00Z")
commitFile("three")
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || !changed {
t.Fatalf("FAIL: a pull after a new commit: changed %v err %v", changed, err)
}
if n := timelineCount(t, store); n != 3 {
t.Errorf("FAIL: after the pull the store holds %d check-ins, want 3", n)
}
assertHead("three")
gitIn(t, up, "reset", "-q", "--hard", "HEAD~1")
// A rewrite may point to an older-dated commit. The default view must
// still follow the source head (zit#252), not the abandoned leaf.
t.Setenv("GIT_AUTHOR_DATE", "2001-01-02T00:00:00Z")
t.Setenv("GIT_COMMITTER_DATE", "2001-01-02T00:00:00Z")
commitFile("rewritten")
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || !changed {
t.Errorf("FAIL: a pull after a force-push: changed %v err %v", changed, err)
}
assertHead("rewritten")
// Nothing of Git stays behind: each pull fetched into a directory
// of its own and removed it.
assertNoGitKept(t, rg)
}
// assertNoGitKept fails when a fetch outlived its pull.
func assertNoGitKept(t *testing.T, rg *registrar) {
t.Helper()
left, _ := filepath.Glob(filepath.Join(rg.dir, "gitimport", "*", "*"))
if len(left) != 0 {
t.Errorf("FAIL: Git data kept after the pull: %v", left)
}
}
// followRig is an upstream with one commit, imported into a store.
func followRig(t *testing.T) (rg *registrar, up, store string, cli *zitcli.Client, guard func(func(lease.Token) error) error) {
t.Helper()
needGitImport(t)
up = t.TempDir()
gitIn(t, up, "init", "-q", "-b", "main")
gitIn(t, up, "commit", "-q", "--allow-empty", "-m", "one")
rg = newTestRegistrar(t, 25)
rg.dir = t.TempDir()
rg.gitTestProtocol = "file"
store = filepath.Join(t.TempDir(), "follow")
if err := rg.importGitClone(context.Background(), api.RepoSpec{Name: "follow", Path: store, Clone: "file://" + up, Git: true, Mirror: true}); err != nil {
t.Fatalf("the first import: %v", err)
}
cli = zitcli.New(store)
guard = func(fn func(lease.Token) error) error { return fn(lease.Unguarded("test")) }
return rg, "file://" + up, store, cli, guard
}
// A pull fetches only recent history, and fetches more only when zit
// says what it has does not reach the store.
func TestAGitPullFetchesShallowAndDeepensOnlyWhenAsked(t *testing.T) {
rg, up, store, cli, guard := followRig(t)
dir := strings.TrimPrefix(up, "file://")
var depths []int
rg.gitOnFetch = func(d int) { depths = append(depths, d) }
rg.gitFirstDepth = 2
gitIn(t, dir, "commit", "-q", "--allow-empty", "-m", "two")
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || !changed {
t.Fatalf("FAIL: one new commit: changed %v err %v", changed, err)
}
if len(depths) != 1 || depths[0] != 2 {
t.Errorf("FAIL: one new commit fetched at depths %v, want [2]", depths)
}
depths = nil
for i := 0; i < 6; i++ {
gitIn(t, dir, "commit", "-q", "--allow-empty", "-m", "more "+strconv.Itoa(i))
}
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || !changed {
t.Fatalf("FAIL: six new commits: changed %v err %v", changed, err)
}
if len(depths) < 2 || depths[0] != 2 || depths[1] <= 2 {
t.Errorf("FAIL: six new commits past a depth of 2 fetched at depths %v; want a deeper second fetch", depths)
}
if n := timelineCount(t, store); n != 8 {
t.Errorf("FAIL: the store holds %d check-ins, want 8", n)
}
assertNoGitKept(t, rg)
}
// A tag added or removed upstream changes the store without a check-in,
// and the pull says so, so the index learns of it.
func TestAGitPullReportsATagOnlyChange(t *testing.T) {
rg, up, store, cli, guard := followRig(t)
dir := strings.TrimPrefix(up, "file://")
gitIn(t, dir, "tag", "v1")
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || !changed {
t.Errorf("FAIL: a new tag: changed %v err %v", changed, err)
}
gitIn(t, dir, "tag", "-d", "v1")
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || !changed {
t.Errorf("FAIL: a deleted tag: changed %v err %v", changed, err)
}
if changed, err := rg.pullGit("follow", store, cli, guard, up); err != nil || changed {
t.Errorf("FAIL: nothing new: changed %v err %v", changed, err)
}
}
// A renamed upstream default is not followed on its own: the pull
// fails, says why, and the store keeps what it had.
func TestAGitPullRefusesARenamedDefaultBranch(t *testing.T) {
rg, up, store, cli, guard := followRig(t)
_, before, err := cli.ImportedHead(context.Background())
if err != nil {
t.Fatal(err)
}
gitIn(t, strings.TrimPrefix(up, "file://"), "branch", "-m", "main", "renamed")
_, err = rg.pullGit("follow", store, cli, guard, up)
if err == nil || !strings.Contains(err.Error(), "refused") {
t.Errorf("FAIL: a renamed default: %v", err)
}
if _, after, herr := cli.ImportedHead(context.Background()); herr != nil || after != before {
t.Errorf("FAIL: the imported head moved to %s (%v), want %s", after, herr, before)
}
assertNoGitKept(t, rg)
}
// An import killed while writing leaves the store unreadable until zit
// finishes it; the next pull does that first, and the store reads again.
func TestAGitPullFinishesAnInterruptedImport(t *testing.T) {
rg, up, store, cli, guard := followRig(t)
dir := strings.TrimPrefix(up, "file://")
payload := make([]byte, 256<<10)
for i := 0; i < 32; i++ {
rand.Read(payload)
if err := os.WriteFile(filepath.Join(dir, "payload.bin"), payload, 0o600); err != nil {
t.Fatal(err)
}
gitIn(t, dir, "add", "payload.bin")
gitIn(t, dir, "commit", "-q", "-m", "payload "+strconv.Itoa(i))
}
src := filepath.Join(t.TempDir(), "src.git")
if out, err := exec.Command("git", "clone", "-q", "--bare", up, src).CombinedOutput(); err != nil {
t.Fatalf("git clone: %v %s", err, out)
}
cmd := exec.Command("zit", "import-git", src, store, "--into", "--no-submanifests")
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(30 * time.Second)
for !zitcli.GitImportPending(store) && time.Now().Before(deadline) && cmd.ProcessState == nil {
time.Sleep(time.Millisecond)
}
cmd.Process.Kill()
cmd.Wait()
if !zitcli.GitImportPending(store) {
t.Skip("the import finished before it could be interrupted")
}
if _, _, err := cli.ImportedHead(context.Background()); err == nil {
t.Errorf("FAIL: an interrupted store still reads")
}
if _, err := rg.pullGit("follow", store, cli, guard, up); err != nil {
t.Fatalf("FAIL: the pull after an interrupted import: %v", err)
}
if zitcli.GitImportPending(store) {
t.Errorf("FAIL: the interrupted import is still pending")
}
if n := timelineCount(t, store); n != 33 {
t.Errorf("FAIL: the store holds %d check-ins, want 33", n)
}
assertNoGitKept(t, rg)
}
// Only the one protocol: an upstream on another is refused by git.
func TestAGitUpstreamSpeaksOnlyHTTPS(t *testing.T) {
needGitImport(t)
up := t.TempDir()
gitIn(t, up, "init", "-q", "-b", "main")
gitIn(t, up, "commit", "-q", "--allow-empty", "-m", "one")
rg := newTestRegistrar(t, 25)
rg.dir = t.TempDir()
err := rg.gitFetch(context.Background(), filepath.Join(t.TempDir(), "w"), up, 0)
if err == nil || !strings.Contains(err.Error(), "not allowed") {
t.Fatalf("FAIL: a local-path upstream with only https allowed: %v", err)
}
}
// The ops refuse what a Git mirror cannot be: one without an https
// address, and one that takes local writes.
func TestAGitMirrorRefusesWhatItCannotBe(t *testing.T) {
g := newSiteRig(t)
for _, body := range []map[string]any{
{"name": "g1", "git": true},
{"name": "g2", "git": true, "clone": "http://example.org/r.git"},
{"name": "g3", "git": true, "clone": "file:///etc"},
} {
if code, out := g.doAs("alice", http.MethodPost, "/api/v1/repos", doorPassword, "", body); code != http.StatusBadRequest {
t.Errorf("FAIL: %v: %d %s", body, code, out)
}
}
if err := g.db.SetMirror("pub", "https://github.com/alice/pub.git", oltp.MinMirrorInterval); err != nil {
t.Fatal(err)
}
if err := g.db.SetMirrorGit("pub"); err != nil {
t.Fatal(err)
}
code, out := g.doAs("alice", http.MethodGet, "/api/v1/r/pub/info", doorPassword, "", nil)
var info struct {
Mirror struct {
Git bool `json:"git"`
} `json:"mirror"`
}
if err := json.Unmarshal([]byte(out), &info); err != nil || code != http.StatusOK || !info.Mirror.Git {
t.Fatalf("FAIL: repo.get lost Git mirror posture: status %d, error %v, body %s", code, err, out)
}
code, out = g.doAs("alice", http.MethodPost, "/api/v1/mirrors/writes", doorPassword, "",
map[string]any{"repo": "pub", "accept": true})
if code != http.StatusBadRequest || !strings.Contains(out, "Git") {
t.Errorf("FAIL: a Git mirror took local writes: %d %s", code, out)
}
if m, _, _ := g.db.MirrorFor("pub"); m.AcceptWrites || !m.Git {
t.Errorf("FAIL: the mirror is now %+v", m)
}
}
// No CI verdict tag is written into a Git mirror: the next import
// would refuse the store.
func TestNoVerdictTagIsWrittenIntoAGitMirror(t *testing.T) {
rg := newTestRegistrar(t, 25)
if err := rg.odb.AddRepo("follow", t.TempDir(), "", oltp.Public); err != nil {
t.Fatal(err)
}
if err := rg.odb.SetMirror("follow", "https://github.com/a/b.git", oltp.MinMirrorInterval); err != nil {
t.Fatal(err)
}
if err := rg.odb.SetMirrorGit("follow"); err != nil {
t.Fatal(err)
}
writes := 0
repos := api.NewRepos(map[string]*api.Repo{"follow": {
Guard: func(fn func(lease.Token) error) error { writes++; return nil },
}})
r := oltp.CIRun{Repo: "follow", Checkin: "abc", RunID: "run1"}
if err := writeVerdict(rg.odb, repos, true, r, ci.Run{Status: "completed"}); err != nil {
t.Fatal(err)
}
if writes != 0 {
t.Errorf("FAIL: a verdict was written into a Git mirror (%d writes)", writes)
}
}
// repo.add carries git through to the registry, and the registry
// records the mirror as a Git one: the two links between the form and
// the Git pull.
func TestAGitCloneIsRecordedAsAGitMirror(t *testing.T) {
g := newSiteRig(t)
code, out := g.doAs("alice", http.MethodPost, "/api/v1/repos", doorPassword, "",
map[string]any{"name": "follow", "clone": "https://github.com/a/b", "git": true})
if code != http.StatusOK && code != http.StatusCreated {
t.Fatalf("repo.add: %d %s", code, out)
}
if len(g.regs.added) != 1 || !g.regs.added[0].Git || !g.regs.added[0].Mirror {
t.Fatalf("FAIL: the registry was handed %+v", g.regs.added)
}
rg := newTestRegistrar(t, 25)
if err := rg.odb.AddRepo("follow", t.TempDir(), "", oltp.Public); err != nil {
t.Fatal(err)
}
rg.recordMirror(g.regs.added[0])
if m, ok, err := rg.odb.MirrorFor("follow"); err != nil || !ok || !m.Git || !m.Tracking() {
t.Errorf("FAIL: recorded as %+v %v %v", m, ok, err)
}
}
// One direction per repository: one that follows a Git upstream is not
// pushed back out to Git.
func TestAFollowingRepositoryIsNotMirroredOut(t *testing.T) {
g := newSiteRig(t)
if err := g.db.SetMirror("pub", "https://github.com/a/b", oltp.MinMirrorInterval); err != nil {
t.Fatal(err)
}
if err := g.db.SetMirrorGit("pub"); err != nil {
t.Fatal(err)
}
code, out := g.doAs("alice", http.MethodPost, "/api/v1/r/pub/git-mirror", doorPassword, "",
map[string]string{"remote": "[email protected]:alice/pub.git"})
if code != http.StatusBadRequest || !strings.Contains(out, "follows") {
t.Errorf("FAIL: a following repository was mirrored out: %d %s", code, out)
}
}