Artifact 886dfbf4609d5e73ba14a7f9243d4911b24cfac65f1f4aa3b4a16f80cbd8812c:
- File
internal/checkout/extract_test.go
— part of check-in
[a71bf662ee]
at
2026-10-18 20:59:23
on branch main
— fix(checkout): record and trust vfile.mtime as fossil does (#444)
* fix(checkout): record and trust vfile.mtime as fossil does
go-libfossil wrote vfile.mtime = 0 everywhere and hashed every file on every scan, so it and fossil rehashed a go-libfossil checkout on every command.
The scan is now fossil's vfile_check_signature: a file whose size matches its artifact (blob.size of mrid) and whose mtime matches vfile.mtime is taken as unchanged without being read, unless ScanHash is given or the mtime-changes setting is off; the scan stores the file's mtime in whole seconds and runs its updates in one transaction. Status, commit, merge, revert and extract use the default scan, as fossil's commands do.
Mtimes are recorded where fossil records them: extract and update store the mtime of each file written with its artifact's content, an unchanged file keeps its mtime through LoadVFile (fossil's update transfer and in-place commit), committed files keep theirs, revert stores the restored file's, and SetMTime stores the time it gives. Undoing a rename drops the mtime, since the file was not checked.
One deliberate difference: an mtime not older than the current second is stored as 0, so the next scan (ours or fossil's) hashes the file and an edit that keeps the size within that second is still seen. A stored 0 is never trusted.
MemStorage now stamps each write with a new mtime from its own clock, as a filesystem does, so in-memory checkouts see same-size edits.
Status on the sqlite repository's 2,220-file checkout: about 250 ms before, about 30 ms after.
* test(checkout): pin the same-second guard against fossil
A same-size edit in the second a file was scanned is committed, and fossil 2.28 reading a row the guard stored as 0 reports the edit. Both tests fail with the guard removed. The guard's comment and the CHANGELOG name it as the one deliberate difference from vfile_check_signature.
* fix(checkout): record the mtime of each write as it happens
Extract and update statted their files only after every file was written and vfile reloaded, so an edit in between was recorded as the written content. They now stat each file right after writing it, as fossil's vfile_to_disk does, and carry that mtime to the row. Revert and the new writtenMTime read the clock before the stat, as the scan does.
A caller's Env without a Clock now gets the real clock, as Storage defaults to the OS; scans read it. The mtime-changes setting is read from the repository only: go-libfossil reads no global fossil settings, which the comment and CHANGELOG now say.
New tests: an edit after extract's or update's write is seen, a stored 0 is never trusted, and a simulated clock at a realistic time drives the mtime fast path. (user: [email protected], size: 20459) [annotate] [blame] [check-ins using] [more...]
package checkout import ( "context" "errors" "strings" "testing" "time" libfossil "github.com/danmestas/go-libfossil/internal/fsltype" "github.com/danmestas/go-libfossil/internal/manifest" "github.com/danmestas/go-libfossil/simio" ) func contains(s, substr string) bool { return strings.Contains(s, substr) } func TestExtract(t *testing.T) { r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() // Use MemStorage to capture extracted files mem := simio.NewMemStorage() co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} // Use a virtual dir for extraction that works with MemStorage co.dir = "/checkout" if err := co.Extract(rid, ExtractOpts{}); err != nil { t.Fatal(err) } // Verify files were written to MemStorage data, err := mem.ReadFile("/checkout/hello.txt") if err != nil { t.Fatal("hello.txt not found:", err) } if string(data) != "hello world\n" { t.Fatalf("hello.txt = %q, want %q", data, "hello world\n") } data2, err := mem.ReadFile("/checkout/src/main.go") if err != nil { t.Fatal("src/main.go not found:", err) } if string(data2) != "package main\n" { t.Fatalf("src/main.go = %q", data2) } data3, err := mem.ReadFile("/checkout/README.md") if err != nil { t.Fatal("README.md not found:", err) } if string(data3) != "# Test\n" { t.Fatalf("README.md = %q", data3) } } func TestExtractDryRun(t *testing.T) { r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() mem := simio.NewMemStorage() co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} co.dir = "/checkout" var callbackFiles []string err = co.Extract(rid, ExtractOpts{ DryRun: true, Callback: func(name string, change UpdateChange) error { callbackFiles = append(callbackFiles, name) return nil }, }) if err != nil { t.Fatal(err) } // Files should NOT be in MemStorage if _, err := mem.ReadFile("/checkout/hello.txt"); err == nil { t.Fatal("DryRun should not write files") } // But callback should have been called if len(callbackFiles) != 3 { t.Fatalf("expected 3 callback calls, got %d", len(callbackFiles)) } } func TestExtractForceProtection(t *testing.T) { r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() mem := simio.NewMemStorage() co.env = &simio.Env{ Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}, } co.dir = "/checkout" // Extract files first. if err := co.Extract(rid, ExtractOpts{Force: true}); err != nil { t.Fatal(err) } // Modify hello.txt on disk so it differs from vfile mhash. if err := mem.WriteFile( "/checkout/hello.txt", []byte("local edit\n"), 0644, ); err != nil { t.Fatal(err) } // Extract again WITHOUT Force — should fail because hello.txt // has local changes. err = co.Extract(rid, ExtractOpts{Force: false}) if err == nil { t.Fatal("Extract without Force should fail over modified files") } if !contains(err.Error(), "local changes") { t.Fatalf("error should mention local changes, got: %v", err) } // Extract with Force=true should succeed. if err := co.Extract(rid, ExtractOpts{Force: true}); err != nil { t.Fatalf("Extract with Force should succeed: %v", err) } } func TestExtractSetMTime(t *testing.T) { r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() mem := simio.NewMemStorage() co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} co.dir = "/checkout" if err := co.Extract(rid, ExtractOpts{SetMTime: true}); err != nil { t.Fatal(err) } // Verify mtime was set on extracted files. info, err := mem.Stat("/checkout/hello.txt") if err != nil { t.Fatal("stat hello.txt:", err) } mtime := info.ModTime() if mtime.IsZero() { t.Fatal("mtime should not be zero when SetMTime is true") } // The checkin timestamp should match what's in the event table. // Just verify it's a valid time (not zero) — the exact value depends // on the test repo's fixed timestamp. if mtime.Year() < 2020 { t.Fatalf("mtime %v looks invalid", mtime) } // Verify all extracted files got the same mtime. info2, _ := mem.Stat("/checkout/src/main.go") if !info2.ModTime().Equal(mtime) { t.Fatalf("src/main.go mtime %v != hello.txt mtime %v", info2.ModTime(), mtime) } } func TestExtractSetMTimeFalse(t *testing.T) { r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() mem := simio.NewMemStorage() co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} co.dir = "/checkout" if err := co.Extract(rid, ExtractOpts{SetMTime: false}); err != nil { t.Fatal(err) } // Verify mtime was NOT set to the check-in time. info, err := mem.Stat("/checkout/hello.txt") if err != nil { t.Fatal("stat hello.txt:", err) } // The write's own time, not the check-in's. if info.ModTime().Equal(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) { t.Fatalf("mtime set to the check-in time when SetMTime is false: %v", info.ModTime()) } } func TestExtractSetMTimeDryRun(t *testing.T) { r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() mem := simio.NewMemStorage() co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} co.dir = "/checkout" // DryRun + SetMTime should not write files or set mtimes. if err := co.Extract(rid, ExtractOpts{SetMTime: true, DryRun: true}); err != nil { t.Fatal(err) } if _, err := mem.ReadFile("/checkout/hello.txt"); err == nil { t.Fatal("DryRun should not write files even with SetMTime") } } func TestExtractObserver(t *testing.T) { // Use a recording observer to verify hooks fire r, cleanup := newTestRepoWithCheckin(t) defer cleanup() dir := t.TempDir() co, err := Create(r, dir, CreateOpts{}) if err != nil { t.Fatal(err) } defer co.Close() rid, _, _ := co.Version() mem := simio.NewMemStorage() type event struct{ name string } var events []event obs := &testObserver{ onExtractStarted: func(ctx context.Context, e ExtractStart) context.Context { events = append(events, event{"started"}) return ctx }, onExtractFileCompleted: func(ctx context.Context, name string, change UpdateChange) { events = append(events, event{"file:" + name}) }, onExtractCompleted: func(ctx context.Context, e ExtractEnd) { events = append(events, event{"completed"}) }, } co.obs = obs co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} co.dir = "/checkout" co.Extract(rid, ExtractOpts{}) if len(events) < 5 { // started + 3 files + completed t.Fatalf("expected at least 5 events, got %d: %v", len(events), events) } if events[0].name != "started" { t.Fatalf("first event should be started, got %s", events[0].name) } if events[len(events)-1].name != "completed" { t.Fatalf("last event should be completed, got %s", events[len(events)-1].name) } } // newCheckoutAtFirstOfTwo builds a repo with two checkins (see // newTestRepoWithTwoCheckins: rid2 edits hello.txt and adds new.txt) and a // checkout with rid1 extracted into memory storage. func newCheckoutAtFirstOfTwo(t *testing.T) (co *Checkout, mem *simio.MemStorage, rid1, rid2 libfossil.FslID) { t.Helper() r, rid1, rid2, cleanup := newTestRepoWithTwoCheckins(t) t.Cleanup(cleanup) co, err := Create(r, t.TempDir(), CreateOpts{}) if err != nil { t.Fatal(err) } t.Cleanup(func() { co.Close() }) if err := setVVar(co.db, "checkout", itoa(int64(rid1))); err != nil { t.Fatal(err) } mem = simio.NewMemStorage() co.env = &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} co.dir = "/checkout" if err := co.Extract(rid1, ExtractOpts{}); err != nil { t.Fatal("extract rid1:", err) } return co, mem, rid1, rid2 } // changedNames scans the checkout's current version and returns the names // of its changed files. func changedNames(t *testing.T, co *Checkout) []string { t.Helper() vid, _, err := co.Version() if err != nil { t.Fatal(err) } var names []string err = co.VisitChanges(vid, true, func(e ChangeEntry) error { names = append(names, e.Name) return nil }) if err != nil { t.Fatal(err) } return names } // Switching a clean checkout to another version is what Extract is for: the // files that differ between the versions are not local changes (#230). func TestExtractSwitchesCleanCheckout(t *testing.T) { co, mem, _, rid2 := newCheckoutAtFirstOfTwo(t) if err := co.Extract(rid2, ExtractOpts{}); err != nil { t.Fatalf("Extract to the next version of a clean checkout: %v", err) } data, err := mem.ReadFile("/checkout/hello.txt") if err != nil { t.Fatal(err) } if string(data) != "hello updated world\n" { t.Fatalf("hello.txt = %q, want rid2's content", data) } if vid := mustVersion(t, co); vid != rid2 { t.Fatalf("Version = %d, want %d", vid, rid2) } } // A refused Extract must leave the checkout exactly as it was: same version, // and the unsaved edit still reported (#230). func TestExtractRefusalLeavesCheckoutUntouched(t *testing.T) { co, mem, rid1, rid2 := newCheckoutAtFirstOfTwo(t) if err := mem.WriteFile("/checkout/README.md", []byte("local edit\n"), 0o644); err != nil { t.Fatal(err) } err := co.Extract(rid2, ExtractOpts{}) if err == nil { t.Fatal("Extract over an unsaved edit succeeded") } if !contains(err.Error(), "local changes") { t.Fatalf("error should mention local changes, got: %v", err) } if vid := mustVersion(t, co); vid != rid1 { t.Fatalf("Version = %d after refusal, want %d", vid, rid1) } if got := changedNames(t, co); len(got) != 1 || got[0] != "README.md" { t.Fatalf("changes after refusal = %q, want only README.md", got) } } // A pending add is unsaved work too, even though the target version would // not write over the file. func TestExtractRefusesOverPendingAdd(t *testing.T) { co, mem, _, rid2 := newCheckoutAtFirstOfTwo(t) if err := mem.WriteFile("/checkout/extra.txt", []byte("extra\n"), 0o644); err != nil { t.Fatal(err) } if _, err := co.Manage(ManageOpts{Paths: []string{"extra.txt"}}); err != nil { t.Fatal(err) } if err := co.Extract(rid2, ExtractOpts{}); err == nil { t.Fatal("Extract over a pending add succeeded") } if got := changedNames(t, co); len(got) != 1 || got[0] != "extra.txt" { t.Fatalf("changes after refusal = %q, want only extra.txt", got) } } // A file the current version does not track, sitting where the target would // write different content, is the user's and must not be clobbered. func TestExtractRefusesOverUntrackedFile(t *testing.T) { co, mem, _, rid2 := newCheckoutAtFirstOfTwo(t) if err := mem.WriteFile("/checkout/new.txt", []byte("mine\n"), 0o644); err != nil { t.Fatal(err) } if err := co.Extract(rid2, ExtractOpts{}); err == nil { t.Fatal("Extract over an untracked file succeeded") } data, err := mem.ReadFile("/checkout/new.txt") if err != nil { t.Fatal(err) } if string(data) != "mine\n" { t.Fatalf("new.txt = %q, untracked file was overwritten", data) } } // Create records the tip as checked out before any file is written. Extracting // an older version into a directory that already holds it must still work: // the files differ from the tip, but none would lose content. This is how // EdgeSync's ExtractTo uses Create + Extract. func TestExtractReextractOlderVersionIntoCreatedCheckout(t *testing.T) { r, rid1, _, cleanup := newTestRepoWithTwoCheckins(t) defer cleanup() mem := simio.NewMemStorage() env := &simio.Env{Storage: mem, Clock: simio.RealClock{}, Rand: simio.CryptoRand{}} for pass := 1; pass <= 2; pass++ { co, err := Create(r, t.TempDir(), CreateOpts{}) if err != nil { t.Fatal(err) } co.env = env co.dir = "/checkout" err = co.Extract(rid1, ExtractOpts{}) co.Close() if err != nil { t.Fatalf("pass %d: Extract(rid1) into a created checkout: %v", pass, err) } } data, err := mem.ReadFile("/checkout/hello.txt") if err != nil { t.Fatal(err) } if string(data) != "hello world\n" { t.Fatalf("hello.txt = %q, want rid1's content", data) } } // A pending merge (a vmerge row) is unsaved work even when every edited file // already matches the target: switching would drop it, and the stale vmerge // row would later give a commit a bogus merge parent. func TestExtractRefusesOverPendingMergeMatchingTarget(t *testing.T) { co, mem, _, rid2 := newCheckoutAtFirstOfTwo(t) if err := mem.WriteFile( "/checkout/hello.txt", []byte("hello updated world\n"), 0o644, ); err != nil { t.Fatal(err) } if _, err := co.db.Exec( "INSERT INTO vmerge(id, merge, mhash) VALUES(0, ?, 'x')", int64(rid2), ); err != nil { t.Fatal(err) } if err := co.Extract(rid2, ExtractOpts{}); err == nil { t.Fatal("Extract over a pending merge succeeded") } } // phantomize turns the stored content of name in version rid into a phantom, // the shape a partial sync leaves: the repo knows the artifact's hash but // holds no content for it. func phantomize(t *testing.T, co *Checkout, rid libfossil.FslID, name string) { t.Helper() files, err := manifest.ListFiles(co.repo, rid) if err != nil { t.Fatal(err) } for _, f := range files { if f.Name != name { continue } db := co.repo.DB() if _, err := db.Exec( "UPDATE blob SET size = -1, content = NULL WHERE uuid = ?", f.UUID, ); err != nil { t.Fatal(err) } if _, err := db.Exec( "INSERT OR IGNORE INTO phantom(rid) SELECT rid FROM blob WHERE uuid = ?", f.UUID, ); err != nil { t.Fatal(err) } return } t.Fatalf("%s is not in version %d", name, rid) } // Extracting a version whose content is partly missing must refuse up front // and leave the checkout as it was, as fossil's checkout does (#231). func TestExtractRefusesVersionWithMissingContent(t *testing.T) { co, mem, rid1, rid2 := newCheckoutAtFirstOfTwo(t) phantomize(t, co, rid2, "new.txt") err := co.Extract(rid2, ExtractOpts{}) if err == nil { t.Fatal("Extract of a version with missing content succeeded") } if !contains(err.Error(), "new.txt") { t.Fatalf("error should name the missing file, got: %v", err) } if vid := mustVersion(t, co); vid != rid1 { t.Fatalf("Version = %d after refusal, want %d", vid, rid1) } var rows int if err := co.db.QueryRow( "SELECT count(*) FROM vfile WHERE vid = ?", int64(rid2), ).Scan(&rows); err != nil { t.Fatal(err) } if rows != 0 { t.Fatalf("vfile holds %d rows for the refused version, want 0", rows) } data, err := mem.ReadFile("/checkout/hello.txt") if err != nil { t.Fatal(err) } if string(data) != "hello world\n" { t.Fatalf("hello.txt = %q, the refused extract wrote files", data) } } // A partly synced tip must not stop Create: the caller may be about to // extract an older, complete version (EdgeSync's ExtractTo does). func TestCreateWithPartialTipThenExtractCompleteVersion(t *testing.T) { r, rid1, rid2, cleanup := newTestRepoWithTwoCheckins(t) defer cleanup() probe := &Checkout{repo: r} phantomize(t, probe, rid2, "new.txt") var observed []error co, err := Create(r, t.TempDir(), CreateOpts{Observer: &testObserver{ onError: func(_ context.Context, err error) { observed = append(observed, err) }, }}) if err != nil { t.Fatalf("Create with a partly synced tip: %v", err) } defer co.Close() if vid := mustVersion(t, co); vid != rid2 { t.Fatalf("Version = %d after Create, want the tip %d", vid, rid2) } if len(observed) != 1 || !errors.Is(observed[0], errMissingContent) { t.Fatalf("observer errors = %v, want one errMissingContent", observed) } var rows int if err := co.db.QueryRow("SELECT count(*) FROM vfile").Scan(&rows); err != nil { t.Fatal(err) } if rows != 0 { t.Fatalf("vfile holds %d rows for a tip it cannot load, want 0", rows) } co.env = &simio.Env{ Storage: simio.NewMemStorage(), Clock: simio.RealClock{}, Rand: simio.CryptoRand{}, } co.dir = "/checkout" if err := co.Extract(rid1, ExtractOpts{}); err != nil { t.Fatalf("Extract of the complete version: %v", err) } if err := co.db.QueryRow( "SELECT count(*) FROM vfile WHERE vid = ?", int64(rid1), ).Scan(&rows); err != nil { t.Fatal(err) } if rows != 3 { t.Fatalf("vfile holds %d rows for rid1 after Extract, want 3", rows) } } // mustVersion returns the checkout's current version, failing the test on // error. func mustVersion(t *testing.T, co *Checkout) libfossil.FslID { t.Helper() vid, _, err := co.Version() if err != nil { t.Fatal(err) } return vid } // A dry run reports what Extract would write and changes nothing: not the // files, not the checkout's file list, not its version (#236). func TestExtractDryRunChangesNothing(t *testing.T) { co, mem, rid1, rid2 := newCheckoutAtFirstOfTwo(t) if err := mem.WriteFile("/checkout/README.md", []byte("local edit\n"), 0o644); err != nil { t.Fatal(err) } var reported []string report := func(name string, _ UpdateChange) error { reported = append(reported, name) return nil } err := co.Extract(rid2, ExtractOpts{DryRun: true, Callback: report}) if err != nil { t.Fatalf("dry run: %v", err) } if len(reported) != 4 { t.Fatalf("dry run reported %v, want rid2's 4 files", reported) } if vid := mustVersion(t, co); vid != rid1 { t.Fatalf("Version = %d after a dry run, want %d", vid, rid1) } if got := changedNames(t, co); len(got) != 1 || got[0] != "README.md" { t.Fatalf("changes after a dry run = %q, want the README.md edit kept", got) } data, err := mem.ReadFile("/checkout/hello.txt") if err != nil { t.Fatal(err) } if string(data) != "hello world\n" { t.Fatalf("hello.txt = %q after a dry run, want it untouched", data) } } // Switching versions ends any pending merge, as fossil's checkout does: a // vmerge row left behind would give the next commit a bogus merge parent // (#236). func TestExtractClearsPendingMergeRecord(t *testing.T) { co, _, _, rid2 := newCheckoutAtFirstOfTwo(t) if _, err := co.db.Exec( "INSERT INTO vmerge(id, merge, mhash) VALUES(0, ?, 'x')", int64(rid2), ); err != nil { t.Fatal(err) } if err := co.Extract(rid2, ExtractOpts{Force: true}); err != nil { t.Fatalf("forced extract: %v", err) } var n int if err := co.db.QueryRow("SELECT count(*) FROM vmerge").Scan(&n); err != nil { t.Fatal(err) } if n != 0 { t.Fatalf("vmerge holds %d rows after switching versions, want 0", n) } } // Re-extracting the current version without Force keeps a pending merge, as // fossil's checkout does when the version does not change (#236). func TestExtractSameVersionKeepsPendingMerge(t *testing.T) { co, _, rid1, rid2 := newCheckoutAtFirstOfTwo(t) if _, err := co.db.Exec( "INSERT INTO vmerge(id, merge, mhash) VALUES(0, ?, 'x')", int64(rid2), ); err != nil { t.Fatal(err) } if err := co.Extract(rid1, ExtractOpts{}); err != nil { t.Fatalf("re-extract: %v", err) } var n int if err := co.db.QueryRow("SELECT count(*) FROM vmerge").Scan(&n); err != nil { t.Fatal(err) } if n != 1 { t.Fatalf("vmerge holds %d rows after re-extracting the same version, want 1", n) } } // A forced extract starts the file list fresh, as fossil's checkout --force // does: a pending rename or removal does not survive it (#236). func TestExtractForceStartsFileListFresh(t *testing.T) { co, _, rid1, _ := newCheckoutAtFirstOfTwo(t) if err := co.Rename(RenameOpts{From: "hello.txt", To: "moved.txt", DoFsMove: true}); err != nil { t.Fatal(err) } if err := co.Unmanage(UnmanageOpts{Paths: []string{"README.md"}}); err != nil { t.Fatal(err) } if err := co.Extract(rid1, ExtractOpts{Force: true}); err != nil { t.Fatalf("forced extract: %v", err) } if got := changedNames(t, co); len(got) != 0 { t.Fatalf("changes after a forced extract = %q, want none", got) } }