Git designed for AI agents.
Review m6: check the pack write deadline on the database clock
Federico Elia948a587609eb
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.mdindex 346caf6..254d81f 100644--- a/doc/graft/STORAGE.md+++ b/doc/graft/STORAGE.md@@ -357,12 +357,18 @@ How it stays safe:- Every "how old" decision uses the database clock: Postgres's `now()`; withSQLite, the host clock, since SQLite serves one host. The clocks of thenodes never decide.-- A writer measures its deadline on its own monotonic clock, so the steps of- a node's wall clock do not matter. The assumption is that, over `gc_grace`,- the writer's clock runs at most `gc_margin` slower than the database's, a- forward jump of the database clock included. With the defaults, 5 minutes- over an hour. Do not step the database host's clock forward by more than- `gc_margin`.+- The writer's deadline is checked on the database clock, in the commit's own+ transaction. The deadline is the database time of the record read that+ preceded the upload plus `gc_grace − gc_margin`. Garbage collection observes+ a pack it may delete on the same clock, and only after that read. So a+ committed row never references a deleted pack, however long the writer+ stopped: a suspended host, a paused VM, a long garbage-collection pause of+ the process.+ - The writer also stops its upload early on its own monotonic clock. That+ only saves work; it is not what makes it safe.+ - The one assumption left is that the database clock does not step backward+ by more than `gc_margin` (5 minutes by default). A forward step is safe:+ it makes the commit check refuse earlier.- `gc_grace` must also be greater than `sync_timeout + gc_margin`, so that async that started from an older checkpoint never meets a pack deleted underit.diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.goindex 14a0546..6609e9e 100644--- a/internal/gitaly/service/storagecore/import.go+++ b/internal/gitaly/service/storagecore/import.go@@ -162,18 +162,22 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportReposvar upload corestore.PackUploadif !pack.Empty() {readAt := time.Now()- if _, err := core.Meta.GetRepository(ctx, key); err != nil {+ record, err := core.Meta.GetRepository(ctx, key)+ if err != nil {if errors.Is(err, corestore.ErrRepositoryNotFound) {return nil, errRegistrationRemoved()}return nil, structerr.NewUnavailable("storage core: %w; %s", err, resumeHint)}+ if record.ID != reg.repo.ID {+ return nil, errRegistrationRemoved()+ }f, err := pack.Open()if err != nil {return nil, structerr.NewInternal("open pack: %w; %s", err, resumeHint)}- upload, err = core.UploadPack(ctx, reg.repo.ID, readAt, pack.SHA256, pack.Size, f)+ upload, err = core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, f)_ = f.Close()if err != nil {return nil, structerr.NewUnavailable("storage core: upload pack: %w; %s", err, resumeHint)@@ -197,13 +201,14 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportReposdefer cancel()res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{- RepoID: reg.repo.ID,- Updates: updates,- Operation: importOperation,- Actor: identity.Actor(ctx, importOperation),- NodeID: core.NodeID,- PackKey: upload.Key,- PackBytes: upload.Bytes,+ RepoID: reg.repo.ID,+ Updates: updates,+ Operation: importOperation,+ Actor: identity.Actor(ctx, importOperation),+ NodeID: core.NodeID,+ PackKey: upload.Key,+ PackBytes: upload.Bytes,+ PackDeadline: upload.DBDeadline,})if err != nil {var conflict *corestore.ErrRefConflictdiff --git a/internal/gitaly/service/storagecore/storagecore_test.go b/internal/gitaly/service/storagecore/storagecore_test.goindex da814f6..a4d2311 100644--- a/internal/gitaly/service/storagecore/storagecore_test.go+++ b/internal/gitaly/service/storagecore/storagecore_test.go@@ -6,6 +6,7 @@ import ("path/filepath""strings""testing"+ "time""github.com/stretchr/testify/require""google.golang.org/grpc"@@ -198,7 +199,7 @@ func (h *harness) commitElsewhere(repoID int64, ref string) git.ObjectID {Operation: "test",Actor: "other-node",NodeID: "node-b",- PackKey: key,+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),PackBytes: pack.Size,})require.NoError(h.t, err)diff --git a/internal/gitaly/service/storagecore/verify_objects_test.go b/internal/gitaly/service/storagecore/verify_objects_test.goindex 8457ae2..dc2c545 100644--- a/internal/gitaly/service/storagecore/verify_objects_test.go+++ b/internal/gitaly/service/storagecore/verify_objects_test.go@@ -59,7 +59,7 @@ func TestVerifyRepositoryReportsObjectsMissingFromStorage(t *testing.T) {Actor: "tester",NodeID: "node-a",Force: true,- PackKey: key,+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),PackBytes: pack.Size,})require.NoError(t, err)@@ -100,7 +100,7 @@ func (h *harness) checkpointAtHead(repoID int64) corestore.Checkpoint {require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, f))require.NoError(h.t, f.Close())- cp, created, err := h.core.Meta.CreateCheckpoint(h.ctx, repoID, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})+ cp, created, err := h.core.Meta.CreateCheckpoint(h.ctx, repoID, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackDeadline: time.Now().Add(time.Hour), PackBytes: pack.Size})require.NoError(h.t, err)require.True(h.t, created)return cpdiff --git a/internal/graft/corestore/checkpoint/checkpoint.go b/internal/graft/corestore/checkpoint/checkpoint.goindex 3d98356..60b6186 100644--- a/internal/graft/corestore/checkpoint/checkpoint.go+++ b/internal/graft/corestore/checkpoint/checkpoint.go@@ -165,7 +165,7 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (cif err != nil {return corestore.Checkpoint{}, false, structerr.NewInternal("open the checkpoint pack: %w", err)}- upload, err := c.core.UploadPack(ctx, record.ID, readAt, pack.SHA256, pack.Size, f)+ upload, err := c.core.UploadPack(ctx, current, readAt, pack.SHA256, pack.Size, f)if closeErr := f.Close(); closeErr != nil && err == nil {err = closeErr}@@ -184,10 +184,11 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (cdefer cancel()cp, created, err := c.core.Meta.CreateCheckpoint(commitCtx, record.ID, corestore.CheckpointInput{- Seq: seq,- Refs: refs,- PackKey: upload.Key,- PackBytes: upload.Bytes,+ Seq: seq,+ Refs: refs,+ PackKey: upload.Key,+ PackBytes: upload.Bytes,+ PackDeadline: upload.DBDeadline,})if err != nil {return corestore.Checkpoint{}, false, errRepository(err)diff --git a/internal/graft/corestore/checkpoint/checkpoint_test.go b/internal/graft/corestore/checkpoint/checkpoint_test.goindex 42bc9a0..c427129 100644--- a/internal/graft/corestore/checkpoint/checkpoint_test.go+++ b/internal/graft/corestore/checkpoint/checkpoint_test.go@@ -167,6 +167,7 @@ func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.}if !pack.Empty() {in.PackKey = corestore.NewPackKey(id, pack.SHA256)+ in.PackDeadline = time.Now().Add(time.Hour)rc, err := pack.Open()require.NoError(t, err)require.NoError(t, e.core.Blobs.Put(e.ctx, in.PackKey, rc))diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.goindex 3c266b1..02aa907 100644--- a/internal/graft/corestore/core.go+++ b/internal/graft/corestore/core.go@@ -126,28 +126,38 @@ type PackUpload struct {// upload plus WriteDeadline(). It keeps the monotonic clock reading of that// read, so that it is compared on the monotonic clock.Deadline time.Time+ // DBDeadline is the same deadline on the database clock: the record's+ // ReadAt plus WriteDeadline(). The commit of the row that references the+ // pack checks it in its transaction (ChangesetInput.PackDeadline,+ // CheckpointInput.PackDeadline): the monotonic clock stops while the host+ // is suspended or the VM paused, the database clock does not.+ DBDeadline time.Time}// UploadPack uploads the pack read from r (size bytes whose SHA-256 is sum)-// under a new owned key of the repository. readAt is the time of a read of the-// repository record that found the repository, taken before that read and-// before the upload; the upload must end before readAt + WriteDeadline(), else-// it is cancelled (nothing is stored) and the error wraps ErrPackDeadline. It is-// the only way the storage core uploads a pack (doc/graft/PLAN7.md, RULE 26).-func (c *Core) UploadPack(ctx context.Context, repoID int64, readAt time.Time, sum [32]byte, size int64, r io.Reader) (PackUpload, error) {+// under a new owned key of the repository. record was returned by a read of the+// repository record that found the repository (GetRepository: its ReadAt is+// the database time of that read), and readAt is the time taken before that+// read; the upload must end before readAt + WriteDeadline(), else it is+// cancelled (nothing is stored) and the error wraps ErrPackDeadline. It is the+// only way the storage core uploads a pack (doc/graft/PLAN7.md, RULE 26).+func (c *Core) UploadPack(ctx context.Context, record Repository, readAt time.Time, sum [32]byte, size int64, r io.Reader) (PackUpload, error) {if c.GCGrace <= 0 || c.GCMargin <= 0 || c.WriteDeadline() <= 0 {return PackUpload{}, fmt.Errorf("storage core: invalid write deadline: gc_grace %s, gc_margin %s", c.GCGrace, c.GCMargin)}+ if record.ReadAt.IsZero() {+ return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: the record carries no database read time", record.ID)+ }deadline := readAt.Add(c.WriteDeadline())if !time.Now().Before(deadline) {- return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: %w", repoID, ErrPackDeadline)+ return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: %w", record.ID, ErrPackDeadline)}putCtx, cancel := context.WithDeadline(ctx, deadline)defer cancel()- key := NewPackKey(repoID, sum)+ key := NewPackKey(record.ID, sum)if err := c.Blobs.Put(putCtx, key, r); err != nil {if ctx.Err() == nil && errors.Is(putCtx.Err(), context.DeadlineExceeded) {return PackUpload{}, fmt.Errorf("storage core: upload %q: %w: %w", key, ErrPackDeadline, err)@@ -155,7 +165,7 @@ func (c *Core) UploadPack(ctx context.Context, repoID int64, readAt time.Time, sreturn PackUpload{}, fmt.Errorf("storage core: upload %q: %w", key, err)}- return PackUpload{Key: key, Bytes: size, Deadline: deadline}, nil+ return PackUpload{Key: key, Bytes: size, Deadline: deadline, DBDeadline: record.ReadAt.Add(c.WriteDeadline())}, nil}// CommitContext returns ctx bounded by the upload's deadline, for the commit ofdiff --git a/internal/graft/corestore/coretest/import.go b/internal/graft/corestore/coretest/import.goindex 3bfe69e..801f9f7 100644--- a/internal/graft/corestore/coretest/import.go+++ b/internal/graft/corestore/coretest/import.go@@ -54,12 +54,13 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProtovar upload corestore.PackUploadif !pack.Empty() {readAt := time.Now()- _, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repoProto))+ record, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repoProto))require.NoError(tb, err)+ require.Equal(tb, registered.ID, record.ID)f, err := pack.Open()require.NoError(tb, err)- upload, err = core.UploadPack(ctx, registered.ID, readAt, pack.SHA256, pack.Size, f)+ upload, err = core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, f)require.NoError(tb, err)require.NoError(tb, f.Close())}@@ -74,13 +75,14 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProtodefer cancel()res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{- RepoID: registered.ID,- Updates: updates,- Operation: "import",- Actor: "import",- NodeID: core.NodeID,- PackKey: upload.Key,- PackBytes: upload.Bytes,+ RepoID: registered.ID,+ Updates: updates,+ Operation: "import",+ Actor: "import",+ NodeID: core.NodeID,+ PackKey: upload.Key,+ PackBytes: upload.Bytes,+ PackDeadline: upload.DBDeadline,})require.NoError(tb, err)require.Equal(tb, uint64(1), res.Seq, "the import is changeset 1")diff --git a/internal/graft/corestore/gate/gate_fix3_test.go b/internal/graft/corestore/gate/gate_fix3_test.goindex fca8f3a..4f5294c 100644--- a/internal/graft/corestore/gate/gate_fix3_test.go+++ b/internal/graft/corestore/gate/gate_fix3_test.go@@ -6,6 +6,7 @@ import ("os""strings""testing"+ "time""github.com/stretchr/testify/require""google.golang.org/grpc"@@ -125,7 +126,7 @@ func TestGateHistoryGuardWithRealSyncerOnFilesBackendCopy(t *testing.T) {}_, err = core.Meta.CommitChangeset(ctx, corestore.ChangesetInput{RepoID: rec.ID, Updates: updates, Operation: "import", Actor: "import", NodeID: core.NodeID,- PackKey: packKey, PackBytes: pack.Size,+ PackKey: packKey, PackDeadline: time.Now().Add(time.Hour), PackBytes: pack.Size,})require.NoError(t, err)require.NoError(t, gitops.WriteWatermark(path, corestore.Watermark{RepoID: rec.ID, Seq: 1}))diff --git a/internal/graft/corestore/gc/gc_test.go b/internal/graft/corestore/gc/gc_test.goindex 134d3d5..35d29e1 100644--- a/internal/graft/corestore/gc/gc_test.go+++ b/internal/graft/corestore/gc/gc_test.go@@ -99,7 +99,7 @@ func (e *env) commit(key string) uint64 {Operation: "test",Actor: "tester",NodeID: "node-a",- PackKey: key,+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),PackBytes: 1,})require.NoError(e.t, err)@@ -111,9 +111,9 @@ func (e *env) checkpoint(seq uint64) string {e.t.Helper()key := e.upload(e.repo.ID)_, created, err := e.core.Meta.CreateCheckpoint(e.ctx, e.repo.ID, corestore.CheckpointInput{- Seq: seq,- Refs: map[string]corestore.Target{"refs/heads/main": corestore.Target(strings.Repeat("a", 40))},- PackKey: key,+ Seq: seq,+ Refs: map[string]corestore.Target{"refs/heads/main": corestore.Target(strings.Repeat("a", 40))},+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),PackBytes: 1,})require.NoError(e.t, err)diff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.goindex ce15d08..24b2cbd 100644--- a/internal/graft/corestore/maintenance/maintenance_test.go+++ b/internal/graft/corestore/maintenance/maintenance_test.go@@ -103,6 +103,7 @@ func (e *env) changeset(updates []corestore.RefUpdate, tips ...git.ObjectID) {in := corestore.ChangesetInput{RepoID: e.repo.ID, Updates: updates, Operation: "test", Actor: "tester", NodeID: "node-a"}if !pack.Empty() {in.PackKey = corestore.NewPackKey(e.repo.ID, pack.SHA256)+ in.PackDeadline = time.Now().Add(time.Hour)rc, err := pack.Open()require.NoError(e.t, err)require.NoError(e.t, e.core.Blobs.Put(ctx, in.PackKey, rc))diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.goindex bd8955b..23f3ad0 100644--- a/internal/graft/corestore/metastore.go+++ b/internal/graft/corestore/metastore.go@@ -23,6 +23,10 @@ type ChangesetInput struct {// objects, empty when there are none. PackBytes is its size.PackKey stringPackBytes int64+ // PackDeadline is the write deadline of the pack on the database clock+ // (PackUpload.DBDeadline). A changeset with a pack is committed only before+ // it, checked in the commit's transaction; one without it is refused.+ PackDeadline time.Time// Changelog, when not nil, appends one changelog entry per effective update// of a branch or tag (IsChangelogRef), in the order of Updates, in the same// transaction. The entries get the next changelog sequences, the actual@@ -80,6 +84,9 @@ type CheckpointInput struct {// pack, PackBytes its size.PackKey stringPackBytes int64+ // PackDeadline is the write deadline of the pack on the database clock+ // (PackUpload.DBDeadline), checked in the record's transaction.+ PackDeadline time.Time}var (diff --git a/internal/graft/corestore/sqlstore/changeset.go b/internal/graft/corestore/sqlstore/changeset.goindex 85b955c..d13fc52 100644--- a/internal/graft/corestore/sqlstore/changeset.go+++ b/internal/graft/corestore/sqlstore/changeset.go@@ -70,6 +70,28 @@ func validateChangeset(in corestore.ChangesetInput) error {return nil}+// checkPackDeadline refuses, with ErrPackDeadline, the commit of a row that+// references an uploaded pack once the database clock reached the pack's write+// deadline (PackUpload.DBDeadline), and the commit of one without a deadline.+// Garbage collection deletes an unreferenced pack gc_grace after it observed it+// on the database clock, which is after the record read the deadline starts+// from: a row committed before the deadline is never a row whose pack is gone,+// whatever the writer's own clock did meanwhile (a suspended host, a paused+// VM).+func (s *Store) checkPackDeadline(ctx context.Context, tx *sql.Tx, deadline time.Time) error {+ if deadline.IsZero() {+ return errors.New("storage core: a row that references a pack must carry the pack's write deadline")+ }+ now, err := s.wallTime(ctx, tx)+ if err != nil {+ return err+ }+ if !now.Before(deadline) {+ return fmt.Errorf("storage core: commit at %s, after the pack's write deadline %s: %w", now.Format(time.RFC3339Nano), deadline.UTC().Format(time.RFC3339Nano), corestore.ErrPackDeadline)+ }+ return nil+}+// commitChangeset records a changeset in the open transaction tx, for the// locked repository repo, and advances repo to the new sequences. It is shared// by CommitChangeset and the initial changeset of RegisterRepository.@@ -126,6 +148,9 @@ func (s *Store) commitChangeset(ctx context.Context, tx *sql.Tx, repo *repoStatevar packKey anyif in.PackKey != "" {+ if err := s.checkPackDeadline(ctx, tx, in.PackDeadline); err != nil {+ return corestore.CommitResult{}, err+ }packKey = in.PackKey}diff --git a/internal/graft/corestore/sqlstore/checkpoint.go b/internal/graft/corestore/sqlstore/checkpoint.goindex 8d2b940..108f5cc 100644--- a/internal/graft/corestore/sqlstore/checkpoint.go+++ b/internal/graft/corestore/sqlstore/checkpoint.go@@ -65,6 +65,9 @@ func (s *Store) CreateCheckpoint(ctx context.Context, repoID int64, in corestorereturn fmt.Errorf("storage core: checkpoint at changeset %d is older than the newest checkpoint at %d of repository %d", in.Seq, newest.Seq, repoID)}+ if err := s.checkPackDeadline(ctx, tx, in.PackDeadline); err != nil {+ return err+ }now, err := s.wallTime(ctx, tx)if err != nil {return errdiff --git a/internal/graft/corestore/sqlstore/checkpoint_large_test.go b/internal/graft/corestore/sqlstore/checkpoint_large_test.goindex bcd2fd4..184f7d1 100644--- a/internal/graft/corestore/sqlstore/checkpoint_large_test.go+++ b/internal/graft/corestore/sqlstore/checkpoint_large_test.go@@ -60,7 +60,7 @@ func TestCheckpointOfManyReferences(t *testing.T) {began := time.Now()_, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{- Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "large checkpoint"), PackBytes: 1,+ Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "large checkpoint"), PackDeadline: time.Now().Add(time.Hour), PackBytes: 1,})took := time.Since(began)close(done)diff --git a/internal/graft/corestore/sqlstore/deadline_test.go b/internal/graft/corestore/sqlstore/deadline_test.gonew file mode 100644index 0000000..5144d86--- /dev/null+++ b/internal/graft/corestore/sqlstore/deadline_test.go@@ -0,0 +1,68 @@+package sqlstore_test++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+)++// TestCommitChecksThePackDeadlineOnTheDatabaseClock verifies the write+// deadline of a pack in the commit transaction (integrated review m6). A+// writer's monotonic clock stops while its host is suspended or its VM paused,+// so the deadline it measured may not have expired for it when, on the+// database clock that garbage collection uses, it has: such a commit is+// refused, and nothing is written. A row with a pack and no deadline is+// refused too.+func TestCommitChecksThePackDeadlineOnTheDatabaseClock(t *testing.T) {+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {+ repo := register(t, ctx, s, testKey("deadline"))+ record, err := s.GetRepository(ctx, repo.Key)+ require.NoError(t, err)+ require.False(t, record.ReadAt.IsZero(), "GetRepository returns the database time of the read")+ require.WithinDuration(t, time.Now(), record.ReadAt, time.Minute)++ changeset := func(deadline time.Time) corestore.ChangesetInput {+ return corestore.ChangesetInput{+ RepoID: repo.ID,+ Updates: []corestore.RefUpdate{create("refs/heads/main", oid(1))},+ Operation: "test",+ Actor: "tester",+ NodeID: "node-a",+ PackKey: ownedKey(repo.ID, "pack"),+ PackBytes: 1,+ PackDeadline: deadline,+ }+ }++ // The writer slept through its deadline: the database clock is past it.+ _, err = s.CommitChangeset(ctx, changeset(record.ReadAt.Add(-time.Second)))+ require.ErrorIs(t, err, corestore.ErrPackDeadline)+ _, err = s.CommitChangeset(ctx, changeset(time.Time{}))+ require.ErrorContains(t, err, "must carry the pack's write deadline")+ requireRefs(t, ctx, s, repo.ID, map[string]corestore.Target{})++ res, err := s.CommitChangeset(ctx, changeset(record.ReadAt.Add(time.Hour)))+ require.NoError(t, err)+ require.Equal(t, uint64(1), res.Seq)++ checkpoint := func(deadline time.Time) corestore.CheckpointInput {+ return corestore.CheckpointInput{+ Seq: 1, Refs: map[string]corestore.Target{"refs/heads/main": oid(1)},+ PackKey: ownedKey(repo.ID, "checkpoint"), PackBytes: 1, PackDeadline: deadline,+ }+ }+ _, _, err = s.CreateCheckpoint(ctx, repo.ID, checkpoint(record.ReadAt.Add(-time.Second)))+ require.ErrorIs(t, err, corestore.ErrPackDeadline)+ _, ok, err := s.NewestCheckpoint(ctx, repo.ID)+ require.NoError(t, err)+ require.False(t, ok, "nothing was recorded")+ _, created, err := s.CreateCheckpoint(ctx, repo.ID, checkpoint(record.ReadAt.Add(time.Hour)))+ require.NoError(t, err)+ require.True(t, created)+ })+}diff --git a/internal/graft/corestore/sqlstore/gc_test.go b/internal/graft/corestore/sqlstore/gc_test.goindex f1ef5c1..050c055 100644--- a/internal/graft/corestore/sqlstore/gc_test.go+++ b/internal/graft/corestore/sqlstore/gc_test.go@@ -38,7 +38,7 @@ func commitPack(ctx context.Context, s *sqlstore.Store, repoID int64, u corestorOperation: "test",Actor: "tester",NodeID: "node-a",- PackKey: packKey,+ PackKey: packKey, PackDeadline: time.Now().Add(time.Hour),PackBytes: 1,})}@@ -170,7 +170,7 @@ func TestCheckpointLifecycle(t *testing.T) {"refs/heads/main": oid(2),"refs/tags/\xff\xfev1": oid(7),}- in := corestore.CheckpointInput{Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "checkpoint 2"), PackBytes: 42}+ in := corestore.CheckpointInput{Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "checkpoint 2"), PackDeadline: time.Now().Add(time.Hour), PackBytes: 42}cp, created, err := s.CreateCheckpoint(ctx, repo.ID, in)require.NoError(t, err)@@ -191,22 +191,22 @@ func TestCheckpointLifecycle(t *testing.T) {// A second build at the same sequence keeps the first checkpoint and its pack.again, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{- Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "another build of 2"),+ Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "another build of 2"), PackDeadline: time.Now().Add(time.Hour),})require.NoError(t, err)require.False(t, created)require.Equal(t, cp, again)for name, bad := range map[string]corestore.CheckpointInput{- "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x")},- "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x")},- "older than the newest": {Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "x")},- "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x")},- "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x")},+ "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},+ "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},+ "older than the newest": {Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},+ "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},+ "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x"), PackDeadline: time.Now().Add(time.Hour)},"no pack key": {Seq: 3, Refs: refs},- "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x")},- "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x")},- "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x")},+ "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},+ "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},+ "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},} {_, _, err := s.CreateCheckpoint(ctx, repo.ID, bad)require.Error(t, err, name)@@ -217,7 +217,7 @@ func TestCheckpointLifecycle(t *testing.T) {}cp3, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{- Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": oid(3)}, PackKey: ownedKey(repo.ID, "checkpoint 3"),+ Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": oid(3)}, PackKey: ownedKey(repo.ID, "checkpoint 3"), PackDeadline: time.Now().Add(time.Hour),})require.NoError(t, err)require.True(t, created)@@ -236,7 +236,7 @@ func TestCheckpointLifecycle(t *testing.T) {require.False(t, deleted)_, _, err = s.CreateCheckpoint(ctx, repo.ID+1000, corestore.CheckpointInput{- Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID+1000, "unknown"),+ Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID+1000, "unknown"), PackDeadline: time.Now().Add(time.Hour),})require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)})@@ -252,7 +252,7 @@ func TestSyncBoundaryUsesTheDatabaseClock(t *testing.T) {for seq := uint64(1); seq <= 3; seq++ {_, _, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{- Seq: seq, Refs: map[string]corestore.Target{"refs/heads/main": oid(int(seq))}, PackKey: ownedKey(repo.ID, fmt.Sprint(seq)),+ Seq: seq, Refs: map[string]corestore.Target{"refs/heads/main": oid(int(seq))}, PackKey: ownedKey(repo.ID, fmt.Sprint(seq)), PackDeadline: time.Now().Add(time.Hour),})require.NoError(t, err)}@@ -394,7 +394,7 @@ func TestDeleteRepositoryRecordsTombstone(t *testing.T) {// Rows of milestone 3 go with the repository._, _, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{- Seq: 2, Refs: map[string]corestore.Target{"refs/heads/main": oid(2)}, PackKey: ownedKey(repo.ID, "cp"),+ Seq: 2, Refs: map[string]corestore.Target{"refs/heads/main": oid(2)}, PackKey: ownedKey(repo.ID, "cp"), PackDeadline: time.Now().Add(time.Hour),})require.NoError(t, err)require.NoError(t, s.RecordOrphans(ctx, repo.ID, []string{ownedKey(repo.ID, "orphan")}))@@ -703,7 +703,7 @@ func TestLockOrderConcurrentWriters(t *testing.T) {}cp, created, err := a.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{- Seq: head, Refs: refs, PackKey: ownedKey(repo.ID, fmt.Sprint("checkpoint ", head)),+ Seq: head, Refs: refs, PackKey: ownedKey(repo.ID, fmt.Sprint("checkpoint ", head)), PackDeadline: time.Now().Add(time.Hour),})switch {case err == nil && created:diff --git a/internal/graft/corestore/sqlstore/repository.go b/internal/graft/corestore/sqlstore/repository.goindex 92e0fdd..d7a5707 100644--- a/internal/graft/corestore/sqlstore/repository.go+++ b/internal/graft/corestore/sqlstore/repository.go@@ -393,11 +393,35 @@ func (s *Store) GetRepository(ctx context.Context, key corestore.RepoKey) (coresreturn corestore.Repository{}, corestore.ErrRepositoryNotFound}- r, err := s.selectRepository(ctx, s.db, "storage = ? AND relative_path = ?", []any{key.Storage, key.RelativePath}, false)+ // The database time of the read (Repository.ReadAt): Postgres reads its+ // clock in the same statement; SQLite's clock is the host's, read before.+ readAt := time.Now().UTC()+ query := "SELECT " + repoColumns + " FROM graft_repositories WHERE storage = ? AND relative_path = ?"+ if s.dialect == Postgres {+ query = "SELECT " + repoColumns + ", clock_timestamp() FROM graft_repositories WHERE storage = ? AND relative_path = ?"+ }+ row := s.db.QueryRowContext(ctx, s.bind(query), key.Storage, key.RelativePath)+ var clock any+ r, err := scanRepoState(func(dest ...any) error {+ if s.dialect == Postgres {+ dest = append(dest, &clock)+ }+ return row.Scan(dest...)+ })+ if errors.Is(err, sql.ErrNoRows) {+ return corestore.Repository{}, corestore.ErrRepositoryNotFound+ }if err != nil {- return corestore.Repository{}, err+ return corestore.Repository{}, fmt.Errorf("select repository: %w", err)+ }+ if s.dialect == Postgres {+ if readAt, err = scanTime(clock); err != nil {+ return corestore.Repository{}, fmt.Errorf("read the database time: %w", err)+ }}- return toRepository(r), nil+ repo := toRepository(r)+ repo.ReadAt = readAt+ return repo, nil}// DeleteRepository implements corestore.MetaStore.diff --git a/internal/graft/corestore/syncer/syncer_checkpoint_test.go b/internal/graft/corestore/syncer/syncer_checkpoint_test.goindex 27015a5..be05aa7 100644--- a/internal/graft/corestore/syncer/syncer_checkpoint_test.go+++ b/internal/graft/corestore/syncer/syncer_checkpoint_test.go@@ -41,7 +41,7 @@ func (e *env) checkpoint(t *testing.T, id int64, src source) corestore.Checkpoinrequire.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))require.NoError(t, rc.Close())- cp, created, err := e.core.Meta.CreateCheckpoint(e.ctx, id, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})+ cp, created, err := e.core.Meta.CreateCheckpoint(e.ctx, id, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackDeadline: time.Now().Add(time.Hour), PackBytes: pack.Size})require.NoError(t, err)require.True(t, created)return cpdiff --git a/internal/graft/corestore/syncer/syncer_import_test.go b/internal/graft/corestore/syncer/syncer_import_test.goindex 2099ad2..e67e57e 100644--- a/internal/graft/corestore/syncer/syncer_import_test.go+++ b/internal/graft/corestore/syncer/syncer_import_test.go@@ -3,6 +3,7 @@ package syncer_testimport ("os""testing"+ "time""github.com/stretchr/testify/require""google.golang.org/grpc/codes"@@ -119,7 +120,7 @@ func testSyncerIncompleteImport(t *testing.T, dialect sqlstore.Dialect, backendOperation: "import",Actor: "tester",NodeID: "node-a",- PackKey: packKey,+ PackKey: packKey, PackDeadline: time.Now().Add(time.Hour),PackBytes: pack.Size,})require.NoError(t, err)diff --git a/internal/graft/corestore/syncer/syncer_race_test.go b/internal/graft/corestore/syncer/syncer_race_test.goindex da05b2d..375d810 100644--- a/internal/graft/corestore/syncer/syncer_race_test.go+++ b/internal/graft/corestore/syncer/syncer_race_test.go@@ -37,6 +37,7 @@ func (e *env) commitFrom(t *testing.T, id int64, node string, src source, updatedefer func() { require.NoError(t, pack.Remove()) }()if !pack.Empty() {in.PackKey = corestore.NewPackKey(id, pack.SHA256)+ in.PackDeadline = time.Now().Add(time.Hour)in.PackBytes = pack.Sizerc, err := pack.Open()require.NoError(t, err)diff --git a/internal/graft/corestore/syncer/syncer_test.go b/internal/graft/corestore/syncer/syncer_test.goindex 6535475..314e060 100644--- a/internal/graft/corestore/syncer/syncer_test.go+++ b/internal/graft/corestore/syncer/syncer_test.go@@ -191,6 +191,7 @@ func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))require.NoError(t, rc.Close())in.PackKey = key+ in.PackDeadline = time.Now().Add(time.Hour)in.PackBytes = pack.Size}diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.goindex 5723064..42a5d00 100644--- a/internal/graft/corestore/types.go+++ b/internal/graft/corestore/types.go@@ -131,6 +131,12 @@ type Repository struct {// LogSeq is the sequence of the last changelog entry, 0 when there is none.LogSeq uint64CreatedAt time.Time+ // ReadAt is the database time of the read that returned the record (set by+ // GetRepository), at or before the read. A pack uploaded after the read is+ // first observable by garbage collection after ReadAt, so a row that+ // references it is committed before ReadAt + Core.WriteDeadline() on the+ // database clock (PackUpload.DBDeadline, checked in the commit).+ ReadAt time.Time}// Changeset is one recorded changeset of the write-ahead log.diff --git a/internal/graft/corestore/upload_test.go b/internal/graft/corestore/upload_test.goindex d9e080f..6efd8e7 100644--- a/internal/graft/corestore/upload_test.go+++ b/internal/graft/corestore/upload_test.go@@ -44,8 +44,10 @@ func TestUploadPackUsesAnOwnedKeyAndTheWriteDeadline(t *testing.T) {core := uploadCore(blobs, time.Hour, 5*time.Minute)sum := sha256.Sum256([]byte("pack"))readAt := time.Now()+ // The database clock of the read, which the commit checks the deadline on.+ record := Repository{ID: 7, ReadAt: time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC)}- upload, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))+ upload, err := core.UploadPack(context.Background(), record, readAt, sum, 4, strings.NewReader("pack"))require.NoError(t, err)require.True(t, IsOwnedPackKey(upload.Key))id, gotSum, _ := ParsePackKey(upload.Key)@@ -53,11 +55,12 @@ func TestUploadPackUsesAnOwnedKeyAndTheWriteDeadline(t *testing.T) {require.Equal(t, sum, gotSum)require.Equal(t, int64(4), upload.Bytes)require.Equal(t, readAt.Add(55*time.Minute), upload.Deadline)+ require.Equal(t, record.ReadAt.Add(55*time.Minute), upload.DBDeadline)require.Equal(t, []string{upload.Key}, blobs.puts)require.WithinDuration(t, upload.Deadline, blobs.deadline, 0, "the Put runs under the write deadline")// Every upload gets its own key, also for the same bytes.- again, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))+ again, err := core.UploadPack(context.Background(), record, readAt, sum, 4, strings.NewReader("pack"))require.NoError(t, err)require.NotEqual(t, upload.Key, again.Key)@@ -73,13 +76,14 @@ func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {t.Parallel()sum := sha256.Sum256([]byte("late"))+ record := Repository{ID: 1, ReadAt: time.Now()}t.Run("deadline passed before the upload", func(t *testing.T) {t.Parallel()blobs := &fakeBlobs{}core := uploadCore(blobs, time.Hour, 5*time.Minute)- _, err := core.UploadPack(context.Background(), 1, time.Now().Add(-time.Hour), sum, 4, strings.NewReader("late"))+ _, err := core.UploadPack(context.Background(), record, time.Now().Add(-time.Hour), sum, 4, strings.NewReader("late"))require.ErrorIs(t, err, ErrPackDeadline)require.Empty(t, blobs.puts, "nothing is uploaded")})@@ -89,7 +93,7 @@ func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {blobs := &fakeBlobs{block: true}core := uploadCore(blobs, 100*time.Millisecond, 50*time.Millisecond)- _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 4, strings.NewReader("late"))+ _, err := core.UploadPack(context.Background(), record, time.Now(), sum, 4, strings.NewReader("late"))require.ErrorIs(t, err, ErrPackDeadline, "the Put is cancelled at the deadline")})@@ -100,10 +104,20 @@ func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {ctx, cancel := context.WithCancel(context.Background())cancel()- _, err := core.UploadPack(ctx, 1, time.Now(), sum, 4, strings.NewReader("late"))+ _, err := core.UploadPack(ctx, record, time.Now(), sum, 4, strings.NewReader("late"))require.ErrorIs(t, err, context.Canceled)require.NotErrorIs(t, err, ErrPackDeadline, "the caller's cancellation is not a deadline")})++ t.Run("a record without a database read time", func(t *testing.T) {+ t.Parallel()+ blobs := &fakeBlobs{}+ core := uploadCore(blobs, time.Hour, 5*time.Minute)++ _, err := core.UploadPack(context.Background(), Repository{ID: 1}, time.Now(), sum, 4, strings.NewReader("late"))+ require.ErrorContains(t, err, "no database read time")+ require.Empty(t, blobs.puts, "nothing is uploaded: its commit could not be bounded")+ })}func TestUploadPackRequiresTheGCTimes(t *testing.T) {@@ -115,7 +129,7 @@ func TestUploadPackRequiresTheGCTimes(t *testing.T) {uploadCore(&fakeBlobs{}, time.Hour, 0),uploadCore(&fakeBlobs{}, time.Minute, time.Hour),} {- _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 1, strings.NewReader("x"))+ _, err := core.UploadPack(context.Background(), Repository{ID: 1, ReadAt: time.Now()}, time.Now(), sum, 1, strings.NewReader("x"))require.ErrorContains(t, err, "invalid write deadline")}}diff --git a/internal/graft/corestore/voter/server_test.go b/internal/graft/corestore/voter/server_test.goindex c0c24d6..0ecd124 100644--- a/internal/graft/corestore/voter/server_test.go+++ b/internal/graft/corestore/voter/server_test.go@@ -6,6 +6,7 @@ import ("sync""sync/atomic""testing"+ "time""github.com/stretchr/testify/require"@@ -164,7 +165,7 @@ func TestAuthoritativeServer(t *testing.T) {Operation: "land",Actor: "other-node",NodeID: "node-b",- PackKey: key,+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),PackBytes: pack.Size,})return errdiff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.goindex 6d3853c..8f3342c 100644--- a/internal/graft/corestore/voter/voter.go+++ b/internal/graft/corestore/voter/voter.go@@ -259,7 +259,7 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, chreturn nil, nil}- pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)+ pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates)if err != nil {return nil, err}@@ -457,21 +457,22 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, changeif prepared != nil && prepared.ok && prepared.repoID == record.ID && prepared.usable(time.Now()) {pack = prepared.pack} else {- pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)+ pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates)if err != nil {return v.unavailable(err)}}in := corestore.ChangesetInput{- RepoID: record.ID,- Updates: refUpdates,- Operation: meta.Operation,- Actor: meta.Actor,- NodeID: v.core.NodeID,- Force: shadow,- PackKey: pack.Key,- PackBytes: pack.Bytes,+ RepoID: record.ID,+ Updates: refUpdates,+ Operation: meta.Operation,+ Actor: meta.Actor,+ NodeID: v.core.NodeID,+ Force: shadow,+ PackKey: pack.Key,+ PackBytes: pack.Bytes,+ PackDeadline: pack.DBDeadline,}if err := v.fault(faultAfterUpload); err != nil {@@ -678,7 +679,8 @@ func newTips(updates []inputUpdate) []git.ObjectID {// local reference reaches (invariant I1, gitops.BuildPack); in shadow mode,// everything that the values the database records reach (buildPackAgainst),// because local references can hold values the storage core never recorded.-func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, repoID int64, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {+func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, record corestore.Repository, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {+ repoID := record.IDtips := newTips(updates)if len(tips) == 0 {return corestore.PackUpload{}, nil@@ -720,7 +722,7 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storag}defer func() { _ = r.Close() }()- upload, err := v.core.UploadPack(ctx, repoID, readAt, pack.SHA256, pack.Size, r)+ upload, err := v.core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, r)if err != nil {return corestore.PackUpload{}, err}