Skip to content

Git designed for AI agents.

Review m6: check the pack write deadline on the database clock

Federico Elia948a587609eb
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index 346caf6..254d81f 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -357,12 +357,18 @@ How it stays safe:
- Every "how old" decision uses the database clock: Postgres's `now()`; with
SQLite, the host clock, since SQLite serves one host. The clocks of the
nodes never decide.
-- A writer measures its deadline on its own monotonic clock, so the steps of
- a node's wall clock do not matter. The assumption is that, over `gc_grace`,
- the writer's clock runs at most `gc_margin` slower than the database's, a
- forward jump of the database clock included. With the defaults, 5 minutes
- over an hour. Do not step the database host's clock forward by more than
- `gc_margin`.
+- The writer's deadline is checked on the database clock, in the commit's own
+ transaction. The deadline is the database time of the record read that
+ preceded the upload plus `gc_grace − gc_margin`. Garbage collection observes
+ a pack it may delete on the same clock, and only after that read. So a
+ committed row never references a deleted pack, however long the writer
+ stopped: a suspended host, a paused VM, a long garbage-collection pause of
+ the process.
+ - The writer also stops its upload early on its own monotonic clock. That
+ only saves work; it is not what makes it safe.
+ - The one assumption left is that the database clock does not step backward
+ by more than `gc_margin` (5 minutes by default). A forward step is safe:
+ it makes the commit check refuse earlier.
- `gc_grace` must also be greater than `sync_timeout + gc_margin`, so that a
sync that started from an older checkpoint never meets a pack deleted under
it.
diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.go
index 14a0546..6609e9e 100644
--- a/internal/gitaly/service/storagecore/import.go
+++ b/internal/gitaly/service/storagecore/import.go
@@ -162,18 +162,22 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
var upload corestore.PackUpload
if !pack.Empty() {
readAt := time.Now()
- if _, err := core.Meta.GetRepository(ctx, key); err != nil {
+ record, err := core.Meta.GetRepository(ctx, key)
+ if err != nil {
if errors.Is(err, corestore.ErrRepositoryNotFound) {
return nil, errRegistrationRemoved()
}
return nil, structerr.NewUnavailable("storage core: %w; %s", err, resumeHint)
}
+ if record.ID != reg.repo.ID {
+ return nil, errRegistrationRemoved()
+ }
f, err := pack.Open()
if err != nil {
return nil, structerr.NewInternal("open pack: %w; %s", err, resumeHint)
}
- upload, err = core.UploadPack(ctx, reg.repo.ID, readAt, pack.SHA256, pack.Size, f)
+ upload, err = core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, f)
_ = f.Close()
if err != nil {
return nil, structerr.NewUnavailable("storage core: upload pack: %w; %s", err, resumeHint)
@@ -197,13 +201,14 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
defer cancel()
res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{
- RepoID: reg.repo.ID,
- Updates: updates,
- Operation: importOperation,
- Actor: identity.Actor(ctx, importOperation),
- NodeID: core.NodeID,
- PackKey: upload.Key,
- PackBytes: upload.Bytes,
+ RepoID: reg.repo.ID,
+ Updates: updates,
+ Operation: importOperation,
+ Actor: identity.Actor(ctx, importOperation),
+ NodeID: core.NodeID,
+ PackKey: upload.Key,
+ PackBytes: upload.Bytes,
+ PackDeadline: upload.DBDeadline,
})
if err != nil {
var conflict *corestore.ErrRefConflict
diff --git a/internal/gitaly/service/storagecore/storagecore_test.go b/internal/gitaly/service/storagecore/storagecore_test.go
index da814f6..a4d2311 100644
--- a/internal/gitaly/service/storagecore/storagecore_test.go
+++ b/internal/gitaly/service/storagecore/storagecore_test.go
@@ -6,6 +6,7 @@ import (
"path/filepath"
"strings"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
@@ -198,7 +199,7 @@ func (h *harness) commitElsewhere(repoID int64, ref string) git.ObjectID {
Operation: "test",
Actor: "other-node",
NodeID: "node-b",
- PackKey: key,
+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),
PackBytes: pack.Size,
})
require.NoError(h.t, err)
diff --git a/internal/gitaly/service/storagecore/verify_objects_test.go b/internal/gitaly/service/storagecore/verify_objects_test.go
index 8457ae2..dc2c545 100644
--- a/internal/gitaly/service/storagecore/verify_objects_test.go
+++ b/internal/gitaly/service/storagecore/verify_objects_test.go
@@ -59,7 +59,7 @@ func TestVerifyRepositoryReportsObjectsMissingFromStorage(t *testing.T) {
Actor: "tester",
NodeID: "node-a",
Force: true,
- PackKey: key,
+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),
PackBytes: pack.Size,
})
require.NoError(t, err)
@@ -100,7 +100,7 @@ func (h *harness) checkpointAtHead(repoID int64) corestore.Checkpoint {
require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, f))
require.NoError(h.t, f.Close())
- cp, created, err := h.core.Meta.CreateCheckpoint(h.ctx, repoID, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})
+ cp, created, err := h.core.Meta.CreateCheckpoint(h.ctx, repoID, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackDeadline: time.Now().Add(time.Hour), PackBytes: pack.Size})
require.NoError(h.t, err)
require.True(h.t, created)
return cp
diff --git a/internal/graft/corestore/checkpoint/checkpoint.go b/internal/graft/corestore/checkpoint/checkpoint.go
index 3d98356..60b6186 100644
--- a/internal/graft/corestore/checkpoint/checkpoint.go
+++ b/internal/graft/corestore/checkpoint/checkpoint.go
@@ -165,7 +165,7 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (c
if err != nil {
return corestore.Checkpoint{}, false, structerr.NewInternal("open the checkpoint pack: %w", err)
}
- upload, err := c.core.UploadPack(ctx, record.ID, readAt, pack.SHA256, pack.Size, f)
+ upload, err := c.core.UploadPack(ctx, current, readAt, pack.SHA256, pack.Size, f)
if closeErr := f.Close(); closeErr != nil && err == nil {
err = closeErr
}
@@ -184,10 +184,11 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (c
defer cancel()
cp, created, err := c.core.Meta.CreateCheckpoint(commitCtx, record.ID, corestore.CheckpointInput{
- Seq: seq,
- Refs: refs,
- PackKey: upload.Key,
- PackBytes: upload.Bytes,
+ Seq: seq,
+ Refs: refs,
+ PackKey: upload.Key,
+ PackBytes: upload.Bytes,
+ PackDeadline: upload.DBDeadline,
})
if err != nil {
return corestore.Checkpoint{}, false, errRepository(err)
diff --git a/internal/graft/corestore/checkpoint/checkpoint_test.go b/internal/graft/corestore/checkpoint/checkpoint_test.go
index 42bc9a0..c427129 100644
--- a/internal/graft/corestore/checkpoint/checkpoint_test.go
+++ b/internal/graft/corestore/checkpoint/checkpoint_test.go
@@ -167,6 +167,7 @@ func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.
}
if !pack.Empty() {
in.PackKey = corestore.NewPackKey(id, pack.SHA256)
+ in.PackDeadline = time.Now().Add(time.Hour)
rc, err := pack.Open()
require.NoError(t, err)
require.NoError(t, e.core.Blobs.Put(e.ctx, in.PackKey, rc))
diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.go
index 3c266b1..02aa907 100644
--- a/internal/graft/corestore/core.go
+++ b/internal/graft/corestore/core.go
@@ -126,28 +126,38 @@ type PackUpload struct {
// upload plus WriteDeadline(). It keeps the monotonic clock reading of that
// read, so that it is compared on the monotonic clock.
Deadline time.Time
+ // DBDeadline is the same deadline on the database clock: the record's
+ // ReadAt plus WriteDeadline(). The commit of the row that references the
+ // pack checks it in its transaction (ChangesetInput.PackDeadline,
+ // CheckpointInput.PackDeadline): the monotonic clock stops while the host
+ // is suspended or the VM paused, the database clock does not.
+ DBDeadline time.Time
}
// UploadPack uploads the pack read from r (size bytes whose SHA-256 is sum)
-// under a new owned key of the repository. readAt is the time of a read of the
-// repository record that found the repository, taken before that read and
-// before the upload; the upload must end before readAt + WriteDeadline(), else
-// it is cancelled (nothing is stored) and the error wraps ErrPackDeadline. It is
-// the only way the storage core uploads a pack (doc/graft/PLAN7.md, RULE 26).
-func (c *Core) UploadPack(ctx context.Context, repoID int64, readAt time.Time, sum [32]byte, size int64, r io.Reader) (PackUpload, error) {
+// under a new owned key of the repository. record was returned by a read of the
+// repository record that found the repository (GetRepository: its ReadAt is
+// the database time of that read), and readAt is the time taken before that
+// read; the upload must end before readAt + WriteDeadline(), else it is
+// cancelled (nothing is stored) and the error wraps ErrPackDeadline. It is the
+// only way the storage core uploads a pack (doc/graft/PLAN7.md, RULE 26).
+func (c *Core) UploadPack(ctx context.Context, record Repository, readAt time.Time, sum [32]byte, size int64, r io.Reader) (PackUpload, error) {
if c.GCGrace <= 0 || c.GCMargin <= 0 || c.WriteDeadline() <= 0 {
return PackUpload{}, fmt.Errorf("storage core: invalid write deadline: gc_grace %s, gc_margin %s", c.GCGrace, c.GCMargin)
}
+ if record.ReadAt.IsZero() {
+ return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: the record carries no database read time", record.ID)
+ }
deadline := readAt.Add(c.WriteDeadline())
if !time.Now().Before(deadline) {
- return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: %w", repoID, ErrPackDeadline)
+ return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: %w", record.ID, ErrPackDeadline)
}
putCtx, cancel := context.WithDeadline(ctx, deadline)
defer cancel()
- key := NewPackKey(repoID, sum)
+ key := NewPackKey(record.ID, sum)
if err := c.Blobs.Put(putCtx, key, r); err != nil {
if ctx.Err() == nil && errors.Is(putCtx.Err(), context.DeadlineExceeded) {
return PackUpload{}, fmt.Errorf("storage core: upload %q: %w: %w", key, ErrPackDeadline, err)
@@ -155,7 +165,7 @@ func (c *Core) UploadPack(ctx context.Context, repoID int64, readAt time.Time, s
return PackUpload{}, fmt.Errorf("storage core: upload %q: %w", key, err)
}
- return PackUpload{Key: key, Bytes: size, Deadline: deadline}, nil
+ return PackUpload{Key: key, Bytes: size, Deadline: deadline, DBDeadline: record.ReadAt.Add(c.WriteDeadline())}, nil
}
// CommitContext returns ctx bounded by the upload's deadline, for the commit of
diff --git a/internal/graft/corestore/coretest/import.go b/internal/graft/corestore/coretest/import.go
index 3bfe69e..801f9f7 100644
--- a/internal/graft/corestore/coretest/import.go
+++ b/internal/graft/corestore/coretest/import.go
@@ -54,12 +54,13 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProto
var upload corestore.PackUpload
if !pack.Empty() {
readAt := time.Now()
- _, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repoProto))
+ record, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repoProto))
require.NoError(tb, err)
+ require.Equal(tb, registered.ID, record.ID)
f, err := pack.Open()
require.NoError(tb, err)
- upload, err = core.UploadPack(ctx, registered.ID, readAt, pack.SHA256, pack.Size, f)
+ upload, err = core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, f)
require.NoError(tb, err)
require.NoError(tb, f.Close())
}
@@ -74,13 +75,14 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProto
defer cancel()
res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{
- RepoID: registered.ID,
- Updates: updates,
- Operation: "import",
- Actor: "import",
- NodeID: core.NodeID,
- PackKey: upload.Key,
- PackBytes: upload.Bytes,
+ RepoID: registered.ID,
+ Updates: updates,
+ Operation: "import",
+ Actor: "import",
+ NodeID: core.NodeID,
+ PackKey: upload.Key,
+ PackBytes: upload.Bytes,
+ PackDeadline: upload.DBDeadline,
})
require.NoError(tb, err)
require.Equal(tb, uint64(1), res.Seq, "the import is changeset 1")
diff --git a/internal/graft/corestore/gate/gate_fix3_test.go b/internal/graft/corestore/gate/gate_fix3_test.go
index fca8f3a..4f5294c 100644
--- a/internal/graft/corestore/gate/gate_fix3_test.go
+++ b/internal/graft/corestore/gate/gate_fix3_test.go
@@ -6,6 +6,7 @@ import (
"os"
"strings"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
@@ -125,7 +126,7 @@ func TestGateHistoryGuardWithRealSyncerOnFilesBackendCopy(t *testing.T) {
}
_, err = core.Meta.CommitChangeset(ctx, corestore.ChangesetInput{
RepoID: rec.ID, Updates: updates, Operation: "import", Actor: "import", NodeID: core.NodeID,
- PackKey: packKey, PackBytes: pack.Size,
+ PackKey: packKey, PackDeadline: time.Now().Add(time.Hour), PackBytes: pack.Size,
})
require.NoError(t, err)
require.NoError(t, gitops.WriteWatermark(path, corestore.Watermark{RepoID: rec.ID, Seq: 1}))
diff --git a/internal/graft/corestore/gc/gc_test.go b/internal/graft/corestore/gc/gc_test.go
index 134d3d5..35d29e1 100644
--- a/internal/graft/corestore/gc/gc_test.go
+++ b/internal/graft/corestore/gc/gc_test.go
@@ -99,7 +99,7 @@ func (e *env) commit(key string) uint64 {
Operation: "test",
Actor: "tester",
NodeID: "node-a",
- PackKey: key,
+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),
PackBytes: 1,
})
require.NoError(e.t, err)
@@ -111,9 +111,9 @@ func (e *env) checkpoint(seq uint64) string {
e.t.Helper()
key := e.upload(e.repo.ID)
_, created, err := e.core.Meta.CreateCheckpoint(e.ctx, e.repo.ID, corestore.CheckpointInput{
- Seq: seq,
- Refs: map[string]corestore.Target{"refs/heads/main": corestore.Target(strings.Repeat("a", 40))},
- PackKey: key,
+ Seq: seq,
+ Refs: map[string]corestore.Target{"refs/heads/main": corestore.Target(strings.Repeat("a", 40))},
+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),
PackBytes: 1,
})
require.NoError(e.t, err)
diff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.go
index ce15d08..24b2cbd 100644
--- a/internal/graft/corestore/maintenance/maintenance_test.go
+++ b/internal/graft/corestore/maintenance/maintenance_test.go
@@ -103,6 +103,7 @@ func (e *env) changeset(updates []corestore.RefUpdate, tips ...git.ObjectID) {
in := corestore.ChangesetInput{RepoID: e.repo.ID, Updates: updates, Operation: "test", Actor: "tester", NodeID: "node-a"}
if !pack.Empty() {
in.PackKey = corestore.NewPackKey(e.repo.ID, pack.SHA256)
+ in.PackDeadline = time.Now().Add(time.Hour)
rc, err := pack.Open()
require.NoError(e.t, err)
require.NoError(e.t, e.core.Blobs.Put(ctx, in.PackKey, rc))
diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.go
index bd8955b..23f3ad0 100644
--- a/internal/graft/corestore/metastore.go
+++ b/internal/graft/corestore/metastore.go
@@ -23,6 +23,10 @@ type ChangesetInput struct {
// objects, empty when there are none. PackBytes is its size.
PackKey string
PackBytes int64
+ // PackDeadline is the write deadline of the pack on the database clock
+ // (PackUpload.DBDeadline). A changeset with a pack is committed only before
+ // it, checked in the commit's transaction; one without it is refused.
+ PackDeadline time.Time
// Changelog, when not nil, appends one changelog entry per effective update
// of a branch or tag (IsChangelogRef), in the order of Updates, in the same
// transaction. The entries get the next changelog sequences, the actual
@@ -80,6 +84,9 @@ type CheckpointInput struct {
// pack, PackBytes its size.
PackKey string
PackBytes int64
+ // PackDeadline is the write deadline of the pack on the database clock
+ // (PackUpload.DBDeadline), checked in the record's transaction.
+ PackDeadline time.Time
}
var (
diff --git a/internal/graft/corestore/sqlstore/changeset.go b/internal/graft/corestore/sqlstore/changeset.go
index 85b955c..d13fc52 100644
--- a/internal/graft/corestore/sqlstore/changeset.go
+++ b/internal/graft/corestore/sqlstore/changeset.go
@@ -70,6 +70,28 @@ func validateChangeset(in corestore.ChangesetInput) error {
return nil
}
+// checkPackDeadline refuses, with ErrPackDeadline, the commit of a row that
+// references an uploaded pack once the database clock reached the pack's write
+// deadline (PackUpload.DBDeadline), and the commit of one without a deadline.
+// Garbage collection deletes an unreferenced pack gc_grace after it observed it
+// on the database clock, which is after the record read the deadline starts
+// from: a row committed before the deadline is never a row whose pack is gone,
+// whatever the writer's own clock did meanwhile (a suspended host, a paused
+// VM).
+func (s *Store) checkPackDeadline(ctx context.Context, tx *sql.Tx, deadline time.Time) error {
+ if deadline.IsZero() {
+ return errors.New("storage core: a row that references a pack must carry the pack's write deadline")
+ }
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+ if !now.Before(deadline) {
+ return fmt.Errorf("storage core: commit at %s, after the pack's write deadline %s: %w", now.Format(time.RFC3339Nano), deadline.UTC().Format(time.RFC3339Nano), corestore.ErrPackDeadline)
+ }
+ return nil
+}
+
// commitChangeset records a changeset in the open transaction tx, for the
// locked repository repo, and advances repo to the new sequences. It is shared
// by CommitChangeset and the initial changeset of RegisterRepository.
@@ -126,6 +148,9 @@ func (s *Store) commitChangeset(ctx context.Context, tx *sql.Tx, repo *repoState
var packKey any
if in.PackKey != "" {
+ if err := s.checkPackDeadline(ctx, tx, in.PackDeadline); err != nil {
+ return corestore.CommitResult{}, err
+ }
packKey = in.PackKey
}
diff --git a/internal/graft/corestore/sqlstore/checkpoint.go b/internal/graft/corestore/sqlstore/checkpoint.go
index 8d2b940..108f5cc 100644
--- a/internal/graft/corestore/sqlstore/checkpoint.go
+++ b/internal/graft/corestore/sqlstore/checkpoint.go
@@ -65,6 +65,9 @@ func (s *Store) CreateCheckpoint(ctx context.Context, repoID int64, in corestore
return fmt.Errorf("storage core: checkpoint at changeset %d is older than the newest checkpoint at %d of repository %d", in.Seq, newest.Seq, repoID)
}
+ if err := s.checkPackDeadline(ctx, tx, in.PackDeadline); err != nil {
+ return err
+ }
now, err := s.wallTime(ctx, tx)
if err != nil {
return err
diff --git a/internal/graft/corestore/sqlstore/checkpoint_large_test.go b/internal/graft/corestore/sqlstore/checkpoint_large_test.go
index bcd2fd4..184f7d1 100644
--- a/internal/graft/corestore/sqlstore/checkpoint_large_test.go
+++ b/internal/graft/corestore/sqlstore/checkpoint_large_test.go
@@ -60,7 +60,7 @@ func TestCheckpointOfManyReferences(t *testing.T) {
began := time.Now()
_, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
- Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "large checkpoint"), PackBytes: 1,
+ Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "large checkpoint"), PackDeadline: time.Now().Add(time.Hour), PackBytes: 1,
})
took := time.Since(began)
close(done)
diff --git a/internal/graft/corestore/sqlstore/deadline_test.go b/internal/graft/corestore/sqlstore/deadline_test.go
new file mode 100644
index 0000000..5144d86
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/deadline_test.go
@@ -0,0 +1,68 @@
+package sqlstore_test
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+)
+
+// TestCommitChecksThePackDeadlineOnTheDatabaseClock verifies the write
+// deadline of a pack in the commit transaction (integrated review m6). A
+// writer's monotonic clock stops while its host is suspended or its VM paused,
+// so the deadline it measured may not have expired for it when, on the
+// database clock that garbage collection uses, it has: such a commit is
+// refused, and nothing is written. A row with a pack and no deadline is
+// refused too.
+func TestCommitChecksThePackDeadlineOnTheDatabaseClock(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo := register(t, ctx, s, testKey("deadline"))
+ record, err := s.GetRepository(ctx, repo.Key)
+ require.NoError(t, err)
+ require.False(t, record.ReadAt.IsZero(), "GetRepository returns the database time of the read")
+ require.WithinDuration(t, time.Now(), record.ReadAt, time.Minute)
+
+ changeset := func(deadline time.Time) corestore.ChangesetInput {
+ return corestore.ChangesetInput{
+ RepoID: repo.ID,
+ Updates: []corestore.RefUpdate{create("refs/heads/main", oid(1))},
+ Operation: "test",
+ Actor: "tester",
+ NodeID: "node-a",
+ PackKey: ownedKey(repo.ID, "pack"),
+ PackBytes: 1,
+ PackDeadline: deadline,
+ }
+ }
+
+ // The writer slept through its deadline: the database clock is past it.
+ _, err = s.CommitChangeset(ctx, changeset(record.ReadAt.Add(-time.Second)))
+ require.ErrorIs(t, err, corestore.ErrPackDeadline)
+ _, err = s.CommitChangeset(ctx, changeset(time.Time{}))
+ require.ErrorContains(t, err, "must carry the pack's write deadline")
+ requireRefs(t, ctx, s, repo.ID, map[string]corestore.Target{})
+
+ res, err := s.CommitChangeset(ctx, changeset(record.ReadAt.Add(time.Hour)))
+ require.NoError(t, err)
+ require.Equal(t, uint64(1), res.Seq)
+
+ checkpoint := func(deadline time.Time) corestore.CheckpointInput {
+ return corestore.CheckpointInput{
+ Seq: 1, Refs: map[string]corestore.Target{"refs/heads/main": oid(1)},
+ PackKey: ownedKey(repo.ID, "checkpoint"), PackBytes: 1, PackDeadline: deadline,
+ }
+ }
+ _, _, err = s.CreateCheckpoint(ctx, repo.ID, checkpoint(record.ReadAt.Add(-time.Second)))
+ require.ErrorIs(t, err, corestore.ErrPackDeadline)
+ _, ok, err := s.NewestCheckpoint(ctx, repo.ID)
+ require.NoError(t, err)
+ require.False(t, ok, "nothing was recorded")
+ _, created, err := s.CreateCheckpoint(ctx, repo.ID, checkpoint(record.ReadAt.Add(time.Hour)))
+ require.NoError(t, err)
+ require.True(t, created)
+ })
+}
diff --git a/internal/graft/corestore/sqlstore/gc_test.go b/internal/graft/corestore/sqlstore/gc_test.go
index f1ef5c1..050c055 100644
--- a/internal/graft/corestore/sqlstore/gc_test.go
+++ b/internal/graft/corestore/sqlstore/gc_test.go
@@ -38,7 +38,7 @@ func commitPack(ctx context.Context, s *sqlstore.Store, repoID int64, u corestor
Operation: "test",
Actor: "tester",
NodeID: "node-a",
- PackKey: packKey,
+ PackKey: packKey, PackDeadline: time.Now().Add(time.Hour),
PackBytes: 1,
})
}
@@ -170,7 +170,7 @@ func TestCheckpointLifecycle(t *testing.T) {
"refs/heads/main": oid(2),
"refs/tags/\xff\xfev1": oid(7),
}
- in := corestore.CheckpointInput{Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "checkpoint 2"), PackBytes: 42}
+ in := corestore.CheckpointInput{Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "checkpoint 2"), PackDeadline: time.Now().Add(time.Hour), PackBytes: 42}
cp, created, err := s.CreateCheckpoint(ctx, repo.ID, in)
require.NoError(t, err)
@@ -191,22 +191,22 @@ func TestCheckpointLifecycle(t *testing.T) {
// A second build at the same sequence keeps the first checkpoint and its pack.
again, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
- Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "another build of 2"),
+ Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "another build of 2"), PackDeadline: time.Now().Add(time.Hour),
})
require.NoError(t, err)
require.False(t, created)
require.Equal(t, cp, again)
for name, bad := range map[string]corestore.CheckpointInput{
- "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x")},
- "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x")},
- "older than the newest": {Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "x")},
- "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x")},
- "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x")},
+ "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "older than the newest": {Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x"), PackDeadline: time.Now().Add(time.Hour)},
"no pack key": {Seq: 3, Refs: refs},
- "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x")},
- "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x")},
- "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x")},
+ "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
} {
_, _, err := s.CreateCheckpoint(ctx, repo.ID, bad)
require.Error(t, err, name)
@@ -217,7 +217,7 @@ func TestCheckpointLifecycle(t *testing.T) {
}
cp3, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
- Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": oid(3)}, PackKey: ownedKey(repo.ID, "checkpoint 3"),
+ Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": oid(3)}, PackKey: ownedKey(repo.ID, "checkpoint 3"), PackDeadline: time.Now().Add(time.Hour),
})
require.NoError(t, err)
require.True(t, created)
@@ -236,7 +236,7 @@ func TestCheckpointLifecycle(t *testing.T) {
require.False(t, deleted)
_, _, err = s.CreateCheckpoint(ctx, repo.ID+1000, corestore.CheckpointInput{
- Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID+1000, "unknown"),
+ Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID+1000, "unknown"), PackDeadline: time.Now().Add(time.Hour),
})
require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
})
@@ -252,7 +252,7 @@ func TestSyncBoundaryUsesTheDatabaseClock(t *testing.T) {
for seq := uint64(1); seq <= 3; seq++ {
_, _, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
- Seq: seq, Refs: map[string]corestore.Target{"refs/heads/main": oid(int(seq))}, PackKey: ownedKey(repo.ID, fmt.Sprint(seq)),
+ Seq: seq, Refs: map[string]corestore.Target{"refs/heads/main": oid(int(seq))}, PackKey: ownedKey(repo.ID, fmt.Sprint(seq)), PackDeadline: time.Now().Add(time.Hour),
})
require.NoError(t, err)
}
@@ -394,7 +394,7 @@ func TestDeleteRepositoryRecordsTombstone(t *testing.T) {
// Rows of milestone 3 go with the repository.
_, _, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
- Seq: 2, Refs: map[string]corestore.Target{"refs/heads/main": oid(2)}, PackKey: ownedKey(repo.ID, "cp"),
+ Seq: 2, Refs: map[string]corestore.Target{"refs/heads/main": oid(2)}, PackKey: ownedKey(repo.ID, "cp"), PackDeadline: time.Now().Add(time.Hour),
})
require.NoError(t, err)
require.NoError(t, s.RecordOrphans(ctx, repo.ID, []string{ownedKey(repo.ID, "orphan")}))
@@ -703,7 +703,7 @@ func TestLockOrderConcurrentWriters(t *testing.T) {
}
cp, created, err := a.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
- Seq: head, Refs: refs, PackKey: ownedKey(repo.ID, fmt.Sprint("checkpoint ", head)),
+ Seq: head, Refs: refs, PackKey: ownedKey(repo.ID, fmt.Sprint("checkpoint ", head)), PackDeadline: time.Now().Add(time.Hour),
})
switch {
case err == nil && created:
diff --git a/internal/graft/corestore/sqlstore/repository.go b/internal/graft/corestore/sqlstore/repository.go
index 92e0fdd..d7a5707 100644
--- a/internal/graft/corestore/sqlstore/repository.go
+++ b/internal/graft/corestore/sqlstore/repository.go
@@ -393,11 +393,35 @@ func (s *Store) GetRepository(ctx context.Context, key corestore.RepoKey) (cores
return corestore.Repository{}, corestore.ErrRepositoryNotFound
}
- r, err := s.selectRepository(ctx, s.db, "storage = ? AND relative_path = ?", []any{key.Storage, key.RelativePath}, false)
+ // The database time of the read (Repository.ReadAt): Postgres reads its
+ // clock in the same statement; SQLite's clock is the host's, read before.
+ readAt := time.Now().UTC()
+ query := "SELECT " + repoColumns + " FROM graft_repositories WHERE storage = ? AND relative_path = ?"
+ if s.dialect == Postgres {
+ query = "SELECT " + repoColumns + ", clock_timestamp() FROM graft_repositories WHERE storage = ? AND relative_path = ?"
+ }
+ row := s.db.QueryRowContext(ctx, s.bind(query), key.Storage, key.RelativePath)
+ var clock any
+ r, err := scanRepoState(func(dest ...any) error {
+ if s.dialect == Postgres {
+ dest = append(dest, &clock)
+ }
+ return row.Scan(dest...)
+ })
+ if errors.Is(err, sql.ErrNoRows) {
+ return corestore.Repository{}, corestore.ErrRepositoryNotFound
+ }
if err != nil {
- return corestore.Repository{}, err
+ return corestore.Repository{}, fmt.Errorf("select repository: %w", err)
+ }
+ if s.dialect == Postgres {
+ if readAt, err = scanTime(clock); err != nil {
+ return corestore.Repository{}, fmt.Errorf("read the database time: %w", err)
+ }
}
- return toRepository(r), nil
+ repo := toRepository(r)
+ repo.ReadAt = readAt
+ return repo, nil
}
// DeleteRepository implements corestore.MetaStore.
diff --git a/internal/graft/corestore/syncer/syncer_checkpoint_test.go b/internal/graft/corestore/syncer/syncer_checkpoint_test.go
index 27015a5..be05aa7 100644
--- a/internal/graft/corestore/syncer/syncer_checkpoint_test.go
+++ b/internal/graft/corestore/syncer/syncer_checkpoint_test.go
@@ -41,7 +41,7 @@ func (e *env) checkpoint(t *testing.T, id int64, src source) corestore.Checkpoin
require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))
require.NoError(t, rc.Close())
- cp, created, err := e.core.Meta.CreateCheckpoint(e.ctx, id, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})
+ cp, created, err := e.core.Meta.CreateCheckpoint(e.ctx, id, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackDeadline: time.Now().Add(time.Hour), PackBytes: pack.Size})
require.NoError(t, err)
require.True(t, created)
return cp
diff --git a/internal/graft/corestore/syncer/syncer_import_test.go b/internal/graft/corestore/syncer/syncer_import_test.go
index 2099ad2..e67e57e 100644
--- a/internal/graft/corestore/syncer/syncer_import_test.go
+++ b/internal/graft/corestore/syncer/syncer_import_test.go
@@ -3,6 +3,7 @@ package syncer_test
import (
"os"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
@@ -119,7 +120,7 @@ func testSyncerIncompleteImport(t *testing.T, dialect sqlstore.Dialect, backend
Operation: "import",
Actor: "tester",
NodeID: "node-a",
- PackKey: packKey,
+ PackKey: packKey, PackDeadline: time.Now().Add(time.Hour),
PackBytes: pack.Size,
})
require.NoError(t, err)
diff --git a/internal/graft/corestore/syncer/syncer_race_test.go b/internal/graft/corestore/syncer/syncer_race_test.go
index da05b2d..375d810 100644
--- a/internal/graft/corestore/syncer/syncer_race_test.go
+++ b/internal/graft/corestore/syncer/syncer_race_test.go
@@ -37,6 +37,7 @@ func (e *env) commitFrom(t *testing.T, id int64, node string, src source, update
defer func() { require.NoError(t, pack.Remove()) }()
if !pack.Empty() {
in.PackKey = corestore.NewPackKey(id, pack.SHA256)
+ in.PackDeadline = time.Now().Add(time.Hour)
in.PackBytes = pack.Size
rc, err := pack.Open()
require.NoError(t, err)
diff --git a/internal/graft/corestore/syncer/syncer_test.go b/internal/graft/corestore/syncer/syncer_test.go
index 6535475..314e060 100644
--- a/internal/graft/corestore/syncer/syncer_test.go
+++ b/internal/graft/corestore/syncer/syncer_test.go
@@ -191,6 +191,7 @@ func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.
require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))
require.NoError(t, rc.Close())
in.PackKey = key
+ in.PackDeadline = time.Now().Add(time.Hour)
in.PackBytes = pack.Size
}
diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.go
index 5723064..42a5d00 100644
--- a/internal/graft/corestore/types.go
+++ b/internal/graft/corestore/types.go
@@ -131,6 +131,12 @@ type Repository struct {
// LogSeq is the sequence of the last changelog entry, 0 when there is none.
LogSeq uint64
CreatedAt time.Time
+ // ReadAt is the database time of the read that returned the record (set by
+ // GetRepository), at or before the read. A pack uploaded after the read is
+ // first observable by garbage collection after ReadAt, so a row that
+ // references it is committed before ReadAt + Core.WriteDeadline() on the
+ // database clock (PackUpload.DBDeadline, checked in the commit).
+ ReadAt time.Time
}
// Changeset is one recorded changeset of the write-ahead log.
diff --git a/internal/graft/corestore/upload_test.go b/internal/graft/corestore/upload_test.go
index d9e080f..6efd8e7 100644
--- a/internal/graft/corestore/upload_test.go
+++ b/internal/graft/corestore/upload_test.go
@@ -44,8 +44,10 @@ func TestUploadPackUsesAnOwnedKeyAndTheWriteDeadline(t *testing.T) {
core := uploadCore(blobs, time.Hour, 5*time.Minute)
sum := sha256.Sum256([]byte("pack"))
readAt := time.Now()
+ // The database clock of the read, which the commit checks the deadline on.
+ record := Repository{ID: 7, ReadAt: time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC)}
- upload, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))
+ upload, err := core.UploadPack(context.Background(), record, readAt, sum, 4, strings.NewReader("pack"))
require.NoError(t, err)
require.True(t, IsOwnedPackKey(upload.Key))
id, gotSum, _ := ParsePackKey(upload.Key)
@@ -53,11 +55,12 @@ func TestUploadPackUsesAnOwnedKeyAndTheWriteDeadline(t *testing.T) {
require.Equal(t, sum, gotSum)
require.Equal(t, int64(4), upload.Bytes)
require.Equal(t, readAt.Add(55*time.Minute), upload.Deadline)
+ require.Equal(t, record.ReadAt.Add(55*time.Minute), upload.DBDeadline)
require.Equal(t, []string{upload.Key}, blobs.puts)
require.WithinDuration(t, upload.Deadline, blobs.deadline, 0, "the Put runs under the write deadline")
// Every upload gets its own key, also for the same bytes.
- again, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))
+ again, err := core.UploadPack(context.Background(), record, readAt, sum, 4, strings.NewReader("pack"))
require.NoError(t, err)
require.NotEqual(t, upload.Key, again.Key)
@@ -73,13 +76,14 @@ func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {
t.Parallel()
sum := sha256.Sum256([]byte("late"))
+ record := Repository{ID: 1, ReadAt: time.Now()}
t.Run("deadline passed before the upload", func(t *testing.T) {
t.Parallel()
blobs := &fakeBlobs{}
core := uploadCore(blobs, time.Hour, 5*time.Minute)
- _, err := core.UploadPack(context.Background(), 1, time.Now().Add(-time.Hour), sum, 4, strings.NewReader("late"))
+ _, err := core.UploadPack(context.Background(), record, time.Now().Add(-time.Hour), sum, 4, strings.NewReader("late"))
require.ErrorIs(t, err, ErrPackDeadline)
require.Empty(t, blobs.puts, "nothing is uploaded")
})
@@ -89,7 +93,7 @@ func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {
blobs := &fakeBlobs{block: true}
core := uploadCore(blobs, 100*time.Millisecond, 50*time.Millisecond)
- _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 4, strings.NewReader("late"))
+ _, err := core.UploadPack(context.Background(), record, time.Now(), sum, 4, strings.NewReader("late"))
require.ErrorIs(t, err, ErrPackDeadline, "the Put is cancelled at the deadline")
})
@@ -100,10 +104,20 @@ func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
- _, err := core.UploadPack(ctx, 1, time.Now(), sum, 4, strings.NewReader("late"))
+ _, err := core.UploadPack(ctx, record, time.Now(), sum, 4, strings.NewReader("late"))
require.ErrorIs(t, err, context.Canceled)
require.NotErrorIs(t, err, ErrPackDeadline, "the caller's cancellation is not a deadline")
})
+
+ t.Run("a record without a database read time", func(t *testing.T) {
+ t.Parallel()
+ blobs := &fakeBlobs{}
+ core := uploadCore(blobs, time.Hour, 5*time.Minute)
+
+ _, err := core.UploadPack(context.Background(), Repository{ID: 1}, time.Now(), sum, 4, strings.NewReader("late"))
+ require.ErrorContains(t, err, "no database read time")
+ require.Empty(t, blobs.puts, "nothing is uploaded: its commit could not be bounded")
+ })
}
func TestUploadPackRequiresTheGCTimes(t *testing.T) {
@@ -115,7 +129,7 @@ func TestUploadPackRequiresTheGCTimes(t *testing.T) {
uploadCore(&fakeBlobs{}, time.Hour, 0),
uploadCore(&fakeBlobs{}, time.Minute, time.Hour),
} {
- _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 1, strings.NewReader("x"))
+ _, err := core.UploadPack(context.Background(), Repository{ID: 1, ReadAt: time.Now()}, time.Now(), sum, 1, strings.NewReader("x"))
require.ErrorContains(t, err, "invalid write deadline")
}
}
diff --git a/internal/graft/corestore/voter/server_test.go b/internal/graft/corestore/voter/server_test.go
index c0c24d6..0ecd124 100644
--- a/internal/graft/corestore/voter/server_test.go
+++ b/internal/graft/corestore/voter/server_test.go
@@ -6,6 +6,7 @@ import (
"sync"
"sync/atomic"
"testing"
+ "time"
"github.com/stretchr/testify/require"
@@ -164,7 +165,7 @@ func TestAuthoritativeServer(t *testing.T) {
Operation: "land",
Actor: "other-node",
NodeID: "node-b",
- PackKey: key,
+ PackKey: key, PackDeadline: time.Now().Add(time.Hour),
PackBytes: pack.Size,
})
return err
diff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.go
index 6d3853c..8f3342c 100644
--- a/internal/graft/corestore/voter/voter.go
+++ b/internal/graft/corestore/voter/voter.go
@@ -259,7 +259,7 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, ch
return nil, nil
}
- pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)
+ pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates)
if err != nil {
return nil, err
}
@@ -457,21 +457,22 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change
if prepared != nil && prepared.ok && prepared.repoID == record.ID && prepared.usable(time.Now()) {
pack = prepared.pack
} else {
- pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)
+ pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record, readAt, updates)
if err != nil {
return v.unavailable(err)
}
}
in := corestore.ChangesetInput{
- RepoID: record.ID,
- Updates: refUpdates,
- Operation: meta.Operation,
- Actor: meta.Actor,
- NodeID: v.core.NodeID,
- Force: shadow,
- PackKey: pack.Key,
- PackBytes: pack.Bytes,
+ RepoID: record.ID,
+ Updates: refUpdates,
+ Operation: meta.Operation,
+ Actor: meta.Actor,
+ NodeID: v.core.NodeID,
+ Force: shadow,
+ PackKey: pack.Key,
+ PackBytes: pack.Bytes,
+ PackDeadline: pack.DBDeadline,
}
if err := v.fault(faultAfterUpload); err != nil {
@@ -678,7 +679,8 @@ func newTips(updates []inputUpdate) []git.ObjectID {
// local reference reaches (invariant I1, gitops.BuildPack); in shadow mode,
// everything that the values the database records reach (buildPackAgainst),
// because local references can hold values the storage core never recorded.
-func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, repoID int64, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {
+func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, record corestore.Repository, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {
+ repoID := record.ID
tips := newTips(updates)
if len(tips) == 0 {
return corestore.PackUpload{}, nil
@@ -720,7 +722,7 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storag
}
defer func() { _ = r.Close() }()
- upload, err := v.core.UploadPack(ctx, repoID, readAt, pack.SHA256, pack.Size, r)
+ upload, err := v.core.UploadPack(ctx, record, readAt, pack.SHA256, pack.Size, r)
if err != nil {
return corestore.PackUpload{}, err
}