Git designed for AI agents.
Review m4: every deleted row gets a tombstone; reconcile the old leaks
Federico Elia0b32470b743b
diff --git a/cmd/graft/storage.go b/cmd/graft/storage.goindex 19ab142..b54a552 100644--- a/cmd/graft/storage.go+++ b/cmd/graft/storage.go@@ -183,6 +183,18 @@ func storageGCRemoved(ctx context.Context, out io.Writer, o *options, f *storage}fmt.Fprintf(out, "%s %d objects (%d bytes) of %d removed repositories of storage %s\n",verb, res.PacksDeleted, res.BytesReclaimed, res.Repositories, displayText(client.Storage()))+ if res.LeaksFound > 0 {+ recorded := "recorded"+ if res.DryRun {+ recorded = "would record"+ }+ fmt.Fprintf(out, "found %d repository IDs with objects and no row or tombstone; %s %d tombstones (their objects are deleted after gc_grace)\n",+ res.LeaksFound, recorded, res.LeaksTombstoned)+ }+ if res.IDsAheadOfAllocator > 0 {+ fmt.Fprintf(out, "warning: %d repository IDs above the database's allocator have objects in the bucket (a database restored from an older backup?); they are left alone\n",+ res.IDsAheadOfAllocator)+ }return nil}diff --git a/cmd/graft/storage_test.go b/cmd/graft/storage_test.goindex 633c350..e2d6006 100644--- a/cmd/graft/storage_test.go+++ b/cmd/graft/storage_test.go@@ -1,6 +1,7 @@package mainimport (+ "crypto/sha256""encoding/json""path/filepath""strings"@@ -144,6 +145,20 @@ func TestCLIStorageMaintenanceAgainstServer(t *testing.T) {code, stdout, stderr = runCLI(t, append([]string{"storage", "gc", "--removed"}, server...)...)require.Equal(t, exitOK, code, stderr)require.Equal(t, "deleted 0 objects (0 bytes) of 0 removed repositories of storage "+repo.GetStorageName()+"\n", stdout)++ // Objects of a repository ID the database has not allocated (a database+ // restored from an older backup) are reported, never touched.+ record, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repo))+ require.NoError(t, err)+ ahead := corestore.NewPackKey(record.ID+2, sha256.Sum256([]byte("ahead")))+ require.NoError(t, core.Blobs.Put(ctx, ahead, strings.NewReader("ahead")))+ code, stdout, stderr = runCLI(t, append([]string{"storage", "gc", "--removed"}, server...)...)+ require.Equal(t, exitOK, code, stderr)+ require.Equal(t, "deleted 0 objects (0 bytes) of 0 removed repositories of storage "+repo.GetStorageName()+"\n"++ "warning: 1 repository IDs above the database's allocator have objects in the bucket (a database restored from an older backup?); they are left alone\n", stdout)+ exists, err := core.Blobs.Exists(ctx, ahead)+ require.NoError(t, err)+ require.True(t, exists)}// TestCLIStorageCoreOff checks that a server without a storage core reportsdiff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex 37e976b..72877b7 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -482,43 +482,112 @@ guarded nor swept).blocked every checkpoint. It is removed with its errors, metric, tests and`gitops.IsReachable`. `CheckpointInput` has no root set, and the checkpointbuild reads no reviews.-* **LOCK ORDER (PM check 3).** Every transaction that must serialize with a- changeset, or with a review write that requires its roots, takes the- repository row **as its first statement**, before it reads or writes any- child table. On Postgres that is `SELECT ... FOR NO KEY UPDATE`; SQLite's- immediate transaction holds the database write lock from `BEGIN`. The- transactions: `CommitChangeset`, `RegisterRepository`, `ImportHistory`,- `InsertReview`, `UpdateReview` with `requireRoots`, and the lease methods.- * Transactions that touch only child rows without changing their `repo_id`- never lock the repository row and never upgrade a lock, so they cannot form- a cycle with the first group: `UpdateReview` without `requireRoots`,- `ReportApplied`, the orphan observations, `DeleteCheckpoint`, and since the- integrated review `CreateCheckpoint`.- * Insert-only child transactions take Postgres's implicit `KEY SHARE` on the- parent and nothing after it. `FOR NO KEY UPDATE` does not conflict with- `KEY SHARE`, so they never hold up a writer. Deleting the repository still- waits for them.- * **Changed by the integrated review (M4).** `CreateCheckpoint` used to lock- the row and insert its snapshot one statement per reference. At 100k- references on Postgres a writer of the repository waited 38–46 s, locally.- * It now reads the row without locking it, and writes the snapshot 1000- references per statement. Nothing in it needs the writers excluded:- * the head only grows;- * the primary key makes one checkpoint per sequence: a concurrent build of- the same sequence inserts nothing (`ON CONFLICT DO NOTHING`) and returns- the one that committed;- * a checkpoint below a newer one is never the newest, so no seed reads it.- * Measured at 100k references (`TestCheckpointOfManyReferences`):- * Postgres: the record takes 1.6–1.7 s, while about 500 changesets commit,- the longest in 15–20 ms;- * SQLite: 0.23–0.27 s, before 0.63–0.70 s. SQLite has one write lock for- the whole database, so a writer waits for that. Tested with `-race` on Postgres and SQLite- (`TestLockOrderConcurrentWriters`): concurrent lands, review creations and- head moves (requiring their roots), a sweeper that deletes keep-around- references until the writers are done, comments, checkpoint commits and- leases on one repository. The test checks that no deadlock surfaces (the- retry hook counts them, and the count must stay at zero), the invariant at- the end, and dense sequences.+* **LOCK ORDER (PM check 3; self-contained, as of the integrated review).**+ *What it guarantees.* The write transactions of the storage core on one+ repository never deadlock with each other, except the one case named below,+ which Postgres detects and the store retries. The rule also gives the+ serialisation that the invariants need: a changeset, a review write that+ requires its roots, and the conditions checked with them (the review roots,+ `max_open_reviews`) see each other in a total order.++ *Background.* The database has one row per repository (`graft_repositories`,+ "the row") and child tables keyed by `repo_id` (references, changesets,+ changelog, reviews, nodes, checkpoints and their references, orphan+ observations, leases), all with `ON DELETE CASCADE` foreign keys. SQLite has+ one write lock for the whole database, taken at `BEGIN IMMEDIATE`: its write+ transactions run one at a time and cannot deadlock. The rest is about+ Postgres, where locks are per row. There, an insert of a child row takes+ `FOR KEY SHARE` on the parent row (the foreign key check) until the+ transaction ends; `FOR NO KEY UPDATE` does not conflict with `FOR KEY SHARE`;+ a `DELETE` of the row takes `FOR UPDATE`, which does.++ *The rule.*+ 1. **Row-first transactions.** A transaction that must serialise with the+ writers of the repository locks the row **as its first statement**, before+ it reads or writes a child table: `SELECT ... FOR NO KEY UPDATE` (Postgres;+ on SQLite the write lock is already held). These are `CommitChangeset`,+ `RegisterRepository` (it inserts the row), `ImportHistory`, `InsertReview`+ (always; it allocates the review ID, checks the roots when required and+ counts the open reviews for `max_open_reviews`), `UpdateReview` with+ `requireRoots`, the three lease methods, and `DeleteRepository` (which+ then deletes the row: the only upgrade, to `FOR UPDATE`, of a lock it+ already holds; the trigger `graft_repositories_tombstone` inserts the+ tombstone, a table without a foreign key).+ 2. **Child-only transactions.** Every other write transaction never locks the+ row explicitly and never changes a `repo_id`:+ * insert-only: `CreateCheckpoint` (since M4, below), `RecordOrphans`,+ `ReportApplied` (an upsert), and the reconciliation's `RecordLeak`+ (`graft_gc_removed` has no foreign key);+ * update or delete of child rows only: `UpdateReview` without+ `requireRoots` (comments, verdicts, closes; one row, with a version+ check), `ForgetOrphans`, `DeleteCheckpoint` (one checkpoint row, whose+ references cascade), `DeleteRemovedRepository`;+ * tables of their own: `ObserveAllocator` and `AdvanceReconcile`+ (`graft_gc_allocator_marks`, the one row of `graft_gc_reconcile`).++ *Why no cycle.* A cycle needs every transaction in it to hold a lock while it+ waits for the next. Take the waits one by one:+ * A row-first transaction waits for the row before it holds anything of the+ repository, so that wait closes no cycle. Holding the row, it can wait only+ for a child row that another transaction holds.+ * The row-first transactions that hold the row run one at a time, so the+ child row it waits for is held by a child-only transaction.+ * A child-only transaction waits on the row only at its first insert (the+ key share), while it holds nothing yet: no cycle through that wait. Its+ other waits are for child rows.+ * The child-only transactions that hold one child row and wait for another+ are the multi-row ones, and each kind touches rows nobody else holds at the+ same time: `ForgetOrphans` and `RecordOrphans` run in a collection, which+ holds the repository's maintenance lease, so one at a time per repository;+ `CreateCheckpoint` inserts rows of its own sequence (a concurrent build of+ the same sequence waits on the primary key and then does nothing).+ * The single-row ones (`UpdateReview` without roots, `DeleteCheckpoint`'s+ checkpoint row, `ReportApplied`) wait for nothing once they hold their row.+ So a chain of waits never comes back to its first transaction, except+ through the cascade of a repository removal, below.++ *The exception: removing a repository.* `DeleteRepository` holds the row+ `FOR UPDATE` and its cascade deletes child rows in the database's order. A+ child-only transaction that deletes several child rows in its own order+ (`ForgetOrphans`) can then hold one row the cascade needs while waiting for+ another the cascade holds. Postgres detects the cycle after+ `deadlock_timeout` and aborts one transaction with SQLSTATE 40P01, which the+ store retries like a serialisation failure (`isRetryable`, `withTx`): the+ outcome is that of a serial order (the repository is gone, or the+ observations were forgotten first). This costs a second at worst, only while+ a repository is removed during its collection, and changes no result.++ *Changed by the integrated review (M4).* `CreateCheckpoint` used to be+ row-first, and inserted its snapshot one statement per reference: at 100k+ references on Postgres a writer of the repository waited 38–46 s, locally.+ It now reads the row without locking it and writes the snapshot 1000+ references per statement. Nothing in it needs the writers excluded:+ * the head only grows, so a sequence at or below the head read stays so;+ * the primary key makes one checkpoint per sequence: a concurrent build of+ the same sequence inserts nothing (`ON CONFLICT DO NOTHING`) and returns the+ one that committed, and a newer checkpoint covers the request and is+ returned too (m7);+ * a checkpoint below a newer one is never the newest, so no seed reads it;+ * a concurrent removal of the repository either waits for it (its key share)+ or makes its first insert fail on the foreign key: nothing is recorded,+ and its pack is an orphan of a removed repository, collected with it;+ * the pack's write deadline is checked in the same transaction on the+ database clock (`PackDeadline`, m6), as for a changeset.+ Measured at 100k references (`TestCheckpointOfManyReferences`): on Postgres+ the record takes 1.6–1.7 s while about 500 changesets commit, the longest in+ 15–20 ms; on SQLite 0.23–0.27 s (before: 0.63–0.70 s), during which writers+ wait for SQLite's one write lock.++ *Tests.* `TestLockOrderConcurrentWriters`, with `-race` on Postgres and+ SQLite: concurrent lands, review creations and head moves (requiring their+ roots), a sweeper that deletes keep-around references until the writers are+ done, comments, checkpoint commits and leases on one repository. It checks+ that no deadlock surfaces (the retry hook counts them, and the count must stay+ at zero), the invariant at the end, and dense sequences.+ `TestConcurrentCheckpointsOfOneSequence` (six concurrent records of one+ sequence), `TestCheckpointOfManyReferences` (writers during a large record),+ `TestCreateIsBoundedByTheOpenReviews` (concurrent creates against the+ limit, two handles).* **Files.** `internal/graft/corestore/reviewroots/**` (Protect, Sweep,Backfill), `internal/gitaly/service/review/**` (the roots writes and theclose release), `internal/graft/corestore/dbreview/**`@@ -1448,6 +1517,78 @@ prefix after the sweep (the PACK OWNERSHIP RULE makes this an enforced bound, noan assumption). A crash after a partial prefix deletion leaves the tombstone; thenext run finishes it. No lease (the repository is gone and its ID is never reused).+**Every deletion has a tombstone (integrated review m4, approved by the PM).**+Before migration `20261019000000_graft_storage_reconcile`, only `DeleteRepository`+wrote the tombstone; a row deleted otherwise (the repair procedure of STORAGE.md,+"State and verify", deletes it by hand) left its prefix in the bucket forever.+The migration adds the trigger `graft_repositories_tombstone`: `AFTER DELETE` on+`graft_repositories`, it records the tombstone of the deleted row on the database+clock (`clock_timestamp()`; SQLite's `'now'`, the host clock that SQLite's+`wallTime` reads too), in the deleting transaction, whoever deletes.+`DeleteRepository` relies on it. On a tombstone of the same ID (a reconciled one,+below) the trigger replaces it: the grace of a removal starts at the removal.++**Reconciliation of the leaks from before the migration.**+`gc.Collector.Reconcile` records a tombstone with an empty key (`RecordLeak`) for+a repository ID that has objects under `repos/<id>/` and neither a row nor a+tombstone; `CollectRemoved` then treats it like any removal+(`ListRemovedRepositories` returns the keyless tombstones in every storage's+list). It runs in the background maintenance before `CollectRemoved` (with `gc =+"auto"`) and in `CollectRemovedRepositories` (`graft storage gc --removed`).+* *Which IDs.* The allocator (Postgres sequence, SQLite `sqlite_sequence`) has+ no time per ID, so every run records its position with the database time+ (`graft_gc_allocator_marks`, `ObserveAllocator`), and examines only IDs at or+ below the greatest position observed at least W ago (`Bound`): IDs allocated+ at least W ago. It keeps the newest such mark and those after it.+* *Proof that a creation in flight is never collected.* Invariant U: an object+ under `repos/<id>/` is written only by `Core.UploadPack(record, readAt, …)`,+ whose record is a read that found the row of `id` (RULE 26: the only uploader;+ `Blobs.Put` has no other caller outside tests); so a creation+ writes nothing before its row commits, and its prefix is empty: the+ reconciliation, which tombstones only a prefix with objects, leaves it alone.+ Two more barriers, for a writer that would break U: (1) an ID younger than W+ is not examined (the PM's guard; on Postgres the sequence counts an+ allocation before its row commits, on SQLite an uncommitted allocation is+ invisible and the single write lock serialises it with the reconciliation);+ (2) a tombstone whose ID has a row is not a removal: `ListRemovedRepositories`+ excludes it, so a creation that commits after it was tombstoned keeps its+ objects, and its later deletion replaces the tombstone with a fresh grace.+ Test: `TestReconcileNeverCollectsACreationInFlight` holds a Postgres+ transaction between the allocation and the commit of a row, with an object+ under its prefix, across reconciliations: not examined while younger than W;+ examined after, tombstoned, then committed, and not collected.+* *Safety of a tombstoned leak.* No row for the ID now, and objects under its+ prefix: by U the row was committed once and has been deleted (IDs are never+ reused: Postgres `BIGSERIAL`, SQLite `AUTOINCREMENT`). A writer that read the+ row before the deletion uploads until `t_r + W`; the tombstone's `removed_at`+ is after the deletion, so the collection, `gc_grace > W` later, comes after+ every such upload, as for any removal.+* *Bounded, paged, idempotent, resumable.* A run examines at most 256 IDs after+ its cursor (`graft_gc_reconcile.done_through`, `UnrecordedIDs`), reads at most+ one object of each prefix (one page of a listing, stopped at the first+ object), records tombstones with "do nothing on conflict" and then moves the+ cursor past the batch. A run that stopped leaves the cursor, and the next run+ examines the batch again, recording nothing twice. With the trigger, a new+ ID never needs a second look: its deletion has its tombstone. After the+ backlog, a run examines only the IDs allocated since the last one.+* *IDs above the allocator* (a database restored from an older backup: the+ allocator will hand those IDs out again) are probed, 16 above the position,+ counted and logged, never touched.+* Metrics: `graft_storage_maintenance_leaks_found_total`,+ `graft_storage_maintenance_leaks_tombstoned_total`,+ `graft_storage_maintenance_ids_ahead_of_allocator`, and the step+ `reconcile` of `graft_storage_maintenance_failures_total`; the RPC returns+ `leaks_found`, `leaks_tombstoned`, `ids_ahead_of_allocator`.+* Why not an operator RPC that lists the bucket: the leaks then depend on an+ operator remembering to run it, and a listing of the whole bucket costs O(all+ objects) on every run. Why not a full listing in the background: the same+ cost, every interval. The cursor makes the cost O(new IDs) after the backlog.+* Tests: `TestDeletedRowGetsATombstone`, `TestReconcileTombstonesLeaks`,+ `TestReconcileIsPagedAndResumable`, `TestReconcileNeverCollectsACreationInFlight`+ (both dialects where the dialect allows; mutants without the W guard or+ without the row exclusion fail them), `TestRunOnceReconcilesLeaks`+ (metrics), `TestCollectRemovedRepositoriesReconcilesLeaks` (RPC).+### Hydration from a checkpoint (syncer, WP76)`hydrate` seeds from `NewestCheckpoint`:diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.mdindex 86f6b5c..a8ce3c6 100644--- a/doc/graft/STORAGE.md+++ b/doc/graft/STORAGE.md@@ -350,7 +350,11 @@ AND relative_path = '<relative path>'`; its changesets, references, changelogand reviews go with it) and run `graft storage import` again. The newregistration gets a new repository ID and a full pack, and takes the changelogand the reviews from the files again, so do this before switching to-authoritative mode.+authoritative mode. The deletion records the tombstone of the old ID (a+database trigger, whoever deletes the row), so garbage collection deletes the+old objects `gc_grace` later. Rows deleted this way before this release left+their objects in the bucket; garbage collection finds them and records their+tombstones (see `gc --removed` below).### Checkpoints and garbage collection@@ -407,7 +411,15 @@ graft storage gc --removed -address unix:/var/opt/graft/gitaly.sock [-storage deof the repository (the background one, another node) holds it, or `Aborted`when the lease is lost during the collection.- `gc --removed` deletes every object of the repositories of the storage that- were removed at least `gc_grace` ago.+ were removed at least `gc_grace` ago. It first reconciles the bucket: a+ repository ID allocated at least `gc_grace − gc_margin` ago that has objects+ in the bucket and neither a row nor a tombstone (a row deleted by hand before+ this release) gets a tombstone, so its objects go `gc_grace` later. A run+ examines at most 256 IDs, from where the previous run stopped, and reads at+ most one object of each; it prints what it found. Objects of IDs above the+ database's allocator (a database restored from an older backup) are reported+ with a warning and never touched: the database will hand those IDs out again.+ The background maintenance does the same with `gc = "auto"`.- `--dry-run` reports what would be deleted and changes nothing. It is a usageerror on the subcommands that do not read it, like `--connectivity` outside`verify` and `--removed` outside `gc`.@@ -418,12 +430,13 @@ can never call the storage core service, whatever `agent_rpcs` says.### UpgradesUpgrading from v0.2.x to this release changes the storage core's database-(migration `20261018000000_graft_storage_gc`) and the keys of the packs in the+(migrations `20261018000000_graft_storage_gc` and+`20261019000000_graft_storage_reconcile`: the tombstone trigger and the state+of the bucket reconciliation) and the keys of the packs in thebucket (owned keys, see Packs). A node of the old version cannot work next toa node of the new one:- **It cannot start.** A v0.2.x node that restarts after the migration- refuses the database, with an error about an unknown migration- (`20261018000000_graft_storage_gc`).+ refuses the database, with an error about the unknown migrations.- **It cannot read.** A v0.2.x node still running reads only the old 64-hexpack keys, so its reads of a repository that a new node wrote fail with"invalid pack key".@@ -712,7 +725,9 @@ without a single divergence.| `graft_storage_maintenance_checkpoints_total` | Checkpoints created by the background maintenance. || `graft_storage_maintenance_packs_deleted_total`, `graft_storage_maintenance_bytes_reclaimed_total` | Bucket objects, and their bytes, deleted by the background garbage collection. || `graft_storage_maintenance_skipped_total{reason}` | Repositories the background maintenance skipped: `no_local_copy`, `nothing_due` (no checkpoint due, no superseded checkpoint, no collection; decided without the lease), `lease_held`. |-| `graft_storage_maintenance_failures_total{step}` | Failed steps of the background maintenance: `collect_removed`, `list`, `lease`, `checkpoint`, `collect`. |+| `graft_storage_maintenance_failures_total{step}` | Failed steps of the background maintenance: `reconcile`, `collect_removed`, `list`, `lease`, `checkpoint`, `collect`. |+| `graft_storage_maintenance_leaks_found_total`, `graft_storage_maintenance_leaks_tombstoned_total` | Repository IDs with objects in the bucket and neither a row nor a tombstone, found by the reconciliation, and the tombstones it recorded for them. |+| `graft_storage_maintenance_ids_ahead_of_allocator` | Repository IDs above the database's allocator with objects in the bucket at the last reconciliation (a database restored from an older backup); never touched. Alert on a value above 0. || `graft_review_roots_backfilled_total` | Keep-around references of open-review commits written by the backfill (daily housekeeping and before every checkpoint). || `graft_review_roots_deleted_total` | Keep-around references deleted because no open review uses them. || `graft_review_roots_missing_total` | Open-review commits the backfill found missing from the repository; the review can only be closed. |diff --git a/internal/gitaly/service/storagecore/maintenance.go b/internal/gitaly/service/storagecore/maintenance.goindex d677830..f75637c 100644--- a/internal/gitaly/service/storagecore/maintenance.go+++ b/internal/gitaly/service/storagecore/maintenance.go@@ -124,14 +124,22 @@ func (s *server) CollectRemovedRepositories(ctx context.Context, req *gitalypb.Creturn nil, err}- report, err := gc.New(core, s.deps.GetLogger()).CollectRemoved(ctx, storageName, req.GetDryRun())+ collector := gc.New(core, s.deps.GetLogger())+ reconciled, err := collector.Reconcile(ctx, req.GetDryRun())+ if err != nil {+ return nil, err+ }+ report, err := collector.CollectRemoved(ctx, storageName, req.GetDryRun())if err != nil {return nil, err}return &gitalypb.CollectRemovedRepositoriesResponse{- Repositories: uint32(report.RemovedRepos),- PacksDeleted: uint32(report.PacksDeleted),- BytesReclaimed: report.BytesReclaimed,- DryRun: report.DryRun,+ Repositories: uint32(report.RemovedRepos),+ PacksDeleted: uint32(report.PacksDeleted),+ BytesReclaimed: report.BytesReclaimed,+ DryRun: report.DryRun,+ LeaksFound: uint32(reconciled.LeaksFound),+ LeaksTombstoned: uint32(reconciled.LeaksTombstoned),+ IdsAheadOfAllocator: uint32(reconciled.AheadOfAllocator),}, nil}diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.goindex c1fff6f..40c2459 100644--- a/internal/gitaly/service/storagecore/maintenance_test.go+++ b/internal/gitaly/service/storagecore/maintenance_test.go@@ -21,12 +21,31 @@ import ("github.com/feder-cr/git-ai/proto/go/gitalypb")-// shortGC makes garbage collection wait a short grace, so that a test sees an-// orphan deleted. The writers' deadline (grace - margin) stays long enough for-// the import's upload under load.-func shortGC(core *corestore.Core) {- core.GCGrace = time.Second- core.GCMargin = 100 * time.Millisecond+// The tests age rows on the database clock instead of waiting with a short+// gc_grace: the writers' deadline W is gc_grace - gc_margin, and an import or+// a shadow write under the race detector on a loaded host does not fit in a W+// of a second.++// exec runs a statement with "?" placeholders on the database.+func (h *harness) exec(query string, args ...any) {+ h.t.Helper()+ store := h.core.Meta.(*sqlstore.Store)+ if store.Dialect() == sqlstore.Postgres {+ for n := 1; strings.Contains(query, "?"); n++ {+ query = strings.Replace(query, "?", fmt.Sprintf("$%d", n), 1)+ }+ }+ _, err := store.DB().ExecContext(h.ctx, query, args...)+ require.NoError(h.t, err)+}++// longAgo returns the database time value of more than gc_grace before now.+func (h *harness) longAgo() any {+ t := time.Now().Add(-h.core.GCGrace - time.Minute).UTC()+ if h.core.Meta.(*sqlstore.Store).Dialect() == sqlstore.SQLite {+ return t.Format(time.RFC3339Nano)+ }+ return t}// putOrphan stores an object under the repository's pack prefix that no row@@ -101,7 +120,7 @@ func TestCreateCheckpointRefusals(t *testing.T) {func TestCollectGarbage(t *testing.T) {t.Parallel()forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {- h := setup(t, mode, dialect, shortGC)+ h := setup(t, mode, dialect)h.seed()repoID := h.importRepo()orphan := h.putOrphan(repoID, "the pack of a write that died")@@ -117,11 +136,11 @@ func TestCollectGarbage(t *testing.T) {require.Equal(t, uint32(1), first.GetOrphansObserved())require.True(t, h.exists(orphan), "observed, not yet deleted")- require.Eventually(t, func() bool {- resp, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})- require.NoError(t, err)- return resp.GetOrphansDeleted() == 1 && resp.GetBytesReclaimed() > 0- }, 30*time.Second, 50*time.Millisecond)+ h.exec("UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ?", h.longAgo(), repoID)+ resp, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})+ require.NoError(t, err)+ require.Equal(t, uint32(1), resp.GetOrphansDeleted())+ require.Positive(t, resp.GetBytesReclaimed())require.False(t, h.exists(orphan))require.Empty(t, h.verify(true).GetDivergences(), "the repository is intact")})@@ -159,7 +178,7 @@ func TestCollectGarbageRefusals(t *testing.T) {func TestCollectRemovedRepositories(t *testing.T) {t.Parallel()forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {- h := setup(t, mode, dialect, shortGC)+ h := setup(t, mode, dialect)h.seed()repoID := h.importRepo()packs := h.packKeys(repoID)@@ -170,11 +189,58 @@ func TestCollectRemovedRepositories(t *testing.T) {require.True(t, removed)storageName := h.cfg.Storages[0].Name- require.Eventually(t, func() bool {+ collect := func() *gitalypb.CollectRemovedRepositoriesResponse {+ resp, err := h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: storageName})+ require.NoError(t, err)+ return resp+ }+ require.Zero(t, collect().GetRepositories(), "the tombstone is younger than gc_grace")+ h.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", h.longAgo(), repoID)+ resp := collect()+ require.Equal(t, uint32(1), resp.GetRepositories())+ require.GreaterOrEqual(t, resp.GetPacksDeleted(), uint32(len(packs)))+ require.Empty(t, h.packKeys(repoID))+ })+}++// TestCollectRemovedRepositoriesReconcilesLeaks: the objects of a repository+// whose row went without a tombstone (deleted by hand before the trigger+// existed) are found by the reconciliation of the RPC once the allocation is W+// old, get a tombstone, and are deleted once it is gc_grace old (integrated+// review m4).+func TestCollectRemovedRepositoriesReconcilesLeaks(t *testing.T) {+ t.Parallel()+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ h := setup(t, corestore.ModeAuthoritative, dialect)+ h.seed()+ repoID := h.importRepo()+ packs := h.packKeys(repoID)+ require.NotEmpty(t, packs)++ h.exec("DELETE FROM graft_repositories WHERE repo_id = ?", repoID)+ h.exec("DELETE FROM graft_gc_removed WHERE repo_id = ?", repoID)++ storageName := h.cfg.Storages[0].Name+ collect := func() *gitalypb.CollectRemovedRepositoriesResponse {resp, err := h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: storageName})require.NoError(t, err)- return resp.GetRepositories() == 1 && resp.GetPacksDeleted() >= uint32(len(packs))- }, 30*time.Second, 50*time.Millisecond)+ require.Zero(t, resp.GetIdsAheadOfAllocator())+ return resp+ }+ require.Zero(t, collect().GetLeaksFound(), "the allocation is not W old yet")++ // The allocator was observed at the ID longer than W ago.+ h.exec("DELETE FROM graft_gc_allocator_marks")+ h.exec("INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)", h.longAgo(), repoID)+ resp := collect()+ require.Equal(t, uint32(1), resp.GetLeaksFound())+ require.Equal(t, uint32(1), resp.GetLeaksTombstoned())+ require.Zero(t, resp.GetRepositories(), "the new tombstone is younger than gc_grace")++ h.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", h.longAgo(), repoID)+ resp = collect()+ require.Equal(t, uint32(1), resp.GetRepositories())+ require.Zero(t, resp.GetLeaksFound(), "examined once")require.Empty(t, h.packKeys(repoID))})}diff --git a/internal/graft/corestore/gc/export_test.go b/internal/graft/corestore/gc/export_test.goindex d233db0..d936946 100644--- a/internal/graft/corestore/gc/export_test.go+++ b/internal/graft/corestore/gc/export_test.go@@ -1,4 +1,6 @@package gc-// SetFaultHook sets the hook called at the fault points of c's runs.func SetFaultHook(c *Collector, hook func(point string) error) { c.faultHook = hook }++// ReconcileBatch is reconcileBatch, for the tests.+const ReconcileBatch = reconcileBatchdiff --git a/internal/graft/corestore/gc/gc.go b/internal/graft/corestore/gc/gc.goindex 6b6315f..a2bb5d3 100644--- a/internal/graft/corestore/gc/gc.go+++ b/internal/graft/corestore/gc/gc.go@@ -64,6 +64,16 @@ type Report struct {BytesReclaimed int64// RemovedRepos are the removed repositories whose objects were deleted.RemovedRepos int+ // LeaksFound are the repository IDs of the reconciled range that have+ // objects in the bucket and neither a row nor a tombstone.+ LeaksFound int+ // LeaksTombstoned are the tombstones the reconciliation recorded for them.+ LeaksTombstoned int+ // AheadOfAllocator are the repository IDs above the allocator's position+ // that have objects in the bucket (a database restored from an older+ // backup). They are reported and never touched: the allocator will hand+ // them out again.+ AheadOfAllocator int// DryRun tells that nothing was deleted or recorded.DryRun bool}@@ -353,6 +363,107 @@ func (c *Collector) CollectRemoved(ctx context.Context, storage string, dryRun breturn report, nil}+const (+ // reconcileBatch bounds the repository IDs one reconciliation examines.+ reconcileBatch = 256+ // aheadProbe is the number of IDs above the allocator's position whose+ // prefixes a reconciliation checks.+ aheadProbe = 16+)++// errFound stops a listing at its first object.+var errFound = errors.New("found")++// hasObjects reports whether the bucket has an object under the repository's+// prefix. It reads one object at most (one page of a paged listing).+func (c *Collector) hasObjects(ctx context.Context, repoID int64) (bool, error) {+ err := c.core.Blobs.List(ctx, repoPrefix(repoID), func(corestore.BlobInfo) error { return errFound })+ switch {+ case errors.Is(err, errFound):+ return true, nil+ case err != nil:+ return false, errBucket(err)+ }+ return false, nil+}++// Reconcile records a tombstone for every repository ID whose row was deleted+// without one (before the trigger graft_repositories_tombstone existed) and+// whose objects are still in the bucket; CollectRemoved then deletes them+// like those of any removed repository (doc/graft/PLAN7.md, "Removed+// repositories", reconciliation). It examines the IDs after its cursor, at+// most reconcileBatch per run, and only those the allocator handed out at+// least the write deadline W ago on the database clock: a creation still in+// flight is never examined (on Postgres an allocated ID precedes its row). An+// ID without a row or a tombstone gets one only when its prefix has objects;+// the cursor then moves past the batch, so a run that stopped is resumed by+// the next one and nothing is examined twice. IDs above the allocator that+// have objects are counted and logged, never touched. A dry run records+// nothing.+func (c *Collector) Reconcile(ctx context.Context, dryRun bool) (Report, error) {+ if err := c.validTimes(); err != nil {+ return Report{}, err+ }++ view, err := c.core.Meta.ObserveAllocator(ctx, c.core.WriteDeadline(), dryRun)+ if err != nil {+ return Report{}, errDatabase(err)+ }+ report := Report{DryRun: dryRun}++ for id := view.Position + 1; id <= view.Position+aheadProbe; id++ {+ found, err := c.hasObjects(ctx, id)+ if err != nil {+ return report, err+ }+ if found {+ report.AheadOfAllocator+++ c.logger.WithFields(log.Fields{+ "repository_id": id,+ "allocator_position": view.Position,+ }).WarnContext(ctx, "storage core: the bucket has objects of a repository ID the database has not allocated; was the database restored from an older backup? They are left alone")+ }+ }++ upTo := min(view.Bound, view.DoneThrough+reconcileBatch)+ if upTo <= view.DoneThrough {+ return report, nil+ }+ ids, err := c.core.Meta.UnrecordedIDs(ctx, view.DoneThrough, upTo)+ if err != nil {+ return report, errDatabase(err)+ }+ for _, id := range ids {+ found, err := c.hasObjects(ctx, id)+ if err != nil {+ return report, err+ }+ if !found {+ continue+ }+ report.LeaksFound+++ if dryRun {+ continue+ }+ recorded, err := c.core.Meta.RecordLeak(ctx, id)+ if err != nil {+ return report, errDatabase(err)+ }+ if recorded {+ report.LeaksTombstoned+++ c.logger.WithField("repository_id", id).InfoContext(ctx,+ "storage core: the bucket has objects of a repository whose row was deleted without a tombstone; they are collected after gc_grace")+ }+ }+ if dryRun {+ return report, nil+ }+ if err := c.core.Meta.AdvanceReconcile(ctx, upTo); err != nil {+ return report, errDatabase(err)+ }+ return report, nil+}+// validTimes refuses to collect with times that would break the proofs.func (c *Collector) validTimes() error {if c.core.GCGrace <= 0 || c.core.GCMargin <= 0 || c.core.WriteDeadline() <= 0 {diff --git a/internal/graft/corestore/gc/gc_test.go b/internal/graft/corestore/gc/gc_test.goindex 35d29e1..778fecb 100644--- a/internal/graft/corestore/gc/gc_test.go+++ b/internal/graft/corestore/gc/gc_test.go@@ -6,6 +6,7 @@ import ("crypto/sha256""errors""fmt"+ "slices""sort""strings""sync"@@ -487,3 +488,202 @@ func TestCollectRemoved(t *testing.T) {require.Equal(t, []string{kept}, e.objects(e.repo.ID), "other repositories are untouched")})}++// tombstones returns the tombstones of every storage, whatever their age.+func (e *env) tombstones() map[int64]corestore.RemovedRepository {+ e.t.Helper()+ removed, err := e.core.Meta.ListRemovedRepositories(e.ctx, "", 0)+ require.NoError(e.t, err)+ out := map[int64]corestore.RemovedRepository{}+ for _, r := range removed {+ out[r.RepoID] = r+ }+ return out+}++// leak registers a repository, stores objects under its prefix and deletes+// its row and its tombstone, as a row deleted by hand before the trigger+// existed left them. It returns the repository ID.+func (e *env) leak(name string, objects int) int64 {+ e.t.Helper()+ repo := e.register(name)+ for range objects {+ e.upload(repo.ID)+ }+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", repo.ID)+ e.exec("DELETE FROM graft_gc_removed WHERE repo_id = ?", repo.ID)+ return repo.ID+}++// allocator returns the allocator view without recording an observation.+func (e *env) allocator() corestore.AllocatorView {+ e.t.Helper()+ view, err := e.core.Meta.ObserveAllocator(e.ctx, e.core.WriteDeadline(), true)+ require.NoError(e.t, err)+ return view+}++// ageAllocator replaces the observations of the allocator by one of its+// current position made d ago.+func (e *env) ageAllocator(d time.Duration) {+ e.t.Helper()+ position := e.allocator().Position+ e.exec("DELETE FROM graft_gc_allocator_marks")+ e.exec("INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)", e.past(d), position)+}++func (e *env) reconcile(dryRun bool) gc.Report {+ e.t.Helper()+ report, err := e.gc.Reconcile(e.ctx, dryRun)+ require.NoError(e.t, err)+ return report+}++// TestDeletedRowGetsATombstone: a repository row deleted by hand (the repair+// of STORAGE.md, "State and verify") gets its tombstone from the trigger, on+// the database clock, and its objects are collected like those of a removed+// repository (integrated review m4).+func TestDeletedRowGetsATombstone(t *testing.T) {+ forEachDialect(t, func(t *testing.T, e *env) {+ repo := e.register("by-hand")+ e.upload(repo.ID)+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", repo.ID)++ tombstone, ok := e.tombstones()[repo.ID]+ require.True(t, ok, "the trigger records the tombstone")+ require.Equal(t, repo.Key, tombstone.Key)+ require.WithinDuration(t, time.Now(), tombstone.RemovedAt, time.Minute)++ e.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", e.past(grace+time.Minute), repo.ID)+ report, err := e.gc.CollectRemoved(e.ctx, "default", false)+ require.NoError(t, err)+ require.Equal(t, 1, report.RemovedRepos)+ require.Empty(t, e.objects(repo.ID))+ })+}++// TestReconcileTombstonesLeaks: the objects of a row deleted without a+// tombstone get one, only for IDs allocated at least W ago, only when the+// prefix has objects, once; the tombstone has no key and is collected by+// every storage's collection after gc_grace (integrated review m4).+func TestReconcileTombstonesLeaks(t *testing.T) {+ forEachDialect(t, func(t *testing.T, e *env) {+ live := e.changeset()+ leaked := e.leak("leaked", 2)+ empty := e.leak("empty", 0)++ require.Equal(t, gc.Report{}, e.reconcile(false), "no allocation is W old yet")+ require.Len(t, e.objects(leaked), 2)++ e.ageAllocator(grace)+ require.Equal(t, gc.Report{LeaksFound: 1, DryRun: true}, e.reconcile(true))+ require.NotContains(t, e.tombstones(), leaked, "a dry run records nothing")++ require.Equal(t, gc.Report{LeaksFound: 1, LeaksTombstoned: 1}, e.reconcile(false))+ tombstones := e.tombstones()+ require.Contains(t, tombstones, leaked)+ require.Equal(t, corestore.RepoKey{}, tombstones[leaked].Key, "the key went with the row")+ require.NotContains(t, tombstones, empty, "an ID without objects needs nothing")+ require.Equal(t, e.allocator().Position, e.allocator().DoneThrough, "the cursor is past the reconciled IDs")++ require.Equal(t, gc.Report{}, e.reconcile(false), "a second run examines nothing again")++ report, err := e.gc.CollectRemoved(e.ctx, "default", false)+ require.NoError(t, err)+ require.Zero(t, report.RemovedRepos, "the tombstone is younger than gc_grace")+ e.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", e.past(grace+time.Minute), leaked)+ report, err = e.gc.CollectRemoved(e.ctx, "default", false)+ require.NoError(t, err)+ require.Equal(t, 1, report.RemovedRepos, "a storage's collection takes the tombstones without a key")+ require.Empty(t, e.objects(leaked))+ require.Equal(t, []string{live}, e.objects(e.repo.ID), "the live repository is untouched")+ })+}++// TestReconcileIsPagedAndResumable: a run examines at most one batch of IDs+// after its cursor; the next run goes on from there, and IDs above the+// allocator with objects are reported and never touched.+func TestReconcileIsPagedAndResumable(t *testing.T) {+ forEachDialect(t, func(t *testing.T, e *env) {+ const position = 2*gc.ReconcileBatch + 40+ if e.store.Dialect() == sqlstore.Postgres {+ e.exec(fmt.Sprintf("SELECT setval('graft_repositories_repo_id_seq', %d)", position))+ } else {+ e.exec("UPDATE sqlite_sequence SET seq = ? WHERE name = 'graft_repositories'", position)+ }+ first, second := int64(5), int64(gc.ReconcileBatch+10)+ e.upload(first)+ e.upload(second)+ ahead := e.upload(position + 3)+ e.ageAllocator(grace)++ require.Equal(t, gc.Report{LeaksFound: 1, LeaksTombstoned: 1, AheadOfAllocator: 1}, e.reconcile(false))+ require.Contains(t, e.tombstones(), first)+ require.NotContains(t, e.tombstones(), second, "beyond the first batch")+ require.EqualValues(t, gc.ReconcileBatch, e.allocator().DoneThrough)++ // A run that stopped after recording a tombstone, before its cursor+ // moved: the next one does not count it again.+ _, err := e.core.Meta.RecordLeak(e.ctx, second)+ require.NoError(t, err)+ require.Equal(t, gc.Report{AheadOfAllocator: 1}, e.reconcile(false))+ require.EqualValues(t, 2*gc.ReconcileBatch, e.allocator().DoneThrough)+ require.Equal(t, gc.Report{AheadOfAllocator: 1}, e.reconcile(false))+ require.EqualValues(t, position, e.allocator().DoneThrough)++ require.NotContains(t, e.tombstones(), int64(position+3))+ require.Equal(t, []string{ahead}, e.objects(position+3), "never touched")+ })+}++// TestReconcileNeverCollectsACreationInFlight holds a creation between the+// allocation of its ID and the commit of its row across reconciliations, on+// Postgres, whose allocator counts the ID before the row commits. (SQLite's+// allocator is transactional: an uncommitted creation's ID is not allocated+// for any other reader, and the single writer lock serialises the creation+// with the reconciliation's writes.) An object written under the ID before+// the commit, which no writer of the storage core does, is the worst case.+// The ID is not examined while its allocation is younger than W; examined+// later it may get a tombstone, but once the row commits the tombstone is not+// a removal: it is never collected, and a later deletion of the row starts a+// fresh grace.+func TestReconcileNeverCollectsACreationInFlight(t *testing.T) {+ if !slices.Contains(coretest.Dialects(), sqlstore.Postgres) {+ t.Skip("the allocator counts uncommitted creations on Postgres only, which is not configured")+ }+ e := newEnv(t, sqlstore.Postgres)++ tx, err := e.store.DB().BeginTx(e.ctx, nil)+ require.NoError(t, err)+ defer func() { _ = tx.Rollback() }()+ var id int64+ require.NoError(t, tx.QueryRowContext(e.ctx, `+ INSERT INTO graft_repositories (storage, relative_path, object_format, head_seq, log_seq, next_review_id, created_at)+ VALUES ('default', 'in-flight.git', 'sha1', 0, 0, 1, now()) RETURNING repo_id`).Scan(&id))+ key := e.upload(id)++ require.Equal(t, e.allocator().Position, id, "the allocator counts the creation")+ require.Equal(t, gc.Report{}, e.reconcile(false), "an allocation younger than W is not examined")+ require.NotContains(t, e.tombstones(), id)++ // A creation that outlives W: its ID is examined and gets a tombstone.+ e.ageAllocator(grace)+ require.Equal(t, gc.Report{LeaksFound: 1, LeaksTombstoned: 1}, e.reconcile(false))+ require.NoError(t, tx.Commit())++ e.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", e.past(grace+time.Minute), id)+ report, err := e.gc.CollectRemoved(e.ctx, "default", false)+ require.NoError(t, err)+ require.Zero(t, report.RemovedRepos, "a tombstone whose ID has a row is not a removal")+ require.Equal(t, []string{key}, e.objects(id))++ // Its deletion is a removal, with a grace from the deletion.+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", id)+ tombstone := e.tombstones()[id]+ require.Equal(t, "in-flight.git", tombstone.Key.RelativePath)+ require.WithinDuration(t, time.Now(), tombstone.RemovedAt, time.Minute)+ report, err = e.gc.CollectRemoved(e.ctx, "default", false)+ require.NoError(t, err)+ require.Zero(t, report.RemovedRepos)+ require.Equal(t, []string{key}, e.objects(id))+}diff --git a/internal/graft/corestore/maintenance/maintenance.go b/internal/graft/corestore/maintenance/maintenance.goindex 33f6b78..986666e 100644--- a/internal/graft/corestore/maintenance/maintenance.go+++ b/internal/graft/corestore/maintenance/maintenance.go@@ -40,6 +40,7 @@ const releaseTimeout = 10 * time.Second// Steps, the label of graft_storage_maintenance_failures_total.const (+ stepReconcile = "reconcile"stepCollectRemoved = "collect_removed"stepList = "list"stepLease = "lease"@@ -93,6 +94,9 @@ type Runner struct {bytes prometheus.Counterskipped *prometheus.CounterVecfailures *prometheus.CounterVec+ leaksFound prometheus.Counter+ leaksMarked prometheus.Counter+ ahead prometheus.Gauge}var _ prometheus.Collector = (*Runner)(nil)@@ -148,8 +152,20 @@ func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collec}, []string{"reason"}),failures: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "graft_storage_maintenance_failures_total",- Help: "Failed steps of the background maintenance, by step (collect_removed, list, lease, checkpoint, collect).",+ Help: "Failed steps of the background maintenance, by step (reconcile, collect_removed, list, lease, checkpoint, collect).",}, []string{"step"}),+ leaksFound: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "graft_storage_maintenance_leaks_found_total",+ Help: "Repository IDs whose objects are in the bucket without a row or a tombstone (a row deleted without one), found by the reconciliation.",+ }),+ leaksMarked: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "graft_storage_maintenance_leaks_tombstoned_total",+ Help: "Tombstones the reconciliation recorded for leaked repository IDs; their objects are collected after gc_grace.",+ }),+ ahead: prometheus.NewGauge(prometheus.GaugeOpts{+ Name: "graft_storage_maintenance_ids_ahead_of_allocator",+ Help: "Repository IDs above the database's allocator that have objects in the bucket, at the last reconciliation (a database restored from an older backup); never touched.",+ }),}for _, result := range []string{"ok", "error"} {r.runs.WithLabelValues(result)@@ -157,7 +173,7 @@ func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collecfor _, reason := range []string{"no_local_copy", "nothing_due", "lease_held"} {r.skipped.WithLabelValues(reason)}- for _, step := range []string{stepCollectRemoved, stepList, stepLease, stepCheckpoint, stepCollect} {+ for _, step := range []string{stepReconcile, stepCollectRemoved, stepList, stepLease, stepCheckpoint, stepCollect} {r.failures.WithLabelValues(step)}return r@@ -199,6 +215,16 @@ func (r *Runner) RunOnce(ctx context.Context) error {errs = append(errs, err)}+ if r.cfg.Collect {+ // Before the removed repositories are collected, so that a leak's+ // tombstone is among them once it is gc_grace old.+ report, err := r.collector.Reconcile(ctx, false)+ if err != nil {+ fail(stepReconcile, err, log.Fields{})+ }+ r.countReconcile(report)+ }+for _, storageName := range r.storages {if !r.cfg.Collect {break@@ -416,6 +442,13 @@ func (r *Runner) count(report gc.Report) {r.bytes.Add(float64(report.BytesReclaimed))}+// countReconcile records what a reconciliation found.+func (r *Runner) countReconcile(report gc.Report) {+ r.leaksFound.Add(float64(report.LeaksFound))+ r.leaksMarked.Add(float64(report.LeaksTombstoned))+ r.ahead.Set(float64(report.AheadOfAllocator))+}+// Describe implements prometheus.Collector.func (r *Runner) Describe(descs chan<- *prometheus.Desc) {r.runs.Describe(descs)@@ -424,6 +457,9 @@ func (r *Runner) Describe(descs chan<- *prometheus.Desc) {r.bytes.Describe(descs)r.skipped.Describe(descs)r.failures.Describe(descs)+ r.leaksFound.Describe(descs)+ r.leaksMarked.Describe(descs)+ r.ahead.Describe(descs)}// Collect implements prometheus.Collector.@@ -434,4 +470,7 @@ func (r *Runner) Collect(metrics chan<- prometheus.Metric) {r.bytes.Collect(metrics)r.skipped.Collect(metrics)r.failures.Collect(metrics)+ r.leaksFound.Collect(metrics)+ r.leaksMarked.Collect(metrics)+ r.ahead.Collect(metrics)}diff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.goindex 2ec7a3e..6ffce63 100644--- a/internal/graft/corestore/maintenance/maintenance_test.go+++ b/internal/graft/corestore/maintenance/maintenance_test.go@@ -6,6 +6,7 @@ import ("crypto/sha256""fmt""os"+ "strings""sync/atomic""testing""time"@@ -157,13 +158,14 @@ func (e *env) metric(name, label string) float64 {if family.GetName() != name {continue}+ // A metric is a counter or a gauge; the other value is zero.for _, m := range family.GetMetric() {if label == "" {- return m.GetCounter().GetValue()+ return m.GetCounter().GetValue() + m.GetGauge().GetValue()}for _, l := range m.GetLabel() {if l.GetValue() == label {- return m.GetCounter().GetValue()+ return m.GetCounter().GetValue() + m.GetGauge().GetValue()}}}@@ -171,6 +173,45 @@ func (e *env) metric(name, label string) float64 {return 0}+// exec runs a statement with "?" placeholders on the database of the env.+func (e *env) exec(query string, args ...any) {+ e.t.Helper()+ store := e.core.Meta.(*sqlstore.Store)+ if store.Dialect() == sqlstore.Postgres {+ for n := 1; strings.Contains(query, "?"); n++ {+ query = strings.Replace(query, "?", fmt.Sprintf("$%d", n), 1)+ }+ }+ _, err := store.DB().ExecContext(testhelper.Context(e.t), query, args...)+ require.NoError(e.t, err)+}++// past returns the database time value of d before now.+func (e *env) past(d time.Duration) any {+ t := time.Now().Add(-d).UTC()+ if e.core.Meta.(*sqlstore.Store).Dialect() == sqlstore.SQLite {+ return t.Format(time.RFC3339Nano)+ }+ return t+}++// The tests age rows on the database clock instead of waiting with a short+// gc_grace: the writers' deadline W is gc_grace - gc_margin, and a checkpoint+// built under the race detector on a loaded host does not fit in a W of a+// fraction of a second.++// ageCheckpoints makes every checkpoint of the repository older than+// gc_grace.+func (e *env) ageCheckpoints() {+ e.exec("UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ?", e.past(e.core.GCGrace+time.Minute), e.repo.ID)+}++// observedLongAgo records that a collection observed the object unreferenced+// more than gc_grace ago.+func (e *env) observedLongAgo(key string) {+ e.exec("INSERT INTO graft_gc_orphans (repo_id, pack_key, first_seen) VALUES (?, ?, ?)", e.repo.ID, key, e.past(e.core.GCGrace+time.Minute))+}+// config3 is a configuration with the checkpoints and the garbage collection// on, and the given checkpoint thresholds.func config3(changesets uint64, packBytes int64) maintenance.Config {@@ -187,7 +228,7 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {ctx := testhelper.Context(t)cfg := config3(1, 0)cfg.Collect = false- e := newEnv(t, coretest.Options{GCGrace: 200 * time.Millisecond, GCMargin: 50 * time.Millisecond}, cfg)+ e := newEnv(t, coretest.Options{}, cfg)var built []corestore.Checkpointfor i := 0; i < 3; i++ {@@ -204,10 +245,11 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {}newest := built[len(built)-1]- require.Eventually(t, func() bool {- require.NoError(t, e.runner.RunOnce(ctx))- return len(e.checkpoints()) == 1- }, 30*time.Second, 50*time.Millisecond, "the superseded checkpoints are dropped")+ require.NoError(t, e.runner.RunOnce(ctx))+ require.Len(t, e.checkpoints(), 3, "no checkpoint is gc_grace old yet")+ e.ageCheckpoints()+ require.NoError(t, e.runner.RunOnce(ctx))+ require.Len(t, e.checkpoints(), 1, "the superseded checkpoints are dropped")require.Equal(t, newest.Seq, e.checkpoints()[0].Seq)keys := e.bucketKeys()@@ -226,6 +268,39 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {require.Equal(t, before+1, e.metric("graft_storage_maintenance_skipped_total", "nothing_due"))}+// TestRunOnceReconcilesLeaks: with the collection on, a run reconciles the+// bucket before it collects the removed repositories, and its metrics count+// the leaks found and tombstoned and the IDs ahead of the allocator.+func TestRunOnceReconcilesLeaks(t *testing.T) {+ t.Parallel()+ ctx := testhelper.Context(t)+ e := newEnv(t, coretest.Options{}, config3(0, 0))+ put := func(repoID int64) {+ content := []byte(fmt.Sprintf("objects of %d", repoID))+ require.NoError(t, e.core.Blobs.Put(ctx, corestore.NewPackKey(repoID, sha256.Sum256(content)), bytes.NewReader(content)))+ }++ // A row deleted by hand before the trigger existed: no tombstone.+ leaked, _, err := e.core.Meta.RegisterRepository(ctx, corestore.RepoKey{Storage: "default", RelativePath: "leaked.git"}, "sha1", corestore.InitialState{})+ require.NoError(t, err)+ put(leaked.ID)+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", leaked.ID)+ e.exec("DELETE FROM graft_gc_removed WHERE repo_id = ?", leaked.ID)+ put(leaked.ID + 2)++ // The allocator was observed at its position longer than W ago.+ e.exec("INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)", e.past(e.core.WriteDeadline()+time.Minute), leaked.ID)++ require.NoError(t, e.runner.RunOnce(ctx))+ require.Equal(t, 1.0, e.metric("graft_storage_maintenance_leaks_found_total", ""))+ require.Equal(t, 1.0, e.metric("graft_storage_maintenance_leaks_tombstoned_total", ""))+ require.Equal(t, 1.0, e.metric("graft_storage_maintenance_ids_ahead_of_allocator", ""))+ removed, err := e.core.Meta.ListRemovedRepositories(ctx, "", 0)+ require.NoError(t, err)+ require.Len(t, removed, 1)+ require.Equal(t, leaked.ID, removed[0].RepoID)+}+// leaseExpiry returns the expiry of the repository's maintenance lease, which// holder must hold.func (e *env) leaseExpiry(holder string) time.Time {@@ -277,33 +352,33 @@ func TestRunOnceDoesOnlyWhatIsOn(t *testing.T) {require.NoError(t, e.core.Blobs.Put(ctx, key, bytes.NewReader(content)))return key}- opts := coretest.Options{GCGrace: 200 * time.Millisecond, GCMargin: 50 * time.Millisecond}+ exists := func(e *env, key string) bool {+ ok, err := e.core.Blobs.Exists(ctx, key)+ require.NoError(t, err)+ return ok+ }+ // An orphan that a collection would delete now: it was observed more than+ // gc_grace ago.checkpointsOnly := config3(1, 0)checkpointsOnly.Collect = false- e := newEnv(t, opts, checkpointsOnly)+ e := newEnv(t, coretest.Options{}, checkpointsOnly)e.fresh()key := orphan(e)- require.Never(t, func() bool {- require.NoError(t, e.runner.RunOnce(ctx))- exists, err := e.core.Blobs.Exists(ctx, key)- require.NoError(t, err)- return !exists- }, time.Second, 100*time.Millisecond, "checkpoints alone delete nothing")+ e.observedLongAgo(key)+ require.NoError(t, e.runner.RunOnce(ctx))+ require.True(t, exists(e, key), "checkpoints alone delete nothing")_, ok := e.newestCheckpoint()require.True(t, ok)collectOnly := config3(1, 0)collectOnly.Checkpoints = false- e = newEnv(t, opts, collectOnly)+ e = newEnv(t, coretest.Options{}, collectOnly)e.fresh()key = orphan(e)- require.Eventually(t, func() bool {- require.NoError(t, e.runner.RunOnce(ctx))- exists, err := e.core.Blobs.Exists(ctx, key)- require.NoError(t, err)- return !exists- }, 30*time.Second, 100*time.Millisecond)+ e.observedLongAgo(key)+ require.NoError(t, e.runner.RunOnce(ctx))+ require.False(t, exists(e, key), "the collection deletes it")_, ok = e.newestCheckpoint()require.False(t, ok, "the collection alone creates no checkpoint")}@@ -393,7 +468,7 @@ func TestRunOnceSkipsARepositoryWhoseLeaseIsHeld(t *testing.T) {func TestRunOnceCollectsAnOrphanPack(t *testing.T) {t.Parallel()ctx := testhelper.Context(t)- e := newEnv(t, coretest.Options{GCGrace: 200 * time.Millisecond, GCMargin: 50 * time.Millisecond}, config3(0, 0))+ e := newEnv(t, coretest.Options{}, config3(0, 0))e.fresh()content := []byte("an orphan pack")@@ -406,12 +481,11 @@ func TestRunOnceCollectsAnOrphanPack(t *testing.T) {require.NoError(t, err)require.True(t, exists, "observed, not yet deleted")- require.Eventually(t, func() bool {- require.NoError(t, e.runner.RunOnce(ctx))- exists, err := e.core.Blobs.Exists(ctx, key)- require.NoError(t, err)- return !exists- }, 30*time.Second, 100*time.Millisecond)+ e.exec("UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ?", e.past(e.core.GCGrace+time.Minute), e.repo.ID)+ require.NoError(t, e.runner.RunOnce(ctx))+ exists, err = e.core.Blobs.Exists(ctx, key)+ require.NoError(t, err)+ require.False(t, exists, "deleted gc_grace after its observation")require.Equal(t, float64(1), e.metric("graft_storage_maintenance_packs_deleted_total", ""))require.Equal(t, float64(len(content)), e.metric("graft_storage_maintenance_bytes_reclaimed_total", ""))}diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.goindex 4f7dbdf..f085511 100644--- a/internal/graft/corestore/metastore.go+++ b/internal/graft/corestore/metastore.go@@ -242,12 +242,31 @@ type MetaStore interface {ReleaseMaintenanceLease(ctx context.Context, repoID int64, holder string) error// ListRemovedRepositories returns the tombstones of the storage (of every- // storage when it is empty) recorded at least minAge ago on the database- // clock, ascending by repository ID.+ // storage when it is empty), and those of an unknown key (RecordLeak),+ // recorded at least minAge ago on the database clock, ascending by+ // repository ID. A tombstone whose ID has a row (a reconciled ID whose+ // creation committed after all) is not returned.ListRemovedRepositories(ctx context.Context, storage string, minAge time.Duration) ([]RemovedRepository, error)// DeleteRemovedRepository removes a tombstone; a missing one is success.DeleteRemovedRepository(ctx context.Context, repoID int64) error+ // ObserveAllocator reads the repository ID allocator and the+ // reconciliation cursor and, unless dryRun, records the observation+ // (AllocatorView.Now, Position) and drops the observations older than the+ // one Bound comes from, in one transaction (doc/graft/PLAN7.md, "Removed+ // repositories", reconciliation).+ ObserveAllocator(ctx context.Context, minAge time.Duration, dryRun bool) (AllocatorView, error)+ // UnrecordedIDs returns the repository IDs in (after, upTo] that have+ // neither a row nor a tombstone, ascending.+ UnrecordedIDs(ctx context.Context, after, upTo int64) ([]int64, error)+ // RecordLeak records a tombstone for repoID, with an unknown key (empty+ // storage and relative path) and the database time, unless the ID has a+ // row or a tombstone; it reports whether it recorded one.+ RecordLeak(ctx context.Context, repoID int64) (bool, error)+ // AdvanceReconcile moves the reconciliation cursor to doneThrough unless+ // it is there or further already.+ AdvanceReconcile(ctx context.Context, doneThrough int64) error+// Close releases the database.Close() error}diff --git a/internal/graft/corestore/sqlstore/maintenance.go b/internal/graft/corestore/sqlstore/maintenance.goindex 09e66c8..52d8abf 100644--- a/internal/graft/corestore/sqlstore/maintenance.go+++ b/internal/graft/corestore/sqlstore/maintenance.go@@ -172,13 +172,16 @@ func (s *Store) ListRemovedRepositories(ctx context.Context, storage string, minreturn err}- query := "SELECT repo_id, storage, relative_path, removed_at FROM graft_gc_removed"+ // A reconciled tombstone (RecordLeak) has an empty key: it belongs to+ // every storage's list. One whose ID has a row is not a removal.+ query := `SELECT g.repo_id, g.storage, g.relative_path, g.removed_at FROM graft_gc_removed g+ WHERE NOT EXISTS (SELECT 1 FROM graft_repositories r WHERE r.repo_id = g.repo_id)`var args []anyif storage != "" {- query += " WHERE storage = ?"+ query += " AND (g.storage = ? OR g.storage = '')"args = append(args, storage)}- query += " ORDER BY repo_id"+ query += " ORDER BY g.repo_id"rows, err := tx.QueryContext(ctx, s.bind(query), args...)if err != nil {diff --git a/internal/graft/corestore/sqlstore/migrations.go b/internal/graft/corestore/sqlstore/migrations.goindex 0576ed6..4a1481e 100644--- a/internal/graft/corestore/sqlstore/migrations.go+++ b/internal/graft/corestore/sqlstore/migrations.go@@ -147,6 +147,35 @@ var sqliteMigrations = []*migrate.Migration{`DROP TABLE graft_checkpoints`,},},+ {+ // Integrated review m4 (doc/graft/PLAN7.md, "Removed repositories"): every+ // deletion of a repository row records its tombstone, also one made by+ // hand, replacing a reconciled one of the same ID (its grace starts at+ // the deletion); and the state of the reconciliation that tombstones the+ // bucket prefixes of rows deleted without one before this migration.+ Id: "20261019000000_graft_storage_reconcile",+ Up: []string{+ `CREATE TRIGGER graft_repositories_tombstone AFTER DELETE ON graft_repositories+ BEGIN+ INSERT OR REPLACE INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)+ VALUES (OLD.repo_id, OLD.storage, OLD.relative_path, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'));+ END`,+ `CREATE TABLE graft_gc_allocator_marks (+ observed_at TEXT NOT NULL PRIMARY KEY,+ position INTEGER NOT NULL+ )`,+ `CREATE TABLE graft_gc_reconcile (+ id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),+ done_through INTEGER NOT NULL+ )`,+ `INSERT INTO graft_gc_reconcile (id, done_through) VALUES (1, 0)`,+ },+ Down: []string{+ `DROP TABLE graft_gc_reconcile`,+ `DROP TABLE graft_gc_allocator_marks`,+ `DROP TRIGGER graft_repositories_tombstone`,+ },+ },}// notOwnedPackKeyMessage is the error of the trigger that refuses a changeset@@ -294,4 +323,36 @@ var postgresMigrations = []*migrate.Migration{`DROP TABLE graft_checkpoints`,},},+ {+ // See the SQLite migration of the same ID.+ Id: "20261019000000_graft_storage_reconcile",+ Up: []string{+ `CREATE FUNCTION graft_repositories_tombstone() RETURNS trigger LANGUAGE plpgsql AS $$+ BEGIN+ INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)+ VALUES (OLD.repo_id, OLD.storage, OLD.relative_path, clock_timestamp())+ ON CONFLICT (repo_id) DO UPDATE SET storage = EXCLUDED.storage,+ relative_path = EXCLUDED.relative_path, removed_at = EXCLUDED.removed_at;+ RETURN OLD;+ END+ $$`,+ `CREATE TRIGGER graft_repositories_tombstone AFTER DELETE ON graft_repositories+ FOR EACH ROW EXECUTE FUNCTION graft_repositories_tombstone()`,+ `CREATE TABLE graft_gc_allocator_marks (+ observed_at TIMESTAMPTZ NOT NULL PRIMARY KEY,+ position BIGINT NOT NULL+ )`,+ `CREATE TABLE graft_gc_reconcile (+ id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),+ done_through BIGINT NOT NULL+ )`,+ `INSERT INTO graft_gc_reconcile (id, done_through) VALUES (1, 0)`,+ },+ Down: []string{+ `DROP TABLE graft_gc_reconcile`,+ `DROP TABLE graft_gc_allocator_marks`,+ `DROP TRIGGER graft_repositories_tombstone ON graft_repositories`,+ `DROP FUNCTION graft_repositories_tombstone()`,+ },+ },}diff --git a/internal/graft/corestore/sqlstore/reconcile.go b/internal/graft/corestore/sqlstore/reconcile.gonew file mode 100644index 0000000..794d8d9--- /dev/null+++ b/internal/graft/corestore/sqlstore/reconcile.go@@ -0,0 +1,212 @@+package sqlstore++import (+ "context"+ "database/sql"+ "fmt"+ "time"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+)++// allocatorPosition returns the greatest repository ID allocated, 0 for none.+// The Postgres sequence is not transactional: it counts the IDs of creations+// whose row is not committed yet. SQLite's is: an uncommitted creation's ID is+// not counted, and a rolled back one is allocated again.+func (s *Store) allocatorPosition(ctx context.Context, q querier) (int64, error) {+ query := "SELECT COALESCE((SELECT seq FROM sqlite_sequence WHERE name = 'graft_repositories'), 0)"+ if s.dialect == Postgres {+ query = "SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM graft_repositories_repo_id_seq"+ }+ var position int64+ if err := q.QueryRowContext(ctx, query).Scan(&position); err != nil {+ return 0, fmt.Errorf("read the repository ID allocator: %w", err)+ }+ return position, nil+}++// ObserveAllocator implements corestore.MetaStore.+func (s *Store) ObserveAllocator(ctx context.Context, minAge time.Duration, dryRun bool) (corestore.AllocatorView, error) {+ if minAge < 0 {+ return corestore.AllocatorView{}, fmt.Errorf("storage core: negative allocation age %s", minAge)+ }++ var view corestore.AllocatorView+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {+ view = corestore.AllocatorView{}++ now, err := s.wallTime(ctx, tx)+ if err != nil {+ return err+ }+ position, err := s.allocatorPosition(ctx, tx)+ if err != nil {+ return err+ }+ if err := tx.QueryRowContext(ctx, "SELECT done_through FROM graft_gc_reconcile WHERE id = 1").Scan(&view.DoneThrough); err != nil {+ return fmt.Errorf("read the reconciliation cursor: %w", err)+ }++ // The marks are few (the newest one at least minAge old and those+ // after it), and compared here: SQLite stores times as text whose+ // fractions do not sort.+ type mark struct {+ key any+ observedAt time.Time+ position int64+ }+ rows, err := tx.QueryContext(ctx, "SELECT observed_at, position FROM graft_gc_allocator_marks")+ if err != nil {+ return fmt.Errorf("read the allocator marks: %w", err)+ }+ var marks []mark+ for rows.Next() {+ var m mark+ if err := rows.Scan(&m.key, &m.position); err != nil {+ _ = rows.Close()+ return fmt.Errorf("scan an allocator mark: %w", err)+ }+ if m.observedAt, err = scanTime(m.key); err != nil {+ _ = rows.Close()+ return err+ }+ marks = append(marks, m)+ }+ if err := rows.Close(); err != nil {+ return fmt.Errorf("read the allocator marks: %w", err)+ }+ if err := rows.Err(); err != nil {+ return fmt.Errorf("read the allocator marks: %w", err)+ }++ limit := now.Add(-minAge)+ var newest *mark+ for i := range marks {+ m := &marks[i]+ if m.observedAt.After(limit) {+ continue+ }+ if m.position > view.Bound {+ view.Bound = m.position+ }+ if newest == nil || m.observedAt.After(newest.observedAt) {+ newest = m+ }+ }+ view.Now, view.Position = now, position+ if dryRun {+ return nil+ }++ if _, err := tx.ExecContext(ctx, s.bind(`+ INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)+ ON CONFLICT (observed_at) DO NOTHING+ `), s.timeArg(now), position); err != nil {+ return fmt.Errorf("record the allocator mark: %w", err)+ }+ // A mark older than the newest eligible one can never give a greater+ // bound: positions only grow.+ for _, m := range marks {+ if newest == nil || !m.observedAt.Before(newest.observedAt) {+ continue+ }+ if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_gc_allocator_marks WHERE observed_at = ?"), m.key); err != nil {+ return fmt.Errorf("drop an old allocator mark: %w", err)+ }+ }+ return nil+ })+ if err != nil {+ return corestore.AllocatorView{}, err+ }+ return view, nil+}++// UnrecordedIDs implements corestore.MetaStore.+func (s *Store) UnrecordedIDs(ctx context.Context, after, upTo int64) ([]int64, error) {+ if upTo <= after {+ return nil, nil+ }++ var ids []int64+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {+ ids = nil++ recorded := map[int64]struct{}{}+ for _, query := range []string{+ "SELECT repo_id FROM graft_repositories WHERE repo_id > ? AND repo_id <= ?",+ "SELECT repo_id FROM graft_gc_removed WHERE repo_id > ? AND repo_id <= ?",+ } {+ rows, err := tx.QueryContext(ctx, s.bind(query), after, upTo)+ if err != nil {+ return fmt.Errorf("read the recorded repository IDs: %w", err)+ }+ for rows.Next() {+ var id int64+ if err := rows.Scan(&id); err != nil {+ _ = rows.Close()+ return fmt.Errorf("scan a repository ID: %w", err)+ }+ recorded[id] = struct{}{}+ }+ if err := rows.Close(); err != nil {+ return fmt.Errorf("read the recorded repository IDs: %w", err)+ }+ if err := rows.Err(); err != nil {+ return fmt.Errorf("read the recorded repository IDs: %w", err)+ }+ }+ for id := after + 1; id <= upTo; id++ {+ if _, ok := recorded[id]; !ok {+ ids = append(ids, id)+ }+ }+ return nil+ })+ if err != nil {+ return nil, err+ }+ return ids, nil+}++// RecordLeak implements corestore.MetaStore.+func (s *Store) RecordLeak(ctx context.Context, repoID int64) (bool, error) {+ var recorded bool+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {+ recorded = false++ now, err := s.wallTime(ctx, tx)+ if err != nil {+ return err+ }+ res, err := tx.ExecContext(ctx, s.bind(`+ INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)+ SELECT ?, '', '', ? WHERE NOT EXISTS (SELECT 1 FROM graft_repositories WHERE repo_id = ?)+ ON CONFLICT (repo_id) DO NOTHING+ `), repoID, s.timeArg(now), repoID)+ if err != nil {+ return fmt.Errorf("record the tombstone of leaked repository %d: %w", repoID, err)+ }+ n, err := res.RowsAffected()+ if err != nil {+ return fmt.Errorf("record the tombstone of leaked repository %d: %w", repoID, err)+ }+ recorded = n == 1+ return nil+ })+ if err != nil {+ return false, err+ }+ return recorded, nil+}++// AdvanceReconcile implements corestore.MetaStore.+func (s *Store) AdvanceReconcile(ctx context.Context, doneThrough int64) error {+ return s.withTx(ctx, nil, func(tx *sql.Tx) error {+ if _, err := tx.ExecContext(ctx, s.bind("UPDATE graft_gc_reconcile SET done_through = ? WHERE id = 1 AND done_through < ?"),+ doneThrough, doneThrough); err != nil {+ return fmt.Errorf("advance the reconciliation cursor: %w", err)+ }+ return nil+ })+}diff --git a/internal/graft/corestore/sqlstore/repository.go b/internal/graft/corestore/sqlstore/repository.goindex d7a5707..ed65581 100644--- a/internal/graft/corestore/sqlstore/repository.go+++ b/internal/graft/corestore/sqlstore/repository.go@@ -442,24 +442,15 @@ func (s *Store) DeleteRepository(ctx context.Context, key corestore.RepoKey) (boreturn err}- now, err := s.wallTime(ctx, tx)- if err != nil {- return err- }-- // The cascade removes everything recorded for the repository; the- // tombstone, which has no foreign key, outlives it so that garbage- // collection deletes the repository's objects from the bucket- // (doc/graft/PLAN7.md, "Removed repositories").+ // The cascade removes everything recorded for the repository. The+ // trigger graft_repositories_tombstone records its tombstone, on the+ // database clock, in this transaction; the tombstone has no foreign key+ // and outlives the row, so that garbage collection deletes the+ // repository's objects from the bucket (doc/graft/PLAN7.md, "Removed+ // repositories"). A row deleted by hand gets one the same way.if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_repositories WHERE repo_id = ?"), repo.id); err != nil {return fmt.Errorf("delete repository: %w", err)}- if _, err := tx.ExecContext(ctx, s.bind(`- INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)- VALUES (?, ?, ?, ?)- `), repo.id, repo.key.Storage, repo.key.RelativePath, s.timeArg(now)); err != nil {- return fmt.Errorf("record the tombstone of repository %d: %w", repo.id, err)- }deleted = truereturn nildiff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.goindex 42a5d00..fa78bdc 100644--- a/internal/graft/corestore/types.go+++ b/internal/graft/corestore/types.go@@ -176,11 +176,31 @@ type Checkpoint struct {// are still to be deleted from the bucket.type RemovedRepository struct {RepoID int64- Key RepoKey+ // Key is empty for a tombstone that the reconciliation recorded: the row+ // was deleted without one, and its key with it.+ Key RepoKey// RemovedAt is a database time (doc/graft/PLAN7.md, CLOCK RULE).RemovedAt time.Time}+// AllocatorView is an observation of the repository ID allocator by the+// reconciliation of the bucket (MetaStore.ObserveAllocator).+type AllocatorView struct {+ // Now is the database time of the observation.+ Now time.Time+ // Position is the greatest repository ID allocated, 0 for none. On+ // Postgres it counts the allocations of creations whose row is not+ // committed yet, or never will be; on SQLite only committed ones.+ Position int64+ // Bound is the greatest position observed at least minAge before Now, 0+ // when there is none: every ID at or below it was allocated at least+ // minAge ago.+ Bound int64+ // DoneThrough is the reconciliation cursor: every ID at or below it is+ // reconciled.+ DoneThrough int64+}+// GCState is what garbage collection decides on for one repository, read in// one database snapshot (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 5).type GCState struct {diff --git a/internal/graft/e2e/storage_gc_test.go b/internal/graft/e2e/storage_gc_test.goindex 9f9ad89..bdc940b 100644--- a/internal/graft/e2e/storage_gc_test.go+++ b/internal/graft/e2e/storage_gc_test.go@@ -4,6 +4,7 @@ import ("bytes""crypto/sha256""fmt"+ "strings""testing""time"@@ -15,6 +16,33 @@ import ("github.com/feder-cr/git-ai/sdk")+// The tests age rows on the database clock instead of waiting with a short+// gc_grace: the writers' deadline W is gc_grace - gc_margin, and an agent's+// write under the race detector on a loaded host does not fit in a W of a+// second or two.++// exec runs a statement with "?" placeholders on the cluster's database.+func (n storageNode) exec(t *testing.T, query string, args ...any) {+ t.Helper()+ store := n.core.Meta.(*sqlstore.Store)+ if store.Dialect() == sqlstore.Postgres {+ for i := 1; strings.Contains(query, "?"); i++ {+ query = strings.Replace(query, "?", fmt.Sprintf("$%d", i), 1)+ }+ }+ _, err := store.DB().ExecContext(n.ctx, query, args...)+ require.NoError(t, err)+}++// longAgo returns the database time value of more than gc_grace before now.+func (n storageNode) longAgo() any {+ t := time.Now().Add(-n.core.GCGrace - time.Minute).UTC()+ if n.core.Meta.(*sqlstore.Store).Dialect() == sqlstore.SQLite {+ return t.Format(time.RFC3339Nano)+ }+ return t+}+// TestStorageGCAcrossNodes runs the compaction of milestone 3 on a cluster of// real servers sharing one database and one bucket (doc/graft/PLAN7.md): node// A checkpoints a repository written on two nodes, node B collects the packs@@ -26,12 +54,6 @@ func TestStorageGCAcrossNodes(t *testing.T) {forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {graft, tokens := workerGraft(1)cluster := newStorageCluster(t, dialect, corestore.ModeAuthoritative, unbatched(graft))- // A short grace, so that the test sees packs collected. The writers'- // deadline (grace - margin) stays long enough for an upload under load.- cluster.adjust = func(core *corestore.Core) {- core.GCGrace = 2 * time.Second- core.GCMargin = 200 * time.Millisecond- }a := cluster.addNode(t)b := cluster.addNode(t)@@ -58,13 +80,14 @@ func TestStorageGCAcrossNodes(t *testing.T) {require.Equal(t, a.record(t, rel).HeadSeq, cp.Sequence)// Node B collects once the checkpoint is gc_grace old.- var collected *sdk.StorageGC- require.Eventually(t, func() bool {- res, err := b.operator(t, rel).StorageGC(b.ctx, false)- require.NoError(t, err)- collected = res- return res.Boundary == cp.Sequence && res.CoveredDeleted > 0- }, 60*time.Second, 200*time.Millisecond)+ repoID := a.record(t, rel).ID+ collected, err := b.operator(t, rel).StorageGC(b.ctx, false)+ require.NoError(t, err)+ require.Zero(t, collected.Boundary, "the checkpoint is younger than gc_grace")+ a.exec(t, "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ?", a.longAgo(), repoID)+ collected, err = b.operator(t, rel).StorageGC(b.ctx, false)+ require.NoError(t, err)+ require.Equal(t, cp.Sequence, collected.Boundary)require.Equal(t, uint32(len(before)), collected.CoveredDeleted, "every changeset pack is covered by the checkpoint")inBucket := bucketPacks(t, cluster.bucketURL)for key := range before {@@ -100,27 +123,30 @@ func TestStorageGCAcrossNodes(t *testing.T) {// An orphan (the pack of a write that died before its database commit)// is deleted gc_grace after a collection observed it.orphan := []byte("the pack of a write that died")- orphanKey := corestore.NewPackKey(a.record(t, rel).ID, sha256.Sum256(orphan))+ orphanKey := corestore.NewPackKey(repoID, sha256.Sum256(orphan))require.NoError(t, a.core.Blobs.Put(a.ctx, orphanKey, bytes.NewReader(orphan)))- require.Eventually(t, func() bool {- res, err := a.operator(t, rel).StorageGC(a.ctx, false)- require.NoError(t, err)- return res.OrphansDeleted == 1- }, 60*time.Second, 200*time.Millisecond)+ res, err := a.operator(t, rel).StorageGC(a.ctx, false)+ require.NoError(t, err)+ require.Equal(t, uint32(1), res.OrphansObserved)+ a.exec(t, "UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ?", a.longAgo(), repoID)+ res, err = a.operator(t, rel).StorageGC(a.ctx, false)+ require.NoError(t, err)+ require.Equal(t, uint32(1), res.OrphansDeleted)require.False(t, bucketPacks(t, cluster.bucketURL)[orphanKey])// A removed repository's objects go once its tombstone is gc_grace old.- repoID := a.record(t, rel).ID_, err = gitalypb.NewRepositoryServiceClient(a.conn(t)).RemoveRepository(a.setupCtx, &gitalypb.RemoveRepositoryRequest{Repository: a.repository(rel)})require.NoError(t, err)client, err := sdk.Dial(a.ctx, a.cfg.SocketPath, sdk.WithStorage(a.cfg.Storages[0].Name), sdk.WithToken(sharedTestToken))require.NoError(t, err)t.Cleanup(func() { _ = client.Close() })- require.Eventually(t, func() bool {- res, err := client.StorageGCRemoved(a.ctx, "", false)- require.NoError(t, err)- return res.Repositories == 1- }, 60*time.Second, 200*time.Millisecond)+ removed, err := client.StorageGCRemoved(a.ctx, "", false)+ require.NoError(t, err)+ require.Zero(t, removed.Repositories, "the tombstone is younger than gc_grace")+ a.exec(t, "UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", a.longAgo(), repoID)+ removed, err = client.StorageGCRemoved(a.ctx, "", false)+ require.NoError(t, err)+ require.Equal(t, uint32(1), removed.Repositories)prefix := fmt.Sprintf("repos/%d/", repoID)for key := range bucketPacks(t, cluster.bucketURL) {require.NotContains(t, key, prefix, "an object of the removed repository is left")@@ -139,10 +165,6 @@ func TestStorageCheckpointBackfillsReviewRoots(t *testing.T) {forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {graft, tokens := workerGraft(1)cluster := newStorageCluster(t, dialect, corestore.ModeAuthoritative, unbatched(graft))- cluster.adjust = func(core *corestore.Core) {- core.GCGrace = 2 * time.Second- core.GCMargin = 200 * time.Millisecond- }a := cluster.addNode(t)rel := a.createRepository(t).GetRelativePath()@@ -174,11 +196,11 @@ func TestStorageCheckpointBackfillsReviewRoots(t *testing.T) {require.Equal(t, corestore.Target(base), snapshot[corestore.KeepAroundRef(base)], "the checkpoint holds the review base")covered := a.referencedPacks(t, rel)- require.Eventually(t, func() bool {- res, err := a.operator(t, rel).StorageGC(a.ctx, false)- require.NoError(t, err)- return res.Boundary == cp.Sequence && res.CoveredDeleted > 0- }, 60*time.Second, 200*time.Millisecond)+ a.exec(t, "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ?", a.longAgo(), repoID)+ res, err := a.operator(t, rel).StorageGC(a.ctx, false)+ require.NoError(t, err)+ require.Equal(t, cp.Sequence, res.Boundary)+ require.Equal(t, uint32(len(covered)), res.CoveredDeleted)inBucket := bucketPacks(t, cluster.bucketURL)for key := range covered {require.False(t, inBucket[key], "covered pack %s deleted", key)diff --git a/proto/go/gitalypb/storage_core.pb.go b/proto/go/gitalypb/storage_core.pb.goindex 3623dd1..d557dbe 100644--- a/proto/go/gitalypb/storage_core.pb.go+++ b/proto/go/gitalypb/storage_core.pb.go@@ -781,9 +781,20 @@ type CollectRemovedRepositoriesResponse struct {// bytes_reclaimed is the size of those objects.BytesReclaimed int64 `protobuf:"varint,3,opt,name=bytes_reclaimed,json=bytesReclaimed,proto3" json:"bytes_reclaimed,omitempty"`// dry_run tells that nothing was deleted or recorded.- DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`- unknownFields protoimpl.UnknownFields- sizeCache protoimpl.SizeCache+ DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`+ // leaks_found is the number of repository IDs the reconciliation found with+ // objects in the bucket and neither a row nor a tombstone (a row deleted+ // without one).+ LeaksFound uint32 `protobuf:"varint,5,opt,name=leaks_found,json=leaksFound,proto3" json:"leaks_found,omitempty"`+ // leaks_tombstoned is the number of tombstones the reconciliation recorded+ // for them; their objects are deleted once the tombstone is gc_grace old.+ LeaksTombstoned uint32 `protobuf:"varint,6,opt,name=leaks_tombstoned,json=leaksTombstoned,proto3" json:"leaks_tombstoned,omitempty"`+ // ids_ahead_of_allocator is the number of repository IDs above the+ // database's allocator that have objects in the bucket (a database restored+ // from an older backup). They are never touched.+ IdsAheadOfAllocator uint32 `protobuf:"varint,7,opt,name=ids_ahead_of_allocator,json=idsAheadOfAllocator,proto3" json:"ids_ahead_of_allocator,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache}func (x *CollectRemovedRepositoriesResponse) Reset() {@@ -844,6 +855,27 @@ func (x *CollectRemovedRepositoriesResponse) GetDryRun() bool {return false}+func (x *CollectRemovedRepositoriesResponse) GetLeaksFound() uint32 {+ if x != nil {+ return x.LeaksFound+ }+ return 0+}++func (x *CollectRemovedRepositoriesResponse) GetLeaksTombstoned() uint32 {+ if x != nil {+ return x.LeaksTombstoned+ }+ return 0+}++func (x *CollectRemovedRepositoriesResponse) GetIdsAheadOfAllocator() uint32 {+ if x != nil {+ return x.IdsAheadOfAllocator+ }+ return 0+}+// ReferenceDivergence is a reference whose local value differs from the// database.type VerifyRepositoryResponse_ReferenceDivergence struct {@@ -983,12 +1015,16 @@ const file_storage_core_proto_rawDesc = "" +"\adry_run\x18\t \x01(\bR\x06dryRun\"e\n" +"!CollectRemovedRepositoriesRequest\x12'\n" +"\fstorage_name\x18\x01 \x01(\tB\x04\x88\xc6,\x01R\vstorageName\x12\x17\n" +- "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xaf\x01\n" ++ "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xb0\x02\n" +"\"CollectRemovedRepositoriesResponse\x12\"\n" +"\frepositories\x18\x01 \x01(\rR\frepositories\x12#\n" +"\rpacks_deleted\x18\x02 \x01(\rR\fpacksDeleted\x12'\n" +"\x0fbytes_reclaimed\x18\x03 \x01(\x03R\x0ebytesReclaimed\x12\x17\n" +- "\adry_run\x18\x04 \x01(\bR\x06dryRun2\xe5\x04\n" ++ "\adry_run\x18\x04 \x01(\bR\x06dryRun\x12\x1f\n" ++ "\vleaks_found\x18\x05 \x01(\rR\n" ++ "leaksFound\x12)\n" ++ "\x10leaks_tombstoned\x18\x06 \x01(\rR\x0fleaksTombstoned\x123\n" ++ "\x16ids_ahead_of_allocator\x18\a \x01(\rR\x13idsAheadOfAllocator2\xe5\x04\n" +"\x12StorageCoreService\x12]\n" +"\x10ImportRepository\x12\x1f.gitaly.ImportRepositoryRequest\x1a .gitaly.ImportRepositoryResponse\"\x06\xfa\x97(\x02\b\x01\x12Z\n" +"\x0fGetStorageState\x12\x1e.gitaly.GetStorageStateRequest\x1a\x1f.gitaly.GetStorageStateResponse\"\x06\xfa\x97(\x02\b\x02\x12]\n" +diff --git a/proto/go/gitalypb/storage_core_grpc.pb.go b/proto/go/gitalypb/storage_core_grpc.pb.goindex 461fa24..0bfea99 100644--- a/proto/go/gitalypb/storage_core_grpc.pb.go+++ b/proto/go/gitalypb/storage_core_grpc.pb.go@@ -67,8 +67,10 @@ type StorageCoreServiceClient interface {// or while another maintenance of the repository holds its lease.CollectGarbage(ctx context.Context, in *CollectGarbageRequest, opts ...grpc.CallOption) (*CollectGarbageResponse, error)// CollectRemovedRepositories deletes every object of the repositories of a- // storage that were removed at least gc_grace ago. It fails with- // FailedPrecondition when the storage core is off.+ // storage that were removed at least gc_grace ago. It first reconciles the+ // bucket: a repository ID with objects and neither a row nor a tombstone+ // gets a tombstone. It fails with FailedPrecondition when the storage core+ // is off.CollectRemovedRepositories(ctx context.Context, in *CollectRemovedRepositoriesRequest, opts ...grpc.CallOption) (*CollectRemovedRepositoriesResponse, error)}@@ -180,8 +182,10 @@ type StorageCoreServiceServer interface {// or while another maintenance of the repository holds its lease.CollectGarbage(context.Context, *CollectGarbageRequest) (*CollectGarbageResponse, error)// CollectRemovedRepositories deletes every object of the repositories of a- // storage that were removed at least gc_grace ago. It fails with- // FailedPrecondition when the storage core is off.+ // storage that were removed at least gc_grace ago. It first reconciles the+ // bucket: a repository ID with objects and neither a row nor a tombstone+ // gets a tombstone. It fails with FailedPrecondition when the storage core+ // is off.CollectRemovedRepositories(context.Context, *CollectRemovedRepositoriesRequest) (*CollectRemovedRepositoriesResponse, error)mustEmbedUnimplementedStorageCoreServiceServer()}diff --git a/proto/storage_core.proto b/proto/storage_core.protoindex 50a1c24..5830b58 100644--- a/proto/storage_core.proto+++ b/proto/storage_core.proto@@ -69,8 +69,10 @@ service StorageCoreService {}// CollectRemovedRepositories deletes every object of the repositories of a- // storage that were removed at least gc_grace ago. It fails with- // FailedPrecondition when the storage core is off.+ // storage that were removed at least gc_grace ago. It first reconciles the+ // bucket: a repository ID with objects and neither a row nor a tombstone+ // gets a tombstone. It fails with FailedPrecondition when the storage core+ // is off.rpc CollectRemovedRepositories(CollectRemovedRepositoriesRequest) returns (CollectRemovedRepositoriesResponse) {option (op_type) = {op: MUTATOR@@ -229,4 +231,15 @@ message CollectRemovedRepositoriesResponse {int64 bytes_reclaimed = 3;// dry_run tells that nothing was deleted or recorded.bool dry_run = 4;+ // leaks_found is the number of repository IDs the reconciliation found with+ // objects in the bucket and neither a row nor a tombstone (a row deleted+ // without one).+ uint32 leaks_found = 5;+ // leaks_tombstoned is the number of tombstones the reconciliation recorded+ // for them; their objects are deleted once the tombstone is gc_grace old.+ uint32 leaks_tombstoned = 6;+ // ids_ahead_of_allocator is the number of repository IDs above the+ // database's allocator that have objects in the bucket (a database restored+ // from an older backup). They are never touched.+ uint32 ids_ahead_of_allocator = 7;}diff --git a/sdk/storage.go b/sdk/storage.goindex 8e0ddd8..624bc1f 100644--- a/sdk/storage.go+++ b/sdk/storage.go@@ -196,6 +196,16 @@ type StorageGCRemoved struct {BytesReclaimed int64 `json:"bytes_reclaimed"`// DryRun tells that nothing was deleted or recorded.DryRun bool `json:"dry_run"`+ // LeaksFound is the number of repository IDs the reconciliation found with+ // objects in the bucket and neither a row nor a tombstone.+ LeaksFound uint32 `json:"leaks_found"`+ // LeaksTombstoned is the number of tombstones it recorded for them; their+ // objects are deleted once the tombstone is gc_grace old.+ LeaksTombstoned uint32 `json:"leaks_tombstoned"`+ // IDsAheadOfAllocator is the number of repository IDs above the+ // database's allocator that have objects in the bucket (a database+ // restored from an older backup); they are never touched.+ IDsAheadOfAllocator uint32 `json:"ids_ahead_of_allocator"`}// StorageCheckpoint checkpoints the repository at its current database head:@@ -244,7 +254,9 @@ func (r *Repo) StorageGC(ctx context.Context, dryRun bool) (*StorageGC, error) {}// StorageGCRemoved deletes every object of the repositories of the storage-// that were removed at least gc_grace ago (with dryRun, only reports them). An+// that were removed at least gc_grace ago (with dryRun, only reports them),+// after it reconciled the bucket (a repository ID with objects and neither a+// row nor a tombstone gets a tombstone). An// empty storage is the client's storage (WithStorage). It fails with// FailedPrecondition when the storage core is off.func (c *Client) StorageGCRemoved(ctx context.Context, storage string, dryRun bool) (*StorageGCRemoved, error) {@@ -259,10 +271,13 @@ func (c *Client) StorageGCRemoved(ctx context.Context, storage string, dryRun boreturn nil, err}return &StorageGCRemoved{- Repositories: resp.GetRepositories(),- PacksDeleted: resp.GetPacksDeleted(),- BytesReclaimed: resp.GetBytesReclaimed(),- DryRun: resp.GetDryRun(),+ Repositories: resp.GetRepositories(),+ PacksDeleted: resp.GetPacksDeleted(),+ BytesReclaimed: resp.GetBytesReclaimed(),+ DryRun: resp.GetDryRun(),+ LeaksFound: resp.GetLeaksFound(),+ LeaksTombstoned: resp.GetLeaksTombstoned(),+ IDsAheadOfAllocator: resp.GetIdsAheadOfAllocator(),}, nil}diff --git a/sdk/storage_test.go b/sdk/storage_test.goindex 0ebc310..6ef140e 100644--- a/sdk/storage_test.go+++ b/sdk/storage_test.go@@ -156,9 +156,12 @@ func TestStorageGC(t *testing.T) {func TestStorageGCRemoved(t *testing.T) {fake := &fakeStorageCore{removedResp: &gitalypb.CollectRemovedRepositoriesResponse{- Repositories: 2,- PacksDeleted: 9,- BytesReclaimed: 4096,+ Repositories: 2,+ PacksDeleted: 9,+ BytesReclaimed: 4096,+ LeaksFound: 3,+ LeaksTombstoned: 2,+ IdsAheadOfAllocator: 1,}}client := dialFake(t, func(srv *grpc.Server) {gitalypb.RegisterStorageCoreServiceServer(srv, fake)@@ -166,7 +169,10 @@ func TestStorageGCRemoved(t *testing.T) {res, err := client.StorageGCRemoved(t.Context(), "", false)require.NoError(t, err)- require.Equal(t, &sdk.StorageGCRemoved{Repositories: 2, PacksDeleted: 9, BytesReclaimed: 4096}, res)+ require.Equal(t, &sdk.StorageGCRemoved{+ Repositories: 2, PacksDeleted: 9, BytesReclaimed: 4096,+ LeaksFound: 3, LeaksTombstoned: 2, IDsAheadOfAllocator: 1,+ }, res)_, err = client.StorageGCRemoved(t.Context(), "other", true)require.NoError(t, err)