Skip to content

Git designed for AI agents.

Review m4: every deleted row gets a tombstone; reconcile the old leaks

Federico Elia0b32470b743b
diff --git a/cmd/graft/storage.go b/cmd/graft/storage.go
index 19ab142..b54a552 100644
--- a/cmd/graft/storage.go
+++ b/cmd/graft/storage.go
@@ -183,6 +183,18 @@ func storageGCRemoved(ctx context.Context, out io.Writer, o *options, f *storage
}
fmt.Fprintf(out, "%s %d objects (%d bytes) of %d removed repositories of storage %s\n",
verb, res.PacksDeleted, res.BytesReclaimed, res.Repositories, displayText(client.Storage()))
+ if res.LeaksFound > 0 {
+ recorded := "recorded"
+ if res.DryRun {
+ recorded = "would record"
+ }
+ fmt.Fprintf(out, "found %d repository IDs with objects and no row or tombstone; %s %d tombstones (their objects are deleted after gc_grace)\n",
+ res.LeaksFound, recorded, res.LeaksTombstoned)
+ }
+ if res.IDsAheadOfAllocator > 0 {
+ fmt.Fprintf(out, "warning: %d repository IDs above the database's allocator have objects in the bucket (a database restored from an older backup?); they are left alone\n",
+ res.IDsAheadOfAllocator)
+ }
return nil
}
diff --git a/cmd/graft/storage_test.go b/cmd/graft/storage_test.go
index 633c350..e2d6006 100644
--- a/cmd/graft/storage_test.go
+++ b/cmd/graft/storage_test.go
@@ -1,6 +1,7 @@
package main
import (
+ "crypto/sha256"
"encoding/json"
"path/filepath"
"strings"
@@ -144,6 +145,20 @@ func TestCLIStorageMaintenanceAgainstServer(t *testing.T) {
code, stdout, stderr = runCLI(t, append([]string{"storage", "gc", "--removed"}, server...)...)
require.Equal(t, exitOK, code, stderr)
require.Equal(t, "deleted 0 objects (0 bytes) of 0 removed repositories of storage "+repo.GetStorageName()+"\n", stdout)
+
+ // Objects of a repository ID the database has not allocated (a database
+ // restored from an older backup) are reported, never touched.
+ record, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repo))
+ require.NoError(t, err)
+ ahead := corestore.NewPackKey(record.ID+2, sha256.Sum256([]byte("ahead")))
+ require.NoError(t, core.Blobs.Put(ctx, ahead, strings.NewReader("ahead")))
+ code, stdout, stderr = runCLI(t, append([]string{"storage", "gc", "--removed"}, server...)...)
+ require.Equal(t, exitOK, code, stderr)
+ require.Equal(t, "deleted 0 objects (0 bytes) of 0 removed repositories of storage "+repo.GetStorageName()+"\n"+
+ "warning: 1 repository IDs above the database's allocator have objects in the bucket (a database restored from an older backup?); they are left alone\n", stdout)
+ exists, err := core.Blobs.Exists(ctx, ahead)
+ require.NoError(t, err)
+ require.True(t, exists)
}
// TestCLIStorageCoreOff checks that a server without a storage core reports
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index 37e976b..72877b7 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -482,43 +482,112 @@ guarded nor swept).
blocked every checkpoint. It is removed with its errors, metric, tests and
`gitops.IsReachable`. `CheckpointInput` has no root set, and the checkpoint
build reads no reviews.
-* **LOCK ORDER (PM check 3).** Every transaction that must serialize with a
- changeset, or with a review write that requires its roots, takes the
- repository row **as its first statement**, before it reads or writes any
- child table. On Postgres that is `SELECT ... FOR NO KEY UPDATE`; SQLite's
- immediate transaction holds the database write lock from `BEGIN`. The
- transactions: `CommitChangeset`, `RegisterRepository`, `ImportHistory`,
- `InsertReview`, `UpdateReview` with `requireRoots`, and the lease methods.
- * Transactions that touch only child rows without changing their `repo_id`
- never lock the repository row and never upgrade a lock, so they cannot form
- a cycle with the first group: `UpdateReview` without `requireRoots`,
- `ReportApplied`, the orphan observations, `DeleteCheckpoint`, and since the
- integrated review `CreateCheckpoint`.
- * Insert-only child transactions take Postgres's implicit `KEY SHARE` on the
- parent and nothing after it. `FOR NO KEY UPDATE` does not conflict with
- `KEY SHARE`, so they never hold up a writer. Deleting the repository still
- waits for them.
- * **Changed by the integrated review (M4).** `CreateCheckpoint` used to lock
- the row and insert its snapshot one statement per reference. At 100k
- references on Postgres a writer of the repository waited 38–46 s, locally.
- * It now reads the row without locking it, and writes the snapshot 1000
- references per statement. Nothing in it needs the writers excluded:
- * the head only grows;
- * the primary key makes one checkpoint per sequence: a concurrent build of
- the same sequence inserts nothing (`ON CONFLICT DO NOTHING`) and returns
- the one that committed;
- * a checkpoint below a newer one is never the newest, so no seed reads it.
- * Measured at 100k references (`TestCheckpointOfManyReferences`):
- * Postgres: the record takes 1.6–1.7 s, while about 500 changesets commit,
- the longest in 15–20 ms;
- * SQLite: 0.23–0.27 s, before 0.63–0.70 s. SQLite has one write lock for
- the whole database, so a writer waits for that. Tested with `-race` on Postgres and SQLite
- (`TestLockOrderConcurrentWriters`): concurrent lands, review creations and
- head moves (requiring their roots), a sweeper that deletes keep-around
- references until the writers are done, comments, checkpoint commits and
- leases on one repository. The test checks that no deadlock surfaces (the
- retry hook counts them, and the count must stay at zero), the invariant at
- the end, and dense sequences.
+* **LOCK ORDER (PM check 3; self-contained, as of the integrated review).**
+ *What it guarantees.* The write transactions of the storage core on one
+ repository never deadlock with each other, except the one case named below,
+ which Postgres detects and the store retries. The rule also gives the
+ serialisation that the invariants need: a changeset, a review write that
+ requires its roots, and the conditions checked with them (the review roots,
+ `max_open_reviews`) see each other in a total order.
+
+ *Background.* The database has one row per repository (`graft_repositories`,
+ "the row") and child tables keyed by `repo_id` (references, changesets,
+ changelog, reviews, nodes, checkpoints and their references, orphan
+ observations, leases), all with `ON DELETE CASCADE` foreign keys. SQLite has
+ one write lock for the whole database, taken at `BEGIN IMMEDIATE`: its write
+ transactions run one at a time and cannot deadlock. The rest is about
+ Postgres, where locks are per row. There, an insert of a child row takes
+ `FOR KEY SHARE` on the parent row (the foreign key check) until the
+ transaction ends; `FOR NO KEY UPDATE` does not conflict with `FOR KEY SHARE`;
+ a `DELETE` of the row takes `FOR UPDATE`, which does.
+
+ *The rule.*
+ 1. **Row-first transactions.** A transaction that must serialise with the
+ writers of the repository locks the row **as its first statement**, before
+ it reads or writes a child table: `SELECT ... FOR NO KEY UPDATE` (Postgres;
+ on SQLite the write lock is already held). These are `CommitChangeset`,
+ `RegisterRepository` (it inserts the row), `ImportHistory`, `InsertReview`
+ (always; it allocates the review ID, checks the roots when required and
+ counts the open reviews for `max_open_reviews`), `UpdateReview` with
+ `requireRoots`, the three lease methods, and `DeleteRepository` (which
+ then deletes the row: the only upgrade, to `FOR UPDATE`, of a lock it
+ already holds; the trigger `graft_repositories_tombstone` inserts the
+ tombstone, a table without a foreign key).
+ 2. **Child-only transactions.** Every other write transaction never locks the
+ row explicitly and never changes a `repo_id`:
+ * insert-only: `CreateCheckpoint` (since M4, below), `RecordOrphans`,
+ `ReportApplied` (an upsert), and the reconciliation's `RecordLeak`
+ (`graft_gc_removed` has no foreign key);
+ * update or delete of child rows only: `UpdateReview` without
+ `requireRoots` (comments, verdicts, closes; one row, with a version
+ check), `ForgetOrphans`, `DeleteCheckpoint` (one checkpoint row, whose
+ references cascade), `DeleteRemovedRepository`;
+ * tables of their own: `ObserveAllocator` and `AdvanceReconcile`
+ (`graft_gc_allocator_marks`, the one row of `graft_gc_reconcile`).
+
+ *Why no cycle.* A cycle needs every transaction in it to hold a lock while it
+ waits for the next. Take the waits one by one:
+ * A row-first transaction waits for the row before it holds anything of the
+ repository, so that wait closes no cycle. Holding the row, it can wait only
+ for a child row that another transaction holds.
+ * The row-first transactions that hold the row run one at a time, so the
+ child row it waits for is held by a child-only transaction.
+ * A child-only transaction waits on the row only at its first insert (the
+ key share), while it holds nothing yet: no cycle through that wait. Its
+ other waits are for child rows.
+ * The child-only transactions that hold one child row and wait for another
+ are the multi-row ones, and each kind touches rows nobody else holds at the
+ same time: `ForgetOrphans` and `RecordOrphans` run in a collection, which
+ holds the repository's maintenance lease, so one at a time per repository;
+ `CreateCheckpoint` inserts rows of its own sequence (a concurrent build of
+ the same sequence waits on the primary key and then does nothing).
+ * The single-row ones (`UpdateReview` without roots, `DeleteCheckpoint`'s
+ checkpoint row, `ReportApplied`) wait for nothing once they hold their row.
+ So a chain of waits never comes back to its first transaction, except
+ through the cascade of a repository removal, below.
+
+ *The exception: removing a repository.* `DeleteRepository` holds the row
+ `FOR UPDATE` and its cascade deletes child rows in the database's order. A
+ child-only transaction that deletes several child rows in its own order
+ (`ForgetOrphans`) can then hold one row the cascade needs while waiting for
+ another the cascade holds. Postgres detects the cycle after
+ `deadlock_timeout` and aborts one transaction with SQLSTATE 40P01, which the
+ store retries like a serialisation failure (`isRetryable`, `withTx`): the
+ outcome is that of a serial order (the repository is gone, or the
+ observations were forgotten first). This costs a second at worst, only while
+ a repository is removed during its collection, and changes no result.
+
+ *Changed by the integrated review (M4).* `CreateCheckpoint` used to be
+ row-first, and inserted its snapshot one statement per reference: at 100k
+ references on Postgres a writer of the repository waited 38–46 s, locally.
+ It now reads the row without locking it and writes the snapshot 1000
+ references per statement. Nothing in it needs the writers excluded:
+ * the head only grows, so a sequence at or below the head read stays so;
+ * the primary key makes one checkpoint per sequence: a concurrent build of
+ the same sequence inserts nothing (`ON CONFLICT DO NOTHING`) and returns the
+ one that committed, and a newer checkpoint covers the request and is
+ returned too (m7);
+ * a checkpoint below a newer one is never the newest, so no seed reads it;
+ * a concurrent removal of the repository either waits for it (its key share)
+ or makes its first insert fail on the foreign key: nothing is recorded,
+ and its pack is an orphan of a removed repository, collected with it;
+ * the pack's write deadline is checked in the same transaction on the
+ database clock (`PackDeadline`, m6), as for a changeset.
+ Measured at 100k references (`TestCheckpointOfManyReferences`): on Postgres
+ the record takes 1.6–1.7 s while about 500 changesets commit, the longest in
+ 15–20 ms; on SQLite 0.23–0.27 s (before: 0.63–0.70 s), during which writers
+ wait for SQLite's one write lock.
+
+ *Tests.* `TestLockOrderConcurrentWriters`, with `-race` on Postgres and
+ SQLite: concurrent lands, review creations and head moves (requiring their
+ roots), a sweeper that deletes keep-around references until the writers are
+ done, comments, checkpoint commits and leases on one repository. It checks
+ that no deadlock surfaces (the retry hook counts them, and the count must stay
+ at zero), the invariant at the end, and dense sequences.
+ `TestConcurrentCheckpointsOfOneSequence` (six concurrent records of one
+ sequence), `TestCheckpointOfManyReferences` (writers during a large record),
+ `TestCreateIsBoundedByTheOpenReviews` (concurrent creates against the
+ limit, two handles).
* **Files.** `internal/graft/corestore/reviewroots/**` (Protect, Sweep,
Backfill), `internal/gitaly/service/review/**` (the roots writes and the
close release), `internal/graft/corestore/dbreview/**`
@@ -1448,6 +1517,78 @@ prefix after the sweep (the PACK OWNERSHIP RULE makes this an enforced bound, no
an assumption). A crash after a partial prefix deletion leaves the tombstone; the
next run finishes it. No lease (the repository is gone and its ID is never reused).
+**Every deletion has a tombstone (integrated review m4, approved by the PM).**
+Before migration `20261019000000_graft_storage_reconcile`, only `DeleteRepository`
+wrote the tombstone; a row deleted otherwise (the repair procedure of STORAGE.md,
+"State and verify", deletes it by hand) left its prefix in the bucket forever.
+The migration adds the trigger `graft_repositories_tombstone`: `AFTER DELETE` on
+`graft_repositories`, it records the tombstone of the deleted row on the database
+clock (`clock_timestamp()`; SQLite's `'now'`, the host clock that SQLite's
+`wallTime` reads too), in the deleting transaction, whoever deletes.
+`DeleteRepository` relies on it. On a tombstone of the same ID (a reconciled one,
+below) the trigger replaces it: the grace of a removal starts at the removal.
+
+**Reconciliation of the leaks from before the migration.**
+`gc.Collector.Reconcile` records a tombstone with an empty key (`RecordLeak`) for
+a repository ID that has objects under `repos/<id>/` and neither a row nor a
+tombstone; `CollectRemoved` then treats it like any removal
+(`ListRemovedRepositories` returns the keyless tombstones in every storage's
+list). It runs in the background maintenance before `CollectRemoved` (with `gc =
+"auto"`) and in `CollectRemovedRepositories` (`graft storage gc --removed`).
+* *Which IDs.* The allocator (Postgres sequence, SQLite `sqlite_sequence`) has
+ no time per ID, so every run records its position with the database time
+ (`graft_gc_allocator_marks`, `ObserveAllocator`), and examines only IDs at or
+ below the greatest position observed at least W ago (`Bound`): IDs allocated
+ at least W ago. It keeps the newest such mark and those after it.
+* *Proof that a creation in flight is never collected.* Invariant U: an object
+ under `repos/<id>/` is written only by `Core.UploadPack(record, readAt, …)`,
+ whose record is a read that found the row of `id` (RULE 26: the only uploader;
+ `Blobs.Put` has no other caller outside tests); so a creation
+ writes nothing before its row commits, and its prefix is empty: the
+ reconciliation, which tombstones only a prefix with objects, leaves it alone.
+ Two more barriers, for a writer that would break U: (1) an ID younger than W
+ is not examined (the PM's guard; on Postgres the sequence counts an
+ allocation before its row commits, on SQLite an uncommitted allocation is
+ invisible and the single write lock serialises it with the reconciliation);
+ (2) a tombstone whose ID has a row is not a removal: `ListRemovedRepositories`
+ excludes it, so a creation that commits after it was tombstoned keeps its
+ objects, and its later deletion replaces the tombstone with a fresh grace.
+ Test: `TestReconcileNeverCollectsACreationInFlight` holds a Postgres
+ transaction between the allocation and the commit of a row, with an object
+ under its prefix, across reconciliations: not examined while younger than W;
+ examined after, tombstoned, then committed, and not collected.
+* *Safety of a tombstoned leak.* No row for the ID now, and objects under its
+ prefix: by U the row was committed once and has been deleted (IDs are never
+ reused: Postgres `BIGSERIAL`, SQLite `AUTOINCREMENT`). A writer that read the
+ row before the deletion uploads until `t_r + W`; the tombstone's `removed_at`
+ is after the deletion, so the collection, `gc_grace > W` later, comes after
+ every such upload, as for any removal.
+* *Bounded, paged, idempotent, resumable.* A run examines at most 256 IDs after
+ its cursor (`graft_gc_reconcile.done_through`, `UnrecordedIDs`), reads at most
+ one object of each prefix (one page of a listing, stopped at the first
+ object), records tombstones with "do nothing on conflict" and then moves the
+ cursor past the batch. A run that stopped leaves the cursor, and the next run
+ examines the batch again, recording nothing twice. With the trigger, a new
+ ID never needs a second look: its deletion has its tombstone. After the
+ backlog, a run examines only the IDs allocated since the last one.
+* *IDs above the allocator* (a database restored from an older backup: the
+ allocator will hand those IDs out again) are probed, 16 above the position,
+ counted and logged, never touched.
+* Metrics: `graft_storage_maintenance_leaks_found_total`,
+ `graft_storage_maintenance_leaks_tombstoned_total`,
+ `graft_storage_maintenance_ids_ahead_of_allocator`, and the step
+ `reconcile` of `graft_storage_maintenance_failures_total`; the RPC returns
+ `leaks_found`, `leaks_tombstoned`, `ids_ahead_of_allocator`.
+* Why not an operator RPC that lists the bucket: the leaks then depend on an
+ operator remembering to run it, and a listing of the whole bucket costs O(all
+ objects) on every run. Why not a full listing in the background: the same
+ cost, every interval. The cursor makes the cost O(new IDs) after the backlog.
+* Tests: `TestDeletedRowGetsATombstone`, `TestReconcileTombstonesLeaks`,
+ `TestReconcileIsPagedAndResumable`, `TestReconcileNeverCollectsACreationInFlight`
+ (both dialects where the dialect allows; mutants without the W guard or
+ without the row exclusion fail them), `TestRunOnceReconcilesLeaks`
+ (metrics), `TestCollectRemovedRepositoriesReconcilesLeaks` (RPC).
+
### Hydration from a checkpoint (syncer, WP76)
`hydrate` seeds from `NewestCheckpoint`:
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index 86f6b5c..a8ce3c6 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -350,7 +350,11 @@ AND relative_path = '<relative path>'`; its changesets, references, changelog
and reviews go with it) and run `graft storage import` again. The new
registration gets a new repository ID and a full pack, and takes the changelog
and the reviews from the files again, so do this before switching to
-authoritative mode.
+authoritative mode. The deletion records the tombstone of the old ID (a
+database trigger, whoever deletes the row), so garbage collection deletes the
+old objects `gc_grace` later. Rows deleted this way before this release left
+their objects in the bucket; garbage collection finds them and records their
+tombstones (see `gc --removed` below).
### Checkpoints and garbage collection
@@ -407,7 +411,15 @@ graft storage gc --removed -address unix:/var/opt/graft/gitaly.sock [-storage de
of the repository (the background one, another node) holds it, or `Aborted`
when the lease is lost during the collection.
- `gc --removed` deletes every object of the repositories of the storage that
- were removed at least `gc_grace` ago.
+ were removed at least `gc_grace` ago. It first reconciles the bucket: a
+ repository ID allocated at least `gc_grace − gc_margin` ago that has objects
+ in the bucket and neither a row nor a tombstone (a row deleted by hand before
+ this release) gets a tombstone, so its objects go `gc_grace` later. A run
+ examines at most 256 IDs, from where the previous run stopped, and reads at
+ most one object of each; it prints what it found. Objects of IDs above the
+ database's allocator (a database restored from an older backup) are reported
+ with a warning and never touched: the database will hand those IDs out again.
+ The background maintenance does the same with `gc = "auto"`.
- `--dry-run` reports what would be deleted and changes nothing. It is a usage
error on the subcommands that do not read it, like `--connectivity` outside
`verify` and `--removed` outside `gc`.
@@ -418,12 +430,13 @@ can never call the storage core service, whatever `agent_rpcs` says.
### Upgrades
Upgrading from v0.2.x to this release changes the storage core's database
-(migration `20261018000000_graft_storage_gc`) and the keys of the packs in the
+(migrations `20261018000000_graft_storage_gc` and
+`20261019000000_graft_storage_reconcile`: the tombstone trigger and the state
+of the bucket reconciliation) and the keys of the packs in the
bucket (owned keys, see Packs). A node of the old version cannot work next to
a node of the new one:
- **It cannot start.** A v0.2.x node that restarts after the migration
- refuses the database, with an error about an unknown migration
- (`20261018000000_graft_storage_gc`).
+ refuses the database, with an error about the unknown migrations.
- **It cannot read.** A v0.2.x node still running reads only the old 64-hex
pack keys, so its reads of a repository that a new node wrote fail with
"invalid pack key".
@@ -712,7 +725,9 @@ without a single divergence.
| `graft_storage_maintenance_checkpoints_total` | Checkpoints created by the background maintenance. |
| `graft_storage_maintenance_packs_deleted_total`, `graft_storage_maintenance_bytes_reclaimed_total` | Bucket objects, and their bytes, deleted by the background garbage collection. |
| `graft_storage_maintenance_skipped_total{reason}` | Repositories the background maintenance skipped: `no_local_copy`, `nothing_due` (no checkpoint due, no superseded checkpoint, no collection; decided without the lease), `lease_held`. |
-| `graft_storage_maintenance_failures_total{step}` | Failed steps of the background maintenance: `collect_removed`, `list`, `lease`, `checkpoint`, `collect`. |
+| `graft_storage_maintenance_failures_total{step}` | Failed steps of the background maintenance: `reconcile`, `collect_removed`, `list`, `lease`, `checkpoint`, `collect`. |
+| `graft_storage_maintenance_leaks_found_total`, `graft_storage_maintenance_leaks_tombstoned_total` | Repository IDs with objects in the bucket and neither a row nor a tombstone, found by the reconciliation, and the tombstones it recorded for them. |
+| `graft_storage_maintenance_ids_ahead_of_allocator` | Repository IDs above the database's allocator with objects in the bucket at the last reconciliation (a database restored from an older backup); never touched. Alert on a value above 0. |
| `graft_review_roots_backfilled_total` | Keep-around references of open-review commits written by the backfill (daily housekeeping and before every checkpoint). |
| `graft_review_roots_deleted_total` | Keep-around references deleted because no open review uses them. |
| `graft_review_roots_missing_total` | Open-review commits the backfill found missing from the repository; the review can only be closed. |
diff --git a/internal/gitaly/service/storagecore/maintenance.go b/internal/gitaly/service/storagecore/maintenance.go
index d677830..f75637c 100644
--- a/internal/gitaly/service/storagecore/maintenance.go
+++ b/internal/gitaly/service/storagecore/maintenance.go
@@ -124,14 +124,22 @@ func (s *server) CollectRemovedRepositories(ctx context.Context, req *gitalypb.C
return nil, err
}
- report, err := gc.New(core, s.deps.GetLogger()).CollectRemoved(ctx, storageName, req.GetDryRun())
+ collector := gc.New(core, s.deps.GetLogger())
+ reconciled, err := collector.Reconcile(ctx, req.GetDryRun())
+ if err != nil {
+ return nil, err
+ }
+ report, err := collector.CollectRemoved(ctx, storageName, req.GetDryRun())
if err != nil {
return nil, err
}
return &gitalypb.CollectRemovedRepositoriesResponse{
- Repositories: uint32(report.RemovedRepos),
- PacksDeleted: uint32(report.PacksDeleted),
- BytesReclaimed: report.BytesReclaimed,
- DryRun: report.DryRun,
+ Repositories: uint32(report.RemovedRepos),
+ PacksDeleted: uint32(report.PacksDeleted),
+ BytesReclaimed: report.BytesReclaimed,
+ DryRun: report.DryRun,
+ LeaksFound: uint32(reconciled.LeaksFound),
+ LeaksTombstoned: uint32(reconciled.LeaksTombstoned),
+ IdsAheadOfAllocator: uint32(reconciled.AheadOfAllocator),
}, nil
}
diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.go
index c1fff6f..40c2459 100644
--- a/internal/gitaly/service/storagecore/maintenance_test.go
+++ b/internal/gitaly/service/storagecore/maintenance_test.go
@@ -21,12 +21,31 @@ import (
"github.com/feder-cr/git-ai/proto/go/gitalypb"
)
-// shortGC makes garbage collection wait a short grace, so that a test sees an
-// orphan deleted. The writers' deadline (grace - margin) stays long enough for
-// the import's upload under load.
-func shortGC(core *corestore.Core) {
- core.GCGrace = time.Second
- core.GCMargin = 100 * time.Millisecond
+// The tests age rows on the database clock instead of waiting with a short
+// gc_grace: the writers' deadline W is gc_grace - gc_margin, and an import or
+// a shadow write under the race detector on a loaded host does not fit in a W
+// of a second.
+
+// exec runs a statement with "?" placeholders on the database.
+func (h *harness) exec(query string, args ...any) {
+ h.t.Helper()
+ store := h.core.Meta.(*sqlstore.Store)
+ if store.Dialect() == sqlstore.Postgres {
+ for n := 1; strings.Contains(query, "?"); n++ {
+ query = strings.Replace(query, "?", fmt.Sprintf("$%d", n), 1)
+ }
+ }
+ _, err := store.DB().ExecContext(h.ctx, query, args...)
+ require.NoError(h.t, err)
+}
+
+// longAgo returns the database time value of more than gc_grace before now.
+func (h *harness) longAgo() any {
+ t := time.Now().Add(-h.core.GCGrace - time.Minute).UTC()
+ if h.core.Meta.(*sqlstore.Store).Dialect() == sqlstore.SQLite {
+ return t.Format(time.RFC3339Nano)
+ }
+ return t
}
// putOrphan stores an object under the repository's pack prefix that no row
@@ -101,7 +120,7 @@ func TestCreateCheckpointRefusals(t *testing.T) {
func TestCollectGarbage(t *testing.T) {
t.Parallel()
forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {
- h := setup(t, mode, dialect, shortGC)
+ h := setup(t, mode, dialect)
h.seed()
repoID := h.importRepo()
orphan := h.putOrphan(repoID, "the pack of a write that died")
@@ -117,11 +136,11 @@ func TestCollectGarbage(t *testing.T) {
require.Equal(t, uint32(1), first.GetOrphansObserved())
require.True(t, h.exists(orphan), "observed, not yet deleted")
- require.Eventually(t, func() bool {
- resp, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
- require.NoError(t, err)
- return resp.GetOrphansDeleted() == 1 && resp.GetBytesReclaimed() > 0
- }, 30*time.Second, 50*time.Millisecond)
+ h.exec("UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ?", h.longAgo(), repoID)
+ resp, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
+ require.NoError(t, err)
+ require.Equal(t, uint32(1), resp.GetOrphansDeleted())
+ require.Positive(t, resp.GetBytesReclaimed())
require.False(t, h.exists(orphan))
require.Empty(t, h.verify(true).GetDivergences(), "the repository is intact")
})
@@ -159,7 +178,7 @@ func TestCollectGarbageRefusals(t *testing.T) {
func TestCollectRemovedRepositories(t *testing.T) {
t.Parallel()
forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {
- h := setup(t, mode, dialect, shortGC)
+ h := setup(t, mode, dialect)
h.seed()
repoID := h.importRepo()
packs := h.packKeys(repoID)
@@ -170,11 +189,58 @@ func TestCollectRemovedRepositories(t *testing.T) {
require.True(t, removed)
storageName := h.cfg.Storages[0].Name
- require.Eventually(t, func() bool {
+ collect := func() *gitalypb.CollectRemovedRepositoriesResponse {
+ resp, err := h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: storageName})
+ require.NoError(t, err)
+ return resp
+ }
+ require.Zero(t, collect().GetRepositories(), "the tombstone is younger than gc_grace")
+ h.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", h.longAgo(), repoID)
+ resp := collect()
+ require.Equal(t, uint32(1), resp.GetRepositories())
+ require.GreaterOrEqual(t, resp.GetPacksDeleted(), uint32(len(packs)))
+ require.Empty(t, h.packKeys(repoID))
+ })
+}
+
+// TestCollectRemovedRepositoriesReconcilesLeaks: the objects of a repository
+// whose row went without a tombstone (deleted by hand before the trigger
+// existed) are found by the reconciliation of the RPC once the allocation is W
+// old, get a tombstone, and are deleted once it is gc_grace old (integrated
+// review m4).
+func TestCollectRemovedRepositoriesReconcilesLeaks(t *testing.T) {
+ t.Parallel()
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ h := setup(t, corestore.ModeAuthoritative, dialect)
+ h.seed()
+ repoID := h.importRepo()
+ packs := h.packKeys(repoID)
+ require.NotEmpty(t, packs)
+
+ h.exec("DELETE FROM graft_repositories WHERE repo_id = ?", repoID)
+ h.exec("DELETE FROM graft_gc_removed WHERE repo_id = ?", repoID)
+
+ storageName := h.cfg.Storages[0].Name
+ collect := func() *gitalypb.CollectRemovedRepositoriesResponse {
resp, err := h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: storageName})
require.NoError(t, err)
- return resp.GetRepositories() == 1 && resp.GetPacksDeleted() >= uint32(len(packs))
- }, 30*time.Second, 50*time.Millisecond)
+ require.Zero(t, resp.GetIdsAheadOfAllocator())
+ return resp
+ }
+ require.Zero(t, collect().GetLeaksFound(), "the allocation is not W old yet")
+
+ // The allocator was observed at the ID longer than W ago.
+ h.exec("DELETE FROM graft_gc_allocator_marks")
+ h.exec("INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)", h.longAgo(), repoID)
+ resp := collect()
+ require.Equal(t, uint32(1), resp.GetLeaksFound())
+ require.Equal(t, uint32(1), resp.GetLeaksTombstoned())
+ require.Zero(t, resp.GetRepositories(), "the new tombstone is younger than gc_grace")
+
+ h.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", h.longAgo(), repoID)
+ resp = collect()
+ require.Equal(t, uint32(1), resp.GetRepositories())
+ require.Zero(t, resp.GetLeaksFound(), "examined once")
require.Empty(t, h.packKeys(repoID))
})
}
diff --git a/internal/graft/corestore/gc/export_test.go b/internal/graft/corestore/gc/export_test.go
index d233db0..d936946 100644
--- a/internal/graft/corestore/gc/export_test.go
+++ b/internal/graft/corestore/gc/export_test.go
@@ -1,4 +1,6 @@
package gc
-// SetFaultHook sets the hook called at the fault points of c's runs.
func SetFaultHook(c *Collector, hook func(point string) error) { c.faultHook = hook }
+
+// ReconcileBatch is reconcileBatch, for the tests.
+const ReconcileBatch = reconcileBatch
diff --git a/internal/graft/corestore/gc/gc.go b/internal/graft/corestore/gc/gc.go
index 6b6315f..a2bb5d3 100644
--- a/internal/graft/corestore/gc/gc.go
+++ b/internal/graft/corestore/gc/gc.go
@@ -64,6 +64,16 @@ type Report struct {
BytesReclaimed int64
// RemovedRepos are the removed repositories whose objects were deleted.
RemovedRepos int
+ // LeaksFound are the repository IDs of the reconciled range that have
+ // objects in the bucket and neither a row nor a tombstone.
+ LeaksFound int
+ // LeaksTombstoned are the tombstones the reconciliation recorded for them.
+ LeaksTombstoned int
+ // AheadOfAllocator are the repository IDs above the allocator's position
+ // that have objects in the bucket (a database restored from an older
+ // backup). They are reported and never touched: the allocator will hand
+ // them out again.
+ AheadOfAllocator int
// DryRun tells that nothing was deleted or recorded.
DryRun bool
}
@@ -353,6 +363,107 @@ func (c *Collector) CollectRemoved(ctx context.Context, storage string, dryRun b
return report, nil
}
+const (
+ // reconcileBatch bounds the repository IDs one reconciliation examines.
+ reconcileBatch = 256
+ // aheadProbe is the number of IDs above the allocator's position whose
+ // prefixes a reconciliation checks.
+ aheadProbe = 16
+)
+
+// errFound stops a listing at its first object.
+var errFound = errors.New("found")
+
+// hasObjects reports whether the bucket has an object under the repository's
+// prefix. It reads one object at most (one page of a paged listing).
+func (c *Collector) hasObjects(ctx context.Context, repoID int64) (bool, error) {
+ err := c.core.Blobs.List(ctx, repoPrefix(repoID), func(corestore.BlobInfo) error { return errFound })
+ switch {
+ case errors.Is(err, errFound):
+ return true, nil
+ case err != nil:
+ return false, errBucket(err)
+ }
+ return false, nil
+}
+
+// Reconcile records a tombstone for every repository ID whose row was deleted
+// without one (before the trigger graft_repositories_tombstone existed) and
+// whose objects are still in the bucket; CollectRemoved then deletes them
+// like those of any removed repository (doc/graft/PLAN7.md, "Removed
+// repositories", reconciliation). It examines the IDs after its cursor, at
+// most reconcileBatch per run, and only those the allocator handed out at
+// least the write deadline W ago on the database clock: a creation still in
+// flight is never examined (on Postgres an allocated ID precedes its row). An
+// ID without a row or a tombstone gets one only when its prefix has objects;
+// the cursor then moves past the batch, so a run that stopped is resumed by
+// the next one and nothing is examined twice. IDs above the allocator that
+// have objects are counted and logged, never touched. A dry run records
+// nothing.
+func (c *Collector) Reconcile(ctx context.Context, dryRun bool) (Report, error) {
+ if err := c.validTimes(); err != nil {
+ return Report{}, err
+ }
+
+ view, err := c.core.Meta.ObserveAllocator(ctx, c.core.WriteDeadline(), dryRun)
+ if err != nil {
+ return Report{}, errDatabase(err)
+ }
+ report := Report{DryRun: dryRun}
+
+ for id := view.Position + 1; id <= view.Position+aheadProbe; id++ {
+ found, err := c.hasObjects(ctx, id)
+ if err != nil {
+ return report, err
+ }
+ if found {
+ report.AheadOfAllocator++
+ c.logger.WithFields(log.Fields{
+ "repository_id": id,
+ "allocator_position": view.Position,
+ }).WarnContext(ctx, "storage core: the bucket has objects of a repository ID the database has not allocated; was the database restored from an older backup? They are left alone")
+ }
+ }
+
+ upTo := min(view.Bound, view.DoneThrough+reconcileBatch)
+ if upTo <= view.DoneThrough {
+ return report, nil
+ }
+ ids, err := c.core.Meta.UnrecordedIDs(ctx, view.DoneThrough, upTo)
+ if err != nil {
+ return report, errDatabase(err)
+ }
+ for _, id := range ids {
+ found, err := c.hasObjects(ctx, id)
+ if err != nil {
+ return report, err
+ }
+ if !found {
+ continue
+ }
+ report.LeaksFound++
+ if dryRun {
+ continue
+ }
+ recorded, err := c.core.Meta.RecordLeak(ctx, id)
+ if err != nil {
+ return report, errDatabase(err)
+ }
+ if recorded {
+ report.LeaksTombstoned++
+ c.logger.WithField("repository_id", id).InfoContext(ctx,
+ "storage core: the bucket has objects of a repository whose row was deleted without a tombstone; they are collected after gc_grace")
+ }
+ }
+ if dryRun {
+ return report, nil
+ }
+ if err := c.core.Meta.AdvanceReconcile(ctx, upTo); err != nil {
+ return report, errDatabase(err)
+ }
+ return report, nil
+}
+
// validTimes refuses to collect with times that would break the proofs.
func (c *Collector) validTimes() error {
if c.core.GCGrace <= 0 || c.core.GCMargin <= 0 || c.core.WriteDeadline() <= 0 {
diff --git a/internal/graft/corestore/gc/gc_test.go b/internal/graft/corestore/gc/gc_test.go
index 35d29e1..778fecb 100644
--- a/internal/graft/corestore/gc/gc_test.go
+++ b/internal/graft/corestore/gc/gc_test.go
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"errors"
"fmt"
+ "slices"
"sort"
"strings"
"sync"
@@ -487,3 +488,202 @@ func TestCollectRemoved(t *testing.T) {
require.Equal(t, []string{kept}, e.objects(e.repo.ID), "other repositories are untouched")
})
}
+
+// tombstones returns the tombstones of every storage, whatever their age.
+func (e *env) tombstones() map[int64]corestore.RemovedRepository {
+ e.t.Helper()
+ removed, err := e.core.Meta.ListRemovedRepositories(e.ctx, "", 0)
+ require.NoError(e.t, err)
+ out := map[int64]corestore.RemovedRepository{}
+ for _, r := range removed {
+ out[r.RepoID] = r
+ }
+ return out
+}
+
+// leak registers a repository, stores objects under its prefix and deletes
+// its row and its tombstone, as a row deleted by hand before the trigger
+// existed left them. It returns the repository ID.
+func (e *env) leak(name string, objects int) int64 {
+ e.t.Helper()
+ repo := e.register(name)
+ for range objects {
+ e.upload(repo.ID)
+ }
+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", repo.ID)
+ e.exec("DELETE FROM graft_gc_removed WHERE repo_id = ?", repo.ID)
+ return repo.ID
+}
+
+// allocator returns the allocator view without recording an observation.
+func (e *env) allocator() corestore.AllocatorView {
+ e.t.Helper()
+ view, err := e.core.Meta.ObserveAllocator(e.ctx, e.core.WriteDeadline(), true)
+ require.NoError(e.t, err)
+ return view
+}
+
+// ageAllocator replaces the observations of the allocator by one of its
+// current position made d ago.
+func (e *env) ageAllocator(d time.Duration) {
+ e.t.Helper()
+ position := e.allocator().Position
+ e.exec("DELETE FROM graft_gc_allocator_marks")
+ e.exec("INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)", e.past(d), position)
+}
+
+func (e *env) reconcile(dryRun bool) gc.Report {
+ e.t.Helper()
+ report, err := e.gc.Reconcile(e.ctx, dryRun)
+ require.NoError(e.t, err)
+ return report
+}
+
+// TestDeletedRowGetsATombstone: a repository row deleted by hand (the repair
+// of STORAGE.md, "State and verify") gets its tombstone from the trigger, on
+// the database clock, and its objects are collected like those of a removed
+// repository (integrated review m4).
+func TestDeletedRowGetsATombstone(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, e *env) {
+ repo := e.register("by-hand")
+ e.upload(repo.ID)
+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", repo.ID)
+
+ tombstone, ok := e.tombstones()[repo.ID]
+ require.True(t, ok, "the trigger records the tombstone")
+ require.Equal(t, repo.Key, tombstone.Key)
+ require.WithinDuration(t, time.Now(), tombstone.RemovedAt, time.Minute)
+
+ e.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", e.past(grace+time.Minute), repo.ID)
+ report, err := e.gc.CollectRemoved(e.ctx, "default", false)
+ require.NoError(t, err)
+ require.Equal(t, 1, report.RemovedRepos)
+ require.Empty(t, e.objects(repo.ID))
+ })
+}
+
+// TestReconcileTombstonesLeaks: the objects of a row deleted without a
+// tombstone get one, only for IDs allocated at least W ago, only when the
+// prefix has objects, once; the tombstone has no key and is collected by
+// every storage's collection after gc_grace (integrated review m4).
+func TestReconcileTombstonesLeaks(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, e *env) {
+ live := e.changeset()
+ leaked := e.leak("leaked", 2)
+ empty := e.leak("empty", 0)
+
+ require.Equal(t, gc.Report{}, e.reconcile(false), "no allocation is W old yet")
+ require.Len(t, e.objects(leaked), 2)
+
+ e.ageAllocator(grace)
+ require.Equal(t, gc.Report{LeaksFound: 1, DryRun: true}, e.reconcile(true))
+ require.NotContains(t, e.tombstones(), leaked, "a dry run records nothing")
+
+ require.Equal(t, gc.Report{LeaksFound: 1, LeaksTombstoned: 1}, e.reconcile(false))
+ tombstones := e.tombstones()
+ require.Contains(t, tombstones, leaked)
+ require.Equal(t, corestore.RepoKey{}, tombstones[leaked].Key, "the key went with the row")
+ require.NotContains(t, tombstones, empty, "an ID without objects needs nothing")
+ require.Equal(t, e.allocator().Position, e.allocator().DoneThrough, "the cursor is past the reconciled IDs")
+
+ require.Equal(t, gc.Report{}, e.reconcile(false), "a second run examines nothing again")
+
+ report, err := e.gc.CollectRemoved(e.ctx, "default", false)
+ require.NoError(t, err)
+ require.Zero(t, report.RemovedRepos, "the tombstone is younger than gc_grace")
+ e.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", e.past(grace+time.Minute), leaked)
+ report, err = e.gc.CollectRemoved(e.ctx, "default", false)
+ require.NoError(t, err)
+ require.Equal(t, 1, report.RemovedRepos, "a storage's collection takes the tombstones without a key")
+ require.Empty(t, e.objects(leaked))
+ require.Equal(t, []string{live}, e.objects(e.repo.ID), "the live repository is untouched")
+ })
+}
+
+// TestReconcileIsPagedAndResumable: a run examines at most one batch of IDs
+// after its cursor; the next run goes on from there, and IDs above the
+// allocator with objects are reported and never touched.
+func TestReconcileIsPagedAndResumable(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, e *env) {
+ const position = 2*gc.ReconcileBatch + 40
+ if e.store.Dialect() == sqlstore.Postgres {
+ e.exec(fmt.Sprintf("SELECT setval('graft_repositories_repo_id_seq', %d)", position))
+ } else {
+ e.exec("UPDATE sqlite_sequence SET seq = ? WHERE name = 'graft_repositories'", position)
+ }
+ first, second := int64(5), int64(gc.ReconcileBatch+10)
+ e.upload(first)
+ e.upload(second)
+ ahead := e.upload(position + 3)
+ e.ageAllocator(grace)
+
+ require.Equal(t, gc.Report{LeaksFound: 1, LeaksTombstoned: 1, AheadOfAllocator: 1}, e.reconcile(false))
+ require.Contains(t, e.tombstones(), first)
+ require.NotContains(t, e.tombstones(), second, "beyond the first batch")
+ require.EqualValues(t, gc.ReconcileBatch, e.allocator().DoneThrough)
+
+ // A run that stopped after recording a tombstone, before its cursor
+ // moved: the next one does not count it again.
+ _, err := e.core.Meta.RecordLeak(e.ctx, second)
+ require.NoError(t, err)
+ require.Equal(t, gc.Report{AheadOfAllocator: 1}, e.reconcile(false))
+ require.EqualValues(t, 2*gc.ReconcileBatch, e.allocator().DoneThrough)
+ require.Equal(t, gc.Report{AheadOfAllocator: 1}, e.reconcile(false))
+ require.EqualValues(t, position, e.allocator().DoneThrough)
+
+ require.NotContains(t, e.tombstones(), int64(position+3))
+ require.Equal(t, []string{ahead}, e.objects(position+3), "never touched")
+ })
+}
+
+// TestReconcileNeverCollectsACreationInFlight holds a creation between the
+// allocation of its ID and the commit of its row across reconciliations, on
+// Postgres, whose allocator counts the ID before the row commits. (SQLite's
+// allocator is transactional: an uncommitted creation's ID is not allocated
+// for any other reader, and the single writer lock serialises the creation
+// with the reconciliation's writes.) An object written under the ID before
+// the commit, which no writer of the storage core does, is the worst case.
+// The ID is not examined while its allocation is younger than W; examined
+// later it may get a tombstone, but once the row commits the tombstone is not
+// a removal: it is never collected, and a later deletion of the row starts a
+// fresh grace.
+func TestReconcileNeverCollectsACreationInFlight(t *testing.T) {
+ if !slices.Contains(coretest.Dialects(), sqlstore.Postgres) {
+ t.Skip("the allocator counts uncommitted creations on Postgres only, which is not configured")
+ }
+ e := newEnv(t, sqlstore.Postgres)
+
+ tx, err := e.store.DB().BeginTx(e.ctx, nil)
+ require.NoError(t, err)
+ defer func() { _ = tx.Rollback() }()
+ var id int64
+ require.NoError(t, tx.QueryRowContext(e.ctx, `
+ INSERT INTO graft_repositories (storage, relative_path, object_format, head_seq, log_seq, next_review_id, created_at)
+ VALUES ('default', 'in-flight.git', 'sha1', 0, 0, 1, now()) RETURNING repo_id`).Scan(&id))
+ key := e.upload(id)
+
+ require.Equal(t, e.allocator().Position, id, "the allocator counts the creation")
+ require.Equal(t, gc.Report{}, e.reconcile(false), "an allocation younger than W is not examined")
+ require.NotContains(t, e.tombstones(), id)
+
+ // A creation that outlives W: its ID is examined and gets a tombstone.
+ e.ageAllocator(grace)
+ require.Equal(t, gc.Report{LeaksFound: 1, LeaksTombstoned: 1}, e.reconcile(false))
+ require.NoError(t, tx.Commit())
+
+ e.exec("UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", e.past(grace+time.Minute), id)
+ report, err := e.gc.CollectRemoved(e.ctx, "default", false)
+ require.NoError(t, err)
+ require.Zero(t, report.RemovedRepos, "a tombstone whose ID has a row is not a removal")
+ require.Equal(t, []string{key}, e.objects(id))
+
+ // Its deletion is a removal, with a grace from the deletion.
+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", id)
+ tombstone := e.tombstones()[id]
+ require.Equal(t, "in-flight.git", tombstone.Key.RelativePath)
+ require.WithinDuration(t, time.Now(), tombstone.RemovedAt, time.Minute)
+ report, err = e.gc.CollectRemoved(e.ctx, "default", false)
+ require.NoError(t, err)
+ require.Zero(t, report.RemovedRepos)
+ require.Equal(t, []string{key}, e.objects(id))
+}
diff --git a/internal/graft/corestore/maintenance/maintenance.go b/internal/graft/corestore/maintenance/maintenance.go
index 33f6b78..986666e 100644
--- a/internal/graft/corestore/maintenance/maintenance.go
+++ b/internal/graft/corestore/maintenance/maintenance.go
@@ -40,6 +40,7 @@ const releaseTimeout = 10 * time.Second
// Steps, the label of graft_storage_maintenance_failures_total.
const (
+ stepReconcile = "reconcile"
stepCollectRemoved = "collect_removed"
stepList = "list"
stepLease = "lease"
@@ -93,6 +94,9 @@ type Runner struct {
bytes prometheus.Counter
skipped *prometheus.CounterVec
failures *prometheus.CounterVec
+ leaksFound prometheus.Counter
+ leaksMarked prometheus.Counter
+ ahead prometheus.Gauge
}
var _ prometheus.Collector = (*Runner)(nil)
@@ -148,8 +152,20 @@ func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collec
}, []string{"reason"}),
failures: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "graft_storage_maintenance_failures_total",
- Help: "Failed steps of the background maintenance, by step (collect_removed, list, lease, checkpoint, collect).",
+ Help: "Failed steps of the background maintenance, by step (reconcile, collect_removed, list, lease, checkpoint, collect).",
}, []string{"step"}),
+ leaksFound: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "graft_storage_maintenance_leaks_found_total",
+ Help: "Repository IDs whose objects are in the bucket without a row or a tombstone (a row deleted without one), found by the reconciliation.",
+ }),
+ leaksMarked: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "graft_storage_maintenance_leaks_tombstoned_total",
+ Help: "Tombstones the reconciliation recorded for leaked repository IDs; their objects are collected after gc_grace.",
+ }),
+ ahead: prometheus.NewGauge(prometheus.GaugeOpts{
+ Name: "graft_storage_maintenance_ids_ahead_of_allocator",
+ Help: "Repository IDs above the database's allocator that have objects in the bucket, at the last reconciliation (a database restored from an older backup); never touched.",
+ }),
}
for _, result := range []string{"ok", "error"} {
r.runs.WithLabelValues(result)
@@ -157,7 +173,7 @@ func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collec
for _, reason := range []string{"no_local_copy", "nothing_due", "lease_held"} {
r.skipped.WithLabelValues(reason)
}
- for _, step := range []string{stepCollectRemoved, stepList, stepLease, stepCheckpoint, stepCollect} {
+ for _, step := range []string{stepReconcile, stepCollectRemoved, stepList, stepLease, stepCheckpoint, stepCollect} {
r.failures.WithLabelValues(step)
}
return r
@@ -199,6 +215,16 @@ func (r *Runner) RunOnce(ctx context.Context) error {
errs = append(errs, err)
}
+ if r.cfg.Collect {
+ // Before the removed repositories are collected, so that a leak's
+ // tombstone is among them once it is gc_grace old.
+ report, err := r.collector.Reconcile(ctx, false)
+ if err != nil {
+ fail(stepReconcile, err, log.Fields{})
+ }
+ r.countReconcile(report)
+ }
+
for _, storageName := range r.storages {
if !r.cfg.Collect {
break
@@ -416,6 +442,13 @@ func (r *Runner) count(report gc.Report) {
r.bytes.Add(float64(report.BytesReclaimed))
}
+// countReconcile records what a reconciliation found.
+func (r *Runner) countReconcile(report gc.Report) {
+ r.leaksFound.Add(float64(report.LeaksFound))
+ r.leaksMarked.Add(float64(report.LeaksTombstoned))
+ r.ahead.Set(float64(report.AheadOfAllocator))
+}
+
// Describe implements prometheus.Collector.
func (r *Runner) Describe(descs chan<- *prometheus.Desc) {
r.runs.Describe(descs)
@@ -424,6 +457,9 @@ func (r *Runner) Describe(descs chan<- *prometheus.Desc) {
r.bytes.Describe(descs)
r.skipped.Describe(descs)
r.failures.Describe(descs)
+ r.leaksFound.Describe(descs)
+ r.leaksMarked.Describe(descs)
+ r.ahead.Describe(descs)
}
// Collect implements prometheus.Collector.
@@ -434,4 +470,7 @@ func (r *Runner) Collect(metrics chan<- prometheus.Metric) {
r.bytes.Collect(metrics)
r.skipped.Collect(metrics)
r.failures.Collect(metrics)
+ r.leaksFound.Collect(metrics)
+ r.leaksMarked.Collect(metrics)
+ r.ahead.Collect(metrics)
}
diff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.go
index 2ec7a3e..6ffce63 100644
--- a/internal/graft/corestore/maintenance/maintenance_test.go
+++ b/internal/graft/corestore/maintenance/maintenance_test.go
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"fmt"
"os"
+ "strings"
"sync/atomic"
"testing"
"time"
@@ -157,13 +158,14 @@ func (e *env) metric(name, label string) float64 {
if family.GetName() != name {
continue
}
+ // A metric is a counter or a gauge; the other value is zero.
for _, m := range family.GetMetric() {
if label == "" {
- return m.GetCounter().GetValue()
+ return m.GetCounter().GetValue() + m.GetGauge().GetValue()
}
for _, l := range m.GetLabel() {
if l.GetValue() == label {
- return m.GetCounter().GetValue()
+ return m.GetCounter().GetValue() + m.GetGauge().GetValue()
}
}
}
@@ -171,6 +173,45 @@ func (e *env) metric(name, label string) float64 {
return 0
}
+// exec runs a statement with "?" placeholders on the database of the env.
+func (e *env) exec(query string, args ...any) {
+ e.t.Helper()
+ store := e.core.Meta.(*sqlstore.Store)
+ if store.Dialect() == sqlstore.Postgres {
+ for n := 1; strings.Contains(query, "?"); n++ {
+ query = strings.Replace(query, "?", fmt.Sprintf("$%d", n), 1)
+ }
+ }
+ _, err := store.DB().ExecContext(testhelper.Context(e.t), query, args...)
+ require.NoError(e.t, err)
+}
+
+// past returns the database time value of d before now.
+func (e *env) past(d time.Duration) any {
+ t := time.Now().Add(-d).UTC()
+ if e.core.Meta.(*sqlstore.Store).Dialect() == sqlstore.SQLite {
+ return t.Format(time.RFC3339Nano)
+ }
+ return t
+}
+
+// The tests age rows on the database clock instead of waiting with a short
+// gc_grace: the writers' deadline W is gc_grace - gc_margin, and a checkpoint
+// built under the race detector on a loaded host does not fit in a W of a
+// fraction of a second.
+
+// ageCheckpoints makes every checkpoint of the repository older than
+// gc_grace.
+func (e *env) ageCheckpoints() {
+ e.exec("UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ?", e.past(e.core.GCGrace+time.Minute), e.repo.ID)
+}
+
+// observedLongAgo records that a collection observed the object unreferenced
+// more than gc_grace ago.
+func (e *env) observedLongAgo(key string) {
+ e.exec("INSERT INTO graft_gc_orphans (repo_id, pack_key, first_seen) VALUES (?, ?, ?)", e.repo.ID, key, e.past(e.core.GCGrace+time.Minute))
+}
+
// config3 is a configuration with the checkpoints and the garbage collection
// on, and the given checkpoint thresholds.
func config3(changesets uint64, packBytes int64) maintenance.Config {
@@ -187,7 +228,7 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {
ctx := testhelper.Context(t)
cfg := config3(1, 0)
cfg.Collect = false
- e := newEnv(t, coretest.Options{GCGrace: 200 * time.Millisecond, GCMargin: 50 * time.Millisecond}, cfg)
+ e := newEnv(t, coretest.Options{}, cfg)
var built []corestore.Checkpoint
for i := 0; i < 3; i++ {
@@ -204,10 +245,11 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {
}
newest := built[len(built)-1]
- require.Eventually(t, func() bool {
- require.NoError(t, e.runner.RunOnce(ctx))
- return len(e.checkpoints()) == 1
- }, 30*time.Second, 50*time.Millisecond, "the superseded checkpoints are dropped")
+ require.NoError(t, e.runner.RunOnce(ctx))
+ require.Len(t, e.checkpoints(), 3, "no checkpoint is gc_grace old yet")
+ e.ageCheckpoints()
+ require.NoError(t, e.runner.RunOnce(ctx))
+ require.Len(t, e.checkpoints(), 1, "the superseded checkpoints are dropped")
require.Equal(t, newest.Seq, e.checkpoints()[0].Seq)
keys := e.bucketKeys()
@@ -226,6 +268,39 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {
require.Equal(t, before+1, e.metric("graft_storage_maintenance_skipped_total", "nothing_due"))
}
+// TestRunOnceReconcilesLeaks: with the collection on, a run reconciles the
+// bucket before it collects the removed repositories, and its metrics count
+// the leaks found and tombstoned and the IDs ahead of the allocator.
+func TestRunOnceReconcilesLeaks(t *testing.T) {
+ t.Parallel()
+ ctx := testhelper.Context(t)
+ e := newEnv(t, coretest.Options{}, config3(0, 0))
+ put := func(repoID int64) {
+ content := []byte(fmt.Sprintf("objects of %d", repoID))
+ require.NoError(t, e.core.Blobs.Put(ctx, corestore.NewPackKey(repoID, sha256.Sum256(content)), bytes.NewReader(content)))
+ }
+
+ // A row deleted by hand before the trigger existed: no tombstone.
+ leaked, _, err := e.core.Meta.RegisterRepository(ctx, corestore.RepoKey{Storage: "default", RelativePath: "leaked.git"}, "sha1", corestore.InitialState{})
+ require.NoError(t, err)
+ put(leaked.ID)
+ e.exec("DELETE FROM graft_repositories WHERE repo_id = ?", leaked.ID)
+ e.exec("DELETE FROM graft_gc_removed WHERE repo_id = ?", leaked.ID)
+ put(leaked.ID + 2)
+
+ // The allocator was observed at its position longer than W ago.
+ e.exec("INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)", e.past(e.core.WriteDeadline()+time.Minute), leaked.ID)
+
+ require.NoError(t, e.runner.RunOnce(ctx))
+ require.Equal(t, 1.0, e.metric("graft_storage_maintenance_leaks_found_total", ""))
+ require.Equal(t, 1.0, e.metric("graft_storage_maintenance_leaks_tombstoned_total", ""))
+ require.Equal(t, 1.0, e.metric("graft_storage_maintenance_ids_ahead_of_allocator", ""))
+ removed, err := e.core.Meta.ListRemovedRepositories(ctx, "", 0)
+ require.NoError(t, err)
+ require.Len(t, removed, 1)
+ require.Equal(t, leaked.ID, removed[0].RepoID)
+}
+
// leaseExpiry returns the expiry of the repository's maintenance lease, which
// holder must hold.
func (e *env) leaseExpiry(holder string) time.Time {
@@ -277,33 +352,33 @@ func TestRunOnceDoesOnlyWhatIsOn(t *testing.T) {
require.NoError(t, e.core.Blobs.Put(ctx, key, bytes.NewReader(content)))
return key
}
- opts := coretest.Options{GCGrace: 200 * time.Millisecond, GCMargin: 50 * time.Millisecond}
+ exists := func(e *env, key string) bool {
+ ok, err := e.core.Blobs.Exists(ctx, key)
+ require.NoError(t, err)
+ return ok
+ }
+ // An orphan that a collection would delete now: it was observed more than
+ // gc_grace ago.
checkpointsOnly := config3(1, 0)
checkpointsOnly.Collect = false
- e := newEnv(t, opts, checkpointsOnly)
+ e := newEnv(t, coretest.Options{}, checkpointsOnly)
e.fresh()
key := orphan(e)
- require.Never(t, func() bool {
- require.NoError(t, e.runner.RunOnce(ctx))
- exists, err := e.core.Blobs.Exists(ctx, key)
- require.NoError(t, err)
- return !exists
- }, time.Second, 100*time.Millisecond, "checkpoints alone delete nothing")
+ e.observedLongAgo(key)
+ require.NoError(t, e.runner.RunOnce(ctx))
+ require.True(t, exists(e, key), "checkpoints alone delete nothing")
_, ok := e.newestCheckpoint()
require.True(t, ok)
collectOnly := config3(1, 0)
collectOnly.Checkpoints = false
- e = newEnv(t, opts, collectOnly)
+ e = newEnv(t, coretest.Options{}, collectOnly)
e.fresh()
key = orphan(e)
- require.Eventually(t, func() bool {
- require.NoError(t, e.runner.RunOnce(ctx))
- exists, err := e.core.Blobs.Exists(ctx, key)
- require.NoError(t, err)
- return !exists
- }, 30*time.Second, 100*time.Millisecond)
+ e.observedLongAgo(key)
+ require.NoError(t, e.runner.RunOnce(ctx))
+ require.False(t, exists(e, key), "the collection deletes it")
_, ok = e.newestCheckpoint()
require.False(t, ok, "the collection alone creates no checkpoint")
}
@@ -393,7 +468,7 @@ func TestRunOnceSkipsARepositoryWhoseLeaseIsHeld(t *testing.T) {
func TestRunOnceCollectsAnOrphanPack(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
- e := newEnv(t, coretest.Options{GCGrace: 200 * time.Millisecond, GCMargin: 50 * time.Millisecond}, config3(0, 0))
+ e := newEnv(t, coretest.Options{}, config3(0, 0))
e.fresh()
content := []byte("an orphan pack")
@@ -406,12 +481,11 @@ func TestRunOnceCollectsAnOrphanPack(t *testing.T) {
require.NoError(t, err)
require.True(t, exists, "observed, not yet deleted")
- require.Eventually(t, func() bool {
- require.NoError(t, e.runner.RunOnce(ctx))
- exists, err := e.core.Blobs.Exists(ctx, key)
- require.NoError(t, err)
- return !exists
- }, 30*time.Second, 100*time.Millisecond)
+ e.exec("UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ?", e.past(e.core.GCGrace+time.Minute), e.repo.ID)
+ require.NoError(t, e.runner.RunOnce(ctx))
+ exists, err = e.core.Blobs.Exists(ctx, key)
+ require.NoError(t, err)
+ require.False(t, exists, "deleted gc_grace after its observation")
require.Equal(t, float64(1), e.metric("graft_storage_maintenance_packs_deleted_total", ""))
require.Equal(t, float64(len(content)), e.metric("graft_storage_maintenance_bytes_reclaimed_total", ""))
}
diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.go
index 4f7dbdf..f085511 100644
--- a/internal/graft/corestore/metastore.go
+++ b/internal/graft/corestore/metastore.go
@@ -242,12 +242,31 @@ type MetaStore interface {
ReleaseMaintenanceLease(ctx context.Context, repoID int64, holder string) error
// ListRemovedRepositories returns the tombstones of the storage (of every
- // storage when it is empty) recorded at least minAge ago on the database
- // clock, ascending by repository ID.
+ // storage when it is empty), and those of an unknown key (RecordLeak),
+ // recorded at least minAge ago on the database clock, ascending by
+ // repository ID. A tombstone whose ID has a row (a reconciled ID whose
+ // creation committed after all) is not returned.
ListRemovedRepositories(ctx context.Context, storage string, minAge time.Duration) ([]RemovedRepository, error)
// DeleteRemovedRepository removes a tombstone; a missing one is success.
DeleteRemovedRepository(ctx context.Context, repoID int64) error
+ // ObserveAllocator reads the repository ID allocator and the
+ // reconciliation cursor and, unless dryRun, records the observation
+ // (AllocatorView.Now, Position) and drops the observations older than the
+ // one Bound comes from, in one transaction (doc/graft/PLAN7.md, "Removed
+ // repositories", reconciliation).
+ ObserveAllocator(ctx context.Context, minAge time.Duration, dryRun bool) (AllocatorView, error)
+ // UnrecordedIDs returns the repository IDs in (after, upTo] that have
+ // neither a row nor a tombstone, ascending.
+ UnrecordedIDs(ctx context.Context, after, upTo int64) ([]int64, error)
+ // RecordLeak records a tombstone for repoID, with an unknown key (empty
+ // storage and relative path) and the database time, unless the ID has a
+ // row or a tombstone; it reports whether it recorded one.
+ RecordLeak(ctx context.Context, repoID int64) (bool, error)
+ // AdvanceReconcile moves the reconciliation cursor to doneThrough unless
+ // it is there or further already.
+ AdvanceReconcile(ctx context.Context, doneThrough int64) error
+
// Close releases the database.
Close() error
}
diff --git a/internal/graft/corestore/sqlstore/maintenance.go b/internal/graft/corestore/sqlstore/maintenance.go
index 09e66c8..52d8abf 100644
--- a/internal/graft/corestore/sqlstore/maintenance.go
+++ b/internal/graft/corestore/sqlstore/maintenance.go
@@ -172,13 +172,16 @@ func (s *Store) ListRemovedRepositories(ctx context.Context, storage string, min
return err
}
- query := "SELECT repo_id, storage, relative_path, removed_at FROM graft_gc_removed"
+ // A reconciled tombstone (RecordLeak) has an empty key: it belongs to
+ // every storage's list. One whose ID has a row is not a removal.
+ query := `SELECT g.repo_id, g.storage, g.relative_path, g.removed_at FROM graft_gc_removed g
+ WHERE NOT EXISTS (SELECT 1 FROM graft_repositories r WHERE r.repo_id = g.repo_id)`
var args []any
if storage != "" {
- query += " WHERE storage = ?"
+ query += " AND (g.storage = ? OR g.storage = '')"
args = append(args, storage)
}
- query += " ORDER BY repo_id"
+ query += " ORDER BY g.repo_id"
rows, err := tx.QueryContext(ctx, s.bind(query), args...)
if err != nil {
diff --git a/internal/graft/corestore/sqlstore/migrations.go b/internal/graft/corestore/sqlstore/migrations.go
index 0576ed6..4a1481e 100644
--- a/internal/graft/corestore/sqlstore/migrations.go
+++ b/internal/graft/corestore/sqlstore/migrations.go
@@ -147,6 +147,35 @@ var sqliteMigrations = []*migrate.Migration{
`DROP TABLE graft_checkpoints`,
},
},
+ {
+ // Integrated review m4 (doc/graft/PLAN7.md, "Removed repositories"): every
+ // deletion of a repository row records its tombstone, also one made by
+ // hand, replacing a reconciled one of the same ID (its grace starts at
+ // the deletion); and the state of the reconciliation that tombstones the
+ // bucket prefixes of rows deleted without one before this migration.
+ Id: "20261019000000_graft_storage_reconcile",
+ Up: []string{
+ `CREATE TRIGGER graft_repositories_tombstone AFTER DELETE ON graft_repositories
+ BEGIN
+ INSERT OR REPLACE INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)
+ VALUES (OLD.repo_id, OLD.storage, OLD.relative_path, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'));
+ END`,
+ `CREATE TABLE graft_gc_allocator_marks (
+ observed_at TEXT NOT NULL PRIMARY KEY,
+ position INTEGER NOT NULL
+ )`,
+ `CREATE TABLE graft_gc_reconcile (
+ id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),
+ done_through INTEGER NOT NULL
+ )`,
+ `INSERT INTO graft_gc_reconcile (id, done_through) VALUES (1, 0)`,
+ },
+ Down: []string{
+ `DROP TABLE graft_gc_reconcile`,
+ `DROP TABLE graft_gc_allocator_marks`,
+ `DROP TRIGGER graft_repositories_tombstone`,
+ },
+ },
}
// notOwnedPackKeyMessage is the error of the trigger that refuses a changeset
@@ -294,4 +323,36 @@ var postgresMigrations = []*migrate.Migration{
`DROP TABLE graft_checkpoints`,
},
},
+ {
+ // See the SQLite migration of the same ID.
+ Id: "20261019000000_graft_storage_reconcile",
+ Up: []string{
+ `CREATE FUNCTION graft_repositories_tombstone() RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN
+ INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)
+ VALUES (OLD.repo_id, OLD.storage, OLD.relative_path, clock_timestamp())
+ ON CONFLICT (repo_id) DO UPDATE SET storage = EXCLUDED.storage,
+ relative_path = EXCLUDED.relative_path, removed_at = EXCLUDED.removed_at;
+ RETURN OLD;
+ END
+ $$`,
+ `CREATE TRIGGER graft_repositories_tombstone AFTER DELETE ON graft_repositories
+ FOR EACH ROW EXECUTE FUNCTION graft_repositories_tombstone()`,
+ `CREATE TABLE graft_gc_allocator_marks (
+ observed_at TIMESTAMPTZ NOT NULL PRIMARY KEY,
+ position BIGINT NOT NULL
+ )`,
+ `CREATE TABLE graft_gc_reconcile (
+ id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),
+ done_through BIGINT NOT NULL
+ )`,
+ `INSERT INTO graft_gc_reconcile (id, done_through) VALUES (1, 0)`,
+ },
+ Down: []string{
+ `DROP TABLE graft_gc_reconcile`,
+ `DROP TABLE graft_gc_allocator_marks`,
+ `DROP TRIGGER graft_repositories_tombstone ON graft_repositories`,
+ `DROP FUNCTION graft_repositories_tombstone()`,
+ },
+ },
}
diff --git a/internal/graft/corestore/sqlstore/reconcile.go b/internal/graft/corestore/sqlstore/reconcile.go
new file mode 100644
index 0000000..794d8d9
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/reconcile.go
@@ -0,0 +1,212 @@
+package sqlstore
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "time"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+)
+
+// allocatorPosition returns the greatest repository ID allocated, 0 for none.
+// The Postgres sequence is not transactional: it counts the IDs of creations
+// whose row is not committed yet. SQLite's is: an uncommitted creation's ID is
+// not counted, and a rolled back one is allocated again.
+func (s *Store) allocatorPosition(ctx context.Context, q querier) (int64, error) {
+ query := "SELECT COALESCE((SELECT seq FROM sqlite_sequence WHERE name = 'graft_repositories'), 0)"
+ if s.dialect == Postgres {
+ query = "SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM graft_repositories_repo_id_seq"
+ }
+ var position int64
+ if err := q.QueryRowContext(ctx, query).Scan(&position); err != nil {
+ return 0, fmt.Errorf("read the repository ID allocator: %w", err)
+ }
+ return position, nil
+}
+
+// ObserveAllocator implements corestore.MetaStore.
+func (s *Store) ObserveAllocator(ctx context.Context, minAge time.Duration, dryRun bool) (corestore.AllocatorView, error) {
+ if minAge < 0 {
+ return corestore.AllocatorView{}, fmt.Errorf("storage core: negative allocation age %s", minAge)
+ }
+
+ var view corestore.AllocatorView
+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ view = corestore.AllocatorView{}
+
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+ position, err := s.allocatorPosition(ctx, tx)
+ if err != nil {
+ return err
+ }
+ if err := tx.QueryRowContext(ctx, "SELECT done_through FROM graft_gc_reconcile WHERE id = 1").Scan(&view.DoneThrough); err != nil {
+ return fmt.Errorf("read the reconciliation cursor: %w", err)
+ }
+
+ // The marks are few (the newest one at least minAge old and those
+ // after it), and compared here: SQLite stores times as text whose
+ // fractions do not sort.
+ type mark struct {
+ key any
+ observedAt time.Time
+ position int64
+ }
+ rows, err := tx.QueryContext(ctx, "SELECT observed_at, position FROM graft_gc_allocator_marks")
+ if err != nil {
+ return fmt.Errorf("read the allocator marks: %w", err)
+ }
+ var marks []mark
+ for rows.Next() {
+ var m mark
+ if err := rows.Scan(&m.key, &m.position); err != nil {
+ _ = rows.Close()
+ return fmt.Errorf("scan an allocator mark: %w", err)
+ }
+ if m.observedAt, err = scanTime(m.key); err != nil {
+ _ = rows.Close()
+ return err
+ }
+ marks = append(marks, m)
+ }
+ if err := rows.Close(); err != nil {
+ return fmt.Errorf("read the allocator marks: %w", err)
+ }
+ if err := rows.Err(); err != nil {
+ return fmt.Errorf("read the allocator marks: %w", err)
+ }
+
+ limit := now.Add(-minAge)
+ var newest *mark
+ for i := range marks {
+ m := &marks[i]
+ if m.observedAt.After(limit) {
+ continue
+ }
+ if m.position > view.Bound {
+ view.Bound = m.position
+ }
+ if newest == nil || m.observedAt.After(newest.observedAt) {
+ newest = m
+ }
+ }
+ view.Now, view.Position = now, position
+ if dryRun {
+ return nil
+ }
+
+ if _, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_gc_allocator_marks (observed_at, position) VALUES (?, ?)
+ ON CONFLICT (observed_at) DO NOTHING
+ `), s.timeArg(now), position); err != nil {
+ return fmt.Errorf("record the allocator mark: %w", err)
+ }
+ // A mark older than the newest eligible one can never give a greater
+ // bound: positions only grow.
+ for _, m := range marks {
+ if newest == nil || !m.observedAt.Before(newest.observedAt) {
+ continue
+ }
+ if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_gc_allocator_marks WHERE observed_at = ?"), m.key); err != nil {
+ return fmt.Errorf("drop an old allocator mark: %w", err)
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return corestore.AllocatorView{}, err
+ }
+ return view, nil
+}
+
+// UnrecordedIDs implements corestore.MetaStore.
+func (s *Store) UnrecordedIDs(ctx context.Context, after, upTo int64) ([]int64, error) {
+ if upTo <= after {
+ return nil, nil
+ }
+
+ var ids []int64
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ ids = nil
+
+ recorded := map[int64]struct{}{}
+ for _, query := range []string{
+ "SELECT repo_id FROM graft_repositories WHERE repo_id > ? AND repo_id <= ?",
+ "SELECT repo_id FROM graft_gc_removed WHERE repo_id > ? AND repo_id <= ?",
+ } {
+ rows, err := tx.QueryContext(ctx, s.bind(query), after, upTo)
+ if err != nil {
+ return fmt.Errorf("read the recorded repository IDs: %w", err)
+ }
+ for rows.Next() {
+ var id int64
+ if err := rows.Scan(&id); err != nil {
+ _ = rows.Close()
+ return fmt.Errorf("scan a repository ID: %w", err)
+ }
+ recorded[id] = struct{}{}
+ }
+ if err := rows.Close(); err != nil {
+ return fmt.Errorf("read the recorded repository IDs: %w", err)
+ }
+ if err := rows.Err(); err != nil {
+ return fmt.Errorf("read the recorded repository IDs: %w", err)
+ }
+ }
+ for id := after + 1; id <= upTo; id++ {
+ if _, ok := recorded[id]; !ok {
+ ids = append(ids, id)
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ return ids, nil
+}
+
+// RecordLeak implements corestore.MetaStore.
+func (s *Store) RecordLeak(ctx context.Context, repoID int64) (bool, error) {
+ var recorded bool
+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ recorded = false
+
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+ res, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)
+ SELECT ?, '', '', ? WHERE NOT EXISTS (SELECT 1 FROM graft_repositories WHERE repo_id = ?)
+ ON CONFLICT (repo_id) DO NOTHING
+ `), repoID, s.timeArg(now), repoID)
+ if err != nil {
+ return fmt.Errorf("record the tombstone of leaked repository %d: %w", repoID, err)
+ }
+ n, err := res.RowsAffected()
+ if err != nil {
+ return fmt.Errorf("record the tombstone of leaked repository %d: %w", repoID, err)
+ }
+ recorded = n == 1
+ return nil
+ })
+ if err != nil {
+ return false, err
+ }
+ return recorded, nil
+}
+
+// AdvanceReconcile implements corestore.MetaStore.
+func (s *Store) AdvanceReconcile(ctx context.Context, doneThrough int64) error {
+ return s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ if _, err := tx.ExecContext(ctx, s.bind("UPDATE graft_gc_reconcile SET done_through = ? WHERE id = 1 AND done_through < ?"),
+ doneThrough, doneThrough); err != nil {
+ return fmt.Errorf("advance the reconciliation cursor: %w", err)
+ }
+ return nil
+ })
+}
diff --git a/internal/graft/corestore/sqlstore/repository.go b/internal/graft/corestore/sqlstore/repository.go
index d7a5707..ed65581 100644
--- a/internal/graft/corestore/sqlstore/repository.go
+++ b/internal/graft/corestore/sqlstore/repository.go
@@ -442,24 +442,15 @@ func (s *Store) DeleteRepository(ctx context.Context, key corestore.RepoKey) (bo
return err
}
- now, err := s.wallTime(ctx, tx)
- if err != nil {
- return err
- }
-
- // The cascade removes everything recorded for the repository; the
- // tombstone, which has no foreign key, outlives it so that garbage
- // collection deletes the repository's objects from the bucket
- // (doc/graft/PLAN7.md, "Removed repositories").
+ // The cascade removes everything recorded for the repository. The
+ // trigger graft_repositories_tombstone records its tombstone, on the
+ // database clock, in this transaction; the tombstone has no foreign key
+ // and outlives the row, so that garbage collection deletes the
+ // repository's objects from the bucket (doc/graft/PLAN7.md, "Removed
+ // repositories"). A row deleted by hand gets one the same way.
if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_repositories WHERE repo_id = ?"), repo.id); err != nil {
return fmt.Errorf("delete repository: %w", err)
}
- if _, err := tx.ExecContext(ctx, s.bind(`
- INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)
- VALUES (?, ?, ?, ?)
- `), repo.id, repo.key.Storage, repo.key.RelativePath, s.timeArg(now)); err != nil {
- return fmt.Errorf("record the tombstone of repository %d: %w", repo.id, err)
- }
deleted = true
return nil
diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.go
index 42a5d00..fa78bdc 100644
--- a/internal/graft/corestore/types.go
+++ b/internal/graft/corestore/types.go
@@ -176,11 +176,31 @@ type Checkpoint struct {
// are still to be deleted from the bucket.
type RemovedRepository struct {
RepoID int64
- Key RepoKey
+ // Key is empty for a tombstone that the reconciliation recorded: the row
+ // was deleted without one, and its key with it.
+ Key RepoKey
// RemovedAt is a database time (doc/graft/PLAN7.md, CLOCK RULE).
RemovedAt time.Time
}
+// AllocatorView is an observation of the repository ID allocator by the
+// reconciliation of the bucket (MetaStore.ObserveAllocator).
+type AllocatorView struct {
+ // Now is the database time of the observation.
+ Now time.Time
+ // Position is the greatest repository ID allocated, 0 for none. On
+ // Postgres it counts the allocations of creations whose row is not
+ // committed yet, or never will be; on SQLite only committed ones.
+ Position int64
+ // Bound is the greatest position observed at least minAge before Now, 0
+ // when there is none: every ID at or below it was allocated at least
+ // minAge ago.
+ Bound int64
+ // DoneThrough is the reconciliation cursor: every ID at or below it is
+ // reconciled.
+ DoneThrough int64
+}
+
// GCState is what garbage collection decides on for one repository, read in
// one database snapshot (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 5).
type GCState struct {
diff --git a/internal/graft/e2e/storage_gc_test.go b/internal/graft/e2e/storage_gc_test.go
index 9f9ad89..bdc940b 100644
--- a/internal/graft/e2e/storage_gc_test.go
+++ b/internal/graft/e2e/storage_gc_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/sha256"
"fmt"
+ "strings"
"testing"
"time"
@@ -15,6 +16,33 @@ import (
"github.com/feder-cr/git-ai/sdk"
)
+// The tests age rows on the database clock instead of waiting with a short
+// gc_grace: the writers' deadline W is gc_grace - gc_margin, and an agent's
+// write under the race detector on a loaded host does not fit in a W of a
+// second or two.
+
+// exec runs a statement with "?" placeholders on the cluster's database.
+func (n storageNode) exec(t *testing.T, query string, args ...any) {
+ t.Helper()
+ store := n.core.Meta.(*sqlstore.Store)
+ if store.Dialect() == sqlstore.Postgres {
+ for i := 1; strings.Contains(query, "?"); i++ {
+ query = strings.Replace(query, "?", fmt.Sprintf("$%d", i), 1)
+ }
+ }
+ _, err := store.DB().ExecContext(n.ctx, query, args...)
+ require.NoError(t, err)
+}
+
+// longAgo returns the database time value of more than gc_grace before now.
+func (n storageNode) longAgo() any {
+ t := time.Now().Add(-n.core.GCGrace - time.Minute).UTC()
+ if n.core.Meta.(*sqlstore.Store).Dialect() == sqlstore.SQLite {
+ return t.Format(time.RFC3339Nano)
+ }
+ return t
+}
+
// TestStorageGCAcrossNodes runs the compaction of milestone 3 on a cluster of
// real servers sharing one database and one bucket (doc/graft/PLAN7.md): node
// A checkpoints a repository written on two nodes, node B collects the packs
@@ -26,12 +54,6 @@ func TestStorageGCAcrossNodes(t *testing.T) {
forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
graft, tokens := workerGraft(1)
cluster := newStorageCluster(t, dialect, corestore.ModeAuthoritative, unbatched(graft))
- // A short grace, so that the test sees packs collected. The writers'
- // deadline (grace - margin) stays long enough for an upload under load.
- cluster.adjust = func(core *corestore.Core) {
- core.GCGrace = 2 * time.Second
- core.GCMargin = 200 * time.Millisecond
- }
a := cluster.addNode(t)
b := cluster.addNode(t)
@@ -58,13 +80,14 @@ func TestStorageGCAcrossNodes(t *testing.T) {
require.Equal(t, a.record(t, rel).HeadSeq, cp.Sequence)
// Node B collects once the checkpoint is gc_grace old.
- var collected *sdk.StorageGC
- require.Eventually(t, func() bool {
- res, err := b.operator(t, rel).StorageGC(b.ctx, false)
- require.NoError(t, err)
- collected = res
- return res.Boundary == cp.Sequence && res.CoveredDeleted > 0
- }, 60*time.Second, 200*time.Millisecond)
+ repoID := a.record(t, rel).ID
+ collected, err := b.operator(t, rel).StorageGC(b.ctx, false)
+ require.NoError(t, err)
+ require.Zero(t, collected.Boundary, "the checkpoint is younger than gc_grace")
+ a.exec(t, "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ?", a.longAgo(), repoID)
+ collected, err = b.operator(t, rel).StorageGC(b.ctx, false)
+ require.NoError(t, err)
+ require.Equal(t, cp.Sequence, collected.Boundary)
require.Equal(t, uint32(len(before)), collected.CoveredDeleted, "every changeset pack is covered by the checkpoint")
inBucket := bucketPacks(t, cluster.bucketURL)
for key := range before {
@@ -100,27 +123,30 @@ func TestStorageGCAcrossNodes(t *testing.T) {
// An orphan (the pack of a write that died before its database commit)
// is deleted gc_grace after a collection observed it.
orphan := []byte("the pack of a write that died")
- orphanKey := corestore.NewPackKey(a.record(t, rel).ID, sha256.Sum256(orphan))
+ orphanKey := corestore.NewPackKey(repoID, sha256.Sum256(orphan))
require.NoError(t, a.core.Blobs.Put(a.ctx, orphanKey, bytes.NewReader(orphan)))
- require.Eventually(t, func() bool {
- res, err := a.operator(t, rel).StorageGC(a.ctx, false)
- require.NoError(t, err)
- return res.OrphansDeleted == 1
- }, 60*time.Second, 200*time.Millisecond)
+ res, err := a.operator(t, rel).StorageGC(a.ctx, false)
+ require.NoError(t, err)
+ require.Equal(t, uint32(1), res.OrphansObserved)
+ a.exec(t, "UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ?", a.longAgo(), repoID)
+ res, err = a.operator(t, rel).StorageGC(a.ctx, false)
+ require.NoError(t, err)
+ require.Equal(t, uint32(1), res.OrphansDeleted)
require.False(t, bucketPacks(t, cluster.bucketURL)[orphanKey])
// A removed repository's objects go once its tombstone is gc_grace old.
- repoID := a.record(t, rel).ID
_, err = gitalypb.NewRepositoryServiceClient(a.conn(t)).RemoveRepository(a.setupCtx, &gitalypb.RemoveRepositoryRequest{Repository: a.repository(rel)})
require.NoError(t, err)
client, err := sdk.Dial(a.ctx, a.cfg.SocketPath, sdk.WithStorage(a.cfg.Storages[0].Name), sdk.WithToken(sharedTestToken))
require.NoError(t, err)
t.Cleanup(func() { _ = client.Close() })
- require.Eventually(t, func() bool {
- res, err := client.StorageGCRemoved(a.ctx, "", false)
- require.NoError(t, err)
- return res.Repositories == 1
- }, 60*time.Second, 200*time.Millisecond)
+ removed, err := client.StorageGCRemoved(a.ctx, "", false)
+ require.NoError(t, err)
+ require.Zero(t, removed.Repositories, "the tombstone is younger than gc_grace")
+ a.exec(t, "UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", a.longAgo(), repoID)
+ removed, err = client.StorageGCRemoved(a.ctx, "", false)
+ require.NoError(t, err)
+ require.Equal(t, uint32(1), removed.Repositories)
prefix := fmt.Sprintf("repos/%d/", repoID)
for key := range bucketPacks(t, cluster.bucketURL) {
require.NotContains(t, key, prefix, "an object of the removed repository is left")
@@ -139,10 +165,6 @@ func TestStorageCheckpointBackfillsReviewRoots(t *testing.T) {
forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
graft, tokens := workerGraft(1)
cluster := newStorageCluster(t, dialect, corestore.ModeAuthoritative, unbatched(graft))
- cluster.adjust = func(core *corestore.Core) {
- core.GCGrace = 2 * time.Second
- core.GCMargin = 200 * time.Millisecond
- }
a := cluster.addNode(t)
rel := a.createRepository(t).GetRelativePath()
@@ -174,11 +196,11 @@ func TestStorageCheckpointBackfillsReviewRoots(t *testing.T) {
require.Equal(t, corestore.Target(base), snapshot[corestore.KeepAroundRef(base)], "the checkpoint holds the review base")
covered := a.referencedPacks(t, rel)
- require.Eventually(t, func() bool {
- res, err := a.operator(t, rel).StorageGC(a.ctx, false)
- require.NoError(t, err)
- return res.Boundary == cp.Sequence && res.CoveredDeleted > 0
- }, 60*time.Second, 200*time.Millisecond)
+ a.exec(t, "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ?", a.longAgo(), repoID)
+ res, err := a.operator(t, rel).StorageGC(a.ctx, false)
+ require.NoError(t, err)
+ require.Equal(t, cp.Sequence, res.Boundary)
+ require.Equal(t, uint32(len(covered)), res.CoveredDeleted)
inBucket := bucketPacks(t, cluster.bucketURL)
for key := range covered {
require.False(t, inBucket[key], "covered pack %s deleted", key)
diff --git a/proto/go/gitalypb/storage_core.pb.go b/proto/go/gitalypb/storage_core.pb.go
index 3623dd1..d557dbe 100644
--- a/proto/go/gitalypb/storage_core.pb.go
+++ b/proto/go/gitalypb/storage_core.pb.go
@@ -781,9 +781,20 @@ type CollectRemovedRepositoriesResponse struct {
// bytes_reclaimed is the size of those objects.
BytesReclaimed int64 `protobuf:"varint,3,opt,name=bytes_reclaimed,json=bytesReclaimed,proto3" json:"bytes_reclaimed,omitempty"`
// dry_run tells that nothing was deleted or recorded.
- DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
+ // leaks_found is the number of repository IDs the reconciliation found with
+ // objects in the bucket and neither a row nor a tombstone (a row deleted
+ // without one).
+ LeaksFound uint32 `protobuf:"varint,5,opt,name=leaks_found,json=leaksFound,proto3" json:"leaks_found,omitempty"`
+ // leaks_tombstoned is the number of tombstones the reconciliation recorded
+ // for them; their objects are deleted once the tombstone is gc_grace old.
+ LeaksTombstoned uint32 `protobuf:"varint,6,opt,name=leaks_tombstoned,json=leaksTombstoned,proto3" json:"leaks_tombstoned,omitempty"`
+ // ids_ahead_of_allocator is the number of repository IDs above the
+ // database's allocator that have objects in the bucket (a database restored
+ // from an older backup). They are never touched.
+ IdsAheadOfAllocator uint32 `protobuf:"varint,7,opt,name=ids_ahead_of_allocator,json=idsAheadOfAllocator,proto3" json:"ids_ahead_of_allocator,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
}
func (x *CollectRemovedRepositoriesResponse) Reset() {
@@ -844,6 +855,27 @@ func (x *CollectRemovedRepositoriesResponse) GetDryRun() bool {
return false
}
+func (x *CollectRemovedRepositoriesResponse) GetLeaksFound() uint32 {
+ if x != nil {
+ return x.LeaksFound
+ }
+ return 0
+}
+
+func (x *CollectRemovedRepositoriesResponse) GetLeaksTombstoned() uint32 {
+ if x != nil {
+ return x.LeaksTombstoned
+ }
+ return 0
+}
+
+func (x *CollectRemovedRepositoriesResponse) GetIdsAheadOfAllocator() uint32 {
+ if x != nil {
+ return x.IdsAheadOfAllocator
+ }
+ return 0
+}
+
// ReferenceDivergence is a reference whose local value differs from the
// database.
type VerifyRepositoryResponse_ReferenceDivergence struct {
@@ -983,12 +1015,16 @@ const file_storage_core_proto_rawDesc = "" +
"\adry_run\x18\t \x01(\bR\x06dryRun\"e\n" +
"!CollectRemovedRepositoriesRequest\x12'\n" +
"\fstorage_name\x18\x01 \x01(\tB\x04\x88\xc6,\x01R\vstorageName\x12\x17\n" +
- "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xaf\x01\n" +
+ "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xb0\x02\n" +
"\"CollectRemovedRepositoriesResponse\x12\"\n" +
"\frepositories\x18\x01 \x01(\rR\frepositories\x12#\n" +
"\rpacks_deleted\x18\x02 \x01(\rR\fpacksDeleted\x12'\n" +
"\x0fbytes_reclaimed\x18\x03 \x01(\x03R\x0ebytesReclaimed\x12\x17\n" +
- "\adry_run\x18\x04 \x01(\bR\x06dryRun2\xe5\x04\n" +
+ "\adry_run\x18\x04 \x01(\bR\x06dryRun\x12\x1f\n" +
+ "\vleaks_found\x18\x05 \x01(\rR\n" +
+ "leaksFound\x12)\n" +
+ "\x10leaks_tombstoned\x18\x06 \x01(\rR\x0fleaksTombstoned\x123\n" +
+ "\x16ids_ahead_of_allocator\x18\a \x01(\rR\x13idsAheadOfAllocator2\xe5\x04\n" +
"\x12StorageCoreService\x12]\n" +
"\x10ImportRepository\x12\x1f.gitaly.ImportRepositoryRequest\x1a .gitaly.ImportRepositoryResponse\"\x06\xfa\x97(\x02\b\x01\x12Z\n" +
"\x0fGetStorageState\x12\x1e.gitaly.GetStorageStateRequest\x1a\x1f.gitaly.GetStorageStateResponse\"\x06\xfa\x97(\x02\b\x02\x12]\n" +
diff --git a/proto/go/gitalypb/storage_core_grpc.pb.go b/proto/go/gitalypb/storage_core_grpc.pb.go
index 461fa24..0bfea99 100644
--- a/proto/go/gitalypb/storage_core_grpc.pb.go
+++ b/proto/go/gitalypb/storage_core_grpc.pb.go
@@ -67,8 +67,10 @@ type StorageCoreServiceClient interface {
// or while another maintenance of the repository holds its lease.
CollectGarbage(ctx context.Context, in *CollectGarbageRequest, opts ...grpc.CallOption) (*CollectGarbageResponse, error)
// CollectRemovedRepositories deletes every object of the repositories of a
- // storage that were removed at least gc_grace ago. It fails with
- // FailedPrecondition when the storage core is off.
+ // storage that were removed at least gc_grace ago. It first reconciles the
+ // bucket: a repository ID with objects and neither a row nor a tombstone
+ // gets a tombstone. It fails with FailedPrecondition when the storage core
+ // is off.
CollectRemovedRepositories(ctx context.Context, in *CollectRemovedRepositoriesRequest, opts ...grpc.CallOption) (*CollectRemovedRepositoriesResponse, error)
}
@@ -180,8 +182,10 @@ type StorageCoreServiceServer interface {
// or while another maintenance of the repository holds its lease.
CollectGarbage(context.Context, *CollectGarbageRequest) (*CollectGarbageResponse, error)
// CollectRemovedRepositories deletes every object of the repositories of a
- // storage that were removed at least gc_grace ago. It fails with
- // FailedPrecondition when the storage core is off.
+ // storage that were removed at least gc_grace ago. It first reconciles the
+ // bucket: a repository ID with objects and neither a row nor a tombstone
+ // gets a tombstone. It fails with FailedPrecondition when the storage core
+ // is off.
CollectRemovedRepositories(context.Context, *CollectRemovedRepositoriesRequest) (*CollectRemovedRepositoriesResponse, error)
mustEmbedUnimplementedStorageCoreServiceServer()
}
diff --git a/proto/storage_core.proto b/proto/storage_core.proto
index 50a1c24..5830b58 100644
--- a/proto/storage_core.proto
+++ b/proto/storage_core.proto
@@ -69,8 +69,10 @@ service StorageCoreService {
}
// CollectRemovedRepositories deletes every object of the repositories of a
- // storage that were removed at least gc_grace ago. It fails with
- // FailedPrecondition when the storage core is off.
+ // storage that were removed at least gc_grace ago. It first reconciles the
+ // bucket: a repository ID with objects and neither a row nor a tombstone
+ // gets a tombstone. It fails with FailedPrecondition when the storage core
+ // is off.
rpc CollectRemovedRepositories(CollectRemovedRepositoriesRequest) returns (CollectRemovedRepositoriesResponse) {
option (op_type) = {
op: MUTATOR
@@ -229,4 +231,15 @@ message CollectRemovedRepositoriesResponse {
int64 bytes_reclaimed = 3;
// dry_run tells that nothing was deleted or recorded.
bool dry_run = 4;
+ // leaks_found is the number of repository IDs the reconciliation found with
+ // objects in the bucket and neither a row nor a tombstone (a row deleted
+ // without one).
+ uint32 leaks_found = 5;
+ // leaks_tombstoned is the number of tombstones the reconciliation recorded
+ // for them; their objects are deleted once the tombstone is gc_grace old.
+ uint32 leaks_tombstoned = 6;
+ // ids_ahead_of_allocator is the number of repository IDs above the
+ // database's allocator that have objects in the bucket (a database restored
+ // from an older backup). They are never touched.
+ uint32 ids_ahead_of_allocator = 7;
}
diff --git a/sdk/storage.go b/sdk/storage.go
index 8e0ddd8..624bc1f 100644
--- a/sdk/storage.go
+++ b/sdk/storage.go
@@ -196,6 +196,16 @@ type StorageGCRemoved struct {
BytesReclaimed int64 `json:"bytes_reclaimed"`
// DryRun tells that nothing was deleted or recorded.
DryRun bool `json:"dry_run"`
+ // LeaksFound is the number of repository IDs the reconciliation found with
+ // objects in the bucket and neither a row nor a tombstone.
+ LeaksFound uint32 `json:"leaks_found"`
+ // LeaksTombstoned is the number of tombstones it recorded for them; their
+ // objects are deleted once the tombstone is gc_grace old.
+ LeaksTombstoned uint32 `json:"leaks_tombstoned"`
+ // IDsAheadOfAllocator is the number of repository IDs above the
+ // database's allocator that have objects in the bucket (a database
+ // restored from an older backup); they are never touched.
+ IDsAheadOfAllocator uint32 `json:"ids_ahead_of_allocator"`
}
// StorageCheckpoint checkpoints the repository at its current database head:
@@ -244,7 +254,9 @@ func (r *Repo) StorageGC(ctx context.Context, dryRun bool) (*StorageGC, error) {
}
// StorageGCRemoved deletes every object of the repositories of the storage
-// that were removed at least gc_grace ago (with dryRun, only reports them). An
+// that were removed at least gc_grace ago (with dryRun, only reports them),
+// after it reconciled the bucket (a repository ID with objects and neither a
+// row nor a tombstone gets a tombstone). An
// empty storage is the client's storage (WithStorage). It fails with
// FailedPrecondition when the storage core is off.
func (c *Client) StorageGCRemoved(ctx context.Context, storage string, dryRun bool) (*StorageGCRemoved, error) {
@@ -259,10 +271,13 @@ func (c *Client) StorageGCRemoved(ctx context.Context, storage string, dryRun bo
return nil, err
}
return &StorageGCRemoved{
- Repositories: resp.GetRepositories(),
- PacksDeleted: resp.GetPacksDeleted(),
- BytesReclaimed: resp.GetBytesReclaimed(),
- DryRun: resp.GetDryRun(),
+ Repositories: resp.GetRepositories(),
+ PacksDeleted: resp.GetPacksDeleted(),
+ BytesReclaimed: resp.GetBytesReclaimed(),
+ DryRun: resp.GetDryRun(),
+ LeaksFound: resp.GetLeaksFound(),
+ LeaksTombstoned: resp.GetLeaksTombstoned(),
+ IDsAheadOfAllocator: resp.GetIdsAheadOfAllocator(),
}, nil
}
diff --git a/sdk/storage_test.go b/sdk/storage_test.go
index 0ebc310..6ef140e 100644
--- a/sdk/storage_test.go
+++ b/sdk/storage_test.go
@@ -156,9 +156,12 @@ func TestStorageGC(t *testing.T) {
func TestStorageGCRemoved(t *testing.T) {
fake := &fakeStorageCore{removedResp: &gitalypb.CollectRemovedRepositoriesResponse{
- Repositories: 2,
- PacksDeleted: 9,
- BytesReclaimed: 4096,
+ Repositories: 2,
+ PacksDeleted: 9,
+ BytesReclaimed: 4096,
+ LeaksFound: 3,
+ LeaksTombstoned: 2,
+ IdsAheadOfAllocator: 1,
}}
client := dialFake(t, func(srv *grpc.Server) {
gitalypb.RegisterStorageCoreServiceServer(srv, fake)
@@ -166,7 +169,10 @@ func TestStorageGCRemoved(t *testing.T) {
res, err := client.StorageGCRemoved(t.Context(), "", false)
require.NoError(t, err)
- require.Equal(t, &sdk.StorageGCRemoved{Repositories: 2, PacksDeleted: 9, BytesReclaimed: 4096}, res)
+ require.Equal(t, &sdk.StorageGCRemoved{
+ Repositories: 2, PacksDeleted: 9, BytesReclaimed: 4096,
+ LeaksFound: 3, LeaksTombstoned: 2, IDsAheadOfAllocator: 1,
+ }, res)
_, err = client.StorageGCRemoved(t.Context(), "other", true)
require.NoError(t, err)