Skip to content

Git designed for AI agents.

sqlstore: milestone-3 schema and store methods (PLAN7 WP71)

Federico Elia4786cd768175
diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.go
index 6f4ac60..7097217 100644
--- a/internal/graft/corestore/metastore.go
+++ b/internal/graft/corestore/metastore.go
@@ -3,6 +3,7 @@ package corestore
import (
"context"
"errors"
+ "time"
"github.com/feder-cr/git-ai/internal/graft/changelog"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
@@ -67,6 +68,23 @@ type InitialState struct {
Reviews []*gitalypb.Review
}
+// CheckpointInput is the state MetaStore.CreateCheckpoint records.
+type CheckpointInput struct {
+ // Seq is the changeset sequence of the reference snapshot: 1..head sequence,
+ // greater than the newest checkpoint's.
+ Seq uint64
+ // Refs is the reference snapshot at Seq (every existing reference, HEAD
+ // included); it holds no absent value and is not empty.
+ Refs map[string]Target
+ // PackKey is the owned key (NewPackKey) of the already uploaded consolidated
+ // pack, PackBytes its size.
+ PackKey string
+ PackBytes int64
+ // Roots are the head and base commits of the open reviews that the pack
+ // includes (doc/graft/PLAN7.md, REVIEW ROOTS RULE R2).
+ Roots []string
+}
+
var (
// ErrReviewNotFound is returned for an unknown review ID.
ErrReviewNotFound = errors.New("storage core: review not found")
@@ -84,9 +102,15 @@ type MetaStore interface {
RegisterRepository(ctx context.Context, key RepoKey, objectFormat string, init InitialState) (Repository, CommitResult, error)
// GetRepository returns ErrRepositoryNotFound for an unknown key.
GetRepository(ctx context.Context, key RepoKey) (Repository, error)
- // DeleteRepository removes the repository and everything recorded for it.
- // It returns false when the repository was not known.
+ // DeleteRepository removes the repository and everything recorded for it,
+ // and records a tombstone (ListRemovedRepositories) in the same transaction
+ // so that garbage collection deletes its objects from the bucket. It returns
+ // false, writing nothing, when the repository was not known.
DeleteRepository(ctx context.Context, key RepoKey) (bool, error)
+ // ListRepositories returns the repositories of the storage whose import is
+ // complete (head sequence > 0), ascending by ID; every storage's when storage
+ // is empty.
+ ListRepositories(ctx context.Context, storage string) ([]Repository, error)
// CommitChangeset checks every update against the current references and
// records the changeset, the reference changes and the changelog entries
@@ -135,6 +159,61 @@ type MetaStore interface {
// version+1. It returns ErrReviewVersionConflict otherwise.
UpdateReview(ctx context.Context, repoID int64, id string, version int64, r *gitalypb.Review) error
+ // CreateCheckpoint records a checkpoint and its reference snapshot in one
+ // transaction, with the database time as its creation time. Its first
+ // statement locks the repository row (doc/graft/PLAN7.md, LOCK ORDER). It
+ // requires Seq <= head sequence, Seq greater than the newest checkpoint's,
+ // an owned pack key of the repository and a valid snapshot, else an error and
+ // nothing is written. A checkpoint at Seq that exists already is returned
+ // unchanged with created false (its pack is kept; the caller's becomes an
+ // orphan). It requires every open review's head and base commit to be in
+ // Roots, else ErrCheckpointRootsChanged and nothing is written.
+ // ErrRepositoryNotFound for an unknown repository.
+ CreateCheckpoint(ctx context.Context, repoID int64, in CheckpointInput) (cp Checkpoint, created bool, err error)
+ // NewestCheckpoint returns the checkpoint with the greatest sequence; ok is
+ // false when the repository has none.
+ NewestCheckpoint(ctx context.Context, repoID int64) (cp Checkpoint, ok bool, err error)
+ // SyncBoundary returns the sequence of the newest checkpoint created at
+ // least age ago on the database clock, 0 when there is none
+ // (doc/graft/PLAN7.md, GC BOUNDARY rule). ErrRepositoryNotFound for an unknown
+ // repository.
+ SyncBoundary(ctx context.Context, repoID int64, age time.Duration) (uint64, error)
+ // CheckpointRefs returns the reference snapshot of the checkpoint at seq,
+ // ErrCheckpointNotFound when there is none.
+ CheckpointRefs(ctx context.Context, repoID int64, seq uint64) (map[string]Target, error)
+ // DeleteCheckpoint removes the checkpoint at seq and its snapshot; false when
+ // there was none. The caller deletes the pack afterwards.
+ DeleteCheckpoint(ctx context.Context, repoID int64, seq uint64) (bool, error)
+
+ // GCSnapshot reads what garbage collection decides on, in one read
+ // transaction (see GCState). ErrRepositoryNotFound for an unknown repository.
+ GCSnapshot(ctx context.Context, repoID int64) (GCState, error)
+ // RecordOrphans records the database time as the first observation of every
+ // key that has none; existing observations are kept.
+ RecordOrphans(ctx context.Context, repoID int64, keys []string) error
+ // ForgetOrphans removes the observations of the keys.
+ ForgetOrphans(ctx context.Context, repoID int64, keys []string) error
+
+ // AcquireMaintenanceLease takes the repository's maintenance lease for
+ // holder until the database time plus ttl, by compare-and-swap: it succeeds
+ // when there is no lease, when it has expired, or when holder holds it.
+ // RenewMaintenanceLease extends it when holder holds it (held false
+ // otherwise); ReleaseMaintenanceLease frees it when holder holds it. A lease
+ // only keeps two nodes from doing the same maintenance at once: no read,
+ // write or correctness argument depends on it. ErrRepositoryNotFound for an
+ // unknown repository, except from ReleaseMaintenanceLease, which has nothing
+ // to free then.
+ AcquireMaintenanceLease(ctx context.Context, repoID int64, holder string, ttl time.Duration) (acquired bool, err error)
+ RenewMaintenanceLease(ctx context.Context, repoID int64, holder string, ttl time.Duration) (held bool, err error)
+ ReleaseMaintenanceLease(ctx context.Context, repoID int64, holder string) error
+
+ // ListRemovedRepositories returns the tombstones of the storage (of every
+ // storage when it is empty) recorded at least minAge ago on the database
+ // clock, ascending by repository ID.
+ ListRemovedRepositories(ctx context.Context, storage string, minAge time.Duration) ([]RemovedRepository, error)
+ // DeleteRemovedRepository removes a tombstone; a missing one is success.
+ DeleteRemovedRepository(ctx context.Context, repoID int64) error
+
// Close releases the database.
Close() error
}
diff --git a/internal/graft/corestore/sqlstore/checkpoint.go b/internal/graft/corestore/sqlstore/checkpoint.go
new file mode 100644
index 0000000..5787967
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/checkpoint.go
@@ -0,0 +1,376 @@
+package sqlstore
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "math"
+ "sort"
+ "time"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+)
+
+// CreateCheckpoint implements corestore.MetaStore.
+func (s *Store) CreateCheckpoint(ctx context.Context, repoID int64, in corestore.CheckpointInput) (corestore.Checkpoint, bool, error) {
+ if err := validateCheckpoint(repoID, in); err != nil {
+ return corestore.Checkpoint{}, false, err
+ }
+
+ var (
+ cp corestore.Checkpoint
+ created bool
+ )
+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ cp, created = corestore.Checkpoint{}, false
+
+ // LOCK ORDER (doc/graft/PLAN7.md): the repository row first. It serializes
+ // the checkpoint with changesets and with the review writes that move a
+ // review's commits, which the roots check below relies on.
+ repo, err := s.selectRepository(ctx, tx, "repo_id = ?", []any{repoID}, true)
+ if err != nil {
+ return err
+ }
+ if in.Seq > repo.headSeq {
+ return fmt.Errorf("storage core: checkpoint at changeset %d is beyond the head %d of repository %d", in.Seq, repo.headSeq, repoID)
+ }
+
+ existing, found, err := s.checkpointAt(ctx, tx, repoID, in.Seq)
+ if err != nil {
+ return err
+ }
+ if found {
+ cp = existing
+ return nil
+ }
+
+ newest, ok, err := s.newestCheckpoint(ctx, tx, repoID)
+ if err != nil {
+ return err
+ }
+ if ok && newest.Seq > in.Seq {
+ return fmt.Errorf("storage core: checkpoint at changeset %d is older than the newest checkpoint at %d of repository %d", in.Seq, newest.Seq, repoID)
+ }
+
+ if err := s.checkReviewRoots(ctx, tx, repoID, in.Roots); err != nil {
+ return err
+ }
+
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+
+ if _, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_checkpoints (repo_id, seq, pack_key, pack_bytes, created_at)
+ VALUES (?, ?, ?, ?, ?)
+ `), repoID, int64(in.Seq), in.PackKey, in.PackBytes, s.timeArg(now)); err != nil {
+ return fmt.Errorf("insert checkpoint: %w", err)
+ }
+
+ names := make([]string, 0, len(in.Refs))
+ for name := range in.Refs {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ for ordinal, name := range names {
+ if _, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_checkpoint_refs (repo_id, seq, ordinal, name, target)
+ VALUES (?, ?, ?, ?, ?)
+ `), repoID, int64(in.Seq), ordinal, bytesArg([]byte(name)), string(in.Refs[name])); err != nil {
+ return fmt.Errorf("insert checkpoint reference: %w", err)
+ }
+ }
+
+ cp = corestore.Checkpoint{
+ RepoID: repoID,
+ Seq: in.Seq,
+ PackKey: in.PackKey,
+ PackBytes: in.PackBytes,
+ CreatedAt: now,
+ }
+ created = true
+ return nil
+ })
+ if err != nil {
+ return corestore.Checkpoint{}, false, err
+ }
+
+ return cp, created, nil
+}
+
+// validateCheckpoint checks a checkpoint before anything is read.
+func validateCheckpoint(repoID int64, in corestore.CheckpointInput) error {
+ if in.Seq == 0 || in.Seq > math.MaxInt64 {
+ return fmt.Errorf("storage core: invalid checkpoint sequence %d", in.Seq)
+ }
+
+ id, _, ok := corestore.ParsePackKey(in.PackKey)
+ if !ok || id != repoID || !corestore.IsOwnedPackKey(in.PackKey) {
+ return fmt.Errorf("storage core: checkpoint pack key %q is not an owned pack key of repository %d", in.PackKey, repoID)
+ }
+ if in.PackBytes < 0 {
+ return fmt.Errorf("storage core: invalid checkpoint pack size %d", in.PackBytes)
+ }
+
+ if len(in.Refs) == 0 {
+ return errors.New("storage core: checkpoint has no references")
+ }
+ for name, target := range in.Refs {
+ if name == "" {
+ return errors.New("storage core: checkpoint reference with an empty name")
+ }
+ // A snapshot holds the references that exist; the target column is text
+ // in both dialects (see validateChangeset).
+ if target.Absent() || !storableText(string(target)) {
+ return fmt.Errorf("storage core: checkpoint reference %q has an invalid target %q", name, target)
+ }
+ }
+
+ for _, root := range in.Roots {
+ if root == "" || !storableText(root) {
+ return fmt.Errorf("storage core: invalid checkpoint root %q", root)
+ }
+ }
+
+ return nil
+}
+
+// checkReviewRoots refuses the checkpoint when an open review has a head or
+// base commit that is not one of the roots the checkpoint's pack includes: the
+// review was created, or moved its head, during the build (doc/graft/PLAN7.md,
+// REVIEW ROOTS RULE R2). The caller holds the repository row, which every
+// review write that moves a review's commits takes first, so the reviews read
+// here are the ones the checkpoint commits with.
+func (s *Store) checkReviewRoots(ctx context.Context, tx *sql.Tx, repoID int64, roots []string) error {
+ included := make(map[string]struct{}, len(roots))
+ for _, root := range roots {
+ included[root] = struct{}{}
+ }
+
+ rows, err := tx.QueryContext(ctx, s.bind("SELECT body FROM graft_reviews WHERE repo_id = ? AND closed = ?"), repoID, false)
+ if err != nil {
+ return fmt.Errorf("read open reviews: %w", err)
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var body string
+ if err := rows.Scan(&body); err != nil {
+ return fmt.Errorf("scan review: %w", err)
+ }
+ review, err := unmarshalReview(body)
+ if err != nil {
+ return err
+ }
+ for _, commit := range []string{review.GetHeadCommitId(), review.GetBaseCommitId()} {
+ if commit == "" {
+ continue
+ }
+ if _, ok := included[commit]; !ok {
+ return fmt.Errorf("review %s commit %s: %w", review.GetId(), commit, corestore.ErrCheckpointRootsChanged)
+ }
+ }
+ }
+
+ return rows.Err()
+}
+
+// checkpointColumns are the columns read by scanCheckpoint, in order.
+const checkpointColumns = "seq, pack_key, pack_bytes, created_at"
+
+// scanCheckpoint reads a row of checkpointColumns.
+func scanCheckpoint(repoID int64, scan func(...any) error) (corestore.Checkpoint, error) {
+ var (
+ seq int64
+ cp corestore.Checkpoint
+ createdRaw any
+ )
+ if err := scan(&seq, &cp.PackKey, &cp.PackBytes, &createdRaw); err != nil {
+ return corestore.Checkpoint{}, err
+ }
+
+ created, err := scanTime(createdRaw)
+ if err != nil {
+ return corestore.Checkpoint{}, err
+ }
+
+ cp.RepoID = repoID
+ cp.Seq = uint64(seq)
+ cp.CreatedAt = created
+ return cp, nil
+}
+
+// checkpointAt reads the checkpoint at seq; found is false when there is none.
+func (s *Store) checkpointAt(ctx context.Context, q querier, repoID int64, seq uint64) (corestore.Checkpoint, bool, error) {
+ cp, err := scanCheckpoint(repoID, q.QueryRowContext(ctx, s.bind(
+ "SELECT "+checkpointColumns+" FROM graft_checkpoints WHERE repo_id = ? AND seq = ?"),
+ repoID, int64(seq)).Scan)
+ if errors.Is(err, sql.ErrNoRows) {
+ return corestore.Checkpoint{}, false, nil
+ }
+ if err != nil {
+ return corestore.Checkpoint{}, false, fmt.Errorf("read checkpoint: %w", err)
+ }
+ return cp, true, nil
+}
+
+// newestCheckpoint reads the checkpoint with the greatest sequence; ok is false
+// when there is none.
+func (s *Store) newestCheckpoint(ctx context.Context, q querier, repoID int64) (corestore.Checkpoint, bool, error) {
+ cp, err := scanCheckpoint(repoID, q.QueryRowContext(ctx, s.bind(
+ "SELECT "+checkpointColumns+" FROM graft_checkpoints WHERE repo_id = ? ORDER BY seq DESC LIMIT 1"),
+ repoID).Scan)
+ if errors.Is(err, sql.ErrNoRows) {
+ return corestore.Checkpoint{}, false, nil
+ }
+ if err != nil {
+ return corestore.Checkpoint{}, false, fmt.Errorf("read newest checkpoint: %w", err)
+ }
+ return cp, true, nil
+}
+
+// listCheckpoints reads every checkpoint of the repository, ascending by
+// sequence.
+func (s *Store) listCheckpoints(ctx context.Context, q querier, repoID int64) ([]corestore.Checkpoint, error) {
+ rows, err := q.QueryContext(ctx, s.bind(
+ "SELECT "+checkpointColumns+" FROM graft_checkpoints WHERE repo_id = ? ORDER BY seq"), repoID)
+ if err != nil {
+ return nil, fmt.Errorf("list checkpoints: %w", err)
+ }
+ defer rows.Close()
+
+ var checkpoints []corestore.Checkpoint
+ for rows.Next() {
+ cp, err := scanCheckpoint(repoID, rows.Scan)
+ if err != nil {
+ return nil, fmt.Errorf("scan checkpoint: %w", err)
+ }
+ checkpoints = append(checkpoints, cp)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, fmt.Errorf("list checkpoints: %w", err)
+ }
+ return checkpoints, nil
+}
+
+// requireRepository returns ErrRepositoryNotFound when the repository does not
+// exist.
+func (s *Store) requireRepository(ctx context.Context, q querier, repoID int64) error {
+ var id int64
+ err := q.QueryRowContext(ctx, s.bind("SELECT repo_id FROM graft_repositories WHERE repo_id = ?"), repoID).Scan(&id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return corestore.ErrRepositoryNotFound
+ }
+ if err != nil {
+ return fmt.Errorf("read repository: %w", err)
+ }
+ return nil
+}
+
+// NewestCheckpoint implements corestore.MetaStore.
+func (s *Store) NewestCheckpoint(ctx context.Context, repoID int64) (corestore.Checkpoint, bool, error) {
+ var (
+ cp corestore.Checkpoint
+ ok bool
+ )
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ var err error
+ cp, ok, err = s.newestCheckpoint(ctx, tx, repoID)
+ return err
+ })
+ if err != nil {
+ return corestore.Checkpoint{}, false, err
+ }
+ return cp, ok, nil
+}
+
+// SyncBoundary implements corestore.MetaStore.
+func (s *Store) SyncBoundary(ctx context.Context, repoID int64, age time.Duration) (uint64, error) {
+ if age < 0 {
+ return 0, fmt.Errorf("storage core: negative checkpoint age %s", age)
+ }
+
+ hostNow := time.Now()
+ var boundary uint64
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ boundary = 0
+
+ now, err := s.snapshotTime(ctx, tx, hostNow)
+ if err != nil {
+ return err
+ }
+ if err := s.requireRepository(ctx, tx, repoID); err != nil {
+ return err
+ }
+ checkpoints, err := s.listCheckpoints(ctx, tx, repoID)
+ if err != nil {
+ return err
+ }
+
+ limit := now.Add(-age)
+ for _, cp := range checkpoints {
+ if !cp.CreatedAt.After(limit) && cp.Seq > boundary {
+ boundary = cp.Seq
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return 0, err
+ }
+ return boundary, nil
+}
+
+// CheckpointRefs implements corestore.MetaStore.
+func (s *Store) CheckpointRefs(ctx context.Context, repoID int64, seq uint64) (map[string]corestore.Target, error) {
+ if seq > math.MaxInt64 {
+ return nil, corestore.ErrCheckpointNotFound
+ }
+
+ var refs map[string]corestore.Target
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ refs = map[string]corestore.Target{}
+
+ if _, found, err := s.checkpointAt(ctx, tx, repoID, seq); err != nil {
+ return err
+ } else if !found {
+ return corestore.ErrCheckpointNotFound
+ }
+
+ return readReferences(ctx, s, tx,
+ "SELECT name, target FROM graft_checkpoint_refs WHERE repo_id = ? AND seq = ? ORDER BY ordinal",
+ []any{repoID, int64(seq)}, refs)
+ })
+ if err != nil {
+ return nil, err
+ }
+ return refs, nil
+}
+
+// DeleteCheckpoint implements corestore.MetaStore. It touches only the
+// checkpoint rows and never the repository row (doc/graft/PLAN7.md, LOCK ORDER).
+func (s *Store) DeleteCheckpoint(ctx context.Context, repoID int64, seq uint64) (bool, error) {
+ if seq > math.MaxInt64 {
+ return false, nil
+ }
+
+ var deleted bool
+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ res, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_checkpoints WHERE repo_id = ? AND seq = ?"), repoID, int64(seq))
+ if err != nil {
+ return fmt.Errorf("delete checkpoint: %w", err)
+ }
+ n, err := res.RowsAffected()
+ if err != nil {
+ return fmt.Errorf("delete checkpoint: %w", err)
+ }
+ deleted = n > 0
+ return nil
+ })
+ if err != nil {
+ return false, err
+ }
+ return deleted, nil
+}
diff --git a/internal/graft/corestore/sqlstore/clock.go b/internal/graft/corestore/sqlstore/clock.go
new file mode 100644
index 0000000..4b05a3a
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/clock.go
@@ -0,0 +1,47 @@
+package sqlstore
+
+import (
+ "context"
+ "fmt"
+ "time"
+)
+
+// The CLOCK RULE of doc/graft/PLAN7.md: every time that decides a garbage
+// collection action (a checkpoint's creation, an orphan's first observation, a
+// tombstone, a lease) is a time of the database, written and compared on the
+// database's clock, so that the clocks of the nodes never enter the decision.
+// SQLite serves the nodes of one host, whose clock it is.
+
+// wallTime returns the current database time, for a time that a write records.
+// On Postgres it is clock_timestamp(), the time of the statement rather than of
+// the start of the transaction, so that a recorded time is never earlier than
+// what the transaction waited for before. On SQLite it is the host clock.
+func (s *Store) wallTime(ctx context.Context, q querier) (time.Time, error) {
+ if s.dialect != Postgres {
+ return time.Now().UTC(), nil
+ }
+
+ var raw any
+ if err := q.QueryRowContext(ctx, "SELECT clock_timestamp()").Scan(&raw); err != nil {
+ return time.Time{}, fmt.Errorf("read the database time: %w", err)
+ }
+ return scanTime(raw)
+}
+
+// snapshotTime returns the database time of a read transaction: a time no
+// later than its snapshot, so that every row committed before that time is
+// visible in it. On Postgres it is now(), the start of the transaction, which
+// precedes the snapshot taken by the transaction's first statement. On SQLite
+// it is hostNow, the host clock that the caller read before the transaction
+// began (a deferred transaction takes its snapshot at its first read).
+func (s *Store) snapshotTime(ctx context.Context, q querier, hostNow time.Time) (time.Time, error) {
+ if s.dialect != Postgres {
+ return hostNow.UTC(), nil
+ }
+
+ var raw any
+ if err := q.QueryRowContext(ctx, "SELECT now()").Scan(&raw); err != nil {
+ return time.Time{}, fmt.Errorf("read the database time: %w", err)
+ }
+ return scanTime(raw)
+}
diff --git a/internal/graft/corestore/sqlstore/export_test.go b/internal/graft/corestore/sqlstore/export_test.go
new file mode 100644
index 0000000..ac033c2
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/export_test.go
@@ -0,0 +1,31 @@
+package sqlstore
+
+import (
+ "context"
+ "errors"
+
+ "github.com/jackc/pgx/v5/pgconn"
+)
+
+// OpenV010 opens the database with only the schema of v0.1.0 (the first
+// migration), as a v0.1.0 binary left it: no checkpoints, no tombstones and no
+// owned-key trigger.
+func OpenV010(ctx context.Context, dsn string) (*Store, error) {
+ return open(ctx, dsn, 1)
+}
+
+// SetRetryHook installs fn as the hook called for every retried transaction and
+// returns the function that removes it.
+func SetRetryHook(fn func(err error)) (restore func()) {
+ retryHook = fn
+ return func() { retryHook = nil }
+}
+
+// IsDeadlock reports whether err is a Postgres deadlock (40P01).
+func IsDeadlock(err error) bool {
+ var pe *pgconn.PgError
+ return errors.As(err, &pe) && pe.Code == "40P01"
+}
+
+// NotOwnedPackKeyMessage is the documented error of the owned-key trigger.
+const NotOwnedPackKeyMessage = notOwnedPackKeyMessage
diff --git a/internal/graft/corestore/sqlstore/gc.go b/internal/graft/corestore/sqlstore/gc.go
new file mode 100644
index 0000000..de77b73
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/gc.go
@@ -0,0 +1,165 @@
+package sqlstore
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+)
+
+// GCSnapshot implements corestore.MetaStore. Everything is read in one read
+// transaction, with the database time of its snapshot: the garbage collector's
+// proof that no live pack is deleted compares that time with the observations
+// and relies on every row committed before it being visible
+// (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 5).
+func (s *Store) GCSnapshot(ctx context.Context, repoID int64) (corestore.GCState, error) {
+ hostNow := time.Now()
+
+ var st corestore.GCState
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ st = corestore.GCState{Packs: map[string]uint64{}, Orphans: map[string]time.Time{}}
+
+ now, err := s.snapshotTime(ctx, tx, hostNow)
+ if err != nil {
+ return err
+ }
+ st.Now = now
+
+ if err := s.requireRepository(ctx, tx, repoID); err != nil {
+ return err
+ }
+
+ // The greatest referencing sequence decides a key's liveness: a legacy key
+ // can be shared by several changesets (doc/graft/PLAN7.md, change G).
+ rows, err := tx.QueryContext(ctx, s.bind(`
+ SELECT pack_key, MAX(seq) FROM graft_changesets
+ WHERE repo_id = ? AND pack_key IS NOT NULL
+ GROUP BY pack_key
+ `), repoID)
+ if err != nil {
+ return fmt.Errorf("read changeset packs: %w", err)
+ }
+ for rows.Next() {
+ var (
+ key string
+ seq int64
+ )
+ if err := rows.Scan(&key, &seq); err != nil {
+ rows.Close()
+ return fmt.Errorf("scan changeset pack: %w", err)
+ }
+ st.Packs[key] = uint64(seq)
+ }
+ if err := rows.Err(); err != nil {
+ rows.Close()
+ return fmt.Errorf("read changeset packs: %w", err)
+ }
+ rows.Close()
+
+ if st.Checkpoints, err = s.listCheckpoints(ctx, tx, repoID); err != nil {
+ return err
+ }
+
+ orphans, err := tx.QueryContext(ctx, s.bind("SELECT pack_key, first_seen FROM graft_gc_orphans WHERE repo_id = ?"), repoID)
+ if err != nil {
+ return fmt.Errorf("read orphan observations: %w", err)
+ }
+ defer orphans.Close()
+ for orphans.Next() {
+ var (
+ key string
+ seenRaw any
+ )
+ if err := orphans.Scan(&key, &seenRaw); err != nil {
+ return fmt.Errorf("scan orphan observation: %w", err)
+ }
+ seen, err := scanTime(seenRaw)
+ if err != nil {
+ return err
+ }
+ st.Orphans[key] = seen
+ }
+ return orphans.Err()
+ })
+ if err != nil {
+ return corestore.GCState{}, err
+ }
+ return st, nil
+}
+
+// RecordOrphans implements corestore.MetaStore. The first observation is the
+// database time of the insert, which is later than the listing that found the
+// keys. It touches only observation rows (doc/graft/PLAN7.md, LOCK ORDER).
+func (s *Store) RecordOrphans(ctx context.Context, repoID int64, keys []string) error {
+ if len(keys) == 0 {
+ return nil
+ }
+ if err := validateOrphanKeys(keys); err != nil {
+ return err
+ }
+
+ return s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+
+ for _, key := range keys {
+ if _, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_gc_orphans (repo_id, pack_key, first_seen)
+ VALUES (?, ?, ?)
+ ON CONFLICT (repo_id, pack_key) DO NOTHING
+ `), repoID, key, s.timeArg(now)); err != nil {
+ return fmt.Errorf("record orphan observation: %w", err)
+ }
+ }
+ return nil
+ })
+}
+
+// ForgetOrphans implements corestore.MetaStore. It touches only observation
+// rows (doc/graft/PLAN7.md, LOCK ORDER).
+func (s *Store) ForgetOrphans(ctx context.Context, repoID int64, keys []string) error {
+ if len(keys) == 0 {
+ return nil
+ }
+ if err := validateOrphanKeys(keys); err != nil {
+ return err
+ }
+
+ return s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ for start := 0; start < len(keys); start += maxNamesPerQuery {
+ chunk := keys[start:min(start+maxNamesPerQuery, len(keys))]
+
+ args := make([]any, 0, len(chunk)+1)
+ args = append(args, repoID)
+ placeholders := make([]string, len(chunk))
+ for i, key := range chunk {
+ placeholders[i] = "?"
+ args = append(args, key)
+ }
+
+ if _, err := tx.ExecContext(ctx, s.bind(
+ "DELETE FROM graft_gc_orphans WHERE repo_id = ? AND pack_key IN ("+strings.Join(placeholders, ", ")+")"),
+ args...); err != nil {
+ return fmt.Errorf("forget orphan observations: %w", err)
+ }
+ }
+ return nil
+ })
+}
+
+// validateOrphanKeys checks that the keys can be stored in the text column of
+// both dialects: an observation that cannot be recorded would make its object
+// impossible to collect, so such a key is an error instead of being skipped.
+func validateOrphanKeys(keys []string) error {
+ for _, key := range keys {
+ if key == "" || !storableText(key) {
+ return fmt.Errorf("storage core: bucket key %q cannot be recorded as an orphan", key)
+ }
+ }
+ return nil
+}
diff --git a/internal/graft/corestore/sqlstore/gc_test.go b/internal/graft/corestore/sqlstore/gc_test.go
new file mode 100644
index 0000000..2eabb60
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/gc_test.go
@@ -0,0 +1,694 @@
+package sqlstore_test
+
+import (
+ "context"
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/coretest"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/sync/errgroup"
+)
+
+// ownedKey returns a new owned pack key of the repository for content.
+func ownedKey(repoID int64, content string) string {
+ return corestore.NewPackKey(repoID, sha256.Sum256([]byte(content)))
+}
+
+// legacyKey returns a key in the format of v0.1.0, which shared pack objects.
+func legacyKey(repoID int64, content string) string {
+ return fmt.Sprintf("repos/%d/packs/%x.pack", repoID, sha256.Sum256([]byte(content)))
+}
+
+// commitPack commits one reference update with a pack key.
+func commitPack(ctx context.Context, s *sqlstore.Store, repoID int64, u corestore.RefUpdate, packKey string) (corestore.CommitResult, error) {
+ return s.CommitChangeset(ctx, corestore.ChangesetInput{
+ RepoID: repoID,
+ Updates: []corestore.RefUpdate{u},
+ Operation: "test",
+ Actor: "tester",
+ NodeID: "node-a",
+ PackKey: packKey,
+ PackBytes: 1,
+ })
+}
+
+// execSQL runs a statement written with "?" placeholders directly on the
+// database of the store, for tests that set up rows the API cannot (an old
+// time, a row of an earlier version).
+func execSQL(t *testing.T, ctx context.Context, s *sqlstore.Store, query string, args ...any) {
+ t.Helper()
+
+ if s.Dialect() == sqlstore.Postgres {
+ var b strings.Builder
+ n := 0
+ for _, r := range query {
+ if r == '?' {
+ n++
+ fmt.Fprintf(&b, "$%d", n)
+ continue
+ }
+ b.WriteRune(r)
+ }
+ query = b.String()
+ }
+
+ _, err := s.DB().ExecContext(ctx, query, args...)
+ require.NoError(t, err)
+}
+
+// timeValue returns t as the dialect stores times.
+func timeValue(s *sqlstore.Store, t time.Time) any {
+ if s.Dialect() == sqlstore.SQLite {
+ return t.UTC().Format(time.RFC3339Nano)
+ }
+ return t.UTC()
+}
+
+// headRepo registers a repository and commits n changesets on refs/heads/main,
+// each with its own owned pack key, which it returns in sequence order.
+func headRepo(t *testing.T, ctx context.Context, s *sqlstore.Store, name string, n int) (corestore.Repository, []string) {
+ t.Helper()
+
+ repo := register(t, ctx, s, testKey(name))
+ keys := make([]string, n)
+ prev := corestore.Target("")
+ for i := range n {
+ keys[i] = ownedKey(repo.ID, fmt.Sprintf("%s pack %d", name, i))
+ next := oid(i + 1)
+ _, err := commitPack(ctx, s, repo.ID, update("refs/heads/main", prev, next), keys[i])
+ require.NoError(t, err)
+ prev = next
+ }
+ return repo, keys
+}
+
+func TestOwnedPackKeyTrigger(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo := register(t, ctx, s, testKey("trigger"))
+
+ for _, key := range []string{legacyKey(repo.ID, "shared"), "not-a-pack-key"} {
+ _, err := commitPack(ctx, s, repo.ID, create("refs/heads/main", oid(1)), key)
+ require.ErrorContains(t, err, sqlstore.NotOwnedPackKeyMessage, key)
+ }
+ // Nothing of the refused changesets was written.
+ requireRefs(t, ctx, s, repo.ID, map[string]corestore.Target{})
+
+ res, err := commitPack(ctx, s, repo.ID, create("refs/heads/main", oid(1)), ownedKey(repo.ID, "owned"))
+ require.NoError(t, err)
+ require.Equal(t, uint64(1), res.Seq)
+
+ // A changeset without new objects has no pack key.
+ res, err = commit(ctx, s, repo.ID, update("refs/heads/main", oid(1), oid(2)))
+ require.NoError(t, err)
+ require.Equal(t, uint64(2), res.Seq)
+ })
+}
+
+func TestV010DatabaseMigrates(t *testing.T) {
+ for _, dialect := range coretest.Dialects() {
+ t.Run(string(dialect), func(t *testing.T) {
+ ctx := testhelper.Context(t)
+ dsn := coretest.DatabaseURL(t, dialect)
+
+ // A database as v0.1.0 left it: one pack object shared by changesets 1
+ // and 3 (content-addressed dedup), and no owned-key trigger yet.
+ old, err := sqlstore.OpenV010(ctx, dsn)
+ require.NoError(t, err)
+ repo, _, err := old.RegisterRepository(ctx, testKey("v010"), "sha1", corestore.InitialState{})
+ require.NoError(t, err)
+ shared, other := legacyKey(repo.ID, "shared"), legacyKey(repo.ID, "other")
+ for i, key := range []string{shared, other, shared} {
+ _, err := commitPack(ctx, old, repo.ID, force("refs/heads/main", oid(i+1)), key)
+ require.NoError(t, err)
+ }
+ require.NoError(t, old.Close())
+
+ // The current binary applies migration 2 at Open.
+ s := openStore(t, ctx, dsn)
+
+ changesets, err := s.ListChangesets(ctx, repo.ID, 0, 10)
+ require.NoError(t, err)
+ require.Len(t, changesets, 3)
+ require.Equal(t, []string{shared, other, shared},
+ []string{changesets[0].PackKey, changesets[1].PackKey, changesets[2].PackKey}, "legacy rows stay readable")
+
+ st, err := s.GCSnapshot(ctx, repo.ID)
+ require.NoError(t, err)
+ require.Equal(t, map[string]uint64{shared: 3, other: 2}, st.Packs, "a shared key is live up to its greatest sequence")
+ require.Empty(t, st.Checkpoints)
+ require.Empty(t, st.Orphans)
+
+ _, err = commitPack(ctx, s, repo.ID, force("refs/heads/main", oid(9)), legacyKey(repo.ID, "new"))
+ require.ErrorContains(t, err, sqlstore.NotOwnedPackKeyMessage)
+ _, err = commitPack(ctx, s, repo.ID, force("refs/heads/main", oid(9)), ownedKey(repo.ID, "new"))
+ require.NoError(t, err)
+ })
+ }
+}
+
+func TestCheckpointLifecycle(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo, _ := headRepo(t, ctx, s, "checkpoint", 3)
+
+ _, ok, err := s.NewestCheckpoint(ctx, repo.ID)
+ require.NoError(t, err)
+ require.False(t, ok)
+
+ refs := map[string]corestore.Target{
+ "HEAD": corestore.SymbolicTarget("refs/heads/main"),
+ "refs/heads/main": oid(2),
+ "refs/tags/\xff\xfev1": oid(7),
+ }
+ in := corestore.CheckpointInput{Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "checkpoint 2"), PackBytes: 42}
+
+ cp, created, err := s.CreateCheckpoint(ctx, repo.ID, in)
+ require.NoError(t, err)
+ require.True(t, created)
+ require.Equal(t, uint64(2), cp.Seq)
+ require.Equal(t, in.PackKey, cp.PackKey)
+ require.Equal(t, int64(42), cp.PackBytes)
+ require.False(t, cp.CreatedAt.IsZero())
+
+ got, err := s.CheckpointRefs(ctx, repo.ID, 2)
+ require.NoError(t, err)
+ require.Equal(t, refs, got, "the snapshot round-trips, non-UTF-8 names and symbolic targets included")
+
+ newest, ok, err := s.NewestCheckpoint(ctx, repo.ID)
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.Equal(t, cp, newest)
+
+ // A second build at the same sequence keeps the first checkpoint and its pack.
+ again, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: 2, Refs: refs, PackKey: ownedKey(repo.ID, "another build of 2"),
+ })
+ require.NoError(t, err)
+ require.False(t, created)
+ require.Equal(t, cp, again)
+
+ for name, bad := range map[string]corestore.CheckpointInput{
+ "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x")},
+ "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x")},
+ "older than the newest": {Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "x")},
+ "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x")},
+ "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x")},
+ "no pack key": {Seq: 3, Refs: refs},
+ "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x")},
+ "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x")},
+ "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x")},
+ "empty root": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID, "x"), Roots: []string{""}},
+ } {
+ _, _, err := s.CreateCheckpoint(ctx, repo.ID, bad)
+ require.Error(t, err, name)
+
+ newest, _, err := s.NewestCheckpoint(ctx, repo.ID)
+ require.NoError(t, err)
+ require.Equal(t, uint64(2), newest.Seq, "%s wrote nothing", name)
+ }
+
+ cp3, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": oid(3)}, PackKey: ownedKey(repo.ID, "checkpoint 3"),
+ })
+ require.NoError(t, err)
+ require.True(t, created)
+
+ st, err := s.GCSnapshot(ctx, repo.ID)
+ require.NoError(t, err)
+ require.Equal(t, []corestore.Checkpoint{cp, cp3}, st.Checkpoints)
+
+ deleted, err := s.DeleteCheckpoint(ctx, repo.ID, 2)
+ require.NoError(t, err)
+ require.True(t, deleted)
+ _, err = s.CheckpointRefs(ctx, repo.ID, 2)
+ require.ErrorIs(t, err, corestore.ErrCheckpointNotFound)
+ deleted, err = s.DeleteCheckpoint(ctx, repo.ID, 2)
+ require.NoError(t, err)
+ require.False(t, deleted)
+
+ _, _, err = s.CreateCheckpoint(ctx, repo.ID+1000, corestore.CheckpointInput{
+ Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID+1000, "unknown"),
+ })
+ require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
+ })
+}
+
+func TestCheckpointRequiresOpenReviewRoots(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo, _ := headRepo(t, ctx, s, "roots", 1)
+
+ head, base := string(oid(10)), string(oid(11))
+ insert := func(r *gitalypb.Review) {
+ _, err := s.InsertReview(ctx, repo.ID, func(string) (*gitalypb.Review, error) { return r, nil })
+ require.NoError(t, err)
+ }
+ insert(&gitalypb.Review{TargetBranch: []byte("main"), HeadCommitId: head, BaseCommitId: base})
+ insert(&gitalypb.Review{TargetBranch: []byte("main"), HeadCommitId: string(oid(12)), State: gitalypb.Review_STATE_CLOSED})
+ insert(&gitalypb.Review{TargetBranch: []byte("main"), HeadCommitId: string(oid(13)), State: gitalypb.Review_STATE_LANDED})
+
+ input := func(roots ...string) corestore.CheckpointInput {
+ return corestore.CheckpointInput{
+ Seq: 1, Refs: map[string]corestore.Target{"refs/heads/main": oid(1)}, PackKey: ownedKey(repo.ID, "roots"), Roots: roots,
+ }
+ }
+
+ for _, roots := range [][]string{nil, {head}, {base}} {
+ _, _, err := s.CreateCheckpoint(ctx, repo.ID, input(roots...))
+ require.ErrorIs(t, err, corestore.ErrCheckpointRootsChanged, "roots %v", roots)
+
+ _, ok, err := s.NewestCheckpoint(ctx, repo.ID)
+ require.NoError(t, err)
+ require.False(t, ok, "nothing recorded with roots %v", roots)
+ }
+
+ // Closed and landed reviews are not roots.
+ _, created, err := s.CreateCheckpoint(ctx, repo.ID, input(head, base))
+ require.NoError(t, err)
+ require.True(t, created)
+ })
+}
+
+func TestSyncBoundaryUsesTheDatabaseClock(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo, _ := headRepo(t, ctx, s, "boundary", 3)
+
+ boundary, err := s.SyncBoundary(ctx, repo.ID, 0)
+ require.NoError(t, err)
+ require.Zero(t, boundary, "no checkpoint")
+
+ for seq := uint64(1); seq <= 3; seq++ {
+ _, _, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: seq, Refs: map[string]corestore.Target{"refs/heads/main": oid(int(seq))}, PackKey: ownedKey(repo.ID, fmt.Sprint(seq)),
+ })
+ require.NoError(t, err)
+ }
+ now := time.Now()
+ execSQL(t, ctx, s, "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ? AND seq = ?", timeValue(s, now.Add(-3*time.Hour)), repo.ID, 1)
+ execSQL(t, ctx, s, "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ? AND seq = ?", timeValue(s, now.Add(-90*time.Minute)), repo.ID, 2)
+
+ for _, tc := range []struct {
+ age time.Duration
+ want uint64
+ }{
+ {age: 0, want: 3},
+ {age: time.Hour, want: 2},
+ {age: 2 * time.Hour, want: 1},
+ {age: 4 * time.Hour, want: 0},
+ } {
+ boundary, err := s.SyncBoundary(ctx, repo.ID, tc.age)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, boundary, "age %s", tc.age)
+ }
+
+ _, err = s.SyncBoundary(ctx, repo.ID, -time.Second)
+ require.Error(t, err)
+ _, err = s.SyncBoundary(ctx, repo.ID+1000, 0)
+ require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
+ })
+}
+
+func TestOrphanObservations(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo, keys := headRepo(t, ctx, s, "orphans", 2)
+ x, y := ownedKey(repo.ID, "x"), "repos/1/packs/leftover.tmp"
+
+ require.NoError(t, s.RecordOrphans(ctx, repo.ID, []string{x, y}))
+
+ st, err := s.GCSnapshot(ctx, repo.ID)
+ require.NoError(t, err)
+ require.Equal(t, map[string]uint64{keys[0]: 1, keys[1]: 2}, st.Packs)
+ require.Len(t, st.Orphans, 2)
+ for key, seen := range st.Orphans {
+ require.False(t, seen.After(st.Now), "%s was observed before the snapshot", key)
+ }
+
+ // A second observation keeps the first one's time.
+ old := time.Now().Add(-2 * time.Hour)
+ execSQL(t, ctx, s, "UPDATE graft_gc_orphans SET first_seen = ? WHERE repo_id = ? AND pack_key = ?", timeValue(s, old), repo.ID, x)
+ require.NoError(t, s.RecordOrphans(ctx, repo.ID, []string{x}))
+ st, err = s.GCSnapshot(ctx, repo.ID)
+ require.NoError(t, err)
+ require.WithinDuration(t, old, st.Orphans[x], time.Millisecond)
+
+ require.NoError(t, s.ForgetOrphans(ctx, repo.ID, []string{x}))
+ st, err = s.GCSnapshot(ctx, repo.ID)
+ require.NoError(t, err)
+ require.NotContains(t, st.Orphans, x)
+ require.Contains(t, st.Orphans, y)
+
+ require.Error(t, s.RecordOrphans(ctx, repo.ID, []string{"repos/1/packs/\xff.pack"}), "an unrecordable key is an error")
+ require.NoError(t, s.RecordOrphans(ctx, repo.ID, nil))
+ require.NoError(t, s.ForgetOrphans(ctx, repo.ID, nil))
+
+ _, err = s.GCSnapshot(ctx, repo.ID+1000)
+ require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
+ })
+}
+
+func TestMaintenanceLease(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, a *sqlstore.Store) {
+ // Two nodes on one database.
+ b := openStore(t, ctx, coreDSN(t, a))
+ repo, _ := headRepo(t, ctx, a, "lease", 1)
+ ttl := time.Hour
+
+ acquire := func(s *sqlstore.Store, holder string) bool {
+ t.Helper()
+ ok, err := s.AcquireMaintenanceLease(ctx, repo.ID, holder, ttl)
+ require.NoError(t, err)
+ return ok
+ }
+ renew := func(s *sqlstore.Store, holder string) bool {
+ t.Helper()
+ ok, err := s.RenewMaintenanceLease(ctx, repo.ID, holder, ttl)
+ require.NoError(t, err)
+ return ok
+ }
+
+ require.True(t, acquire(a, "node-a"))
+ require.False(t, acquire(b, "node-b"), "held by node-a")
+ require.True(t, acquire(a, "node-a"), "the holder acquires again")
+ require.False(t, renew(b, "node-b"))
+ require.True(t, renew(a, "node-a"))
+
+ require.NoError(t, b.ReleaseMaintenanceLease(ctx, repo.ID, "node-b"), "releasing a lease one does not hold changes nothing")
+ require.False(t, acquire(b, "node-b"))
+ require.NoError(t, a.ReleaseMaintenanceLease(ctx, repo.ID, "node-a"))
+ require.True(t, acquire(b, "node-b"))
+
+ // An expired lease (its holder died) can be taken over.
+ execSQL(t, ctx, a, "UPDATE graft_gc_leases SET expires_at = ? WHERE repo_id = ?", timeValue(a, time.Now().Add(-time.Minute)), repo.ID)
+ require.True(t, acquire(a, "node-a"))
+ require.False(t, renew(b, "node-b"), "node-b lost the lease")
+
+ _, err := a.AcquireMaintenanceLease(ctx, repo.ID, "node-a", 0)
+ require.Error(t, err)
+ _, err = a.AcquireMaintenanceLease(ctx, repo.ID, "", ttl)
+ require.Error(t, err)
+ _, err = a.AcquireMaintenanceLease(ctx, repo.ID+1000, "node-a", ttl)
+ require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
+ _, err = a.RenewMaintenanceLease(ctx, repo.ID+1000, "node-a", ttl)
+ require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
+ require.NoError(t, a.ReleaseMaintenanceLease(ctx, repo.ID+1000, "node-a"))
+
+ // Exactly one of concurrent acquisitions from both nodes wins.
+ require.NoError(t, a.ReleaseMaintenanceLease(ctx, repo.ID, "node-a"))
+ var winners atomic.Int32
+ var g errgroup.Group
+ for i := range 16 {
+ s := a
+ if i%2 == 1 {
+ s = b
+ }
+ g.Go(func() error {
+ ok, err := s.AcquireMaintenanceLease(ctx, repo.ID, fmt.Sprintf("node-%d", i), ttl)
+ if ok {
+ winners.Add(1)
+ }
+ return err
+ })
+ }
+ require.NoError(t, g.Wait())
+ require.Equal(t, int32(1), winners.Load())
+ })
+}
+
+func TestDeleteRepositoryRecordsTombstone(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo, _ := headRepo(t, ctx, s, "removed", 2)
+ key := testKey("removed")
+
+ // Rows of milestone 3 go with the repository.
+ _, _, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: 2, Refs: map[string]corestore.Target{"refs/heads/main": oid(2)}, PackKey: ownedKey(repo.ID, "cp"),
+ })
+ require.NoError(t, err)
+ require.NoError(t, s.RecordOrphans(ctx, repo.ID, []string{ownedKey(repo.ID, "orphan")}))
+ ok, err := s.AcquireMaintenanceLease(ctx, repo.ID, "node-a", time.Hour)
+ require.NoError(t, err)
+ require.True(t, ok)
+
+ deleted, err := s.DeleteRepository(ctx, key)
+ require.NoError(t, err)
+ require.True(t, deleted)
+
+ _, err = s.GCSnapshot(ctx, repo.ID)
+ require.ErrorIs(t, err, corestore.ErrRepositoryNotFound)
+ for _, table := range []string{"graft_checkpoints", "graft_checkpoint_refs", "graft_gc_orphans", "graft_gc_leases", "graft_changesets"} {
+ var n int
+ query := "SELECT COUNT(*) FROM " + table + " WHERE repo_id = ?"
+ if s.Dialect() == sqlstore.Postgres {
+ query = strings.Replace(query, "?", "$1", 1)
+ }
+ require.NoError(t, s.DB().QueryRowContext(ctx, query, repo.ID).Scan(&n))
+ require.Zero(t, n, "%s rows of the removed repository", table)
+ }
+
+ removed, err := s.ListRemovedRepositories(ctx, "", 0)
+ require.NoError(t, err)
+ require.Len(t, removed, 1)
+ require.Equal(t, repo.ID, removed[0].RepoID)
+ require.Equal(t, key, removed[0].Key)
+
+ // The age filter is on the database clock.
+ removed, err = s.ListRemovedRepositories(ctx, "default", time.Hour)
+ require.NoError(t, err)
+ require.Empty(t, removed, "too young")
+ execSQL(t, ctx, s, "UPDATE graft_gc_removed SET removed_at = ? WHERE repo_id = ?", timeValue(s, time.Now().Add(-2*time.Hour)), repo.ID)
+ removed, err = s.ListRemovedRepositories(ctx, "default", time.Hour)
+ require.NoError(t, err)
+ require.Len(t, removed, 1)
+ removed, err = s.ListRemovedRepositories(ctx, "other-storage", 0)
+ require.NoError(t, err)
+ require.Empty(t, removed)
+
+ // Deleting an unknown repository records nothing.
+ deleted, err = s.DeleteRepository(ctx, key)
+ require.NoError(t, err)
+ require.False(t, deleted)
+ removed, err = s.ListRemovedRepositories(ctx, "", 0)
+ require.NoError(t, err)
+ require.Len(t, removed, 1)
+
+ // A repository created again at the path is a new one.
+ again := register(t, ctx, s, key)
+ require.NotEqual(t, repo.ID, again.ID)
+
+ require.NoError(t, s.DeleteRemovedRepository(ctx, repo.ID))
+ require.NoError(t, s.DeleteRemovedRepository(ctx, repo.ID), "idempotent")
+ removed, err = s.ListRemovedRepositories(ctx, "", 0)
+ require.NoError(t, err)
+ require.Empty(t, removed)
+ })
+}
+
+func TestListRepositories(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ register(t, ctx, s, testKey("incomplete-import")) // head sequence 0
+ first, _ := headRepo(t, ctx, s, "first", 1)
+ second, _ := headRepo(t, ctx, s, "second", 2)
+
+ otherKey := corestore.RepoKey{Storage: "other", RelativePath: "@hashed/other.git"}
+ other, _, err := s.RegisterRepository(ctx, otherKey, "sha256", corestore.InitialState{})
+ require.NoError(t, err)
+ _, err = commit(ctx, s, other.ID, create("HEAD", corestore.SymbolicTarget("refs/heads/main")))
+ require.NoError(t, err)
+
+ ids := func(storage string) []int64 {
+ repos, err := s.ListRepositories(ctx, storage)
+ require.NoError(t, err)
+ var ids []int64
+ for _, r := range repos {
+ require.NotZero(t, r.HeadSeq)
+ ids = append(ids, r.ID)
+ }
+ return ids
+ }
+
+ require.Equal(t, []int64{first.ID, second.ID, other.ID}, ids(""))
+ require.Equal(t, []int64{first.ID, second.ID}, ids("default"))
+ require.Equal(t, []int64{other.ID}, ids("other"))
+ require.Empty(t, ids("missing"))
+ })
+}
+
+// TestLockOrderConcurrentWriters runs changesets, review creations, review
+// updates, checkpoint commits and lease operations on one repository from two
+// nodes at once: every transaction that serializes on the repository takes its
+// row first (doc/graft/PLAN7.md, LOCK ORDER), so Postgres never reports a
+// deadlock, and every invariant holds at the end.
+func TestLockOrderConcurrentWriters(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, a *sqlstore.Store) {
+ b := openStore(t, ctx, coreDSN(t, a))
+ repo, _ := headRepo(t, ctx, a, "lock-order", 1)
+
+ var deadlocks atomic.Int32
+ restore := sqlstore.SetRetryHook(func(err error) {
+ if sqlstore.IsDeadlock(err) {
+ deadlocks.Add(1)
+ }
+ })
+ defer restore()
+
+ _, err := a.InsertReview(ctx, repo.ID, func(string) (*gitalypb.Review, error) {
+ return &gitalypb.Review{TargetBranch: []byte("main"), HeadCommitId: string(oid(500))}, nil
+ })
+ require.NoError(t, err)
+
+ const (
+ landsPerNode = 30
+ reviewsPerNode = 15
+ updates = 15
+ checkpoints = 10
+ )
+
+ var (
+ mu sync.Mutex
+ createdSeqs []uint64
+ rootsRefused int
+ unexpected []error
+ recordUnexpected = func(err error) { mu.Lock(); unexpected = append(unexpected, err); mu.Unlock() }
+ nodes = []*sqlstore.Store{a, b}
+ g errgroup.Group
+ )
+
+ for n, s := range nodes {
+ g.Go(func() error {
+ for i := range landsPerNode {
+ ref := fmt.Sprintf("refs/heads/node-%d-%d", n, i)
+ if _, err := commitPack(ctx, s, repo.ID, create(ref, oid(1000+i)), ownedKey(repo.ID, ref)); err != nil {
+ recordUnexpected(fmt.Errorf("land %s: %w", ref, err))
+ }
+ }
+ return nil
+ })
+ g.Go(func() error {
+ for i := range reviewsPerNode {
+ head := string(oid(2000 + 100*n + i))
+ if _, err := s.InsertReview(ctx, repo.ID, func(string) (*gitalypb.Review, error) {
+ return &gitalypb.Review{TargetBranch: []byte("main"), HeadCommitId: head}, nil
+ }); err != nil {
+ recordUnexpected(fmt.Errorf("insert review: %w", err))
+ }
+ }
+ return nil
+ })
+ }
+
+ // Updates of one review that do not move its commits (a comment): a
+ // version compare-and-swap on the review row only.
+ g.Go(func() error {
+ for done := 0; done < updates; {
+ r, version, err := b.GetReview(ctx, repo.ID, "1")
+ if err != nil {
+ recordUnexpected(fmt.Errorf("get review: %w", err))
+ return nil
+ }
+ r.Title = []byte(fmt.Sprintf("comment %d", done))
+ switch err := b.UpdateReview(ctx, repo.ID, "1", version, r); {
+ case err == nil:
+ done++
+ case errors.Is(err, corestore.ErrReviewVersionConflict):
+ default:
+ recordUnexpected(fmt.Errorf("update review: %w", err))
+ return nil
+ }
+ }
+ return nil
+ })
+
+ // Checkpoints at the head, with the open reviews read before the commit as
+ // roots: a review created in between makes the commit refuse (R2).
+ g.Go(func() error {
+ for range checkpoints {
+ refs, head, err := a.Refs(ctx, repo.ID)
+ if err != nil {
+ recordUnexpected(err)
+ return nil
+ }
+ open, err := a.ListReviews(ctx, repo.ID, "", false, 0)
+ if err != nil {
+ recordUnexpected(err)
+ return nil
+ }
+ var roots []string
+ for _, r := range open {
+ roots = append(roots, r.GetHeadCommitId())
+ }
+
+ cp, created, err := a.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: head, Refs: refs, PackKey: ownedKey(repo.ID, fmt.Sprint("checkpoint ", head)), Roots: roots,
+ })
+ switch {
+ case err == nil && created:
+ mu.Lock()
+ createdSeqs = append(createdSeqs, cp.Seq)
+ mu.Unlock()
+ case err == nil:
+ case errors.Is(err, corestore.ErrCheckpointRootsChanged):
+ mu.Lock()
+ rootsRefused++
+ mu.Unlock()
+ default:
+ recordUnexpected(fmt.Errorf("checkpoint at %d: %w", head, err))
+ }
+ }
+ return nil
+ })
+
+ g.Go(func() error {
+ for i := range 20 {
+ holder := fmt.Sprintf("node-%d", i%2)
+ if _, err := nodes[i%2].AcquireMaintenanceLease(ctx, repo.ID, holder, time.Minute); err != nil {
+ recordUnexpected(fmt.Errorf("acquire lease: %w", err))
+ }
+ if err := nodes[i%2].ReleaseMaintenanceLease(ctx, repo.ID, holder); err != nil {
+ recordUnexpected(fmt.Errorf("release lease: %w", err))
+ }
+ }
+ return nil
+ })
+
+ require.NoError(t, g.Wait())
+ require.Empty(t, unexpected)
+ require.Zero(t, deadlocks.Load(), "a transaction deadlocked")
+
+ // Dense changesets: the first one plus every land.
+ final, err := a.GetRepository(ctx, testKey("lock-order"))
+ require.NoError(t, err)
+ require.Equal(t, uint64(1+2*landsPerNode), final.HeadSeq)
+ changesets, err := a.ListChangesets(ctx, repo.ID, 0, final.HeadSeq)
+ require.NoError(t, err)
+ require.Len(t, changesets, int(final.HeadSeq))
+
+ // Dense review IDs: the first one plus every creation.
+ reviews, err := a.ListReviews(ctx, repo.ID, "", true, 0)
+ require.NoError(t, err)
+ require.Len(t, reviews, 1+2*reviewsPerNode)
+
+ // Checkpoints are increasing and never beyond the head; every one the
+ // checkpoint goroutine saw created is recorded.
+ st, err := a.GCSnapshot(ctx, repo.ID)
+ require.NoError(t, err)
+ var recorded []uint64
+ for i, cp := range st.Checkpoints {
+ require.LessOrEqual(t, cp.Seq, final.HeadSeq)
+ if i > 0 {
+ require.Greater(t, cp.Seq, st.Checkpoints[i-1].Seq)
+ }
+ recorded = append(recorded, cp.Seq)
+ }
+ require.Equal(t, createdSeqs, recorded)
+ t.Logf("checkpoints created %d, refused because a review changed %d", len(createdSeqs), rootsRefused)
+ })
+}
diff --git a/internal/graft/corestore/sqlstore/maintenance.go b/internal/graft/corestore/sqlstore/maintenance.go
new file mode 100644
index 0000000..09e66c8
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/maintenance.go
@@ -0,0 +1,222 @@
+package sqlstore
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+)
+
+// lease is the maintenance lease of a repository as stored.
+type lease struct {
+ holder string
+ expiresAt time.Time
+}
+
+// readLease reads the lease of the repository; ok is false when there is none.
+func (s *Store) readLease(ctx context.Context, q querier, repoID int64) (lease, bool, error) {
+ var (
+ l lease
+ expiresRaw any
+ )
+ err := q.QueryRowContext(ctx, s.bind("SELECT holder, expires_at FROM graft_gc_leases WHERE repo_id = ?"), repoID).
+ Scan(&l.holder, &expiresRaw)
+ if errors.Is(err, sql.ErrNoRows) {
+ return lease{}, false, nil
+ }
+ if err != nil {
+ return lease{}, false, fmt.Errorf("read maintenance lease: %w", err)
+ }
+ if l.expiresAt, err = scanTime(expiresRaw); err != nil {
+ return lease{}, false, err
+ }
+ return l, true, nil
+}
+
+// validateLease checks the arguments of the lease methods.
+func validateLease(holder string, ttl time.Duration) error {
+ if holder == "" || !storableText(holder) {
+ return fmt.Errorf("storage core: invalid maintenance lease holder %q", holder)
+ }
+ if ttl <= 0 {
+ return fmt.Errorf("storage core: invalid maintenance lease duration %s", ttl)
+ }
+ return nil
+}
+
+// AcquireMaintenanceLease implements corestore.MetaStore.
+func (s *Store) AcquireMaintenanceLease(ctx context.Context, repoID int64, holder string, ttl time.Duration) (bool, error) {
+ if err := validateLease(holder, ttl); err != nil {
+ return false, err
+ }
+
+ var acquired bool
+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ acquired = false
+
+ // LOCK ORDER (doc/graft/PLAN7.md): the repository row first; it also makes
+ // the read and the write of the lease one compare-and-swap.
+ if _, err := s.selectRepository(ctx, tx, "repo_id = ?", []any{repoID}, true); err != nil {
+ return err
+ }
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+
+ current, ok, err := s.readLease(ctx, tx, repoID)
+ if err != nil {
+ return err
+ }
+ if ok && current.holder != holder && current.expiresAt.After(now) {
+ return nil
+ }
+
+ if _, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_gc_leases (repo_id, holder, acquired_at, expires_at)
+ VALUES (?, ?, ?, ?)
+ ON CONFLICT (repo_id) DO UPDATE SET
+ holder = excluded.holder,
+ acquired_at = excluded.acquired_at,
+ expires_at = excluded.expires_at
+ `), repoID, holder, s.timeArg(now), s.timeArg(now.Add(ttl))); err != nil {
+ return fmt.Errorf("write maintenance lease: %w", err)
+ }
+ acquired = true
+ return nil
+ })
+ if err != nil {
+ return false, err
+ }
+ return acquired, nil
+}
+
+// RenewMaintenanceLease implements corestore.MetaStore.
+func (s *Store) RenewMaintenanceLease(ctx context.Context, repoID int64, holder string, ttl time.Duration) (bool, error) {
+ if err := validateLease(holder, ttl); err != nil {
+ return false, err
+ }
+
+ var held bool
+ err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ held = false
+
+ if _, err := s.selectRepository(ctx, tx, "repo_id = ?", []any{repoID}, true); err != nil {
+ return err
+ }
+ now, err := s.wallTime(ctx, tx)
+ if err != nil {
+ return err
+ }
+
+ current, ok, err := s.readLease(ctx, tx, repoID)
+ if err != nil {
+ return err
+ }
+ if !ok || current.holder != holder {
+ return nil
+ }
+
+ if _, err := tx.ExecContext(ctx, s.bind("UPDATE graft_gc_leases SET expires_at = ? WHERE repo_id = ? AND holder = ?"),
+ s.timeArg(now.Add(ttl)), repoID, holder); err != nil {
+ return fmt.Errorf("renew maintenance lease: %w", err)
+ }
+ held = true
+ return nil
+ })
+ if err != nil {
+ return false, err
+ }
+ return held, nil
+}
+
+// ReleaseMaintenanceLease implements corestore.MetaStore.
+func (s *Store) ReleaseMaintenanceLease(ctx context.Context, repoID int64, holder string) error {
+ if holder == "" || !storableText(holder) {
+ return fmt.Errorf("storage core: invalid maintenance lease holder %q", holder)
+ }
+
+ return s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ _, err := s.selectRepository(ctx, tx, "repo_id = ?", []any{repoID}, true)
+ if errors.Is(err, corestore.ErrRepositoryNotFound) {
+ // The lease went with the repository row.
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+
+ if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_gc_leases WHERE repo_id = ? AND holder = ?"), repoID, holder); err != nil {
+ return fmt.Errorf("release maintenance lease: %w", err)
+ }
+ return nil
+ })
+}
+
+// ListRemovedRepositories implements corestore.MetaStore.
+func (s *Store) ListRemovedRepositories(ctx context.Context, storage string, minAge time.Duration) ([]corestore.RemovedRepository, error) {
+ if minAge < 0 {
+ return nil, fmt.Errorf("storage core: negative tombstone age %s", minAge)
+ }
+
+ hostNow := time.Now()
+ var removed []corestore.RemovedRepository
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ removed = nil
+
+ now, err := s.snapshotTime(ctx, tx, hostNow)
+ if err != nil {
+ return err
+ }
+
+ query := "SELECT repo_id, storage, relative_path, removed_at FROM graft_gc_removed"
+ var args []any
+ if storage != "" {
+ query += " WHERE storage = ?"
+ args = append(args, storage)
+ }
+ query += " ORDER BY repo_id"
+
+ rows, err := tx.QueryContext(ctx, s.bind(query), args...)
+ if err != nil {
+ return fmt.Errorf("list removed repositories: %w", err)
+ }
+ defer rows.Close()
+
+ limit := now.Add(-minAge)
+ for rows.Next() {
+ var (
+ r corestore.RemovedRepository
+ removedRaw any
+ )
+ if err := rows.Scan(&r.RepoID, &r.Key.Storage, &r.Key.RelativePath, &removedRaw); err != nil {
+ return fmt.Errorf("scan removed repository: %w", err)
+ }
+ if r.RemovedAt, err = scanTime(removedRaw); err != nil {
+ return err
+ }
+ if r.RemovedAt.After(limit) {
+ continue
+ }
+ removed = append(removed, r)
+ }
+ return rows.Err()
+ })
+ if err != nil {
+ return nil, err
+ }
+ return removed, nil
+}
+
+// DeleteRemovedRepository implements corestore.MetaStore.
+func (s *Store) DeleteRemovedRepository(ctx context.Context, repoID int64) error {
+ return s.withTx(ctx, nil, func(tx *sql.Tx) error {
+ if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_gc_removed WHERE repo_id = ?"), repoID); err != nil {
+ return fmt.Errorf("delete tombstone: %w", err)
+ }
+ return nil
+ })
+}
diff --git a/internal/graft/corestore/sqlstore/migrations.go b/internal/graft/corestore/sqlstore/migrations.go
index 2e61276..0576ed6 100644
--- a/internal/graft/corestore/sqlstore/migrations.go
+++ b/internal/graft/corestore/sqlstore/migrations.go
@@ -90,8 +90,71 @@ var sqliteMigrations = []*migrate.Migration{
`DROP TABLE graft_repositories`,
},
},
+ {
+ // Milestone 3 (doc/graft/PLAN7.md, "Data model"): checkpoints, the garbage
+ // collector's orphan observations, maintenance leases, the tombstones of
+ // removed repositories, and the trigger that refuses a pack key that is not
+ // owned (PACK OWNERSHIP RULE 3).
+ Id: "20261018000000_graft_storage_gc",
+ Up: []string{
+ `CREATE TABLE graft_checkpoints (
+ repo_id INTEGER NOT NULL REFERENCES graft_repositories (repo_id) ON DELETE CASCADE,
+ seq INTEGER NOT NULL,
+ pack_key TEXT NOT NULL,
+ pack_bytes INTEGER NOT NULL DEFAULT 0,
+ created_at TEXT NOT NULL,
+ PRIMARY KEY (repo_id, seq)
+ )`,
+ `CREATE TABLE graft_checkpoint_refs (
+ repo_id INTEGER NOT NULL,
+ seq INTEGER NOT NULL,
+ ordinal INTEGER NOT NULL,
+ name BLOB NOT NULL,
+ target TEXT NOT NULL,
+ PRIMARY KEY (repo_id, seq, ordinal),
+ FOREIGN KEY (repo_id, seq) REFERENCES graft_checkpoints (repo_id, seq) ON DELETE CASCADE
+ )`,
+ `CREATE TABLE graft_gc_orphans (
+ repo_id INTEGER NOT NULL REFERENCES graft_repositories (repo_id) ON DELETE CASCADE,
+ pack_key TEXT NOT NULL,
+ first_seen TEXT NOT NULL,
+ PRIMARY KEY (repo_id, pack_key)
+ )`,
+ `CREATE TABLE graft_gc_leases (
+ repo_id INTEGER NOT NULL PRIMARY KEY REFERENCES graft_repositories (repo_id) ON DELETE CASCADE,
+ holder TEXT NOT NULL,
+ acquired_at TEXT NOT NULL,
+ expires_at TEXT NOT NULL
+ )`,
+ `CREATE TABLE graft_gc_removed (
+ repo_id INTEGER NOT NULL PRIMARY KEY,
+ storage TEXT NOT NULL,
+ relative_path TEXT NOT NULL,
+ removed_at TEXT NOT NULL
+ )`,
+ `CREATE TRIGGER graft_changesets_owned_pack BEFORE INSERT ON graft_changesets
+ WHEN NEW.pack_key IS NOT NULL AND NEW.pack_key NOT LIKE 'repos/%/packs/%-%.pack'
+ BEGIN
+ SELECT RAISE(ABORT, '` + notOwnedPackKeyMessage + `');
+ END`,
+ },
+ Down: []string{
+ `DROP TRIGGER graft_changesets_owned_pack`,
+ `DROP TABLE graft_gc_removed`,
+ `DROP TABLE graft_gc_leases`,
+ `DROP TABLE graft_gc_orphans`,
+ `DROP TABLE graft_checkpoint_refs`,
+ `DROP TABLE graft_checkpoints`,
+ },
+ },
}
+// notOwnedPackKeyMessage is the error of the trigger that refuses a changeset
+// whose pack key is not owned: a node that still runs a binary from before
+// milestone 3 shares pack objects between writes (doc/graft/PLAN7.md, PACK
+// OWNERSHIP RULE 3).
+const notOwnedPackKeyMessage = "storage core: the pack key is not an owned key; every node must run the same version"
+
// postgresMigrations is the same schema for Postgres.
var postgresMigrations = []*migrate.Migration{
{
@@ -171,4 +234,64 @@ var postgresMigrations = []*migrate.Migration{
`DROP TABLE graft_repositories`,
},
},
+ {
+ // See the SQLite migration of the same ID.
+ Id: "20261018000000_graft_storage_gc",
+ Up: []string{
+ `CREATE TABLE graft_checkpoints (
+ repo_id BIGINT NOT NULL REFERENCES graft_repositories (repo_id) ON DELETE CASCADE,
+ seq BIGINT NOT NULL,
+ pack_key TEXT NOT NULL,
+ pack_bytes BIGINT NOT NULL DEFAULT 0,
+ created_at TIMESTAMPTZ NOT NULL,
+ PRIMARY KEY (repo_id, seq)
+ )`,
+ `CREATE TABLE graft_checkpoint_refs (
+ repo_id BIGINT NOT NULL,
+ seq BIGINT NOT NULL,
+ ordinal INTEGER NOT NULL,
+ name BYTEA NOT NULL,
+ target TEXT NOT NULL,
+ PRIMARY KEY (repo_id, seq, ordinal),
+ FOREIGN KEY (repo_id, seq) REFERENCES graft_checkpoints (repo_id, seq) ON DELETE CASCADE
+ )`,
+ `CREATE TABLE graft_gc_orphans (
+ repo_id BIGINT NOT NULL REFERENCES graft_repositories (repo_id) ON DELETE CASCADE,
+ pack_key TEXT NOT NULL,
+ first_seen TIMESTAMPTZ NOT NULL,
+ PRIMARY KEY (repo_id, pack_key)
+ )`,
+ `CREATE TABLE graft_gc_leases (
+ repo_id BIGINT NOT NULL PRIMARY KEY REFERENCES graft_repositories (repo_id) ON DELETE CASCADE,
+ holder TEXT NOT NULL,
+ acquired_at TIMESTAMPTZ NOT NULL,
+ expires_at TIMESTAMPTZ NOT NULL
+ )`,
+ `CREATE TABLE graft_gc_removed (
+ repo_id BIGINT NOT NULL PRIMARY KEY,
+ storage TEXT NOT NULL,
+ relative_path TEXT NOT NULL,
+ removed_at TIMESTAMPTZ NOT NULL
+ )`,
+ `CREATE FUNCTION graft_changesets_owned_pack() RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN
+ IF NEW.pack_key IS NOT NULL AND NEW.pack_key NOT LIKE 'repos/%/packs/%-%.pack' THEN
+ RAISE EXCEPTION '` + notOwnedPackKeyMessage + `' USING ERRCODE = 'check_violation';
+ END IF;
+ RETURN NEW;
+ END
+ $$`,
+ `CREATE TRIGGER graft_changesets_owned_pack BEFORE INSERT ON graft_changesets
+ FOR EACH ROW EXECUTE FUNCTION graft_changesets_owned_pack()`,
+ },
+ Down: []string{
+ `DROP TRIGGER graft_changesets_owned_pack ON graft_changesets`,
+ `DROP FUNCTION graft_changesets_owned_pack()`,
+ `DROP TABLE graft_gc_removed`,
+ `DROP TABLE graft_gc_leases`,
+ `DROP TABLE graft_gc_orphans`,
+ `DROP TABLE graft_checkpoint_refs`,
+ `DROP TABLE graft_checkpoints`,
+ },
+ },
}
diff --git a/internal/graft/corestore/sqlstore/repository.go b/internal/graft/corestore/sqlstore/repository.go
index e33014a..0601593 100644
--- a/internal/graft/corestore/sqlstore/repository.go
+++ b/internal/graft/corestore/sqlstore/repository.go
@@ -37,6 +37,18 @@ func (s *Store) selectRepository(ctx context.Context, q querier, where string, a
query += " FOR UPDATE"
}
+ r, err := scanRepoState(q.QueryRowContext(ctx, s.bind(query), args...).Scan)
+ if errors.Is(err, sql.ErrNoRows) {
+ return repoState{}, corestore.ErrRepositoryNotFound
+ }
+ if err != nil {
+ return repoState{}, fmt.Errorf("select repository: %w", err)
+ }
+ return r, nil
+}
+
+// scanRepoState reads a row of repoColumns.
+func scanRepoState(scan func(...any) error) (repoState, error) {
var (
r repoState
storage, relativePath string
@@ -44,21 +56,16 @@ func (s *Store) selectRepository(ctx context.Context, q querier, where string, a
nextReviewID int64
createdRaw any
)
- err := q.QueryRowContext(ctx, s.bind(query), args...).Scan(
- &r.id, &storage, &relativePath, &r.objectFormat, &headSeq, &logSeq, &nextReviewID, &createdRaw,
- )
- if errors.Is(err, sql.ErrNoRows) {
- return repoState{}, corestore.ErrRepositoryNotFound
- }
- if err != nil {
- return repoState{}, fmt.Errorf("select repository: %w", err)
+ if err := scan(&r.id, &storage, &relativePath, &r.objectFormat, &headSeq, &logSeq, &nextReviewID, &createdRaw); err != nil {
+ return repoState{}, err
}
- r.createdAt, err = scanTime(createdRaw)
+ createdAt, err := scanTime(createdRaw)
if err != nil {
- return repoState{}, fmt.Errorf("select repository: %w", err)
+ return repoState{}, err
}
+ r.createdAt = createdAt
r.key = corestore.RepoKey{Storage: storage, RelativePath: relativePath}
r.headSeq = uint64(headSeq)
r.logSeq = uint64(logSeq)
@@ -393,16 +400,36 @@ func (s *Store) DeleteRepository(ctx context.Context, key corestore.RepoKey) (bo
var deleted bool
err := s.withTx(ctx, nil, func(tx *sql.Tx) error {
- res, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_repositories WHERE storage = ? AND relative_path = ?"),
- key.Storage, key.RelativePath)
+ deleted = false
+
+ repo, err := s.selectRepository(ctx, tx, "storage = ? AND relative_path = ?", []any{key.Storage, key.RelativePath}, true)
+ if errors.Is(err, corestore.ErrRepositoryNotFound) {
+ return nil
+ }
if err != nil {
- return fmt.Errorf("delete repository: %w", err)
+ return err
}
- n, err := res.RowsAffected()
+
+ now, err := s.wallTime(ctx, tx)
if err != nil {
+ return err
+ }
+
+ // The cascade removes everything recorded for the repository; the
+ // tombstone, which has no foreign key, outlives it so that garbage
+ // collection deletes the repository's objects from the bucket
+ // (doc/graft/PLAN7.md, "Removed repositories").
+ if _, err := tx.ExecContext(ctx, s.bind("DELETE FROM graft_repositories WHERE repo_id = ?"), repo.id); err != nil {
return fmt.Errorf("delete repository: %w", err)
}
- deleted = n > 0
+ if _, err := tx.ExecContext(ctx, s.bind(`
+ INSERT INTO graft_gc_removed (repo_id, storage, relative_path, removed_at)
+ VALUES (?, ?, ?, ?)
+ `), repo.id, repo.key.Storage, repo.key.RelativePath, s.timeArg(now)); err != nil {
+ return fmt.Errorf("record the tombstone of repository %d: %w", repo.id, err)
+ }
+
+ deleted = true
return nil
})
if err != nil {
@@ -411,6 +438,41 @@ func (s *Store) DeleteRepository(ctx context.Context, key corestore.RepoKey) (bo
return deleted, nil
}
+// ListRepositories implements corestore.MetaStore.
+func (s *Store) ListRepositories(ctx context.Context, storage string) ([]corestore.Repository, error) {
+ query := "SELECT " + repoColumns + " FROM graft_repositories WHERE head_seq > 0"
+ var args []any
+ if storage != "" {
+ query += " AND storage = ?"
+ args = append(args, storage)
+ }
+ query += " ORDER BY repo_id"
+
+ var repos []corestore.Repository
+ err := s.withReadTx(ctx, func(tx *sql.Tx) error {
+ repos = nil
+
+ rows, err := tx.QueryContext(ctx, s.bind(query), args...)
+ if err != nil {
+ return fmt.Errorf("list repositories: %w", err)
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ r, err := scanRepoState(rows.Scan)
+ if err != nil {
+ return fmt.Errorf("scan repository: %w", err)
+ }
+ repos = append(repos, toRepository(r))
+ }
+ return rows.Err()
+ })
+ if err != nil {
+ return nil, err
+ }
+ return repos, nil
+}
+
// Refs implements corestore.MetaStore.
func (s *Store) Refs(ctx context.Context, repoID int64) (map[string]corestore.Target, uint64, error) {
var (
diff --git a/internal/graft/corestore/sqlstore/store.go b/internal/graft/corestore/sqlstore/store.go
index 8ce2a84..9581487 100644
--- a/internal/graft/corestore/sqlstore/store.go
+++ b/internal/graft/corestore/sqlstore/store.go
@@ -113,6 +113,13 @@ func ParseDSN(dsn string) (Dialect, string, error) {
// the lock is acquired in slices of sqliteBusySlice, never longer than the
// context's deadline.
func Open(ctx context.Context, dsn string) (*Store, error) {
+ return open(ctx, dsn, 0)
+}
+
+// open is Open applying only the first migrations of the schema when
+// migrations is greater than 0 (tests build a database as an earlier version
+// left it), every migration otherwise.
+func open(ctx context.Context, dsn string, migrations int) (*Store, error) {
dialect, target, err := ParseDSN(dsn)
if err != nil {
return nil, err
@@ -141,7 +148,7 @@ func Open(ctx context.Context, dsn string) (*Store, error) {
return nil, fmt.Errorf("ping %s database: %w", dialect, err)
}
- if err := applyMigrations(ctx, db, dialect); err != nil {
+ if err := applyMigrations(ctx, db, dialect, migrations); err != nil {
_ = db.Close()
return nil, fmt.Errorf("apply migrations: %w", err)
}
@@ -186,10 +193,11 @@ func ping(ctx context.Context, db *sql.DB, dialect Dialect) error {
}
}
-// applyMigrations brings the schema of the dialect up to date. Postgres
-// processes are serialized with an advisory lock held on a dedicated
-// connection; SQLite processes that race on a new database retry.
-func applyMigrations(ctx context.Context, db *sql.DB, dialect Dialect) error {
+// applyMigrations brings the schema of the dialect up to date, or up to its
+// first limit migrations when limit is greater than 0. Postgres processes are
+// serialized with an advisory lock held on a dedicated connection; SQLite
+// processes that race on a new database retry.
+func applyMigrations(ctx context.Context, db *sql.DB, dialect Dialect, limit int) error {
migrations := sqliteMigrations
name := "sqlite3"
attempts := maxMigrationAttempts
@@ -197,7 +205,12 @@ func applyMigrations(ctx context.Context, db *sql.DB, dialect Dialect) error {
migrations = postgresMigrations
name = "postgres"
attempts = 1
+ }
+ if limit > 0 {
+ migrations = migrations[:min(limit, len(migrations))]
+ }
+ if dialect == Postgres {
conn, err := db.Conn(ctx)
if err != nil {
return fmt.Errorf("get migration connection: %w", err)
@@ -332,6 +345,11 @@ func (s *Store) withReadTx(ctx context.Context, fn func(*sql.Tx) error) error {
return s.retryTx(ctx, s.readDB, s.readTxOptions(), fn)
}
+// retryHook, when set by a test, is called with the error of every transaction
+// that is retried, so that a test can assert which kind of conflict occurred
+// (doc/graft/PLAN7.md, LOCK ORDER: no deadlock). It is nil in production.
+var retryHook func(err error)
+
func (s *Store) retryTx(ctx context.Context, db *sql.DB, opts *sql.TxOptions, fn func(*sql.Tx) error) error {
var err error
for attempt := 0; ; attempt++ {
@@ -339,6 +357,9 @@ func (s *Store) retryTx(ctx context.Context, db *sql.DB, opts *sql.TxOptions, fn
if err == nil || attempt >= maxTxRetries || !isRetryable(err) || ctx.Err() != nil {
return err
}
+ if retryHook != nil {
+ retryHook(err)
+ }
if serr := sleep(ctx, attempt); serr != nil {
return err
}
diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.go
index 51fe9c8..f7895c4 100644
--- a/internal/graft/corestore/types.go
+++ b/internal/graft/corestore/types.go
@@ -152,6 +152,45 @@ type Changeset struct {
Updates []AppliedUpdate
}
+// Checkpoint is a consolidated state of a repository at a changeset sequence:
+// one pack with every object reachable from the references at Seq and from the
+// roots of the reviews open when it was built, plus the reference snapshot at
+// Seq (MetaStore.CheckpointRefs). Hydration starts from the newest checkpoint
+// and replays only the changesets after it (doc/graft/PLAN7.md).
+type Checkpoint struct {
+ RepoID int64
+ Seq uint64
+ PackKey string
+ PackBytes int64
+ // CreatedAt is a database time (doc/graft/PLAN7.md, CLOCK RULE).
+ CreatedAt time.Time
+}
+
+// RemovedRepository is the tombstone of a removed repository whose objects
+// are still to be deleted from the bucket.
+type RemovedRepository struct {
+ RepoID int64
+ Key RepoKey
+ // RemovedAt is a database time (doc/graft/PLAN7.md, CLOCK RULE).
+ RemovedAt time.Time
+}
+
+// GCState is what garbage collection decides on for one repository, read in
+// one database snapshot (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 5).
+type GCState struct {
+ // Now is the database time of the snapshot: every row committed before it is
+ // part of the snapshot.
+ Now time.Time
+ // Packs maps every changeset pack key to the greatest sequence of the
+ // changesets that reference it (a legacy key can be shared by several).
+ Packs map[string]uint64
+ // Checkpoints are the checkpoints, ascending by sequence.
+ Checkpoints []Checkpoint
+ // Orphans maps the keys observed unreferenced to the database time they were
+ // first observed.
+ Orphans map[string]time.Time
+}
+
// RefConflict is one reference whose current value did not satisfy its
// RefUpdate.
type RefConflict struct {
@@ -185,6 +224,14 @@ var (
// ErrBlobExists is returned by BlobStore.Put for a key that exists already:
// objects are never overwritten nor shared between writers.
ErrBlobExists = errors.New("storage core: blob already exists")
+ // ErrCheckpointNotFound is returned for a checkpoint sequence the
+ // repository has no checkpoint at.
+ ErrCheckpointNotFound = errors.New("storage core: checkpoint not found")
+ // ErrCheckpointRootsChanged is returned by MetaStore.CreateCheckpoint when an
+ // open review has a head or base commit that the checkpoint's pack does not
+ // include: the review was created or moved during the build
+ // (doc/graft/PLAN7.md, REVIEW ROOTS RULE R2).
+ ErrCheckpointRootsChanged = errors.New("storage core: the open reviews changed during the checkpoint build")
// ErrBlobCorrupt is returned by the reader of BlobStore.Get when the content
// does not hash to its key.
ErrBlobCorrupt = errors.New("storage core: blob content does not match its key")