Skip to content

Git designed for AI agents.

WP78: CreateCheckpoint, CollectGarbage and CollectRemovedRepositories

Federico Elia03ef6c21aecc
diff --git a/cmd/graft/main.go b/cmd/graft/main.go
index e5461f8..f0f2f24 100644
--- a/cmd/graft/main.go
+++ b/cmd/graft/main.go
@@ -265,7 +265,7 @@ commands:
cat REV:PATH print a file at a revision
search QUERY search file contents (--regex, -i, --path, --context, -n, --rev, --json)
review SUBCOMMAND create, show, list, comment, approve, request-changes, update, close (--branch, --title, --json)
- storage SUBCOMMAND import, state, verify a repository in the storage core (--connectivity, --json)
+ storage SUBCOMMAND import, state, verify, checkpoint, gc a repository in the storage core; gc --removed for a storage (--connectivity, --dry-run, --json)
environment: GRAFT_ADDRESS, GRAFT_TOKEN, GRAFT_STORAGE, GRAFT_USER, GRAFT_AUTHOR
diff --git a/cmd/graft/storage.go b/cmd/graft/storage.go
index 753ba85..842c89b 100644
--- a/cmd/graft/storage.go
+++ b/cmd/graft/storage.go
@@ -9,20 +9,23 @@ import (
"io"
"unicode/utf8"
+ "github.com/feder-cr/git-ai/internal/graft/workspace"
"github.com/feder-cr/git-ai/sdk"
)
// storageSynopsis is the command line of graft storage.
-const storageSynopsis = "usage: graft storage import|state|verify [-repo R] [--connectivity] [--json]"
+const storageSynopsis = "usage: graft storage import|state|verify|checkpoint|gc [-repo R] [--connectivity] [--dry-run] [--removed] [--json]"
// storageUsage lists the subcommands of graft storage.
-const storageUsage = "storage requires a subcommand: import, state or verify (" + storageSynopsis + ")"
+const storageUsage = "storage requires a subcommand: import, state, verify, checkpoint or gc (" + storageSynopsis + ")"
// storageFlags holds the flags of graft storage. Each subcommand reads only
// some of them, as its help text says.
type storageFlags struct {
repo string
connectivity bool
+ dryRun bool
+ removed bool
}
// storageCommand returns the storage command, the operator's view of the
@@ -37,7 +40,9 @@ func storageCommand() command {
}
fs.StringVar(&f.repo, "repo", "", "repository path, used without a workspace together with -address and -storage")
fs.BoolVar(&f.connectivity, "connectivity", false, "verify: also check the connectivity of the local repository (git fsck --connectivity-only) and that the object storage holds every object of the database references")
- fs.BoolVar(&o.jsonOut, "json", false, "import, state, verify: print the result as one JSON object")
+ fs.BoolVar(&f.dryRun, "dry-run", false, "gc: report what would be deleted, delete nothing")
+ fs.BoolVar(&f.removed, "removed", false, "gc: collect the removed repositories of the storage (-storage) instead of one repository; takes no -repo")
+ fs.BoolVar(&o.jsonOut, "json", false, "print the result as one JSON object")
},
run: func(ctx context.Context, out io.Writer, o *options, args []string) error {
return cmdStorage(ctx, out, o, f, args)
@@ -59,10 +64,124 @@ func cmdStorage(ctx context.Context, out io.Writer, o *options, f *storageFlags,
return storageState(ctx, out, o, f, rest)
case "verify":
return storageVerify(ctx, out, o, f, rest)
+ case "checkpoint":
+ return storageCheckpoint(ctx, out, o, f, rest)
+ case "gc":
+ return storageGC(ctx, out, o, f, rest)
}
return usagef("unknown storage subcommand %q: %s", sub, storageUsage)
}
+func storageCheckpoint(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {
+ if len(args) != 0 {
+ return usagef("storage checkpoint takes flags only: checkpoint [-repo R] [--json]")
+ }
+ repo, err := openStorage(ctx, o, f)
+ if err != nil {
+ return err
+ }
+ res, err := repo.StorageCheckpoint(ctx)
+ if err != nil {
+ return mapError(err)
+ }
+ if o.jsonOut {
+ return writeStorageJSON(out, res)
+ }
+ switch {
+ case res.Sequence == 0:
+ fmt.Fprintln(out, "no checkpoint: the repository has no object to consolidate")
+ case res.Created:
+ fmt.Fprintf(out, "checkpoint created at changeset %d: %d references, pack %d bytes\n", res.Sequence, res.References, res.PackBytes)
+ default:
+ fmt.Fprintf(out, "checkpoint at changeset %d exists already: %d references, pack %d bytes\n", res.Sequence, res.References, res.PackBytes)
+ }
+ return nil
+}
+
+func storageGC(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {
+ if len(args) != 0 {
+ return usagef("storage gc takes flags only: gc [-repo R] [--dry-run] [--json], or gc --removed [-storage S] [--dry-run] [--json]")
+ }
+ if f.removed {
+ if f.repo != "" {
+ return usagef("storage gc --removed collects the removed repositories of a storage and takes no -repo")
+ }
+ return storageGCRemoved(ctx, out, o, f)
+ }
+
+ repo, err := openStorage(ctx, o, f)
+ if err != nil {
+ return err
+ }
+ res, err := repo.StorageGC(ctx, f.dryRun)
+ if err != nil {
+ return mapError(err)
+ }
+ if o.jsonOut {
+ return writeStorageJSON(out, res)
+ }
+ verb := "deleted"
+ if res.DryRun {
+ verb = "would delete"
+ }
+ fmt.Fprintf(out, "%s %d objects (%d bytes): %d covered by the checkpoint at changeset %d, %d superseded checkpoints, %d orphans\n",
+ verb, res.PacksDeleted, res.BytesReclaimed, res.CoveredDeleted, res.Boundary, res.CheckpointsDropped, res.OrphansDeleted)
+ fmt.Fprintf(out, "orphans observed now %d, still in their grace period %d\n", res.OrphansObserved, res.OrphansInGrace)
+ return nil
+}
+
+func storageGCRemoved(ctx context.Context, out io.Writer, o *options, f *storageFlags) error {
+ client, err := openStorageClient(ctx, o)
+ if err != nil {
+ return err
+ }
+ res, err := client.StorageGCRemoved(ctx, "", f.dryRun)
+ if err != nil {
+ return mapError(err)
+ }
+ if o.jsonOut {
+ return writeStorageJSON(out, res)
+ }
+ verb := "deleted"
+ if res.DryRun {
+ verb = "would delete"
+ }
+ fmt.Fprintf(out, "%s %d objects (%d bytes) of %d removed repositories of storage %s\n",
+ verb, res.PacksDeleted, res.BytesReclaimed, res.Repositories, displayText(client.Storage()))
+ return nil
+}
+
+// openStorageClient returns the client of a storage-wide command: the server
+// and the storage of the workspace, or without a workspace -address and
+// -storage.
+func openStorageClient(ctx context.Context, o *options) (*sdk.Client, error) {
+ var t target
+ if root, err := workspaceRoot(o); err == nil {
+ if meta, err := workspace.ReadMeta(root); err == nil {
+ t = target{Address: pick(o.address, meta.Address), Token: o.token, Storage: pick(o.storage, meta.Storage), Repo: meta.Repo, User: o.user}
+ }
+ }
+ if t.Address == "" {
+ if o.address == "" {
+ return nil, usagef("storage gc --removed needs a workspace or -address")
+ }
+ t = target{Address: o.address, Token: o.token, Storage: o.storage, User: o.user}
+ }
+
+ // A storage-wide command names no repository, which the workspace backend
+ // requires: the client is dialed with the same options.
+ opts := []sdk.DialOption{sdk.WithStorage(pick(t.Storage, defaultStorage))}
+ if t.Token != "" {
+ opts = append(opts, sdk.WithToken(t.Token))
+ }
+ client, err := sdk.Dial(ctx, t.Address, opts...)
+ if err != nil {
+ return nil, err
+ }
+ registerCloser(ctx, client)
+ return client, nil
+}
+
func storageImport(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {
if len(args) != 0 {
return usagef("storage import takes flags only: import [-repo R] [--json]")
diff --git a/cmd/graft/storage_test.go b/cmd/graft/storage_test.go
index 29fa929..4ccad23 100644
--- a/cmd/graft/storage_test.go
+++ b/cmd/graft/storage_test.go
@@ -99,6 +99,53 @@ func TestCLIStorageAgainstServer(t *testing.T) {
require.Equal(t, []sdk.StorageDivergence{{Reference: "refs/heads/rogue", LocalTarget: main.String()}}, verify.Divergences)
}
+// TestCLIStorageMaintenanceAgainstServer drives graft storage checkpoint and gc
+// against a real server whose storage core runs in authoritative mode. It runs
+// serially because the CLI tests share the package-level backend factory.
+func TestCLIStorageMaintenanceAgainstServer(t *testing.T) {
+ ctx := testhelper.Context(t)
+ cfg := testcfg.Build(t)
+ cfg.Graft.Standalone = true
+ core := coretest.Open(t, coretest.Options{Mode: corestore.ModeAuthoritative})
+ addr := testserver.RunGitalyServer(t, cfg, setup.RegisterAll, testserver.WithGraftStorage(core))
+
+ repo, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
+ server := []string{"-address", addr, "-storage", repo.GetStorageName()}
+ target := append(append([]string{}, server...), "-repo", repo.GetRelativePath())
+ cli := func(args ...string) (int, string, string) {
+ return runCLI(t, append(append([]string{"storage"}, args...), target...)...)
+ }
+
+ code, _, stderr := cli("import")
+ require.Equal(t, exitOK, code, stderr)
+
+ code, stdout, stderr := cli("checkpoint")
+ require.Equal(t, exitOK, code, stderr)
+ require.Regexp(t, `^checkpoint created at changeset 1: 2 references, pack [1-9][0-9]* bytes\n$`, stdout)
+
+ code, stdout, stderr = cli("checkpoint", "--json")
+ require.Equal(t, exitOK, code, stderr)
+ var checkpoint sdk.StorageCheckpoint
+ require.NoError(t, json.Unmarshal([]byte(stdout), &checkpoint))
+ require.False(t, checkpoint.Created, "a checkpoint at the head exists")
+ require.EqualValues(t, 1, checkpoint.Sequence)
+
+ code, stdout, stderr = cli("gc", "--dry-run", "--json")
+ require.Equal(t, exitOK, code, stderr)
+ var gc sdk.StorageGC
+ require.NoError(t, json.Unmarshal([]byte(stdout), &gc))
+ require.True(t, gc.DryRun)
+
+ code, stdout, stderr = cli("gc")
+ require.Equal(t, exitOK, code, stderr)
+ require.Contains(t, stdout, "deleted 0 objects (0 bytes)")
+
+ code, stdout, stderr = runCLI(t, append([]string{"storage", "gc", "--removed"}, server...)...)
+ require.Equal(t, exitOK, code, stderr)
+ require.Equal(t, "deleted 0 objects (0 bytes) of 0 removed repositories of storage "+repo.GetStorageName()+"\n", stdout)
+}
+
// TestCLIStorageCoreOff checks that a server without a storage core reports
// mode off and refuses an import.
func TestCLIStorageCoreOff(t *testing.T) {
@@ -121,11 +168,15 @@ func TestCLIStorageUsageErrorsDoNotContactServer(t *testing.T) {
dir := filepath.Join(t.TempDir(), "missing")
cases := map[string][]string{
- "no subcommand": {"storage", "-C", dir},
- "unknown subcommand": {"storage", "repair", "-C", dir},
- "import with arg": {"storage", "import", "-C", dir, "extra"},
- "state with arg": {"storage", "state", "-C", dir, "extra"},
- "verify with arg": {"storage", "verify", "-C", dir, "extra"},
+ "no subcommand": {"storage", "-C", dir},
+ "unknown subcommand": {"storage", "repair", "-C", dir},
+ "import with arg": {"storage", "import", "-C", dir, "extra"},
+ "state with arg": {"storage", "state", "-C", dir, "extra"},
+ "verify with arg": {"storage", "verify", "-C", dir, "extra"},
+ "checkpoint with arg": {"storage", "checkpoint", "-C", dir, "extra"},
+ "gc with arg": {"storage", "gc", "-C", dir, "extra"},
+ "gc removed and repo": {"storage", "gc", "--removed", "-C", dir, "-repo", "a.git"},
+ "gc removed without server": {"storage", "gc", "--removed", "-C", dir},
}
for name, args := range cases {
code, _, stderr := runCLI(t, args...)
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index 683cca1..ca96608 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -1536,6 +1536,29 @@ graft storage gc --removed [-storage S] [--dry-run] [--json]
`gc -repo R` touches only that repository (answer 1); `--removed` is the storage-wide
sweep of removed repositories and takes no `-repo`.
+**As implemented (WP78).**
+* `CreateCheckpointResponse` has no `review_commits`: since the REVIEW ROOTS
+ RULE the review commits are references, counted in `references`.
+* `CollectGarbage` takes the lease as `maintenance.RequestLeaseHolder`:
+ `node_id/PID/request-N`, unique per request. So a request excludes the
+ background runs (`node_id/PID`), and the other requests, of its own process
+ too.
+* The SDK methods are in `sdk/storage.go`, already among the operator-only
+ sources of the agent allowlist test, not in a new `sdk/storage_gc.go`.
+* `graft storage gc --removed` dials a client of the server without a
+ repository: the workspace backend requires one.
+* The handlers map the errors of `checkpoint` and `gc`, which carry their
+ codes. Off is `FailedPrecondition`; a checkpoint in shadow mode is
+ `FailedPrecondition`; an untracked repository is `NotFound`; an unknown
+ storage is `InvalidArgument`.
+* Tests:
+ * the handlers on SQLite and Postgres: created and idempotent, refusals, an
+ orphan collected after the grace, dry run, lease held by another holder
+ and released after the request, removed repositories;
+ * the SDK against a fake;
+ * the CLI against a real authoritative server (`checkpoint`, `gc`,
+ `gc --dry-run --json`, `gc --removed`) and its usage errors.
+
## Rejected alternatives
* **Local-only hidden keep-alive references for review roots** (option b of the
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index 197fb36..7f2950d 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -320,6 +320,38 @@ registration gets a new repository ID and a full pack, and takes the changelog
and the reviews from the files again, so do this before switching to
authoritative mode.
+### Checkpoints and garbage collection
+
+```sh
+graft storage checkpoint -address unix:/var/opt/graft/gitaly.sock -repo @hashed/ab/cd/x.git [--json]
+graft storage gc -address unix:/var/opt/graft/gitaly.sock -repo @hashed/ab/cd/x.git [--dry-run] [--json]
+graft storage gc --removed -address unix:/var/opt/graft/gitaly.sock [-storage default] [--dry-run] [--json]
+```
+
+- `checkpoint` (authoritative mode) consolidates the repository at its current
+ database head: one pack of every object reachable from its references, and
+ the reference snapshot. A fresh node then hydrates from the checkpoint
+ instead of replaying every changeset, and garbage collection can delete the
+ packs the checkpoint covers. A checkpoint that exists at the head is
+ reported, not built again.
+- `gc` deletes the packs of one repository that no reader needs any more:
+ - the changeset packs covered by a checkpoint at least `gc_grace` old, and
+ the checkpoints that one supersedes;
+ - the objects no database row references, once garbage collection observed
+ them unreferenced at least `gc_grace` earlier (the packs of writes that
+ died before their database commit). The first run only observes them.
+
+ It runs in shadow mode too (orphans only: there are no checkpoints). It holds
+ the repository's maintenance lease while it runs, and fails with
+ `FailedPrecondition` when another maintenance of the repository (the
+ background one, another node) holds it.
+- `gc --removed` deletes every object of the repositories of the storage that
+ were removed at least `gc_grace` ago.
+- `--dry-run` reports what would be deleted and changes nothing.
+
+These commands are for operators: they need the shared token, and an agent
+token is refused.
+
### Switching from shadow to authoritative
1. Run every node in shadow mode and import every repository
@@ -622,9 +654,10 @@ show:
packs, and there is no virtual filesystem.
- **Checkpoints and pack garbage collection run in the background only when
`maintenance = "auto"`** (authoritative mode; off by default). Without it,
- packs are never deleted and hydration replays every changeset after the
- newest checkpoint, from 1 when there is none. The on-demand `graft storage`
- commands for both are not available yet.
+ they run when an operator runs `graft storage checkpoint` and `graft storage
+ gc` (see [Checkpoints and garbage collection](#checkpoints-and-garbage-collection));
+ otherwise packs are never deleted and hydration replays every changeset after
+ the newest checkpoint, from 1 when there is none.
- **Praefect is not supported in authoritative mode.** Configuration validation
requires `[graft] standalone = true` for authoritative mode. Shadow mode
works with or without Praefect.
diff --git a/internal/gitaly/service/storagecore/maintenance.go b/internal/gitaly/service/storagecore/maintenance.go
new file mode 100644
index 0000000..564edcf
--- /dev/null
+++ b/internal/gitaly/service/storagecore/maintenance.go
@@ -0,0 +1,139 @@
+package storagecore
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/feder-cr/git-ai/internal/gitaly/storage"
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/checkpoint"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/gc"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/maintenance"
+ "github.com/feder-cr/git-ai/internal/structerr"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+)
+
+// releaseTimeout bounds the release of a maintenance lease after a request.
+const releaseTimeout = 10 * time.Second
+
+// CreateCheckpoint implements gitalypb.StorageCoreServiceServer
+// (doc/graft/PLAN7.md, "Checkpoint creation").
+func (s *server) CreateCheckpoint(ctx context.Context, req *gitalypb.CreateCheckpointRequest) (*gitalypb.CreateCheckpointResponse, error) {
+ repo := req.GetRepository()
+ if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
+ return nil, structerr.NewInvalidArgument("%w", err)
+ }
+ core, err := s.enabledCore()
+ if err != nil {
+ return nil, err
+ }
+
+ cp := checkpoint.New(core, s.deps.GetLogger(), s.deps.GetLocator(), s.deps.GetRepositoryFactory(), s.deps.GetGraftStorageSyncer())
+ created, isNew, err := cp.Create(ctx, repo)
+ if err != nil {
+ return nil, err
+ }
+
+ resp := &gitalypb.CreateCheckpointResponse{
+ CheckpointSequence: created.Seq,
+ Created: isNew,
+ PackBytes: created.PackBytes,
+ }
+ if created.Seq == 0 {
+ // No object to consolidate: there is no checkpoint.
+ return resp, nil
+ }
+ refs, err := core.Meta.CheckpointRefs(ctx, created.RepoID, created.Seq)
+ if err != nil {
+ return nil, structerr.NewUnavailable("storage core: checkpoint references: %w", err)
+ }
+ resp.References = uint32(len(refs))
+ return resp, nil
+}
+
+// CollectGarbage implements gitalypb.StorageCoreServiceServer
+// (doc/graft/PLAN7.md, "Pack GC"). It holds the repository's maintenance lease
+// while it collects, so that it does not run alongside the background
+// maintenance or another request; correctness does not depend on the lease.
+func (s *server) CollectGarbage(ctx context.Context, req *gitalypb.CollectGarbageRequest) (*gitalypb.CollectGarbageResponse, error) {
+ repo := req.GetRepository()
+ if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
+ return nil, structerr.NewInvalidArgument("%w", err)
+ }
+ core, err := s.enabledCore()
+ if err != nil {
+ return nil, err
+ }
+ ttl, err := s.deps.GetCfg().Graft.Storage.LeaseTTLOrDefault()
+ if err != nil {
+ return nil, structerr.NewInternal("%w", err)
+ }
+
+ record, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repo))
+ if errors.Is(err, corestore.ErrRepositoryNotFound) {
+ return nil, structerr.NewNotFound("repository is not tracked by the storage core")
+ }
+ if err != nil {
+ return nil, structerr.NewUnavailable("storage core: %w", err)
+ }
+
+ holder := maintenance.RequestLeaseHolder(core)
+ acquired, err := core.Meta.AcquireMaintenanceLease(ctx, record.ID, holder, ttl)
+ if errors.Is(err, corestore.ErrRepositoryNotFound) {
+ return nil, structerr.NewNotFound("repository is not tracked by the storage core")
+ }
+ if err != nil {
+ return nil, structerr.NewUnavailable("storage core: maintenance lease: %w", err)
+ }
+ if !acquired {
+ return nil, structerr.NewFailedPrecondition("another maintenance operation of this repository is in progress")
+ }
+ defer func() {
+ releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
+ defer cancel()
+ if err := core.Meta.ReleaseMaintenanceLease(releaseCtx, record.ID, holder); err != nil {
+ s.deps.GetLogger().WithError(err).WarnContext(ctx, "storage core: release the maintenance lease; it expires by itself")
+ }
+ }()
+
+ report, err := gc.New(core, s.deps.GetLogger()).CollectRepository(ctx, record.ID, req.GetDryRun())
+ if err != nil {
+ return nil, err
+ }
+ return &gitalypb.CollectGarbageResponse{
+ Boundary: report.Boundary,
+ CoveredDeleted: uint32(report.CoveredDeleted),
+ CheckpointsDropped: uint32(report.CheckpointsDropped),
+ OrphansDeleted: uint32(report.OrphansDeleted),
+ OrphansObserved: uint32(report.OrphansObserved),
+ OrphansInGrace: uint32(report.OrphansInGrace),
+ PacksDeleted: uint32(report.PacksDeleted),
+ BytesReclaimed: report.BytesReclaimed,
+ DryRun: report.DryRun,
+ }, nil
+}
+
+// CollectRemovedRepositories implements gitalypb.StorageCoreServiceServer
+// (doc/graft/PLAN7.md, "Removed repositories").
+func (s *server) CollectRemovedRepositories(ctx context.Context, req *gitalypb.CollectRemovedRepositoriesRequest) (*gitalypb.CollectRemovedRepositoriesResponse, error) {
+ storageName := req.GetStorageName()
+ if _, err := s.deps.GetLocator().GetStorageByName(ctx, storageName); err != nil {
+ return nil, structerr.NewInvalidArgument("%w", err)
+ }
+ core, err := s.enabledCore()
+ if err != nil {
+ return nil, err
+ }
+
+ report, err := gc.New(core, s.deps.GetLogger()).CollectRemoved(ctx, storageName, req.GetDryRun())
+ if err != nil {
+ return nil, err
+ }
+ return &gitalypb.CollectRemovedRepositoriesResponse{
+ Repositories: uint32(report.RemovedRepos),
+ PacksDeleted: uint32(report.PacksDeleted),
+ BytesReclaimed: report.BytesReclaimed,
+ DryRun: report.DryRun,
+ }, nil
+}
diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.go
new file mode 100644
index 0000000..91ca797
--- /dev/null
+++ b/internal/gitaly/service/storagecore/maintenance_test.go
@@ -0,0 +1,190 @@
+package storagecore
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+)
+
+// shortGC makes garbage collection wait a short grace, so that a test sees an
+// orphan deleted. The writers' deadline (grace - margin) stays long enough for
+// the import's upload under load.
+func shortGC(core *corestore.Core) {
+ core.GCGrace = time.Second
+ core.GCMargin = 100 * time.Millisecond
+}
+
+// putOrphan stores an object under the repository's pack prefix that no row
+// references, as a write that died before its database commit leaves.
+func (h *harness) putOrphan(repoID int64, content string) string {
+ h.t.Helper()
+ key := corestore.NewPackKey(repoID, sha256.Sum256([]byte(content)))
+ require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, bytes.NewReader([]byte(content))))
+ return key
+}
+
+func (h *harness) exists(key string) bool {
+ h.t.Helper()
+ ok, err := h.core.Blobs.Exists(h.ctx, key)
+ require.NoError(h.t, err)
+ return ok
+}
+
+func TestCreateCheckpoint(t *testing.T) {
+ t.Parallel()
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ h := setup(t, corestore.ModeAuthoritative, dialect)
+ h.seed()
+ repoID := h.importRepo()
+
+ resp, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: h.repo})
+ require.NoError(t, err)
+ record, err := h.core.Meta.GetRepository(h.ctx, corestore.KeyOf(h.repo))
+ require.NoError(t, err)
+ require.Equal(t, repoID, record.ID)
+ require.True(t, resp.GetCreated())
+ require.Equal(t, record.HeadSeq, resp.GetCheckpointSequence())
+ require.Positive(t, resp.GetPackBytes())
+ require.Equal(t, uint32(len(h.localRefs())), resp.GetReferences(), "HEAD, the branch and the tag")
+
+ again, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: h.repo})
+ require.NoError(t, err)
+ require.False(t, again.GetCreated(), "a checkpoint at the head exists")
+ require.Equal(t, resp.GetCheckpointSequence(), again.GetCheckpointSequence())
+ })
+}
+
+func TestCreateCheckpointRefusals(t *testing.T) {
+ t.Parallel()
+
+ off := setup(t, corestore.ModeOff, sqlstore.SQLite)
+ _, err := off.client.CreateCheckpoint(off.ctx, &gitalypb.CreateCheckpointRequest{Repository: off.repo})
+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)
+
+ shadow := setup(t, corestore.ModeShadow, sqlstore.SQLite)
+ shadow.seed()
+ shadow.importRepo()
+ _, err = shadow.client.CreateCheckpoint(shadow.ctx, &gitalypb.CreateCheckpointRequest{Repository: shadow.repo})
+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "checkpoints are authoritative mode only: %v", err)
+
+ untracked := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)
+ _, err = untracked.client.CreateCheckpoint(untracked.ctx, &gitalypb.CreateCheckpointRequest{Repository: untracked.repo})
+ require.Equal(t, codes.NotFound, status.Code(err), "%v", err)
+
+ _, err = untracked.client.CreateCheckpoint(untracked.ctx, &gitalypb.CreateCheckpointRequest{})
+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)
+}
+
+func TestCollectGarbage(t *testing.T) {
+ t.Parallel()
+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {
+ h := setup(t, mode, dialect, shortGC)
+ h.seed()
+ repoID := h.importRepo()
+ orphan := h.putOrphan(repoID, "the pack of a write that died")
+
+ dry, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo, DryRun: true})
+ require.NoError(t, err)
+ require.True(t, dry.GetDryRun())
+ require.Equal(t, uint32(0), dry.GetPacksDeleted())
+ require.True(t, h.exists(orphan))
+
+ first, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
+ require.NoError(t, err)
+ require.Equal(t, uint32(1), first.GetOrphansObserved())
+ require.True(t, h.exists(orphan), "observed, not yet deleted")
+
+ require.Eventually(t, func() bool {
+ resp, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
+ require.NoError(t, err)
+ return resp.GetOrphansDeleted() == 1 && resp.GetBytesReclaimed() > 0
+ }, 30*time.Second, 50*time.Millisecond)
+ require.False(t, h.exists(orphan))
+ require.Empty(t, h.verify(true).GetDivergences(), "the repository is intact")
+ })
+}
+
+func TestCollectGarbageRefusals(t *testing.T) {
+ t.Parallel()
+
+ off := setup(t, corestore.ModeOff, sqlstore.SQLite)
+ _, err := off.client.CollectGarbage(off.ctx, &gitalypb.CollectGarbageRequest{Repository: off.repo})
+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)
+
+ h := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)
+ _, err = h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
+ require.Equal(t, codes.NotFound, status.Code(err), "untracked: %v", err)
+
+ h.seed()
+ repoID := h.importRepo()
+ acquired, err := h.core.Meta.AcquireMaintenanceLease(h.ctx, repoID, "node-b/1", time.Hour)
+ require.NoError(t, err)
+ require.True(t, acquired)
+ _, err = h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)
+ require.Contains(t, err.Error(), "another maintenance operation of this repository is in progress")
+
+ require.NoError(t, h.core.Meta.ReleaseMaintenanceLease(h.ctx, repoID, "node-b/1"))
+ _, err = h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})
+ require.NoError(t, err)
+ // The request released its lease.
+ acquired, err = h.core.Meta.AcquireMaintenanceLease(h.ctx, repoID, "node-b/1", time.Hour)
+ require.NoError(t, err)
+ require.True(t, acquired)
+}
+
+func TestCollectRemovedRepositories(t *testing.T) {
+ t.Parallel()
+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {
+ h := setup(t, mode, dialect, shortGC)
+ h.seed()
+ repoID := h.importRepo()
+ packs := h.packKeys(repoID)
+ require.NotEmpty(t, packs)
+
+ removed, err := h.core.Meta.DeleteRepository(h.ctx, corestore.KeyOf(h.repo))
+ require.NoError(t, err)
+ require.True(t, removed)
+
+ storageName := h.cfg.Storages[0].Name
+ require.Eventually(t, func() bool {
+ resp, err := h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: storageName})
+ require.NoError(t, err)
+ return resp.GetRepositories() == 1 && resp.GetPacksDeleted() >= uint32(len(packs))
+ }, 30*time.Second, 50*time.Millisecond)
+ require.Empty(t, h.packKeys(repoID))
+ })
+}
+
+func TestCollectRemovedRepositoriesRefusals(t *testing.T) {
+ t.Parallel()
+
+ off := setup(t, corestore.ModeOff, sqlstore.SQLite)
+ _, err := off.client.CollectRemovedRepositories(off.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: off.cfg.Storages[0].Name})
+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)
+
+ h := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)
+ _, err = h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: "no-such-storage"})
+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)
+}
+
+// packKeys returns the objects of the repository in the bucket.
+func (h *harness) packKeys(repoID int64) map[string]bool {
+ h.t.Helper()
+ keys := map[string]bool{}
+ require.NoError(h.t, h.core.Blobs.List(h.ctx, fmt.Sprintf("repos/%d/", repoID), func(info corestore.BlobInfo) error {
+ keys[info.Key] = true
+ return nil
+ }))
+ return keys
+}
diff --git a/internal/graft/corestore/maintenance/maintenance.go b/internal/graft/corestore/maintenance/maintenance.go
index ef2c7a7..db09ec5 100644
--- a/internal/graft/corestore/maintenance/maintenance.go
+++ b/internal/graft/corestore/maintenance/maintenance.go
@@ -18,6 +18,7 @@ import (
"fmt"
"os"
"sync"
+ "sync/atomic"
"time"
"github.com/prometheus/client_golang/prometheus"
@@ -83,6 +84,23 @@ type Runner struct {
var _ prometheus.Collector = (*Runner)(nil)
+// LeaseHolder names the background maintenance of this process in the
+// maintenance leases of core: the node ID and the process ID, so that two
+// processes of one node (a graceful upgrade) never both hold a lease.
+func LeaseHolder(core *corestore.Core) string {
+ return fmt.Sprintf("%s/%d", core.NodeID, os.Getpid())
+}
+
+// requests numbers the on-demand maintenance operations of the process.
+var requests atomic.Uint64
+
+// RequestLeaseHolder names one on-demand maintenance operation of this process
+// (a StorageCoreService RPC): a holder of its own, so that it excludes the
+// background runs and the other operations.
+func RequestLeaseHolder(core *corestore.Core) string {
+ return fmt.Sprintf("%s/request-%d", LeaseHolder(core), requests.Add(1))
+}
+
// New returns the runner of core for the node's storages.
func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collector, fresh corestore.Freshener, logger log.Logger, cfg Config, storages []string) *Runner {
r := &Runner{
@@ -93,7 +111,7 @@ func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collec
logger: logger,
cfg: cfg,
storages: storages,
- holder: fmt.Sprintf("%s/%d", core.NodeID, os.Getpid()),
+ holder: LeaseHolder(core),
runs: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "graft_storage_maintenance_runs_total",
Help: "Runs of the background maintenance of the storage core, by result (ok, error: at least one step failed).",
diff --git a/proto/go/gitalypb/storage_core.pb.go b/proto/go/gitalypb/storage_core.pb.go
index 42f7fb3..3623dd1 100644
--- a/proto/go/gitalypb/storage_core.pb.go
+++ b/proto/go/gitalypb/storage_core.pb.go
@@ -416,6 +416,434 @@ func (x *VerifyRepositoryResponse) GetConnectivityError() string {
return ""
}
+// CreateCheckpointRequest is the request of CreateCheckpoint.
+type CreateCheckpointRequest struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // repository is the repository to checkpoint.
+ Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CreateCheckpointRequest) Reset() {
+ *x = CreateCheckpointRequest{}
+ mi := &file_storage_core_proto_msgTypes[6]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CreateCheckpointRequest) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CreateCheckpointRequest) ProtoMessage() {}
+
+func (x *CreateCheckpointRequest) ProtoReflect() protoreflect.Message {
+ mi := &file_storage_core_proto_msgTypes[6]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CreateCheckpointRequest.ProtoReflect.Descriptor instead.
+func (*CreateCheckpointRequest) Descriptor() ([]byte, []int) {
+ return file_storage_core_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *CreateCheckpointRequest) GetRepository() *Repository {
+ if x != nil {
+ return x.Repository
+ }
+ return nil
+}
+
+// CreateCheckpointResponse is the response of CreateCheckpoint.
+type CreateCheckpointResponse struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // checkpoint_sequence is the changeset sequence the checkpoint is at; 0 when
+ // the repository has no object to consolidate (every reference is symbolic).
+ CheckpointSequence uint64 `protobuf:"varint,1,opt,name=checkpoint_sequence,json=checkpointSequence,proto3" json:"checkpoint_sequence,omitempty"`
+ // created is false when a checkpoint at the head existed already.
+ Created bool `protobuf:"varint,2,opt,name=created,proto3" json:"created,omitempty"`
+ // pack_bytes is the size of the checkpoint's pack.
+ PackBytes int64 `protobuf:"varint,3,opt,name=pack_bytes,json=packBytes,proto3" json:"pack_bytes,omitempty"`
+ // references is the number of references in the checkpoint's snapshot.
+ References uint32 `protobuf:"varint,4,opt,name=references,proto3" json:"references,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CreateCheckpointResponse) Reset() {
+ *x = CreateCheckpointResponse{}
+ mi := &file_storage_core_proto_msgTypes[7]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CreateCheckpointResponse) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CreateCheckpointResponse) ProtoMessage() {}
+
+func (x *CreateCheckpointResponse) ProtoReflect() protoreflect.Message {
+ mi := &file_storage_core_proto_msgTypes[7]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CreateCheckpointResponse.ProtoReflect.Descriptor instead.
+func (*CreateCheckpointResponse) Descriptor() ([]byte, []int) {
+ return file_storage_core_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *CreateCheckpointResponse) GetCheckpointSequence() uint64 {
+ if x != nil {
+ return x.CheckpointSequence
+ }
+ return 0
+}
+
+func (x *CreateCheckpointResponse) GetCreated() bool {
+ if x != nil {
+ return x.Created
+ }
+ return false
+}
+
+func (x *CreateCheckpointResponse) GetPackBytes() int64 {
+ if x != nil {
+ return x.PackBytes
+ }
+ return 0
+}
+
+func (x *CreateCheckpointResponse) GetReferences() uint32 {
+ if x != nil {
+ return x.References
+ }
+ return 0
+}
+
+// CollectGarbageRequest is the request of CollectGarbage.
+type CollectGarbageRequest struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // repository is the repository whose packs are collected.
+ Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"`
+ // dry_run reports what would be deleted and changes nothing.
+ DryRun bool `protobuf:"varint,2,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CollectGarbageRequest) Reset() {
+ *x = CollectGarbageRequest{}
+ mi := &file_storage_core_proto_msgTypes[8]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CollectGarbageRequest) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CollectGarbageRequest) ProtoMessage() {}
+
+func (x *CollectGarbageRequest) ProtoReflect() protoreflect.Message {
+ mi := &file_storage_core_proto_msgTypes[8]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CollectGarbageRequest.ProtoReflect.Descriptor instead.
+func (*CollectGarbageRequest) Descriptor() ([]byte, []int) {
+ return file_storage_core_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *CollectGarbageRequest) GetRepository() *Repository {
+ if x != nil {
+ return x.Repository
+ }
+ return nil
+}
+
+func (x *CollectGarbageRequest) GetDryRun() bool {
+ if x != nil {
+ return x.DryRun
+ }
+ return false
+}
+
+// CollectGarbageResponse is the response of CollectGarbage.
+type CollectGarbageResponse struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // boundary is the sequence of the newest checkpoint at least gc_grace old;
+ // 0 when there is none.
+ Boundary uint64 `protobuf:"varint,1,opt,name=boundary,proto3" json:"boundary,omitempty"`
+ // covered_deleted is the number of changeset packs covered by the boundary.
+ CoveredDeleted uint32 `protobuf:"varint,2,opt,name=covered_deleted,json=coveredDeleted,proto3" json:"covered_deleted,omitempty"`
+ // checkpoints_dropped is the number of checkpoints superseded by the boundary.
+ CheckpointsDropped uint32 `protobuf:"varint,3,opt,name=checkpoints_dropped,json=checkpointsDropped,proto3" json:"checkpoints_dropped,omitempty"`
+ // orphans_deleted is the number of unreferenced objects observed at least
+ // gc_grace ago.
+ OrphansDeleted uint32 `protobuf:"varint,4,opt,name=orphans_deleted,json=orphansDeleted,proto3" json:"orphans_deleted,omitempty"`
+ // orphans_observed is the number of unreferenced objects seen for the first
+ // time.
+ OrphansObserved uint32 `protobuf:"varint,5,opt,name=orphans_observed,json=orphansObserved,proto3" json:"orphans_observed,omitempty"`
+ // orphans_in_grace is the number of unreferenced objects observed less than
+ // gc_grace ago.
+ OrphansInGrace uint32 `protobuf:"varint,6,opt,name=orphans_in_grace,json=orphansInGrace,proto3" json:"orphans_in_grace,omitempty"`
+ // packs_deleted is the number of objects deleted (to delete in a dry run).
+ PacksDeleted uint32 `protobuf:"varint,7,opt,name=packs_deleted,json=packsDeleted,proto3" json:"packs_deleted,omitempty"`
+ // bytes_reclaimed is the size of those objects.
+ BytesReclaimed int64 `protobuf:"varint,8,opt,name=bytes_reclaimed,json=bytesReclaimed,proto3" json:"bytes_reclaimed,omitempty"`
+ // dry_run tells that nothing was deleted or recorded.
+ DryRun bool `protobuf:"varint,9,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CollectGarbageResponse) Reset() {
+ *x = CollectGarbageResponse{}
+ mi := &file_storage_core_proto_msgTypes[9]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CollectGarbageResponse) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CollectGarbageResponse) ProtoMessage() {}
+
+func (x *CollectGarbageResponse) ProtoReflect() protoreflect.Message {
+ mi := &file_storage_core_proto_msgTypes[9]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CollectGarbageResponse.ProtoReflect.Descriptor instead.
+func (*CollectGarbageResponse) Descriptor() ([]byte, []int) {
+ return file_storage_core_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *CollectGarbageResponse) GetBoundary() uint64 {
+ if x != nil {
+ return x.Boundary
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetCoveredDeleted() uint32 {
+ if x != nil {
+ return x.CoveredDeleted
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetCheckpointsDropped() uint32 {
+ if x != nil {
+ return x.CheckpointsDropped
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetOrphansDeleted() uint32 {
+ if x != nil {
+ return x.OrphansDeleted
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetOrphansObserved() uint32 {
+ if x != nil {
+ return x.OrphansObserved
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetOrphansInGrace() uint32 {
+ if x != nil {
+ return x.OrphansInGrace
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetPacksDeleted() uint32 {
+ if x != nil {
+ return x.PacksDeleted
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetBytesReclaimed() int64 {
+ if x != nil {
+ return x.BytesReclaimed
+ }
+ return 0
+}
+
+func (x *CollectGarbageResponse) GetDryRun() bool {
+ if x != nil {
+ return x.DryRun
+ }
+ return false
+}
+
+// CollectRemovedRepositoriesRequest is the request of
+// CollectRemovedRepositories.
+type CollectRemovedRepositoriesRequest struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // storage_name is the storage whose removed repositories are collected.
+ StorageName string `protobuf:"bytes,1,opt,name=storage_name,json=storageName,proto3" json:"storage_name,omitempty"`
+ // dry_run reports what would be deleted and changes nothing.
+ DryRun bool `protobuf:"varint,2,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CollectRemovedRepositoriesRequest) Reset() {
+ *x = CollectRemovedRepositoriesRequest{}
+ mi := &file_storage_core_proto_msgTypes[10]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CollectRemovedRepositoriesRequest) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CollectRemovedRepositoriesRequest) ProtoMessage() {}
+
+func (x *CollectRemovedRepositoriesRequest) ProtoReflect() protoreflect.Message {
+ mi := &file_storage_core_proto_msgTypes[10]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CollectRemovedRepositoriesRequest.ProtoReflect.Descriptor instead.
+func (*CollectRemovedRepositoriesRequest) Descriptor() ([]byte, []int) {
+ return file_storage_core_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *CollectRemovedRepositoriesRequest) GetStorageName() string {
+ if x != nil {
+ return x.StorageName
+ }
+ return ""
+}
+
+func (x *CollectRemovedRepositoriesRequest) GetDryRun() bool {
+ if x != nil {
+ return x.DryRun
+ }
+ return false
+}
+
+// CollectRemovedRepositoriesResponse is the response of
+// CollectRemovedRepositories.
+type CollectRemovedRepositoriesResponse struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // repositories is the number of removed repositories whose objects were
+ // deleted (to delete in a dry run).
+ Repositories uint32 `protobuf:"varint,1,opt,name=repositories,proto3" json:"repositories,omitempty"`
+ // packs_deleted is the number of objects deleted.
+ PacksDeleted uint32 `protobuf:"varint,2,opt,name=packs_deleted,json=packsDeleted,proto3" json:"packs_deleted,omitempty"`
+ // bytes_reclaimed is the size of those objects.
+ BytesReclaimed int64 `protobuf:"varint,3,opt,name=bytes_reclaimed,json=bytesReclaimed,proto3" json:"bytes_reclaimed,omitempty"`
+ // dry_run tells that nothing was deleted or recorded.
+ DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CollectRemovedRepositoriesResponse) Reset() {
+ *x = CollectRemovedRepositoriesResponse{}
+ mi := &file_storage_core_proto_msgTypes[11]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CollectRemovedRepositoriesResponse) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CollectRemovedRepositoriesResponse) ProtoMessage() {}
+
+func (x *CollectRemovedRepositoriesResponse) ProtoReflect() protoreflect.Message {
+ mi := &file_storage_core_proto_msgTypes[11]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CollectRemovedRepositoriesResponse.ProtoReflect.Descriptor instead.
+func (*CollectRemovedRepositoriesResponse) Descriptor() ([]byte, []int) {
+ return file_storage_core_proto_rawDescGZIP(), []int{11}
+}
+
+func (x *CollectRemovedRepositoriesResponse) GetRepositories() uint32 {
+ if x != nil {
+ return x.Repositories
+ }
+ return 0
+}
+
+func (x *CollectRemovedRepositoriesResponse) GetPacksDeleted() uint32 {
+ if x != nil {
+ return x.PacksDeleted
+ }
+ return 0
+}
+
+func (x *CollectRemovedRepositoriesResponse) GetBytesReclaimed() int64 {
+ if x != nil {
+ return x.BytesReclaimed
+ }
+ return 0
+}
+
+func (x *CollectRemovedRepositoriesResponse) GetDryRun() bool {
+ if x != nil {
+ return x.DryRun
+ }
+ return false
+}
+
// ReferenceDivergence is a reference whose local value differs from the
// database.
type VerifyRepositoryResponse_ReferenceDivergence struct {
@@ -433,7 +861,7 @@ type VerifyRepositoryResponse_ReferenceDivergence struct {
func (x *VerifyRepositoryResponse_ReferenceDivergence) Reset() {
*x = VerifyRepositoryResponse_ReferenceDivergence{}
- mi := &file_storage_core_proto_msgTypes[6]
+ mi := &file_storage_core_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -445,7 +873,7 @@ func (x *VerifyRepositoryResponse_ReferenceDivergence) String() string {
func (*VerifyRepositoryResponse_ReferenceDivergence) ProtoMessage() {}
func (x *VerifyRepositoryResponse_ReferenceDivergence) ProtoReflect() protoreflect.Message {
- mi := &file_storage_core_proto_msgTypes[6]
+ mi := &file_storage_core_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -525,11 +953,49 @@ const file_storage_core_proto_rawDesc = "" +
"\x13ReferenceDivergence\x12\x1c\n" +
"\treference\x18\x01 \x01(\fR\treference\x12!\n" +
"\flocal_target\x18\x02 \x01(\tR\vlocalTarget\x12'\n" +
- "\x0fdatabase_target\x18\x03 \x01(\tR\x0edatabaseTarget2\xae\x02\n" +
+ "\x0fdatabase_target\x18\x03 \x01(\tR\x0edatabaseTarget\"S\n" +
+ "\x17CreateCheckpointRequest\x128\n" +
+ "\n" +
+ "repository\x18\x01 \x01(\v2\x12.gitaly.RepositoryB\x04\x98\xc6,\x01R\n" +
+ "repository\"\xa4\x01\n" +
+ "\x18CreateCheckpointResponse\x12/\n" +
+ "\x13checkpoint_sequence\x18\x01 \x01(\x04R\x12checkpointSequence\x12\x18\n" +
+ "\acreated\x18\x02 \x01(\bR\acreated\x12\x1d\n" +
+ "\n" +
+ "pack_bytes\x18\x03 \x01(\x03R\tpackBytes\x12\x1e\n" +
+ "\n" +
+ "references\x18\x04 \x01(\rR\n" +
+ "references\"j\n" +
+ "\x15CollectGarbageRequest\x128\n" +
+ "\n" +
+ "repository\x18\x01 \x01(\v2\x12.gitaly.RepositoryB\x04\x98\xc6,\x01R\n" +
+ "repository\x12\x17\n" +
+ "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xf3\x02\n" +
+ "\x16CollectGarbageResponse\x12\x1a\n" +
+ "\bboundary\x18\x01 \x01(\x04R\bboundary\x12'\n" +
+ "\x0fcovered_deleted\x18\x02 \x01(\rR\x0ecoveredDeleted\x12/\n" +
+ "\x13checkpoints_dropped\x18\x03 \x01(\rR\x12checkpointsDropped\x12'\n" +
+ "\x0forphans_deleted\x18\x04 \x01(\rR\x0eorphansDeleted\x12)\n" +
+ "\x10orphans_observed\x18\x05 \x01(\rR\x0forphansObserved\x12(\n" +
+ "\x10orphans_in_grace\x18\x06 \x01(\rR\x0eorphansInGrace\x12#\n" +
+ "\rpacks_deleted\x18\a \x01(\rR\fpacksDeleted\x12'\n" +
+ "\x0fbytes_reclaimed\x18\b \x01(\x03R\x0ebytesReclaimed\x12\x17\n" +
+ "\adry_run\x18\t \x01(\bR\x06dryRun\"e\n" +
+ "!CollectRemovedRepositoriesRequest\x12'\n" +
+ "\fstorage_name\x18\x01 \x01(\tB\x04\x88\xc6,\x01R\vstorageName\x12\x17\n" +
+ "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xaf\x01\n" +
+ "\"CollectRemovedRepositoriesResponse\x12\"\n" +
+ "\frepositories\x18\x01 \x01(\rR\frepositories\x12#\n" +
+ "\rpacks_deleted\x18\x02 \x01(\rR\fpacksDeleted\x12'\n" +
+ "\x0fbytes_reclaimed\x18\x03 \x01(\x03R\x0ebytesReclaimed\x12\x17\n" +
+ "\adry_run\x18\x04 \x01(\bR\x06dryRun2\xe5\x04\n" +
"\x12StorageCoreService\x12]\n" +
"\x10ImportRepository\x12\x1f.gitaly.ImportRepositoryRequest\x1a .gitaly.ImportRepositoryResponse\"\x06\xfa\x97(\x02\b\x01\x12Z\n" +
"\x0fGetStorageState\x12\x1e.gitaly.GetStorageStateRequest\x1a\x1f.gitaly.GetStorageStateResponse\"\x06\xfa\x97(\x02\b\x02\x12]\n" +
- "\x10VerifyRepository\x12\x1f.gitaly.VerifyRepositoryRequest\x1a .gitaly.VerifyRepositoryResponse\"\x06\xfa\x97(\x02\b\x02B.Z,github.com/feder-cr/git-ai/proto/go/gitalypbb\x06proto3"
+ "\x10VerifyRepository\x12\x1f.gitaly.VerifyRepositoryRequest\x1a .gitaly.VerifyRepositoryResponse\"\x06\xfa\x97(\x02\b\x02\x12]\n" +
+ "\x10CreateCheckpoint\x12\x1f.gitaly.CreateCheckpointRequest\x1a .gitaly.CreateCheckpointResponse\"\x06\xfa\x97(\x02\b\x01\x12W\n" +
+ "\x0eCollectGarbage\x12\x1d.gitaly.CollectGarbageRequest\x1a\x1e.gitaly.CollectGarbageResponse\"\x06\xfa\x97(\x02\b\x01\x12}\n" +
+ "\x1aCollectRemovedRepositories\x12).gitaly.CollectRemovedRepositoriesRequest\x1a*.gitaly.CollectRemovedRepositoriesResponse\"\b\xfa\x97(\x04\b\x01\x10\x02B.Z,github.com/feder-cr/git-ai/proto/go/gitalypbb\x06proto3"
var (
file_storage_core_proto_rawDescOnce sync.Once
@@ -543,7 +1009,7 @@ func file_storage_core_proto_rawDescGZIP() []byte {
return file_storage_core_proto_rawDescData
}
-var file_storage_core_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
+var file_storage_core_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
var file_storage_core_proto_goTypes = []any{
(*ImportRepositoryRequest)(nil), // 0: gitaly.ImportRepositoryRequest
(*ImportRepositoryResponse)(nil), // 1: gitaly.ImportRepositoryResponse
@@ -551,25 +1017,39 @@ var file_storage_core_proto_goTypes = []any{
(*GetStorageStateResponse)(nil), // 3: gitaly.GetStorageStateResponse
(*VerifyRepositoryRequest)(nil), // 4: gitaly.VerifyRepositoryRequest
(*VerifyRepositoryResponse)(nil), // 5: gitaly.VerifyRepositoryResponse
- (*VerifyRepositoryResponse_ReferenceDivergence)(nil), // 6: gitaly.VerifyRepositoryResponse.ReferenceDivergence
- (*Repository)(nil), // 7: gitaly.Repository
+ (*CreateCheckpointRequest)(nil), // 6: gitaly.CreateCheckpointRequest
+ (*CreateCheckpointResponse)(nil), // 7: gitaly.CreateCheckpointResponse
+ (*CollectGarbageRequest)(nil), // 8: gitaly.CollectGarbageRequest
+ (*CollectGarbageResponse)(nil), // 9: gitaly.CollectGarbageResponse
+ (*CollectRemovedRepositoriesRequest)(nil), // 10: gitaly.CollectRemovedRepositoriesRequest
+ (*CollectRemovedRepositoriesResponse)(nil), // 11: gitaly.CollectRemovedRepositoriesResponse
+ (*VerifyRepositoryResponse_ReferenceDivergence)(nil), // 12: gitaly.VerifyRepositoryResponse.ReferenceDivergence
+ (*Repository)(nil), // 13: gitaly.Repository
}
var file_storage_core_proto_depIdxs = []int32{
- 7, // 0: gitaly.ImportRepositoryRequest.repository:type_name -> gitaly.Repository
- 7, // 1: gitaly.GetStorageStateRequest.repository:type_name -> gitaly.Repository
- 7, // 2: gitaly.VerifyRepositoryRequest.repository:type_name -> gitaly.Repository
- 6, // 3: gitaly.VerifyRepositoryResponse.divergences:type_name -> gitaly.VerifyRepositoryResponse.ReferenceDivergence
- 0, // 4: gitaly.StorageCoreService.ImportRepository:input_type -> gitaly.ImportRepositoryRequest
- 2, // 5: gitaly.StorageCoreService.GetStorageState:input_type -> gitaly.GetStorageStateRequest
- 4, // 6: gitaly.StorageCoreService.VerifyRepository:input_type -> gitaly.VerifyRepositoryRequest
- 1, // 7: gitaly.StorageCoreService.ImportRepository:output_type -> gitaly.ImportRepositoryResponse
- 3, // 8: gitaly.StorageCoreService.GetStorageState:output_type -> gitaly.GetStorageStateResponse
- 5, // 9: gitaly.StorageCoreService.VerifyRepository:output_type -> gitaly.VerifyRepositoryResponse
- 7, // [7:10] is the sub-list for method output_type
- 4, // [4:7] is the sub-list for method input_type
- 4, // [4:4] is the sub-list for extension type_name
- 4, // [4:4] is the sub-list for extension extendee
- 0, // [0:4] is the sub-list for field type_name
+ 13, // 0: gitaly.ImportRepositoryRequest.repository:type_name -> gitaly.Repository
+ 13, // 1: gitaly.GetStorageStateRequest.repository:type_name -> gitaly.Repository
+ 13, // 2: gitaly.VerifyRepositoryRequest.repository:type_name -> gitaly.Repository
+ 12, // 3: gitaly.VerifyRepositoryResponse.divergences:type_name -> gitaly.VerifyRepositoryResponse.ReferenceDivergence
+ 13, // 4: gitaly.CreateCheckpointRequest.repository:type_name -> gitaly.Repository
+ 13, // 5: gitaly.CollectGarbageRequest.repository:type_name -> gitaly.Repository
+ 0, // 6: gitaly.StorageCoreService.ImportRepository:input_type -> gitaly.ImportRepositoryRequest
+ 2, // 7: gitaly.StorageCoreService.GetStorageState:input_type -> gitaly.GetStorageStateRequest
+ 4, // 8: gitaly.StorageCoreService.VerifyRepository:input_type -> gitaly.VerifyRepositoryRequest
+ 6, // 9: gitaly.StorageCoreService.CreateCheckpoint:input_type -> gitaly.CreateCheckpointRequest
+ 8, // 10: gitaly.StorageCoreService.CollectGarbage:input_type -> gitaly.CollectGarbageRequest
+ 10, // 11: gitaly.StorageCoreService.CollectRemovedRepositories:input_type -> gitaly.CollectRemovedRepositoriesRequest
+ 1, // 12: gitaly.StorageCoreService.ImportRepository:output_type -> gitaly.ImportRepositoryResponse
+ 3, // 13: gitaly.StorageCoreService.GetStorageState:output_type -> gitaly.GetStorageStateResponse
+ 5, // 14: gitaly.StorageCoreService.VerifyRepository:output_type -> gitaly.VerifyRepositoryResponse
+ 7, // 15: gitaly.StorageCoreService.CreateCheckpoint:output_type -> gitaly.CreateCheckpointResponse
+ 9, // 16: gitaly.StorageCoreService.CollectGarbage:output_type -> gitaly.CollectGarbageResponse
+ 11, // 17: gitaly.StorageCoreService.CollectRemovedRepositories:output_type -> gitaly.CollectRemovedRepositoriesResponse
+ 12, // [12:18] is the sub-list for method output_type
+ 6, // [6:12] is the sub-list for method input_type
+ 6, // [6:6] is the sub-list for extension type_name
+ 6, // [6:6] is the sub-list for extension extendee
+ 0, // [0:6] is the sub-list for field type_name
}
func init() { file_storage_core_proto_init() }
@@ -585,7 +1065,7 @@ func file_storage_core_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_storage_core_proto_rawDesc), len(file_storage_core_proto_rawDesc)),
NumEnums: 0,
- NumMessages: 7,
+ NumMessages: 13,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/proto/go/gitalypb/storage_core_grpc.pb.go b/proto/go/gitalypb/storage_core_grpc.pb.go
index 55763d1..461fa24 100644
--- a/proto/go/gitalypb/storage_core_grpc.pb.go
+++ b/proto/go/gitalypb/storage_core_grpc.pb.go
@@ -19,9 +19,12 @@ import (
const _ = grpc.SupportPackageIsVersion9
const (
- StorageCoreService_ImportRepository_FullMethodName = "/gitaly.StorageCoreService/ImportRepository"
- StorageCoreService_GetStorageState_FullMethodName = "/gitaly.StorageCoreService/GetStorageState"
- StorageCoreService_VerifyRepository_FullMethodName = "/gitaly.StorageCoreService/VerifyRepository"
+ StorageCoreService_ImportRepository_FullMethodName = "/gitaly.StorageCoreService/ImportRepository"
+ StorageCoreService_GetStorageState_FullMethodName = "/gitaly.StorageCoreService/GetStorageState"
+ StorageCoreService_VerifyRepository_FullMethodName = "/gitaly.StorageCoreService/VerifyRepository"
+ StorageCoreService_CreateCheckpoint_FullMethodName = "/gitaly.StorageCoreService/CreateCheckpoint"
+ StorageCoreService_CollectGarbage_FullMethodName = "/gitaly.StorageCoreService/CollectGarbage"
+ StorageCoreService_CollectRemovedRepositories_FullMethodName = "/gitaly.StorageCoreService/CollectRemovedRepositories"
)
// StorageCoreServiceClient is the client API for StorageCoreService service.
@@ -50,6 +53,23 @@ type StorageCoreServiceClient interface {
// optionally checks the connectivity of the local repository. In
// authoritative mode the local repository is caught up first.
VerifyRepository(ctx context.Context, in *VerifyRepositoryRequest, opts ...grpc.CallOption) (*VerifyRepositoryResponse, error)
+ // CreateCheckpoint builds a checkpoint of the repository at its current
+ // database head and records it: one pack of every object reachable from the
+ // references at the head, with the reference snapshot, so that hydration
+ // starts from it and garbage collection can delete the packs it covers. It
+ // fails with FailedPrecondition outside authoritative mode.
+ CreateCheckpoint(ctx context.Context, in *CreateCheckpointRequest, opts ...grpc.CallOption) (*CreateCheckpointResponse, error)
+ // CollectGarbage deletes the packs of the repository that no reader needs
+ // any more: the objects no row references once they were observed so at
+ // least gc_grace ago, the changeset packs covered by a checkpoint at least
+ // gc_grace old, and the checkpoints that one supersedes. It touches no other
+ // repository. It fails with FailedPrecondition when the storage core is off,
+ // or while another maintenance of the repository holds its lease.
+ CollectGarbage(ctx context.Context, in *CollectGarbageRequest, opts ...grpc.CallOption) (*CollectGarbageResponse, error)
+ // CollectRemovedRepositories deletes every object of the repositories of a
+ // storage that were removed at least gc_grace ago. It fails with
+ // FailedPrecondition when the storage core is off.
+ CollectRemovedRepositories(ctx context.Context, in *CollectRemovedRepositoriesRequest, opts ...grpc.CallOption) (*CollectRemovedRepositoriesResponse, error)
}
type storageCoreServiceClient struct {
@@ -90,6 +110,36 @@ func (c *storageCoreServiceClient) VerifyRepository(ctx context.Context, in *Ver
return out, nil
}
+func (c *storageCoreServiceClient) CreateCheckpoint(ctx context.Context, in *CreateCheckpointRequest, opts ...grpc.CallOption) (*CreateCheckpointResponse, error) {
+ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+ out := new(CreateCheckpointResponse)
+ err := c.cc.Invoke(ctx, StorageCoreService_CreateCheckpoint_FullMethodName, in, out, cOpts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *storageCoreServiceClient) CollectGarbage(ctx context.Context, in *CollectGarbageRequest, opts ...grpc.CallOption) (*CollectGarbageResponse, error) {
+ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+ out := new(CollectGarbageResponse)
+ err := c.cc.Invoke(ctx, StorageCoreService_CollectGarbage_FullMethodName, in, out, cOpts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+func (c *storageCoreServiceClient) CollectRemovedRepositories(ctx context.Context, in *CollectRemovedRepositoriesRequest, opts ...grpc.CallOption) (*CollectRemovedRepositoriesResponse, error) {
+ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+ out := new(CollectRemovedRepositoriesResponse)
+ err := c.cc.Invoke(ctx, StorageCoreService_CollectRemovedRepositories_FullMethodName, in, out, cOpts...)
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
// StorageCoreServiceServer is the server API for StorageCoreService service.
// All implementations must embed UnimplementedStorageCoreServiceServer
// for forward compatibility.
@@ -116,6 +166,23 @@ type StorageCoreServiceServer interface {
// optionally checks the connectivity of the local repository. In
// authoritative mode the local repository is caught up first.
VerifyRepository(context.Context, *VerifyRepositoryRequest) (*VerifyRepositoryResponse, error)
+ // CreateCheckpoint builds a checkpoint of the repository at its current
+ // database head and records it: one pack of every object reachable from the
+ // references at the head, with the reference snapshot, so that hydration
+ // starts from it and garbage collection can delete the packs it covers. It
+ // fails with FailedPrecondition outside authoritative mode.
+ CreateCheckpoint(context.Context, *CreateCheckpointRequest) (*CreateCheckpointResponse, error)
+ // CollectGarbage deletes the packs of the repository that no reader needs
+ // any more: the objects no row references once they were observed so at
+ // least gc_grace ago, the changeset packs covered by a checkpoint at least
+ // gc_grace old, and the checkpoints that one supersedes. It touches no other
+ // repository. It fails with FailedPrecondition when the storage core is off,
+ // or while another maintenance of the repository holds its lease.
+ CollectGarbage(context.Context, *CollectGarbageRequest) (*CollectGarbageResponse, error)
+ // CollectRemovedRepositories deletes every object of the repositories of a
+ // storage that were removed at least gc_grace ago. It fails with
+ // FailedPrecondition when the storage core is off.
+ CollectRemovedRepositories(context.Context, *CollectRemovedRepositoriesRequest) (*CollectRemovedRepositoriesResponse, error)
mustEmbedUnimplementedStorageCoreServiceServer()
}
@@ -135,6 +202,15 @@ func (UnimplementedStorageCoreServiceServer) GetStorageState(context.Context, *G
func (UnimplementedStorageCoreServiceServer) VerifyRepository(context.Context, *VerifyRepositoryRequest) (*VerifyRepositoryResponse, error) {
return nil, status.Error(codes.Unimplemented, "method VerifyRepository not implemented")
}
+func (UnimplementedStorageCoreServiceServer) CreateCheckpoint(context.Context, *CreateCheckpointRequest) (*CreateCheckpointResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "method CreateCheckpoint not implemented")
+}
+func (UnimplementedStorageCoreServiceServer) CollectGarbage(context.Context, *CollectGarbageRequest) (*CollectGarbageResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "method CollectGarbage not implemented")
+}
+func (UnimplementedStorageCoreServiceServer) CollectRemovedRepositories(context.Context, *CollectRemovedRepositoriesRequest) (*CollectRemovedRepositoriesResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "method CollectRemovedRepositories not implemented")
+}
func (UnimplementedStorageCoreServiceServer) mustEmbedUnimplementedStorageCoreServiceServer() {}
func (UnimplementedStorageCoreServiceServer) testEmbeddedByValue() {}
@@ -210,6 +286,60 @@ func _StorageCoreService_VerifyRepository_Handler(srv interface{}, ctx context.C
return interceptor(ctx, in, info, handler)
}
+func _StorageCoreService_CreateCheckpoint_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(CreateCheckpointRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(StorageCoreServiceServer).CreateCheckpoint(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: StorageCoreService_CreateCheckpoint_FullMethodName,
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(StorageCoreServiceServer).CreateCheckpoint(ctx, req.(*CreateCheckpointRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _StorageCoreService_CollectGarbage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(CollectGarbageRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(StorageCoreServiceServer).CollectGarbage(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: StorageCoreService_CollectGarbage_FullMethodName,
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(StorageCoreServiceServer).CollectGarbage(ctx, req.(*CollectGarbageRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
+func _StorageCoreService_CollectRemovedRepositories_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+ in := new(CollectRemovedRepositoriesRequest)
+ if err := dec(in); err != nil {
+ return nil, err
+ }
+ if interceptor == nil {
+ return srv.(StorageCoreServiceServer).CollectRemovedRepositories(ctx, in)
+ }
+ info := &grpc.UnaryServerInfo{
+ Server: srv,
+ FullMethod: StorageCoreService_CollectRemovedRepositories_FullMethodName,
+ }
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+ return srv.(StorageCoreServiceServer).CollectRemovedRepositories(ctx, req.(*CollectRemovedRepositoriesRequest))
+ }
+ return interceptor(ctx, in, info, handler)
+}
+
// StorageCoreService_ServiceDesc is the grpc.ServiceDesc for StorageCoreService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -229,6 +359,18 @@ var StorageCoreService_ServiceDesc = grpc.ServiceDesc{
MethodName: "VerifyRepository",
Handler: _StorageCoreService_VerifyRepository_Handler,
},
+ {
+ MethodName: "CreateCheckpoint",
+ Handler: _StorageCoreService_CreateCheckpoint_Handler,
+ },
+ {
+ MethodName: "CollectGarbage",
+ Handler: _StorageCoreService_CollectGarbage_Handler,
+ },
+ {
+ MethodName: "CollectRemovedRepositories",
+ Handler: _StorageCoreService_CollectRemovedRepositories_Handler,
+ },
},
Streams: []grpc.StreamDesc{},
Metadata: "storage_core.proto",
diff --git a/proto/storage_core.proto b/proto/storage_core.proto
index 3f2da65..50a1c24 100644
--- a/proto/storage_core.proto
+++ b/proto/storage_core.proto
@@ -44,6 +44,39 @@ service StorageCoreService {
op: ACCESSOR
};
}
+
+ // CreateCheckpoint builds a checkpoint of the repository at its current
+ // database head and records it: one pack of every object reachable from the
+ // references at the head, with the reference snapshot, so that hydration
+ // starts from it and garbage collection can delete the packs it covers. It
+ // fails with FailedPrecondition outside authoritative mode.
+ rpc CreateCheckpoint(CreateCheckpointRequest) returns (CreateCheckpointResponse) {
+ option (op_type) = {
+ op: MUTATOR
+ };
+ }
+
+ // CollectGarbage deletes the packs of the repository that no reader needs
+ // any more: the objects no row references once they were observed so at
+ // least gc_grace ago, the changeset packs covered by a checkpoint at least
+ // gc_grace old, and the checkpoints that one supersedes. It touches no other
+ // repository. It fails with FailedPrecondition when the storage core is off,
+ // or while another maintenance of the repository holds its lease.
+ rpc CollectGarbage(CollectGarbageRequest) returns (CollectGarbageResponse) {
+ option (op_type) = {
+ op: MUTATOR
+ };
+ }
+
+ // CollectRemovedRepositories deletes every object of the repositories of a
+ // storage that were removed at least gc_grace ago. It fails with
+ // FailedPrecondition when the storage core is off.
+ rpc CollectRemovedRepositories(CollectRemovedRepositoriesRequest) returns (CollectRemovedRepositoriesResponse) {
+ option (op_type) = {
+ op: MUTATOR
+ scope_level: STORAGE
+ };
+ }
}
// ImportRepositoryRequest is the request of ImportRepository.
@@ -121,3 +154,79 @@ message VerifyRepositoryResponse {
// connectivity_error describes a failed connectivity check.
string connectivity_error = 4;
}
+
+// CreateCheckpointRequest is the request of CreateCheckpoint.
+message CreateCheckpointRequest {
+ // repository is the repository to checkpoint.
+ Repository repository = 1 [(target_repository)=true];
+}
+
+// CreateCheckpointResponse is the response of CreateCheckpoint.
+message CreateCheckpointResponse {
+ // checkpoint_sequence is the changeset sequence the checkpoint is at; 0 when
+ // the repository has no object to consolidate (every reference is symbolic).
+ uint64 checkpoint_sequence = 1;
+ // created is false when a checkpoint at the head existed already.
+ bool created = 2;
+ // pack_bytes is the size of the checkpoint's pack.
+ int64 pack_bytes = 3;
+ // references is the number of references in the checkpoint's snapshot.
+ uint32 references = 4;
+}
+
+// CollectGarbageRequest is the request of CollectGarbage.
+message CollectGarbageRequest {
+ // repository is the repository whose packs are collected.
+ Repository repository = 1 [(target_repository)=true];
+ // dry_run reports what would be deleted and changes nothing.
+ bool dry_run = 2;
+}
+
+// CollectGarbageResponse is the response of CollectGarbage.
+message CollectGarbageResponse {
+ // boundary is the sequence of the newest checkpoint at least gc_grace old;
+ // 0 when there is none.
+ uint64 boundary = 1;
+ // covered_deleted is the number of changeset packs covered by the boundary.
+ uint32 covered_deleted = 2;
+ // checkpoints_dropped is the number of checkpoints superseded by the boundary.
+ uint32 checkpoints_dropped = 3;
+ // orphans_deleted is the number of unreferenced objects observed at least
+ // gc_grace ago.
+ uint32 orphans_deleted = 4;
+ // orphans_observed is the number of unreferenced objects seen for the first
+ // time.
+ uint32 orphans_observed = 5;
+ // orphans_in_grace is the number of unreferenced objects observed less than
+ // gc_grace ago.
+ uint32 orphans_in_grace = 6;
+ // packs_deleted is the number of objects deleted (to delete in a dry run).
+ uint32 packs_deleted = 7;
+ // bytes_reclaimed is the size of those objects.
+ int64 bytes_reclaimed = 8;
+ // dry_run tells that nothing was deleted or recorded.
+ bool dry_run = 9;
+}
+
+// CollectRemovedRepositoriesRequest is the request of
+// CollectRemovedRepositories.
+message CollectRemovedRepositoriesRequest {
+ // storage_name is the storage whose removed repositories are collected.
+ string storage_name = 1 [(storage)=true];
+ // dry_run reports what would be deleted and changes nothing.
+ bool dry_run = 2;
+}
+
+// CollectRemovedRepositoriesResponse is the response of
+// CollectRemovedRepositories.
+message CollectRemovedRepositoriesResponse {
+ // repositories is the number of removed repositories whose objects were
+ // deleted (to delete in a dry run).
+ uint32 repositories = 1;
+ // packs_deleted is the number of objects deleted.
+ uint32 packs_deleted = 2;
+ // bytes_reclaimed is the size of those objects.
+ int64 bytes_reclaimed = 3;
+ // dry_run tells that nothing was deleted or recorded.
+ bool dry_run = 4;
+}
diff --git a/sdk/storage.go b/sdk/storage.go
index 55f54e5..8e0ddd8 100644
--- a/sdk/storage.go
+++ b/sdk/storage.go
@@ -147,6 +147,125 @@ func (r *Repo) StorageVerify(ctx context.Context, connectivity bool) (*StorageVe
return out, nil
}
+// StorageCheckpoint is the outcome of checkpointing a repository.
+type StorageCheckpoint struct {
+ // Sequence is the changeset sequence the checkpoint is at; 0 when the
+ // repository has no object to consolidate (every reference is symbolic).
+ Sequence uint64 `json:"sequence"`
+ // Created is false when a checkpoint at the head existed already.
+ Created bool `json:"created"`
+ // PackBytes is the size of the checkpoint's pack.
+ PackBytes int64 `json:"pack_bytes"`
+ // References is the number of references in the checkpoint's snapshot.
+ References uint32 `json:"references"`
+}
+
+// StorageGC is what garbage collection of one repository deleted, or would
+// delete in a dry run.
+type StorageGC struct {
+ // Boundary is the sequence of the newest checkpoint at least gc_grace old,
+ // 0 when there is none.
+ Boundary uint64 `json:"boundary"`
+ // CoveredDeleted are the changeset packs covered by the boundary.
+ CoveredDeleted uint32 `json:"covered_deleted"`
+ // CheckpointsDropped are the checkpoints superseded by the boundary.
+ CheckpointsDropped uint32 `json:"checkpoints_dropped"`
+ // OrphansDeleted are the unreferenced objects observed at least gc_grace ago.
+ OrphansDeleted uint32 `json:"orphans_deleted"`
+ // OrphansObserved are the unreferenced objects seen for the first time.
+ OrphansObserved uint32 `json:"orphans_observed"`
+ // OrphansInGrace are the unreferenced objects observed less than gc_grace ago.
+ OrphansInGrace uint32 `json:"orphans_in_grace"`
+ // PacksDeleted counts every object deleted.
+ PacksDeleted uint32 `json:"packs_deleted"`
+ // BytesReclaimed is the size of those objects.
+ BytesReclaimed int64 `json:"bytes_reclaimed"`
+ // DryRun tells that nothing was deleted or recorded.
+ DryRun bool `json:"dry_run"`
+}
+
+// StorageGCRemoved is what garbage collection of a storage's removed
+// repositories deleted, or would delete in a dry run.
+type StorageGCRemoved struct {
+ // Repositories is the number of removed repositories whose objects were
+ // deleted.
+ Repositories uint32 `json:"repositories"`
+ // PacksDeleted counts every object deleted.
+ PacksDeleted uint32 `json:"packs_deleted"`
+ // BytesReclaimed is the size of those objects.
+ BytesReclaimed int64 `json:"bytes_reclaimed"`
+ // DryRun tells that nothing was deleted or recorded.
+ DryRun bool `json:"dry_run"`
+}
+
+// StorageCheckpoint checkpoints the repository at its current database head:
+// one pack of every object reachable from its references, with the reference
+// snapshot, so that hydration starts from it and garbage collection can delete
+// the packs it covers. It fails with FailedPrecondition outside authoritative
+// mode.
+func (r *Repo) StorageCheckpoint(ctx context.Context) (*StorageCheckpoint, error) {
+ resp, err := r.storageCore().CreateCheckpoint(ctx, &gitalypb.CreateCheckpointRequest{
+ Repository: r.repository(),
+ })
+ if err != nil {
+ return nil, err
+ }
+ return &StorageCheckpoint{
+ Sequence: resp.GetCheckpointSequence(),
+ Created: resp.GetCreated(),
+ PackBytes: resp.GetPackBytes(),
+ References: resp.GetReferences(),
+ }, nil
+}
+
+// StorageGC deletes the packs of the repository that no reader needs any more
+// (with dryRun, only reports them). It touches no other repository. It fails
+// with FailedPrecondition when the storage core is off or another maintenance
+// of the repository is in progress.
+func (r *Repo) StorageGC(ctx context.Context, dryRun bool) (*StorageGC, error) {
+ resp, err := r.storageCore().CollectGarbage(ctx, &gitalypb.CollectGarbageRequest{
+ Repository: r.repository(),
+ DryRun: dryRun,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return &StorageGC{
+ Boundary: resp.GetBoundary(),
+ CoveredDeleted: resp.GetCoveredDeleted(),
+ CheckpointsDropped: resp.GetCheckpointsDropped(),
+ OrphansDeleted: resp.GetOrphansDeleted(),
+ OrphansObserved: resp.GetOrphansObserved(),
+ OrphansInGrace: resp.GetOrphansInGrace(),
+ PacksDeleted: resp.GetPacksDeleted(),
+ BytesReclaimed: resp.GetBytesReclaimed(),
+ DryRun: resp.GetDryRun(),
+ }, nil
+}
+
+// StorageGCRemoved deletes every object of the repositories of the storage
+// that were removed at least gc_grace ago (with dryRun, only reports them). An
+// empty storage is the client's storage (WithStorage). It fails with
+// FailedPrecondition when the storage core is off.
+func (c *Client) StorageGCRemoved(ctx context.Context, storage string, dryRun bool) (*StorageGCRemoved, error) {
+ if storage == "" {
+ storage = c.storage
+ }
+ resp, err := gitalypb.NewStorageCoreServiceClient(c.conn).CollectRemovedRepositories(ctx, &gitalypb.CollectRemovedRepositoriesRequest{
+ StorageName: storage,
+ DryRun: dryRun,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return &StorageGCRemoved{
+ Repositories: resp.GetRepositories(),
+ PacksDeleted: resp.GetPacksDeleted(),
+ BytesReclaimed: resp.GetBytesReclaimed(),
+ DryRun: resp.GetDryRun(),
+ }, nil
+}
+
// storageCore returns the storage core service client on the connection of the client.
func (r *Repo) storageCore() gitalypb.StorageCoreServiceClient {
return gitalypb.NewStorageCoreServiceClient(r.client.conn)
diff --git a/sdk/storage_test.go b/sdk/storage_test.go
index bb04acc..0ebc310 100644
--- a/sdk/storage_test.go
+++ b/sdk/storage_test.go
@@ -19,15 +19,51 @@ import (
type fakeStorageCore struct {
gitalypb.UnimplementedStorageCoreServiceServer
- importResp *gitalypb.ImportRepositoryResponse
- stateResp *gitalypb.GetStorageStateResponse
- verifyResp *gitalypb.VerifyRepositoryResponse
- err error
+ importResp *gitalypb.ImportRepositoryResponse
+ stateResp *gitalypb.GetStorageStateResponse
+ verifyResp *gitalypb.VerifyRepositoryResponse
+ checkpointResp *gitalypb.CreateCheckpointResponse
+ gcResp *gitalypb.CollectGarbageResponse
+ removedResp *gitalypb.CollectRemovedRepositoriesResponse
+ err error
- mu sync.Mutex
- importReqs []*gitalypb.ImportRepositoryRequest
- stateReqs []*gitalypb.GetStorageStateRequest
- verifyReqs []*gitalypb.VerifyRepositoryRequest
+ mu sync.Mutex
+ importReqs []*gitalypb.ImportRepositoryRequest
+ stateReqs []*gitalypb.GetStorageStateRequest
+ verifyReqs []*gitalypb.VerifyRepositoryRequest
+ checkpointReqs []*gitalypb.CreateCheckpointRequest
+ gcReqs []*gitalypb.CollectGarbageRequest
+ removedReqs []*gitalypb.CollectRemovedRepositoriesRequest
+}
+
+func (f *fakeStorageCore) CreateCheckpoint(_ context.Context, req *gitalypb.CreateCheckpointRequest) (*gitalypb.CreateCheckpointResponse, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.checkpointReqs = append(f.checkpointReqs, req)
+ if f.err != nil {
+ return nil, f.err
+ }
+ return f.checkpointResp, nil
+}
+
+func (f *fakeStorageCore) CollectGarbage(_ context.Context, req *gitalypb.CollectGarbageRequest) (*gitalypb.CollectGarbageResponse, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.gcReqs = append(f.gcReqs, req)
+ if f.err != nil {
+ return nil, f.err
+ }
+ return f.gcResp, nil
+}
+
+func (f *fakeStorageCore) CollectRemovedRepositories(_ context.Context, req *gitalypb.CollectRemovedRepositoriesRequest) (*gitalypb.CollectRemovedRepositoriesResponse, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.removedReqs = append(f.removedReqs, req)
+ if f.err != nil {
+ return nil, f.err
+ }
+ return f.removedResp, nil
}
func (f *fakeStorageCore) ImportRepository(_ context.Context, req *gitalypb.ImportRepositoryRequest) (*gitalypb.ImportRepositoryResponse, error) {
@@ -70,6 +106,88 @@ func storageRepo(t *testing.T, fake *fakeStorageCore) *sdk.Repo {
return client.Repo("@hashed/ab/cd/repo.git")
}
+func TestStorageCheckpoint(t *testing.T) {
+ fake := &fakeStorageCore{checkpointResp: &gitalypb.CreateCheckpointResponse{
+ CheckpointSequence: 42,
+ Created: true,
+ PackBytes: 8192,
+ References: 5,
+ }}
+ repo := storageRepo(t, fake)
+
+ res, err := repo.StorageCheckpoint(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, &sdk.StorageCheckpoint{Sequence: 42, Created: true, PackBytes: 8192, References: 5}, res)
+ require.Len(t, fake.checkpointReqs, 1)
+ require.Equal(t, "@hashed/ab/cd/repo.git", fake.checkpointReqs[0].GetRepository().GetRelativePath())
+}
+
+func TestStorageGC(t *testing.T) {
+ fake := &fakeStorageCore{gcResp: &gitalypb.CollectGarbageResponse{
+ Boundary: 7,
+ CoveredDeleted: 3,
+ CheckpointsDropped: 1,
+ OrphansDeleted: 2,
+ OrphansObserved: 4,
+ OrphansInGrace: 5,
+ PacksDeleted: 6,
+ BytesReclaimed: 1 << 20,
+ DryRun: true,
+ }}
+ repo := storageRepo(t, fake)
+
+ res, err := repo.StorageGC(t.Context(), true)
+ require.NoError(t, err)
+ require.Equal(t, &sdk.StorageGC{
+ Boundary: 7,
+ CoveredDeleted: 3,
+ CheckpointsDropped: 1,
+ OrphansDeleted: 2,
+ OrphansObserved: 4,
+ OrphansInGrace: 5,
+ PacksDeleted: 6,
+ BytesReclaimed: 1 << 20,
+ DryRun: true,
+ }, res)
+ require.Len(t, fake.gcReqs, 1)
+ require.True(t, fake.gcReqs[0].GetDryRun())
+ require.Equal(t, "default", fake.gcReqs[0].GetRepository().GetStorageName())
+}
+
+func TestStorageGCRemoved(t *testing.T) {
+ fake := &fakeStorageCore{removedResp: &gitalypb.CollectRemovedRepositoriesResponse{
+ Repositories: 2,
+ PacksDeleted: 9,
+ BytesReclaimed: 4096,
+ }}
+ client := dialFake(t, func(srv *grpc.Server) {
+ gitalypb.RegisterStorageCoreServiceServer(srv, fake)
+ }, sdk.WithStorage("default"))
+
+ res, err := client.StorageGCRemoved(t.Context(), "", false)
+ require.NoError(t, err)
+ require.Equal(t, &sdk.StorageGCRemoved{Repositories: 2, PacksDeleted: 9, BytesReclaimed: 4096}, res)
+
+ _, err = client.StorageGCRemoved(t.Context(), "other", true)
+ require.NoError(t, err)
+
+ require.Len(t, fake.removedReqs, 2)
+ require.Equal(t, "default", fake.removedReqs[0].GetStorageName(), "the client's storage by default")
+ require.False(t, fake.removedReqs[0].GetDryRun())
+ require.Equal(t, "other", fake.removedReqs[1].GetStorageName())
+ require.True(t, fake.removedReqs[1].GetDryRun())
+}
+
+func TestStorageMaintenanceErrorsAreReturned(t *testing.T) {
+ fake := &fakeStorageCore{err: status.Error(codes.FailedPrecondition, "the storage core is off")}
+ repo := storageRepo(t, fake)
+
+ _, err := repo.StorageCheckpoint(t.Context())
+ require.Equal(t, codes.FailedPrecondition, status.Code(err))
+ _, err = repo.StorageGC(t.Context(), false)
+ require.Equal(t, codes.FailedPrecondition, status.Code(err))
+}
+
func TestStorageImport(t *testing.T) {
fake := &fakeStorageCore{importResp: &gitalypb.ImportRepositoryResponse{
Sequence: 1,