Git designed for AI agents.
WP78: CreateCheckpoint, CollectGarbage and CollectRemovedRepositories
Federico Elia03ef6c21aecc
diff --git a/cmd/graft/main.go b/cmd/graft/main.goindex e5461f8..f0f2f24 100644--- a/cmd/graft/main.go+++ b/cmd/graft/main.go@@ -265,7 +265,7 @@ commands:cat REV:PATH print a file at a revisionsearch QUERY search file contents (--regex, -i, --path, --context, -n, --rev, --json)review SUBCOMMAND create, show, list, comment, approve, request-changes, update, close (--branch, --title, --json)- storage SUBCOMMAND import, state, verify a repository in the storage core (--connectivity, --json)+ storage SUBCOMMAND import, state, verify, checkpoint, gc a repository in the storage core; gc --removed for a storage (--connectivity, --dry-run, --json)environment: GRAFT_ADDRESS, GRAFT_TOKEN, GRAFT_STORAGE, GRAFT_USER, GRAFT_AUTHORdiff --git a/cmd/graft/storage.go b/cmd/graft/storage.goindex 753ba85..842c89b 100644--- a/cmd/graft/storage.go+++ b/cmd/graft/storage.go@@ -9,20 +9,23 @@ import ("io""unicode/utf8"+ "github.com/feder-cr/git-ai/internal/graft/workspace""github.com/feder-cr/git-ai/sdk")// storageSynopsis is the command line of graft storage.-const storageSynopsis = "usage: graft storage import|state|verify [-repo R] [--connectivity] [--json]"+const storageSynopsis = "usage: graft storage import|state|verify|checkpoint|gc [-repo R] [--connectivity] [--dry-run] [--removed] [--json]"// storageUsage lists the subcommands of graft storage.-const storageUsage = "storage requires a subcommand: import, state or verify (" + storageSynopsis + ")"+const storageUsage = "storage requires a subcommand: import, state, verify, checkpoint or gc (" + storageSynopsis + ")"// storageFlags holds the flags of graft storage. Each subcommand reads only// some of them, as its help text says.type storageFlags struct {repo stringconnectivity bool+ dryRun bool+ removed bool}// storageCommand returns the storage command, the operator's view of the@@ -37,7 +40,9 @@ func storageCommand() command {}fs.StringVar(&f.repo, "repo", "", "repository path, used without a workspace together with -address and -storage")fs.BoolVar(&f.connectivity, "connectivity", false, "verify: also check the connectivity of the local repository (git fsck --connectivity-only) and that the object storage holds every object of the database references")- fs.BoolVar(&o.jsonOut, "json", false, "import, state, verify: print the result as one JSON object")+ fs.BoolVar(&f.dryRun, "dry-run", false, "gc: report what would be deleted, delete nothing")+ fs.BoolVar(&f.removed, "removed", false, "gc: collect the removed repositories of the storage (-storage) instead of one repository; takes no -repo")+ fs.BoolVar(&o.jsonOut, "json", false, "print the result as one JSON object")},run: func(ctx context.Context, out io.Writer, o *options, args []string) error {return cmdStorage(ctx, out, o, f, args)@@ -59,10 +64,124 @@ func cmdStorage(ctx context.Context, out io.Writer, o *options, f *storageFlags,return storageState(ctx, out, o, f, rest)case "verify":return storageVerify(ctx, out, o, f, rest)+ case "checkpoint":+ return storageCheckpoint(ctx, out, o, f, rest)+ case "gc":+ return storageGC(ctx, out, o, f, rest)}return usagef("unknown storage subcommand %q: %s", sub, storageUsage)}+func storageCheckpoint(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {+ if len(args) != 0 {+ return usagef("storage checkpoint takes flags only: checkpoint [-repo R] [--json]")+ }+ repo, err := openStorage(ctx, o, f)+ if err != nil {+ return err+ }+ res, err := repo.StorageCheckpoint(ctx)+ if err != nil {+ return mapError(err)+ }+ if o.jsonOut {+ return writeStorageJSON(out, res)+ }+ switch {+ case res.Sequence == 0:+ fmt.Fprintln(out, "no checkpoint: the repository has no object to consolidate")+ case res.Created:+ fmt.Fprintf(out, "checkpoint created at changeset %d: %d references, pack %d bytes\n", res.Sequence, res.References, res.PackBytes)+ default:+ fmt.Fprintf(out, "checkpoint at changeset %d exists already: %d references, pack %d bytes\n", res.Sequence, res.References, res.PackBytes)+ }+ return nil+}++func storageGC(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {+ if len(args) != 0 {+ return usagef("storage gc takes flags only: gc [-repo R] [--dry-run] [--json], or gc --removed [-storage S] [--dry-run] [--json]")+ }+ if f.removed {+ if f.repo != "" {+ return usagef("storage gc --removed collects the removed repositories of a storage and takes no -repo")+ }+ return storageGCRemoved(ctx, out, o, f)+ }++ repo, err := openStorage(ctx, o, f)+ if err != nil {+ return err+ }+ res, err := repo.StorageGC(ctx, f.dryRun)+ if err != nil {+ return mapError(err)+ }+ if o.jsonOut {+ return writeStorageJSON(out, res)+ }+ verb := "deleted"+ if res.DryRun {+ verb = "would delete"+ }+ fmt.Fprintf(out, "%s %d objects (%d bytes): %d covered by the checkpoint at changeset %d, %d superseded checkpoints, %d orphans\n",+ verb, res.PacksDeleted, res.BytesReclaimed, res.CoveredDeleted, res.Boundary, res.CheckpointsDropped, res.OrphansDeleted)+ fmt.Fprintf(out, "orphans observed now %d, still in their grace period %d\n", res.OrphansObserved, res.OrphansInGrace)+ return nil+}++func storageGCRemoved(ctx context.Context, out io.Writer, o *options, f *storageFlags) error {+ client, err := openStorageClient(ctx, o)+ if err != nil {+ return err+ }+ res, err := client.StorageGCRemoved(ctx, "", f.dryRun)+ if err != nil {+ return mapError(err)+ }+ if o.jsonOut {+ return writeStorageJSON(out, res)+ }+ verb := "deleted"+ if res.DryRun {+ verb = "would delete"+ }+ fmt.Fprintf(out, "%s %d objects (%d bytes) of %d removed repositories of storage %s\n",+ verb, res.PacksDeleted, res.BytesReclaimed, res.Repositories, displayText(client.Storage()))+ return nil+}++// openStorageClient returns the client of a storage-wide command: the server+// and the storage of the workspace, or without a workspace -address and+// -storage.+func openStorageClient(ctx context.Context, o *options) (*sdk.Client, error) {+ var t target+ if root, err := workspaceRoot(o); err == nil {+ if meta, err := workspace.ReadMeta(root); err == nil {+ t = target{Address: pick(o.address, meta.Address), Token: o.token, Storage: pick(o.storage, meta.Storage), Repo: meta.Repo, User: o.user}+ }+ }+ if t.Address == "" {+ if o.address == "" {+ return nil, usagef("storage gc --removed needs a workspace or -address")+ }+ t = target{Address: o.address, Token: o.token, Storage: o.storage, User: o.user}+ }++ // A storage-wide command names no repository, which the workspace backend+ // requires: the client is dialed with the same options.+ opts := []sdk.DialOption{sdk.WithStorage(pick(t.Storage, defaultStorage))}+ if t.Token != "" {+ opts = append(opts, sdk.WithToken(t.Token))+ }+ client, err := sdk.Dial(ctx, t.Address, opts...)+ if err != nil {+ return nil, err+ }+ registerCloser(ctx, client)+ return client, nil+}+func storageImport(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {if len(args) != 0 {return usagef("storage import takes flags only: import [-repo R] [--json]")diff --git a/cmd/graft/storage_test.go b/cmd/graft/storage_test.goindex 29fa929..4ccad23 100644--- a/cmd/graft/storage_test.go+++ b/cmd/graft/storage_test.go@@ -99,6 +99,53 @@ func TestCLIStorageAgainstServer(t *testing.T) {require.Equal(t, []sdk.StorageDivergence{{Reference: "refs/heads/rogue", LocalTarget: main.String()}}, verify.Divergences)}+// TestCLIStorageMaintenanceAgainstServer drives graft storage checkpoint and gc+// against a real server whose storage core runs in authoritative mode. It runs+// serially because the CLI tests share the package-level backend factory.+func TestCLIStorageMaintenanceAgainstServer(t *testing.T) {+ ctx := testhelper.Context(t)+ cfg := testcfg.Build(t)+ cfg.Graft.Standalone = true+ core := coretest.Open(t, coretest.Options{Mode: corestore.ModeAuthoritative})+ addr := testserver.RunGitalyServer(t, cfg, setup.RegisterAll, testserver.WithGraftStorage(core))++ repo, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))+ server := []string{"-address", addr, "-storage", repo.GetStorageName()}+ target := append(append([]string{}, server...), "-repo", repo.GetRelativePath())+ cli := func(args ...string) (int, string, string) {+ return runCLI(t, append(append([]string{"storage"}, args...), target...)...)+ }++ code, _, stderr := cli("import")+ require.Equal(t, exitOK, code, stderr)++ code, stdout, stderr := cli("checkpoint")+ require.Equal(t, exitOK, code, stderr)+ require.Regexp(t, `^checkpoint created at changeset 1: 2 references, pack [1-9][0-9]* bytes\n$`, stdout)++ code, stdout, stderr = cli("checkpoint", "--json")+ require.Equal(t, exitOK, code, stderr)+ var checkpoint sdk.StorageCheckpoint+ require.NoError(t, json.Unmarshal([]byte(stdout), &checkpoint))+ require.False(t, checkpoint.Created, "a checkpoint at the head exists")+ require.EqualValues(t, 1, checkpoint.Sequence)++ code, stdout, stderr = cli("gc", "--dry-run", "--json")+ require.Equal(t, exitOK, code, stderr)+ var gc sdk.StorageGC+ require.NoError(t, json.Unmarshal([]byte(stdout), &gc))+ require.True(t, gc.DryRun)++ code, stdout, stderr = cli("gc")+ require.Equal(t, exitOK, code, stderr)+ require.Contains(t, stdout, "deleted 0 objects (0 bytes)")++ code, stdout, stderr = runCLI(t, append([]string{"storage", "gc", "--removed"}, server...)...)+ require.Equal(t, exitOK, code, stderr)+ require.Equal(t, "deleted 0 objects (0 bytes) of 0 removed repositories of storage "+repo.GetStorageName()+"\n", stdout)+}+// TestCLIStorageCoreOff checks that a server without a storage core reports// mode off and refuses an import.func TestCLIStorageCoreOff(t *testing.T) {@@ -121,11 +168,15 @@ func TestCLIStorageUsageErrorsDoNotContactServer(t *testing.T) {dir := filepath.Join(t.TempDir(), "missing")cases := map[string][]string{- "no subcommand": {"storage", "-C", dir},- "unknown subcommand": {"storage", "repair", "-C", dir},- "import with arg": {"storage", "import", "-C", dir, "extra"},- "state with arg": {"storage", "state", "-C", dir, "extra"},- "verify with arg": {"storage", "verify", "-C", dir, "extra"},+ "no subcommand": {"storage", "-C", dir},+ "unknown subcommand": {"storage", "repair", "-C", dir},+ "import with arg": {"storage", "import", "-C", dir, "extra"},+ "state with arg": {"storage", "state", "-C", dir, "extra"},+ "verify with arg": {"storage", "verify", "-C", dir, "extra"},+ "checkpoint with arg": {"storage", "checkpoint", "-C", dir, "extra"},+ "gc with arg": {"storage", "gc", "-C", dir, "extra"},+ "gc removed and repo": {"storage", "gc", "--removed", "-C", dir, "-repo", "a.git"},+ "gc removed without server": {"storage", "gc", "--removed", "-C", dir},}for name, args := range cases {code, _, stderr := runCLI(t, args...)diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex 683cca1..ca96608 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -1536,6 +1536,29 @@ graft storage gc --removed [-storage S] [--dry-run] [--json]`gc -repo R` touches only that repository (answer 1); `--removed` is the storage-widesweep of removed repositories and takes no `-repo`.+**As implemented (WP78).**+* `CreateCheckpointResponse` has no `review_commits`: since the REVIEW ROOTS+ RULE the review commits are references, counted in `references`.+* `CollectGarbage` takes the lease as `maintenance.RequestLeaseHolder`:+ `node_id/PID/request-N`, unique per request. So a request excludes the+ background runs (`node_id/PID`), and the other requests, of its own process+ too.+* The SDK methods are in `sdk/storage.go`, already among the operator-only+ sources of the agent allowlist test, not in a new `sdk/storage_gc.go`.+* `graft storage gc --removed` dials a client of the server without a+ repository: the workspace backend requires one.+* The handlers map the errors of `checkpoint` and `gc`, which carry their+ codes. Off is `FailedPrecondition`; a checkpoint in shadow mode is+ `FailedPrecondition`; an untracked repository is `NotFound`; an unknown+ storage is `InvalidArgument`.+* Tests:+ * the handlers on SQLite and Postgres: created and idempotent, refusals, an+ orphan collected after the grace, dry run, lease held by another holder+ and released after the request, removed repositories;+ * the SDK against a fake;+ * the CLI against a real authoritative server (`checkpoint`, `gc`,+ `gc --dry-run --json`, `gc --removed`) and its usage errors.+## Rejected alternatives* **Local-only hidden keep-alive references for review roots** (option b of thediff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.mdindex 197fb36..7f2950d 100644--- a/doc/graft/STORAGE.md+++ b/doc/graft/STORAGE.md@@ -320,6 +320,38 @@ registration gets a new repository ID and a full pack, and takes the changelogand the reviews from the files again, so do this before switching toauthoritative mode.+### Checkpoints and garbage collection++```sh+graft storage checkpoint -address unix:/var/opt/graft/gitaly.sock -repo @hashed/ab/cd/x.git [--json]+graft storage gc -address unix:/var/opt/graft/gitaly.sock -repo @hashed/ab/cd/x.git [--dry-run] [--json]+graft storage gc --removed -address unix:/var/opt/graft/gitaly.sock [-storage default] [--dry-run] [--json]+```++- `checkpoint` (authoritative mode) consolidates the repository at its current+ database head: one pack of every object reachable from its references, and+ the reference snapshot. A fresh node then hydrates from the checkpoint+ instead of replaying every changeset, and garbage collection can delete the+ packs the checkpoint covers. A checkpoint that exists at the head is+ reported, not built again.+- `gc` deletes the packs of one repository that no reader needs any more:+ - the changeset packs covered by a checkpoint at least `gc_grace` old, and+ the checkpoints that one supersedes;+ - the objects no database row references, once garbage collection observed+ them unreferenced at least `gc_grace` earlier (the packs of writes that+ died before their database commit). The first run only observes them.++ It runs in shadow mode too (orphans only: there are no checkpoints). It holds+ the repository's maintenance lease while it runs, and fails with+ `FailedPrecondition` when another maintenance of the repository (the+ background one, another node) holds it.+- `gc --removed` deletes every object of the repositories of the storage that+ were removed at least `gc_grace` ago.+- `--dry-run` reports what would be deleted and changes nothing.++These commands are for operators: they need the shared token, and an agent+token is refused.+### Switching from shadow to authoritative1. Run every node in shadow mode and import every repository@@ -622,9 +654,10 @@ show:packs, and there is no virtual filesystem.- **Checkpoints and pack garbage collection run in the background only when`maintenance = "auto"`** (authoritative mode; off by default). Without it,- packs are never deleted and hydration replays every changeset after the- newest checkpoint, from 1 when there is none. The on-demand `graft storage`- commands for both are not available yet.+ they run when an operator runs `graft storage checkpoint` and `graft storage+ gc` (see [Checkpoints and garbage collection](#checkpoints-and-garbage-collection));+ otherwise packs are never deleted and hydration replays every changeset after+ the newest checkpoint, from 1 when there is none.- **Praefect is not supported in authoritative mode.** Configuration validationrequires `[graft] standalone = true` for authoritative mode. Shadow modeworks with or without Praefect.diff --git a/internal/gitaly/service/storagecore/maintenance.go b/internal/gitaly/service/storagecore/maintenance.gonew file mode 100644index 0000000..564edcf--- /dev/null+++ b/internal/gitaly/service/storagecore/maintenance.go@@ -0,0 +1,139 @@+package storagecore++import (+ "context"+ "errors"+ "time"++ "github.com/feder-cr/git-ai/internal/gitaly/storage"+ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/checkpoint"+ "github.com/feder-cr/git-ai/internal/graft/corestore/gc"+ "github.com/feder-cr/git-ai/internal/graft/corestore/maintenance"+ "github.com/feder-cr/git-ai/internal/structerr"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// releaseTimeout bounds the release of a maintenance lease after a request.+const releaseTimeout = 10 * time.Second++// CreateCheckpoint implements gitalypb.StorageCoreServiceServer+// (doc/graft/PLAN7.md, "Checkpoint creation").+func (s *server) CreateCheckpoint(ctx context.Context, req *gitalypb.CreateCheckpointRequest) (*gitalypb.CreateCheckpointResponse, error) {+ repo := req.GetRepository()+ if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {+ return nil, structerr.NewInvalidArgument("%w", err)+ }+ core, err := s.enabledCore()+ if err != nil {+ return nil, err+ }++ cp := checkpoint.New(core, s.deps.GetLogger(), s.deps.GetLocator(), s.deps.GetRepositoryFactory(), s.deps.GetGraftStorageSyncer())+ created, isNew, err := cp.Create(ctx, repo)+ if err != nil {+ return nil, err+ }++ resp := &gitalypb.CreateCheckpointResponse{+ CheckpointSequence: created.Seq,+ Created: isNew,+ PackBytes: created.PackBytes,+ }+ if created.Seq == 0 {+ // No object to consolidate: there is no checkpoint.+ return resp, nil+ }+ refs, err := core.Meta.CheckpointRefs(ctx, created.RepoID, created.Seq)+ if err != nil {+ return nil, structerr.NewUnavailable("storage core: checkpoint references: %w", err)+ }+ resp.References = uint32(len(refs))+ return resp, nil+}++// CollectGarbage implements gitalypb.StorageCoreServiceServer+// (doc/graft/PLAN7.md, "Pack GC"). It holds the repository's maintenance lease+// while it collects, so that it does not run alongside the background+// maintenance or another request; correctness does not depend on the lease.+func (s *server) CollectGarbage(ctx context.Context, req *gitalypb.CollectGarbageRequest) (*gitalypb.CollectGarbageResponse, error) {+ repo := req.GetRepository()+ if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {+ return nil, structerr.NewInvalidArgument("%w", err)+ }+ core, err := s.enabledCore()+ if err != nil {+ return nil, err+ }+ ttl, err := s.deps.GetCfg().Graft.Storage.LeaseTTLOrDefault()+ if err != nil {+ return nil, structerr.NewInternal("%w", err)+ }++ record, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repo))+ if errors.Is(err, corestore.ErrRepositoryNotFound) {+ return nil, structerr.NewNotFound("repository is not tracked by the storage core")+ }+ if err != nil {+ return nil, structerr.NewUnavailable("storage core: %w", err)+ }++ holder := maintenance.RequestLeaseHolder(core)+ acquired, err := core.Meta.AcquireMaintenanceLease(ctx, record.ID, holder, ttl)+ if errors.Is(err, corestore.ErrRepositoryNotFound) {+ return nil, structerr.NewNotFound("repository is not tracked by the storage core")+ }+ if err != nil {+ return nil, structerr.NewUnavailable("storage core: maintenance lease: %w", err)+ }+ if !acquired {+ return nil, structerr.NewFailedPrecondition("another maintenance operation of this repository is in progress")+ }+ defer func() {+ releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)+ defer cancel()+ if err := core.Meta.ReleaseMaintenanceLease(releaseCtx, record.ID, holder); err != nil {+ s.deps.GetLogger().WithError(err).WarnContext(ctx, "storage core: release the maintenance lease; it expires by itself")+ }+ }()++ report, err := gc.New(core, s.deps.GetLogger()).CollectRepository(ctx, record.ID, req.GetDryRun())+ if err != nil {+ return nil, err+ }+ return &gitalypb.CollectGarbageResponse{+ Boundary: report.Boundary,+ CoveredDeleted: uint32(report.CoveredDeleted),+ CheckpointsDropped: uint32(report.CheckpointsDropped),+ OrphansDeleted: uint32(report.OrphansDeleted),+ OrphansObserved: uint32(report.OrphansObserved),+ OrphansInGrace: uint32(report.OrphansInGrace),+ PacksDeleted: uint32(report.PacksDeleted),+ BytesReclaimed: report.BytesReclaimed,+ DryRun: report.DryRun,+ }, nil+}++// CollectRemovedRepositories implements gitalypb.StorageCoreServiceServer+// (doc/graft/PLAN7.md, "Removed repositories").+func (s *server) CollectRemovedRepositories(ctx context.Context, req *gitalypb.CollectRemovedRepositoriesRequest) (*gitalypb.CollectRemovedRepositoriesResponse, error) {+ storageName := req.GetStorageName()+ if _, err := s.deps.GetLocator().GetStorageByName(ctx, storageName); err != nil {+ return nil, structerr.NewInvalidArgument("%w", err)+ }+ core, err := s.enabledCore()+ if err != nil {+ return nil, err+ }++ report, err := gc.New(core, s.deps.GetLogger()).CollectRemoved(ctx, storageName, req.GetDryRun())+ if err != nil {+ return nil, err+ }+ return &gitalypb.CollectRemovedRepositoriesResponse{+ Repositories: uint32(report.RemovedRepos),+ PacksDeleted: uint32(report.PacksDeleted),+ BytesReclaimed: report.BytesReclaimed,+ DryRun: report.DryRun,+ }, nil+}diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.gonew file mode 100644index 0000000..91ca797--- /dev/null+++ b/internal/gitaly/service/storagecore/maintenance_test.go@@ -0,0 +1,190 @@+package storagecore++import (+ "bytes"+ "crypto/sha256"+ "fmt"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// shortGC makes garbage collection wait a short grace, so that a test sees an+// orphan deleted. The writers' deadline (grace - margin) stays long enough for+// the import's upload under load.+func shortGC(core *corestore.Core) {+ core.GCGrace = time.Second+ core.GCMargin = 100 * time.Millisecond+}++// putOrphan stores an object under the repository's pack prefix that no row+// references, as a write that died before its database commit leaves.+func (h *harness) putOrphan(repoID int64, content string) string {+ h.t.Helper()+ key := corestore.NewPackKey(repoID, sha256.Sum256([]byte(content)))+ require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, bytes.NewReader([]byte(content))))+ return key+}++func (h *harness) exists(key string) bool {+ h.t.Helper()+ ok, err := h.core.Blobs.Exists(h.ctx, key)+ require.NoError(h.t, err)+ return ok+}++func TestCreateCheckpoint(t *testing.T) {+ t.Parallel()+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ h := setup(t, corestore.ModeAuthoritative, dialect)+ h.seed()+ repoID := h.importRepo()++ resp, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: h.repo})+ require.NoError(t, err)+ record, err := h.core.Meta.GetRepository(h.ctx, corestore.KeyOf(h.repo))+ require.NoError(t, err)+ require.Equal(t, repoID, record.ID)+ require.True(t, resp.GetCreated())+ require.Equal(t, record.HeadSeq, resp.GetCheckpointSequence())+ require.Positive(t, resp.GetPackBytes())+ require.Equal(t, uint32(len(h.localRefs())), resp.GetReferences(), "HEAD, the branch and the tag")++ again, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: h.repo})+ require.NoError(t, err)+ require.False(t, again.GetCreated(), "a checkpoint at the head exists")+ require.Equal(t, resp.GetCheckpointSequence(), again.GetCheckpointSequence())+ })+}++func TestCreateCheckpointRefusals(t *testing.T) {+ t.Parallel()++ off := setup(t, corestore.ModeOff, sqlstore.SQLite)+ _, err := off.client.CreateCheckpoint(off.ctx, &gitalypb.CreateCheckpointRequest{Repository: off.repo})+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)++ shadow := setup(t, corestore.ModeShadow, sqlstore.SQLite)+ shadow.seed()+ shadow.importRepo()+ _, err = shadow.client.CreateCheckpoint(shadow.ctx, &gitalypb.CreateCheckpointRequest{Repository: shadow.repo})+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "checkpoints are authoritative mode only: %v", err)++ untracked := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)+ _, err = untracked.client.CreateCheckpoint(untracked.ctx, &gitalypb.CreateCheckpointRequest{Repository: untracked.repo})+ require.Equal(t, codes.NotFound, status.Code(err), "%v", err)++ _, err = untracked.client.CreateCheckpoint(untracked.ctx, &gitalypb.CreateCheckpointRequest{})+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)+}++func TestCollectGarbage(t *testing.T) {+ t.Parallel()+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {+ h := setup(t, mode, dialect, shortGC)+ h.seed()+ repoID := h.importRepo()+ orphan := h.putOrphan(repoID, "the pack of a write that died")++ dry, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo, DryRun: true})+ require.NoError(t, err)+ require.True(t, dry.GetDryRun())+ require.Equal(t, uint32(0), dry.GetPacksDeleted())+ require.True(t, h.exists(orphan))++ first, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})+ require.NoError(t, err)+ require.Equal(t, uint32(1), first.GetOrphansObserved())+ require.True(t, h.exists(orphan), "observed, not yet deleted")++ require.Eventually(t, func() bool {+ resp, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})+ require.NoError(t, err)+ return resp.GetOrphansDeleted() == 1 && resp.GetBytesReclaimed() > 0+ }, 30*time.Second, 50*time.Millisecond)+ require.False(t, h.exists(orphan))+ require.Empty(t, h.verify(true).GetDivergences(), "the repository is intact")+ })+}++func TestCollectGarbageRefusals(t *testing.T) {+ t.Parallel()++ off := setup(t, corestore.ModeOff, sqlstore.SQLite)+ _, err := off.client.CollectGarbage(off.ctx, &gitalypb.CollectGarbageRequest{Repository: off.repo})+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)++ h := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)+ _, err = h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})+ require.Equal(t, codes.NotFound, status.Code(err), "untracked: %v", err)++ h.seed()+ repoID := h.importRepo()+ acquired, err := h.core.Meta.AcquireMaintenanceLease(h.ctx, repoID, "node-b/1", time.Hour)+ require.NoError(t, err)+ require.True(t, acquired)+ _, err = h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)+ require.Contains(t, err.Error(), "another maintenance operation of this repository is in progress")++ require.NoError(t, h.core.Meta.ReleaseMaintenanceLease(h.ctx, repoID, "node-b/1"))+ _, err = h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: h.repo})+ require.NoError(t, err)+ // The request released its lease.+ acquired, err = h.core.Meta.AcquireMaintenanceLease(h.ctx, repoID, "node-b/1", time.Hour)+ require.NoError(t, err)+ require.True(t, acquired)+}++func TestCollectRemovedRepositories(t *testing.T) {+ t.Parallel()+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {+ h := setup(t, mode, dialect, shortGC)+ h.seed()+ repoID := h.importRepo()+ packs := h.packKeys(repoID)+ require.NotEmpty(t, packs)++ removed, err := h.core.Meta.DeleteRepository(h.ctx, corestore.KeyOf(h.repo))+ require.NoError(t, err)+ require.True(t, removed)++ storageName := h.cfg.Storages[0].Name+ require.Eventually(t, func() bool {+ resp, err := h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: storageName})+ require.NoError(t, err)+ return resp.GetRepositories() == 1 && resp.GetPacksDeleted() >= uint32(len(packs))+ }, 30*time.Second, 50*time.Millisecond)+ require.Empty(t, h.packKeys(repoID))+ })+}++func TestCollectRemovedRepositoriesRefusals(t *testing.T) {+ t.Parallel()++ off := setup(t, corestore.ModeOff, sqlstore.SQLite)+ _, err := off.client.CollectRemovedRepositories(off.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: off.cfg.Storages[0].Name})+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)++ h := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)+ _, err = h.client.CollectRemovedRepositories(h.ctx, &gitalypb.CollectRemovedRepositoriesRequest{StorageName: "no-such-storage"})+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)+}++// packKeys returns the objects of the repository in the bucket.+func (h *harness) packKeys(repoID int64) map[string]bool {+ h.t.Helper()+ keys := map[string]bool{}+ require.NoError(h.t, h.core.Blobs.List(h.ctx, fmt.Sprintf("repos/%d/", repoID), func(info corestore.BlobInfo) error {+ keys[info.Key] = true+ return nil+ }))+ return keys+}diff --git a/internal/graft/corestore/maintenance/maintenance.go b/internal/graft/corestore/maintenance/maintenance.goindex ef2c7a7..db09ec5 100644--- a/internal/graft/corestore/maintenance/maintenance.go+++ b/internal/graft/corestore/maintenance/maintenance.go@@ -18,6 +18,7 @@ import ("fmt""os""sync"+ "sync/atomic""time""github.com/prometheus/client_golang/prometheus"@@ -83,6 +84,23 @@ type Runner struct {var _ prometheus.Collector = (*Runner)(nil)+// LeaseHolder names the background maintenance of this process in the+// maintenance leases of core: the node ID and the process ID, so that two+// processes of one node (a graceful upgrade) never both hold a lease.+func LeaseHolder(core *corestore.Core) string {+ return fmt.Sprintf("%s/%d", core.NodeID, os.Getpid())+}++// requests numbers the on-demand maintenance operations of the process.+var requests atomic.Uint64++// RequestLeaseHolder names one on-demand maintenance operation of this process+// (a StorageCoreService RPC): a holder of its own, so that it excludes the+// background runs and the other operations.+func RequestLeaseHolder(core *corestore.Core) string {+ return fmt.Sprintf("%s/request-%d", LeaseHolder(core), requests.Add(1))+}+// New returns the runner of core for the node's storages.func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Collector, fresh corestore.Freshener, logger log.Logger, cfg Config, storages []string) *Runner {r := &Runner{@@ -93,7 +111,7 @@ func New(core *corestore.Core, cp *checkpoint.Checkpointer, collector *gc.Colleclogger: logger,cfg: cfg,storages: storages,- holder: fmt.Sprintf("%s/%d", core.NodeID, os.Getpid()),+ holder: LeaseHolder(core),runs: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "graft_storage_maintenance_runs_total",Help: "Runs of the background maintenance of the storage core, by result (ok, error: at least one step failed).",diff --git a/proto/go/gitalypb/storage_core.pb.go b/proto/go/gitalypb/storage_core.pb.goindex 42f7fb3..3623dd1 100644--- a/proto/go/gitalypb/storage_core.pb.go+++ b/proto/go/gitalypb/storage_core.pb.go@@ -416,6 +416,434 @@ func (x *VerifyRepositoryResponse) GetConnectivityError() string {return ""}+// CreateCheckpointRequest is the request of CreateCheckpoint.+type CreateCheckpointRequest struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ // repository is the repository to checkpoint.+ Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CreateCheckpointRequest) Reset() {+ *x = CreateCheckpointRequest{}+ mi := &file_storage_core_proto_msgTypes[6]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CreateCheckpointRequest) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CreateCheckpointRequest) ProtoMessage() {}++func (x *CreateCheckpointRequest) ProtoReflect() protoreflect.Message {+ mi := &file_storage_core_proto_msgTypes[6]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CreateCheckpointRequest.ProtoReflect.Descriptor instead.+func (*CreateCheckpointRequest) Descriptor() ([]byte, []int) {+ return file_storage_core_proto_rawDescGZIP(), []int{6}+}++func (x *CreateCheckpointRequest) GetRepository() *Repository {+ if x != nil {+ return x.Repository+ }+ return nil+}++// CreateCheckpointResponse is the response of CreateCheckpoint.+type CreateCheckpointResponse struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ // checkpoint_sequence is the changeset sequence the checkpoint is at; 0 when+ // the repository has no object to consolidate (every reference is symbolic).+ CheckpointSequence uint64 `protobuf:"varint,1,opt,name=checkpoint_sequence,json=checkpointSequence,proto3" json:"checkpoint_sequence,omitempty"`+ // created is false when a checkpoint at the head existed already.+ Created bool `protobuf:"varint,2,opt,name=created,proto3" json:"created,omitempty"`+ // pack_bytes is the size of the checkpoint's pack.+ PackBytes int64 `protobuf:"varint,3,opt,name=pack_bytes,json=packBytes,proto3" json:"pack_bytes,omitempty"`+ // references is the number of references in the checkpoint's snapshot.+ References uint32 `protobuf:"varint,4,opt,name=references,proto3" json:"references,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CreateCheckpointResponse) Reset() {+ *x = CreateCheckpointResponse{}+ mi := &file_storage_core_proto_msgTypes[7]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CreateCheckpointResponse) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CreateCheckpointResponse) ProtoMessage() {}++func (x *CreateCheckpointResponse) ProtoReflect() protoreflect.Message {+ mi := &file_storage_core_proto_msgTypes[7]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CreateCheckpointResponse.ProtoReflect.Descriptor instead.+func (*CreateCheckpointResponse) Descriptor() ([]byte, []int) {+ return file_storage_core_proto_rawDescGZIP(), []int{7}+}++func (x *CreateCheckpointResponse) GetCheckpointSequence() uint64 {+ if x != nil {+ return x.CheckpointSequence+ }+ return 0+}++func (x *CreateCheckpointResponse) GetCreated() bool {+ if x != nil {+ return x.Created+ }+ return false+}++func (x *CreateCheckpointResponse) GetPackBytes() int64 {+ if x != nil {+ return x.PackBytes+ }+ return 0+}++func (x *CreateCheckpointResponse) GetReferences() uint32 {+ if x != nil {+ return x.References+ }+ return 0+}++// CollectGarbageRequest is the request of CollectGarbage.+type CollectGarbageRequest struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ // repository is the repository whose packs are collected.+ Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"`+ // dry_run reports what would be deleted and changes nothing.+ DryRun bool `protobuf:"varint,2,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CollectGarbageRequest) Reset() {+ *x = CollectGarbageRequest{}+ mi := &file_storage_core_proto_msgTypes[8]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CollectGarbageRequest) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CollectGarbageRequest) ProtoMessage() {}++func (x *CollectGarbageRequest) ProtoReflect() protoreflect.Message {+ mi := &file_storage_core_proto_msgTypes[8]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CollectGarbageRequest.ProtoReflect.Descriptor instead.+func (*CollectGarbageRequest) Descriptor() ([]byte, []int) {+ return file_storage_core_proto_rawDescGZIP(), []int{8}+}++func (x *CollectGarbageRequest) GetRepository() *Repository {+ if x != nil {+ return x.Repository+ }+ return nil+}++func (x *CollectGarbageRequest) GetDryRun() bool {+ if x != nil {+ return x.DryRun+ }+ return false+}++// CollectGarbageResponse is the response of CollectGarbage.+type CollectGarbageResponse struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ // boundary is the sequence of the newest checkpoint at least gc_grace old;+ // 0 when there is none.+ Boundary uint64 `protobuf:"varint,1,opt,name=boundary,proto3" json:"boundary,omitempty"`+ // covered_deleted is the number of changeset packs covered by the boundary.+ CoveredDeleted uint32 `protobuf:"varint,2,opt,name=covered_deleted,json=coveredDeleted,proto3" json:"covered_deleted,omitempty"`+ // checkpoints_dropped is the number of checkpoints superseded by the boundary.+ CheckpointsDropped uint32 `protobuf:"varint,3,opt,name=checkpoints_dropped,json=checkpointsDropped,proto3" json:"checkpoints_dropped,omitempty"`+ // orphans_deleted is the number of unreferenced objects observed at least+ // gc_grace ago.+ OrphansDeleted uint32 `protobuf:"varint,4,opt,name=orphans_deleted,json=orphansDeleted,proto3" json:"orphans_deleted,omitempty"`+ // orphans_observed is the number of unreferenced objects seen for the first+ // time.+ OrphansObserved uint32 `protobuf:"varint,5,opt,name=orphans_observed,json=orphansObserved,proto3" json:"orphans_observed,omitempty"`+ // orphans_in_grace is the number of unreferenced objects observed less than+ // gc_grace ago.+ OrphansInGrace uint32 `protobuf:"varint,6,opt,name=orphans_in_grace,json=orphansInGrace,proto3" json:"orphans_in_grace,omitempty"`+ // packs_deleted is the number of objects deleted (to delete in a dry run).+ PacksDeleted uint32 `protobuf:"varint,7,opt,name=packs_deleted,json=packsDeleted,proto3" json:"packs_deleted,omitempty"`+ // bytes_reclaimed is the size of those objects.+ BytesReclaimed int64 `protobuf:"varint,8,opt,name=bytes_reclaimed,json=bytesReclaimed,proto3" json:"bytes_reclaimed,omitempty"`+ // dry_run tells that nothing was deleted or recorded.+ DryRun bool `protobuf:"varint,9,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CollectGarbageResponse) Reset() {+ *x = CollectGarbageResponse{}+ mi := &file_storage_core_proto_msgTypes[9]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CollectGarbageResponse) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CollectGarbageResponse) ProtoMessage() {}++func (x *CollectGarbageResponse) ProtoReflect() protoreflect.Message {+ mi := &file_storage_core_proto_msgTypes[9]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CollectGarbageResponse.ProtoReflect.Descriptor instead.+func (*CollectGarbageResponse) Descriptor() ([]byte, []int) {+ return file_storage_core_proto_rawDescGZIP(), []int{9}+}++func (x *CollectGarbageResponse) GetBoundary() uint64 {+ if x != nil {+ return x.Boundary+ }+ return 0+}++func (x *CollectGarbageResponse) GetCoveredDeleted() uint32 {+ if x != nil {+ return x.CoveredDeleted+ }+ return 0+}++func (x *CollectGarbageResponse) GetCheckpointsDropped() uint32 {+ if x != nil {+ return x.CheckpointsDropped+ }+ return 0+}++func (x *CollectGarbageResponse) GetOrphansDeleted() uint32 {+ if x != nil {+ return x.OrphansDeleted+ }+ return 0+}++func (x *CollectGarbageResponse) GetOrphansObserved() uint32 {+ if x != nil {+ return x.OrphansObserved+ }+ return 0+}++func (x *CollectGarbageResponse) GetOrphansInGrace() uint32 {+ if x != nil {+ return x.OrphansInGrace+ }+ return 0+}++func (x *CollectGarbageResponse) GetPacksDeleted() uint32 {+ if x != nil {+ return x.PacksDeleted+ }+ return 0+}++func (x *CollectGarbageResponse) GetBytesReclaimed() int64 {+ if x != nil {+ return x.BytesReclaimed+ }+ return 0+}++func (x *CollectGarbageResponse) GetDryRun() bool {+ if x != nil {+ return x.DryRun+ }+ return false+}++// CollectRemovedRepositoriesRequest is the request of+// CollectRemovedRepositories.+type CollectRemovedRepositoriesRequest struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ // storage_name is the storage whose removed repositories are collected.+ StorageName string `protobuf:"bytes,1,opt,name=storage_name,json=storageName,proto3" json:"storage_name,omitempty"`+ // dry_run reports what would be deleted and changes nothing.+ DryRun bool `protobuf:"varint,2,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CollectRemovedRepositoriesRequest) Reset() {+ *x = CollectRemovedRepositoriesRequest{}+ mi := &file_storage_core_proto_msgTypes[10]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CollectRemovedRepositoriesRequest) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CollectRemovedRepositoriesRequest) ProtoMessage() {}++func (x *CollectRemovedRepositoriesRequest) ProtoReflect() protoreflect.Message {+ mi := &file_storage_core_proto_msgTypes[10]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CollectRemovedRepositoriesRequest.ProtoReflect.Descriptor instead.+func (*CollectRemovedRepositoriesRequest) Descriptor() ([]byte, []int) {+ return file_storage_core_proto_rawDescGZIP(), []int{10}+}++func (x *CollectRemovedRepositoriesRequest) GetStorageName() string {+ if x != nil {+ return x.StorageName+ }+ return ""+}++func (x *CollectRemovedRepositoriesRequest) GetDryRun() bool {+ if x != nil {+ return x.DryRun+ }+ return false+}++// CollectRemovedRepositoriesResponse is the response of+// CollectRemovedRepositories.+type CollectRemovedRepositoriesResponse struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ // repositories is the number of removed repositories whose objects were+ // deleted (to delete in a dry run).+ Repositories uint32 `protobuf:"varint,1,opt,name=repositories,proto3" json:"repositories,omitempty"`+ // packs_deleted is the number of objects deleted.+ PacksDeleted uint32 `protobuf:"varint,2,opt,name=packs_deleted,json=packsDeleted,proto3" json:"packs_deleted,omitempty"`+ // bytes_reclaimed is the size of those objects.+ BytesReclaimed int64 `protobuf:"varint,3,opt,name=bytes_reclaimed,json=bytesReclaimed,proto3" json:"bytes_reclaimed,omitempty"`+ // dry_run tells that nothing was deleted or recorded.+ DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CollectRemovedRepositoriesResponse) Reset() {+ *x = CollectRemovedRepositoriesResponse{}+ mi := &file_storage_core_proto_msgTypes[11]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CollectRemovedRepositoriesResponse) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CollectRemovedRepositoriesResponse) ProtoMessage() {}++func (x *CollectRemovedRepositoriesResponse) ProtoReflect() protoreflect.Message {+ mi := &file_storage_core_proto_msgTypes[11]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CollectRemovedRepositoriesResponse.ProtoReflect.Descriptor instead.+func (*CollectRemovedRepositoriesResponse) Descriptor() ([]byte, []int) {+ return file_storage_core_proto_rawDescGZIP(), []int{11}+}++func (x *CollectRemovedRepositoriesResponse) GetRepositories() uint32 {+ if x != nil {+ return x.Repositories+ }+ return 0+}++func (x *CollectRemovedRepositoriesResponse) GetPacksDeleted() uint32 {+ if x != nil {+ return x.PacksDeleted+ }+ return 0+}++func (x *CollectRemovedRepositoriesResponse) GetBytesReclaimed() int64 {+ if x != nil {+ return x.BytesReclaimed+ }+ return 0+}++func (x *CollectRemovedRepositoriesResponse) GetDryRun() bool {+ if x != nil {+ return x.DryRun+ }+ return false+}+// ReferenceDivergence is a reference whose local value differs from the// database.type VerifyRepositoryResponse_ReferenceDivergence struct {@@ -433,7 +861,7 @@ type VerifyRepositoryResponse_ReferenceDivergence struct {func (x *VerifyRepositoryResponse_ReferenceDivergence) Reset() {*x = VerifyRepositoryResponse_ReferenceDivergence{}- mi := &file_storage_core_proto_msgTypes[6]+ mi := &file_storage_core_proto_msgTypes[12]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -445,7 +873,7 @@ func (x *VerifyRepositoryResponse_ReferenceDivergence) String() string {func (*VerifyRepositoryResponse_ReferenceDivergence) ProtoMessage() {}func (x *VerifyRepositoryResponse_ReferenceDivergence) ProtoReflect() protoreflect.Message {- mi := &file_storage_core_proto_msgTypes[6]+ mi := &file_storage_core_proto_msgTypes[12]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -525,11 +953,49 @@ const file_storage_core_proto_rawDesc = "" +"\x13ReferenceDivergence\x12\x1c\n" +"\treference\x18\x01 \x01(\fR\treference\x12!\n" +"\flocal_target\x18\x02 \x01(\tR\vlocalTarget\x12'\n" +- "\x0fdatabase_target\x18\x03 \x01(\tR\x0edatabaseTarget2\xae\x02\n" ++ "\x0fdatabase_target\x18\x03 \x01(\tR\x0edatabaseTarget\"S\n" ++ "\x17CreateCheckpointRequest\x128\n" ++ "\n" ++ "repository\x18\x01 \x01(\v2\x12.gitaly.RepositoryB\x04\x98\xc6,\x01R\n" ++ "repository\"\xa4\x01\n" ++ "\x18CreateCheckpointResponse\x12/\n" ++ "\x13checkpoint_sequence\x18\x01 \x01(\x04R\x12checkpointSequence\x12\x18\n" ++ "\acreated\x18\x02 \x01(\bR\acreated\x12\x1d\n" ++ "\n" ++ "pack_bytes\x18\x03 \x01(\x03R\tpackBytes\x12\x1e\n" ++ "\n" ++ "references\x18\x04 \x01(\rR\n" ++ "references\"j\n" ++ "\x15CollectGarbageRequest\x128\n" ++ "\n" ++ "repository\x18\x01 \x01(\v2\x12.gitaly.RepositoryB\x04\x98\xc6,\x01R\n" ++ "repository\x12\x17\n" ++ "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xf3\x02\n" ++ "\x16CollectGarbageResponse\x12\x1a\n" ++ "\bboundary\x18\x01 \x01(\x04R\bboundary\x12'\n" ++ "\x0fcovered_deleted\x18\x02 \x01(\rR\x0ecoveredDeleted\x12/\n" ++ "\x13checkpoints_dropped\x18\x03 \x01(\rR\x12checkpointsDropped\x12'\n" ++ "\x0forphans_deleted\x18\x04 \x01(\rR\x0eorphansDeleted\x12)\n" ++ "\x10orphans_observed\x18\x05 \x01(\rR\x0forphansObserved\x12(\n" ++ "\x10orphans_in_grace\x18\x06 \x01(\rR\x0eorphansInGrace\x12#\n" ++ "\rpacks_deleted\x18\a \x01(\rR\fpacksDeleted\x12'\n" ++ "\x0fbytes_reclaimed\x18\b \x01(\x03R\x0ebytesReclaimed\x12\x17\n" ++ "\adry_run\x18\t \x01(\bR\x06dryRun\"e\n" ++ "!CollectRemovedRepositoriesRequest\x12'\n" ++ "\fstorage_name\x18\x01 \x01(\tB\x04\x88\xc6,\x01R\vstorageName\x12\x17\n" ++ "\adry_run\x18\x02 \x01(\bR\x06dryRun\"\xaf\x01\n" ++ "\"CollectRemovedRepositoriesResponse\x12\"\n" ++ "\frepositories\x18\x01 \x01(\rR\frepositories\x12#\n" ++ "\rpacks_deleted\x18\x02 \x01(\rR\fpacksDeleted\x12'\n" ++ "\x0fbytes_reclaimed\x18\x03 \x01(\x03R\x0ebytesReclaimed\x12\x17\n" ++ "\adry_run\x18\x04 \x01(\bR\x06dryRun2\xe5\x04\n" +"\x12StorageCoreService\x12]\n" +"\x10ImportRepository\x12\x1f.gitaly.ImportRepositoryRequest\x1a .gitaly.ImportRepositoryResponse\"\x06\xfa\x97(\x02\b\x01\x12Z\n" +"\x0fGetStorageState\x12\x1e.gitaly.GetStorageStateRequest\x1a\x1f.gitaly.GetStorageStateResponse\"\x06\xfa\x97(\x02\b\x02\x12]\n" +- "\x10VerifyRepository\x12\x1f.gitaly.VerifyRepositoryRequest\x1a .gitaly.VerifyRepositoryResponse\"\x06\xfa\x97(\x02\b\x02B.Z,github.com/feder-cr/git-ai/proto/go/gitalypbb\x06proto3"+ "\x10VerifyRepository\x12\x1f.gitaly.VerifyRepositoryRequest\x1a .gitaly.VerifyRepositoryResponse\"\x06\xfa\x97(\x02\b\x02\x12]\n" ++ "\x10CreateCheckpoint\x12\x1f.gitaly.CreateCheckpointRequest\x1a .gitaly.CreateCheckpointResponse\"\x06\xfa\x97(\x02\b\x01\x12W\n" ++ "\x0eCollectGarbage\x12\x1d.gitaly.CollectGarbageRequest\x1a\x1e.gitaly.CollectGarbageResponse\"\x06\xfa\x97(\x02\b\x01\x12}\n" ++ "\x1aCollectRemovedRepositories\x12).gitaly.CollectRemovedRepositoriesRequest\x1a*.gitaly.CollectRemovedRepositoriesResponse\"\b\xfa\x97(\x04\b\x01\x10\x02B.Z,github.com/feder-cr/git-ai/proto/go/gitalypbb\x06proto3"var (file_storage_core_proto_rawDescOnce sync.Once@@ -543,7 +1009,7 @@ func file_storage_core_proto_rawDescGZIP() []byte {return file_storage_core_proto_rawDescData}-var file_storage_core_proto_msgTypes = make([]protoimpl.MessageInfo, 7)+var file_storage_core_proto_msgTypes = make([]protoimpl.MessageInfo, 13)var file_storage_core_proto_goTypes = []any{(*ImportRepositoryRequest)(nil), // 0: gitaly.ImportRepositoryRequest(*ImportRepositoryResponse)(nil), // 1: gitaly.ImportRepositoryResponse@@ -551,25 +1017,39 @@ var file_storage_core_proto_goTypes = []any{(*GetStorageStateResponse)(nil), // 3: gitaly.GetStorageStateResponse(*VerifyRepositoryRequest)(nil), // 4: gitaly.VerifyRepositoryRequest(*VerifyRepositoryResponse)(nil), // 5: gitaly.VerifyRepositoryResponse- (*VerifyRepositoryResponse_ReferenceDivergence)(nil), // 6: gitaly.VerifyRepositoryResponse.ReferenceDivergence- (*Repository)(nil), // 7: gitaly.Repository+ (*CreateCheckpointRequest)(nil), // 6: gitaly.CreateCheckpointRequest+ (*CreateCheckpointResponse)(nil), // 7: gitaly.CreateCheckpointResponse+ (*CollectGarbageRequest)(nil), // 8: gitaly.CollectGarbageRequest+ (*CollectGarbageResponse)(nil), // 9: gitaly.CollectGarbageResponse+ (*CollectRemovedRepositoriesRequest)(nil), // 10: gitaly.CollectRemovedRepositoriesRequest+ (*CollectRemovedRepositoriesResponse)(nil), // 11: gitaly.CollectRemovedRepositoriesResponse+ (*VerifyRepositoryResponse_ReferenceDivergence)(nil), // 12: gitaly.VerifyRepositoryResponse.ReferenceDivergence+ (*Repository)(nil), // 13: gitaly.Repository}var file_storage_core_proto_depIdxs = []int32{- 7, // 0: gitaly.ImportRepositoryRequest.repository:type_name -> gitaly.Repository- 7, // 1: gitaly.GetStorageStateRequest.repository:type_name -> gitaly.Repository- 7, // 2: gitaly.VerifyRepositoryRequest.repository:type_name -> gitaly.Repository- 6, // 3: gitaly.VerifyRepositoryResponse.divergences:type_name -> gitaly.VerifyRepositoryResponse.ReferenceDivergence- 0, // 4: gitaly.StorageCoreService.ImportRepository:input_type -> gitaly.ImportRepositoryRequest- 2, // 5: gitaly.StorageCoreService.GetStorageState:input_type -> gitaly.GetStorageStateRequest- 4, // 6: gitaly.StorageCoreService.VerifyRepository:input_type -> gitaly.VerifyRepositoryRequest- 1, // 7: gitaly.StorageCoreService.ImportRepository:output_type -> gitaly.ImportRepositoryResponse- 3, // 8: gitaly.StorageCoreService.GetStorageState:output_type -> gitaly.GetStorageStateResponse- 5, // 9: gitaly.StorageCoreService.VerifyRepository:output_type -> gitaly.VerifyRepositoryResponse- 7, // [7:10] is the sub-list for method output_type- 4, // [4:7] is the sub-list for method input_type- 4, // [4:4] is the sub-list for extension type_name- 4, // [4:4] is the sub-list for extension extendee- 0, // [0:4] is the sub-list for field type_name+ 13, // 0: gitaly.ImportRepositoryRequest.repository:type_name -> gitaly.Repository+ 13, // 1: gitaly.GetStorageStateRequest.repository:type_name -> gitaly.Repository+ 13, // 2: gitaly.VerifyRepositoryRequest.repository:type_name -> gitaly.Repository+ 12, // 3: gitaly.VerifyRepositoryResponse.divergences:type_name -> gitaly.VerifyRepositoryResponse.ReferenceDivergence+ 13, // 4: gitaly.CreateCheckpointRequest.repository:type_name -> gitaly.Repository+ 13, // 5: gitaly.CollectGarbageRequest.repository:type_name -> gitaly.Repository+ 0, // 6: gitaly.StorageCoreService.ImportRepository:input_type -> gitaly.ImportRepositoryRequest+ 2, // 7: gitaly.StorageCoreService.GetStorageState:input_type -> gitaly.GetStorageStateRequest+ 4, // 8: gitaly.StorageCoreService.VerifyRepository:input_type -> gitaly.VerifyRepositoryRequest+ 6, // 9: gitaly.StorageCoreService.CreateCheckpoint:input_type -> gitaly.CreateCheckpointRequest+ 8, // 10: gitaly.StorageCoreService.CollectGarbage:input_type -> gitaly.CollectGarbageRequest+ 10, // 11: gitaly.StorageCoreService.CollectRemovedRepositories:input_type -> gitaly.CollectRemovedRepositoriesRequest+ 1, // 12: gitaly.StorageCoreService.ImportRepository:output_type -> gitaly.ImportRepositoryResponse+ 3, // 13: gitaly.StorageCoreService.GetStorageState:output_type -> gitaly.GetStorageStateResponse+ 5, // 14: gitaly.StorageCoreService.VerifyRepository:output_type -> gitaly.VerifyRepositoryResponse+ 7, // 15: gitaly.StorageCoreService.CreateCheckpoint:output_type -> gitaly.CreateCheckpointResponse+ 9, // 16: gitaly.StorageCoreService.CollectGarbage:output_type -> gitaly.CollectGarbageResponse+ 11, // 17: gitaly.StorageCoreService.CollectRemovedRepositories:output_type -> gitaly.CollectRemovedRepositoriesResponse+ 12, // [12:18] is the sub-list for method output_type+ 6, // [6:12] is the sub-list for method input_type+ 6, // [6:6] is the sub-list for extension type_name+ 6, // [6:6] is the sub-list for extension extendee+ 0, // [0:6] is the sub-list for field type_name}func init() { file_storage_core_proto_init() }@@ -585,7 +1065,7 @@ func file_storage_core_proto_init() {GoPackagePath: reflect.TypeOf(x{}).PkgPath(),RawDescriptor: unsafe.Slice(unsafe.StringData(file_storage_core_proto_rawDesc), len(file_storage_core_proto_rawDesc)),NumEnums: 0,- NumMessages: 7,+ NumMessages: 13,NumExtensions: 0,NumServices: 1,},diff --git a/proto/go/gitalypb/storage_core_grpc.pb.go b/proto/go/gitalypb/storage_core_grpc.pb.goindex 55763d1..461fa24 100644--- a/proto/go/gitalypb/storage_core_grpc.pb.go+++ b/proto/go/gitalypb/storage_core_grpc.pb.go@@ -19,9 +19,12 @@ import (const _ = grpc.SupportPackageIsVersion9const (- StorageCoreService_ImportRepository_FullMethodName = "/gitaly.StorageCoreService/ImportRepository"- StorageCoreService_GetStorageState_FullMethodName = "/gitaly.StorageCoreService/GetStorageState"- StorageCoreService_VerifyRepository_FullMethodName = "/gitaly.StorageCoreService/VerifyRepository"+ StorageCoreService_ImportRepository_FullMethodName = "/gitaly.StorageCoreService/ImportRepository"+ StorageCoreService_GetStorageState_FullMethodName = "/gitaly.StorageCoreService/GetStorageState"+ StorageCoreService_VerifyRepository_FullMethodName = "/gitaly.StorageCoreService/VerifyRepository"+ StorageCoreService_CreateCheckpoint_FullMethodName = "/gitaly.StorageCoreService/CreateCheckpoint"+ StorageCoreService_CollectGarbage_FullMethodName = "/gitaly.StorageCoreService/CollectGarbage"+ StorageCoreService_CollectRemovedRepositories_FullMethodName = "/gitaly.StorageCoreService/CollectRemovedRepositories")// StorageCoreServiceClient is the client API for StorageCoreService service.@@ -50,6 +53,23 @@ type StorageCoreServiceClient interface {// optionally checks the connectivity of the local repository. In// authoritative mode the local repository is caught up first.VerifyRepository(ctx context.Context, in *VerifyRepositoryRequest, opts ...grpc.CallOption) (*VerifyRepositoryResponse, error)+ // CreateCheckpoint builds a checkpoint of the repository at its current+ // database head and records it: one pack of every object reachable from the+ // references at the head, with the reference snapshot, so that hydration+ // starts from it and garbage collection can delete the packs it covers. It+ // fails with FailedPrecondition outside authoritative mode.+ CreateCheckpoint(ctx context.Context, in *CreateCheckpointRequest, opts ...grpc.CallOption) (*CreateCheckpointResponse, error)+ // CollectGarbage deletes the packs of the repository that no reader needs+ // any more: the objects no row references once they were observed so at+ // least gc_grace ago, the changeset packs covered by a checkpoint at least+ // gc_grace old, and the checkpoints that one supersedes. It touches no other+ // repository. It fails with FailedPrecondition when the storage core is off,+ // or while another maintenance of the repository holds its lease.+ CollectGarbage(ctx context.Context, in *CollectGarbageRequest, opts ...grpc.CallOption) (*CollectGarbageResponse, error)+ // CollectRemovedRepositories deletes every object of the repositories of a+ // storage that were removed at least gc_grace ago. It fails with+ // FailedPrecondition when the storage core is off.+ CollectRemovedRepositories(ctx context.Context, in *CollectRemovedRepositoriesRequest, opts ...grpc.CallOption) (*CollectRemovedRepositoriesResponse, error)}type storageCoreServiceClient struct {@@ -90,6 +110,36 @@ func (c *storageCoreServiceClient) VerifyRepository(ctx context.Context, in *Verreturn out, nil}+func (c *storageCoreServiceClient) CreateCheckpoint(ctx context.Context, in *CreateCheckpointRequest, opts ...grpc.CallOption) (*CreateCheckpointResponse, error) {+ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)+ out := new(CreateCheckpointResponse)+ err := c.cc.Invoke(ctx, StorageCoreService_CreateCheckpoint_FullMethodName, in, out, cOpts...)+ if err != nil {+ return nil, err+ }+ return out, nil+}++func (c *storageCoreServiceClient) CollectGarbage(ctx context.Context, in *CollectGarbageRequest, opts ...grpc.CallOption) (*CollectGarbageResponse, error) {+ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)+ out := new(CollectGarbageResponse)+ err := c.cc.Invoke(ctx, StorageCoreService_CollectGarbage_FullMethodName, in, out, cOpts...)+ if err != nil {+ return nil, err+ }+ return out, nil+}++func (c *storageCoreServiceClient) CollectRemovedRepositories(ctx context.Context, in *CollectRemovedRepositoriesRequest, opts ...grpc.CallOption) (*CollectRemovedRepositoriesResponse, error) {+ cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)+ out := new(CollectRemovedRepositoriesResponse)+ err := c.cc.Invoke(ctx, StorageCoreService_CollectRemovedRepositories_FullMethodName, in, out, cOpts...)+ if err != nil {+ return nil, err+ }+ return out, nil+}+// StorageCoreServiceServer is the server API for StorageCoreService service.// All implementations must embed UnimplementedStorageCoreServiceServer// for forward compatibility.@@ -116,6 +166,23 @@ type StorageCoreServiceServer interface {// optionally checks the connectivity of the local repository. In// authoritative mode the local repository is caught up first.VerifyRepository(context.Context, *VerifyRepositoryRequest) (*VerifyRepositoryResponse, error)+ // CreateCheckpoint builds a checkpoint of the repository at its current+ // database head and records it: one pack of every object reachable from the+ // references at the head, with the reference snapshot, so that hydration+ // starts from it and garbage collection can delete the packs it covers. It+ // fails with FailedPrecondition outside authoritative mode.+ CreateCheckpoint(context.Context, *CreateCheckpointRequest) (*CreateCheckpointResponse, error)+ // CollectGarbage deletes the packs of the repository that no reader needs+ // any more: the objects no row references once they were observed so at+ // least gc_grace ago, the changeset packs covered by a checkpoint at least+ // gc_grace old, and the checkpoints that one supersedes. It touches no other+ // repository. It fails with FailedPrecondition when the storage core is off,+ // or while another maintenance of the repository holds its lease.+ CollectGarbage(context.Context, *CollectGarbageRequest) (*CollectGarbageResponse, error)+ // CollectRemovedRepositories deletes every object of the repositories of a+ // storage that were removed at least gc_grace ago. It fails with+ // FailedPrecondition when the storage core is off.+ CollectRemovedRepositories(context.Context, *CollectRemovedRepositoriesRequest) (*CollectRemovedRepositoriesResponse, error)mustEmbedUnimplementedStorageCoreServiceServer()}@@ -135,6 +202,15 @@ func (UnimplementedStorageCoreServiceServer) GetStorageState(context.Context, *Gfunc (UnimplementedStorageCoreServiceServer) VerifyRepository(context.Context, *VerifyRepositoryRequest) (*VerifyRepositoryResponse, error) {return nil, status.Error(codes.Unimplemented, "method VerifyRepository not implemented")}+func (UnimplementedStorageCoreServiceServer) CreateCheckpoint(context.Context, *CreateCheckpointRequest) (*CreateCheckpointResponse, error) {+ return nil, status.Error(codes.Unimplemented, "method CreateCheckpoint not implemented")+}+func (UnimplementedStorageCoreServiceServer) CollectGarbage(context.Context, *CollectGarbageRequest) (*CollectGarbageResponse, error) {+ return nil, status.Error(codes.Unimplemented, "method CollectGarbage not implemented")+}+func (UnimplementedStorageCoreServiceServer) CollectRemovedRepositories(context.Context, *CollectRemovedRepositoriesRequest) (*CollectRemovedRepositoriesResponse, error) {+ return nil, status.Error(codes.Unimplemented, "method CollectRemovedRepositories not implemented")+}func (UnimplementedStorageCoreServiceServer) mustEmbedUnimplementedStorageCoreServiceServer() {}func (UnimplementedStorageCoreServiceServer) testEmbeddedByValue() {}@@ -210,6 +286,60 @@ func _StorageCoreService_VerifyRepository_Handler(srv interface{}, ctx context.Creturn interceptor(ctx, in, info, handler)}+func _StorageCoreService_CreateCheckpoint_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {+ in := new(CreateCheckpointRequest)+ if err := dec(in); err != nil {+ return nil, err+ }+ if interceptor == nil {+ return srv.(StorageCoreServiceServer).CreateCheckpoint(ctx, in)+ }+ info := &grpc.UnaryServerInfo{+ Server: srv,+ FullMethod: StorageCoreService_CreateCheckpoint_FullMethodName,+ }+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {+ return srv.(StorageCoreServiceServer).CreateCheckpoint(ctx, req.(*CreateCheckpointRequest))+ }+ return interceptor(ctx, in, info, handler)+}++func _StorageCoreService_CollectGarbage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {+ in := new(CollectGarbageRequest)+ if err := dec(in); err != nil {+ return nil, err+ }+ if interceptor == nil {+ return srv.(StorageCoreServiceServer).CollectGarbage(ctx, in)+ }+ info := &grpc.UnaryServerInfo{+ Server: srv,+ FullMethod: StorageCoreService_CollectGarbage_FullMethodName,+ }+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {+ return srv.(StorageCoreServiceServer).CollectGarbage(ctx, req.(*CollectGarbageRequest))+ }+ return interceptor(ctx, in, info, handler)+}++func _StorageCoreService_CollectRemovedRepositories_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {+ in := new(CollectRemovedRepositoriesRequest)+ if err := dec(in); err != nil {+ return nil, err+ }+ if interceptor == nil {+ return srv.(StorageCoreServiceServer).CollectRemovedRepositories(ctx, in)+ }+ info := &grpc.UnaryServerInfo{+ Server: srv,+ FullMethod: StorageCoreService_CollectRemovedRepositories_FullMethodName,+ }+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {+ return srv.(StorageCoreServiceServer).CollectRemovedRepositories(ctx, req.(*CollectRemovedRepositoriesRequest))+ }+ return interceptor(ctx, in, info, handler)+}+// StorageCoreService_ServiceDesc is the grpc.ServiceDesc for StorageCoreService service.// It's only intended for direct use with grpc.RegisterService,// and not to be introspected or modified (even as a copy)@@ -229,6 +359,18 @@ var StorageCoreService_ServiceDesc = grpc.ServiceDesc{MethodName: "VerifyRepository",Handler: _StorageCoreService_VerifyRepository_Handler,},+ {+ MethodName: "CreateCheckpoint",+ Handler: _StorageCoreService_CreateCheckpoint_Handler,+ },+ {+ MethodName: "CollectGarbage",+ Handler: _StorageCoreService_CollectGarbage_Handler,+ },+ {+ MethodName: "CollectRemovedRepositories",+ Handler: _StorageCoreService_CollectRemovedRepositories_Handler,+ },},Streams: []grpc.StreamDesc{},Metadata: "storage_core.proto",diff --git a/proto/storage_core.proto b/proto/storage_core.protoindex 3f2da65..50a1c24 100644--- a/proto/storage_core.proto+++ b/proto/storage_core.proto@@ -44,6 +44,39 @@ service StorageCoreService {op: ACCESSOR};}++ // CreateCheckpoint builds a checkpoint of the repository at its current+ // database head and records it: one pack of every object reachable from the+ // references at the head, with the reference snapshot, so that hydration+ // starts from it and garbage collection can delete the packs it covers. It+ // fails with FailedPrecondition outside authoritative mode.+ rpc CreateCheckpoint(CreateCheckpointRequest) returns (CreateCheckpointResponse) {+ option (op_type) = {+ op: MUTATOR+ };+ }++ // CollectGarbage deletes the packs of the repository that no reader needs+ // any more: the objects no row references once they were observed so at+ // least gc_grace ago, the changeset packs covered by a checkpoint at least+ // gc_grace old, and the checkpoints that one supersedes. It touches no other+ // repository. It fails with FailedPrecondition when the storage core is off,+ // or while another maintenance of the repository holds its lease.+ rpc CollectGarbage(CollectGarbageRequest) returns (CollectGarbageResponse) {+ option (op_type) = {+ op: MUTATOR+ };+ }++ // CollectRemovedRepositories deletes every object of the repositories of a+ // storage that were removed at least gc_grace ago. It fails with+ // FailedPrecondition when the storage core is off.+ rpc CollectRemovedRepositories(CollectRemovedRepositoriesRequest) returns (CollectRemovedRepositoriesResponse) {+ option (op_type) = {+ op: MUTATOR+ scope_level: STORAGE+ };+ }}// ImportRepositoryRequest is the request of ImportRepository.@@ -121,3 +154,79 @@ message VerifyRepositoryResponse {// connectivity_error describes a failed connectivity check.string connectivity_error = 4;}++// CreateCheckpointRequest is the request of CreateCheckpoint.+message CreateCheckpointRequest {+ // repository is the repository to checkpoint.+ Repository repository = 1 [(target_repository)=true];+}++// CreateCheckpointResponse is the response of CreateCheckpoint.+message CreateCheckpointResponse {+ // checkpoint_sequence is the changeset sequence the checkpoint is at; 0 when+ // the repository has no object to consolidate (every reference is symbolic).+ uint64 checkpoint_sequence = 1;+ // created is false when a checkpoint at the head existed already.+ bool created = 2;+ // pack_bytes is the size of the checkpoint's pack.+ int64 pack_bytes = 3;+ // references is the number of references in the checkpoint's snapshot.+ uint32 references = 4;+}++// CollectGarbageRequest is the request of CollectGarbage.+message CollectGarbageRequest {+ // repository is the repository whose packs are collected.+ Repository repository = 1 [(target_repository)=true];+ // dry_run reports what would be deleted and changes nothing.+ bool dry_run = 2;+}++// CollectGarbageResponse is the response of CollectGarbage.+message CollectGarbageResponse {+ // boundary is the sequence of the newest checkpoint at least gc_grace old;+ // 0 when there is none.+ uint64 boundary = 1;+ // covered_deleted is the number of changeset packs covered by the boundary.+ uint32 covered_deleted = 2;+ // checkpoints_dropped is the number of checkpoints superseded by the boundary.+ uint32 checkpoints_dropped = 3;+ // orphans_deleted is the number of unreferenced objects observed at least+ // gc_grace ago.+ uint32 orphans_deleted = 4;+ // orphans_observed is the number of unreferenced objects seen for the first+ // time.+ uint32 orphans_observed = 5;+ // orphans_in_grace is the number of unreferenced objects observed less than+ // gc_grace ago.+ uint32 orphans_in_grace = 6;+ // packs_deleted is the number of objects deleted (to delete in a dry run).+ uint32 packs_deleted = 7;+ // bytes_reclaimed is the size of those objects.+ int64 bytes_reclaimed = 8;+ // dry_run tells that nothing was deleted or recorded.+ bool dry_run = 9;+}++// CollectRemovedRepositoriesRequest is the request of+// CollectRemovedRepositories.+message CollectRemovedRepositoriesRequest {+ // storage_name is the storage whose removed repositories are collected.+ string storage_name = 1 [(storage)=true];+ // dry_run reports what would be deleted and changes nothing.+ bool dry_run = 2;+}++// CollectRemovedRepositoriesResponse is the response of+// CollectRemovedRepositories.+message CollectRemovedRepositoriesResponse {+ // repositories is the number of removed repositories whose objects were+ // deleted (to delete in a dry run).+ uint32 repositories = 1;+ // packs_deleted is the number of objects deleted.+ uint32 packs_deleted = 2;+ // bytes_reclaimed is the size of those objects.+ int64 bytes_reclaimed = 3;+ // dry_run tells that nothing was deleted or recorded.+ bool dry_run = 4;+}diff --git a/sdk/storage.go b/sdk/storage.goindex 55f54e5..8e0ddd8 100644--- a/sdk/storage.go+++ b/sdk/storage.go@@ -147,6 +147,125 @@ func (r *Repo) StorageVerify(ctx context.Context, connectivity bool) (*StorageVereturn out, nil}+// StorageCheckpoint is the outcome of checkpointing a repository.+type StorageCheckpoint struct {+ // Sequence is the changeset sequence the checkpoint is at; 0 when the+ // repository has no object to consolidate (every reference is symbolic).+ Sequence uint64 `json:"sequence"`+ // Created is false when a checkpoint at the head existed already.+ Created bool `json:"created"`+ // PackBytes is the size of the checkpoint's pack.+ PackBytes int64 `json:"pack_bytes"`+ // References is the number of references in the checkpoint's snapshot.+ References uint32 `json:"references"`+}++// StorageGC is what garbage collection of one repository deleted, or would+// delete in a dry run.+type StorageGC struct {+ // Boundary is the sequence of the newest checkpoint at least gc_grace old,+ // 0 when there is none.+ Boundary uint64 `json:"boundary"`+ // CoveredDeleted are the changeset packs covered by the boundary.+ CoveredDeleted uint32 `json:"covered_deleted"`+ // CheckpointsDropped are the checkpoints superseded by the boundary.+ CheckpointsDropped uint32 `json:"checkpoints_dropped"`+ // OrphansDeleted are the unreferenced objects observed at least gc_grace ago.+ OrphansDeleted uint32 `json:"orphans_deleted"`+ // OrphansObserved are the unreferenced objects seen for the first time.+ OrphansObserved uint32 `json:"orphans_observed"`+ // OrphansInGrace are the unreferenced objects observed less than gc_grace ago.+ OrphansInGrace uint32 `json:"orphans_in_grace"`+ // PacksDeleted counts every object deleted.+ PacksDeleted uint32 `json:"packs_deleted"`+ // BytesReclaimed is the size of those objects.+ BytesReclaimed int64 `json:"bytes_reclaimed"`+ // DryRun tells that nothing was deleted or recorded.+ DryRun bool `json:"dry_run"`+}++// StorageGCRemoved is what garbage collection of a storage's removed+// repositories deleted, or would delete in a dry run.+type StorageGCRemoved struct {+ // Repositories is the number of removed repositories whose objects were+ // deleted.+ Repositories uint32 `json:"repositories"`+ // PacksDeleted counts every object deleted.+ PacksDeleted uint32 `json:"packs_deleted"`+ // BytesReclaimed is the size of those objects.+ BytesReclaimed int64 `json:"bytes_reclaimed"`+ // DryRun tells that nothing was deleted or recorded.+ DryRun bool `json:"dry_run"`+}++// StorageCheckpoint checkpoints the repository at its current database head:+// one pack of every object reachable from its references, with the reference+// snapshot, so that hydration starts from it and garbage collection can delete+// the packs it covers. It fails with FailedPrecondition outside authoritative+// mode.+func (r *Repo) StorageCheckpoint(ctx context.Context) (*StorageCheckpoint, error) {+ resp, err := r.storageCore().CreateCheckpoint(ctx, &gitalypb.CreateCheckpointRequest{+ Repository: r.repository(),+ })+ if err != nil {+ return nil, err+ }+ return &StorageCheckpoint{+ Sequence: resp.GetCheckpointSequence(),+ Created: resp.GetCreated(),+ PackBytes: resp.GetPackBytes(),+ References: resp.GetReferences(),+ }, nil+}++// StorageGC deletes the packs of the repository that no reader needs any more+// (with dryRun, only reports them). It touches no other repository. It fails+// with FailedPrecondition when the storage core is off or another maintenance+// of the repository is in progress.+func (r *Repo) StorageGC(ctx context.Context, dryRun bool) (*StorageGC, error) {+ resp, err := r.storageCore().CollectGarbage(ctx, &gitalypb.CollectGarbageRequest{+ Repository: r.repository(),+ DryRun: dryRun,+ })+ if err != nil {+ return nil, err+ }+ return &StorageGC{+ Boundary: resp.GetBoundary(),+ CoveredDeleted: resp.GetCoveredDeleted(),+ CheckpointsDropped: resp.GetCheckpointsDropped(),+ OrphansDeleted: resp.GetOrphansDeleted(),+ OrphansObserved: resp.GetOrphansObserved(),+ OrphansInGrace: resp.GetOrphansInGrace(),+ PacksDeleted: resp.GetPacksDeleted(),+ BytesReclaimed: resp.GetBytesReclaimed(),+ DryRun: resp.GetDryRun(),+ }, nil+}++// StorageGCRemoved deletes every object of the repositories of the storage+// that were removed at least gc_grace ago (with dryRun, only reports them). An+// empty storage is the client's storage (WithStorage). It fails with+// FailedPrecondition when the storage core is off.+func (c *Client) StorageGCRemoved(ctx context.Context, storage string, dryRun bool) (*StorageGCRemoved, error) {+ if storage == "" {+ storage = c.storage+ }+ resp, err := gitalypb.NewStorageCoreServiceClient(c.conn).CollectRemovedRepositories(ctx, &gitalypb.CollectRemovedRepositoriesRequest{+ StorageName: storage,+ DryRun: dryRun,+ })+ if err != nil {+ return nil, err+ }+ return &StorageGCRemoved{+ Repositories: resp.GetRepositories(),+ PacksDeleted: resp.GetPacksDeleted(),+ BytesReclaimed: resp.GetBytesReclaimed(),+ DryRun: resp.GetDryRun(),+ }, nil+}+// storageCore returns the storage core service client on the connection of the client.func (r *Repo) storageCore() gitalypb.StorageCoreServiceClient {return gitalypb.NewStorageCoreServiceClient(r.client.conn)diff --git a/sdk/storage_test.go b/sdk/storage_test.goindex bb04acc..0ebc310 100644--- a/sdk/storage_test.go+++ b/sdk/storage_test.go@@ -19,15 +19,51 @@ import (type fakeStorageCore struct {gitalypb.UnimplementedStorageCoreServiceServer- importResp *gitalypb.ImportRepositoryResponse- stateResp *gitalypb.GetStorageStateResponse- verifyResp *gitalypb.VerifyRepositoryResponse- err error+ importResp *gitalypb.ImportRepositoryResponse+ stateResp *gitalypb.GetStorageStateResponse+ verifyResp *gitalypb.VerifyRepositoryResponse+ checkpointResp *gitalypb.CreateCheckpointResponse+ gcResp *gitalypb.CollectGarbageResponse+ removedResp *gitalypb.CollectRemovedRepositoriesResponse+ err error- mu sync.Mutex- importReqs []*gitalypb.ImportRepositoryRequest- stateReqs []*gitalypb.GetStorageStateRequest- verifyReqs []*gitalypb.VerifyRepositoryRequest+ mu sync.Mutex+ importReqs []*gitalypb.ImportRepositoryRequest+ stateReqs []*gitalypb.GetStorageStateRequest+ verifyReqs []*gitalypb.VerifyRepositoryRequest+ checkpointReqs []*gitalypb.CreateCheckpointRequest+ gcReqs []*gitalypb.CollectGarbageRequest+ removedReqs []*gitalypb.CollectRemovedRepositoriesRequest+}++func (f *fakeStorageCore) CreateCheckpoint(_ context.Context, req *gitalypb.CreateCheckpointRequest) (*gitalypb.CreateCheckpointResponse, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.checkpointReqs = append(f.checkpointReqs, req)+ if f.err != nil {+ return nil, f.err+ }+ return f.checkpointResp, nil+}++func (f *fakeStorageCore) CollectGarbage(_ context.Context, req *gitalypb.CollectGarbageRequest) (*gitalypb.CollectGarbageResponse, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.gcReqs = append(f.gcReqs, req)+ if f.err != nil {+ return nil, f.err+ }+ return f.gcResp, nil+}++func (f *fakeStorageCore) CollectRemovedRepositories(_ context.Context, req *gitalypb.CollectRemovedRepositoriesRequest) (*gitalypb.CollectRemovedRepositoriesResponse, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.removedReqs = append(f.removedReqs, req)+ if f.err != nil {+ return nil, f.err+ }+ return f.removedResp, nil}func (f *fakeStorageCore) ImportRepository(_ context.Context, req *gitalypb.ImportRepositoryRequest) (*gitalypb.ImportRepositoryResponse, error) {@@ -70,6 +106,88 @@ func storageRepo(t *testing.T, fake *fakeStorageCore) *sdk.Repo {return client.Repo("@hashed/ab/cd/repo.git")}+func TestStorageCheckpoint(t *testing.T) {+ fake := &fakeStorageCore{checkpointResp: &gitalypb.CreateCheckpointResponse{+ CheckpointSequence: 42,+ Created: true,+ PackBytes: 8192,+ References: 5,+ }}+ repo := storageRepo(t, fake)++ res, err := repo.StorageCheckpoint(t.Context())+ require.NoError(t, err)+ require.Equal(t, &sdk.StorageCheckpoint{Sequence: 42, Created: true, PackBytes: 8192, References: 5}, res)+ require.Len(t, fake.checkpointReqs, 1)+ require.Equal(t, "@hashed/ab/cd/repo.git", fake.checkpointReqs[0].GetRepository().GetRelativePath())+}++func TestStorageGC(t *testing.T) {+ fake := &fakeStorageCore{gcResp: &gitalypb.CollectGarbageResponse{+ Boundary: 7,+ CoveredDeleted: 3,+ CheckpointsDropped: 1,+ OrphansDeleted: 2,+ OrphansObserved: 4,+ OrphansInGrace: 5,+ PacksDeleted: 6,+ BytesReclaimed: 1 << 20,+ DryRun: true,+ }}+ repo := storageRepo(t, fake)++ res, err := repo.StorageGC(t.Context(), true)+ require.NoError(t, err)+ require.Equal(t, &sdk.StorageGC{+ Boundary: 7,+ CoveredDeleted: 3,+ CheckpointsDropped: 1,+ OrphansDeleted: 2,+ OrphansObserved: 4,+ OrphansInGrace: 5,+ PacksDeleted: 6,+ BytesReclaimed: 1 << 20,+ DryRun: true,+ }, res)+ require.Len(t, fake.gcReqs, 1)+ require.True(t, fake.gcReqs[0].GetDryRun())+ require.Equal(t, "default", fake.gcReqs[0].GetRepository().GetStorageName())+}++func TestStorageGCRemoved(t *testing.T) {+ fake := &fakeStorageCore{removedResp: &gitalypb.CollectRemovedRepositoriesResponse{+ Repositories: 2,+ PacksDeleted: 9,+ BytesReclaimed: 4096,+ }}+ client := dialFake(t, func(srv *grpc.Server) {+ gitalypb.RegisterStorageCoreServiceServer(srv, fake)+ }, sdk.WithStorage("default"))++ res, err := client.StorageGCRemoved(t.Context(), "", false)+ require.NoError(t, err)+ require.Equal(t, &sdk.StorageGCRemoved{Repositories: 2, PacksDeleted: 9, BytesReclaimed: 4096}, res)++ _, err = client.StorageGCRemoved(t.Context(), "other", true)+ require.NoError(t, err)++ require.Len(t, fake.removedReqs, 2)+ require.Equal(t, "default", fake.removedReqs[0].GetStorageName(), "the client's storage by default")+ require.False(t, fake.removedReqs[0].GetDryRun())+ require.Equal(t, "other", fake.removedReqs[1].GetStorageName())+ require.True(t, fake.removedReqs[1].GetDryRun())+}++func TestStorageMaintenanceErrorsAreReturned(t *testing.T) {+ fake := &fakeStorageCore{err: status.Error(codes.FailedPrecondition, "the storage core is off")}+ repo := storageRepo(t, fake)++ _, err := repo.StorageCheckpoint(t.Context())+ require.Equal(t, codes.FailedPrecondition, status.Code(err))+ _, err = repo.StorageGC(t.Context(), false)+ require.Equal(t, codes.FailedPrecondition, status.Code(err))+}+func TestStorageImport(t *testing.T) {fake := &fakeStorageCore{importResp: &gitalypb.ImportRepositoryResponse{Sequence: 1,