Git designed for AI agents.
Fourth review: one marker-removal rule, a model test, and the findings
Federico Elia70d4a2b97ba1
diff --git a/cmd/graft/storage.go b/cmd/graft/storage.goindex 842c89b..19ab142 100644--- a/cmd/graft/storage.go+++ b/cmd/graft/storage.go@@ -7,6 +7,7 @@ import ("flag""fmt""io"+ "slices""unicode/utf8""github.com/feder-cr/git-ai/internal/graft/workspace"@@ -20,7 +21,7 @@ const storageSynopsis = "usage: graft storage import|state|verify|checkpoint|gcconst storageUsage = "storage requires a subcommand: import, state, verify, checkpoint or gc (" + storageSynopsis + ")"// storageFlags holds the flags of graft storage. Each subcommand reads only-// some of them, as its help text says.+// some of them, as its help text says, and rejects the others (checkUsed).type storageFlags struct {repo stringconnectivity bool@@ -57,6 +58,11 @@ func cmdStorage(ctx context.Context, out io.Writer, o *options, f *storageFlags,return usagef("%s", storageUsage)}sub, rest := args[0], args[1:]+ if used, known := storageSubcommandFlags[sub]; known {+ if err := f.checkUsed(sub, used); err != nil {+ return err+ }+ }switch sub {case "import":return storageImport(ctx, out, o, f, rest)@@ -72,6 +78,35 @@ func cmdStorage(ctx context.Context, out io.Writer, o *options, f *storageFlags,return usagef("unknown storage subcommand %q: %s", sub, storageUsage)}+// storageSubcommandFlags are the subcommand-specific flags each subcommand+// reads.+var storageSubcommandFlags = map[string][]string{+ "import": nil,+ "state": nil,+ "verify": {"connectivity"},+ "checkpoint": nil,+ "gc": {"dry-run", "removed"},+}++// checkUsed rejects a subcommand-specific flag that the subcommand sub does not+// read: a flag that does nothing must not look accepted (checkpoint --dry-run+// would create a checkpoint).+func (f *storageFlags) checkUsed(sub string, used []string) error {+ for _, flag := range []struct {+ name string+ set bool+ }{+ {"connectivity", f.connectivity},+ {"dry-run", f.dryRun},+ {"removed", f.removed},+ } {+ if flag.set && !slices.Contains(used, flag.name) {+ return usagef("storage %s does not take --%s: %s", sub, flag.name, storageSynopsis)+ }+ }+ return nil+}+func storageCheckpoint(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {if len(args) != 0 {return usagef("storage checkpoint takes flags only: checkpoint [-repo R] [--json]")diff --git a/cmd/graft/storage_test.go b/cmd/graft/storage_test.goindex 4ccad23..633c350 100644--- a/cmd/graft/storage_test.go+++ b/cmd/graft/storage_test.go@@ -177,6 +177,11 @@ func TestCLIStorageUsageErrorsDoNotContactServer(t *testing.T) {"gc with arg": {"storage", "gc", "-C", dir, "extra"},"gc removed and repo": {"storage", "gc", "--removed", "-C", dir, "-repo", "a.git"},"gc removed without server": {"storage", "gc", "--removed", "-C", dir},+ "import dry run": {"storage", "import", "--dry-run", "-C", dir},+ "checkpoint dry run": {"storage", "checkpoint", "--dry-run", "-C", dir},+ "state connectivity": {"storage", "state", "--connectivity", "-C", dir},+ "verify removed": {"storage", "verify", "--removed", "-C", dir},+ "checkpoint removed": {"storage", "checkpoint", "--removed", "-C", dir},}for name, args := range cases {code, _, stderr := runCLI(t, args...)diff --git a/doc/graft/AGENT-SECURITY.md b/doc/graft/AGENT-SECURITY.mdindex 48ecdab..ea5ba18 100644--- a/doc/graft/AGENT-SECURITY.md+++ b/doc/graft/AGENT-SECURITY.md@@ -57,7 +57,8 @@ These can never be enabled, not even through `graft.agent_rpcs`:- `RemoteService` `UpdateRemoteMirror`, `FindRemoteRepository` and`FindRemoteRootRef`, which push to or query a URL the caller chooses.- `CleanupService`, `ObjectPoolService`, `HookService`, `InternalGitaly`,- `PartitionService`.+ `PartitionService`, and `StorageCoreService` (imports, checkpoints and+ garbage collection, which deletes packs of the bucket).Every other RPC that is not on the allowlist (for example the other`OperationService` RPCs such as `UserMergeBranch`, or `SSHService`) is denieddiff --git a/doc/graft/LIMITATIONS.md b/doc/graft/LIMITATIONS.mdindex a43c4e0..349a5d4 100644--- a/doc/graft/LIMITATIONS.md+++ b/doc/graft/LIMITATIONS.md@@ -82,7 +82,7 @@ they are rejected with `PermissionDenied` and need the shared token:- `RemoteService`: `UpdateRemoteMirror`, `FindRemoteRepository`,`FindRemoteRootRef` (they push to or query a URL the caller chooses);- every RPC of `HookService`, `ObjectPoolService`, `CleanupService`,- `PartitionService` and `InternalGitaly`.+ `PartitionService`, `InternalGitaly` and `StorageCoreService`.`CreateRepositoryFromSnapshot` downloads a tar archive from a URL that theclient chooses and extracts it into the new repository without filtering it.diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex c376650..67b3154 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -657,11 +657,12 @@ repository-wide lock.written, which a writing git process always has: `tables.list.lock`(prepare to commit), `<table>.lock` (the tables a compaction merges),`*.temp.*` (a table being written). On top of that there is a floor- of 2 s (`RecoveryGrace`), for the instant between a commit's rename- and the start of its auto-compaction.+ of 2 s (`RecoveryGrace`) from the recovery and from the last git+ activity seen, for the instant between a commit's rename and the+ start of its auto-compaction.* A marker that records an untrusted repository is handled like amarker of another boot: the repository is moved aside if it is a- cache, and its marker is never settled otherwise.+ cache; otherwise its marker is never settled, and it is not served.* Why the grace bound holds: when the previous owner dies, the stdinof its `update-ref --stdin` children closes.* A child that has not been sent `commit` aborts at end of input and@@ -748,6 +749,61 @@ repository-wide lock.* m-4: a test proves the end of a deferred process runs after it exited. Themutation that swaps the two defers fails it.* Clean states are dropped; MarkDirty takes its state from the map again.+* Fourth review (of 32d0ac4 and 03ef6c2). Four reviews in a row found marker+ defects in different branches (Settle, Forget, Recover). The root cause:+ "a marker may go only when that is safe" was re-implemented branch by+ branch. Fixed one level up:+ * One function, `dropMarker`, removes markers and drops states. Every path+ goes through it: Settle, the missing-path case, Forget, recovered markers,+ pruning. It checks the whole condition under the state's file lock (which+ MarkDirty waits for) and `d.mu` (which Begin takes):+ * no live deferred git process (`nLive`);+ * not untrusted;+ * an existing repository only under git's reference lock, after its stack+ was made durable under that lock. A missing repository has no stack, and+ the syncer's repository lock keeps it from being hydrated meanwhile;+ * no git activity in the stack (stack.c's lock and temporary names). Seeing+ some pushes the 2 s floor forward.+ * Finding 1 (MAJOR): the missing-path branch no longer removes a marker that+ a live process needs (moved aside, then hydrated, then the orphan write).+ * Finding 3: a recovered marker of a missing repository is settled. No+ reftable directory means no activity.+ * Finding 5: a state leaves the map only in `dropMarker`, in the same `d.mu`+ section that checks `nLive`, so a Begin is never lost.+ * Finding 6: the floor counts from the last activity seen.+ * Forget rewrites a marker that is not a plain marker of the running boot.+ The model test found this one: a marker of another boot, kept for a live+ process, made the next restart distrust the hydrated copy.+ * A marker of another boot makes the state untrusted (`errReboot`), so+ nothing settles it before the move.+ * Untrusted means never served: `EnsureFresh`, which the gate runs for every+ repository RPC, answers `Unavailable` for an untrusted path. A repository+ that is not a cache stays in place, unserved, for an operator.+ * Finding 2:+ * `+gitaly` is reserved (`corestore.CheckRelativePath`), in the gate for+ every repository RPC and in every storage core RPC;+ * STORAGE.md has a procedure for importing a kept copy;+ * metrics `graft_storage_kept_untrusted_repositories` and `_bytes`.+ * Finding 4: a local history file that cannot be read means keep (moved to+ `graft-untrusted`), never a failed startup. Database errors still fail the+ recovery and are retried.+ * Finding 7: the downgrade is documented (STORAGE.md, Failures).+ * Finding 8: `StorageCoreService` is never allowed for per-agent tokens.+ * Finding 9: a flag that a storage subcommand does not read is a usage error.+ * Finding 10: CollectGarbage holds its lease with the runner's renewal+ (`maintenance.HoldLease`). A lease lost during the collection is `Aborted`.+ * `TestMarkerLifecycleModel`: 40 seeds × 250 random steps.+ * The steps: MarkDirty; Begin and end; table writes without fsync; Settle,+ final or not; failed fsyncs; move aside and Forget; hydration; a process+ restart with git processes of the previous owner in the middle of a+ write; a kernel restart; acceptance in place.+ * After every step it asserts that a write that may not be durable means a+ marker on disk, and that untrusted equals reported untrusted, with the+ marker kept. The model learns durability from the fsyncs themselves.+ * Mutation check: removing any one condition fails the model or+ `TestDropMarkerGuards`. The conditions: live processes, git's lock, git+ activity, untrusted, the plain-marker rewrite, the missing-path removal,+ the recovered untrusted record, the reboot distrust.**Tests.*** `localrefs`:diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.mdindex abc919f..d0bebb4 100644--- a/doc/graft/STORAGE.md+++ b/doc/graft/STORAGE.md@@ -364,15 +364,18 @@ graft storage gc --removed -address unix:/var/opt/graft/gitaly.sock [-storage dedied before their database commit). The first run only observes them.It runs in shadow mode too (orphans only: there are no checkpoints). It holds- the repository's maintenance lease while it runs, and fails with- `FailedPrecondition` when another maintenance of the repository (the- background one, another node) holds it.+ the repository's maintenance lease while it runs, renewed like the background+ maintenance's, and fails with `FailedPrecondition` when another maintenance+ of the repository (the background one, another node) holds it, or `Aborted`+ when the lease is lost during the collection.- `gc --removed` deletes every object of the repositories of the storage thatwere removed at least `gc_grace` ago.-- `--dry-run` reports what would be deleted and changes nothing.+- `--dry-run` reports what would be deleted and changes nothing. It is a usage+ error on the subcommands that do not read it, like `--connectivity` outside+ `verify` and `--removed` outside `gc`.-These commands are for operators: they need the shared token, and an agent-token is refused.+These commands are for operators: they need the shared token. A per-agent token+can never call the storage core service, whatever `agent_rpcs` says.### Switching from shadow to authoritative@@ -521,16 +524,57 @@ or re-import the repository from scratch (see- *A write the kernel reports as failed* (a writeback error, same boot): thedurable step's fsync fails, the repository is moved aside at once and logged,and the next request hydrates it. The kernel reports such a failure once, so- from then on the repository's marker is never removed: if the move aside- fails, it is retried every second. Other errors of the step (a file that- cannot be opened for a moment) are retried every second and counted.+ from then on the repository is untrusted. Every request for it fails with+ `Unavailable`, and its marker is never removed, until it is moved aside. If+ the move fails, it is retried every second. A repository that is not a+ cache of the storage core (the database cannot rebuild it) stays in place,+ untrusted and not served, for an operator. Other errors of the step (a file+ that cannot be opened for a moment) are retried every second and counted.- *Process restart*: a marker found is kept while the repository's reftabledirectory holds a git lock or a table being written (a git process of the- previous process may still be finishing), and for 2 s at least. A- repository the previous process had found untrusted is moved aside.- - A repository moved aside that holds shadow-mode history not imported yet- goes to `<storage>/+gitaly/graft-untrusted/` and stays there: import what- it holds, or remove it.+ previous process may still be finishing), and for 2 s at least after the+ restart and after the last such file seen. A repository the previous+ process had found untrusted is moved aside.+ - A marker stays while a git process of the node that writes without fsync+ is running, even when its repository was moved aside meanwhile: the+ process names the repository by its path, so a hydrated copy there would+ get its writes.+ - **Kept copies.** A repository moved aside that may hold shadow-mode history+ not imported yet goes to `<storage>/+gitaly/graft-untrusted/<stale-*>/repo.git`+ and stays there. That is history written in shadow mode (file changelog+ entries, file reviews) that the database does not have yet, or history+ files that cannot be read. The repository itself is hydrated from the+ database by its next request.+ `graft_storage_kept_untrusted_repositories` and+ `graft_storage_kept_untrusted_bytes` report the kept copies per storage. No+ request can name a path under `+gitaly/`: the gate and the storage core+ RPCs reject it with `InvalidArgument`.++ To import what a kept copy holds:+ 1. Take the node out of service: no client traffic to it during the+ procedure. `gitaly serve` keeps running.+ 2. Move the repository's current copy at `<storage>/<relative path>` out of+ the storage, then move the kept `repo.git` to `<storage>/<relative path>`.+ 3. `graft storage import -address <node> -storage <storage> -repo <relative path>`.+ It imports the file changelog entries and the reviews the database does+ not have, without touching the references.+ 4. Remove the copy at `<storage>/<relative path>` again: its references are+ not trusted. The next request hydrates the repository from the database.+ `graft storage verify -repo <relative path>` must then report+ `consistent`.+ 5. Remove the kept directory, and put the node back in service.++ A kept copy whose history files cannot be read cannot be imported: repair+ them, or extract what is readable by hand. Removing a kept copy loses only+ its history that was not imported.+ - *Downgrade*: a release without this rule (v0.2.x) ignores the markers. It+ would serve a repository that is untrusted, or dirty from an unclean+ shutdown. Before a downgrade, stop `gitaly serve` cleanly: the final step+ makes every repository durable. Then check that+ `<storage>/+gitaly/graft-ref-durability/` holds no `*.json` marker. A+ marker left there names a repository that is untrusted or that a live+ process kept. Move that repository out of the storage first; its next+ request after the downgrade hydrates it.- *Clean stop*: after the servers stopped, `gitaly serve` makes every dirtyrepository durable and removes its marker, so a clean reboot hydratesnothing.@@ -596,11 +640,12 @@ without a single divergence.| `graft_storage_ref_admission_bypassed_total{writer}` | Writers that went on to git's lock without their turn after `reftable.lockTimeout` (10 s). || `graft_storage_ref_admission_expired_total` | Turns ended by the 10 s bound instead of their release (a git process that died between its phases). || `graft_storage_ref_dirty_repositories` | Repositories with reference writes not yet made durable. |-| `graft_storage_ref_dirty_marked_total`, `graft_storage_ref_dirty_cleared_total` | Dirty markers written and removed by the durable step. |+| `graft_storage_ref_dirty_marked_total`, `graft_storage_ref_dirty_cleared_total` | Dirty markers written, and removed (the writes are durable, or the repository was moved aside, and no git process can still write it). || `graft_storage_ref_writes_inflight` | git processes of this node that write references without fsync and have not exited; their repositories keep their markers. || `graft_storage_ref_durability_owner` | 1 when this process owns the deferred durability of its storages, 0 while another process (the old one of a graceful upgrade) does. || `graft_storage_ref_durable_step_errors_total` | Durable steps that failed with a transient error (not a failed fsync); retried every second. || `graft_storage_ref_untrusted_total{reason}` | Repositories moved aside to be hydrated again: `reboot` (dirty at a kernel restart), `sync_failed` (the durable step's fsync failed). |+| `graft_storage_kept_untrusted_repositories{storage}`, `graft_storage_kept_untrusted_bytes{storage}` | Untrusted copies kept in `+gitaly/graft-untrusted/` for an operator (see Kept copies), and their size. Read at every scrape. |## Benchmarkdiff --git a/internal/gitaly/server/auth/agent_allowlist_test.go b/internal/gitaly/server/auth/agent_allowlist_test.goindex 43a4c2a..6ee54dd 100644--- a/internal/gitaly/server/auth/agent_allowlist_test.go+++ b/internal/gitaly/server/auth/agent_allowlist_test.go@@ -174,6 +174,9 @@ func TestAgentAllowlist_EveryRegisteredRPC(t *testing.T) {gitalypb.ObjectPoolService_DeleteObjectPool_FullMethodName,gitalypb.CleanupService_RewriteHistory_FullMethodName,gitalypb.PartitionService_BackupPartition_FullMethodName,+ gitalypb.StorageCoreService_CollectGarbage_FullMethodName,+ gitalypb.StorageCoreService_CollectRemovedRepositories_FullMethodName,+ gitalypb.StorageCoreService_GetStorageState_FullMethodName,} {require.False(t, allowlist.mayCall(fullMethod), "agent may not call %s", fullMethod)}@@ -350,6 +353,8 @@ func TestNewAgentAllowlist_Validation(t *testing.T) {{desc: "object pool", rpcs: []string{gitalypb.ObjectPoolService_DeleteObjectPool_FullMethodName}, wantErr: "may never be called by an agent"},{desc: "cleanup service", rpcs: []string{gitalypb.CleanupService_RewriteHistory_FullMethodName}, wantErr: "may never be called by an agent"},{desc: "partition service", rpcs: []string{gitalypb.PartitionService_BackupPartition_FullMethodName}, wantErr: "may never be called by an agent"},+ {desc: "storage core garbage collection", rpcs: []string{gitalypb.StorageCoreService_CollectGarbage_FullMethodName}, wantErr: "may never be called by an agent"},+ {desc: "storage core state", rpcs: []string{gitalypb.StorageCoreService_GetStorageState_FullMethodName}, wantErr: "may never be called by an agent"},{desc: "one bad entry among good ones", rpcs: []string{"/gitaly.RepositoryService/RepositorySize", gitalypb.RepositoryService_FetchRemote_FullMethodName}, wantErr: "entry 2"},} {t.Run(tc.desc, func(t *testing.T) {diff --git a/internal/gitaly/server/auth/agent_methods.go b/internal/gitaly/server/auth/agent_methods.goindex 7280115..9fb4faa 100644--- a/internal/gitaly/server/auth/agent_methods.go+++ b/internal/gitaly/server/auth/agent_methods.go@@ -26,12 +26,16 @@ import (// - CleanupService and PartitionService are administrative: they rewrite and// clean up repositories and partitions outside the reference policy.// - InternalGitaly serves the internal calls between Gitaly nodes.+// - StorageCoreService is the storage core's administration: imports,+// checkpoints and garbage collection, which deletes packs of the bucket.+// It is reserved for operators (doc/graft/STORAGE.md).var agentNeverAllowedServices = []string{gitalypb.HookService_ServiceDesc.ServiceName,gitalypb.ObjectPoolService_ServiceDesc.ServiceName,gitalypb.CleanupService_ServiceDesc.ServiceName,gitalypb.PartitionService_ServiceDesc.ServiceName,gitalypb.InternalGitaly_ServiceDesc.ServiceName,+ gitalypb.StorageCoreService_ServiceDesc.ServiceName,}// agentNeverAllowedMethods are RPCs that no caller authenticated with adiff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.goindex 592174e..491c9a0 100644--- a/internal/gitaly/service/storagecore/import.go+++ b/internal/gitaly/service/storagecore/import.go@@ -61,8 +61,8 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos}repo := req.GetRepository()- if err := s.deps.GetLocator().ValidateRepository(ctx, repo); err != nil {- return nil, structerr.NewInvalidArgument("%w", err)+ if err := s.validateRepository(ctx, repo); err != nil {+ return nil, err}local := s.localRepo(repo)diff --git a/internal/gitaly/service/storagecore/maintenance.go b/internal/gitaly/service/storagecore/maintenance.goindex 564edcf..d677830 100644--- a/internal/gitaly/service/storagecore/maintenance.go+++ b/internal/gitaly/service/storagecore/maintenance.go@@ -3,7 +3,6 @@ package storagecoreimport ("context""errors"- "time""github.com/feder-cr/git-ai/internal/gitaly/storage""github.com/feder-cr/git-ai/internal/graft/corestore"@@ -14,15 +13,12 @@ import ("github.com/feder-cr/git-ai/proto/go/gitalypb")-// releaseTimeout bounds the release of a maintenance lease after a request.-const releaseTimeout = 10 * time.Second-// CreateCheckpoint implements gitalypb.StorageCoreServiceServer// (doc/graft/PLAN7.md, "Checkpoint creation").func (s *server) CreateCheckpoint(ctx context.Context, req *gitalypb.CreateCheckpointRequest) (*gitalypb.CreateCheckpointResponse, error) {repo := req.GetRepository()- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {- return nil, structerr.NewInvalidArgument("%w", err)+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {+ return nil, err}core, err := s.enabledCore()if err != nil {@@ -54,12 +50,13 @@ func (s *server) CreateCheckpoint(ctx context.Context, req *gitalypb.CreateCheck// CollectGarbage implements gitalypb.StorageCoreServiceServer// (doc/graft/PLAN7.md, "Pack GC"). It holds the repository's maintenance lease-// while it collects, so that it does not run alongside the background-// maintenance or another request; correctness does not depend on the lease.+// while it collects, renewed (maintenance.HoldLease), so that it does not run+// alongside the background maintenance or another request; correctness does+// not depend on the lease.func (s *server) CollectGarbage(ctx context.Context, req *gitalypb.CollectGarbageRequest) (*gitalypb.CollectGarbageResponse, error) {repo := req.GetRepository()- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {- return nil, structerr.NewInvalidArgument("%w", err)+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {+ return nil, err}core, err := s.enabledCore()if err != nil {@@ -89,16 +86,17 @@ func (s *server) CollectGarbage(ctx context.Context, req *gitalypb.CollectGarbagif !acquired {return nil, structerr.NewFailedPrecondition("another maintenance operation of this repository is in progress")}- defer func() {- releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)- defer cancel()- if err := core.Meta.ReleaseMaintenanceLease(releaseCtx, record.ID, holder); err != nil {- s.deps.GetLogger().WithError(err).WarnContext(ctx, "storage core: release the maintenance lease; it expires by itself")- }- }()+ // The lease is renewed while the collection runs, like the background+ // maintenance's: a collection longer than lease_ttl keeps others out.+ lost := false+ leaseCtx, release := maintenance.HoldLease(ctx, core, s.deps.GetLogger(), record.ID, holder, ttl, func() { lost = true })- report, err := gc.New(core, s.deps.GetLogger()).CollectRepository(ctx, record.ID, req.GetDryRun())+ report, err := gc.New(core, s.deps.GetLogger()).CollectRepository(leaseCtx, record.ID, req.GetDryRun())+ release()if err != nil {+ if lost {+ return nil, structerr.NewAborted("the maintenance lease was lost to another maintenance operation of this repository: %w", err)+ }return nil, err}return &gitalypb.CollectGarbageResponse{diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.goindex 91ca797..08cdf18 100644--- a/internal/gitaly/service/storagecore/maintenance_test.go+++ b/internal/gitaly/service/storagecore/maintenance_test.go@@ -178,6 +178,39 @@ func TestCollectRemovedRepositoriesRefusals(t *testing.T) {require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)}+// TestStorageCoreRejectsReservedRelativePaths verifies that no storage core+// RPC acts on a repository inside Gitaly's data directory, where the storage+// core keeps untrusted copies for an operator: the gate does not see these+// RPCs, so they check it themselves.+func TestStorageCoreRejectsReservedRelativePaths(t *testing.T) {+ t.Parallel()++ h := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)+ repo := &gitalypb.Repository{StorageName: h.repo.GetStorageName(), RelativePath: "+gitaly/graft-untrusted/stale-1/repo.git"}+ for name, call := range map[string]func() error{+ "GetStorageState": func() error {+ _, err := h.client.GetStorageState(h.ctx, &gitalypb.GetStorageStateRequest{Repository: repo})+ return err+ },+ "VerifyRepository": func() error {+ _, err := h.client.VerifyRepository(h.ctx, &gitalypb.VerifyRepositoryRequest{Repository: repo})+ return err+ },+ "CreateCheckpoint": func() error {+ _, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: repo})+ return err+ },+ "CollectGarbage": func() error {+ _, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: repo})+ return err+ },+ } {+ err := call()+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%s: %v", name, err)+ require.ErrorContains(t, err, "Gitaly's data directory", name)+ }+}+// packKeys returns the objects of the repository in the bucket.func (h *harness) packKeys(repoID int64) map[string]bool {h.t.Helper()diff --git a/internal/gitaly/service/storagecore/server.go b/internal/gitaly/service/storagecore/server.goindex 5c540d6..5f00176 100644--- a/internal/gitaly/service/storagecore/server.go+++ b/internal/gitaly/service/storagecore/server.go@@ -13,6 +13,7 @@ import ("github.com/feder-cr/git-ai/internal/git/gitcmd""github.com/feder-cr/git-ai/internal/git/localrepo""github.com/feder-cr/git-ai/internal/gitaly/service"+ "github.com/feder-cr/git-ai/internal/gitaly/storage""github.com/feder-cr/git-ai/internal/graft/corestore""github.com/feder-cr/git-ai/internal/structerr""github.com/feder-cr/git-ai/proto/go/gitalypb"@@ -37,6 +38,20 @@ func (s *server) enabledCore() (*corestore.Core, error) {return core, nil}+// validateRepository validates the repository of a request: the locator's+// checks (InvalidArgument), and a relative path outside Gitaly's data+// directory of the storage (corestore.CheckRelativePath). The gate does not see+// the storage core's RPCs, so they check it themselves.+func (s *server) validateRepository(ctx context.Context, repo *gitalypb.Repository, opts ...storage.ValidateRepositoryOption) error {+ if err := s.deps.GetLocator().ValidateRepository(ctx, repo, opts...); err != nil {+ return structerr.NewInvalidArgument("%w", err)+ }+ if err := corestore.CheckRelativePath(repo.GetRelativePath()); err != nil {+ return structerr.NewInvalidArgument("%w", err)+ }+ return nil+}+// localRepo returns the local repository of a request.func (s *server) localRepo(repo *gitalypb.Repository) *localrepo.Repo {return s.deps.GetRepositoryFactory().Build(repo)diff --git a/internal/gitaly/service/storagecore/state.go b/internal/gitaly/service/storagecore/state.goindex 652e7f6..ee97f71 100644--- a/internal/gitaly/service/storagecore/state.go+++ b/internal/gitaly/service/storagecore/state.go@@ -15,8 +15,8 @@ import (// that this node has no local copy of (another node may have it).func (s *server) GetStorageState(ctx context.Context, req *gitalypb.GetStorageStateRequest) (*gitalypb.GetStorageStateResponse, error) {repo := req.GetRepository()- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {- return nil, structerr.NewInvalidArgument("%w", err)+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {+ return nil, err}core := s.deps.GetGraftStorage()diff --git a/internal/gitaly/service/storagecore/verify.go b/internal/gitaly/service/storagecore/verify.goindex b417ce0..86c3832 100644--- a/internal/gitaly/service/storagecore/verify.go+++ b/internal/gitaly/service/storagecore/verify.go@@ -33,8 +33,8 @@ func (s *server) VerifyRepository(ctx context.Context, req *gitalypb.VerifyRepos}repo := req.GetRepository()- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {- return nil, structerr.NewInvalidArgument("%w", err)+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {+ return nil, err}if core.Authoritative() {diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.goindex df13121..3c266b1 100644--- a/internal/graft/corestore/core.go+++ b/internal/graft/corestore/core.go@@ -9,6 +9,7 @@ import ("strings""time"+ "github.com/feder-cr/git-ai/internal/gitaly/config""github.com/feder-cr/git-ai/internal/gitaly/storage""github.com/feder-cr/git-ai/internal/graft/corestore/localrefs")@@ -199,6 +200,23 @@ func (c *Core) Close() error {return firstErr}+// ErrReservedRelativePath is returned for a repository named inside Gitaly's+// own data directory of a storage (config.GitalyDataPrefix): the temporary,+// cache and state directories, the dirty markers, and the untrusted copies the+// storage core keeps for an operator. A request naming one could make the+// storage core treat a kept copy as a stale cache and remove it.+var ErrReservedRelativePath = errors.New("the relative path is inside Gitaly's data directory of the storage, which no request may name")++// CheckRelativePath rejects a relative path that names Gitaly's data directory+// of the storage or anything in it (ErrReservedRelativePath).+func CheckRelativePath(relativePath string) error {+ rel := NormalizeRelativePath(relativePath)+ if rel == config.GitalyDataPrefix || strings.HasPrefix(rel, config.GitalyDataPrefix+"/") {+ return fmt.Errorf("%w: %q", ErrReservedRelativePath, relativePath)+ }+ return nil+}+// KeyOf returns the RepoKey of a gitaly repository. The relative path is// normalized the way the storage locator resolves it (filepath.Join of the// storage root and the relative path), so that two spellings of one directorydiff --git a/internal/graft/corestore/gate/gate.go b/internal/graft/corestore/gate/gate.goindex bd190d4..603b2c3 100644--- a/internal/graft/corestore/gate/gate.go+++ b/internal/graft/corestore/gate/gate.go@@ -267,7 +267,9 @@ func targetRepo(mi protoregistry.MethodInfo, msg any) *gitalypb.Repository {// (corestore.KeyOf), while the locator resolves every spelling of a path to// the same directory: without this check "grp//proj.git" and "grp/proj.git"// would be two repositories in the database sharing one local copy. An empty-// path is left to the handler, which rejects it itself.+// path is left to the handler, which rejects it itself. A path inside Gitaly's+// data directory is rejected too (corestore.CheckRelativePath): the stale-cache+// rule must never act on a copy the storage core keeps there.func checkCanonical(repo *gitalypb.Repository) error {rel := repo.GetRelativePath()if rel == "" {@@ -276,6 +278,9 @@ func checkCanonical(repo *gitalypb.Repository) error {if clean := filepath.Clean(rel); clean != rel {return structerr.NewInvalidArgument("relative path %q is not canonical in authoritative storage mode, use %q", rel, clean)}+ if err := corestore.CheckRelativePath(rel); err != nil {+ return structerr.NewInvalidArgument("%w", err)+ }return nil}diff --git a/internal/graft/corestore/gate/gate_stale_test.go b/internal/graft/corestore/gate/gate_stale_test.goindex c2988eb..3faec2e 100644--- a/internal/graft/corestore/gate/gate_stale_test.go+++ b/internal/graft/corestore/gate/gate_stale_test.go@@ -229,11 +229,16 @@ func TestGateRemoveRepositoryDropsReinstalledCache(t *testing.T) {}// The database key is the relative path as given, so the gate only accepts-// relative paths in their cleaned form.-func TestGateRejectsNonCanonicalRelativePath(t *testing.T) {+// relative paths in their cleaned form; and none inside Gitaly's data+// directory, where the storage core keeps untrusted copies that the stale-cache+// rule must never remove.+func TestGateRejectsNonCanonicalOrReservedRelativePath(t *testing.T) {t.Parallel()- for _, rel := range []string{"grp//proj.git", "./grp/proj.git", "grp/proj.git/", "grp/../grp/proj.git", "grp/./proj.git"} {+ for _, rel := range []string{+ "grp//proj.git", "./grp/proj.git", "grp/proj.git/", "grp/../grp/proj.git", "grp/./proj.git",+ "+gitaly", "+gitaly/graft-untrusted/stale-1/repo.git", "+gitaly/tmp/hydrate-1/repo.git",+ } {t.Run(rel, func(t *testing.T) {t.Parallel()diff --git a/internal/graft/corestore/localrefs/durability.go b/internal/graft/corestore/localrefs/durability.goindex 35b7ad0..2a81b48 100644--- a/internal/graft/corestore/localrefs/durability.go+++ b/internal/graft/corestore/localrefs/durability.go@@ -90,6 +90,11 @@ const stackLockTag = "graft durable step "// writes, so the local repository must not be served before it is rebuilt.var ErrUntrusted = errors.New("storage core: the local references may have lost writes")+// errReboot makes a repository untrusted whose marker is of another kernel+// boot: the kernel restarted while its references had writes that were not+// durable, so its stack may be torn.+var errReboot = errors.New("the kernel restarted while the references had writes that were not durable")+// UntrustedError is ErrUntrusted for one repository.type UntrustedError struct {Repo RepoRef@@ -126,9 +131,13 @@ func (e writebackError) Unwrap() error { return e.err }// - before the first reference write that is not durable, the repository// gets a dirty marker, written durably, that holds the identity of the// kernel boot (MarkDirty);-// - a durable step makes the references durable and removes the marker,-// under git's reference lock and only when no transaction of the process is-// live (Settle);+// - a durable step makes the references durable under git's reference lock+// (Settle); one function removes markers, every path through it, and only+// when nothing can leave a write that is not durable in the stack any more+// (dropMarker);+// - a repository whose writes may be lost (a failed fsync, a marker of+// another boot) is untrusted: never served, its marker never removed, until+// it is moved aside (Forget);// - when it takes the ownership, a marker of another boot means the kernel// restarted while the repository had writes that may never have reached the// disk: its reftable stack may be torn, which git reads as an empty@@ -157,13 +166,14 @@ type Durability struct {ownerFiles []*os.Filelocked bool// recoveryGrace is how long a repository recovered dirty keeps its marker- // at least: a git process of the previous owner that outlived it (killed- // with it, but not reaped yet) may still be committing for that long.+ // at least, and how long after git activity was last seen in a stack: a git+ // process of the previous owner that outlived it (killed with it, but not+ // reaped yet) may still be committing for that long.recoveryGrace time.Durationmu sync.Mutex- // repos holds the state of every repository written since the process- // started (one entry per repository; Forget drops it).+ // repos holds the state of every repository that is dirty or that a live+ // git process counts on (one entry per repository; dropMarker drops it).repos map[string]*refStatemarked prometheus.Counter@@ -179,28 +189,37 @@ var _ prometheus.Collector = (*Durability)(nil)// refState is what Durability knows of one local repository, by its path. Its// fields are guarded by Durability.mu, except marker writes and removals,-// which file serializes.+// which file serializes. A state leaves the map only in dropMarker, under its+// file lock, so whoever holds the file lock of the map's state keeps it current+// (lockState).type refState struct {// file serializes the writes and the removal of the marker file.file sync.Mutexrepo RepoRefmarker string- dirty bool+ // dirty is set while the marker may be on disk: from before it is written+ // until dropMarker removed it.+ dirty bool+ // markerPlain is set when the marker on disk is a plain marker of the+ // running boot: neither one that records untrusted nor one of another boot,+ // which would make a restart distrust what is at the path.+ markerPlain bool// nLive counts the git processes of this process that write the// repository's references without fsync and have not exited: any of them// may still write a table or compact the stack.nLive intlastWrite time.Time- // recovered is set for a repository found dirty when the process took the- // ownership: a git process of the previous owner that outlived it may still- // be writing, so its marker stays while git's lock or temporary files are- // there (gitActive), and not before settleAfter.- recovered bool+ // settleAfter is the earliest time the marker may go: the recovery grace of+ // a repository found dirty when the process took the ownership (a git+ // process of the previous owner that outlived it may still be committing),+ // pushed forward whenever git activity is seen in its stack.settleAfter time.Time- // untrusted is the first failed fsync of the repository. The kernel reports- // a failed writeback once, so a later fsync can succeed although writes- // were lost: the repository never becomes clean again, only moved aside.- // The marker records it, so that a restart of the process keeps it.+ // untrusted is the first failed fsync of the repository, or the restart of+ // the kernel that a marker of another boot shows. The kernel reports a+ // failed writeback once, so a later fsync can succeed although writes were+ // lost: the repository is never served and never becomes clean, it is only+ // moved aside (Forget). The marker records a failed fsync, so that a restart+ // of the process keeps it.untrusted error// synced holds the tables made durable during the current dirty period that// the stack still lists. Tables are immutable, so one fsync each is enough.@@ -289,7 +308,7 @@ func newDurability(locator storage.Locator, idle time.Duration) *Durability {}),cleared: prometheus.NewCounter(prometheus.CounterOpts{Name: "graft_storage_ref_dirty_cleared_total",- Help: "Number of dirty markers removed by a durable step.",+ Help: "Number of dirty markers removed: the repository's reference writes are durable, or the repository was moved aside, and no git process can still write it.",}),untrusted: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "graft_storage_ref_untrusted_total",@@ -449,32 +468,15 @@ func (d *Durability) MarkDirty(ctx context.Context, repo RepoRef, path string) ereturn nil}- // The state is the map's after its file lock is taken: a durable step may- // have dropped a clean state meanwhile.- var st *refState- for {- d.mu.Lock()- st = d.stateLocked(path)- d.mu.Unlock()- st.file.Lock()- d.mu.Lock()- current := d.repos[path] == st- if current {- st.lastWrite = time.Now()- }- dirty := st.dirty- d.mu.Unlock()- if !current {- st.file.Unlock()- continue- }- if dirty {- st.file.Unlock()- return nil- }- break- }+ st := d.lockState(path)defer st.file.Unlock()+ d.mu.Lock()+ st.lastWrite = time.Now()+ dirty := st.dirty+ d.mu.Unlock()+ if dirty {+ return nil+ }markerPath, err := d.markerPath(ctx, repo, path)if err != nil {@@ -492,12 +494,37 @@ func (d *Durability) MarkDirty(ctx context.Context, repo RepoRef, path string) edefer d.mu.Unlock()st.repo = repost.marker = markerPath+ st.markerPlain = trued.marked.Inc()+ d.setDirtyLocked(st)+ return nil+}++// lockState returns the state of the repository at path, created when there is+// none, with its file lock held: the state stays the map's until the lock is+// released, because only dropMarker removes a state, under its file lock.+func (d *Durability) lockState(path string) *refState {+ for {+ d.mu.Lock()+ st := d.stateLocked(path)+ d.mu.Unlock()+ st.file.Lock()+ d.mu.Lock()+ current := d.repos[path] == st+ d.mu.Unlock()+ if current {+ return st+ }+ st.file.Unlock()+ }+}++// setDirtyLocked marks the state dirty. The caller holds d.mu.+func (d *Durability) setDirtyLocked(st *refState) {if !st.dirty {st.dirty = trued.dirty.Inc()}- return nil}// Begin records that a git process that writes the references of the@@ -505,7 +532,8 @@ func (d *Durability) MarkDirty(ctx context.Context, repo RepoRef, path string) e// once the process has exited. Until then the process may write a table or// compact the stack, whatever hooks git runs or skips (a failed prepare, a// failed commit, a kill), so the durable step keeps the marker. end may be-// called more than once.+// called more than once. A state with a live process never leaves the map+// (dropMarker), so end always counts down the state Begin counted up.func (d *Durability) Begin(path string) (end func()) {if d == nil {return func() {}@@ -522,15 +550,26 @@ func (d *Durability) Begin(path string) (end func()) {once.Do(func() {d.mu.Lock()defer d.mu.Unlock()- // Forget may have dropped the state; the count belongs to it.- if d.repos[path] == st {- st.nLive--- d.inflight.Dec()- }+ st.nLive--+ d.inflight.Dec()})}}+// Untrusted reports whether the local repository at path may have lost+// reference writes: it must not be served until it is moved aside and+// hydrated again.+func (d *Durability) Untrusted(path string) bool {+ if d == nil {+ return false+ }++ d.mu.Lock()+ defer d.mu.Unlock()+ st := d.repos[path]+ return st != nil && st.untrusted != nil+}+// Dirty returns the paths of the dirty repositories.func (d *Durability) Dirty() []string {if d == nil {@@ -576,13 +615,14 @@ func (d *Durability) CountStepError() {// durable. A failed fsync is an *UntrustedError (the caller moves the// repository aside at once); any other error is transient. When the repository// has been idle for the idle period, or final is set (the process stops), and-// no transaction of the process is live, it then removes the marker: the-// decision is taken under git's reference lock, taken without waiting (a busy-// lock leaves the marker for the next step), and the marker file is removed-// after the lock is released, under the state's file lock that a writer's-// MarkDirty waits for. A repository that no longer exists (moved aside as a-// stale cache) loses its marker. The caller holds the syncer's repository lock,-// so no syncer write and no replacement of the repository runs meanwhile.+// no git process of the process is live, it then takes git's reference lock+// without waiting (a busy lock leaves the marker for the next step), makes the+// stack durable again under it, and lets dropMarker decide. A repository that+// no longer exists (moved aside as a stale cache) has no stack, and its marker+// goes by the same rule: it stays while a git process that names the path is+// live. A clean state with no live process leaves the map. The caller holds the+// syncer's repository lock, so no syncer write and no replacement of the+// repository runs meanwhile.func (d *Durability) Settle(path string, final bool) error {if !d.Deferring() {return nil@@ -590,7 +630,7 @@ func (d *Durability) Settle(path string, final bool) error {d.mu.Lock()st := d.repos[path]- if st == nil || !st.dirty {+ if st == nil {d.mu.Unlock()return nil}@@ -601,34 +641,33 @@ func (d *Durability) Settle(path string, final bool) error {d.mu.Unlock()return &UntrustedError{Repo: repo, Path: path, Err: untrusted}}- now := time.Now()- idle := st.nLive == 0 && now.After(st.settleAfter) && (final || now.Sub(st.lastWrite) >= d.idle)- recovered := st.recovered+ dirty := st.dirty+ due := st.nLive == 0 && (final || time.Since(st.lastWrite) >= d.idle)d.mu.Unlock()- if idle && recovered {- // A git process of the previous owner may still be writing.- active, err := gitActive(path)- if err != nil {- return err- }- idle = !active- }+ exists := trueif _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) {- st.file.Lock()+ exists = false+ } else if err != nil {+ return fmt.Errorf("stat the repository: %w", err)+ }+ if !dirty || !exists {+ // A repository that is gone has no stack; a clean state only leaves the+ // map. A transaction that marked an existing repository meanwhile leaves+ // it to the next step, under git's reference lock (dropMarker refuses+ // otherwise).+ st := d.lockState(path)defer st.file.Unlock()- d.clear(st)- if err := d.removeMarker(st); err != nil {- return err- }- d.prune(path, st)- return nil+ _, err := d.dropMarker(path, st, false)+ return err}if err := d.syncStack(path, st); err != nil {- return d.stepError(st, repo, path, err)+ st := d.lockState(path)+ defer st.file.Unlock()+ return d.stepErrorLocked(st, repo, path, err)}- if !idle {+ if !due {return nil}@@ -639,49 +678,113 @@ func (d *Durability) Settle(path string, final bool) error {if !ok {return nil}-- cleared, err := d.settleLocked(path, st)- if err != nil {- _ = lock.release()- return d.stepError(st, repo, path, err)- }- if !cleared {- return lock.release()+ err = d.settleLocked(path, repo)+ if releaseErr := lock.release(); err == nil {+ err = releaseErr}+ return err+}- // The state is clean from here on, and a writer that takes git's lock now- // writes a new marker: it waits for the file lock, held until the old- // marker is removed.+// settleLocked makes the stack durable under git's reference lock, which the+// caller holds (no transaction commits meanwhile), and removes the marker when+// dropMarker finds it safe. The marker file goes before git's lock is+// released, under the state's file lock, which a writer's MarkDirty waits for.+func (d *Durability) settleLocked(path string, repo RepoRef) error {+ st := d.lockState(path)defer st.file.Unlock()- if err := lock.release(); err != nil {- return err++ if err := d.syncStack(path, st); err != nil {+ return d.stepErrorLocked(st, repo, path, err)}- if err := d.removeMarker(st); err != nil {- return err+ if d.crashAt == crashDurableStep {+ os.Exit(crashExitCode)}- d.prune(path, st)- return nil+ _, err := d.dropMarker(path, st, true)+ return err}-// prune drops the state of a repository that is clean and has no live git-// process, so that the states do not grow with every repository ever written.-// The caller holds the state's file lock: MarkDirty takes the state again from-// the map after its own.-func (d *Durability) prune(path string, st *refState) {+// dropMarker is the only code that removes a dirty marker or drops a state+// from the map. It removes the marker of the repository at path only when+// nothing can still leave a reference write that is not durable in its stack:+//+// - no git process of this process that writes the references without fsync+// is live (nLive);+// - the repository is not untrusted;+// - when the repository exists, the caller holds git's reference lock+// (ownLock) and made the stack durable under it (settleLocked): no git+// process can write meanwhile, and the ones that exited before are in what+// was made durable. A repository that does not exist has no stack, and a+// git process cannot write it (the caller holds the syncer's repository+// lock, so it is not hydrated meanwhile);+// - no git process writes the stack now (gitActive: a git lock or a table+// being written; seeing one also pushes settleAfter a recovery grace+// forward), and settleAfter has passed.+//+// The caller holds the state's file lock, which MarkDirty waits for before it+// reads whether the repository is dirty. The checks and the state change are+// made under d.mu, which Begin takes. It reports whether the marker was+// removed. A clean state that no live git process counts on and that is+// trusted leaves the map, so that the states do not grow with every repository+// ever written.+func (d *Durability) dropMarker(path string, st *refState, ownLock bool) (bool, error) {+ exists := true+ if _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) {+ exists = false+ } else if err != nil {+ return false, fmt.Errorf("stat the repository: %w", err)+ }+ active := false+ if exists {+ var err error+ if active, err = gitActive(path, ownLock); err != nil {+ return false, err+ }+ }++ d.mu.Lock()+ now := time.Now()+ if active {+ if until := now.Add(d.recoveryGrace); until.After(st.settleAfter) {+ st.settleAfter = until+ }+ }+ safe := st.dirty && st.nLive == 0 && st.untrusted == nil && (ownLock || !exists) && !active && !now.Before(st.settleAfter)+ if safe {+ st.dirty = false+ d.dirty.Dec()+ st.synced = map[string]struct{}{}+ }+ markerPath := st.marker+ d.mu.Unlock()++ if safe && markerPath != "" {+ if err := os.Remove(markerPath); err != nil && !errors.Is(err, fs.ErrNotExist) {+ d.mu.Lock()+ d.setDirtyLocked(st)+ d.mu.Unlock()+ return false, fmt.Errorf("remove the dirty marker: %w", err)+ }+ }+ if safe {+ d.cleared.Inc()+ }+d.mu.Lock()defer d.mu.Unlock()- if d.repos[path] == st && !st.dirty && st.untrusted == nil && st.nLive == 0 {+ if d.repos[path] == st && !st.dirty && st.nLive == 0 && st.untrusted == nil {delete(d.repos, path)}+ return safe, nil}-// stepError is the error of a durable step: an *UntrustedError for a failed-// fsync, which makes the repository untrusted for good, the error itself-// otherwise. The first failure is written into the marker, so that the+// stepErrorLocked is the error of a durable step: an *UntrustedError for a+// failed fsync, which makes the repository untrusted for good, the error+// itself otherwise. The first failure is written into the marker, so that the// repository stays untrusted across a restart of the process (the next fsync// would succeed); when even that write fails, the disk is failing and the-// state of this process keeps it untrusted.-func (d *Durability) stepError(st *refState, repo RepoRef, path string, err error) error {+// state of this process keeps it untrusted. The caller holds the state's file+// lock.+func (d *Durability) stepErrorLocked(st *refState, repo RepoRef, path string, err error) error {var writeback writebackErrorif !errors.As(err, &writeback) {return err@@ -693,32 +796,42 @@ func (d *Durability) stepError(st *refState, repo RepoRef, path string, err errost.untrusted = err}markerPath := st.marker+ dirty := st.dirtyd.mu.Unlock()- if first && markerPath != "" {- st.file.Lock()+ if first && dirty && markerPath != "" {content, encodeErr := json.Marshal(marker{RepoRef: repo, BootID: d.bootID, Untrusted: err.Error()})if encodeErr == nil {encodeErr = d.writeDurably(markerPath, content)}- st.file.Unlock()if encodeErr != nil {return &UntrustedError{Repo: repo, Path: path, Err: fmt.Errorf("%w (recording it in the marker failed: %w)", err, encodeErr)}}+ d.mu.Lock()+ st.markerPlain = false+ d.mu.Unlock()}return &UntrustedError{Repo: repo, Path: path, Err: err}}// gitActive reports whether a git process may be writing the reftable stack of-// the repository at path: git's lock of the list, a lock of a table being-// compacted, or a table being written is there.-func gitActive(path string) (bool, error) {+// the repository at path: git's lock of the list (unless ownLock: the caller+// holds it), a lock of a table being compacted, or a table being written is+// there (the names of git's reftable/stack.c). A repository without a reftable+// directory has no stack to write.+func gitActive(path string, ownLock bool) (bool, error) {entries, err := os.ReadDir(filepath.Join(path, "reftable"))+ if errors.Is(err, fs.ErrNotExist) {+ return false, nil+ }if err != nil {return false, fmt.Errorf("read the reftable directory: %w", err)}for _, entry := range entries {name := entry.Name()+ if ownLock && name == "tables.list.lock" {+ continue+ }if strings.HasSuffix(name, ".lock") || strings.Contains(name, ".temp.") {return true, nil}@@ -726,55 +839,6 @@ func gitActive(path string) (bool, error) {return false, nil}-// settleLocked makes the tables written since the first pass durable and, when-// no transaction of the process is live, marks the state clean with its file-// lock held (the caller releases it). The caller holds git's reference lock: no-// transaction can take it meanwhile, and the live transactions are counted from-// before they take it until their end.-func (d *Durability) settleLocked(path string, st *refState) (bool, error) {- d.mu.Lock()- busy := st.nLive > 0 || st.untrusted != nil- d.mu.Unlock()- if busy {- return false, nil- }-- if err := d.syncStack(path, st); err != nil {- return false, err- }- if d.crashAt == crashDurableStep {- os.Exit(crashExitCode)- }-- st.file.Lock()- d.clear(st)- return true, nil-}--// clear marks the state clean. The caller holds the state's file lock.-func (d *Durability) clear(st *refState) {- d.mu.Lock()- defer d.mu.Unlock()- if st.dirty {- st.dirty = false- d.cleared.Inc()- d.dirty.Dec()- }- st.synced = map[string]struct{}{}-}--// removeMarker removes the marker file of a clean state. The caller holds the-// state's file lock.-func (d *Durability) removeMarker(st *refState) error {- if st.marker == "" {- return nil- }- if err := os.Remove(st.marker); err != nil && !errors.Is(err, fs.ErrNotExist) {- return fmt.Errorf("remove the dirty marker: %w", err)- }- return nil-}-// stackLock is git's reference lock of a repository, taken by the durable// step.type stackLock struct {@@ -939,9 +1003,34 @@ func (d *Durability) syncStack(path string, st *refState) error {return nil}-// Forget drops the state and the marker of the repository at path, which the-// caller moved aside under the syncer's repository lock. A writer that still-// writes to the moved repository writes to a directory that is removed.+// Tracked returns the paths of the repositories that have a state: the dirty+// ones and the clean ones that a live git process still counts on. The durable+// steps visit them all, so that a clean state leaves the map once its last+// process ended.+func (d *Durability) Tracked() []string {+ if d == nil {+ return nil+ }++ d.mu.Lock()+ defer d.mu.Unlock()+ paths := make([]string, 0, len(d.repos))+ for path := range d.repos {+ paths = append(paths, path)+ }+ return paths+}++// Forget hands the path of a repository over to what comes there next: the+// caller moved the local repository aside under the syncer's repository lock,+// or, after a kernel restart, decided to keep in place a repository that is not+// a cache of the storage core. The state stops being untrusted, since what made+// it so is no longer at path, and the marker goes by dropMarker's rule. A git+// process of this process that is still live names the repository by its path,+// so its later writes land in what is there now (the hydrated repository, say):+// the marker then stays, plain (without the untrusted record, so that a restart+// does not move the new repository aside), until the process exited and a+// durable step made the stack durable.func (d *Durability) Forget(ctx context.Context, repo RepoRef, path string) error {if d == nil {return nil@@ -952,56 +1041,74 @@ func (d *Durability) Forget(ctx context.Context, repo RepoRef, path string) erroreturn err}+ st := d.lockState(path)+ defer st.file.Unlock()+d.mu.Lock()- st := d.repos[path]+ // A plain marker of this boot is on disk when the state was dirty with one;+ // otherwise one is written if the marker has to stay.+ plainOnDisk := st.dirty && st.markerPlain && st.marker == markerPath+ st.repo = repo+ st.marker = markerPath+ d.setDirtyLocked(st)+ st.untrusted = nil+ st.synced = map[string]struct{}{}d.mu.Unlock()- if st != nil {- st.file.Lock()- defer st.file.Unlock()- }+ dropped, err := d.forgetLocked(path, st, repo)d.mu.Lock()- live := st != nil && d.repos[path] == st && st.nLive > 0+ untrusted := st.untrusted != nild.mu.Unlock()- if live {- // A git process that writes without fsync is still running. It names the- // repository by its path, so its later writes land in what is there now- // (the hydrated repository, say): the repository stays dirty, with a plain- // marker, until it exited and a durable step made the stack durable.- content, err := json.Marshal(marker{RepoRef: repo, BootID: d.bootID})- if err != nil {- return fmt.Errorf("encode the dirty marker: %w", err)- }- if err := d.writeDurably(markerPath, content); err != nil {- return fmt.Errorf("write the dirty marker: %w", err)- }- d.mu.Lock()- defer d.mu.Unlock()- if !st.dirty {- st.dirty = true- d.dirty.Inc()- }- st.repo = repo- st.marker = markerPath- st.untrusted = nil- st.recovered = false- st.synced = map[string]struct{}{}- return nil+ if dropped || plainOnDisk || untrusted {+ // An untrusted state (a failed fsync of a repository kept in place) has+ // its record in the marker, or is kept untrusted by this process.+ return err}- if err := os.Remove(markerPath); err != nil && !errors.Is(err, fs.ErrNotExist) {- return fmt.Errorf("remove the dirty marker: %w", err)+ content, encodeErr := json.Marshal(marker{RepoRef: repo, BootID: d.bootID})+ if encodeErr == nil {+ encodeErr = d.writeDurably(markerPath, content)}-d.mu.Lock()defer d.mu.Unlock()- if st != nil && d.repos[path] == st {- if st.dirty {- d.dirty.Dec()+ if encodeErr != nil {+ // The state is dirty, so a writer's MarkDirty would not write the+ // marker: without one on disk, the repository at path cannot be+ // trusted. It is moved aside again, and Forget retried.+ if st.untrusted == nil {+ st.untrusted = fmt.Errorf("write the dirty marker: %w", encodeErr)}- delete(d.repos, path)+ return errors.Join(err, st.untrusted)}- return nil+ st.markerPlain = true+ return err+}++// forgetLocked lets dropMarker decide the marker of a forgotten repository.+// One kept in place is made durable under git's reference lock first, as a+// durable step does; when the lock is busy the marker stays for the next step.+// The caller holds the state's file lock.+func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (bool, error) {+ if _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) {+ return d.dropMarker(path, st, false)+ } else if err != nil {+ return false, fmt.Errorf("stat the repository: %w", err)+ }++ lock, ok, err := lockStack(path)+ if err != nil || !ok {+ return false, err+ }+ dropped := false+ if err = d.syncStack(path, st); err != nil {+ err = d.stepErrorLocked(st, repo, path, err)+ } else {+ dropped, err = d.dropMarker(path, st, true)+ }+ if releaseErr := lock.release(); err == nil {+ err = releaseErr+ }+ return dropped, err}// Recovered is a dirty repository found by Recover.@@ -1018,14 +1125,16 @@ type Recovered struct {}// Recover reads the dirty markers of the storages; only the owner calls it,-// when it takes the ownership. A marker of the running boot makes its-// repository dirty again, to be settled once no git process of the previous-// owner can still write it (gitActive, and the recovery grace), or untrusted-// again when the marker says so; what a durable step of the previous owner+// when it takes the ownership. Every marker makes its repository dirty again.+// One of the running boot is settled once no git process of the previous owner+// can still write the stack (dropMarker: gitActive, and the recovery grace),+// unless the marker records it untrusted; one of another boot makes the+// repository untrusted (errReboot). What a durable step of the previous owner// left (its reference lock, its temporary files) is removed. It returns every// repository found; the caller moves aside the ones of another boot and the-// untrusted ones before serving. A marker that cannot be read is an error: its-// repository is unknown, and serving could read a torn stack.+// untrusted ones before serving (Forget then hands their paths over). A marker+// that cannot be read is an error: its repository is unknown, and serving+// could read a torn stack.func (d *Durability) Recover(ctx context.Context, storages []string) ([]Recovered, error) {if d == nil || !d.ownsLocks() {return nil, nil@@ -1076,29 +1185,33 @@ func (d *Durability) Recover(ctx context.Context, storages []string) ([]Recoverereturn nil, fmt.Errorf("dirty marker %q: %w", markerPath, err)}+ if err := removeStaleStackLock(path); err != nil {+ return nil, err+ }+ if err := removeStepTemporaries(path); err != nil {+ return nil, err+ }+reboot := m.BootID != d.bootID+ st := d.lockState(path)+ d.mu.Lock()+ d.setDirtyLocked(st)+ st.repo = m.RepoRef+ st.marker = markerPath+ st.markerPlain = !reboot && m.Untrusted == ""+ switch {+ case m.Untrusted != "":+ st.untrusted = errors.New(m.Untrusted)+ case reboot:+ st.untrusted = errReboot+ }if !reboot {- if err := removeStaleStackLock(path); err != nil {- return nil, err- }- if err := removeStepTemporaries(path); err != nil {- return nil, err- }- d.mu.Lock()- st := d.stateLocked(path)- if !st.dirty {- st.dirty = true- d.dirty.Inc()- }- st.repo = m.RepoRef- st.marker = markerPath- st.recovered = true+ // A git process of the previous owner, of this boot, may still be+ // committing (dropMarker).st.settleAfter = time.Now().Add(d.recoveryGrace)- if m.Untrusted != "" {- st.untrusted = errors.New(m.Untrusted)- }- d.mu.Unlock()}+ d.mu.Unlock()+ st.file.Unlock()found = append(found, Recovered{Repo: m.RepoRef, Path: path, Reboot: reboot, Untrusted: m.Untrusted != ""})}}diff --git a/internal/graft/corestore/localrefs/durability_model_test.go b/internal/graft/corestore/localrefs/durability_model_test.gonew file mode 100644index 0000000..85c732d--- /dev/null+++ b/internal/graft/corestore/localrefs/durability_model_test.go@@ -0,0 +1,492 @@+package localrefs++import (+ "errors"+ "fmt"+ "io/fs"+ "math/rand"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++// TestMarkerLifecycleModel drives random interleavings of everything that+// touches a dirty marker against a reference model: transactions that mark the+// repository (MarkDirty), git processes that start and exit (Begin and end) and+// write tables without fsync, durable steps (Settle, final or not), failed+// fsyncs, the repository moved aside (Forget) and hydrated again, restarts of+// the process (Recover of the same boot, with git processes of the previous+// owner still finishing a write) and kernel restarts (Recover of another+// boot), and the acceptance in place of a repository after a kernel restart.+// After every step it asserts the two invariants of the LOCAL REFERENCE WRITES+// RULE (doc/graft/PLAN7.md):+//+// 1. while a git process may still write the stack without fsync (it marked+// the repository and has not exited, or it is a process of the previous+// owner in the middle of a write), or the stack at the path holds a table,+// a table list or a directory entry written without fsync since, the+// marker is on disk;+// 2. an untrusted repository is reported untrusted (the syncer never serves+// it) and keeps its marker, until it is moved aside or accepted.+//+// The model learns what is durable from the fsyncs the durability makes (its+// sync function), not from what the code under test reports. The steps are+// sequential: what git's reference lock excludes (a write while a durable step+// holds it) is not modelled; dropMarker refuses to remove the marker of an+// existing repository without that lock.+func TestMarkerLifecycleModel(t *testing.T) {+ t.Parallel()++ for seed := int64(1); seed <= 40; seed++ {+ t.Run(fmt.Sprint(seed), func(t *testing.T) {+ t.Parallel()+ runMarkerModel(t, seed, 250)+ })+ }+}++// TestDropMarkerGuards verifies the guards of dropMarker that no caller+// reaches in the model: the callers settle or move aside an untrusted+// repository first, and take git's reference lock for an existing one.+func TestDropMarkerGuards(t *testing.T) {+ t.Parallel()+ f := setupDurability(t)+ d := f.durability(t, "boot-a", 0)+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))++ st := d.lockState(f.path)+ defer st.file.Unlock()++ d.mu.Lock()+ st.untrusted = errors.New("input/output error")+ d.mu.Unlock()+ dropped, err := d.dropMarker(f.path, st, true)+ require.NoError(t, err)+ require.False(t, dropped, "an untrusted repository keeps its marker")++ d.mu.Lock()+ st.untrusted = nil+ d.mu.Unlock()+ dropped, err = d.dropMarker(f.path, st, false)+ require.NoError(t, err)+ require.False(t, dropped, "the marker of an existing repository goes only under git's reference lock")+ require.FileExists(t, f.markerPath(t, d))++ dropped, err = d.dropMarker(f.path, st, true)+ require.NoError(t, err)+ require.True(t, dropped)+ require.NoFileExists(t, f.markerPath(t, d))+}++// TestMovedRepositoryKeepsTheMarkerOfALiveProcess is the fourth review's+// sequence: a repository is moved aside while a git process that marked it+// still runs, the marker stays through a durable step of the missing path and+// through the hydration, so the process's later write is covered; it goes once+// the process exited and a step made the stack durable.+func TestMovedRepositoryKeepsTheMarkerOfALiveProcess(t *testing.T) {+ t.Parallel()+ f := setupDurability(t)+ d := f.durability(t, "boot-a", 0)+ template := filepath.Join(t.TempDir(), "template.git")+ copyTree(t, f.path, template)++ end := d.Begin(f.path)+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))+ require.NoError(t, os.Rename(f.path, filepath.Join(t.TempDir(), "aside")))+ require.NoError(t, d.Forget(f.ctx, f.repo, f.path))+ require.FileExists(t, f.markerPath(t, d))++ require.NoError(t, d.Settle(f.path, true))+ require.FileExists(t, f.markerPath(t, d), "the process names the path: a hydration there gets its writes")++ copyTree(t, template, f.path)+ require.NoError(t, d.Settle(f.path, true))+ require.FileExists(t, f.markerPath(t, d))++ end()+ require.NoError(t, d.Settle(f.path, true))+ require.NoFileExists(t, f.markerPath(t, d))+ require.Empty(t, d.Tracked())+}++// TestRecoveredMarkerOfAMissingRepositoryIsSettled verifies that a marker found+// at the recovery for a repository that no longer exists is removed by the+// next durable step, instead of failing on its missing stack at every step.+func TestRecoveredMarkerOfAMissingRepositoryIsSettled(t *testing.T) {+ t.Parallel()+ f := setupDurability(t)++ old := f.durability(t, "boot-a", 0)+ require.NoError(t, old.MarkDirty(f.ctx, f.repo, f.path))+ require.NoError(t, old.Close())+ require.NoError(t, os.Rename(f.path, filepath.Join(t.TempDir(), "aside")))++ next := f.durability(t, "boot-a", 0)+ _, err := next.Recover(f.ctx, []string{f.repo.Storage})+ require.NoError(t, err)+ require.NoError(t, next.Settle(f.path, false))+ require.NoFileExists(t, f.markerPath(t, next))+ require.Empty(t, next.Tracked())+}++// TestGitActivityPushesTheGraceForward verifies that the recovery grace counts+// from the last git activity seen in the stack, not from the recovery: a git+// process of the previous owner may start its auto-compaction right after its+// commit.+func TestGitActivityPushesTheGraceForward(t *testing.T) {+ t.Parallel()+ f := setupDurability(t)++ old := f.durability(t, "boot-a", 0)+ require.NoError(t, old.MarkDirty(f.ctx, f.repo, f.path))+ require.NoError(t, old.Close())++ next := f.durability(t, "boot-a", 0, WithRecoveryGrace(time.Hour))+ _, err := next.Recover(f.ctx, []string{f.repo.Storage})+ require.NoError(t, err)+ // The grace of the recovery has passed.+ next.mu.Lock()+ next.repos[f.path].settleAfter = time.Now().Add(-time.Second)+ next.mu.Unlock()++ temp := filepath.Join(f.path, "reftable", "0x000000000009-0x000000000009-abcdef01.ref.temp.XyZ123")+ require.NoError(t, os.WriteFile(temp, nil, 0o666))+ before := time.Now()+ require.NoError(t, next.Settle(f.path, true))+ require.FileExists(t, f.markerPath(t, next))++ require.NoError(t, os.Remove(temp))+ require.NoError(t, next.Settle(f.path, true))+ require.FileExists(t, f.markerPath(t, next), "the grace counts from the activity")+ next.mu.Lock()+ require.True(t, next.repos[f.path].settleAfter.After(before.Add(time.Hour-time.Minute)))+ next.mu.Unlock()+}++// modelProc is a git process of the model.+type modelProc struct {+ end func()+ marked bool+ // orphan is a process of an owner that restarted: it is not counted by the+ // running durability, and it only finishes the write it was in (inFlight,+ // the temporary table it writes).+ orphan bool+ inFlight string+}++type markerModel struct {+ t *testing.T+ f durabilityFixture+ rng *rand.Rand+ d *Durability+ boot int++ template string+ asideDir string+ reftable string++ procs []*modelProc+ // pending are the tables at the path written by the model's git processes+ // and not made durable since; listPending and dirPending the table list and+ // the directory entries.+ pending map[string]bool+ listPending bool+ dirPending bool+ // untrusted is set by a failed fsync and by a kernel restart with a marker,+ // rebootUntrusted only by the latter (the repository may then be accepted in+ // place).+ untrusted bool+ rebootUntrusted bool+ // failNext makes the fsyncs of the stack fail during the next durable step.+ failNext bool+ failing bool+ tables int+ moves int+}++func runMarkerModel(t *testing.T, seed int64, steps int) {+ f := setupDurability(t)+ m := &markerModel{+ t: t,+ f: f,+ rng: rand.New(rand.NewSource(seed)),+ template: filepath.Join(t.TempDir(), "template.git"),+ asideDir: t.TempDir(),+ reftable: filepath.Join(f.path, "reftable"),+ pending: map[string]bool{},+ }+ copyTree(t, f.path, m.template)+ m.d = m.newDurability()++ for step := 0; step < steps; step++ {+ op := m.step()+ m.check(fmt.Sprintf("seed %d step %d (%s)", seed, step, op))+ }+}++// sync is the durability's fsync: it fails the stack's files while failing is+// set, and tells the model which files are durable.+func (m *markerModel) sync(file *os.File) error {+ name := file.Name()+ inStack := name == m.reftable || filepath.Dir(name) == m.reftable+ if m.failing && inStack {+ return errors.New("input/output error")+ }+ if err := file.Sync(); err != nil {+ return err+ }+ switch {+ case name == m.reftable:+ m.dirPending = false+ case !inStack:+ case filepath.Base(name) == "tables.list":+ m.listPending = false+ default:+ delete(m.pending, filepath.Base(name))+ }+ return nil+}++func (m *markerModel) newDurability() *Durability {+ m.boot+++ d, err := NewDurability(m.f.locator, 0, WithBootID(fmt.Sprintf("boot-%d", m.boot)), WithSync(m.sync), WithRecoveryGrace(0))+ require.NoError(m.t, err)+ m.t.Cleanup(func() { require.NoError(m.t, d.Close()) })+ m.f.own(m.t, d)+ _, err = d.Recover(m.f.ctx, []string{m.f.repo.Storage})+ require.NoError(m.t, err)+ d.Activate()+ return d+}++func (m *markerModel) exists() bool {+ _, err := os.Lstat(m.f.path)+ if errors.Is(err, fs.ErrNotExist) {+ return false+ }+ require.NoError(m.t, err)+ return true+}++func (m *markerModel) live(match func(*modelProc) bool) []*modelProc {+ var procs []*modelProc+ for _, p := range m.procs {+ if match(p) {+ procs = append(procs, p)+ }+ }+ return procs+}++func (m *markerModel) remove(p *modelProc) {+ for i, q := range m.procs {+ if q == p {+ m.procs = append(m.procs[:i], m.procs[i+1:]...)+ return+ }+ }+}++func (m *markerModel) pick(procs []*modelProc) *modelProc {+ return procs[m.rng.Intn(len(procs))]+}++// step runs one random operation and returns its name.+func (m *markerModel) step() string {+ own := func(p *modelProc) bool { return !p.orphan }+ switch m.rng.Intn(14) {+ case 0, 1:+ if len(m.procs) < 4 {+ m.procs = append(m.procs, &modelProc{end: m.d.Begin(m.f.path)})+ return "begin"+ }+ case 2:+ unmarked := m.live(func(p *modelProc) bool { return own(p) && !p.marked })+ if len(unmarked) > 0 && m.exists() {+ require.NoError(m.t, m.d.MarkDirty(m.f.ctx, m.f.repo, m.f.path))+ m.pick(unmarked).marked = true+ return "mark"+ }+ case 3, 4:+ writers := m.live(func(p *modelProc) bool { return own(p) && p.marked })+ if len(writers) > 0 && m.exists() {+ m.writeTable(m.nextTable(), "")+ return "write"+ }+ case 5:+ procs := m.live(own)+ if len(procs) > 0 {+ p := m.pick(procs)+ p.end()+ m.remove(p)+ return "end"+ }+ case 6, 7:+ m.failing = m.failNext+ m.failNext = false+ err := m.d.Settle(m.f.path, m.rng.Intn(2) == 0)+ m.failing = false+ if errors.Is(err, ErrUntrusted) {+ m.untrusted = true+ } else {+ require.NoError(m.t, err)+ }+ return "settle"+ case 8:+ m.failNext = true+ return "fail the next fsyncs"+ case 9:+ if m.exists() {+ m.moves+++ require.NoError(m.t, os.Rename(m.f.path, filepath.Join(m.asideDir, fmt.Sprint(m.moves))))+ require.NoError(m.t, m.d.Forget(m.f.ctx, m.f.repo, m.f.path))+ m.pending = map[string]bool{}+ m.listPending, m.dirPending = false, false+ m.untrusted, m.rebootUntrusted = false, false+ return "move aside"+ }+ case 10:+ if !m.exists() {+ // A hydration writes the repository durably before it moves it in.+ copyTree(m.t, m.template, m.f.path)+ return "hydrate"+ }+ case 11:+ return m.restart()+ case 12:+ if orphans := m.live(func(p *modelProc) bool { return p.orphan }); len(orphans) > 0 {+ p := m.pick(orphans)+ if m.exists() && m.rng.Intn(4) != 0 {+ m.writeTable(strings.TrimSuffix(filepath.Base(p.inFlight), ".temp.XyZ123"), p.inFlight)+ }+ // Otherwise the process is killed: its temporary table stays.+ m.remove(p)+ return "orphan exits"+ }+ case 13:+ if m.rebootUntrusted && m.exists() {+ // The repository is not a cache of the storage core: kept in place.+ require.NoError(m.t, m.d.Forget(m.f.ctx, m.f.repo, m.f.path))+ m.pending = map[string]bool{}+ m.listPending, m.dirPending = false, false+ m.untrusted, m.rebootUntrusted = false, false+ return "accept in place"+ }+ return m.reboot()+ }+ return "nothing"+}++func (m *markerModel) nextTable() string {+ m.tables+++ return fmt.Sprintf("0x%012x-0x%012x-model%04d.ref", 100+m.tables, 100+m.tables, m.tables)+}++// writeTable writes a table and lists it, as git does, without fsync. A table+// written through a temporary file (temp) is renamed into place.+func (m *markerModel) writeTable(name, temp string) {+ table := filepath.Join(m.reftable, name)+ if temp != "" {+ if err := os.Rename(temp, table); errors.Is(err, fs.ErrNotExist) {+ // The repository it was writing was moved aside meanwhile.+ return+ } else {+ require.NoError(m.t, err)+ }+ } else {+ require.NoError(m.t, os.WriteFile(table, []byte(name), 0o666))+ }+ list := filepath.Join(m.reftable, "tables.list")+ content, err := os.ReadFile(list)+ require.NoError(m.t, err)+ require.NoError(m.t, os.WriteFile(list+".model", append(content, []byte(name+"\n")...), 0o666))+ require.NoError(m.t, os.Rename(list+".model", list))+ m.pending[name] = true+ m.listPending, m.dirPending = true, true+}++// restart restarts the process in the same boot: its git processes that may+// write become orphans, half of them in the middle of a table write.+func (m *markerModel) restart() string {+ require.NoError(m.t, m.d.Close())+ var orphans []*modelProc+ for _, p := range m.procs {+ if p.orphan || !p.marked || !m.exists() || m.rng.Intn(2) == 0 {+ continue+ }+ temp := filepath.Join(m.reftable, m.nextTable()+".temp.XyZ123")+ require.NoError(m.t, os.WriteFile(temp, []byte("in flight"), 0o666))+ orphans = append(orphans, &modelProc{orphan: true, inFlight: temp})+ }+ m.procs = orphans+ m.boot--+ m.d = m.newDurability()+ return "restart"+}++// reboot restarts the kernel: every process dies, and a repository with a+// marker becomes untrusted (its writes may never have reached the disk).+func (m *markerModel) reboot() string {+ require.NoError(m.t, m.d.Close())+ _, err := os.Stat(m.f.markerPath(m.t, m.d))+ marked := err == nil+ m.procs = nil+ if marked {+ m.untrusted = true+ m.rebootUntrusted = true+ }+ m.d = m.newDurability()+ return "reboot"+}++// check asserts the invariants.+func (m *markerModel) check(where string) {+ _, err := os.Stat(m.f.markerPath(m.t, m.d))+ markerOnDisk := err == nil++ // A process of the previous owner may still write only while its temporary+ // table is there: git renames it into the stack by its path.+ mayWrite := len(m.live(func(p *modelProc) bool {+ if p.orphan {+ _, err := os.Stat(p.inFlight)+ return err == nil+ }+ return p.marked+ })) > 0+ unsynced := m.exists() && (len(m.pending) > 0 || m.listPending || m.dirPending)+ if mayWrite || unsynced {+ require.True(m.t, markerOnDisk, "%s: a write may not be durable (writers %t, pending %v, list %t, dir %t) but the marker is gone",+ where, mayWrite, m.pending, m.listPending, m.dirPending)+ }+ require.Equal(m.t, m.untrusted, m.d.Untrusted(m.f.path), "%s: untrusted", where)+ if m.untrusted {+ require.True(m.t, markerOnDisk, "%s: an untrusted repository lost its marker", where)+ }+}++// copyTree copies the directory src to dst.+func copyTree(t *testing.T, src, dst string) {+ t.Helper()+ require.NoError(t, filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ rel, err := filepath.Rel(src, path)+ if err != nil {+ return err+ }+ target := filepath.Join(dst, rel)+ if entry.IsDir() {+ return os.MkdirAll(target, 0o777)+ }+ content, err := os.ReadFile(path)+ if err != nil {+ return err+ }+ return os.WriteFile(target, content, 0o666)+ }))+}diff --git a/internal/graft/corestore/localrefs/durability_test.go b/internal/graft/corestore/localrefs/durability_test.goindex 974714c..e13f786 100644--- a/internal/graft/corestore/localrefs/durability_test.go+++ b/internal/graft/corestore/localrefs/durability_test.go@@ -585,14 +585,19 @@ func TestRecoverTellsAKernelRestartFromAProcessRestart(t *testing.T) {require.NoError(t, sameBoot.Settle(f.path, false))require.NoFileExists(t, f.markerPath(t, sameBoot))- // Another boot: the writes may not have reached the disk.+ // Another boot: the writes may not have reached the disk. The repository is+ // untrusted until the caller moves it aside (Forget): never served, its+ // marker never removed by a durable step.require.NoError(t, sameBoot.MarkDirty(f.ctx, f.repo, f.path))require.NoError(t, sameBoot.Close())otherBoot := f.durability(t, "boot-b", 0)found, err = otherBoot.Recover(f.ctx, storages)require.NoError(t, err)require.Equal(t, []Recovered{{Repo: f.repo, Path: f.path, Reboot: true}}, found)- require.Empty(t, otherBoot.Dirty())+ require.Equal(t, []string{f.path}, otherBoot.Dirty())+ require.True(t, otherBoot.Untrusted(f.path))+ require.ErrorIs(t, otherBoot.Settle(f.path, true), ErrUntrusted)+ require.FileExists(t, f.markerPath(t, otherBoot))}// TestRecoverRemovesWhatADurableStepThatDiedLeft verifies that the referencediff --git a/internal/graft/corestore/maintenance/maintenance.go b/internal/graft/corestore/maintenance/maintenance.goindex db09ec5..3f79212 100644--- a/internal/graft/corestore/maintenance/maintenance.go+++ b/internal/graft/corestore/maintenance/maintenance.go@@ -245,17 +245,10 @@ func (r *Runner) maintain(ctx context.Context, record corestore.Repository) (strreturn "", nil}- leaseCtx, cancel := context.WithCancel(ctx)- renewed := r.renew(leaseCtx, cancel, record.ID)- defer func() {- cancel()- <-renewed- releaseCtx, cancelRelease := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)- defer cancelRelease()- if err := r.core.Meta.ReleaseMaintenanceLease(releaseCtx, record.ID, r.holder); err != nil {- r.logger.WithError(err).WithField("repository_id", record.ID).Warn("storage core: release the maintenance lease; it expires by itself")- }- }()+ leaseCtx, release := HoldLease(ctx, r.core, r.logger, record.ID, r.holder, r.cfg.LeaseTTL, func() {+ r.failures.WithLabelValues(stepLease).Inc()+ })+ defer release()due, err := r.checkpointDue(leaseCtx, record)if err != nil {@@ -282,36 +275,49 @@ func (r *Runner) maintain(ctx context.Context, record corestore.Repository) (strreturn "", nil}-// renew renews the lease of repoID every third of its time to live until ctx-// ends, and cancels the run (cancel) when the lease is lost: another process-// may then do the same work, which is wasted but correct. The returned channel-// is closed when it stops.-func (r *Runner) renew(ctx context.Context, cancel context.CancelFunc, repoID int64) <-chan struct{} {+// HoldLease keeps the maintenance lease of repoID, which holder acquired with+// time to live ttl, for the work done under the returned context: it renews the+// lease every third of ttl, and when the lease is lost it calls lost and+// cancels the context (another process may then do the same work, which is+// wasted but correct). release stops the renewal, waits for it, and releases+// the lease; the background maintenance and the CollectGarbage request use it+// alike.+func HoldLease(ctx context.Context, core *corestore.Core, logger log.Logger, repoID int64, holder string, ttl time.Duration, lost func()) (context.Context, func()) {+ leaseCtx, cancel := context.WithCancel(ctx)done := make(chan struct{})go func() {defer close(done)- ticker := time.NewTicker(r.cfg.LeaseTTL / 3)+ ticker := time.NewTicker(ttl / 3)defer ticker.Stop()for {select {- case <-ctx.Done():+ case <-leaseCtx.Done():returncase <-ticker.C:}- held, err := r.core.Meta.RenewMaintenanceLease(ctx, repoID, r.holder, r.cfg.LeaseTTL)- if ctx.Err() != nil {+ held, err := core.Meta.RenewMaintenanceLease(leaseCtx, repoID, holder, ttl)+ if leaseCtx.Err() != nil {return}if err != nil || !held {- r.logger.WithError(err).WithField("repository_id", repoID).WithField("held", held).- Warn("storage core: the maintenance lease was lost, the run of the repository stops")- r.failures.WithLabelValues(stepLease).Inc()+ logger.WithError(err).WithField("repository_id", repoID).WithField("held", held).+ Warn("storage core: the maintenance lease was lost, the work on the repository stops")+ lost()cancel()return}}}()- return done++ return leaseCtx, func() {+ cancel()+ <-done+ releaseCtx, cancelRelease := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)+ defer cancelRelease()+ if err := core.Meta.ReleaseMaintenanceLease(releaseCtx, repoID, holder); err != nil {+ logger.WithError(err).WithField("repository_id", repoID).Warn("storage core: release the maintenance lease; it expires by itself")+ }+ }}// checkpointDue reports whether the repository needs a checkpoint: enoughdiff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.goindex c45be6d..5f4b418 100644--- a/internal/graft/corestore/maintenance/maintenance_test.go+++ b/internal/graft/corestore/maintenance/maintenance_test.go@@ -6,6 +6,7 @@ import ("crypto/sha256""fmt""os"+ "sync/atomic""testing""time"@@ -329,6 +330,38 @@ func TestRunOnceStopsWhenTheLeaseIsLost(t *testing.T) {require.Equal(t, float64(1), e.metric("graft_storage_maintenance_runs_total", "error"))}+// TestHoldLeaseRenewsUntilReleased verifies the lease the background+// maintenance and the CollectGarbage request hold while they work: no other+// holder takes it for several times its time to live, and it is free once+// released.+func TestHoldLeaseRenewsUntilReleased(t *testing.T) {+ t.Parallel()+ ctx := testhelper.Context(t)+ e := newEnv(t, coretest.Options{}, config3(0, 0))+ const ttl = 300 * time.Millisecond+ const holder = "node-a/1/request-1"++ acquired, err := e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, holder, ttl)+ require.NoError(t, err)+ require.True(t, acquired)+ var lost atomic.Bool+ leaseCtx, release := maintenance.HoldLease(ctx, e.core, testhelper.NewLogger(t), e.repo.ID, holder, ttl, func() { lost.Store(true) })++ require.Never(t, func() bool {+ acquired, err := e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, "node-b/1", ttl)+ require.NoError(t, err)+ return acquired+ }, 5*ttl, ttl/10, "the lease is renewed")+ require.NoError(t, leaseCtx.Err())+ require.False(t, lost.Load())++ release()+ require.ErrorIs(t, leaseCtx.Err(), context.Canceled)+ acquired, err = e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, "node-b/1", ttl)+ require.NoError(t, err)+ require.True(t, acquired, "released")+}+func (e *env) bucketKeys() map[string]bool {e.t.Helper()keys := map[string]bool{}diff --git a/internal/graft/corestore/syncer/durability.go b/internal/graft/corestore/syncer/durability.goindex f428854..e9795d1 100644--- a/internal/graft/corestore/syncer/durability.go+++ b/internal/graft/corestore/syncer/durability.go@@ -4,11 +4,13 @@ import ("context""errors""fmt"+ "io/fs""os""path/filepath""sync""time"+ "github.com/prometheus/client_golang/prometheus""google.golang.org/grpc/codes""google.golang.org/grpc/status"@@ -41,6 +43,8 @@ func (s *Syncer) StartLocalReferences(ctx context.Context, storages []string) (sif !d.Enabled() {return func() {}, nil}+ kept := append([]string(nil), storages...)+ s.keptStorages.Store(&kept)owned, err := d.TryOwn(ctx, storages)if err != nil {@@ -73,7 +77,7 @@ func (s *Syncer) StartLocalReferences(ctx context.Context, storages []string) (s}finalCtx, cancelFinal := context.WithTimeout(context.WithoutCancel(ctx), finalStepTimeout)defer cancelFinal()- for _, path := range d.Dirty() {+ for _, path := range d.Tracked() {s.durableStep(finalCtx, path, true)}}, nil@@ -111,7 +115,7 @@ func (s *Syncer) runDurableSteps(ctx context.Context, storages []string, recovers.logger.Info("storage core: this process owns the deferred durability of the storages now")}- for _, path := range d.Dirty() {+ for _, path := range d.Tracked() {s.durableStep(ctx, path, false)}}@@ -131,8 +135,9 @@ func (s *Syncer) runDurableSteps(ctx context.Context, storages []string, recover//// Only a cache of the storage core is moved (storageCoreCache). Any other// marker of another boot (the repository was removed, or replaced by one being-// imported, whose local copy may be the only one) is dropped; an untrusted one-// is kept, with its repository untrusted, and logged.+// imported, whose local copy may be the only one) is handed over (Forget); an+// untrusted one is kept, with its repository untrusted, so never served, and+// logged.func (s *Syncer) RecoverLocalReferences(ctx context.Context, storages []string) error {d := s.core.RefDurabilityfound, err := d.Recover(ctx, storages)@@ -157,12 +162,12 @@ func (s *Syncer) RecoverLocalReferences(ctx context.Context, storages []string)if rec.Reboot {s.logger.WithField("repository_path", rec.Path).Warn("storage core: dropped the dirty marker of a local repository that is not a cache of the storage core")- if err := d.Forget(ctx, rec.Repo, rec.Path); err != nil {+ if err := s.forget(ctx, rec.Repo, rec.Path); err != nil {return err}} else {s.logger.WithField("repository_path", rec.Path).- Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place, untrusted")+ Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place, untrusted, and not served")}continue}@@ -177,11 +182,13 @@ func (s *Syncer) RecoverLocalReferences(ctx context.Context, storages []string)// storageCoreCache reports whether the local repository at path is a cache of// the storage core that a hydration can rebuild: the database holds it with// changesets (an import that is not complete has none, and its local copy may-// be the only one). keep reports that the copy holds history written in shadow-// mode and not imported yet (checkReplaceable, the guard of a sync's+// be the only one). keep reports that the copy may hold history written in+// shadow mode and not imported yet (checkReplaceable, the guard of a sync's// hydration): the copy is moved aside all the same, so that it is never served,-// but kept, so that the history is not lost. An error (the database, or the-// repository's files, cannot be read) is returned: the caller tries again.+// but kept, so that the history is not lost. A local changelog or review file+// that cannot be read (Internal) means keep too: whether it holds such history+// is unknown, and a corrupt file must not stop the node. An error of the+// database is returned: the caller tries again.func (s *Syncer) storageCoreCache(ctx context.Context, repo localrefs.RepoRef, path string) (cache, keep bool, err error) {r, err := s.core.Meta.GetRepository(ctx, corestore.RepoKey{Storage: repo.Storage, RelativePath: repo.RelativePath})if errors.Is(err, corestore.ErrRepositoryNotFound) {@@ -202,7 +209,12 @@ func (s *Syncer) storageCoreCache(ctx context.Context, repo localrefs.RepoRef, pwm, ok := s.readWatermark(path)if err := s.checkReplaceable(ctx, r, path, wm, ok); err != nil {- if status.Code(err) == codes.FailedPrecondition {+ switch status.Code(err) {+ case codes.FailedPrecondition:+ return true, true, nil+ case codes.Internal:+ s.logger.WithError(err).WithField("repository_path", path).+ Error("storage core: the history files of an untrusted local repository cannot be read; it is kept for an operator")return true, true, nil}return false, false, err@@ -210,6 +222,16 @@ func (s *Syncer) storageCoreCache(ctx context.Context, repo localrefs.RepoRef, preturn true, false, nil}+// forget is Forget under the repository lock.+func (s *Syncer) forget(ctx context.Context, repo localrefs.RepoRef, path string) error {+ release, err := s.lock(ctx, path)+ if err != nil {+ return err+ }+ defer release()+ return s.core.RefDurability.Forget(ctx, repo, path)+}+// durableStep makes the references of the repository at path durable under the// repository lock (final: without waiting for the repository to be idle). When// the kernel reports that a write failed, the repository is moved aside at@@ -253,9 +275,9 @@ func (s *Syncer) durableStep(ctx context.Context, path string, final bool) {return}if !cache {- // Its marker stays (the untrusted state is sticky): after a reboot the- // recovery decides again.- logger.Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place")+ // Its marker stays and it is not served (the untrusted state is sticky):+ // after a restart the recovery decides again.+ logger.Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place and not served")return}if err := s.distrustLocked(ctx, untrusted.Repo, path, localrefs.ReasonSyncFailed, untrusted, keep); err != nil {@@ -281,6 +303,72 @@ func (s *Syncer) distrust(ctx context.Context, repo localrefs.RepoRef, path, rea// operator imports what they hold or removes them.const KeptDirName = "+gitaly/graft-untrusted"+var (+ keptRepositoriesDesc = prometheus.NewDesc(+ "graft_storage_kept_untrusted_repositories",+ "Number of untrusted local repositories kept in "+KeptDirName+" for an operator, by storage: they hold history not imported yet (doc/graft/STORAGE.md, kept copies).",+ []string{"storage"}, nil)+ keptBytesDesc = prometheus.NewDesc(+ "graft_storage_kept_untrusted_bytes",+ "Bytes of the untrusted local repositories kept in "+KeptDirName+", by storage.",+ []string{"storage"}, nil)+)++// collectKept reports the kept untrusted copies of the storages. It reads the+// directories at every scrape: they are empty unless an operator has copies to+// import, which the metric is there to show. A storage whose directory cannot+// be read is logged and left out.+func (s *Syncer) collectKept(metrics chan<- prometheus.Metric) {+ storages := s.keptStorages.Load()+ if storages == nil {+ return+ }+ for _, storageName := range *storages {+ root, err := s.locator.GetStorageByName(context.Background(), storageName)+ if err != nil {+ s.logger.WithError(err).WithField("storage", storageName).Warn("storage core: measure the kept untrusted repositories")+ continue+ }+ repos, size, err := keptSize(filepath.Join(root, KeptDirName))+ if err != nil {+ s.logger.WithError(err).WithField("storage", storageName).Warn("storage core: measure the kept untrusted repositories")+ continue+ }+ metrics <- prometheus.MustNewConstMetric(keptRepositoriesDesc, prometheus.GaugeValue, float64(repos), storageName)+ metrics <- prometheus.MustNewConstMetric(keptBytesDesc, prometheus.GaugeValue, float64(size), storageName)+ }+}++// keptSize returns the number of kept repositories in dir (one directory each)+// and the bytes of their files.+func keptSize(dir string) (int, int64, error) {+ entries, err := os.ReadDir(dir)+ if errors.Is(err, fs.ErrNotExist) {+ return 0, 0, nil+ }+ if err != nil {+ return 0, 0, err+ }+ var size int64+ err = filepath.WalkDir(dir, func(_ string, entry fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if entry.Type().IsRegular() {+ info, err := entry.Info()+ if err != nil {+ return err+ }+ size += info.Size()+ }+ return nil+ })+ if err != nil {+ return 0, 0, err+ }+ return len(entries), size, nil+}+// distrustLocked moves the local repository at path out of the way, then drops// its dirty marker: the next request hydrates it. It goes into the temporary// directory of its storage and is removed there in the background, or, withdiff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.goindex d55a668..13936fc 100644--- a/internal/graft/corestore/syncer/syncer.go+++ b/internal/graft/corestore/syncer/syncer.go@@ -132,6 +132,10 @@ type Syncer struct {syncChangesets prometheus.CountersyncBytes prometheus.CountersyncSeeds prometheus.Counter++ // keptStorages are the storages whose kept untrusted copies (KeptDirName)+ // the metrics report, set by StartLocalReferences.+ keptStorages atomic.Pointer[[]string]}type repoLock struct {@@ -307,6 +311,8 @@ func (s *Syncer) Describe(descs chan<- *prometheus.Desc) {s.syncChangesets.Describe(descs)s.syncBytes.Describe(descs)s.syncSeeds.Describe(descs)+ descs <- keptRepositoriesDesc+ descs <- keptBytesDesc}// Collect implements prometheus.Collector.@@ -319,6 +325,7 @@ func (s *Syncer) Collect(metrics chan<- prometheus.Metric) {s.syncChangesets.Collect(metrics)s.syncBytes.Collect(metrics)s.syncSeeds.Collect(metrics)+ s.collectKept(metrics)}func (s *Syncer) syncTimeout() time.Duration {@@ -349,6 +356,12 @@ func (s *Syncer) sync(ctx context.Context, key corestore.RepoKey) error {if err := ctx.Err(); err != nil {return structerr.NewUnavailable("storage core: sync: %w", err)}+ // A repository whose references may have lost writes is never served:+ // the durable step moves a cache aside (the next request hydrates it), and+ // any other one waits for an operator.+ if s.core.RefDurability.Untrusted(path) {+ return structerr.NewUnavailable("storage core: the local references of %s may have lost writes; the repository is not served until it is rebuilt", key)+ }// The record is read again on every round: a restart can mean that the// repository was removed and registered again under a new ID meanwhile, anddiff --git a/internal/graft/corestore/syncer/syncer_localrefs_test.go b/internal/graft/corestore/syncer/syncer_localrefs_test.goindex a2e2a3b..8afc50a 100644--- a/internal/graft/corestore/syncer/syncer_localrefs_test.go+++ b/internal/graft/corestore/syncer/syncer_localrefs_test.go@@ -11,6 +11,8 @@ import ("github.com/prometheus/client_golang/prometheus""github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest"@@ -320,6 +322,75 @@ func TestSyncerUntrustedRepositoryWithShadowHistoryIsKept(t *testing.T) {require.Empty(t, e.markers(t))}+// TestSyncerNeverServesAnUntrustedRepository verifies that a repository whose+// references may have lost writes is not served while it is still in place+// (its move aside failed): every request is Unavailable until the next durable+// step moves it aside, and the request after that hydrates it.+func TestSyncerNeverServesAnUntrustedRepository(t *testing.T) {+ t.Parallel()++ var failing atomic.Bool+ e, repo, _ := caughtUpDirty(t, localrefs.WithSync(failReferences(&failing)))+ tmpDir, err := e.locator.TempDir(e.storageName)+ require.NoError(t, err)+ require.NoError(t, os.MkdirAll(tmpDir, 0o700))++ failing.Store(true)+ require.NoError(t, os.Chmod(tmpDir, 0o500))+ e.sy.DurableStep(e.ctx, e.localPath(), false)+ require.NoError(t, os.Chmod(tmpDir, 0o700))+ failing.Store(false)+ require.DirExists(t, e.localPath(), "the move aside failed")++ err = e.sy.EnsureFresh(e.ctx, e.pbRepo(localRelPath))+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)++ e.sy.DurableStep(e.ctx, e.localPath(), false)+ require.NoDirExists(t, e.localPath())+ require.NoError(t, e.sy.EnsureFresh(e.ctx, e.pbRepo(localRelPath)))+ e.verifySynced(t, repo)+}++// TestSyncerRecoveryKeepsACopyWhoseHistoryCannotBeRead verifies that a local+// changelog that cannot be read does not stop the recovery, so the node: the+// untrusted copy is moved aside and kept for an operator, as one with history+// not imported yet, and the kept-copies metrics report it.+func TestSyncerRecoveryKeepsACopyWhoseHistoryCannotBeRead(t *testing.T) {+ t.Parallel()++ e, repo, _ := caughtUpDirty(t, localrefs.WithBootID("boot-a"))+ // A directory where the changelog file is: reading it fails.+ require.NoError(t, os.MkdirAll(changelog.Path(e.localPath()), 0o700))++ coretest.EnableLocalRefs(t, e.core, e.cfg, localrefs.WithBootID("boot-b"))+ e.sy = syncer.New(e.core, e.logger, e.locator, e.repoFactory, e.gitCmdFactory)+ stop, err := e.sy.StartLocalReferences(e.ctx, []string{e.storageName})+ require.NoError(t, err)+ defer stop()++ require.NoDirExists(t, e.localPath(), "never served")+ kept, err := filepath.Glob(filepath.Join(e.storagePath, syncer.KeptDirName, "*", "repo.git"))+ require.NoError(t, err)+ require.Len(t, kept, 1)+ require.Empty(t, e.markers(t))++ reg := prometheus.NewPedanticRegistry()+ require.NoError(t, reg.Register(e.sy))+ families, err := reg.Gather()+ require.NoError(t, err)+ values := map[string]float64{}+ for _, family := range families {+ for _, metric := range family.GetMetric() {+ values[family.GetName()] = metric.GetGauge().GetValue()+ }+ }+ require.Equal(t, float64(1), values["graft_storage_kept_untrusted_repositories"])+ require.Positive(t, values["graft_storage_kept_untrusted_bytes"])++ require.NoError(t, e.sy.EnsureFresh(e.ctx, e.pbRepo(localRelPath)))+ e.verifySynced(t, repo)+}+// TestSyncerUntrustedRepositoryThatIsGoneLosesItsMarker verifies that an// untrusted repository that was moved away meanwhile (a stale cache) does not// keep its marker forever.