Skip to content

Git designed for AI agents.

Fourth review: one marker-removal rule, a model test, and the findings

Federico Elia70d4a2b97ba1
diff --git a/cmd/graft/storage.go b/cmd/graft/storage.go
index 842c89b..19ab142 100644
--- a/cmd/graft/storage.go
+++ b/cmd/graft/storage.go
@@ -7,6 +7,7 @@ import (
"flag"
"fmt"
"io"
+ "slices"
"unicode/utf8"
"github.com/feder-cr/git-ai/internal/graft/workspace"
@@ -20,7 +21,7 @@ const storageSynopsis = "usage: graft storage import|state|verify|checkpoint|gc
const storageUsage = "storage requires a subcommand: import, state, verify, checkpoint or gc (" + storageSynopsis + ")"
// storageFlags holds the flags of graft storage. Each subcommand reads only
-// some of them, as its help text says.
+// some of them, as its help text says, and rejects the others (checkUsed).
type storageFlags struct {
repo string
connectivity bool
@@ -57,6 +58,11 @@ func cmdStorage(ctx context.Context, out io.Writer, o *options, f *storageFlags,
return usagef("%s", storageUsage)
}
sub, rest := args[0], args[1:]
+ if used, known := storageSubcommandFlags[sub]; known {
+ if err := f.checkUsed(sub, used); err != nil {
+ return err
+ }
+ }
switch sub {
case "import":
return storageImport(ctx, out, o, f, rest)
@@ -72,6 +78,35 @@ func cmdStorage(ctx context.Context, out io.Writer, o *options, f *storageFlags,
return usagef("unknown storage subcommand %q: %s", sub, storageUsage)
}
+// storageSubcommandFlags are the subcommand-specific flags each subcommand
+// reads.
+var storageSubcommandFlags = map[string][]string{
+ "import": nil,
+ "state": nil,
+ "verify": {"connectivity"},
+ "checkpoint": nil,
+ "gc": {"dry-run", "removed"},
+}
+
+// checkUsed rejects a subcommand-specific flag that the subcommand sub does not
+// read: a flag that does nothing must not look accepted (checkpoint --dry-run
+// would create a checkpoint).
+func (f *storageFlags) checkUsed(sub string, used []string) error {
+ for _, flag := range []struct {
+ name string
+ set bool
+ }{
+ {"connectivity", f.connectivity},
+ {"dry-run", f.dryRun},
+ {"removed", f.removed},
+ } {
+ if flag.set && !slices.Contains(used, flag.name) {
+ return usagef("storage %s does not take --%s: %s", sub, flag.name, storageSynopsis)
+ }
+ }
+ return nil
+}
+
func storageCheckpoint(ctx context.Context, out io.Writer, o *options, f *storageFlags, args []string) error {
if len(args) != 0 {
return usagef("storage checkpoint takes flags only: checkpoint [-repo R] [--json]")
diff --git a/cmd/graft/storage_test.go b/cmd/graft/storage_test.go
index 4ccad23..633c350 100644
--- a/cmd/graft/storage_test.go
+++ b/cmd/graft/storage_test.go
@@ -177,6 +177,11 @@ func TestCLIStorageUsageErrorsDoNotContactServer(t *testing.T) {
"gc with arg": {"storage", "gc", "-C", dir, "extra"},
"gc removed and repo": {"storage", "gc", "--removed", "-C", dir, "-repo", "a.git"},
"gc removed without server": {"storage", "gc", "--removed", "-C", dir},
+ "import dry run": {"storage", "import", "--dry-run", "-C", dir},
+ "checkpoint dry run": {"storage", "checkpoint", "--dry-run", "-C", dir},
+ "state connectivity": {"storage", "state", "--connectivity", "-C", dir},
+ "verify removed": {"storage", "verify", "--removed", "-C", dir},
+ "checkpoint removed": {"storage", "checkpoint", "--removed", "-C", dir},
}
for name, args := range cases {
code, _, stderr := runCLI(t, args...)
diff --git a/doc/graft/AGENT-SECURITY.md b/doc/graft/AGENT-SECURITY.md
index 48ecdab..ea5ba18 100644
--- a/doc/graft/AGENT-SECURITY.md
+++ b/doc/graft/AGENT-SECURITY.md
@@ -57,7 +57,8 @@ These can never be enabled, not even through `graft.agent_rpcs`:
- `RemoteService` `UpdateRemoteMirror`, `FindRemoteRepository` and
`FindRemoteRootRef`, which push to or query a URL the caller chooses.
- `CleanupService`, `ObjectPoolService`, `HookService`, `InternalGitaly`,
- `PartitionService`.
+ `PartitionService`, and `StorageCoreService` (imports, checkpoints and
+ garbage collection, which deletes packs of the bucket).
Every other RPC that is not on the allowlist (for example the other
`OperationService` RPCs such as `UserMergeBranch`, or `SSHService`) is denied
diff --git a/doc/graft/LIMITATIONS.md b/doc/graft/LIMITATIONS.md
index a43c4e0..349a5d4 100644
--- a/doc/graft/LIMITATIONS.md
+++ b/doc/graft/LIMITATIONS.md
@@ -82,7 +82,7 @@ they are rejected with `PermissionDenied` and need the shared token:
- `RemoteService`: `UpdateRemoteMirror`, `FindRemoteRepository`,
`FindRemoteRootRef` (they push to or query a URL the caller chooses);
- every RPC of `HookService`, `ObjectPoolService`, `CleanupService`,
- `PartitionService` and `InternalGitaly`.
+ `PartitionService`, `InternalGitaly` and `StorageCoreService`.
`CreateRepositoryFromSnapshot` downloads a tar archive from a URL that the
client chooses and extracts it into the new repository without filtering it.
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index c376650..67b3154 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -657,11 +657,12 @@ repository-wide lock.
written, which a writing git process always has: `tables.list.lock`
(prepare to commit), `<table>.lock` (the tables a compaction merges),
`*.temp.*` (a table being written). On top of that there is a floor
- of 2 s (`RecoveryGrace`), for the instant between a commit's rename
- and the start of its auto-compaction.
+ of 2 s (`RecoveryGrace`) from the recovery and from the last git
+ activity seen, for the instant between a commit's rename and the
+ start of its auto-compaction.
* A marker that records an untrusted repository is handled like a
marker of another boot: the repository is moved aside if it is a
- cache, and its marker is never settled otherwise.
+ cache; otherwise its marker is never settled, and it is not served.
* Why the grace bound holds: when the previous owner dies, the stdin
of its `update-ref --stdin` children closes.
* A child that has not been sent `commit` aborts at end of input and
@@ -748,6 +749,61 @@ repository-wide lock.
* m-4: a test proves the end of a deferred process runs after it exited. The
mutation that swaps the two defers fails it.
* Clean states are dropped; MarkDirty takes its state from the map again.
+* Fourth review (of 32d0ac4 and 03ef6c2). Four reviews in a row found marker
+ defects in different branches (Settle, Forget, Recover). The root cause:
+ "a marker may go only when that is safe" was re-implemented branch by
+ branch. Fixed one level up:
+ * One function, `dropMarker`, removes markers and drops states. Every path
+ goes through it: Settle, the missing-path case, Forget, recovered markers,
+ pruning. It checks the whole condition under the state's file lock (which
+ MarkDirty waits for) and `d.mu` (which Begin takes):
+ * no live deferred git process (`nLive`);
+ * not untrusted;
+ * an existing repository only under git's reference lock, after its stack
+ was made durable under that lock. A missing repository has no stack, and
+ the syncer's repository lock keeps it from being hydrated meanwhile;
+ * no git activity in the stack (stack.c's lock and temporary names). Seeing
+ some pushes the 2 s floor forward.
+ * Finding 1 (MAJOR): the missing-path branch no longer removes a marker that
+ a live process needs (moved aside, then hydrated, then the orphan write).
+ * Finding 3: a recovered marker of a missing repository is settled. No
+ reftable directory means no activity.
+ * Finding 5: a state leaves the map only in `dropMarker`, in the same `d.mu`
+ section that checks `nLive`, so a Begin is never lost.
+ * Finding 6: the floor counts from the last activity seen.
+ * Forget rewrites a marker that is not a plain marker of the running boot.
+ The model test found this one: a marker of another boot, kept for a live
+ process, made the next restart distrust the hydrated copy.
+ * A marker of another boot makes the state untrusted (`errReboot`), so
+ nothing settles it before the move.
+ * Untrusted means never served: `EnsureFresh`, which the gate runs for every
+ repository RPC, answers `Unavailable` for an untrusted path. A repository
+ that is not a cache stays in place, unserved, for an operator.
+ * Finding 2:
+ * `+gitaly` is reserved (`corestore.CheckRelativePath`), in the gate for
+ every repository RPC and in every storage core RPC;
+ * STORAGE.md has a procedure for importing a kept copy;
+ * metrics `graft_storage_kept_untrusted_repositories` and `_bytes`.
+ * Finding 4: a local history file that cannot be read means keep (moved to
+ `graft-untrusted`), never a failed startup. Database errors still fail the
+ recovery and are retried.
+ * Finding 7: the downgrade is documented (STORAGE.md, Failures).
+ * Finding 8: `StorageCoreService` is never allowed for per-agent tokens.
+ * Finding 9: a flag that a storage subcommand does not read is a usage error.
+ * Finding 10: CollectGarbage holds its lease with the runner's renewal
+ (`maintenance.HoldLease`). A lease lost during the collection is `Aborted`.
+ * `TestMarkerLifecycleModel`: 40 seeds × 250 random steps.
+ * The steps: MarkDirty; Begin and end; table writes without fsync; Settle,
+ final or not; failed fsyncs; move aside and Forget; hydration; a process
+ restart with git processes of the previous owner in the middle of a
+ write; a kernel restart; acceptance in place.
+ * After every step it asserts that a write that may not be durable means a
+ marker on disk, and that untrusted equals reported untrusted, with the
+ marker kept. The model learns durability from the fsyncs themselves.
+ * Mutation check: removing any one condition fails the model or
+ `TestDropMarkerGuards`. The conditions: live processes, git's lock, git
+ activity, untrusted, the plain-marker rewrite, the missing-path removal,
+ the recovered untrusted record, the reboot distrust.
**Tests.**
* `localrefs`:
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index abc919f..d0bebb4 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -364,15 +364,18 @@ graft storage gc --removed -address unix:/var/opt/graft/gitaly.sock [-storage de
died before their database commit). The first run only observes them.
It runs in shadow mode too (orphans only: there are no checkpoints). It holds
- the repository's maintenance lease while it runs, and fails with
- `FailedPrecondition` when another maintenance of the repository (the
- background one, another node) holds it.
+ the repository's maintenance lease while it runs, renewed like the background
+ maintenance's, and fails with `FailedPrecondition` when another maintenance
+ of the repository (the background one, another node) holds it, or `Aborted`
+ when the lease is lost during the collection.
- `gc --removed` deletes every object of the repositories of the storage that
were removed at least `gc_grace` ago.
-- `--dry-run` reports what would be deleted and changes nothing.
+- `--dry-run` reports what would be deleted and changes nothing. It is a usage
+ error on the subcommands that do not read it, like `--connectivity` outside
+ `verify` and `--removed` outside `gc`.
-These commands are for operators: they need the shared token, and an agent
-token is refused.
+These commands are for operators: they need the shared token. A per-agent token
+can never call the storage core service, whatever `agent_rpcs` says.
### Switching from shadow to authoritative
@@ -521,16 +524,57 @@ or re-import the repository from scratch (see
- *A write the kernel reports as failed* (a writeback error, same boot): the
durable step's fsync fails, the repository is moved aside at once and logged,
and the next request hydrates it. The kernel reports such a failure once, so
- from then on the repository's marker is never removed: if the move aside
- fails, it is retried every second. Other errors of the step (a file that
- cannot be opened for a moment) are retried every second and counted.
+ from then on the repository is untrusted. Every request for it fails with
+ `Unavailable`, and its marker is never removed, until it is moved aside. If
+ the move fails, it is retried every second. A repository that is not a
+ cache of the storage core (the database cannot rebuild it) stays in place,
+ untrusted and not served, for an operator. Other errors of the step (a file
+ that cannot be opened for a moment) are retried every second and counted.
- *Process restart*: a marker found is kept while the repository's reftable
directory holds a git lock or a table being written (a git process of the
- previous process may still be finishing), and for 2 s at least. A
- repository the previous process had found untrusted is moved aside.
- - A repository moved aside that holds shadow-mode history not imported yet
- goes to `<storage>/+gitaly/graft-untrusted/` and stays there: import what
- it holds, or remove it.
+ previous process may still be finishing), and for 2 s at least after the
+ restart and after the last such file seen. A repository the previous
+ process had found untrusted is moved aside.
+ - A marker stays while a git process of the node that writes without fsync
+ is running, even when its repository was moved aside meanwhile: the
+ process names the repository by its path, so a hydrated copy there would
+ get its writes.
+ - **Kept copies.** A repository moved aside that may hold shadow-mode history
+ not imported yet goes to `<storage>/+gitaly/graft-untrusted/<stale-*>/repo.git`
+ and stays there. That is history written in shadow mode (file changelog
+ entries, file reviews) that the database does not have yet, or history
+ files that cannot be read. The repository itself is hydrated from the
+ database by its next request.
+ `graft_storage_kept_untrusted_repositories` and
+ `graft_storage_kept_untrusted_bytes` report the kept copies per storage. No
+ request can name a path under `+gitaly/`: the gate and the storage core
+ RPCs reject it with `InvalidArgument`.
+
+ To import what a kept copy holds:
+ 1. Take the node out of service: no client traffic to it during the
+ procedure. `gitaly serve` keeps running.
+ 2. Move the repository's current copy at `<storage>/<relative path>` out of
+ the storage, then move the kept `repo.git` to `<storage>/<relative path>`.
+ 3. `graft storage import -address <node> -storage <storage> -repo <relative path>`.
+ It imports the file changelog entries and the reviews the database does
+ not have, without touching the references.
+ 4. Remove the copy at `<storage>/<relative path>` again: its references are
+ not trusted. The next request hydrates the repository from the database.
+ `graft storage verify -repo <relative path>` must then report
+ `consistent`.
+ 5. Remove the kept directory, and put the node back in service.
+
+ A kept copy whose history files cannot be read cannot be imported: repair
+ them, or extract what is readable by hand. Removing a kept copy loses only
+ its history that was not imported.
+ - *Downgrade*: a release without this rule (v0.2.x) ignores the markers. It
+ would serve a repository that is untrusted, or dirty from an unclean
+ shutdown. Before a downgrade, stop `gitaly serve` cleanly: the final step
+ makes every repository durable. Then check that
+ `<storage>/+gitaly/graft-ref-durability/` holds no `*.json` marker. A
+ marker left there names a repository that is untrusted or that a live
+ process kept. Move that repository out of the storage first; its next
+ request after the downgrade hydrates it.
- *Clean stop*: after the servers stopped, `gitaly serve` makes every dirty
repository durable and removes its marker, so a clean reboot hydrates
nothing.
@@ -596,11 +640,12 @@ without a single divergence.
| `graft_storage_ref_admission_bypassed_total{writer}` | Writers that went on to git's lock without their turn after `reftable.lockTimeout` (10 s). |
| `graft_storage_ref_admission_expired_total` | Turns ended by the 10 s bound instead of their release (a git process that died between its phases). |
| `graft_storage_ref_dirty_repositories` | Repositories with reference writes not yet made durable. |
-| `graft_storage_ref_dirty_marked_total`, `graft_storage_ref_dirty_cleared_total` | Dirty markers written and removed by the durable step. |
+| `graft_storage_ref_dirty_marked_total`, `graft_storage_ref_dirty_cleared_total` | Dirty markers written, and removed (the writes are durable, or the repository was moved aside, and no git process can still write it). |
| `graft_storage_ref_writes_inflight` | git processes of this node that write references without fsync and have not exited; their repositories keep their markers. |
| `graft_storage_ref_durability_owner` | 1 when this process owns the deferred durability of its storages, 0 while another process (the old one of a graceful upgrade) does. |
| `graft_storage_ref_durable_step_errors_total` | Durable steps that failed with a transient error (not a failed fsync); retried every second. |
| `graft_storage_ref_untrusted_total{reason}` | Repositories moved aside to be hydrated again: `reboot` (dirty at a kernel restart), `sync_failed` (the durable step's fsync failed). |
+| `graft_storage_kept_untrusted_repositories{storage}`, `graft_storage_kept_untrusted_bytes{storage}` | Untrusted copies kept in `+gitaly/graft-untrusted/` for an operator (see Kept copies), and their size. Read at every scrape. |
## Benchmark
diff --git a/internal/gitaly/server/auth/agent_allowlist_test.go b/internal/gitaly/server/auth/agent_allowlist_test.go
index 43a4c2a..6ee54dd 100644
--- a/internal/gitaly/server/auth/agent_allowlist_test.go
+++ b/internal/gitaly/server/auth/agent_allowlist_test.go
@@ -174,6 +174,9 @@ func TestAgentAllowlist_EveryRegisteredRPC(t *testing.T) {
gitalypb.ObjectPoolService_DeleteObjectPool_FullMethodName,
gitalypb.CleanupService_RewriteHistory_FullMethodName,
gitalypb.PartitionService_BackupPartition_FullMethodName,
+ gitalypb.StorageCoreService_CollectGarbage_FullMethodName,
+ gitalypb.StorageCoreService_CollectRemovedRepositories_FullMethodName,
+ gitalypb.StorageCoreService_GetStorageState_FullMethodName,
} {
require.False(t, allowlist.mayCall(fullMethod), "agent may not call %s", fullMethod)
}
@@ -350,6 +353,8 @@ func TestNewAgentAllowlist_Validation(t *testing.T) {
{desc: "object pool", rpcs: []string{gitalypb.ObjectPoolService_DeleteObjectPool_FullMethodName}, wantErr: "may never be called by an agent"},
{desc: "cleanup service", rpcs: []string{gitalypb.CleanupService_RewriteHistory_FullMethodName}, wantErr: "may never be called by an agent"},
{desc: "partition service", rpcs: []string{gitalypb.PartitionService_BackupPartition_FullMethodName}, wantErr: "may never be called by an agent"},
+ {desc: "storage core garbage collection", rpcs: []string{gitalypb.StorageCoreService_CollectGarbage_FullMethodName}, wantErr: "may never be called by an agent"},
+ {desc: "storage core state", rpcs: []string{gitalypb.StorageCoreService_GetStorageState_FullMethodName}, wantErr: "may never be called by an agent"},
{desc: "one bad entry among good ones", rpcs: []string{"/gitaly.RepositoryService/RepositorySize", gitalypb.RepositoryService_FetchRemote_FullMethodName}, wantErr: "entry 2"},
} {
t.Run(tc.desc, func(t *testing.T) {
diff --git a/internal/gitaly/server/auth/agent_methods.go b/internal/gitaly/server/auth/agent_methods.go
index 7280115..9fb4faa 100644
--- a/internal/gitaly/server/auth/agent_methods.go
+++ b/internal/gitaly/server/auth/agent_methods.go
@@ -26,12 +26,16 @@ import (
// - CleanupService and PartitionService are administrative: they rewrite and
// clean up repositories and partitions outside the reference policy.
// - InternalGitaly serves the internal calls between Gitaly nodes.
+// - StorageCoreService is the storage core's administration: imports,
+// checkpoints and garbage collection, which deletes packs of the bucket.
+// It is reserved for operators (doc/graft/STORAGE.md).
var agentNeverAllowedServices = []string{
gitalypb.HookService_ServiceDesc.ServiceName,
gitalypb.ObjectPoolService_ServiceDesc.ServiceName,
gitalypb.CleanupService_ServiceDesc.ServiceName,
gitalypb.PartitionService_ServiceDesc.ServiceName,
gitalypb.InternalGitaly_ServiceDesc.ServiceName,
+ gitalypb.StorageCoreService_ServiceDesc.ServiceName,
}
// agentNeverAllowedMethods are RPCs that no caller authenticated with a
diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.go
index 592174e..491c9a0 100644
--- a/internal/gitaly/service/storagecore/import.go
+++ b/internal/gitaly/service/storagecore/import.go
@@ -61,8 +61,8 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
}
repo := req.GetRepository()
- if err := s.deps.GetLocator().ValidateRepository(ctx, repo); err != nil {
- return nil, structerr.NewInvalidArgument("%w", err)
+ if err := s.validateRepository(ctx, repo); err != nil {
+ return nil, err
}
local := s.localRepo(repo)
diff --git a/internal/gitaly/service/storagecore/maintenance.go b/internal/gitaly/service/storagecore/maintenance.go
index 564edcf..d677830 100644
--- a/internal/gitaly/service/storagecore/maintenance.go
+++ b/internal/gitaly/service/storagecore/maintenance.go
@@ -3,7 +3,6 @@ package storagecore
import (
"context"
"errors"
- "time"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/graft/corestore"
@@ -14,15 +13,12 @@ import (
"github.com/feder-cr/git-ai/proto/go/gitalypb"
)
-// releaseTimeout bounds the release of a maintenance lease after a request.
-const releaseTimeout = 10 * time.Second
-
// CreateCheckpoint implements gitalypb.StorageCoreServiceServer
// (doc/graft/PLAN7.md, "Checkpoint creation").
func (s *server) CreateCheckpoint(ctx context.Context, req *gitalypb.CreateCheckpointRequest) (*gitalypb.CreateCheckpointResponse, error) {
repo := req.GetRepository()
- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
- return nil, structerr.NewInvalidArgument("%w", err)
+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
+ return nil, err
}
core, err := s.enabledCore()
if err != nil {
@@ -54,12 +50,13 @@ func (s *server) CreateCheckpoint(ctx context.Context, req *gitalypb.CreateCheck
// CollectGarbage implements gitalypb.StorageCoreServiceServer
// (doc/graft/PLAN7.md, "Pack GC"). It holds the repository's maintenance lease
-// while it collects, so that it does not run alongside the background
-// maintenance or another request; correctness does not depend on the lease.
+// while it collects, renewed (maintenance.HoldLease), so that it does not run
+// alongside the background maintenance or another request; correctness does
+// not depend on the lease.
func (s *server) CollectGarbage(ctx context.Context, req *gitalypb.CollectGarbageRequest) (*gitalypb.CollectGarbageResponse, error) {
repo := req.GetRepository()
- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
- return nil, structerr.NewInvalidArgument("%w", err)
+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
+ return nil, err
}
core, err := s.enabledCore()
if err != nil {
@@ -89,16 +86,17 @@ func (s *server) CollectGarbage(ctx context.Context, req *gitalypb.CollectGarbag
if !acquired {
return nil, structerr.NewFailedPrecondition("another maintenance operation of this repository is in progress")
}
- defer func() {
- releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
- defer cancel()
- if err := core.Meta.ReleaseMaintenanceLease(releaseCtx, record.ID, holder); err != nil {
- s.deps.GetLogger().WithError(err).WarnContext(ctx, "storage core: release the maintenance lease; it expires by itself")
- }
- }()
+ // The lease is renewed while the collection runs, like the background
+ // maintenance's: a collection longer than lease_ttl keeps others out.
+ lost := false
+ leaseCtx, release := maintenance.HoldLease(ctx, core, s.deps.GetLogger(), record.ID, holder, ttl, func() { lost = true })
- report, err := gc.New(core, s.deps.GetLogger()).CollectRepository(ctx, record.ID, req.GetDryRun())
+ report, err := gc.New(core, s.deps.GetLogger()).CollectRepository(leaseCtx, record.ID, req.GetDryRun())
+ release()
if err != nil {
+ if lost {
+ return nil, structerr.NewAborted("the maintenance lease was lost to another maintenance operation of this repository: %w", err)
+ }
return nil, err
}
return &gitalypb.CollectGarbageResponse{
diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.go
index 91ca797..08cdf18 100644
--- a/internal/gitaly/service/storagecore/maintenance_test.go
+++ b/internal/gitaly/service/storagecore/maintenance_test.go
@@ -178,6 +178,39 @@ func TestCollectRemovedRepositoriesRefusals(t *testing.T) {
require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)
}
+// TestStorageCoreRejectsReservedRelativePaths verifies that no storage core
+// RPC acts on a repository inside Gitaly's data directory, where the storage
+// core keeps untrusted copies for an operator: the gate does not see these
+// RPCs, so they check it themselves.
+func TestStorageCoreRejectsReservedRelativePaths(t *testing.T) {
+ t.Parallel()
+
+ h := setup(t, corestore.ModeAuthoritative, sqlstore.SQLite)
+ repo := &gitalypb.Repository{StorageName: h.repo.GetStorageName(), RelativePath: "+gitaly/graft-untrusted/stale-1/repo.git"}
+ for name, call := range map[string]func() error{
+ "GetStorageState": func() error {
+ _, err := h.client.GetStorageState(h.ctx, &gitalypb.GetStorageStateRequest{Repository: repo})
+ return err
+ },
+ "VerifyRepository": func() error {
+ _, err := h.client.VerifyRepository(h.ctx, &gitalypb.VerifyRepositoryRequest{Repository: repo})
+ return err
+ },
+ "CreateCheckpoint": func() error {
+ _, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: repo})
+ return err
+ },
+ "CollectGarbage": func() error {
+ _, err := h.client.CollectGarbage(h.ctx, &gitalypb.CollectGarbageRequest{Repository: repo})
+ return err
+ },
+ } {
+ err := call()
+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%s: %v", name, err)
+ require.ErrorContains(t, err, "Gitaly's data directory", name)
+ }
+}
+
// packKeys returns the objects of the repository in the bucket.
func (h *harness) packKeys(repoID int64) map[string]bool {
h.t.Helper()
diff --git a/internal/gitaly/service/storagecore/server.go b/internal/gitaly/service/storagecore/server.go
index 5c540d6..5f00176 100644
--- a/internal/gitaly/service/storagecore/server.go
+++ b/internal/gitaly/service/storagecore/server.go
@@ -13,6 +13,7 @@ import (
"github.com/feder-cr/git-ai/internal/git/gitcmd"
"github.com/feder-cr/git-ai/internal/git/localrepo"
"github.com/feder-cr/git-ai/internal/gitaly/service"
+ "github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/graft/corestore"
"github.com/feder-cr/git-ai/internal/structerr"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
@@ -37,6 +38,20 @@ func (s *server) enabledCore() (*corestore.Core, error) {
return core, nil
}
+// validateRepository validates the repository of a request: the locator's
+// checks (InvalidArgument), and a relative path outside Gitaly's data
+// directory of the storage (corestore.CheckRelativePath). The gate does not see
+// the storage core's RPCs, so they check it themselves.
+func (s *server) validateRepository(ctx context.Context, repo *gitalypb.Repository, opts ...storage.ValidateRepositoryOption) error {
+ if err := s.deps.GetLocator().ValidateRepository(ctx, repo, opts...); err != nil {
+ return structerr.NewInvalidArgument("%w", err)
+ }
+ if err := corestore.CheckRelativePath(repo.GetRelativePath()); err != nil {
+ return structerr.NewInvalidArgument("%w", err)
+ }
+ return nil
+}
+
// localRepo returns the local repository of a request.
func (s *server) localRepo(repo *gitalypb.Repository) *localrepo.Repo {
return s.deps.GetRepositoryFactory().Build(repo)
diff --git a/internal/gitaly/service/storagecore/state.go b/internal/gitaly/service/storagecore/state.go
index 652e7f6..ee97f71 100644
--- a/internal/gitaly/service/storagecore/state.go
+++ b/internal/gitaly/service/storagecore/state.go
@@ -15,8 +15,8 @@ import (
// that this node has no local copy of (another node may have it).
func (s *server) GetStorageState(ctx context.Context, req *gitalypb.GetStorageStateRequest) (*gitalypb.GetStorageStateResponse, error) {
repo := req.GetRepository()
- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
- return nil, structerr.NewInvalidArgument("%w", err)
+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
+ return nil, err
}
core := s.deps.GetGraftStorage()
diff --git a/internal/gitaly/service/storagecore/verify.go b/internal/gitaly/service/storagecore/verify.go
index b417ce0..86c3832 100644
--- a/internal/gitaly/service/storagecore/verify.go
+++ b/internal/gitaly/service/storagecore/verify.go
@@ -33,8 +33,8 @@ func (s *server) VerifyRepository(ctx context.Context, req *gitalypb.VerifyRepos
}
repo := req.GetRepository()
- if err := s.deps.GetLocator().ValidateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
- return nil, structerr.NewInvalidArgument("%w", err)
+ if err := s.validateRepository(ctx, repo, storage.WithSkipRepositoryExistenceCheck()); err != nil {
+ return nil, err
}
if core.Authoritative() {
diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.go
index df13121..3c266b1 100644
--- a/internal/graft/corestore/core.go
+++ b/internal/graft/corestore/core.go
@@ -9,6 +9,7 @@ import (
"strings"
"time"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/graft/corestore/localrefs"
)
@@ -199,6 +200,23 @@ func (c *Core) Close() error {
return firstErr
}
+// ErrReservedRelativePath is returned for a repository named inside Gitaly's
+// own data directory of a storage (config.GitalyDataPrefix): the temporary,
+// cache and state directories, the dirty markers, and the untrusted copies the
+// storage core keeps for an operator. A request naming one could make the
+// storage core treat a kept copy as a stale cache and remove it.
+var ErrReservedRelativePath = errors.New("the relative path is inside Gitaly's data directory of the storage, which no request may name")
+
+// CheckRelativePath rejects a relative path that names Gitaly's data directory
+// of the storage or anything in it (ErrReservedRelativePath).
+func CheckRelativePath(relativePath string) error {
+ rel := NormalizeRelativePath(relativePath)
+ if rel == config.GitalyDataPrefix || strings.HasPrefix(rel, config.GitalyDataPrefix+"/") {
+ return fmt.Errorf("%w: %q", ErrReservedRelativePath, relativePath)
+ }
+ return nil
+}
+
// KeyOf returns the RepoKey of a gitaly repository. The relative path is
// normalized the way the storage locator resolves it (filepath.Join of the
// storage root and the relative path), so that two spellings of one directory
diff --git a/internal/graft/corestore/gate/gate.go b/internal/graft/corestore/gate/gate.go
index bd190d4..603b2c3 100644
--- a/internal/graft/corestore/gate/gate.go
+++ b/internal/graft/corestore/gate/gate.go
@@ -267,7 +267,9 @@ func targetRepo(mi protoregistry.MethodInfo, msg any) *gitalypb.Repository {
// (corestore.KeyOf), while the locator resolves every spelling of a path to
// the same directory: without this check "grp//proj.git" and "grp/proj.git"
// would be two repositories in the database sharing one local copy. An empty
-// path is left to the handler, which rejects it itself.
+// path is left to the handler, which rejects it itself. A path inside Gitaly's
+// data directory is rejected too (corestore.CheckRelativePath): the stale-cache
+// rule must never act on a copy the storage core keeps there.
func checkCanonical(repo *gitalypb.Repository) error {
rel := repo.GetRelativePath()
if rel == "" {
@@ -276,6 +278,9 @@ func checkCanonical(repo *gitalypb.Repository) error {
if clean := filepath.Clean(rel); clean != rel {
return structerr.NewInvalidArgument("relative path %q is not canonical in authoritative storage mode, use %q", rel, clean)
}
+ if err := corestore.CheckRelativePath(rel); err != nil {
+ return structerr.NewInvalidArgument("%w", err)
+ }
return nil
}
diff --git a/internal/graft/corestore/gate/gate_stale_test.go b/internal/graft/corestore/gate/gate_stale_test.go
index c2988eb..3faec2e 100644
--- a/internal/graft/corestore/gate/gate_stale_test.go
+++ b/internal/graft/corestore/gate/gate_stale_test.go
@@ -229,11 +229,16 @@ func TestGateRemoveRepositoryDropsReinstalledCache(t *testing.T) {
}
// The database key is the relative path as given, so the gate only accepts
-// relative paths in their cleaned form.
-func TestGateRejectsNonCanonicalRelativePath(t *testing.T) {
+// relative paths in their cleaned form; and none inside Gitaly's data
+// directory, where the storage core keeps untrusted copies that the stale-cache
+// rule must never remove.
+func TestGateRejectsNonCanonicalOrReservedRelativePath(t *testing.T) {
t.Parallel()
- for _, rel := range []string{"grp//proj.git", "./grp/proj.git", "grp/proj.git/", "grp/../grp/proj.git", "grp/./proj.git"} {
+ for _, rel := range []string{
+ "grp//proj.git", "./grp/proj.git", "grp/proj.git/", "grp/../grp/proj.git", "grp/./proj.git",
+ "+gitaly", "+gitaly/graft-untrusted/stale-1/repo.git", "+gitaly/tmp/hydrate-1/repo.git",
+ } {
t.Run(rel, func(t *testing.T) {
t.Parallel()
diff --git a/internal/graft/corestore/localrefs/durability.go b/internal/graft/corestore/localrefs/durability.go
index 35b7ad0..2a81b48 100644
--- a/internal/graft/corestore/localrefs/durability.go
+++ b/internal/graft/corestore/localrefs/durability.go
@@ -90,6 +90,11 @@ const stackLockTag = "graft durable step "
// writes, so the local repository must not be served before it is rebuilt.
var ErrUntrusted = errors.New("storage core: the local references may have lost writes")
+// errReboot makes a repository untrusted whose marker is of another kernel
+// boot: the kernel restarted while its references had writes that were not
+// durable, so its stack may be torn.
+var errReboot = errors.New("the kernel restarted while the references had writes that were not durable")
+
// UntrustedError is ErrUntrusted for one repository.
type UntrustedError struct {
Repo RepoRef
@@ -126,9 +131,13 @@ func (e writebackError) Unwrap() error { return e.err }
// - before the first reference write that is not durable, the repository
// gets a dirty marker, written durably, that holds the identity of the
// kernel boot (MarkDirty);
-// - a durable step makes the references durable and removes the marker,
-// under git's reference lock and only when no transaction of the process is
-// live (Settle);
+// - a durable step makes the references durable under git's reference lock
+// (Settle); one function removes markers, every path through it, and only
+// when nothing can leave a write that is not durable in the stack any more
+// (dropMarker);
+// - a repository whose writes may be lost (a failed fsync, a marker of
+// another boot) is untrusted: never served, its marker never removed, until
+// it is moved aside (Forget);
// - when it takes the ownership, a marker of another boot means the kernel
// restarted while the repository had writes that may never have reached the
// disk: its reftable stack may be torn, which git reads as an empty
@@ -157,13 +166,14 @@ type Durability struct {
ownerFiles []*os.File
locked bool
// recoveryGrace is how long a repository recovered dirty keeps its marker
- // at least: a git process of the previous owner that outlived it (killed
- // with it, but not reaped yet) may still be committing for that long.
+ // at least, and how long after git activity was last seen in a stack: a git
+ // process of the previous owner that outlived it (killed with it, but not
+ // reaped yet) may still be committing for that long.
recoveryGrace time.Duration
mu sync.Mutex
- // repos holds the state of every repository written since the process
- // started (one entry per repository; Forget drops it).
+ // repos holds the state of every repository that is dirty or that a live
+ // git process counts on (one entry per repository; dropMarker drops it).
repos map[string]*refState
marked prometheus.Counter
@@ -179,28 +189,37 @@ var _ prometheus.Collector = (*Durability)(nil)
// refState is what Durability knows of one local repository, by its path. Its
// fields are guarded by Durability.mu, except marker writes and removals,
-// which file serializes.
+// which file serializes. A state leaves the map only in dropMarker, under its
+// file lock, so whoever holds the file lock of the map's state keeps it current
+// (lockState).
type refState struct {
// file serializes the writes and the removal of the marker file.
file sync.Mutex
repo RepoRef
marker string
- dirty bool
+ // dirty is set while the marker may be on disk: from before it is written
+ // until dropMarker removed it.
+ dirty bool
+ // markerPlain is set when the marker on disk is a plain marker of the
+ // running boot: neither one that records untrusted nor one of another boot,
+ // which would make a restart distrust what is at the path.
+ markerPlain bool
// nLive counts the git processes of this process that write the
// repository's references without fsync and have not exited: any of them
// may still write a table or compact the stack.
nLive int
lastWrite time.Time
- // recovered is set for a repository found dirty when the process took the
- // ownership: a git process of the previous owner that outlived it may still
- // be writing, so its marker stays while git's lock or temporary files are
- // there (gitActive), and not before settleAfter.
- recovered bool
+ // settleAfter is the earliest time the marker may go: the recovery grace of
+ // a repository found dirty when the process took the ownership (a git
+ // process of the previous owner that outlived it may still be committing),
+ // pushed forward whenever git activity is seen in its stack.
settleAfter time.Time
- // untrusted is the first failed fsync of the repository. The kernel reports
- // a failed writeback once, so a later fsync can succeed although writes
- // were lost: the repository never becomes clean again, only moved aside.
- // The marker records it, so that a restart of the process keeps it.
+ // untrusted is the first failed fsync of the repository, or the restart of
+ // the kernel that a marker of another boot shows. The kernel reports a
+ // failed writeback once, so a later fsync can succeed although writes were
+ // lost: the repository is never served and never becomes clean, it is only
+ // moved aside (Forget). The marker records a failed fsync, so that a restart
+ // of the process keeps it.
untrusted error
// synced holds the tables made durable during the current dirty period that
// the stack still lists. Tables are immutable, so one fsync each is enough.
@@ -289,7 +308,7 @@ func newDurability(locator storage.Locator, idle time.Duration) *Durability {
}),
cleared: prometheus.NewCounter(prometheus.CounterOpts{
Name: "graft_storage_ref_dirty_cleared_total",
- Help: "Number of dirty markers removed by a durable step.",
+ Help: "Number of dirty markers removed: the repository's reference writes are durable, or the repository was moved aside, and no git process can still write it.",
}),
untrusted: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "graft_storage_ref_untrusted_total",
@@ -449,32 +468,15 @@ func (d *Durability) MarkDirty(ctx context.Context, repo RepoRef, path string) e
return nil
}
- // The state is the map's after its file lock is taken: a durable step may
- // have dropped a clean state meanwhile.
- var st *refState
- for {
- d.mu.Lock()
- st = d.stateLocked(path)
- d.mu.Unlock()
- st.file.Lock()
- d.mu.Lock()
- current := d.repos[path] == st
- if current {
- st.lastWrite = time.Now()
- }
- dirty := st.dirty
- d.mu.Unlock()
- if !current {
- st.file.Unlock()
- continue
- }
- if dirty {
- st.file.Unlock()
- return nil
- }
- break
- }
+ st := d.lockState(path)
defer st.file.Unlock()
+ d.mu.Lock()
+ st.lastWrite = time.Now()
+ dirty := st.dirty
+ d.mu.Unlock()
+ if dirty {
+ return nil
+ }
markerPath, err := d.markerPath(ctx, repo, path)
if err != nil {
@@ -492,12 +494,37 @@ func (d *Durability) MarkDirty(ctx context.Context, repo RepoRef, path string) e
defer d.mu.Unlock()
st.repo = repo
st.marker = markerPath
+ st.markerPlain = true
d.marked.Inc()
+ d.setDirtyLocked(st)
+ return nil
+}
+
+// lockState returns the state of the repository at path, created when there is
+// none, with its file lock held: the state stays the map's until the lock is
+// released, because only dropMarker removes a state, under its file lock.
+func (d *Durability) lockState(path string) *refState {
+ for {
+ d.mu.Lock()
+ st := d.stateLocked(path)
+ d.mu.Unlock()
+ st.file.Lock()
+ d.mu.Lock()
+ current := d.repos[path] == st
+ d.mu.Unlock()
+ if current {
+ return st
+ }
+ st.file.Unlock()
+ }
+}
+
+// setDirtyLocked marks the state dirty. The caller holds d.mu.
+func (d *Durability) setDirtyLocked(st *refState) {
if !st.dirty {
st.dirty = true
d.dirty.Inc()
}
- return nil
}
// Begin records that a git process that writes the references of the
@@ -505,7 +532,8 @@ func (d *Durability) MarkDirty(ctx context.Context, repo RepoRef, path string) e
// once the process has exited. Until then the process may write a table or
// compact the stack, whatever hooks git runs or skips (a failed prepare, a
// failed commit, a kill), so the durable step keeps the marker. end may be
-// called more than once.
+// called more than once. A state with a live process never leaves the map
+// (dropMarker), so end always counts down the state Begin counted up.
func (d *Durability) Begin(path string) (end func()) {
if d == nil {
return func() {}
@@ -522,15 +550,26 @@ func (d *Durability) Begin(path string) (end func()) {
once.Do(func() {
d.mu.Lock()
defer d.mu.Unlock()
- // Forget may have dropped the state; the count belongs to it.
- if d.repos[path] == st {
- st.nLive--
- d.inflight.Dec()
- }
+ st.nLive--
+ d.inflight.Dec()
})
}
}
+// Untrusted reports whether the local repository at path may have lost
+// reference writes: it must not be served until it is moved aside and
+// hydrated again.
+func (d *Durability) Untrusted(path string) bool {
+ if d == nil {
+ return false
+ }
+
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ st := d.repos[path]
+ return st != nil && st.untrusted != nil
+}
+
// Dirty returns the paths of the dirty repositories.
func (d *Durability) Dirty() []string {
if d == nil {
@@ -576,13 +615,14 @@ func (d *Durability) CountStepError() {
// durable. A failed fsync is an *UntrustedError (the caller moves the
// repository aside at once); any other error is transient. When the repository
// has been idle for the idle period, or final is set (the process stops), and
-// no transaction of the process is live, it then removes the marker: the
-// decision is taken under git's reference lock, taken without waiting (a busy
-// lock leaves the marker for the next step), and the marker file is removed
-// after the lock is released, under the state's file lock that a writer's
-// MarkDirty waits for. A repository that no longer exists (moved aside as a
-// stale cache) loses its marker. The caller holds the syncer's repository lock,
-// so no syncer write and no replacement of the repository runs meanwhile.
+// no git process of the process is live, it then takes git's reference lock
+// without waiting (a busy lock leaves the marker for the next step), makes the
+// stack durable again under it, and lets dropMarker decide. A repository that
+// no longer exists (moved aside as a stale cache) has no stack, and its marker
+// goes by the same rule: it stays while a git process that names the path is
+// live. A clean state with no live process leaves the map. The caller holds the
+// syncer's repository lock, so no syncer write and no replacement of the
+// repository runs meanwhile.
func (d *Durability) Settle(path string, final bool) error {
if !d.Deferring() {
return nil
@@ -590,7 +630,7 @@ func (d *Durability) Settle(path string, final bool) error {
d.mu.Lock()
st := d.repos[path]
- if st == nil || !st.dirty {
+ if st == nil {
d.mu.Unlock()
return nil
}
@@ -601,34 +641,33 @@ func (d *Durability) Settle(path string, final bool) error {
d.mu.Unlock()
return &UntrustedError{Repo: repo, Path: path, Err: untrusted}
}
- now := time.Now()
- idle := st.nLive == 0 && now.After(st.settleAfter) && (final || now.Sub(st.lastWrite) >= d.idle)
- recovered := st.recovered
+ dirty := st.dirty
+ due := st.nLive == 0 && (final || time.Since(st.lastWrite) >= d.idle)
d.mu.Unlock()
- if idle && recovered {
- // A git process of the previous owner may still be writing.
- active, err := gitActive(path)
- if err != nil {
- return err
- }
- idle = !active
- }
+ exists := true
if _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) {
- st.file.Lock()
+ exists = false
+ } else if err != nil {
+ return fmt.Errorf("stat the repository: %w", err)
+ }
+ if !dirty || !exists {
+ // A repository that is gone has no stack; a clean state only leaves the
+ // map. A transaction that marked an existing repository meanwhile leaves
+ // it to the next step, under git's reference lock (dropMarker refuses
+ // otherwise).
+ st := d.lockState(path)
defer st.file.Unlock()
- d.clear(st)
- if err := d.removeMarker(st); err != nil {
- return err
- }
- d.prune(path, st)
- return nil
+ _, err := d.dropMarker(path, st, false)
+ return err
}
if err := d.syncStack(path, st); err != nil {
- return d.stepError(st, repo, path, err)
+ st := d.lockState(path)
+ defer st.file.Unlock()
+ return d.stepErrorLocked(st, repo, path, err)
}
- if !idle {
+ if !due {
return nil
}
@@ -639,49 +678,113 @@ func (d *Durability) Settle(path string, final bool) error {
if !ok {
return nil
}
-
- cleared, err := d.settleLocked(path, st)
- if err != nil {
- _ = lock.release()
- return d.stepError(st, repo, path, err)
- }
- if !cleared {
- return lock.release()
+ err = d.settleLocked(path, repo)
+ if releaseErr := lock.release(); err == nil {
+ err = releaseErr
}
+ return err
+}
- // The state is clean from here on, and a writer that takes git's lock now
- // writes a new marker: it waits for the file lock, held until the old
- // marker is removed.
+// settleLocked makes the stack durable under git's reference lock, which the
+// caller holds (no transaction commits meanwhile), and removes the marker when
+// dropMarker finds it safe. The marker file goes before git's lock is
+// released, under the state's file lock, which a writer's MarkDirty waits for.
+func (d *Durability) settleLocked(path string, repo RepoRef) error {
+ st := d.lockState(path)
defer st.file.Unlock()
- if err := lock.release(); err != nil {
- return err
+
+ if err := d.syncStack(path, st); err != nil {
+ return d.stepErrorLocked(st, repo, path, err)
}
- if err := d.removeMarker(st); err != nil {
- return err
+ if d.crashAt == crashDurableStep {
+ os.Exit(crashExitCode)
}
- d.prune(path, st)
- return nil
+ _, err := d.dropMarker(path, st, true)
+ return err
}
-// prune drops the state of a repository that is clean and has no live git
-// process, so that the states do not grow with every repository ever written.
-// The caller holds the state's file lock: MarkDirty takes the state again from
-// the map after its own.
-func (d *Durability) prune(path string, st *refState) {
+// dropMarker is the only code that removes a dirty marker or drops a state
+// from the map. It removes the marker of the repository at path only when
+// nothing can still leave a reference write that is not durable in its stack:
+//
+// - no git process of this process that writes the references without fsync
+// is live (nLive);
+// - the repository is not untrusted;
+// - when the repository exists, the caller holds git's reference lock
+// (ownLock) and made the stack durable under it (settleLocked): no git
+// process can write meanwhile, and the ones that exited before are in what
+// was made durable. A repository that does not exist has no stack, and a
+// git process cannot write it (the caller holds the syncer's repository
+// lock, so it is not hydrated meanwhile);
+// - no git process writes the stack now (gitActive: a git lock or a table
+// being written; seeing one also pushes settleAfter a recovery grace
+// forward), and settleAfter has passed.
+//
+// The caller holds the state's file lock, which MarkDirty waits for before it
+// reads whether the repository is dirty. The checks and the state change are
+// made under d.mu, which Begin takes. It reports whether the marker was
+// removed. A clean state that no live git process counts on and that is
+// trusted leaves the map, so that the states do not grow with every repository
+// ever written.
+func (d *Durability) dropMarker(path string, st *refState, ownLock bool) (bool, error) {
+ exists := true
+ if _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) {
+ exists = false
+ } else if err != nil {
+ return false, fmt.Errorf("stat the repository: %w", err)
+ }
+ active := false
+ if exists {
+ var err error
+ if active, err = gitActive(path, ownLock); err != nil {
+ return false, err
+ }
+ }
+
+ d.mu.Lock()
+ now := time.Now()
+ if active {
+ if until := now.Add(d.recoveryGrace); until.After(st.settleAfter) {
+ st.settleAfter = until
+ }
+ }
+ safe := st.dirty && st.nLive == 0 && st.untrusted == nil && (ownLock || !exists) && !active && !now.Before(st.settleAfter)
+ if safe {
+ st.dirty = false
+ d.dirty.Dec()
+ st.synced = map[string]struct{}{}
+ }
+ markerPath := st.marker
+ d.mu.Unlock()
+
+ if safe && markerPath != "" {
+ if err := os.Remove(markerPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ d.mu.Lock()
+ d.setDirtyLocked(st)
+ d.mu.Unlock()
+ return false, fmt.Errorf("remove the dirty marker: %w", err)
+ }
+ }
+ if safe {
+ d.cleared.Inc()
+ }
+
d.mu.Lock()
defer d.mu.Unlock()
- if d.repos[path] == st && !st.dirty && st.untrusted == nil && st.nLive == 0 {
+ if d.repos[path] == st && !st.dirty && st.nLive == 0 && st.untrusted == nil {
delete(d.repos, path)
}
+ return safe, nil
}
-// stepError is the error of a durable step: an *UntrustedError for a failed
-// fsync, which makes the repository untrusted for good, the error itself
-// otherwise. The first failure is written into the marker, so that the
+// stepErrorLocked is the error of a durable step: an *UntrustedError for a
+// failed fsync, which makes the repository untrusted for good, the error
+// itself otherwise. The first failure is written into the marker, so that the
// repository stays untrusted across a restart of the process (the next fsync
// would succeed); when even that write fails, the disk is failing and the
-// state of this process keeps it untrusted.
-func (d *Durability) stepError(st *refState, repo RepoRef, path string, err error) error {
+// state of this process keeps it untrusted. The caller holds the state's file
+// lock.
+func (d *Durability) stepErrorLocked(st *refState, repo RepoRef, path string, err error) error {
var writeback writebackError
if !errors.As(err, &writeback) {
return err
@@ -693,32 +796,42 @@ func (d *Durability) stepError(st *refState, repo RepoRef, path string, err erro
st.untrusted = err
}
markerPath := st.marker
+ dirty := st.dirty
d.mu.Unlock()
- if first && markerPath != "" {
- st.file.Lock()
+ if first && dirty && markerPath != "" {
content, encodeErr := json.Marshal(marker{RepoRef: repo, BootID: d.bootID, Untrusted: err.Error()})
if encodeErr == nil {
encodeErr = d.writeDurably(markerPath, content)
}
- st.file.Unlock()
if encodeErr != nil {
return &UntrustedError{Repo: repo, Path: path, Err: fmt.Errorf("%w (recording it in the marker failed: %w)", err, encodeErr)}
}
+ d.mu.Lock()
+ st.markerPlain = false
+ d.mu.Unlock()
}
return &UntrustedError{Repo: repo, Path: path, Err: err}
}
// gitActive reports whether a git process may be writing the reftable stack of
-// the repository at path: git's lock of the list, a lock of a table being
-// compacted, or a table being written is there.
-func gitActive(path string) (bool, error) {
+// the repository at path: git's lock of the list (unless ownLock: the caller
+// holds it), a lock of a table being compacted, or a table being written is
+// there (the names of git's reftable/stack.c). A repository without a reftable
+// directory has no stack to write.
+func gitActive(path string, ownLock bool) (bool, error) {
entries, err := os.ReadDir(filepath.Join(path, "reftable"))
+ if errors.Is(err, fs.ErrNotExist) {
+ return false, nil
+ }
if err != nil {
return false, fmt.Errorf("read the reftable directory: %w", err)
}
for _, entry := range entries {
name := entry.Name()
+ if ownLock && name == "tables.list.lock" {
+ continue
+ }
if strings.HasSuffix(name, ".lock") || strings.Contains(name, ".temp.") {
return true, nil
}
@@ -726,55 +839,6 @@ func gitActive(path string) (bool, error) {
return false, nil
}
-// settleLocked makes the tables written since the first pass durable and, when
-// no transaction of the process is live, marks the state clean with its file
-// lock held (the caller releases it). The caller holds git's reference lock: no
-// transaction can take it meanwhile, and the live transactions are counted from
-// before they take it until their end.
-func (d *Durability) settleLocked(path string, st *refState) (bool, error) {
- d.mu.Lock()
- busy := st.nLive > 0 || st.untrusted != nil
- d.mu.Unlock()
- if busy {
- return false, nil
- }
-
- if err := d.syncStack(path, st); err != nil {
- return false, err
- }
- if d.crashAt == crashDurableStep {
- os.Exit(crashExitCode)
- }
-
- st.file.Lock()
- d.clear(st)
- return true, nil
-}
-
-// clear marks the state clean. The caller holds the state's file lock.
-func (d *Durability) clear(st *refState) {
- d.mu.Lock()
- defer d.mu.Unlock()
- if st.dirty {
- st.dirty = false
- d.cleared.Inc()
- d.dirty.Dec()
- }
- st.synced = map[string]struct{}{}
-}
-
-// removeMarker removes the marker file of a clean state. The caller holds the
-// state's file lock.
-func (d *Durability) removeMarker(st *refState) error {
- if st.marker == "" {
- return nil
- }
- if err := os.Remove(st.marker); err != nil && !errors.Is(err, fs.ErrNotExist) {
- return fmt.Errorf("remove the dirty marker: %w", err)
- }
- return nil
-}
-
// stackLock is git's reference lock of a repository, taken by the durable
// step.
type stackLock struct {
@@ -939,9 +1003,34 @@ func (d *Durability) syncStack(path string, st *refState) error {
return nil
}
-// Forget drops the state and the marker of the repository at path, which the
-// caller moved aside under the syncer's repository lock. A writer that still
-// writes to the moved repository writes to a directory that is removed.
+// Tracked returns the paths of the repositories that have a state: the dirty
+// ones and the clean ones that a live git process still counts on. The durable
+// steps visit them all, so that a clean state leaves the map once its last
+// process ended.
+func (d *Durability) Tracked() []string {
+ if d == nil {
+ return nil
+ }
+
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ paths := make([]string, 0, len(d.repos))
+ for path := range d.repos {
+ paths = append(paths, path)
+ }
+ return paths
+}
+
+// Forget hands the path of a repository over to what comes there next: the
+// caller moved the local repository aside under the syncer's repository lock,
+// or, after a kernel restart, decided to keep in place a repository that is not
+// a cache of the storage core. The state stops being untrusted, since what made
+// it so is no longer at path, and the marker goes by dropMarker's rule. A git
+// process of this process that is still live names the repository by its path,
+// so its later writes land in what is there now (the hydrated repository, say):
+// the marker then stays, plain (without the untrusted record, so that a restart
+// does not move the new repository aside), until the process exited and a
+// durable step made the stack durable.
func (d *Durability) Forget(ctx context.Context, repo RepoRef, path string) error {
if d == nil {
return nil
@@ -952,56 +1041,74 @@ func (d *Durability) Forget(ctx context.Context, repo RepoRef, path string) erro
return err
}
+ st := d.lockState(path)
+ defer st.file.Unlock()
+
d.mu.Lock()
- st := d.repos[path]
+ // A plain marker of this boot is on disk when the state was dirty with one;
+ // otherwise one is written if the marker has to stay.
+ plainOnDisk := st.dirty && st.markerPlain && st.marker == markerPath
+ st.repo = repo
+ st.marker = markerPath
+ d.setDirtyLocked(st)
+ st.untrusted = nil
+ st.synced = map[string]struct{}{}
d.mu.Unlock()
- if st != nil {
- st.file.Lock()
- defer st.file.Unlock()
- }
+ dropped, err := d.forgetLocked(path, st, repo)
d.mu.Lock()
- live := st != nil && d.repos[path] == st && st.nLive > 0
+ untrusted := st.untrusted != nil
d.mu.Unlock()
- if live {
- // A git process that writes without fsync is still running. It names the
- // repository by its path, so its later writes land in what is there now
- // (the hydrated repository, say): the repository stays dirty, with a plain
- // marker, until it exited and a durable step made the stack durable.
- content, err := json.Marshal(marker{RepoRef: repo, BootID: d.bootID})
- if err != nil {
- return fmt.Errorf("encode the dirty marker: %w", err)
- }
- if err := d.writeDurably(markerPath, content); err != nil {
- return fmt.Errorf("write the dirty marker: %w", err)
- }
- d.mu.Lock()
- defer d.mu.Unlock()
- if !st.dirty {
- st.dirty = true
- d.dirty.Inc()
- }
- st.repo = repo
- st.marker = markerPath
- st.untrusted = nil
- st.recovered = false
- st.synced = map[string]struct{}{}
- return nil
+ if dropped || plainOnDisk || untrusted {
+ // An untrusted state (a failed fsync of a repository kept in place) has
+ // its record in the marker, or is kept untrusted by this process.
+ return err
}
- if err := os.Remove(markerPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
- return fmt.Errorf("remove the dirty marker: %w", err)
+ content, encodeErr := json.Marshal(marker{RepoRef: repo, BootID: d.bootID})
+ if encodeErr == nil {
+ encodeErr = d.writeDurably(markerPath, content)
}
-
d.mu.Lock()
defer d.mu.Unlock()
- if st != nil && d.repos[path] == st {
- if st.dirty {
- d.dirty.Dec()
+ if encodeErr != nil {
+ // The state is dirty, so a writer's MarkDirty would not write the
+ // marker: without one on disk, the repository at path cannot be
+ // trusted. It is moved aside again, and Forget retried.
+ if st.untrusted == nil {
+ st.untrusted = fmt.Errorf("write the dirty marker: %w", encodeErr)
}
- delete(d.repos, path)
+ return errors.Join(err, st.untrusted)
}
- return nil
+ st.markerPlain = true
+ return err
+}
+
+// forgetLocked lets dropMarker decide the marker of a forgotten repository.
+// One kept in place is made durable under git's reference lock first, as a
+// durable step does; when the lock is busy the marker stays for the next step.
+// The caller holds the state's file lock.
+func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (bool, error) {
+ if _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) {
+ return d.dropMarker(path, st, false)
+ } else if err != nil {
+ return false, fmt.Errorf("stat the repository: %w", err)
+ }
+
+ lock, ok, err := lockStack(path)
+ if err != nil || !ok {
+ return false, err
+ }
+ dropped := false
+ if err = d.syncStack(path, st); err != nil {
+ err = d.stepErrorLocked(st, repo, path, err)
+ } else {
+ dropped, err = d.dropMarker(path, st, true)
+ }
+ if releaseErr := lock.release(); err == nil {
+ err = releaseErr
+ }
+ return dropped, err
}
// Recovered is a dirty repository found by Recover.
@@ -1018,14 +1125,16 @@ type Recovered struct {
}
// Recover reads the dirty markers of the storages; only the owner calls it,
-// when it takes the ownership. A marker of the running boot makes its
-// repository dirty again, to be settled once no git process of the previous
-// owner can still write it (gitActive, and the recovery grace), or untrusted
-// again when the marker says so; what a durable step of the previous owner
+// when it takes the ownership. Every marker makes its repository dirty again.
+// One of the running boot is settled once no git process of the previous owner
+// can still write the stack (dropMarker: gitActive, and the recovery grace),
+// unless the marker records it untrusted; one of another boot makes the
+// repository untrusted (errReboot). What a durable step of the previous owner
// left (its reference lock, its temporary files) is removed. It returns every
// repository found; the caller moves aside the ones of another boot and the
-// untrusted ones before serving. A marker that cannot be read is an error: its
-// repository is unknown, and serving could read a torn stack.
+// untrusted ones before serving (Forget then hands their paths over). A marker
+// that cannot be read is an error: its repository is unknown, and serving
+// could read a torn stack.
func (d *Durability) Recover(ctx context.Context, storages []string) ([]Recovered, error) {
if d == nil || !d.ownsLocks() {
return nil, nil
@@ -1076,29 +1185,33 @@ func (d *Durability) Recover(ctx context.Context, storages []string) ([]Recovere
return nil, fmt.Errorf("dirty marker %q: %w", markerPath, err)
}
+ if err := removeStaleStackLock(path); err != nil {
+ return nil, err
+ }
+ if err := removeStepTemporaries(path); err != nil {
+ return nil, err
+ }
+
reboot := m.BootID != d.bootID
+ st := d.lockState(path)
+ d.mu.Lock()
+ d.setDirtyLocked(st)
+ st.repo = m.RepoRef
+ st.marker = markerPath
+ st.markerPlain = !reboot && m.Untrusted == ""
+ switch {
+ case m.Untrusted != "":
+ st.untrusted = errors.New(m.Untrusted)
+ case reboot:
+ st.untrusted = errReboot
+ }
if !reboot {
- if err := removeStaleStackLock(path); err != nil {
- return nil, err
- }
- if err := removeStepTemporaries(path); err != nil {
- return nil, err
- }
- d.mu.Lock()
- st := d.stateLocked(path)
- if !st.dirty {
- st.dirty = true
- d.dirty.Inc()
- }
- st.repo = m.RepoRef
- st.marker = markerPath
- st.recovered = true
+ // A git process of the previous owner, of this boot, may still be
+ // committing (dropMarker).
st.settleAfter = time.Now().Add(d.recoveryGrace)
- if m.Untrusted != "" {
- st.untrusted = errors.New(m.Untrusted)
- }
- d.mu.Unlock()
}
+ d.mu.Unlock()
+ st.file.Unlock()
found = append(found, Recovered{Repo: m.RepoRef, Path: path, Reboot: reboot, Untrusted: m.Untrusted != ""})
}
}
diff --git a/internal/graft/corestore/localrefs/durability_model_test.go b/internal/graft/corestore/localrefs/durability_model_test.go
new file mode 100644
index 0000000..85c732d
--- /dev/null
+++ b/internal/graft/corestore/localrefs/durability_model_test.go
@@ -0,0 +1,492 @@
+package localrefs
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestMarkerLifecycleModel drives random interleavings of everything that
+// touches a dirty marker against a reference model: transactions that mark the
+// repository (MarkDirty), git processes that start and exit (Begin and end) and
+// write tables without fsync, durable steps (Settle, final or not), failed
+// fsyncs, the repository moved aside (Forget) and hydrated again, restarts of
+// the process (Recover of the same boot, with git processes of the previous
+// owner still finishing a write) and kernel restarts (Recover of another
+// boot), and the acceptance in place of a repository after a kernel restart.
+// After every step it asserts the two invariants of the LOCAL REFERENCE WRITES
+// RULE (doc/graft/PLAN7.md):
+//
+// 1. while a git process may still write the stack without fsync (it marked
+// the repository and has not exited, or it is a process of the previous
+// owner in the middle of a write), or the stack at the path holds a table,
+// a table list or a directory entry written without fsync since, the
+// marker is on disk;
+// 2. an untrusted repository is reported untrusted (the syncer never serves
+// it) and keeps its marker, until it is moved aside or accepted.
+//
+// The model learns what is durable from the fsyncs the durability makes (its
+// sync function), not from what the code under test reports. The steps are
+// sequential: what git's reference lock excludes (a write while a durable step
+// holds it) is not modelled; dropMarker refuses to remove the marker of an
+// existing repository without that lock.
+func TestMarkerLifecycleModel(t *testing.T) {
+ t.Parallel()
+
+ for seed := int64(1); seed <= 40; seed++ {
+ t.Run(fmt.Sprint(seed), func(t *testing.T) {
+ t.Parallel()
+ runMarkerModel(t, seed, 250)
+ })
+ }
+}
+
+// TestDropMarkerGuards verifies the guards of dropMarker that no caller
+// reaches in the model: the callers settle or move aside an untrusted
+// repository first, and take git's reference lock for an existing one.
+func TestDropMarkerGuards(t *testing.T) {
+ t.Parallel()
+ f := setupDurability(t)
+ d := f.durability(t, "boot-a", 0)
+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
+
+ st := d.lockState(f.path)
+ defer st.file.Unlock()
+
+ d.mu.Lock()
+ st.untrusted = errors.New("input/output error")
+ d.mu.Unlock()
+ dropped, err := d.dropMarker(f.path, st, true)
+ require.NoError(t, err)
+ require.False(t, dropped, "an untrusted repository keeps its marker")
+
+ d.mu.Lock()
+ st.untrusted = nil
+ d.mu.Unlock()
+ dropped, err = d.dropMarker(f.path, st, false)
+ require.NoError(t, err)
+ require.False(t, dropped, "the marker of an existing repository goes only under git's reference lock")
+ require.FileExists(t, f.markerPath(t, d))
+
+ dropped, err = d.dropMarker(f.path, st, true)
+ require.NoError(t, err)
+ require.True(t, dropped)
+ require.NoFileExists(t, f.markerPath(t, d))
+}
+
+// TestMovedRepositoryKeepsTheMarkerOfALiveProcess is the fourth review's
+// sequence: a repository is moved aside while a git process that marked it
+// still runs, the marker stays through a durable step of the missing path and
+// through the hydration, so the process's later write is covered; it goes once
+// the process exited and a step made the stack durable.
+func TestMovedRepositoryKeepsTheMarkerOfALiveProcess(t *testing.T) {
+ t.Parallel()
+ f := setupDurability(t)
+ d := f.durability(t, "boot-a", 0)
+ template := filepath.Join(t.TempDir(), "template.git")
+ copyTree(t, f.path, template)
+
+ end := d.Begin(f.path)
+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
+ require.NoError(t, os.Rename(f.path, filepath.Join(t.TempDir(), "aside")))
+ require.NoError(t, d.Forget(f.ctx, f.repo, f.path))
+ require.FileExists(t, f.markerPath(t, d))
+
+ require.NoError(t, d.Settle(f.path, true))
+ require.FileExists(t, f.markerPath(t, d), "the process names the path: a hydration there gets its writes")
+
+ copyTree(t, template, f.path)
+ require.NoError(t, d.Settle(f.path, true))
+ require.FileExists(t, f.markerPath(t, d))
+
+ end()
+ require.NoError(t, d.Settle(f.path, true))
+ require.NoFileExists(t, f.markerPath(t, d))
+ require.Empty(t, d.Tracked())
+}
+
+// TestRecoveredMarkerOfAMissingRepositoryIsSettled verifies that a marker found
+// at the recovery for a repository that no longer exists is removed by the
+// next durable step, instead of failing on its missing stack at every step.
+func TestRecoveredMarkerOfAMissingRepositoryIsSettled(t *testing.T) {
+ t.Parallel()
+ f := setupDurability(t)
+
+ old := f.durability(t, "boot-a", 0)
+ require.NoError(t, old.MarkDirty(f.ctx, f.repo, f.path))
+ require.NoError(t, old.Close())
+ require.NoError(t, os.Rename(f.path, filepath.Join(t.TempDir(), "aside")))
+
+ next := f.durability(t, "boot-a", 0)
+ _, err := next.Recover(f.ctx, []string{f.repo.Storage})
+ require.NoError(t, err)
+ require.NoError(t, next.Settle(f.path, false))
+ require.NoFileExists(t, f.markerPath(t, next))
+ require.Empty(t, next.Tracked())
+}
+
+// TestGitActivityPushesTheGraceForward verifies that the recovery grace counts
+// from the last git activity seen in the stack, not from the recovery: a git
+// process of the previous owner may start its auto-compaction right after its
+// commit.
+func TestGitActivityPushesTheGraceForward(t *testing.T) {
+ t.Parallel()
+ f := setupDurability(t)
+
+ old := f.durability(t, "boot-a", 0)
+ require.NoError(t, old.MarkDirty(f.ctx, f.repo, f.path))
+ require.NoError(t, old.Close())
+
+ next := f.durability(t, "boot-a", 0, WithRecoveryGrace(time.Hour))
+ _, err := next.Recover(f.ctx, []string{f.repo.Storage})
+ require.NoError(t, err)
+ // The grace of the recovery has passed.
+ next.mu.Lock()
+ next.repos[f.path].settleAfter = time.Now().Add(-time.Second)
+ next.mu.Unlock()
+
+ temp := filepath.Join(f.path, "reftable", "0x000000000009-0x000000000009-abcdef01.ref.temp.XyZ123")
+ require.NoError(t, os.WriteFile(temp, nil, 0o666))
+ before := time.Now()
+ require.NoError(t, next.Settle(f.path, true))
+ require.FileExists(t, f.markerPath(t, next))
+
+ require.NoError(t, os.Remove(temp))
+ require.NoError(t, next.Settle(f.path, true))
+ require.FileExists(t, f.markerPath(t, next), "the grace counts from the activity")
+ next.mu.Lock()
+ require.True(t, next.repos[f.path].settleAfter.After(before.Add(time.Hour-time.Minute)))
+ next.mu.Unlock()
+}
+
+// modelProc is a git process of the model.
+type modelProc struct {
+ end func()
+ marked bool
+ // orphan is a process of an owner that restarted: it is not counted by the
+ // running durability, and it only finishes the write it was in (inFlight,
+ // the temporary table it writes).
+ orphan bool
+ inFlight string
+}
+
+type markerModel struct {
+ t *testing.T
+ f durabilityFixture
+ rng *rand.Rand
+ d *Durability
+ boot int
+
+ template string
+ asideDir string
+ reftable string
+
+ procs []*modelProc
+ // pending are the tables at the path written by the model's git processes
+ // and not made durable since; listPending and dirPending the table list and
+ // the directory entries.
+ pending map[string]bool
+ listPending bool
+ dirPending bool
+ // untrusted is set by a failed fsync and by a kernel restart with a marker,
+ // rebootUntrusted only by the latter (the repository may then be accepted in
+ // place).
+ untrusted bool
+ rebootUntrusted bool
+ // failNext makes the fsyncs of the stack fail during the next durable step.
+ failNext bool
+ failing bool
+ tables int
+ moves int
+}
+
+func runMarkerModel(t *testing.T, seed int64, steps int) {
+ f := setupDurability(t)
+ m := &markerModel{
+ t: t,
+ f: f,
+ rng: rand.New(rand.NewSource(seed)),
+ template: filepath.Join(t.TempDir(), "template.git"),
+ asideDir: t.TempDir(),
+ reftable: filepath.Join(f.path, "reftable"),
+ pending: map[string]bool{},
+ }
+ copyTree(t, f.path, m.template)
+ m.d = m.newDurability()
+
+ for step := 0; step < steps; step++ {
+ op := m.step()
+ m.check(fmt.Sprintf("seed %d step %d (%s)", seed, step, op))
+ }
+}
+
+// sync is the durability's fsync: it fails the stack's files while failing is
+// set, and tells the model which files are durable.
+func (m *markerModel) sync(file *os.File) error {
+ name := file.Name()
+ inStack := name == m.reftable || filepath.Dir(name) == m.reftable
+ if m.failing && inStack {
+ return errors.New("input/output error")
+ }
+ if err := file.Sync(); err != nil {
+ return err
+ }
+ switch {
+ case name == m.reftable:
+ m.dirPending = false
+ case !inStack:
+ case filepath.Base(name) == "tables.list":
+ m.listPending = false
+ default:
+ delete(m.pending, filepath.Base(name))
+ }
+ return nil
+}
+
+func (m *markerModel) newDurability() *Durability {
+ m.boot++
+ d, err := NewDurability(m.f.locator, 0, WithBootID(fmt.Sprintf("boot-%d", m.boot)), WithSync(m.sync), WithRecoveryGrace(0))
+ require.NoError(m.t, err)
+ m.t.Cleanup(func() { require.NoError(m.t, d.Close()) })
+ m.f.own(m.t, d)
+ _, err = d.Recover(m.f.ctx, []string{m.f.repo.Storage})
+ require.NoError(m.t, err)
+ d.Activate()
+ return d
+}
+
+func (m *markerModel) exists() bool {
+ _, err := os.Lstat(m.f.path)
+ if errors.Is(err, fs.ErrNotExist) {
+ return false
+ }
+ require.NoError(m.t, err)
+ return true
+}
+
+func (m *markerModel) live(match func(*modelProc) bool) []*modelProc {
+ var procs []*modelProc
+ for _, p := range m.procs {
+ if match(p) {
+ procs = append(procs, p)
+ }
+ }
+ return procs
+}
+
+func (m *markerModel) remove(p *modelProc) {
+ for i, q := range m.procs {
+ if q == p {
+ m.procs = append(m.procs[:i], m.procs[i+1:]...)
+ return
+ }
+ }
+}
+
+func (m *markerModel) pick(procs []*modelProc) *modelProc {
+ return procs[m.rng.Intn(len(procs))]
+}
+
+// step runs one random operation and returns its name.
+func (m *markerModel) step() string {
+ own := func(p *modelProc) bool { return !p.orphan }
+ switch m.rng.Intn(14) {
+ case 0, 1:
+ if len(m.procs) < 4 {
+ m.procs = append(m.procs, &modelProc{end: m.d.Begin(m.f.path)})
+ return "begin"
+ }
+ case 2:
+ unmarked := m.live(func(p *modelProc) bool { return own(p) && !p.marked })
+ if len(unmarked) > 0 && m.exists() {
+ require.NoError(m.t, m.d.MarkDirty(m.f.ctx, m.f.repo, m.f.path))
+ m.pick(unmarked).marked = true
+ return "mark"
+ }
+ case 3, 4:
+ writers := m.live(func(p *modelProc) bool { return own(p) && p.marked })
+ if len(writers) > 0 && m.exists() {
+ m.writeTable(m.nextTable(), "")
+ return "write"
+ }
+ case 5:
+ procs := m.live(own)
+ if len(procs) > 0 {
+ p := m.pick(procs)
+ p.end()
+ m.remove(p)
+ return "end"
+ }
+ case 6, 7:
+ m.failing = m.failNext
+ m.failNext = false
+ err := m.d.Settle(m.f.path, m.rng.Intn(2) == 0)
+ m.failing = false
+ if errors.Is(err, ErrUntrusted) {
+ m.untrusted = true
+ } else {
+ require.NoError(m.t, err)
+ }
+ return "settle"
+ case 8:
+ m.failNext = true
+ return "fail the next fsyncs"
+ case 9:
+ if m.exists() {
+ m.moves++
+ require.NoError(m.t, os.Rename(m.f.path, filepath.Join(m.asideDir, fmt.Sprint(m.moves))))
+ require.NoError(m.t, m.d.Forget(m.f.ctx, m.f.repo, m.f.path))
+ m.pending = map[string]bool{}
+ m.listPending, m.dirPending = false, false
+ m.untrusted, m.rebootUntrusted = false, false
+ return "move aside"
+ }
+ case 10:
+ if !m.exists() {
+ // A hydration writes the repository durably before it moves it in.
+ copyTree(m.t, m.template, m.f.path)
+ return "hydrate"
+ }
+ case 11:
+ return m.restart()
+ case 12:
+ if orphans := m.live(func(p *modelProc) bool { return p.orphan }); len(orphans) > 0 {
+ p := m.pick(orphans)
+ if m.exists() && m.rng.Intn(4) != 0 {
+ m.writeTable(strings.TrimSuffix(filepath.Base(p.inFlight), ".temp.XyZ123"), p.inFlight)
+ }
+ // Otherwise the process is killed: its temporary table stays.
+ m.remove(p)
+ return "orphan exits"
+ }
+ case 13:
+ if m.rebootUntrusted && m.exists() {
+ // The repository is not a cache of the storage core: kept in place.
+ require.NoError(m.t, m.d.Forget(m.f.ctx, m.f.repo, m.f.path))
+ m.pending = map[string]bool{}
+ m.listPending, m.dirPending = false, false
+ m.untrusted, m.rebootUntrusted = false, false
+ return "accept in place"
+ }
+ return m.reboot()
+ }
+ return "nothing"
+}
+
+func (m *markerModel) nextTable() string {
+ m.tables++
+ return fmt.Sprintf("0x%012x-0x%012x-model%04d.ref", 100+m.tables, 100+m.tables, m.tables)
+}
+
+// writeTable writes a table and lists it, as git does, without fsync. A table
+// written through a temporary file (temp) is renamed into place.
+func (m *markerModel) writeTable(name, temp string) {
+ table := filepath.Join(m.reftable, name)
+ if temp != "" {
+ if err := os.Rename(temp, table); errors.Is(err, fs.ErrNotExist) {
+ // The repository it was writing was moved aside meanwhile.
+ return
+ } else {
+ require.NoError(m.t, err)
+ }
+ } else {
+ require.NoError(m.t, os.WriteFile(table, []byte(name), 0o666))
+ }
+ list := filepath.Join(m.reftable, "tables.list")
+ content, err := os.ReadFile(list)
+ require.NoError(m.t, err)
+ require.NoError(m.t, os.WriteFile(list+".model", append(content, []byte(name+"\n")...), 0o666))
+ require.NoError(m.t, os.Rename(list+".model", list))
+ m.pending[name] = true
+ m.listPending, m.dirPending = true, true
+}
+
+// restart restarts the process in the same boot: its git processes that may
+// write become orphans, half of them in the middle of a table write.
+func (m *markerModel) restart() string {
+ require.NoError(m.t, m.d.Close())
+ var orphans []*modelProc
+ for _, p := range m.procs {
+ if p.orphan || !p.marked || !m.exists() || m.rng.Intn(2) == 0 {
+ continue
+ }
+ temp := filepath.Join(m.reftable, m.nextTable()+".temp.XyZ123")
+ require.NoError(m.t, os.WriteFile(temp, []byte("in flight"), 0o666))
+ orphans = append(orphans, &modelProc{orphan: true, inFlight: temp})
+ }
+ m.procs = orphans
+ m.boot--
+ m.d = m.newDurability()
+ return "restart"
+}
+
+// reboot restarts the kernel: every process dies, and a repository with a
+// marker becomes untrusted (its writes may never have reached the disk).
+func (m *markerModel) reboot() string {
+ require.NoError(m.t, m.d.Close())
+ _, err := os.Stat(m.f.markerPath(m.t, m.d))
+ marked := err == nil
+ m.procs = nil
+ if marked {
+ m.untrusted = true
+ m.rebootUntrusted = true
+ }
+ m.d = m.newDurability()
+ return "reboot"
+}
+
+// check asserts the invariants.
+func (m *markerModel) check(where string) {
+ _, err := os.Stat(m.f.markerPath(m.t, m.d))
+ markerOnDisk := err == nil
+
+ // A process of the previous owner may still write only while its temporary
+ // table is there: git renames it into the stack by its path.
+ mayWrite := len(m.live(func(p *modelProc) bool {
+ if p.orphan {
+ _, err := os.Stat(p.inFlight)
+ return err == nil
+ }
+ return p.marked
+ })) > 0
+ unsynced := m.exists() && (len(m.pending) > 0 || m.listPending || m.dirPending)
+ if mayWrite || unsynced {
+ require.True(m.t, markerOnDisk, "%s: a write may not be durable (writers %t, pending %v, list %t, dir %t) but the marker is gone",
+ where, mayWrite, m.pending, m.listPending, m.dirPending)
+ }
+ require.Equal(m.t, m.untrusted, m.d.Untrusted(m.f.path), "%s: untrusted", where)
+ if m.untrusted {
+ require.True(m.t, markerOnDisk, "%s: an untrusted repository lost its marker", where)
+ }
+}
+
+// copyTree copies the directory src to dst.
+func copyTree(t *testing.T, src, dst string) {
+ t.Helper()
+ require.NoError(t, filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ rel, err := filepath.Rel(src, path)
+ if err != nil {
+ return err
+ }
+ target := filepath.Join(dst, rel)
+ if entry.IsDir() {
+ return os.MkdirAll(target, 0o777)
+ }
+ content, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(target, content, 0o666)
+ }))
+}
diff --git a/internal/graft/corestore/localrefs/durability_test.go b/internal/graft/corestore/localrefs/durability_test.go
index 974714c..e13f786 100644
--- a/internal/graft/corestore/localrefs/durability_test.go
+++ b/internal/graft/corestore/localrefs/durability_test.go
@@ -585,14 +585,19 @@ func TestRecoverTellsAKernelRestartFromAProcessRestart(t *testing.T) {
require.NoError(t, sameBoot.Settle(f.path, false))
require.NoFileExists(t, f.markerPath(t, sameBoot))
- // Another boot: the writes may not have reached the disk.
+ // Another boot: the writes may not have reached the disk. The repository is
+ // untrusted until the caller moves it aside (Forget): never served, its
+ // marker never removed by a durable step.
require.NoError(t, sameBoot.MarkDirty(f.ctx, f.repo, f.path))
require.NoError(t, sameBoot.Close())
otherBoot := f.durability(t, "boot-b", 0)
found, err = otherBoot.Recover(f.ctx, storages)
require.NoError(t, err)
require.Equal(t, []Recovered{{Repo: f.repo, Path: f.path, Reboot: true}}, found)
- require.Empty(t, otherBoot.Dirty())
+ require.Equal(t, []string{f.path}, otherBoot.Dirty())
+ require.True(t, otherBoot.Untrusted(f.path))
+ require.ErrorIs(t, otherBoot.Settle(f.path, true), ErrUntrusted)
+ require.FileExists(t, f.markerPath(t, otherBoot))
}
// TestRecoverRemovesWhatADurableStepThatDiedLeft verifies that the reference
diff --git a/internal/graft/corestore/maintenance/maintenance.go b/internal/graft/corestore/maintenance/maintenance.go
index db09ec5..3f79212 100644
--- a/internal/graft/corestore/maintenance/maintenance.go
+++ b/internal/graft/corestore/maintenance/maintenance.go
@@ -245,17 +245,10 @@ func (r *Runner) maintain(ctx context.Context, record corestore.Repository) (str
return "", nil
}
- leaseCtx, cancel := context.WithCancel(ctx)
- renewed := r.renew(leaseCtx, cancel, record.ID)
- defer func() {
- cancel()
- <-renewed
- releaseCtx, cancelRelease := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
- defer cancelRelease()
- if err := r.core.Meta.ReleaseMaintenanceLease(releaseCtx, record.ID, r.holder); err != nil {
- r.logger.WithError(err).WithField("repository_id", record.ID).Warn("storage core: release the maintenance lease; it expires by itself")
- }
- }()
+ leaseCtx, release := HoldLease(ctx, r.core, r.logger, record.ID, r.holder, r.cfg.LeaseTTL, func() {
+ r.failures.WithLabelValues(stepLease).Inc()
+ })
+ defer release()
due, err := r.checkpointDue(leaseCtx, record)
if err != nil {
@@ -282,36 +275,49 @@ func (r *Runner) maintain(ctx context.Context, record corestore.Repository) (str
return "", nil
}
-// renew renews the lease of repoID every third of its time to live until ctx
-// ends, and cancels the run (cancel) when the lease is lost: another process
-// may then do the same work, which is wasted but correct. The returned channel
-// is closed when it stops.
-func (r *Runner) renew(ctx context.Context, cancel context.CancelFunc, repoID int64) <-chan struct{} {
+// HoldLease keeps the maintenance lease of repoID, which holder acquired with
+// time to live ttl, for the work done under the returned context: it renews the
+// lease every third of ttl, and when the lease is lost it calls lost and
+// cancels the context (another process may then do the same work, which is
+// wasted but correct). release stops the renewal, waits for it, and releases
+// the lease; the background maintenance and the CollectGarbage request use it
+// alike.
+func HoldLease(ctx context.Context, core *corestore.Core, logger log.Logger, repoID int64, holder string, ttl time.Duration, lost func()) (context.Context, func()) {
+ leaseCtx, cancel := context.WithCancel(ctx)
done := make(chan struct{})
go func() {
defer close(done)
- ticker := time.NewTicker(r.cfg.LeaseTTL / 3)
+ ticker := time.NewTicker(ttl / 3)
defer ticker.Stop()
for {
select {
- case <-ctx.Done():
+ case <-leaseCtx.Done():
return
case <-ticker.C:
}
- held, err := r.core.Meta.RenewMaintenanceLease(ctx, repoID, r.holder, r.cfg.LeaseTTL)
- if ctx.Err() != nil {
+ held, err := core.Meta.RenewMaintenanceLease(leaseCtx, repoID, holder, ttl)
+ if leaseCtx.Err() != nil {
return
}
if err != nil || !held {
- r.logger.WithError(err).WithField("repository_id", repoID).WithField("held", held).
- Warn("storage core: the maintenance lease was lost, the run of the repository stops")
- r.failures.WithLabelValues(stepLease).Inc()
+ logger.WithError(err).WithField("repository_id", repoID).WithField("held", held).
+ Warn("storage core: the maintenance lease was lost, the work on the repository stops")
+ lost()
cancel()
return
}
}
}()
- return done
+
+ return leaseCtx, func() {
+ cancel()
+ <-done
+ releaseCtx, cancelRelease := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
+ defer cancelRelease()
+ if err := core.Meta.ReleaseMaintenanceLease(releaseCtx, repoID, holder); err != nil {
+ logger.WithError(err).WithField("repository_id", repoID).Warn("storage core: release the maintenance lease; it expires by itself")
+ }
+ }
}
// checkpointDue reports whether the repository needs a checkpoint: enough
diff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.go
index c45be6d..5f4b418 100644
--- a/internal/graft/corestore/maintenance/maintenance_test.go
+++ b/internal/graft/corestore/maintenance/maintenance_test.go
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"fmt"
"os"
+ "sync/atomic"
"testing"
"time"
@@ -329,6 +330,38 @@ func TestRunOnceStopsWhenTheLeaseIsLost(t *testing.T) {
require.Equal(t, float64(1), e.metric("graft_storage_maintenance_runs_total", "error"))
}
+// TestHoldLeaseRenewsUntilReleased verifies the lease the background
+// maintenance and the CollectGarbage request hold while they work: no other
+// holder takes it for several times its time to live, and it is free once
+// released.
+func TestHoldLeaseRenewsUntilReleased(t *testing.T) {
+ t.Parallel()
+ ctx := testhelper.Context(t)
+ e := newEnv(t, coretest.Options{}, config3(0, 0))
+ const ttl = 300 * time.Millisecond
+ const holder = "node-a/1/request-1"
+
+ acquired, err := e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, holder, ttl)
+ require.NoError(t, err)
+ require.True(t, acquired)
+ var lost atomic.Bool
+ leaseCtx, release := maintenance.HoldLease(ctx, e.core, testhelper.NewLogger(t), e.repo.ID, holder, ttl, func() { lost.Store(true) })
+
+ require.Never(t, func() bool {
+ acquired, err := e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, "node-b/1", ttl)
+ require.NoError(t, err)
+ return acquired
+ }, 5*ttl, ttl/10, "the lease is renewed")
+ require.NoError(t, leaseCtx.Err())
+ require.False(t, lost.Load())
+
+ release()
+ require.ErrorIs(t, leaseCtx.Err(), context.Canceled)
+ acquired, err = e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, "node-b/1", ttl)
+ require.NoError(t, err)
+ require.True(t, acquired, "released")
+}
+
func (e *env) bucketKeys() map[string]bool {
e.t.Helper()
keys := map[string]bool{}
diff --git a/internal/graft/corestore/syncer/durability.go b/internal/graft/corestore/syncer/durability.go
index f428854..e9795d1 100644
--- a/internal/graft/corestore/syncer/durability.go
+++ b/internal/graft/corestore/syncer/durability.go
@@ -4,11 +4,13 @@ import (
"context"
"errors"
"fmt"
+ "io/fs"
"os"
"path/filepath"
"sync"
"time"
+ "github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -41,6 +43,8 @@ func (s *Syncer) StartLocalReferences(ctx context.Context, storages []string) (s
if !d.Enabled() {
return func() {}, nil
}
+ kept := append([]string(nil), storages...)
+ s.keptStorages.Store(&kept)
owned, err := d.TryOwn(ctx, storages)
if err != nil {
@@ -73,7 +77,7 @@ func (s *Syncer) StartLocalReferences(ctx context.Context, storages []string) (s
}
finalCtx, cancelFinal := context.WithTimeout(context.WithoutCancel(ctx), finalStepTimeout)
defer cancelFinal()
- for _, path := range d.Dirty() {
+ for _, path := range d.Tracked() {
s.durableStep(finalCtx, path, true)
}
}, nil
@@ -111,7 +115,7 @@ func (s *Syncer) runDurableSteps(ctx context.Context, storages []string, recover
s.logger.Info("storage core: this process owns the deferred durability of the storages now")
}
- for _, path := range d.Dirty() {
+ for _, path := range d.Tracked() {
s.durableStep(ctx, path, false)
}
}
@@ -131,8 +135,9 @@ func (s *Syncer) runDurableSteps(ctx context.Context, storages []string, recover
//
// Only a cache of the storage core is moved (storageCoreCache). Any other
// marker of another boot (the repository was removed, or replaced by one being
-// imported, whose local copy may be the only one) is dropped; an untrusted one
-// is kept, with its repository untrusted, and logged.
+// imported, whose local copy may be the only one) is handed over (Forget); an
+// untrusted one is kept, with its repository untrusted, so never served, and
+// logged.
func (s *Syncer) RecoverLocalReferences(ctx context.Context, storages []string) error {
d := s.core.RefDurability
found, err := d.Recover(ctx, storages)
@@ -157,12 +162,12 @@ func (s *Syncer) RecoverLocalReferences(ctx context.Context, storages []string)
if rec.Reboot {
s.logger.WithField("repository_path", rec.Path).
Warn("storage core: dropped the dirty marker of a local repository that is not a cache of the storage core")
- if err := d.Forget(ctx, rec.Repo, rec.Path); err != nil {
+ if err := s.forget(ctx, rec.Repo, rec.Path); err != nil {
return err
}
} else {
s.logger.WithField("repository_path", rec.Path).
- Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place, untrusted")
+ Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place, untrusted, and not served")
}
continue
}
@@ -177,11 +182,13 @@ func (s *Syncer) RecoverLocalReferences(ctx context.Context, storages []string)
// storageCoreCache reports whether the local repository at path is a cache of
// the storage core that a hydration can rebuild: the database holds it with
// changesets (an import that is not complete has none, and its local copy may
-// be the only one). keep reports that the copy holds history written in shadow
-// mode and not imported yet (checkReplaceable, the guard of a sync's
+// be the only one). keep reports that the copy may hold history written in
+// shadow mode and not imported yet (checkReplaceable, the guard of a sync's
// hydration): the copy is moved aside all the same, so that it is never served,
-// but kept, so that the history is not lost. An error (the database, or the
-// repository's files, cannot be read) is returned: the caller tries again.
+// but kept, so that the history is not lost. A local changelog or review file
+// that cannot be read (Internal) means keep too: whether it holds such history
+// is unknown, and a corrupt file must not stop the node. An error of the
+// database is returned: the caller tries again.
func (s *Syncer) storageCoreCache(ctx context.Context, repo localrefs.RepoRef, path string) (cache, keep bool, err error) {
r, err := s.core.Meta.GetRepository(ctx, corestore.RepoKey{Storage: repo.Storage, RelativePath: repo.RelativePath})
if errors.Is(err, corestore.ErrRepositoryNotFound) {
@@ -202,7 +209,12 @@ func (s *Syncer) storageCoreCache(ctx context.Context, repo localrefs.RepoRef, p
wm, ok := s.readWatermark(path)
if err := s.checkReplaceable(ctx, r, path, wm, ok); err != nil {
- if status.Code(err) == codes.FailedPrecondition {
+ switch status.Code(err) {
+ case codes.FailedPrecondition:
+ return true, true, nil
+ case codes.Internal:
+ s.logger.WithError(err).WithField("repository_path", path).
+ Error("storage core: the history files of an untrusted local repository cannot be read; it is kept for an operator")
return true, true, nil
}
return false, false, err
@@ -210,6 +222,16 @@ func (s *Syncer) storageCoreCache(ctx context.Context, repo localrefs.RepoRef, p
return true, false, nil
}
+// forget is Forget under the repository lock.
+func (s *Syncer) forget(ctx context.Context, repo localrefs.RepoRef, path string) error {
+ release, err := s.lock(ctx, path)
+ if err != nil {
+ return err
+ }
+ defer release()
+ return s.core.RefDurability.Forget(ctx, repo, path)
+}
+
// durableStep makes the references of the repository at path durable under the
// repository lock (final: without waiting for the repository to be idle). When
// the kernel reports that a write failed, the repository is moved aside at
@@ -253,9 +275,9 @@ func (s *Syncer) durableStep(ctx context.Context, path string, final bool) {
return
}
if !cache {
- // Its marker stays (the untrusted state is sticky): after a reboot the
- // recovery decides again.
- logger.Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place")
+ // Its marker stays and it is not served (the untrusted state is sticky):
+ // after a restart the recovery decides again.
+ logger.Error("storage core: the references of a repository that is not a cache of the storage core may have lost writes; it is kept in place and not served")
return
}
if err := s.distrustLocked(ctx, untrusted.Repo, path, localrefs.ReasonSyncFailed, untrusted, keep); err != nil {
@@ -281,6 +303,72 @@ func (s *Syncer) distrust(ctx context.Context, repo localrefs.RepoRef, path, rea
// operator imports what they hold or removes them.
const KeptDirName = "+gitaly/graft-untrusted"
+var (
+ keptRepositoriesDesc = prometheus.NewDesc(
+ "graft_storage_kept_untrusted_repositories",
+ "Number of untrusted local repositories kept in "+KeptDirName+" for an operator, by storage: they hold history not imported yet (doc/graft/STORAGE.md, kept copies).",
+ []string{"storage"}, nil)
+ keptBytesDesc = prometheus.NewDesc(
+ "graft_storage_kept_untrusted_bytes",
+ "Bytes of the untrusted local repositories kept in "+KeptDirName+", by storage.",
+ []string{"storage"}, nil)
+)
+
+// collectKept reports the kept untrusted copies of the storages. It reads the
+// directories at every scrape: they are empty unless an operator has copies to
+// import, which the metric is there to show. A storage whose directory cannot
+// be read is logged and left out.
+func (s *Syncer) collectKept(metrics chan<- prometheus.Metric) {
+ storages := s.keptStorages.Load()
+ if storages == nil {
+ return
+ }
+ for _, storageName := range *storages {
+ root, err := s.locator.GetStorageByName(context.Background(), storageName)
+ if err != nil {
+ s.logger.WithError(err).WithField("storage", storageName).Warn("storage core: measure the kept untrusted repositories")
+ continue
+ }
+ repos, size, err := keptSize(filepath.Join(root, KeptDirName))
+ if err != nil {
+ s.logger.WithError(err).WithField("storage", storageName).Warn("storage core: measure the kept untrusted repositories")
+ continue
+ }
+ metrics <- prometheus.MustNewConstMetric(keptRepositoriesDesc, prometheus.GaugeValue, float64(repos), storageName)
+ metrics <- prometheus.MustNewConstMetric(keptBytesDesc, prometheus.GaugeValue, float64(size), storageName)
+ }
+}
+
+// keptSize returns the number of kept repositories in dir (one directory each)
+// and the bytes of their files.
+func keptSize(dir string) (int, int64, error) {
+ entries, err := os.ReadDir(dir)
+ if errors.Is(err, fs.ErrNotExist) {
+ return 0, 0, nil
+ }
+ if err != nil {
+ return 0, 0, err
+ }
+ var size int64
+ err = filepath.WalkDir(dir, func(_ string, entry fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if entry.Type().IsRegular() {
+ info, err := entry.Info()
+ if err != nil {
+ return err
+ }
+ size += info.Size()
+ }
+ return nil
+ })
+ if err != nil {
+ return 0, 0, err
+ }
+ return len(entries), size, nil
+}
+
// distrustLocked moves the local repository at path out of the way, then drops
// its dirty marker: the next request hydrates it. It goes into the temporary
// directory of its storage and is removed there in the background, or, with
diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.go
index d55a668..13936fc 100644
--- a/internal/graft/corestore/syncer/syncer.go
+++ b/internal/graft/corestore/syncer/syncer.go
@@ -132,6 +132,10 @@ type Syncer struct {
syncChangesets prometheus.Counter
syncBytes prometheus.Counter
syncSeeds prometheus.Counter
+
+ // keptStorages are the storages whose kept untrusted copies (KeptDirName)
+ // the metrics report, set by StartLocalReferences.
+ keptStorages atomic.Pointer[[]string]
}
type repoLock struct {
@@ -307,6 +311,8 @@ func (s *Syncer) Describe(descs chan<- *prometheus.Desc) {
s.syncChangesets.Describe(descs)
s.syncBytes.Describe(descs)
s.syncSeeds.Describe(descs)
+ descs <- keptRepositoriesDesc
+ descs <- keptBytesDesc
}
// Collect implements prometheus.Collector.
@@ -319,6 +325,7 @@ func (s *Syncer) Collect(metrics chan<- prometheus.Metric) {
s.syncChangesets.Collect(metrics)
s.syncBytes.Collect(metrics)
s.syncSeeds.Collect(metrics)
+ s.collectKept(metrics)
}
func (s *Syncer) syncTimeout() time.Duration {
@@ -349,6 +356,12 @@ func (s *Syncer) sync(ctx context.Context, key corestore.RepoKey) error {
if err := ctx.Err(); err != nil {
return structerr.NewUnavailable("storage core: sync: %w", err)
}
+ // A repository whose references may have lost writes is never served:
+ // the durable step moves a cache aside (the next request hydrates it), and
+ // any other one waits for an operator.
+ if s.core.RefDurability.Untrusted(path) {
+ return structerr.NewUnavailable("storage core: the local references of %s may have lost writes; the repository is not served until it is rebuilt", key)
+ }
// The record is read again on every round: a restart can mean that the
// repository was removed and registered again under a new ID meanwhile, and
diff --git a/internal/graft/corestore/syncer/syncer_localrefs_test.go b/internal/graft/corestore/syncer/syncer_localrefs_test.go
index a2e2a3b..8afc50a 100644
--- a/internal/graft/corestore/syncer/syncer_localrefs_test.go
+++ b/internal/graft/corestore/syncer/syncer_localrefs_test.go
@@ -11,6 +11,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gittest"
@@ -320,6 +322,75 @@ func TestSyncerUntrustedRepositoryWithShadowHistoryIsKept(t *testing.T) {
require.Empty(t, e.markers(t))
}
+// TestSyncerNeverServesAnUntrustedRepository verifies that a repository whose
+// references may have lost writes is not served while it is still in place
+// (its move aside failed): every request is Unavailable until the next durable
+// step moves it aside, and the request after that hydrates it.
+func TestSyncerNeverServesAnUntrustedRepository(t *testing.T) {
+ t.Parallel()
+
+ var failing atomic.Bool
+ e, repo, _ := caughtUpDirty(t, localrefs.WithSync(failReferences(&failing)))
+ tmpDir, err := e.locator.TempDir(e.storageName)
+ require.NoError(t, err)
+ require.NoError(t, os.MkdirAll(tmpDir, 0o700))
+
+ failing.Store(true)
+ require.NoError(t, os.Chmod(tmpDir, 0o500))
+ e.sy.DurableStep(e.ctx, e.localPath(), false)
+ require.NoError(t, os.Chmod(tmpDir, 0o700))
+ failing.Store(false)
+ require.DirExists(t, e.localPath(), "the move aside failed")
+
+ err = e.sy.EnsureFresh(e.ctx, e.pbRepo(localRelPath))
+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)
+
+ e.sy.DurableStep(e.ctx, e.localPath(), false)
+ require.NoDirExists(t, e.localPath())
+ require.NoError(t, e.sy.EnsureFresh(e.ctx, e.pbRepo(localRelPath)))
+ e.verifySynced(t, repo)
+}
+
+// TestSyncerRecoveryKeepsACopyWhoseHistoryCannotBeRead verifies that a local
+// changelog that cannot be read does not stop the recovery, so the node: the
+// untrusted copy is moved aside and kept for an operator, as one with history
+// not imported yet, and the kept-copies metrics report it.
+func TestSyncerRecoveryKeepsACopyWhoseHistoryCannotBeRead(t *testing.T) {
+ t.Parallel()
+
+ e, repo, _ := caughtUpDirty(t, localrefs.WithBootID("boot-a"))
+ // A directory where the changelog file is: reading it fails.
+ require.NoError(t, os.MkdirAll(changelog.Path(e.localPath()), 0o700))
+
+ coretest.EnableLocalRefs(t, e.core, e.cfg, localrefs.WithBootID("boot-b"))
+ e.sy = syncer.New(e.core, e.logger, e.locator, e.repoFactory, e.gitCmdFactory)
+ stop, err := e.sy.StartLocalReferences(e.ctx, []string{e.storageName})
+ require.NoError(t, err)
+ defer stop()
+
+ require.NoDirExists(t, e.localPath(), "never served")
+ kept, err := filepath.Glob(filepath.Join(e.storagePath, syncer.KeptDirName, "*", "repo.git"))
+ require.NoError(t, err)
+ require.Len(t, kept, 1)
+ require.Empty(t, e.markers(t))
+
+ reg := prometheus.NewPedanticRegistry()
+ require.NoError(t, reg.Register(e.sy))
+ families, err := reg.Gather()
+ require.NoError(t, err)
+ values := map[string]float64{}
+ for _, family := range families {
+ for _, metric := range family.GetMetric() {
+ values[family.GetName()] = metric.GetGauge().GetValue()
+ }
+ }
+ require.Equal(t, float64(1), values["graft_storage_kept_untrusted_repositories"])
+ require.Positive(t, values["graft_storage_kept_untrusted_bytes"])
+
+ require.NoError(t, e.sy.EnsureFresh(e.ctx, e.pbRepo(localRelPath)))
+ e.verifySynced(t, repo)
+}
+
// TestSyncerUntrustedRepositoryThatIsGoneLosesItsMarker verifies that an
// untrusted repository that was moved away meanwhile (a stale cache) does not
// keep its marker forever.