Skip to content

Git designed for AI agents.

graft: a queue of tickets for the lands that lose a round

Federico Elia4fe5536624be
diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.go
index 3153ab5..ce5ccfd 100644
--- a/internal/gitaly/service/operations/land_preresolve.go
+++ b/internal/gitaly/service/operations/land_preresolve.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "sort"
"strings"
"time"
@@ -70,6 +71,9 @@ type landPreResolution struct {
// verifyAll says that a verification is configured: every resolution then needs a verified tree
// to land, a rule's as well as a model's (15.9).
verifyAll bool
+ // ticket is the land's place in the queue of its files, once it has lost a round (land_tickets.go).
+ // The lane lets the holder's verdicts land before those of any other land on the same files.
+ ticket *landTicket
}
func newLandPreResolution() *landPreResolution {
@@ -141,6 +145,9 @@ type landNeedsResolution struct {
// the batch ran out of attempts: the tree it was about to land is another one, and the land goes
// out again as it does for an unverified tree.
moved bool
+ // queued says that another land holds the ticket of one of the files (paths): this one waits for
+ // its turn before it resolves again.
+ queued bool
}
func (e *landNeedsResolution) Error() string {
@@ -262,7 +269,15 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
return ctx.Err() == nil && errors.Is(resolveCtx.Err(), context.DeadlineExceeded)
}
rounds := s.resolveCfg.ResolveRoundsOrDefault()
+ used := 0
+ // The land keeps its ticket until it ends, whatever the ending: it lands, the deadline stops it, or
+ // it fails.
+ defer func() {
+ pre.ticket.release()
+ landResolutionRounds.Observe(float64(used))
+ }()
for round := 1; round <= rounds; round++ {
+ used = round
if err := s.landPreResolve(resolveCtx, in, committer, pre); err != nil {
// The deadline ended a step of the round: the listing, the tree, the resolution or the
// verification. A refusal of the policy is not a deadline.
@@ -281,17 +296,85 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
}
return nil, err
}
- resp, err := s.landOnce(ctx, in, committer, nil, pre)
- if !errors.As(err, &need) {
- return resp, err
+ // Another land holds the ticket of one of these files: this one does not enter the lane, which would
+ // only hand it back (and which every land of the branch waits for), and waits for its turn.
+ if paths := pre.currentPaths(); s.landTicketsBlock(in, paths, pre) {
+ need = &landNeedsResolution{paths: paths, queued: true}
+ } else {
+ resp, err := s.landOnce(ctx, in, committer, nil, pre)
+ if !errors.As(err, &need) {
+ return resp, err
+ }
}
if deadline() {
+ pre.ticket.release()
return s.landFinalPass(ctx, in, committer, pre, landStoppedReason(need, timeout))
}
+ // The land lost the round, to a land that changed its files (or to the tree that moved, with a
+ // verification): it takes its turn on them. From now on it is the only land of the branch that
+ // resolves these files again, and the others wait without calling the resolver.
+ if err := s.landTakeTicket(resolveCtx, in, pre, need); err != nil {
+ if deadline() {
+ pre.ticket.release()
+ return s.landFinalPass(ctx, in, committer, pre, landQueueReason(timeout))
+ }
+ switch {
+ case errors.Is(ctx.Err(), context.DeadlineExceeded):
+ return nil, structerr.NewDeadlineExceeded("waiting for the other lands on the same files: %w", err)
+ case ctx.Err() != nil:
+ return nil, structerr.NewCanceled("waiting for the other lands on the same files: %w", err)
+ }
+ return nil, err
+ }
}
+ pre.ticket.release()
return s.landFinalPass(ctx, in, committer, pre, landStoppedReason(need, timeout))
}
+// currentPaths are the paths of the conflicting files the land has at the latest tip it looked at.
+func (p *landPreResolution) currentPaths() []string {
+ paths := make([]string, 0, len(p.current))
+ for k := range p.current {
+ paths = append(paths, k.path)
+ }
+ sort.Strings(paths)
+ return paths
+}
+
+// landTicketsBlock reports whether another land holds the ticket of one of paths.
+func (s *Server) landTicketsBlock(in *gitalypb.UserLandRequest, paths []string, pre *landPreResolution) bool {
+ if len(paths) == 0 {
+ return false
+ }
+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
+ return landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket)
+}
+
+// landTakeTicket makes the land hold the tickets of the files it needs resolved, waiting for the lands
+// that lost a round before it (land_tickets.go). A land that holds tickets for all of them keeps them.
+// One that now needs more files gives its tickets back and takes the larger set at the tail: tickets
+// are taken in sorted order, so that no two lands wait for each other.
+func (s *Server) landTakeTicket(ctx context.Context, in *gitalypb.UserLandRequest, pre *landPreResolution, need *landNeedsResolution) error {
+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
+ keys := landTicketKeys(scope, need.paths, s.resolveCfg.Verify.Enabled)
+ if pre.ticket.covers(keys) {
+ return nil
+ }
+ pre.ticket.release()
+ pre.ticket = nil
+ t, err := landTickets.acquire(ctx, keys)
+ if err != nil {
+ return err
+ }
+ pre.ticket = t
+ return nil
+}
+
+// landQueueReason is why a land that waited for its turn on its files could not go on.
+func landQueueReason(timeout time.Duration) string {
+ return fmt.Sprintf("the time ran out waiting for the other lands on the same files (deadline of %s)", timeout)
+}
+
// landCallerGaveUp is the status of a land whose caller gave up, or whose own deadline passed, while
// it waited for a slot or verified: the caller's cancellation, not an error of the server.
func landCallerGaveUp(ctx context.Context, err error) error {
diff --git a/internal/gitaly/service/operations/land_queue_test.go b/internal/gitaly/service/operations/land_queue_test.go
new file mode 100644
index 0000000..0415253
--- /dev/null
+++ b/internal/gitaly/service/operations/land_queue_test.go
@@ -0,0 +1,223 @@
+package operations
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/feder-cr/git-ai/internal/git"
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+)
+
+// The tests of the queue of the lands that lose a round (land_tickets.go): N lands that conflict on
+// one file, a resolver that unions the two sides (a rule: the file keeps every line), and the lands
+// that wait for their turn without calling the resolver.
+
+// unionResolver writes a resolver command that resolves every file of the request with the union of the
+// lines of the two sides, as a rule, and counts its runs in dir/calls.log. The runs are concurrent, so
+// each reads its own request file.
+func unionResolver(t *testing.T) (string, []string) {
+ t.Helper()
+
+ dir := t.TempDir()
+ script := fmt.Sprintf(`#!/bin/sh
+DIR='%s'
+echo x >> "$DIR/calls.log"
+cat > "$DIR/req-$$.json"
+python3 - "$DIR/req-$$.json" <<'PY'
+import base64, json, sys
+req = json.load(open(sys.argv[1]))
+files = []
+for f in req["files"]:
+ ours = base64.b64decode(f["ours"]["content"]).decode().splitlines()
+ theirs = base64.b64decode(f["theirs"]["content"]).decode().splitlines()
+ merged = ours + [l for l in theirs if l not in ours]
+ files.append({"path": f["path"], "action": "write", "mode": "100644", "origin": "rule",
+ "content": base64.b64encode(("\n".join(merged) + "\n").encode()).decode()})
+json.dump({"version": 1, "resolver": "graft-resolve/union", "files": files}, sys.stdout)
+PY
+`, dir)
+ path := filepath.Join(dir, "resolver.sh")
+ require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
+ return dir, []string{path}
+}
+
+// unionCalls is the number of runs of the resolver of dir.
+func unionCalls(t *testing.T, dir string) int {
+ t.Helper()
+
+ raw, err := os.ReadFile(filepath.Join(dir, "calls.log"))
+ if os.IsNotExist(err) {
+ return 0
+ }
+ require.NoError(t, err)
+ return len(raw) / 2
+}
+
+// queueFixture is a repository where main (tip) changed a and b, and n commits from the base each
+// change a in their own way (conflict on a only; b is left as the base has it, so that it merges
+// clean), plus one commit that changes b only.
+type queueFixture struct {
+ resolveFixture
+ dir string
+ onA []git.ObjectID
+ onlyB git.ObjectID
+ scope string
+ keyOfA string
+ keyOfB string
+ resolver config.GraftResolver
+}
+
+func newQueueFixture(t *testing.T, n int, tune func(*config.GraftResolver)) queueFixture {
+ t.Helper()
+
+ dir, argv := unionResolver(t)
+ rcfg := enabledResolver(argv)
+ if tune != nil {
+ tune(&rcfg)
+ }
+ fx := newResolveFixtureWith(t, rcfg,
+ []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "base\n"}, {Mode: "100644", Path: "b", Content: "base\n"}},
+ []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "ours\n"}, {Mode: "100644", Path: "b", Content: "ours\n"}},
+ []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "theirs-0\n"}, {Mode: "100644", Path: "b", Content: "base\n"}},
+ )
+ q := queueFixture{resolveFixture: fx, dir: dir, resolver: rcfg}
+ q.onA = append(q.onA, fx.commit)
+ for i := 1; i < n; i++ {
+ q.onA = append(q.onA, gittest.WriteCommit(t, fx.cfg, fx.repoPath, gittest.WithParents(fx.base),
+ gittest.WithTreeEntries(
+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: fmt.Sprintf("theirs-%d\n", i)},
+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"})))
+ }
+ q.onlyB = gittest.WriteCommit(t, fx.cfg, fx.repoPath, gittest.WithParents(fx.base),
+ gittest.WithTreeEntries(
+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "base\n"},
+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "theirs-b\n"}))
+ q.scope = landTicketScope(fx.repoProto.GetStorageName(), fx.repoProto.GetRelativePath(), []byte(git.DefaultBranch))
+ q.keyOfA = landTicketKeys(q.scope, []string{"a"}, false)[0]
+ q.keyOfB = landTicketKeys(q.scope, []string{"b"}, false)[0]
+ return q
+}
+
+func (q queueFixture) land(commit git.ObjectID) (*gitalypb.UserLandResponse, error) {
+ req := q.request("Land " + commit.String())
+ req.CommitId = commit.String()
+ return q.client.UserLand(q.ctx, req)
+}
+
+// branchBlob reads a file at the tip of the branch.
+func (q queueFixture) branchBlob(t *testing.T, path string) string {
+ t.Helper()
+
+ return strings.TrimSpace(string(gittest.Exec(t, q.cfg, "-C", q.repoPath, "show", git.DefaultBranch+":"+path)))
+}
+
+// TestUserLand_theLandsOnAHotFileAllLand: more lands than resolve_rounds conflict on one file. Without an
+// order each one loses a round to every land that lands before it, and the last ones run out of rounds;
+// with the queue each loses at most one, and the resolver is called for a land after its first round only
+// when it is its turn.
+func TestUserLand_theLandsOnAHotFileAllLand(t *testing.T) {
+ t.Parallel()
+
+ const lands = 8
+ q := newQueueFixture(t, lands, func(c *config.GraftResolver) { c.ResolveRounds = 3 })
+
+ var wg sync.WaitGroup
+ errs := make([]error, lands)
+ for i, c := range q.onA {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ _, errs[i] = q.land(c)
+ }()
+ }
+ wg.Wait()
+ for i, err := range errs {
+ require.NoError(t, err, "land %d of %d (resolve_rounds is %d)", i+1, lands, 3)
+ }
+
+ // Every land is in the file: the union of all the sides.
+ got := strings.Split(q.branchBlob(t, "a"), "\n")
+ require.Len(t, got, lands+1, "ours and the %d changes: %v", lands, got)
+ // The resolver was called for the first round of every land, and once more for each land that lost
+ // it: about 2N, against N*(N+1)/2 without the queue.
+ require.LessOrEqual(t, unionCalls(t, q.dir), 2*lands, "resolver calls")
+ require.Empty(t, landTickets.keysOf(q.scope), "no ticket is kept")
+}
+
+// TestUserLand_aLandWaitsForItsTurnWithoutTheResolver: another land holds the ticket of the file. A land on
+// the same file resolves once, then waits: the resolver is not called while it waits, and it lands when
+// the holder is done, with the verdict it already had. A land on another file is not delayed.
+func TestUserLand_aLandWaitsForItsTurnWithoutTheResolver(t *testing.T) {
+ t.Parallel()
+
+ q := newQueueFixture(t, 1, nil)
+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})
+ require.NoError(t, err)
+ defer holder.release()
+
+ // A land on b only passes by, whoever holds a.
+ _, err = q.land(q.onlyB)
+ require.NoError(t, err, "a land on another file is not delayed")
+ require.Equal(t, 1, unionCalls(t, q.dir))
+
+ done := make(chan error, 1)
+ go func() {
+ _, err := q.land(q.onA[0])
+ done <- err
+ }()
+ require.Eventually(t, func() bool { return landTickets.waiting(q.keyOfA) == 1 }, 30*time.Second, 5*time.Millisecond, "the land never queued")
+ callsWhenQueued := unionCalls(t, q.dir)
+ require.Equal(t, 2, callsWhenQueued, "the first round of the land resolved once")
+ select {
+ case err := <-done:
+ t.Fatalf("the land went past the holder of its file: %v", err)
+ case <-time.After(500 * time.Millisecond):
+ }
+ require.Equal(t, callsWhenQueued, unionCalls(t, q.dir), "a waiting land does not call the resolver")
+
+ holder.release()
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(30 * time.Second):
+ t.Fatal("the land did not land when the holder was done")
+ }
+ require.Equal(t, callsWhenQueued, unionCalls(t, q.dir), "it landed with the verdict it had: the tip did not change the file")
+ require.Equal(t, []string{"ours", "theirs-0"}, strings.Split(q.branchBlob(t, "a"), "\n")[:2])
+}
+
+// TestUserLand_aLandThatWaitsPastItsDeadlineGetsThePackage: the holder of the file does not finish in
+// time. The land that waits ends at its own deadline with the package that says why, and does not land
+// past the holder.
+func TestUserLand_aLandThatWaitsPastItsDeadlineGetsThePackage(t *testing.T) {
+ t.Parallel()
+
+ q := newQueueFixture(t, 1, func(c *config.GraftResolver) { c.Timeout = "2s" })
+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})
+ require.NoError(t, err)
+ defer holder.release()
+
+ start := time.Now()
+ _, err = q.land(q.onA[0])
+ require.Less(t, time.Since(start), 30*time.Second)
+ requireMergeConflict(t, err)
+ pkg := landConflictDetail(err)
+ require.NotNil(t, pkg, "the package, not a bare deadline")
+ f := packageFile(t, pkg, "a")
+ require.Equal(t, gitalypb.LandConflictFile_OUTCOME_NOT_ATTEMPTED, f.GetOutcome())
+ require.Contains(t, f.GetReason(), "waiting for the other lands on the same files")
+ requireBranchAt(t, q.cfg, q.repoPath, git.DefaultBranch, q.tip)
+ require.Equal(t, 0, landTickets.waiting(q.keyOfA), "the land left the queue")
+
+ // The holder is still the holder.
+ require.True(t, landTickets.blocked([]string{q.keyOfA}, nil))
+}
diff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.go
index 50585fc..fdded7a 100644
--- a/internal/gitaly/service/operations/land_resolve.go
+++ b/internal/gitaly/service/operations/land_resolve.go
@@ -146,11 +146,23 @@ func (s *Server) landResolveAt(
var reusedNames []string
var sent []resolver.File
var mine []*landFileState
+ // A land that ran out of time waiting for its turn on its files (land_tickets.go) is ended with the
+ // package, and does not land past the lands that hold the tickets: its files are not attempted, and
+ // say why.
+ queuedBehind := false
+ if pre.unverifiable != "" {
+ paths := make([]string, 0, len(files))
+ for _, f := range files {
+ paths = append(paths, f.conflict.path)
+ }
+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
+ queuedBehind = landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket)
+ }
for _, f := range files {
if !f.sent() {
continue
}
- if v, ok := pre.verdicts[landKeyOf(f.conflict)]; ok {
+ if v, ok := pre.verdicts[landKeyOf(f.conflict)]; ok && !queuedBehind {
mine = append(mine, f)
reused[f.conflict.path] = v.file
reusedNames = append(reusedNames, v.resolver)
@@ -486,6 +498,17 @@ func (s *Server) landApplyVerdicts(
return landResolution{}, err
}
pre.setCurrent(l.files)
+ // A land that lost a round holds the ticket of its files and is the one whose verdict lands: any
+ // other land on the same files waits for its turn (land_tickets.go), instead of landing first and
+ // invalidating the holder's verdict once more.
+ paths := make([]string, 0, len(l.files))
+ for _, f := range l.files {
+ paths = append(paths, f.conflict.path)
+ }
+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
+ if landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket) {
+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: paths, queued: true}
+ }
for _, f := range l.files {
if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {
missing = append(missing, f.conflict.path)
diff --git a/internal/gitaly/service/operations/land_tickets.go b/internal/gitaly/service/operations/land_tickets.go
new file mode 100644
index 0000000..373f846
--- /dev/null
+++ b/internal/gitaly/service/operations/land_tickets.go
@@ -0,0 +1,233 @@
+package operations
+
+import (
+ "context"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// The tickets of the lands that lose a round (PLAN10 15.8).
+//
+// A land with conflicts is resolved out of the lane, and a file's verdict is valid only for the exact
+// sides of the file, so every landing that changes a hot file invalidates the verdicts of all the
+// lands that wait for it. Without an order they all resolve again after every landing: N lands on one
+// file cost about N*N/2 resolver calls, which fill the resolve slots, and which land first is nobody's
+// choice, so a land can lose every round until resolve_rounds ends it.
+//
+// A land that has lost a round therefore takes a ticket: a first-in first-out queue per file (per
+// branch when a verification is configured, since the tree that is verified is the whole branch's). The
+// holder is the only land that calls the resolver for those files after losing a round; the others wait
+// without a request. The holder keeps the ticket until it lands, its deadline ends, or it fails: nothing
+// takes it away. The lane hands back, to wait in the queue, any other land with a verdict on a ticketed
+// file (landApplyVerdicts), so that the holder's verdict is the one that lands. The queue is in memory
+// and per process, like the lanes: against another node of the same branch (authoritative storage) the
+// lands go on as they did, in rounds, without the guarantee.
+
+// landTicketWait is the time a land waited for the tickets of its files.
+var landTicketWait = promauto.NewHistogram(prometheus.HistogramOpts{
+ Name: "graft_land_ticket_wait_seconds",
+ Help: "Time a land that lost a round of its conflict resolution waited for its turn on its files.",
+ Buckets: prometheus.ExponentialBuckets(0.005, 4, 9),
+})
+
+// landResolutionRounds is the number of rounds the lands with conflicts used.
+var landResolutionRounds = promauto.NewHistogram(prometheus.HistogramOpts{
+ Name: "graft_land_resolution_rounds",
+ Help: "Rounds (resolve, then the lane) a land with conflicts needed.",
+ Buckets: []float64{1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 64},
+})
+
+// landTicketRegistry is the tickets of the process.
+type landTicketRegistry struct {
+ mu sync.Mutex
+ keys map[string]*landTicketKey
+}
+
+// landTicketKey is one file (or branch) and its queue.
+type landTicketKey struct {
+ holder *landTicket
+ waiters []*landTicketWaiter
+}
+
+type landTicketWaiter struct {
+ ticket *landTicket
+ ready chan struct{}
+}
+
+// landTicket is the right to resolve a set of files of a branch before the other lands that lost a round.
+type landTicket struct {
+ reg *landTicketRegistry
+ keys []string // sorted: the order they are taken in, so that two lands cannot wait for each other
+
+ mu sync.Mutex
+ acquired []string // the keys held, in the order taken
+ released bool
+}
+
+// landTickets are the tickets of the process, shared by all the servers of it.
+var landTickets = &landTicketRegistry{keys: map[string]*landTicketKey{}}
+
+// landTicketKeys names the tickets of the files of a branch: one per path, or the one of the branch when
+// branchWide. scope is the repository and the branch.
+func landTicketKeys(scope string, paths []string, branchWide bool) []string {
+ if branchWide || len(paths) == 0 {
+ return []string{scope + "\x00"}
+ }
+ seen := make(map[string]bool, len(paths))
+ keys := make([]string, 0, len(paths))
+ for _, p := range paths {
+ k := scope + "\x00" + p
+ if !seen[k] {
+ seen[k] = true
+ keys = append(keys, k)
+ }
+ }
+ sort.Strings(keys)
+ return keys
+}
+
+// landTicketScope is what the tickets of a land are about: its repository and its branch.
+func landTicketScope(storage, relativePath string, branch []byte) string {
+ return strings.Join([]string{storage, relativePath, string(branch)}, "\x00")
+}
+
+// acquire waits for the tickets of keys, first in first out on each, and takes them in sorted order. It
+// returns when it holds all of them, or with ctx's error holding none.
+func (r *landTicketRegistry) acquire(ctx context.Context, keys []string) (*landTicket, error) {
+ start := time.Now()
+ t := &landTicket{reg: r, keys: keys}
+ for _, key := range keys {
+ if err := r.take(ctx, t, key); err != nil {
+ t.release()
+ landTicketWait.Observe(time.Since(start).Seconds())
+ return nil, err
+ }
+ }
+ landTicketWait.Observe(time.Since(start).Seconds())
+ return t, nil
+}
+
+// take makes t the holder of key, waiting for the holders and the waiters before it.
+func (r *landTicketRegistry) take(ctx context.Context, t *landTicket, key string) error {
+ r.mu.Lock()
+ k := r.keys[key]
+ if k == nil {
+ k = &landTicketKey{}
+ r.keys[key] = k
+ }
+ if k.holder == nil && len(k.waiters) == 0 {
+ k.holder = t
+ r.mu.Unlock()
+ t.noteAcquired(key)
+ return nil
+ }
+ w := &landTicketWaiter{ticket: t, ready: make(chan struct{})}
+ k.waiters = append(k.waiters, w)
+ r.mu.Unlock()
+
+ select {
+ case <-w.ready:
+ t.noteAcquired(key)
+ return nil
+ case <-ctx.Done():
+ r.mu.Lock()
+ for i, x := range k.waiters {
+ if x == w {
+ // Still queued: leave it.
+ k.waiters = append(k.waiters[:i], k.waiters[i+1:]...)
+ r.mu.Unlock()
+ return ctx.Err()
+ }
+ }
+ r.mu.Unlock()
+ // The key was handed over while the context ended: it is held, and the caller releases it.
+ t.noteAcquired(key)
+ return ctx.Err()
+ }
+}
+
+func (t *landTicket) noteAcquired(key string) {
+ t.mu.Lock()
+ t.acquired = append(t.acquired, key)
+ t.mu.Unlock()
+}
+
+// release gives every key back, to the next waiter of each; a second call does nothing.
+func (t *landTicket) release() {
+ if t == nil {
+ return
+ }
+ t.mu.Lock()
+ if t.released {
+ t.mu.Unlock()
+ return
+ }
+ t.released = true
+ acquired := t.acquired
+ t.mu.Unlock()
+
+ r := t.reg
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for i := len(acquired) - 1; i >= 0; i-- {
+ k := r.keys[acquired[i]]
+ if k == nil || k.holder != t {
+ continue
+ }
+ if len(k.waiters) == 0 {
+ delete(r.keys, acquired[i])
+ continue
+ }
+ next := k.waiters[0]
+ k.waiters = k.waiters[1:]
+ k.holder = next.ticket
+ close(next.ready)
+ }
+}
+
+// covers reports whether t holds a ticket for every one of keys.
+func (t *landTicket) covers(keys []string) bool {
+ if t == nil {
+ return false
+ }
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ held := make(map[string]bool, len(t.acquired))
+ for _, k := range t.acquired {
+ held[k] = true
+ }
+ for _, k := range keys {
+ if !held[k] {
+ return false
+ }
+ }
+ return true
+}
+
+// blocked reports whether another land holds a ticket for one of keys: the land that asks, which is me
+// when it holds tickets (nil otherwise), must wait for its turn.
+func (r *landTicketRegistry) blocked(keys []string, me *landTicket) bool {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for _, key := range keys {
+ if k := r.keys[key]; k != nil && k.holder != nil && k.holder != me {
+ return true
+ }
+ }
+ return false
+}
+
+// waiting returns the number of lands that wait for key, for the tests.
+func (r *landTicketRegistry) waiting(key string) int {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if k := r.keys[key]; k != nil {
+ return len(k.waiters)
+ }
+ return 0
+}
diff --git a/internal/gitaly/service/operations/land_tickets_test.go b/internal/gitaly/service/operations/land_tickets_test.go
new file mode 100644
index 0000000..81e063f
--- /dev/null
+++ b/internal/gitaly/service/operations/land_tickets_test.go
@@ -0,0 +1,150 @@
+package operations
+
+import (
+ "context"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+// keysOf returns the keys of scope that still have a holder or a waiter, for the tests.
+func (r *landTicketRegistry) keysOf(scope string) []string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ var out []string
+ for k := range r.keys {
+ if strings.HasPrefix(k, scope+"\x00") {
+ out = append(out, k)
+ }
+ }
+ return out
+}
+
+func newTestTickets() *landTicketRegistry {
+ return &landTicketRegistry{keys: map[string]*landTicketKey{}}
+}
+
+// TestLandTickets_firstInFirstOut: the holder of a key keeps it until it releases it, and the waiters
+// get it one after the other in the order they asked.
+func TestLandTickets_firstInFirstOut(t *testing.T) {
+ t.Parallel()
+
+ r := newTestTickets()
+ keys := landTicketKeys("repo", []string{"a"}, false)
+ first, err := r.acquire(context.Background(), keys)
+ require.NoError(t, err)
+
+ var mu sync.Mutex
+ var order []int
+ var wg sync.WaitGroup
+ tickets := make([]*landTicket, 5)
+ for i := range tickets {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ tk, err := r.acquire(context.Background(), keys)
+ require.NoError(t, err)
+ mu.Lock()
+ order = append(order, i)
+ mu.Unlock()
+ tickets[i] = tk
+ tk.release()
+ }()
+ // They ask one after the other.
+ require.Eventually(t, func() bool { return r.waiting(keys[0]) == i+1 }, 5*time.Second, time.Millisecond)
+ }
+ require.True(t, r.blocked(keys, nil), "the holder blocks the others")
+ require.False(t, r.blocked(keys, first), "and not itself")
+ first.release()
+ wg.Wait()
+ require.Equal(t, []int{0, 1, 2, 3, 4}, order)
+ require.False(t, r.blocked(keys, nil), "nothing is kept of a key without holder or waiter")
+ require.Empty(t, r.keys)
+}
+
+// TestLandTickets_aWaiterThatGivesUpLeavesTheQueue: the context ends while the land waits; it holds
+// nothing, and the next waiter is not stuck behind it.
+func TestLandTickets_aWaiterThatGivesUpLeavesTheQueue(t *testing.T) {
+ t.Parallel()
+
+ r := newTestTickets()
+ keys := landTicketKeys("repo", []string{"a"}, false)
+ holder, err := r.acquire(context.Background(), keys)
+ require.NoError(t, err)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ gone := make(chan error, 1)
+ go func() {
+ _, err := r.acquire(ctx, keys)
+ gone <- err
+ }()
+ require.Eventually(t, func() bool { return r.waiting(keys[0]) == 1 }, 5*time.Second, time.Millisecond)
+
+ next := make(chan *landTicket, 1)
+ go func() {
+ tk, _ := r.acquire(context.Background(), keys)
+ next <- tk
+ }()
+ require.Eventually(t, func() bool { return r.waiting(keys[0]) == 2 }, 5*time.Second, time.Millisecond)
+
+ cancel()
+ require.ErrorIs(t, <-gone, context.Canceled)
+ require.Equal(t, 1, r.waiting(keys[0]))
+ holder.release()
+ tk := <-next
+ require.NotNil(t, tk)
+ tk.release()
+ require.Empty(t, r.keys)
+}
+
+// TestLandTickets_orderedKeysDoNotDeadlock: lands that need the same files in another order take them
+// in the same (sorted) order, so none of them waits for the other while holding what it needs.
+func TestLandTickets_orderedKeysDoNotDeadlock(t *testing.T) {
+ t.Parallel()
+
+ r := newTestTickets()
+ var wg sync.WaitGroup
+ for i := 0; i < 40; i++ {
+ paths := []string{"a", "b", "c"}
+ if i%2 == 1 {
+ paths = []string{"c", "b", "a"}
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cancel()
+ tk, err := r.acquire(ctx, landTicketKeys("repo", paths, false))
+ require.NoError(t, err)
+ time.Sleep(time.Millisecond)
+ tk.release()
+ }()
+ }
+ wg.Wait()
+ require.Empty(t, r.keys)
+}
+
+// TestLandTickets_keys: a ticket is per file, or per branch when a verification is configured (the tree
+// that is verified is the whole branch's), and a land with no file of its own takes the branch's.
+func TestLandTickets_keys(t *testing.T) {
+ t.Parallel()
+
+ require.Equal(t, landTicketKeys("r", []string{"b", "a", "b"}, false), []string{"r\x00a", "r\x00b"})
+ require.Equal(t, []string{"r\x00"}, landTicketKeys("r", []string{"a"}, true))
+ require.Equal(t, []string{"r\x00"}, landTicketKeys("r", nil, false))
+ require.NotEqual(t, landTicketScope("s", "x.git", []byte("main")), landTicketScope("s", "x.git", []byte("dev")))
+
+ r := newTestTickets()
+ tk, err := r.acquire(context.Background(), landTicketKeys("r", []string{"a", "b"}, false))
+ require.NoError(t, err)
+ require.True(t, tk.covers(landTicketKeys("r", []string{"a"}, false)))
+ require.False(t, tk.covers(landTicketKeys("r", []string{"a", "c"}, false)))
+ require.False(t, r.blocked(landTicketKeys("r", []string{"c"}, false), nil), "another file is free")
+ require.True(t, r.blocked(landTicketKeys("r", []string{"c", "b"}, false), nil))
+ tk.release()
+ tk.release() // a second release does nothing
+ require.False(t, (*landTicket)(nil).covers(nil))
+}