Git designed for AI agents.
graft: a queue of tickets for the lands that lose a round
Federico Elia4fe5536624be
diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.goindex 3153ab5..ce5ccfd 100644--- a/internal/gitaly/service/operations/land_preresolve.go+++ b/internal/gitaly/service/operations/land_preresolve.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "sort""strings""time"@@ -70,6 +71,9 @@ type landPreResolution struct {// verifyAll says that a verification is configured: every resolution then needs a verified tree// to land, a rule's as well as a model's (15.9).verifyAll bool+ // ticket is the land's place in the queue of its files, once it has lost a round (land_tickets.go).+ // The lane lets the holder's verdicts land before those of any other land on the same files.+ ticket *landTicket}func newLandPreResolution() *landPreResolution {@@ -141,6 +145,9 @@ type landNeedsResolution struct {// the batch ran out of attempts: the tree it was about to land is another one, and the land goes// out again as it does for an unverified tree.moved bool+ // queued says that another land holds the ticket of one of the files (paths): this one waits for+ // its turn before it resolves again.+ queued bool}func (e *landNeedsResolution) Error() string {@@ -262,7 +269,15 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequestreturn ctx.Err() == nil && errors.Is(resolveCtx.Err(), context.DeadlineExceeded)}rounds := s.resolveCfg.ResolveRoundsOrDefault()+ used := 0+ // The land keeps its ticket until it ends, whatever the ending: it lands, the deadline stops it, or+ // it fails.+ defer func() {+ pre.ticket.release()+ landResolutionRounds.Observe(float64(used))+ }()for round := 1; round <= rounds; round++ {+ used = roundif err := s.landPreResolve(resolveCtx, in, committer, pre); err != nil {// The deadline ended a step of the round: the listing, the tree, the resolution or the// verification. A refusal of the policy is not a deadline.@@ -281,17 +296,85 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest}return nil, err}- resp, err := s.landOnce(ctx, in, committer, nil, pre)- if !errors.As(err, &need) {- return resp, err+ // Another land holds the ticket of one of these files: this one does not enter the lane, which would+ // only hand it back (and which every land of the branch waits for), and waits for its turn.+ if paths := pre.currentPaths(); s.landTicketsBlock(in, paths, pre) {+ need = &landNeedsResolution{paths: paths, queued: true}+ } else {+ resp, err := s.landOnce(ctx, in, committer, nil, pre)+ if !errors.As(err, &need) {+ return resp, err+ }}if deadline() {+ pre.ticket.release()return s.landFinalPass(ctx, in, committer, pre, landStoppedReason(need, timeout))}+ // The land lost the round, to a land that changed its files (or to the tree that moved, with a+ // verification): it takes its turn on them. From now on it is the only land of the branch that+ // resolves these files again, and the others wait without calling the resolver.+ if err := s.landTakeTicket(resolveCtx, in, pre, need); err != nil {+ if deadline() {+ pre.ticket.release()+ return s.landFinalPass(ctx, in, committer, pre, landQueueReason(timeout))+ }+ switch {+ case errors.Is(ctx.Err(), context.DeadlineExceeded):+ return nil, structerr.NewDeadlineExceeded("waiting for the other lands on the same files: %w", err)+ case ctx.Err() != nil:+ return nil, structerr.NewCanceled("waiting for the other lands on the same files: %w", err)+ }+ return nil, err+ }}+ pre.ticket.release()return s.landFinalPass(ctx, in, committer, pre, landStoppedReason(need, timeout))}+// currentPaths are the paths of the conflicting files the land has at the latest tip it looked at.+func (p *landPreResolution) currentPaths() []string {+ paths := make([]string, 0, len(p.current))+ for k := range p.current {+ paths = append(paths, k.path)+ }+ sort.Strings(paths)+ return paths+}++// landTicketsBlock reports whether another land holds the ticket of one of paths.+func (s *Server) landTicketsBlock(in *gitalypb.UserLandRequest, paths []string, pre *landPreResolution) bool {+ if len(paths) == 0 {+ return false+ }+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())+ return landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket)+}++// landTakeTicket makes the land hold the tickets of the files it needs resolved, waiting for the lands+// that lost a round before it (land_tickets.go). A land that holds tickets for all of them keeps them.+// One that now needs more files gives its tickets back and takes the larger set at the tail: tickets+// are taken in sorted order, so that no two lands wait for each other.+func (s *Server) landTakeTicket(ctx context.Context, in *gitalypb.UserLandRequest, pre *landPreResolution, need *landNeedsResolution) error {+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())+ keys := landTicketKeys(scope, need.paths, s.resolveCfg.Verify.Enabled)+ if pre.ticket.covers(keys) {+ return nil+ }+ pre.ticket.release()+ pre.ticket = nil+ t, err := landTickets.acquire(ctx, keys)+ if err != nil {+ return err+ }+ pre.ticket = t+ return nil+}++// landQueueReason is why a land that waited for its turn on its files could not go on.+func landQueueReason(timeout time.Duration) string {+ return fmt.Sprintf("the time ran out waiting for the other lands on the same files (deadline of %s)", timeout)+}+// landCallerGaveUp is the status of a land whose caller gave up, or whose own deadline passed, while// it waited for a slot or verified: the caller's cancellation, not an error of the server.func landCallerGaveUp(ctx context.Context, err error) error {diff --git a/internal/gitaly/service/operations/land_queue_test.go b/internal/gitaly/service/operations/land_queue_test.gonew file mode 100644index 0000000..0415253--- /dev/null+++ b/internal/gitaly/service/operations/land_queue_test.go@@ -0,0 +1,223 @@+package operations++import (+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// The tests of the queue of the lands that lose a round (land_tickets.go): N lands that conflict on+// one file, a resolver that unions the two sides (a rule: the file keeps every line), and the lands+// that wait for their turn without calling the resolver.++// unionResolver writes a resolver command that resolves every file of the request with the union of the+// lines of the two sides, as a rule, and counts its runs in dir/calls.log. The runs are concurrent, so+// each reads its own request file.+func unionResolver(t *testing.T) (string, []string) {+ t.Helper()++ dir := t.TempDir()+ script := fmt.Sprintf(`#!/bin/sh+DIR='%s'+echo x >> "$DIR/calls.log"+cat > "$DIR/req-$$.json"+python3 - "$DIR/req-$$.json" <<'PY'+import base64, json, sys+req = json.load(open(sys.argv[1]))+files = []+for f in req["files"]:+ ours = base64.b64decode(f["ours"]["content"]).decode().splitlines()+ theirs = base64.b64decode(f["theirs"]["content"]).decode().splitlines()+ merged = ours + [l for l in theirs if l not in ours]+ files.append({"path": f["path"], "action": "write", "mode": "100644", "origin": "rule",+ "content": base64.b64encode(("\n".join(merged) + "\n").encode()).decode()})+json.dump({"version": 1, "resolver": "graft-resolve/union", "files": files}, sys.stdout)+PY+`, dir)+ path := filepath.Join(dir, "resolver.sh")+ require.NoError(t, os.WriteFile(path, []byte(script), 0o755))+ return dir, []string{path}+}++// unionCalls is the number of runs of the resolver of dir.+func unionCalls(t *testing.T, dir string) int {+ t.Helper()++ raw, err := os.ReadFile(filepath.Join(dir, "calls.log"))+ if os.IsNotExist(err) {+ return 0+ }+ require.NoError(t, err)+ return len(raw) / 2+}++// queueFixture is a repository where main (tip) changed a and b, and n commits from the base each+// change a in their own way (conflict on a only; b is left as the base has it, so that it merges+// clean), plus one commit that changes b only.+type queueFixture struct {+ resolveFixture+ dir string+ onA []git.ObjectID+ onlyB git.ObjectID+ scope string+ keyOfA string+ keyOfB string+ resolver config.GraftResolver+}++func newQueueFixture(t *testing.T, n int, tune func(*config.GraftResolver)) queueFixture {+ t.Helper()++ dir, argv := unionResolver(t)+ rcfg := enabledResolver(argv)+ if tune != nil {+ tune(&rcfg)+ }+ fx := newResolveFixtureWith(t, rcfg,+ []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "base\n"}, {Mode: "100644", Path: "b", Content: "base\n"}},+ []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "ours\n"}, {Mode: "100644", Path: "b", Content: "ours\n"}},+ []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "theirs-0\n"}, {Mode: "100644", Path: "b", Content: "base\n"}},+ )+ q := queueFixture{resolveFixture: fx, dir: dir, resolver: rcfg}+ q.onA = append(q.onA, fx.commit)+ for i := 1; i < n; i++ {+ q.onA = append(q.onA, gittest.WriteCommit(t, fx.cfg, fx.repoPath, gittest.WithParents(fx.base),+ gittest.WithTreeEntries(+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: fmt.Sprintf("theirs-%d\n", i)},+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"})))+ }+ q.onlyB = gittest.WriteCommit(t, fx.cfg, fx.repoPath, gittest.WithParents(fx.base),+ gittest.WithTreeEntries(+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "base\n"},+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "theirs-b\n"}))+ q.scope = landTicketScope(fx.repoProto.GetStorageName(), fx.repoProto.GetRelativePath(), []byte(git.DefaultBranch))+ q.keyOfA = landTicketKeys(q.scope, []string{"a"}, false)[0]+ q.keyOfB = landTicketKeys(q.scope, []string{"b"}, false)[0]+ return q+}++func (q queueFixture) land(commit git.ObjectID) (*gitalypb.UserLandResponse, error) {+ req := q.request("Land " + commit.String())+ req.CommitId = commit.String()+ return q.client.UserLand(q.ctx, req)+}++// branchBlob reads a file at the tip of the branch.+func (q queueFixture) branchBlob(t *testing.T, path string) string {+ t.Helper()++ return strings.TrimSpace(string(gittest.Exec(t, q.cfg, "-C", q.repoPath, "show", git.DefaultBranch+":"+path)))+}++// TestUserLand_theLandsOnAHotFileAllLand: more lands than resolve_rounds conflict on one file. Without an+// order each one loses a round to every land that lands before it, and the last ones run out of rounds;+// with the queue each loses at most one, and the resolver is called for a land after its first round only+// when it is its turn.+func TestUserLand_theLandsOnAHotFileAllLand(t *testing.T) {+ t.Parallel()++ const lands = 8+ q := newQueueFixture(t, lands, func(c *config.GraftResolver) { c.ResolveRounds = 3 })++ var wg sync.WaitGroup+ errs := make([]error, lands)+ for i, c := range q.onA {+ wg.Add(1)+ go func() {+ defer wg.Done()+ _, errs[i] = q.land(c)+ }()+ }+ wg.Wait()+ for i, err := range errs {+ require.NoError(t, err, "land %d of %d (resolve_rounds is %d)", i+1, lands, 3)+ }++ // Every land is in the file: the union of all the sides.+ got := strings.Split(q.branchBlob(t, "a"), "\n")+ require.Len(t, got, lands+1, "ours and the %d changes: %v", lands, got)+ // The resolver was called for the first round of every land, and once more for each land that lost+ // it: about 2N, against N*(N+1)/2 without the queue.+ require.LessOrEqual(t, unionCalls(t, q.dir), 2*lands, "resolver calls")+ require.Empty(t, landTickets.keysOf(q.scope), "no ticket is kept")+}++// TestUserLand_aLandWaitsForItsTurnWithoutTheResolver: another land holds the ticket of the file. A land on+// the same file resolves once, then waits: the resolver is not called while it waits, and it lands when+// the holder is done, with the verdict it already had. A land on another file is not delayed.+func TestUserLand_aLandWaitsForItsTurnWithoutTheResolver(t *testing.T) {+ t.Parallel()++ q := newQueueFixture(t, 1, nil)+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})+ require.NoError(t, err)+ defer holder.release()++ // A land on b only passes by, whoever holds a.+ _, err = q.land(q.onlyB)+ require.NoError(t, err, "a land on another file is not delayed")+ require.Equal(t, 1, unionCalls(t, q.dir))++ done := make(chan error, 1)+ go func() {+ _, err := q.land(q.onA[0])+ done <- err+ }()+ require.Eventually(t, func() bool { return landTickets.waiting(q.keyOfA) == 1 }, 30*time.Second, 5*time.Millisecond, "the land never queued")+ callsWhenQueued := unionCalls(t, q.dir)+ require.Equal(t, 2, callsWhenQueued, "the first round of the land resolved once")+ select {+ case err := <-done:+ t.Fatalf("the land went past the holder of its file: %v", err)+ case <-time.After(500 * time.Millisecond):+ }+ require.Equal(t, callsWhenQueued, unionCalls(t, q.dir), "a waiting land does not call the resolver")++ holder.release()+ select {+ case err := <-done:+ require.NoError(t, err)+ case <-time.After(30 * time.Second):+ t.Fatal("the land did not land when the holder was done")+ }+ require.Equal(t, callsWhenQueued, unionCalls(t, q.dir), "it landed with the verdict it had: the tip did not change the file")+ require.Equal(t, []string{"ours", "theirs-0"}, strings.Split(q.branchBlob(t, "a"), "\n")[:2])+}++// TestUserLand_aLandThatWaitsPastItsDeadlineGetsThePackage: the holder of the file does not finish in+// time. The land that waits ends at its own deadline with the package that says why, and does not land+// past the holder.+func TestUserLand_aLandThatWaitsPastItsDeadlineGetsThePackage(t *testing.T) {+ t.Parallel()++ q := newQueueFixture(t, 1, func(c *config.GraftResolver) { c.Timeout = "2s" })+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})+ require.NoError(t, err)+ defer holder.release()++ start := time.Now()+ _, err = q.land(q.onA[0])+ require.Less(t, time.Since(start), 30*time.Second)+ requireMergeConflict(t, err)+ pkg := landConflictDetail(err)+ require.NotNil(t, pkg, "the package, not a bare deadline")+ f := packageFile(t, pkg, "a")+ require.Equal(t, gitalypb.LandConflictFile_OUTCOME_NOT_ATTEMPTED, f.GetOutcome())+ require.Contains(t, f.GetReason(), "waiting for the other lands on the same files")+ requireBranchAt(t, q.cfg, q.repoPath, git.DefaultBranch, q.tip)+ require.Equal(t, 0, landTickets.waiting(q.keyOfA), "the land left the queue")++ // The holder is still the holder.+ require.True(t, landTickets.blocked([]string{q.keyOfA}, nil))+}diff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.goindex 50585fc..fdded7a 100644--- a/internal/gitaly/service/operations/land_resolve.go+++ b/internal/gitaly/service/operations/land_resolve.go@@ -146,11 +146,23 @@ func (s *Server) landResolveAt(var reusedNames []stringvar sent []resolver.Filevar mine []*landFileState+ // A land that ran out of time waiting for its turn on its files (land_tickets.go) is ended with the+ // package, and does not land past the lands that hold the tickets: its files are not attempted, and+ // say why.+ queuedBehind := false+ if pre.unverifiable != "" {+ paths := make([]string, 0, len(files))+ for _, f := range files {+ paths = append(paths, f.conflict.path)+ }+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())+ queuedBehind = landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket)+ }for _, f := range files {if !f.sent() {continue}- if v, ok := pre.verdicts[landKeyOf(f.conflict)]; ok {+ if v, ok := pre.verdicts[landKeyOf(f.conflict)]; ok && !queuedBehind {mine = append(mine, f)reused[f.conflict.path] = v.filereusedNames = append(reusedNames, v.resolver)@@ -486,6 +498,17 @@ func (s *Server) landApplyVerdicts(return landResolution{}, err}pre.setCurrent(l.files)+ // A land that lost a round holds the ticket of its files and is the one whose verdict lands: any+ // other land on the same files waits for its turn (land_tickets.go), instead of landing first and+ // invalidating the holder's verdict once more.+ paths := make([]string, 0, len(l.files))+ for _, f := range l.files {+ paths = append(paths, f.conflict.path)+ }+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())+ if landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket) {+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: paths, queued: true}+ }for _, f := range l.files {if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {missing = append(missing, f.conflict.path)diff --git a/internal/gitaly/service/operations/land_tickets.go b/internal/gitaly/service/operations/land_tickets.gonew file mode 100644index 0000000..373f846--- /dev/null+++ b/internal/gitaly/service/operations/land_tickets.go@@ -0,0 +1,233 @@+package operations++import (+ "context"+ "sort"+ "strings"+ "sync"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++// The tickets of the lands that lose a round (PLAN10 15.8).+//+// A land with conflicts is resolved out of the lane, and a file's verdict is valid only for the exact+// sides of the file, so every landing that changes a hot file invalidates the verdicts of all the+// lands that wait for it. Without an order they all resolve again after every landing: N lands on one+// file cost about N*N/2 resolver calls, which fill the resolve slots, and which land first is nobody's+// choice, so a land can lose every round until resolve_rounds ends it.+//+// A land that has lost a round therefore takes a ticket: a first-in first-out queue per file (per+// branch when a verification is configured, since the tree that is verified is the whole branch's). The+// holder is the only land that calls the resolver for those files after losing a round; the others wait+// without a request. The holder keeps the ticket until it lands, its deadline ends, or it fails: nothing+// takes it away. The lane hands back, to wait in the queue, any other land with a verdict on a ticketed+// file (landApplyVerdicts), so that the holder's verdict is the one that lands. The queue is in memory+// and per process, like the lanes: against another node of the same branch (authoritative storage) the+// lands go on as they did, in rounds, without the guarantee.++// landTicketWait is the time a land waited for the tickets of its files.+var landTicketWait = promauto.NewHistogram(prometheus.HistogramOpts{+ Name: "graft_land_ticket_wait_seconds",+ Help: "Time a land that lost a round of its conflict resolution waited for its turn on its files.",+ Buckets: prometheus.ExponentialBuckets(0.005, 4, 9),+})++// landResolutionRounds is the number of rounds the lands with conflicts used.+var landResolutionRounds = promauto.NewHistogram(prometheus.HistogramOpts{+ Name: "graft_land_resolution_rounds",+ Help: "Rounds (resolve, then the lane) a land with conflicts needed.",+ Buckets: []float64{1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 64},+})++// landTicketRegistry is the tickets of the process.+type landTicketRegistry struct {+ mu sync.Mutex+ keys map[string]*landTicketKey+}++// landTicketKey is one file (or branch) and its queue.+type landTicketKey struct {+ holder *landTicket+ waiters []*landTicketWaiter+}++type landTicketWaiter struct {+ ticket *landTicket+ ready chan struct{}+}++// landTicket is the right to resolve a set of files of a branch before the other lands that lost a round.+type landTicket struct {+ reg *landTicketRegistry+ keys []string // sorted: the order they are taken in, so that two lands cannot wait for each other++ mu sync.Mutex+ acquired []string // the keys held, in the order taken+ released bool+}++// landTickets are the tickets of the process, shared by all the servers of it.+var landTickets = &landTicketRegistry{keys: map[string]*landTicketKey{}}++// landTicketKeys names the tickets of the files of a branch: one per path, or the one of the branch when+// branchWide. scope is the repository and the branch.+func landTicketKeys(scope string, paths []string, branchWide bool) []string {+ if branchWide || len(paths) == 0 {+ return []string{scope + "\x00"}+ }+ seen := make(map[string]bool, len(paths))+ keys := make([]string, 0, len(paths))+ for _, p := range paths {+ k := scope + "\x00" + p+ if !seen[k] {+ seen[k] = true+ keys = append(keys, k)+ }+ }+ sort.Strings(keys)+ return keys+}++// landTicketScope is what the tickets of a land are about: its repository and its branch.+func landTicketScope(storage, relativePath string, branch []byte) string {+ return strings.Join([]string{storage, relativePath, string(branch)}, "\x00")+}++// acquire waits for the tickets of keys, first in first out on each, and takes them in sorted order. It+// returns when it holds all of them, or with ctx's error holding none.+func (r *landTicketRegistry) acquire(ctx context.Context, keys []string) (*landTicket, error) {+ start := time.Now()+ t := &landTicket{reg: r, keys: keys}+ for _, key := range keys {+ if err := r.take(ctx, t, key); err != nil {+ t.release()+ landTicketWait.Observe(time.Since(start).Seconds())+ return nil, err+ }+ }+ landTicketWait.Observe(time.Since(start).Seconds())+ return t, nil+}++// take makes t the holder of key, waiting for the holders and the waiters before it.+func (r *landTicketRegistry) take(ctx context.Context, t *landTicket, key string) error {+ r.mu.Lock()+ k := r.keys[key]+ if k == nil {+ k = &landTicketKey{}+ r.keys[key] = k+ }+ if k.holder == nil && len(k.waiters) == 0 {+ k.holder = t+ r.mu.Unlock()+ t.noteAcquired(key)+ return nil+ }+ w := &landTicketWaiter{ticket: t, ready: make(chan struct{})}+ k.waiters = append(k.waiters, w)+ r.mu.Unlock()++ select {+ case <-w.ready:+ t.noteAcquired(key)+ return nil+ case <-ctx.Done():+ r.mu.Lock()+ for i, x := range k.waiters {+ if x == w {+ // Still queued: leave it.+ k.waiters = append(k.waiters[:i], k.waiters[i+1:]...)+ r.mu.Unlock()+ return ctx.Err()+ }+ }+ r.mu.Unlock()+ // The key was handed over while the context ended: it is held, and the caller releases it.+ t.noteAcquired(key)+ return ctx.Err()+ }+}++func (t *landTicket) noteAcquired(key string) {+ t.mu.Lock()+ t.acquired = append(t.acquired, key)+ t.mu.Unlock()+}++// release gives every key back, to the next waiter of each; a second call does nothing.+func (t *landTicket) release() {+ if t == nil {+ return+ }+ t.mu.Lock()+ if t.released {+ t.mu.Unlock()+ return+ }+ t.released = true+ acquired := t.acquired+ t.mu.Unlock()++ r := t.reg+ r.mu.Lock()+ defer r.mu.Unlock()+ for i := len(acquired) - 1; i >= 0; i-- {+ k := r.keys[acquired[i]]+ if k == nil || k.holder != t {+ continue+ }+ if len(k.waiters) == 0 {+ delete(r.keys, acquired[i])+ continue+ }+ next := k.waiters[0]+ k.waiters = k.waiters[1:]+ k.holder = next.ticket+ close(next.ready)+ }+}++// covers reports whether t holds a ticket for every one of keys.+func (t *landTicket) covers(keys []string) bool {+ if t == nil {+ return false+ }+ t.mu.Lock()+ defer t.mu.Unlock()+ held := make(map[string]bool, len(t.acquired))+ for _, k := range t.acquired {+ held[k] = true+ }+ for _, k := range keys {+ if !held[k] {+ return false+ }+ }+ return true+}++// blocked reports whether another land holds a ticket for one of keys: the land that asks, which is me+// when it holds tickets (nil otherwise), must wait for its turn.+func (r *landTicketRegistry) blocked(keys []string, me *landTicket) bool {+ r.mu.Lock()+ defer r.mu.Unlock()+ for _, key := range keys {+ if k := r.keys[key]; k != nil && k.holder != nil && k.holder != me {+ return true+ }+ }+ return false+}++// waiting returns the number of lands that wait for key, for the tests.+func (r *landTicketRegistry) waiting(key string) int {+ r.mu.Lock()+ defer r.mu.Unlock()+ if k := r.keys[key]; k != nil {+ return len(k.waiters)+ }+ return 0+}diff --git a/internal/gitaly/service/operations/land_tickets_test.go b/internal/gitaly/service/operations/land_tickets_test.gonew file mode 100644index 0000000..81e063f--- /dev/null+++ b/internal/gitaly/service/operations/land_tickets_test.go@@ -0,0 +1,150 @@+package operations++import (+ "context"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++// keysOf returns the keys of scope that still have a holder or a waiter, for the tests.+func (r *landTicketRegistry) keysOf(scope string) []string {+ r.mu.Lock()+ defer r.mu.Unlock()+ var out []string+ for k := range r.keys {+ if strings.HasPrefix(k, scope+"\x00") {+ out = append(out, k)+ }+ }+ return out+}++func newTestTickets() *landTicketRegistry {+ return &landTicketRegistry{keys: map[string]*landTicketKey{}}+}++// TestLandTickets_firstInFirstOut: the holder of a key keeps it until it releases it, and the waiters+// get it one after the other in the order they asked.+func TestLandTickets_firstInFirstOut(t *testing.T) {+ t.Parallel()++ r := newTestTickets()+ keys := landTicketKeys("repo", []string{"a"}, false)+ first, err := r.acquire(context.Background(), keys)+ require.NoError(t, err)++ var mu sync.Mutex+ var order []int+ var wg sync.WaitGroup+ tickets := make([]*landTicket, 5)+ for i := range tickets {+ wg.Add(1)+ go func() {+ defer wg.Done()+ tk, err := r.acquire(context.Background(), keys)+ require.NoError(t, err)+ mu.Lock()+ order = append(order, i)+ mu.Unlock()+ tickets[i] = tk+ tk.release()+ }()+ // They ask one after the other.+ require.Eventually(t, func() bool { return r.waiting(keys[0]) == i+1 }, 5*time.Second, time.Millisecond)+ }+ require.True(t, r.blocked(keys, nil), "the holder blocks the others")+ require.False(t, r.blocked(keys, first), "and not itself")+ first.release()+ wg.Wait()+ require.Equal(t, []int{0, 1, 2, 3, 4}, order)+ require.False(t, r.blocked(keys, nil), "nothing is kept of a key without holder or waiter")+ require.Empty(t, r.keys)+}++// TestLandTickets_aWaiterThatGivesUpLeavesTheQueue: the context ends while the land waits; it holds+// nothing, and the next waiter is not stuck behind it.+func TestLandTickets_aWaiterThatGivesUpLeavesTheQueue(t *testing.T) {+ t.Parallel()++ r := newTestTickets()+ keys := landTicketKeys("repo", []string{"a"}, false)+ holder, err := r.acquire(context.Background(), keys)+ require.NoError(t, err)++ ctx, cancel := context.WithCancel(context.Background())+ gone := make(chan error, 1)+ go func() {+ _, err := r.acquire(ctx, keys)+ gone <- err+ }()+ require.Eventually(t, func() bool { return r.waiting(keys[0]) == 1 }, 5*time.Second, time.Millisecond)++ next := make(chan *landTicket, 1)+ go func() {+ tk, _ := r.acquire(context.Background(), keys)+ next <- tk+ }()+ require.Eventually(t, func() bool { return r.waiting(keys[0]) == 2 }, 5*time.Second, time.Millisecond)++ cancel()+ require.ErrorIs(t, <-gone, context.Canceled)+ require.Equal(t, 1, r.waiting(keys[0]))+ holder.release()+ tk := <-next+ require.NotNil(t, tk)+ tk.release()+ require.Empty(t, r.keys)+}++// TestLandTickets_orderedKeysDoNotDeadlock: lands that need the same files in another order take them+// in the same (sorted) order, so none of them waits for the other while holding what it needs.+func TestLandTickets_orderedKeysDoNotDeadlock(t *testing.T) {+ t.Parallel()++ r := newTestTickets()+ var wg sync.WaitGroup+ for i := 0; i < 40; i++ {+ paths := []string{"a", "b", "c"}+ if i%2 == 1 {+ paths = []string{"c", "b", "a"}+ }+ wg.Add(1)+ go func() {+ defer wg.Done()+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)+ defer cancel()+ tk, err := r.acquire(ctx, landTicketKeys("repo", paths, false))+ require.NoError(t, err)+ time.Sleep(time.Millisecond)+ tk.release()+ }()+ }+ wg.Wait()+ require.Empty(t, r.keys)+}++// TestLandTickets_keys: a ticket is per file, or per branch when a verification is configured (the tree+// that is verified is the whole branch's), and a land with no file of its own takes the branch's.+func TestLandTickets_keys(t *testing.T) {+ t.Parallel()++ require.Equal(t, landTicketKeys("r", []string{"b", "a", "b"}, false), []string{"r\x00a", "r\x00b"})+ require.Equal(t, []string{"r\x00"}, landTicketKeys("r", []string{"a"}, true))+ require.Equal(t, []string{"r\x00"}, landTicketKeys("r", nil, false))+ require.NotEqual(t, landTicketScope("s", "x.git", []byte("main")), landTicketScope("s", "x.git", []byte("dev")))++ r := newTestTickets()+ tk, err := r.acquire(context.Background(), landTicketKeys("r", []string{"a", "b"}, false))+ require.NoError(t, err)+ require.True(t, tk.covers(landTicketKeys("r", []string{"a"}, false)))+ require.False(t, tk.covers(landTicketKeys("r", []string{"a", "c"}, false)))+ require.False(t, r.blocked(landTicketKeys("r", []string{"c"}, false), nil), "another file is free")+ require.True(t, r.blocked(landTicketKeys("r", []string{"c", "b"}, false), nil))+ tk.release()+ tk.release() // a second release does nothing+ require.False(t, (*landTicket)(nil).covers(nil))+}