Git designed for AI agents.
graft: a model land gives its ticket back, the holder has priority, and all the inline time is the batch's
Federico Eliaf603f034d332
diff --git a/internal/gitaly/service/operations/land_inline.go b/internal/gitaly/service/operations/land_inline.goindex 5c40e27..c647abc 100644--- a/internal/gitaly/service/operations/land_inline.go+++ b/internal/gitaly/service/operations/land_inline.go@@ -30,19 +30,24 @@ import (// refuses to call one even if it were to ignore RulesOnly; v0.2.0 has no gating and may call one;// - the lane never waits: the call takes a slot of a pool of its own (a model that holds all the// resolve slots does not stop the rules), and if there is none the member goes out of the lane;-// - the time the lane may lose to calls that fail is bounded once per batch (landInlineBudget): a call-// is given inline_timeout (default 1s) or what is left of the budget, and a call that fails, in time or-// not, is charged to it; once it is spent the other members of the batch go out of the lane without a-// call. A call that succeeds costs the batch nothing but its own time;-// - a member that the rules cannot resolve (the resolver says so) goes out of the lane without a ticket:-// without a verification what a model resolves never lands, so a ticket would protect nothing;+// - the time the lane spends in inline calls is bounded once per batch attempt (landInlineBudget): a call+// is given inline_timeout (default 1s) or what is left of the budget, and all the time of every call,+// one that succeeds as well as one that fails, is charged to it; once it is spent the other members of+// the batch go out of the lane without a call, where the queue guarantees their progress. Fast rules+// (milliseconds) do not notice it;+// - a member that the rules cannot resolve (the resolver says so) goes out of the lane without a ticket,+// and releases the one it took: without a verification what a model resolves never lands, so a ticket+// would protect nothing and the lands on its file would wait for the model;+// - a land that lost a round out of the lane holds its files with priority until it is done: the lane hands+// back the inline resolutions of the others on them, which would change the files under its verdict at+// every round (a rule slower than inline_timeout would never land on a hot file);// - a resolver that does not echo RulesOnly is an older graft-resolve, which may have called a model:// the answer is dropped, and the lane never resolves inline again for the life of the process;// - with a verification (verify.enabled) the lane never resolves: every tree is verified out of the lane;// - the answer is validated like any other (the client's acceptance rules, the modes, the origin), and// decideLand is the same: an answer that is not a rule's is never taken here.-// landInlineBudget is the time a batch attempt may lose to inline calls that fail (land_inline.go). The+// landInlineBudget is the time a batch attempt may spend in inline calls (land_inline.go). The// lane applies the members of an attempt one after the other, so it needs no lock.type landInlineBudget struct {lost time.Duration@@ -148,8 +153,10 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReqdefer cancel()start := time.Now()resp, err := slot.Resolve(callCtx, req)+ // All the time of the lane in the call is the batch's, whether the call succeeds or not: a rule that takes+ // 0.9 s and succeeds stalls a batch of N members for N * 0.9 s as much as one that fails.+ budget.lost += time.Since(start)if err != nil {- budget.lost += time.Since(start)var refusal *resolver.UnresolvableErrorif errors.As(err, &refusal) || errors.Is(err, resolver.ErrUnresolvable) {// The rules cannot resolve a file of the member.@@ -158,10 +165,7 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReqs.logger.WithError(err).WithField("branch", string(in.GetBranch())).Debug("graft: inline resolution handed back")return landInlineSlow}- fail := func(result landInlineResult) landInlineResult {- budget.lost += time.Since(start)- return result- }+ fail := func(result landInlineResult) landInlineResult { return result }if !resp.RulesOnly {if s.landInlineOff.CompareAndSwap(false, true) {s.logger.WithField("resolver", resp.Resolver).Warn("graft: the resolver did not honour a request for its rules only (an older graft-resolve): the lane resolves out of the lane from now on")@@ -192,6 +196,34 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReqreturn landInlineResolved}+// landListingKeys are the tickets of the files of a listing.+func (s *Server) landListingKeys(in *gitalypb.UserLandRequest, l landListing) []string {+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())+ return landTicketKeys(scope, landListingPaths(l), s.resolveCfg.Verify.Enabled)+}++func landListingPaths(l landListing) []string {+ paths := make([]string, 0, len(l.files))+ for _, f := range l.files {+ paths = append(paths, f.conflict.path)+ }+ return paths+}++// landWillNotLand reports whether the files the land has now carry a model's verdict (or a held one) and no+// verification is configured: such a land ends with the package, so no other land gives way to it.+func (s *Server) landWillNotLand(pre *landPreResolution) bool {+ if s.resolveCfg.Verify.Enabled {+ return false+ }+ for k := range pre.current {+ if v, ok := pre.verdicts[k]; ok && (v.file.FromModel() || v.file.Hold != "") {+ return true+ }+ }+ return false+}+// landHasModelVerdict reports whether a file of the listing has a model's (or a held) verdict. Without a// verification such a land does not land, so no other land has to give way to it.func landHasModelVerdict(s *Server, l landListing, pre *landPreResolution) bool {diff --git a/internal/gitaly/service/operations/land_inline_bounds_test.go b/internal/gitaly/service/operations/land_inline_bounds_test.goindex f5fa466..a86b860 100644--- a/internal/gitaly/service/operations/land_inline_bounds_test.go+++ b/internal/gitaly/service/operations/land_inline_bounds_test.go@@ -7,6 +7,7 @@ import ("path/filepath""strings""sync"+ "sync/atomic""testing""time"@@ -134,18 +135,20 @@ func TestLandResolveInline_aBatchLosesAtMostItsBudget(t *testing.T) {require.Equal(t, 2, inlineCalls(t, p.dir))}-// TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing: a batch of members that the rules resolve-// quickly is not cut short by the time those calls take.-func TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing(t *testing.T) {+// TestLandResolveInline_aBatchOfFastRulesIsNotCutShort: members that the rules resolve in milliseconds, against a+// budget that is large next to them, are all resolved in the lane; the time of their calls is charged all the same+// (the other side of TestLandResolveInline_aSlowRuleThatSucceedsIsChargedToTheBudget).+func TestLandResolveInline_aBatchOfFastRulesIsNotCutShort(t *testing.T) {t.Parallel()- p := newInlineProbe(t, "", nil)+ p := newInlineProbe(t, "", func(c *config.GraftResolver) { c.InlineTimeout = "30s" })budget := &landInlineBudget{}ctx := withLandInlineBudget(p.fx.ctx, budget)for i := 0; i < 4; i++ {require.Equal(t, landInlineResolved, p.resolve(ctx, t, newLandPreResolution()), "member %d", i+1)}- require.Zero(t, budget.lost)+ require.Positive(t, budget.lost, "the time of the calls is charged")+ require.Less(t, budget.lost, 30*time.Second)}// TestLandResolveInline_neverWaitsForASlot: the lane takes a slot of its own pool without waiting. With the@@ -416,3 +419,137 @@ func TestUserLand_withNoInlineSlotAllTheLandsStillLand(t *testing.T) {require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), lands+1)require.Empty(t, landTickets.keysOf(q.scope))}++// TestUserLand_aModelLandReleasesTheTicketItQueuedFor: a land that needs a model queues behind a holder (no inline+// slot is free, so the lane could not try the rules) and takes the ticket; the next lane round finds that the+// rules cannot resolve it, and it must give the ticket back before it calls the model out of the lane: a model's+// resolution never lands without a verification, and the lands on the file would wait for it.+func TestUserLand_aModelLandReleasesTheTicketItQueuedFor(t *testing.T) {+ t.Parallel()++ var s *Server+ operationServerHooks.Store(t.Name(), func(served *Server) { s = served })+ q := newQueueFixtureMode(t, 1, "model", func(c *config.GraftResolver) { c.InlineTimeout = "" })+ require.NotNil(t, s)++ var own []interface{ Release() }+ for {+ slot, ok := s.conflictResolver.TryAcquireInlineSlot()+ if !ok {+ break+ }+ own = append(own, slot)+ }+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})+ require.NoError(t, err)+ defer holder.release()++ model := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),+ gittest.WithTreeEntries(+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "model-1\n"},+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))+ done := make(chan error, 1)+ go func() {+ _, err := q.land(model)+ done <- err+ }()+ require.Eventually(t, func() bool { return landTickets.waiting(q.keyOfA) == 1 }, 30*time.Second, 5*time.Millisecond, "the model land queued")+ for _, slot := range own {+ slot.Release()+ }+ holder.release()++ // The model request is in flight: the one that is not for the rules only.+ require.Eventually(t, func() bool { return inlineCalls(t, q.dir) >= 1 && unionCalls(t, q.dir)-inlineCalls(t, q.dir) >= 1 }, 30*time.Second, 5*time.Millisecond)+ require.False(t, landTickets.blocked([]string{q.keyOfA}, nil), "the land that needs a model holds the ticket of the file during the model call")++ require.NoError(t, os.WriteFile(filepath.Join(q.dir, "release"), nil, 0o644))+ select {+ case err := <-done:+ requireMergeConflict(t, err)+ case <-time.After(60 * time.Second):+ t.Fatal("the model land did not end")+ }+}++// TestLandResolveInline_aSlowRuleThatSucceedsIsChargedToTheBudget: the rule takes 0.4 s and ends, and the lane gives it+// 1 s: six members of one batch attempt do not stall the lane for 6 x 0.4 s. All the time of the calls is the+// batch's, and once it is spent the rest go out of the lane without a call.+func TestLandResolveInline_aSlowRuleThatSucceedsIsChargedToTheBudget(t *testing.T) {+ t.Parallel()++ const members = 6+ p := newInlineProbe(t, "slowok", func(c *config.GraftResolver) { c.InlineTimeout = "1s" })+ budget := &landInlineBudget{}+ ctx := withLandInlineBudget(p.fx.ctx, budget)++ start := time.Now()+ resolved := 0+ for i := 0; i < members; i++ {+ if p.resolve(ctx, t, newLandPreResolution()) == landInlineResolved {+ resolved+++ }+ }+ stalled := time.Since(start)++ require.Positive(t, resolved, "a rule that ends in time is taken")+ require.Less(t, resolved, members, "but not for every member")+ require.Less(t, inlineCalls(t, p.dir), members, "the members that found the budget spent did not call")+ require.Less(t, stalled, 1800*time.Millisecond, "the lane stalled for the budget, not for 6 x 0.4 s")+ require.GreaterOrEqual(t, budget.lost, 400*time.Millisecond)+}++// TestUserLand_aRuleSlowerThanTheTimeoutLandsOnAHotFile: a rule takes 2 s, longer than inline_timeout, on a file that+// other lands change all the time (each resolved by a fast rule in the lane). Its verdict would be stale at every+// round; once it has lost a round it holds the file with priority, the inline resolutions of the others wait+// behind it, and it lands while the stream of the others is still going. All of them land.+func TestUserLand_aRuleSlowerThanTheTimeoutLandsOnAHotFile(t *testing.T) {+ t.Parallel()++ const stream = 60+ q := newQueueFixtureMode(t, stream, "slowrule", func(c *config.GraftResolver) { c.InlineTimeout = "500ms" })+ slow := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),+ gittest.WithTreeEntries(+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "slowrule-1\n"},+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))++ slowDone := make(chan error, 1)+ go func() {+ _, err := q.land(slow)+ slowDone <- err+ }()+ // The slow land is resolving out of the lane (the lane gave up its rule at the timeout).+ require.Eventually(t, func() bool { return unionCalls(t, q.dir)-inlineCalls(t, q.dir) >= 1 }, 60*time.Second, 5*time.Millisecond)++ var landed atomic.Int32+ streamDone := make(chan error, 1)+ go func() {+ for _, c := range q.onA {+ if _, err := q.land(c); err != nil {+ streamDone <- err+ return+ }+ landed.Add(1)+ }+ streamDone <- nil+ }()++ select {+ case err := <-slowDone:+ require.NoError(t, err, "the slow rule's land")+ case <-time.After(120 * time.Second):+ t.Fatal("the land with the slow rule did not end")+ }+ require.Less(t, int(landed.Load()), stream, "the slow land landed while the stream of the others was going: they gave way")++ select {+ case err := <-streamDone:+ require.NoError(t, err)+ case <-time.After(120 * time.Second):+ t.Fatal("the others did not end")+ }+ got := q.branchBlob(t, "a")+ require.Contains(t, got, "slowrule-1")+ require.Len(t, strings.Split(got, "\n"), stream+2, "ours, the slow rule's line and every other land's")+ require.Empty(t, landTickets.keysOf(q.scope))+}diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.goindex 01fc73a..a8e3baf 100644--- a/internal/gitaly/service/operations/land_preresolve.go+++ b/internal/gitaly/service/operations/land_preresolve.go@@ -340,7 +340,14 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest// merely did not find the rules enough takes no turn: the lane has not lost anything, and without a// verification what the model resolves next never lands, so a ticket would protect nothing. Only a// lane round that was told to wait for its turn does.- if laneRound && !need.queued || need.rulesCannot {+ if need.rulesCannot {+ // The rules cannot resolve it: what is next is a model's call out of the lane, which never lands+ // without a verification. It holds no ticket while it runs, even the one it took when it queued.+ pre.ticket.release()+ pre.ticket = nil+ continue+ }+ if laneRound && !need.queued {continue}if err := s.landTakeTicket(resolveCtx, in, pre, need); err != nil {@@ -356,6 +363,11 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest}return nil, err}+ if !laneRound {+ // It lost a round out of the lane: its resolution is the one that must land, and the inline+ // resolutions of the others give way to it until it is done.+ pre.ticket.setPriority()+ }// Back to the lane at once with the ticket: the rules are resolved there, no pre-resolution first.lane = laneRound && need.queued}@@ -379,7 +391,7 @@ func (p *landPreResolution) currentPaths() []string {// landTicketsBlock reports whether another land holds the ticket of one of paths.func (s *Server) landTicketsBlock(in *gitalypb.UserLandRequest, paths []string, pre *landPreResolution) bool {- if len(paths) == 0 {+ if len(paths) == 0 || s.landWillNotLand(pre) {return false}scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())diff --git a/internal/gitaly/service/operations/land_queue_test.go b/internal/gitaly/service/operations/land_queue_test.goindex bf1110f..edc3cab 100644--- a/internal/gitaly/service/operations/land_queue_test.go+++ b/internal/gitaly/service/operations/land_queue_test.go@@ -32,6 +32,9 @@ func unionResolver(t *testing.T) (string, []string) {// unionResolverMode is unionResolver with a mode:// - "slow": takes 3 s to answer a request for the rules only (a rule that does not end in time);+// - "slowok": takes 0.4 s to answer a request for the rules only, and answers (a rule that is slow but ends);+// - "slowrule": a file whose theirs side says "slowrule" takes 2 s to resolve, for any request (a rule that+// is slower than inline_timeout and succeeds);// - "slowonce": only the first request for the rules only takes 3 s (one hiccup);// - "noecho": does not say that it honoured the request (an older graft-resolve);// - "echo-model", "echo-noorigin", "echo-hold": echoes the request for the rules only, but answers with a@@ -57,6 +60,8 @@ if rules:open(dir + "/inline.log", "a").write("x\n")if mode == "slow":time.sleep(3)+ if mode == "slowok":+ time.sleep(0.4)if mode == "slowonce":try:os.close(os.open(dir + "/slowed", os.O_CREAT | os.O_EXCL))@@ -76,6 +81,8 @@ for f in req["files"]:ours = base64.b64decode(f["ours"]["content"]).decode().splitlines()theirs = base64.b64decode(f["theirs"]["content"]).decode().splitlines()model = any("model" in l for l in theirs)+ if mode == "slowrule" and any("slowrule" in l for l in theirs):+ time.sleep(2)if mode == "model" and model:if rules:sys.stderr.write("needs a model")diff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.goindex 50e3ab3..8efec82 100644--- a/internal/gitaly/service/operations/land_resolve.go+++ b/internal/gitaly/service/operations/land_resolve.go@@ -517,6 +517,11 @@ func (s *Server) landApplyVerdicts(// the batch, as it did before the resolution left the lane. What a rule cannot resolve goes out.inline := landInlineNotTriedif len(missing) > 0 && pre != nil && s.conflictResolver != nil && l.bases <= 1 {+ // A land that lost a round out of the lane holds the files with priority: resolving them here would+ // change them under its verdict once more, so the lands that would give way and wait their turn.+ if s.landInlineEnabled() && landTickets.priorityBlocked(s.landListingKeys(in, l), pre.ticket) {+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: landListingPaths(l), queued: true}+ }inline = s.landResolveInline(ctx, in, tip, commitID, l, pre)if inline == landInlineResolved {missing = missing[:0]diff --git a/internal/gitaly/service/operations/land_tickets.go b/internal/gitaly/service/operations/land_tickets.goindex 10c5aa7..83dbd47 100644--- a/internal/gitaly/service/operations/land_tickets.go+++ b/internal/gitaly/service/operations/land_tickets.go@@ -67,6 +67,10 @@ type landTicket struct {mu sync.Mutexacquired []string // the keys held, in the order takenreleased bool+ // priority says that the holder resolves its files out of the lane after losing a round: until it ends, the+ // lane hands back the inline resolutions of the other lands on them, which would change the files again and+ // make its verdict stale at every round (priorityBlocked).+ priority bool}// landTickets are the tickets of the process, shared by all the servers of it.@@ -212,6 +216,32 @@ func (t *landTicket) covers(keys []string) bool {return true}+// setPriority makes the ticket one that the inline resolutions of the other lands give way to.+func (t *landTicket) setPriority() {+ t.mu.Lock()+ t.priority = true+ t.mu.Unlock()+}++func (t *landTicket) hasPriority() bool {+ t.mu.Lock()+ defer t.mu.Unlock()+ return t.priority && !t.released+}++// priorityBlocked reports whether another land holds, with priority, a ticket for one of keys: the land that+// asks must not change those files in the lane until the holder is done.+func (r *landTicketRegistry) priorityBlocked(keys []string, me *landTicket) bool {+ r.mu.Lock()+ defer r.mu.Unlock()+ for _, key := range keys {+ if k := r.keys[key]; k != nil && k.holder != nil && k.holder != me && k.holder.hasPriority() {+ return true+ }+ }+ return false+}+// blocked reports whether another land holds a ticket for one of keys: the land that asks, which is me// when it holds tickets (nil otherwise), must wait for its turn.func (r *landTicketRegistry) blocked(keys []string, me *landTicket) bool {diff --git a/internal/graft/resolver/client.go b/internal/graft/resolver/client.goindex e3fc918..fe6de72 100644--- a/internal/graft/resolver/client.go+++ b/internal/graft/resolver/client.go@@ -84,13 +84,13 @@ func (e *UnresolvableError) Is(target error) bool {return target == ErrUnresolvable}-// Client calls the configured resolver. It is safe for concurrent use. A nil-// *Client is the disabled resolver.// minInlineSlots is the least size of the pool of the lane's own calls: they are rules, which take// milliseconds, and each lane (one per branch) needs one at a time, so that the pool of one server is// not the bottleneck of its branches.const minInlineSlots = 8+// Client calls the configured resolver. It is safe for concurrent use. A nil+// *Client is the disabled resolver.type Client struct {command []stringurl string