Skip to content

Git designed for AI agents.

graft: a model land gives its ticket back, the holder has priority, and all the inline time is the batch's

Federico Eliaf603f034d332
diff --git a/internal/gitaly/service/operations/land_inline.go b/internal/gitaly/service/operations/land_inline.go
index 5c40e27..c647abc 100644
--- a/internal/gitaly/service/operations/land_inline.go
+++ b/internal/gitaly/service/operations/land_inline.go
@@ -30,19 +30,24 @@ import (
// refuses to call one even if it were to ignore RulesOnly; v0.2.0 has no gating and may call one;
// - the lane never waits: the call takes a slot of a pool of its own (a model that holds all the
// resolve slots does not stop the rules), and if there is none the member goes out of the lane;
-// - the time the lane may lose to calls that fail is bounded once per batch (landInlineBudget): a call
-// is given inline_timeout (default 1s) or what is left of the budget, and a call that fails, in time or
-// not, is charged to it; once it is spent the other members of the batch go out of the lane without a
-// call. A call that succeeds costs the batch nothing but its own time;
-// - a member that the rules cannot resolve (the resolver says so) goes out of the lane without a ticket:
-// without a verification what a model resolves never lands, so a ticket would protect nothing;
+// - the time the lane spends in inline calls is bounded once per batch attempt (landInlineBudget): a call
+// is given inline_timeout (default 1s) or what is left of the budget, and all the time of every call,
+// one that succeeds as well as one that fails, is charged to it; once it is spent the other members of
+// the batch go out of the lane without a call, where the queue guarantees their progress. Fast rules
+// (milliseconds) do not notice it;
+// - a member that the rules cannot resolve (the resolver says so) goes out of the lane without a ticket,
+// and releases the one it took: without a verification what a model resolves never lands, so a ticket
+// would protect nothing and the lands on its file would wait for the model;
+// - a land that lost a round out of the lane holds its files with priority until it is done: the lane hands
+// back the inline resolutions of the others on them, which would change the files under its verdict at
+// every round (a rule slower than inline_timeout would never land on a hot file);
// - a resolver that does not echo RulesOnly is an older graft-resolve, which may have called a model:
// the answer is dropped, and the lane never resolves inline again for the life of the process;
// - with a verification (verify.enabled) the lane never resolves: every tree is verified out of the lane;
// - the answer is validated like any other (the client's acceptance rules, the modes, the origin), and
// decideLand is the same: an answer that is not a rule's is never taken here.
-// landInlineBudget is the time a batch attempt may lose to inline calls that fail (land_inline.go). The
+// landInlineBudget is the time a batch attempt may spend in inline calls (land_inline.go). The
// lane applies the members of an attempt one after the other, so it needs no lock.
type landInlineBudget struct {
lost time.Duration
@@ -148,8 +153,10 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReq
defer cancel()
start := time.Now()
resp, err := slot.Resolve(callCtx, req)
+ // All the time of the lane in the call is the batch's, whether the call succeeds or not: a rule that takes
+ // 0.9 s and succeeds stalls a batch of N members for N * 0.9 s as much as one that fails.
+ budget.lost += time.Since(start)
if err != nil {
- budget.lost += time.Since(start)
var refusal *resolver.UnresolvableError
if errors.As(err, &refusal) || errors.Is(err, resolver.ErrUnresolvable) {
// The rules cannot resolve a file of the member.
@@ -158,10 +165,7 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReq
s.logger.WithError(err).WithField("branch", string(in.GetBranch())).Debug("graft: inline resolution handed back")
return landInlineSlow
}
- fail := func(result landInlineResult) landInlineResult {
- budget.lost += time.Since(start)
- return result
- }
+ fail := func(result landInlineResult) landInlineResult { return result }
if !resp.RulesOnly {
if s.landInlineOff.CompareAndSwap(false, true) {
s.logger.WithField("resolver", resp.Resolver).Warn("graft: the resolver did not honour a request for its rules only (an older graft-resolve): the lane resolves out of the lane from now on")
@@ -192,6 +196,34 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReq
return landInlineResolved
}
+// landListingKeys are the tickets of the files of a listing.
+func (s *Server) landListingKeys(in *gitalypb.UserLandRequest, l landListing) []string {
+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
+ return landTicketKeys(scope, landListingPaths(l), s.resolveCfg.Verify.Enabled)
+}
+
+func landListingPaths(l landListing) []string {
+ paths := make([]string, 0, len(l.files))
+ for _, f := range l.files {
+ paths = append(paths, f.conflict.path)
+ }
+ return paths
+}
+
+// landWillNotLand reports whether the files the land has now carry a model's verdict (or a held one) and no
+// verification is configured: such a land ends with the package, so no other land gives way to it.
+func (s *Server) landWillNotLand(pre *landPreResolution) bool {
+ if s.resolveCfg.Verify.Enabled {
+ return false
+ }
+ for k := range pre.current {
+ if v, ok := pre.verdicts[k]; ok && (v.file.FromModel() || v.file.Hold != "") {
+ return true
+ }
+ }
+ return false
+}
+
// landHasModelVerdict reports whether a file of the listing has a model's (or a held) verdict. Without a
// verification such a land does not land, so no other land has to give way to it.
func landHasModelVerdict(s *Server, l landListing, pre *landPreResolution) bool {
diff --git a/internal/gitaly/service/operations/land_inline_bounds_test.go b/internal/gitaly/service/operations/land_inline_bounds_test.go
index f5fa466..a86b860 100644
--- a/internal/gitaly/service/operations/land_inline_bounds_test.go
+++ b/internal/gitaly/service/operations/land_inline_bounds_test.go
@@ -7,6 +7,7 @@ import (
"path/filepath"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -134,18 +135,20 @@ func TestLandResolveInline_aBatchLosesAtMostItsBudget(t *testing.T) {
require.Equal(t, 2, inlineCalls(t, p.dir))
}
-// TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing: a batch of members that the rules resolve
-// quickly is not cut short by the time those calls take.
-func TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing(t *testing.T) {
+// TestLandResolveInline_aBatchOfFastRulesIsNotCutShort: members that the rules resolve in milliseconds, against a
+// budget that is large next to them, are all resolved in the lane; the time of their calls is charged all the same
+// (the other side of TestLandResolveInline_aSlowRuleThatSucceedsIsChargedToTheBudget).
+func TestLandResolveInline_aBatchOfFastRulesIsNotCutShort(t *testing.T) {
t.Parallel()
- p := newInlineProbe(t, "", nil)
+ p := newInlineProbe(t, "", func(c *config.GraftResolver) { c.InlineTimeout = "30s" })
budget := &landInlineBudget{}
ctx := withLandInlineBudget(p.fx.ctx, budget)
for i := 0; i < 4; i++ {
require.Equal(t, landInlineResolved, p.resolve(ctx, t, newLandPreResolution()), "member %d", i+1)
}
- require.Zero(t, budget.lost)
+ require.Positive(t, budget.lost, "the time of the calls is charged")
+ require.Less(t, budget.lost, 30*time.Second)
}
// TestLandResolveInline_neverWaitsForASlot: the lane takes a slot of its own pool without waiting. With the
@@ -416,3 +419,137 @@ func TestUserLand_withNoInlineSlotAllTheLandsStillLand(t *testing.T) {
require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), lands+1)
require.Empty(t, landTickets.keysOf(q.scope))
}
+
+// TestUserLand_aModelLandReleasesTheTicketItQueuedFor: a land that needs a model queues behind a holder (no inline
+// slot is free, so the lane could not try the rules) and takes the ticket; the next lane round finds that the
+// rules cannot resolve it, and it must give the ticket back before it calls the model out of the lane: a model's
+// resolution never lands without a verification, and the lands on the file would wait for it.
+func TestUserLand_aModelLandReleasesTheTicketItQueuedFor(t *testing.T) {
+ t.Parallel()
+
+ var s *Server
+ operationServerHooks.Store(t.Name(), func(served *Server) { s = served })
+ q := newQueueFixtureMode(t, 1, "model", func(c *config.GraftResolver) { c.InlineTimeout = "" })
+ require.NotNil(t, s)
+
+ var own []interface{ Release() }
+ for {
+ slot, ok := s.conflictResolver.TryAcquireInlineSlot()
+ if !ok {
+ break
+ }
+ own = append(own, slot)
+ }
+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})
+ require.NoError(t, err)
+ defer holder.release()
+
+ model := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),
+ gittest.WithTreeEntries(
+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "model-1\n"},
+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))
+ done := make(chan error, 1)
+ go func() {
+ _, err := q.land(model)
+ done <- err
+ }()
+ require.Eventually(t, func() bool { return landTickets.waiting(q.keyOfA) == 1 }, 30*time.Second, 5*time.Millisecond, "the model land queued")
+ for _, slot := range own {
+ slot.Release()
+ }
+ holder.release()
+
+ // The model request is in flight: the one that is not for the rules only.
+ require.Eventually(t, func() bool { return inlineCalls(t, q.dir) >= 1 && unionCalls(t, q.dir)-inlineCalls(t, q.dir) >= 1 }, 30*time.Second, 5*time.Millisecond)
+ require.False(t, landTickets.blocked([]string{q.keyOfA}, nil), "the land that needs a model holds the ticket of the file during the model call")
+
+ require.NoError(t, os.WriteFile(filepath.Join(q.dir, "release"), nil, 0o644))
+ select {
+ case err := <-done:
+ requireMergeConflict(t, err)
+ case <-time.After(60 * time.Second):
+ t.Fatal("the model land did not end")
+ }
+}
+
+// TestLandResolveInline_aSlowRuleThatSucceedsIsChargedToTheBudget: the rule takes 0.4 s and ends, and the lane gives it
+// 1 s: six members of one batch attempt do not stall the lane for 6 x 0.4 s. All the time of the calls is the
+// batch's, and once it is spent the rest go out of the lane without a call.
+func TestLandResolveInline_aSlowRuleThatSucceedsIsChargedToTheBudget(t *testing.T) {
+ t.Parallel()
+
+ const members = 6
+ p := newInlineProbe(t, "slowok", func(c *config.GraftResolver) { c.InlineTimeout = "1s" })
+ budget := &landInlineBudget{}
+ ctx := withLandInlineBudget(p.fx.ctx, budget)
+
+ start := time.Now()
+ resolved := 0
+ for i := 0; i < members; i++ {
+ if p.resolve(ctx, t, newLandPreResolution()) == landInlineResolved {
+ resolved++
+ }
+ }
+ stalled := time.Since(start)
+
+ require.Positive(t, resolved, "a rule that ends in time is taken")
+ require.Less(t, resolved, members, "but not for every member")
+ require.Less(t, inlineCalls(t, p.dir), members, "the members that found the budget spent did not call")
+ require.Less(t, stalled, 1800*time.Millisecond, "the lane stalled for the budget, not for 6 x 0.4 s")
+ require.GreaterOrEqual(t, budget.lost, 400*time.Millisecond)
+}
+
+// TestUserLand_aRuleSlowerThanTheTimeoutLandsOnAHotFile: a rule takes 2 s, longer than inline_timeout, on a file that
+// other lands change all the time (each resolved by a fast rule in the lane). Its verdict would be stale at every
+// round; once it has lost a round it holds the file with priority, the inline resolutions of the others wait
+// behind it, and it lands while the stream of the others is still going. All of them land.
+func TestUserLand_aRuleSlowerThanTheTimeoutLandsOnAHotFile(t *testing.T) {
+ t.Parallel()
+
+ const stream = 60
+ q := newQueueFixtureMode(t, stream, "slowrule", func(c *config.GraftResolver) { c.InlineTimeout = "500ms" })
+ slow := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),
+ gittest.WithTreeEntries(
+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "slowrule-1\n"},
+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))
+
+ slowDone := make(chan error, 1)
+ go func() {
+ _, err := q.land(slow)
+ slowDone <- err
+ }()
+ // The slow land is resolving out of the lane (the lane gave up its rule at the timeout).
+ require.Eventually(t, func() bool { return unionCalls(t, q.dir)-inlineCalls(t, q.dir) >= 1 }, 60*time.Second, 5*time.Millisecond)
+
+ var landed atomic.Int32
+ streamDone := make(chan error, 1)
+ go func() {
+ for _, c := range q.onA {
+ if _, err := q.land(c); err != nil {
+ streamDone <- err
+ return
+ }
+ landed.Add(1)
+ }
+ streamDone <- nil
+ }()
+
+ select {
+ case err := <-slowDone:
+ require.NoError(t, err, "the slow rule's land")
+ case <-time.After(120 * time.Second):
+ t.Fatal("the land with the slow rule did not end")
+ }
+ require.Less(t, int(landed.Load()), stream, "the slow land landed while the stream of the others was going: they gave way")
+
+ select {
+ case err := <-streamDone:
+ require.NoError(t, err)
+ case <-time.After(120 * time.Second):
+ t.Fatal("the others did not end")
+ }
+ got := q.branchBlob(t, "a")
+ require.Contains(t, got, "slowrule-1")
+ require.Len(t, strings.Split(got, "\n"), stream+2, "ours, the slow rule's line and every other land's")
+ require.Empty(t, landTickets.keysOf(q.scope))
+}
diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.go
index 01fc73a..a8e3baf 100644
--- a/internal/gitaly/service/operations/land_preresolve.go
+++ b/internal/gitaly/service/operations/land_preresolve.go
@@ -340,7 +340,14 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
// merely did not find the rules enough takes no turn: the lane has not lost anything, and without a
// verification what the model resolves next never lands, so a ticket would protect nothing. Only a
// lane round that was told to wait for its turn does.
- if laneRound && !need.queued || need.rulesCannot {
+ if need.rulesCannot {
+ // The rules cannot resolve it: what is next is a model's call out of the lane, which never lands
+ // without a verification. It holds no ticket while it runs, even the one it took when it queued.
+ pre.ticket.release()
+ pre.ticket = nil
+ continue
+ }
+ if laneRound && !need.queued {
continue
}
if err := s.landTakeTicket(resolveCtx, in, pre, need); err != nil {
@@ -356,6 +363,11 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
}
return nil, err
}
+ if !laneRound {
+ // It lost a round out of the lane: its resolution is the one that must land, and the inline
+ // resolutions of the others give way to it until it is done.
+ pre.ticket.setPriority()
+ }
// Back to the lane at once with the ticket: the rules are resolved there, no pre-resolution first.
lane = laneRound && need.queued
}
@@ -379,7 +391,7 @@ func (p *landPreResolution) currentPaths() []string {
// landTicketsBlock reports whether another land holds the ticket of one of paths.
func (s *Server) landTicketsBlock(in *gitalypb.UserLandRequest, paths []string, pre *landPreResolution) bool {
- if len(paths) == 0 {
+ if len(paths) == 0 || s.landWillNotLand(pre) {
return false
}
scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
diff --git a/internal/gitaly/service/operations/land_queue_test.go b/internal/gitaly/service/operations/land_queue_test.go
index bf1110f..edc3cab 100644
--- a/internal/gitaly/service/operations/land_queue_test.go
+++ b/internal/gitaly/service/operations/land_queue_test.go
@@ -32,6 +32,9 @@ func unionResolver(t *testing.T) (string, []string) {
// unionResolverMode is unionResolver with a mode:
// - "slow": takes 3 s to answer a request for the rules only (a rule that does not end in time);
+// - "slowok": takes 0.4 s to answer a request for the rules only, and answers (a rule that is slow but ends);
+// - "slowrule": a file whose theirs side says "slowrule" takes 2 s to resolve, for any request (a rule that
+// is slower than inline_timeout and succeeds);
// - "slowonce": only the first request for the rules only takes 3 s (one hiccup);
// - "noecho": does not say that it honoured the request (an older graft-resolve);
// - "echo-model", "echo-noorigin", "echo-hold": echoes the request for the rules only, but answers with a
@@ -57,6 +60,8 @@ if rules:
open(dir + "/inline.log", "a").write("x\n")
if mode == "slow":
time.sleep(3)
+ if mode == "slowok":
+ time.sleep(0.4)
if mode == "slowonce":
try:
os.close(os.open(dir + "/slowed", os.O_CREAT | os.O_EXCL))
@@ -76,6 +81,8 @@ for f in req["files"]:
ours = base64.b64decode(f["ours"]["content"]).decode().splitlines()
theirs = base64.b64decode(f["theirs"]["content"]).decode().splitlines()
model = any("model" in l for l in theirs)
+ if mode == "slowrule" and any("slowrule" in l for l in theirs):
+ time.sleep(2)
if mode == "model" and model:
if rules:
sys.stderr.write("needs a model")
diff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.go
index 50e3ab3..8efec82 100644
--- a/internal/gitaly/service/operations/land_resolve.go
+++ b/internal/gitaly/service/operations/land_resolve.go
@@ -517,6 +517,11 @@ func (s *Server) landApplyVerdicts(
// the batch, as it did before the resolution left the lane. What a rule cannot resolve goes out.
inline := landInlineNotTried
if len(missing) > 0 && pre != nil && s.conflictResolver != nil && l.bases <= 1 {
+ // A land that lost a round out of the lane holds the files with priority: resolving them here would
+ // change them under its verdict once more, so the lands that would give way and wait their turn.
+ if s.landInlineEnabled() && landTickets.priorityBlocked(s.landListingKeys(in, l), pre.ticket) {
+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: landListingPaths(l), queued: true}
+ }
inline = s.landResolveInline(ctx, in, tip, commitID, l, pre)
if inline == landInlineResolved {
missing = missing[:0]
diff --git a/internal/gitaly/service/operations/land_tickets.go b/internal/gitaly/service/operations/land_tickets.go
index 10c5aa7..83dbd47 100644
--- a/internal/gitaly/service/operations/land_tickets.go
+++ b/internal/gitaly/service/operations/land_tickets.go
@@ -67,6 +67,10 @@ type landTicket struct {
mu sync.Mutex
acquired []string // the keys held, in the order taken
released bool
+ // priority says that the holder resolves its files out of the lane after losing a round: until it ends, the
+ // lane hands back the inline resolutions of the other lands on them, which would change the files again and
+ // make its verdict stale at every round (priorityBlocked).
+ priority bool
}
// landTickets are the tickets of the process, shared by all the servers of it.
@@ -212,6 +216,32 @@ func (t *landTicket) covers(keys []string) bool {
return true
}
+// setPriority makes the ticket one that the inline resolutions of the other lands give way to.
+func (t *landTicket) setPriority() {
+ t.mu.Lock()
+ t.priority = true
+ t.mu.Unlock()
+}
+
+func (t *landTicket) hasPriority() bool {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ return t.priority && !t.released
+}
+
+// priorityBlocked reports whether another land holds, with priority, a ticket for one of keys: the land that
+// asks must not change those files in the lane until the holder is done.
+func (r *landTicketRegistry) priorityBlocked(keys []string, me *landTicket) bool {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for _, key := range keys {
+ if k := r.keys[key]; k != nil && k.holder != nil && k.holder != me && k.holder.hasPriority() {
+ return true
+ }
+ }
+ return false
+}
+
// blocked reports whether another land holds a ticket for one of keys: the land that asks, which is me
// when it holds tickets (nil otherwise), must wait for its turn.
func (r *landTicketRegistry) blocked(keys []string, me *landTicket) bool {
diff --git a/internal/graft/resolver/client.go b/internal/graft/resolver/client.go
index e3fc918..fe6de72 100644
--- a/internal/graft/resolver/client.go
+++ b/internal/graft/resolver/client.go
@@ -84,13 +84,13 @@ func (e *UnresolvableError) Is(target error) bool {
return target == ErrUnresolvable
}
-// Client calls the configured resolver. It is safe for concurrent use. A nil
-// *Client is the disabled resolver.
// minInlineSlots is the least size of the pool of the lane's own calls: they are rules, which take
// milliseconds, and each lane (one per branch) needs one at a time, so that the pool of one server is
// not the bottleneck of its branches.
const minInlineSlots = 8
+// Client calls the configured resolver. It is safe for concurrent use. A nil
+// *Client is the disabled resolver.
type Client struct {
command []string
url string