Git designed for AI agents.
graft: bound the inline resolution of the lane and take no ticket for what only a model resolves
Federico Eliae3b60e3f79b0
diff --git a/internal/gitaly/service/operations/land_attempt.go b/internal/gitaly/service/operations/land_attempt.goindex 6a2bbba..070538e 100644--- a/internal/gitaly/service/operations/land_attempt.go+++ b/internal/gitaly/service/operations/land_attempt.go@@ -61,6 +61,8 @@ type landingAttempt struct {observe landPhaseObserver// mode is how the attempt makes its objects durable.mode landPublishMode+ // inline is the time the members of the attempt may lose to inline calls that fail (land_inline.go).+ inline landInlineBudget}// batchRebaser replays the commits of the changes of an attempt. It is rebase.Engine.@@ -239,7 +241,7 @@ func (a *landingAttempt) decide(ctx context.Context, tip git.ObjectID, req *land// UserLand to be resolved out of the lane. When the land fails otherwise, the request gets// the merge conflict it would have got without resolution, with the conflict package as a// second detail when the server listed the conflicts.- res, resolveErr := a.server.landApplyVerdicts(ctx, in, a.repo, tip, commitID, req.committer, req.preResolution)+ res, resolveErr := a.server.landApplyVerdicts(withLandInlineBudget(ctx, &a.inline), in, a.repo, tip, commitID, req.committer, req.preResolution)var need *landNeedsResolutionif errors.As(resolveErr, &need) {return landStep{}, needdiff --git a/internal/gitaly/service/operations/land_inline.go b/internal/gitaly/service/operations/land_inline.goindex 7c2e0fe..5c40e27 100644--- a/internal/gitaly/service/operations/land_inline.go+++ b/internal/gitaly/service/operations/land_inline.go@@ -2,7 +2,9 @@ package operationsimport ("context"+ "errors""fmt"+ "time""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/gitaly/hook"@@ -23,16 +25,57 @@ import (//// - the request asks for the rules only (Request.RulesOnly): graft-resolve never calls a model for it,// and says so in its answer (Response.RulesOnly). The split follows what the resolution is, not how-// long the last one took;-// - the call is bounded by inline_timeout (default 1s); if it does not end in time, or does not resolve-// every missing file as a rule, the member goes back out of the lane, as it did before, to be resolved-// there, with the queue of land_tickets.go;-// - a resolver that does not echo RulesOnly is an older graft-resolve, which may have called a model: the-// answer is dropped, and the lane never resolves inline again for the life of the process;+// long the last one took. The request announces no capability of the server but the origin, so that+// a graft-resolve that gates the models on the capabilities (any since the capabilities were added)+// refuses to call one even if it were to ignore RulesOnly; v0.2.0 has no gating and may call one;+// - the lane never waits: the call takes a slot of a pool of its own (a model that holds all the+// resolve slots does not stop the rules), and if there is none the member goes out of the lane;+// - the time the lane may lose to calls that fail is bounded once per batch (landInlineBudget): a call+// is given inline_timeout (default 1s) or what is left of the budget, and a call that fails, in time or+// not, is charged to it; once it is spent the other members of the batch go out of the lane without a+// call. A call that succeeds costs the batch nothing but its own time;+// - a member that the rules cannot resolve (the resolver says so) goes out of the lane without a ticket:+// without a verification what a model resolves never lands, so a ticket would protect nothing;+// - a resolver that does not echo RulesOnly is an older graft-resolve, which may have called a model:+// the answer is dropped, and the lane never resolves inline again for the life of the process;// - with a verification (verify.enabled) the lane never resolves: every tree is verified out of the lane;// - the answer is validated like any other (the client's acceptance rules, the modes, the origin), and// decideLand is the same: an answer that is not a rule's is never taken here.+// landInlineBudget is the time a batch attempt may lose to inline calls that fail (land_inline.go). The+// lane applies the members of an attempt one after the other, so it needs no lock.+type landInlineBudget struct {+ lost time.Duration+}++type landInlineBudgetKey struct{}++// withLandInlineBudget returns ctx carrying b, for the members of one batch attempt to share.+func withLandInlineBudget(ctx context.Context, b *landInlineBudget) context.Context {+ return context.WithValue(ctx, landInlineBudgetKey{}, b)+}++func landInlineBudgetFrom(ctx context.Context) *landInlineBudget {+ b, _ := ctx.Value(landInlineBudgetKey{}).(*landInlineBudget)+ return b+}++// landInlineResult is what became of the attempt to resolve a member inline.+type landInlineResult int++const (+ // landInlineNotTried: the lane does not resolve this member inline.+ landInlineNotTried landInlineResult = iota+ // landInlineResolved: every missing file has a verdict now.+ landInlineResolved+ // landInlineRulesCannot: the rules cannot resolve a file of the member (or it may not be sent): out of+ // the lane, for a model or the package.+ landInlineRulesCannot+ // landInlineSlow: the call did not end in time, found no slot, or the batch had spent its budget, or the+ // resolver is an older one: out of the lane, to be resolved there.+ landInlineSlow+)+// landInlineEnabled reports whether the lane resolves the rules inline: it is configured (inline_timeout is// not zero), there is no verification, and the resolver has not shown itself to be an older one.func (s *Server) landInlineEnabled() bool {@@ -41,18 +84,17 @@ func (s *Server) landInlineEnabled() bool {}// landResolveInline resolves, in the lane, the conflicting files of the member that have no verdict, as-// the rules would, and keeps the verdicts. It reports whether every one of them has a verdict now; it-// changes nothing when it reports false.-func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandRequest, tip, commitID git.ObjectID, l landListing, pre *landPreResolution) bool {+// the rules would, and keeps the verdicts. It changes nothing unless it returns landInlineResolved.+func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandRequest, tip, commitID git.ObjectID, l landListing, pre *landPreResolution) landInlineResult {if !s.landInlineEnabled() || pre.unverifiable != "" {- return false+ return landInlineNotTried}timeout, _ := s.resolveCfg.InlineTimeoutOrDefault()// An agent that may not update the branch gets nothing resolved, as out of the lane (the policy first):// the land goes out of the lane, which refuses it with the policy's message.reference := git.NewReferenceNameFromBranchName(string(in.GetBranch()))if err := hook.CheckAgentPolicy(ctx, s.graft, fmt.Sprintf("%s %s %s\n", tip, commitID, reference)); err != nil {- return false+ return landInlineNotTried}var pending []*landFileState@@ -64,63 +106,102 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReqif !f.sent() {// A file that may not be sent to the resolver is not resolved: the land goes out of the lane// and says why.- return false+ return landInlineRulesCannot}pending = append(pending, f)sent = append(sent, landRequestFile(f))}if len(pending) == 0 {- return false+ return landInlineNotTried}+ budget := landInlineBudgetFrom(ctx)+ if budget == nil {+ budget = &landInlineBudget{}+ }+ allowed := min(timeout, timeout-budget.lost)+ if allowed <= 0 {+ return landInlineSlow+ }+ slot, ok := s.conflictResolver.TryAcquireInlineSlot()+ if !ok {+ return landInlineSlow+ }+ defer slot.Release()+req := resolver.Request{Version: resolver.ProtocolVersion,Repository: resolver.Repository{Storage: in.GetRepository().GetStorageName(),RelativePath: in.GetRepository().GetRelativePath(),},- Branch: string(in.GetBranch()),- Commit: commitID.String(),- Messages: resolver.Messages{Commit: l.commitMessage},- Actor: identity.Actor(ctx, ""),- Files: sent,- Capabilities: resolver.ServerCapabilities(),+ Branch: string(in.GetBranch()),+ Commit: commitID.String(),+ Messages: resolver.Messages{Commit: l.commitMessage},+ Actor: identity.Actor(ctx, ""),+ Files: sent,+ // Only the origin: a graft-resolve that gates a model on the other capabilities refuses to call one.+ Capabilities: []string{resolver.CapabilityOrigin},RulesOnly: true,}- callCtx, cancel := context.WithTimeout(ctx, timeout)+ callCtx, cancel := context.WithTimeout(ctx, allowed)defer cancel()- resp, err := s.landResolveSlotted(callCtx, req)+ start := time.Now()+ resp, err := slot.Resolve(callCtx, req)if err != nil {- // Out of time, or the rules cannot resolve a file: the member is resolved out of the lane.+ budget.lost += time.Since(start)+ var refusal *resolver.UnresolvableError+ if errors.As(err, &refusal) || errors.Is(err, resolver.ErrUnresolvable) {+ // The rules cannot resolve a file of the member.+ return landInlineRulesCannot+ }s.logger.WithError(err).WithField("branch", string(in.GetBranch())).Debug("graft: inline resolution handed back")- return false+ return landInlineSlow+ }+ fail := func(result landInlineResult) landInlineResult {+ budget.lost += time.Since(start)+ return result}if !resp.RulesOnly {if s.landInlineOff.CompareAndSwap(false, true) {s.logger.WithField("resolver", resp.Resolver).Warn("graft: the resolver did not honour a request for its rules only (an older graft-resolve): the lane resolves out of the lane from now on")}- return false+ return fail(landInlineSlow)}if !landResolverNameValid(resp.Resolver) {- return false+ return fail(landInlineSlow)}landNormalizeOrigins(resp)byPath := make(map[string]resolver.ResponseFile, len(resp.Files))for _, rf := range resp.Files {if rf.FromModel() || rf.Hold != "" {// Never a model's in the lane, whatever was asked.- return false+ return fail(landInlineRulesCannot)}byPath[rf.Path] = rf}for _, f := range pending {rf, ok := byPath[f.conflict.path]if !ok || landModeOfSides(f.conflict, rf) != nil {- return false+ return fail(landInlineRulesCannot)}}for _, f := range pending {pre.verdicts[landKeyOf(f.conflict)] = landVerdict{file: byPath[f.conflict.path], resolver: resp.Resolver}}- return true+ return landInlineResolved+}++// landHasModelVerdict reports whether a file of the listing has a model's (or a held) verdict. Without a+// verification such a land does not land, so no other land has to give way to it.+func landHasModelVerdict(s *Server, l landListing, pre *landPreResolution) bool {+ if s.resolveCfg.Verify.Enabled {+ return false+ }+ for _, f := range l.files {+ if v, ok := pre.verdicts[landKeyOf(f.conflict)]; ok && (v.file.FromModel() || v.file.Hold != "") {+ return true+ }+ }+ return false}diff --git a/internal/gitaly/service/operations/land_inline_bounds_test.go b/internal/gitaly/service/operations/land_inline_bounds_test.gonew file mode 100644index 0000000..cc713cc--- /dev/null+++ b/internal/gitaly/service/operations/land_inline_bounds_test.go@@ -0,0 +1,384 @@+package operations++import (+ "context"+ "encoding/json"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// The tests of the bounds of the inline resolution (land_inline.go): what the lane gives away to a call+// that fails, what it takes from a reply, and what it never waits for.++// inlineProbe is a listing of the fixture's conflicts, classified at its tip, with a server that resolves+// inline, for the tests that call landResolveInline directly.+type inlineProbe struct {+ fx resolveFixture+ s *Server+ dir string+ l landListing+}++func newInlineProbe(t *testing.T, mode string, tune func(*config.GraftResolver)) inlineProbe {+ t.Helper()++ dir, argv := unionResolverMode(t, mode)+ rcfg := enabledResolver(argv)+ rcfg.InlineTimeout = ""+ if tune != nil {+ tune(&rcfg)+ }+ fx := newResolveFixture(t, rcfg)+ s := fx.server(t, rcfg)+ l, err := landListConflicts(fx.ctx, fx.repo, fx.tip, fx.commit, 1<<20)+ require.NoError(t, err)+ require.NoError(t, s.landClassify(fx.ctx, fx.repo, l.files, l.base, fx.tip, fx.commit, newLandClassifyCache(l.files, fx.tip)))+ return inlineProbe{fx: fx, s: s, dir: dir, l: l}+}++func (p inlineProbe) resolve(ctx context.Context, t *testing.T, pre *landPreResolution) landInlineResult {+ t.Helper()+ return p.s.landResolveInline(ctx, p.fx.request("Land"), p.fx.tip, p.fx.commit, p.l, pre)+}++// TestLandResolveInline_aReplyThatIsNotTheRulesIsRejected: a resolver that echoes rules_only but answers with+// a model's file, a file that says nothing of its origin, or a file it holds, is not taken: the member goes out+// of the lane as one the rules cannot resolve, and nothing of the reply is kept. The guard is the lane's own;+// decideLand would refuse such a land too, but only after the lane had taken the verdict.+func TestLandResolveInline_aReplyThatIsNotTheRulesIsRejected(t *testing.T) {+ t.Parallel()++ for _, mode := range []string{"echo-model", "echo-noorigin", "echo-hold"} {+ t.Run(mode, func(t *testing.T) {+ t.Parallel()++ p := newInlineProbe(t, mode, nil)+ pre := newLandPreResolution()+ require.Equal(t, landInlineRulesCannot, p.resolve(p.fx.ctx, t, pre))+ require.Empty(t, pre.verdicts, "nothing of the reply is kept")+ require.False(t, p.s.landInlineOff.Load(), "the resolver did honour the request")+ require.Equal(t, 1, inlineCalls(t, p.dir))+ })+ }+}++// TestLandResolveInline_announcesOnlyTheOrigin: the request asks for the rules only and announces no+// capability but the origin, so that a graft-resolve that gates the models on the capabilities refuses to+// call one even if it were to ignore rules_only.+func TestLandResolveInline_announcesOnlyTheOrigin(t *testing.T) {+ t.Parallel()++ p := newInlineProbe(t, "", nil)+ require.Equal(t, landInlineResolved, p.resolve(p.fx.ctx, t, newLandPreResolution()))++ reqs, err := filepath.Glob(filepath.Join(p.dir, "req-*.json"))+ require.NoError(t, err)+ require.Len(t, reqs, 1)+ raw, err := os.ReadFile(reqs[0])+ require.NoError(t, err)+ var req struct {+ RulesOnly bool `json:"rules_only"`+ Capabilities []string `json:"capabilities"`+ }+ require.NoError(t, json.Unmarshal(raw, &req))+ require.True(t, req.RulesOnly)+ require.Equal(t, []string{"origin"}, req.Capabilities)+}++// TestLandResolveInline_aResolverThatRefusesIsRulesCannot: the refusal of the rules is not a slow call: the+// member is one that only a model can resolve, and the lane charges the batch nothing but the call.+func TestLandResolveInline_aResolverThatRefusesIsRulesCannot(t *testing.T) {+ t.Parallel()++ p := newInlineProbe(t, "refuse", nil)+ pre := newLandPreResolution()+ require.Equal(t, landInlineRulesCannot, p.resolve(p.fx.ctx, t, pre))+ require.Empty(t, pre.verdicts)+ require.False(t, p.s.landInlineOff.Load())+}++// TestLandResolveInline_aBatchLosesAtMostItsBudget: the rule takes 3 s and the lane gives it 300 ms. Six+// members of one batch attempt (one budget) lose that once: the first call is cancelled, the other five are+// handed back without a call. Measured on the time the lane spends in the calls, which is what stalls the+// batch: six members at 300 ms each would be 1.8 s.+func TestLandResolveInline_aBatchLosesAtMostItsBudget(t *testing.T) {+ t.Parallel()++ const members = 6+ p := newInlineProbe(t, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })+ budget := &landInlineBudget{}+ ctx := withLandInlineBudget(p.fx.ctx, budget)++ start := time.Now()+ for i := 0; i < members; i++ {+ require.Equal(t, landInlineSlow, p.resolve(ctx, t, newLandPreResolution()), "member %d", i+1)+ }+ stalled := time.Since(start)++ require.Equal(t, 1, inlineCalls(t, p.dir), "only the first member called: the others found the budget spent")+ require.GreaterOrEqual(t, budget.lost, 250*time.Millisecond)+ require.Less(t, stalled, 1500*time.Millisecond, "the lane stalled once for the call, not once per member")++ // The next batch attempt has a budget of its own.+ require.Equal(t, landInlineSlow, p.resolve(withLandInlineBudget(p.fx.ctx, &landInlineBudget{}), t, newLandPreResolution()))+ require.Equal(t, 2, inlineCalls(t, p.dir))+}++// TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing: a batch of members that the rules resolve+// quickly is not cut short by the time those calls take.+func TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing(t *testing.T) {+ t.Parallel()++ p := newInlineProbe(t, "", nil)+ budget := &landInlineBudget{}+ ctx := withLandInlineBudget(p.fx.ctx, budget)+ for i := 0; i < 4; i++ {+ require.Equal(t, landInlineResolved, p.resolve(ctx, t, newLandPreResolution()), "member %d", i+1)+ }+ require.Zero(t, budget.lost)+}++// TestLandResolveInline_neverWaitsForASlot: the lane takes a slot of its own pool without waiting. With the+// pool taken the member goes out of the lane at once, without a call; and a model that holds every resolve+// slot of the resolver does not stop the rules.+func TestLandResolveInline_neverWaitsForASlot(t *testing.T) {+ t.Parallel()++ p := newInlineProbe(t, "", nil)++ // Every resolve slot is held by models: the rules go on.+ var held []interface{ Release() }+ for i := 0; i < p.s.resolveCfg.ResolveConcurrencyOrDefault(); i++ {+ slot, err := p.s.conflictResolver.AcquireResolveSlot(p.fx.ctx)+ require.NoError(t, err)+ held = append(held, slot)+ }+ require.Equal(t, landInlineResolved, p.resolve(p.fx.ctx, t, newLandPreResolution()), "the resolve pool is not the lane's")+ for _, slot := range held {+ slot.Release()+ }+ calls := inlineCalls(t, p.dir)++ // The lane's own pool is taken (by the other members in flight): no wait, no call.+ var own []interface{ Release() }+ for {+ slot, ok := p.s.conflictResolver.TryAcquireInlineSlot()+ if !ok {+ break+ }+ own = append(own, slot)+ }+ require.NotEmpty(t, own)+ done := make(chan landInlineResult, 1)+ go func() { done <- p.resolve(p.fx.ctx, t, newLandPreResolution()) }()+ select {+ case res := <-done:+ require.Equal(t, landInlineSlow, res)+ case <-time.After(10 * time.Second):+ t.Fatal("the lane waited for a slot")+ }+ require.Equal(t, calls, inlineCalls(t, p.dir), "no call without a slot")+ for _, slot := range own {+ slot.Release()+ }+ require.Equal(t, landInlineResolved, p.resolve(p.fx.ctx, t, newLandPreResolution()), "the slots were given back")+}++// TestUserLand_aLandNeedingAModelDoesNotHoldBackTheUnionLands: one land on a hot file needs a model (the+// rules cannot resolve it, and the model takes as long as it takes: here, until the test says so), and+// six others on the same file are resolved by a rule. Without a verification what the model resolves never+// lands, so the land that needs it takes no ticket and the others are not made to wait for it: all six+// land while the model is still working, and no ticket is held.+func TestUserLand_aLandNeedingAModelDoesNotHoldBackTheUnionLands(t *testing.T) {+ t.Parallel()++ q := newQueueFixtureMode(t, 6, "model", func(c *config.GraftResolver) { c.InlineTimeout = "" })+ model := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),+ gittest.WithTreeEntries(+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "model-1\n"},+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))++ modelDone := make(chan error, 1)+ go func() {+ _, err := q.land(model)+ modelDone <- err+ }()+ // The model land is at the resolver, out of the lane: its request is the one that is not for the rules.+ require.Eventually(t, func() bool { return unionCalls(t, q.dir)-inlineCalls(t, q.dir) >= 1 }, 30*time.Second, 10*time.Millisecond)++ landed := make(chan []error, 1)+ go func() { landed <- q.landAll(q.onA) }()+ select {+ case errs := <-landed:+ for i, err := range errs {+ require.NoError(t, err, "union land %d", i+1)+ }+ case <-time.After(60 * time.Second):+ t.Fatal("the union lands waited for the land that needs a model")+ }+ require.Empty(t, landTickets.keysOf(q.scope), "the land that needs a model holds no ticket")+ select {+ case err := <-modelDone:+ t.Fatalf("the model land ended before the model answered: %v", err)+ default:+ }+ require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), 7)++ // The model answers: its resolution is a proposal, as without a verification it always was.+ require.NoError(t, os.WriteFile(filepath.Join(q.dir, "release"), nil, 0o644))+ select {+ case err := <-modelDone:+ requireMergeConflict(t, err)+ require.NotNil(t, landConflictDetail(err), "the package with the proposal")+ case <-time.After(60 * time.Second):+ t.Fatal("the model land did not end")+ }+ require.Empty(t, landTickets.keysOf(q.scope))+}++// TestUserLand_aTicketOnTheFileDoesNotStopTheRulesInTheLane: another land holds the ticket of the file,+// and the lane resolves the rules of this one inline all the same: the ticket serves the out-of-lane+// resolution, and a rule does not wait for it. The block in the lane comes after the inline attempt; it+// hands back only what the lane could not resolve.+func TestUserLand_aTicketOnTheFileDoesNotStopTheRulesInTheLane(t *testing.T) {+ t.Parallel()++ q := newQueueFixture(t, 2, inlineOn)+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})+ require.NoError(t, err)+ defer holder.release()++ done := make(chan error, 1)+ go func() {+ _, err := q.land(q.onA[0])+ done <- err+ }()+ select {+ case err := <-done:+ require.NoError(t, err)+ case <-time.After(30 * time.Second):+ t.Fatal("the land waited for the holder of the ticket")+ }+ require.Equal(t, 1, inlineCalls(t, q.dir), "resolved in the lane")+ require.Equal(t, 0, landTickets.waiting(q.keyOfA), "it never queued")+ require.True(t, landTickets.blocked([]string{q.keyOfA}, nil), "the holder is still the holder")+}++// TestUserLand_aSlowRuleWaitsForTheTicketHolder: when the lane could not resolve inline (the rule did not end in+// time), the land goes out of the lane and, with another land holding the ticket of the file, waits for its+// turn there as before.+func TestUserLand_aSlowRuleWaitsForTheTicketHolder(t *testing.T) {+ t.Parallel()++ q := newQueueFixtureMode(t, 1, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})+ require.NoError(t, err)+ defer holder.release()++ done := make(chan error, 1)+ go func() {+ _, err := q.land(q.onA[0])+ done <- err+ }()+ require.Eventually(t, func() bool { return landTickets.waiting(q.keyOfA) == 1 }, 30*time.Second, 5*time.Millisecond, "the land never queued")+ select {+ case err := <-done:+ t.Fatalf("the land went past the holder of its file: %v", err)+ case <-time.After(300 * time.Millisecond):+ }+ holder.release()+ select {+ case err := <-done:+ require.NoError(t, err)+ case <-time.After(60 * time.Second):+ t.Fatal("the land did not land when the holder was done")+ }+}++// TestUserLand_oneFailedInlineCallDoesNotStartAConvoy: one rule call hiccups (3 s, cut at 300 ms) in the middle+// of a hot file's lands. The members the lane handed back are resolved out of the lane and come back, but no+// file stays ticketed: the next wave is resolved in the lane, at once, without one request out of it.+func TestUserLand_oneFailedInlineCallDoesNotStartAConvoy(t *testing.T) {+ t.Parallel()++ const lands = 8+ q := newQueueFixtureMode(t, 2*lands, "slowonce", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })++ for i, err := range q.landAll(q.onA[:lands]) {+ require.NoError(t, err, "land %d of the first wave", i+1)+ }+ require.Empty(t, landTickets.keysOf(q.scope), "no ticket is kept after the hiccup")+ outOfLane := unionCalls(t, q.dir) - inlineCalls(t, q.dir)+ inline := inlineCalls(t, q.dir)++ for i, err := range q.landAll(q.onA[lands:]) {+ require.NoError(t, err, "land %d of the second wave", i+1)+ }+ require.Equal(t, outOfLane, unionCalls(t, q.dir)-inlineCalls(t, q.dir), "the second wave never left the lane")+ require.Greater(t, inlineCalls(t, q.dir), inline)+ require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), 2*lands+1)+ require.Empty(t, landTickets.keysOf(q.scope))+}++// TestUserLand_theLaneRoundDoesNotCountTowardTheRounds: with resolve_rounds of 1 a land that only a model can+// resolve still leaves the lane for the pre-resolution: it ends with the proposal, not with the file+// NOT_ATTEMPTED and nothing proposed.+func TestUserLand_theLaneRoundDoesNotCountTowardTheRounds(t *testing.T) {+ t.Parallel()++ q := newQueueFixtureMode(t, 1, "model", func(c *config.GraftResolver) {+ c.InlineTimeout = ""+ c.ResolveRounds = 1+ })+ require.NoError(t, os.WriteFile(filepath.Join(q.dir, "release"), nil, 0o644))+ model := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),+ gittest.WithTreeEntries(+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "model-1\n"},+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))++ _, err := q.land(model)+ requireMergeConflict(t, err)+ pkg := landConflictDetail(err)+ require.NotNil(t, pkg)+ require.Equal(t, gitalypb.LandConflictFile_OUTCOME_PROPOSED, packageFile(t, pkg, "a").GetOutcome(), "the model was asked")+}++// TestUserLand_aLaneRoundThatHandsBackTakesNoTicket: the rule does not end in time, so the lane hands the land+// back; it is resolved out of the lane and comes back, and at no round does it hold a ticket (a ticket would+// make the lands on its file wait for it, for a hiccup of the lane).+func TestUserLand_aLaneRoundThatHandsBackTakesNoTicket(t *testing.T) {+ t.Parallel()++ var s *Server+ operationServerHooks.Store(t.Name(), func(served *Server) { s = served })+ q := newQueueFixtureMode(t, 1, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })+ require.NotNil(t, s)++ var mu sync.Mutex+ var held [][]string+ s.landLaneHook = func() {+ mu.Lock()+ defer mu.Unlock()+ held = append(held, landTickets.keysOf(q.scope))+ }+ _, err := q.land(q.onA[0])+ require.NoError(t, err)++ mu.Lock()+ defer mu.Unlock()+ require.GreaterOrEqual(t, len(held), 2, "the land went through the lane, out of it, and back")+ for round, keys := range held {+ require.Empty(t, keys, "ticket held at the lane round %d", round+1)+ }+ require.Positive(t, inlineCalls(t, q.dir))+}diff --git a/internal/gitaly/service/operations/land_inline_test.go b/internal/gitaly/service/operations/land_inline_test.goindex d763eb0..711e342 100644--- a/internal/gitaly/service/operations/land_inline_test.go+++ b/internal/gitaly/service/operations/land_inline_test.go@@ -5,7 +5,6 @@ import ("strings""sync""testing"- "time""github.com/stretchr/testify/require"@@ -64,30 +63,33 @@ func TestUserLand_theRulesAreResolvedInTheLane(t *testing.T) {require.Len(t, strings.Split(on.branchBlob(t, "a"), "\n"), lands+1, "every land is in the file")require.Positive(t, inlineCalls(t, on.dir), "the lane asked for the rules only")- off := newQueueFixture(t, lands, tune(false))+ // The control does not depend on who is faster: the resolver answers the two lands only once both have+ // asked, so both resolved against the same tip, and the one that lands second has a stale verdict and,+ // with one round, no other.+ off := newQueueFixtureMode(t, 2, "barrier", tune(false))failed := 0for _, err := range off.landAll(off.onA) {if err != nil {failed++}}- require.Positive(t, failed, "the control: without the inline resolution the cap of 1 round ends lands")+ require.Equal(t, 1, failed, "the control: without the inline resolution the cap of 1 round ends the land that lost it")require.Zero(t, inlineCalls(t, off.dir))}// TestUserLand_aRuleThatDoesNotEndInTimeIsHandedBack: the rule takes 3 s and the lane gives it 300 ms: the-// member goes out of the lane, and is resolved there (the resolver answers the ordinary request at once).+// members go out of the lane, and are resolved there (the resolver answers the ordinary request at once).+// How long the lane stalls is measured by TestLandResolveInline_aBatchLosesAtMostItsBudget.func TestUserLand_aRuleThatDoesNotEndInTimeIsHandedBack(t *testing.T) {t.Parallel()q := newQueueFixtureMode(t, 3, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })- start := time.Now()for i, err := range q.landAll(q.onA) {require.NoError(t, err, "land %d", i+1)}- require.Less(t, time.Since(start), 60*time.Second)require.Positive(t, inlineCalls(t, q.dir), "the lane tried")require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), 4)+ require.Empty(t, landTickets.keysOf(q.scope), "no ticket is kept")}// TestUserLand_anOlderResolverIsNeverAskedAgain: a resolver that does not say that it honoured the request@@ -173,12 +175,12 @@ func TestLandResolveInline_neverForAnAgentThePolicyDenies(t *testing.T) {require.NoError(t, err)require.NoError(t, s.landClassify(ctx, fx.repo, l.files, l.base, fx.tip, fx.commit, newLandClassifyCache(l.files, fx.tip)))pre := newLandPreResolution()- require.False(t, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre), "denied by the policy")+ require.Equal(t, landInlineNotTried, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre), "denied by the policy")require.Zero(t, unionCalls(t, dir), "nothing was resolved")// The same agent where the policy lets it update the branch.s.graft.Policy = []config.GraftPolicyRule{{Actor: "agent-7", Refs: []string{"refs/heads/**"}, Allow: true}}- require.True(t, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre))+ require.Equal(t, landInlineResolved, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre))require.Equal(t, 1, unionCalls(t, dir))}diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.goindex 6e5417b..01fc73a 100644--- a/internal/gitaly/service/operations/land_preresolve.go+++ b/internal/gitaly/service/operations/land_preresolve.go@@ -148,6 +148,10 @@ type landNeedsResolution struct {// queued says that another land holds the ticket of one of the files (paths): this one waits for// its turn before it resolves again.queued bool+ // rulesCannot says that the lane asked the rules to resolve the files and they cannot (or a file may+ // not be sent): without a verification what a model resolves never lands, so the land is resolved out+ // of the lane for the proposals and ends there, without a ticket.+ rulesCannot bool}func (e *landNeedsResolution) Error() string {@@ -280,11 +284,22 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest// without conflicts needs no pre-resolution at all, and one with conflicts that a rule resolves is// resolved there, in its batch. What the rules cannot resolve is handed back, and resolved out of the// lane in the next round.- laneFirst := s.landInlineEnabled()- for round := 1; round <= rounds; round++ {- used = round+ //+ // A round that enters the lane without a pre-resolution (a lane round) is not one of resolve_rounds: it+ // resolves nothing out of the lane, so with resolve_rounds = 1 the land that needs a model still+ // leaves the lane. A land that waited for its turn on a file goes back to the lane too, where the+ // rules are resolved inline whoever holds the ticket. A lane round is bounded all the same.+ lane := s.landInlineEnabled()+ laneRoundsLeft := landMaxLaneRounds+ for round := 1; round <= rounds; {+ used+++ laneRound := lane && laneRoundsLeft > 0+ lane = falsevar preErr error- if !(round == 1 && laneFirst) {+ if laneRound {+ laneRoundsLeft--+ } else {+ round++preErr = s.landPreResolve(resolveCtx, in, committer, pre)}if err := preErr; err != nil {@@ -321,7 +336,13 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest}// The land lost the round, to a land that changed its files (or to the tree that moved, with a// verification): it takes its turn on them. From now on it is the only land of the branch that- // resolves these files again, and the others wait without calling the resolver.+ // resolves these files again, and the others wait without calling the resolver. A lane round that+ // merely did not find the rules enough takes no turn: the lane has not lost anything, and without a+ // verification what the model resolves next never lands, so a ticket would protect nothing. Only a+ // lane round that was told to wait for its turn does.+ if laneRound && !need.queued || need.rulesCannot {+ continue+ }if err := s.landTakeTicket(resolveCtx, in, pre, need); err != nil {if deadline() {pre.ticket.release()@@ -335,11 +356,17 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest}return nil, err}+ // Back to the lane at once with the ticket: the rules are resolved there, no pre-resolution first.+ lane = laneRound && need.queued}pre.ticket.release()return s.landFinalPass(ctx, in, committer, pre, landStoppedReason(need, timeout))}+// landMaxLaneRounds bounds the rounds that enter the lane without a pre-resolution, which do not count+// toward resolve_rounds: the first, and the ones after a wait for a turn.+const landMaxLaneRounds = 4+// currentPaths are the paths of the conflicting files the land has at the latest tip it looked at.func (p *landPreResolution) currentPaths() []string {paths := make([]string, 0, len(p.current))diff --git a/internal/gitaly/service/operations/land_queue_test.go b/internal/gitaly/service/operations/land_queue_test.goindex bddcb16..bf1110f 100644--- a/internal/gitaly/service/operations/land_queue_test.go+++ b/internal/gitaly/service/operations/land_queue_test.go@@ -30,9 +30,16 @@ func unionResolver(t *testing.T) (string, []string) {return unionResolverMode(t, "")}-// unionResolverMode is unionResolver with a mode: "slow" takes 3 s to answer a request for the rules only-// (a rule that does not end in time), "noecho" does not say that it honoured the request (an older-// graft-resolve).+// unionResolverMode is unionResolver with a mode:+// - "slow": takes 3 s to answer a request for the rules only (a rule that does not end in time);+// - "slowonce": only the first request for the rules only takes 3 s (one hiccup);+// - "noecho": does not say that it honoured the request (an older graft-resolve);+// - "echo-model", "echo-noorigin", "echo-hold": echoes the request for the rules only, but answers with a+// model's file, one without an origin, or one with a hold;+// - "refuse": refuses every request for the rules only (the rules cannot resolve it);+// - "model": refuses the requests for the rules only on a file whose theirs side says "model", and+// answers the others with a model's resolution once dir/release exists (a model that takes a while);+// - "barrier": answers a request that is not for the rules only once two requests have come.func unionResolverMode(t *testing.T, mode string) (string, []string) {t.Helper()@@ -41,23 +48,58 @@ func unionResolverMode(t *testing.T, mode string) (string, []string) {DIR='%s'echo x >> "$DIR/calls.log"cat > "$DIR/req-$$.json"-python3 - "$DIR/req-$$.json" "$DIR" '%s' <<'PY'-import base64, json, sys, time+exec python3 - "$DIR/req-$$.json" "$DIR" '%s' <<'PY'+import base64, json, os, sys, timereq = json.load(open(sys.argv[1]))+dir, mode = sys.argv[2], sys.argv[3]rules = bool(req.get("rules_only"))if rules:- open(sys.argv[2] + "/inline.log", "a").write("x\n")- if sys.argv[3] == "slow":+ open(dir + "/inline.log", "a").write("x\n")+ if mode == "slow":time.sleep(3)+ if mode == "slowonce":+ try:+ os.close(os.open(dir + "/slowed", os.O_CREAT | os.O_EXCL))+ time.sleep(3)+ except FileExistsError:+ pass+ if mode == "refuse":+ sys.stderr.write("the rules cannot resolve it")+ sys.exit(1)+elif mode == "barrier":+ for _ in range(200):+ if len(open(dir + "/calls.log").read().split()) >= 2:+ break+ time.sleep(0.05)files = []for f in req["files"]:ours = base64.b64decode(f["ours"]["content"]).decode().splitlines()theirs = base64.b64decode(f["theirs"]["content"]).decode().splitlines()- merged = ours + [l for l in theirs if l not in ours]- files.append({"path": f["path"], "action": "write", "mode": "100644", "origin": "rule",- "content": base64.b64encode(("\n".join(merged) + "\n").encode()).decode()})+ model = any("model" in l for l in theirs)+ if mode == "model" and model:+ if rules:+ sys.stderr.write("needs a model")+ sys.exit(1)+ for _ in range(400):+ if os.path.exists(dir + "/release"):+ break+ time.sleep(0.05)+ merged = theirs+ origin = "model"+ else:+ merged = ours + [l for l in theirs if l not in ours]+ origin = "rule"+ entry = {"path": f["path"], "action": "write", "mode": "100644", "origin": origin,+ "content": base64.b64encode(("\n".join(merged) + "\n").encode()).decode()}+ if mode == "echo-model":+ entry["origin"] = "model"+ if mode == "echo-noorigin":+ del entry["origin"]+ if mode == "echo-hold":+ entry["hold"] = "held by the second model"+ files.append(entry)resp = {"version": 1, "resolver": "graft-resolve/union", "files": files}-if sys.argv[3] != "noecho":+if mode != "noecho":resp["rules_only"] = rulesjson.dump(resp, sys.stdout)PYdiff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.goindex ca9fe17..50e3ab3 100644--- a/internal/gitaly/service/operations/land_resolve.go+++ b/internal/gitaly/service/operations/land_resolve.go@@ -498,17 +498,6 @@ func (s *Server) landApplyVerdicts(return landResolution{}, err}pre.setCurrent(l.files)- // A land that lost a round holds the ticket of its files and is the one whose verdict lands: any- // other land on the same files waits for its turn (land_tickets.go), instead of landing first and- // invalidating the holder's verdict once more.- paths := make([]string, 0, len(l.files))- for _, f := range l.files {- paths = append(paths, f.conflict.path)- }- scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())- if landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket) {- return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: paths, queued: true}- }for _, f := range l.files {if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {missing = append(missing, f.conflict.path)@@ -526,16 +515,39 @@ func (s *Server) landApplyVerdicts(}// The rules are resolved here, in the lane, within a bound of time (land_inline.go): the member rides// the batch, as it did before the resolution left the lane. What a rule cannot resolve goes out.- if len(missing) > 0 && pre != nil && s.conflictResolver != nil && l.bases <= 1 && s.landResolveInline(ctx, in, tip, commitID, l, pre) {- missing = missing[:0]- for _, f := range l.files {- if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {- missing = append(missing, f.conflict.path)+ inline := landInlineNotTried+ if len(missing) > 0 && pre != nil && s.conflictResolver != nil && l.bases <= 1 {+ inline = s.landResolveInline(ctx, in, tip, commitID, l, pre)+ if inline == landInlineResolved {+ missing = missing[:0]+ for _, f := range l.files {+ if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {+ missing = append(missing, f.conflict.path)+ }}}}+ // What the rules cannot resolve, without a verification, is resolved out of the lane for the proposals+ // and ends there: what a model resolves never lands, so no ticket protects anything.+ rulesCannot := inline == landInlineRulesCannot && !s.resolveCfg.Verify.Enabled+ // A land that lost a round holds the ticket of its files and is the one whose verdict lands: any other+ // land on the same files, whose verdicts were made out of the lane, waits for its turn+ // (land_tickets.go), instead of landing first and invalidating the holder's verdict once more. A land+ // that the lane has just resolved inline needs no turn: the rules do not wait for the tickets.+ // Without a verification, a land whose files have a model's verdict does not land either: it is not+ // waited for.+ if pre != nil && s.conflictResolver != nil && l.bases <= 1 && inline != landInlineResolved && !rulesCannot && !landHasModelVerdict(s, l, pre) {+ paths := make([]string, 0, len(l.files))+ for _, f := range l.files {+ paths = append(paths, f.conflict.path)+ }+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())+ if landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket) {+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: paths, queued: true}+ }+ }if len(missing) > 0 {- return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: missing}+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: missing, rulesCannot: rulesCannot}}reused := map[string]resolver.ResponseFile{}diff --git a/internal/gitaly/service/operations/land_tickets.go b/internal/gitaly/service/operations/land_tickets.goindex 373f846..10c5aa7 100644--- a/internal/gitaly/service/operations/land_tickets.go+++ b/internal/gitaly/service/operations/land_tickets.go@@ -190,13 +190,16 @@ func (t *landTicket) release() {}}-// covers reports whether t holds a ticket for every one of keys.+// covers reports whether t holds a ticket for every one of keys: a released ticket holds none.func (t *landTicket) covers(keys []string) bool {if t == nil {return false}t.mu.Lock()defer t.mu.Unlock()+ if t.released {+ return false+ }held := make(map[string]bool, len(t.acquired))for _, k := range t.acquired {held[k] = truediff --git a/internal/gitaly/service/operations/land_tickets_test.go b/internal/gitaly/service/operations/land_tickets_test.goindex 81e063f..1a14ff0 100644--- a/internal/gitaly/service/operations/land_tickets_test.go+++ b/internal/gitaly/service/operations/land_tickets_test.go@@ -146,5 +146,6 @@ func TestLandTickets_keys(t *testing.T) {require.True(t, r.blocked(landTicketKeys("r", []string{"c", "b"}, false), nil))tk.release()tk.release() // a second release does nothing+ require.False(t, tk.covers(landTicketKeys("r", []string{"a"}, false)), "a released ticket covers nothing")require.False(t, (*landTicket)(nil).covers(nil))}diff --git a/internal/graft/resolver/check.go b/internal/graft/resolver/check.goindex f2ee760..2604c6a 100644--- a/internal/graft/resolver/check.go+++ b/internal/graft/resolver/check.go@@ -93,6 +93,23 @@ var resolveSlotWait = promauto.NewHistogram(prometheus.HistogramOpts{type ResolveSlot struct {c *Clientreleased bool+ // inline says that the slot is one of the pool of the lane's own calls.+ inline bool+}++// TryAcquireInlineSlot takes a slot of the pool of the calls with which the lane resolves the rules, without+// waiting: it reports false when all are taken, and the lane goes on without the call. Release it when the+// request is done.+func (c *Client) TryAcquireInlineSlot() (*ResolveSlot, bool) {+ if c == nil {+ return nil, false+ }+ select {+ case c.inlineSlots <- struct{}{}:+ return &ResolveSlot{c: c, inline: true}, true+ default:+ return nil, false+ }}// AcquireResolveSlot waits for a resolve slot. The wait counts against the deadline of ctx: when@@ -118,6 +135,10 @@ func (s *ResolveSlot) Release() {return}s.released = true+ if s.inline {+ <-s.c.inlineSlots+ return+ }<-s.c.resolveSlots}diff --git a/internal/graft/resolver/client.go b/internal/graft/resolver/client.goindex 34940bb..e3fc918 100644--- a/internal/graft/resolver/client.go+++ b/internal/graft/resolver/client.go@@ -86,6 +86,11 @@ func (e *UnresolvableError) Is(target error) bool {// Client calls the configured resolver. It is safe for concurrent use. A nil// *Client is the disabled resolver.+// minInlineSlots is the least size of the pool of the lane's own calls: they are rules, which take+// milliseconds, and each lane (one per branch) needs one at a time, so that the pool of one server is+// not the bottleneck of its branches.+const minInlineSlots = 8+type Client struct {command []stringurl string@@ -102,6 +107,9 @@ type Client struct {// the short checks (PLAN10 15.8).checkSlots chan struct{}resolveSlots chan struct{}+ // inlineSlots bounds the calls with which the lane resolves the rules itself (TryAcquireInlineSlot): a pool+ // of its own, which is never waited for, so that a model that holds the resolve slots does not stop the rules.+ inlineSlots chan struct{}// sendRules decide the files that are never sent (SendRules).sendRules *SendRules}@@ -159,6 +167,7 @@ func New(cfg config.GraftResolver) (*Client, error) {sendRules: rules,checkSlots: make(chan struct{}, cfg.CheckConcurrencyOrDefault()),resolveSlots: make(chan struct{}, cfg.ResolveConcurrencyOrDefault()),+ inlineSlots: make(chan struct{}, max(minInlineSlots, cfg.ResolveConcurrencyOrDefault())),}if cfg.URL != "" {