Skip to content

Git designed for AI agents.

graft: bound the inline resolution of the lane and take no ticket for what only a model resolves

Federico Eliae3b60e3f79b0
diff --git a/internal/gitaly/service/operations/land_attempt.go b/internal/gitaly/service/operations/land_attempt.go
index 6a2bbba..070538e 100644
--- a/internal/gitaly/service/operations/land_attempt.go
+++ b/internal/gitaly/service/operations/land_attempt.go
@@ -61,6 +61,8 @@ type landingAttempt struct {
observe landPhaseObserver
// mode is how the attempt makes its objects durable.
mode landPublishMode
+ // inline is the time the members of the attempt may lose to inline calls that fail (land_inline.go).
+ inline landInlineBudget
}
// batchRebaser replays the commits of the changes of an attempt. It is rebase.Engine.
@@ -239,7 +241,7 @@ func (a *landingAttempt) decide(ctx context.Context, tip git.ObjectID, req *land
// UserLand to be resolved out of the lane. When the land fails otherwise, the request gets
// the merge conflict it would have got without resolution, with the conflict package as a
// second detail when the server listed the conflicts.
- res, resolveErr := a.server.landApplyVerdicts(ctx, in, a.repo, tip, commitID, req.committer, req.preResolution)
+ res, resolveErr := a.server.landApplyVerdicts(withLandInlineBudget(ctx, &a.inline), in, a.repo, tip, commitID, req.committer, req.preResolution)
var need *landNeedsResolution
if errors.As(resolveErr, &need) {
return landStep{}, need
diff --git a/internal/gitaly/service/operations/land_inline.go b/internal/gitaly/service/operations/land_inline.go
index 7c2e0fe..5c40e27 100644
--- a/internal/gitaly/service/operations/land_inline.go
+++ b/internal/gitaly/service/operations/land_inline.go
@@ -2,7 +2,9 @@ package operations
import (
"context"
+ "errors"
"fmt"
+ "time"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/gitaly/hook"
@@ -23,16 +25,57 @@ import (
//
// - the request asks for the rules only (Request.RulesOnly): graft-resolve never calls a model for it,
// and says so in its answer (Response.RulesOnly). The split follows what the resolution is, not how
-// long the last one took;
-// - the call is bounded by inline_timeout (default 1s); if it does not end in time, or does not resolve
-// every missing file as a rule, the member goes back out of the lane, as it did before, to be resolved
-// there, with the queue of land_tickets.go;
-// - a resolver that does not echo RulesOnly is an older graft-resolve, which may have called a model: the
-// answer is dropped, and the lane never resolves inline again for the life of the process;
+// long the last one took. The request announces no capability of the server but the origin, so that
+// a graft-resolve that gates the models on the capabilities (any since the capabilities were added)
+// refuses to call one even if it were to ignore RulesOnly; v0.2.0 has no gating and may call one;
+// - the lane never waits: the call takes a slot of a pool of its own (a model that holds all the
+// resolve slots does not stop the rules), and if there is none the member goes out of the lane;
+// - the time the lane may lose to calls that fail is bounded once per batch (landInlineBudget): a call
+// is given inline_timeout (default 1s) or what is left of the budget, and a call that fails, in time or
+// not, is charged to it; once it is spent the other members of the batch go out of the lane without a
+// call. A call that succeeds costs the batch nothing but its own time;
+// - a member that the rules cannot resolve (the resolver says so) goes out of the lane without a ticket:
+// without a verification what a model resolves never lands, so a ticket would protect nothing;
+// - a resolver that does not echo RulesOnly is an older graft-resolve, which may have called a model:
+// the answer is dropped, and the lane never resolves inline again for the life of the process;
// - with a verification (verify.enabled) the lane never resolves: every tree is verified out of the lane;
// - the answer is validated like any other (the client's acceptance rules, the modes, the origin), and
// decideLand is the same: an answer that is not a rule's is never taken here.
+// landInlineBudget is the time a batch attempt may lose to inline calls that fail (land_inline.go). The
+// lane applies the members of an attempt one after the other, so it needs no lock.
+type landInlineBudget struct {
+ lost time.Duration
+}
+
+type landInlineBudgetKey struct{}
+
+// withLandInlineBudget returns ctx carrying b, for the members of one batch attempt to share.
+func withLandInlineBudget(ctx context.Context, b *landInlineBudget) context.Context {
+ return context.WithValue(ctx, landInlineBudgetKey{}, b)
+}
+
+func landInlineBudgetFrom(ctx context.Context) *landInlineBudget {
+ b, _ := ctx.Value(landInlineBudgetKey{}).(*landInlineBudget)
+ return b
+}
+
+// landInlineResult is what became of the attempt to resolve a member inline.
+type landInlineResult int
+
+const (
+ // landInlineNotTried: the lane does not resolve this member inline.
+ landInlineNotTried landInlineResult = iota
+ // landInlineResolved: every missing file has a verdict now.
+ landInlineResolved
+ // landInlineRulesCannot: the rules cannot resolve a file of the member (or it may not be sent): out of
+ // the lane, for a model or the package.
+ landInlineRulesCannot
+ // landInlineSlow: the call did not end in time, found no slot, or the batch had spent its budget, or the
+ // resolver is an older one: out of the lane, to be resolved there.
+ landInlineSlow
+)
+
// landInlineEnabled reports whether the lane resolves the rules inline: it is configured (inline_timeout is
// not zero), there is no verification, and the resolver has not shown itself to be an older one.
func (s *Server) landInlineEnabled() bool {
@@ -41,18 +84,17 @@ func (s *Server) landInlineEnabled() bool {
}
// landResolveInline resolves, in the lane, the conflicting files of the member that have no verdict, as
-// the rules would, and keeps the verdicts. It reports whether every one of them has a verdict now; it
-// changes nothing when it reports false.
-func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandRequest, tip, commitID git.ObjectID, l landListing, pre *landPreResolution) bool {
+// the rules would, and keeps the verdicts. It changes nothing unless it returns landInlineResolved.
+func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandRequest, tip, commitID git.ObjectID, l landListing, pre *landPreResolution) landInlineResult {
if !s.landInlineEnabled() || pre.unverifiable != "" {
- return false
+ return landInlineNotTried
}
timeout, _ := s.resolveCfg.InlineTimeoutOrDefault()
// An agent that may not update the branch gets nothing resolved, as out of the lane (the policy first):
// the land goes out of the lane, which refuses it with the policy's message.
reference := git.NewReferenceNameFromBranchName(string(in.GetBranch()))
if err := hook.CheckAgentPolicy(ctx, s.graft, fmt.Sprintf("%s %s %s\n", tip, commitID, reference)); err != nil {
- return false
+ return landInlineNotTried
}
var pending []*landFileState
@@ -64,63 +106,102 @@ func (s *Server) landResolveInline(ctx context.Context, in *gitalypb.UserLandReq
if !f.sent() {
// A file that may not be sent to the resolver is not resolved: the land goes out of the lane
// and says why.
- return false
+ return landInlineRulesCannot
}
pending = append(pending, f)
sent = append(sent, landRequestFile(f))
}
if len(pending) == 0 {
- return false
+ return landInlineNotTried
}
+ budget := landInlineBudgetFrom(ctx)
+ if budget == nil {
+ budget = &landInlineBudget{}
+ }
+ allowed := min(timeout, timeout-budget.lost)
+ if allowed <= 0 {
+ return landInlineSlow
+ }
+ slot, ok := s.conflictResolver.TryAcquireInlineSlot()
+ if !ok {
+ return landInlineSlow
+ }
+ defer slot.Release()
+
req := resolver.Request{
Version: resolver.ProtocolVersion,
Repository: resolver.Repository{
Storage: in.GetRepository().GetStorageName(),
RelativePath: in.GetRepository().GetRelativePath(),
},
- Branch: string(in.GetBranch()),
- Commit: commitID.String(),
- Messages: resolver.Messages{Commit: l.commitMessage},
- Actor: identity.Actor(ctx, ""),
- Files: sent,
- Capabilities: resolver.ServerCapabilities(),
+ Branch: string(in.GetBranch()),
+ Commit: commitID.String(),
+ Messages: resolver.Messages{Commit: l.commitMessage},
+ Actor: identity.Actor(ctx, ""),
+ Files: sent,
+ // Only the origin: a graft-resolve that gates a model on the other capabilities refuses to call one.
+ Capabilities: []string{resolver.CapabilityOrigin},
RulesOnly: true,
}
- callCtx, cancel := context.WithTimeout(ctx, timeout)
+ callCtx, cancel := context.WithTimeout(ctx, allowed)
defer cancel()
- resp, err := s.landResolveSlotted(callCtx, req)
+ start := time.Now()
+ resp, err := slot.Resolve(callCtx, req)
if err != nil {
- // Out of time, or the rules cannot resolve a file: the member is resolved out of the lane.
+ budget.lost += time.Since(start)
+ var refusal *resolver.UnresolvableError
+ if errors.As(err, &refusal) || errors.Is(err, resolver.ErrUnresolvable) {
+ // The rules cannot resolve a file of the member.
+ return landInlineRulesCannot
+ }
s.logger.WithError(err).WithField("branch", string(in.GetBranch())).Debug("graft: inline resolution handed back")
- return false
+ return landInlineSlow
+ }
+ fail := func(result landInlineResult) landInlineResult {
+ budget.lost += time.Since(start)
+ return result
}
if !resp.RulesOnly {
if s.landInlineOff.CompareAndSwap(false, true) {
s.logger.WithField("resolver", resp.Resolver).Warn("graft: the resolver did not honour a request for its rules only (an older graft-resolve): the lane resolves out of the lane from now on")
}
- return false
+ return fail(landInlineSlow)
}
if !landResolverNameValid(resp.Resolver) {
- return false
+ return fail(landInlineSlow)
}
landNormalizeOrigins(resp)
byPath := make(map[string]resolver.ResponseFile, len(resp.Files))
for _, rf := range resp.Files {
if rf.FromModel() || rf.Hold != "" {
// Never a model's in the lane, whatever was asked.
- return false
+ return fail(landInlineRulesCannot)
}
byPath[rf.Path] = rf
}
for _, f := range pending {
rf, ok := byPath[f.conflict.path]
if !ok || landModeOfSides(f.conflict, rf) != nil {
- return false
+ return fail(landInlineRulesCannot)
}
}
for _, f := range pending {
pre.verdicts[landKeyOf(f.conflict)] = landVerdict{file: byPath[f.conflict.path], resolver: resp.Resolver}
}
- return true
+ return landInlineResolved
+}
+
+// landHasModelVerdict reports whether a file of the listing has a model's (or a held) verdict. Without a
+// verification such a land does not land, so no other land has to give way to it.
+func landHasModelVerdict(s *Server, l landListing, pre *landPreResolution) bool {
+ if s.resolveCfg.Verify.Enabled {
+ return false
+ }
+ for _, f := range l.files {
+ if v, ok := pre.verdicts[landKeyOf(f.conflict)]; ok && (v.file.FromModel() || v.file.Hold != "") {
+ return true
+ }
+ }
+ return false
}
diff --git a/internal/gitaly/service/operations/land_inline_bounds_test.go b/internal/gitaly/service/operations/land_inline_bounds_test.go
new file mode 100644
index 0000000..cc713cc
--- /dev/null
+++ b/internal/gitaly/service/operations/land_inline_bounds_test.go
@@ -0,0 +1,384 @@
+package operations
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+)
+
+// The tests of the bounds of the inline resolution (land_inline.go): what the lane gives away to a call
+// that fails, what it takes from a reply, and what it never waits for.
+
+// inlineProbe is a listing of the fixture's conflicts, classified at its tip, with a server that resolves
+// inline, for the tests that call landResolveInline directly.
+type inlineProbe struct {
+ fx resolveFixture
+ s *Server
+ dir string
+ l landListing
+}
+
+func newInlineProbe(t *testing.T, mode string, tune func(*config.GraftResolver)) inlineProbe {
+ t.Helper()
+
+ dir, argv := unionResolverMode(t, mode)
+ rcfg := enabledResolver(argv)
+ rcfg.InlineTimeout = ""
+ if tune != nil {
+ tune(&rcfg)
+ }
+ fx := newResolveFixture(t, rcfg)
+ s := fx.server(t, rcfg)
+ l, err := landListConflicts(fx.ctx, fx.repo, fx.tip, fx.commit, 1<<20)
+ require.NoError(t, err)
+ require.NoError(t, s.landClassify(fx.ctx, fx.repo, l.files, l.base, fx.tip, fx.commit, newLandClassifyCache(l.files, fx.tip)))
+ return inlineProbe{fx: fx, s: s, dir: dir, l: l}
+}
+
+func (p inlineProbe) resolve(ctx context.Context, t *testing.T, pre *landPreResolution) landInlineResult {
+ t.Helper()
+ return p.s.landResolveInline(ctx, p.fx.request("Land"), p.fx.tip, p.fx.commit, p.l, pre)
+}
+
+// TestLandResolveInline_aReplyThatIsNotTheRulesIsRejected: a resolver that echoes rules_only but answers with
+// a model's file, a file that says nothing of its origin, or a file it holds, is not taken: the member goes out
+// of the lane as one the rules cannot resolve, and nothing of the reply is kept. The guard is the lane's own;
+// decideLand would refuse such a land too, but only after the lane had taken the verdict.
+func TestLandResolveInline_aReplyThatIsNotTheRulesIsRejected(t *testing.T) {
+ t.Parallel()
+
+ for _, mode := range []string{"echo-model", "echo-noorigin", "echo-hold"} {
+ t.Run(mode, func(t *testing.T) {
+ t.Parallel()
+
+ p := newInlineProbe(t, mode, nil)
+ pre := newLandPreResolution()
+ require.Equal(t, landInlineRulesCannot, p.resolve(p.fx.ctx, t, pre))
+ require.Empty(t, pre.verdicts, "nothing of the reply is kept")
+ require.False(t, p.s.landInlineOff.Load(), "the resolver did honour the request")
+ require.Equal(t, 1, inlineCalls(t, p.dir))
+ })
+ }
+}
+
+// TestLandResolveInline_announcesOnlyTheOrigin: the request asks for the rules only and announces no
+// capability but the origin, so that a graft-resolve that gates the models on the capabilities refuses to
+// call one even if it were to ignore rules_only.
+func TestLandResolveInline_announcesOnlyTheOrigin(t *testing.T) {
+ t.Parallel()
+
+ p := newInlineProbe(t, "", nil)
+ require.Equal(t, landInlineResolved, p.resolve(p.fx.ctx, t, newLandPreResolution()))
+
+ reqs, err := filepath.Glob(filepath.Join(p.dir, "req-*.json"))
+ require.NoError(t, err)
+ require.Len(t, reqs, 1)
+ raw, err := os.ReadFile(reqs[0])
+ require.NoError(t, err)
+ var req struct {
+ RulesOnly bool `json:"rules_only"`
+ Capabilities []string `json:"capabilities"`
+ }
+ require.NoError(t, json.Unmarshal(raw, &req))
+ require.True(t, req.RulesOnly)
+ require.Equal(t, []string{"origin"}, req.Capabilities)
+}
+
+// TestLandResolveInline_aResolverThatRefusesIsRulesCannot: the refusal of the rules is not a slow call: the
+// member is one that only a model can resolve, and the lane charges the batch nothing but the call.
+func TestLandResolveInline_aResolverThatRefusesIsRulesCannot(t *testing.T) {
+ t.Parallel()
+
+ p := newInlineProbe(t, "refuse", nil)
+ pre := newLandPreResolution()
+ require.Equal(t, landInlineRulesCannot, p.resolve(p.fx.ctx, t, pre))
+ require.Empty(t, pre.verdicts)
+ require.False(t, p.s.landInlineOff.Load())
+}
+
+// TestLandResolveInline_aBatchLosesAtMostItsBudget: the rule takes 3 s and the lane gives it 300 ms. Six
+// members of one batch attempt (one budget) lose that once: the first call is cancelled, the other five are
+// handed back without a call. Measured on the time the lane spends in the calls, which is what stalls the
+// batch: six members at 300 ms each would be 1.8 s.
+func TestLandResolveInline_aBatchLosesAtMostItsBudget(t *testing.T) {
+ t.Parallel()
+
+ const members = 6
+ p := newInlineProbe(t, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })
+ budget := &landInlineBudget{}
+ ctx := withLandInlineBudget(p.fx.ctx, budget)
+
+ start := time.Now()
+ for i := 0; i < members; i++ {
+ require.Equal(t, landInlineSlow, p.resolve(ctx, t, newLandPreResolution()), "member %d", i+1)
+ }
+ stalled := time.Since(start)
+
+ require.Equal(t, 1, inlineCalls(t, p.dir), "only the first member called: the others found the budget spent")
+ require.GreaterOrEqual(t, budget.lost, 250*time.Millisecond)
+ require.Less(t, stalled, 1500*time.Millisecond, "the lane stalled once for the call, not once per member")
+
+ // The next batch attempt has a budget of its own.
+ require.Equal(t, landInlineSlow, p.resolve(withLandInlineBudget(p.fx.ctx, &landInlineBudget{}), t, newLandPreResolution()))
+ require.Equal(t, 2, inlineCalls(t, p.dir))
+}
+
+// TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing: a batch of members that the rules resolve
+// quickly is not cut short by the time those calls take.
+func TestLandResolveInline_aCallThatSucceedsCostsTheBudgetNothing(t *testing.T) {
+ t.Parallel()
+
+ p := newInlineProbe(t, "", nil)
+ budget := &landInlineBudget{}
+ ctx := withLandInlineBudget(p.fx.ctx, budget)
+ for i := 0; i < 4; i++ {
+ require.Equal(t, landInlineResolved, p.resolve(ctx, t, newLandPreResolution()), "member %d", i+1)
+ }
+ require.Zero(t, budget.lost)
+}
+
+// TestLandResolveInline_neverWaitsForASlot: the lane takes a slot of its own pool without waiting. With the
+// pool taken the member goes out of the lane at once, without a call; and a model that holds every resolve
+// slot of the resolver does not stop the rules.
+func TestLandResolveInline_neverWaitsForASlot(t *testing.T) {
+ t.Parallel()
+
+ p := newInlineProbe(t, "", nil)
+
+ // Every resolve slot is held by models: the rules go on.
+ var held []interface{ Release() }
+ for i := 0; i < p.s.resolveCfg.ResolveConcurrencyOrDefault(); i++ {
+ slot, err := p.s.conflictResolver.AcquireResolveSlot(p.fx.ctx)
+ require.NoError(t, err)
+ held = append(held, slot)
+ }
+ require.Equal(t, landInlineResolved, p.resolve(p.fx.ctx, t, newLandPreResolution()), "the resolve pool is not the lane's")
+ for _, slot := range held {
+ slot.Release()
+ }
+ calls := inlineCalls(t, p.dir)
+
+ // The lane's own pool is taken (by the other members in flight): no wait, no call.
+ var own []interface{ Release() }
+ for {
+ slot, ok := p.s.conflictResolver.TryAcquireInlineSlot()
+ if !ok {
+ break
+ }
+ own = append(own, slot)
+ }
+ require.NotEmpty(t, own)
+ done := make(chan landInlineResult, 1)
+ go func() { done <- p.resolve(p.fx.ctx, t, newLandPreResolution()) }()
+ select {
+ case res := <-done:
+ require.Equal(t, landInlineSlow, res)
+ case <-time.After(10 * time.Second):
+ t.Fatal("the lane waited for a slot")
+ }
+ require.Equal(t, calls, inlineCalls(t, p.dir), "no call without a slot")
+ for _, slot := range own {
+ slot.Release()
+ }
+ require.Equal(t, landInlineResolved, p.resolve(p.fx.ctx, t, newLandPreResolution()), "the slots were given back")
+}
+
+// TestUserLand_aLandNeedingAModelDoesNotHoldBackTheUnionLands: one land on a hot file needs a model (the
+// rules cannot resolve it, and the model takes as long as it takes: here, until the test says so), and
+// six others on the same file are resolved by a rule. Without a verification what the model resolves never
+// lands, so the land that needs it takes no ticket and the others are not made to wait for it: all six
+// land while the model is still working, and no ticket is held.
+func TestUserLand_aLandNeedingAModelDoesNotHoldBackTheUnionLands(t *testing.T) {
+ t.Parallel()
+
+ q := newQueueFixtureMode(t, 6, "model", func(c *config.GraftResolver) { c.InlineTimeout = "" })
+ model := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),
+ gittest.WithTreeEntries(
+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "model-1\n"},
+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))
+
+ modelDone := make(chan error, 1)
+ go func() {
+ _, err := q.land(model)
+ modelDone <- err
+ }()
+ // The model land is at the resolver, out of the lane: its request is the one that is not for the rules.
+ require.Eventually(t, func() bool { return unionCalls(t, q.dir)-inlineCalls(t, q.dir) >= 1 }, 30*time.Second, 10*time.Millisecond)
+
+ landed := make(chan []error, 1)
+ go func() { landed <- q.landAll(q.onA) }()
+ select {
+ case errs := <-landed:
+ for i, err := range errs {
+ require.NoError(t, err, "union land %d", i+1)
+ }
+ case <-time.After(60 * time.Second):
+ t.Fatal("the union lands waited for the land that needs a model")
+ }
+ require.Empty(t, landTickets.keysOf(q.scope), "the land that needs a model holds no ticket")
+ select {
+ case err := <-modelDone:
+ t.Fatalf("the model land ended before the model answered: %v", err)
+ default:
+ }
+ require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), 7)
+
+ // The model answers: its resolution is a proposal, as without a verification it always was.
+ require.NoError(t, os.WriteFile(filepath.Join(q.dir, "release"), nil, 0o644))
+ select {
+ case err := <-modelDone:
+ requireMergeConflict(t, err)
+ require.NotNil(t, landConflictDetail(err), "the package with the proposal")
+ case <-time.After(60 * time.Second):
+ t.Fatal("the model land did not end")
+ }
+ require.Empty(t, landTickets.keysOf(q.scope))
+}
+
+// TestUserLand_aTicketOnTheFileDoesNotStopTheRulesInTheLane: another land holds the ticket of the file,
+// and the lane resolves the rules of this one inline all the same: the ticket serves the out-of-lane
+// resolution, and a rule does not wait for it. The block in the lane comes after the inline attempt; it
+// hands back only what the lane could not resolve.
+func TestUserLand_aTicketOnTheFileDoesNotStopTheRulesInTheLane(t *testing.T) {
+ t.Parallel()
+
+ q := newQueueFixture(t, 2, inlineOn)
+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})
+ require.NoError(t, err)
+ defer holder.release()
+
+ done := make(chan error, 1)
+ go func() {
+ _, err := q.land(q.onA[0])
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(30 * time.Second):
+ t.Fatal("the land waited for the holder of the ticket")
+ }
+ require.Equal(t, 1, inlineCalls(t, q.dir), "resolved in the lane")
+ require.Equal(t, 0, landTickets.waiting(q.keyOfA), "it never queued")
+ require.True(t, landTickets.blocked([]string{q.keyOfA}, nil), "the holder is still the holder")
+}
+
+// TestUserLand_aSlowRuleWaitsForTheTicketHolder: when the lane could not resolve inline (the rule did not end in
+// time), the land goes out of the lane and, with another land holding the ticket of the file, waits for its
+// turn there as before.
+func TestUserLand_aSlowRuleWaitsForTheTicketHolder(t *testing.T) {
+ t.Parallel()
+
+ q := newQueueFixtureMode(t, 1, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })
+ holder, err := landTickets.acquire(q.ctx, []string{q.keyOfA})
+ require.NoError(t, err)
+ defer holder.release()
+
+ done := make(chan error, 1)
+ go func() {
+ _, err := q.land(q.onA[0])
+ done <- err
+ }()
+ require.Eventually(t, func() bool { return landTickets.waiting(q.keyOfA) == 1 }, 30*time.Second, 5*time.Millisecond, "the land never queued")
+ select {
+ case err := <-done:
+ t.Fatalf("the land went past the holder of its file: %v", err)
+ case <-time.After(300 * time.Millisecond):
+ }
+ holder.release()
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(60 * time.Second):
+ t.Fatal("the land did not land when the holder was done")
+ }
+}
+
+// TestUserLand_oneFailedInlineCallDoesNotStartAConvoy: one rule call hiccups (3 s, cut at 300 ms) in the middle
+// of a hot file's lands. The members the lane handed back are resolved out of the lane and come back, but no
+// file stays ticketed: the next wave is resolved in the lane, at once, without one request out of it.
+func TestUserLand_oneFailedInlineCallDoesNotStartAConvoy(t *testing.T) {
+ t.Parallel()
+
+ const lands = 8
+ q := newQueueFixtureMode(t, 2*lands, "slowonce", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })
+
+ for i, err := range q.landAll(q.onA[:lands]) {
+ require.NoError(t, err, "land %d of the first wave", i+1)
+ }
+ require.Empty(t, landTickets.keysOf(q.scope), "no ticket is kept after the hiccup")
+ outOfLane := unionCalls(t, q.dir) - inlineCalls(t, q.dir)
+ inline := inlineCalls(t, q.dir)
+
+ for i, err := range q.landAll(q.onA[lands:]) {
+ require.NoError(t, err, "land %d of the second wave", i+1)
+ }
+ require.Equal(t, outOfLane, unionCalls(t, q.dir)-inlineCalls(t, q.dir), "the second wave never left the lane")
+ require.Greater(t, inlineCalls(t, q.dir), inline)
+ require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), 2*lands+1)
+ require.Empty(t, landTickets.keysOf(q.scope))
+}
+
+// TestUserLand_theLaneRoundDoesNotCountTowardTheRounds: with resolve_rounds of 1 a land that only a model can
+// resolve still leaves the lane for the pre-resolution: it ends with the proposal, not with the file
+// NOT_ATTEMPTED and nothing proposed.
+func TestUserLand_theLaneRoundDoesNotCountTowardTheRounds(t *testing.T) {
+ t.Parallel()
+
+ q := newQueueFixtureMode(t, 1, "model", func(c *config.GraftResolver) {
+ c.InlineTimeout = ""
+ c.ResolveRounds = 1
+ })
+ require.NoError(t, os.WriteFile(filepath.Join(q.dir, "release"), nil, 0o644))
+ model := gittest.WriteCommit(t, q.cfg, q.repoPath, gittest.WithParents(q.base),
+ gittest.WithTreeEntries(
+ gittest.TreeEntry{Mode: "100644", Path: "a", Content: "model-1\n"},
+ gittest.TreeEntry{Mode: "100644", Path: "b", Content: "base\n"}))
+
+ _, err := q.land(model)
+ requireMergeConflict(t, err)
+ pkg := landConflictDetail(err)
+ require.NotNil(t, pkg)
+ require.Equal(t, gitalypb.LandConflictFile_OUTCOME_PROPOSED, packageFile(t, pkg, "a").GetOutcome(), "the model was asked")
+}
+
+// TestUserLand_aLaneRoundThatHandsBackTakesNoTicket: the rule does not end in time, so the lane hands the land
+// back; it is resolved out of the lane and comes back, and at no round does it hold a ticket (a ticket would
+// make the lands on its file wait for it, for a hiccup of the lane).
+func TestUserLand_aLaneRoundThatHandsBackTakesNoTicket(t *testing.T) {
+ t.Parallel()
+
+ var s *Server
+ operationServerHooks.Store(t.Name(), func(served *Server) { s = served })
+ q := newQueueFixtureMode(t, 1, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })
+ require.NotNil(t, s)
+
+ var mu sync.Mutex
+ var held [][]string
+ s.landLaneHook = func() {
+ mu.Lock()
+ defer mu.Unlock()
+ held = append(held, landTickets.keysOf(q.scope))
+ }
+ _, err := q.land(q.onA[0])
+ require.NoError(t, err)
+
+ mu.Lock()
+ defer mu.Unlock()
+ require.GreaterOrEqual(t, len(held), 2, "the land went through the lane, out of it, and back")
+ for round, keys := range held {
+ require.Empty(t, keys, "ticket held at the lane round %d", round+1)
+ }
+ require.Positive(t, inlineCalls(t, q.dir))
+}
diff --git a/internal/gitaly/service/operations/land_inline_test.go b/internal/gitaly/service/operations/land_inline_test.go
index d763eb0..711e342 100644
--- a/internal/gitaly/service/operations/land_inline_test.go
+++ b/internal/gitaly/service/operations/land_inline_test.go
@@ -5,7 +5,6 @@ import (
"strings"
"sync"
"testing"
- "time"
"github.com/stretchr/testify/require"
@@ -64,30 +63,33 @@ func TestUserLand_theRulesAreResolvedInTheLane(t *testing.T) {
require.Len(t, strings.Split(on.branchBlob(t, "a"), "\n"), lands+1, "every land is in the file")
require.Positive(t, inlineCalls(t, on.dir), "the lane asked for the rules only")
- off := newQueueFixture(t, lands, tune(false))
+ // The control does not depend on who is faster: the resolver answers the two lands only once both have
+ // asked, so both resolved against the same tip, and the one that lands second has a stale verdict and,
+ // with one round, no other.
+ off := newQueueFixtureMode(t, 2, "barrier", tune(false))
failed := 0
for _, err := range off.landAll(off.onA) {
if err != nil {
failed++
}
}
- require.Positive(t, failed, "the control: without the inline resolution the cap of 1 round ends lands")
+ require.Equal(t, 1, failed, "the control: without the inline resolution the cap of 1 round ends the land that lost it")
require.Zero(t, inlineCalls(t, off.dir))
}
// TestUserLand_aRuleThatDoesNotEndInTimeIsHandedBack: the rule takes 3 s and the lane gives it 300 ms: the
-// member goes out of the lane, and is resolved there (the resolver answers the ordinary request at once).
+// members go out of the lane, and are resolved there (the resolver answers the ordinary request at once).
+// How long the lane stalls is measured by TestLandResolveInline_aBatchLosesAtMostItsBudget.
func TestUserLand_aRuleThatDoesNotEndInTimeIsHandedBack(t *testing.T) {
t.Parallel()
q := newQueueFixtureMode(t, 3, "slow", func(c *config.GraftResolver) { c.InlineTimeout = "300ms" })
- start := time.Now()
for i, err := range q.landAll(q.onA) {
require.NoError(t, err, "land %d", i+1)
}
- require.Less(t, time.Since(start), 60*time.Second)
require.Positive(t, inlineCalls(t, q.dir), "the lane tried")
require.Len(t, strings.Split(q.branchBlob(t, "a"), "\n"), 4)
+ require.Empty(t, landTickets.keysOf(q.scope), "no ticket is kept")
}
// TestUserLand_anOlderResolverIsNeverAskedAgain: a resolver that does not say that it honoured the request
@@ -173,12 +175,12 @@ func TestLandResolveInline_neverForAnAgentThePolicyDenies(t *testing.T) {
require.NoError(t, err)
require.NoError(t, s.landClassify(ctx, fx.repo, l.files, l.base, fx.tip, fx.commit, newLandClassifyCache(l.files, fx.tip)))
pre := newLandPreResolution()
- require.False(t, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre), "denied by the policy")
+ require.Equal(t, landInlineNotTried, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre), "denied by the policy")
require.Zero(t, unionCalls(t, dir), "nothing was resolved")
// The same agent where the policy lets it update the branch.
s.graft.Policy = []config.GraftPolicyRule{{Actor: "agent-7", Refs: []string{"refs/heads/**"}, Allow: true}}
- require.True(t, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre))
+ require.Equal(t, landInlineResolved, s.landResolveInline(ctx, fx.request("Land"), fx.tip, fx.commit, l, pre))
require.Equal(t, 1, unionCalls(t, dir))
}
diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.go
index 6e5417b..01fc73a 100644
--- a/internal/gitaly/service/operations/land_preresolve.go
+++ b/internal/gitaly/service/operations/land_preresolve.go
@@ -148,6 +148,10 @@ type landNeedsResolution struct {
// queued says that another land holds the ticket of one of the files (paths): this one waits for
// its turn before it resolves again.
queued bool
+ // rulesCannot says that the lane asked the rules to resolve the files and they cannot (or a file may
+ // not be sent): without a verification what a model resolves never lands, so the land is resolved out
+ // of the lane for the proposals and ends there, without a ticket.
+ rulesCannot bool
}
func (e *landNeedsResolution) Error() string {
@@ -280,11 +284,22 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
// without conflicts needs no pre-resolution at all, and one with conflicts that a rule resolves is
// resolved there, in its batch. What the rules cannot resolve is handed back, and resolved out of the
// lane in the next round.
- laneFirst := s.landInlineEnabled()
- for round := 1; round <= rounds; round++ {
- used = round
+ //
+ // A round that enters the lane without a pre-resolution (a lane round) is not one of resolve_rounds: it
+ // resolves nothing out of the lane, so with resolve_rounds = 1 the land that needs a model still
+ // leaves the lane. A land that waited for its turn on a file goes back to the lane too, where the
+ // rules are resolved inline whoever holds the ticket. A lane round is bounded all the same.
+ lane := s.landInlineEnabled()
+ laneRoundsLeft := landMaxLaneRounds
+ for round := 1; round <= rounds; {
+ used++
+ laneRound := lane && laneRoundsLeft > 0
+ lane = false
var preErr error
- if !(round == 1 && laneFirst) {
+ if laneRound {
+ laneRoundsLeft--
+ } else {
+ round++
preErr = s.landPreResolve(resolveCtx, in, committer, pre)
}
if err := preErr; err != nil {
@@ -321,7 +336,13 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
}
// The land lost the round, to a land that changed its files (or to the tree that moved, with a
// verification): it takes its turn on them. From now on it is the only land of the branch that
- // resolves these files again, and the others wait without calling the resolver.
+ // resolves these files again, and the others wait without calling the resolver. A lane round that
+ // merely did not find the rules enough takes no turn: the lane has not lost anything, and without a
+ // verification what the model resolves next never lands, so a ticket would protect nothing. Only a
+ // lane round that was told to wait for its turn does.
+ if laneRound && !need.queued || need.rulesCannot {
+ continue
+ }
if err := s.landTakeTicket(resolveCtx, in, pre, need); err != nil {
if deadline() {
pre.ticket.release()
@@ -335,11 +356,17 @@ func (s *Server) landResolving(ctx context.Context, in *gitalypb.UserLandRequest
}
return nil, err
}
+ // Back to the lane at once with the ticket: the rules are resolved there, no pre-resolution first.
+ lane = laneRound && need.queued
}
pre.ticket.release()
return s.landFinalPass(ctx, in, committer, pre, landStoppedReason(need, timeout))
}
+// landMaxLaneRounds bounds the rounds that enter the lane without a pre-resolution, which do not count
+// toward resolve_rounds: the first, and the ones after a wait for a turn.
+const landMaxLaneRounds = 4
+
// currentPaths are the paths of the conflicting files the land has at the latest tip it looked at.
func (p *landPreResolution) currentPaths() []string {
paths := make([]string, 0, len(p.current))
diff --git a/internal/gitaly/service/operations/land_queue_test.go b/internal/gitaly/service/operations/land_queue_test.go
index bddcb16..bf1110f 100644
--- a/internal/gitaly/service/operations/land_queue_test.go
+++ b/internal/gitaly/service/operations/land_queue_test.go
@@ -30,9 +30,16 @@ func unionResolver(t *testing.T) (string, []string) {
return unionResolverMode(t, "")
}
-// unionResolverMode is unionResolver with a mode: "slow" takes 3 s to answer a request for the rules only
-// (a rule that does not end in time), "noecho" does not say that it honoured the request (an older
-// graft-resolve).
+// unionResolverMode is unionResolver with a mode:
+// - "slow": takes 3 s to answer a request for the rules only (a rule that does not end in time);
+// - "slowonce": only the first request for the rules only takes 3 s (one hiccup);
+// - "noecho": does not say that it honoured the request (an older graft-resolve);
+// - "echo-model", "echo-noorigin", "echo-hold": echoes the request for the rules only, but answers with a
+// model's file, one without an origin, or one with a hold;
+// - "refuse": refuses every request for the rules only (the rules cannot resolve it);
+// - "model": refuses the requests for the rules only on a file whose theirs side says "model", and
+// answers the others with a model's resolution once dir/release exists (a model that takes a while);
+// - "barrier": answers a request that is not for the rules only once two requests have come.
func unionResolverMode(t *testing.T, mode string) (string, []string) {
t.Helper()
@@ -41,23 +48,58 @@ func unionResolverMode(t *testing.T, mode string) (string, []string) {
DIR='%s'
echo x >> "$DIR/calls.log"
cat > "$DIR/req-$$.json"
-python3 - "$DIR/req-$$.json" "$DIR" '%s' <<'PY'
-import base64, json, sys, time
+exec python3 - "$DIR/req-$$.json" "$DIR" '%s' <<'PY'
+import base64, json, os, sys, time
req = json.load(open(sys.argv[1]))
+dir, mode = sys.argv[2], sys.argv[3]
rules = bool(req.get("rules_only"))
if rules:
- open(sys.argv[2] + "/inline.log", "a").write("x\n")
- if sys.argv[3] == "slow":
+ open(dir + "/inline.log", "a").write("x\n")
+ if mode == "slow":
time.sleep(3)
+ if mode == "slowonce":
+ try:
+ os.close(os.open(dir + "/slowed", os.O_CREAT | os.O_EXCL))
+ time.sleep(3)
+ except FileExistsError:
+ pass
+ if mode == "refuse":
+ sys.stderr.write("the rules cannot resolve it")
+ sys.exit(1)
+elif mode == "barrier":
+ for _ in range(200):
+ if len(open(dir + "/calls.log").read().split()) >= 2:
+ break
+ time.sleep(0.05)
files = []
for f in req["files"]:
ours = base64.b64decode(f["ours"]["content"]).decode().splitlines()
theirs = base64.b64decode(f["theirs"]["content"]).decode().splitlines()
- merged = ours + [l for l in theirs if l not in ours]
- files.append({"path": f["path"], "action": "write", "mode": "100644", "origin": "rule",
- "content": base64.b64encode(("\n".join(merged) + "\n").encode()).decode()})
+ model = any("model" in l for l in theirs)
+ if mode == "model" and model:
+ if rules:
+ sys.stderr.write("needs a model")
+ sys.exit(1)
+ for _ in range(400):
+ if os.path.exists(dir + "/release"):
+ break
+ time.sleep(0.05)
+ merged = theirs
+ origin = "model"
+ else:
+ merged = ours + [l for l in theirs if l not in ours]
+ origin = "rule"
+ entry = {"path": f["path"], "action": "write", "mode": "100644", "origin": origin,
+ "content": base64.b64encode(("\n".join(merged) + "\n").encode()).decode()}
+ if mode == "echo-model":
+ entry["origin"] = "model"
+ if mode == "echo-noorigin":
+ del entry["origin"]
+ if mode == "echo-hold":
+ entry["hold"] = "held by the second model"
+ files.append(entry)
resp = {"version": 1, "resolver": "graft-resolve/union", "files": files}
-if sys.argv[3] != "noecho":
+if mode != "noecho":
resp["rules_only"] = rules
json.dump(resp, sys.stdout)
PY
diff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.go
index ca9fe17..50e3ab3 100644
--- a/internal/gitaly/service/operations/land_resolve.go
+++ b/internal/gitaly/service/operations/land_resolve.go
@@ -498,17 +498,6 @@ func (s *Server) landApplyVerdicts(
return landResolution{}, err
}
pre.setCurrent(l.files)
- // A land that lost a round holds the ticket of its files and is the one whose verdict lands: any
- // other land on the same files waits for its turn (land_tickets.go), instead of landing first and
- // invalidating the holder's verdict once more.
- paths := make([]string, 0, len(l.files))
- for _, f := range l.files {
- paths = append(paths, f.conflict.path)
- }
- scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
- if landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket) {
- return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: paths, queued: true}
- }
for _, f := range l.files {
if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {
missing = append(missing, f.conflict.path)
@@ -526,16 +515,39 @@ func (s *Server) landApplyVerdicts(
}
// The rules are resolved here, in the lane, within a bound of time (land_inline.go): the member rides
// the batch, as it did before the resolution left the lane. What a rule cannot resolve goes out.
- if len(missing) > 0 && pre != nil && s.conflictResolver != nil && l.bases <= 1 && s.landResolveInline(ctx, in, tip, commitID, l, pre) {
- missing = missing[:0]
- for _, f := range l.files {
- if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {
- missing = append(missing, f.conflict.path)
+ inline := landInlineNotTried
+ if len(missing) > 0 && pre != nil && s.conflictResolver != nil && l.bases <= 1 {
+ inline = s.landResolveInline(ctx, in, tip, commitID, l, pre)
+ if inline == landInlineResolved {
+ missing = missing[:0]
+ for _, f := range l.files {
+ if _, ok := pre.verdicts[landKeyOf(f.conflict)]; !ok || !f.sent() {
+ missing = append(missing, f.conflict.path)
+ }
}
}
}
+ // What the rules cannot resolve, without a verification, is resolved out of the lane for the proposals
+ // and ends there: what a model resolves never lands, so no ticket protects anything.
+ rulesCannot := inline == landInlineRulesCannot && !s.resolveCfg.Verify.Enabled
+ // A land that lost a round holds the ticket of its files and is the one whose verdict lands: any other
+ // land on the same files, whose verdicts were made out of the lane, waits for its turn
+ // (land_tickets.go), instead of landing first and invalidating the holder's verdict once more. A land
+ // that the lane has just resolved inline needs no turn: the rules do not wait for the tickets.
+ // Without a verification, a land whose files have a model's verdict does not land either: it is not
+ // waited for.
+ if pre != nil && s.conflictResolver != nil && l.bases <= 1 && inline != landInlineResolved && !rulesCannot && !landHasModelVerdict(s, l, pre) {
+ paths := make([]string, 0, len(l.files))
+ for _, f := range l.files {
+ paths = append(paths, f.conflict.path)
+ }
+ scope := landTicketScope(in.GetRepository().GetStorageName(), in.GetRepository().GetRelativePath(), in.GetBranch())
+ if landTickets.blocked(landTicketKeys(scope, paths, s.resolveCfg.Verify.Enabled), pre.ticket) {
+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: paths, queued: true}
+ }
+ }
if len(missing) > 0 {
- return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: missing}
+ return landResolution{}, &landNeedsResolution{tip: tip, commit: commitID, paths: missing, rulesCannot: rulesCannot}
}
reused := map[string]resolver.ResponseFile{}
diff --git a/internal/gitaly/service/operations/land_tickets.go b/internal/gitaly/service/operations/land_tickets.go
index 373f846..10c5aa7 100644
--- a/internal/gitaly/service/operations/land_tickets.go
+++ b/internal/gitaly/service/operations/land_tickets.go
@@ -190,13 +190,16 @@ func (t *landTicket) release() {
}
}
-// covers reports whether t holds a ticket for every one of keys.
+// covers reports whether t holds a ticket for every one of keys: a released ticket holds none.
func (t *landTicket) covers(keys []string) bool {
if t == nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
+ if t.released {
+ return false
+ }
held := make(map[string]bool, len(t.acquired))
for _, k := range t.acquired {
held[k] = true
diff --git a/internal/gitaly/service/operations/land_tickets_test.go b/internal/gitaly/service/operations/land_tickets_test.go
index 81e063f..1a14ff0 100644
--- a/internal/gitaly/service/operations/land_tickets_test.go
+++ b/internal/gitaly/service/operations/land_tickets_test.go
@@ -146,5 +146,6 @@ func TestLandTickets_keys(t *testing.T) {
require.True(t, r.blocked(landTicketKeys("r", []string{"c", "b"}, false), nil))
tk.release()
tk.release() // a second release does nothing
+ require.False(t, tk.covers(landTicketKeys("r", []string{"a"}, false)), "a released ticket covers nothing")
require.False(t, (*landTicket)(nil).covers(nil))
}
diff --git a/internal/graft/resolver/check.go b/internal/graft/resolver/check.go
index f2ee760..2604c6a 100644
--- a/internal/graft/resolver/check.go
+++ b/internal/graft/resolver/check.go
@@ -93,6 +93,23 @@ var resolveSlotWait = promauto.NewHistogram(prometheus.HistogramOpts{
type ResolveSlot struct {
c *Client
released bool
+ // inline says that the slot is one of the pool of the lane's own calls.
+ inline bool
+}
+
+// TryAcquireInlineSlot takes a slot of the pool of the calls with which the lane resolves the rules, without
+// waiting: it reports false when all are taken, and the lane goes on without the call. Release it when the
+// request is done.
+func (c *Client) TryAcquireInlineSlot() (*ResolveSlot, bool) {
+ if c == nil {
+ return nil, false
+ }
+ select {
+ case c.inlineSlots <- struct{}{}:
+ return &ResolveSlot{c: c, inline: true}, true
+ default:
+ return nil, false
+ }
}
// AcquireResolveSlot waits for a resolve slot. The wait counts against the deadline of ctx: when
@@ -118,6 +135,10 @@ func (s *ResolveSlot) Release() {
return
}
s.released = true
+ if s.inline {
+ <-s.c.inlineSlots
+ return
+ }
<-s.c.resolveSlots
}
diff --git a/internal/graft/resolver/client.go b/internal/graft/resolver/client.go
index 34940bb..e3fc918 100644
--- a/internal/graft/resolver/client.go
+++ b/internal/graft/resolver/client.go
@@ -86,6 +86,11 @@ func (e *UnresolvableError) Is(target error) bool {
// Client calls the configured resolver. It is safe for concurrent use. A nil
// *Client is the disabled resolver.
+// minInlineSlots is the least size of the pool of the lane's own calls: they are rules, which take
+// milliseconds, and each lane (one per branch) needs one at a time, so that the pool of one server is
+// not the bottleneck of its branches.
+const minInlineSlots = 8
+
type Client struct {
command []string
url string
@@ -102,6 +107,9 @@ type Client struct {
// the short checks (PLAN10 15.8).
checkSlots chan struct{}
resolveSlots chan struct{}
+ // inlineSlots bounds the calls with which the lane resolves the rules itself (TryAcquireInlineSlot): a pool
+ // of its own, which is never waited for, so that a model that holds the resolve slots does not stop the rules.
+ inlineSlots chan struct{}
// sendRules decide the files that are never sent (SendRules).
sendRules *SendRules
}
@@ -159,6 +167,7 @@ func New(cfg config.GraftResolver) (*Client, error) {
sendRules: rules,
checkSlots: make(chan struct{}, cfg.CheckConcurrencyOrDefault()),
resolveSlots: make(chan struct{}, cfg.ResolveConcurrencyOrDefault()),
+ inlineSlots: make(chan struct{}, max(minInlineSlots, cfg.ResolveConcurrencyOrDefault())),
}
if cfg.URL != "" {