Skip to content

Git designed for AI agents.

graft: e2e counts the verify runs of the agreement gate; nits

Federico Elia0116ff79f3cc
diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.go
index 4330286..3153ab5 100644
--- a/internal/gitaly/service/operations/land_preresolve.go
+++ b/internal/gitaly/service/operations/land_preresolve.go
@@ -371,8 +371,10 @@ func (s *Server) landFinalPass(
func landFinalPassGaveUp(in *gitalypb.UserLandRequest, needs *landNeedsResolution, passes int) error {
return landKeptMovingError(in, landConflict{
reference: git.NewReferenceNameFromBranchName(string(in.GetBranch())),
- oldOID: needs.tip,
- newOID: needs.tip,
- cause: errors.New("the branch kept moving while the land was being ended"),
+ // The tip the last pass resolved at is the one the branch moved away from; where it moved to is
+ // not known here (the lane handed the land back without a compare-and-swap), so newOID is empty.
+ oldOID: needs.tip,
+ newOID: "",
+ cause: errors.New("the branch kept moving while the land was being ended"),
}, passes)
}
diff --git a/internal/graft/e2e/proposals_agree_test.go b/internal/graft/e2e/proposals_agree_test.go
index 289be94..cabd679 100644
--- a/internal/graft/e2e/proposals_agree_test.go
+++ b/internal/graft/e2e/proposals_agree_test.go
@@ -26,8 +26,9 @@ sys.stdout.write("===== BEGIN PATCHED CODE %s =====\n%s===== END PATCHED CODE %s
// agreeResolverConfig writes a resolve.toml with the llm strategy of fakeModel and an agree_with model
// (second, the source of a model), and returns the graft configuration that uses the real
-// graft-resolve with it and a verification that always passes.
-func agreeResolverConfig(t *testing.T, second string) exampleConfig {
+// graft-resolve with it and a verification that always passes, and the file in which each run of that
+// verification leaves a line.
+func agreeResolverConfig(t *testing.T, second string) (exampleConfig, string) {
t.Helper()
dir := t.TempDir()
@@ -52,13 +53,28 @@ command = ["python3", "-I", "`+other+`"]
model = "fake-b"
`), 0o600))
+ runs := filepath.Join(dir, "verify-runs")
+ verify := filepath.Join(dir, "verify.sh")
+ require.NoError(t, os.WriteFile(verify, []byte("#!/bin/sh\necho x >> '"+runs+"'\nexit 0\n"), 0o755))
+
ex := loadExample(t)
ex.graft.Land.Resolver = config.GraftResolver{
Enabled: true,
Command: []string{buildAgentsTool(t, "./cmd/graft-resolve"), "--config", resolveConfig},
- Verify: config.GraftResolverVerify{Enabled: true, Command: []string{"/bin/true"}},
+ Verify: config.GraftResolverVerify{Enabled: true, Command: []string{verify}},
+ }
+ return ex, runs
+}
+
+// verifyRunsOf counts the lines that the runs of the verification left in runs.
+func verifyRunsOf(t *testing.T, runs string) int {
+ t.Helper()
+ raw, err := os.ReadFile(runs)
+ if os.IsNotExist(err) {
+ return 0
}
- return ex
+ require.NoError(t, err)
+ return len(raw) / 2
}
// TestAgreeWithTheModelsThatAgreeLandThroughVerify and ...ThatDisagreeAreHeld are the agreement gate
@@ -69,7 +85,7 @@ model = "fake-b"
func TestAgreeWithTheModelsThatAgreeLandThroughVerify(t *testing.T) {
t.Parallel()
- ex := agreeResolverConfig(t, fakeModel)
+ ex, runs := agreeResolverConfig(t, fakeModel)
s := startGitaly(t, ex.graft, ex.sharedToken)
repo := s.repoAs(t, exampleTokens["planner"])
ctx := s.ctx
@@ -88,13 +104,14 @@ func TestAgreeWithTheModelsThatAgreeLandThroughVerify(t *testing.T) {
require.NoError(t, err, "two models that agree, and a verification that passes: the resolution lands")
msg := s.git(t, "log", "-1", "--format=%B", res.LandedCommitID)
require.Contains(t, msg, "Graft-Resolved-By: graft-resolve/model")
- require.Contains(t, msg, "Graft-Checked-By: verify:true")
+ require.Contains(t, msg, "Graft-Checked-By: verify:verify.sh")
+ require.Equal(t, 1, verifyRunsOf(t, runs), "the tree that lands was verified once")
}
func TestAgreeWithTheModelsThatDisagreeAreHeld(t *testing.T) {
t.Parallel()
- ex := agreeResolverConfig(t, otherModel)
+ ex, runs := agreeResolverConfig(t, otherModel)
s := startGitaly(t, ex.graft, ex.sharedToken)
repo := s.repoAs(t, exampleTokens["planner"])
ctx := s.ctx
@@ -123,6 +140,7 @@ func TestAgreeWithTheModelsThatDisagreeAreHeld(t *testing.T) {
require.Contains(t, f.GetReason(), "fake-b")
require.NotNil(t, f.GetProposal(), "the first model's proposal is kept for the agent")
require.Contains(t, f.GetProposal().GetExplanation(), "held: the second model (fake-b) disagreed")
+ require.Equal(t, 0, verifyRunsOf(t, runs), "a held resolution is not even verified: verify.command never ran")
}
// TestStorageProposalsInAuthoritativeMode: the references of the proposals are written and pruned
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.go
index fb82d76..b9c5248 100644
--- a/tools/graft-agents/cmd/graft-resolve/bench.go
+++ b/tools/graft-agents/cmd/graft-resolve/bench.go
@@ -186,9 +186,9 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
fmt.Fprintln(stderr, "graft-resolve bench run: a chain with an agree_with model has two prices: use --pricing openrouter")
return 2
}
- cap := *maxUSD
+ perModel := *maxUSD
if len(agreeModels) > 0 {
- cap /= 2
+ perModel /= 2
}
var prices, agreePrices resolve.Prices
switch *pricingSource {
@@ -208,12 +208,12 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
fmt.Fprintf(stderr, "graft-resolve bench run: unknown --pricing %q, want openrouter or fixed\n", *pricingSource)
return 2
}
- if budget, err = resolve.NewBudget(cap, prices, ""); err != nil {
+ if budget, err = resolve.NewBudget(perModel, prices, ""); err != nil {
fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
return 1
}
if len(agreeModels) > 0 {
- if agreeBudget, err = resolve.NewBudget(cap, agreePrices, ""); err != nil {
+ if agreeBudget, err = resolve.NewBudget(perModel, agreePrices, ""); err != nil {
fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
return 1
}
@@ -223,10 +223,10 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
return 1
}
fmt.Fprintf(stderr, "cap $%.2f; $%g per million input tokens, $%g per million output tokens, $%g per request\n",
- cap, prices.InputPerMillion, prices.OutputPerMillion, prices.PerRequest)
+ perModel, prices.InputPerMillion, prices.OutputPerMillion, prices.PerRequest)
if agreeBudget != nil {
fmt.Fprintf(stderr, "cap $%.2f for the agree_with model; $%g per million input tokens, $%g per million output tokens, $%g per request\n",
- cap, agreePrices.InputPerMillion, agreePrices.OutputPerMillion, agreePrices.PerRequest)
+ perModel, agreePrices.InputPerMillion, agreePrices.OutputPerMillion, agreePrices.PerRequest)
}
}