Git designed for AI agents.
graft: e2e counts the verify runs of the agreement gate; nits
Federico Elia0116ff79f3cc
diff --git a/internal/gitaly/service/operations/land_preresolve.go b/internal/gitaly/service/operations/land_preresolve.goindex 4330286..3153ab5 100644--- a/internal/gitaly/service/operations/land_preresolve.go+++ b/internal/gitaly/service/operations/land_preresolve.go@@ -371,8 +371,10 @@ func (s *Server) landFinalPass(func landFinalPassGaveUp(in *gitalypb.UserLandRequest, needs *landNeedsResolution, passes int) error {return landKeptMovingError(in, landConflict{reference: git.NewReferenceNameFromBranchName(string(in.GetBranch())),- oldOID: needs.tip,- newOID: needs.tip,- cause: errors.New("the branch kept moving while the land was being ended"),+ // The tip the last pass resolved at is the one the branch moved away from; where it moved to is+ // not known here (the lane handed the land back without a compare-and-swap), so newOID is empty.+ oldOID: needs.tip,+ newOID: "",+ cause: errors.New("the branch kept moving while the land was being ended"),}, passes)}diff --git a/internal/graft/e2e/proposals_agree_test.go b/internal/graft/e2e/proposals_agree_test.goindex 289be94..cabd679 100644--- a/internal/graft/e2e/proposals_agree_test.go+++ b/internal/graft/e2e/proposals_agree_test.go@@ -26,8 +26,9 @@ sys.stdout.write("===== BEGIN PATCHED CODE %s =====\n%s===== END PATCHED CODE %s// agreeResolverConfig writes a resolve.toml with the llm strategy of fakeModel and an agree_with model// (second, the source of a model), and returns the graft configuration that uses the real-// graft-resolve with it and a verification that always passes.-func agreeResolverConfig(t *testing.T, second string) exampleConfig {+// graft-resolve with it and a verification that always passes, and the file in which each run of that+// verification leaves a line.+func agreeResolverConfig(t *testing.T, second string) (exampleConfig, string) {t.Helper()dir := t.TempDir()@@ -52,13 +53,28 @@ command = ["python3", "-I", "`+other+`"]model = "fake-b"`), 0o600))+ runs := filepath.Join(dir, "verify-runs")+ verify := filepath.Join(dir, "verify.sh")+ require.NoError(t, os.WriteFile(verify, []byte("#!/bin/sh\necho x >> '"+runs+"'\nexit 0\n"), 0o755))+ex := loadExample(t)ex.graft.Land.Resolver = config.GraftResolver{Enabled: true,Command: []string{buildAgentsTool(t, "./cmd/graft-resolve"), "--config", resolveConfig},- Verify: config.GraftResolverVerify{Enabled: true, Command: []string{"/bin/true"}},+ Verify: config.GraftResolverVerify{Enabled: true, Command: []string{verify}},+ }+ return ex, runs+}++// verifyRunsOf counts the lines that the runs of the verification left in runs.+func verifyRunsOf(t *testing.T, runs string) int {+ t.Helper()+ raw, err := os.ReadFile(runs)+ if os.IsNotExist(err) {+ return 0}- return ex+ require.NoError(t, err)+ return len(raw) / 2}// TestAgreeWithTheModelsThatAgreeLandThroughVerify and ...ThatDisagreeAreHeld are the agreement gate@@ -69,7 +85,7 @@ model = "fake-b"func TestAgreeWithTheModelsThatAgreeLandThroughVerify(t *testing.T) {t.Parallel()- ex := agreeResolverConfig(t, fakeModel)+ ex, runs := agreeResolverConfig(t, fakeModel)s := startGitaly(t, ex.graft, ex.sharedToken)repo := s.repoAs(t, exampleTokens["planner"])ctx := s.ctx@@ -88,13 +104,14 @@ func TestAgreeWithTheModelsThatAgreeLandThroughVerify(t *testing.T) {require.NoError(t, err, "two models that agree, and a verification that passes: the resolution lands")msg := s.git(t, "log", "-1", "--format=%B", res.LandedCommitID)require.Contains(t, msg, "Graft-Resolved-By: graft-resolve/model")- require.Contains(t, msg, "Graft-Checked-By: verify:true")+ require.Contains(t, msg, "Graft-Checked-By: verify:verify.sh")+ require.Equal(t, 1, verifyRunsOf(t, runs), "the tree that lands was verified once")}func TestAgreeWithTheModelsThatDisagreeAreHeld(t *testing.T) {t.Parallel()- ex := agreeResolverConfig(t, otherModel)+ ex, runs := agreeResolverConfig(t, otherModel)s := startGitaly(t, ex.graft, ex.sharedToken)repo := s.repoAs(t, exampleTokens["planner"])ctx := s.ctx@@ -123,6 +140,7 @@ func TestAgreeWithTheModelsThatDisagreeAreHeld(t *testing.T) {require.Contains(t, f.GetReason(), "fake-b")require.NotNil(t, f.GetProposal(), "the first model's proposal is kept for the agent")require.Contains(t, f.GetProposal().GetExplanation(), "held: the second model (fake-b) disagreed")+ require.Equal(t, 0, verifyRunsOf(t, runs), "a held resolution is not even verified: verify.command never ran")}// TestStorageProposalsInAuthoritativeMode: the references of the proposals are written and pruneddiff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.goindex fb82d76..b9c5248 100644--- a/tools/graft-agents/cmd/graft-resolve/bench.go+++ b/tools/graft-agents/cmd/graft-resolve/bench.go@@ -186,9 +186,9 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intfmt.Fprintln(stderr, "graft-resolve bench run: a chain with an agree_with model has two prices: use --pricing openrouter")return 2}- cap := *maxUSD+ perModel := *maxUSDif len(agreeModels) > 0 {- cap /= 2+ perModel /= 2}var prices, agreePrices resolve.Pricesswitch *pricingSource {@@ -208,12 +208,12 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intfmt.Fprintf(stderr, "graft-resolve bench run: unknown --pricing %q, want openrouter or fixed\n", *pricingSource)return 2}- if budget, err = resolve.NewBudget(cap, prices, ""); err != nil {+ if budget, err = resolve.NewBudget(perModel, prices, ""); err != nil {fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)return 1}if len(agreeModels) > 0 {- if agreeBudget, err = resolve.NewBudget(cap, agreePrices, ""); err != nil {+ if agreeBudget, err = resolve.NewBudget(perModel, agreePrices, ""); err != nil {fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)return 1}@@ -223,10 +223,10 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intreturn 1}fmt.Fprintf(stderr, "cap $%.2f; $%g per million input tokens, $%g per million output tokens, $%g per request\n",- cap, prices.InputPerMillion, prices.OutputPerMillion, prices.PerRequest)+ perModel, prices.InputPerMillion, prices.OutputPerMillion, prices.PerRequest)if agreeBudget != nil {fmt.Fprintf(stderr, "cap $%.2f for the agree_with model; $%g per million input tokens, $%g per million output tokens, $%g per request\n",- cap, agreePrices.InputPerMillion, agreePrices.OutputPerMillion, agreePrices.PerRequest)+ perModel, agreePrices.InputPerMillion, agreePrices.OutputPerMillion, agreePrices.PerRequest)}}