Git designed for AI agents.
graft: the agreement gate decides "clean" from structured findings, and cancels the second model
Federico Eliab6efa99a745c
diff --git a/tools/graft-agents/resolve/llm_agree_test.go b/tools/graft-agents/resolve/llm_agree_test.goindex d2e119b..7ebf471 100644--- a/tools/graft-agents/resolve/llm_agree_test.go+++ b/tools/graft-agents/resolve/llm_agree_test.go@@ -7,7 +7,9 @@ import ("errors""regexp""strings"+ "sync/atomic""testing"+ "time""github.com/feder-cr/git-ai/tools/graft-agents/llm""github.com/feder-cr/git-ai/tools/graft-agents/protocol"@@ -133,15 +135,30 @@ func TestAgreeWithASecondModelThatFailsHolds(t *testing.T) {func TestAgreeWithAFirstModelThatFailsFailsTheFile(t *testing.T) {wp19SetAnswer(agreeFail("provider down"))- b := agreeSetB(agreeAnswer("merged\n"))+ // The second model is slow: it answers only when its context ends. The first model failing cancels+ // it, and the file is not kept waiting for it.+ var cancelled atomic.Bool+ b := agreeSetB(func(ctx context.Context, req llm.Request) (*llm.Response, error) {+ select {+ case <-ctx.Done():+ cancelled.Store(true)+ return nil, ctx.Err()+ case <-time.After(30 * time.Second):+ return wp19Answer("merged\n"), nil+ }+ })+ start := time.Now()_, err := resolveOne(t)var unresolved *UnresolvedErrorif !errors.As(err, &unresolved) {t.Fatalf("err = %v", err)}- // The second model was asked at the same time and its call is waited for: the cost is real.- if b.calls() != 1 {- t.Fatalf("calls of the second model = %d", b.calls())+ if time.Since(start) > 10*time.Second {+ t.Fatalf("the file waited %s for the second model", time.Since(start))+ }+ // The second model was asked at the same time, its call was cancelled, and it was waited for.+ if b.calls() != 1 || !cancelled.Load() {+ t.Fatalf("calls of the second model = %d, cancelled = %v", b.calls(), cancelled.Load())}}@@ -149,8 +166,8 @@ func TestAgreeWithAnUncleanReportHoldsEvenWhenTheModelsAgree(t *testing.T) {r := wp19Build(t, agreeConfig)s := r.bindings[0].Strategy.(*llmStrategy)clean := LLMResult{Content: []byte("x\n"), Report: []string{"keeps every uncontested change of both sides (2)"}}- dirty := LLMResult{Content: []byte("x\n"), Report: []string{"region 1: drops theirs' change: the line \"y\" it adds is missing"}}- repeats := LLMResult{Content: []byte("x\n"), Report: []string{"region 2: duplicates a line both sides added: \"z\""}}+ dirty := LLMResult{Content: []byte("x\n"), Report: []string{"region 1: drops theirs' change: the line \"y\" it adds is missing"}, Findings: 1}+ repeats := LLMResult{Content: []byte("x\n"), Report: []string{"region 2: duplicates a line both sides added: \"z\""}, Findings: 1}run := func(res LLMResult) *agreeRun { return &agreeRun{res: res} }if why := s.agreement(clean, run(clean)); why != "" {@@ -232,25 +249,42 @@ func TestAgreeWithConfig(t *testing.T) {}}-func TestLLMReportClean(t *testing.T) {- for report, want := range map[string]bool{- "": true,- "keeps every uncontested change of both sides (3)": true,- "2 contested changes resolved freely: nothing line-level to verify": true,- "1 of 2 regions were answered again after findings of the deterministic checks": true,- "region 1: drops ours' change 3-4: the line \"a\" it adds is missing": false,- "region 2: duplicates a line both sides added: \"b\"": false,- } {- var lines []string- if report != "" {- lines = []string{report}- }- if got := LLMReportClean(lines); got != want {- t.Errorf("LLMReportClean(%q) = %v", report, got)+// TestLLMFindingsSurviveTheCutOfTheReport: the report is cut at 16 lines, so a drop in a late region can+// be missing from its text, and the agree_with gate must not read the text. Findings counts them all.+func TestLLMFindingsSurviveTheCutOfTheReport(t *testing.T) {+ var comps []llmComposition+ for i := 0; i < 17; i++ {+ comps = append(comps, llmComposition{neither: true})+ }+ late := llmComposition{problems: []string{"drops THEIRS' change at base line 6: the line \"T6\" it adds is missing"}}+ comps = append(comps, late)++ report := llmReport(comps, 0)+ if len(report) != 16 {+ t.Fatalf("the report is not cut: %d lines", len(report))+ }+ for _, line := range report {+ if strings.Contains(line, "drops") {+ t.Fatalf("the late drop is in the first 16 lines, the test proves nothing: %v", report)}}- if !LLMReportClean([]string{"a", "b", "region 3: the answer has lines found in neither side"}) {- t.Error("a line found in neither side is a free resolution, not a drop")+ if got := llmFindings(comps); got != 1 {+ t.Fatalf("findings = %d, want the late drop", got)+ }++ // The gate holds a file whose report text is clean and whose findings are not.+ r := wp19Build(t, agreeConfig)+ s := r.bindings[0].Strategy.(*llmStrategy)+ text := LLMResult{Content: []byte("x\n"), Report: report, Findings: llmFindings(comps)}+ clean := LLMResult{Content: []byte("x\n"), Report: report}+ if why := s.agreement(text, &agreeRun{res: clean}); !strings.HasPrefix(why, "held: a composition report was not clean") {+ t.Fatalf("a drop past the cut is not held: %q", why)+ }+ if why := s.agreement(clean, &agreeRun{res: text}); !strings.HasPrefix(why, "held: a composition report was not clean") {+ t.Fatalf("a drop past the cut of the second model is not held: %q", why)+ }+ if why := s.agreement(clean, &agreeRun{res: clean}); why != "" {+ t.Fatalf("two answers without findings are held: %q", why)}}@@ -267,3 +301,38 @@ func TestLLMNormalize(t *testing.T) {}}}++// TestAgreeWithNeedsAServerThatHolds: with agree_with a file may carry a hold, which only a server with+// the capability "hold" never lands. A server that proposes but cannot hold gets no model's resolution+// (the models are not called), and one that holds does.+func TestAgreeWithNeedsAServerThatHolds(t *testing.T) {+ a := wp19SetAnswer(agreeAnswer("merged\n"))+ b := agreeSetB(agreeAnswer("merged\n"))+ r := wp19Build(t, agreeConfig)++ noHold := wp19Request(wp19Conflict("a.txt"))+ noHold.Capabilities = []string{protocol.CapabilityOrigin, protocol.CapabilityProposal, protocol.CapabilityVerify}+ _, err := r.Resolve(context.Background(), noHold)+ var unresolved *UnresolvedError+ if !errors.As(err, &unresolved) || !strings.Contains(err.Error(), "the server cannot hold a model's resolution (agree_with); upgrade the server") {+ t.Fatalf("err = %v", err)+ }+ if a.calls() != 0 || b.calls() != 0 {+ t.Fatalf("the models were called for a server that cannot hold: %d, %d", a.calls(), b.calls())+ }++ // Without agree_with the same server is enough.+ single := wp19Build(t, wp19LLMConfig)+ if _, err := single.Resolve(context.Background(), noHold); err != nil {+ t.Fatalf("a single model for a server that proposes: %v", err)+ }++ // A server of this release.+ if _, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt"))); err != nil {+ t.Fatal(err)+ }+ // The first model served the single-model run above as well.+ if a.calls() != 2 || b.calls() != 1 {+ t.Fatalf("calls %d, %d", a.calls(), b.calls())+ }+}diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.goindex e59f99a..fbf317e 100644--- a/tools/graft-agents/resolve/llm_core.go+++ b/tools/graft-agents/resolve/llm_core.go@@ -128,6 +128,10 @@ type LLMResult struct {// drops and repeats of the two changes, and how many contested regions it resolved freely. It is// a report and not a gate, and goes to the agent with the proposal.Report []string+ // Findings is the number of drops and duplicates that the composition check found in the regions+ // (each one is a line of the report, but the report is cut at 16 lines and Findings is not). The+ // agree_with gate decides "clean" from it, never from the text of the report.+ Findings int// Calls and Repairs are the model calls of the file and those of them that were repair rounds.Calls, Repairs int}@@ -279,19 +283,6 @@ func (s *LLMRegionStats) Absorb(o *LLMRegionStats, prefix string) {}}-// LLMReportClean reports whether a composition report holds no finding: no line says that the answer-// drops a change of a side or repeats a line both added (llmReport words them first). It is the-// "clean report" of the agree_with gate (PLAN10 15.12): a report is not a gate for a single model,-// and is one of the two conditions under which two models' agreement is trusted.-func LLMReportClean(report []string) bool {- for _, line := range report {- if strings.Contains(line, "drops") || strings.Contains(line, "duplicates") {- return false- }- }- return true-}-// LLMNormalize is a file as the agree_with gate and the judges compare it: line endings as LF, no// whitespace at the end of a line, no blank line at the end of the file. The bench normalizes// with it too, so that "the same answer" means the same thing everywhere.diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.goindex 8470873..712db9f 100644--- a/tools/graft-agents/resolve/llm_patch.go+++ b/tools/graft-agents/resolve/llm_patch.go@@ -429,6 +429,16 @@ func llmRegionAtLine(spans map[int][2]int, line int) (int, bool) {return best, found}+// llmFindings counts the drops and the duplicates of the regions: what the report says first, whatever+// of it the cut of the report leaves.+func llmFindings(comps []llmComposition) int {+ n := 0+ for _, c := range comps {+ n += len(c.problems)+ }+ return n+}+// llmReport words the composition checks of the regions (llmCompose) as the report that goes with// the proposal: what is wrong first, in the words a repair round quotes, then what it keeps.func llmReport(comps []llmComposition, repairs int) []string {@@ -678,6 +688,7 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptiHunks: len(conflicting),RegionsEqualOurs: equalOurs,Report: report,+ Findings: llmFindings(comps),Calls: calls,Repairs: repairs,}, nildiff --git a/tools/graft-agents/resolve/llm_strategy.go b/tools/graft-agents/resolve/llm_strategy.goindex bae360f..ef09670 100644--- a/tools/graft-agents/resolve/llm_strategy.go+++ b/tools/graft-agents/resolve/llm_strategy.go@@ -48,7 +48,7 @@ import (// With agree_with, both models resolve each file at the same time with the same prompt, and the// first model's resolution is proposed as always (it lands only through verify.command). It is// HELD, and never lands, unless the two results are the same file after normalization-// (LLMNormalize) and both composition reports are clean (LLMReportClean): the response carries+// (LLMNormalize) and neither composition check found a drop or a duplicate (LLMResult.Findings): the response carries// Hold, the report says that the second model disagreed, and the server neither verifies nor// lands the file. A model that fails makes the file unresolved when it is the first, and holds it// when it is the second. A budget on one model needs a budget on the other.@@ -226,8 +226,12 @@ func (s *llmStrategy) Resolve(ctx context.Context, f File, req protocol.Request)defer cancel()res, err := ResolveLLM(first, s.provider, lf, s.opt)if second != nil {- // The second model is waited for whatever the first one did: its calls are paid for, and- // its usage belongs to the file.+ // A first model that failed or declined leaves nothing for the second one to be compared with:+ // its context is cancelled. It is waited for whatever happened: the calls it made are paid for,+ // and its usage belongs to the file.+ if err != nil {+ second.cancel()+ }second.wait()if stats != nil {stats.Absorb(second.stats, "agree_with: ")@@ -254,22 +258,25 @@ func (s *llmStrategy) Resolve(ctx context.Context, f File, req protocol.Request)// agreeRun is the second model resolving a file, started by agree.start.type agreeRun struct {- done chan struct{}- res LLMResult- err error- stats *LLMRegionStats+ done chan struct{}+ cancel context.CancelFunc+ res LLMResult+ err error+ stats *LLMRegionStats}// start resolves lf with the side's model in the background. When counting, the run keeps its own// stats, which the caller absorbs into its own once the run is waited for.func (s *llmSide) start(ctx context.Context, lf LLMFile, counting bool) *agreeRun {- r := &agreeRun{done: make(chan struct{})}+ ctx, cancel := context.WithCancel(ctx)+ r := &agreeRun{done: make(chan struct{}), cancel: cancel}if counting {r.stats = &LLMRegionStats{}ctx = WithLLMRegionStats(ctx, r.stats)}go func() {defer close(r.done)+ defer cancel()defer func() {if p := recover(); p != nil {r.err = fmt.Errorf("resolve: the second model panicked: %v", p)@@ -287,16 +294,17 @@ func (r *agreeRun) wait() { <-r.done }// agreement says why the first model's resolution is not the one both models give, or "" when the// two agree: the same file after the judge's normalization (LLMNormalize), and neither report-// holds a finding (LLMReportClean). The first model's answer is the one that is proposed.+// found a drop or a duplicate (LLMResult.Findings, never the text of the report, which is cut). The+// first model's answer is the one that is proposed, byte for byte.func (s *llmStrategy) agreement(first LLMResult, second *agreeRun) string {switch {case second.err != nil && errors.Is(second.err, ErrLLMNotApplicable):return fmt.Sprintf("held: the second model (%s) could not take the file", s.agree.model)case second.err != nil:return fmt.Sprintf("held: the second model (%s) gave no usable answer: %s", s.agree.model, llmClip(second.err.Error(), 200))- case !LLMReportClean(first.Report):+ case first.Findings > 0:return fmt.Sprintf("held: a composition report was not clean: the first model (%s) drops or repeats a change", s.model)- case !LLMReportClean(second.res.Report):+ case second.res.Findings > 0:return fmt.Sprintf("held: a composition report was not clean: the second model (%s) drops or repeats a change", s.agree.model)case !bytes.Equal(LLMNormalize(first.Content), LLMNormalize(second.res.Content)):return fmt.Sprintf("held: the second model (%s) disagreed: it resolved the file differently from the first (%s)", s.agree.model, s.model)@@ -304,6 +312,9 @@ func (s *llmStrategy) agreement(first LLMResult, second *agreeRun) string {return ""}+// holds reports whether the strategy's answers may carry a hold (agree_with is set).+func (s *llmStrategy) holds() bool { return s.agree != nil }+// syntaxChecked marks the llm answers for the syntax check (resolveFile).func (s *llmStrategy) syntaxChecked() {}