Skip to content

Git designed for AI agents.

graft: the agreement gate decides "clean" from structured findings, and cancels the second model

Federico Eliab6efa99a745c
diff --git a/tools/graft-agents/resolve/llm_agree_test.go b/tools/graft-agents/resolve/llm_agree_test.go
index d2e119b..7ebf471 100644
--- a/tools/graft-agents/resolve/llm_agree_test.go
+++ b/tools/graft-agents/resolve/llm_agree_test.go
@@ -7,7 +7,9 @@ import (
"errors"
"regexp"
"strings"
+ "sync/atomic"
"testing"
+ "time"
"github.com/feder-cr/git-ai/tools/graft-agents/llm"
"github.com/feder-cr/git-ai/tools/graft-agents/protocol"
@@ -133,15 +135,30 @@ func TestAgreeWithASecondModelThatFailsHolds(t *testing.T) {
func TestAgreeWithAFirstModelThatFailsFailsTheFile(t *testing.T) {
wp19SetAnswer(agreeFail("provider down"))
- b := agreeSetB(agreeAnswer("merged\n"))
+ // The second model is slow: it answers only when its context ends. The first model failing cancels
+ // it, and the file is not kept waiting for it.
+ var cancelled atomic.Bool
+ b := agreeSetB(func(ctx context.Context, req llm.Request) (*llm.Response, error) {
+ select {
+ case <-ctx.Done():
+ cancelled.Store(true)
+ return nil, ctx.Err()
+ case <-time.After(30 * time.Second):
+ return wp19Answer("merged\n"), nil
+ }
+ })
+ start := time.Now()
_, err := resolveOne(t)
var unresolved *UnresolvedError
if !errors.As(err, &unresolved) {
t.Fatalf("err = %v", err)
}
- // The second model was asked at the same time and its call is waited for: the cost is real.
- if b.calls() != 1 {
- t.Fatalf("calls of the second model = %d", b.calls())
+ if time.Since(start) > 10*time.Second {
+ t.Fatalf("the file waited %s for the second model", time.Since(start))
+ }
+ // The second model was asked at the same time, its call was cancelled, and it was waited for.
+ if b.calls() != 1 || !cancelled.Load() {
+ t.Fatalf("calls of the second model = %d, cancelled = %v", b.calls(), cancelled.Load())
}
}
@@ -149,8 +166,8 @@ func TestAgreeWithAnUncleanReportHoldsEvenWhenTheModelsAgree(t *testing.T) {
r := wp19Build(t, agreeConfig)
s := r.bindings[0].Strategy.(*llmStrategy)
clean := LLMResult{Content: []byte("x\n"), Report: []string{"keeps every uncontested change of both sides (2)"}}
- dirty := LLMResult{Content: []byte("x\n"), Report: []string{"region 1: drops theirs' change: the line \"y\" it adds is missing"}}
- repeats := LLMResult{Content: []byte("x\n"), Report: []string{"region 2: duplicates a line both sides added: \"z\""}}
+ dirty := LLMResult{Content: []byte("x\n"), Report: []string{"region 1: drops theirs' change: the line \"y\" it adds is missing"}, Findings: 1}
+ repeats := LLMResult{Content: []byte("x\n"), Report: []string{"region 2: duplicates a line both sides added: \"z\""}, Findings: 1}
run := func(res LLMResult) *agreeRun { return &agreeRun{res: res} }
if why := s.agreement(clean, run(clean)); why != "" {
@@ -232,25 +249,42 @@ func TestAgreeWithConfig(t *testing.T) {
}
}
-func TestLLMReportClean(t *testing.T) {
- for report, want := range map[string]bool{
- "": true,
- "keeps every uncontested change of both sides (3)": true,
- "2 contested changes resolved freely: nothing line-level to verify": true,
- "1 of 2 regions were answered again after findings of the deterministic checks": true,
- "region 1: drops ours' change 3-4: the line \"a\" it adds is missing": false,
- "region 2: duplicates a line both sides added: \"b\"": false,
- } {
- var lines []string
- if report != "" {
- lines = []string{report}
- }
- if got := LLMReportClean(lines); got != want {
- t.Errorf("LLMReportClean(%q) = %v", report, got)
+// TestLLMFindingsSurviveTheCutOfTheReport: the report is cut at 16 lines, so a drop in a late region can
+// be missing from its text, and the agree_with gate must not read the text. Findings counts them all.
+func TestLLMFindingsSurviveTheCutOfTheReport(t *testing.T) {
+ var comps []llmComposition
+ for i := 0; i < 17; i++ {
+ comps = append(comps, llmComposition{neither: true})
+ }
+ late := llmComposition{problems: []string{"drops THEIRS' change at base line 6: the line \"T6\" it adds is missing"}}
+ comps = append(comps, late)
+
+ report := llmReport(comps, 0)
+ if len(report) != 16 {
+ t.Fatalf("the report is not cut: %d lines", len(report))
+ }
+ for _, line := range report {
+ if strings.Contains(line, "drops") {
+ t.Fatalf("the late drop is in the first 16 lines, the test proves nothing: %v", report)
}
}
- if !LLMReportClean([]string{"a", "b", "region 3: the answer has lines found in neither side"}) {
- t.Error("a line found in neither side is a free resolution, not a drop")
+ if got := llmFindings(comps); got != 1 {
+ t.Fatalf("findings = %d, want the late drop", got)
+ }
+
+ // The gate holds a file whose report text is clean and whose findings are not.
+ r := wp19Build(t, agreeConfig)
+ s := r.bindings[0].Strategy.(*llmStrategy)
+ text := LLMResult{Content: []byte("x\n"), Report: report, Findings: llmFindings(comps)}
+ clean := LLMResult{Content: []byte("x\n"), Report: report}
+ if why := s.agreement(text, &agreeRun{res: clean}); !strings.HasPrefix(why, "held: a composition report was not clean") {
+ t.Fatalf("a drop past the cut is not held: %q", why)
+ }
+ if why := s.agreement(clean, &agreeRun{res: text}); !strings.HasPrefix(why, "held: a composition report was not clean") {
+ t.Fatalf("a drop past the cut of the second model is not held: %q", why)
+ }
+ if why := s.agreement(clean, &agreeRun{res: clean}); why != "" {
+ t.Fatalf("two answers without findings are held: %q", why)
}
}
@@ -267,3 +301,38 @@ func TestLLMNormalize(t *testing.T) {
}
}
}
+
+// TestAgreeWithNeedsAServerThatHolds: with agree_with a file may carry a hold, which only a server with
+// the capability "hold" never lands. A server that proposes but cannot hold gets no model's resolution
+// (the models are not called), and one that holds does.
+func TestAgreeWithNeedsAServerThatHolds(t *testing.T) {
+ a := wp19SetAnswer(agreeAnswer("merged\n"))
+ b := agreeSetB(agreeAnswer("merged\n"))
+ r := wp19Build(t, agreeConfig)
+
+ noHold := wp19Request(wp19Conflict("a.txt"))
+ noHold.Capabilities = []string{protocol.CapabilityOrigin, protocol.CapabilityProposal, protocol.CapabilityVerify}
+ _, err := r.Resolve(context.Background(), noHold)
+ var unresolved *UnresolvedError
+ if !errors.As(err, &unresolved) || !strings.Contains(err.Error(), "the server cannot hold a model's resolution (agree_with); upgrade the server") {
+ t.Fatalf("err = %v", err)
+ }
+ if a.calls() != 0 || b.calls() != 0 {
+ t.Fatalf("the models were called for a server that cannot hold: %d, %d", a.calls(), b.calls())
+ }
+
+ // Without agree_with the same server is enough.
+ single := wp19Build(t, wp19LLMConfig)
+ if _, err := single.Resolve(context.Background(), noHold); err != nil {
+ t.Fatalf("a single model for a server that proposes: %v", err)
+ }
+
+ // A server of this release.
+ if _, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt"))); err != nil {
+ t.Fatal(err)
+ }
+ // The first model served the single-model run above as well.
+ if a.calls() != 2 || b.calls() != 1 {
+ t.Fatalf("calls %d, %d", a.calls(), b.calls())
+ }
+}
diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.go
index e59f99a..fbf317e 100644
--- a/tools/graft-agents/resolve/llm_core.go
+++ b/tools/graft-agents/resolve/llm_core.go
@@ -128,6 +128,10 @@ type LLMResult struct {
// drops and repeats of the two changes, and how many contested regions it resolved freely. It is
// a report and not a gate, and goes to the agent with the proposal.
Report []string
+ // Findings is the number of drops and duplicates that the composition check found in the regions
+ // (each one is a line of the report, but the report is cut at 16 lines and Findings is not). The
+ // agree_with gate decides "clean" from it, never from the text of the report.
+ Findings int
// Calls and Repairs are the model calls of the file and those of them that were repair rounds.
Calls, Repairs int
}
@@ -279,19 +283,6 @@ func (s *LLMRegionStats) Absorb(o *LLMRegionStats, prefix string) {
}
}
-// LLMReportClean reports whether a composition report holds no finding: no line says that the answer
-// drops a change of a side or repeats a line both added (llmReport words them first). It is the
-// "clean report" of the agree_with gate (PLAN10 15.12): a report is not a gate for a single model,
-// and is one of the two conditions under which two models' agreement is trusted.
-func LLMReportClean(report []string) bool {
- for _, line := range report {
- if strings.Contains(line, "drops") || strings.Contains(line, "duplicates") {
- return false
- }
- }
- return true
-}
-
// LLMNormalize is a file as the agree_with gate and the judges compare it: line endings as LF, no
// whitespace at the end of a line, no blank line at the end of the file. The bench normalizes
// with it too, so that "the same answer" means the same thing everywhere.
diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.go
index 8470873..712db9f 100644
--- a/tools/graft-agents/resolve/llm_patch.go
+++ b/tools/graft-agents/resolve/llm_patch.go
@@ -429,6 +429,16 @@ func llmRegionAtLine(spans map[int][2]int, line int) (int, bool) {
return best, found
}
+// llmFindings counts the drops and the duplicates of the regions: what the report says first, whatever
+// of it the cut of the report leaves.
+func llmFindings(comps []llmComposition) int {
+ n := 0
+ for _, c := range comps {
+ n += len(c.problems)
+ }
+ return n
+}
+
// llmReport words the composition checks of the regions (llmCompose) as the report that goes with
// the proposal: what is wrong first, in the words a repair round quotes, then what it keeps.
func llmReport(comps []llmComposition, repairs int) []string {
@@ -678,6 +688,7 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOpti
Hunks: len(conflicting),
RegionsEqualOurs: equalOurs,
Report: report,
+ Findings: llmFindings(comps),
Calls: calls,
Repairs: repairs,
}, nil
diff --git a/tools/graft-agents/resolve/llm_strategy.go b/tools/graft-agents/resolve/llm_strategy.go
index bae360f..ef09670 100644
--- a/tools/graft-agents/resolve/llm_strategy.go
+++ b/tools/graft-agents/resolve/llm_strategy.go
@@ -48,7 +48,7 @@ import (
// With agree_with, both models resolve each file at the same time with the same prompt, and the
// first model's resolution is proposed as always (it lands only through verify.command). It is
// HELD, and never lands, unless the two results are the same file after normalization
-// (LLMNormalize) and both composition reports are clean (LLMReportClean): the response carries
+// (LLMNormalize) and neither composition check found a drop or a duplicate (LLMResult.Findings): the response carries
// Hold, the report says that the second model disagreed, and the server neither verifies nor
// lands the file. A model that fails makes the file unresolved when it is the first, and holds it
// when it is the second. A budget on one model needs a budget on the other.
@@ -226,8 +226,12 @@ func (s *llmStrategy) Resolve(ctx context.Context, f File, req protocol.Request)
defer cancel()
res, err := ResolveLLM(first, s.provider, lf, s.opt)
if second != nil {
- // The second model is waited for whatever the first one did: its calls are paid for, and
- // its usage belongs to the file.
+ // A first model that failed or declined leaves nothing for the second one to be compared with:
+ // its context is cancelled. It is waited for whatever happened: the calls it made are paid for,
+ // and its usage belongs to the file.
+ if err != nil {
+ second.cancel()
+ }
second.wait()
if stats != nil {
stats.Absorb(second.stats, "agree_with: ")
@@ -254,22 +258,25 @@ func (s *llmStrategy) Resolve(ctx context.Context, f File, req protocol.Request)
// agreeRun is the second model resolving a file, started by agree.start.
type agreeRun struct {
- done chan struct{}
- res LLMResult
- err error
- stats *LLMRegionStats
+ done chan struct{}
+ cancel context.CancelFunc
+ res LLMResult
+ err error
+ stats *LLMRegionStats
}
// start resolves lf with the side's model in the background. When counting, the run keeps its own
// stats, which the caller absorbs into its own once the run is waited for.
func (s *llmSide) start(ctx context.Context, lf LLMFile, counting bool) *agreeRun {
- r := &agreeRun{done: make(chan struct{})}
+ ctx, cancel := context.WithCancel(ctx)
+ r := &agreeRun{done: make(chan struct{}), cancel: cancel}
if counting {
r.stats = &LLMRegionStats{}
ctx = WithLLMRegionStats(ctx, r.stats)
}
go func() {
defer close(r.done)
+ defer cancel()
defer func() {
if p := recover(); p != nil {
r.err = fmt.Errorf("resolve: the second model panicked: %v", p)
@@ -287,16 +294,17 @@ func (r *agreeRun) wait() { <-r.done }
// agreement says why the first model's resolution is not the one both models give, or "" when the
// two agree: the same file after the judge's normalization (LLMNormalize), and neither report
-// holds a finding (LLMReportClean). The first model's answer is the one that is proposed.
+// found a drop or a duplicate (LLMResult.Findings, never the text of the report, which is cut). The
+// first model's answer is the one that is proposed, byte for byte.
func (s *llmStrategy) agreement(first LLMResult, second *agreeRun) string {
switch {
case second.err != nil && errors.Is(second.err, ErrLLMNotApplicable):
return fmt.Sprintf("held: the second model (%s) could not take the file", s.agree.model)
case second.err != nil:
return fmt.Sprintf("held: the second model (%s) gave no usable answer: %s", s.agree.model, llmClip(second.err.Error(), 200))
- case !LLMReportClean(first.Report):
+ case first.Findings > 0:
return fmt.Sprintf("held: a composition report was not clean: the first model (%s) drops or repeats a change", s.model)
- case !LLMReportClean(second.res.Report):
+ case second.res.Findings > 0:
return fmt.Sprintf("held: a composition report was not clean: the second model (%s) drops or repeats a change", s.agree.model)
case !bytes.Equal(LLMNormalize(first.Content), LLMNormalize(second.res.Content)):
return fmt.Sprintf("held: the second model (%s) disagreed: it resolved the file differently from the first (%s)", s.agree.model, s.model)
@@ -304,6 +312,9 @@ func (s *llmStrategy) agreement(first LLMResult, second *agreeRun) string {
return ""
}
+// holds reports whether the strategy's answers may carry a hold (agree_with is set).
+func (s *llmStrategy) holds() bool { return s.agree != nil }
+
// syntaxChecked marks the llm answers for the syntax check (resolveFile).
func (s *llmStrategy) syntaxChecked() {}