Skip to content

Git designed for AI agents.

resolve: send CRLF files as LF; keep rejected answers for audit

Federico Elia2c2bef507ef8
diff --git a/doc/graft/PLAN10.md b/doc/graft/PLAN10.md
index 39edb16..99d10d6 100644
--- a/doc/graft/PLAN10.md
+++ b/doc/graft/PLAN10.md
@@ -703,6 +703,16 @@ Deviations, measured or stated:
* (e) Leniency: a region at the end of a file may end without a newline when neither side has a
final newline; synthmerge requires the newline.
+**R1 result and addendum** (r1-20261019-101514, real spend $0.099): strict error 41.8% (R0 on the
+same 300, re-judged: 53.8%); both accepted, wrong->correct 37 and correct->wrong 13, McNemar
+p = 0.0009; the bound with reorder and whitespace counted correct 32.9%; human-outside 29 cases,
+rate without them 33.2%. Of the 42 anchor rejections, 22 were CRLF files, and no CRLF file was
+accepted: the context went to the model with CR and had to come back byte for byte, which models
+do not do. Our bug, fixed before R2: a CRLF file is sent as LF and its answer converted back
+(mixed line endings are sent as they are, and the report counts the cases by line endings). The
+bench keeps the raw text of every rejected answer (<id>.rejected.txt beside the answers, 0600), so
+that rejections can be classified. R1b, R1 with the fix, is the baseline of R2.
+
**Measurement.** Luna, effort low, max_tokens 8000, routing pinned to the standard tiers, the same
300 paired cases (`--sample 300 --seed 1`, the first 300 of the Luna 600), cap $1.20 a run, the
whole series under $5 real, and a run over $1.00 real stops the series until the PM is told.
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.md
index 40b517d..5747787 100644
--- a/doc/graft/RESOLVE.md
+++ b/doc/graft/RESOLVE.md
@@ -607,7 +607,10 @@ differently there is one model call, whose request holds:
conflicting regions of the file for reference, and the directives last,
closest to the answer.
-The model answers in plain text, the patched code between a BEGIN and an END
+A file whose sides all break their lines with CRLF reaches the model with LF,
+since a model answers with LF; its answer is read as LF (CRLF in it too) and
+the resolved file is CRLF again. A file with mixed line endings is sent as it
+is. The model answers in plain text, the patched code between a BEGIN and an END
line that carry the request's random token: no JSON, no response format, and
no explanation. The last such block of the answer is read, since a model may
draft or quote before it answers. The answer is accepted for a region only
diff --git a/tools/graft-agents/resolve/bench/r1_test.go b/tools/graft-agents/resolve/bench/r1_test.go
index 43d29df..958160c 100644
--- a/tools/graft-agents/resolve/bench/r1_test.go
+++ b/tools/graft-agents/resolve/bench/r1_test.go
@@ -5,6 +5,8 @@ package bench
import (
"context"
"fmt"
+ "os"
+ "path/filepath"
"regexp"
"strings"
"testing"
@@ -161,3 +163,69 @@ func TestRunnerCountsRegionsEqualToOurs(t *testing.T) {
t.Fatalf("markdown:\n%s", md.String())
}
}
+
+// benchBadProvider answers every region with text that holds no patched code.
+type benchBadProvider struct{}
+
+func (benchBadProvider) Name() string { return "bench:bad" }
+
+func (benchBadProvider) Complete(context.Context, llm.Request) (*llm.Response, error) {
+ return &llm.Response{Text: "I cannot do that.", Model: "bench-bad"}, nil
+}
+
+func init() {
+ llm.Register("bench-bad-fake", func(llm.Config) (llm.Provider, error) { return benchBadProvider{}, nil })
+}
+
+// TestRunnerKeepsRejectedAnswers: the raw text of a rejected model answer is kept beside the
+// answers, readable by its owner only, and the report counts the cases by line endings.
+func TestRunnerKeepsRejectedAnswers(t *testing.T) {
+ f := newFixture(t)
+ buildHistory(f)
+ head := f.git("rev-parse", "HEAD")
+ cases, _, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg, err := resolve.ParseConfig([]byte("[[strategy]]\ntype = \"llm\"\n\n[strategy.llm]\ntype = \"bench-bad-fake\"\nmodel = \"m\"\n"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ res, err := cfg.Build()
+ if err != nil {
+ t.Fatal(err)
+ }
+ answers := filepath.Join(t.TempDir(), "answers")
+ if err := os.MkdirAll(answers, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ results, err := Runner{Resolver: res, Clone: func(string) string { return f.dir }, AnswersDir: answers}.Run(context.Background(), cases)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].Outcome != OutcomeRejected || results[0].LineEndings != resolve.LineEndingsLF {
+ t.Fatalf("results %+v, want one rejected LF case", results)
+ }
+ path := filepath.Join(answers, results[0].ID+".rejected.txt")
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(raw), "I cannot do that.") || !strings.Contains(string(raw), "has no patched code") {
+ t.Fatalf("rejected.txt = %q", raw)
+ }
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if info.Mode().Perm() != 0o600 {
+ t.Fatalf("mode %v, want 0600", info.Mode().Perm())
+ }
+ var md strings.Builder
+ if err := NewReport("bad", results).WriteMarkdown(&md); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(md.String(), "Content conflicts by line endings (CRLF goes to the model as LF; mixed is sent as it is): lf 1 (correct 0, wrong 0, rejected 1).") {
+ t.Fatalf("markdown:\n%s", md.String())
+ }
+}
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
index 976d629..74def8b 100644
--- a/tools/graft-agents/resolve/bench/report.go
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -6,6 +6,8 @@ import (
"sort"
"strings"
"time"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"
)
// Counts aggregates the outcomes of a set of cases.
@@ -233,6 +235,24 @@ func (rep Report) WriteMarkdown(w io.Writer) error {
rep.Total.EqualsOurs, rep.Total.Accepted(), rep.equalsOurs(OutcomeCorrect), rep.equalsOurs(OutcomeWrong))
fmt.Fprintf(&b, "\nRegions of the accepted model answers answered with OURS' region unchanged: %d of %d.\n",
rep.Total.RegionsEqualOurs, rep.Total.Regions)
+ endings := map[string]Counts{}
+ for _, r := range rep.Results {
+ if r.LineEndings == "" {
+ continue
+ }
+ c := endings[r.LineEndings]
+ c.add(r)
+ endings[r.LineEndings] = c
+ }
+ var lines []string
+ for _, e := range []string{resolve.LineEndingsLF, resolve.LineEndingsCRLF, resolve.LineEndingsMixed} {
+ if c, ok := endings[e]; ok {
+ lines = append(lines, fmt.Sprintf("%s %d (correct %d, wrong %d, rejected %d)", e, c.Cases, c.Correct, c.Wrong, c.Rejected))
+ }
+ }
+ if len(lines) > 0 {
+ fmt.Fprintf(&b, "\nContent conflicts by line endings (CRLF goes to the model as LF; mixed is sent as it is): %s.\n", strings.Join(lines, ", "))
+ }
if len(rep.Rejected) > 0 {
reasons := make([]string, 0, len(rep.Rejected))
for r := range rep.Rejected {
diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.go
index 70b729e..75d5088 100644
--- a/tools/graft-agents/resolve/bench/run.go
+++ b/tools/graft-agents/resolve/bench/run.go
@@ -63,9 +63,12 @@ type Result struct {
// Regions and RegionsEqualOurs count, for an accepted answer of an llm strategy, the
// conflicting regions the model answered and those it answered with OURS' region unchanged
// (synthmerge's no-change check, per region).
- Regions int `json:"regions,omitempty"`
- RegionsEqualOurs int `json:"regions_equal_ours,omitempty"`
- Strategy string `json:"strategy,omitempty"`
+ Regions int `json:"regions,omitempty"`
+ RegionsEqualOurs int `json:"regions_equal_ours,omitempty"`
+ // LineEndings are those of the sides of a content conflict (resolve.LLMLineEndings): the llm
+ // strategy sends a CRLF file as LF, and a mixed one as it is.
+ LineEndings string `json:"line_endings,omitempty"`
+ Strategy string `json:"strategy,omitempty"`
// Answer is the action of an accepted resolution ("write" or "delete"); with
// Runner.AnswersDir its content is stored for audit and Rejudge.
Answer string `json:"answer,omitempty"`
@@ -233,6 +236,9 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, er
Limits: protocol.Limits{MaxFileBytes: maxBytes},
}
+ if c.Kind == protocol.KindContent && sides["ours"] != nil && sides["theirs"] != nil {
+ res.LineEndings = resolve.LLMLineEndings(sideBytes(sides["base"]), sideBytes(sides["ours"]), sideBytes(sides["theirs"]))
+ }
start := time.Now()
regionStats := &resolve.LLMRegionStats{}
resp, err := r.Resolver.Resolve(resolve.WithLLMRegionStats(ctx, regionStats), req)
@@ -243,6 +249,13 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, er
return res, err
}
res.Outcome, res.Strategy, res.Reason = classifyFailure(unresolved.Failures[0])
+ // The raw text of a rejected model answer is kept beside the answers, for audit.
+ if rejected := regionStats.Rejected(); r.AnswersDir != "" && len(rejected) > 0 {
+ raw := strings.Join(rejected, "\n===== next rejected answer =====\n")
+ if err := os.WriteFile(filepath.Join(r.AnswersDir, c.ID+".rejected.txt"), []byte(raw), 0o600); err != nil {
+ return res, err
+ }
+ }
// A content conflict with more regions than the llm strategy sends is declined before
// any call: it is not sent, for that reason.
if res.Outcome == OutcomeAbstained && c.Kind == protocol.KindContent && sides["ours"] != nil && sides["theirs"] != nil {
diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.go
index 2e77d15..9fe573c 100644
--- a/tools/graft-agents/resolve/llm_core.go
+++ b/tools/graft-agents/resolve/llm_core.go
@@ -135,6 +135,21 @@ type LLMRegionStats struct {
mu sync.Mutex
regions, equalOurs int
recorded bool
+ rejected []string
+}
+
+// RecordRejected keeps the raw text of a model answer that was rejected, with why, for audit.
+func (s *LLMRegionStats) RecordRejected(text string) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.rejected = append(s.rejected, text)
+}
+
+// Rejected returns the raw texts of the rejected answers, in order.
+func (s *LLMRegionStats) Rejected() []string {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return append([]string(nil), s.rejected...)
}
// Record sets the counts of the last file resolved.
@@ -196,7 +211,54 @@ func ResolveLLM(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptions)
return LLMResult{}, err
}
- return llmResolvePatch(ctx, p, f, opt, nonce)
+ // A file whose sides all end their lines with CRLF reaches the model with LF: a model answers
+ // with LF, and the context lines must come back exactly. The answer is read as LF (CRLF in
+ // it is taken as LF too) and the result is CRLF again. A file with mixed line endings is
+ // sent as it is.
+ crlf := LLMLineEndings(f.Base, f.Ours, f.Theirs) == LineEndingsCRLF
+ if crlf {
+ f.Base, f.Ours, f.Theirs = llmToLF(f.Base), llmToLF(f.Ours), llmToLF(f.Theirs)
+ }
+ res, err := llmResolvePatch(ctx, p, f, opt, nonce, crlf)
+ if err != nil {
+ return LLMResult{}, err
+ }
+ if crlf {
+ res.Content = bytes.ReplaceAll(res.Content, []byte("\n"), []byte("\r\n"))
+ }
+ return res, nil
+}
+
+// The line endings of a file's sides (LLMLineEndings).
+const (
+ LineEndingsLF = "lf"
+ LineEndingsCRLF = "crlf"
+ LineEndingsMixed = "mixed"
+)
+
+// LLMLineEndings returns the line endings of the sides that are present (nil is absent): CRLF when
+// every line break of every side is CRLF and there is one, LF when none is, and mixed otherwise.
+func LLMLineEndings(sides ...[]byte) string {
+ breaks, crlf := 0, 0
+ for _, s := range sides {
+ breaks += bytes.Count(s, []byte("\n"))
+ crlf += bytes.Count(s, []byte("\r\n"))
+ }
+ switch {
+ case crlf == 0:
+ return LineEndingsLF
+ case crlf == breaks:
+ return LineEndingsCRLF
+ }
+ return LineEndingsMixed
+}
+
+// llmToLF returns b with CRLF line breaks as LF, keeping an absent side nil.
+func llmToLF(b []byte) []byte {
+ if b == nil {
+ return nil
+ }
+ return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
}
// llmNonce returns a random token that marks the data sections of a prompt.
diff --git a/tools/graft-agents/resolve/llm_gate2_test.go b/tools/graft-agents/resolve/llm_gate2_test.go
index ba77155..09a1a2b 100644
--- a/tools/graft-agents/resolve/llm_gate2_test.go
+++ b/tools/graft-agents/resolve/llm_gate2_test.go
@@ -66,25 +66,100 @@ func TestLLMStrategyDeclinesInvalidUTF8WithoutCall(t *testing.T) {
}
}
-// TestResolveLLMKeepsCRLFLineEndings: the context of a CRLF file is CRLF, an answer that keeps
-// CRLF lands as it is, and one whose region is not newline-terminated is refused rather than
-// patched up (the newline rule).
+// TestResolveLLMKeepsCRLFLineEndings: a file whose sides are all CRLF reaches the model as LF, a
+// model that answers with LF, as models do, gets its answer accepted, and the result is CRLF
+// again; an answer that echoes CRLF is read as LF and accepted too; the newline rule still holds.
func TestResolveLLMKeepsCRLFLineEndings(t *testing.T) {
base := strings.ReplaceAll(numbered("line", 30), "\n", "\r\n")
ours := strings.Replace(base, "line 15\r\n", "o\r\n", 1)
theirs := strings.Replace(base, "line 15\r\n", "t\r\n", 1)
f := LLMFile{Path: "crlf.txt", Base: []byte(base), Ours: []byte(ours), Theirs: []byte(theirs)}
+ want := strings.Replace(base, "line 15\r\n", "merged\r\n", 1)
- res, err := ResolveLLM(context.Background(), llmFixed("merged\r\n"), f, LLMOptions{})
+ // A realistic model: whatever it reads, it answers with LF line breaks.
+ p := &llmFake{answer: func(_ int, req llm.Request) (*llm.Response, error) {
+ return llmRawAnswer(strings.ReplaceAll(patchWrap(t, req, "merged\n"), "\r", "")), nil
+ }}
+ res, err := ResolveLLM(context.Background(), p, f, LLMOptions{})
if err != nil {
t.Fatal(err)
}
- content := string(res.Content)
- if want := strings.Replace(base, "line 15\r\n", "merged\r\n", 1); content != want {
- t.Fatalf("content = %q", content)
+ if string(res.Content) != want {
+ t.Fatalf("content = %q", res.Content)
+ }
+ if strings.Contains(p.reqs[0].System+p.reqs[0].Prompt, "\r") {
+ t.Fatal("a CRLF file reached the model with CR")
+ }
+
+ // A model that echoes CRLF in the code: its answer is read as LF.
+ echo := &llmFake{answer: func(_ int, req llm.Request) (*llm.Response, error) {
+ n := patchNonceRE.FindStringSubmatch(req.Prompt)[1]
+ body := "line 12\r\nline 13\r\nline 14\r\nmerged\r\nline 16\r\nline 17\r\nline 18\r\n"
+ return llmRawAnswer("===== BEGIN PATCHED CODE " + n + " =====\n" + body + "===== END PATCHED CODE " + n + " =====\n"), nil
+ }}
+ res, err = ResolveLLM(context.Background(), echo, f, LLMOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(res.Content) != want {
+ t.Fatalf("content of a CRLF echo = %q", res.Content)
}
if _, err := ResolveLLM(context.Background(), llmFixed("merged"), f, LLMOptions{}); !errors.Is(err, ErrLLMRejected) {
t.Fatalf("err = %v, want the newline rule", err)
}
}
+
+// TestResolveLLMLineEndingsBySidesPresent: an add/add conflict (no base) whose sides are CRLF is a
+// CRLF file; a file with mixed line endings is sent as it is.
+func TestResolveLLMLineEndingsBySidesPresent(t *testing.T) {
+ ours := strings.ReplaceAll(numbered("line", 20), "\n", "\r\n")
+ theirs := strings.Replace(ours, "line 10\r\n", "t\r\n", 1)
+ ours = strings.Replace(ours, "line 10\r\n", "o\r\n", 1)
+ p := llmFixed("merged\n")
+ res, err := ResolveLLM(context.Background(), p, LLMFile{Path: "a.txt", Ours: []byte(ours), Theirs: []byte(theirs)}, LLMOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(p.reqs[0].Prompt, "\r") || !strings.Contains(string(res.Content), "merged\r\n") {
+ t.Fatalf("add/add CRLF: prompt has CR %v, content %q", strings.Contains(p.reqs[0].Prompt, "\r"), res.Content)
+ }
+
+ mixed := strings.Replace(ours, "line 3\r\n", "line 3\n", 1)
+ p = llmFixed("merged\n")
+ _, _ = ResolveLLM(context.Background(), p, LLMFile{Path: "a.txt", Ours: []byte(mixed), Theirs: []byte(theirs)}, LLMOptions{})
+ if len(p.reqs) == 0 || !strings.Contains(p.reqs[0].Prompt, "\r") {
+ t.Fatal("a file with mixed line endings was not sent as it is")
+ }
+
+ for _, tc := range []struct {
+ sides [][]byte
+ want string
+ }{
+ {[][]byte{[]byte("a\nb\n"), nil, []byte("c\n")}, LineEndingsLF},
+ {[][]byte{[]byte("a\r\nb\r\n"), nil, []byte("c\r\nd")}, LineEndingsCRLF},
+ {[][]byte{[]byte("a\r\nb\n"), []byte("c\r\n")}, LineEndingsMixed},
+ {[][]byte{[]byte("a\r\n"), []byte("c\n")}, LineEndingsMixed},
+ {[][]byte{[]byte("no break"), nil}, LineEndingsLF},
+ } {
+ if got := LLMLineEndings(tc.sides...); got != tc.want {
+ t.Fatalf("%q: %s, want %s", tc.sides, got, tc.want)
+ }
+ }
+}
+
+// TestResolveLLMRecordsRejectedAnswers: the raw text of a rejected answer reaches the
+// LLMRegionStats of the context, with why it was rejected.
+func TestResolveLLMRecordsRejectedAnswers(t *testing.T) {
+ stats := &LLMRegionStats{}
+ f := LLMFile{Path: "a.txt", Base: []byte("base\n"), Ours: []byte("ours\n"), Theirs: []byte("theirs\n")}
+ raw := &llmFake{answer: func(int, llm.Request) (*llm.Response, error) { return llmRawAnswer("no block at all"), nil }}
+ _, err := ResolveLLM(WithLLMRegionStats(context.Background(), stats), raw, f, LLMOptions{})
+ if !errors.Is(err, ErrLLMRejected) {
+ t.Fatalf("err = %v", err)
+ }
+ got := stats.Rejected()
+ if len(got) != 1 || !strings.Contains(got[0], "no block at all") || !strings.Contains(got[0], "has no patched code") {
+ t.Fatalf("rejected = %q", got)
+ }
+}
diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.go
index 802d836..5f43c9f 100644
--- a/tools/graft-agents/resolve/llm_patch.go
+++ b/tools/graft-agents/resolve/llm_patch.go
@@ -398,7 +398,9 @@ func LLMConflictRegions(base, ours, theirs []byte) int {
// MaxRegions conflicting regions, or a region whose request is over MaxPromptBytes, is declined
// before any call. The file is resolved only when every region is: the first rejected region
// rejects the file, and no partial resolution is returned.
-func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptions, nonce string) (LLMResult, error) {
+// crlf says that the file was CRLF and is sent as LF (ResolveLLM): CRLF in an answer is read as LF.
+// The raw text of a rejected answer goes to the LLMRegionStats of ctx, if any, for audit.
+func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptions, nonce string, crlf bool) (LLMResult, error) {
base := llmSplitLines(f.Base)
oursEdits, err := llmDiff(base, llmSplitLines(f.Ours), defaultLLMMaxDiffCells)
if err != nil {
@@ -451,16 +453,26 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOpti
if err != nil {
return LLMResult{}, err
}
+ rejected := func(err error) error {
+ err = fmt.Errorf("region %d: %w", k+1, err)
+ if stats := llmRegionStatsFrom(ctx); stats != nil {
+ stats.RecordRejected(fmt.Sprintf("%v\n----- raw answer -----\n%s", err, resp.Text))
+ }
+ return err
+ }
answer, err := llmPatchAnswer(resp.Text, nonce)
if err != nil {
- return LLMResult{}, fmt.Errorf("region %d: %w", k+1, err)
+ return LLMResult{}, rejected(err)
+ }
+ if crlf {
+ answer = strings.ReplaceAll(answer, "\r\n", "\n")
}
region, err := llmCheckPatched(answer, material[k], r.ours, r.theirs)
if err != nil {
- return LLMResult{}, fmt.Errorf("region %d: %w", k+1, err)
+ return LLMResult{}, rejected(err)
}
if err := llmCheckAnswer(region, opt); err != nil {
- return LLMResult{}, fmt.Errorf("region %d: %w", k+1, err)
+ return LLMResult{}, rejected(err)
}
r.answer = region
if region == r.ours {