Git designed for AI agents.
resolve: send CRLF files as LF; keep rejected answers for audit
Federico Elia2c2bef507ef8
diff --git a/doc/graft/PLAN10.md b/doc/graft/PLAN10.mdindex 39edb16..99d10d6 100644--- a/doc/graft/PLAN10.md+++ b/doc/graft/PLAN10.md@@ -703,6 +703,16 @@ Deviations, measured or stated:* (e) Leniency: a region at the end of a file may end without a newline when neither side has afinal newline; synthmerge requires the newline.+**R1 result and addendum** (r1-20261019-101514, real spend $0.099): strict error 41.8% (R0 on the+same 300, re-judged: 53.8%); both accepted, wrong->correct 37 and correct->wrong 13, McNemar+p = 0.0009; the bound with reorder and whitespace counted correct 32.9%; human-outside 29 cases,+rate without them 33.2%. Of the 42 anchor rejections, 22 were CRLF files, and no CRLF file was+accepted: the context went to the model with CR and had to come back byte for byte, which models+do not do. Our bug, fixed before R2: a CRLF file is sent as LF and its answer converted back+(mixed line endings are sent as they are, and the report counts the cases by line endings). The+bench keeps the raw text of every rejected answer (<id>.rejected.txt beside the answers, 0600), so+that rejections can be classified. R1b, R1 with the fix, is the baseline of R2.+**Measurement.** Luna, effort low, max_tokens 8000, routing pinned to the standard tiers, the same300 paired cases (`--sample 300 --seed 1`, the first 300 of the Luna 600), cap $1.20 a run, thewhole series under $5 real, and a run over $1.00 real stops the series until the PM is told.diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.mdindex 40b517d..5747787 100644--- a/doc/graft/RESOLVE.md+++ b/doc/graft/RESOLVE.md@@ -607,7 +607,10 @@ differently there is one model call, whose request holds:conflicting regions of the file for reference, and the directives last,closest to the answer.-The model answers in plain text, the patched code between a BEGIN and an END+A file whose sides all break their lines with CRLF reaches the model with LF,+since a model answers with LF; its answer is read as LF (CRLF in it too) and+the resolved file is CRLF again. A file with mixed line endings is sent as it+is. The model answers in plain text, the patched code between a BEGIN and an ENDline that carry the request's random token: no JSON, no response format, andno explanation. The last such block of the answer is read, since a model maydraft or quote before it answers. The answer is accepted for a region onlydiff --git a/tools/graft-agents/resolve/bench/r1_test.go b/tools/graft-agents/resolve/bench/r1_test.goindex 43d29df..958160c 100644--- a/tools/graft-agents/resolve/bench/r1_test.go+++ b/tools/graft-agents/resolve/bench/r1_test.go@@ -5,6 +5,8 @@ package benchimport ("context""fmt"+ "os"+ "path/filepath""regexp""strings""testing"@@ -161,3 +163,69 @@ func TestRunnerCountsRegionsEqualToOurs(t *testing.T) {t.Fatalf("markdown:\n%s", md.String())}}++// benchBadProvider answers every region with text that holds no patched code.+type benchBadProvider struct{}++func (benchBadProvider) Name() string { return "bench:bad" }++func (benchBadProvider) Complete(context.Context, llm.Request) (*llm.Response, error) {+ return &llm.Response{Text: "I cannot do that.", Model: "bench-bad"}, nil+}++func init() {+ llm.Register("bench-bad-fake", func(llm.Config) (llm.Provider, error) { return benchBadProvider{}, nil })+}++// TestRunnerKeepsRejectedAnswers: the raw text of a rejected model answer is kept beside the+// answers, readable by its owner only, and the report counts the cases by line endings.+func TestRunnerKeepsRejectedAnswers(t *testing.T) {+ f := newFixture(t)+ buildHistory(f)+ head := f.git("rev-parse", "HEAD")+ cases, _, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})+ if err != nil {+ t.Fatal(err)+ }+ cfg, err := resolve.ParseConfig([]byte("[[strategy]]\ntype = \"llm\"\n\n[strategy.llm]\ntype = \"bench-bad-fake\"\nmodel = \"m\"\n"))+ if err != nil {+ t.Fatal(err)+ }+ res, err := cfg.Build()+ if err != nil {+ t.Fatal(err)+ }+ answers := filepath.Join(t.TempDir(), "answers")+ if err := os.MkdirAll(answers, 0o700); err != nil {+ t.Fatal(err)+ }+ results, err := Runner{Resolver: res, Clone: func(string) string { return f.dir }, AnswersDir: answers}.Run(context.Background(), cases)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 || results[0].Outcome != OutcomeRejected || results[0].LineEndings != resolve.LineEndingsLF {+ t.Fatalf("results %+v, want one rejected LF case", results)+ }+ path := filepath.Join(answers, results[0].ID+".rejected.txt")+ raw, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(raw), "I cannot do that.") || !strings.Contains(string(raw), "has no patched code") {+ t.Fatalf("rejected.txt = %q", raw)+ }+ info, err := os.Stat(path)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm() != 0o600 {+ t.Fatalf("mode %v, want 0600", info.Mode().Perm())+ }+ var md strings.Builder+ if err := NewReport("bad", results).WriteMarkdown(&md); err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(md.String(), "Content conflicts by line endings (CRLF goes to the model as LF; mixed is sent as it is): lf 1 (correct 0, wrong 0, rejected 1).") {+ t.Fatalf("markdown:\n%s", md.String())+ }+}diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex 976d629..74def8b 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -6,6 +6,8 @@ import ("sort""strings""time"++ "github.com/feder-cr/git-ai/tools/graft-agents/resolve")// Counts aggregates the outcomes of a set of cases.@@ -233,6 +235,24 @@ func (rep Report) WriteMarkdown(w io.Writer) error {rep.Total.EqualsOurs, rep.Total.Accepted(), rep.equalsOurs(OutcomeCorrect), rep.equalsOurs(OutcomeWrong))fmt.Fprintf(&b, "\nRegions of the accepted model answers answered with OURS' region unchanged: %d of %d.\n",rep.Total.RegionsEqualOurs, rep.Total.Regions)+ endings := map[string]Counts{}+ for _, r := range rep.Results {+ if r.LineEndings == "" {+ continue+ }+ c := endings[r.LineEndings]+ c.add(r)+ endings[r.LineEndings] = c+ }+ var lines []string+ for _, e := range []string{resolve.LineEndingsLF, resolve.LineEndingsCRLF, resolve.LineEndingsMixed} {+ if c, ok := endings[e]; ok {+ lines = append(lines, fmt.Sprintf("%s %d (correct %d, wrong %d, rejected %d)", e, c.Cases, c.Correct, c.Wrong, c.Rejected))+ }+ }+ if len(lines) > 0 {+ fmt.Fprintf(&b, "\nContent conflicts by line endings (CRLF goes to the model as LF; mixed is sent as it is): %s.\n", strings.Join(lines, ", "))+ }if len(rep.Rejected) > 0 {reasons := make([]string, 0, len(rep.Rejected))for r := range rep.Rejected {diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.goindex 70b729e..75d5088 100644--- a/tools/graft-agents/resolve/bench/run.go+++ b/tools/graft-agents/resolve/bench/run.go@@ -63,9 +63,12 @@ type Result struct {// Regions and RegionsEqualOurs count, for an accepted answer of an llm strategy, the// conflicting regions the model answered and those it answered with OURS' region unchanged// (synthmerge's no-change check, per region).- Regions int `json:"regions,omitempty"`- RegionsEqualOurs int `json:"regions_equal_ours,omitempty"`- Strategy string `json:"strategy,omitempty"`+ Regions int `json:"regions,omitempty"`+ RegionsEqualOurs int `json:"regions_equal_ours,omitempty"`+ // LineEndings are those of the sides of a content conflict (resolve.LLMLineEndings): the llm+ // strategy sends a CRLF file as LF, and a mixed one as it is.+ LineEndings string `json:"line_endings,omitempty"`+ Strategy string `json:"strategy,omitempty"`// Answer is the action of an accepted resolution ("write" or "delete"); with// Runner.AnswersDir its content is stored for audit and Rejudge.Answer string `json:"answer,omitempty"`@@ -233,6 +236,9 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, erLimits: protocol.Limits{MaxFileBytes: maxBytes},}+ if c.Kind == protocol.KindContent && sides["ours"] != nil && sides["theirs"] != nil {+ res.LineEndings = resolve.LLMLineEndings(sideBytes(sides["base"]), sideBytes(sides["ours"]), sideBytes(sides["theirs"]))+ }start := time.Now()regionStats := &resolve.LLMRegionStats{}resp, err := r.Resolver.Resolve(resolve.WithLLMRegionStats(ctx, regionStats), req)@@ -243,6 +249,13 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, erreturn res, err}res.Outcome, res.Strategy, res.Reason = classifyFailure(unresolved.Failures[0])+ // The raw text of a rejected model answer is kept beside the answers, for audit.+ if rejected := regionStats.Rejected(); r.AnswersDir != "" && len(rejected) > 0 {+ raw := strings.Join(rejected, "\n===== next rejected answer =====\n")+ if err := os.WriteFile(filepath.Join(r.AnswersDir, c.ID+".rejected.txt"), []byte(raw), 0o600); err != nil {+ return res, err+ }+ }// A content conflict with more regions than the llm strategy sends is declined before// any call: it is not sent, for that reason.if res.Outcome == OutcomeAbstained && c.Kind == protocol.KindContent && sides["ours"] != nil && sides["theirs"] != nil {diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.goindex 2e77d15..9fe573c 100644--- a/tools/graft-agents/resolve/llm_core.go+++ b/tools/graft-agents/resolve/llm_core.go@@ -135,6 +135,21 @@ type LLMRegionStats struct {mu sync.Mutexregions, equalOurs intrecorded bool+ rejected []string+}++// RecordRejected keeps the raw text of a model answer that was rejected, with why, for audit.+func (s *LLMRegionStats) RecordRejected(text string) {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.rejected = append(s.rejected, text)+}++// Rejected returns the raw texts of the rejected answers, in order.+func (s *LLMRegionStats) Rejected() []string {+ s.mu.Lock()+ defer s.mu.Unlock()+ return append([]string(nil), s.rejected...)}// Record sets the counts of the last file resolved.@@ -196,7 +211,54 @@ func ResolveLLM(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptions)return LLMResult{}, err}- return llmResolvePatch(ctx, p, f, opt, nonce)+ // A file whose sides all end their lines with CRLF reaches the model with LF: a model answers+ // with LF, and the context lines must come back exactly. The answer is read as LF (CRLF in+ // it is taken as LF too) and the result is CRLF again. A file with mixed line endings is+ // sent as it is.+ crlf := LLMLineEndings(f.Base, f.Ours, f.Theirs) == LineEndingsCRLF+ if crlf {+ f.Base, f.Ours, f.Theirs = llmToLF(f.Base), llmToLF(f.Ours), llmToLF(f.Theirs)+ }+ res, err := llmResolvePatch(ctx, p, f, opt, nonce, crlf)+ if err != nil {+ return LLMResult{}, err+ }+ if crlf {+ res.Content = bytes.ReplaceAll(res.Content, []byte("\n"), []byte("\r\n"))+ }+ return res, nil+}++// The line endings of a file's sides (LLMLineEndings).+const (+ LineEndingsLF = "lf"+ LineEndingsCRLF = "crlf"+ LineEndingsMixed = "mixed"+)++// LLMLineEndings returns the line endings of the sides that are present (nil is absent): CRLF when+// every line break of every side is CRLF and there is one, LF when none is, and mixed otherwise.+func LLMLineEndings(sides ...[]byte) string {+ breaks, crlf := 0, 0+ for _, s := range sides {+ breaks += bytes.Count(s, []byte("\n"))+ crlf += bytes.Count(s, []byte("\r\n"))+ }+ switch {+ case crlf == 0:+ return LineEndingsLF+ case crlf == breaks:+ return LineEndingsCRLF+ }+ return LineEndingsMixed+}++// llmToLF returns b with CRLF line breaks as LF, keeping an absent side nil.+func llmToLF(b []byte) []byte {+ if b == nil {+ return nil+ }+ return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))}// llmNonce returns a random token that marks the data sections of a prompt.diff --git a/tools/graft-agents/resolve/llm_gate2_test.go b/tools/graft-agents/resolve/llm_gate2_test.goindex ba77155..09a1a2b 100644--- a/tools/graft-agents/resolve/llm_gate2_test.go+++ b/tools/graft-agents/resolve/llm_gate2_test.go@@ -66,25 +66,100 @@ func TestLLMStrategyDeclinesInvalidUTF8WithoutCall(t *testing.T) {}}-// TestResolveLLMKeepsCRLFLineEndings: the context of a CRLF file is CRLF, an answer that keeps-// CRLF lands as it is, and one whose region is not newline-terminated is refused rather than-// patched up (the newline rule).+// TestResolveLLMKeepsCRLFLineEndings: a file whose sides are all CRLF reaches the model as LF, a+// model that answers with LF, as models do, gets its answer accepted, and the result is CRLF+// again; an answer that echoes CRLF is read as LF and accepted too; the newline rule still holds.func TestResolveLLMKeepsCRLFLineEndings(t *testing.T) {base := strings.ReplaceAll(numbered("line", 30), "\n", "\r\n")ours := strings.Replace(base, "line 15\r\n", "o\r\n", 1)theirs := strings.Replace(base, "line 15\r\n", "t\r\n", 1)f := LLMFile{Path: "crlf.txt", Base: []byte(base), Ours: []byte(ours), Theirs: []byte(theirs)}+ want := strings.Replace(base, "line 15\r\n", "merged\r\n", 1)- res, err := ResolveLLM(context.Background(), llmFixed("merged\r\n"), f, LLMOptions{})+ // A realistic model: whatever it reads, it answers with LF line breaks.+ p := &llmFake{answer: func(_ int, req llm.Request) (*llm.Response, error) {+ return llmRawAnswer(strings.ReplaceAll(patchWrap(t, req, "merged\n"), "\r", "")), nil+ }}+ res, err := ResolveLLM(context.Background(), p, f, LLMOptions{})if err != nil {t.Fatal(err)}- content := string(res.Content)- if want := strings.Replace(base, "line 15\r\n", "merged\r\n", 1); content != want {- t.Fatalf("content = %q", content)+ if string(res.Content) != want {+ t.Fatalf("content = %q", res.Content)+ }+ if strings.Contains(p.reqs[0].System+p.reqs[0].Prompt, "\r") {+ t.Fatal("a CRLF file reached the model with CR")+ }++ // A model that echoes CRLF in the code: its answer is read as LF.+ echo := &llmFake{answer: func(_ int, req llm.Request) (*llm.Response, error) {+ n := patchNonceRE.FindStringSubmatch(req.Prompt)[1]+ body := "line 12\r\nline 13\r\nline 14\r\nmerged\r\nline 16\r\nline 17\r\nline 18\r\n"+ return llmRawAnswer("===== BEGIN PATCHED CODE " + n + " =====\n" + body + "===== END PATCHED CODE " + n + " =====\n"), nil+ }}+ res, err = ResolveLLM(context.Background(), echo, f, LLMOptions{})+ if err != nil {+ t.Fatal(err)+ }+ if string(res.Content) != want {+ t.Fatalf("content of a CRLF echo = %q", res.Content)}if _, err := ResolveLLM(context.Background(), llmFixed("merged"), f, LLMOptions{}); !errors.Is(err, ErrLLMRejected) {t.Fatalf("err = %v, want the newline rule", err)}}++// TestResolveLLMLineEndingsBySidesPresent: an add/add conflict (no base) whose sides are CRLF is a+// CRLF file; a file with mixed line endings is sent as it is.+func TestResolveLLMLineEndingsBySidesPresent(t *testing.T) {+ ours := strings.ReplaceAll(numbered("line", 20), "\n", "\r\n")+ theirs := strings.Replace(ours, "line 10\r\n", "t\r\n", 1)+ ours = strings.Replace(ours, "line 10\r\n", "o\r\n", 1)+ p := llmFixed("merged\n")+ res, err := ResolveLLM(context.Background(), p, LLMFile{Path: "a.txt", Ours: []byte(ours), Theirs: []byte(theirs)}, LLMOptions{})+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(p.reqs[0].Prompt, "\r") || !strings.Contains(string(res.Content), "merged\r\n") {+ t.Fatalf("add/add CRLF: prompt has CR %v, content %q", strings.Contains(p.reqs[0].Prompt, "\r"), res.Content)+ }++ mixed := strings.Replace(ours, "line 3\r\n", "line 3\n", 1)+ p = llmFixed("merged\n")+ _, _ = ResolveLLM(context.Background(), p, LLMFile{Path: "a.txt", Ours: []byte(mixed), Theirs: []byte(theirs)}, LLMOptions{})+ if len(p.reqs) == 0 || !strings.Contains(p.reqs[0].Prompt, "\r") {+ t.Fatal("a file with mixed line endings was not sent as it is")+ }++ for _, tc := range []struct {+ sides [][]byte+ want string+ }{+ {[][]byte{[]byte("a\nb\n"), nil, []byte("c\n")}, LineEndingsLF},+ {[][]byte{[]byte("a\r\nb\r\n"), nil, []byte("c\r\nd")}, LineEndingsCRLF},+ {[][]byte{[]byte("a\r\nb\n"), []byte("c\r\n")}, LineEndingsMixed},+ {[][]byte{[]byte("a\r\n"), []byte("c\n")}, LineEndingsMixed},+ {[][]byte{[]byte("no break"), nil}, LineEndingsLF},+ } {+ if got := LLMLineEndings(tc.sides...); got != tc.want {+ t.Fatalf("%q: %s, want %s", tc.sides, got, tc.want)+ }+ }+}++// TestResolveLLMRecordsRejectedAnswers: the raw text of a rejected answer reaches the+// LLMRegionStats of the context, with why it was rejected.+func TestResolveLLMRecordsRejectedAnswers(t *testing.T) {+ stats := &LLMRegionStats{}+ f := LLMFile{Path: "a.txt", Base: []byte("base\n"), Ours: []byte("ours\n"), Theirs: []byte("theirs\n")}+ raw := &llmFake{answer: func(int, llm.Request) (*llm.Response, error) { return llmRawAnswer("no block at all"), nil }}+ _, err := ResolveLLM(WithLLMRegionStats(context.Background(), stats), raw, f, LLMOptions{})+ if !errors.Is(err, ErrLLMRejected) {+ t.Fatalf("err = %v", err)+ }+ got := stats.Rejected()+ if len(got) != 1 || !strings.Contains(got[0], "no block at all") || !strings.Contains(got[0], "has no patched code") {+ t.Fatalf("rejected = %q", got)+ }+}diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.goindex 802d836..5f43c9f 100644--- a/tools/graft-agents/resolve/llm_patch.go+++ b/tools/graft-agents/resolve/llm_patch.go@@ -398,7 +398,9 @@ func LLMConflictRegions(base, ours, theirs []byte) int {// MaxRegions conflicting regions, or a region whose request is over MaxPromptBytes, is declined// before any call. The file is resolved only when every region is: the first rejected region// rejects the file, and no partial resolution is returned.-func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptions, nonce string) (LLMResult, error) {+// crlf says that the file was CRLF and is sent as LF (ResolveLLM): CRLF in an answer is read as LF.+// The raw text of a rejected answer goes to the LLMRegionStats of ctx, if any, for audit.+func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptions, nonce string, crlf bool) (LLMResult, error) {base := llmSplitLines(f.Base)oursEdits, err := llmDiff(base, llmSplitLines(f.Ours), defaultLLMMaxDiffCells)if err != nil {@@ -451,16 +453,26 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptiif err != nil {return LLMResult{}, err}+ rejected := func(err error) error {+ err = fmt.Errorf("region %d: %w", k+1, err)+ if stats := llmRegionStatsFrom(ctx); stats != nil {+ stats.RecordRejected(fmt.Sprintf("%v\n----- raw answer -----\n%s", err, resp.Text))+ }+ return err+ }answer, err := llmPatchAnswer(resp.Text, nonce)if err != nil {- return LLMResult{}, fmt.Errorf("region %d: %w", k+1, err)+ return LLMResult{}, rejected(err)+ }+ if crlf {+ answer = strings.ReplaceAll(answer, "\r\n", "\n")}region, err := llmCheckPatched(answer, material[k], r.ours, r.theirs)if err != nil {- return LLMResult{}, fmt.Errorf("region %d: %w", k+1, err)+ return LLMResult{}, rejected(err)}if err := llmCheckAnswer(region, opt); err != nil {- return LLMResult{}, fmt.Errorf("region %d: %w", k+1, err)+ return LLMResult{}, rejected(err)}r.answer = regionif region == r.ours {