Skip to content

Git designed for AI agents.

graft: llm strategy option repair_composition (false = repair only a broken answer), to be measured

Federico Elia7aca1a284548
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.md
index 95f474f..70d66f2 100644
--- a/doc/graft/RESOLVE.md
+++ b/doc/graft/RESOLVE.md
@@ -852,6 +852,11 @@ own on the `[[strategy]]` table, on by default (PLAN10 15.10):
(the composition check), the first syntax error of the file mapped to its
region. A region is asked again once at most; the repaired answer is kept if
it passes the rules and has fewer findings.
+- `repair_composition`: with `false` the repair round is spent only on an answer
+ that is broken (a rule it breaks, a syntax error of the assembled file) and
+ never on an accepted one whose composition check found a drop or a duplicate;
+ the finding is only reported. No effect when `repair` is `false`. It exists to
+ be measured (PLAN10 15.10) and is removed or becomes the rule once it is.
A proposal also carries the composition report of its answer (what it keeps,
drops and repeats of the two sides' changes, in exact words), as the start of
diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.go
index 95ba381..b54e621 100644
--- a/tools/graft-agents/resolve/llm_core.go
+++ b/tools/graft-agents/resolve/llm_core.go
@@ -81,6 +81,11 @@ type LLMOptions struct {
NoAbstain bool
// NoRepair makes no second call: an answer that the deterministic checks reject is rejected.
NoRepair bool
+ // NoCompositionRepair leaves an answer that the rules and the syntax accept alone: the repair
+ // round is then spent only on an answer that is broken (a rule it breaks, a syntax error of the
+ // assembled file), never on a drop or a duplicate that the composition check found in an
+ // accepted one. Without effect when NoRepair is set.
+ NoCompositionRepair bool
// MaxContextBytes caps the symmetric context of a region. Zero means the default.
MaxContextBytes int
// NoGitRegions takes the regions from our own line diff and not from git's merge (llmFileRegions).
diff --git a/tools/graft-agents/resolve/llm_design_test.go b/tools/graft-agents/resolve/llm_design_test.go
index 2c7688b..35bb4cc 100644
--- a/tools/graft-agents/resolve/llm_design_test.go
+++ b/tools/graft-agents/resolve/llm_design_test.go
@@ -338,6 +338,24 @@ func TestLLMDesignRepairOnTheCompositionAndTheReport(t *testing.T) {
if err != nil || len(none.reqs) != 1 || res.Repairs != 0 || !strings.Contains(strings.Join(res.Report, "\n"), drop) {
t.Fatalf("err %v, calls %d, report %q", err, len(none.reqs), res.Report)
}
+
+ // An accepted answer is left alone when the repair is limited to broken answers: the finding is
+ // reported, nothing is asked again; a broken answer is still repaired.
+ only := &llmFake{answer: func(int, llm.Request) (*llm.Response, error) { return llmFakeAnswer("O5\nline 6\n"), nil }}
+ res, err = ResolveLLM(context.Background(), only, f, LLMOptions{NoCompositionRepair: true})
+ if err != nil || len(only.reqs) != 1 || res.Repairs != 0 || !strings.Contains(strings.Join(res.Report, "\n"), drop) || !strings.Contains(string(res.Content), "O5\nline 6\n") {
+ t.Fatalf("err %v, calls %d, repairs %d, report %q", err, len(only.reqs), res.Repairs, res.Report)
+ }
+ brokenFirst := &llmFake{answer: func(i int, _ llm.Request) (*llm.Response, error) {
+ if i == 0 {
+ return llmFakeAnswer("<<<<<<< markers are not an answer\n"), nil
+ }
+ return llmFakeAnswer("O5\nT6\n"), nil
+ }}
+ res, err = ResolveLLM(context.Background(), brokenFirst, f, LLMOptions{NoCompositionRepair: true})
+ if err != nil || len(brokenFirst.reqs) != 2 || res.Repairs != 1 || !strings.Contains(string(res.Content), "O5\nT6\n") {
+ t.Fatalf("a broken answer was not repaired: err %v, calls %d", err, len(brokenFirst.reqs))
+ }
}
// TestLLMComposition: the check on regions, per case of its definition (llm_compose.go).
@@ -453,10 +471,11 @@ func TestLLMDesignAtTheResolver(t *testing.T) {
// The options of the design step come from the strategy table, on unless switched off.
for key, check := range map[string]func(LLMOptions) bool{
- "symmetric_context": func(o LLMOptions) bool { return o.NoSymmetricContext },
- "abstain": func(o LLMOptions) bool { return o.NoAbstain },
- "repair": func(o LLMOptions) bool { return o.NoRepair },
- "git_regions": func(o LLMOptions) bool { return o.NoGitRegions },
+ "symmetric_context": func(o LLMOptions) bool { return o.NoSymmetricContext },
+ "abstain": func(o LLMOptions) bool { return o.NoAbstain },
+ "repair": func(o LLMOptions) bool { return o.NoRepair },
+ "repair_composition": func(o LLMOptions) bool { return o.NoCompositionRepair },
+ "git_regions": func(o LLMOptions) bool { return o.NoGitRegions },
} {
cfg := strings.Replace(wp19LLMConfig, `paths = ["*.txt", "**/*.txt"]`, "paths = [\"*.txt\"]\n"+key+" = false", 1)
s := wp19BuildStrategy(t, cfg)
diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.go
index 8af2e7a..8f0a933 100644
--- a/tools/graft-agents/resolve/llm_patch.go
+++ b/tools/graft-agents/resolve/llm_patch.go
@@ -715,7 +715,7 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOpti
}
r.answer = region
comps[k] = llmCompose(r, r.start, region)
- if !opt.NoRepair && !repaired[k] && len(comps[k].problems) > 0 {
+ if !opt.NoRepair && !opt.NoCompositionRepair && !repaired[k] && len(comps[k].problems) > 0 {
// The composition found a drop or a duplicate: one repair round, with the findings. The
// repaired answer replaces the first only if it is valid and has fewer findings.
_, _, region2, reason2, err2 := repair(k, material[k].head+region+material[k].tail, comps[k].problems)
diff --git a/tools/graft-agents/resolve/llm_strategy.go b/tools/graft-agents/resolve/llm_strategy.go
index 04e73d5..b06bced 100644
--- a/tools/graft-agents/resolve/llm_strategy.go
+++ b/tools/graft-agents/resolve/llm_strategy.go
@@ -27,6 +27,7 @@ import (
// symmetric_context = true # optional: OURS' change and intent beside THEIRS' (PLAN10 15.10)
// abstain = true # optional: the model may answer that the two changes cannot both hold
// repair = true # optional: one repair round, driven by deterministic findings
+// repair_composition = true # optional: false = repair only a broken answer, never an accepted one
// git_regions = true # optional: the regions are git's own merge (git merge-file), not our diff
// max_context_bytes = 4096 # optional: cap on the symmetric context of a region
// call_timeout = "120s" # optional: bound on one model call (one region)
@@ -85,7 +86,7 @@ func newLLMStrategy(spec StrategySpec) (Strategy, error) {
if _, ok := spec.Options["min_confidence"]; ok {
return nil, fmt.Errorf("strategy %q: min_confidence is gone: the model reports no confidence and an answer is accepted on the checks alone; remove the option", spec.Name)
}
- if err := allowOnly(spec, "llm", "max_prompt_bytes", "context_lines", "max_answer_bytes", "max_regions", "call_timeout", "symmetric_context", "abstain", "repair", "git_regions", "max_context_bytes"); err != nil {
+ if err := allowOnly(spec, "llm", "max_prompt_bytes", "context_lines", "max_answer_bytes", "max_regions", "call_timeout", "symmetric_context", "abstain", "repair", "repair_composition", "git_regions", "max_context_bytes"); err != nil {
return nil, err
}
@@ -107,7 +108,7 @@ func newLLMStrategy(spec StrategySpec) (Strategy, error) {
for _, o := range []struct {
key string
off *bool
- }{{"symmetric_context", &opt.NoSymmetricContext}, {"abstain", &opt.NoAbstain}, {"repair", &opt.NoRepair}, {"git_regions", &opt.NoGitRegions}} {
+ }{{"symmetric_context", &opt.NoSymmetricContext}, {"abstain", &opt.NoAbstain}, {"repair", &opt.NoRepair}, {"repair_composition", &opt.NoCompositionRepair}, {"git_regions", &opt.NoGitRegions}} {
if v, ok := spec.Options[o.key]; ok {
on, isBool := v.(bool)
if !isBool {