Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(bench): preserve complete agent patches against the original task base (#2095, from #2063)

xerj-orgacfa29562df6
diff --git a/tools/xerj-code/swebench/README.md b/tools/xerj-code/swebench/README.md
index 4df0d2206..67e18e95f 100644
--- a/tools/xerj-code/swebench/README.md
+++ b/tools/xerj-code/swebench/README.md
@@ -89,4 +89,6 @@ alongside the **token/cost delta** from the `--tokens-out` logs.
## Files
- `gen_predictions.py` — generates the `native`/`xerj` prediction JSONLs (+ token logs).
+ Patches compare with the original task base and include staged, committed and
+ new non-ignored files, including binary changes; ignored artifacts stay out.
- scoring — the official `swebench.harness.run_evaluation` (not vendored here on purpose).
diff --git a/tools/xerj-code/swebench/gen_predictions.py b/tools/xerj-code/swebench/gen_predictions.py
index b42e4df53..51ef2614e 100644
--- a/tools/xerj-code/swebench/gen_predictions.py
+++ b/tools/xerj-code/swebench/gen_predictions.py
@@ -58,6 +58,8 @@ def retrieve_context(repo_dir, instance_id, problem, k=6, chars=8000):
def run_agent(repo_dir, problem, ref_block, timeout, max_turns):
+ base = sh(["git", "rev-parse", "HEAD"], cwd=repo_dir)
+ base.check_returncode()
ref = ""
if ref_block:
ref = ("\n\nXERJ retrieved these passages from THIS repository — the code you "
@@ -83,8 +85,12 @@ def run_agent(repo_dir, problem, ref_block, timeout, max_turns):
model = list(d.get("modelUsage", {}).keys())
except Exception:
pass
- diff = sh(["git", "diff"], cwd=repo_dir).stdout
- return diff, usage, model, el
+ # This is a disposable task clone. Include new files without ignoring
+ # staged edits or commits the agent made relative to the pinned base.
+ sh(["git", "add", "--intent-to-add", "--all"], cwd=repo_dir).check_returncode()
+ diff = sh(["git", "diff", "--binary", base.stdout.strip()], cwd=repo_dir)
+ diff.check_returncode()
+ return diff.stdout, usage, model, el
def main():
diff --git a/tools/xerj-code/swebench/test_patch_capture.py b/tools/xerj-code/swebench/test_patch_capture.py
new file mode 100644
index 000000000..ebe73e5f2
--- /dev/null
+++ b/tools/xerj-code/swebench/test_patch_capture.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Git patch round trips with a recording agent adapter, no paid model calls."""
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import gen_predictions
+
+_REAL_RUN = subprocess.run
+
+
+def git(repo, *args, **kwargs):
+ return _REAL_RUN(["git", *args], cwd=repo, check=True,
+ capture_output=True, text=True, **kwargs).stdout.strip()
+
+
+class PatchCaptureTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.repo = self.root / "repo"
+ self.repo.mkdir()
+ git(self.repo, "init", "--quiet")
+ git(self.repo, "config", "user.name", "Fixture")
+ git(self.repo, "config", "user.email", "[email protected]")
+ git(self.repo, "config", "core.hooksPath", ".git/hooks")
+ (self.repo / "existing.py").write_text("value = 'before'\n")
+ (self.repo / "removed.py").write_text("obsolete = True\n")
+ (self.repo / ".gitignore").write_text("ignored.log\n")
+ git(self.repo, "add", ".")
+ git(self.repo, "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "fixture")
+
+ def run_recording_agent(self, edit):
+ def agent_or_git(cmd, **kwargs):
+ if cmd[0] != "claude":
+ return _REAL_RUN(cmd, **kwargs)
+ edit()
+ return subprocess.CompletedProcess(cmd, 0, json.dumps({"usage": {}, "modelUsage": {"fixture": {}}}), "")
+
+ with patch.object(gen_predictions.subprocess, "run", agent_or_git):
+ return gen_predictions.run_agent(str(self.repo), "Fixture issue", None, 5, 1)[0]
+
+ def test_staged_new_and_deleted_source_round_trip_to_a_clean_base(self):
+ clean = self.root / "clean"
+ git(self.root, "clone", "--quiet", str(self.repo), str(clean))
+
+ def edit():
+ (self.repo / "existing.py").write_text("value = 'after'\n")
+ git(self.repo, "add", "existing.py")
+ (self.repo / "new_module.py").write_text("new_value = 42\n")
+ (self.repo / "data.bin").write_bytes(b"\x00fixture\xff")
+ (self.repo / "removed.py").unlink()
+
+ diff = self.run_recording_agent(edit)
+ git(clean, "apply", "--check", "-", input=diff)
+ git(clean, "apply", "-", input=diff)
+ self.assertEqual((clean / "existing.py").read_text(), "value = 'after'\n")
+ self.assertEqual((clean / "new_module.py").read_text(), "new_value = 42\n")
+ self.assertFalse((clean / "removed.py").exists())
+ self.assertEqual((clean / "data.bin").read_bytes(), b"\x00fixture\xff")
+
+ def test_agent_commits_are_compared_with_the_original_base(self):
+ def edit():
+ (self.repo / "existing.py").write_text("value = 'committed fix'\n")
+ git(self.repo, "add", "existing.py")
+ git(self.repo, "-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "agent fixture")
+
+ diff = self.run_recording_agent(edit)
+ self.assertIn("value = 'committed fix'", diff)
+
+ def test_ignored_agent_artifacts_remain_outside_the_prediction(self):
+ def edit():
+ (self.repo / "existing.py").write_text("value = 'after'\n")
+ (self.repo / "ignored.log").write_text("fixture artifact\n")
+
+ diff = self.run_recording_agent(edit)
+ self.assertIn("value = 'after'", diff)
+ self.assertNotIn("ignored.log", diff)
+
+ def test_unchanged_agent_result_still_has_an_empty_patch(self):
+ self.assertEqual(self.run_recording_agent(lambda: None), "")
+
+
+if __name__ == "__main__":
+ unittest.main()