Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

Contextbench evaluation for leann (#313)

Jane Hu886d56a44289
diff --git a/benchmarks/contextbench/.gitignore b/benchmarks/contextbench/.gitignore
new file mode 100644
index 0000000..e914675
--- /dev/null
+++ b/benchmarks/contextbench/.gitignore
@@ -0,0 +1,25 @@
+.DS_Store
+__pycache__/
+*.py[cod]
+
+.venv/
+.eval-venv/
+.mitmproxy-venv/
+
+.leann/
+scripts/.leann/
+scripts/contextbench_eval_repos/
+scripts/contextbench_work_dir_claude/
+scripts/contextbench_work_dir_claude_overlap160/
+scripts/scripts/
+
+logs/
+traces/
+*.log
+*.jsonl
+eval_report.json
+contextbench_accuracy_report.json
+prepare_repos_with_leann_failures.jsonl
+task_metrics.jsonl
+task_session_usage.log
+all_predictions*.jsonl
diff --git a/benchmarks/contextbench/README.md b/benchmarks/contextbench/README.md
new file mode 100644
index 0000000..439ef2c
--- /dev/null
+++ b/benchmarks/contextbench/README.md
@@ -0,0 +1,103 @@
+# ContextBench LEANN Runner
+
+This directory keeps a small local runner around the upstream ContextBench repo.
+
+## Kept Files
+
+- `contextbench_official_repo/`: upstream ContextBench code and data.
+- `scripts/*.py`: local preparation, run, and evaluation scripts.
+- `mitmproxy_addons/trace_recorder.py`: HTTP trace recorder used while Claude runs.
+- `requirements-run.txt`: extra Python dependencies for these local scripts.
+
+Generated directories such as `.venv/`, `.mitmproxy-venv/`, `traces/`,
+`logs/`, `scripts/contextbench_work_dir_*`, and `scripts/contextbench_eval_repos/`
+can be deleted and regenerated.
+
+## 1. Create Python Environment
+
+Run from this directory:
+
+```bash
+python3.11 -m venv .venv
+source .venv/bin/activate
+pip install -r contextbench_official_repo/requirements.txt
+pip install -r requirements-run.txt
+```
+
+## 2. Install Runtime CLIs
+
+Install LEANN:
+
+```bash
+uv tool install leann-core --with leann
+```
+
+Install `mitmdump` in a separate environment:
+
+```bash
+python3.11 -m venv .mitmproxy-venv
+.mitmproxy-venv/bin/python -m pip install mitmproxy
+```
+
+The run script also expects:
+
+- `claude` CLI available on `PATH`.
+- Node/npm available for `npx ccusage`.
+- A Claude login session or `ANTHROPIC_API_KEY` in the environment.
+- If using LEANN MCP mode, a Claude MCP server named `leann-server` or
+ `LEANN_MCP_SERVER`/`CLAUDE_MCP_CONFIG_PATH` configured accordingly.
+
+## 3. Prepare Repos And LEANN Indexes
+
+```bash
+cd scripts
+WORK_ROOT=contextbench_work_dir_claude python prepare_repos_with_leann.py
+```
+
+## 4. Run Selected Tasks
+
+```bash
+cd scripts
+LEANN_ENABLED=1 \
+WORK_ROOT=contextbench_work_dir_claude \
+OUTPUT_FILE=all_predictions_claude.jsonl \
+python batch_run_selected.py
+```
+
+Run without LEANN:
+
+```bash
+LEANN_ENABLED=0 \
+WORK_ROOT=contextbench_work_dir_claude \
+OUTPUT_FILE=all_predictions_claude_baseline.jsonl \
+python batch_run_selected.py
+```
+
+Run specific IDs without editing the script:
+
+```bash
+SELECTED_IDS=id1,id2 python batch_run_selected.py
+```
+
+## 5. Evaluate Results
+
+Context retrieval metrics:
+
+```bash
+cd ".../contextbench_official_repo"
+
+PYTHONPATH=. python -m contextbench.evaluate \
+ --gold data/full.parquet \
+ --pred "../scripts/all_predictions_claude.jsonl" \
+ --cache "../scripts/contextbench_eval_repos" \
+ --out "../scripts/contextbench_official_eval_claude.jsonl" \
+ 2>&1 | tee "../scripts/contextbench_official_eval_claude.log"
+```
+
+## 6. Clean Generated Files
+
+```bash
+rm -rf .venv .mitmproxy-venv .eval-venv .leann .pycache_tmp logs traces
+rm -rf scripts/.leann scripts/scripts
+rm -rf scripts/contextbench_eval_repos scripts/contextbench_work_dir_claude scripts/contextbench_work_dir_claude_overlap160
+```
diff --git a/benchmarks/contextbench/mitmproxy_addons/trace_recorder.py b/benchmarks/contextbench/mitmproxy_addons/trace_recorder.py
new file mode 100644
index 0000000..942bdfd
--- /dev/null
+++ b/benchmarks/contextbench/mitmproxy_addons/trace_recorder.py
@@ -0,0 +1,72 @@
+import json
+import os
+import time
+from pathlib import Path
+
+from mitmproxy import http
+
+TASK_ID = os.environ.get("TASK_INSTANCE", "unknown_task")
+TRACE_DIR = Path(
+ os.environ.get("TRACE_DIR", Path(__file__).resolve().parents[1] / "traces" / "raw")
+)
+TRACE_DIR.mkdir(parents=True, exist_ok=True)
+HOST_FILTER = os.environ.get("TRACE_HOST_FILTER", "").strip().lower()
+
+OUTPUT_FILE = TRACE_DIR / f"{TASK_ID}_trace.jsonl"
+
+
+def _should_record(flow: http.HTTPFlow) -> bool:
+ host = (flow.request.host or "").lower()
+
+ # filter out telemetry traffic
+ if "statsig" in host:
+ return False
+
+ if HOST_FILTER and HOST_FILTER not in host:
+ return False
+
+ return True
+
+
+def request(flow: http.HTTPFlow):
+ if "statsig" in flow.request.pretty_host:
+ flow.response = http.Response.make(204)
+ return
+
+
+def response(flow: http.HTTPFlow):
+ should = _should_record(flow)
+
+ if not should:
+ return
+
+ if flow.response and flow.response.stream:
+ flow.response.stream = False
+
+ try:
+ entry = {
+ "task_id": TASK_ID,
+ "timestamp": time.time(),
+ "id": flow.id,
+ "request": {
+ "method": flow.request.method,
+ "url": flow.request.pretty_url,
+ "headers": dict(flow.request.headers),
+ "text": flow.request.get_text(strict=False) or "",
+ },
+ "response": {
+ "status_code": flow.response.status_code if flow.response else None,
+ "headers": dict(flow.response.headers) if flow.response else {},
+ "text": flow.response.get_text(strict=False) if flow.response else "",
+ },
+ }
+
+ with open(OUTPUT_FILE, "a", encoding="utf-8") as f:
+ f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+ except Exception as e:
+ print(f"DEBUG: Error recording trace: {e}")
+ import traceback
+
+ traceback.print_exc()
+ pass
diff --git a/benchmarks/contextbench/scripts/auto_run.py b/benchmarks/contextbench/scripts/auto_run.py
new file mode 100644
index 0000000..7a456e8
--- /dev/null
+++ b/benchmarks/contextbench/scripts/auto_run.py
@@ -0,0 +1,1034 @@
+import json
+import os
+import re
+import shutil
+import signal
+import socket
+import subprocess
+import time
+from pathlib import Path
+from typing import Optional
+
+import pexpect
+from datasets import load_dataset
+from git import Repo
+
+DATASET_NAME = os.environ.get("DATASET_NAME", "Contextbench/ContextBench")
+DATASET_SPLIT = os.environ.get("DATASET_SPLIT", "train")
+
+
+# Parse integer environment variables with validation and defaults.
+def _get_int_env(name: str, default: int, min_value: int = 1) -> int:
+ raw = os.environ.get(name, "").strip()
+ if not raw:
+ return default
+ try:
+ value = int(raw)
+ except ValueError:
+ print(f"⚠️ Invalid {name}={raw!r}; falling back to {default}")
+ return default
+ if value < min_value:
+ print(f"⚠️ {name} must be >= {min_value}; falling back to {default}")
+ return default
+ return value
+
+
+LEANN_TOP_K = _get_int_env("LEANN_TOP_K", 5, min_value=1)
+USAGE_LOG_FILE = os.environ.get("USAGE_LOG_FILE", "task_session_usage.log").strip()
+TASK_METRICS_FILE = os.environ.get("TASK_METRICS_FILE", "task_metrics.jsonl").strip()
+PREFETCH_STRICT = os.environ.get("PREFETCH_STRICT", "1").strip() != "0"
+
+
+# Get uncommitted changes compared to the current HEAD commit.
+def get_git_diff(repo_dir: Path) -> str:
+ try:
+ repo = Repo(repo_dir)
+ return repo.git.diff(repo.head.commit)
+ except Exception as e:
+ print(f"⚠️ Diff Error: {e}")
+ return ""
+
+
+# Prepare repository state for one ContextBench task instance.
+def setup_task_environment(
+ instance_id: str, repo_url: str, work_root: Path, task: Optional[dict] = None
+) -> Path:
+ print(f"🚀 [1/4] Preparing environment: {instance_id}")
+ if not task:
+ ds = load_dataset(DATASET_NAME, split=DATASET_SPLIT)
+ task = next((x for x in ds if x["instance_id"] == instance_id), None)
+ if not task:
+ raise RuntimeError(f"Instance {instance_id} not found in dataset.")
+
+ target_dir = work_root / instance_id
+ if not target_dir.exists():
+ Repo.clone_from(repo_url, target_dir)
+
+ repo = Repo(target_dir)
+ repo.git.reset("--hard")
+ # Preserve LEANN indexes if present.
+ repo.git.clean("-fdx", "-e", ".leann/")
+ repo.git.checkout(task["base_commit"])
+ (target_dir / "PROBLEM.md").write_text(task["problem_statement"], encoding="utf-8")
+ return target_dir
+
+
+# Pre-download all task repositories before model execution starts.
+def prefetch_task_repositories(tasks: list[dict], work_root: Path) -> None:
+ if not tasks:
+ print("📦 [Prefetch] No tasks to download.")
+ return
+
+ print(f"📦 [Prefetch] Downloading repositories for {len(tasks)} tasks...")
+ failures: list[str] = []
+
+ for i, task in enumerate(tasks, start=1):
+ instance_id = task.get("instance_id", "").strip()
+ repo_url = task.get("repo_url", "").strip()
+ target_dir = work_root / instance_id
+
+ if not instance_id or not repo_url:
+ failures.append(f"task[{i}] missing instance_id/repo_url")
+ print(f" [{i}/{len(tasks)}] ❌ Invalid task metadata; skipping.")
+ continue
+
+ if target_dir.exists():
+ # A previous interrupted clone can leave a broken .git dir.
+ # Treat only repos with a valid HEAD as reusable.
+ try:
+ repo = Repo(target_dir)
+ repo.git.rev_parse("HEAD")
+ print(f" [{i}/{len(tasks)}] ✅ Already present: {instance_id}")
+ continue
+ except Exception:
+ print(f" [{i}/{len(tasks)}] ♻️ Found incomplete repo, re-cloning: {instance_id}")
+ shutil.rmtree(target_dir, ignore_errors=True)
+
+ cloned = False
+ last_error: Optional[Exception] = None
+ for attempt in range(1, 3 + 1):
+ try:
+ print(f" [{i}/{len(tasks)}] ⬇️ Cloning: {instance_id} (attempt {attempt}/3)")
+ Repo.clone_from(repo_url, target_dir)
+ print(f" [{i}/{len(tasks)}] ✅ Cloned: {instance_id}")
+ cloned = True
+ break
+ except Exception as e:
+ last_error = e
+ print(f" [{i}/{len(tasks)}] ❌ Clone attempt failed: {instance_id} -> {e}")
+ shutil.rmtree(target_dir, ignore_errors=True)
+ if attempt < 3:
+ sleep_s = 4 * attempt
+ print(f" [{i}/{len(tasks)}] ⏳ Retrying in {sleep_s}s...")
+ time.sleep(sleep_s)
+
+ if not cloned:
+ failures.append(f"{instance_id} ({last_error})")
+
+ if failures:
+ sample = ", ".join(failures[:3])
+ extra = f" (+{len(failures) - 3} more)" if len(failures) > 3 else ""
+ message = f"Repository prefetch failed for {len(failures)} task(s): {sample}{extra}"
+ if PREFETCH_STRICT:
+ raise RuntimeError(message)
+ print(f"⚠️ {message}")
+ print("⚠️ PREFETCH_STRICT=0; continuing with available repositories.")
+ print("⚠️ [Prefetch] Completed with partial failures.")
+ return
+
+ print("✅ [Prefetch] All repositories are ready.")
+
+
+# Wait until a local TCP port starts accepting connections.
+def wait_for_port(port: int, timeout: int = 10):
+ start = time.time()
+ while time.time() - start < timeout:
+ try:
+ with socket.create_connection(("127.0.0.1", port), timeout=0.2):
+ return
+ except OSError:
+ time.sleep(0.1)
+ print(f"⚠️ Warning: Proxy port {port} did not open in {timeout}s")
+
+
+# Resolve mitmproxy CA cert path for Python HTTPS clients behind proxy.
+def _resolve_mitm_ca_cert() -> Optional[Path]:
+ env_cert = os.environ.get("MITM_CA_CERT_PATH", "").strip()
+ candidates = []
+ if env_cert:
+ candidates.append(Path(env_cert).expanduser())
+ candidates.extend(
+ [
+ Path.home() / ".mitmproxy" / "mitmproxy-ca-cert.pem",
+ Path.home() / ".mitmproxy" / "mitmproxy-ca-cert.cer",
+ ]
+ )
+ for cert_path in candidates:
+ if cert_path.exists():
+ return cert_path
+ return None
+
+
+def _resolve_mitmdump_bin() -> str:
+ env_bin = os.environ.get("MITMDUMP_BIN", "").strip()
+ if env_bin:
+ return env_bin
+
+ # Prefer a dedicated local mitmproxy venv to avoid dependency conflicts
+ # with the main project environment.
+ project_root = Path(__file__).resolve().parents[1]
+ local_bin = project_root / ".mitmproxy-venv" / "bin" / "mitmdump"
+ if local_bin.exists():
+ return str(local_bin)
+
+ return "mitmdump"
+
+
+# Start mitmproxy traffic recorder for the current task.
+def start_mitmproxy(instance_id: str, mitm_script_path: str) -> subprocess.Popen:
+ print("🕵️ [2/4] Starting Traffic Recorder...")
+ env = os.environ.copy()
+ env["TASK_INSTANCE"] = instance_id
+ mitmdump_bin = _resolve_mitmdump_bin()
+ cmd = [mitmdump_bin, "--no-http2", "-s", mitm_script_path, "-q"]
+ try:
+ process = subprocess.Popen(cmd, env=env, preexec_fn=os.setsid)
+ except FileNotFoundError as exc:
+ raise RuntimeError(
+ "mitmdump not found. Install mitmproxy in a dedicated env with:\n"
+ " python3 -m venv .mitmproxy-venv\n"
+ " .mitmproxy-venv/bin/python -m pip install mitmproxy\n"
+ "or set MITMDUMP_BIN to an existing mitmdump path."
+ ) from exc
+ wait_for_port(8080)
+ return process
+
+
+# Check whether a server name exists in dict/list MCP server containers.
+def _server_in_container(servers_obj, server_name: str) -> bool:
+ if isinstance(servers_obj, dict):
+ return server_name in servers_obj
+ if isinstance(servers_obj, list):
+ for item in servers_obj:
+ if isinstance(item, dict) and item.get("name") == server_name:
+ return True
+ return False
+
+
+# Recursively scan a config tree for a specific MCP server entry.
+def _has_mcp_server_config(node, server_name: str, parent_key: str = "") -> bool:
+ if isinstance(node, dict):
+ for key, value in node.items():
+ key_l = key.lower()
+ if key_l in {"mcpservers", "mcp_servers"} and _server_in_container(value, server_name):
+ return True
+ if (
+ parent_key.lower() == "mcp"
+ and key_l == "servers"
+ and _server_in_container(value, server_name)
+ ):
+ return True
+ if _has_mcp_server_config(value, server_name, key):
+ return True
+ elif isinstance(node, list):
+ for item in node:
+ if _has_mcp_server_config(item, server_name, parent_key):
+ return True
+ return False
+
+
+# Resolve Claude settings path that defines the required MCP server.
+def resolve_claude_mcp_config(server_name: str) -> Optional[Path]:
+ env_path = os.environ.get("CLAUDE_MCP_CONFIG_PATH", "").strip()
+ candidates = []
+ if env_path:
+ candidates.append(Path(env_path).expanduser())
+ candidates.extend(
+ [
+ Path.home() / ".claude" / "settings.json",
+ Path.home() / ".config" / "claude" / "settings.json",
+ Path.home() / ".config" / "claude-code" / "settings.json",
+ Path.home() / ".claude.json",
+ ]
+ )
+
+ seen = set()
+ for cfg_path in candidates:
+ cfg_path = cfg_path.resolve()
+ if str(cfg_path) in seen:
+ continue
+ seen.add(str(cfg_path))
+ if not cfg_path.exists():
+ continue
+ try:
+ data = json.loads(cfg_path.read_text(encoding="utf-8"))
+ except Exception:
+ continue
+ if _has_mcp_server_config(data, server_name):
+ return cfg_path
+ return None
+
+
+def _normalize_mcp_servers(servers_obj) -> dict[str, dict]:
+ servers: dict[str, dict] = {}
+ if isinstance(servers_obj, dict):
+ for name, config in servers_obj.items():
+ if isinstance(name, str) and isinstance(config, dict):
+ servers[name] = config
+ elif isinstance(servers_obj, list):
+ for item in servers_obj:
+ if not isinstance(item, dict):
+ continue
+ name = item.get("name")
+ if not isinstance(name, str) or not name:
+ continue
+ config = dict(item)
+ config.pop("name", None)
+ servers[name] = config
+ return servers
+
+
+def _extract_mcp_servers(node, parent_key: str = "") -> dict[str, dict]:
+ servers: dict[str, dict] = {}
+ if isinstance(node, dict):
+ for key, value in node.items():
+ key_l = key.lower()
+ if key_l in {"mcpservers", "mcp_servers"}:
+ servers.update(_normalize_mcp_servers(value))
+ continue
+ if parent_key.lower() == "mcp" and key_l == "servers":
+ servers.update(_normalize_mcp_servers(value))
+ continue
+ servers.update(_extract_mcp_servers(value, key))
+ elif isinstance(node, list):
+ for item in node:
+ servers.update(_extract_mcp_servers(item, parent_key))
+ return servers
+
+
+def build_strict_mcp_config_without_server(server_name: str) -> Optional[str]:
+ cfg_path = resolve_claude_mcp_config(server_name)
+ if not cfg_path:
+ return None
+ try:
+ data = json.loads(cfg_path.read_text(encoding="utf-8"))
+ except Exception:
+ return None
+ servers = _extract_mcp_servers(data)
+ servers.pop(server_name, None)
+ return json.dumps({"mcpServers": servers}, separators=(",", ":"))
+
+
+# Decide whether LEANN/MCP integration is available for this task repo.
+def resolve_leann_integration(target_dir: Path) -> dict[str, str]:
+ leann_enabled = os.environ.get("LEANN_ENABLED", "1") != "0"
+ use_mcp = os.environ.get("LEANN_USE_MCP", "1") != "0"
+ mcp_server_name = os.environ.get("LEANN_MCP_SERVER", "leann-server")
+ leann_index_exists = (target_dir / ".leann").exists()
+
+ mode = "none"
+ if leann_enabled and leann_index_exists:
+ if use_mcp:
+ mode = "mcp"
+ print(
+ f" -> 🔍 LEANN MCP enabled (server: {mcp_server_name}, "
+ f"forced top_k={LEANN_TOP_K})"
+ )
+ else:
+ print(" -> ⚠️ LEANN_USE_MCP=0 but CLI mode is disabled; continuing without LEANN")
+ elif leann_enabled:
+ print(" -> ⚠️ LEANN enabled but no .leann index found; continuing without LEANN")
+
+ print(mode)
+ return {
+ "mode": mode,
+ "mcp_server_name": mcp_server_name,
+ }
+
+
+# Build the initial Claude prompt from PROBLEM.md plus optional LEANN hints.
+def build_initial_prompt(
+ target_dir: Path, leann_info: dict[str, str], instance_id: str = ""
+) -> str:
+ problem_file = target_dir / "PROBLEM.md"
+ try:
+ problem_text = problem_file.read_text(encoding="utf-8").strip()
+ if leann_info.get("mode") == "mcp":
+ leann_info.get("mcp_server_name", "leann-server")
+ mcp_hint = (
+ f"Your LEANN index name is: {instance_id}\n"
+ "Use LEANN MCP as a cost-aware semantic entry-point router. "
+ "Before broad exploration, call leann_search once. "
+ "Use top_k=5 by default, top_k=10 only for clearly multi-hop tasks spanning multiple subsystems; never use top_k>10. "
+ "Use show_metadata=false.\n"
+ "After LEANN, open the top 1-3 likely implementation/source files first, preferring source files over tests/docs/examples/generated files. "
+ "Do not open many retrieved files just because they were returned. "
+ "After identifying the likely fix location, use targeted Grep to find existing callers, API/server entry points, tests, and config files that may need to be updated. "
+ "This targeted Grep is preferred over additional LEANN once concrete symbols, functions, file paths, config keys, or error strings are known.\n"
+ "Run at most one additional leann_search only if no plausible implementation entry point is found or a required subsystem is clearly missing. "
+ "The second query must be more specific, using concrete identifiers/literals found so far. "
+ "Use show_metadata=false unless metadata is necessary to disambiguate files. "
+ "Never run more than two leann_search calls total.\n"
+ )
+ return f"{mcp_hint}\n\n{problem_text}"
+ return problem_text
+ except Exception as e:
+ raise RuntimeError(f"Failed to read problem statement from {problem_file}: {e}")
+
+
+# Run Claude CLI autonomously in the prepared repository.
+def run_claude_autonomous(
+ target_dir: Path,
+ model: str,
+ leann_info: Optional[dict[str, str]] = None,
+ instance_id: str = "",
+):
+ print("🤖 [3/4] Launching Claude Code")
+ if (target_dir / ".claude").exists():
+ shutil.rmtree(target_dir / ".claude")
+
+ if leann_info is None:
+ leann_info = resolve_leann_integration(target_dir)
+ server_name = leann_info.get("mcp_server_name", "leann-server")
+ if leann_info.get("mode") == "mcp":
+ cfg_path = resolve_claude_mcp_config(server_name)
+ if not cfg_path:
+ raise RuntimeError(
+ f"LEANN_USE_MCP=1 but MCP server '{server_name}' was not found in Claude config. "
+ "Set CLAUDE_MCP_CONFIG_PATH or configure this server in Claude settings."
+ )
+ print(f" -> ✅ Claude MCP config found for '{server_name}': {cfg_path}")
+ initial_prompt = build_initial_prompt(target_dir, leann_info, instance_id=instance_id)
+
+ env = os.environ.copy()
+ env.update(
+ {
+ "HTTP_PROXY": "http://127.0.0.1:8080",
+ "HTTPS_PROXY": "http://127.0.0.1:8080",
+ "NODE_TLS_REJECT_UNAUTHORIZED": "0",
+ }
+ )
+
+ claude_args = ["-p", initial_prompt, "--dangerously-skip-permissions"]
+ if model:
+ claude_args.extend(["--model", model])
+ if leann_info.get("mode") != "mcp":
+ filtered_mcp_config = build_strict_mcp_config_without_server(server_name)
+ if filtered_mcp_config is not None:
+ claude_args.extend(
+ [
+ "--strict-mcp-config",
+ "--mcp-config",
+ filtered_mcp_config,
+ ]
+ )
+ print(
+ f" -> 🚫 Non-LEANN mode; removed MCP server '{server_name}' from strict MCP config"
+ )
+
+ child = pexpect.spawn(
+ "claude", claude_args, cwd=target_dir, env=env, encoding="utf-8", timeout=1200
+ )
+
+ try:
+ while True:
+ index = child.expect(
+ [r"\[y/n\]", r"Allow execution", pexpect.EOF, pexpect.TIMEOUT], timeout=5
+ )
+ if index in [0, 1]:
+ child.sendline("y")
+ elif index == 2:
+ break
+ except Exception as e:
+ print(f"❌ Interaction error: {e}")
+ finally:
+ if child.isalive():
+ child.terminate(force=True)
+
+
+# ---------------------------------------------------------------------------
+# Trajectory extraction: parse mitmproxy trace → ContextBench traj_data
+# ---------------------------------------------------------------------------
+
+
+def _parse_sse_tool_uses(text: str) -> list[dict]:
+ """Reconstruct tool_use calls from a streamed SSE response body."""
+ tool_blocks: dict[int, dict] = {} # index -> {name, partial_json}
+ completed: list[dict] = []
+
+ for line in text.split("\n"):
+ if not line.startswith("data: "):
+ continue
+ data_str = line[6:]
+ try:
+ event = json.loads(data_str)
+ except Exception:
+ continue
+
+ event_type = event.get("type", "")
+
+ if event_type == "content_block_start":
+ idx = event.get("index", 0)
+ block = event.get("content_block", {})
+ if block.get("type") == "tool_use":
+ tool_blocks[idx] = {"name": block.get("name", ""), "partial_json": ""}
+
+ elif event_type == "content_block_delta":
+ idx = event.get("index", 0)
+ delta = event.get("delta", {})
+ if idx in tool_blocks and delta.get("type") == "input_json_delta":
+ tool_blocks[idx]["partial_json"] += delta.get("partial_json", "")
+
+ elif event_type == "content_block_stop":
+ idx = event.get("index", 0)
+ if idx in tool_blocks:
+ tool = tool_blocks.pop(idx)
+ try:
+ input_data = json.loads(tool["partial_json"]) if tool["partial_json"] else {}
+ except Exception:
+ input_data = {}
+ completed.append({"name": tool["name"], "input": input_data})
+
+ return completed
+
+
+def _parse_json_tool_uses(text: str) -> list[dict]:
+ """Extract tool_use calls from a non-streamed JSON response body."""
+ try:
+ resp = json.loads(text)
+ except Exception:
+ return []
+ completed = []
+ for block in resp.get("content", []):
+ if block.get("type") == "tool_use":
+ completed.append({"name": block.get("name", ""), "input": block.get("input", {})})
+ return completed
+
+
+def _make_relative(file_path: str, target_dir: Path) -> str:
+ """Convert an absolute file path to one relative to target_dir."""
+ try:
+ return str(Path(file_path).relative_to(target_dir.resolve()))
+ except ValueError:
+ return file_path
+
+
+def _is_leann_search_tool(name: str) -> bool:
+ return bool(name) and (
+ name == "leann_search" or name.endswith("__leann_search") or "leann_search" in name
+ )
+
+
+def _extract_trace_artifacts(
+ trace_path: Path,
+ target_dir: Path,
+ since_timestamp: Optional[float] = None,
+) -> dict:
+ """
+ Parse the mitmproxy JSONL trace once and derive both ContextBench trajectory data
+ and LEANN usage statistics.
+
+ Each Anthropic /messages response that contains tool_use calls becomes one pred_step.
+ Read calls with line offsets are mapped to line spans; other file-touching tool calls
+ (Glob, Grep, Bash cat/head) contribute to pred_files only.
+ """
+ pred_steps: list[dict] = []
+ leann_search_calls = 0
+
+ if not trace_path.exists():
+ print(f"⚠️ Trace file not found: {trace_path}")
+ return {
+ "traj_data": _empty_traj_data(),
+ "leann_tool_used": False,
+ "leann_search_calls": 0,
+ }
+
+ with open(trace_path, encoding="utf-8") as f:
+ for raw_line in f:
+ try:
+ entry = json.loads(raw_line)
+ except Exception:
+ continue
+
+ ts = entry.get("timestamp")
+ if (
+ since_timestamp is not None
+ and isinstance(ts, (int, float))
+ and ts < since_timestamp
+ ):
+ continue
+
+ url = entry.get("request", {}).get("url", "")
+ if "anthropic.com" not in url or "/messages" not in url:
+ continue
+
+ response_text = entry.get("response", {}).get("text", "") or ""
+ if not response_text:
+ continue
+
+ # Support both streaming (SSE) and non-streaming responses.
+ if "data: " in response_text:
+ tool_uses = _parse_sse_tool_uses(response_text)
+ else:
+ tool_uses = _parse_json_tool_uses(response_text)
+
+ if not tool_uses:
+ continue
+
+ step_files: list[str] = []
+ step_spans: dict[str, list[dict]] = {}
+
+ for tool in tool_uses:
+ name = tool.get("name", "")
+ inp = tool.get("input", {}) or {}
+
+ if _is_leann_search_tool(name):
+ leann_search_calls += 1
+
+ if name == "Read":
+ raw_path = inp.get("file_path", "")
+ if not raw_path:
+ continue
+ rel = _make_relative(raw_path, target_dir)
+ if rel not in step_files:
+ step_files.append(rel)
+ offset = inp.get("offset")
+ limit = inp.get("limit")
+ if offset is not None:
+ start = int(offset)
+ end = start + int(limit) if limit is not None else start + 2000
+ step_spans.setdefault(rel, []).append({"start": start, "end": end})
+
+ elif name in ("Grep", "Glob"):
+ # These don't return exact spans; record any explicit path argument.
+ raw_path = inp.get("path", "")
+ if raw_path and not raw_path.endswith(("*", "/")):
+ rel = _make_relative(raw_path, target_dir)
+ if rel not in step_files:
+ step_files.append(rel)
+
+ elif name == "Bash":
+ # Heuristic: detect `cat`, `head`, `tail` calls on files.
+ cmd = inp.get("command", "") or ""
+ for m in re.finditer(
+ r"(?:cat|head|tail|sed|awk)\s+(?:-[^\s]+\s+)*([^\s|><;]+)", cmd
+ ):
+ candidate = m.group(1)
+ if "/" in candidate or candidate.endswith(
+ (".py", ".go", ".ts", ".js", ".java", ".c", ".cpp", ".rs")
+ ):
+ rel = _make_relative(candidate, target_dir)
+ if rel not in step_files:
+ step_files.append(rel)
+
+ if step_files:
+ pred_steps.append(
+ {
+ "files": step_files,
+ "spans": step_spans,
+ "symbols": {},
+ }
+ )
+
+ # Aggregate across all steps.
+ all_files: list[str] = []
+ all_spans: dict[str, list[dict]] = {}
+ for step in pred_steps:
+ for f in step["files"]:
+ if f not in all_files:
+ all_files.append(f)
+ for f, spans in step["spans"].items():
+ all_spans.setdefault(f, []).extend(spans)
+
+ return {
+ "traj_data": {
+ "pred_steps": pred_steps,
+ "pred_files": all_files,
+ "pred_spans": all_spans,
+ "pred_symbols": {},
+ },
+ "leann_tool_used": leann_search_calls > 0,
+ "leann_search_calls": leann_search_calls,
+ }
+
+
+def extract_trajectory_from_traces(
+ trace_path: Path,
+ target_dir: Path,
+ since_timestamp: Optional[float] = None,
+) -> dict:
+ return _extract_trace_artifacts(
+ trace_path,
+ target_dir,
+ since_timestamp=since_timestamp,
+ )["traj_data"]
+
+
+def detect_messages_api_error(
+ trace_path: Path,
+ since_timestamp: Optional[float] = None,
+) -> Optional[str]:
+ """Return a concise API error message if /v1/messages requests failed."""
+ if not trace_path.exists():
+ return None
+ try:
+ with open(trace_path, encoding="utf-8") as f:
+ for raw_line in f:
+ try:
+ entry = json.loads(raw_line)
+ except Exception:
+ continue
+ ts = entry.get("timestamp")
+ if (
+ since_timestamp is not None
+ and isinstance(ts, (int, float))
+ and ts < since_timestamp
+ ):
+ continue
+
+ url = entry.get("request", {}).get("url", "")
+ if "/v1/messages" not in url:
+ continue
+ response = entry.get("response", {}) or {}
+ status_code = response.get("status_code")
+ if not isinstance(status_code, int) or status_code < 400:
+ continue
+ text = response.get("text", "") or ""
+ err_type = ""
+ err_message = ""
+ try:
+ payload = json.loads(text)
+ error_obj = payload.get("error", {}) if isinstance(payload, dict) else {}
+ if isinstance(error_obj, dict):
+ err_type = str(error_obj.get("type", "") or "")
+ err_message = str(error_obj.get("message", "") or "")
+ except Exception:
+ pass
+ detail = f"/v1/messages returned HTTP {status_code}"
+ if err_type:
+ detail += f" ({err_type})"
+ if err_message:
+ detail += f": {err_message}"
+ return detail
+ except Exception:
+ return None
+ return None
+
+
+def _empty_traj_data() -> dict:
+ return {"pred_steps": [], "pred_files": [], "pred_spans": {}, "pred_symbols": {}}
+
+
+# ---------------------------------------------------------------------------
+# Usage tracking (mirrored from swebench runner)
+# ---------------------------------------------------------------------------
+
+
+def _blank_usage() -> dict:
+ return {
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "cache_creation_tokens": 0,
+ "cache_read_tokens": 0,
+ "total_tokens": 0,
+ "cost_usd": 0.0,
+ }
+
+
+def _normalize_usage(raw: Optional[dict]) -> dict:
+ base = _blank_usage()
+ if not isinstance(raw, dict):
+ return base
+ base["input_tokens"] = int(raw.get("inputTokens", 0) or 0)
+ base["output_tokens"] = int(raw.get("outputTokens", 0) or 0)
+ base["cache_creation_tokens"] = int(raw.get("cacheCreationTokens", 0) or 0)
+ base["cache_read_tokens"] = int(raw.get("cacheReadTokens", 0) or 0)
+ base["total_tokens"] = int(raw.get("totalTokens", 0) or 0)
+ base["cost_usd"] = float(raw.get("totalCost", 0) or 0.0)
+ return base
+
+
+def _usage_delta(before: dict, after: dict) -> dict:
+ return {
+ "input_tokens": after["input_tokens"] - before["input_tokens"],
+ "output_tokens": after["output_tokens"] - before["output_tokens"],
+ "cache_creation_tokens": after["cache_creation_tokens"] - before["cache_creation_tokens"],
+ "cache_read_tokens": after["cache_read_tokens"] - before["cache_read_tokens"],
+ "total_tokens": after["total_tokens"] - before["total_tokens"],
+ "cost_usd": round(after["cost_usd"] - before["cost_usd"], 6),
+ }
+
+
+def _has_positive_delta(usage: dict) -> bool:
+ return (
+ usage["total_tokens"] > 0
+ or usage["input_tokens"] > 0
+ or usage["output_tokens"] > 0
+ or usage["cache_creation_tokens"] > 0
+ or usage["cache_read_tokens"] > 0
+ or usage["cost_usd"] > 0
+ )
+
+
+def _format_usage(usage: dict) -> str:
+ return (
+ f"input: {usage['input_tokens']:,}, output: {usage['output_tokens']:,}, "
+ f"cache_read: {usage['cache_read_tokens']:,}, cache_create: {usage['cache_creation_tokens']:,} "
+ f"total: {usage['total_tokens']:,}, cost: ${usage['cost_usd']:.4f}"
+ )
+
+
+def append_usage_log(line: str) -> None:
+ if not USAGE_LOG_FILE:
+ return
+ log_path = Path(USAGE_LOG_FILE).expanduser()
+ try:
+ if log_path.parent and str(log_path.parent) != ".":
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+ with log_path.open("a", encoding="utf-8") as f:
+ f.write(f"{line}\n")
+ except Exception as e:
+ print(f"⚠️ Failed to write usage log {log_path}: {e}")
+
+
+def append_task_metrics(entry: dict) -> None:
+ if not TASK_METRICS_FILE:
+ return
+ metrics_path = Path(TASK_METRICS_FILE).expanduser()
+ try:
+ if metrics_path.parent and str(metrics_path.parent) != ".":
+ metrics_path.parent.mkdir(parents=True, exist_ok=True)
+ with metrics_path.open("a", encoding="utf-8") as f:
+ f.write(json.dumps(entry) + "\n")
+ except Exception as e:
+ print(f"⚠️ Failed to write task metrics {metrics_path}: {e}")
+
+
+def get_ccusage_sessions() -> Optional[dict[str, dict]]:
+ try:
+ result = subprocess.run(
+ ["npx", "ccusage", "session", "--json", "--offline"],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ if result.returncode != 0:
+ stderr = (result.stderr or "").strip()
+ raise RuntimeError(stderr or "ccusage returned non-zero exit code")
+ data = json.loads(result.stdout or "{}")
+ sessions = data.get("sessions", [])
+ if not isinstance(sessions, list):
+ return {}
+ by_session: dict[str, dict] = {}
+ for row in sessions:
+ if not isinstance(row, dict):
+ continue
+ session_id = str(row.get("sessionId", "")).strip()
+ if not session_id:
+ continue
+ by_session[session_id] = _normalize_usage(row)
+ return by_session
+ except Exception as e:
+ print(f"⚠️ Failed to get ccusage sessions: {e}")
+ return None
+
+
+def compute_usage_diff(
+ before_sessions: Optional[dict[str, dict]],
+ after_sessions: Optional[dict[str, dict]],
+ instance_id: str,
+) -> Optional[dict]:
+ if before_sessions is None or after_sessions is None:
+ return None
+
+ changed_sessions: dict[str, dict] = {}
+ all_session_ids = set(before_sessions.keys()) | set(after_sessions.keys())
+ for session_id in all_session_ids:
+ before = before_sessions.get(session_id, _blank_usage())
+ after = after_sessions.get(session_id, _blank_usage())
+ delta = _usage_delta(before, after)
+ if _has_positive_delta(delta):
+ changed_sessions[session_id] = delta
+
+ if not changed_sessions:
+ return None
+
+ instance_lower = instance_id.lower()
+ subagent_ids = [
+ sid for sid in changed_sessions if sid.lower() == "subagents" or "subagent" in sid.lower()
+ ]
+ non_subagent_items = [
+ (sid, usage) for sid, usage in changed_sessions.items() if sid not in subagent_ids
+ ]
+ matched_non_subagent = [
+ (sid, usage)
+ for sid, usage in non_subagent_items
+ if sid.lower() in instance_lower or instance_lower in sid.lower()
+ ]
+ primary_candidates = matched_non_subagent or non_subagent_items
+ if not primary_candidates:
+ return None
+
+ primary_session_id, session_usage = max(
+ primary_candidates, key=lambda x: (x[1]["total_tokens"], x[1]["cost_usd"])
+ )
+ return {
+ "session_id": primary_session_id,
+ "input_tokens": session_usage["input_tokens"],
+ "output_tokens": session_usage["output_tokens"],
+ "cache_creation_tokens": session_usage["cache_creation_tokens"],
+ "cache_read_tokens": session_usage["cache_read_tokens"],
+ "total_tokens": session_usage["total_tokens"],
+ "cost_usd": session_usage["cost_usd"],
+ "session_usage": session_usage,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Main task runner
+# ---------------------------------------------------------------------------
+
+
+def run_single_task(
+ instance_id: str,
+ repo_url: str,
+ work_root: Path,
+ mitm_script_path: str,
+ trace_dir: Path,
+ model: str = "",
+ task: Optional[dict] = None,
+) -> tuple:
+ """
+ Run one ContextBench instance end-to-end.
+
+ Returns: (patch, latency_seconds, agent_seconds, traj_data, usage)
+ - patch : git diff string (the model's code changes)
+ - latency_seconds : end-to-end wall-clock seconds
+ - agent_seconds : Claude agent runtime only
+ - traj_data : ContextBench trajectory dict (pred_steps, pred_files, …)
+ - usage : token/cost usage dict, or None
+ """
+ print(f"\n🌊 Starting Claude Pipeline for {instance_id}")
+ work_root = Path(work_root)
+ trace_dir = Path(trace_dir)
+ mitm_proc = None
+ start_time = time.time()
+ usage_before: Optional[dict[str, dict]] = None
+ usage: Optional[dict] = None
+ status = "failed"
+ error_message: Optional[str] = None
+ claude_start_ts: Optional[float] = None
+ claude_end_ts: Optional[float] = None
+ leann_mode: Optional[str] = None
+ leann_tool_used = False
+ leann_search_calls = 0
+
+ try:
+ target_dir = setup_task_environment(instance_id, repo_url, work_root, task=task)
+ leann_info = resolve_leann_integration(target_dir)
+ leann_mode = leann_info.get("mode")
+ mitm_proc = start_mitmproxy(instance_id, str(mitm_script_path))
+
+ usage_before = get_ccusage_sessions()
+ claude_start_ts = time.time()
+ run_claude_autonomous(target_dir, model, leann_info=leann_info, instance_id=instance_id)
+ claude_end_ts = time.time()
+ usage_after = get_ccusage_sessions()
+
+ trace_path = trace_dir / f"{instance_id}_trace.jsonl"
+ api_error = detect_messages_api_error(trace_path, since_timestamp=claude_start_ts)
+ if api_error:
+ raise RuntimeError(
+ f"Claude API request failed for {instance_id}: {api_error}. "
+ "Set MODEL/CLAUDE_MODEL to an available model for your account."
+ )
+
+ usage = compute_usage_diff(usage_before, usage_after, instance_id=instance_id)
+ if usage:
+ task_usage_line = (
+ f"💰 Task session usage ({usage['session_id']}) — "
+ f"{_format_usage(usage['session_usage'])}"
+ )
+ print(task_usage_line)
+
+ print(f"📍 [4/4] Extracting trajectory from trace: {trace_path.name}")
+ trace_artifacts = _extract_trace_artifacts(
+ trace_path,
+ target_dir,
+ since_timestamp=claude_start_ts,
+ )
+ traj_data = trace_artifacts["traj_data"]
+ leann_tool_used = bool(trace_artifacts["leann_tool_used"])
+ leann_search_calls = int(trace_artifacts["leann_search_calls"])
+ print(
+ f" -> {len(traj_data['pred_steps'])} steps, "
+ f"{len(traj_data['pred_files'])} unique files, "
+ f"leann_calls={leann_search_calls}"
+ )
+
+ patch = get_git_diff(target_dir)
+ elapsed = time.time() - start_time
+ agent_elapsed = (
+ max(0.0, claude_end_ts - claude_start_ts)
+ if claude_start_ts is not None and claude_end_ts is not None
+ else None
+ )
+ print(f"⏱️ Task completed in {elapsed:.1f}s ({elapsed / 60:.1f}min)")
+ status = "succeeded"
+ return patch, elapsed, agent_elapsed, traj_data, usage
+ except Exception as e:
+ error_message = str(e)
+ raise
+
+ finally:
+ elapsed = time.time() - start_time
+ agent_elapsed = None
+ if claude_start_ts is not None:
+ agent_end = claude_end_ts if claude_end_ts is not None else time.time()
+ agent_elapsed = max(0.0, agent_end - claude_start_ts)
+ if usage is None and usage_before is not None:
+ usage_after = get_ccusage_sessions()
+ usage = compute_usage_diff(usage_before, usage_after, instance_id=instance_id)
+
+ metrics_entry: dict = {
+ "instance_id": instance_id,
+ "repo_url": repo_url,
+ "model": model or "cli-default",
+ "status": status,
+ "latency_seconds": round(elapsed, 1),
+ "timestamp_unix": int(time.time()),
+ "token_usage": usage,
+ "leann_mode": leann_mode,
+ "leann_tool_used": leann_tool_used,
+ "leann_search_calls": leann_search_calls,
+ }
+ if agent_elapsed is not None:
+ metrics_entry["agent_seconds"] = round(agent_elapsed, 1)
+ if error_message:
+ metrics_entry["error"] = error_message
+ append_task_metrics(metrics_entry)
+
+ metric_line = (
+ f"📊 Task metrics ({instance_id}) — status={status}, "
+ f"agent={agent_elapsed:.1f}s, latency={elapsed:.1f}s"
+ if agent_elapsed is not None
+ else f"📊 Task metrics ({instance_id}) — status={status}, latency={elapsed:.1f}s"
+ )
+ if usage:
+ metric_line += (
+ f", session={usage['session_id']}, {_format_usage(usage['session_usage'])}"
+ )
+ else:
+ metric_line += ", token usage unavailable"
+ metric_line += (
+ f", leann_mode={leann_mode}, "
+ f"leann_tool_used={str(leann_tool_used).lower()}, "
+ f"leann_search_calls={leann_search_calls}"
+ )
+ if error_message:
+ metric_line += f", error={error_message}"
+ print(metric_line)
+ append_usage_log(metric_line)
+
+ if mitm_proc:
+ os.killpg(os.getpgid(mitm_proc.pid), signal.SIGTERM)
diff --git a/benchmarks/contextbench/scripts/batch_run_random.py b/benchmarks/contextbench/scripts/batch_run_random.py
new file mode 100644
index 0000000..b1f1a4c
--- /dev/null
+++ b/benchmarks/contextbench/scripts/batch_run_random.py
@@ -0,0 +1,140 @@
+import json
+import os
+import random
+import subprocess
+import time
+from pathlib import Path
+
+from auto_run import prefetch_task_repositories
+from datasets import load_dataset
+
+ROOT = Path(__file__).resolve().parents[1]
+OUTPUT_FILE = os.environ.get("OUTPUT_FILE", "all_predictions_claude.jsonl")
+NUM_TASKS = int(os.environ.get("NUM_TASKS", "31"))
+WORK_ROOT = os.environ.get("WORK_ROOT", "contextbench_work_dir_claude")
+MODEL = os.environ.get("MODEL", os.environ.get("CLAUDE_MODEL", "")).strip()
+DATASET_NAME = os.environ.get("DATASET_NAME", "Contextbench/ContextBench")
+DATASET_SPLIT = os.environ.get("DATASET_SPLIT", "train")
+# Optionally restrict to one benchmark split: Verified | Pro | Poly | Multi
+BENCH_FILTER = os.environ.get("BENCH_FILTER", "Pro").strip()
+# Optionally restrict random sampling to one repo (supports partial match),
+# e.g. "django/django" or "sympy".
+REPO_FILTER = os.environ.get("REPO_FILTER", "").strip().lower()
+PREFETCH_REPOS = os.environ.get("PREFETCH_REPOS", "4").strip() != "0"
+MITM_SCRIPT = ROOT / "mitmproxy_addons" / "trace_recorder.py"
+TRACE_DIR = ROOT / "traces" / "raw"
+
+
+def cleanup_residuals():
+ print("🧹 Cleaning up residual processes (Claude & Mitm)...")
+ try:
+ subprocess.run(["pkill", "-f", "claude"], stderr=subprocess.DEVNULL)
+ subprocess.run(["pkill", "-f", "mitmdump"], stderr=subprocess.DEVNULL)
+ time.sleep(2)
+ except Exception:
+ pass
+
+
+def main():
+ Path(WORK_ROOT).mkdir(parents=True, exist_ok=True)
+ TRACE_DIR.mkdir(parents=True, exist_ok=True)
+
+ print(f"🧠 Model: {MODEL or '(Claude CLI default)'}")
+ print(f"🔎 Bench filter: {BENCH_FILTER or '(all)'}")
+ if REPO_FILTER:
+ print(f"📁 Repo filter: {REPO_FILTER}")
+
+ api_key = os.environ.get("ANTHROPIC_API_KEY")
+ if api_key:
+ print(f"🔑 Using API key from environment: {api_key[:20]}...")
+ else:
+ print("🔐 ANTHROPIC_API_KEY not set; using Claude CLI logged-in session.")
+
+ existing_ids: set = set()
+ output_path = Path(OUTPUT_FILE)
+ if output_path.exists():
+ with open(output_path) as f:
+ for line in f:
+ try:
+ data = json.loads(line)
+ existing_ids.add(data["instance_id"])
+ except Exception:
+ continue
+ print(f"✅ Found {len(existing_ids)} already completed tasks.")
+
+ print(f"📚 Loading dataset: {DATASET_NAME} ({DATASET_SPLIT})...")
+ ds = load_dataset(DATASET_NAME, split=DATASET_SPLIT)
+
+ pending_tasks = [
+ t
+ for t in ds
+ if t["instance_id"] not in existing_ids
+ and (not BENCH_FILTER or t.get("source", "") == BENCH_FILTER)
+ and (
+ not REPO_FILTER
+ or REPO_FILTER in (t.get("repo", "") or "").lower()
+ or REPO_FILTER in (t.get("repo_url", "") or "").lower()
+ )
+ ]
+
+ if not pending_tasks:
+ print("🎉 No pending tasks to run!")
+ return
+
+ selected_tasks = random.sample(pending_tasks, min(NUM_TASKS, len(pending_tasks)))
+ print(f"🚀 Randomly selected {len(selected_tasks)} tasks to process.")
+ if PREFETCH_REPOS:
+ prefetch_task_repositories(selected_tasks, Path(WORK_ROOT))
+ else:
+ print("⏭️ PREFETCH_REPOS=0; skipping prefetch step.")
+
+ for i, task in enumerate(selected_tasks):
+ instance_id = task["instance_id"]
+ repo_url = task["repo_url"]
+
+ print(f"\n{'-' * 60}")
+ print(f"📦 [{i + 1}/{len(selected_tasks)}] Running: {instance_id}")
+ print(f" repo: {repo_url} source: {task.get('source', '?')}")
+
+ # try:
+ # patch, elapsed, traj_data, usage = run_single_task(
+ # instance_id=instance_id,
+ # repo_url=repo_url,
+ # work_root=WORK_ROOT,
+ # mitm_script_path=str(MITM_SCRIPT),
+ # trace_dir=TRACE_DIR,
+ # model=MODEL,
+ # task=task,
+ # )
+
+ # result_entry = {
+ # "instance_id": instance_id,
+ # "model_patch": patch if patch else "",
+ # "model_name_or_path": "claude-code-cli",
+ # "elapsed_seconds": round(elapsed, 1),
+ # "traj_data": traj_data,
+ # "token_usage": usage,
+ # }
+
+ # with open(OUTPUT_FILE, "a", encoding="utf-8") as f:
+ # f.write(json.dumps(result_entry) + "\n")
+ # print(f"✅ Result saved for {instance_id}")
+ # success_count += 1
+
+ # except Exception as e:
+ # print(f"❌ Error processing {instance_id}: {e}")
+ # failure_count += 1
+ # finally:
+ # cleanup_residuals()
+ # print("💤 Cooldown...")
+ # time.sleep(20)
+
+ # print(
+ # f"\n✅ Finished {len(selected_tasks)} random tasks: "
+ # f"{success_count} succeeded, {failure_count} failed. "
+ # f"Results in {OUTPUT_FILE}"
+ # )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/benchmarks/contextbench/scripts/batch_run_selected.py b/benchmarks/contextbench/scripts/batch_run_selected.py
new file mode 100644
index 0000000..8854850
--- /dev/null
+++ b/benchmarks/contextbench/scripts/batch_run_selected.py
@@ -0,0 +1,208 @@
+import json
+import os
+import subprocess
+import time
+from pathlib import Path
+
+from auto_run import prefetch_task_repositories, run_single_task
+from datasets import load_dataset
+
+ROOT = Path(__file__).resolve().parents[1]
+OUTPUT_FILE = os.environ.get("OUTPUT_FILE", "all_predictions_claude.jsonl")
+WORK_ROOT = os.environ.get("WORK_ROOT", "contextbench_work_dir_claude")
+MODEL = os.environ.get("MODEL", os.environ.get("CLAUDE_MODEL", "")).strip()
+DATASET_NAME = os.environ.get("DATASET_NAME", "Contextbench/ContextBench")
+DATASET_SPLIT = os.environ.get("DATASET_SPLIT", "train")
+BENCH_FILTER = os.environ.get("BENCH_FILTER", "").strip() # e.g. "Verified", "Pro", "Poly", "Multi"
+PREFETCH_REPOS = os.environ.get("PREFETCH_REPOS", "1").strip() != "0"
+MITM_SCRIPT = ROOT / "mitmproxy_addons" / "trace_recorder.py"
+TRACE_DIR = ROOT / "traces" / "raw"
+
+# Instances to run. Set instance_ids here or pass via SELECTED_IDS env var (comma-separated).
+SELECTED_IDS = [
+ # "SWE-Bench-Pro__python__maintenance__bugfix__19a1fba2",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__2464eadb",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__38dc8f4e",
+ # "SWE-Bench-Pro__javascript__maintenance__bugfix__2bfb5681",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__71253eae",
+ # "SWE-Bench-Pro__javascript__maintenance__bugfix__93b583ae",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__dcc84d4c",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__462b957d",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__9af74069",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__7b688a35",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__64fffdfa",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__22a1484c",
+ # "SWE-Bench-Pro__go__maintenance__bugfix__1177cd53",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__a4287775",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__ba13492e",
+ # "SWE-Bench-Pro__go__maintenance__bugfix__b91d5788",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__091dae2f",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__b6eff698",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__fcb506a5",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__3cfd9a02",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__4c132bfd",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__7c2efe8a",
+ "SWE-Bench-Pro__go__maintenance__bugfix__40a717e5",
+ "SWE-Bench-Pro__go__maintenance__bugfix__52d866b3",
+ "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92",
+ "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd",
+ "SWE-Bench-Pro__javascript__maintenance__bugfix__82518720",
+ "SWE-Bench-Pro__javascript__maintenance__bugfix__e31ec45c",
+ "SWE-Bench-Pro__python__maintenance__bugfix__07bb383a",
+ "SWE-Bench-Pro__python__maintenance__bugfix__0bac5789",
+ "SWE-Bench-Pro__python__maintenance__bugfix__18d7bbbc",
+ "SWE-Bench-Pro__python__maintenance__bugfix__1cf3e889",
+ "SWE-Bench-Pro__python__maintenance__bugfix__20dad82b",
+ "SWE-Bench-Pro__python__maintenance__bugfix__20f502e0",
+ "SWE-Bench-Pro__python__maintenance__bugfix__509a20d9",
+ "SWE-Bench-Pro__python__maintenance__bugfix__53ca6a30",
+ "SWE-Bench-Pro__python__maintenance__bugfix__552343cd",
+ "SWE-Bench-Pro__python__maintenance__bugfix__5b2cf9bb",
+ "SWE-Bench-Pro__python__maintenance__bugfix__66e05eaa",
+ "SWE-Bench-Pro__python__maintenance__bugfix__6ebb54dc",
+ "SWE-Bench-Pro__python__maintenance__bugfix__87bfb374",
+ "SWE-Bench-Pro__python__maintenance__bugfix__89932d58",
+ "SWE-Bench-Pro__python__maintenance__bugfix__942d0b14",
+ "SWE-Bench-Pro__python__maintenance__bugfix__983f2896",
+ "SWE-Bench-Pro__python__maintenance__bugfix__a984b409",
+ "SWE-Bench-Pro__python__maintenance__bugfix__aa07d0c3",
+ "SWE-Bench-Pro__python__maintenance__bugfix__cf01f471",
+ "SWE-Bench-Pro__python__maintenance__bugfix__d2506f10",
+ "SWE-Bench-Pro__python__maintenance__bugfix__e579f2f0",
+ "SWE-Bench-Pro__python__maintenance__bugfix__eafb1f0b",
+ "SWE-Bench-Pro__python__maintenance__bugfix__ef8756b1",
+ "SWE-Bench-Pro__python__maintenance__bugfix__f87209f8",
+ "SWE-Bench-Pro__python__maintenance__bugfix__ff79bafd",
+]
+
+if os.environ.get("SELECTED_IDS"):
+ SELECTED_IDS = [x.strip() for x in os.environ["SELECTED_IDS"].split(",") if x.strip()]
+
+
+def cleanup_residuals():
+ print("🧹 Cleaning up residual processes (Claude & Mitm)...")
+ try:
+ subprocess.run(["pkill", "-f", "claude"], stderr=subprocess.DEVNULL)
+ subprocess.run(["pkill", "-f", "mitmdump"], stderr=subprocess.DEVNULL)
+ time.sleep(2)
+ except Exception:
+ pass
+
+
+def main():
+ if not SELECTED_IDS:
+ print(
+ "⚠️ Warning: SELECTED_IDS list is empty. Add instance IDs to the script or set SELECTED_IDS env var."
+ )
+ return
+
+ Path(WORK_ROOT).mkdir(parents=True, exist_ok=True)
+ TRACE_DIR.mkdir(parents=True, exist_ok=True)
+
+ print(f"🧠 Model: {MODEL or '(Claude CLI default)'}")
+ if BENCH_FILTER:
+ print(f"🔎 Bench filter: {BENCH_FILTER}")
+
+ api_key = os.environ.get("ANTHROPIC_API_KEY")
+ if api_key:
+ print(f"🔑 Using API key from environment: {api_key[:20]}...")
+ else:
+ print("🔐 ANTHROPIC_API_KEY not set; using Claude CLI logged-in session.")
+
+ # Load already-completed instance IDs to support resuming.
+ existing_ids: set = set()
+ output_path = Path(OUTPUT_FILE)
+ if output_path.exists():
+ with open(output_path) as f:
+ for line in f:
+ try:
+ data = json.loads(line)
+ existing_ids.add(data["instance_id"])
+ except Exception:
+ continue
+ print(f"✅ Found {len(existing_ids)} already completed tasks.")
+
+ print(f"📚 Loading dataset: {DATASET_NAME} ({DATASET_SPLIT})...")
+ ds = load_dataset(DATASET_NAME, split=DATASET_SPLIT)
+
+ # Build a lookup dict for fast access.
+ task_lookup = {t["instance_id"]: t for t in ds}
+
+ selected_tasks = []
+ for iid in SELECTED_IDS:
+ if iid in existing_ids:
+ print(f"⏩ Skipping {iid} (already completed)")
+ continue
+ task = task_lookup.get(iid)
+ if task is None:
+ print(f"⚠️ Instance {iid} not found in dataset; skipping.")
+ continue
+ if BENCH_FILTER and task.get("source", "") != BENCH_FILTER:
+ print(f"⏩ Skipping {iid} (source={task.get('source')} != {BENCH_FILTER})")
+ continue
+ selected_tasks.append(task)
+
+ if not selected_tasks:
+ print("🎉 No pending selected tasks to run!")
+ return
+
+ print(f"🚀 Selected {len(selected_tasks)} tasks to process.")
+ if PREFETCH_REPOS:
+ prefetch_task_repositories(selected_tasks, Path(WORK_ROOT))
+ else:
+ print("⏭️ PREFETCH_REPOS=0; skipping prefetch step.")
+ success_count = 0
+ failure_count = 0
+
+ for i, task in enumerate(selected_tasks):
+ instance_id = task["instance_id"]
+ repo_url = task["repo_url"]
+
+ print(f"\n{'-' * 60}")
+ print(f"📦 [{i + 1}/{len(selected_tasks)}] Running: {instance_id}")
+ print(f" repo: {repo_url} source: {task.get('source', '?')}")
+
+ try:
+ patch, elapsed, agent_seconds, traj_data, usage = run_single_task(
+ instance_id=instance_id,
+ repo_url=repo_url,
+ work_root=WORK_ROOT,
+ mitm_script_path=str(MITM_SCRIPT),
+ trace_dir=TRACE_DIR,
+ model=MODEL,
+ task=task,
+ )
+
+ result_entry = {
+ "instance_id": instance_id,
+ "model_patch": patch if patch else "",
+ "model_name_or_path": "claude-code-cli",
+ "elapsed_seconds": round(elapsed, 1),
+ "latency_seconds": round(elapsed, 1),
+ "agent_seconds": round(agent_seconds, 1) if agent_seconds is not None else None,
+ "traj_data": traj_data,
+ "token_usage": usage,
+ }
+
+ with open(OUTPUT_FILE, "a", encoding="utf-8") as f:
+ f.write(json.dumps(result_entry) + "\n")
+ print(f"✅ Result saved for {instance_id}")
+ success_count += 1
+
+ except Exception as e:
+ print(f"❌ Error processing {instance_id}: {e}")
+ failure_count += 1
+ finally:
+ cleanup_residuals()
+ print("💤 Cooldown...")
+ time.sleep(20)
+
+ print(
+ f"\n✅ Finished {len(selected_tasks)} selected tasks: "
+ f"{success_count} succeeded, {failure_count} failed. "
+ f"Results in {OUTPUT_FILE}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/benchmarks/contextbench/scripts/prepare_repos_with_leann.py b/benchmarks/contextbench/scripts/prepare_repos_with_leann.py
new file mode 100644
index 0000000..f85c8f2
--- /dev/null
+++ b/benchmarks/contextbench/scripts/prepare_repos_with_leann.py
@@ -0,0 +1,462 @@
+"""
+Prepare ContextBench repositories and build LEANN indexes.
+
+For each selected ContextBench instance:
+ 1. Clone the repo into <WORK_ROOT>/<instance_id>
+ 2. Checkout base_commit
+ 3. Build LEANN index under <WORK_ROOT>/<instance_id>/.leann/
+
+Usage:
+ cd scripts
+ python prepare_repos_with_leann.py
+"""
+
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import time
+from pathlib import Path
+from typing import Optional
+
+from datasets import load_dataset
+from git import Repo
+
+WORK_ROOT = os.environ.get("WORK_ROOT", "contextbench_work_dir_claude")
+LEANN_BIN = os.environ.get("LEANN_BIN", "leann")
+DATASET_NAME = os.environ.get("DATASET_NAME", "Contextbench/ContextBench")
+DATASET_SPLIT = os.environ.get("DATASET_SPLIT", "train")
+BENCH_FILTER = os.environ.get("BENCH_FILTER", "").strip() # Verified | Pro | Poly | Multi
+LEANN_SOURCE_EXTENSIONS = os.environ.get(
+ "LEANN_SOURCE_EXTENSIONS",
+ "py,go,js,jsx,ts,tsx,java,kt,kts,rs,rb,php,cs,c,cc,cpp,h,hpp,m,mm,swift,scala,sh,sql,lua,r",
+)
+
+# LEANN index build parameters — override via env vars for sweeping configs.
+# ast-chunk-size is in non-whitespace CHARACTERS (not tokens). bge-base-en-v1.5
+# has a 512-token limit; at ~1.2 tokens/char: 300 chars + 64 overlap ≈ 436 tokens.
+LEANN_EMBEDDING_MODEL = os.environ.get(
+ "LEANN_EMBEDDING_MODEL", "jinaai/jina-embeddings-v2-base-code"
+)
+LEANN_AST_CHUNK_SIZE = os.environ.get("LEANN_AST_CHUNK_SIZE", "600")
+LEANN_AST_CHUNK_OVERLAP = os.environ.get("LEANN_AST_CHUNK_OVERLAP", "96")
+# Set to "0" to disable vendor/generated exclusion (e.g. for ablation experiments).
+LEANN_EXCLUDE_VENDOR = os.environ.get("LEANN_EXCLUDE_VENDOR", "1").strip() != "0"
+# Set to "1" to exclude test files (e.g. for ablation experiments). Default off
+# to avoid missing bugfix targets that touch test/fixture/spec files.
+LEANN_EXCLUDE_TESTS = os.environ.get("LEANN_EXCLUDE_TESTS", "0").strip() != "0"
+FAILED_INSTANCES_LOG = os.environ.get(
+ "FAILED_INSTANCES_LOG", "prepare_repos_with_leann_failures.jsonl"
+).strip()
+
+# Fill in ContextBench instance_ids. Leave empty to prepare all tasks
+# (optionally filtered by BENCH_FILTER).
+SELECTED_IDS: list[str] = [
+ # "SWE-Bench-Pro__python__maintenance__bugfix__19a1fba2",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__2464eadb",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__38dc8f4e",
+ # "SWE-Bench-Pro__javascript__maintenance__bugfix__2bfb5681",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__71253eae",
+ # "SWE-Bench-Pro__javascript__maintenance__bugfix__93b583ae",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__dcc84d4c",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__462b957d",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__9af74069",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__7b688a35",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__64fffdfa",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__22a1484c",
+ # "SWE-Bench-Pro__go__maintenance__bugfix__1177cd53",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__a4287775",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__ba13492e",
+ # "SWE-Bench-Pro__go__maintenance__bugfix__b91d5788",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__091dae2f",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__b6eff698",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__fcb506a5",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__3cfd9a02",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__4c132bfd",
+ # "SWE-Bench-Pro__python__maintenance__bugfix__7c2efe8a",
+ "SWE-Bench-Pro__go__maintenance__bugfix__40a717e5",
+ "SWE-Bench-Pro__go__maintenance__bugfix__52d866b3",
+ "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92",
+ "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd",
+ "SWE-Bench-Pro__javascript__maintenance__bugfix__82518720",
+ "SWE-Bench-Pro__javascript__maintenance__bugfix__e31ec45c",
+ "SWE-Bench-Pro__python__maintenance__bugfix__07bb383a",
+ "SWE-Bench-Pro__python__maintenance__bugfix__0bac5789",
+ "SWE-Bench-Pro__python__maintenance__bugfix__18d7bbbc",
+ "SWE-Bench-Pro__python__maintenance__bugfix__1cf3e889",
+ "SWE-Bench-Pro__python__maintenance__bugfix__20dad82b",
+ "SWE-Bench-Pro__python__maintenance__bugfix__20f502e0",
+ "SWE-Bench-Pro__python__maintenance__bugfix__509a20d9",
+ "SWE-Bench-Pro__python__maintenance__bugfix__53ca6a30",
+ "SWE-Bench-Pro__python__maintenance__bugfix__552343cd",
+ "SWE-Bench-Pro__python__maintenance__bugfix__5b2cf9bb",
+ "SWE-Bench-Pro__python__maintenance__bugfix__66e05eaa",
+ "SWE-Bench-Pro__python__maintenance__bugfix__6ebb54dc",
+ "SWE-Bench-Pro__python__maintenance__bugfix__87bfb374",
+ "SWE-Bench-Pro__python__maintenance__bugfix__89932d58",
+ "SWE-Bench-Pro__python__maintenance__bugfix__942d0b14",
+ "SWE-Bench-Pro__python__maintenance__bugfix__983f2896",
+ "SWE-Bench-Pro__python__maintenance__bugfix__a984b409",
+ "SWE-Bench-Pro__python__maintenance__bugfix__aa07d0c3",
+ "SWE-Bench-Pro__python__maintenance__bugfix__cf01f471",
+ "SWE-Bench-Pro__python__maintenance__bugfix__d2506f10",
+ "SWE-Bench-Pro__python__maintenance__bugfix__e579f2f0",
+ "SWE-Bench-Pro__python__maintenance__bugfix__eafb1f0b",
+ "SWE-Bench-Pro__python__maintenance__bugfix__ef8756b1",
+ "SWE-Bench-Pro__python__maintenance__bugfix__f87209f8",
+ "SWE-Bench-Pro__python__maintenance__bugfix__ff79bafd",
+]
+
+if os.environ.get("SELECTED_IDS"):
+ SELECTED_IDS = [x.strip() for x in os.environ["SELECTED_IDS"].split(",") if x.strip()]
+
+# Vendor and generated-code directory/file patterns to exclude from the index.
+# These are third-party or machine-generated files that are never the target of
+# a bugfix, so indexing them only adds noise to search results.
+_VENDOR_DIR_PATTERNS = (
+ "vendor/",
+ "node_modules/",
+ "third_party/",
+ "thirdparty/",
+ "externals/",
+ ".cache/",
+)
+_GENERATED_FILE_PATTERNS = (
+ "_pb.go",
+ ".pb.go",
+ "_gen.go",
+ ".pb.cc",
+ ".pb.h",
+)
+# Filenames like `zz_generated.deepcopy.go` end in `.go`, not `zz_generated`;
+# match these as path substrings (controller-gen / k8s-style outputs).
+_GENERATED_FILE_SUBSTRINGS = ("zz_generated",)
+
+# Test file path/name patterns to exclude from the index.
+_TEST_PATH_PATTERNS = (
+ "/test/",
+ "/tests/",
+ "/__tests__/",
+ "/spec/",
+ "/testdata/",
+ "/test_",
+ "/fixtures/",
+)
+_TEST_FILE_PATTERNS = (
+ "_test.py",
+ "_test.go",
+ ".test.js",
+ ".test.ts",
+ ".test.jsx",
+ ".test.tsx",
+ ".spec.js",
+ ".spec.ts",
+ ".spec.jsx",
+ ".spec.tsx",
+ "_spec.rb",
+)
+
+
+def _run_command(cmd: list[str], cwd: Path) -> subprocess.CompletedProcess:
+ return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
+
+
+def _read_json_file(path: Path) -> Optional[dict]:
+ if not path.exists():
+ return None
+ try:
+ return json.loads(path.read_text(encoding="utf-8"))
+ except Exception:
+ return None
+
+
+def _count_index_chunks(repo_dir: Path, instance_id: str) -> Optional[int]:
+ ids_path = repo_dir / ".leann" / "indexes" / instance_id / "documents.ids.txt"
+ if not ids_path.exists():
+ return None
+ with ids_path.open("r", encoding="utf-8") as f:
+ return sum(1 for _ in f)
+
+
+def _print_subprocess_output(label: str, text: str, max_lines: int = 20) -> None:
+ lines = [line for line in (text or "").splitlines() if line.strip()]
+ if not lines:
+ return
+ print(f" 📄 {label}:")
+ for line in lines[:max_lines]:
+ print(f" {line}")
+ if len(lines) > max_lines:
+ print(f" ... ({len(lines) - max_lines} more lines)")
+
+
+def _write_failure_report(failures: list[dict]) -> Optional[Path]:
+ if not failures:
+ return None
+ report_path = Path(FAILED_INSTANCES_LOG)
+ report_path.parent.mkdir(parents=True, exist_ok=True)
+ with report_path.open("w", encoding="utf-8") as f:
+ for item in failures:
+ f.write(json.dumps(item, ensure_ascii=True) + "\n")
+ return report_path
+
+
+def _load_tasks() -> list[dict]:
+ print(f"📚 Loading dataset: {DATASET_NAME} ({DATASET_SPLIT})...")
+ ds = load_dataset(DATASET_NAME, split=DATASET_SPLIT)
+ tasks: list[dict] = list(ds)
+ if BENCH_FILTER:
+ tasks = [t for t in tasks if t.get("source", "") == BENCH_FILTER]
+
+ if SELECTED_IDS:
+ task_lookup = {t["instance_id"]: t for t in tasks}
+ selected: list[dict] = []
+ for iid in SELECTED_IDS:
+ task = task_lookup.get(iid)
+ if not task:
+ print(f"⚠️ Instance not found in dataset/split/filter: {iid}")
+ continue
+ selected.append(task)
+ return selected
+
+ return tasks
+
+
+def _is_pytest_style_test_py(normalized_path: str) -> bool:
+ """True for pytest-style modules: basename test_*.py (e.g. test_foo.py)."""
+ name = Path(normalized_path).name
+ return name.startswith("test_") and name.lower().endswith(".py")
+
+
+def build_leann_index(instance_id: str, repo_dir: Path) -> tuple[bool, Optional[str]]:
+ print(f" 🔍 Building LEANN index for {instance_id}...")
+
+ result = _run_command(["git", "ls-files"], cwd=repo_dir)
+ if result.returncode != 0:
+ error = f"Could not list git files: {result.stderr.strip()}"
+ print(f" ⚠️ {error}")
+ return False, error
+
+ tracked_files = [f for f in result.stdout.strip().split("\n") if f]
+ allowed_exts = {
+ f".{ext.strip().lstrip('.').lower()}"
+ for ext in LEANN_SOURCE_EXTENSIONS.split(",")
+ if ext.strip()
+ }
+ source_files = [f for f in tracked_files if Path(f).suffix.lower() in allowed_exts]
+
+ # Exclude vendor/generated files — they are never bugfix targets and add noise.
+ if LEANN_EXCLUDE_VENDOR:
+ before = len(source_files)
+ normalized = [f.replace("\\", "/") for f in source_files]
+ source_files = [
+ f
+ for f, n in zip(source_files, normalized)
+ if not any(pat in n for pat in _VENDOR_DIR_PATTERNS)
+ and not any(n.endswith(pat) for pat in _GENERATED_FILE_PATTERNS)
+ and not any(sub in n for sub in _GENERATED_FILE_SUBSTRINGS)
+ ]
+ excluded = before - len(source_files)
+ if excluded:
+ print(
+ f" 🚫 Excluded {excluded} vendor/generated files ({before} → {len(source_files)})"
+ )
+
+ # Exclude test files — they are rarely bugfix targets and consistently rank
+ # high in semantic search due to mirroring production code patterns.
+ if LEANN_EXCLUDE_TESTS:
+ before = len(source_files)
+ normalized = [f.replace("\\", "/") for f in source_files]
+ source_files = [
+ f
+ for f, n in zip(source_files, normalized)
+ if not any(pat in n for pat in _TEST_PATH_PATTERNS)
+ and not _is_pytest_style_test_py(n)
+ and not any(n.endswith(pat) for pat in _TEST_FILE_PATTERNS)
+ ]
+ excluded = before - len(source_files)
+ if excluded:
+ print(f" 🚫 Excluded {excluded} test files ({before} → {len(source_files)})")
+
+ if not source_files:
+ error = f"No source files found for extensions: {sorted(allowed_exts)}"
+ print(f" ⚠️ {error}")
+ return False, error
+
+ # Derive --file-types from the actual extensions present after filtering,
+ # so all indexed file types benefit from AST-aware chunking.
+ indexed_exts = sorted(
+ {Path(f).suffix.lstrip(".").lower() for f in source_files if Path(f).suffix}
+ )
+ file_types_arg = ",".join(indexed_exts)
+
+ print(f" 📊 Found {len(source_files)} source files (types: {file_types_arg})")
+ leann_cmd = [
+ LEANN_BIN,
+ "build",
+ instance_id,
+ "--docs",
+ *source_files,
+ "--embedding-mode",
+ "sentence-transformers",
+ "--embedding-model",
+ LEANN_EMBEDDING_MODEL,
+ "--backend",
+ "hnsw",
+ "--file-types",
+ file_types_arg,
+ "--force",
+ "--ast-chunk-size",
+ LEANN_AST_CHUNK_SIZE,
+ "--ast-chunk-overlap",
+ LEANN_AST_CHUNK_OVERLAP,
+ "--use-ast-chunking",
+ "--no-recompute",
+ ]
+ debug_cmd = [
+ LEANN_BIN,
+ "build",
+ instance_id,
+ "--docs",
+ f"<{len(source_files)} files>",
+ *leann_cmd[len(["leann", "build", instance_id, "--docs"]) + len(source_files) :],
+ ]
+ print(f" 🧪 LEANN command: {shlex.join(debug_cmd)}")
+
+ try:
+ proc = subprocess.run(
+ leann_cmd,
+ cwd=repo_dir,
+ capture_output=True,
+ text=True,
+ timeout=1800,
+ env={
+ **os.environ,
+ "LEANN_EMBEDDING_DEVICE": os.environ.get("LEANN_EMBEDDING_DEVICE", "mps"),
+ "LEANN_BATCH_SIZE": os.environ.get("LEANN_BATCH_SIZE", "32"),
+ },
+ )
+ except subprocess.TimeoutExpired:
+ error = "LEANN build timed out"
+ print(f" ⏰ {error}")
+ return False, error
+ except Exception as e:
+ error = f"LEANN error: {e}"
+ print(f" ❌ {error}")
+ return False, error
+
+ _print_subprocess_output("LEANN stdout", proc.stdout)
+ _print_subprocess_output("LEANN stderr", proc.stderr)
+
+ if proc.returncode != 0:
+ stderr = (proc.stderr or "").strip()
+ print(f" ❌ LEANN build failed: {stderr}")
+ return False, f"LEANN build failed: {stderr}"
+
+ meta_path = repo_dir / ".leann" / "indexes" / instance_id / "documents.leann.meta.json"
+ meta = _read_json_file(meta_path)
+ if meta:
+ print(f" 🧠 Embedding model in index: {meta.get('embedding_model', 'unknown')}")
+ print(" 🌲 AST chunking requested: yes (--use-ast-chunking)")
+ chunk_count = _count_index_chunks(repo_dir, instance_id)
+ if chunk_count is not None:
+ print(f" 🧩 Indexed chunks: {chunk_count}")
+
+ print(" ✅ LEANN index built successfully")
+ return True, None
+
+
+def prepare_single_task(task: dict) -> tuple[bool, Optional[str]]:
+ instance_id = task["instance_id"]
+ repo_url = task["repo_url"]
+ base_commit = task["base_commit"]
+ target_dir = Path(WORK_ROOT) / instance_id
+
+ print(f"\n{'=' * 72}")
+ print(f"📦 Preparing: {instance_id}")
+ print(f" repo: {repo_url}")
+ print(f" commit: {base_commit[:12]}...")
+ print(f"{'=' * 72}")
+
+ if not target_dir.exists():
+ print(f" 📥 Cloning {repo_url}...")
+ try:
+ Repo.clone_from(repo_url, target_dir)
+ except Exception as e:
+ print(f" ❌ Clone failed: {e}")
+ return False, f"Clone failed: {e}"
+ else:
+ print(" ✓ Repo already exists")
+
+ try:
+ repo = Repo(target_dir)
+ print(f" 🔀 Checking out {base_commit[:8]}...")
+ repo.git.reset("--hard")
+ repo.git.checkout(base_commit)
+ repo.git.clean("-fdx", "-e", ".leann/")
+ (target_dir / "PROBLEM.md").write_text(task.get("problem_statement", ""), encoding="utf-8")
+ except Exception as e:
+ print(f" ❌ Checkout/clean failed: {e}")
+ return False, f"Checkout/clean failed: {e}"
+
+ return build_leann_index(instance_id, target_dir)
+
+
+def main():
+ print("🚀 ContextBench Repository Preparation with LEANN Indexing")
+ print("=" * 72)
+
+ leann_path = shutil.which(LEANN_BIN)
+ if not leann_path:
+ print("❌ LEANN not found. Install: uv tool install leann-core --with leann")
+ return
+ print(f"✅ LEANN found: {leann_path}")
+
+ tasks = _load_tasks()
+ if not tasks:
+ print("⚠️ No tasks selected. Set SELECTED_IDS or adjust BENCH_FILTER.")
+ return
+
+ Path(WORK_ROOT).mkdir(parents=True, exist_ok=True)
+ print(f"\n📂 Work root: {WORK_ROOT}")
+ print(f"🎯 Tasks to prepare: {len(tasks)}")
+ if BENCH_FILTER:
+ print(f"🔎 Bench filter: {BENCH_FILTER}")
+
+ success_count = 0
+ fail_count = 0
+ failures: list[dict] = []
+ for i, task in enumerate(tasks, start=1):
+ print(f"\n[{i}/{len(tasks)}]")
+ succeeded, error = prepare_single_task(task)
+ if succeeded:
+ success_count += 1
+ else:
+ fail_count += 1
+ failures.append(
+ {
+ "instance_id": task.get("instance_id", ""),
+ "repo_url": task.get("repo_url", ""),
+ "base_commit": task.get("base_commit", ""),
+ "error": error or "unknown error",
+ "failed_at_unix": int(time.time()),
+ }
+ )
+
+ print(f"\n{'=' * 72}")
+ print(f"🎉 Done! ✅ {success_count} succeeded ❌ {fail_count} failed")
+ if failures:
+ print("\nFailed instances:")
+ for item in failures:
+ print(f" - {item['instance_id']}: {item['error']}")
+ report_path = _write_failure_report(failures)
+ if report_path is not None:
+ print(f"\n📝 Failure report written to: {report_path}")
+ print("\nNext steps:")
+ print(f" 1. Verify indexes: ls {WORK_ROOT}/*/.leann")
+ print(" 2. Run with LEANN: LEANN_ENABLED=1 python batch_run_selected.py")
+ print(" 3. Baseline run: LEANN_ENABLED=0 python batch_run_selected.py")
+
+
+if __name__ == "__main__":
+ main()