Skip to content

Personal AI, On Personal Devices

Python

fix(evals): clean up AI stack runtime harness (#320)

Avanika Narayana689be0c69a9
diff --git a/scripts/experiments/a1_seed_feedback.py b/scripts/experiments/a1_seed_feedback.py
deleted file mode 100644
index c6565d92..00000000
--- a/scripts/experiments/a1_seed_feedback.py
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env python3
-"""A1: Seed feedback on all traces in the local M1 traces.db.
-
-- Judges each unscored trace with Sonnet 4.6 using the calibration-validated prompt
-- Parallelized via ThreadPoolExecutor (8 workers) for I/O-bound API calls
-- Writes feedback to local traces.db via TraceStore.update_feedback
-- Logs every call to a JSONL audit file
-- Idempotent: skips traces that already have feedback (safe to re-run)
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import re
-import sqlite3
-import sys
-import threading
-import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime
-from pathlib import Path
-
-from openjarvis.core.types import Message, Role
-from openjarvis.engine.cloud import CloudEngine
-from openjarvis.traces.store import TraceStore
-
-HOME = Path(os.environ.get("OPENJARVIS_HOME", "/scratch/user/jonsaadfalcon/openjarvis-m1"))
-DB = HOME / "traces.db"
-LOG = HOME / "a1_feedback_log.jsonl"
-MODEL = "claude-sonnet-4-6"
-MAX_WORKERS = 8
-
-JUDGE_PROMPT = """\
-You are evaluating whether an AI agent successfully completed its assigned task.
-
-Assign a SCORE from the set {{0.2, 0.4, 0.6, 0.8}} using this rubric:
-
-- 0.8 = Clean success. Task completed correctly. Minor stylistic issues don't affect correctness.
-- 0.6 = Partial. Real progress made but the answer has real gaps — missed requirement, incomplete output, recovered from errors but final result imperfect.
-- 0.4 = Poor. Some progress but the result is clearly incomplete, wrong, or the agent got mostly stuck.
-- 0.2 = Failure. Agent crashed, got stuck in a loop, gave up, hit a budget/poll/token limit before finishing, or produced no usable result.
-
-IMPORTANT: Do not trust the agent's own self-assessment. Agents often narrate "I am stuck" or "I hit an error" — those are failure signals. Agents sometimes claim success when the actual output is incomplete — look at the concrete result, not the rhetoric.
-
-TASK QUERY (first 1200 chars):
-<<<
-{query}
->>>
-
-AGENT FINAL RESULT (first 2500 chars):
-<<<
-{result_head}
->>>
-{tail_section}
-Respond in EXACTLY this format, nothing else:
-SCORE=<one of 0.2, 0.4, 0.6, 0.8>
-REASON=<one brief sentence>
-"""
-
-
-SCORE_RE = re.compile(r"SCORE\s*=\s*(0?\.[2468])", re.IGNORECASE)
-REASON_RE = re.compile(r"REASON\s*=\s*(.+?)\s*$", re.IGNORECASE | re.DOTALL)
-
-
-def build_prompt(query: str, result: str) -> str:
- q = (query or "")[:1200]
- head = (result or "")[:2500]
- if result and len(result) > 3000:
- tail = f"\nAGENT FINAL RESULT (last 500 chars):\n<<<\n{result[-500:]}\n>>>\n"
- else:
- tail = ""
- return JUDGE_PROMPT.format(query=q, result_head=head, tail_section=tail)
-
-
-def judge_one(ce: CloudEngine, trace_id: str, query: str, result: str) -> dict:
- prompt = build_prompt(query, result)
- t0 = time.time()
- try:
- resp = ce.generate(
- messages=[Message(role=Role.USER, content=prompt)],
- model=MODEL,
- max_tokens=150,
- temperature=0.0,
- )
- content = resp.get("content", "") or ""
- cost = resp.get("cost_usd", 0.0) or 0.0
- usage = resp.get("usage", {}) or {}
- in_tok = usage.get("prompt_tokens", 0) or usage.get("input_tokens", 0)
- out_tok = usage.get("completion_tokens", 0) or usage.get("output_tokens", 0)
-
- m = SCORE_RE.search(content)
- score = float(m.group(1)) if m else None
- mr = REASON_RE.search(content)
- reason = mr.group(1).strip() if mr else "(parse failed)"
-
- return {
- "trace_id": trace_id,
- "score": score,
- "reason": reason,
- "raw": content,
- "cost": cost,
- "input_tokens": in_tok,
- "output_tokens": out_tok,
- "elapsed": time.time() - t0,
- "judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
- "error": None,
- }
- except Exception as e:
- return {
- "trace_id": trace_id,
- "score": None,
- "reason": None,
- "raw": None,
- "cost": 0.0,
- "input_tokens": 0,
- "output_tokens": 0,
- "elapsed": time.time() - t0,
- "judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
- "error": f"{type(e).__name__}: {e}",
- }
-
-
-def main() -> int:
- if not os.environ.get("ANTHROPIC_API_KEY"):
- print("ERROR: ANTHROPIC_API_KEY not set", file=sys.stderr)
- return 1
-
- store = TraceStore(DB)
- # Use the TraceStore's connection for the initial read
- conn = store._conn
- conn.row_factory = sqlite3.Row
-
- # Pull all traces that need scoring
- rows = list(conn.execute(
- "SELECT trace_id, query, result, agent, model FROM traces "
- "WHERE feedback IS NULL"
- ))
- already_scored = conn.execute(
- "SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL"
- ).fetchone()[0]
-
- print(f"traces.db: {conn.execute('SELECT COUNT(*) FROM traces').fetchone()[0]} total")
- print(f" already scored: {already_scored}")
- print(f" to judge: {len(rows)}")
- print(f"parallelism: {MAX_WORKERS} workers")
- print(f"log: {LOG}")
-
- if not rows:
- print("Nothing to do.")
- return 0
-
- ce = CloudEngine()
- write_lock = threading.Lock()
- log_lock = threading.Lock()
- log_fp = open(LOG, "a", encoding="utf-8")
-
- # Write a header line marking this run
- log_fp.write(json.dumps({
- "_run_started": datetime.utcnow().isoformat(timespec="seconds") + "Z",
- "model": MODEL,
- "workers": MAX_WORKERS,
- "to_judge": len(rows),
- "already_scored": already_scored,
- }) + "\n")
- log_fp.flush()
-
- total_cost = 0.0
- done = 0
- errors = 0
- score_counts: dict = {}
- t_start = time.time()
-
- def on_result(res: dict) -> None:
- nonlocal total_cost, done, errors
- trace_id = res["trace_id"]
-
- # Write feedback to DB (if we got a valid score)
- if res["score"] is not None and res["error"] is None:
- with write_lock:
- store.update_feedback(trace_id, res["score"])
- conn.commit()
- else:
- errors += 1
-
- with log_lock:
- log_fp.write(json.dumps(res) + "\n")
- log_fp.flush()
-
- total_cost += res["cost"] or 0.0
- done += 1
- score_counts[res["score"]] = score_counts.get(res["score"], 0) + 1
-
- # Progress every 50 or on error
- if done % 50 == 0 or res["error"]:
- elapsed = time.time() - t_start
- rate = done / max(0.001, elapsed)
- eta = (len(rows) - done) / max(0.001, rate)
- tag = "ERR " if res["error"] else " "
- print(
- f"{tag}[{done:4}/{len(rows)}] score={res['score']} "
- f"cost=${total_cost:.3f} "
- f"rate={rate:.1f}/s eta={eta:.0f}s "
- f"errors={errors}"
- )
- if res["error"]:
- print(f" ERROR on {trace_id[:12]}: {res['error']}")
-
- with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
- futures = [
- pool.submit(judge_one, ce, r["trace_id"], r["query"] or "", r["result"] or "")
- for r in rows
- ]
- for fut in as_completed(futures):
- res = fut.result()
- on_result(res)
-
- log_fp.close()
-
- elapsed = time.time() - t_start
- print(f"\n{'='*60}")
- print(f"A1 COMPLETE in {elapsed:.1f}s ({elapsed/60:.1f}min)")
- print(f"Total cost: ${total_cost:.4f}")
- print(f"Errors: {errors}/{len(rows)}")
- print(f"Score distribution:")
- for k in sorted(score_counts.keys(), key=lambda x: (x is None, x)):
- v = score_counts[k]
- pct = 100 * v / len(rows)
- label = {0.2: "failure", 0.4: "poor", 0.6: "partial", 0.8: "clean", None: "ERROR"}.get(k, "?")
- print(f" {k} ({label}): {v} ({pct:.1f}%)")
-
- # Verify by re-counting from DB
- with_fb = conn.execute(
- "SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL"
- ).fetchone()[0]
- above_gate = conn.execute(
- "SELECT COUNT(*) FROM traces WHERE feedback >= 0.7"
- ).fetchone()[0]
- print(f"\nPost-A1 DB state:")
- print(f" traces with feedback: {with_fb}")
- print(f" traces passing 0.7 gate (eligible for personal benchmark): {above_gate}")
- return 0 if errors == 0 else 2
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/experiments/generate_distillation_configs.py b/scripts/experiments/generate_distillation_configs.py
deleted file mode 100644
index 1ef98274..00000000
--- a/scripts/experiments/generate_distillation_configs.py
+++ /dev/null
@@ -1,304 +0,0 @@
-#!/usr/bin/env python3
-"""Generate all distillation experiment TOML configs.
-
-Produces configs for 7 experiment axes × multiple settings.
-Run: python scripts/experiments/generate_distillation_configs.py
-"""
-
-from __future__ import annotations
-
-import itertools
-from pathlib import Path
-
-CONFIGS_DIR = Path("src/openjarvis/evals/configs/distillation")
-
-# ── Teacher models ──────────────────────────────────────────────────────────
-TEACHERS = {
- "opus": {
- "model": "claude-opus-4-6",
- "engine": "cloud",
- "provider": "anthropic",
- },
- "gpt54": {
- "model": "gpt-5.4",
- "engine": "cloud",
- "provider": "openai",
- },
- "gemini": {
- "model": "gemini-3.1-pro-preview",
- "engine": "cloud",
- "provider": "google",
- },
- "qwen397b": {
- "model": "Qwen/Qwen3.5-397B-A17B-FP8",
- "engine": "vllm",
- "provider": "local",
- "note": "# Requires 8×H100, vLLM serve on port 8010",
- },
-}
-
-# ── Student models ──────────────────────────────────────────────────────────
-# Served via vLLM on this H100 node. 27B uses the FP8 weights that fit on a
-# single H100; 2B and 9B use standard FP16.
-STUDENTS = {
- "2b": {"model": "Qwen/Qwen3.5-2B", "engine": "vllm", "port": 8000},
- "9b": {"model": "Qwen/Qwen3.5-9B", "engine": "vllm", "port": 8001},
- "27b": {"model": "Qwen/Qwen3.5-27B-FP8", "engine": "vllm", "port": 8002},
-}
-
-# ── Benchmarks ──────────────────────────────────────────────────────────────
-BENCHMARKS = {
- "pb": "pinchbench",
- "tc15": "toolcall15",
- "tb": "taubench",
-}
-
-# ── Data configs ────────────────────────────────────────────────────────────
-DATA_CONFIGS = {
- "C1": {
- "desc": "Zero test data — external traces only (GeneralThought + ADP)",
- "trace_source": "external",
- "benchmark_queries_visible": False,
- },
- "C2": {
- "desc": "Test queries only — benchmark traces visible, answers hidden",
- "trace_source": "benchmark",
- "benchmark_queries_visible": True,
- },
- "C3": {
- "desc": "Test queries + external — both benchmark and external traces",
- "trace_source": "both",
- "benchmark_queries_visible": True,
- },
-}
-
-# ── Budget presets ──────────────────────────────────────────────────────────
-BUDGETS = {
- "minimal": {"max_tool_calls": 5, "max_cost": 0.50},
- "standard": {"max_tool_calls": 15, "max_cost": 2.00},
- "thorough": {"max_tool_calls": 30, "max_cost": 5.00},
- "exhaustive": {"max_tool_calls": 50, "max_cost": 10.00},
-}
-
-# ── Gate presets ────────────────────────────────────────────────────────────
-GATES = {
- "permissive": {"min_improvement": 0.0, "max_regression": 0.10},
- "standard": {"min_improvement": 0.0, "max_regression": 0.05},
- "strict": {"min_improvement": 0.02, "max_regression": 0.02},
- "none": {"min_improvement": -1.0, "max_regression": 1.0},
-}
-
-# ── Autonomy presets ────────────────────────────────────────────────────────
-AUTONOMY_MODES = ["auto", "tiered", "manual"]
-
-
-def render_config(
- *,
- experiment: str,
- teacher_key: str,
- student_key: str,
- benchmark_key: str,
- data_config_key: str = "C2",
- budget_key: str = "standard",
- gate_key: str = "standard",
- autonomy: str = "auto",
- iterative_sessions: int = 1,
-) -> str:
- """Render a TOML config string."""
- teacher = TEACHERS[teacher_key]
- student = STUDENTS[student_key]
- benchmark = BENCHMARKS[benchmark_key]
- data_cfg = DATA_CONFIGS[data_config_key]
- budget = BUDGETS[budget_key]
- gate = GATES[gate_key]
-
- note = teacher.get("note", "")
- note_line = f"\n{note}" if note else ""
-
- return f"""\
-# Distillation Experiment Config
-# Experiment: {experiment}
-# Teacher: {teacher["model"]} ({teacher["provider"]})
-# Student: {student["model"]} ({student["engine"]})
-# Benchmark: {benchmark}
-# Data config: {data_config_key} — {data_cfg["desc"]}
-# Budget: {budget_key} ({budget["max_tool_calls"]} tool calls, ${budget["max_cost"]:.2f})
-# Gate: {gate_key} (min_improvement={gate["min_improvement"]}, max_regression={gate["max_regression"]})
-# Autonomy: {autonomy}
-# Iterative sessions: {iterative_sessions}
-{note_line}
-
-[intelligence]
-default_model = "{student["model"]}"
-
-[engine]
-default = "{student["engine"]}"
-
-[engine.vllm]
-host = "http://localhost:{student.get("port", 8000)}"
-
-[learning.distillation]
-enabled = true
-autonomy_mode = "{autonomy}"
-teacher_model = "{teacher["model"]}"
-max_cost_per_session_usd = {budget["max_cost"]}
-max_tool_calls_per_diagnosis = {budget["max_tool_calls"]}
-
-[learning.distillation.gate]
-min_improvement = {gate["min_improvement"]}
-max_regression = {gate["max_regression"]}
-benchmark_subsample_size = 50
-
-[learning.distillation.benchmark]
-synthesis_feedback_threshold = 0.7
-max_benchmark_size = 200
-
-[learning.distillation.experiment]
-# Metadata for the experiment runner (not read by distillation itself)
-experiment_id = "{experiment}"
-teacher_key = "{teacher_key}"
-student_key = "{student_key}"
-benchmark = "{benchmark}"
-data_config = "{data_config_key}"
-trace_source = "{data_cfg["trace_source"]}"
-benchmark_queries_visible = {str(data_cfg["benchmark_queries_visible"]).lower()}
-budget_key = "{budget_key}"
-gate_key = "{gate_key}"
-iterative_sessions = {iterative_sessions}
-"""
-
-
-def write_config(subdir: str, filename: str, content: str) -> Path:
- path = CONFIGS_DIR / subdir / filename
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text(content, encoding="utf-8")
- return path
-
-
-def generate_all() -> int:
- count = 0
-
- # ── Exp 1a: Teacher Model Ablation ────────────────────────────────────
- # Fix: S-9b, B-standard, A-auto, G-standard, I-single
- # Vary: teacher × benchmark × data_config
- for teacher_key, bench_key, dc_key in itertools.product(
- TEACHERS, BENCHMARKS, DATA_CONFIGS
- ):
- filename = f"{teacher_key}-9b-{bench_key}-{dc_key}.toml"
- content = render_config(
- experiment=f"exp1a-teacher/{teacher_key}-{bench_key}-{dc_key}",
- teacher_key=teacher_key,
- student_key="9b",
- benchmark_key=bench_key,
- data_config_key=dc_key,
- )
- write_config("exp1a-teacher", filename, content)
- count += 1
-
- # ── Exp 1b: Budget Ablation ───────────────────────────────────────────
- # Fix: S-9b, T-sonnet(opus for quality), A-auto, G-standard, I-single
- # Vary: budget × benchmark × data_config
- for budget_key, bench_key, dc_key in itertools.product(
- BUDGETS, BENCHMARKS, DATA_CONFIGS
- ):
- filename = f"{budget_key}-9b-{bench_key}-{dc_key}.toml"
- content = render_config(
- experiment=f"exp1b-budget/{budget_key}-{bench_key}-{dc_key}",
- teacher_key="opus",
- student_key="9b",
- benchmark_key=bench_key,
- data_config_key=dc_key,
- budget_key=budget_key,
- )
- write_config("exp1b-budget", filename, content)
- count += 1
-
- # ── Exp 1c: Student Model Scaling ─────────────────────────────────────
- # Fix: T-opus, B-standard, A-auto, G-standard, I-single
- # Vary: student × benchmark × data_config
- for student_key, bench_key, dc_key in itertools.product(
- STUDENTS, BENCHMARKS, DATA_CONFIGS
- ):
- filename = f"opus-{student_key}-{bench_key}-{dc_key}.toml"
- content = render_config(
- experiment=f"exp1c-student/opus-{student_key}-{bench_key}-{dc_key}",
- teacher_key="opus",
- student_key=student_key,
- benchmark_key=bench_key,
- data_config_key=dc_key,
- )
- write_config("exp1c-student", filename, content)
- count += 1
-
- # ── Exp 2a: Gate Strictness ───────────────────────────────────────────
- # Fix: S-9b, T-opus, B-standard, A-auto, I-single
- # Vary: gate × benchmark
- for gate_key, bench_key in itertools.product(GATES, BENCHMARKS):
- filename = f"{gate_key}-9b-{bench_key}.toml"
- content = render_config(
- experiment=f"exp2a-gate/{gate_key}-{bench_key}",
- teacher_key="opus",
- student_key="9b",
- benchmark_key=bench_key,
- gate_key=gate_key,
- )
- write_config("exp2a-gate", filename, content)
- count += 1
-
- # ── Exp 2b: Autonomy Mode ────────────────────────────────────────────
- # Fix: S-9b, T-opus, B-standard, G-standard, I-single
- # Vary: autonomy × benchmark
- for autonomy, bench_key in itertools.product(AUTONOMY_MODES, BENCHMARKS):
- filename = f"{autonomy}-9b-{bench_key}.toml"
- content = render_config(
- experiment=f"exp2b-autonomy/{autonomy}-{bench_key}",
- teacher_key="opus",
- student_key="9b",
- benchmark_key=bench_key,
- autonomy=autonomy,
- )
- write_config("exp2b-autonomy", filename, content)
- count += 1
-
- # ── Exp 3a: Iterative Sessions ───────────────────────────────────────
- # Fix: S-9b, T-opus, B-standard, A-auto, G-standard
- # Vary: number of chained sessions × benchmark
- for n_sessions, bench_key in itertools.product([1, 3, 5], BENCHMARKS):
- filename = f"iter{n_sessions}-9b-{bench_key}.toml"
- content = render_config(
- experiment=f"exp3a-iterative/iter{n_sessions}-{bench_key}",
- teacher_key="opus",
- student_key="9b",
- benchmark_key=bench_key,
- iterative_sessions=n_sessions,
- )
- write_config("exp3a-iterative", filename, content)
- count += 1
-
- # ── Exp 3b: Cross-Benchmark Transfer ─────────────────────────────────
- # Optimize using traces from benchmark X, eval on benchmark Y
- for opt_bench, eval_bench in itertools.permutations(BENCHMARKS, 2):
- filename = f"opt-{opt_bench}-eval-{eval_bench}-9b.toml"
- content = render_config(
- experiment=f"exp3b-transfer/opt-{opt_bench}-eval-{eval_bench}",
- teacher_key="opus",
- student_key="9b",
- benchmark_key=opt_bench, # Traces from this benchmark
- )
- # Add eval benchmark as metadata
- content += f'\neval_benchmark = "{BENCHMARKS[eval_bench]}"\n'
- write_config("exp3b-transfer", filename, content)
- count += 1
-
- return count
-
-
-if __name__ == "__main__":
- n = generate_all()
- print(f"Generated {n} config files in {CONFIGS_DIR}/")
-
- # Print summary
- for subdir in sorted(CONFIGS_DIR.iterdir()):
- if subdir.is_dir():
- files = list(subdir.glob("*.toml"))
- print(f" {subdir.name}/: {len(files)} configs")
diff --git a/scripts/experiments/m2_collect_results.py b/scripts/experiments/m2_collect_results.py
deleted file mode 100644
index 41319399..00000000
--- a/scripts/experiments/m2_collect_results.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python3
-"""M2: Collect distilled eval results and produce comparison table.
-
-Reads .summary.json files from results/neurips-2026/{distilled,baselines}/
-and produces a before/after comparison against the Step 1 baseline numbers.
-"""
-
-from __future__ import annotations
-
-import json
-import sys
-from pathlib import Path
-
-# Jon's Step 1 baselines — used when a local baseline result doesn't exist
-STEP1_BASELINES = {
- "2b": {"toolcall15": 33.3, "pinchbench": 69.6, "livecodebench": 5.6, "taubench": 70.0, "taubench-telecom": 0.0, "gaia": 0.0, "liveresearch": 0.0, "liveresearchbench": None},
- "9b": {"toolcall15": 46.7, "pinchbench": 95.7, "livecodebench": 17.6, "taubench": 85.0, "taubench-telecom": 80.0, "gaia": 38.0, "liveresearch": 75.0, "liveresearchbench": None},
- "27b": {"toolcall15": 40.0, "pinchbench": 65.2, "livecodebench": 20.0, "taubench": 75.0, "taubench-telecom": 75.0, "gaia": 48.0, "liveresearch": 66.7, "liveresearchbench": None},
-}
-
-# Which benchmarks go through the agent layer (where distillation edits actually apply)
-AGENT_BENCHMARKS = {"pinchbench", "gaia", "liveresearch"}
-
-DISTILLED_ROOT = Path("results/neurips-2026/distilled")
-BASELINE_ROOT = Path("results/neurips-2026/baselines")
-
-
-def find_summary(root: Path, size: str, bench: str) -> Path | None:
- """Find the summary JSON for a model × benchmark run."""
- # Expected path: root/qwen-{size}/{bench}/{bench}_Qwen-Qwen3.5-{size}.summary.json
- candidates = list(root.glob(f"qwen-{size}/{bench}/*.summary.json"))
- return candidates[0] if candidates else None
-
-
-def load_accuracy(summary_path: Path) -> float | None:
- """Extract overall accuracy from a summary.json file."""
- try:
- d = json.loads(summary_path.read_text())
- # The summary has various shapes; try a few
- for key in ["overall_accuracy", "accuracy", "overall_score"]:
- if key in d:
- return float(d[key]) * 100 if d[key] <= 1.0 else float(d[key])
- # Try nested
- if "results" in d:
- for r in d["results"]:
- if "accuracy" in r:
- return float(r["accuracy"]) * 100 if r["accuracy"] <= 1.0 else float(r["accuracy"])
- except Exception as e:
- print(f" error reading {summary_path}: {e}", file=sys.stderr)
- return None
-
-
-def main() -> int:
- print("=" * 100)
- print("M2 Distilled vs Baseline Comparison")
- print("=" * 100)
- print()
- print(f"{'Model':8} {'Benchmark':20} {'Baseline':>10} {'Distilled':>10} {'Delta':>10} {'Agent?':>10}")
- print("-" * 100)
-
- benchmarks = [
- "toolcall15", "pinchbench", "livecodebench",
- "taubench", "taubench-telecom",
- "gaia", "liveresearch", "liveresearchbench",
- ]
-
- summary_rows = []
- for size in ["2b", "9b", "27b"]:
- for bench in benchmarks:
- # Baseline: prefer local file (if run this session), fall back to Step 1 numbers
- baseline_path = find_summary(BASELINE_ROOT, size, bench)
- if baseline_path:
- baseline = load_accuracy(baseline_path)
- base_source = "local"
- else:
- baseline = STEP1_BASELINES[size].get(bench)
- base_source = "step1"
-
- # Distilled: must be local from this session
- distilled_path = find_summary(DISTILLED_ROOT, size, bench)
- distilled = load_accuracy(distilled_path) if distilled_path else None
-
- # Format
- b_str = f"{baseline:.1f}%" if baseline is not None else "—"
- d_str = f"{distilled:.1f}%" if distilled is not None else "pending"
- if baseline is not None and distilled is not None:
- delta = distilled - baseline
- d_sign = "+" if delta >= 0 else ""
- delta_str = f"{d_sign}{delta:.1f}%"
- else:
- delta_str = "—"
- agent = "AGENT" if bench in AGENT_BENCHMARKS else "direct"
-
- print(f"qwen-{size:4} {bench:20} {b_str:>10} {d_str:>10} {delta_str:>10} {agent:>10}")
- summary_rows.append({
- "model": f"qwen-{size}",
- "benchmark": bench,
- "baseline": baseline,
- "distilled": distilled,
- "delta": distilled - baseline if (baseline is not None and distilled is not None) else None,
- "agent_benchmark": bench in AGENT_BENCHMARKS,
- })
- print()
-
- # Aggregate: agent vs direct benchmark deltas
- print("=" * 100)
- print("Aggregate deltas by benchmark type (paper finding)")
- print("=" * 100)
- agent_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and r["agent_benchmark"]]
- direct_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and not r["agent_benchmark"]]
- if agent_deltas:
- mean_agent = sum(agent_deltas) / len(agent_deltas)
- print(f"Agent benchmarks (PB, GAIA, DeepResearchBench): mean delta = {mean_agent:+.2f}% over {len(agent_deltas)} runs")
- if direct_deltas:
- mean_direct = sum(direct_deltas) / len(direct_deltas)
- print(f"Direct benchmarks (TC15, TauB, TBTel, LRB, LCB): mean delta = {mean_direct:+.2f}% over {len(direct_deltas)} runs")
- print()
-
- # Completion progress
- expected = 24
- distilled_count = sum(1 for r in summary_rows if r["distilled"] is not None)
- print(f"Distilled runs complete: {distilled_count}/{expected}")
-
- # Save JSON
- out = Path("results/neurips-2026/distillation-m2/m2_comparison.json")
- out.parent.mkdir(parents=True, exist_ok=True)
- out.write_text(json.dumps(summary_rows, indent=2, default=str))
- print(f"Full data: {out}")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/experiments/m2_create_distilled_configs.py b/scripts/experiments/m2_create_distilled_configs.py
deleted file mode 100644
index a52e2eda..00000000
--- a/scripts/experiments/m2_create_distilled_configs.py
+++ /dev/null
@@ -1,307 +0,0 @@
-#!/usr/bin/env python3
-"""M2: Create distilled eval configs from M1 consensus edits.
-
-Generates 24 distilled configs (3 models × 8 benchmarks) by cloning
-baseline configs and applying the consensus edits from M1. Also creates
-4 missing baseline configs (livecodebench-qwen-9b, liveresearchbench-*).
-
-Usage: python scripts/experiments/m2_create_distilled_configs.py
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-
-CONFIGS_DIR = Path("src/openjarvis/evals/configs")
-M2_DIR = CONFIGS_DIR / "distillation" / "m2"
-
-# ── Consensus values from M1 (1,131 edits) ──────────────────────────────
-DISTILLED_TEMP = 0.2 # 84/134 votes (agent benchmarks only)
-DISTILLED_MAX_TURNS = 15 # 56/125 votes (close: 25 had 49)
-REMOVE_TOOLS = {"shell_exec", "http_request"} # 13 + 6 votes
-
-# ── Model specs ──────────────────────────────────────────────────────────
-MODELS = {
- "2b": {"name": "Qwen/Qwen3.5-2B", "num_gpus": 1, "port": 8000},
- "9b": {"name": "Qwen/Qwen3.5-9B", "num_gpus": 1, "port": 8001},
- "27b": {"name": "Qwen/Qwen3.5-27B-FP8", "num_gpus": 1, "port": 8002},
-}
-
-# ── Benchmark specs ──────────────────────────────────────────────────────
-# Each benchmark defines its baseline config and what changes in distilled.
-BENCHMARKS = {
- "toolcall15": {
- "backend": "jarvis-direct",
- "baseline_temp": 0.0,
- "distilled_temp": 0.0, # CONTROL: no change for coding
- "max_tokens": 4096,
- "max_samples": None,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "extra_benchmark_fields": {},
- },
- "pinchbench": {
- "backend": "jarvis-agent",
- "agent": "native_openhands",
- "baseline_temp": 0.6,
- "distilled_temp": DISTILLED_TEMP,
- "max_tokens": 8192,
- "max_samples": None,
- "judge_model": "claude-opus-4-5",
- "judge_engine": "cloud",
- "baseline_tools": [
- "think", "file_read", "file_write", "web_search", "shell_exec",
- "code_interpreter", "browser_navigate", "image_generate",
- "calculator", "http_request", "pdf_extract",
- ],
- "extra_benchmark_fields": {},
- },
- "taubench": {
- "backend": "jarvis-direct",
- "baseline_temp": 0.7,
- "distilled_temp": DISTILLED_TEMP,
- "max_tokens": 4096,
- "max_samples": 20,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "extra_benchmark_fields": {"split": "airline,retail"},
- },
- "taubench-telecom": {
- "benchmark_name": "taubench", # same benchmark, different split
- "backend": "jarvis-direct",
- "baseline_temp": 0.7,
- "distilled_temp": DISTILLED_TEMP,
- "max_tokens": 4096,
- "max_samples": 20,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "extra_benchmark_fields": {"split": "telecom"},
- },
- "gaia": {
- "backend": "jarvis-agent",
- "agent": "monitor_operative",
- "baseline_temp": 0.6,
- "distilled_temp": DISTILLED_TEMP,
- "max_tokens": 8192,
- "max_samples": 50,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "baseline_tools": [
- "think", "calculator", "code_interpreter", "web_search", "file_read",
- ],
- "extra_benchmark_fields": {},
- },
- "liveresearch": {
- "backend": "jarvis-agent",
- "agent": "monitor_operative",
- "baseline_temp": 0.6,
- "distilled_temp": DISTILLED_TEMP,
- "max_tokens": 16384,
- "max_samples": 50,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "baseline_tools": [
- "web_search", "file_read", "file_write", "code_interpreter", "think",
- ],
- "extra_benchmark_fields": {},
- },
- "liveresearchbench": {
- "backend": "jarvis-direct",
- "baseline_temp": 0.0,
- "distilled_temp": 0.0, # CONTROL: reasoning benchmark
- "max_tokens": 8192,
- "max_samples": 50,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "extra_benchmark_fields": {},
- },
- "livecodebench": {
- "backend": "jarvis-direct",
- "baseline_temp": 0.0,
- "distilled_temp": 0.0, # CONTROL: coding benchmark
- "max_tokens": 4096,
- "max_samples": 20,
- "judge_model": "gpt-5-mini-2025-08-07",
- "judge_engine": "cloud",
- "extra_benchmark_fields": {},
- },
-}
-
-
-def render_config(
- *,
- comment: str,
- meta_name: str,
- description: str,
- temperature: float,
- max_tokens: int,
- judge_model: str,
- judge_engine: str,
- output_dir: str,
- model_name: str,
- model_engine: str,
- num_gpus: int,
- benchmark_name: str,
- backend: str,
- agent: str | None = None,
- tools: list[str] | None = None,
- max_samples: int | None = None,
- extra_benchmark: dict | None = None,
- seed: int = 42,
-) -> str:
- lines = [f"# {comment}"]
- lines.append(f'[meta]\nname = "{meta_name}"\ndescription = "{description}"\n')
- lines.append(f"[defaults]\ntemperature = {temperature}\nmax_tokens = {max_tokens}\n")
- lines.append(f'[judge]\nmodel = "{judge_model}"\ntemperature = 0.0')
- if judge_engine:
- lines.append(f'engine = "{judge_engine}"')
- lines.append(f"max_tokens = 4096\n")
- lines.append(f'[run]\nmax_workers = 1\noutput_dir = "{output_dir}"\nseed = {seed}\n')
- lines.append(f'[[models]]\nname = "{model_name}"\nengine = "{model_engine}"\nnum_gpus = {num_gpus}\n')
- lines.append(f'[[benchmarks]]\nname = "{benchmark_name}"\nbackend = "{backend}"')
- if agent:
- lines.append(f'agent = "{agent}"')
- if max_samples:
- lines.append(f"max_samples = {max_samples}")
- if tools:
- tools_str = ", ".join(f'"{t}"' for t in tools)
- lines.append(f"tools = [{tools_str}]")
- if extra_benchmark:
- for k, v in extra_benchmark.items():
- if isinstance(v, str):
- lines.append(f'{k} = "{v}"')
- else:
- lines.append(f"{k} = {v}")
- lines.append("")
- return "\n".join(lines)
-
-
-def make_size_label(size: str) -> str:
- return {"2b": "qwen-2b", "9b": "qwen-9b", "27b": "qwen-27b"}[size]
-
-
-def generate_missing_baselines() -> int:
- """Create baseline configs that don't exist yet."""
- count = 0
-
- # livecodebench-qwen-9b (missing)
- p = CONFIGS_DIR / "livecodebench-qwen-9b.toml"
- if not p.exists():
- b = BENCHMARKS["livecodebench"]
- m = MODELS["9b"]
- p.write_text(render_config(
- comment="LiveCodeBench eval: Qwen3.5-9B (vLLM, 1 GPU)",
- meta_name="livecodebench-qwen-9b",
- description="LiveCodeBench on Qwen/Qwen3.5-9B (vLLM, 1 GPU)",
- temperature=b["baseline_temp"],
- max_tokens=b["max_tokens"],
- judge_model=b["judge_model"],
- judge_engine=b["judge_engine"],
- output_dir="results/neurips-2026/baselines/qwen-9b/livecodebench/",
- model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"],
- benchmark_name="livecodebench", backend=b["backend"],
- max_samples=b["max_samples"],
- ))
- count += 1
- print(f" created {p}")
-
- # liveresearchbench-qwen-{2b,9b,27b}
- for size, m in MODELS.items():
- sl = make_size_label(size)
- p = CONFIGS_DIR / f"liveresearchbench-{sl}.toml"
- if not p.exists():
- b = BENCHMARKS["liveresearchbench"]
- p.write_text(render_config(
- comment=f"LiveResearchBench (Salesforce): Qwen3.5-{size.upper()} (vLLM)",
- meta_name=f"liveresearchbench-{sl}",
- description=f"LiveResearchBench on {m['name']} (vLLM)",
- temperature=b["baseline_temp"],
- max_tokens=b["max_tokens"],
- judge_model=b["judge_model"],
- judge_engine=b["judge_engine"],
- output_dir=f"results/neurips-2026/baselines/{sl}/liveresearchbench/",
- model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"],
- benchmark_name="liveresearchbench", backend=b["backend"],
- max_samples=b["max_samples"],
- ))
- count += 1
- print(f" created {p}")
-
- return count
-
-
-def generate_distilled_configs() -> int:
- """Create distilled configs for all 24 model × benchmark combos."""
- M2_DIR.mkdir(parents=True, exist_ok=True)
- count = 0
-
- for size, model in MODELS.items():
- sl = make_size_label(size)
- for bench_key, bench in BENCHMARKS.items():
- bench_name = bench.get("benchmark_name", bench_key)
- is_agent = bench["backend"] == "jarvis-agent"
- temp = bench["distilled_temp"]
-
- # Tool list: remove broken tools for agent benchmarks
- tools = None
- if is_agent and "baseline_tools" in bench:
- tools = [t for t in bench["baseline_tools"]
- if t not in REMOVE_TOOLS]
-
- fname = f"{bench_key}-{sl}-distilled.toml"
- out_path = M2_DIR / fname
-
- # Determine what changed for the comment
- changes = []
- if temp != bench["baseline_temp"]:
- changes.append(f"temp {bench['baseline_temp']}→{temp}")
- if tools and set(tools) != set(bench.get("baseline_tools", [])):
- removed = set(bench.get("baseline_tools", [])) - set(tools)
- changes.append(f"removed {removed}")
- if is_agent:
- changes.append(f"max_turns 10→{DISTILLED_MAX_TURNS} (via OPENJARVIS_CONFIG)")
- change_str = "; ".join(changes) if changes else "CONTROL (no change)"
-
- out_path.write_text(render_config(
- comment=f"M2 DISTILLED: {bench_key} × {model['name']} — {change_str}",
- meta_name=f"{bench_key}-{sl}-distilled",
- description=f"Distilled {bench_key} on {model['name']}",
- temperature=temp,
- max_tokens=bench["max_tokens"],
- judge_model=bench["judge_model"],
- judge_engine=bench["judge_engine"],
- output_dir=f"results/neurips-2026/distilled/{sl}/{bench_key}/",
- model_name=model["name"], model_engine="vllm", num_gpus=model["num_gpus"],
- benchmark_name=bench_name, backend=bench["backend"],
- agent=bench.get("agent"),
- tools=tools,
- max_samples=bench.get("max_samples"),
- extra_benchmark=bench.get("extra_benchmark_fields"),
- ))
- count += 1
-
- return count
-
-
-if __name__ == "__main__":
- print("=== Creating missing baseline configs ===")
- n_base = generate_missing_baselines()
- print(f"Created {n_base} missing baseline configs\n")
-
- print("=== Creating distilled M2 configs ===")
- n_dist = generate_distilled_configs()
- print(f"Created {n_dist} distilled configs in {M2_DIR}/\n")
-
- # Summary
- print("=== Change matrix ===")
- print(f"{'Benchmark':20} {'Backend':14} {'Temp Δ':12} {'Tool Δ':20} {'max_turns Δ':12}")
- print("-" * 80)
- for bk, b in BENCHMARKS.items():
- is_agent = b["backend"] == "jarvis-agent"
- temp_change = f"{b['baseline_temp']}→{b['distilled_temp']}" if b["baseline_temp"] != b["distilled_temp"] else "—"
- tool_change = "—"
- if is_agent and "baseline_tools" in b:
- removed = REMOVE_TOOLS & set(b.get("baseline_tools", []))
- tool_change = f"-{removed}" if removed else "—"
- mt_change = f"10→{DISTILLED_MAX_TURNS}" if is_agent else "—"
- print(f"{bk:20} {b['backend']:14} {temp_change:12} {str(tool_change):20} {mt_change:12}")
diff --git a/scripts/experiments/m2_run_distilled_evals.sh b/scripts/experiments/m2_run_distilled_evals.sh
deleted file mode 100644
index 9e17e48b..00000000
--- a/scripts/experiments/m2_run_distilled_evals.sh
+++ /dev/null
@@ -1,183 +0,0 @@
-#!/usr/bin/env bash
-# ──────────────────────────────────────────────────────────────────────────────
-# M2: Run distilled eval configs — resumable, agent-benchmarks-first ordering
-#
-# Assumes vLLM already running: 2B:8000, 9B:8001, 27B-FP8:8002
-#
-# Usage:
-# bash m2_run_distilled_evals.sh # all
-# bash m2_run_distilled_evals.sh 9b # 9b only
-# bash m2_run_distilled_evals.sh 9b gaia # 9b + gaia only
-# ──────────────────────────────────────────────────────────────────────────────
-
-set -uo pipefail
-
-VENV=".venv/bin/python"
-M2_CONFIGS="src/openjarvis/evals/configs/distillation/m2"
-BASELINE_CONFIGS="src/openjarvis/evals/configs"
-M2_HOME="/scratch/user/jonsaadfalcon/openjarvis-m2"
-MODEL_FILTER=${1:-all}
-BENCH_FILTER=${2:-all}
-FORCE=${FORCE:-0} # set FORCE=1 to re-run completed configs
-
-RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
-log() { echo -e "${BLUE}[m2]${NC} $*"; }
-ok() { echo -e "${GREEN}[ OK ]${NC} $*"; }
-warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
-fail() { echo -e "${RED}[FAIL]${NC} $*"; }
-skip() { echo -e "${YELLOW}[SKIP]${NC} $*"; }
-
-declare -A MODEL_PORT=( [2b]=8000 [9b]=8001 [27b]=8002 )
-
-# Agent benchmarks FIRST (where distillation impact is expected)
-AGENT_BENCHMARKS="pinchbench gaia liveresearch"
-DIRECT_BENCHMARKS="toolcall15 taubench taubench-telecom livecodebench liveresearchbench"
-ALL_BENCHMARKS="${AGENT_BENCHMARKS} ${DIRECT_BENCHMARKS}"
-
-check_vllm() {
- for size in 2b 9b 27b; do
- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
- local port=${MODEL_PORT[$size]}
- if ! curl -sf "http://localhost:${port}/v1/models" >/dev/null 2>&1; then
- fail "vLLM ${size} not responding on port ${port}"
- return 1
- fi
- ok "vLLM ${size} healthy on port ${port}"
- done
-}
-
-# Check if a run already completed (summary.json exists AND has a real accuracy).
-# A summary with errors=total_samples (like my earlier broken-routing tests)
-# is treated as incomplete and re-run.
-is_complete() {
- local summary_path=$1
- [ -f "$summary_path" ] || return 1
- python3 -c "
-import json, sys
-try:
- d = json.load(open('$summary_path'))
- total = d.get('total_samples', 0)
- errors = d.get('errors', 0)
- scored = d.get('scored_samples', 0)
- # Complete if at least some samples were scored successfully
- sys.exit(0 if scored > 0 else 1)
-except Exception:
- sys.exit(1)
-" 2>/dev/null
-}
-
-run_eval() {
- local config_path=$1 label=$2 size=$3 summary_path=$4 use_distilled=${5:-false}
-
- if [ "$FORCE" != "1" ] && is_complete "$summary_path"; then
- skip "${label} [already complete]"
- return 0
- fi
-
- local oj_config="${M2_HOME}/config-baseline-${size}.toml"
- [ "$use_distilled" = "true" ] && oj_config="${M2_HOME}/config-${size}.toml"
-
- log "Running: ${label} [$(basename ${oj_config})]"
- env OPENJARVIS_CONFIG="${oj_config}" ${VENV} -m openjarvis.evals run -c "${config_path}" 2>&1
- local rc=$?
- if [ $rc -eq 0 ] && is_complete "$summary_path"; then
- ok "Done: ${label}"
- else
- warn "Failed: ${label} (rc=$rc)"
- fi
-}
-
-# Derive the expected summary.json path for a given distilled config
-summary_for_distilled() {
- local bench=$1 size=$2
- # Output dir from the config template: results/neurips-2026/distilled/qwen-{size}/{bench}/
- # Summary file pattern: {bench}_Qwen-Qwen3.5-{size}.summary.json
- local model_slug
- if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8"
- else model_slug="Qwen-Qwen3.5-${size}"; fi
- # Uppercase B for the model slug
- model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g')
- # For taubench-telecom, the benchmark name in output is "taubench" not "taubench-telecom"
- local bench_fname=$bench
- [ "$bench" = "taubench-telecom" ] && bench_fname="taubench"
- echo "results/neurips-2026/distilled/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json"
-}
-
-summary_for_baseline() {
- local bench=$1 size=$2
- local model_slug
- if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8"
- else model_slug="Qwen-Qwen3.5-${size}"; fi
- model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g')
- local bench_fname=$bench
- [ "$bench" = "taubench-telecom" ] && bench_fname="taubench"
- echo "results/neurips-2026/baselines/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json"
-}
-
-log "M2 Distilled Eval Runner (resumable, agent-first)"
-log "Model filter: ${MODEL_FILTER} Benchmark filter: ${BENCH_FILTER} FORCE=${FORCE}"
-
-check_vllm || exit 1
-
-start_time=$(date +%s)
-
-# Phase B1: DISTILLED agent benchmarks (highest-priority: PinchBench, GAIA, DeepResearchBench)
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-log "Phase B1: DISTILLED agent benchmarks (9 runs — the critical data)"
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-for size in 2b 9b 27b; do
- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
- for bench in ${AGENT_BENCHMARKS}; do
- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue
- cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml"
- sum=$(summary_for_distilled "$bench" "$size")
- [ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true
- done
-done
-
-# Phase B2: DISTILLED direct benchmarks (controls — should show minimal delta)
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-log "Phase B2: DISTILLED direct benchmarks (15 runs — controls)"
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-for size in 2b 9b 27b; do
- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
- for bench in ${DIRECT_BENCHMARKS}; do
- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue
- cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml"
- sum=$(summary_for_distilled "$bench" "$size")
- [ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true
- done
-done
-
-# Phase A: LiveResearchBench baselines (last, since Step 1 baselines exist for other benchmarks)
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-log "Phase A: LiveResearchBench baselines (3 runs — new benchmark only)"
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-for size in 2b 9b 27b; do
- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "liveresearchbench" ] && continue
- cfg="${BASELINE_CONFIGS}/liveresearchbench-qwen-${size}.toml"
- sum=$(summary_for_baseline "liveresearchbench" "$size")
- [ -f "$cfg" ] && run_eval "$cfg" "BASELINE liveresearchbench-qwen-${size}" "${size}" "$sum" false
-done
-
-# Phase C: Spot-check 2 baselines against Jon's Step 1 numbers
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-log "Phase C: Spot-check baselines (TC15-9B, GAIA-9B)"
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-if [ "$MODEL_FILTER" = "all" ] || [ "$MODEL_FILTER" = "9b" ]; then
- for bench in toolcall15 gaia; do
- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue
- cfg="${BASELINE_CONFIGS}/${bench}-qwen-9b.toml"
- sum=$(summary_for_baseline "$bench" "9b")
- [ -f "$cfg" ] && run_eval "$cfg" "SPOTCHECK ${bench}-qwen-9b" "9b" "$sum" false
- done
-fi
-
-end_time=$(date +%s)
-elapsed=$((end_time - start_time))
-
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-ok "M2 complete in ${elapsed}s ($(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"
-log "Distilled results: results/neurips-2026/distilled/"
-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
diff --git a/scripts/experiments/m3_hill_climb.py b/scripts/experiments/m3_hill_climb.py
deleted file mode 100644
index 3c785182..00000000
--- a/scripts/experiments/m3_hill_climb.py
+++ /dev/null
@@ -1,581 +0,0 @@
-#!/usr/bin/env python3
-"""M3: Empirical hill-climbing with an LLM proposer.
-
-Replaces M1's open-loop "aggregate consensus across sessions" with a closed
-loop:
-
- For each target (student, benchmark, agent):
- for round in 1..N:
- edit = teacher.propose_one(history_with_measured_deltas)
- score_new = eval_subsample(apply(edit))
- if score_new > current_score: accept
-
-Every proposal is empirically verified before the next is proposed, and the
-teacher sees measured deltas (not just traces) in its context.
-
-Usage:
- python scripts/experiments/m3_hill_climb.py \\
- --student 9b --benchmark liveresearch \\
- --rounds 4 --k-subsample 8 --k-final 30
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import re
-import subprocess
-import sys
-import tempfile
-import time
-from dataclasses import asdict, dataclass, field
-from pathlib import Path
-from typing import Any
-
-from openjarvis.core.types import Message, Role
-from openjarvis.engine.cloud import CloudEngine
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# Config & constants
-# ═══════════════════════════════════════════════════════════════════════════
-
-STUDENT = {
- "2b": {"name": "Qwen/Qwen3.5-2B", "port": 8000, "gpu": 4},
- "9b": {"name": "Qwen/Qwen3.5-9B", "port": 8001, "gpu": 5},
- "27b": {"name": "Qwen/Qwen3.5-27B-FP8", "port": 8002, "gpu": 6},
-}
-
-# Per-benchmark defaults (backend, baseline config)
-BENCHMARK = {
- "liveresearch": {
- "backend": "jarvis-agent",
- "agent": "monitor_operative",
- "baseline_temp": 0.6,
- "baseline_max_tokens": 16384,
- "baseline_max_turns": 10,
- "baseline_tools": ["web_search", "file_read", "file_write",
- "code_interpreter", "think"],
- "max_samples_final": 50, # final eval
- "judge": "gpt-5-mini-2025-08-07",
- },
- "gaia": {
- "backend": "jarvis-agent",
- "agent": "monitor_operative",
- "baseline_temp": 0.6,
- "baseline_max_tokens": 8192,
- "baseline_max_turns": 10,
- "baseline_tools": ["think", "calculator", "code_interpreter",
- "web_search", "file_read"],
- "max_samples_final": 50,
- "judge": "gpt-5-mini-2025-08-07",
- },
- "pinchbench": {
- "backend": "jarvis-agent",
- "agent": "native_openhands",
- "baseline_temp": 0.6,
- "baseline_max_tokens": 8192,
- "baseline_max_turns": 10,
- "baseline_tools": ["think", "file_read", "file_write", "web_search",
- "shell_exec", "code_interpreter", "browser_navigate",
- "image_generate", "calculator", "http_request",
- "pdf_extract"],
- "max_samples_final": 23,
- "judge": "claude-opus-4-5",
- },
-}
-
-AVAILABLE_TOOLS = [
- "think", "file_read", "file_write", "web_search", "shell_exec",
- "code_interpreter", "browser_navigate", "image_generate", "calculator",
- "http_request", "pdf_extract", "pdf_reader", "list_directory",
-]
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# Proposer (LLM)
-# ═══════════════════════════════════════════════════════════════════════════
-
-
-PROPOSER_SYSTEM = """\
-You are optimizing an OpenJarvis agent configuration for maximum accuracy on \
-a benchmark. You propose ONE config edit per round. After each proposal, the \
-edit is applied and the benchmark is run on a subsample; you then see the \
-measured score delta and decide the next edit.
-
-Your job is to find the config that maximizes measured accuracy.
-
-EDIT GRAMMAR — return JSON with "op" and the parameter fields at top level:
-
-{"op": "<op_name>", <param_fields>, "rationale": "<one sentence>"}
-
-Valid ops and their parameter fields:
-
-1. set_temperature: "value" (float, 0.0..1.0)
-2. set_max_turns: "value" (int, 1..100)
-3. set_max_tokens: "value" (int, 512..32768)
-4. add_tool: "tool_name" (string; must be in AVAILABLE_TOOLS, not already active)
-5. remove_tool: "tool_name" (string; must be in current tools)
-6. noop: (no params; propose only if you believe no further edit will help)
-
-CONCRETE EXAMPLES:
- {"op": "set_temperature", "value": 0.3, "rationale": "Reduce loop risk."}
- {"op": "set_max_turns", "value": 20, "rationale": "More turns for research tasks."}
- {"op": "add_tool", "tool_name": "pdf_extract", "rationale": "Tasks require PDF reading."}
- {"op": "remove_tool", "tool_name": "shell_exec", "rationale": "Tool is broken in this env."}
- {"op": "noop", "rationale": "Current config seems optimal."}
-
-EXPLORATION BIAS:
- The config space has 5 distinct axes: temperature, max_turns, max_tokens,
- tool additions (add_tool), tool removals (remove_tool). Before proposing a
- second edit on an axis you've already tried, consider whether an untried
- axis might reveal a larger gain. Tool-list edits (add/remove) often matter
- more than numeric hyperparameters on benchmarks where the agent uses tools.
-
-Return ONLY the JSON object, no preamble, no code fences."""
-
-
-def build_user_prompt(
- *, benchmark: str, student: str, agent: str,
- baseline_score: float, current_score: float, current_config: dict,
- edit_history: list[dict], available_tools: list[str],
- sample_queries: list[str],
-) -> str:
- hist_lines = []
- for i, e in enumerate(edit_history, 1):
- delta = e["score_after"] - e["score_before"]
- status = "ACCEPTED" if e["accepted"] else "REJECTED"
- hist_lines.append(
- f" Round {i}: {json.dumps(e['edit'])} "
- f"→ score {e['score_before']:.1f}% → {e['score_after']:.1f}% "
- f"(Δ {delta:+.1f}, {status})"
- )
- hist = "\n".join(hist_lines) if hist_lines else " (no edits yet — baseline is the starting point)"
-
- samples = "\n".join(f" - {q[:150]}..." for q in sample_queries[:3])
-
- unused_tools = [t for t in available_tools if t not in current_config["tools"]]
-
- return f"""\
-TARGET:
- student: {student} (vLLM-served Qwen3.5)
- benchmark: {benchmark}
- agent: {agent} (uses OpenAI-format structured tool calls)
-
-CURRENT CONFIG:
- temperature = {current_config['temperature']}
- max_turns = {current_config['max_turns']}
- max_tokens = {current_config['max_tokens']}
- tools = {current_config['tools']}
-
-TOOLS NOT CURRENTLY ACTIVE (available to add):
- {unused_tools}
-
-BASELINE (unedited) SCORE: {baseline_score:.1f}%
-CURRENT BEST SCORE: {current_score:.1f}%
-
-EDIT HISTORY (with measured deltas):
-{hist}
-
-SAMPLE TASKS FROM THIS BENCHMARK:
-{samples}
-
-Propose ONE edit that you predict will improve the measured accuracy. Consider \
-the edit history — do not repeat proposals that were rejected. If you believe \
-further edits will not help, propose noop.
-
-Return JSON only."""
-
-
-def call_proposer(
- engine: CloudEngine, system: str, user: str, model: str = "claude-sonnet-4-6"
-) -> dict:
- resp = engine.generate(
- messages=[
- Message(role=Role.SYSTEM, content=system),
- Message(role=Role.USER, content=user),
- ],
- model=model, max_tokens=600, temperature=0.3,
- )
- content = (resp.get("content") or "").strip()
- # Be forgiving about code fences
- if content.startswith("```"):
- content = re.sub(r"^```(?:json)?\s*", "", content)
- content = re.sub(r"\s*```\s*$", "", content)
- # Find the JSON object
- m = re.search(r"\{[\s\S]*\}", content)
- if not m:
- raise ValueError(f"No JSON object found in proposer output: {content[:200]}")
- return json.loads(m.group(0))
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# Edit application & evaluation
-# ═══════════════════════════════════════════════════════════════════════════
-
-
-@dataclass
-class Config:
- temperature: float
- max_turns: int
- max_tokens: int
- tools: list[str]
-
-
-def apply_edit(cfg: Config, edit: dict) -> Config:
- """Apply an edit. Tolerant of both flat and nested (params) forms."""
- op = edit["op"]
- # Merge top-level edit fields with params for flat-or-nested tolerance
- p = {**edit.get("params", {}), **{k: v for k, v in edit.items()
- if k not in ("op", "params", "rationale")}}
- new = Config(
- temperature=cfg.temperature, max_turns=cfg.max_turns,
- max_tokens=cfg.max_tokens, tools=list(cfg.tools),
- )
- if op == "noop":
- return new
- if op == "set_temperature":
- new.temperature = float(p["value"])
- elif op == "set_max_turns":
- new.max_turns = int(p["value"])
- elif op == "set_max_tokens":
- new.max_tokens = int(p["value"])
- elif op == "add_tool":
- tool = p["tool_name"]
- if tool not in new.tools:
- new.tools.append(tool)
- elif op == "remove_tool":
- tool = p["tool_name"]
- new.tools = [t for t in new.tools if t != tool]
- else:
- raise ValueError(f"unknown edit op: {op}")
- return new
-
-
-def write_eval_toml(
- *, bench: str, bench_spec: dict, student: dict, cfg: Config,
- k_samples: int, output_dir: Path,
-) -> Path:
- agent_line = f'agent = "{bench_spec["agent"]}"' if bench_spec.get("agent") else ""
- tools_str = "[" + ", ".join(f'"{t}"' for t in cfg.tools) + "]"
- toml = f"""\
-[meta]
-name = "m3-{bench}-{student['name'].replace('/', '-')}"
-description = "M3 hill-climb round"
-
-[defaults]
-temperature = {cfg.temperature}
-max_tokens = {cfg.max_tokens}
-
-[judge]
-model = "{bench_spec['judge']}"
-temperature = 0.0
-max_tokens = 4096
-engine = "cloud"
-
-[run]
-max_workers = 1
-output_dir = "{output_dir}"
-seed = 42
-
-[[models]]
-name = "{student['name']}"
-engine = "vllm"
-num_gpus = 1
-
-[[benchmarks]]
-name = "{bench}"
-backend = "{bench_spec['backend']}"
-{agent_line}
-max_samples = {k_samples}
-tools = {tools_str}
-"""
- path = output_dir / "eval.toml"
- output_dir.mkdir(parents=True, exist_ok=True)
- path.write_text(toml)
- return path
-
-
-def write_openjarvis_config(home: Path, port: int, max_turns: int) -> Path:
- p = home / "global-config.toml"
- p.parent.mkdir(parents=True, exist_ok=True)
- p.write_text(f"""\
-[agent]
-max_turns = {max_turns}
-
-[engine]
-default = "vllm"
-
-[engine.vllm]
-host = "http://localhost:{port}"
-""")
- return p
-
-
-def run_eval(eval_toml: Path, oj_config: Path) -> tuple[float, int, int]:
- """Run one eval. Returns (accuracy_pct, scored, total)."""
- env = {**os.environ, "OPENJARVIS_CONFIG": str(oj_config)}
- result = subprocess.run(
- [".venv/bin/python", "-m", "openjarvis.evals", "run", "-c", str(eval_toml)],
- capture_output=True, text=True, env=env, timeout=7200,
- )
- # Find the summary.json
- out_dir = eval_toml.parent
- sums = list(out_dir.glob("**/*.summary.json"))
- if not sums:
- print(f"[m3] WARNING: no summary.json in {out_dir}")
- print(f"[m3] stderr tail: {result.stderr[-500:]}")
- return 0.0, 0, 0
- d = json.loads(sums[0].read_text())
- acc = d.get("accuracy", 0.0)
- acc_pct = acc * 100 if acc <= 1.0 else acc
- return acc_pct, d.get("scored_samples", 0), d.get("total_samples", 0)
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# Benchmark sample loader (for proposer context)
-# ═══════════════════════════════════════════════════════════════════════════
-
-
-def load_sample_queries(bench: str, n: int = 3) -> list[str]:
- try:
- if bench == "liveresearch":
- from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset
- ds = LiveResearchBenchDataset()
- ds.load(max_samples=n)
- elif bench == "pinchbench":
- from openjarvis.evals.datasets.pinchbench import PinchBenchDataset
- ds = PinchBenchDataset()
- elif bench == "gaia":
- from openjarvis.evals.datasets.gaia import GAIADataset
- ds = GAIADataset()
- if hasattr(ds, "load"):
- ds.load(max_samples=n)
- else:
- return []
- return [r.problem for r in list(ds.iter_records())[:n]]
- except Exception as e:
- print(f"[m3] WARNING: could not load samples for {bench}: {e}")
- return []
-
-
-# ═══════════════════════════════════════════════════════════════════════════
-# Main hill-climb loop
-# ═══════════════════════════════════════════════════════════════════════════
-
-
-def hill_climb(args) -> dict:
- bench_spec = BENCHMARK[args.benchmark]
- student = STUDENT[args.student]
-
- # State dir (resumable)
- base_dir = Path(args.out_dir) / f"{args.student}-{args.benchmark}"
- base_dir.mkdir(parents=True, exist_ok=True)
- state_path = base_dir / "state.json"
-
- # Load or initialize state
- if state_path.exists() and not args.fresh:
- state = json.loads(state_path.read_text())
- print(f"[m3] Resumed state from round {len(state['history'])}/{args.rounds}")
- else:
- baseline_cfg_dict = {
- "temperature": bench_spec["baseline_temp"],
- "max_turns": bench_spec["baseline_max_turns"],
- "max_tokens": bench_spec["baseline_max_tokens"],
- "tools": list(bench_spec["baseline_tools"]),
- }
- # ALWAYS measure the baseline today first (unless user provides --trust-baseline).
- # This prevents anchoring to an unreproducible Step 1 number.
- # We measure at k=k_final so the final delta is like-for-like.
- if args.trust_baseline and args.baseline_score is not None:
- measured_baseline = args.baseline_score
- measured_baseline_k = None
- print(f"[m3] Trusting provided baseline score: {measured_baseline:.1f}% "
- f"(--trust-baseline set)")
- else:
- print(f"[m3] Measuring today's baseline with k={args.k_final} "
- f"(matches k_final for clean like-for-like delta)...")
- bl_dir = base_dir / "baseline_measure"
- bl_toml = write_eval_toml(
- bench=args.benchmark, bench_spec=bench_spec, student=student,
- cfg=Config(**baseline_cfg_dict),
- k_samples=args.k_final, output_dir=bl_dir,
- )
- bl_oj = write_openjarvis_config(bl_dir, student["port"], baseline_cfg_dict["max_turns"])
- t0 = time.monotonic()
- measured_baseline, bl_scored, bl_total = run_eval(bl_toml, bl_oj)
- measured_baseline_k = bl_scored
- print(f"[m3] Measured baseline: {measured_baseline:.1f}% "
- f"({bl_scored}/{bl_total}) in {(time.monotonic() - t0)/60:.1f} min")
- if args.baseline_score is not None:
- print(f"[m3] (reference: --baseline-score was {args.baseline_score:.1f}%, "
- f"drift = {measured_baseline - args.baseline_score:+.1f})")
-
- state = {
- "args": vars(args),
- "benchmark": args.benchmark,
- "student": student["name"],
- "agent": bench_spec["agent"],
- "baseline_config": baseline_cfg_dict,
- "baseline_score": measured_baseline,
- "baseline_score_reference": args.baseline_score,
- "current_config": dict(baseline_cfg_dict),
- "current_score": measured_baseline,
- "history": [],
- }
-
- # Save state helper
- def save():
- state_path.write_text(json.dumps(state, indent=2, default=str))
-
- save()
-
- # Sample queries for proposer context
- sample_queries = load_sample_queries(args.benchmark)
-
- engine = CloudEngine()
-
- # Hill-climb rounds
- for round_num in range(len(state["history"]) + 1, args.rounds + 1):
- print(f"\n{'═' * 70}")
- print(f"[m3] Round {round_num}/{args.rounds}")
- print(f"[m3] current_score = {state['current_score']:.1f}%")
- print(f"[m3] current_config = {state['current_config']}")
-
- # Propose
- user_prompt = build_user_prompt(
- benchmark=args.benchmark, student=student["name"],
- agent=bench_spec["agent"],
- baseline_score=state["baseline_score"],
- current_score=state["current_score"],
- current_config=state["current_config"],
- edit_history=state["history"],
- available_tools=AVAILABLE_TOOLS,
- sample_queries=sample_queries,
- )
- try:
- edit = call_proposer(engine, PROPOSER_SYSTEM, user_prompt,
- model=args.proposer_model)
- except Exception as e:
- print(f"[m3] Proposer failed: {e}. Ending hill-climb.")
- break
-
- print(f"[m3] Proposed: {json.dumps(edit)}")
-
- if edit.get("op") == "noop":
- print(f"[m3] Teacher proposed noop; stopping.")
- break
-
- # Apply and evaluate subsample
- try:
- candidate = apply_edit(Config(**state["current_config"]), edit)
- except Exception as e:
- print(f"[m3] apply_edit failed: {e}. Recording as malformed edit.")
- # Record as a rejected malformed edit so teacher won't repeat
- state["history"].append({
- "round": round_num, "edit": edit,
- "config_after": None,
- "score_before": state["current_score"],
- "score_after": state["current_score"], # no change
- "scored": 0, "total": 0, "elapsed_seconds": 0,
- "accepted": False,
- "error": f"malformed_edit: {e}",
- })
- save()
- continue
-
- round_dir = base_dir / f"round_{round_num}"
- eval_toml = write_eval_toml(
- bench=args.benchmark, bench_spec=bench_spec, student=student,
- cfg=candidate, k_samples=args.k_subsample,
- output_dir=round_dir,
- )
- oj_cfg = write_openjarvis_config(round_dir, student["port"], candidate.max_turns)
-
- print(f"[m3] Running k={args.k_subsample} subsample eval...")
- t0 = time.monotonic()
- acc, scored, total = run_eval(eval_toml, oj_cfg)
- elapsed = time.monotonic() - t0
- print(f"[m3] Subsample score: {acc:.1f}% ({scored}/{total}) in {elapsed/60:.1f} min")
-
- score_before = state["current_score"]
- delta = acc - score_before
- accepted = delta > args.accept_threshold
-
- # Record history
- state["history"].append({
- "round": round_num, "edit": edit,
- "config_after": asdict(candidate),
- "score_before": score_before, "score_after": acc,
- "scored": scored, "total": total, "elapsed_seconds": elapsed,
- "accepted": accepted,
- })
-
- if accepted:
- state["current_config"] = asdict(candidate)
- state["current_score"] = acc
- print(f"[m3] ACCEPTED (Δ={delta:+.1f})")
- else:
- print(f"[m3] REJECTED (Δ={delta:+.1f} ≤ {args.accept_threshold})")
-
- save()
-
- # Final eval with current config
- print(f"\n{'═' * 70}")
- print(f"[m3] Final eval with best config: {state['current_config']}")
- final_dir = base_dir / "final"
- final_cfg = Config(**state["current_config"])
- eval_toml = write_eval_toml(
- bench=args.benchmark, bench_spec=bench_spec, student=student,
- cfg=final_cfg, k_samples=args.k_final,
- output_dir=final_dir,
- )
- oj_cfg = write_openjarvis_config(final_dir, student["port"], final_cfg.max_turns)
-
- t0 = time.monotonic()
- final_acc, final_scored, final_total = run_eval(eval_toml, oj_cfg)
- elapsed = time.monotonic() - t0
-
- state["final_score"] = final_acc
- state["final_scored"] = final_scored
- state["final_total"] = final_total
- state["final_elapsed_seconds"] = elapsed
- save()
-
- print(f"\n{'═' * 70}")
- print(f"[m3] DONE")
- print(f"[m3] baseline = {state['baseline_score']:.1f}%")
- print(f"[m3] final = {final_acc:.1f}% ({final_scored}/{final_total}) in {elapsed/60:.1f} min")
- print(f"[m3] Δ vs baseline = {final_acc - state['baseline_score']:+.1f}")
- print(f"[m3] state: {state_path}")
-
- return state
-
-
-def main() -> int:
- ap = argparse.ArgumentParser()
- ap.add_argument("--student", required=True, choices=list(STUDENT))
- ap.add_argument("--benchmark", required=True, choices=list(BENCHMARK))
- ap.add_argument("--rounds", type=int, default=4)
- ap.add_argument("--k-subsample", type=int, default=8)
- ap.add_argument("--k-final", type=int, default=30)
- ap.add_argument("--baseline-score", type=float, default=None,
- help="Optional reference baseline (shown alongside measured). "
- "Hill-climb always measures today's baseline unless --trust-baseline.")
- ap.add_argument("--trust-baseline", action="store_true",
- help="Skip baseline re-measurement, trust --baseline-score.")
- ap.add_argument("--accept-threshold", type=float, default=0.0,
- help="Accept edit if score Δ > this (default: 0)")
- ap.add_argument("--proposer-model", default="claude-sonnet-4-6")
- ap.add_argument("--out-dir", default="results/neurips-2026/distillation-m3")
- ap.add_argument("--fresh", action="store_true",
- help="Overwrite existing state and start fresh")
- args = ap.parse_args()
-
- result = hill_climb(args)
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/experiments/run_distillation_experiments.sh b/scripts/experiments/run_distillation_experiments.sh
deleted file mode 100755
index 961db01c..00000000
--- a/scripts/experiments/run_distillation_experiments.sh
+++ /dev/null
@@ -1,307 +0,0 @@
-#!/usr/bin/env bash
-# ──────────────────────────────────────────────────────────────────────────────
-# Run distillation ablation experiments
-#
-# Prerequisites:
-# - Ollama running with qwen3.5:{2b,9b,27b}
-# - ANTHROPIC_API_KEY set (for Opus teacher)
-# - OPENAI_API_KEY set (for GPT-5.4 teacher)
-# - GOOGLE_API_KEY set (for Gemini teacher)
-# - For Qwen-397B teacher: vLLM serving on port 8010 with 8×H100
-# - Traces seeded with feedback (run A1 blocker first)
-# - jarvis learning init already run
-#
-# Usage:
-# bash scripts/experiments/run_distillation_experiments.sh # Run all
-# bash scripts/experiments/run_distillation_experiments.sh exp1a # Run Phase 1a only
-# bash scripts/experiments/run_distillation_experiments.sh exp1a opus # Single config
-# ──────────────────────────────────────────────────────────────────────────────
-
-set -euo pipefail
-
-CONFIGS_DIR="src/openjarvis/evals/configs/distillation"
-RESULTS_DIR="results/neurips-2026/agent-optimization/distillation"
-EXPERIMENT=${1:-all}
-FILTER=${2:-}
-
-# ── Colors ───────────────────────────────────────────────────────────────────
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m'
-
-log() { echo -e "${BLUE}[distill]${NC} $*"; }
-ok() { echo -e "${GREEN}[ OK ]${NC} $*"; }
-warn() { echo -e "${YELLOW}[ WARN ]${NC} $*"; }
-fail() { echo -e "${RED}[ FAIL ]${NC} $*"; }
-
-# ── Preflight checks ────────────────────────────────────────────────────────
-check_prereqs() {
- log "Preflight checks..."
-
- # Check API keys
- if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
- warn "ANTHROPIC_API_KEY not set — Opus teacher experiments will fail"
- fi
- if [ -z "${OPENAI_API_KEY:-}" ]; then
- warn "OPENAI_API_KEY not set — GPT-5.4 teacher experiments will fail"
- fi
- if [ -z "${GOOGLE_API_KEY:-}" ]; then
- warn "GOOGLE_API_KEY not set — Gemini teacher experiments will fail"
- fi
-
- # Check Ollama
- if ! ollama list &>/dev/null; then
- fail "Ollama not running. Start it first."
- exit 1
- fi
-
- # Check student models
- for model in qwen3.5:2b qwen3.5:9b qwen3.5:27b; do
- if ! ollama list 2>/dev/null | grep -q "$model"; then
- warn "Model $model not found in Ollama. Pull with: ollama pull $model"
- fi
- done
-
- # Check distillation init
- if [ ! -d "$HOME/.openjarvis/learning" ]; then
- log "Running jarvis learning init..."
- uv run jarvis learning init
- fi
-
- ok "Preflight complete"
-}
-
-# ── Run a single distillation session ────────────────────────────────────────
-run_session() {
- local config_file=$1
- local experiment_name
- experiment_name=$(basename "$(dirname "$config_file")")
- local config_name
- config_name=$(basename "${config_file%.toml}")
- local output_dir="${RESULTS_DIR}/${experiment_name}/${config_name}"
-
- # Skip if already completed
- if [ -f "${output_dir}/session/session.json" ]; then
- ok "SKIP ${experiment_name}/${config_name} (already done)"
- return 0
- fi
-
- log "──────────────────────────────────────────────────────"
- log "Experiment: ${experiment_name}/${config_name}"
- log "Config: ${config_file}"
- log "Output: ${output_dir}"
- log "──────────────────────────────────────────────────────"
-
- mkdir -p "${output_dir}"
-
- # Extract metadata from config
- local teacher_model
- teacher_model=$(grep 'teacher_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
- local student_model
- student_model=$(grep 'default_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
- local benchmark
- benchmark=$(grep '^benchmark ' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
- local data_config
- data_config=$(grep 'data_config' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
- local iterative
- iterative=$(grep 'iterative_sessions' "$config_file" | head -1 | sed 's/.*= *//')
-
- log "Teacher: ${teacher_model}"
- log "Student: ${student_model}"
- log "Data: ${data_config:-C2}"
- log "Iter: ${iterative:-1}"
-
- # ── Step 1: Seed traces based on data config ─────────────────────────
- # (In a full implementation, this would filter/prepare the TraceStore
- # based on C1/C2/C3. For now we use whatever traces exist.)
-
- # ── Step 2: Run distillation session ─────────────────────────────────
- local n_sessions=${iterative:-1}
- local session_num=1
- local prev_session_id=""
-
- while [ "$session_num" -le "$n_sessions" ]; do
- log "Session ${session_num}/${n_sessions}..."
-
- local session_output="${output_dir}/session_${session_num}"
- mkdir -p "${session_output}"
-
- # Run the distillation session via Python
- # (jarvis learning run doesn't support all config params yet,
- # so we call the orchestrator directly)
- uv run python << PYEOF > "${session_output}/run.log" 2>&1 || true
-import json, os, shutil, sys
-from pathlib import Path
-
-from openjarvis.engine.cloud import CloudEngine
-from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend
-from openjarvis.traces.store import TraceStore
-from openjarvis.learning.distillation.checkpoint.store import CheckpointStore
-from openjarvis.learning.distillation.models import AutonomyMode
-from openjarvis.learning.distillation.orchestrator import DistillationOrchestrator
-from openjarvis.learning.distillation.storage.session_store import SessionStore
-from openjarvis.learning.distillation.student_runner import (
- VLLMStudentRunner,
- build_benchmark_samples_from_traces,
-)
-from openjarvis.learning.distillation.triggers import OnDemandTrigger
-from openjarvis.learning.optimize.feedback.judge import TraceJudge
-
-home = Path(os.environ.get("OPENJARVIS_HOME", str(Path.home() / ".openjarvis")))
-
-# Read config params
-teacher_model = "${teacher_model}"
-student_model = "${student_model}"
-autonomy = "auto"
-max_cost = float("$(grep 'max_cost_per_session_usd' "$config_file" | head -1 | sed 's/.*= *//')")
-max_tools = int("$(grep 'max_tool_calls_per_diagnosis' "$config_file" | head -1 | sed 's/.*= *//')")
-
-# Real student runner via vLLM
-vllm_host = os.environ.get("VLLM_HOST", "http://localhost:8001")
-student_runner = VLLMStudentRunner(
- host=vllm_host,
- model=student_model,
-)
-
-# Real judge via cloud LLM
-cloud_engine = CloudEngine()
-judge_backend = JarvisDirectBackend(engine_key="cloud")
-judge = TraceJudge(backend=judge_backend, model="gpt-5-mini-2025-08-07")
-
-# Build benchmark samples from existing traces
-trace_store = TraceStore(home / "traces.db")
-benchmark_samples = build_benchmark_samples_from_traces(trace_store, limit=50)
-
-orch = DistillationOrchestrator(
- teacher_engine=cloud_engine,
- teacher_model=teacher_model,
- trace_store=trace_store,
- benchmark_samples=benchmark_samples,
- student_runner=student_runner,
- judge=judge,
- session_store=SessionStore(home / "learning" / "learning.db"),
- checkpoint_store=CheckpointStore(home),
- openjarvis_home=home,
- autonomy_mode=AutonomyMode.AUTO,
- scorer=None,
- min_traces=10,
- max_cost_usd=max_cost,
- max_tool_calls=max_tools,
-)
-session = orch.run(OnDemandTrigger())
-
-# Save results
-result = {
- "session_id": session.id,
- "status": session.status.value,
- "cost_usd": session.teacher_cost_usd,
- "edits_total": len(session.edit_outcomes),
- "edits_applied": len([o for o in session.edit_outcomes if o.status == "applied"]),
- "edits_rejected": len([o for o in session.edit_outcomes if o.status == "rejected_by_gate"]),
- "error": session.error,
-}
-Path("${session_output}/result.json").write_text(json.dumps(result, indent=2))
-
-# Copy session artifacts
-sd = home / "learning" / "sessions" / session.id
-if sd.exists():
- shutil.copytree(sd, Path("${session_output}/artifacts"), dirs_exist_ok=True)
-
-print(json.dumps(result, indent=2))
-PYEOF
-
- # Check result
- if [ -f "${session_output}/result.json" ]; then
- local status
- status=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['status'])")
- local cost
- cost=$(python3 -c "import json; print(f\"\${json.load(open('${session_output}/result.json'))['cost_usd']:.4f}\")")
- local applied
- applied=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['edits_applied'])")
-
- if [ "$status" = "completed" ]; then
- ok "Session ${session_num}: status=${status}, cost=\$${cost}, applied=${applied}"
- else
- warn "Session ${session_num}: status=${status}, cost=\$${cost}"
- fi
- else
- fail "Session ${session_num}: no result.json (check ${session_output}/run.log)"
- fi
-
- session_num=$((session_num + 1))
- done
-
- ok "Done: ${experiment_name}/${config_name}"
-}
-
-# ── Run experiment group ─────────────────────────────────────────────────────
-run_experiment() {
- local exp_dir=$1
- local exp_name
- exp_name=$(basename "$exp_dir")
-
- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
- log "EXPERIMENT GROUP: ${exp_name}"
- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-
- local count=0
- local total
- total=$(ls "${exp_dir}"/*.toml 2>/dev/null | wc -l)
-
- for config in "${exp_dir}"/*.toml; do
- [ -f "$config" ] || continue
-
- # Apply filter if specified
- if [ -n "${FILTER}" ] && ! echo "$config" | grep -q "${FILTER}"; then
- continue
- fi
-
- count=$((count + 1))
- log "[${count}/${total}] $(basename "$config")"
- run_session "$config"
- done
-
- ok "Experiment group ${exp_name}: ${count} configs processed"
-}
-
-# ── Main ─────────────────────────────────────────────────────────────────────
-main() {
- check_prereqs
-
- log "Starting distillation experiments"
- log "Experiment filter: ${EXPERIMENT}"
- log "Config filter: ${FILTER:-none}"
-
- local start_time
- start_time=$(date +%s)
-
- if [ "$EXPERIMENT" = "all" ]; then
- # Run in priority order
- for exp in exp1a-teacher exp1b-budget exp1c-student \
- exp2a-gate exp2b-autonomy \
- exp3a-iterative exp3b-transfer; do
- if [ -d "${CONFIGS_DIR}/${exp}" ]; then
- run_experiment "${CONFIGS_DIR}/${exp}"
- fi
- done
- elif [ -d "${CONFIGS_DIR}/${EXPERIMENT}" ]; then
- run_experiment "${CONFIGS_DIR}/${EXPERIMENT}"
- else
- fail "Unknown experiment: ${EXPERIMENT}"
- echo "Available: exp1a-teacher exp1b-budget exp1c-student exp2a-gate exp2b-autonomy exp3a-iterative exp3b-transfer"
- exit 1
- fi
-
- local end_time
- end_time=$(date +%s)
- local elapsed=$((end_time - start_time))
-
- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
- ok "All experiments complete in ${elapsed}s"
- log "Results in: ${RESULTS_DIR}/"
- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-}
-
-main "$@"
diff --git a/scripts/experiments/run_track_b_gepa_dspy.sh b/scripts/experiments/run_track_b_gepa_dspy.sh
deleted file mode 100644
index a628be4a..00000000
--- a/scripts/experiments/run_track_b_gepa_dspy.sh
+++ /dev/null
@@ -1,375 +0,0 @@
-#!/usr/bin/env bash
-# =============================================================================
-# Track B: GEPA/DSPy Agent Optimization
-# NeurIPS 2026 — Agent Optimization Experiments
-#
-# Runs GEPA and DSPy BootstrapFewShot across:
-# Models: qwen-9b, qwen-27b, qwen-35b
-# Benchmarks: toolcall15, pinchbench, taubench
-#
-# Usage:
-# bash scripts/experiments/run_track_b_gepa_dspy.sh
-# bash scripts/experiments/run_track_b_gepa_dspy.sh --model qwen-9b --benchmark pinchbench
-# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer gepa
-# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer dspy
-#
-# Expected runtime: ~2-4 hours per GEPA run, ~1-2 hours per DSPy run
-# Total wall-clock: ~12-18 hours (parallelized across GPUs)
-# Estimated API cost: ~$90 GEPA + ~$90 DSPy = ~$180 total
-#
-# =============================================================================
-# vLLM Serving Commands (run these BEFORE this script on the GPU node)
-# =============================================================================
-#
-# GPU 0 — Qwen-9B (1x GPU):
-# CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen2.5-7B-Instruct \
-# --model Qwen/Qwen2.5-7B-Instruct \
-# --served-model-name qwen-9b \
-# --port 8001 --host 0.0.0.0 \
-# --max-model-len 32768 --gpu-memory-utilization 0.9 &
-#
-# GPU 1 — Qwen-27B (1-2x GPU):
-# CUDA_VISIBLE_DEVICES=1,2 vllm serve Qwen/Qwen2.5-32B-Instruct \
-# --model Qwen/Qwen2.5-32B-Instruct \
-# --served-model-name qwen-27b \
-# --port 8002 --host 0.0.0.0 \
-# --tensor-parallel-size 2 \
-# --max-model-len 32768 --gpu-memory-utilization 0.9 &
-#
-# GPU 3 — Qwen-35B (1-2x GPU):
-# CUDA_VISIBLE_DEVICES=3,4 vllm serve Qwen/Qwen2.5-72B-Instruct \
-# --model Qwen/Qwen2.5-72B-Instruct \
-# --served-model-name qwen-35b \
-# --port 8003 --host 0.0.0.0 \
-# --tensor-parallel-size 2 \
-# --max-model-len 32768 --gpu-memory-utilization 0.9 &
-#
-# Wait for all servers to be healthy:
-# sleep 60 && curl -s http://localhost:8001/health && \
-# curl -s http://localhost:8002/health && \
-# curl -s http://localhost:8003/health
-#
-# =============================================================================
-
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# Defaults
-# ---------------------------------------------------------------------------
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/agent-optimization"
-LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs"
-TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
-
-ALL_MODELS=(qwen-9b qwen-27b qwen-35b)
-ALL_BENCHMARKS=(toolcall15 pinchbench taubench)
-ALL_OPTIMIZERS=(gepa dspy)
-
-# Override defaults with CLI flags
-FILTER_MODEL=""
-FILTER_BENCHMARK=""
-FILTER_OPTIMIZER=""
-
-# GEPA settings
-GEPA_TRIALS=20
-GEPA_MAX_SAMPLES=50
-GEPA_OPTIMIZER_MODEL="claude-sonnet-4-6"
-
-# DSPy settings
-DSPY_OPTIMIZER="BootstrapFewShotWithRandomSearch"
-DSPY_TEACHER_LM="claude-sonnet-4-6"
-
-# ---------------------------------------------------------------------------
-# Parse CLI flags
-# ---------------------------------------------------------------------------
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --model)
- FILTER_MODEL="$2"; shift 2 ;;
- --benchmark)
- FILTER_BENCHMARK="$2"; shift 2 ;;
- --optimizer)
- FILTER_OPTIMIZER="$2"; shift 2 ;;
- --gepa-trials)
- GEPA_TRIALS="$2"; shift 2 ;;
- --gepa-max-samples)
- GEPA_MAX_SAMPLES="$2"; shift 2 ;;
- --dspy-optimizer)
- DSPY_OPTIMIZER="$2"; shift 2 ;;
- -h|--help)
- sed -n '2,30p' "$0" | grep '^#' | sed 's/^# \?//'
- exit 0 ;;
- *)
- echo "Unknown flag: $1"; exit 1 ;;
- esac
-done
-
-# Apply filters
-if [[ -n "$FILTER_MODEL" ]]; then
- ALL_MODELS=("$FILTER_MODEL")
-fi
-if [[ -n "$FILTER_BENCHMARK" ]]; then
- ALL_BENCHMARKS=("$FILTER_BENCHMARK")
-fi
-if [[ -n "$FILTER_OPTIMIZER" ]]; then
- ALL_OPTIMIZERS=("$FILTER_OPTIMIZER")
-fi
-
-# ---------------------------------------------------------------------------
-# Logging helpers
-# ---------------------------------------------------------------------------
-mkdir -p "$LOG_DIR"
-LOG_FILE="${LOG_DIR}/track_b_${TIMESTAMP}.log"
-
-log() {
- local level="$1"; shift
- local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"
- echo "$msg"
- echo "$msg" >> "$LOG_FILE"
-}
-
-log_info() { log "INFO " "$@"; }
-log_ok() { log "OK " "$@"; }
-log_warn() { log "WARN " "$@"; }
-log_error() { log "ERROR" "$@"; }
-
-# ---------------------------------------------------------------------------
-# Environment setup
-# ---------------------------------------------------------------------------
-setup_env() {
- log_info "=== Track B: GEPA/DSPy Optimization ==="
- log_info "Repo: $REPO_ROOT"
- log_info "Log: $LOG_FILE"
- log_info "Models: ${ALL_MODELS[*]}"
- log_info "Benchmarks: ${ALL_BENCHMARKS[*]}"
- log_info "Optimizers: ${ALL_OPTIMIZERS[*]}"
- echo ""
-
- # Check we are in the repo root
- if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then
- log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)"
- exit 1
- fi
-
- # Install/sync dependencies
- log_info "Running uv sync..."
- cd "$REPO_ROOT"
- uv sync --extra dev 2>&1 | tail -5
- log_ok "uv sync complete"
-
- # Check required API keys
- if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then
- log_error "ANTHROPIC_API_KEY is not set. Required for the optimizer teacher model."
- log_error " export ANTHROPIC_API_KEY=sk-ant-..."
- exit 1
- fi
- log_ok "ANTHROPIC_API_KEY is set"
-
- # Optional: OpenAI key (used if teacher_lm is an OpenAI model)
- if [[ -z "${OPENAI_API_KEY:-}" ]]; then
- log_warn "OPENAI_API_KEY not set (only needed if using OpenAI teacher models)"
- fi
-
- echo ""
- log_info "Model-to-port mapping (vLLM must be pre-started on these ports):"
- log_info " qwen-9b -> http://localhost:8001"
- log_info " qwen-27b -> http://localhost:8002"
- log_info " qwen-35b -> http://localhost:8003"
- echo ""
-}
-
-# ---------------------------------------------------------------------------
-# Model port lookup
-# ---------------------------------------------------------------------------
-model_port() {
- case "$1" in
- qwen-9b) echo 8001 ;;
- qwen-27b) echo 8002 ;;
- qwen-35b) echo 8003 ;;
- *)
- log_error "Unknown model: $1"
- exit 1 ;;
- esac
-}
-
-# ---------------------------------------------------------------------------
-# Health check: verify the vLLM server for a model is reachable
-# ---------------------------------------------------------------------------
-check_server_health() {
- local model="$1"
- local port
- port="$(model_port "$model")"
- local url="http://localhost:${port}/health"
- if curl -sf "$url" > /dev/null 2>&1; then
- log_ok "vLLM server for $model is healthy at port $port"
- return 0
- else
- log_error "vLLM server for $model NOT reachable at $url"
- log_error "Start it with the vLLM commands in the script header."
- return 1
- fi
-}
-
-# ---------------------------------------------------------------------------
-# GEPA optimization for one (model, benchmark) pair
-# ---------------------------------------------------------------------------
-run_gepa() {
- local model="$1"
- local bench="$2"
- local port
- port="$(model_port "$model")"
- local out_dir="${RESULTS_BASE}/gepa/${model}/${bench}"
-
- log_info "--- GEPA: $model × $bench ---"
- log_info " Output dir: $out_dir"
- log_info " Trials: $GEPA_TRIALS Max-samples: $GEPA_MAX_SAMPLES"
- mkdir -p "$out_dir"
-
- # Record start time
- local t0
- t0="$(date +%s)"
-
- OPENAI_API_BASE="http://localhost:${port}/v1" \
- uv run jarvis optimize run \
- --benchmark "$bench" \
- --model "$model" \
- --optimizer-model "$GEPA_OPTIMIZER_MODEL" \
- --trials "$GEPA_TRIALS" \
- --max-samples "$GEPA_MAX_SAMPLES" \
- --output-dir "$out_dir" \
- 2>&1 | tee -a "$LOG_FILE"
-
- local exit_code=${PIPESTATUS[0]}
- local t1
- t1="$(date +%s)"
- local elapsed=$(( t1 - t0 ))
-
- if [[ $exit_code -eq 0 ]]; then
- log_ok "GEPA $model/$bench done in ${elapsed}s"
- else
- log_error "GEPA $model/$bench FAILED (exit $exit_code) after ${elapsed}s"
- return $exit_code
- fi
-}
-
-# ---------------------------------------------------------------------------
-# DSPy optimization for one (model, benchmark) pair
-# ---------------------------------------------------------------------------
-run_dspy() {
- local model="$1"
- local bench="$2"
- local port
- port="$(model_port "$model")"
- local out_dir="${RESULTS_BASE}/dspy/${model}/${bench}"
-
- log_info "--- DSPy: $model × $bench ---"
- log_info " Output dir: $out_dir"
- log_info " Teleprompter: $DSPY_OPTIMIZER Teacher: $DSPY_TEACHER_LM"
- mkdir -p "$out_dir"
-
- local t0
- t0="$(date +%s)"
-
- OPENAI_API_BASE="http://localhost:${port}/v1" \
- uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
-import sys
-from openjarvis.learning.agents.dspy_optimizer import DSPyAgentOptimizer
-from openjarvis.core.config import DSPyOptimizerConfig
-from openjarvis.traces.store import TraceStore
-
-store = TraceStore()
-config = DSPyOptimizerConfig(
- optimizer="${DSPY_OPTIMIZER}",
- teacher_lm="${DSPY_TEACHER_LM}",
- config_dir="${out_dir}",
- benchmark="${bench}",
- agent_filter="${model}",
-)
-result = DSPyAgentOptimizer(config).optimize(store)
-print(f"DSPy result for ${model}/${bench}: {result}")
-if result.get("status") not in ("ok", "success", "done"):
- sys.exit(1)
-PYEOF
-
- local exit_code=${PIPESTATUS[0]}
- local t1
- t1="$(date +%s)"
- local elapsed=$(( t1 - t0 ))
-
- if [[ $exit_code -eq 0 ]]; then
- log_ok "DSPy $model/$bench done in ${elapsed}s"
- else
- log_error "DSPy $model/$bench FAILED (exit $exit_code) after ${elapsed}s"
- return $exit_code
- fi
-}
-
-# ---------------------------------------------------------------------------
-# Summary: print result file locations
-# ---------------------------------------------------------------------------
-print_summary() {
- echo ""
- log_info "=== Track B Complete ==="
- log_info "Results written to:"
- for opt in "${ALL_OPTIMIZERS[@]}"; do
- for model in "${ALL_MODELS[@]}"; do
- for bench in "${ALL_BENCHMARKS[@]}"; do
- local out_dir="${RESULTS_BASE}/${opt}/${model}/${bench}"
- if [[ -d "$out_dir" ]]; then
- log_ok " $opt/$model/$bench -> $out_dir"
- else
- log_warn " $opt/$model/$bench -> MISSING ($out_dir)"
- fi
- done
- done
- done
- log_info "Full log: $LOG_FILE"
-}
-
-# ---------------------------------------------------------------------------
-# Main
-# ---------------------------------------------------------------------------
-main() {
- setup_env
-
- local failed=0
-
- # Pre-flight: check vLLM servers for all target models
- log_info "Checking vLLM server health..."
- for model in "${ALL_MODELS[@]}"; do
- check_server_health "$model" || failed=$(( failed + 1 ))
- done
- if [[ $failed -gt 0 ]]; then
- log_error "$failed vLLM server(s) not reachable. Start them first (see header)."
- exit 1
- fi
- echo ""
-
- # Run all requested (optimizer, model, benchmark) combinations
- for opt in "${ALL_OPTIMIZERS[@]}"; do
- log_info "=========================================="
- log_info "Optimizer: $opt"
- log_info "=========================================="
- for model in "${ALL_MODELS[@]}"; do
- for bench in "${ALL_BENCHMARKS[@]}"; do
- case "$opt" in
- gepa) run_gepa "$model" "$bench" || failed=$(( failed + 1 )) ;;
- dspy) run_dspy "$model" "$bench" || failed=$(( failed + 1 )) ;;
- *) log_error "Unknown optimizer: $opt"; failed=$(( failed + 1 )) ;;
- esac
- echo ""
- done
- done
- done
-
- print_summary
-
- if [[ $failed -gt 0 ]]; then
- log_error "$failed run(s) failed. Check log for details: $LOG_FILE"
- exit 1
- fi
-
- log_ok "All Track B runs completed successfully."
-}
-
-main "$@"
diff --git a/scripts/experiments/run_track_d_lora_sft.sh b/scripts/experiments/run_track_d_lora_sft.sh
deleted file mode 100644
index e90b7415..00000000
--- a/scripts/experiments/run_track_d_lora_sft.sh
+++ /dev/null
@@ -1,778 +0,0 @@
-#!/usr/bin/env bash
-# =============================================================================
-# Track D: LoRA / SFT Intelligence Optimization
-# NeurIPS 2026 — Intelligence Optimization Experiments
-#
-# Runs LoRA and SFT fine-tuning across:
-# LoRA models: Qwen-2B, Qwen-9B, Qwen-27B
-# SFT models: Qwen-2B, Qwen-9B
-#
-# After training, runs fast-benchmark eval on every checkpoint.
-#
-# Usage:
-# bash scripts/experiments/run_track_d_lora_sft.sh
-# bash scripts/experiments/run_track_d_lora_sft.sh --method lora
-# bash scripts/experiments/run_track_d_lora_sft.sh --method sft --model qwen-2b
-# bash scripts/experiments/run_track_d_lora_sft.sh --skip-eval
-#
-# Expected runtime per run:
-# Qwen-2B LoRA (~4-8 h, 1x H100, GPU 0)
-# Qwen-9B LoRA (~8-16 h, 1x H100, GPU 1)
-# Qwen-27B LoRA (~16-24 h, 2x H100, GPUs 2-3)
-# Qwen-2B SFT (~8-16 h, 1x H100, GPU 4)
-# Qwen-9B SFT (~16-24 h, 2x H100, GPUs 5-6)
-#
-# Total wall-clock (all in parallel): ~24 h on a 7-8x H100 node
-# Total GPU-hours: ~100-200 H100-hours
-#
-# =============================================================================
-# GPU Allocation — recommended for a single 8x H100 node
-# =============================================================================
-#
-# Run D1 (Qwen-2B LoRA) on GPU 0:
-# CUDA_VISIBLE_DEVICES=0 bash scripts/experiments/run_track_d_lora_sft.sh \
-# --method lora --model qwen-2b &
-#
-# Run D2 (Qwen-9B LoRA) on GPU 1:
-# CUDA_VISIBLE_DEVICES=1 bash scripts/experiments/run_track_d_lora_sft.sh \
-# --method lora --model qwen-9b &
-#
-# Run D3 (Qwen-27B LoRA) on GPUs 2-3:
-# CUDA_VISIBLE_DEVICES=2,3 bash scripts/experiments/run_track_d_lora_sft.sh \
-# --method lora --model qwen-27b &
-#
-# Run D4 (Qwen-2B SFT) on GPU 4:
-# CUDA_VISIBLE_DEVICES=4 bash scripts/experiments/run_track_d_lora_sft.sh \
-# --method sft --model qwen-2b &
-#
-# Run D5 (Qwen-9B SFT) on GPUs 5-6:
-# CUDA_VISIBLE_DEVICES=5,6 bash scripts/experiments/run_track_d_lora_sft.sh \
-# --method sft --model qwen-9b &
-#
-# =============================================================================
-# Training datasets (downloaded automatically if HF_TOKEN is set)
-# =============================================================================
-#
-# Primary agentic traces:
-# - neulab/agent-data-collection
-# - GAIR/AgentInstruct
-#
-# Supplementary reasoning:
-# - GeneralThought-430K-filtered
-# - GLM-4.7-flash SFT traces (168K + 57K)
-#
-# =============================================================================
-
-set -euo pipefail
-
-# ---------------------------------------------------------------------------
-# Defaults
-# ---------------------------------------------------------------------------
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/intelligence-optimization"
-LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs"
-TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
-
-# Default: run all methods and models
-ALL_LORA_MODELS=(qwen-2b qwen-9b qwen-27b)
-ALL_SFT_MODELS=(qwen-2b qwen-9b)
-FAST_BENCHMARKS=(toolcall15 pinchbench taubench)
-
-# CLI overrides
-FILTER_METHOD=""
-FILTER_MODEL=""
-SKIP_EVAL=false
-
-# Training hyperparameters (override per model below)
-LORA_RANK=64
-LORA_ALPHA=128
-LORA_DROPOUT=0.05
-LORA_EPOCHS=3
-LORA_LR=2e-4
-LORA_BATCH_SIZE=8
-LORA_GRAD_ACCUM=4
-
-SFT_EPOCHS=3
-SFT_LR=1e-5
-SFT_BATCH_SIZE=4
-SFT_GRAD_ACCUM=8
-
-MAX_SEQ_LEN=8192
-
-# Fast-eval settings
-EVAL_MAX_SAMPLES=20
-
-# ---------------------------------------------------------------------------
-# HuggingFace model IDs
-# ---------------------------------------------------------------------------
-model_hf_id() {
- case "$1" in
- qwen-2b) echo "Qwen/Qwen2.5-1.5B-Instruct" ;;
- qwen-9b) echo "Qwen/Qwen2.5-7B-Instruct" ;;
- qwen-27b) echo "Qwen/Qwen2.5-32B-Instruct" ;;
- *)
- echo "ERROR: unknown model $1" >&2
- exit 1 ;;
- esac
-}
-
-# ---------------------------------------------------------------------------
-# Parse CLI flags
-# ---------------------------------------------------------------------------
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --method)
- FILTER_METHOD="$2"; shift 2 ;;
- --model)
- FILTER_MODEL="$2"; shift 2 ;;
- --skip-eval)
- SKIP_EVAL=true; shift ;;
- --lora-rank)
- LORA_RANK="$2"; shift 2 ;;
- --lora-epochs)
- LORA_EPOCHS="$2"; shift 2 ;;
- --sft-epochs)
- SFT_EPOCHS="$2"; shift 2 ;;
- -h|--help)
- sed -n '2,35p' "$0" | grep '^#' | sed 's/^# \?//'
- exit 0 ;;
- *)
- echo "Unknown flag: $1"; exit 1 ;;
- esac
-done
-
-# Apply model filter
-if [[ -n "$FILTER_MODEL" ]]; then
- ALL_LORA_MODELS=("$FILTER_MODEL")
- ALL_SFT_MODELS=("$FILTER_MODEL")
-fi
-
-# Apply method filter
-RUN_LORA=true
-RUN_SFT=true
-if [[ "$FILTER_METHOD" == "lora" ]]; then
- RUN_SFT=false
-elif [[ "$FILTER_METHOD" == "sft" ]]; then
- RUN_LORA=false
-fi
-
-# ---------------------------------------------------------------------------
-# Logging helpers
-# ---------------------------------------------------------------------------
-mkdir -p "$LOG_DIR"
-LOG_FILE="${LOG_DIR}/track_d_${TIMESTAMP}.log"
-
-log() {
- local level="$1"; shift
- local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"
- echo "$msg"
- echo "$msg" >> "$LOG_FILE"
-}
-
-log_info() { log "INFO " "$@"; }
-log_ok() { log "OK " "$@"; }
-log_warn() { log "WARN " "$@"; }
-log_error() { log "ERROR" "$@"; }
-
-# ---------------------------------------------------------------------------
-# Environment setup
-# ---------------------------------------------------------------------------
-setup_env() {
- log_info "=== Track D: LoRA/SFT Training ==="
- log_info "Repo: $REPO_ROOT"
- log_info "Log: $LOG_FILE"
- log_info "Results: $RESULTS_BASE"
- echo ""
-
- # Check we are in the repo root
- if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then
- log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)"
- exit 1
- fi
-
- # Install/sync dependencies
- log_info "Running uv sync..."
- cd "$REPO_ROOT"
- uv sync --extra dev 2>&1 | tail -5
- log_ok "uv sync complete"
-
- # HuggingFace token — required to download gated Qwen models
- if [[ -z "${HF_TOKEN:-}" ]]; then
- log_warn "HF_TOKEN is not set."
- log_warn " If Qwen models are gated, set: export HF_TOKEN=hf_..."
- log_warn " Or pre-download them with: huggingface-cli download <model>"
- else
- log_ok "HF_TOKEN is set"
- # Log in so huggingface_hub uses the token
- uv run python -c "
-import huggingface_hub
-huggingface_hub.login(token='${HF_TOKEN}', add_to_git_credential=False)
-print('Logged in to HuggingFace Hub')
-" 2>&1 | tee -a "$LOG_FILE"
- fi
-
- # Check for GPU
- if ! command -v nvidia-smi &>/dev/null; then
- log_warn "nvidia-smi not found — ensure CUDA is available for training."
- else
- log_info "GPU status:"
- nvidia-smi --query-gpu=index,name,memory.total,memory.free \
- --format=csv,noheader 2>&1 | while IFS= read -r line; do
- log_info " $line"
- done
- fi
-
- # Check for trl / peft / transformers (training stack)
- uv run python -c "
-import importlib, sys
-missing = []
-for pkg in ['transformers', 'peft', 'trl', 'datasets', 'accelerate', 'bitsandbytes']:
- if importlib.util.find_spec(pkg) is None:
- missing.append(pkg)
-if missing:
- print('MISSING packages:', missing)
- sys.exit(1)
-else:
- print('Training stack OK: transformers, peft, trl, datasets, accelerate, bitsandbytes')
-" 2>&1 | tee -a "$LOG_FILE" || {
- log_error "Some training packages are missing. Install with:"
- log_error " uv add transformers peft trl datasets accelerate bitsandbytes"
- exit 1
- }
-
- echo ""
-}
-
-# ---------------------------------------------------------------------------
-# Download / verify training dataset
-# ---------------------------------------------------------------------------
-prepare_dataset() {
- local method="$1"
- local model="$2"
- local data_dir="${REPO_ROOT}/results/neurips-2026/training-data"
- mkdir -p "$data_dir"
-
- log_info "Preparing training dataset for $method/$model..."
-
- uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
-from datasets import load_dataset
-import json
-from pathlib import Path
-
-data_dir = Path("${data_dir}")
-output_path = data_dir / "${method}_${model}_train.jsonl"
-
-if output_path.exists():
- lines = output_path.read_text().count('\n')
- print(f"Dataset already exists: {output_path} ({lines} examples)")
-else:
- print("Downloading neulab/agent-data-collection...")
- ds = load_dataset("neulab/agent-data-collection", split="train")
- print(f"Raw dataset size: {len(ds)}")
-
- # Filter to reasonable-length examples for agentic fine-tuning
- examples = []
- for ex in ds:
- messages = ex.get("messages") or ex.get("conversations") or []
- if not messages:
- continue
- total_len = sum(len(str(m)) for m in messages)
- if 200 <= total_len <= 16000:
- examples.append({"messages": messages})
-
- print(f"Filtered dataset size: {len(examples)}")
-
- with output_path.open("w") as f:
- for ex in examples:
- f.write(json.dumps(ex) + "\n")
- print(f"Saved to {output_path}")
-PYEOF
-
- echo "${data_dir}/${method}_${model}_train.jsonl"
-}
-
-# ---------------------------------------------------------------------------
-# LoRA training for one model
-# ---------------------------------------------------------------------------
-run_lora() {
- local model="$1"
- local hf_id
- hf_id="$(model_hf_id "$model")"
- local out_dir="${RESULTS_BASE}/lora/${model}"
- local checkpoint_dir="${out_dir}/checkpoint"
- mkdir -p "$out_dir" "$checkpoint_dir"
-
- log_info "=========================================="
- log_info "LoRA training: $model ($hf_id)"
- log_info " Output: $out_dir"
- log_info " LoRA rank=$LORA_RANK alpha=$LORA_ALPHA dropout=$LORA_DROPOUT"
- log_info " Epochs=$LORA_EPOCHS LR=$LORA_LR batch=$LORA_BATCH_SIZE grad_accum=$LORA_GRAD_ACCUM"
- log_info "=========================================="
-
- local dataset_path
- dataset_path="$(prepare_dataset lora "$model")"
-
- local t0
- t0="$(date +%s)"
-
- # Number of GPUs visible
- local n_gpus
- n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)"
- log_info "Training on $n_gpus GPU(s)"
-
- if [[ "$n_gpus" -gt 1 ]]; then
- LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus"
- else
- LAUNCHER="uv run python"
- fi
-
- $LAUNCHER - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
-import json
-import math
-import os
-from pathlib import Path
-
-import torch
-from datasets import load_dataset
-from peft import LoraConfig, TaskType, get_peft_model
-from transformers import (
- AutoModelForCausalLM,
- AutoTokenizer,
- DataCollatorForSeq2Seq,
- Trainer,
- TrainingArguments,
-)
-
-hf_id = "${hf_id}"
-out_dir = "${out_dir}"
-ckpt = "${checkpoint_dir}"
-data_path = "${dataset_path}"
-
-# ---- Tokenizer ----
-print(f"Loading tokenizer: {hf_id}")
-tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True)
-if tokenizer.pad_token is None:
- tokenizer.pad_token = tokenizer.eos_token
-
-# ---- Dataset ----
-print(f"Loading dataset: {data_path}")
-raw = load_dataset("json", data_files=data_path, split="train")
-raw = raw.train_test_split(test_size=0.02, seed=42)
-
-def tokenize(example):
- messages = example["messages"]
- text = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=False,
- )
- enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN})
- enc["labels"] = enc["input_ids"].copy()
- return enc
-
-print("Tokenizing dataset...")
-tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4)
-print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}")
-
-# ---- Model ----
-print(f"Loading model: {hf_id}")
-model = AutoModelForCausalLM.from_pretrained(
- hf_id,
- torch_dtype=torch.bfloat16,
- trust_remote_code=True,
- device_map="auto",
-)
-
-# ---- LoRA ----
-lora_config = LoraConfig(
- task_type=TaskType.CAUSAL_LM,
- r=${LORA_RANK},
- lora_alpha=${LORA_ALPHA},
- lora_dropout=${LORA_DROPOUT},
- target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
- "gate_proj", "up_proj", "down_proj"],
- bias="none",
-)
-model = get_peft_model(model, lora_config)
-model.print_trainable_parameters()
-
-# ---- Training args ----
-steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${LORA_BATCH_SIZE} * ${LORA_GRAD_ACCUM}))
-total_steps = steps_per_epoch * ${LORA_EPOCHS}
-save_steps = max(50, steps_per_epoch // 2)
-
-args = TrainingArguments(
- output_dir=ckpt,
- num_train_epochs=${LORA_EPOCHS},
- per_device_train_batch_size=${LORA_BATCH_SIZE},
- gradient_accumulation_steps=${LORA_GRAD_ACCUM},
- learning_rate=${LORA_LR},
- warmup_ratio=0.05,
- lr_scheduler_type="cosine",
- logging_steps=10,
- save_steps=save_steps,
- eval_strategy="steps",
- eval_steps=save_steps,
- load_best_model_at_end=True,
- metric_for_best_model="eval_loss",
- bf16=True,
- gradient_checkpointing=True,
- dataloader_num_workers=4,
- report_to="none",
- save_total_limit=3,
-)
-
-# ---- Trainer ----
-collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)
-trainer = Trainer(
- model=model,
- args=args,
- train_dataset=tok_ds["train"],
- eval_dataset=tok_ds["test"],
- data_collator=collator,
-)
-
-print(f"Starting LoRA training: {total_steps} total steps")
-trainer.train()
-
-# Save final adapter
-final_adapter = os.path.join(out_dir, "lora_adapter_final")
-model.save_pretrained(final_adapter)
-tokenizer.save_pretrained(final_adapter)
-print(f"Final LoRA adapter saved to {final_adapter}")
-
-# Save training metadata
-meta = {
- "model": hf_id,
- "method": "lora",
- "lora_rank": ${LORA_RANK},
- "lora_alpha": ${LORA_ALPHA},
- "epochs": ${LORA_EPOCHS},
- "train_examples": len(tok_ds["train"]),
- "final_train_loss": trainer.state.log_history[-1].get("loss"),
-}
-with open(os.path.join(out_dir, "training_meta.json"), "w") as f:
- import json; json.dump(meta, f, indent=2)
-print("Training metadata saved.")
-PYEOF
-
- local exit_code=${PIPESTATUS[0]}
- local t1
- t1="$(date +%s)"
- local elapsed=$(( t1 - t0 ))
-
- if [[ $exit_code -eq 0 ]]; then
- log_ok "LoRA $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"
- return 0
- else
- log_error "LoRA $model FAILED (exit $exit_code) after ${elapsed}s"
- return $exit_code
- fi
-}
-
-# ---------------------------------------------------------------------------
-# SFT (full fine-tuning) for one model
-# ---------------------------------------------------------------------------
-run_sft() {
- local model="$1"
- local hf_id
- hf_id="$(model_hf_id "$model")"
- local out_dir="${RESULTS_BASE}/sft/${model}"
- local checkpoint_dir="${out_dir}/checkpoint"
- mkdir -p "$out_dir" "$checkpoint_dir"
-
- log_info "=========================================="
- log_info "SFT training: $model ($hf_id)"
- log_info " Output: $out_dir"
- log_info " Epochs=$SFT_EPOCHS LR=$SFT_LR batch=$SFT_BATCH_SIZE grad_accum=$SFT_GRAD_ACCUM"
- log_info "=========================================="
-
- local dataset_path
- dataset_path="$(prepare_dataset sft "$model")"
-
- local t0
- t0="$(date +%s)"
-
- local n_gpus
- n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)"
- log_info "Training on $n_gpus GPU(s)"
-
- if [[ "$n_gpus" -gt 1 ]]; then
- LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus"
- else
- LAUNCHER="uv run python"
- fi
-
- $LAUNCHER - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
-import json
-import math
-import os
-from pathlib import Path
-
-import torch
-from datasets import load_dataset
-from transformers import (
- AutoModelForCausalLM,
- AutoTokenizer,
- DataCollatorForSeq2Seq,
- Trainer,
- TrainingArguments,
-)
-
-hf_id = "${hf_id}"
-out_dir = "${out_dir}"
-ckpt = "${checkpoint_dir}"
-data_path = "${dataset_path}"
-
-# ---- Tokenizer ----
-print(f"Loading tokenizer: {hf_id}")
-tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True)
-if tokenizer.pad_token is None:
- tokenizer.pad_token = tokenizer.eos_token
-
-# ---- Dataset ----
-print(f"Loading dataset: {data_path}")
-raw = load_dataset("json", data_files=data_path, split="train")
-raw = raw.train_test_split(test_size=0.02, seed=42)
-
-def tokenize(example):
- messages = example["messages"]
- text = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=False,
- )
- enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN})
- enc["labels"] = enc["input_ids"].copy()
- return enc
-
-print("Tokenizing dataset...")
-tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4)
-print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}")
-
-# ---- Model (4-bit quantized to fit on fewer GPUs) ----
-print(f"Loading model: {hf_id}")
-from transformers import BitsAndBytesConfig
-bnb_config = BitsAndBytesConfig(
- load_in_4bit=True,
- bnb_4bit_compute_dtype=torch.bfloat16,
- bnb_4bit_quant_type="nf4",
- bnb_4bit_use_double_quant=True,
-)
-model = AutoModelForCausalLM.from_pretrained(
- hf_id,
- quantization_config=bnb_config,
- trust_remote_code=True,
- device_map="auto",
-)
-model.config.use_cache = False
-
-# ---- Training args ----
-steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${SFT_BATCH_SIZE} * ${SFT_GRAD_ACCUM}))
-total_steps = steps_per_epoch * ${SFT_EPOCHS}
-save_steps = max(50, steps_per_epoch // 2)
-
-args = TrainingArguments(
- output_dir=ckpt,
- num_train_epochs=${SFT_EPOCHS},
- per_device_train_batch_size=${SFT_BATCH_SIZE},
- gradient_accumulation_steps=${SFT_GRAD_ACCUM},
- learning_rate=${SFT_LR},
- warmup_ratio=0.03,
- lr_scheduler_type="cosine",
- logging_steps=10,
- save_steps=save_steps,
- eval_strategy="steps",
- eval_steps=save_steps,
- load_best_model_at_end=True,
- metric_for_best_model="eval_loss",
- bf16=True,
- gradient_checkpointing=True,
- dataloader_num_workers=4,
- report_to="none",
- save_total_limit=3,
-)
-
-# ---- Trainer ----
-collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)
-trainer = Trainer(
- model=model,
- args=args,
- train_dataset=tok_ds["train"],
- eval_dataset=tok_ds["test"],
- data_collator=collator,
-)
-
-print(f"Starting SFT training: {total_steps} total steps")
-trainer.train()
-
-# Save final model
-final_model = os.path.join(out_dir, "sft_model_final")
-model.save_pretrained(final_model)
-tokenizer.save_pretrained(final_model)
-print(f"Final SFT model saved to {final_model}")
-
-# Save training metadata
-meta = {
- "model": hf_id,
- "method": "sft",
- "epochs": ${SFT_EPOCHS},
- "train_examples": len(tok_ds["train"]),
- "final_train_loss": trainer.state.log_history[-1].get("loss"),
-}
-with open(os.path.join(out_dir, "training_meta.json"), "w") as f:
- import json; json.dump(meta, f, indent=2)
-print("Training metadata saved.")
-PYEOF
-
- local exit_code=${PIPESTATUS[0]}
- local t1
- t1="$(date +%s)"
- local elapsed=$(( t1 - t0 ))
-
- if [[ $exit_code -eq 0 ]]; then
- log_ok "SFT $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"
- return 0
- else
- log_error "SFT $model FAILED (exit $exit_code) after ${elapsed}s"
- return $exit_code
- fi
-}
-
-# ---------------------------------------------------------------------------
-# Post-training eval: run fast benchmarks on a checkpoint
-# ---------------------------------------------------------------------------
-run_eval() {
- local method="$1" # lora or sft
- local model="$2"
- local out_dir="${RESULTS_BASE}/${method}/${model}"
-
- if [[ "$SKIP_EVAL" == "true" ]]; then
- log_info "Skipping eval (--skip-eval)"
- return 0
- fi
-
- # Locate the final checkpoint / adapter
- local checkpoint=""
- if [[ "$method" == "lora" ]]; then
- checkpoint="${out_dir}/lora_adapter_final"
- else
- checkpoint="${out_dir}/sft_model_final"
- fi
-
- if [[ ! -d "$checkpoint" ]]; then
- log_warn "Checkpoint not found for $method/$model at $checkpoint — skipping eval"
- return 0
- fi
-
- log_info "--- Post-training eval: $method/$model ---"
- log_info " Checkpoint: $checkpoint"
-
- for bench in "${FAST_BENCHMARKS[@]}"; do
- local eval_out="${out_dir}/eval/${bench}"
- mkdir -p "$eval_out"
- log_info " Eval: $bench -> $eval_out"
-
- uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
-import subprocess, sys
-
-cmd = [
- "uv", "run", "python", "-m", "openjarvis.evals", "run",
- "--model-path", "${checkpoint}",
- "--model-id", "${model}-${method}",
- "--benchmark", "${bench}",
- "--max-samples", "${EVAL_MAX_SAMPLES}",
- "--output", "${eval_out}",
-]
-print("Running:", " ".join(cmd))
-result = subprocess.run(cmd, capture_output=False)
-sys.exit(result.returncode)
-PYEOF
-
- local eval_exit=${PIPESTATUS[0]}
- if [[ $eval_exit -eq 0 ]]; then
- log_ok " Eval $method/$model/$bench OK"
- else
- log_warn " Eval $method/$model/$bench returned exit $eval_exit (non-fatal)"
- fi
- done
-}
-
-# ---------------------------------------------------------------------------
-# Summary
-# ---------------------------------------------------------------------------
-print_summary() {
- echo ""
- log_info "=== Track D Complete ==="
- log_info "Results written to:"
-
- if [[ "$RUN_LORA" == "true" ]]; then
- for model in "${ALL_LORA_MODELS[@]}"; do
- local out="${RESULTS_BASE}/lora/${model}"
- if [[ -d "$out" ]]; then
- log_ok " lora/$model -> $out"
- else
- log_warn " lora/$model -> MISSING ($out)"
- fi
- done
- fi
-
- if [[ "$RUN_SFT" == "true" ]]; then
- for model in "${ALL_SFT_MODELS[@]}"; do
- local out="${RESULTS_BASE}/sft/${model}"
- if [[ -d "$out" ]]; then
- log_ok " sft/$model -> $out"
- else
- log_warn " sft/$model -> MISSING ($out)"
- fi
- done
- fi
-
- log_info "Full log: $LOG_FILE"
-}
-
-# ---------------------------------------------------------------------------
-# Main
-# ---------------------------------------------------------------------------
-main() {
- setup_env
-
- local failed=0
-
- # ---- LoRA runs ----
- if [[ "$RUN_LORA" == "true" ]]; then
- log_info "=========================================="
- log_info "Starting LoRA training runs"
- log_info "Models: ${ALL_LORA_MODELS[*]}"
- log_info "=========================================="
- for model in "${ALL_LORA_MODELS[@]}"; do
- run_lora "$model" || { failed=$(( failed + 1 )); log_error "LoRA $model failed, continuing..."; }
- run_eval lora "$model"
- echo ""
- done
- fi
-
- # ---- SFT runs ----
- if [[ "$RUN_SFT" == "true" ]]; then
- log_info "=========================================="
- log_info "Starting SFT training runs"
- log_info "Models: ${ALL_SFT_MODELS[*]}"
- log_info "=========================================="
- for model in "${ALL_SFT_MODELS[@]}"; do
- run_sft "$model" || { failed=$(( failed + 1 )); log_error "SFT $model failed, continuing..."; }
- run_eval sft "$model"
- echo ""
- done
- fi
-
- print_summary
-
- if [[ $failed -gt 0 ]]; then
- log_error "$failed training run(s) failed. Check log: $LOG_FILE"
- exit 1
- fi
-
- log_ok "All Track D runs completed successfully."
-}
-
-main "$@"
diff --git a/scripts/oauth_all.py b/scripts/oauth_all.py
index 7e969c27..50755171 100644
--- a/scripts/oauth_all.py
+++ b/scripts/oauth_all.py
@@ -14,7 +14,7 @@ import os
import webbrowser
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
from urllib.parse import parse_qs, urlencode, urlparse
import httpx
@@ -44,16 +44,31 @@ def _load_creds(filename: str) -> Dict[str, str]:
_google = _load_creds("google.json")
-GOOGLE_CLIENT_ID = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"]
-GOOGLE_CLIENT_SECRET = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "") or _google["client_secret"]
+GOOGLE_CLIENT_ID = (
+ os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"]
+)
+GOOGLE_CLIENT_SECRET = (
+ os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "")
+ or _google["client_secret"]
+)
_strava = _load_creds("strava.json")
-STRAVA_CLIENT_ID = os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"]
-STRAVA_CLIENT_SECRET = os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "") or _strava["client_secret"]
+STRAVA_CLIENT_ID = (
+ os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"]
+)
+STRAVA_CLIENT_SECRET = (
+ os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "")
+ or _strava["client_secret"]
+)
_spotify = _load_creds("spotify.json")
-SPOTIFY_CLIENT_ID = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"]
-SPOTIFY_CLIENT_SECRET = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "") or _spotify["client_secret"]
+SPOTIFY_CLIENT_ID = (
+ os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"]
+)
+SPOTIFY_CLIENT_SECRET = (
+ os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "")
+ or _spotify["client_secret"]
+)
# ── Generic OAuth helpers ────────────────────────────────────────────────────
@@ -128,7 +143,7 @@ def do_google() -> None:
"prompt": "consent",
})
)
- print(f" Opening browser...")
+ print(" Opening browser...")
webbrowser.open(url)
code = _wait_for_code()
@@ -176,7 +191,7 @@ def do_strava() -> None:
"scope": "activity:read_all",
})
)
- print(f" Opening browser...")
+ print(" Opening browser...")
webbrowser.open(url)
code = _wait_for_code()
@@ -208,7 +223,6 @@ def do_strava() -> None:
def _make_self_signed_cert() -> tuple[str, str]:
"""Generate a temporary self-signed cert for localhost HTTPS."""
- import ssl
import subprocess
import tempfile
@@ -288,7 +302,7 @@ def do_spotify() -> None:
"scope": "user-read-recently-played",
})
)
- print(f" Opening browser...")
+ print(" Opening browser...")
webbrowser.open(url)
code = _wait_for_code(port=spotify_port)
@@ -323,7 +337,9 @@ def do_spotify() -> None:
# ── Main ─────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Run OAuth flows for OpenJarvis connectors")
+ parser = argparse.ArgumentParser(
+ description="Run OAuth flows for OpenJarvis connectors"
+ )
parser.add_argument("--google", action="store_true", help="Only Google")
parser.add_argument("--strava", action="store_true", help="Only Strava")
parser.add_argument("--spotify", action="store_true", help="Only Spotify")
diff --git a/scripts/smoke_framework_comparison.sh b/scripts/smoke_framework_comparison.sh
index f9ad6faa..38791d5e 100755
--- a/scripts/smoke_framework_comparison.sh
+++ b/scripts/smoke_framework_comparison.sh
@@ -47,27 +47,45 @@ print(' all pins OK')
echo "==> Running one-task smoke per (framework, benchmark)"
mkdir -p results/smoke
+CONFIG_DIR="results/smoke/configs"
+mkdir -p "$CONFIG_DIR"
SMOKE_BENCHES=(toolcall15 pinchbench gaia)
SMOKE_FRAMEWORKS=(hermes openclaw openjarvis)
SMOKE_MODEL="qwen-9b"
for fwk in "${SMOKE_FRAMEWORKS[@]}"; do
for bench in "${SMOKE_BENCHES[@]}"; do
- config="src/openjarvis/evals/configs/framework_comparison/${bench}-${fwk}-${SMOKE_MODEL}.toml"
- if [ ! -f "$config" ]; then
- echo " ! missing config: $config (run make_configs --all-tier1 (or materialize the configs you need))"
- continue
- fi
+ uv run python -m openjarvis.evals.comparison.make_configs \
+ --framework "$fwk" \
+ --model "$SMOKE_MODEL" \
+ --benchmark "$bench" \
+ --output-dir "$CONFIG_DIR" >/dev/null
+ config="${CONFIG_DIR}/${bench}-${fwk}-${SMOKE_MODEL}.toml"
+ run_dir="results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/"
+ JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" uv run python - "$config" "$run_dir" <<'PY'
+from pathlib import Path
+import re
+import sys
+
+path = Path(sys.argv[1])
+run_dir = sys.argv[2]
+text = path.read_text()
+text = re.sub(r'^output_dir = ".*"$', f'output_dir = "{run_dir}"', text, flags=re.M)
+text = re.sub(r"^max_samples = \d+$", "max_samples = 1", text, flags=re.M)
+path.write_text(text)
+PY
echo " ▸ $fwk × $bench"
- uv run python -m openjarvis.evals run --config "$config" --max-samples 1 \
- --output-dir "results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/" \
+ mkdir -p "$run_dir"
+ JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" \
+ JARVIS_BACKEND_API_KEY="${JARVIS_BACKEND_API_KEY:-smoke}" \
+ uv run python -m openjarvis.evals run --config "$config" \
|| echo " FAILED (continuing)"
done
done
echo "==> Generating T1 from smoke results"
uv run python -m openjarvis.evals.comparison.table_gen \
- --results-glob "results/smoke/**/summary.json" \
+ --results-glob "results/smoke/**/*.summary.json" \
--tables T1 \
--output-dir results/smoke/tables/
diff --git a/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs b/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs
index d199a740..242e68fc 100644
--- a/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs
+++ b/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs
@@ -6,13 +6,14 @@
// --task <prompt> --model <m> --base-url <url> --api-key <k> \
// --output-json <path> [--workspace <path>]
//
-// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw chat --message <task> --json`
-// (or equivalent non-interactive entry). Emits a JSON dict matching the
+// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw agent --local
+// --message <task> --json`. Emits a JSON dict matching the
// _RunnerOutput Pydantic schema in _subprocess_runner.py.
-import { writeFileSync, existsSync } from 'node:fs';
+import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
import { spawn } from 'node:child_process';
import { join } from 'node:path';
+import { tmpdir } from 'node:os';
import { argv, env, exit, chdir } from 'node:process';
function parseArgs(args) {
@@ -47,12 +48,30 @@ async function main() {
return 3;
}
+ const requestedModel = String(args.model || '').includes('/')
+ ? String(args.model || '')
+ : `ollama/${args.model}`;
+ const runDir = mkdtempSync(join(tmpdir(), 'openjarvis-openclaw-'));
+ const configPath = join(runDir, 'openclaw.json');
+ writeFileSync(configPath, JSON.stringify({
+ agents: {
+ defaults: {
+ model: { primary: requestedModel },
+ models: { [requestedModel]: {} },
+ },
+ },
+ }));
+
const childEnv = {
...env,
OPENCLAW_MODEL: args.model,
OPENCLAW_BASE_URL: args.base_url,
OPENCLAW_API_KEY: args.api_key,
+ OPENCLAW_CONFIG_PATH: configPath,
};
+ if (requestedModel.startsWith('ollama/')) {
+ childEnv.OLLAMA_API_KEY = args.api_key || 'ollama';
+ }
// Use the SAME node executable that's running this script — picking up
// 'node' from PATH can resolve to a system Node too old for OpenClaw
@@ -95,11 +114,18 @@ async function main() {
// { response: str, usage: {...}, messages: [{...}], tool_calls: int }
try {
const parsed = JSON.parse(stdout);
- output.content = parsed.response || '';
- output.usage = parsed.usage || {};
- output.trajectory = parsed.messages || [];
+ const payloads = Array.isArray(parsed.payloads) ? parsed.payloads : [];
+ const messages = Array.isArray(parsed.messages) ? parsed.messages : [];
+ const usage = parsed.usage || parsed.meta?.agentMeta?.usage || {};
+ output.content = parsed.response || payloads.map((p) => p.text || '').join('\n');
+ output.usage = {
+ prompt_tokens: usage.prompt_tokens ?? usage.input ?? 0,
+ completion_tokens: usage.completion_tokens ?? usage.output ?? 0,
+ total_tokens: usage.total_tokens ?? usage.total ?? 0,
+ };
+ output.trajectory = messages;
output.tool_calls = parsed.tool_calls || 0;
- output.turn_count = (parsed.messages || []).filter(
+ output.turn_count = messages.filter(
(m) => m.role === 'assistant'
).length;
} catch (e) {
diff --git a/src/openjarvis/evals/comparison/table_gen.py b/src/openjarvis/evals/comparison/table_gen.py
index b224590a..575a1660 100644
--- a/src/openjarvis/evals/comparison/table_gen.py
+++ b/src/openjarvis/evals/comparison/table_gen.py
@@ -38,6 +38,10 @@ class MixedCommitError(TableGenError):
"""Raised when one (framework, model, benchmark) cell has multiple commits."""
+class NoResultsError(TableGenError):
+ """Raised when no valid summary files were loaded."""
+
+
class _MetricStats(BaseModel):
mean: float
std: float
@@ -395,6 +399,11 @@ def main(results_glob: str, tables: str, output_dir: Optional[Path]) -> None:
click.echo(
f"Loaded {len(frame.df)} metric rows; {frame.unloadable_count} files skipped."
)
+ if frame.df.is_empty():
+ raise click.ClickException(
+ "No valid summary files matched --results-glob; refusing to emit empty "
+ "tables."
+ )
requested = [t.strip() for t in tables.split(",") if t.strip()]
for name in requested:
diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py
index 851a8c8e..9b03d220 100644
--- a/src/openjarvis/evals/core/runner.py
+++ b/src/openjarvis/evals/core/runner.py
@@ -307,6 +307,38 @@ class EvalRunner:
def _process_one(self, record: EvalRecord) -> EvalResult:
"""Process a single evaluation sample."""
cfg = self._config
+
+ def _backend_error_result(full: dict, message: str) -> EvalResult:
+ usage = full.get("usage", {}) or {}
+ energy_j = full.get("energy_joules", 0.0) or 0.0
+ power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0
+ return EvalResult(
+ record_id=record.record_id,
+ model_answer=full.get("content", "") or "",
+ error=message,
+ latency_seconds=full.get("latency_seconds", 0.0) or 0.0,
+ prompt_tokens=usage.get("prompt_tokens", 0),
+ completion_tokens=usage.get("completion_tokens", 0),
+ cost_usd=full.get("cost_usd", 0.0) or 0.0,
+ ttft=full.get("ttft", 0.0) or 0.0,
+ energy_joules=energy_j,
+ power_watts=power_w,
+ gpu_utilization_pct=full.get("gpu_utilization_pct", 0.0) or 0.0,
+ throughput_tok_per_sec=full.get("throughput_tok_per_sec", 0.0)
+ or 0.0,
+ trace_data=full.get("trace_data"),
+ framework=full.get(
+ "framework",
+ getattr(self._backend, "framework_name", "openjarvis"),
+ ),
+ framework_commit=full.get(
+ "framework_commit",
+ getattr(self._backend, "framework_commit_value", "") or "",
+ ),
+ tool_calls=int(full.get("tool_calls", 0) or 0),
+ turn_count=int(full.get("turn_count", 0) or 0),
+ )
+
try:
gen_kwargs: dict = dict(
model=cfg.model,
@@ -327,6 +359,8 @@ class EvalRunner:
**gen_kwargs,
)
full = full or {}
+ if full.get("error"):
+ return _backend_error_result(full, str(full["error"]))
all_tool_results = list(
full.get(
"tool_results",
@@ -346,6 +380,8 @@ class EvalRunner:
**gen_kwargs,
)
sfull = sfull or {}
+ if sfull.get("error"):
+ return _backend_error_result(sfull, str(sfull["error"]))
all_tool_results.extend(
sfull.get("tool_results", []),
)
@@ -363,6 +399,8 @@ class EvalRunner:
**gen_kwargs,
)
full = full or {}
+ if full.get("error"):
+ return _backend_error_result(full, str(full["error"]))
content = full.get("content", "")
is_correct, scoring_meta = self._scorer.score(
record,
@@ -378,7 +416,7 @@ class EvalRunner:
# 'or 0.0' handles the case where the key is present but the
# value is None -- .get(default) only kicks in when missing.
energy_j = full.get("energy_joules", 0.0) or 0.0
- power_w = full.get("power_watts", 0.0) or 0.0
+ power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0
throughput = full.get("throughput_tok_per_sec", 0.0) or 0.0
accuracy_score = 1.0 if is_correct else 0.0
diff --git a/tests/evals/comparison/test_export_to_table_gen_roundtrip.py b/tests/evals/comparison/test_export_to_table_gen_roundtrip.py
index fbe0c378..9ce5679b 100644
--- a/tests/evals/comparison/test_export_to_table_gen_roundtrip.py
+++ b/tests/evals/comparison/test_export_to_table_gen_roundtrip.py
@@ -122,7 +122,7 @@ class TestSummaryToDictEmitsRequiredFields:
completion_tokens=50,
cost_usd=0.001,
energy_joules=10.0,
- power_watts=20.0,
+ power_watts=30.0,
framework="hermes",
framework_commit="5d3be898a",
tool_calls=2,
@@ -156,6 +156,7 @@ class TestSummaryToDictEmitsRequiredFields:
assert d["n_tasks"] == 5
assert "accuracy" in d["metrics"]
assert "latency_seconds" in d["metrics"]
+ assert d["metrics"]["peak_power_w"]["mean"] == 30.0
assert d["metrics"]["accuracy"]["n"] == 5
assert d["metrics"]["latency_seconds"]["n"] == 5
# Existing rich schema also present (unchanged)
diff --git a/tests/evals/comparison/test_openclaw_runner_contract.py b/tests/evals/comparison/test_openclaw_runner_contract.py
new file mode 100644
index 00000000..0db4f5c1
--- /dev/null
+++ b/tests/evals/comparison/test_openclaw_runner_contract.py
@@ -0,0 +1,72 @@
+"""Contract tests for the OpenClaw subprocess runner."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+def test_openclaw_runner_parses_real_agent_json_shape(tmp_path: Path) -> None:
+ node = shutil.which("node")
+ if node is None:
+ pytest.skip("node not available")
+
+ fake_openclaw = tmp_path / "openclaw.mjs"
+ fake_openclaw.write_text(
+ "\n".join(
+ [
+ "console.log(JSON.stringify({",
+ " payloads: [{ text: 'hello from openclaw', mediaUrl: null }],",
+ " meta: {",
+ " agentMeta: { usage: { input: 11, output: 7, total: 18 } },",
+ " },",
+ "}));",
+ ]
+ ),
+ encoding="utf-8",
+ )
+
+ runner = (
+ Path(__file__).resolve().parents[3]
+ / "src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs"
+ )
+ out_json = tmp_path / "out.json"
+ env = {
+ "OPENCLAW_PATH": str(tmp_path),
+ "HOME": str(tmp_path / "home"),
+ }
+ result = subprocess.run(
+ [
+ node,
+ str(runner),
+ "--task",
+ "Say hello.",
+ "--model",
+ "qwen3:0.6b",
+ "--base-url",
+ "http://127.0.0.1:11434/v1",
+ "--api-key",
+ "ollama",
+ "--output-json",
+ str(out_json),
+ ],
+ env=env,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ check=False,
+ )
+
+ assert result.returncode == 0, result.stderr
+ data = json.loads(out_json.read_text(encoding="utf-8"))
+ assert data["content"] == "hello from openclaw"
+ assert data["usage"] == {
+ "prompt_tokens": 11,
+ "completion_tokens": 7,
+ "total_tokens": 18,
+ }
+ assert data["error"] is None
diff --git a/tests/evals/comparison/test_runner_error_path_framework.py b/tests/evals/comparison/test_runner_error_path_framework.py
index c6272e39..8280dd7a 100644
--- a/tests/evals/comparison/test_runner_error_path_framework.py
+++ b/tests/evals/comparison/test_runner_error_path_framework.py
@@ -9,6 +9,7 @@ happy-path EvalResult construction.
from __future__ import annotations
+import json
from typing import Any, Dict, Iterable, List, Optional, Tuple
from openjarvis.evals.core.dataset import DatasetProvider
@@ -33,6 +34,57 @@ class _FailingBackend:
pass
+class _PeakOnlyBackend:
+ """Mock external-style backend that reports peak_power_w but no power_watts."""
+
+ backend_id = "hermes"
+ framework_name = "hermes"
+ framework_commit_value = "abc12345"
+
+ def generate(self, *args: Any, **kwargs: Any) -> str:
+ return "hi"
+
+ def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
+ return {
+ "content": "hi",
+ "usage": {},
+ "latency_seconds": 1.0,
+ "energy_joules": 10.0,
+ "peak_power_w": 42.0,
+ "framework": "hermes",
+ "framework_commit": "abc12345",
+ }
+
+ def close(self) -> None:
+ pass
+
+
+class _BackendErrorPayload:
+ """Mock external backend that returns an error payload instead of raising."""
+
+ backend_id = "hermes"
+ framework_name = "hermes"
+ framework_commit_value = "abc12345"
+
+ def generate(self, *args: Any, **kwargs: Any) -> str:
+ return ""
+
+ def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
+ return {
+ "content": "",
+ "usage": {"prompt_tokens": 3, "completion_tokens": 0},
+ "latency_seconds": 1.5,
+ "energy_joules": None,
+ "peak_power_w": 17.0,
+ "framework": "hermes",
+ "framework_commit": "abc12345",
+ "error": "backend reported failure",
+ }
+
+ def close(self) -> None:
+ pass
+
+
class _SingleRecordDataset(DatasetProvider):
dataset_id = "mock"
dataset_name = "mock"
@@ -141,6 +193,70 @@ class TestErrorPathFrameworkPropagation:
assert results[0].error is not None
+class TestExternalTelemetryPropagation:
+ def test_peak_power_falls_back_to_power_watts(self, tmp_path: Any) -> None:
+ """External backends report peak_power_w; runner should preserve it
+ in EvalResult.power_watts so summary/table metrics are populated."""
+ cfg = RunConfig(
+ benchmark="mock",
+ backend="hermes",
+ model="mock-model",
+ max_samples=1,
+ max_workers=1,
+ output_path=str(tmp_path / "out.jsonl"),
+ )
+ runner = EvalRunner(
+ config=cfg,
+ dataset=_SingleRecordDataset(),
+ backend=_PeakOnlyBackend(), # type: ignore[arg-type]
+ scorer=_NoopScorer(),
+ )
+ runner.run()
+
+ assert runner.results[0].power_watts == 42.0
+ summary = json.loads((tmp_path / "out.summary.json").read_text())
+ assert summary["metrics"]["peak_power_w"]["mean"] == 42.0
+
+
+class TestBackendErrorPayloadPropagation:
+ def test_backend_error_payload_marks_result_error(self, tmp_path: Any) -> None:
+ """Backends can return structured error payloads without raising;
+ those must count as errors rather than scored empty answers."""
+ cfg = RunConfig(
+ benchmark="mock",
+ backend="hermes",
+ model="mock-model",
+ max_samples=1,
+ max_workers=1,
+ output_path=str(tmp_path / "out.jsonl"),
+ )
+ runner = EvalRunner(
+ config=cfg,
+ dataset=_SingleRecordDataset(),
+ backend=_BackendErrorPayload(), # type: ignore[arg-type]
+ scorer=_NoopScorer(),
+ )
+ summary = runner.run()
+
+ result = runner.results[0]
+ assert result.error == "backend reported failure"
+ assert result.is_correct is None
+ assert result.framework == "hermes"
+ assert result.framework_commit == "abc12345"
+ assert result.power_watts == 17.0
+ assert result.prompt_tokens == 3
+ assert summary.errors == 1
+ assert summary.scored_samples == 0
+
+ summary_data = json.loads((tmp_path / "out.summary.json").read_text())
+ assert summary_data["errors"] == 1
+ assert summary_data["metrics"]["accuracy"] == {
+ "mean": 0.0,
+ "std": 0.0,
+ "n": 0,
+ }
+
+
class _MockBackendWithCommit:
"""Mock backend that exposes framework_name AND framework_commit_value."""
diff --git a/tests/evals/comparison/test_table_gen.py b/tests/evals/comparison/test_table_gen.py
index 18c2fcf8..bf82daa5 100644
--- a/tests/evals/comparison/test_table_gen.py
+++ b/tests/evals/comparison/test_table_gen.py
@@ -13,6 +13,7 @@ from openjarvis.evals.comparison.table_gen import ( # noqa: E402
MixedCommitError,
ResultsFrame,
load_results,
+ main,
)
@@ -70,6 +71,23 @@ class TestLoadResults:
with pytest.raises(MixedCommitError, match="abc123.*zzz999"):
load_results(str(tmp_path / "**" / "summary.json"))
+ def test_cli_refuses_empty_result_set(self, tmp_path: Path) -> None:
+ from click.testing import CliRunner
+
+ result = CliRunner().invoke(
+ main,
+ [
+ "--results-glob",
+ str(tmp_path / "**" / "summary.json"),
+ "--tables",
+ "T1",
+ "--output-dir",
+ str(tmp_path / "tables"),
+ ],
+ )
+ assert result.exit_code != 0
+ assert "No valid summary files matched" in result.output
+
class TestRenderBooktabs:
def test_emits_valid_tabular(self) -> None: