Personal AI, On Personal Devices
Python
fix(evals): clean up AI stack runtime harness (#320)
Avanika Narayana689be0c69a9
diff --git a/scripts/experiments/a1_seed_feedback.py b/scripts/experiments/a1_seed_feedback.pydeleted file mode 100644index c6565d92..00000000--- a/scripts/experiments/a1_seed_feedback.py+++ /dev/null@@ -1,246 +0,0 @@-#!/usr/bin/env python3-"""A1: Seed feedback on all traces in the local M1 traces.db.--- Judges each unscored trace with Sonnet 4.6 using the calibration-validated prompt-- Parallelized via ThreadPoolExecutor (8 workers) for I/O-bound API calls-- Writes feedback to local traces.db via TraceStore.update_feedback-- Logs every call to a JSONL audit file-- Idempotent: skips traces that already have feedback (safe to re-run)-"""--from __future__ import annotations--import json-import os-import re-import sqlite3-import sys-import threading-import time-from concurrent.futures import ThreadPoolExecutor, as_completed-from datetime import datetime-from pathlib import Path--from openjarvis.core.types import Message, Role-from openjarvis.engine.cloud import CloudEngine-from openjarvis.traces.store import TraceStore--HOME = Path(os.environ.get("OPENJARVIS_HOME", "/scratch/user/jonsaadfalcon/openjarvis-m1"))-DB = HOME / "traces.db"-LOG = HOME / "a1_feedback_log.jsonl"-MODEL = "claude-sonnet-4-6"-MAX_WORKERS = 8--JUDGE_PROMPT = """\-You are evaluating whether an AI agent successfully completed its assigned task.--Assign a SCORE from the set {{0.2, 0.4, 0.6, 0.8}} using this rubric:--- 0.8 = Clean success. Task completed correctly. Minor stylistic issues don't affect correctness.-- 0.6 = Partial. Real progress made but the answer has real gaps — missed requirement, incomplete output, recovered from errors but final result imperfect.-- 0.4 = Poor. Some progress but the result is clearly incomplete, wrong, or the agent got mostly stuck.-- 0.2 = Failure. Agent crashed, got stuck in a loop, gave up, hit a budget/poll/token limit before finishing, or produced no usable result.--IMPORTANT: Do not trust the agent's own self-assessment. Agents often narrate "I am stuck" or "I hit an error" — those are failure signals. Agents sometimes claim success when the actual output is incomplete — look at the concrete result, not the rhetoric.--TASK QUERY (first 1200 chars):-<<<-{query}->>>--AGENT FINAL RESULT (first 2500 chars):-<<<-{result_head}->>>-{tail_section}-Respond in EXACTLY this format, nothing else:-SCORE=<one of 0.2, 0.4, 0.6, 0.8>-REASON=<one brief sentence>-"""---SCORE_RE = re.compile(r"SCORE\s*=\s*(0?\.[2468])", re.IGNORECASE)-REASON_RE = re.compile(r"REASON\s*=\s*(.+?)\s*$", re.IGNORECASE | re.DOTALL)---def build_prompt(query: str, result: str) -> str:- q = (query or "")[:1200]- head = (result or "")[:2500]- if result and len(result) > 3000:- tail = f"\nAGENT FINAL RESULT (last 500 chars):\n<<<\n{result[-500:]}\n>>>\n"- else:- tail = ""- return JUDGE_PROMPT.format(query=q, result_head=head, tail_section=tail)---def judge_one(ce: CloudEngine, trace_id: str, query: str, result: str) -> dict:- prompt = build_prompt(query, result)- t0 = time.time()- try:- resp = ce.generate(- messages=[Message(role=Role.USER, content=prompt)],- model=MODEL,- max_tokens=150,- temperature=0.0,- )- content = resp.get("content", "") or ""- cost = resp.get("cost_usd", 0.0) or 0.0- usage = resp.get("usage", {}) or {}- in_tok = usage.get("prompt_tokens", 0) or usage.get("input_tokens", 0)- out_tok = usage.get("completion_tokens", 0) or usage.get("output_tokens", 0)-- m = SCORE_RE.search(content)- score = float(m.group(1)) if m else None- mr = REASON_RE.search(content)- reason = mr.group(1).strip() if mr else "(parse failed)"-- return {- "trace_id": trace_id,- "score": score,- "reason": reason,- "raw": content,- "cost": cost,- "input_tokens": in_tok,- "output_tokens": out_tok,- "elapsed": time.time() - t0,- "judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",- "error": None,- }- except Exception as e:- return {- "trace_id": trace_id,- "score": None,- "reason": None,- "raw": None,- "cost": 0.0,- "input_tokens": 0,- "output_tokens": 0,- "elapsed": time.time() - t0,- "judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",- "error": f"{type(e).__name__}: {e}",- }---def main() -> int:- if not os.environ.get("ANTHROPIC_API_KEY"):- print("ERROR: ANTHROPIC_API_KEY not set", file=sys.stderr)- return 1-- store = TraceStore(DB)- # Use the TraceStore's connection for the initial read- conn = store._conn- conn.row_factory = sqlite3.Row-- # Pull all traces that need scoring- rows = list(conn.execute(- "SELECT trace_id, query, result, agent, model FROM traces "- "WHERE feedback IS NULL"- ))- already_scored = conn.execute(- "SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL"- ).fetchone()[0]-- print(f"traces.db: {conn.execute('SELECT COUNT(*) FROM traces').fetchone()[0]} total")- print(f" already scored: {already_scored}")- print(f" to judge: {len(rows)}")- print(f"parallelism: {MAX_WORKERS} workers")- print(f"log: {LOG}")-- if not rows:- print("Nothing to do.")- return 0-- ce = CloudEngine()- write_lock = threading.Lock()- log_lock = threading.Lock()- log_fp = open(LOG, "a", encoding="utf-8")-- # Write a header line marking this run- log_fp.write(json.dumps({- "_run_started": datetime.utcnow().isoformat(timespec="seconds") + "Z",- "model": MODEL,- "workers": MAX_WORKERS,- "to_judge": len(rows),- "already_scored": already_scored,- }) + "\n")- log_fp.flush()-- total_cost = 0.0- done = 0- errors = 0- score_counts: dict = {}- t_start = time.time()-- def on_result(res: dict) -> None:- nonlocal total_cost, done, errors- trace_id = res["trace_id"]-- # Write feedback to DB (if we got a valid score)- if res["score"] is not None and res["error"] is None:- with write_lock:- store.update_feedback(trace_id, res["score"])- conn.commit()- else:- errors += 1-- with log_lock:- log_fp.write(json.dumps(res) + "\n")- log_fp.flush()-- total_cost += res["cost"] or 0.0- done += 1- score_counts[res["score"]] = score_counts.get(res["score"], 0) + 1-- # Progress every 50 or on error- if done % 50 == 0 or res["error"]:- elapsed = time.time() - t_start- rate = done / max(0.001, elapsed)- eta = (len(rows) - done) / max(0.001, rate)- tag = "ERR " if res["error"] else " "- print(- f"{tag}[{done:4}/{len(rows)}] score={res['score']} "- f"cost=${total_cost:.3f} "- f"rate={rate:.1f}/s eta={eta:.0f}s "- f"errors={errors}"- )- if res["error"]:- print(f" ERROR on {trace_id[:12]}: {res['error']}")-- with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:- futures = [- pool.submit(judge_one, ce, r["trace_id"], r["query"] or "", r["result"] or "")- for r in rows- ]- for fut in as_completed(futures):- res = fut.result()- on_result(res)-- log_fp.close()-- elapsed = time.time() - t_start- print(f"\n{'='*60}")- print(f"A1 COMPLETE in {elapsed:.1f}s ({elapsed/60:.1f}min)")- print(f"Total cost: ${total_cost:.4f}")- print(f"Errors: {errors}/{len(rows)}")- print(f"Score distribution:")- for k in sorted(score_counts.keys(), key=lambda x: (x is None, x)):- v = score_counts[k]- pct = 100 * v / len(rows)- label = {0.2: "failure", 0.4: "poor", 0.6: "partial", 0.8: "clean", None: "ERROR"}.get(k, "?")- print(f" {k} ({label}): {v} ({pct:.1f}%)")-- # Verify by re-counting from DB- with_fb = conn.execute(- "SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL"- ).fetchone()[0]- above_gate = conn.execute(- "SELECT COUNT(*) FROM traces WHERE feedback >= 0.7"- ).fetchone()[0]- print(f"\nPost-A1 DB state:")- print(f" traces with feedback: {with_fb}")- print(f" traces passing 0.7 gate (eligible for personal benchmark): {above_gate}")- return 0 if errors == 0 else 2---if __name__ == "__main__":- sys.exit(main())diff --git a/scripts/experiments/generate_distillation_configs.py b/scripts/experiments/generate_distillation_configs.pydeleted file mode 100644index 1ef98274..00000000--- a/scripts/experiments/generate_distillation_configs.py+++ /dev/null@@ -1,304 +0,0 @@-#!/usr/bin/env python3-"""Generate all distillation experiment TOML configs.--Produces configs for 7 experiment axes × multiple settings.-Run: python scripts/experiments/generate_distillation_configs.py-"""--from __future__ import annotations--import itertools-from pathlib import Path--CONFIGS_DIR = Path("src/openjarvis/evals/configs/distillation")--# ── Teacher models ──────────────────────────────────────────────────────────-TEACHERS = {- "opus": {- "model": "claude-opus-4-6",- "engine": "cloud",- "provider": "anthropic",- },- "gpt54": {- "model": "gpt-5.4",- "engine": "cloud",- "provider": "openai",- },- "gemini": {- "model": "gemini-3.1-pro-preview",- "engine": "cloud",- "provider": "google",- },- "qwen397b": {- "model": "Qwen/Qwen3.5-397B-A17B-FP8",- "engine": "vllm",- "provider": "local",- "note": "# Requires 8×H100, vLLM serve on port 8010",- },-}--# ── Student models ──────────────────────────────────────────────────────────-# Served via vLLM on this H100 node. 27B uses the FP8 weights that fit on a-# single H100; 2B and 9B use standard FP16.-STUDENTS = {- "2b": {"model": "Qwen/Qwen3.5-2B", "engine": "vllm", "port": 8000},- "9b": {"model": "Qwen/Qwen3.5-9B", "engine": "vllm", "port": 8001},- "27b": {"model": "Qwen/Qwen3.5-27B-FP8", "engine": "vllm", "port": 8002},-}--# ── Benchmarks ──────────────────────────────────────────────────────────────-BENCHMARKS = {- "pb": "pinchbench",- "tc15": "toolcall15",- "tb": "taubench",-}--# ── Data configs ────────────────────────────────────────────────────────────-DATA_CONFIGS = {- "C1": {- "desc": "Zero test data — external traces only (GeneralThought + ADP)",- "trace_source": "external",- "benchmark_queries_visible": False,- },- "C2": {- "desc": "Test queries only — benchmark traces visible, answers hidden",- "trace_source": "benchmark",- "benchmark_queries_visible": True,- },- "C3": {- "desc": "Test queries + external — both benchmark and external traces",- "trace_source": "both",- "benchmark_queries_visible": True,- },-}--# ── Budget presets ──────────────────────────────────────────────────────────-BUDGETS = {- "minimal": {"max_tool_calls": 5, "max_cost": 0.50},- "standard": {"max_tool_calls": 15, "max_cost": 2.00},- "thorough": {"max_tool_calls": 30, "max_cost": 5.00},- "exhaustive": {"max_tool_calls": 50, "max_cost": 10.00},-}--# ── Gate presets ────────────────────────────────────────────────────────────-GATES = {- "permissive": {"min_improvement": 0.0, "max_regression": 0.10},- "standard": {"min_improvement": 0.0, "max_regression": 0.05},- "strict": {"min_improvement": 0.02, "max_regression": 0.02},- "none": {"min_improvement": -1.0, "max_regression": 1.0},-}--# ── Autonomy presets ────────────────────────────────────────────────────────-AUTONOMY_MODES = ["auto", "tiered", "manual"]---def render_config(- *,- experiment: str,- teacher_key: str,- student_key: str,- benchmark_key: str,- data_config_key: str = "C2",- budget_key: str = "standard",- gate_key: str = "standard",- autonomy: str = "auto",- iterative_sessions: int = 1,-) -> str:- """Render a TOML config string."""- teacher = TEACHERS[teacher_key]- student = STUDENTS[student_key]- benchmark = BENCHMARKS[benchmark_key]- data_cfg = DATA_CONFIGS[data_config_key]- budget = BUDGETS[budget_key]- gate = GATES[gate_key]-- note = teacher.get("note", "")- note_line = f"\n{note}" if note else ""-- return f"""\-# Distillation Experiment Config-# Experiment: {experiment}-# Teacher: {teacher["model"]} ({teacher["provider"]})-# Student: {student["model"]} ({student["engine"]})-# Benchmark: {benchmark}-# Data config: {data_config_key} — {data_cfg["desc"]}-# Budget: {budget_key} ({budget["max_tool_calls"]} tool calls, ${budget["max_cost"]:.2f})-# Gate: {gate_key} (min_improvement={gate["min_improvement"]}, max_regression={gate["max_regression"]})-# Autonomy: {autonomy}-# Iterative sessions: {iterative_sessions}-{note_line}--[intelligence]-default_model = "{student["model"]}"--[engine]-default = "{student["engine"]}"--[engine.vllm]-host = "http://localhost:{student.get("port", 8000)}"--[learning.distillation]-enabled = true-autonomy_mode = "{autonomy}"-teacher_model = "{teacher["model"]}"-max_cost_per_session_usd = {budget["max_cost"]}-max_tool_calls_per_diagnosis = {budget["max_tool_calls"]}--[learning.distillation.gate]-min_improvement = {gate["min_improvement"]}-max_regression = {gate["max_regression"]}-benchmark_subsample_size = 50--[learning.distillation.benchmark]-synthesis_feedback_threshold = 0.7-max_benchmark_size = 200--[learning.distillation.experiment]-# Metadata for the experiment runner (not read by distillation itself)-experiment_id = "{experiment}"-teacher_key = "{teacher_key}"-student_key = "{student_key}"-benchmark = "{benchmark}"-data_config = "{data_config_key}"-trace_source = "{data_cfg["trace_source"]}"-benchmark_queries_visible = {str(data_cfg["benchmark_queries_visible"]).lower()}-budget_key = "{budget_key}"-gate_key = "{gate_key}"-iterative_sessions = {iterative_sessions}-"""---def write_config(subdir: str, filename: str, content: str) -> Path:- path = CONFIGS_DIR / subdir / filename- path.parent.mkdir(parents=True, exist_ok=True)- path.write_text(content, encoding="utf-8")- return path---def generate_all() -> int:- count = 0-- # ── Exp 1a: Teacher Model Ablation ────────────────────────────────────- # Fix: S-9b, B-standard, A-auto, G-standard, I-single- # Vary: teacher × benchmark × data_config- for teacher_key, bench_key, dc_key in itertools.product(- TEACHERS, BENCHMARKS, DATA_CONFIGS- ):- filename = f"{teacher_key}-9b-{bench_key}-{dc_key}.toml"- content = render_config(- experiment=f"exp1a-teacher/{teacher_key}-{bench_key}-{dc_key}",- teacher_key=teacher_key,- student_key="9b",- benchmark_key=bench_key,- data_config_key=dc_key,- )- write_config("exp1a-teacher", filename, content)- count += 1-- # ── Exp 1b: Budget Ablation ───────────────────────────────────────────- # Fix: S-9b, T-sonnet(opus for quality), A-auto, G-standard, I-single- # Vary: budget × benchmark × data_config- for budget_key, bench_key, dc_key in itertools.product(- BUDGETS, BENCHMARKS, DATA_CONFIGS- ):- filename = f"{budget_key}-9b-{bench_key}-{dc_key}.toml"- content = render_config(- experiment=f"exp1b-budget/{budget_key}-{bench_key}-{dc_key}",- teacher_key="opus",- student_key="9b",- benchmark_key=bench_key,- data_config_key=dc_key,- budget_key=budget_key,- )- write_config("exp1b-budget", filename, content)- count += 1-- # ── Exp 1c: Student Model Scaling ─────────────────────────────────────- # Fix: T-opus, B-standard, A-auto, G-standard, I-single- # Vary: student × benchmark × data_config- for student_key, bench_key, dc_key in itertools.product(- STUDENTS, BENCHMARKS, DATA_CONFIGS- ):- filename = f"opus-{student_key}-{bench_key}-{dc_key}.toml"- content = render_config(- experiment=f"exp1c-student/opus-{student_key}-{bench_key}-{dc_key}",- teacher_key="opus",- student_key=student_key,- benchmark_key=bench_key,- data_config_key=dc_key,- )- write_config("exp1c-student", filename, content)- count += 1-- # ── Exp 2a: Gate Strictness ───────────────────────────────────────────- # Fix: S-9b, T-opus, B-standard, A-auto, I-single- # Vary: gate × benchmark- for gate_key, bench_key in itertools.product(GATES, BENCHMARKS):- filename = f"{gate_key}-9b-{bench_key}.toml"- content = render_config(- experiment=f"exp2a-gate/{gate_key}-{bench_key}",- teacher_key="opus",- student_key="9b",- benchmark_key=bench_key,- gate_key=gate_key,- )- write_config("exp2a-gate", filename, content)- count += 1-- # ── Exp 2b: Autonomy Mode ────────────────────────────────────────────- # Fix: S-9b, T-opus, B-standard, G-standard, I-single- # Vary: autonomy × benchmark- for autonomy, bench_key in itertools.product(AUTONOMY_MODES, BENCHMARKS):- filename = f"{autonomy}-9b-{bench_key}.toml"- content = render_config(- experiment=f"exp2b-autonomy/{autonomy}-{bench_key}",- teacher_key="opus",- student_key="9b",- benchmark_key=bench_key,- autonomy=autonomy,- )- write_config("exp2b-autonomy", filename, content)- count += 1-- # ── Exp 3a: Iterative Sessions ───────────────────────────────────────- # Fix: S-9b, T-opus, B-standard, A-auto, G-standard- # Vary: number of chained sessions × benchmark- for n_sessions, bench_key in itertools.product([1, 3, 5], BENCHMARKS):- filename = f"iter{n_sessions}-9b-{bench_key}.toml"- content = render_config(- experiment=f"exp3a-iterative/iter{n_sessions}-{bench_key}",- teacher_key="opus",- student_key="9b",- benchmark_key=bench_key,- iterative_sessions=n_sessions,- )- write_config("exp3a-iterative", filename, content)- count += 1-- # ── Exp 3b: Cross-Benchmark Transfer ─────────────────────────────────- # Optimize using traces from benchmark X, eval on benchmark Y- for opt_bench, eval_bench in itertools.permutations(BENCHMARKS, 2):- filename = f"opt-{opt_bench}-eval-{eval_bench}-9b.toml"- content = render_config(- experiment=f"exp3b-transfer/opt-{opt_bench}-eval-{eval_bench}",- teacher_key="opus",- student_key="9b",- benchmark_key=opt_bench, # Traces from this benchmark- )- # Add eval benchmark as metadata- content += f'\neval_benchmark = "{BENCHMARKS[eval_bench]}"\n'- write_config("exp3b-transfer", filename, content)- count += 1-- return count---if __name__ == "__main__":- n = generate_all()- print(f"Generated {n} config files in {CONFIGS_DIR}/")-- # Print summary- for subdir in sorted(CONFIGS_DIR.iterdir()):- if subdir.is_dir():- files = list(subdir.glob("*.toml"))- print(f" {subdir.name}/: {len(files)} configs")diff --git a/scripts/experiments/m2_collect_results.py b/scripts/experiments/m2_collect_results.pydeleted file mode 100644index 41319399..00000000--- a/scripts/experiments/m2_collect_results.py+++ /dev/null@@ -1,133 +0,0 @@-#!/usr/bin/env python3-"""M2: Collect distilled eval results and produce comparison table.--Reads .summary.json files from results/neurips-2026/{distilled,baselines}/-and produces a before/after comparison against the Step 1 baseline numbers.-"""--from __future__ import annotations--import json-import sys-from pathlib import Path--# Jon's Step 1 baselines — used when a local baseline result doesn't exist-STEP1_BASELINES = {- "2b": {"toolcall15": 33.3, "pinchbench": 69.6, "livecodebench": 5.6, "taubench": 70.0, "taubench-telecom": 0.0, "gaia": 0.0, "liveresearch": 0.0, "liveresearchbench": None},- "9b": {"toolcall15": 46.7, "pinchbench": 95.7, "livecodebench": 17.6, "taubench": 85.0, "taubench-telecom": 80.0, "gaia": 38.0, "liveresearch": 75.0, "liveresearchbench": None},- "27b": {"toolcall15": 40.0, "pinchbench": 65.2, "livecodebench": 20.0, "taubench": 75.0, "taubench-telecom": 75.0, "gaia": 48.0, "liveresearch": 66.7, "liveresearchbench": None},-}--# Which benchmarks go through the agent layer (where distillation edits actually apply)-AGENT_BENCHMARKS = {"pinchbench", "gaia", "liveresearch"}--DISTILLED_ROOT = Path("results/neurips-2026/distilled")-BASELINE_ROOT = Path("results/neurips-2026/baselines")---def find_summary(root: Path, size: str, bench: str) -> Path | None:- """Find the summary JSON for a model × benchmark run."""- # Expected path: root/qwen-{size}/{bench}/{bench}_Qwen-Qwen3.5-{size}.summary.json- candidates = list(root.glob(f"qwen-{size}/{bench}/*.summary.json"))- return candidates[0] if candidates else None---def load_accuracy(summary_path: Path) -> float | None:- """Extract overall accuracy from a summary.json file."""- try:- d = json.loads(summary_path.read_text())- # The summary has various shapes; try a few- for key in ["overall_accuracy", "accuracy", "overall_score"]:- if key in d:- return float(d[key]) * 100 if d[key] <= 1.0 else float(d[key])- # Try nested- if "results" in d:- for r in d["results"]:- if "accuracy" in r:- return float(r["accuracy"]) * 100 if r["accuracy"] <= 1.0 else float(r["accuracy"])- except Exception as e:- print(f" error reading {summary_path}: {e}", file=sys.stderr)- return None---def main() -> int:- print("=" * 100)- print("M2 Distilled vs Baseline Comparison")- print("=" * 100)- print()- print(f"{'Model':8} {'Benchmark':20} {'Baseline':>10} {'Distilled':>10} {'Delta':>10} {'Agent?':>10}")- print("-" * 100)-- benchmarks = [- "toolcall15", "pinchbench", "livecodebench",- "taubench", "taubench-telecom",- "gaia", "liveresearch", "liveresearchbench",- ]-- summary_rows = []- for size in ["2b", "9b", "27b"]:- for bench in benchmarks:- # Baseline: prefer local file (if run this session), fall back to Step 1 numbers- baseline_path = find_summary(BASELINE_ROOT, size, bench)- if baseline_path:- baseline = load_accuracy(baseline_path)- base_source = "local"- else:- baseline = STEP1_BASELINES[size].get(bench)- base_source = "step1"-- # Distilled: must be local from this session- distilled_path = find_summary(DISTILLED_ROOT, size, bench)- distilled = load_accuracy(distilled_path) if distilled_path else None-- # Format- b_str = f"{baseline:.1f}%" if baseline is not None else "—"- d_str = f"{distilled:.1f}%" if distilled is not None else "pending"- if baseline is not None and distilled is not None:- delta = distilled - baseline- d_sign = "+" if delta >= 0 else ""- delta_str = f"{d_sign}{delta:.1f}%"- else:- delta_str = "—"- agent = "AGENT" if bench in AGENT_BENCHMARKS else "direct"-- print(f"qwen-{size:4} {bench:20} {b_str:>10} {d_str:>10} {delta_str:>10} {agent:>10}")- summary_rows.append({- "model": f"qwen-{size}",- "benchmark": bench,- "baseline": baseline,- "distilled": distilled,- "delta": distilled - baseline if (baseline is not None and distilled is not None) else None,- "agent_benchmark": bench in AGENT_BENCHMARKS,- })- print()-- # Aggregate: agent vs direct benchmark deltas- print("=" * 100)- print("Aggregate deltas by benchmark type (paper finding)")- print("=" * 100)- agent_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and r["agent_benchmark"]]- direct_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and not r["agent_benchmark"]]- if agent_deltas:- mean_agent = sum(agent_deltas) / len(agent_deltas)- print(f"Agent benchmarks (PB, GAIA, DeepResearchBench): mean delta = {mean_agent:+.2f}% over {len(agent_deltas)} runs")- if direct_deltas:- mean_direct = sum(direct_deltas) / len(direct_deltas)- print(f"Direct benchmarks (TC15, TauB, TBTel, LRB, LCB): mean delta = {mean_direct:+.2f}% over {len(direct_deltas)} runs")- print()-- # Completion progress- expected = 24- distilled_count = sum(1 for r in summary_rows if r["distilled"] is not None)- print(f"Distilled runs complete: {distilled_count}/{expected}")-- # Save JSON- out = Path("results/neurips-2026/distillation-m2/m2_comparison.json")- out.parent.mkdir(parents=True, exist_ok=True)- out.write_text(json.dumps(summary_rows, indent=2, default=str))- print(f"Full data: {out}")- return 0---if __name__ == "__main__":- sys.exit(main())diff --git a/scripts/experiments/m2_create_distilled_configs.py b/scripts/experiments/m2_create_distilled_configs.pydeleted file mode 100644index a52e2eda..00000000--- a/scripts/experiments/m2_create_distilled_configs.py+++ /dev/null@@ -1,307 +0,0 @@-#!/usr/bin/env python3-"""M2: Create distilled eval configs from M1 consensus edits.--Generates 24 distilled configs (3 models × 8 benchmarks) by cloning-baseline configs and applying the consensus edits from M1. Also creates-4 missing baseline configs (livecodebench-qwen-9b, liveresearchbench-*).--Usage: python scripts/experiments/m2_create_distilled_configs.py-"""--from __future__ import annotations--from pathlib import Path--CONFIGS_DIR = Path("src/openjarvis/evals/configs")-M2_DIR = CONFIGS_DIR / "distillation" / "m2"--# ── Consensus values from M1 (1,131 edits) ──────────────────────────────-DISTILLED_TEMP = 0.2 # 84/134 votes (agent benchmarks only)-DISTILLED_MAX_TURNS = 15 # 56/125 votes (close: 25 had 49)-REMOVE_TOOLS = {"shell_exec", "http_request"} # 13 + 6 votes--# ── Model specs ──────────────────────────────────────────────────────────-MODELS = {- "2b": {"name": "Qwen/Qwen3.5-2B", "num_gpus": 1, "port": 8000},- "9b": {"name": "Qwen/Qwen3.5-9B", "num_gpus": 1, "port": 8001},- "27b": {"name": "Qwen/Qwen3.5-27B-FP8", "num_gpus": 1, "port": 8002},-}--# ── Benchmark specs ──────────────────────────────────────────────────────-# Each benchmark defines its baseline config and what changes in distilled.-BENCHMARKS = {- "toolcall15": {- "backend": "jarvis-direct",- "baseline_temp": 0.0,- "distilled_temp": 0.0, # CONTROL: no change for coding- "max_tokens": 4096,- "max_samples": None,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "extra_benchmark_fields": {},- },- "pinchbench": {- "backend": "jarvis-agent",- "agent": "native_openhands",- "baseline_temp": 0.6,- "distilled_temp": DISTILLED_TEMP,- "max_tokens": 8192,- "max_samples": None,- "judge_model": "claude-opus-4-5",- "judge_engine": "cloud",- "baseline_tools": [- "think", "file_read", "file_write", "web_search", "shell_exec",- "code_interpreter", "browser_navigate", "image_generate",- "calculator", "http_request", "pdf_extract",- ],- "extra_benchmark_fields": {},- },- "taubench": {- "backend": "jarvis-direct",- "baseline_temp": 0.7,- "distilled_temp": DISTILLED_TEMP,- "max_tokens": 4096,- "max_samples": 20,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "extra_benchmark_fields": {"split": "airline,retail"},- },- "taubench-telecom": {- "benchmark_name": "taubench", # same benchmark, different split- "backend": "jarvis-direct",- "baseline_temp": 0.7,- "distilled_temp": DISTILLED_TEMP,- "max_tokens": 4096,- "max_samples": 20,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "extra_benchmark_fields": {"split": "telecom"},- },- "gaia": {- "backend": "jarvis-agent",- "agent": "monitor_operative",- "baseline_temp": 0.6,- "distilled_temp": DISTILLED_TEMP,- "max_tokens": 8192,- "max_samples": 50,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "baseline_tools": [- "think", "calculator", "code_interpreter", "web_search", "file_read",- ],- "extra_benchmark_fields": {},- },- "liveresearch": {- "backend": "jarvis-agent",- "agent": "monitor_operative",- "baseline_temp": 0.6,- "distilled_temp": DISTILLED_TEMP,- "max_tokens": 16384,- "max_samples": 50,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "baseline_tools": [- "web_search", "file_read", "file_write", "code_interpreter", "think",- ],- "extra_benchmark_fields": {},- },- "liveresearchbench": {- "backend": "jarvis-direct",- "baseline_temp": 0.0,- "distilled_temp": 0.0, # CONTROL: reasoning benchmark- "max_tokens": 8192,- "max_samples": 50,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "extra_benchmark_fields": {},- },- "livecodebench": {- "backend": "jarvis-direct",- "baseline_temp": 0.0,- "distilled_temp": 0.0, # CONTROL: coding benchmark- "max_tokens": 4096,- "max_samples": 20,- "judge_model": "gpt-5-mini-2025-08-07",- "judge_engine": "cloud",- "extra_benchmark_fields": {},- },-}---def render_config(- *,- comment: str,- meta_name: str,- description: str,- temperature: float,- max_tokens: int,- judge_model: str,- judge_engine: str,- output_dir: str,- model_name: str,- model_engine: str,- num_gpus: int,- benchmark_name: str,- backend: str,- agent: str | None = None,- tools: list[str] | None = None,- max_samples: int | None = None,- extra_benchmark: dict | None = None,- seed: int = 42,-) -> str:- lines = [f"# {comment}"]- lines.append(f'[meta]\nname = "{meta_name}"\ndescription = "{description}"\n')- lines.append(f"[defaults]\ntemperature = {temperature}\nmax_tokens = {max_tokens}\n")- lines.append(f'[judge]\nmodel = "{judge_model}"\ntemperature = 0.0')- if judge_engine:- lines.append(f'engine = "{judge_engine}"')- lines.append(f"max_tokens = 4096\n")- lines.append(f'[run]\nmax_workers = 1\noutput_dir = "{output_dir}"\nseed = {seed}\n')- lines.append(f'[[models]]\nname = "{model_name}"\nengine = "{model_engine}"\nnum_gpus = {num_gpus}\n')- lines.append(f'[[benchmarks]]\nname = "{benchmark_name}"\nbackend = "{backend}"')- if agent:- lines.append(f'agent = "{agent}"')- if max_samples:- lines.append(f"max_samples = {max_samples}")- if tools:- tools_str = ", ".join(f'"{t}"' for t in tools)- lines.append(f"tools = [{tools_str}]")- if extra_benchmark:- for k, v in extra_benchmark.items():- if isinstance(v, str):- lines.append(f'{k} = "{v}"')- else:- lines.append(f"{k} = {v}")- lines.append("")- return "\n".join(lines)---def make_size_label(size: str) -> str:- return {"2b": "qwen-2b", "9b": "qwen-9b", "27b": "qwen-27b"}[size]---def generate_missing_baselines() -> int:- """Create baseline configs that don't exist yet."""- count = 0-- # livecodebench-qwen-9b (missing)- p = CONFIGS_DIR / "livecodebench-qwen-9b.toml"- if not p.exists():- b = BENCHMARKS["livecodebench"]- m = MODELS["9b"]- p.write_text(render_config(- comment="LiveCodeBench eval: Qwen3.5-9B (vLLM, 1 GPU)",- meta_name="livecodebench-qwen-9b",- description="LiveCodeBench on Qwen/Qwen3.5-9B (vLLM, 1 GPU)",- temperature=b["baseline_temp"],- max_tokens=b["max_tokens"],- judge_model=b["judge_model"],- judge_engine=b["judge_engine"],- output_dir="results/neurips-2026/baselines/qwen-9b/livecodebench/",- model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"],- benchmark_name="livecodebench", backend=b["backend"],- max_samples=b["max_samples"],- ))- count += 1- print(f" created {p}")-- # liveresearchbench-qwen-{2b,9b,27b}- for size, m in MODELS.items():- sl = make_size_label(size)- p = CONFIGS_DIR / f"liveresearchbench-{sl}.toml"- if not p.exists():- b = BENCHMARKS["liveresearchbench"]- p.write_text(render_config(- comment=f"LiveResearchBench (Salesforce): Qwen3.5-{size.upper()} (vLLM)",- meta_name=f"liveresearchbench-{sl}",- description=f"LiveResearchBench on {m['name']} (vLLM)",- temperature=b["baseline_temp"],- max_tokens=b["max_tokens"],- judge_model=b["judge_model"],- judge_engine=b["judge_engine"],- output_dir=f"results/neurips-2026/baselines/{sl}/liveresearchbench/",- model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"],- benchmark_name="liveresearchbench", backend=b["backend"],- max_samples=b["max_samples"],- ))- count += 1- print(f" created {p}")-- return count---def generate_distilled_configs() -> int:- """Create distilled configs for all 24 model × benchmark combos."""- M2_DIR.mkdir(parents=True, exist_ok=True)- count = 0-- for size, model in MODELS.items():- sl = make_size_label(size)- for bench_key, bench in BENCHMARKS.items():- bench_name = bench.get("benchmark_name", bench_key)- is_agent = bench["backend"] == "jarvis-agent"- temp = bench["distilled_temp"]-- # Tool list: remove broken tools for agent benchmarks- tools = None- if is_agent and "baseline_tools" in bench:- tools = [t for t in bench["baseline_tools"]- if t not in REMOVE_TOOLS]-- fname = f"{bench_key}-{sl}-distilled.toml"- out_path = M2_DIR / fname-- # Determine what changed for the comment- changes = []- if temp != bench["baseline_temp"]:- changes.append(f"temp {bench['baseline_temp']}→{temp}")- if tools and set(tools) != set(bench.get("baseline_tools", [])):- removed = set(bench.get("baseline_tools", [])) - set(tools)- changes.append(f"removed {removed}")- if is_agent:- changes.append(f"max_turns 10→{DISTILLED_MAX_TURNS} (via OPENJARVIS_CONFIG)")- change_str = "; ".join(changes) if changes else "CONTROL (no change)"-- out_path.write_text(render_config(- comment=f"M2 DISTILLED: {bench_key} × {model['name']} — {change_str}",- meta_name=f"{bench_key}-{sl}-distilled",- description=f"Distilled {bench_key} on {model['name']}",- temperature=temp,- max_tokens=bench["max_tokens"],- judge_model=bench["judge_model"],- judge_engine=bench["judge_engine"],- output_dir=f"results/neurips-2026/distilled/{sl}/{bench_key}/",- model_name=model["name"], model_engine="vllm", num_gpus=model["num_gpus"],- benchmark_name=bench_name, backend=bench["backend"],- agent=bench.get("agent"),- tools=tools,- max_samples=bench.get("max_samples"),- extra_benchmark=bench.get("extra_benchmark_fields"),- ))- count += 1-- return count---if __name__ == "__main__":- print("=== Creating missing baseline configs ===")- n_base = generate_missing_baselines()- print(f"Created {n_base} missing baseline configs\n")-- print("=== Creating distilled M2 configs ===")- n_dist = generate_distilled_configs()- print(f"Created {n_dist} distilled configs in {M2_DIR}/\n")-- # Summary- print("=== Change matrix ===")- print(f"{'Benchmark':20} {'Backend':14} {'Temp Δ':12} {'Tool Δ':20} {'max_turns Δ':12}")- print("-" * 80)- for bk, b in BENCHMARKS.items():- is_agent = b["backend"] == "jarvis-agent"- temp_change = f"{b['baseline_temp']}→{b['distilled_temp']}" if b["baseline_temp"] != b["distilled_temp"] else "—"- tool_change = "—"- if is_agent and "baseline_tools" in b:- removed = REMOVE_TOOLS & set(b.get("baseline_tools", []))- tool_change = f"-{removed}" if removed else "—"- mt_change = f"10→{DISTILLED_MAX_TURNS}" if is_agent else "—"- print(f"{bk:20} {b['backend']:14} {temp_change:12} {str(tool_change):20} {mt_change:12}")diff --git a/scripts/experiments/m2_run_distilled_evals.sh b/scripts/experiments/m2_run_distilled_evals.shdeleted file mode 100644index 9e17e48b..00000000--- a/scripts/experiments/m2_run_distilled_evals.sh+++ /dev/null@@ -1,183 +0,0 @@-#!/usr/bin/env bash-# ──────────────────────────────────────────────────────────────────────────────-# M2: Run distilled eval configs — resumable, agent-benchmarks-first ordering-#-# Assumes vLLM already running: 2B:8000, 9B:8001, 27B-FP8:8002-#-# Usage:-# bash m2_run_distilled_evals.sh # all-# bash m2_run_distilled_evals.sh 9b # 9b only-# bash m2_run_distilled_evals.sh 9b gaia # 9b + gaia only-# ──────────────────────────────────────────────────────────────────────────────--set -uo pipefail--VENV=".venv/bin/python"-M2_CONFIGS="src/openjarvis/evals/configs/distillation/m2"-BASELINE_CONFIGS="src/openjarvis/evals/configs"-M2_HOME="/scratch/user/jonsaadfalcon/openjarvis-m2"-MODEL_FILTER=${1:-all}-BENCH_FILTER=${2:-all}-FORCE=${FORCE:-0} # set FORCE=1 to re-run completed configs--RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'-log() { echo -e "${BLUE}[m2]${NC} $*"; }-ok() { echo -e "${GREEN}[ OK ]${NC} $*"; }-warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }-fail() { echo -e "${RED}[FAIL]${NC} $*"; }-skip() { echo -e "${YELLOW}[SKIP]${NC} $*"; }--declare -A MODEL_PORT=( [2b]=8000 [9b]=8001 [27b]=8002 )--# Agent benchmarks FIRST (where distillation impact is expected)-AGENT_BENCHMARKS="pinchbench gaia liveresearch"-DIRECT_BENCHMARKS="toolcall15 taubench taubench-telecom livecodebench liveresearchbench"-ALL_BENCHMARKS="${AGENT_BENCHMARKS} ${DIRECT_BENCHMARKS}"--check_vllm() {- for size in 2b 9b 27b; do- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue- local port=${MODEL_PORT[$size]}- if ! curl -sf "http://localhost:${port}/v1/models" >/dev/null 2>&1; then- fail "vLLM ${size} not responding on port ${port}"- return 1- fi- ok "vLLM ${size} healthy on port ${port}"- done-}--# Check if a run already completed (summary.json exists AND has a real accuracy).-# A summary with errors=total_samples (like my earlier broken-routing tests)-# is treated as incomplete and re-run.-is_complete() {- local summary_path=$1- [ -f "$summary_path" ] || return 1- python3 -c "-import json, sys-try:- d = json.load(open('$summary_path'))- total = d.get('total_samples', 0)- errors = d.get('errors', 0)- scored = d.get('scored_samples', 0)- # Complete if at least some samples were scored successfully- sys.exit(0 if scored > 0 else 1)-except Exception:- sys.exit(1)-" 2>/dev/null-}--run_eval() {- local config_path=$1 label=$2 size=$3 summary_path=$4 use_distilled=${5:-false}-- if [ "$FORCE" != "1" ] && is_complete "$summary_path"; then- skip "${label} [already complete]"- return 0- fi-- local oj_config="${M2_HOME}/config-baseline-${size}.toml"- [ "$use_distilled" = "true" ] && oj_config="${M2_HOME}/config-${size}.toml"-- log "Running: ${label} [$(basename ${oj_config})]"- env OPENJARVIS_CONFIG="${oj_config}" ${VENV} -m openjarvis.evals run -c "${config_path}" 2>&1- local rc=$?- if [ $rc -eq 0 ] && is_complete "$summary_path"; then- ok "Done: ${label}"- else- warn "Failed: ${label} (rc=$rc)"- fi-}--# Derive the expected summary.json path for a given distilled config-summary_for_distilled() {- local bench=$1 size=$2- # Output dir from the config template: results/neurips-2026/distilled/qwen-{size}/{bench}/- # Summary file pattern: {bench}_Qwen-Qwen3.5-{size}.summary.json- local model_slug- if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8"- else model_slug="Qwen-Qwen3.5-${size}"; fi- # Uppercase B for the model slug- model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g')- # For taubench-telecom, the benchmark name in output is "taubench" not "taubench-telecom"- local bench_fname=$bench- [ "$bench" = "taubench-telecom" ] && bench_fname="taubench"- echo "results/neurips-2026/distilled/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json"-}--summary_for_baseline() {- local bench=$1 size=$2- local model_slug- if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8"- else model_slug="Qwen-Qwen3.5-${size}"; fi- model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g')- local bench_fname=$bench- [ "$bench" = "taubench-telecom" ] && bench_fname="taubench"- echo "results/neurips-2026/baselines/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json"-}--log "M2 Distilled Eval Runner (resumable, agent-first)"-log "Model filter: ${MODEL_FILTER} Benchmark filter: ${BENCH_FILTER} FORCE=${FORCE}"--check_vllm || exit 1--start_time=$(date +%s)--# Phase B1: DISTILLED agent benchmarks (highest-priority: PinchBench, GAIA, DeepResearchBench)-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-log "Phase B1: DISTILLED agent benchmarks (9 runs — the critical data)"-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-for size in 2b 9b 27b; do- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue- for bench in ${AGENT_BENCHMARKS}; do- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue- cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml"- sum=$(summary_for_distilled "$bench" "$size")- [ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true- done-done--# Phase B2: DISTILLED direct benchmarks (controls — should show minimal delta)-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-log "Phase B2: DISTILLED direct benchmarks (15 runs — controls)"-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-for size in 2b 9b 27b; do- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue- for bench in ${DIRECT_BENCHMARKS}; do- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue- cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml"- sum=$(summary_for_distilled "$bench" "$size")- [ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true- done-done--# Phase A: LiveResearchBench baselines (last, since Step 1 baselines exist for other benchmarks)-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-log "Phase A: LiveResearchBench baselines (3 runs — new benchmark only)"-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-for size in 2b 9b 27b; do- [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "liveresearchbench" ] && continue- cfg="${BASELINE_CONFIGS}/liveresearchbench-qwen-${size}.toml"- sum=$(summary_for_baseline "liveresearchbench" "$size")- [ -f "$cfg" ] && run_eval "$cfg" "BASELINE liveresearchbench-qwen-${size}" "${size}" "$sum" false-done--# Phase C: Spot-check 2 baselines against Jon's Step 1 numbers-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-log "Phase C: Spot-check baselines (TC15-9B, GAIA-9B)"-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-if [ "$MODEL_FILTER" = "all" ] || [ "$MODEL_FILTER" = "9b" ]; then- for bench in toolcall15 gaia; do- [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue- cfg="${BASELINE_CONFIGS}/${bench}-qwen-9b.toml"- sum=$(summary_for_baseline "$bench" "9b")- [ -f "$cfg" ] && run_eval "$cfg" "SPOTCHECK ${bench}-qwen-9b" "9b" "$sum" false- done-fi--end_time=$(date +%s)-elapsed=$((end_time - start_time))--log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-ok "M2 complete in ${elapsed}s ($(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"-log "Distilled results: results/neurips-2026/distilled/"-log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"diff --git a/scripts/experiments/m3_hill_climb.py b/scripts/experiments/m3_hill_climb.pydeleted file mode 100644index 3c785182..00000000--- a/scripts/experiments/m3_hill_climb.py+++ /dev/null@@ -1,581 +0,0 @@-#!/usr/bin/env python3-"""M3: Empirical hill-climbing with an LLM proposer.--Replaces M1's open-loop "aggregate consensus across sessions" with a closed-loop:-- For each target (student, benchmark, agent):- for round in 1..N:- edit = teacher.propose_one(history_with_measured_deltas)- score_new = eval_subsample(apply(edit))- if score_new > current_score: accept--Every proposal is empirically verified before the next is proposed, and the-teacher sees measured deltas (not just traces) in its context.--Usage:- python scripts/experiments/m3_hill_climb.py \\- --student 9b --benchmark liveresearch \\- --rounds 4 --k-subsample 8 --k-final 30-"""--from __future__ import annotations--import argparse-import json-import os-import re-import subprocess-import sys-import tempfile-import time-from dataclasses import asdict, dataclass, field-from pathlib import Path-from typing import Any--from openjarvis.core.types import Message, Role-from openjarvis.engine.cloud import CloudEngine---# ═══════════════════════════════════════════════════════════════════════════-# Config & constants-# ═══════════════════════════════════════════════════════════════════════════--STUDENT = {- "2b": {"name": "Qwen/Qwen3.5-2B", "port": 8000, "gpu": 4},- "9b": {"name": "Qwen/Qwen3.5-9B", "port": 8001, "gpu": 5},- "27b": {"name": "Qwen/Qwen3.5-27B-FP8", "port": 8002, "gpu": 6},-}--# Per-benchmark defaults (backend, baseline config)-BENCHMARK = {- "liveresearch": {- "backend": "jarvis-agent",- "agent": "monitor_operative",- "baseline_temp": 0.6,- "baseline_max_tokens": 16384,- "baseline_max_turns": 10,- "baseline_tools": ["web_search", "file_read", "file_write",- "code_interpreter", "think"],- "max_samples_final": 50, # final eval- "judge": "gpt-5-mini-2025-08-07",- },- "gaia": {- "backend": "jarvis-agent",- "agent": "monitor_operative",- "baseline_temp": 0.6,- "baseline_max_tokens": 8192,- "baseline_max_turns": 10,- "baseline_tools": ["think", "calculator", "code_interpreter",- "web_search", "file_read"],- "max_samples_final": 50,- "judge": "gpt-5-mini-2025-08-07",- },- "pinchbench": {- "backend": "jarvis-agent",- "agent": "native_openhands",- "baseline_temp": 0.6,- "baseline_max_tokens": 8192,- "baseline_max_turns": 10,- "baseline_tools": ["think", "file_read", "file_write", "web_search",- "shell_exec", "code_interpreter", "browser_navigate",- "image_generate", "calculator", "http_request",- "pdf_extract"],- "max_samples_final": 23,- "judge": "claude-opus-4-5",- },-}--AVAILABLE_TOOLS = [- "think", "file_read", "file_write", "web_search", "shell_exec",- "code_interpreter", "browser_navigate", "image_generate", "calculator",- "http_request", "pdf_extract", "pdf_reader", "list_directory",-]---# ═══════════════════════════════════════════════════════════════════════════-# Proposer (LLM)-# ═══════════════════════════════════════════════════════════════════════════---PROPOSER_SYSTEM = """\-You are optimizing an OpenJarvis agent configuration for maximum accuracy on \-a benchmark. You propose ONE config edit per round. After each proposal, the \-edit is applied and the benchmark is run on a subsample; you then see the \-measured score delta and decide the next edit.--Your job is to find the config that maximizes measured accuracy.--EDIT GRAMMAR — return JSON with "op" and the parameter fields at top level:--{"op": "<op_name>", <param_fields>, "rationale": "<one sentence>"}--Valid ops and their parameter fields:--1. set_temperature: "value" (float, 0.0..1.0)-2. set_max_turns: "value" (int, 1..100)-3. set_max_tokens: "value" (int, 512..32768)-4. add_tool: "tool_name" (string; must be in AVAILABLE_TOOLS, not already active)-5. remove_tool: "tool_name" (string; must be in current tools)-6. noop: (no params; propose only if you believe no further edit will help)--CONCRETE EXAMPLES:- {"op": "set_temperature", "value": 0.3, "rationale": "Reduce loop risk."}- {"op": "set_max_turns", "value": 20, "rationale": "More turns for research tasks."}- {"op": "add_tool", "tool_name": "pdf_extract", "rationale": "Tasks require PDF reading."}- {"op": "remove_tool", "tool_name": "shell_exec", "rationale": "Tool is broken in this env."}- {"op": "noop", "rationale": "Current config seems optimal."}--EXPLORATION BIAS:- The config space has 5 distinct axes: temperature, max_turns, max_tokens,- tool additions (add_tool), tool removals (remove_tool). Before proposing a- second edit on an axis you've already tried, consider whether an untried- axis might reveal a larger gain. Tool-list edits (add/remove) often matter- more than numeric hyperparameters on benchmarks where the agent uses tools.--Return ONLY the JSON object, no preamble, no code fences."""---def build_user_prompt(- *, benchmark: str, student: str, agent: str,- baseline_score: float, current_score: float, current_config: dict,- edit_history: list[dict], available_tools: list[str],- sample_queries: list[str],-) -> str:- hist_lines = []- for i, e in enumerate(edit_history, 1):- delta = e["score_after"] - e["score_before"]- status = "ACCEPTED" if e["accepted"] else "REJECTED"- hist_lines.append(- f" Round {i}: {json.dumps(e['edit'])} "- f"→ score {e['score_before']:.1f}% → {e['score_after']:.1f}% "- f"(Δ {delta:+.1f}, {status})"- )- hist = "\n".join(hist_lines) if hist_lines else " (no edits yet — baseline is the starting point)"-- samples = "\n".join(f" - {q[:150]}..." for q in sample_queries[:3])-- unused_tools = [t for t in available_tools if t not in current_config["tools"]]-- return f"""\-TARGET:- student: {student} (vLLM-served Qwen3.5)- benchmark: {benchmark}- agent: {agent} (uses OpenAI-format structured tool calls)--CURRENT CONFIG:- temperature = {current_config['temperature']}- max_turns = {current_config['max_turns']}- max_tokens = {current_config['max_tokens']}- tools = {current_config['tools']}--TOOLS NOT CURRENTLY ACTIVE (available to add):- {unused_tools}--BASELINE (unedited) SCORE: {baseline_score:.1f}%-CURRENT BEST SCORE: {current_score:.1f}%--EDIT HISTORY (with measured deltas):-{hist}--SAMPLE TASKS FROM THIS BENCHMARK:-{samples}--Propose ONE edit that you predict will improve the measured accuracy. Consider \-the edit history — do not repeat proposals that were rejected. If you believe \-further edits will not help, propose noop.--Return JSON only."""---def call_proposer(- engine: CloudEngine, system: str, user: str, model: str = "claude-sonnet-4-6"-) -> dict:- resp = engine.generate(- messages=[- Message(role=Role.SYSTEM, content=system),- Message(role=Role.USER, content=user),- ],- model=model, max_tokens=600, temperature=0.3,- )- content = (resp.get("content") or "").strip()- # Be forgiving about code fences- if content.startswith("```"):- content = re.sub(r"^```(?:json)?\s*", "", content)- content = re.sub(r"\s*```\s*$", "", content)- # Find the JSON object- m = re.search(r"\{[\s\S]*\}", content)- if not m:- raise ValueError(f"No JSON object found in proposer output: {content[:200]}")- return json.loads(m.group(0))---# ═══════════════════════════════════════════════════════════════════════════-# Edit application & evaluation-# ═══════════════════════════════════════════════════════════════════════════---@dataclass-class Config:- temperature: float- max_turns: int- max_tokens: int- tools: list[str]---def apply_edit(cfg: Config, edit: dict) -> Config:- """Apply an edit. Tolerant of both flat and nested (params) forms."""- op = edit["op"]- # Merge top-level edit fields with params for flat-or-nested tolerance- p = {**edit.get("params", {}), **{k: v for k, v in edit.items()- if k not in ("op", "params", "rationale")}}- new = Config(- temperature=cfg.temperature, max_turns=cfg.max_turns,- max_tokens=cfg.max_tokens, tools=list(cfg.tools),- )- if op == "noop":- return new- if op == "set_temperature":- new.temperature = float(p["value"])- elif op == "set_max_turns":- new.max_turns = int(p["value"])- elif op == "set_max_tokens":- new.max_tokens = int(p["value"])- elif op == "add_tool":- tool = p["tool_name"]- if tool not in new.tools:- new.tools.append(tool)- elif op == "remove_tool":- tool = p["tool_name"]- new.tools = [t for t in new.tools if t != tool]- else:- raise ValueError(f"unknown edit op: {op}")- return new---def write_eval_toml(- *, bench: str, bench_spec: dict, student: dict, cfg: Config,- k_samples: int, output_dir: Path,-) -> Path:- agent_line = f'agent = "{bench_spec["agent"]}"' if bench_spec.get("agent") else ""- tools_str = "[" + ", ".join(f'"{t}"' for t in cfg.tools) + "]"- toml = f"""\-[meta]-name = "m3-{bench}-{student['name'].replace('/', '-')}"-description = "M3 hill-climb round"--[defaults]-temperature = {cfg.temperature}-max_tokens = {cfg.max_tokens}--[judge]-model = "{bench_spec['judge']}"-temperature = 0.0-max_tokens = 4096-engine = "cloud"--[run]-max_workers = 1-output_dir = "{output_dir}"-seed = 42--[[models]]-name = "{student['name']}"-engine = "vllm"-num_gpus = 1--[[benchmarks]]-name = "{bench}"-backend = "{bench_spec['backend']}"-{agent_line}-max_samples = {k_samples}-tools = {tools_str}-"""- path = output_dir / "eval.toml"- output_dir.mkdir(parents=True, exist_ok=True)- path.write_text(toml)- return path---def write_openjarvis_config(home: Path, port: int, max_turns: int) -> Path:- p = home / "global-config.toml"- p.parent.mkdir(parents=True, exist_ok=True)- p.write_text(f"""\-[agent]-max_turns = {max_turns}--[engine]-default = "vllm"--[engine.vllm]-host = "http://localhost:{port}"-""")- return p---def run_eval(eval_toml: Path, oj_config: Path) -> tuple[float, int, int]:- """Run one eval. Returns (accuracy_pct, scored, total)."""- env = {**os.environ, "OPENJARVIS_CONFIG": str(oj_config)}- result = subprocess.run(- [".venv/bin/python", "-m", "openjarvis.evals", "run", "-c", str(eval_toml)],- capture_output=True, text=True, env=env, timeout=7200,- )- # Find the summary.json- out_dir = eval_toml.parent- sums = list(out_dir.glob("**/*.summary.json"))- if not sums:- print(f"[m3] WARNING: no summary.json in {out_dir}")- print(f"[m3] stderr tail: {result.stderr[-500:]}")- return 0.0, 0, 0- d = json.loads(sums[0].read_text())- acc = d.get("accuracy", 0.0)- acc_pct = acc * 100 if acc <= 1.0 else acc- return acc_pct, d.get("scored_samples", 0), d.get("total_samples", 0)---# ═══════════════════════════════════════════════════════════════════════════-# Benchmark sample loader (for proposer context)-# ═══════════════════════════════════════════════════════════════════════════---def load_sample_queries(bench: str, n: int = 3) -> list[str]:- try:- if bench == "liveresearch":- from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset- ds = LiveResearchBenchDataset()- ds.load(max_samples=n)- elif bench == "pinchbench":- from openjarvis.evals.datasets.pinchbench import PinchBenchDataset- ds = PinchBenchDataset()- elif bench == "gaia":- from openjarvis.evals.datasets.gaia import GAIADataset- ds = GAIADataset()- if hasattr(ds, "load"):- ds.load(max_samples=n)- else:- return []- return [r.problem for r in list(ds.iter_records())[:n]]- except Exception as e:- print(f"[m3] WARNING: could not load samples for {bench}: {e}")- return []---# ═══════════════════════════════════════════════════════════════════════════-# Main hill-climb loop-# ═══════════════════════════════════════════════════════════════════════════---def hill_climb(args) -> dict:- bench_spec = BENCHMARK[args.benchmark]- student = STUDENT[args.student]-- # State dir (resumable)- base_dir = Path(args.out_dir) / f"{args.student}-{args.benchmark}"- base_dir.mkdir(parents=True, exist_ok=True)- state_path = base_dir / "state.json"-- # Load or initialize state- if state_path.exists() and not args.fresh:- state = json.loads(state_path.read_text())- print(f"[m3] Resumed state from round {len(state['history'])}/{args.rounds}")- else:- baseline_cfg_dict = {- "temperature": bench_spec["baseline_temp"],- "max_turns": bench_spec["baseline_max_turns"],- "max_tokens": bench_spec["baseline_max_tokens"],- "tools": list(bench_spec["baseline_tools"]),- }- # ALWAYS measure the baseline today first (unless user provides --trust-baseline).- # This prevents anchoring to an unreproducible Step 1 number.- # We measure at k=k_final so the final delta is like-for-like.- if args.trust_baseline and args.baseline_score is not None:- measured_baseline = args.baseline_score- measured_baseline_k = None- print(f"[m3] Trusting provided baseline score: {measured_baseline:.1f}% "- f"(--trust-baseline set)")- else:- print(f"[m3] Measuring today's baseline with k={args.k_final} "- f"(matches k_final for clean like-for-like delta)...")- bl_dir = base_dir / "baseline_measure"- bl_toml = write_eval_toml(- bench=args.benchmark, bench_spec=bench_spec, student=student,- cfg=Config(**baseline_cfg_dict),- k_samples=args.k_final, output_dir=bl_dir,- )- bl_oj = write_openjarvis_config(bl_dir, student["port"], baseline_cfg_dict["max_turns"])- t0 = time.monotonic()- measured_baseline, bl_scored, bl_total = run_eval(bl_toml, bl_oj)- measured_baseline_k = bl_scored- print(f"[m3] Measured baseline: {measured_baseline:.1f}% "- f"({bl_scored}/{bl_total}) in {(time.monotonic() - t0)/60:.1f} min")- if args.baseline_score is not None:- print(f"[m3] (reference: --baseline-score was {args.baseline_score:.1f}%, "- f"drift = {measured_baseline - args.baseline_score:+.1f})")-- state = {- "args": vars(args),- "benchmark": args.benchmark,- "student": student["name"],- "agent": bench_spec["agent"],- "baseline_config": baseline_cfg_dict,- "baseline_score": measured_baseline,- "baseline_score_reference": args.baseline_score,- "current_config": dict(baseline_cfg_dict),- "current_score": measured_baseline,- "history": [],- }-- # Save state helper- def save():- state_path.write_text(json.dumps(state, indent=2, default=str))-- save()-- # Sample queries for proposer context- sample_queries = load_sample_queries(args.benchmark)-- engine = CloudEngine()-- # Hill-climb rounds- for round_num in range(len(state["history"]) + 1, args.rounds + 1):- print(f"\n{'═' * 70}")- print(f"[m3] Round {round_num}/{args.rounds}")- print(f"[m3] current_score = {state['current_score']:.1f}%")- print(f"[m3] current_config = {state['current_config']}")-- # Propose- user_prompt = build_user_prompt(- benchmark=args.benchmark, student=student["name"],- agent=bench_spec["agent"],- baseline_score=state["baseline_score"],- current_score=state["current_score"],- current_config=state["current_config"],- edit_history=state["history"],- available_tools=AVAILABLE_TOOLS,- sample_queries=sample_queries,- )- try:- edit = call_proposer(engine, PROPOSER_SYSTEM, user_prompt,- model=args.proposer_model)- except Exception as e:- print(f"[m3] Proposer failed: {e}. Ending hill-climb.")- break-- print(f"[m3] Proposed: {json.dumps(edit)}")-- if edit.get("op") == "noop":- print(f"[m3] Teacher proposed noop; stopping.")- break-- # Apply and evaluate subsample- try:- candidate = apply_edit(Config(**state["current_config"]), edit)- except Exception as e:- print(f"[m3] apply_edit failed: {e}. Recording as malformed edit.")- # Record as a rejected malformed edit so teacher won't repeat- state["history"].append({- "round": round_num, "edit": edit,- "config_after": None,- "score_before": state["current_score"],- "score_after": state["current_score"], # no change- "scored": 0, "total": 0, "elapsed_seconds": 0,- "accepted": False,- "error": f"malformed_edit: {e}",- })- save()- continue-- round_dir = base_dir / f"round_{round_num}"- eval_toml = write_eval_toml(- bench=args.benchmark, bench_spec=bench_spec, student=student,- cfg=candidate, k_samples=args.k_subsample,- output_dir=round_dir,- )- oj_cfg = write_openjarvis_config(round_dir, student["port"], candidate.max_turns)-- print(f"[m3] Running k={args.k_subsample} subsample eval...")- t0 = time.monotonic()- acc, scored, total = run_eval(eval_toml, oj_cfg)- elapsed = time.monotonic() - t0- print(f"[m3] Subsample score: {acc:.1f}% ({scored}/{total}) in {elapsed/60:.1f} min")-- score_before = state["current_score"]- delta = acc - score_before- accepted = delta > args.accept_threshold-- # Record history- state["history"].append({- "round": round_num, "edit": edit,- "config_after": asdict(candidate),- "score_before": score_before, "score_after": acc,- "scored": scored, "total": total, "elapsed_seconds": elapsed,- "accepted": accepted,- })-- if accepted:- state["current_config"] = asdict(candidate)- state["current_score"] = acc- print(f"[m3] ACCEPTED (Δ={delta:+.1f})")- else:- print(f"[m3] REJECTED (Δ={delta:+.1f} ≤ {args.accept_threshold})")-- save()-- # Final eval with current config- print(f"\n{'═' * 70}")- print(f"[m3] Final eval with best config: {state['current_config']}")- final_dir = base_dir / "final"- final_cfg = Config(**state["current_config"])- eval_toml = write_eval_toml(- bench=args.benchmark, bench_spec=bench_spec, student=student,- cfg=final_cfg, k_samples=args.k_final,- output_dir=final_dir,- )- oj_cfg = write_openjarvis_config(final_dir, student["port"], final_cfg.max_turns)-- t0 = time.monotonic()- final_acc, final_scored, final_total = run_eval(eval_toml, oj_cfg)- elapsed = time.monotonic() - t0-- state["final_score"] = final_acc- state["final_scored"] = final_scored- state["final_total"] = final_total- state["final_elapsed_seconds"] = elapsed- save()-- print(f"\n{'═' * 70}")- print(f"[m3] DONE")- print(f"[m3] baseline = {state['baseline_score']:.1f}%")- print(f"[m3] final = {final_acc:.1f}% ({final_scored}/{final_total}) in {elapsed/60:.1f} min")- print(f"[m3] Δ vs baseline = {final_acc - state['baseline_score']:+.1f}")- print(f"[m3] state: {state_path}")-- return state---def main() -> int:- ap = argparse.ArgumentParser()- ap.add_argument("--student", required=True, choices=list(STUDENT))- ap.add_argument("--benchmark", required=True, choices=list(BENCHMARK))- ap.add_argument("--rounds", type=int, default=4)- ap.add_argument("--k-subsample", type=int, default=8)- ap.add_argument("--k-final", type=int, default=30)- ap.add_argument("--baseline-score", type=float, default=None,- help="Optional reference baseline (shown alongside measured). "- "Hill-climb always measures today's baseline unless --trust-baseline.")- ap.add_argument("--trust-baseline", action="store_true",- help="Skip baseline re-measurement, trust --baseline-score.")- ap.add_argument("--accept-threshold", type=float, default=0.0,- help="Accept edit if score Δ > this (default: 0)")- ap.add_argument("--proposer-model", default="claude-sonnet-4-6")- ap.add_argument("--out-dir", default="results/neurips-2026/distillation-m3")- ap.add_argument("--fresh", action="store_true",- help="Overwrite existing state and start fresh")- args = ap.parse_args()-- result = hill_climb(args)- return 0---if __name__ == "__main__":- sys.exit(main())diff --git a/scripts/experiments/run_distillation_experiments.sh b/scripts/experiments/run_distillation_experiments.shdeleted file mode 100755index 961db01c..00000000--- a/scripts/experiments/run_distillation_experiments.sh+++ /dev/null@@ -1,307 +0,0 @@-#!/usr/bin/env bash-# ──────────────────────────────────────────────────────────────────────────────-# Run distillation ablation experiments-#-# Prerequisites:-# - Ollama running with qwen3.5:{2b,9b,27b}-# - ANTHROPIC_API_KEY set (for Opus teacher)-# - OPENAI_API_KEY set (for GPT-5.4 teacher)-# - GOOGLE_API_KEY set (for Gemini teacher)-# - For Qwen-397B teacher: vLLM serving on port 8010 with 8×H100-# - Traces seeded with feedback (run A1 blocker first)-# - jarvis learning init already run-#-# Usage:-# bash scripts/experiments/run_distillation_experiments.sh # Run all-# bash scripts/experiments/run_distillation_experiments.sh exp1a # Run Phase 1a only-# bash scripts/experiments/run_distillation_experiments.sh exp1a opus # Single config-# ──────────────────────────────────────────────────────────────────────────────--set -euo pipefail--CONFIGS_DIR="src/openjarvis/evals/configs/distillation"-RESULTS_DIR="results/neurips-2026/agent-optimization/distillation"-EXPERIMENT=${1:-all}-FILTER=${2:-}--# ── Colors ───────────────────────────────────────────────────────────────────-RED='\033[0;31m'-GREEN='\033[0;32m'-YELLOW='\033[1;33m'-BLUE='\033[0;34m'-NC='\033[0m'--log() { echo -e "${BLUE}[distill]${NC} $*"; }-ok() { echo -e "${GREEN}[ OK ]${NC} $*"; }-warn() { echo -e "${YELLOW}[ WARN ]${NC} $*"; }-fail() { echo -e "${RED}[ FAIL ]${NC} $*"; }--# ── Preflight checks ────────────────────────────────────────────────────────-check_prereqs() {- log "Preflight checks..."-- # Check API keys- if [ -z "${ANTHROPIC_API_KEY:-}" ]; then- warn "ANTHROPIC_API_KEY not set — Opus teacher experiments will fail"- fi- if [ -z "${OPENAI_API_KEY:-}" ]; then- warn "OPENAI_API_KEY not set — GPT-5.4 teacher experiments will fail"- fi- if [ -z "${GOOGLE_API_KEY:-}" ]; then- warn "GOOGLE_API_KEY not set — Gemini teacher experiments will fail"- fi-- # Check Ollama- if ! ollama list &>/dev/null; then- fail "Ollama not running. Start it first."- exit 1- fi-- # Check student models- for model in qwen3.5:2b qwen3.5:9b qwen3.5:27b; do- if ! ollama list 2>/dev/null | grep -q "$model"; then- warn "Model $model not found in Ollama. Pull with: ollama pull $model"- fi- done-- # Check distillation init- if [ ! -d "$HOME/.openjarvis/learning" ]; then- log "Running jarvis learning init..."- uv run jarvis learning init- fi-- ok "Preflight complete"-}--# ── Run a single distillation session ────────────────────────────────────────-run_session() {- local config_file=$1- local experiment_name- experiment_name=$(basename "$(dirname "$config_file")")- local config_name- config_name=$(basename "${config_file%.toml}")- local output_dir="${RESULTS_DIR}/${experiment_name}/${config_name}"-- # Skip if already completed- if [ -f "${output_dir}/session/session.json" ]; then- ok "SKIP ${experiment_name}/${config_name} (already done)"- return 0- fi-- log "──────────────────────────────────────────────────────"- log "Experiment: ${experiment_name}/${config_name}"- log "Config: ${config_file}"- log "Output: ${output_dir}"- log "──────────────────────────────────────────────────────"-- mkdir -p "${output_dir}"-- # Extract metadata from config- local teacher_model- teacher_model=$(grep 'teacher_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')- local student_model- student_model=$(grep 'default_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')- local benchmark- benchmark=$(grep '^benchmark ' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')- local data_config- data_config=$(grep 'data_config' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')- local iterative- iterative=$(grep 'iterative_sessions' "$config_file" | head -1 | sed 's/.*= *//')-- log "Teacher: ${teacher_model}"- log "Student: ${student_model}"- log "Data: ${data_config:-C2}"- log "Iter: ${iterative:-1}"-- # ── Step 1: Seed traces based on data config ─────────────────────────- # (In a full implementation, this would filter/prepare the TraceStore- # based on C1/C2/C3. For now we use whatever traces exist.)-- # ── Step 2: Run distillation session ─────────────────────────────────- local n_sessions=${iterative:-1}- local session_num=1- local prev_session_id=""-- while [ "$session_num" -le "$n_sessions" ]; do- log "Session ${session_num}/${n_sessions}..."-- local session_output="${output_dir}/session_${session_num}"- mkdir -p "${session_output}"-- # Run the distillation session via Python- # (jarvis learning run doesn't support all config params yet,- # so we call the orchestrator directly)- uv run python << PYEOF > "${session_output}/run.log" 2>&1 || true-import json, os, shutil, sys-from pathlib import Path--from openjarvis.engine.cloud import CloudEngine-from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend-from openjarvis.traces.store import TraceStore-from openjarvis.learning.distillation.checkpoint.store import CheckpointStore-from openjarvis.learning.distillation.models import AutonomyMode-from openjarvis.learning.distillation.orchestrator import DistillationOrchestrator-from openjarvis.learning.distillation.storage.session_store import SessionStore-from openjarvis.learning.distillation.student_runner import (- VLLMStudentRunner,- build_benchmark_samples_from_traces,-)-from openjarvis.learning.distillation.triggers import OnDemandTrigger-from openjarvis.learning.optimize.feedback.judge import TraceJudge--home = Path(os.environ.get("OPENJARVIS_HOME", str(Path.home() / ".openjarvis")))--# Read config params-teacher_model = "${teacher_model}"-student_model = "${student_model}"-autonomy = "auto"-max_cost = float("$(grep 'max_cost_per_session_usd' "$config_file" | head -1 | sed 's/.*= *//')")-max_tools = int("$(grep 'max_tool_calls_per_diagnosis' "$config_file" | head -1 | sed 's/.*= *//')")--# Real student runner via vLLM-vllm_host = os.environ.get("VLLM_HOST", "http://localhost:8001")-student_runner = VLLMStudentRunner(- host=vllm_host,- model=student_model,-)--# Real judge via cloud LLM-cloud_engine = CloudEngine()-judge_backend = JarvisDirectBackend(engine_key="cloud")-judge = TraceJudge(backend=judge_backend, model="gpt-5-mini-2025-08-07")--# Build benchmark samples from existing traces-trace_store = TraceStore(home / "traces.db")-benchmark_samples = build_benchmark_samples_from_traces(trace_store, limit=50)--orch = DistillationOrchestrator(- teacher_engine=cloud_engine,- teacher_model=teacher_model,- trace_store=trace_store,- benchmark_samples=benchmark_samples,- student_runner=student_runner,- judge=judge,- session_store=SessionStore(home / "learning" / "learning.db"),- checkpoint_store=CheckpointStore(home),- openjarvis_home=home,- autonomy_mode=AutonomyMode.AUTO,- scorer=None,- min_traces=10,- max_cost_usd=max_cost,- max_tool_calls=max_tools,-)-session = orch.run(OnDemandTrigger())--# Save results-result = {- "session_id": session.id,- "status": session.status.value,- "cost_usd": session.teacher_cost_usd,- "edits_total": len(session.edit_outcomes),- "edits_applied": len([o for o in session.edit_outcomes if o.status == "applied"]),- "edits_rejected": len([o for o in session.edit_outcomes if o.status == "rejected_by_gate"]),- "error": session.error,-}-Path("${session_output}/result.json").write_text(json.dumps(result, indent=2))--# Copy session artifacts-sd = home / "learning" / "sessions" / session.id-if sd.exists():- shutil.copytree(sd, Path("${session_output}/artifacts"), dirs_exist_ok=True)--print(json.dumps(result, indent=2))-PYEOF-- # Check result- if [ -f "${session_output}/result.json" ]; then- local status- status=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['status'])")- local cost- cost=$(python3 -c "import json; print(f\"\${json.load(open('${session_output}/result.json'))['cost_usd']:.4f}\")")- local applied- applied=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['edits_applied'])")-- if [ "$status" = "completed" ]; then- ok "Session ${session_num}: status=${status}, cost=\$${cost}, applied=${applied}"- else- warn "Session ${session_num}: status=${status}, cost=\$${cost}"- fi- else- fail "Session ${session_num}: no result.json (check ${session_output}/run.log)"- fi-- session_num=$((session_num + 1))- done-- ok "Done: ${experiment_name}/${config_name}"-}--# ── Run experiment group ─────────────────────────────────────────────────────-run_experiment() {- local exp_dir=$1- local exp_name- exp_name=$(basename "$exp_dir")-- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"- log "EXPERIMENT GROUP: ${exp_name}"- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-- local count=0- local total- total=$(ls "${exp_dir}"/*.toml 2>/dev/null | wc -l)-- for config in "${exp_dir}"/*.toml; do- [ -f "$config" ] || continue-- # Apply filter if specified- if [ -n "${FILTER}" ] && ! echo "$config" | grep -q "${FILTER}"; then- continue- fi-- count=$((count + 1))- log "[${count}/${total}] $(basename "$config")"- run_session "$config"- done-- ok "Experiment group ${exp_name}: ${count} configs processed"-}--# ── Main ─────────────────────────────────────────────────────────────────────-main() {- check_prereqs-- log "Starting distillation experiments"- log "Experiment filter: ${EXPERIMENT}"- log "Config filter: ${FILTER:-none}"-- local start_time- start_time=$(date +%s)-- if [ "$EXPERIMENT" = "all" ]; then- # Run in priority order- for exp in exp1a-teacher exp1b-budget exp1c-student \- exp2a-gate exp2b-autonomy \- exp3a-iterative exp3b-transfer; do- if [ -d "${CONFIGS_DIR}/${exp}" ]; then- run_experiment "${CONFIGS_DIR}/${exp}"- fi- done- elif [ -d "${CONFIGS_DIR}/${EXPERIMENT}" ]; then- run_experiment "${CONFIGS_DIR}/${EXPERIMENT}"- else- fail "Unknown experiment: ${EXPERIMENT}"- echo "Available: exp1a-teacher exp1b-budget exp1c-student exp2a-gate exp2b-autonomy exp3a-iterative exp3b-transfer"- exit 1- fi-- local end_time- end_time=$(date +%s)- local elapsed=$((end_time - start_time))-- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"- ok "All experiments complete in ${elapsed}s"- log "Results in: ${RESULTS_DIR}/"- log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"-}--main "$@"diff --git a/scripts/experiments/run_track_b_gepa_dspy.sh b/scripts/experiments/run_track_b_gepa_dspy.shdeleted file mode 100644index a628be4a..00000000--- a/scripts/experiments/run_track_b_gepa_dspy.sh+++ /dev/null@@ -1,375 +0,0 @@-#!/usr/bin/env bash-# =============================================================================-# Track B: GEPA/DSPy Agent Optimization-# NeurIPS 2026 — Agent Optimization Experiments-#-# Runs GEPA and DSPy BootstrapFewShot across:-# Models: qwen-9b, qwen-27b, qwen-35b-# Benchmarks: toolcall15, pinchbench, taubench-#-# Usage:-# bash scripts/experiments/run_track_b_gepa_dspy.sh-# bash scripts/experiments/run_track_b_gepa_dspy.sh --model qwen-9b --benchmark pinchbench-# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer gepa-# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer dspy-#-# Expected runtime: ~2-4 hours per GEPA run, ~1-2 hours per DSPy run-# Total wall-clock: ~12-18 hours (parallelized across GPUs)-# Estimated API cost: ~$90 GEPA + ~$90 DSPy = ~$180 total-#-# =============================================================================-# vLLM Serving Commands (run these BEFORE this script on the GPU node)-# =============================================================================-#-# GPU 0 — Qwen-9B (1x GPU):-# CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen2.5-7B-Instruct \-# --model Qwen/Qwen2.5-7B-Instruct \-# --served-model-name qwen-9b \-# --port 8001 --host 0.0.0.0 \-# --max-model-len 32768 --gpu-memory-utilization 0.9 &-#-# GPU 1 — Qwen-27B (1-2x GPU):-# CUDA_VISIBLE_DEVICES=1,2 vllm serve Qwen/Qwen2.5-32B-Instruct \-# --model Qwen/Qwen2.5-32B-Instruct \-# --served-model-name qwen-27b \-# --port 8002 --host 0.0.0.0 \-# --tensor-parallel-size 2 \-# --max-model-len 32768 --gpu-memory-utilization 0.9 &-#-# GPU 3 — Qwen-35B (1-2x GPU):-# CUDA_VISIBLE_DEVICES=3,4 vllm serve Qwen/Qwen2.5-72B-Instruct \-# --model Qwen/Qwen2.5-72B-Instruct \-# --served-model-name qwen-35b \-# --port 8003 --host 0.0.0.0 \-# --tensor-parallel-size 2 \-# --max-model-len 32768 --gpu-memory-utilization 0.9 &-#-# Wait for all servers to be healthy:-# sleep 60 && curl -s http://localhost:8001/health && \-# curl -s http://localhost:8002/health && \-# curl -s http://localhost:8003/health-#-# =============================================================================--set -euo pipefail--# ----------------------------------------------------------------------------# Defaults-# ----------------------------------------------------------------------------REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/agent-optimization"-LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs"-TIMESTAMP="$(date +%Y%m%d_%H%M%S)"--ALL_MODELS=(qwen-9b qwen-27b qwen-35b)-ALL_BENCHMARKS=(toolcall15 pinchbench taubench)-ALL_OPTIMIZERS=(gepa dspy)--# Override defaults with CLI flags-FILTER_MODEL=""-FILTER_BENCHMARK=""-FILTER_OPTIMIZER=""--# GEPA settings-GEPA_TRIALS=20-GEPA_MAX_SAMPLES=50-GEPA_OPTIMIZER_MODEL="claude-sonnet-4-6"--# DSPy settings-DSPY_OPTIMIZER="BootstrapFewShotWithRandomSearch"-DSPY_TEACHER_LM="claude-sonnet-4-6"--# ----------------------------------------------------------------------------# Parse CLI flags-# ----------------------------------------------------------------------------while [[ $# -gt 0 ]]; do- case "$1" in- --model)- FILTER_MODEL="$2"; shift 2 ;;- --benchmark)- FILTER_BENCHMARK="$2"; shift 2 ;;- --optimizer)- FILTER_OPTIMIZER="$2"; shift 2 ;;- --gepa-trials)- GEPA_TRIALS="$2"; shift 2 ;;- --gepa-max-samples)- GEPA_MAX_SAMPLES="$2"; shift 2 ;;- --dspy-optimizer)- DSPY_OPTIMIZER="$2"; shift 2 ;;- -h|--help)- sed -n '2,30p' "$0" | grep '^#' | sed 's/^# \?//'- exit 0 ;;- *)- echo "Unknown flag: $1"; exit 1 ;;- esac-done--# Apply filters-if [[ -n "$FILTER_MODEL" ]]; then- ALL_MODELS=("$FILTER_MODEL")-fi-if [[ -n "$FILTER_BENCHMARK" ]]; then- ALL_BENCHMARKS=("$FILTER_BENCHMARK")-fi-if [[ -n "$FILTER_OPTIMIZER" ]]; then- ALL_OPTIMIZERS=("$FILTER_OPTIMIZER")-fi--# ----------------------------------------------------------------------------# Logging helpers-# ----------------------------------------------------------------------------mkdir -p "$LOG_DIR"-LOG_FILE="${LOG_DIR}/track_b_${TIMESTAMP}.log"--log() {- local level="$1"; shift- local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"- echo "$msg"- echo "$msg" >> "$LOG_FILE"-}--log_info() { log "INFO " "$@"; }-log_ok() { log "OK " "$@"; }-log_warn() { log "WARN " "$@"; }-log_error() { log "ERROR" "$@"; }--# ----------------------------------------------------------------------------# Environment setup-# ----------------------------------------------------------------------------setup_env() {- log_info "=== Track B: GEPA/DSPy Optimization ==="- log_info "Repo: $REPO_ROOT"- log_info "Log: $LOG_FILE"- log_info "Models: ${ALL_MODELS[*]}"- log_info "Benchmarks: ${ALL_BENCHMARKS[*]}"- log_info "Optimizers: ${ALL_OPTIMIZERS[*]}"- echo ""-- # Check we are in the repo root- if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then- log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)"- exit 1- fi-- # Install/sync dependencies- log_info "Running uv sync..."- cd "$REPO_ROOT"- uv sync --extra dev 2>&1 | tail -5- log_ok "uv sync complete"-- # Check required API keys- if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then- log_error "ANTHROPIC_API_KEY is not set. Required for the optimizer teacher model."- log_error " export ANTHROPIC_API_KEY=sk-ant-..."- exit 1- fi- log_ok "ANTHROPIC_API_KEY is set"-- # Optional: OpenAI key (used if teacher_lm is an OpenAI model)- if [[ -z "${OPENAI_API_KEY:-}" ]]; then- log_warn "OPENAI_API_KEY not set (only needed if using OpenAI teacher models)"- fi-- echo ""- log_info "Model-to-port mapping (vLLM must be pre-started on these ports):"- log_info " qwen-9b -> http://localhost:8001"- log_info " qwen-27b -> http://localhost:8002"- log_info " qwen-35b -> http://localhost:8003"- echo ""-}--# ----------------------------------------------------------------------------# Model port lookup-# ----------------------------------------------------------------------------model_port() {- case "$1" in- qwen-9b) echo 8001 ;;- qwen-27b) echo 8002 ;;- qwen-35b) echo 8003 ;;- *)- log_error "Unknown model: $1"- exit 1 ;;- esac-}--# ----------------------------------------------------------------------------# Health check: verify the vLLM server for a model is reachable-# ----------------------------------------------------------------------------check_server_health() {- local model="$1"- local port- port="$(model_port "$model")"- local url="http://localhost:${port}/health"- if curl -sf "$url" > /dev/null 2>&1; then- log_ok "vLLM server for $model is healthy at port $port"- return 0- else- log_error "vLLM server for $model NOT reachable at $url"- log_error "Start it with the vLLM commands in the script header."- return 1- fi-}--# ----------------------------------------------------------------------------# GEPA optimization for one (model, benchmark) pair-# ----------------------------------------------------------------------------run_gepa() {- local model="$1"- local bench="$2"- local port- port="$(model_port "$model")"- local out_dir="${RESULTS_BASE}/gepa/${model}/${bench}"-- log_info "--- GEPA: $model × $bench ---"- log_info " Output dir: $out_dir"- log_info " Trials: $GEPA_TRIALS Max-samples: $GEPA_MAX_SAMPLES"- mkdir -p "$out_dir"-- # Record start time- local t0- t0="$(date +%s)"-- OPENAI_API_BASE="http://localhost:${port}/v1" \- uv run jarvis optimize run \- --benchmark "$bench" \- --model "$model" \- --optimizer-model "$GEPA_OPTIMIZER_MODEL" \- --trials "$GEPA_TRIALS" \- --max-samples "$GEPA_MAX_SAMPLES" \- --output-dir "$out_dir" \- 2>&1 | tee -a "$LOG_FILE"-- local exit_code=${PIPESTATUS[0]}- local t1- t1="$(date +%s)"- local elapsed=$(( t1 - t0 ))-- if [[ $exit_code -eq 0 ]]; then- log_ok "GEPA $model/$bench done in ${elapsed}s"- else- log_error "GEPA $model/$bench FAILED (exit $exit_code) after ${elapsed}s"- return $exit_code- fi-}--# ----------------------------------------------------------------------------# DSPy optimization for one (model, benchmark) pair-# ----------------------------------------------------------------------------run_dspy() {- local model="$1"- local bench="$2"- local port- port="$(model_port "$model")"- local out_dir="${RESULTS_BASE}/dspy/${model}/${bench}"-- log_info "--- DSPy: $model × $bench ---"- log_info " Output dir: $out_dir"- log_info " Teleprompter: $DSPY_OPTIMIZER Teacher: $DSPY_TEACHER_LM"- mkdir -p "$out_dir"-- local t0- t0="$(date +%s)"-- OPENAI_API_BASE="http://localhost:${port}/v1" \- uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"-import sys-from openjarvis.learning.agents.dspy_optimizer import DSPyAgentOptimizer-from openjarvis.core.config import DSPyOptimizerConfig-from openjarvis.traces.store import TraceStore--store = TraceStore()-config = DSPyOptimizerConfig(- optimizer="${DSPY_OPTIMIZER}",- teacher_lm="${DSPY_TEACHER_LM}",- config_dir="${out_dir}",- benchmark="${bench}",- agent_filter="${model}",-)-result = DSPyAgentOptimizer(config).optimize(store)-print(f"DSPy result for ${model}/${bench}: {result}")-if result.get("status") not in ("ok", "success", "done"):- sys.exit(1)-PYEOF-- local exit_code=${PIPESTATUS[0]}- local t1- t1="$(date +%s)"- local elapsed=$(( t1 - t0 ))-- if [[ $exit_code -eq 0 ]]; then- log_ok "DSPy $model/$bench done in ${elapsed}s"- else- log_error "DSPy $model/$bench FAILED (exit $exit_code) after ${elapsed}s"- return $exit_code- fi-}--# ----------------------------------------------------------------------------# Summary: print result file locations-# ----------------------------------------------------------------------------print_summary() {- echo ""- log_info "=== Track B Complete ==="- log_info "Results written to:"- for opt in "${ALL_OPTIMIZERS[@]}"; do- for model in "${ALL_MODELS[@]}"; do- for bench in "${ALL_BENCHMARKS[@]}"; do- local out_dir="${RESULTS_BASE}/${opt}/${model}/${bench}"- if [[ -d "$out_dir" ]]; then- log_ok " $opt/$model/$bench -> $out_dir"- else- log_warn " $opt/$model/$bench -> MISSING ($out_dir)"- fi- done- done- done- log_info "Full log: $LOG_FILE"-}--# ----------------------------------------------------------------------------# Main-# ----------------------------------------------------------------------------main() {- setup_env-- local failed=0-- # Pre-flight: check vLLM servers for all target models- log_info "Checking vLLM server health..."- for model in "${ALL_MODELS[@]}"; do- check_server_health "$model" || failed=$(( failed + 1 ))- done- if [[ $failed -gt 0 ]]; then- log_error "$failed vLLM server(s) not reachable. Start them first (see header)."- exit 1- fi- echo ""-- # Run all requested (optimizer, model, benchmark) combinations- for opt in "${ALL_OPTIMIZERS[@]}"; do- log_info "=========================================="- log_info "Optimizer: $opt"- log_info "=========================================="- for model in "${ALL_MODELS[@]}"; do- for bench in "${ALL_BENCHMARKS[@]}"; do- case "$opt" in- gepa) run_gepa "$model" "$bench" || failed=$(( failed + 1 )) ;;- dspy) run_dspy "$model" "$bench" || failed=$(( failed + 1 )) ;;- *) log_error "Unknown optimizer: $opt"; failed=$(( failed + 1 )) ;;- esac- echo ""- done- done- done-- print_summary-- if [[ $failed -gt 0 ]]; then- log_error "$failed run(s) failed. Check log for details: $LOG_FILE"- exit 1- fi-- log_ok "All Track B runs completed successfully."-}--main "$@"diff --git a/scripts/experiments/run_track_d_lora_sft.sh b/scripts/experiments/run_track_d_lora_sft.shdeleted file mode 100644index e90b7415..00000000--- a/scripts/experiments/run_track_d_lora_sft.sh+++ /dev/null@@ -1,778 +0,0 @@-#!/usr/bin/env bash-# =============================================================================-# Track D: LoRA / SFT Intelligence Optimization-# NeurIPS 2026 — Intelligence Optimization Experiments-#-# Runs LoRA and SFT fine-tuning across:-# LoRA models: Qwen-2B, Qwen-9B, Qwen-27B-# SFT models: Qwen-2B, Qwen-9B-#-# After training, runs fast-benchmark eval on every checkpoint.-#-# Usage:-# bash scripts/experiments/run_track_d_lora_sft.sh-# bash scripts/experiments/run_track_d_lora_sft.sh --method lora-# bash scripts/experiments/run_track_d_lora_sft.sh --method sft --model qwen-2b-# bash scripts/experiments/run_track_d_lora_sft.sh --skip-eval-#-# Expected runtime per run:-# Qwen-2B LoRA (~4-8 h, 1x H100, GPU 0)-# Qwen-9B LoRA (~8-16 h, 1x H100, GPU 1)-# Qwen-27B LoRA (~16-24 h, 2x H100, GPUs 2-3)-# Qwen-2B SFT (~8-16 h, 1x H100, GPU 4)-# Qwen-9B SFT (~16-24 h, 2x H100, GPUs 5-6)-#-# Total wall-clock (all in parallel): ~24 h on a 7-8x H100 node-# Total GPU-hours: ~100-200 H100-hours-#-# =============================================================================-# GPU Allocation — recommended for a single 8x H100 node-# =============================================================================-#-# Run D1 (Qwen-2B LoRA) on GPU 0:-# CUDA_VISIBLE_DEVICES=0 bash scripts/experiments/run_track_d_lora_sft.sh \-# --method lora --model qwen-2b &-#-# Run D2 (Qwen-9B LoRA) on GPU 1:-# CUDA_VISIBLE_DEVICES=1 bash scripts/experiments/run_track_d_lora_sft.sh \-# --method lora --model qwen-9b &-#-# Run D3 (Qwen-27B LoRA) on GPUs 2-3:-# CUDA_VISIBLE_DEVICES=2,3 bash scripts/experiments/run_track_d_lora_sft.sh \-# --method lora --model qwen-27b &-#-# Run D4 (Qwen-2B SFT) on GPU 4:-# CUDA_VISIBLE_DEVICES=4 bash scripts/experiments/run_track_d_lora_sft.sh \-# --method sft --model qwen-2b &-#-# Run D5 (Qwen-9B SFT) on GPUs 5-6:-# CUDA_VISIBLE_DEVICES=5,6 bash scripts/experiments/run_track_d_lora_sft.sh \-# --method sft --model qwen-9b &-#-# =============================================================================-# Training datasets (downloaded automatically if HF_TOKEN is set)-# =============================================================================-#-# Primary agentic traces:-# - neulab/agent-data-collection-# - GAIR/AgentInstruct-#-# Supplementary reasoning:-# - GeneralThought-430K-filtered-# - GLM-4.7-flash SFT traces (168K + 57K)-#-# =============================================================================--set -euo pipefail--# ----------------------------------------------------------------------------# Defaults-# ----------------------------------------------------------------------------REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/intelligence-optimization"-LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs"-TIMESTAMP="$(date +%Y%m%d_%H%M%S)"--# Default: run all methods and models-ALL_LORA_MODELS=(qwen-2b qwen-9b qwen-27b)-ALL_SFT_MODELS=(qwen-2b qwen-9b)-FAST_BENCHMARKS=(toolcall15 pinchbench taubench)--# CLI overrides-FILTER_METHOD=""-FILTER_MODEL=""-SKIP_EVAL=false--# Training hyperparameters (override per model below)-LORA_RANK=64-LORA_ALPHA=128-LORA_DROPOUT=0.05-LORA_EPOCHS=3-LORA_LR=2e-4-LORA_BATCH_SIZE=8-LORA_GRAD_ACCUM=4--SFT_EPOCHS=3-SFT_LR=1e-5-SFT_BATCH_SIZE=4-SFT_GRAD_ACCUM=8--MAX_SEQ_LEN=8192--# Fast-eval settings-EVAL_MAX_SAMPLES=20--# ----------------------------------------------------------------------------# HuggingFace model IDs-# ----------------------------------------------------------------------------model_hf_id() {- case "$1" in- qwen-2b) echo "Qwen/Qwen2.5-1.5B-Instruct" ;;- qwen-9b) echo "Qwen/Qwen2.5-7B-Instruct" ;;- qwen-27b) echo "Qwen/Qwen2.5-32B-Instruct" ;;- *)- echo "ERROR: unknown model $1" >&2- exit 1 ;;- esac-}--# ----------------------------------------------------------------------------# Parse CLI flags-# ----------------------------------------------------------------------------while [[ $# -gt 0 ]]; do- case "$1" in- --method)- FILTER_METHOD="$2"; shift 2 ;;- --model)- FILTER_MODEL="$2"; shift 2 ;;- --skip-eval)- SKIP_EVAL=true; shift ;;- --lora-rank)- LORA_RANK="$2"; shift 2 ;;- --lora-epochs)- LORA_EPOCHS="$2"; shift 2 ;;- --sft-epochs)- SFT_EPOCHS="$2"; shift 2 ;;- -h|--help)- sed -n '2,35p' "$0" | grep '^#' | sed 's/^# \?//'- exit 0 ;;- *)- echo "Unknown flag: $1"; exit 1 ;;- esac-done--# Apply model filter-if [[ -n "$FILTER_MODEL" ]]; then- ALL_LORA_MODELS=("$FILTER_MODEL")- ALL_SFT_MODELS=("$FILTER_MODEL")-fi--# Apply method filter-RUN_LORA=true-RUN_SFT=true-if [[ "$FILTER_METHOD" == "lora" ]]; then- RUN_SFT=false-elif [[ "$FILTER_METHOD" == "sft" ]]; then- RUN_LORA=false-fi--# ----------------------------------------------------------------------------# Logging helpers-# ----------------------------------------------------------------------------mkdir -p "$LOG_DIR"-LOG_FILE="${LOG_DIR}/track_d_${TIMESTAMP}.log"--log() {- local level="$1"; shift- local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"- echo "$msg"- echo "$msg" >> "$LOG_FILE"-}--log_info() { log "INFO " "$@"; }-log_ok() { log "OK " "$@"; }-log_warn() { log "WARN " "$@"; }-log_error() { log "ERROR" "$@"; }--# ----------------------------------------------------------------------------# Environment setup-# ----------------------------------------------------------------------------setup_env() {- log_info "=== Track D: LoRA/SFT Training ==="- log_info "Repo: $REPO_ROOT"- log_info "Log: $LOG_FILE"- log_info "Results: $RESULTS_BASE"- echo ""-- # Check we are in the repo root- if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then- log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)"- exit 1- fi-- # Install/sync dependencies- log_info "Running uv sync..."- cd "$REPO_ROOT"- uv sync --extra dev 2>&1 | tail -5- log_ok "uv sync complete"-- # HuggingFace token — required to download gated Qwen models- if [[ -z "${HF_TOKEN:-}" ]]; then- log_warn "HF_TOKEN is not set."- log_warn " If Qwen models are gated, set: export HF_TOKEN=hf_..."- log_warn " Or pre-download them with: huggingface-cli download <model>"- else- log_ok "HF_TOKEN is set"- # Log in so huggingface_hub uses the token- uv run python -c "-import huggingface_hub-huggingface_hub.login(token='${HF_TOKEN}', add_to_git_credential=False)-print('Logged in to HuggingFace Hub')-" 2>&1 | tee -a "$LOG_FILE"- fi-- # Check for GPU- if ! command -v nvidia-smi &>/dev/null; then- log_warn "nvidia-smi not found — ensure CUDA is available for training."- else- log_info "GPU status:"- nvidia-smi --query-gpu=index,name,memory.total,memory.free \- --format=csv,noheader 2>&1 | while IFS= read -r line; do- log_info " $line"- done- fi-- # Check for trl / peft / transformers (training stack)- uv run python -c "-import importlib, sys-missing = []-for pkg in ['transformers', 'peft', 'trl', 'datasets', 'accelerate', 'bitsandbytes']:- if importlib.util.find_spec(pkg) is None:- missing.append(pkg)-if missing:- print('MISSING packages:', missing)- sys.exit(1)-else:- print('Training stack OK: transformers, peft, trl, datasets, accelerate, bitsandbytes')-" 2>&1 | tee -a "$LOG_FILE" || {- log_error "Some training packages are missing. Install with:"- log_error " uv add transformers peft trl datasets accelerate bitsandbytes"- exit 1- }-- echo ""-}--# ----------------------------------------------------------------------------# Download / verify training dataset-# ----------------------------------------------------------------------------prepare_dataset() {- local method="$1"- local model="$2"- local data_dir="${REPO_ROOT}/results/neurips-2026/training-data"- mkdir -p "$data_dir"-- log_info "Preparing training dataset for $method/$model..."-- uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"-from datasets import load_dataset-import json-from pathlib import Path--data_dir = Path("${data_dir}")-output_path = data_dir / "${method}_${model}_train.jsonl"--if output_path.exists():- lines = output_path.read_text().count('\n')- print(f"Dataset already exists: {output_path} ({lines} examples)")-else:- print("Downloading neulab/agent-data-collection...")- ds = load_dataset("neulab/agent-data-collection", split="train")- print(f"Raw dataset size: {len(ds)}")-- # Filter to reasonable-length examples for agentic fine-tuning- examples = []- for ex in ds:- messages = ex.get("messages") or ex.get("conversations") or []- if not messages:- continue- total_len = sum(len(str(m)) for m in messages)- if 200 <= total_len <= 16000:- examples.append({"messages": messages})-- print(f"Filtered dataset size: {len(examples)}")-- with output_path.open("w") as f:- for ex in examples:- f.write(json.dumps(ex) + "\n")- print(f"Saved to {output_path}")-PYEOF-- echo "${data_dir}/${method}_${model}_train.jsonl"-}--# ----------------------------------------------------------------------------# LoRA training for one model-# ----------------------------------------------------------------------------run_lora() {- local model="$1"- local hf_id- hf_id="$(model_hf_id "$model")"- local out_dir="${RESULTS_BASE}/lora/${model}"- local checkpoint_dir="${out_dir}/checkpoint"- mkdir -p "$out_dir" "$checkpoint_dir"-- log_info "=========================================="- log_info "LoRA training: $model ($hf_id)"- log_info " Output: $out_dir"- log_info " LoRA rank=$LORA_RANK alpha=$LORA_ALPHA dropout=$LORA_DROPOUT"- log_info " Epochs=$LORA_EPOCHS LR=$LORA_LR batch=$LORA_BATCH_SIZE grad_accum=$LORA_GRAD_ACCUM"- log_info "=========================================="-- local dataset_path- dataset_path="$(prepare_dataset lora "$model")"-- local t0- t0="$(date +%s)"-- # Number of GPUs visible- local n_gpus- n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)"- log_info "Training on $n_gpus GPU(s)"-- if [[ "$n_gpus" -gt 1 ]]; then- LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus"- else- LAUNCHER="uv run python"- fi-- $LAUNCHER - <<PYEOF 2>&1 | tee -a "$LOG_FILE"-import json-import math-import os-from pathlib import Path--import torch-from datasets import load_dataset-from peft import LoraConfig, TaskType, get_peft_model-from transformers import (- AutoModelForCausalLM,- AutoTokenizer,- DataCollatorForSeq2Seq,- Trainer,- TrainingArguments,-)--hf_id = "${hf_id}"-out_dir = "${out_dir}"-ckpt = "${checkpoint_dir}"-data_path = "${dataset_path}"--# ---- Tokenizer -----print(f"Loading tokenizer: {hf_id}")-tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True)-if tokenizer.pad_token is None:- tokenizer.pad_token = tokenizer.eos_token--# ---- Dataset -----print(f"Loading dataset: {data_path}")-raw = load_dataset("json", data_files=data_path, split="train")-raw = raw.train_test_split(test_size=0.02, seed=42)--def tokenize(example):- messages = example["messages"]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=False,- )- enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN})- enc["labels"] = enc["input_ids"].copy()- return enc--print("Tokenizing dataset...")-tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4)-print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}")--# ---- Model -----print(f"Loading model: {hf_id}")-model = AutoModelForCausalLM.from_pretrained(- hf_id,- torch_dtype=torch.bfloat16,- trust_remote_code=True,- device_map="auto",-)--# ---- LoRA -----lora_config = LoraConfig(- task_type=TaskType.CAUSAL_LM,- r=${LORA_RANK},- lora_alpha=${LORA_ALPHA},- lora_dropout=${LORA_DROPOUT},- target_modules=["q_proj", "k_proj", "v_proj", "o_proj",- "gate_proj", "up_proj", "down_proj"],- bias="none",-)-model = get_peft_model(model, lora_config)-model.print_trainable_parameters()--# ---- Training args -----steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${LORA_BATCH_SIZE} * ${LORA_GRAD_ACCUM}))-total_steps = steps_per_epoch * ${LORA_EPOCHS}-save_steps = max(50, steps_per_epoch // 2)--args = TrainingArguments(- output_dir=ckpt,- num_train_epochs=${LORA_EPOCHS},- per_device_train_batch_size=${LORA_BATCH_SIZE},- gradient_accumulation_steps=${LORA_GRAD_ACCUM},- learning_rate=${LORA_LR},- warmup_ratio=0.05,- lr_scheduler_type="cosine",- logging_steps=10,- save_steps=save_steps,- eval_strategy="steps",- eval_steps=save_steps,- load_best_model_at_end=True,- metric_for_best_model="eval_loss",- bf16=True,- gradient_checkpointing=True,- dataloader_num_workers=4,- report_to="none",- save_total_limit=3,-)--# ---- Trainer -----collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)-trainer = Trainer(- model=model,- args=args,- train_dataset=tok_ds["train"],- eval_dataset=tok_ds["test"],- data_collator=collator,-)--print(f"Starting LoRA training: {total_steps} total steps")-trainer.train()--# Save final adapter-final_adapter = os.path.join(out_dir, "lora_adapter_final")-model.save_pretrained(final_adapter)-tokenizer.save_pretrained(final_adapter)-print(f"Final LoRA adapter saved to {final_adapter}")--# Save training metadata-meta = {- "model": hf_id,- "method": "lora",- "lora_rank": ${LORA_RANK},- "lora_alpha": ${LORA_ALPHA},- "epochs": ${LORA_EPOCHS},- "train_examples": len(tok_ds["train"]),- "final_train_loss": trainer.state.log_history[-1].get("loss"),-}-with open(os.path.join(out_dir, "training_meta.json"), "w") as f:- import json; json.dump(meta, f, indent=2)-print("Training metadata saved.")-PYEOF-- local exit_code=${PIPESTATUS[0]}- local t1- t1="$(date +%s)"- local elapsed=$(( t1 - t0 ))-- if [[ $exit_code -eq 0 ]]; then- log_ok "LoRA $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"- return 0- else- log_error "LoRA $model FAILED (exit $exit_code) after ${elapsed}s"- return $exit_code- fi-}--# ----------------------------------------------------------------------------# SFT (full fine-tuning) for one model-# ----------------------------------------------------------------------------run_sft() {- local model="$1"- local hf_id- hf_id="$(model_hf_id "$model")"- local out_dir="${RESULTS_BASE}/sft/${model}"- local checkpoint_dir="${out_dir}/checkpoint"- mkdir -p "$out_dir" "$checkpoint_dir"-- log_info "=========================================="- log_info "SFT training: $model ($hf_id)"- log_info " Output: $out_dir"- log_info " Epochs=$SFT_EPOCHS LR=$SFT_LR batch=$SFT_BATCH_SIZE grad_accum=$SFT_GRAD_ACCUM"- log_info "=========================================="-- local dataset_path- dataset_path="$(prepare_dataset sft "$model")"-- local t0- t0="$(date +%s)"-- local n_gpus- n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)"- log_info "Training on $n_gpus GPU(s)"-- if [[ "$n_gpus" -gt 1 ]]; then- LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus"- else- LAUNCHER="uv run python"- fi-- $LAUNCHER - <<PYEOF 2>&1 | tee -a "$LOG_FILE"-import json-import math-import os-from pathlib import Path--import torch-from datasets import load_dataset-from transformers import (- AutoModelForCausalLM,- AutoTokenizer,- DataCollatorForSeq2Seq,- Trainer,- TrainingArguments,-)--hf_id = "${hf_id}"-out_dir = "${out_dir}"-ckpt = "${checkpoint_dir}"-data_path = "${dataset_path}"--# ---- Tokenizer -----print(f"Loading tokenizer: {hf_id}")-tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True)-if tokenizer.pad_token is None:- tokenizer.pad_token = tokenizer.eos_token--# ---- Dataset -----print(f"Loading dataset: {data_path}")-raw = load_dataset("json", data_files=data_path, split="train")-raw = raw.train_test_split(test_size=0.02, seed=42)--def tokenize(example):- messages = example["messages"]- text = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=False,- )- enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN})- enc["labels"] = enc["input_ids"].copy()- return enc--print("Tokenizing dataset...")-tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4)-print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}")--# ---- Model (4-bit quantized to fit on fewer GPUs) -----print(f"Loading model: {hf_id}")-from transformers import BitsAndBytesConfig-bnb_config = BitsAndBytesConfig(- load_in_4bit=True,- bnb_4bit_compute_dtype=torch.bfloat16,- bnb_4bit_quant_type="nf4",- bnb_4bit_use_double_quant=True,-)-model = AutoModelForCausalLM.from_pretrained(- hf_id,- quantization_config=bnb_config,- trust_remote_code=True,- device_map="auto",-)-model.config.use_cache = False--# ---- Training args -----steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${SFT_BATCH_SIZE} * ${SFT_GRAD_ACCUM}))-total_steps = steps_per_epoch * ${SFT_EPOCHS}-save_steps = max(50, steps_per_epoch // 2)--args = TrainingArguments(- output_dir=ckpt,- num_train_epochs=${SFT_EPOCHS},- per_device_train_batch_size=${SFT_BATCH_SIZE},- gradient_accumulation_steps=${SFT_GRAD_ACCUM},- learning_rate=${SFT_LR},- warmup_ratio=0.03,- lr_scheduler_type="cosine",- logging_steps=10,- save_steps=save_steps,- eval_strategy="steps",- eval_steps=save_steps,- load_best_model_at_end=True,- metric_for_best_model="eval_loss",- bf16=True,- gradient_checkpointing=True,- dataloader_num_workers=4,- report_to="none",- save_total_limit=3,-)--# ---- Trainer -----collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)-trainer = Trainer(- model=model,- args=args,- train_dataset=tok_ds["train"],- eval_dataset=tok_ds["test"],- data_collator=collator,-)--print(f"Starting SFT training: {total_steps} total steps")-trainer.train()--# Save final model-final_model = os.path.join(out_dir, "sft_model_final")-model.save_pretrained(final_model)-tokenizer.save_pretrained(final_model)-print(f"Final SFT model saved to {final_model}")--# Save training metadata-meta = {- "model": hf_id,- "method": "sft",- "epochs": ${SFT_EPOCHS},- "train_examples": len(tok_ds["train"]),- "final_train_loss": trainer.state.log_history[-1].get("loss"),-}-with open(os.path.join(out_dir, "training_meta.json"), "w") as f:- import json; json.dump(meta, f, indent=2)-print("Training metadata saved.")-PYEOF-- local exit_code=${PIPESTATUS[0]}- local t1- t1="$(date +%s)"- local elapsed=$(( t1 - t0 ))-- if [[ $exit_code -eq 0 ]]; then- log_ok "SFT $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"- return 0- else- log_error "SFT $model FAILED (exit $exit_code) after ${elapsed}s"- return $exit_code- fi-}--# ----------------------------------------------------------------------------# Post-training eval: run fast benchmarks on a checkpoint-# ----------------------------------------------------------------------------run_eval() {- local method="$1" # lora or sft- local model="$2"- local out_dir="${RESULTS_BASE}/${method}/${model}"-- if [[ "$SKIP_EVAL" == "true" ]]; then- log_info "Skipping eval (--skip-eval)"- return 0- fi-- # Locate the final checkpoint / adapter- local checkpoint=""- if [[ "$method" == "lora" ]]; then- checkpoint="${out_dir}/lora_adapter_final"- else- checkpoint="${out_dir}/sft_model_final"- fi-- if [[ ! -d "$checkpoint" ]]; then- log_warn "Checkpoint not found for $method/$model at $checkpoint — skipping eval"- return 0- fi-- log_info "--- Post-training eval: $method/$model ---"- log_info " Checkpoint: $checkpoint"-- for bench in "${FAST_BENCHMARKS[@]}"; do- local eval_out="${out_dir}/eval/${bench}"- mkdir -p "$eval_out"- log_info " Eval: $bench -> $eval_out"-- uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"-import subprocess, sys--cmd = [- "uv", "run", "python", "-m", "openjarvis.evals", "run",- "--model-path", "${checkpoint}",- "--model-id", "${model}-${method}",- "--benchmark", "${bench}",- "--max-samples", "${EVAL_MAX_SAMPLES}",- "--output", "${eval_out}",-]-print("Running:", " ".join(cmd))-result = subprocess.run(cmd, capture_output=False)-sys.exit(result.returncode)-PYEOF-- local eval_exit=${PIPESTATUS[0]}- if [[ $eval_exit -eq 0 ]]; then- log_ok " Eval $method/$model/$bench OK"- else- log_warn " Eval $method/$model/$bench returned exit $eval_exit (non-fatal)"- fi- done-}--# ----------------------------------------------------------------------------# Summary-# ----------------------------------------------------------------------------print_summary() {- echo ""- log_info "=== Track D Complete ==="- log_info "Results written to:"-- if [[ "$RUN_LORA" == "true" ]]; then- for model in "${ALL_LORA_MODELS[@]}"; do- local out="${RESULTS_BASE}/lora/${model}"- if [[ -d "$out" ]]; then- log_ok " lora/$model -> $out"- else- log_warn " lora/$model -> MISSING ($out)"- fi- done- fi-- if [[ "$RUN_SFT" == "true" ]]; then- for model in "${ALL_SFT_MODELS[@]}"; do- local out="${RESULTS_BASE}/sft/${model}"- if [[ -d "$out" ]]; then- log_ok " sft/$model -> $out"- else- log_warn " sft/$model -> MISSING ($out)"- fi- done- fi-- log_info "Full log: $LOG_FILE"-}--# ----------------------------------------------------------------------------# Main-# ----------------------------------------------------------------------------main() {- setup_env-- local failed=0-- # ---- LoRA runs ----- if [[ "$RUN_LORA" == "true" ]]; then- log_info "=========================================="- log_info "Starting LoRA training runs"- log_info "Models: ${ALL_LORA_MODELS[*]}"- log_info "=========================================="- for model in "${ALL_LORA_MODELS[@]}"; do- run_lora "$model" || { failed=$(( failed + 1 )); log_error "LoRA $model failed, continuing..."; }- run_eval lora "$model"- echo ""- done- fi-- # ---- SFT runs ----- if [[ "$RUN_SFT" == "true" ]]; then- log_info "=========================================="- log_info "Starting SFT training runs"- log_info "Models: ${ALL_SFT_MODELS[*]}"- log_info "=========================================="- for model in "${ALL_SFT_MODELS[@]}"; do- run_sft "$model" || { failed=$(( failed + 1 )); log_error "SFT $model failed, continuing..."; }- run_eval sft "$model"- echo ""- done- fi-- print_summary-- if [[ $failed -gt 0 ]]; then- log_error "$failed training run(s) failed. Check log: $LOG_FILE"- exit 1- fi-- log_ok "All Track D runs completed successfully."-}--main "$@"diff --git a/scripts/oauth_all.py b/scripts/oauth_all.pyindex 7e969c27..50755171 100644--- a/scripts/oauth_all.py+++ b/scripts/oauth_all.py@@ -14,7 +14,7 @@ import osimport webbrowserfrom http.server import BaseHTTPRequestHandler, HTTPServerfrom pathlib import Path-from typing import Any, Dict, List, Optional+from typing import Any, Dict, Listfrom urllib.parse import parse_qs, urlencode, urlparseimport httpx@@ -44,16 +44,31 @@ def _load_creds(filename: str) -> Dict[str, str]:_google = _load_creds("google.json")-GOOGLE_CLIENT_ID = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"]-GOOGLE_CLIENT_SECRET = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "") or _google["client_secret"]+GOOGLE_CLIENT_ID = (+ os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"]+)+GOOGLE_CLIENT_SECRET = (+ os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "")+ or _google["client_secret"]+)_strava = _load_creds("strava.json")-STRAVA_CLIENT_ID = os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"]-STRAVA_CLIENT_SECRET = os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "") or _strava["client_secret"]+STRAVA_CLIENT_ID = (+ os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"]+)+STRAVA_CLIENT_SECRET = (+ os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "")+ or _strava["client_secret"]+)_spotify = _load_creds("spotify.json")-SPOTIFY_CLIENT_ID = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"]-SPOTIFY_CLIENT_SECRET = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "") or _spotify["client_secret"]+SPOTIFY_CLIENT_ID = (+ os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"]+)+SPOTIFY_CLIENT_SECRET = (+ os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "")+ or _spotify["client_secret"]+)# ── Generic OAuth helpers ────────────────────────────────────────────────────@@ -128,7 +143,7 @@ def do_google() -> None:"prompt": "consent",}))- print(f" Opening browser...")+ print(" Opening browser...")webbrowser.open(url)code = _wait_for_code()@@ -176,7 +191,7 @@ def do_strava() -> None:"scope": "activity:read_all",}))- print(f" Opening browser...")+ print(" Opening browser...")webbrowser.open(url)code = _wait_for_code()@@ -208,7 +223,6 @@ def do_strava() -> None:def _make_self_signed_cert() -> tuple[str, str]:"""Generate a temporary self-signed cert for localhost HTTPS."""- import sslimport subprocessimport tempfile@@ -288,7 +302,7 @@ def do_spotify() -> None:"scope": "user-read-recently-played",}))- print(f" Opening browser...")+ print(" Opening browser...")webbrowser.open(url)code = _wait_for_code(port=spotify_port)@@ -323,7 +337,9 @@ def do_spotify() -> None:# ── Main ─────────────────────────────────────────────────────────────────────if __name__ == "__main__":- parser = argparse.ArgumentParser(description="Run OAuth flows for OpenJarvis connectors")+ parser = argparse.ArgumentParser(+ description="Run OAuth flows for OpenJarvis connectors"+ )parser.add_argument("--google", action="store_true", help="Only Google")parser.add_argument("--strava", action="store_true", help="Only Strava")parser.add_argument("--spotify", action="store_true", help="Only Spotify")diff --git a/scripts/smoke_framework_comparison.sh b/scripts/smoke_framework_comparison.shindex f9ad6faa..38791d5e 100755--- a/scripts/smoke_framework_comparison.sh+++ b/scripts/smoke_framework_comparison.sh@@ -47,27 +47,45 @@ print(' all pins OK')echo "==> Running one-task smoke per (framework, benchmark)"mkdir -p results/smoke+CONFIG_DIR="results/smoke/configs"+mkdir -p "$CONFIG_DIR"SMOKE_BENCHES=(toolcall15 pinchbench gaia)SMOKE_FRAMEWORKS=(hermes openclaw openjarvis)SMOKE_MODEL="qwen-9b"for fwk in "${SMOKE_FRAMEWORKS[@]}"; dofor bench in "${SMOKE_BENCHES[@]}"; do- config="src/openjarvis/evals/configs/framework_comparison/${bench}-${fwk}-${SMOKE_MODEL}.toml"- if [ ! -f "$config" ]; then- echo " ! missing config: $config (run make_configs --all-tier1 (or materialize the configs you need))"- continue- fi+ uv run python -m openjarvis.evals.comparison.make_configs \+ --framework "$fwk" \+ --model "$SMOKE_MODEL" \+ --benchmark "$bench" \+ --output-dir "$CONFIG_DIR" >/dev/null+ config="${CONFIG_DIR}/${bench}-${fwk}-${SMOKE_MODEL}.toml"+ run_dir="results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/"+ JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" uv run python - "$config" "$run_dir" <<'PY'+from pathlib import Path+import re+import sys++path = Path(sys.argv[1])+run_dir = sys.argv[2]+text = path.read_text()+text = re.sub(r'^output_dir = ".*"$', f'output_dir = "{run_dir}"', text, flags=re.M)+text = re.sub(r"^max_samples = \d+$", "max_samples = 1", text, flags=re.M)+path.write_text(text)+PYecho " ▸ $fwk × $bench"- uv run python -m openjarvis.evals run --config "$config" --max-samples 1 \- --output-dir "results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/" \+ mkdir -p "$run_dir"+ JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" \+ JARVIS_BACKEND_API_KEY="${JARVIS_BACKEND_API_KEY:-smoke}" \+ uv run python -m openjarvis.evals run --config "$config" \|| echo " FAILED (continuing)"donedoneecho "==> Generating T1 from smoke results"uv run python -m openjarvis.evals.comparison.table_gen \- --results-glob "results/smoke/**/summary.json" \+ --results-glob "results/smoke/**/*.summary.json" \--tables T1 \--output-dir results/smoke/tables/diff --git a/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs b/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjsindex d199a740..242e68fc 100644--- a/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs+++ b/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs@@ -6,13 +6,14 @@// --task <prompt> --model <m> --base-url <url> --api-key <k> \// --output-json <path> [--workspace <path>]//-// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw chat --message <task> --json`-// (or equivalent non-interactive entry). Emits a JSON dict matching the+// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw agent --local+// --message <task> --json`. Emits a JSON dict matching the// _RunnerOutput Pydantic schema in _subprocess_runner.py.-import { writeFileSync, existsSync } from 'node:fs';+import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';import { spawn } from 'node:child_process';import { join } from 'node:path';+import { tmpdir } from 'node:os';import { argv, env, exit, chdir } from 'node:process';function parseArgs(args) {@@ -47,12 +48,30 @@ async function main() {return 3;}+ const requestedModel = String(args.model || '').includes('/')+ ? String(args.model || '')+ : `ollama/${args.model}`;+ const runDir = mkdtempSync(join(tmpdir(), 'openjarvis-openclaw-'));+ const configPath = join(runDir, 'openclaw.json');+ writeFileSync(configPath, JSON.stringify({+ agents: {+ defaults: {+ model: { primary: requestedModel },+ models: { [requestedModel]: {} },+ },+ },+ }));+const childEnv = {...env,OPENCLAW_MODEL: args.model,OPENCLAW_BASE_URL: args.base_url,OPENCLAW_API_KEY: args.api_key,+ OPENCLAW_CONFIG_PATH: configPath,};+ if (requestedModel.startsWith('ollama/')) {+ childEnv.OLLAMA_API_KEY = args.api_key || 'ollama';+ }// Use the SAME node executable that's running this script — picking up// 'node' from PATH can resolve to a system Node too old for OpenClaw@@ -95,11 +114,18 @@ async function main() {// { response: str, usage: {...}, messages: [{...}], tool_calls: int }try {const parsed = JSON.parse(stdout);- output.content = parsed.response || '';- output.usage = parsed.usage || {};- output.trajectory = parsed.messages || [];+ const payloads = Array.isArray(parsed.payloads) ? parsed.payloads : [];+ const messages = Array.isArray(parsed.messages) ? parsed.messages : [];+ const usage = parsed.usage || parsed.meta?.agentMeta?.usage || {};+ output.content = parsed.response || payloads.map((p) => p.text || '').join('\n');+ output.usage = {+ prompt_tokens: usage.prompt_tokens ?? usage.input ?? 0,+ completion_tokens: usage.completion_tokens ?? usage.output ?? 0,+ total_tokens: usage.total_tokens ?? usage.total ?? 0,+ };+ output.trajectory = messages;output.tool_calls = parsed.tool_calls || 0;- output.turn_count = (parsed.messages || []).filter(+ output.turn_count = messages.filter((m) => m.role === 'assistant').length;} catch (e) {diff --git a/src/openjarvis/evals/comparison/table_gen.py b/src/openjarvis/evals/comparison/table_gen.pyindex b224590a..575a1660 100644--- a/src/openjarvis/evals/comparison/table_gen.py+++ b/src/openjarvis/evals/comparison/table_gen.py@@ -38,6 +38,10 @@ class MixedCommitError(TableGenError):"""Raised when one (framework, model, benchmark) cell has multiple commits."""+class NoResultsError(TableGenError):+ """Raised when no valid summary files were loaded."""++class _MetricStats(BaseModel):mean: floatstd: float@@ -395,6 +399,11 @@ def main(results_glob: str, tables: str, output_dir: Optional[Path]) -> None:click.echo(f"Loaded {len(frame.df)} metric rows; {frame.unloadable_count} files skipped.")+ if frame.df.is_empty():+ raise click.ClickException(+ "No valid summary files matched --results-glob; refusing to emit empty "+ "tables."+ )requested = [t.strip() for t in tables.split(",") if t.strip()]for name in requested:diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.pyindex 851a8c8e..9b03d220 100644--- a/src/openjarvis/evals/core/runner.py+++ b/src/openjarvis/evals/core/runner.py@@ -307,6 +307,38 @@ class EvalRunner:def _process_one(self, record: EvalRecord) -> EvalResult:"""Process a single evaluation sample."""cfg = self._config++ def _backend_error_result(full: dict, message: str) -> EvalResult:+ usage = full.get("usage", {}) or {}+ energy_j = full.get("energy_joules", 0.0) or 0.0+ power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0+ return EvalResult(+ record_id=record.record_id,+ model_answer=full.get("content", "") or "",+ error=message,+ latency_seconds=full.get("latency_seconds", 0.0) or 0.0,+ prompt_tokens=usage.get("prompt_tokens", 0),+ completion_tokens=usage.get("completion_tokens", 0),+ cost_usd=full.get("cost_usd", 0.0) or 0.0,+ ttft=full.get("ttft", 0.0) or 0.0,+ energy_joules=energy_j,+ power_watts=power_w,+ gpu_utilization_pct=full.get("gpu_utilization_pct", 0.0) or 0.0,+ throughput_tok_per_sec=full.get("throughput_tok_per_sec", 0.0)+ or 0.0,+ trace_data=full.get("trace_data"),+ framework=full.get(+ "framework",+ getattr(self._backend, "framework_name", "openjarvis"),+ ),+ framework_commit=full.get(+ "framework_commit",+ getattr(self._backend, "framework_commit_value", "") or "",+ ),+ tool_calls=int(full.get("tool_calls", 0) or 0),+ turn_count=int(full.get("turn_count", 0) or 0),+ )+try:gen_kwargs: dict = dict(model=cfg.model,@@ -327,6 +359,8 @@ class EvalRunner:**gen_kwargs,)full = full or {}+ if full.get("error"):+ return _backend_error_result(full, str(full["error"]))all_tool_results = list(full.get("tool_results",@@ -346,6 +380,8 @@ class EvalRunner:**gen_kwargs,)sfull = sfull or {}+ if sfull.get("error"):+ return _backend_error_result(sfull, str(sfull["error"]))all_tool_results.extend(sfull.get("tool_results", []),)@@ -363,6 +399,8 @@ class EvalRunner:**gen_kwargs,)full = full or {}+ if full.get("error"):+ return _backend_error_result(full, str(full["error"]))content = full.get("content", "")is_correct, scoring_meta = self._scorer.score(record,@@ -378,7 +416,7 @@ class EvalRunner:# 'or 0.0' handles the case where the key is present but the# value is None -- .get(default) only kicks in when missing.energy_j = full.get("energy_joules", 0.0) or 0.0- power_w = full.get("power_watts", 0.0) or 0.0+ power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0throughput = full.get("throughput_tok_per_sec", 0.0) or 0.0accuracy_score = 1.0 if is_correct else 0.0diff --git a/tests/evals/comparison/test_export_to_table_gen_roundtrip.py b/tests/evals/comparison/test_export_to_table_gen_roundtrip.pyindex fbe0c378..9ce5679b 100644--- a/tests/evals/comparison/test_export_to_table_gen_roundtrip.py+++ b/tests/evals/comparison/test_export_to_table_gen_roundtrip.py@@ -122,7 +122,7 @@ class TestSummaryToDictEmitsRequiredFields:completion_tokens=50,cost_usd=0.001,energy_joules=10.0,- power_watts=20.0,+ power_watts=30.0,framework="hermes",framework_commit="5d3be898a",tool_calls=2,@@ -156,6 +156,7 @@ class TestSummaryToDictEmitsRequiredFields:assert d["n_tasks"] == 5assert "accuracy" in d["metrics"]assert "latency_seconds" in d["metrics"]+ assert d["metrics"]["peak_power_w"]["mean"] == 30.0assert d["metrics"]["accuracy"]["n"] == 5assert d["metrics"]["latency_seconds"]["n"] == 5# Existing rich schema also present (unchanged)diff --git a/tests/evals/comparison/test_openclaw_runner_contract.py b/tests/evals/comparison/test_openclaw_runner_contract.pynew file mode 100644index 00000000..0db4f5c1--- /dev/null+++ b/tests/evals/comparison/test_openclaw_runner_contract.py@@ -0,0 +1,72 @@+"""Contract tests for the OpenClaw subprocess runner."""++from __future__ import annotations++import json+import shutil+import subprocess+from pathlib import Path++import pytest+++def test_openclaw_runner_parses_real_agent_json_shape(tmp_path: Path) -> None:+ node = shutil.which("node")+ if node is None:+ pytest.skip("node not available")++ fake_openclaw = tmp_path / "openclaw.mjs"+ fake_openclaw.write_text(+ "\n".join(+ [+ "console.log(JSON.stringify({",+ " payloads: [{ text: 'hello from openclaw', mediaUrl: null }],",+ " meta: {",+ " agentMeta: { usage: { input: 11, output: 7, total: 18 } },",+ " },",+ "}));",+ ]+ ),+ encoding="utf-8",+ )++ runner = (+ Path(__file__).resolve().parents[3]+ / "src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs"+ )+ out_json = tmp_path / "out.json"+ env = {+ "OPENCLAW_PATH": str(tmp_path),+ "HOME": str(tmp_path / "home"),+ }+ result = subprocess.run(+ [+ node,+ str(runner),+ "--task",+ "Say hello.",+ "--model",+ "qwen3:0.6b",+ "--base-url",+ "http://127.0.0.1:11434/v1",+ "--api-key",+ "ollama",+ "--output-json",+ str(out_json),+ ],+ env=env,+ capture_output=True,+ text=True,+ timeout=30,+ check=False,+ )++ assert result.returncode == 0, result.stderr+ data = json.loads(out_json.read_text(encoding="utf-8"))+ assert data["content"] == "hello from openclaw"+ assert data["usage"] == {+ "prompt_tokens": 11,+ "completion_tokens": 7,+ "total_tokens": 18,+ }+ assert data["error"] is Nonediff --git a/tests/evals/comparison/test_runner_error_path_framework.py b/tests/evals/comparison/test_runner_error_path_framework.pyindex c6272e39..8280dd7a 100644--- a/tests/evals/comparison/test_runner_error_path_framework.py+++ b/tests/evals/comparison/test_runner_error_path_framework.py@@ -9,6 +9,7 @@ happy-path EvalResult construction.from __future__ import annotations+import jsonfrom typing import Any, Dict, Iterable, List, Optional, Tuplefrom openjarvis.evals.core.dataset import DatasetProvider@@ -33,6 +34,57 @@ class _FailingBackend:pass+class _PeakOnlyBackend:+ """Mock external-style backend that reports peak_power_w but no power_watts."""++ backend_id = "hermes"+ framework_name = "hermes"+ framework_commit_value = "abc12345"++ def generate(self, *args: Any, **kwargs: Any) -> str:+ return "hi"++ def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:+ return {+ "content": "hi",+ "usage": {},+ "latency_seconds": 1.0,+ "energy_joules": 10.0,+ "peak_power_w": 42.0,+ "framework": "hermes",+ "framework_commit": "abc12345",+ }++ def close(self) -> None:+ pass+++class _BackendErrorPayload:+ """Mock external backend that returns an error payload instead of raising."""++ backend_id = "hermes"+ framework_name = "hermes"+ framework_commit_value = "abc12345"++ def generate(self, *args: Any, **kwargs: Any) -> str:+ return ""++ def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:+ return {+ "content": "",+ "usage": {"prompt_tokens": 3, "completion_tokens": 0},+ "latency_seconds": 1.5,+ "energy_joules": None,+ "peak_power_w": 17.0,+ "framework": "hermes",+ "framework_commit": "abc12345",+ "error": "backend reported failure",+ }++ def close(self) -> None:+ pass++class _SingleRecordDataset(DatasetProvider):dataset_id = "mock"dataset_name = "mock"@@ -141,6 +193,70 @@ class TestErrorPathFrameworkPropagation:assert results[0].error is not None+class TestExternalTelemetryPropagation:+ def test_peak_power_falls_back_to_power_watts(self, tmp_path: Any) -> None:+ """External backends report peak_power_w; runner should preserve it+ in EvalResult.power_watts so summary/table metrics are populated."""+ cfg = RunConfig(+ benchmark="mock",+ backend="hermes",+ model="mock-model",+ max_samples=1,+ max_workers=1,+ output_path=str(tmp_path / "out.jsonl"),+ )+ runner = EvalRunner(+ config=cfg,+ dataset=_SingleRecordDataset(),+ backend=_PeakOnlyBackend(), # type: ignore[arg-type]+ scorer=_NoopScorer(),+ )+ runner.run()++ assert runner.results[0].power_watts == 42.0+ summary = json.loads((tmp_path / "out.summary.json").read_text())+ assert summary["metrics"]["peak_power_w"]["mean"] == 42.0+++class TestBackendErrorPayloadPropagation:+ def test_backend_error_payload_marks_result_error(self, tmp_path: Any) -> None:+ """Backends can return structured error payloads without raising;+ those must count as errors rather than scored empty answers."""+ cfg = RunConfig(+ benchmark="mock",+ backend="hermes",+ model="mock-model",+ max_samples=1,+ max_workers=1,+ output_path=str(tmp_path / "out.jsonl"),+ )+ runner = EvalRunner(+ config=cfg,+ dataset=_SingleRecordDataset(),+ backend=_BackendErrorPayload(), # type: ignore[arg-type]+ scorer=_NoopScorer(),+ )+ summary = runner.run()++ result = runner.results[0]+ assert result.error == "backend reported failure"+ assert result.is_correct is None+ assert result.framework == "hermes"+ assert result.framework_commit == "abc12345"+ assert result.power_watts == 17.0+ assert result.prompt_tokens == 3+ assert summary.errors == 1+ assert summary.scored_samples == 0++ summary_data = json.loads((tmp_path / "out.summary.json").read_text())+ assert summary_data["errors"] == 1+ assert summary_data["metrics"]["accuracy"] == {+ "mean": 0.0,+ "std": 0.0,+ "n": 0,+ }++class _MockBackendWithCommit:"""Mock backend that exposes framework_name AND framework_commit_value."""diff --git a/tests/evals/comparison/test_table_gen.py b/tests/evals/comparison/test_table_gen.pyindex 18c2fcf8..bf82daa5 100644--- a/tests/evals/comparison/test_table_gen.py+++ b/tests/evals/comparison/test_table_gen.py@@ -13,6 +13,7 @@ from openjarvis.evals.comparison.table_gen import ( # noqa: E402MixedCommitError,ResultsFrame,load_results,+ main,)@@ -70,6 +71,23 @@ class TestLoadResults:with pytest.raises(MixedCommitError, match="abc123.*zzz999"):load_results(str(tmp_path / "**" / "summary.json"))+ def test_cli_refuses_empty_result_set(self, tmp_path: Path) -> None:+ from click.testing import CliRunner++ result = CliRunner().invoke(+ main,+ [+ "--results-glob",+ str(tmp_path / "**" / "summary.json"),+ "--tables",+ "T1",+ "--output-dir",+ str(tmp_path / "tables"),+ ],+ )+ assert result.exit_code != 0+ assert "No valid summary files matched" in result.output+class TestRenderBooktabs:def test_emits_valid_tabular(self) -> None: