Personal AI, On Personal Devices
Python
feat(evals): parallel TauBench execution + Phase 2 local-model configs (#207)
Jon Saad-Falcon30a29d64f532
diff --git a/src/openjarvis/evals/configs/gaia-nemotron-nano.toml b/src/openjarvis/evals/configs/gaia-nemotron-nano.tomlnew file mode 100644index 00000000..f5afdffe--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml@@ -0,0 +1,30 @@+# GAIA eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+[meta]+name = "gaia-nemotron-nano"+description = "GAIA on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/gaia-nemotron-nano/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/gaia-qwen-27b.toml b/src/openjarvis/evals/configs/gaia-qwen-27b.tomlnew file mode 100644index 00000000..0a1c1a13--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-qwen-27b.toml@@ -0,0 +1,30 @@+# GAIA eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+[meta]+name = "gaia-qwen-27b"+description = "GAIA on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/gaia-qwen-27b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/gaia-trinity-large.toml b/src/openjarvis/evals/configs/gaia-trinity-large.tomlnew file mode 100644index 00000000..5d3ebad7--- /dev/null+++ b/src/openjarvis/evals/configs/gaia-trinity-large.toml@@ -0,0 +1,30 @@+# GAIA eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)+[meta]+name = "gaia-trinity-large"+description = "GAIA on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/gaia-trinity-large/"+seed = 42++[[models]]+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"+engine = "vllm"+num_gpus = 8++[[benchmarks]]+name = "gaia"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.tomlnew file mode 100644index 00000000..63ecf526--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml@@ -0,0 +1,30 @@+# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+[meta]+name = "liveresearch-nemotron-nano"+description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/liveresearch-nemotron-nano/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.tomlnew file mode 100644index 00000000..05857481--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml@@ -0,0 +1,30 @@+# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+[meta]+name = "liveresearch-qwen-27b"+description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.tomlnew file mode 100644index 00000000..27d883d8--- /dev/null+++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml@@ -0,0 +1,30 @@+# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)+[meta]+name = "liveresearch-trinity-large"+description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"++[defaults]+temperature = 0.6+max_tokens = 16384++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+max_tokens = 4096++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"+seed = 42++[[models]]+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"+engine = "vllm"+num_gpus = 8++[[benchmarks]]+name = "liveresearch"+backend = "jarvis-agent"+agent = "monitor_operative"+max_samples = 50+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.tomlnew file mode 100644index 00000000..332c0061--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml@@ -0,0 +1,28 @@+# TauBench V2 Telecom eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)+[meta]+name = "taubench-telecom-nemotron-nano"+description = "TauBench V2 Telecom on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0.7)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 24+output_dir = "results/neurips-2026/baselines/taubench-telecom-nemotron-nano/"+seed = 42++[[models]]+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"+engine = "vllm"+num_gpus = 1++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.tomlnew file mode 100644index 00000000..18b8b518--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml@@ -0,0 +1,28 @@+# TauBench V2 Telecom eval: Qwen3.5-27B-FP8 (vLLM, TP=4)+[meta]+name = "taubench-telecom-qwen-27b"+description = "TauBench V2 Telecom on Qwen/Qwen3.5-27B-FP8 (temperature=0.7)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 8+output_dir = "results/neurips-2026/baselines/taubench-telecom-qwen-27b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-27B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "telecom"diff --git a/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.tomlnew file mode 100644index 00000000..55487f12--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml@@ -0,0 +1,28 @@+# TauBench V2 Telecom eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)+[meta]+name = "taubench-telecom-trinity-large"+description = "TauBench V2 Telecom on arcee-ai/Trinity-Large-Thinking-FP8-Block (temperature=0.7)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 16+output_dir = "results/neurips-2026/baselines/taubench-telecom-trinity-large/"+seed = 42++[[models]]+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"+engine = "vllm"+num_gpus = 8++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "telecom"diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.pyindex 1c8eeadf..499eb721 100644--- a/src/openjarvis/evals/core/runner.py+++ b/src/openjarvis/evals/core/runner.py@@ -131,6 +131,23 @@ class EvalRunner:except AttributeError:self._has_task_env = False+ # Probe whether this task env is thread-safe (no CWD changes, no+ # shared mutable globals). Datasets opt in by setting THREAD_SAFE = True+ # on the env class returned by create_task_env. Default: False (safe).+ self._task_env_thread_safe = False+ if self._has_task_env:+ try:+ # Cheap probe: instantiate-free attribute lookup via a sample record+ sample_records = list(self._dataset.iter_records())+ if sample_records:+ probe_env = self._dataset.create_task_env(sample_records[0])+ if probe_env is not None and getattr(+ type(probe_env), "THREAD_SAFE", False+ ):+ self._task_env_thread_safe = True+ except Exception as exc: # pragma: no cover - probe is best-effort+ LOGGER.debug("Task env thread-safety probe failed: %s", exc)+records = list(self._dataset.iter_records())LOGGER.info("Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s",@@ -171,9 +188,11 @@ class EvalRunner:try:if cfg.episode_mode:self._run_episode_mode(records, progress_callback, total)- elif self._has_task_env:+ elif self._has_task_env and not self._task_env_thread_safe:# Task environments (PinchBench etc.) change CWD —# must process sequentially for thread safety.+ # Envs that opt in via THREAD_SAFE=True fall through to the+ # parallel ThreadPoolExecutor branch below.for record in records:result = self._process_one(record)self._results.append(result)diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.pyindex 896a17f8..ed257f2c 100644--- a/src/openjarvis/evals/datasets/taubench.py+++ b/src/openjarvis/evals/datasets/taubench.py@@ -9,6 +9,7 @@ Reference: https://github.com/sierra-research/tau2-benchfrom __future__ import annotationsimport logging+import osimport subprocessimport sysfrom pathlib import Path@@ -40,11 +41,20 @@ def _ensure_tau2() -> None:capture_output=True,)LOGGER.info("Installing tau2-bench ...")- subprocess.run(- [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],- check=True,- capture_output=True,- )+ # Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs+ # which don't ship pip by default.+ try:+ subprocess.run(+ [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],+ check=True,+ capture_output=True,+ )+ except (subprocess.CalledProcessError, FileNotFoundError):+ subprocess.run(+ ["uv", "pip", "install", "--python", sys.executable, "-e", str(CACHE_DIR)],+ check=True,+ capture_output=True,+ )class TauBenchDataset(DatasetProvider):@@ -68,7 +78,9 @@ class TauBenchDataset(DatasetProvider):self._temperature: float = 0.7self._max_tokens: int = 4096self._user_model: Optional[str] = None- self._num_trials: int = 3 # pass^k: best of k trials per task+ # pass^k: best of k trials per task. Default 3, override via env var+ # OPENJARVIS_TAUBENCH_TRIALS for faster runs (e.g. =1 for 3x speedup).+ self._num_trials: int = int(os.environ.get("OPENJARVIS_TAUBENCH_TRIALS", "3"))self._telemetry: bool = Falseself._gpu_metrics: bool = Falsediff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.pyindex 925f4494..6c4425c6 100644--- a/src/openjarvis/evals/execution/taubench_env.py+++ b/src/openjarvis/evals/execution/taubench_env.py@@ -222,6 +222,12 @@ class TauBenchTaskEnv:runs the simulation, and stores results in record.metadata for the scorer."""+ # Thread-safety marker for the eval runner. tau2 simulations are pure+ # in-process (LLM calls + tau2 orchestrator state held inside this+ # instance) — no CWD changes, no shared mutable globals — so multiple+ # TauBenchTaskEnv instances can run in parallel threads safely.+ THREAD_SAFE = True+def __init__(self,record: EvalRecord,