Skip to content

Personal AI, On Personal Devices

Python

feat(evals): parallel TauBench execution + Phase 2 local-model configs (#207)

Jon Saad-Falcon30a29d64f532
diff --git a/src/openjarvis/evals/configs/gaia-nemotron-nano.toml b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml
new file mode 100644
index 00000000..f5afdffe
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-nemotron-nano.toml
@@ -0,0 +1,30 @@
+# GAIA eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "gaia-nemotron-nano"
+description = "GAIA on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/gaia-nemotron-nano/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/gaia-qwen-27b.toml b/src/openjarvis/evals/configs/gaia-qwen-27b.toml
new file mode 100644
index 00000000..0a1c1a13
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-qwen-27b.toml
@@ -0,0 +1,30 @@
+# GAIA eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "gaia-qwen-27b"
+description = "GAIA on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/gaia-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/gaia-trinity-large.toml b/src/openjarvis/evals/configs/gaia-trinity-large.toml
new file mode 100644
index 00000000..5d3ebad7
--- /dev/null
+++ b/src/openjarvis/evals/configs/gaia-trinity-large.toml
@@ -0,0 +1,30 @@
+# GAIA eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+[meta]
+name = "gaia-trinity-large"
+description = "GAIA on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/gaia-trinity-large/"
+seed = 42
+
+[[models]]
+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "gaia"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
new file mode 100644
index 00000000..63ecf526
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-nemotron-nano.toml
@@ -0,0 +1,30 @@
+# LiveResearchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "liveresearch-nemotron-nano"
+description = "LiveResearchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/liveresearch-nemotron-nano/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
new file mode 100644
index 00000000..05857481
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-qwen-27b.toml
@@ -0,0 +1,30 @@
+# LiveResearchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "liveresearch-qwen-27b"
+description = "LiveResearchBench on Qwen/Qwen3.5-27B-FP8 with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/liveresearch-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/liveresearch-trinity-large.toml b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
new file mode 100644
index 00000000..27d883d8
--- /dev/null
+++ b/src/openjarvis/evals/configs/liveresearch-trinity-large.toml
@@ -0,0 +1,30 @@
+# LiveResearchBench eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+[meta]
+name = "liveresearch-trinity-large"
+description = "LiveResearchBench on arcee-ai/Trinity-Large-Thinking-FP8-Block with monitor_operative agent"
+
+[defaults]
+temperature = 0.6
+max_tokens = 16384
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+max_tokens = 4096
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/liveresearch-trinity-large/"
+seed = 42
+
+[[models]]
+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "liveresearch"
+backend = "jarvis-agent"
+agent = "monitor_operative"
+max_samples = 50
+tools = ["web_search", "file_read", "file_write", "code_interpreter", "think"]
diff --git a/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml
new file mode 100644
index 00000000..332c0061
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-nemotron-nano.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)
+[meta]
+name = "taubench-telecom-nemotron-nano"
+description = "TauBench V2 Telecom on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0.7)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 24
+output_dir = "results/neurips-2026/baselines/taubench-telecom-nemotron-nano/"
+seed = 42
+
+[[models]]
+name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
+engine = "vllm"
+num_gpus = 1
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml
new file mode 100644
index 00000000..18b8b518
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-qwen-27b.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom eval: Qwen3.5-27B-FP8 (vLLM, TP=4)
+[meta]
+name = "taubench-telecom-qwen-27b"
+description = "TauBench V2 Telecom on Qwen/Qwen3.5-27B-FP8 (temperature=0.7)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 8
+output_dir = "results/neurips-2026/baselines/taubench-telecom-qwen-27b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-27B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml
new file mode 100644
index 00000000..55487f12
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-telecom-trinity-large.toml
@@ -0,0 +1,28 @@
+# TauBench V2 Telecom eval: Trinity-Large-Thinking-FP8-Block (vLLM, TP=8)
+[meta]
+name = "taubench-telecom-trinity-large"
+description = "TauBench V2 Telecom on arcee-ai/Trinity-Large-Thinking-FP8-Block (temperature=0.7)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 16
+output_dir = "results/neurips-2026/baselines/taubench-telecom-trinity-large/"
+seed = 42
+
+[[models]]
+name = "arcee-ai/Trinity-Large-Thinking-FP8-Block"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "telecom"
diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py
index 1c8eeadf..499eb721 100644
--- a/src/openjarvis/evals/core/runner.py
+++ b/src/openjarvis/evals/core/runner.py
@@ -131,6 +131,23 @@ class EvalRunner:
except AttributeError:
self._has_task_env = False
+ # Probe whether this task env is thread-safe (no CWD changes, no
+ # shared mutable globals). Datasets opt in by setting THREAD_SAFE = True
+ # on the env class returned by create_task_env. Default: False (safe).
+ self._task_env_thread_safe = False
+ if self._has_task_env:
+ try:
+ # Cheap probe: instantiate-free attribute lookup via a sample record
+ sample_records = list(self._dataset.iter_records())
+ if sample_records:
+ probe_env = self._dataset.create_task_env(sample_records[0])
+ if probe_env is not None and getattr(
+ type(probe_env), "THREAD_SAFE", False
+ ):
+ self._task_env_thread_safe = True
+ except Exception as exc: # pragma: no cover - probe is best-effort
+ LOGGER.debug("Task env thread-safety probe failed: %s", exc)
+
records = list(self._dataset.iter_records())
LOGGER.info(
"Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s",
@@ -171,9 +188,11 @@ class EvalRunner:
try:
if cfg.episode_mode:
self._run_episode_mode(records, progress_callback, total)
- elif self._has_task_env:
+ elif self._has_task_env and not self._task_env_thread_safe:
# Task environments (PinchBench etc.) change CWD —
# must process sequentially for thread safety.
+ # Envs that opt in via THREAD_SAFE=True fall through to the
+ # parallel ThreadPoolExecutor branch below.
for record in records:
result = self._process_one(record)
self._results.append(result)
diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.py
index 896a17f8..ed257f2c 100644
--- a/src/openjarvis/evals/datasets/taubench.py
+++ b/src/openjarvis/evals/datasets/taubench.py
@@ -9,6 +9,7 @@ Reference: https://github.com/sierra-research/tau2-bench
from __future__ import annotations
import logging
+import os
import subprocess
import sys
from pathlib import Path
@@ -40,11 +41,20 @@ def _ensure_tau2() -> None:
capture_output=True,
)
LOGGER.info("Installing tau2-bench ...")
- subprocess.run(
- [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
- check=True,
- capture_output=True,
- )
+ # Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs
+ # which don't ship pip by default.
+ try:
+ subprocess.run(
+ [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
+ check=True,
+ capture_output=True,
+ )
+ except (subprocess.CalledProcessError, FileNotFoundError):
+ subprocess.run(
+ ["uv", "pip", "install", "--python", sys.executable, "-e", str(CACHE_DIR)],
+ check=True,
+ capture_output=True,
+ )
class TauBenchDataset(DatasetProvider):
@@ -68,7 +78,9 @@ class TauBenchDataset(DatasetProvider):
self._temperature: float = 0.7
self._max_tokens: int = 4096
self._user_model: Optional[str] = None
- self._num_trials: int = 3 # pass^k: best of k trials per task
+ # pass^k: best of k trials per task. Default 3, override via env var
+ # OPENJARVIS_TAUBENCH_TRIALS for faster runs (e.g. =1 for 3x speedup).
+ self._num_trials: int = int(os.environ.get("OPENJARVIS_TAUBENCH_TRIALS", "3"))
self._telemetry: bool = False
self._gpu_metrics: bool = False
diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py
index 925f4494..6c4425c6 100644
--- a/src/openjarvis/evals/execution/taubench_env.py
+++ b/src/openjarvis/evals/execution/taubench_env.py
@@ -222,6 +222,12 @@ class TauBenchTaskEnv:
runs the simulation, and stores results in record.metadata for the scorer.
"""
+ # Thread-safety marker for the eval runner. tau2 simulations are pure
+ # in-process (LLM calls + tau2 orchestrator state held inside this
+ # instance) — no CWD changes, no shared mutable globals — so multiple
+ # TauBenchTaskEnv instances can run in parallel threads safely.
+ THREAD_SAFE = True
+
def __init__(
self,
record: EvalRecord,