Skip to content

Personal AI, On Personal Devices

Python

feat(evals): TauBench V2 native integration + GAIA eval configs (#162)

Avanika Narayan450397a2ad4f
diff --git a/src/openjarvis/connectors/slack_connector.py b/src/openjarvis/connectors/slack_connector.py
index f7c4f0dc..65c8a715 100644
--- a/src/openjarvis/connectors/slack_connector.py
+++ b/src/openjarvis/connectors/slack_connector.py
@@ -25,7 +25,10 @@ from openjarvis.tools._stubs import ToolSpec
_SLACK_API_BASE = "https://slack.com/api"
_SLACK_AUTH_ENDPOINT = "https://slack.com/oauth/v2/authorize"
-_SLACK_SCOPES = "channels:read,channels:history,groups:read,groups:history,im:read,im:history,mpim:read,mpim:history,users:read"
+_SLACK_SCOPES = (
+ "channels:read,channels:history,groups:read,groups:history,"
+ "im:read,im:history,mpim:read,mpim:history,users:read"
+)
_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "slack.json")
# ---------------------------------------------------------------------------
diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py
index 4781863a..eea9e7e7 100644
--- a/src/openjarvis/evals/cli.py
+++ b/src/openjarvis/evals/cli.py
@@ -124,6 +124,10 @@ BENCHMARKS = {
"category": "agentic",
"description": "PinchBench real-world agent tasks",
},
+ "taubench": {
+ "category": "agentic",
+ "description": "TauBench multi-turn customer service",
+ },
}
BACKENDS = {
@@ -308,6 +312,10 @@ def _build_dataset(benchmark: str, subset: str | None = None):
from openjarvis.evals.datasets.pinchbench import PinchBenchDataset
return PinchBenchDataset(path=subset)
+ elif benchmark == "taubench":
+ from openjarvis.evals.datasets.taubench import TauBenchDataset
+ domains = subset.split(",") if subset else None
+ return TauBenchDataset(domains=domains)
else:
raise click.ClickException(f"Unknown benchmark: {benchmark}")
@@ -444,6 +452,9 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):
from openjarvis.evals.scorers.pinchbench import PinchBenchScorer
return PinchBenchScorer(judge_backend, judge_model)
+ elif benchmark == "taubench":
+ from openjarvis.evals.scorers.taubench import TauBenchScorer
+ return TauBenchScorer(judge_backend, judge_model)
else:
raise click.ClickException(f"Unknown benchmark: {benchmark}")
@@ -545,6 +556,14 @@ def _run_single(config, console: Optional[Console] = None) -> object:
model=config.model,
)
dataset = _build_dataset(config.benchmark)
+ # Inject engine config for benchmarks that run their own simulation
+ if hasattr(dataset, "set_engine_config"):
+ dataset.set_engine_config(
+ engine_key=config.engine_key,
+ model=config.model,
+ temperature=config.temperature,
+ max_tokens=config.max_tokens,
+ )
judge_engine = getattr(config, "judge_engine", "cloud") or "cloud"
judge_backend = _build_judge_backend(config.judge_model, engine_key=judge_engine)
scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model)
diff --git a/src/openjarvis/evals/configs/taubench-claude-opus.toml b/src/openjarvis/evals/configs/taubench-claude-opus.toml
new file mode 100644
index 00000000..d27f8f30
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-claude-opus.toml
@@ -0,0 +1,29 @@
+# TauBench V2 eval: Claude Opus 4.6 (cloud)
+# Multi-turn customer service benchmark — test split, 3 trials
+
+[meta]
+name = "taubench-claude-opus"
+description = "TauBench V2 on claude-opus-4-6 (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-claude-opus/"
+seed = 42
+
+[[models]]
+name = "claude-opus-4-6"
+engine = "cloud"
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml
new file mode 100644
index 00000000..996e2fba
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml
@@ -0,0 +1,29 @@
+# TauBench V2 eval: Gemini 3.1 Pro Preview (cloud)
+# Multi-turn customer service benchmark — test split, 3 trials
+
+[meta]
+name = "taubench-gemini-31-pro"
+description = "TauBench V2 on gemini-3.1-pro-preview (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-gemini-31-pro/"
+seed = 42
+
+[[models]]
+name = "gemini-3.1-pro-preview"
+engine = "cloud"
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-gpt54.toml b/src/openjarvis/evals/configs/taubench-gpt54.toml
new file mode 100644
index 00000000..f2b3ef7d
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-gpt54.toml
@@ -0,0 +1,29 @@
+# TauBench V2 eval: GPT-5.4 (cloud)
+# Multi-turn customer service benchmark — test split, 3 trials
+
+[meta]
+name = "taubench-gpt54"
+description = "TauBench V2 on gpt-5.4-2026-03-05 (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-gpt54/"
+seed = 42
+
+[[models]]
+name = "gpt-5.4-2026-03-05"
+engine = "cloud"
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-nemotron3-super.toml b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml
new file mode 100644
index 00000000..2afff9a7
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml
@@ -0,0 +1,29 @@
+# TauBench V2 eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang)
+[meta]
+name = "taubench-nemotron3-super"
+description = "TauBench V2 on Nemotron-3-Super-120B-A12B-FP8 (test split, pass^3)"
+
+[defaults]
+temperature = 1.0
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-nemotron3-super/"
+seed = 42
+
+[[models]]
+name = "nemotron"
+engine = "vllm"
+num_gpus = 4
+# SGLang serving on port 8001
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-qwen122b.toml b/src/openjarvis/evals/configs/taubench-qwen122b.toml
new file mode 100644
index 00000000..e403a462
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-qwen122b.toml
@@ -0,0 +1,28 @@
+# TauBench V2 eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)
+[meta]
+name = "taubench-qwen122b"
+description = "TauBench V2 on Qwen/Qwen3.5-122B-A10B-FP8 (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-qwen122b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-122B-A10B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-qwen35b.toml b/src/openjarvis/evals/configs/taubench-qwen35b.toml
new file mode 100644
index 00000000..cb779851
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-qwen35b.toml
@@ -0,0 +1,28 @@
+# TauBench V2 eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)
+[meta]
+name = "taubench-qwen35b"
+description = "TauBench V2 on Qwen/Qwen3.5-35B-A3B-FP8 (test split, pass^3)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-qwen35b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-35B-A3B-FP8"
+engine = "vllm"
+num_gpus = 4
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/configs/taubench-qwen397b.toml b/src/openjarvis/evals/configs/taubench-qwen397b.toml
new file mode 100644
index 00000000..3989aab5
--- /dev/null
+++ b/src/openjarvis/evals/configs/taubench-qwen397b.toml
@@ -0,0 +1,30 @@
+# TauBench V2 eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)
+# Multi-turn customer service benchmark across airline, retail
+
+[meta]
+name = "taubench-qwen397b"
+description = "TauBench V2 on Qwen/Qwen3.5-397B-A17B-FP8 (airline + retail)"
+
+[defaults]
+temperature = 0.7
+max_tokens = 4096
+
+[judge]
+model = "gpt-5-mini-2025-08-07"
+temperature = 0.0
+engine = "cloud"
+
+[run]
+max_workers = 1
+output_dir = "results/taubench-qwen397b/"
+seed = 42
+
+[[models]]
+name = "Qwen/Qwen3.5-397B-A17B-FP8"
+engine = "vllm"
+num_gpus = 8
+
+[[benchmarks]]
+name = "taubench"
+backend = "jarvis-direct"
+split = "airline,retail"
diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py
index 7067477a..20aa9e04 100644
--- a/src/openjarvis/evals/core/config.py
+++ b/src/openjarvis/evals/core/config.py
@@ -61,6 +61,7 @@ KNOWN_BENCHMARKS = {
"doc_qa",
"browser_assistant",
"pinchbench",
+ "taubench",
}
diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.py
new file mode 100644
index 00000000..9cdc4757
--- /dev/null
+++ b/src/openjarvis/evals/datasets/taubench.py
@@ -0,0 +1,234 @@
+"""TauBench V2 dataset provider — multi-turn customer service benchmark.
+
+Wraps the tau2-bench framework for evaluation within OpenJarvis.
+Supports airline, retail, and telecom domains.
+
+Reference: https://github.com/sierra-research/tau2-bench
+"""
+
+from __future__ import annotations
+
+import logging
+import subprocess
+import sys
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+from openjarvis.evals.core.dataset import DatasetProvider
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git"
+CACHE_DIR = Path.home() / ".cache" / "tau2-bench"
+
+DOMAINS = ("airline", "retail", "telecom")
+
+
+def _ensure_tau2() -> None:
+ """Ensure tau2 package is importable; install from cache if needed."""
+ try:
+ import tau2 # noqa: F401
+ except ImportError:
+ # Clone and install from source
+ if not CACHE_DIR.exists():
+ LOGGER.info("Cloning tau2-bench from %s ...", TAU2_REPO)
+ CACHE_DIR.parent.mkdir(parents=True, exist_ok=True)
+ subprocess.run(
+ ["git", "clone", "--depth", "1", TAU2_REPO, str(CACHE_DIR)],
+ check=True,
+ capture_output=True,
+ )
+ LOGGER.info("Installing tau2-bench ...")
+ subprocess.run(
+ [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
+ check=True,
+ capture_output=True,
+ )
+
+
+class TauBenchDataset(DatasetProvider):
+ """TauBench V2 multi-turn customer service benchmark.
+
+ Wraps tau2-bench's task loading and evaluation infrastructure.
+ Each EvalRecord represents a single customer service scenario.
+ """
+
+ dataset_id = "taubench"
+ dataset_name = "TauBench"
+
+ def __init__(
+ self,
+ domains: Optional[List[str]] = None,
+ ) -> None:
+ self._domains = domains or list(DOMAINS)
+ self._records: List[EvalRecord] = []
+ self._engine_key: Optional[str] = None
+ self._model: Optional[str] = None
+ self._temperature: float = 0.7
+ self._max_tokens: int = 4096
+ self._user_model: Optional[str] = None
+ self._num_trials: int = 3 # pass^k: best of k trials per task
+
+ def set_engine_config(
+ self,
+ engine_key: Optional[str] = None,
+ model: Optional[str] = None,
+ temperature: float = 0.7,
+ max_tokens: int = 4096,
+ user_model: Optional[str] = None,
+ num_trials: Optional[int] = None,
+ ) -> None:
+ """Inject engine configuration for the agent. Called by CLI."""
+ if engine_key is not None:
+ self._engine_key = engine_key
+ if model is not None:
+ self._model = model
+ self._temperature = temperature
+ self._max_tokens = max_tokens
+ if user_model is not None:
+ self._user_model = user_model
+ if num_trials is not None:
+ self._num_trials = num_trials
+
+ def verify_requirements(self) -> List[str]:
+ issues: List[str] = []
+ try:
+ _ensure_tau2()
+ except Exception as exc:
+ issues.append(f"tau2-bench not available: {exc}")
+ return issues
+
+ def load(
+ self,
+ *,
+ max_samples: Optional[int] = None,
+ split: Optional[str] = None,
+ seed: Optional[int] = None,
+ ) -> None:
+ _ensure_tau2()
+ from tau2.runner import get_tasks
+
+ # split overrides domains if provided (e.g. "airline,retail")
+ domains = self._domains
+ if split:
+ domains = [d.strip() for d in split.split(",") if d.strip()]
+
+ all_records: List[EvalRecord] = []
+
+ for domain in domains:
+ if domain not in DOMAINS:
+ LOGGER.warning("Unknown TauBench domain: %s", domain)
+ continue
+
+ # Load tasks, filtering to test split when available
+ from tau2.runner import load_task_splits
+ try:
+ task_splits = load_task_splits(domain)
+ test_ids = (
+ set(str(t) for t in task_splits.get("test", []))
+ if task_splits
+ else set()
+ )
+ except Exception:
+ test_ids = set()
+
+ tasks = get_tasks(domain)
+ if test_ids:
+ tasks = [t for t in tasks if str(t.id) in test_ids]
+ LOGGER.info(
+ "TauBench: loaded %d test-split tasks for domain '%s'",
+ len(tasks), domain,
+ )
+ else:
+ LOGGER.info(
+ "TauBench: loaded %d tasks for domain '%s' (no test split)",
+ len(tasks), domain,
+ )
+
+ for task in tasks:
+ # Build the user's reason for calling as the problem prompt
+ user_scenario = task.user_scenario
+ instructions = user_scenario.instructions
+ problem = (
+ f"Domain: {domain}\n"
+ f"Reason for call: {instructions.reason_for_call}\n"
+ f"Known info: {instructions.known_info or 'None'}\n"
+ )
+
+ # Extract evaluation criteria
+ eval_criteria = task.evaluation_criteria
+ actions = []
+ nl_assertions = []
+ communicate_info = []
+ reward_basis = []
+ if eval_criteria:
+ actions = [
+ a.model_dump() if hasattr(a, "model_dump") else a
+ for a in (eval_criteria.actions or [])
+ ]
+ nl_assertions = eval_criteria.nl_assertions or []
+ communicate_info = [
+ c.model_dump() if hasattr(c, "model_dump") else c
+ for c in (eval_criteria.communicate_info or [])
+ ]
+ reward_basis = [
+ r.value if hasattr(r, "value") else r
+ for r in (eval_criteria.reward_basis or [])
+ ]
+
+ record = EvalRecord(
+ record_id=f"{domain}_{task.id}",
+ problem=problem,
+ reference=task.description.purpose if task.description else "",
+ category=domain,
+ subject=f"taubench-{domain}",
+ metadata={
+ "domain": domain,
+ "task_id": task.id,
+ "task_instructions": instructions.task_instructions,
+ "reason_for_call": instructions.reason_for_call,
+ "known_info": instructions.known_info,
+ "unknown_info": instructions.unknown_info,
+ "actions": actions,
+ "nl_assertions": nl_assertions,
+ "communicate_info": communicate_info,
+ "reward_basis": reward_basis,
+ },
+ )
+ all_records.append(record)
+
+ if seed is not None:
+ import random
+ random.Random(seed).shuffle(all_records)
+ if max_samples is not None:
+ all_records = all_records[:max_samples]
+
+ self._records = all_records
+ LOGGER.info(
+ "TauBench: loaded %d total tasks across %s",
+ len(self._records),
+ ", ".join(self._domains),
+ )
+
+ def iter_records(self) -> Iterable[EvalRecord]:
+ return iter(self._records)
+
+ def size(self) -> int:
+ return len(self._records)
+
+ def create_task_env(self, record: EvalRecord):
+ """Create a TauBench task environment for evaluation."""
+ from openjarvis.evals.execution.taubench_env import TauBenchTaskEnv
+ return TauBenchTaskEnv(
+ record,
+ engine_key=self._engine_key,
+ model=self._model,
+ temperature=self._temperature,
+ max_tokens=self._max_tokens,
+ user_model=self._user_model,
+ num_trials=self._num_trials,
+ )
+
+
+__all__ = ["TauBenchDataset"]
diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py
new file mode 100644
index 00000000..cd9b25a8
--- /dev/null
+++ b/src/openjarvis/evals/execution/taubench_env.py
@@ -0,0 +1,369 @@
+"""TauBench task environment — native OpenJarvis agent in tau2 simulation.
+
+Plugs OpenJarvis's inference engine into tau2-bench's orchestrator as a
+``HalfDuplexAgent``, so the multi-turn conversation loop, user simulator,
+domain tools, database, and evaluation all come from tau2-bench while the
+agent's LLM calls go through OpenJarvis.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from types import TracebackType
+from typing import Any, Optional, Type
+
+from openjarvis.evals.core.types import EvalRecord
+
+LOGGER = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Message conversion helpers
+# ---------------------------------------------------------------------------
+
+def _tau2_to_oj_messages(
+ tau2_messages: list,
+) -> list:
+ """Convert tau2 Message objects to OpenJarvis Message objects."""
+ from openjarvis.core.types import Message, Role
+ from openjarvis.core.types import ToolCall as OJToolCall
+
+ oj_msgs: list = []
+ for m in tau2_messages:
+ role_str = getattr(m, "role", "user")
+ if role_str == "system":
+ oj_msgs.append(Message(role=Role.SYSTEM, content=m.content or ""))
+ elif role_str == "user":
+ oj_msgs.append(Message(role=Role.USER, content=m.content or ""))
+ elif role_str == "assistant":
+ tc_list = getattr(m, "tool_calls", None)
+ if tc_list:
+ oj_tool_calls = [
+ OJToolCall(
+ id=tc.id,
+ name=tc.name,
+ arguments=(
+ json.dumps(tc.arguments)
+ if isinstance(tc.arguments, dict)
+ else str(tc.arguments)
+ ),
+ )
+ for tc in tc_list
+ ]
+ oj_msgs.append(
+ Message(
+ role=Role.ASSISTANT,
+ content=m.content or "",
+ tool_calls=oj_tool_calls,
+ )
+ )
+ else:
+ oj_msgs.append(
+ Message(role=Role.ASSISTANT, content=m.content or "")
+ )
+ elif role_str == "tool":
+ # tau2 ToolMessage uses 'id', not 'tool_call_id'
+ raw_id = getattr(m, "id", "") or getattr(m, "tool_call_id", "") or ""
+ import re as _re
+ clean_id = _re.sub(r"[^a-zA-Z0-9_-]", "_", raw_id)
+ oj_msgs.append(
+ Message(
+ role=Role.TOOL,
+ content=m.content or "",
+ tool_call_id=clean_id,
+ name=getattr(m, "name", ""),
+ )
+ )
+ return oj_msgs
+
+
+def _oj_result_to_tau2_msg(result: dict):
+ """Convert OpenJarvis engine.generate() result to a tau2 AssistantMessage."""
+ from tau2.data_model.message import AssistantMessage, ToolCall
+
+ raw_tool_calls = result.get("tool_calls", [])
+ tool_calls = None
+ if raw_tool_calls:
+ tool_calls = [
+ ToolCall(
+ id=tc.get("id", ""),
+ name=tc.get("name", ""),
+ arguments=(
+ json.loads(tc["arguments"])
+ if isinstance(tc.get("arguments"), str)
+ else tc.get("arguments", {})
+ ),
+ requestor="assistant",
+ )
+ for tc in raw_tool_calls
+ ]
+
+ return AssistantMessage(
+ role="assistant",
+ content=result.get("content") or None,
+ tool_calls=tool_calls,
+ cost=result.get("cost_usd", 0.0),
+ )
+
+
+# ---------------------------------------------------------------------------
+# Jarvis-powered tau2 agent
+# ---------------------------------------------------------------------------
+
+class JarvisHalfDuplexAgent:
+ """A tau2 HalfDuplexAgent backed by OpenJarvis's inference engine.
+
+ Replaces tau2's built-in LLMAgent while keeping the same interface
+ so the Orchestrator, UserSimulator, and evaluation work unchanged.
+ """
+
+ STOP = "###STOP###"
+
+ def __init__(
+ self,
+ tools: list,
+ domain_policy: str,
+ engine: Any,
+ model: str,
+ temperature: float = 0.7,
+ max_tokens: int = 4096,
+ ) -> None:
+ self.tools = tools
+ self.domain_policy = domain_policy
+ self._engine = engine
+ self._model = model
+ self._temperature = temperature
+ self._max_tokens = max_tokens
+
+ @property
+ def system_prompt(self) -> str:
+ from tau2.agent.llm_agent import AGENT_INSTRUCTION, SYSTEM_PROMPT
+ return SYSTEM_PROMPT.format(
+ domain_policy=self.domain_policy,
+ agent_instruction=AGENT_INSTRUCTION,
+ )
+
+ def get_init_state(self, message_history=None):
+ from tau2.agent.llm_agent import LLMAgentState
+ from tau2.data_model.message import SystemMessage
+ return LLMAgentState(
+ system_messages=[
+ SystemMessage(role="system", content=self.system_prompt),
+ ],
+ messages=list(message_history) if message_history else [],
+ )
+
+ def generate_next_message(self, message, state):
+ from tau2.data_model.message import MultiToolMessage
+
+ # Add incoming message to conversation state
+ if isinstance(message, MultiToolMessage):
+ state.messages.extend(message.tool_messages)
+ elif message is not None:
+ state.messages.append(message)
+
+ # Convert to OpenJarvis format
+ oj_messages = _tau2_to_oj_messages(
+ state.system_messages + state.messages,
+ )
+
+ # Build OpenAI tool schemas from tau2 tools
+ openai_tools = [t.openai_schema for t in self.tools]
+
+ # Call OpenJarvis engine
+ gen_kwargs: dict = {
+ "model": self._model,
+ "temperature": self._temperature,
+ "max_tokens": self._max_tokens,
+ "tools": openai_tools,
+ }
+ # Disable thinking mode for local models (Qwen3.5 etc.)
+ # to avoid <think> tags interfering with tool call parsing
+ if "qwen" in self._model.lower():
+ gen_kwargs["chat_template_kwargs"] = {
+ "enable_thinking": False,
+ }
+ result = self._engine.generate(oj_messages, **gen_kwargs)
+
+ # Convert result to tau2 AssistantMessage
+ assistant_msg = _oj_result_to_tau2_msg(result)
+ state.messages.append(assistant_msg)
+ return assistant_msg, state
+
+ @classmethod
+ def is_stop(cls, message) -> bool:
+ if hasattr(message, "is_tool_call") and message.is_tool_call():
+ return False
+ content = getattr(message, "content", "") or ""
+ return cls.STOP in content
+
+ def set_seed(self, seed: int) -> None:
+ pass
+
+ def stop(self, *args, **kwargs) -> None:
+ """Cleanup hook called by orchestrator."""
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Task environment
+# ---------------------------------------------------------------------------
+
+class TauBenchTaskEnv:
+ """Per-task environment for TauBench evaluation.
+
+ Creates an OpenJarvis-powered agent, plugs it into tau2's orchestrator,
+ runs the simulation, and stores results in record.metadata for the scorer.
+ """
+
+ def __init__(
+ self,
+ record: EvalRecord,
+ engine_key: Optional[str] = None,
+ model: Optional[str] = None,
+ temperature: float = 0.7,
+ max_tokens: int = 4096,
+ user_model: Optional[str] = None,
+ num_trials: int = 1,
+ ) -> None:
+ self._record = record
+ self._num_trials = num_trials
+ self._engine_key = engine_key
+ self._model = model or "claude-opus-4-6"
+ self._temperature = temperature
+ self._max_tokens = max_tokens
+ self._user_model = user_model or "gpt-5-mini-2025-08-07"
+ self._system = None
+
+ def __enter__(self) -> TauBenchTaskEnv:
+ # Build OpenJarvis system for engine access
+ from openjarvis.system import SystemBuilder
+
+ builder = SystemBuilder()
+ if self._engine_key:
+ builder.engine(self._engine_key)
+ self._system = builder.build()
+
+ # Run the simulation
+ self._run_simulation()
+ return self
+
+ def _run_simulation(self) -> None:
+ from tau2.evaluator.evaluator import EvaluationType
+ from tau2.orchestrator.orchestrator import Orchestrator
+ from tau2.runner.build import build_environment, build_user
+ from tau2.runner.helpers import get_tasks
+ from tau2.runner.simulation import run_simulation
+
+ domain = self._record.metadata["domain"]
+ task_id = self._record.metadata["task_id"]
+
+ tasks = get_tasks(domain, task_ids=[task_id])
+ if not tasks:
+ LOGGER.error("Task %s not found in domain %s", task_id, domain)
+ self._record.metadata["tau_reward"] = 0.0
+ self._record.metadata["is_resolved"] = False
+ return
+
+ task = tasks[0]
+ best_reward = 0.0
+ best_info: dict = {}
+ best_n_messages = 0
+
+ # Run multiple trials, keep best result (pass^k)
+ for trial in range(self._num_trials):
+ try:
+ environment = build_environment(domain)
+
+ agent = JarvisHalfDuplexAgent(
+ tools=environment.get_tools(),
+ domain_policy=environment.get_policy(),
+ engine=self._system.engine,
+ model=self._model,
+ temperature=self._temperature,
+ max_tokens=self._max_tokens,
+ )
+
+ user = build_user(
+ "user_simulator",
+ environment,
+ task,
+ llm=self._user_model,
+ )
+
+ orchestrator = Orchestrator(
+ domain=domain,
+ agent=agent,
+ user=user,
+ environment=environment,
+ task=task,
+ max_steps=200,
+ seed=42 + trial,
+ )
+
+ simulation = run_simulation(
+ orchestrator,
+ evaluation_type=EvaluationType.ALL_WITH_NL_ASSERTIONS,
+ )
+
+ reward = (
+ simulation.reward_info.reward
+ if simulation.reward_info
+ else 0.0
+ )
+ info: dict = {}
+ if simulation.reward_info:
+ info = simulation.reward_info.info or {}
+ if simulation.reward_info.reward_breakdown:
+ info["reward_breakdown"] = {
+ k.value if hasattr(k, "value") else str(k): v
+ for k, v in simulation.reward_info.reward_breakdown.items()
+ }
+ n_messages = len(simulation.messages) if simulation.messages else 0
+
+ trial_label = (
+ f" (trial {trial + 1}/{self._num_trials})"
+ if self._num_trials > 1 else ""
+ )
+ LOGGER.info(
+ "TauBench %s/%s: reward=%.2f messages=%d%s",
+ domain, task_id, reward, n_messages, trial_label,
+ )
+
+ if reward > best_reward:
+ best_reward = reward
+ best_info = info
+ best_n_messages = n_messages
+
+ # Early exit if perfect score
+ if reward >= 1.0:
+ break
+
+ except Exception as exc:
+ LOGGER.error(
+ "TauBench simulation failed for %s/%s: %s",
+ domain, task_id, exc,
+ )
+
+ self._record.metadata["tau_reward"] = best_reward
+ self._record.metadata["tau_info"] = best_info
+ self._record.metadata["tau_n_messages"] = best_n_messages
+ self._record.metadata["tau_num_trials"] = self._num_trials
+ self._record.metadata["is_resolved"] = best_reward >= 0.5
+
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType],
+ ) -> None:
+ if self._system:
+ try:
+ self._system.close()
+ except Exception:
+ pass
+ self._system = None
+
+
+__all__ = ["TauBenchTaskEnv"]
diff --git a/src/openjarvis/evals/scorers/taubench.py b/src/openjarvis/evals/scorers/taubench.py
new file mode 100644
index 00000000..227640de
--- /dev/null
+++ b/src/openjarvis/evals/scorers/taubench.py
@@ -0,0 +1,46 @@
+"""TauBench scorer — wraps tau2-bench's evaluation results.
+
+Since TauBench runs its own simulation loop (agent + user simulator +
+tools + evaluation), the scorer simply reads the reward that was
+computed during task execution and stored in record.metadata.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from openjarvis.evals.core.scorer import Scorer
+from openjarvis.evals.core.types import EvalRecord
+
+
+class TauBenchScorer(Scorer):
+ """TauBench scorer — reads pre-computed rewards from tau2-bench.
+
+ The actual evaluation (DB state checks, action matching,
+ communication checks, NL assertions) is done by tau2-bench's
+ evaluator during simulation. This scorer extracts the result.
+ """
+
+ scorer_id = "taubench"
+
+ def __init__(self, judge_backend: Any = None, judge_model: str = "") -> None:
+ self._judge_backend = judge_backend
+ self._judge_model = judge_model
+
+ def score(
+ self, record: EvalRecord, model_answer: str,
+ ) -> Tuple[Optional[bool], Dict[str, Any]]:
+ reward = record.metadata.get("tau_reward", 0.0)
+ info = record.metadata.get("tau_info", {})
+ n_messages = record.metadata.get("tau_n_messages", 0)
+
+ is_correct = reward >= 0.5
+
+ return is_correct, {
+ "score": reward,
+ "breakdown": info,
+ "notes": f"reward={reward:.2f}, messages={n_messages}",
+ }
+
+
+__all__ = ["TauBenchScorer"]
diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py
index e569fa48..ab856d4d 100644
--- a/tests/evals/test_benchmark_datasets.py
+++ b/tests/evals/test_benchmark_datasets.py
@@ -313,7 +313,7 @@ class TestConfigBenchmarks:
def test_benchmarks_count(self) -> None:
from openjarvis.evals.core.config import KNOWN_BENCHMARKS
- assert len(KNOWN_BENCHMARKS) == 26
+ assert len(KNOWN_BENCHMARKS) == 27
# ---------------------------------------------------------------------------