Personal AI, On Personal Devices
Python
feat(evals): TauBench V2 native integration + GAIA eval configs (#162)
Avanika Narayan450397a2ad4f
diff --git a/src/openjarvis/connectors/slack_connector.py b/src/openjarvis/connectors/slack_connector.pyindex f7c4f0dc..65c8a715 100644--- a/src/openjarvis/connectors/slack_connector.py+++ b/src/openjarvis/connectors/slack_connector.py@@ -25,7 +25,10 @@ from openjarvis.tools._stubs import ToolSpec_SLACK_API_BASE = "https://slack.com/api"_SLACK_AUTH_ENDPOINT = "https://slack.com/oauth/v2/authorize"-_SLACK_SCOPES = "channels:read,channels:history,groups:read,groups:history,im:read,im:history,mpim:read,mpim:history,users:read"+_SLACK_SCOPES = (+ "channels:read,channels:history,groups:read,groups:history,"+ "im:read,im:history,mpim:read,mpim:history,users:read"+)_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "slack.json")# ---------------------------------------------------------------------------diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.pyindex 4781863a..eea9e7e7 100644--- a/src/openjarvis/evals/cli.py+++ b/src/openjarvis/evals/cli.py@@ -124,6 +124,10 @@ BENCHMARKS = {"category": "agentic","description": "PinchBench real-world agent tasks",},+ "taubench": {+ "category": "agentic",+ "description": "TauBench multi-turn customer service",+ },}BACKENDS = {@@ -308,6 +312,10 @@ def _build_dataset(benchmark: str, subset: str | None = None):from openjarvis.evals.datasets.pinchbench import PinchBenchDatasetreturn PinchBenchDataset(path=subset)+ elif benchmark == "taubench":+ from openjarvis.evals.datasets.taubench import TauBenchDataset+ domains = subset.split(",") if subset else None+ return TauBenchDataset(domains=domains)else:raise click.ClickException(f"Unknown benchmark: {benchmark}")@@ -444,6 +452,9 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str):from openjarvis.evals.scorers.pinchbench import PinchBenchScorerreturn PinchBenchScorer(judge_backend, judge_model)+ elif benchmark == "taubench":+ from openjarvis.evals.scorers.taubench import TauBenchScorer+ return TauBenchScorer(judge_backend, judge_model)else:raise click.ClickException(f"Unknown benchmark: {benchmark}")@@ -545,6 +556,14 @@ def _run_single(config, console: Optional[Console] = None) -> object:model=config.model,)dataset = _build_dataset(config.benchmark)+ # Inject engine config for benchmarks that run their own simulation+ if hasattr(dataset, "set_engine_config"):+ dataset.set_engine_config(+ engine_key=config.engine_key,+ model=config.model,+ temperature=config.temperature,+ max_tokens=config.max_tokens,+ )judge_engine = getattr(config, "judge_engine", "cloud") or "cloud"judge_backend = _build_judge_backend(config.judge_model, engine_key=judge_engine)scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model)diff --git a/src/openjarvis/evals/configs/taubench-claude-opus.toml b/src/openjarvis/evals/configs/taubench-claude-opus.tomlnew file mode 100644index 00000000..d27f8f30--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-claude-opus.toml@@ -0,0 +1,29 @@+# TauBench V2 eval: Claude Opus 4.6 (cloud)+# Multi-turn customer service benchmark — test split, 3 trials++[meta]+name = "taubench-claude-opus"+description = "TauBench V2 on claude-opus-4-6 (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-claude-opus/"+seed = 42++[[models]]+name = "claude-opus-4-6"+engine = "cloud"++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml b/src/openjarvis/evals/configs/taubench-gemini-31-pro.tomlnew file mode 100644index 00000000..996e2fba--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml@@ -0,0 +1,29 @@+# TauBench V2 eval: Gemini 3.1 Pro Preview (cloud)+# Multi-turn customer service benchmark — test split, 3 trials++[meta]+name = "taubench-gemini-31-pro"+description = "TauBench V2 on gemini-3.1-pro-preview (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-gemini-31-pro/"+seed = 42++[[models]]+name = "gemini-3.1-pro-preview"+engine = "cloud"++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-gpt54.toml b/src/openjarvis/evals/configs/taubench-gpt54.tomlnew file mode 100644index 00000000..f2b3ef7d--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-gpt54.toml@@ -0,0 +1,29 @@+# TauBench V2 eval: GPT-5.4 (cloud)+# Multi-turn customer service benchmark — test split, 3 trials++[meta]+name = "taubench-gpt54"+description = "TauBench V2 on gpt-5.4-2026-03-05 (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-gpt54/"+seed = 42++[[models]]+name = "gpt-5.4-2026-03-05"+engine = "cloud"++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-nemotron3-super.toml b/src/openjarvis/evals/configs/taubench-nemotron3-super.tomlnew file mode 100644index 00000000..2afff9a7--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml@@ -0,0 +1,29 @@+# TauBench V2 eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang)+[meta]+name = "taubench-nemotron3-super"+description = "TauBench V2 on Nemotron-3-Super-120B-A12B-FP8 (test split, pass^3)"++[defaults]+temperature = 1.0+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-nemotron3-super/"+seed = 42++[[models]]+name = "nemotron"+engine = "vllm"+num_gpus = 4+# SGLang serving on port 8001++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen122b.toml b/src/openjarvis/evals/configs/taubench-qwen122b.tomlnew file mode 100644index 00000000..e403a462--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen122b.toml@@ -0,0 +1,28 @@+# TauBench V2 eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)+[meta]+name = "taubench-qwen122b"+description = "TauBench V2 on Qwen/Qwen3.5-122B-A10B-FP8 (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-qwen122b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-122B-A10B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen35b.toml b/src/openjarvis/evals/configs/taubench-qwen35b.tomlnew file mode 100644index 00000000..cb779851--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen35b.toml@@ -0,0 +1,28 @@+# TauBench V2 eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)+[meta]+name = "taubench-qwen35b"+description = "TauBench V2 on Qwen/Qwen3.5-35B-A3B-FP8 (test split, pass^3)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-qwen35b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-35B-A3B-FP8"+engine = "vllm"+num_gpus = 4++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/configs/taubench-qwen397b.toml b/src/openjarvis/evals/configs/taubench-qwen397b.tomlnew file mode 100644index 00000000..3989aab5--- /dev/null+++ b/src/openjarvis/evals/configs/taubench-qwen397b.toml@@ -0,0 +1,30 @@+# TauBench V2 eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)+# Multi-turn customer service benchmark across airline, retail++[meta]+name = "taubench-qwen397b"+description = "TauBench V2 on Qwen/Qwen3.5-397B-A17B-FP8 (airline + retail)"++[defaults]+temperature = 0.7+max_tokens = 4096++[judge]+model = "gpt-5-mini-2025-08-07"+temperature = 0.0+engine = "cloud"++[run]+max_workers = 1+output_dir = "results/taubench-qwen397b/"+seed = 42++[[models]]+name = "Qwen/Qwen3.5-397B-A17B-FP8"+engine = "vllm"+num_gpus = 8++[[benchmarks]]+name = "taubench"+backend = "jarvis-direct"+split = "airline,retail"diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.pyindex 7067477a..20aa9e04 100644--- a/src/openjarvis/evals/core/config.py+++ b/src/openjarvis/evals/core/config.py@@ -61,6 +61,7 @@ KNOWN_BENCHMARKS = {"doc_qa","browser_assistant","pinchbench",+ "taubench",}diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.pynew file mode 100644index 00000000..9cdc4757--- /dev/null+++ b/src/openjarvis/evals/datasets/taubench.py@@ -0,0 +1,234 @@+"""TauBench V2 dataset provider — multi-turn customer service benchmark.++Wraps the tau2-bench framework for evaluation within OpenJarvis.+Supports airline, retail, and telecom domains.++Reference: https://github.com/sierra-research/tau2-bench+"""++from __future__ import annotations++import logging+import subprocess+import sys+from pathlib import Path+from typing import Iterable, List, Optional++from openjarvis.evals.core.dataset import DatasetProvider+from openjarvis.evals.core.types import EvalRecord++LOGGER = logging.getLogger(__name__)++TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git"+CACHE_DIR = Path.home() / ".cache" / "tau2-bench"++DOMAINS = ("airline", "retail", "telecom")+++def _ensure_tau2() -> None:+ """Ensure tau2 package is importable; install from cache if needed."""+ try:+ import tau2 # noqa: F401+ except ImportError:+ # Clone and install from source+ if not CACHE_DIR.exists():+ LOGGER.info("Cloning tau2-bench from %s ...", TAU2_REPO)+ CACHE_DIR.parent.mkdir(parents=True, exist_ok=True)+ subprocess.run(+ ["git", "clone", "--depth", "1", TAU2_REPO, str(CACHE_DIR)],+ check=True,+ capture_output=True,+ )+ LOGGER.info("Installing tau2-bench ...")+ subprocess.run(+ [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],+ check=True,+ capture_output=True,+ )+++class TauBenchDataset(DatasetProvider):+ """TauBench V2 multi-turn customer service benchmark.++ Wraps tau2-bench's task loading and evaluation infrastructure.+ Each EvalRecord represents a single customer service scenario.+ """++ dataset_id = "taubench"+ dataset_name = "TauBench"++ def __init__(+ self,+ domains: Optional[List[str]] = None,+ ) -> None:+ self._domains = domains or list(DOMAINS)+ self._records: List[EvalRecord] = []+ self._engine_key: Optional[str] = None+ self._model: Optional[str] = None+ self._temperature: float = 0.7+ self._max_tokens: int = 4096+ self._user_model: Optional[str] = None+ self._num_trials: int = 3 # pass^k: best of k trials per task++ def set_engine_config(+ self,+ engine_key: Optional[str] = None,+ model: Optional[str] = None,+ temperature: float = 0.7,+ max_tokens: int = 4096,+ user_model: Optional[str] = None,+ num_trials: Optional[int] = None,+ ) -> None:+ """Inject engine configuration for the agent. Called by CLI."""+ if engine_key is not None:+ self._engine_key = engine_key+ if model is not None:+ self._model = model+ self._temperature = temperature+ self._max_tokens = max_tokens+ if user_model is not None:+ self._user_model = user_model+ if num_trials is not None:+ self._num_trials = num_trials++ def verify_requirements(self) -> List[str]:+ issues: List[str] = []+ try:+ _ensure_tau2()+ except Exception as exc:+ issues.append(f"tau2-bench not available: {exc}")+ return issues++ def load(+ self,+ *,+ max_samples: Optional[int] = None,+ split: Optional[str] = None,+ seed: Optional[int] = None,+ ) -> None:+ _ensure_tau2()+ from tau2.runner import get_tasks++ # split overrides domains if provided (e.g. "airline,retail")+ domains = self._domains+ if split:+ domains = [d.strip() for d in split.split(",") if d.strip()]++ all_records: List[EvalRecord] = []++ for domain in domains:+ if domain not in DOMAINS:+ LOGGER.warning("Unknown TauBench domain: %s", domain)+ continue++ # Load tasks, filtering to test split when available+ from tau2.runner import load_task_splits+ try:+ task_splits = load_task_splits(domain)+ test_ids = (+ set(str(t) for t in task_splits.get("test", []))+ if task_splits+ else set()+ )+ except Exception:+ test_ids = set()++ tasks = get_tasks(domain)+ if test_ids:+ tasks = [t for t in tasks if str(t.id) in test_ids]+ LOGGER.info(+ "TauBench: loaded %d test-split tasks for domain '%s'",+ len(tasks), domain,+ )+ else:+ LOGGER.info(+ "TauBench: loaded %d tasks for domain '%s' (no test split)",+ len(tasks), domain,+ )++ for task in tasks:+ # Build the user's reason for calling as the problem prompt+ user_scenario = task.user_scenario+ instructions = user_scenario.instructions+ problem = (+ f"Domain: {domain}\n"+ f"Reason for call: {instructions.reason_for_call}\n"+ f"Known info: {instructions.known_info or 'None'}\n"+ )++ # Extract evaluation criteria+ eval_criteria = task.evaluation_criteria+ actions = []+ nl_assertions = []+ communicate_info = []+ reward_basis = []+ if eval_criteria:+ actions = [+ a.model_dump() if hasattr(a, "model_dump") else a+ for a in (eval_criteria.actions or [])+ ]+ nl_assertions = eval_criteria.nl_assertions or []+ communicate_info = [+ c.model_dump() if hasattr(c, "model_dump") else c+ for c in (eval_criteria.communicate_info or [])+ ]+ reward_basis = [+ r.value if hasattr(r, "value") else r+ for r in (eval_criteria.reward_basis or [])+ ]++ record = EvalRecord(+ record_id=f"{domain}_{task.id}",+ problem=problem,+ reference=task.description.purpose if task.description else "",+ category=domain,+ subject=f"taubench-{domain}",+ metadata={+ "domain": domain,+ "task_id": task.id,+ "task_instructions": instructions.task_instructions,+ "reason_for_call": instructions.reason_for_call,+ "known_info": instructions.known_info,+ "unknown_info": instructions.unknown_info,+ "actions": actions,+ "nl_assertions": nl_assertions,+ "communicate_info": communicate_info,+ "reward_basis": reward_basis,+ },+ )+ all_records.append(record)++ if seed is not None:+ import random+ random.Random(seed).shuffle(all_records)+ if max_samples is not None:+ all_records = all_records[:max_samples]++ self._records = all_records+ LOGGER.info(+ "TauBench: loaded %d total tasks across %s",+ len(self._records),+ ", ".join(self._domains),+ )++ def iter_records(self) -> Iterable[EvalRecord]:+ return iter(self._records)++ def size(self) -> int:+ return len(self._records)++ def create_task_env(self, record: EvalRecord):+ """Create a TauBench task environment for evaluation."""+ from openjarvis.evals.execution.taubench_env import TauBenchTaskEnv+ return TauBenchTaskEnv(+ record,+ engine_key=self._engine_key,+ model=self._model,+ temperature=self._temperature,+ max_tokens=self._max_tokens,+ user_model=self._user_model,+ num_trials=self._num_trials,+ )+++__all__ = ["TauBenchDataset"]diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.pynew file mode 100644index 00000000..cd9b25a8--- /dev/null+++ b/src/openjarvis/evals/execution/taubench_env.py@@ -0,0 +1,369 @@+"""TauBench task environment — native OpenJarvis agent in tau2 simulation.++Plugs OpenJarvis's inference engine into tau2-bench's orchestrator as a+``HalfDuplexAgent``, so the multi-turn conversation loop, user simulator,+domain tools, database, and evaluation all come from tau2-bench while the+agent's LLM calls go through OpenJarvis.+"""++from __future__ import annotations++import json+import logging+from types import TracebackType+from typing import Any, Optional, Type++from openjarvis.evals.core.types import EvalRecord++LOGGER = logging.getLogger(__name__)+++# ---------------------------------------------------------------------------+# Message conversion helpers+# ---------------------------------------------------------------------------++def _tau2_to_oj_messages(+ tau2_messages: list,+) -> list:+ """Convert tau2 Message objects to OpenJarvis Message objects."""+ from openjarvis.core.types import Message, Role+ from openjarvis.core.types import ToolCall as OJToolCall++ oj_msgs: list = []+ for m in tau2_messages:+ role_str = getattr(m, "role", "user")+ if role_str == "system":+ oj_msgs.append(Message(role=Role.SYSTEM, content=m.content or ""))+ elif role_str == "user":+ oj_msgs.append(Message(role=Role.USER, content=m.content or ""))+ elif role_str == "assistant":+ tc_list = getattr(m, "tool_calls", None)+ if tc_list:+ oj_tool_calls = [+ OJToolCall(+ id=tc.id,+ name=tc.name,+ arguments=(+ json.dumps(tc.arguments)+ if isinstance(tc.arguments, dict)+ else str(tc.arguments)+ ),+ )+ for tc in tc_list+ ]+ oj_msgs.append(+ Message(+ role=Role.ASSISTANT,+ content=m.content or "",+ tool_calls=oj_tool_calls,+ )+ )+ else:+ oj_msgs.append(+ Message(role=Role.ASSISTANT, content=m.content or "")+ )+ elif role_str == "tool":+ # tau2 ToolMessage uses 'id', not 'tool_call_id'+ raw_id = getattr(m, "id", "") or getattr(m, "tool_call_id", "") or ""+ import re as _re+ clean_id = _re.sub(r"[^a-zA-Z0-9_-]", "_", raw_id)+ oj_msgs.append(+ Message(+ role=Role.TOOL,+ content=m.content or "",+ tool_call_id=clean_id,+ name=getattr(m, "name", ""),+ )+ )+ return oj_msgs+++def _oj_result_to_tau2_msg(result: dict):+ """Convert OpenJarvis engine.generate() result to a tau2 AssistantMessage."""+ from tau2.data_model.message import AssistantMessage, ToolCall++ raw_tool_calls = result.get("tool_calls", [])+ tool_calls = None+ if raw_tool_calls:+ tool_calls = [+ ToolCall(+ id=tc.get("id", ""),+ name=tc.get("name", ""),+ arguments=(+ json.loads(tc["arguments"])+ if isinstance(tc.get("arguments"), str)+ else tc.get("arguments", {})+ ),+ requestor="assistant",+ )+ for tc in raw_tool_calls+ ]++ return AssistantMessage(+ role="assistant",+ content=result.get("content") or None,+ tool_calls=tool_calls,+ cost=result.get("cost_usd", 0.0),+ )+++# ---------------------------------------------------------------------------+# Jarvis-powered tau2 agent+# ---------------------------------------------------------------------------++class JarvisHalfDuplexAgent:+ """A tau2 HalfDuplexAgent backed by OpenJarvis's inference engine.++ Replaces tau2's built-in LLMAgent while keeping the same interface+ so the Orchestrator, UserSimulator, and evaluation work unchanged.+ """++ STOP = "###STOP###"++ def __init__(+ self,+ tools: list,+ domain_policy: str,+ engine: Any,+ model: str,+ temperature: float = 0.7,+ max_tokens: int = 4096,+ ) -> None:+ self.tools = tools+ self.domain_policy = domain_policy+ self._engine = engine+ self._model = model+ self._temperature = temperature+ self._max_tokens = max_tokens++ @property+ def system_prompt(self) -> str:+ from tau2.agent.llm_agent import AGENT_INSTRUCTION, SYSTEM_PROMPT+ return SYSTEM_PROMPT.format(+ domain_policy=self.domain_policy,+ agent_instruction=AGENT_INSTRUCTION,+ )++ def get_init_state(self, message_history=None):+ from tau2.agent.llm_agent import LLMAgentState+ from tau2.data_model.message import SystemMessage+ return LLMAgentState(+ system_messages=[+ SystemMessage(role="system", content=self.system_prompt),+ ],+ messages=list(message_history) if message_history else [],+ )++ def generate_next_message(self, message, state):+ from tau2.data_model.message import MultiToolMessage++ # Add incoming message to conversation state+ if isinstance(message, MultiToolMessage):+ state.messages.extend(message.tool_messages)+ elif message is not None:+ state.messages.append(message)++ # Convert to OpenJarvis format+ oj_messages = _tau2_to_oj_messages(+ state.system_messages + state.messages,+ )++ # Build OpenAI tool schemas from tau2 tools+ openai_tools = [t.openai_schema for t in self.tools]++ # Call OpenJarvis engine+ gen_kwargs: dict = {+ "model": self._model,+ "temperature": self._temperature,+ "max_tokens": self._max_tokens,+ "tools": openai_tools,+ }+ # Disable thinking mode for local models (Qwen3.5 etc.)+ # to avoid <think> tags interfering with tool call parsing+ if "qwen" in self._model.lower():+ gen_kwargs["chat_template_kwargs"] = {+ "enable_thinking": False,+ }+ result = self._engine.generate(oj_messages, **gen_kwargs)++ # Convert result to tau2 AssistantMessage+ assistant_msg = _oj_result_to_tau2_msg(result)+ state.messages.append(assistant_msg)+ return assistant_msg, state++ @classmethod+ def is_stop(cls, message) -> bool:+ if hasattr(message, "is_tool_call") and message.is_tool_call():+ return False+ content = getattr(message, "content", "") or ""+ return cls.STOP in content++ def set_seed(self, seed: int) -> None:+ pass++ def stop(self, *args, **kwargs) -> None:+ """Cleanup hook called by orchestrator."""+ pass+++# ---------------------------------------------------------------------------+# Task environment+# ---------------------------------------------------------------------------++class TauBenchTaskEnv:+ """Per-task environment for TauBench evaluation.++ Creates an OpenJarvis-powered agent, plugs it into tau2's orchestrator,+ runs the simulation, and stores results in record.metadata for the scorer.+ """++ def __init__(+ self,+ record: EvalRecord,+ engine_key: Optional[str] = None,+ model: Optional[str] = None,+ temperature: float = 0.7,+ max_tokens: int = 4096,+ user_model: Optional[str] = None,+ num_trials: int = 1,+ ) -> None:+ self._record = record+ self._num_trials = num_trials+ self._engine_key = engine_key+ self._model = model or "claude-opus-4-6"+ self._temperature = temperature+ self._max_tokens = max_tokens+ self._user_model = user_model or "gpt-5-mini-2025-08-07"+ self._system = None++ def __enter__(self) -> TauBenchTaskEnv:+ # Build OpenJarvis system for engine access+ from openjarvis.system import SystemBuilder++ builder = SystemBuilder()+ if self._engine_key:+ builder.engine(self._engine_key)+ self._system = builder.build()++ # Run the simulation+ self._run_simulation()+ return self++ def _run_simulation(self) -> None:+ from tau2.evaluator.evaluator import EvaluationType+ from tau2.orchestrator.orchestrator import Orchestrator+ from tau2.runner.build import build_environment, build_user+ from tau2.runner.helpers import get_tasks+ from tau2.runner.simulation import run_simulation++ domain = self._record.metadata["domain"]+ task_id = self._record.metadata["task_id"]++ tasks = get_tasks(domain, task_ids=[task_id])+ if not tasks:+ LOGGER.error("Task %s not found in domain %s", task_id, domain)+ self._record.metadata["tau_reward"] = 0.0+ self._record.metadata["is_resolved"] = False+ return++ task = tasks[0]+ best_reward = 0.0+ best_info: dict = {}+ best_n_messages = 0++ # Run multiple trials, keep best result (pass^k)+ for trial in range(self._num_trials):+ try:+ environment = build_environment(domain)++ agent = JarvisHalfDuplexAgent(+ tools=environment.get_tools(),+ domain_policy=environment.get_policy(),+ engine=self._system.engine,+ model=self._model,+ temperature=self._temperature,+ max_tokens=self._max_tokens,+ )++ user = build_user(+ "user_simulator",+ environment,+ task,+ llm=self._user_model,+ )++ orchestrator = Orchestrator(+ domain=domain,+ agent=agent,+ user=user,+ environment=environment,+ task=task,+ max_steps=200,+ seed=42 + trial,+ )++ simulation = run_simulation(+ orchestrator,+ evaluation_type=EvaluationType.ALL_WITH_NL_ASSERTIONS,+ )++ reward = (+ simulation.reward_info.reward+ if simulation.reward_info+ else 0.0+ )+ info: dict = {}+ if simulation.reward_info:+ info = simulation.reward_info.info or {}+ if simulation.reward_info.reward_breakdown:+ info["reward_breakdown"] = {+ k.value if hasattr(k, "value") else str(k): v+ for k, v in simulation.reward_info.reward_breakdown.items()+ }+ n_messages = len(simulation.messages) if simulation.messages else 0++ trial_label = (+ f" (trial {trial + 1}/{self._num_trials})"+ if self._num_trials > 1 else ""+ )+ LOGGER.info(+ "TauBench %s/%s: reward=%.2f messages=%d%s",+ domain, task_id, reward, n_messages, trial_label,+ )++ if reward > best_reward:+ best_reward = reward+ best_info = info+ best_n_messages = n_messages++ # Early exit if perfect score+ if reward >= 1.0:+ break++ except Exception as exc:+ LOGGER.error(+ "TauBench simulation failed for %s/%s: %s",+ domain, task_id, exc,+ )++ self._record.metadata["tau_reward"] = best_reward+ self._record.metadata["tau_info"] = best_info+ self._record.metadata["tau_n_messages"] = best_n_messages+ self._record.metadata["tau_num_trials"] = self._num_trials+ self._record.metadata["is_resolved"] = best_reward >= 0.5++ def __exit__(+ self,+ exc_type: Optional[Type[BaseException]],+ exc_val: Optional[BaseException],+ exc_tb: Optional[TracebackType],+ ) -> None:+ if self._system:+ try:+ self._system.close()+ except Exception:+ pass+ self._system = None+++__all__ = ["TauBenchTaskEnv"]diff --git a/src/openjarvis/evals/scorers/taubench.py b/src/openjarvis/evals/scorers/taubench.pynew file mode 100644index 00000000..227640de--- /dev/null+++ b/src/openjarvis/evals/scorers/taubench.py@@ -0,0 +1,46 @@+"""TauBench scorer — wraps tau2-bench's evaluation results.++Since TauBench runs its own simulation loop (agent + user simulator ++tools + evaluation), the scorer simply reads the reward that was+computed during task execution and stored in record.metadata.+"""++from __future__ import annotations++from typing import Any, Dict, Optional, Tuple++from openjarvis.evals.core.scorer import Scorer+from openjarvis.evals.core.types import EvalRecord+++class TauBenchScorer(Scorer):+ """TauBench scorer — reads pre-computed rewards from tau2-bench.++ The actual evaluation (DB state checks, action matching,+ communication checks, NL assertions) is done by tau2-bench's+ evaluator during simulation. This scorer extracts the result.+ """++ scorer_id = "taubench"++ def __init__(self, judge_backend: Any = None, judge_model: str = "") -> None:+ self._judge_backend = judge_backend+ self._judge_model = judge_model++ def score(+ self, record: EvalRecord, model_answer: str,+ ) -> Tuple[Optional[bool], Dict[str, Any]]:+ reward = record.metadata.get("tau_reward", 0.0)+ info = record.metadata.get("tau_info", {})+ n_messages = record.metadata.get("tau_n_messages", 0)++ is_correct = reward >= 0.5++ return is_correct, {+ "score": reward,+ "breakdown": info,+ "notes": f"reward={reward:.2f}, messages={n_messages}",+ }+++__all__ = ["TauBenchScorer"]diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.pyindex e569fa48..ab856d4d 100644--- a/tests/evals/test_benchmark_datasets.py+++ b/tests/evals/test_benchmark_datasets.py@@ -313,7 +313,7 @@ class TestConfigBenchmarks:def test_benchmarks_count(self) -> None:from openjarvis.evals.core.config import KNOWN_BENCHMARKS- assert len(KNOWN_BENCHMARKS) == 26+ assert len(KNOWN_BENCHMARKS) == 27# ---------------------------------------------------------------------------