Skip to content

Personal AI, On Personal Devices

Python

fix(evals): make TauBench dependency explicit (#739)

Elliot Sluskyc9942961adfc
diff --git a/docs/user-guide/evaluations.md b/docs/user-guide/evaluations.md
index fe51934e..7f0587c3 100644
--- a/docs/user-guide/evaluations.md
+++ b/docs/user-guide/evaluations.md
@@ -31,6 +31,16 @@ uv sync --extra dev --extra eval-wandb # Weights & Biases run tracking
uv sync --extra dev --extra eval-sheets # Google Sheets results export
```
+TauBench additionally requires Python 3.12 or newer and the upstream `tau2`
+package. Install the pinned revision explicitly before running that benchmark:
+
+```bash
+uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@fc0055dc4e0a316c3f83133267fbd6faaa770992"
+```
+
+OpenJarvis does not install third-party packages automatically when an
+evaluation is imported or run.
+
!!! note "Python version requirement"
Python 3.10 requires the `tomli` package for TOML config parsing. `openjarvis` declares it as a conditional dependency, so it is installed automatically.
diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.py
index e0874ed9..d56cdfeb 100644
--- a/src/openjarvis/evals/datasets/taubench.py
+++ b/src/openjarvis/evals/datasets/taubench.py
@@ -8,13 +8,12 @@ Reference: https://github.com/sierra-research/tau2-bench
from __future__ import annotations
+import json
import logging
import os
-import subprocess
-import sys
+from importlib import metadata
from typing import Iterable, List, Optional
-from openjarvis.core.paths import get_cache_dir
from openjarvis.evals.core.dataset import DatasetProvider
from openjarvis.evals.core.splits import apply_split
from openjarvis.evals.core.types import EvalRecord
@@ -22,48 +21,50 @@ from openjarvis.evals.core.types import EvalRecord
LOGGER = logging.getLogger(__name__)
TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git"
-CACHE_DIR = get_cache_dir() / "tau2-bench"
+# v1.0.1. Keep the full commit SHA here (rather than a movable tag) so every
+# TauBench setup uses the same third-party code.
+TAU2_REVISION = "fc0055dc4e0a316c3f83133267fbd6faaa770992"
+TAU2_INSTALL_SPEC = f"tau2 @ git+{TAU2_REPO}@{TAU2_REVISION}"
DOMAINS = ("airline", "retail", "telecom")
def _ensure_tau2() -> None:
- """Ensure tau2 package is importable; install from cache if needed."""
+ """Ensure the explicitly installed, pinned tau2 package is importable."""
+ try:
+ distribution = metadata.distribution("tau2")
+ except metadata.PackageNotFoundError as exc:
+ raise ImportError(
+ "TauBench requires tau2, which OpenJarvis does not install at "
+ "runtime. Install the pinned dependency explicitly (Python >=3.12): "
+ f'uv pip install "{TAU2_INSTALL_SPEC}"'
+ ) from exc
+
+ try:
+ direct_url_text = distribution.read_text("direct_url.json")
+ direct_url = json.loads(direct_url_text or "")
+ vcs_info = direct_url.get("vcs_info", {})
+ installed_repo = direct_url.get("url")
+ installed_revision = vcs_info.get("commit_id")
+ except (json.JSONDecodeError, AttributeError):
+ installed_repo = None
+ installed_revision = None
+
+ if installed_repo != TAU2_REPO or installed_revision != TAU2_REVISION:
+ raise ImportError(
+ "The installed tau2 package does not match OpenJarvis's pinned "
+ "source revision. Reinstall it explicitly (Python >=3.12): "
+ f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'
+ )
+
try:
import tau2 # noqa: F401
- except ImportError:
- # Clone and install from source
- if not CACHE_DIR.exists():
- LOGGER.info("Cloning tau2-bench from %s ...", TAU2_REPO)
- CACHE_DIR.parent.mkdir(parents=True, exist_ok=True)
- subprocess.run(
- ["git", "clone", "--depth", "1", TAU2_REPO, str(CACHE_DIR)],
- check=True,
- capture_output=True,
- )
- LOGGER.info("Installing tau2-bench ...")
- # Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs
- # which don't ship pip by default.
- try:
- subprocess.run(
- [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],
- check=True,
- capture_output=True,
- )
- except (subprocess.CalledProcessError, FileNotFoundError):
- subprocess.run(
- [
- "uv",
- "pip",
- "install",
- "--python",
- sys.executable,
- "-e",
- str(CACHE_DIR),
- ],
- check=True,
- capture_output=True,
- )
+ except ImportError as exc:
+ raise ImportError(
+ "The pinned tau2 package is installed but cannot be imported. "
+ "Reinstall it explicitly (Python >=3.12): "
+ f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'
+ ) from exc
class TauBenchDataset(DatasetProvider):
diff --git a/tests/evals/datasets/test_taubench.py b/tests/evals/datasets/test_taubench.py
new file mode 100644
index 00000000..b54e074f
--- /dev/null
+++ b/tests/evals/datasets/test_taubench.py
@@ -0,0 +1,98 @@
+"""Tests for the TauBench optional dependency boundary."""
+
+from __future__ import annotations
+
+import builtins
+import sys
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+
+from openjarvis.evals.datasets import taubench
+
+
+def _mock_direct_url(monkeypatch, direct_url):
+ distribution = Mock()
+ distribution.read_text.return_value = direct_url
+ monkeypatch.setattr(
+ taubench.metadata, "distribution", Mock(return_value=distribution)
+ )
+
+
+def test_ensure_tau2_accepts_the_pinned_source_revision(monkeypatch):
+ monkeypatch.setitem(sys.modules, "tau2", ModuleType("tau2"))
+ _mock_direct_url(
+ monkeypatch,
+ (
+ '{"url": "https://github.com/sierra-research/tau2-bench.git", '
+ '"vcs_info": {"vcs": "git", '
+ f'"commit_id": "{taubench.TAU2_REVISION}"}}}}'
+ ),
+ )
+
+ taubench._ensure_tau2()
+
+
+def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch):
+ monkeypatch.setitem(sys.modules, "tau2", None)
+ monkeypatch.setattr(
+ taubench.metadata,
+ "distribution",
+ Mock(side_effect=taubench.metadata.PackageNotFoundError),
+ )
+
+ with pytest.raises(ImportError) as exc_info:
+ taubench._ensure_tau2()
+
+ message = str(exc_info.value)
+ assert "does not install at runtime" in message
+ assert taubench.TAU2_REVISION in message
+ assert "uv pip install" in message
+
+
+ "direct_url",
+ [
+ # Editable install left behind by the previous runtime installer.
+ '{"url": "file:///home/user/.openjarvis/cache/tau2-bench", '
+ '"dir_info": {"editable": true}}',
+ # A git install from an arbitrary upstream revision.
+ '{"url": "https://github.com/sierra-research/tau2-bench.git", '
+ '"vcs_info": {"vcs": "git", "commit_id": "deadbeef"}}',
+ # Registry installs do not carry PEP 610 direct-origin metadata.
+ None,
+ ],
+)
+def test_ensure_tau2_rejects_unpinned_install(monkeypatch, direct_url):
+ _mock_direct_url(monkeypatch, direct_url)
+ original_import = builtins.__import__
+
+ def guarded_import(name, *args, **kwargs):
+ if name == "tau2":
+ raise AssertionError("unverified tau2 package was imported")
+ return original_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", guarded_import)
+
+ with pytest.raises(ImportError) as exc_info:
+ taubench._ensure_tau2()
+
+ message = str(exc_info.value)
+ assert "does not match" in message
+ assert taubench.TAU2_REVISION in message
+ assert "--force-reinstall" in message
+
+
+def test_verify_requirements_reports_install_instruction(monkeypatch):
+ monkeypatch.setitem(sys.modules, "tau2", None)
+ monkeypatch.setattr(
+ taubench.metadata,
+ "distribution",
+ Mock(side_effect=taubench.metadata.PackageNotFoundError),
+ )
+
+ issues = taubench.TauBenchDataset().verify_requirements()
+
+ assert len(issues) == 1
+ assert taubench.TAU2_REVISION in issues[0]