Personal AI, On Personal Devices
Python
fix(evals): make TauBench dependency explicit (#739)
Elliot Sluskyc9942961adfc
diff --git a/docs/user-guide/evaluations.md b/docs/user-guide/evaluations.mdindex fe51934e..7f0587c3 100644--- a/docs/user-guide/evaluations.md+++ b/docs/user-guide/evaluations.md@@ -31,6 +31,16 @@ uv sync --extra dev --extra eval-wandb # Weights & Biases run trackinguv sync --extra dev --extra eval-sheets # Google Sheets results export```+TauBench additionally requires Python 3.12 or newer and the upstream `tau2`+package. Install the pinned revision explicitly before running that benchmark:++```bash+uv pip install "tau2 @ git+https://github.com/sierra-research/tau2-bench.git@fc0055dc4e0a316c3f83133267fbd6faaa770992"+```++OpenJarvis does not install third-party packages automatically when an+evaluation is imported or run.+!!! note "Python version requirement"Python 3.10 requires the `tomli` package for TOML config parsing. `openjarvis` declares it as a conditional dependency, so it is installed automatically.diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.pyindex e0874ed9..d56cdfeb 100644--- a/src/openjarvis/evals/datasets/taubench.py+++ b/src/openjarvis/evals/datasets/taubench.py@@ -8,13 +8,12 @@ Reference: https://github.com/sierra-research/tau2-benchfrom __future__ import annotations+import jsonimport loggingimport os-import subprocess-import sys+from importlib import metadatafrom typing import Iterable, List, Optional-from openjarvis.core.paths import get_cache_dirfrom openjarvis.evals.core.dataset import DatasetProviderfrom openjarvis.evals.core.splits import apply_splitfrom openjarvis.evals.core.types import EvalRecord@@ -22,48 +21,50 @@ from openjarvis.evals.core.types import EvalRecordLOGGER = logging.getLogger(__name__)TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git"-CACHE_DIR = get_cache_dir() / "tau2-bench"+# v1.0.1. Keep the full commit SHA here (rather than a movable tag) so every+# TauBench setup uses the same third-party code.+TAU2_REVISION = "fc0055dc4e0a316c3f83133267fbd6faaa770992"+TAU2_INSTALL_SPEC = f"tau2 @ git+{TAU2_REPO}@{TAU2_REVISION}"DOMAINS = ("airline", "retail", "telecom")def _ensure_tau2() -> None:- """Ensure tau2 package is importable; install from cache if needed."""+ """Ensure the explicitly installed, pinned tau2 package is importable."""+ try:+ distribution = metadata.distribution("tau2")+ except metadata.PackageNotFoundError as exc:+ raise ImportError(+ "TauBench requires tau2, which OpenJarvis does not install at "+ "runtime. Install the pinned dependency explicitly (Python >=3.12): "+ f'uv pip install "{TAU2_INSTALL_SPEC}"'+ ) from exc++ try:+ direct_url_text = distribution.read_text("direct_url.json")+ direct_url = json.loads(direct_url_text or "")+ vcs_info = direct_url.get("vcs_info", {})+ installed_repo = direct_url.get("url")+ installed_revision = vcs_info.get("commit_id")+ except (json.JSONDecodeError, AttributeError):+ installed_repo = None+ installed_revision = None++ if installed_repo != TAU2_REPO or installed_revision != TAU2_REVISION:+ raise ImportError(+ "The installed tau2 package does not match OpenJarvis's pinned "+ "source revision. Reinstall it explicitly (Python >=3.12): "+ f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'+ )+try:import tau2 # noqa: F401- except ImportError:- # Clone and install from source- if not CACHE_DIR.exists():- LOGGER.info("Cloning tau2-bench from %s ...", TAU2_REPO)- CACHE_DIR.parent.mkdir(parents=True, exist_ok=True)- subprocess.run(- ["git", "clone", "--depth", "1", TAU2_REPO, str(CACHE_DIR)],- check=True,- capture_output=True,- )- LOGGER.info("Installing tau2-bench ...")- # Try `python -m pip` first; fall back to `uv pip` for uv-managed venvs- # which don't ship pip by default.- try:- subprocess.run(- [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)],- check=True,- capture_output=True,- )- except (subprocess.CalledProcessError, FileNotFoundError):- subprocess.run(- [- "uv",- "pip",- "install",- "--python",- sys.executable,- "-e",- str(CACHE_DIR),- ],- check=True,- capture_output=True,- )+ except ImportError as exc:+ raise ImportError(+ "The pinned tau2 package is installed but cannot be imported. "+ "Reinstall it explicitly (Python >=3.12): "+ f'uv pip install --force-reinstall "{TAU2_INSTALL_SPEC}"'+ ) from excclass TauBenchDataset(DatasetProvider):diff --git a/tests/evals/datasets/test_taubench.py b/tests/evals/datasets/test_taubench.pynew file mode 100644index 00000000..b54e074f--- /dev/null+++ b/tests/evals/datasets/test_taubench.py@@ -0,0 +1,98 @@+"""Tests for the TauBench optional dependency boundary."""++from __future__ import annotations++import builtins+import sys+from types import ModuleType+from unittest.mock import Mock++import pytest++from openjarvis.evals.datasets import taubench+++def _mock_direct_url(monkeypatch, direct_url):+ distribution = Mock()+ distribution.read_text.return_value = direct_url+ monkeypatch.setattr(+ taubench.metadata, "distribution", Mock(return_value=distribution)+ )+++def test_ensure_tau2_accepts_the_pinned_source_revision(monkeypatch):+ monkeypatch.setitem(sys.modules, "tau2", ModuleType("tau2"))+ _mock_direct_url(+ monkeypatch,+ (+ '{"url": "https://github.com/sierra-research/tau2-bench.git", '+ '"vcs_info": {"vcs": "git", '+ f'"commit_id": "{taubench.TAU2_REVISION}"}}}}'+ ),+ )++ taubench._ensure_tau2()+++def test_ensure_tau2_requires_explicit_pinned_install(monkeypatch):+ monkeypatch.setitem(sys.modules, "tau2", None)+ monkeypatch.setattr(+ taubench.metadata,+ "distribution",+ Mock(side_effect=taubench.metadata.PackageNotFoundError),+ )++ with pytest.raises(ImportError) as exc_info:+ taubench._ensure_tau2()++ message = str(exc_info.value)+ assert "does not install at runtime" in message+ assert taubench.TAU2_REVISION in message+ assert "uv pip install" in message+++ "direct_url",+ [+ # Editable install left behind by the previous runtime installer.+ '{"url": "file:///home/user/.openjarvis/cache/tau2-bench", '+ '"dir_info": {"editable": true}}',+ # A git install from an arbitrary upstream revision.+ '{"url": "https://github.com/sierra-research/tau2-bench.git", '+ '"vcs_info": {"vcs": "git", "commit_id": "deadbeef"}}',+ # Registry installs do not carry PEP 610 direct-origin metadata.+ None,+ ],+)+def test_ensure_tau2_rejects_unpinned_install(monkeypatch, direct_url):+ _mock_direct_url(monkeypatch, direct_url)+ original_import = builtins.__import__++ def guarded_import(name, *args, **kwargs):+ if name == "tau2":+ raise AssertionError("unverified tau2 package was imported")+ return original_import(name, *args, **kwargs)++ monkeypatch.setattr(builtins, "__import__", guarded_import)++ with pytest.raises(ImportError) as exc_info:+ taubench._ensure_tau2()++ message = str(exc_info.value)+ assert "does not match" in message+ assert taubench.TAU2_REVISION in message+ assert "--force-reinstall" in message+++def test_verify_requirements_reports_install_instruction(monkeypatch):+ monkeypatch.setitem(sys.modules, "tau2", None)+ monkeypatch.setattr(+ taubench.metadata,+ "distribution",+ Mock(side_effect=taubench.metadata.PackageNotFoundError),+ )++ issues = taubench.TauBenchDataset().verify_requirements()++ assert len(issues) == 1+ assert taubench.TAU2_REVISION in issues[0]