The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Tests for Modal sandbox infrastructure fixes (TBLite baseline).
Covers the bugs discovered while setting up TBLite evaluation:
1. Tool resolution — terminal + file tools load correctly
2. CWD fix — host paths get replaced with /root for container backends
3. ephemeral_disk version check
4. ensurepip fix in Modal image builder
5. No swe-rex dependency — uses native Modal SDK
6. /home/ added to host prefix check
7. Vercel sandbox cwd normalization
"""
import os
import sys
from pathlib import Path
import pytest
from tools import approval_context
# Ensure repo root is importable
_repo_root = Path(__file__).resolve().parent.parent.parent
if str(_repo_root) not in sys.path:
sys.path.insert(0, str(_repo_root))
try:
import tools.terminal_tool
_tt_mod = sys.modules["tools.terminal_tool"]
except ImportError:
pytest.skip("hermes-agent tools not importable (missing deps)", allow_module_level=True)
# =========================================================================
# Test 1: Tool resolution includes terminal + file tools
# =========================================================================
class TestToolResolution:
"""Verify get_tool_definitions returns all expected tools for eval."""
def test_terminal_tool_present(self):
"""The terminal tool must be present (not silently dropped)."""
from model_tools import get_tool_definitions
tools = get_tool_definitions(
enabled_toolsets=["terminal", "file"],
quiet_mode=True,
)
names = [t["function"]["name"] for t in tools]
assert "terminal" in names, f"terminal tool missing! Only got: {names}."
# =========================================================================
# Test 2-4: CWD handling for container backends
# =========================================================================
class TestCwdHandling:
"""Verify host paths are sanitized for container backends."""
def test_home_path_replaced_for_modal(self, monkeypatch):
"""TERMINAL_CWD=/home/user/... should be replaced with /root for modal."""
monkeypatch.setenv("TERMINAL_ENV", "modal")
monkeypatch.setenv("TERMINAL_CWD", "/home/dakota/github/hermes-agent")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/root", (
f"Expected /root, got {config['cwd']}. "
"/home/ paths should be replaced for modal backend."
)
def test_users_path_replaced_for_docker_by_default(self, monkeypatch):
"""Docker should keep host paths out of the sandbox unless explicitly enabled."""
monkeypatch.setenv("TERMINAL_ENV", "docker")
monkeypatch.setenv("TERMINAL_CWD", "/Users/someone/projects")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/root", (
f"Expected /root, got {config['cwd']}. "
"Host paths should be discarded for docker backend by default."
)
assert config["host_cwd"] is None
assert config["docker_mount_cwd_to_workspace"] is False
@pytest.mark.platforms("linux")
def test_users_path_maps_to_workspace_for_docker_when_enabled(self, monkeypatch):
"""Docker should map the host cwd into /workspace only when explicitly enabled."""
monkeypatch.setenv("TERMINAL_ENV", "docker")
monkeypatch.setenv("TERMINAL_CWD", "/Users/someone/projects")
monkeypatch.setenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "true")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/workspace"
assert config["host_cwd"] == "/Users/someone/projects"
assert config["docker_mount_cwd_to_workspace"] is True
def test_windows_path_replaced_for_modal(self, monkeypatch):
"""TERMINAL_CWD=C:\\Users\\... should be replaced for modal."""
monkeypatch.setenv("TERMINAL_ENV", "modal")
monkeypatch.setenv("TERMINAL_CWD", "C:\\Users\\someone\\projects")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/root"
def test_host_path_replaced_for_vercel_sandbox(self, monkeypatch):
"""Host paths should be discarded for Vercel Sandbox."""
monkeypatch.setenv("TERMINAL_ENV", "vercel_sandbox")
monkeypatch.setenv("TERMINAL_CWD", "/Users/someone/projects")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/vercel/sandbox"
def test_relative_path_replaced_for_vercel_sandbox(self, monkeypatch):
"""Relative cwd should not map into a remote Vercel sandbox."""
monkeypatch.setenv("TERMINAL_ENV", "vercel_sandbox")
monkeypatch.setenv("TERMINAL_CWD", "src")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/vercel/sandbox"
def test_default_cwd_is_workspace_root_for_vercel_sandbox(self, monkeypatch):
monkeypatch.setenv("TERMINAL_ENV", "vercel_sandbox")
monkeypatch.delenv("TERMINAL_CWD", raising=False)
config = _tt_mod._get_env_config()
assert config["cwd"] == "/vercel/sandbox"
@pytest.mark.parametrize("backend", ["modal", "docker", "daytona"])
def test_default_cwd_is_root_for_container_backends(self, backend, monkeypatch):
"""Container backends should default to /root, not ~."""
monkeypatch.setenv("TERMINAL_ENV", backend)
monkeypatch.delenv("TERMINAL_CWD", raising=False)
monkeypatch.delenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", raising=False)
config = _tt_mod._get_env_config()
assert config["cwd"] == "/root", (
f"Backend {backend}: expected /root default, got {config['cwd']}"
)
@pytest.mark.platforms("linux")
def test_docker_default_cwd_maps_current_directory_when_enabled(self, monkeypatch):
"""Docker should use /workspace when cwd mounting is explicitly enabled."""
monkeypatch.setattr("tools.terminal_tool.os.getcwd", lambda: "/home/user/project")
monkeypatch.setenv("TERMINAL_ENV", "docker")
monkeypatch.setenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "true")
monkeypatch.delenv("TERMINAL_CWD", raising=False)
config = _tt_mod._get_env_config()
assert config["cwd"] == "/workspace"
assert config["host_cwd"] == "/home/user/project"
def test_local_backend_uses_getcwd(self, monkeypatch):
"""Local backend should use os.getcwd(), not /root."""
monkeypatch.setenv("TERMINAL_ENV", "local")
monkeypatch.delenv("TERMINAL_CWD", raising=False)
config = _tt_mod._get_env_config()
assert config["cwd"] == os.getcwd()
def test_create_environment_passes_docker_host_cwd_and_flag(self, monkeypatch):
"""Docker host cwd and mount flag should reach DockerEnvironment."""
captured = {}
sentinel = object()
def _fake_docker_environment(**kwargs):
captured.update(kwargs)
return sentinel
from tools.terminal_tool_backends import _create_environment
monkeypatch.setattr("tools.terminal_tool_backends._DockerEnvironment", _fake_docker_environment)
env = _create_environment(
env_type="docker",
image="python:3.11",
cwd="/workspace",
timeout=60,
container_config={"docker_mount_cwd_to_workspace": True},
host_cwd="/home/user/project",
)
assert env is sentinel
assert captured["cwd"] == "/workspace"
assert captured["host_cwd"] == "/home/user/project"
assert captured["auto_mount_cwd"] is True
def test_ssh_preserves_home_paths(self, monkeypatch):
"""SSH backend should NOT replace /home/ paths (they're valid remotely)."""
monkeypatch.setenv("TERMINAL_ENV", "ssh")
monkeypatch.setenv("TERMINAL_CWD", "/home/remote-user/work")
monkeypatch.setenv("TERMINAL_SSH_HOST", "example.com")
monkeypatch.setenv("TERMINAL_SSH_USER", "user")
config = _tt_mod._get_env_config()
assert config["cwd"] == "/home/remote-user/work", (
"SSH backend should preserve /home/ paths"
)
# =========================================================================
# Test 5: ephemeral_disk version check
# =========================================================================
# =========================================================================
# Test 6: ModalEnvironment defaults
# =========================================================================
# =========================================================================
# Test 7: ensurepip fix in ModalEnvironment
# =========================================================================
# =========================================================================
# Test 8: Host prefix list completeness
# =========================================================================
class TestHostPrefixList:
"""Verify the host prefix list catches common host-only paths.
The prefixes used to live as an inline literal inside ``_get_env_config``;
they now live in the module-level ``_HOST_CWD_PREFIXES`` constant shared by
both the ``_get_env_config`` sanitizer and the override-resolution guard
(``_is_unusable_container_cwd``). Assert the *behavior* (each common host
prefix is flagged as unusable inside a container) rather than grepping a
function's source — the latter is a change-detector that breaks on any
refactor that moves the constant.
"""
def test_all_common_host_paths_flagged_unusable(self):
"""A host path under each user root must be rejected as a container cwd; in-sandbox
absolute paths pass."""
for host_path in ("/Users/me/proj", "/home/me/proj", "C:\\Users\\me", "C:/Users/me"):
assert _tt_mod._is_unusable_container_cwd(host_path) is True, (
f"Host path {host_path!r} should be rejected as a container "
"cwd but was accepted."
)
for sandbox_path in ("/workspace", "/root/proj", "/srv/app"):
assert _tt_mod._is_unusable_container_cwd(sandbox_path) is False
def test_any_windows_drive_letter_is_a_host_cwd(self):
"""The host-shape predicate is platform-independent data: every drive letter, either slash,
is a host path (#60962). On POSIX ``D:\\proj`` is also non-absolute so the container guard
already rejected it; on a Windows host it IS absolute and only this predicate catches it."""
from tools.terminal_tool_config import _is_host_cwd
for host_path in ("C:\\Users\\me", "D:\\proj", "e:/work", "Z:\\"):
assert _is_host_cwd(host_path) is True, host_path
for not_host in ("/workspace", "/srv/app", "relative/dir", "C", "C:"):
assert _is_host_cwd(not_host) is False, not_host
# =========================================================================
# Test 7: Host-bound Docker sandboxes must not bypass dangerous-command
# approval. Isolated Docker keeps the container fast-path; once a host path
# is bind-mounted into the container, a command like `rm -rf /workspace` can
# reach real host files, so it goes through the normal approval flow.
# (PR #6436, @Kolektori)
# =========================================================================
class TestDockerHostBindApproval:
"""Docker host bind mounts disable the container approval fast-path."""
def test_docker_host_access_detection(self):
"""_docker_has_host_access flags bind-mounted host paths only."""
# Isolated docker (no host binds) -> not host access.
assert _tt_mod._docker_has_host_access(
{"env_type": "docker", "docker_volumes": [],
"host_cwd": None, "docker_mount_cwd_to_workspace": False}) is False
# Host-path bind mount -> host access.
assert _tt_mod._docker_has_host_access(
{"env_type": "docker", "docker_volumes": ["/tmp:/hosttmp"]}) is True
# Named volume (not a host path) -> not host access.
assert _tt_mod._docker_has_host_access(
{"env_type": "docker", "docker_volumes": ["myvol:/data"]}) is False
# cwd auto-mount flag -> host access.
assert _tt_mod._docker_has_host_access(
{"env_type": "docker", "host_cwd": "/home/u/p",
"docker_mount_cwd_to_workspace": True}) is True
# Windows host path -> host access.
assert _tt_mod._docker_has_host_access(
{"env_type": "docker", "docker_volumes": ["C:\\Users:/data"]}) is True
# Other container backends never report host access.
assert _tt_mod._docker_has_host_access(
{"env_type": "modal", "docker_volumes": ["/tmp:/x"]}) is False
def test_should_skip_container_guards(self):
"""Docker skips only when isolated; other sandboxes always skip."""
import tools.approval as A
assert A._should_skip_container_guards("docker", has_host_access=False) is True
assert A._should_skip_container_guards("docker", has_host_access=True) is False
assert A._should_skip_container_guards("modal", has_host_access=True) is True
assert A._should_skip_container_guards("daytona") is True
assert A._should_skip_container_guards("local") is False
def test_raising_registry_lookup_keeps_container_guards_on(self, monkeypatch):
"""A registry that raises during the provider lookup must fail soft to guards-on,
not propagate out of the approval predicate."""
from agent import terminal_env_registry as R
import tools.approval as A
def boom(*_a, **_k):
raise RuntimeError("registry down")
monkeypatch.setattr(R._registry, "get_provider", boom)
assert A._should_skip_container_guards("p_disposable") is False
def test_registered_disposable_plugin_skips_container_guards(self):
"""Plugin classification uses its registered provider, not built-in names only."""
from agent import terminal_env_registry
from agent.terminal_env_provider import TerminalEnvironmentProvider
import tools.approval as A
class DisposablePlugin(TerminalEnvironmentProvider):
name = "approval_disposable_plugin"
display_name = "Approval disposable plugin"
def is_available(self):
return True
def create_environment(self, **kwargs):
raise NotImplementedError
provider = DisposablePlugin()
previous = terminal_env_registry.get_provider(provider.name)
terminal_env_registry.register_provider(provider)
try:
assert A._should_skip_container_guards(provider.name) is True
assert A._should_skip_container_guards("unknown_plugin_backend") is False
finally:
terminal_env_registry.restore_registration(provider.name, provider, previous)
class BrokenDisposablePlugin(DisposablePlugin):
name = "broken_approval_disposable_plugin"
@property
def skip_container_guards(self):
raise RuntimeError("broken plugin classification")
broken_provider = BrokenDisposablePlugin()
terminal_env_registry.register_provider(broken_provider)
try:
assert A._should_skip_container_guards(broken_provider.name) is False
finally:
terminal_env_registry.restore_registration(broken_provider.name, broken_provider, None)
def test_isolated_docker_keeps_fast_path(self, monkeypatch):
"""Isolated Docker still bypasses dangerous-command approval."""
import tools.approval as A
self._isolate_approval_state(monkeypatch)
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
res = A.check_all_command_guards("rm -rf /workspace", "docker",
has_host_access=False)
assert res["approved"] is True
@staticmethod
def _isolate_approval_state(monkeypatch):
"""Clear approval state that leaks in from the real user config.
``tools.approval`` loads ``command_allowlist`` into module-level
``_permanent_approved`` at import time. This file imports
``tools.terminal_tool`` at module level (collection time — BEFORE the
hermetic HERMES_HOME fixture runs), so on a dev machine whose real
config permanently allowlists e.g. "delete in root path" the guard
under test silently approves and the assertions flip. CI never has
such an allowlist, making this a local-only flake.
Same import-time freeze applies to ``_YOLO_MODE_FROZEN``: it reads
HERMES_YOLO_MODE off the environment when the module is imported at
collection time, before conftest's per-test env blanking runs. A test
run launched from a --yolo Hermes session (or any shell exporting
HERMES_YOLO_MODE=1) freezes True and every guard auto-approves.
Reset it explicitly so the tests exercise the guard, not the bypass.
"""
import tools.approval as A
monkeypatch.setattr(A, "_permanent_approved", set())
monkeypatch.setattr(A, "_session_approved", {})
monkeypatch.setattr(A, "_YOLO_MODE_FROZEN", False)
monkeypatch.setattr(approval_context, "_get_approval_mode", lambda: "manual")
def test_host_bound_docker_requires_approval(self, monkeypatch):
"""Host-bound Docker dangerous command escalates instead of bypassing."""
import tools.approval as A
self._isolate_approval_state(monkeypatch)
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
res = A.check_all_command_guards("rm -rf /workspace", "docker",
has_host_access=True)
# Must NOT take the silent container fast-path.
assert res.get("approved") is not True
assert res.get("status") == "pending_approval"
def test_execute_code_isolated_docker_keeps_fast_path(self, monkeypatch):
"""Isolated Docker execute_code still bypasses the guard."""
import tools.approval as A
self._isolate_approval_state(monkeypatch)
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
res = A.check_execute_code_guard("import os", "docker",
has_host_access=False)
assert res["approved"] is True
def test_execute_code_host_bound_docker_requires_approval(self, monkeypatch):
"""Host-bound Docker execute_code does not get the container fast-path."""
import tools.approval as A
self._isolate_approval_state(monkeypatch)
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
res = A.check_execute_code_guard(
"import os; os.system('rm -rf /workspace')", "docker",
has_host_access=True)
assert res.get("approved") is not True
assert res.get("status") == "pending_approval"
def test_execute_code_vercel_sandbox_always_skips(self, monkeypatch):
"""vercel_sandbox has no host-bind concept and stays always-skipped."""
import tools.approval as A
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
res = A.check_execute_code_guard("import os", "vercel_sandbox",
has_host_access=True)
assert res["approved"] is True