The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Multiplexed gateway: module-level caches in tools/ and agent/ must not hand profile A's
config/.env-derived value to profile B. Real temp homes, real config.yaml/.env, real modules;
only HTTP transports are stubbed.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
import hermes_yaml as yaml
from agent.secret_scope import build_profile_secret_scope, reset_secret_scope, set_secret_scope
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
# Whether the main model sees images natively is a models.dev catalog verdict. A fresh home has no
# catalog, so without this the vision routing below fetched models.dev live, and a slow or failed
# fetch read as "unknown" -> aux vision (CI: `assert True is False` in profile A).
_CATALOG = {"openai": {"id": "openai", "models": {"gpt-4o": {
"id": "gpt-4o", "tool_call": True, "attachment": True, "modalities": {"input": ["text", "image"]}}}}}
def _make_home(root: Path, cfg: dict, env: str = "") -> Path:
root.mkdir(parents=True, exist_ok=True)
(root / "config.yaml").write_text(yaml.safe_dump(cfg), encoding="utf-8")
(root / ".env").write_text(env, encoding="utf-8")
(root / "cache").mkdir(exist_ok=True)
(root / "models_dev_cache.json").write_text(json.dumps(_CATALOG), encoding="utf-8")
return root
class _scoped:
def __init__(self, home: Path):
self.home = home
def __enter__(self):
self._t1 = set_hermes_home_override(str(self.home))
self._t2 = set_secret_scope(build_profile_secret_scope(self.home))
def __exit__(self, *_):
reset_secret_scope(self._t2)
reset_hermes_home_override(self._t1)
@pytest.fixture
def two_homes(tmp_path, monkeypatch):
a = _make_home(tmp_path / "A", {})
b = _make_home(tmp_path / "A" / "profiles" / "B", {})
monkeypatch.setenv("HERMES_HOME", str(a))
return a, b
def test_home_keyed_caches_serve_each_profile_its_own_config(tmp_path, monkeypatch):
"""One mechanism (dict keyed by home / override bypass) across the sites that read per-profile
config or per-home files: aux-vision routing, learned image cost table, aux
semaphore, MCP lock."""
main = {"model": {"provider": "openai", "model": "gpt-4o"}}
a = _make_home(tmp_path / "A", {**main,
"auxiliary": {"vision": {"provider": "auto"}, "summary": {"max_concurrency": 2}}})
b = _make_home(tmp_path / "A" / "profiles" / "B", {**main,
"auxiliary": {"vision": {"provider": "openai", "model": "gpt-4o-mini"},
"summary": {"max_concurrency": 7}}})
monkeypatch.setenv("HERMES_HOME", str(a))
(a / "cache" / "image_token_costs.json").write_text(json.dumps({"[email protected]": 1000}), encoding="utf-8")
(b / "cache" / "image_token_costs.json").write_text(json.dumps({"[email protected]": 3000}), encoding="utf-8")
import agent.auxiliary_client as ac
import agent.image_token_cost as itc
from agent import models_dev
import tools.computer_use.tool as cu
from tools import mcp_tool_loop
ac._reset_aux_semaphores()
# The catalog memo is process-wide: start cold so the seeded disk catalog is what answers.
monkeypatch.setattr(models_dev, "_models_dev_cache", {})
monkeypatch.setattr(models_dev, "_models_dev_cache_time", 0)
with _scoped(a):
assert cu._should_route_through_aux_vision() is False # no explicit aux vision: native path
assert itc.learned_image_token_cost("m", "http://gw.example/v1") == 1000
sem_a = ac._acquire_sync_aux_semaphore("summary")
sem_a.acquire()
cookie = mcp_tool_loop._try_acquire_mcp_discovery_lock()
lock_a = cookie._fh.name
cookie.release()
with _scoped(b):
assert cu._should_route_through_aux_vision() is True # B named a dedicated vision model
assert itc.learned_image_token_cost("m", "http://gw.example/v1") == 3000
sem_b = ac._acquire_sync_aux_semaphore("summary")
cookie = mcp_tool_loop._try_acquire_mcp_discovery_lock()
lock_b = cookie._fh.name
cookie.release()
assert Path(lock_a).parent == a and Path(lock_b).parent == b
assert sem_b is not sem_a
with _scoped(a):
# B's differently-sized lookup must not have rebuilt the semaphore A is holding.
assert ac._acquire_sync_aux_semaphore("summary") is sem_a
sem_a.release()
def test_endpoint_model_catalog_memo_is_keyed_by_credential(two_homes, monkeypatch):
"""Two profiles, same base_url, different api_key: a per-key gateway's catalog fetched with A's
key must not be served to B from the in-memory memo (the disk memo already lives per home)."""
from contextlib import contextmanager
import httpx
import agent.model_metadata as mm
a, b = two_homes
mm._endpoint_model_metadata_cache.clear()
mm._endpoint_model_metadata_cache_time.clear()
@contextmanager
def stream(url, headers=None, **kwargs):
who = headers.get("Authorization", "").rsplit("-", 1)[-1]
yield httpx.Response(200, request=httpx.Request("GET", url),
json={"data": [{"id": f"model-for-{who}", "context_length": 1}]})
monkeypatch.setattr(mm.model_metadata_http, "stream", stream)
with _scoped(a):
assert set(mm.fetch_endpoint_model_metadata("http://gw.example/v1", api_key="key-A")) == {"model-for-A"}
with _scoped(b):
assert set(mm.fetch_endpoint_model_metadata("http://gw.example/v1", api_key="key-B")) == {"model-for-B"}
with _scoped(a):
assert set(mm.fetch_endpoint_model_metadata("http://gw.example/v1", api_key="key-A")) == {"model-for-A"}