The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Tool-surface routing matrix: every (provider, model, modality) combo.
This is the integration test for the question Teknium asked: regardless
of which provider+model the user picks and whether they pass an
image_url or not, does the tool surface route correctly to the right
endpoint with the right payload shape?
Drives ``_handle_video_generate(args)`` end-to-end — config write →
config read → registry lookup → provider.generate() → outbound HTTP/SDK
call. Stubs fal_client and httpx so we observe routing without hitting
the network.
"""
from __future__ import annotations
import asyncio
import json
import types
from typing import Any, Dict, List
import pytest
import hermes_yaml as yaml
@pytest.fixture(autouse=True)
def _reset_registry():
from agent import video_gen_registry
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
@pytest.fixture
def matrix_env(tmp_path, monkeypatch):
"""Set up HERMES_HOME, stub fal_client + httpx, force plugin discovery."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setenv("FAL_KEY", "test-key")
monkeypatch.setenv("XAI_API_KEY", "test-key")
# This matrix supplies its own SDK fake; lazy installation is neither
# required nor permitted by the hermetic test runner.
monkeypatch.setattr("tools.lazy_deps.ensure", lambda *args, **kwargs: None)
fal_calls: list[dict[str, Any]] = []
xai_calls: list[dict[str, Any]] = []
# fal_client stub
fake_fal = types.ModuleType("fal_client")
def _subscribe(endpoint, arguments=None, with_logs=False):
fal_calls.append({"endpoint": endpoint, "arguments": arguments})
return {"video": {"url": f"https://fake-fal/{endpoint.replace('/','_')}.mp4"}}
fake_fal.subscribe = _subscribe # type: ignore
class _FalHandle:
def iter_events(self, **_):
return iter(())
def __init__(self, result):
self._result = result
def get(self):
return self._result
def _submit(endpoint, arguments=None, headers=None):
fal_calls.append({"endpoint": endpoint, "arguments": arguments})
return _FalHandle({"video": {"url": f"https://fake-fal/{endpoint.replace('/','_')}.mp4"}})
fake_fal.submit = _submit # type: ignore
monkeypatch.setitem(__import__("sys").modules, "fal_client", fake_fal)
# httpx stub for xAI
import httpx
class _Resp:
def __init__(self, p, s=200):
self.status_code = s
self._p = p
self.text = json.dumps(p)
def raise_for_status(self):
if self.status_code >= 400:
raise httpx.HTTPStatusError("err", request=None, response=self) # type: ignore
def json(self):
return self._p
class _Client:
async def __aenter__(self): return self
async def __aexit__(self, *a): return None
async def post(self, url, headers=None, json=None, timeout=None):
xai_calls.append({"url": url, "json": json})
return _Resp({"request_id": "req-1"})
async def get(self, url, headers=None, timeout=None):
payload = xai_calls[-1]["json"]
storage_options = payload.get("storage_options") or {}
return _Resp({
"status": "done",
"video": {
"url": "https://xai-cdn/out.mp4",
"duration": 8,
"file_output": {
"file_id": "file-123",
"filename": storage_options.get("filename", "out.mp4"),
"public_url": "https://xai-files.example/out.mp4",
"public_url_expires_at": 1234567890,
},
},
"model": payload.get("model", "grok-imagine-video"),
})
import plugins.video_gen.xai as xai_plugin
monkeypatch.setattr(xai_plugin.httpx, "AsyncClient", lambda: _Client())
async def _no_sleep(*a, **k): return None
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
# Reset FAL plugin's lazy fal_client cache so it picks up the stub
from plugins.video_gen import fal as fal_plugin
fal_plugin._fal_client = None
# Force discovery
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered(force=True)
return tmp_path, fal_calls, xai_calls
def _invoke_tool(home, cfg: dict, args: dict, tool_name: str = "video_generate") -> dict:
"""Write config, invoke the registered tool handler, return parsed JSON."""
(home / "config.yaml").write_text(yaml.safe_dump(cfg), encoding="utf-8")
import hermes_cli.config as cfg_mod
if hasattr(cfg_mod, "_invalidate_load_config_cache"):
cfg_mod._invalidate_load_config_cache()
from tools.registry import discover_builtin_tools, registry
if tool_name not in registry._tools:
discover_builtin_tools()
handler = registry._tools[tool_name].handler
return json.loads(handler(args))
# ─────────────────────────────────────────────────────────────────────────
# FAL: every family × {text-only, text+image}
# ─────────────────────────────────────────────────────────────────────────
# We parametrize over the catalog so the test discovers new families
# automatically. If someone adds 'sora-2' to FAL_FAMILIES, this matrix
# picks it up — no test changes needed beyond confirming the endpoints.
def _all_fal_families():
from plugins.video_gen.fal import FAL_FAMILIES
return list(FAL_FAMILIES.keys())
def _t2v_fal_families():
"""Families that advertise a text-to-video endpoint."""
from plugins.video_gen.fal import FAL_FAMILIES
return [fid for fid, meta in FAL_FAMILIES.items() if meta.get("text_endpoint")]
@pytest.mark.parametrize("family_id", _t2v_fal_families())
def test_fal_text_only_routes_to_text_endpoint(matrix_env, family_id):
home, fal_calls, _ = matrix_env
from plugins.video_gen.fal import FAL_FAMILIES
result = _invoke_tool(
home,
{"video_gen": {"provider": "fal", "model": family_id}},
{"prompt": "a dog running"},
)
assert result["success"] is True, f"{family_id}: {result.get('error')}"
assert result["modality"] == "text"
assert result["provider"] == "fal"
# Outbound endpoint must be the family's text endpoint
assert len(fal_calls) == 1
endpoint = fal_calls[0]["endpoint"]
assert endpoint == FAL_FAMILIES[family_id]["text_endpoint"]
# Payload must NOT contain any image-shaped key
payload = fal_calls[0]["arguments"] or {}
image_keys = [k for k in payload if "image" in k and "url" in k]
assert not image_keys, f"{family_id} text-only leaked image keys: {image_keys}"
def _i2v_fal_families():
"""Every family that can animate an existing image."""
from plugins.video_gen.fal import FAL_FAMILIES
return [fid for fid, meta in FAL_FAMILIES.items() if meta.get("image_endpoint")]
@pytest.mark.parametrize("family_id", _i2v_fal_families())
def test_fal_image_to_video_routes_to_image_endpoint(matrix_env, family_id):
home, fal_calls, _ = matrix_env
from plugins.video_gen.fal import FAL_FAMILIES
result = _invoke_tool(
home,
{"video_gen": {"provider": "fal", "model": family_id}},
{"prompt": "animate this", "image_url": "https://example.com/i.png"},
)
meta = FAL_FAMILIES[family_id]
assert result["success"] is True, f"{family_id}: {result.get('error')}"
assert result["modality"] == "image"
assert len(fal_calls) == 1
assert fal_calls[0]["endpoint"] == meta["image_endpoint"]
# The image must land under the family's declared key and no other
# (kling v3 4k wants start_image_url; sending both would be a 422).
payload = fal_calls[0]["arguments"] or {}
image_key = meta.get("image_param_key") or "image_url"
assert payload.get(image_key) == "https://example.com/i.png"
other_keys = [k for k in payload if "image" in k and "url" in k and k != image_key]
assert not other_keys, f"{family_id} sent extra image keys: {other_keys}"
# ─────────────────────────────────────────────────────────────────────────
# xAI: text-only / text+image both go to /videos/generations
# (xAI uses one endpoint with an optional 'image' field, not separate URLs)
# ─────────────────────────────────────────────────────────────────────────
def test_xai_text_only_via_tool_surface(matrix_env):
home, _, xai_calls = matrix_env
result = _invoke_tool(
home,
{"video_gen": {"provider": "xai"}},
{"prompt": "a dog running"},
)
assert result["success"] is True
assert result["modality"] == "text"
assert result["provider"] == "xai"
assert len(xai_calls) == 1
assert xai_calls[0]["url"].endswith("/videos/generations")
payload = xai_calls[0]["json"] or {}
assert "image" not in payload
assert "reference_images" not in payload
assert payload["storage_options"]["public_url"] is True
assert "expires_after" not in payload["storage_options"]
assert result["video"] == "https://xai-files.example/out.mp4"
assert result["public_url"] == "https://xai-files.example/out.mp4"
assert result.get("temporary_url") == "https://xai-cdn/out.mp4"
# ─────────────────────────────────────────────────────────────────────────
# models do not choose models (#83080 ruling): the configured video_gen.model is the only selector
# ─────────────────────────────────────────────────────────────────────────
def test_model_is_never_an_agent_choice(matrix_env):
"""No generation tool advertises a ``model`` parameter, and a ``model`` smuggled into the call
is ignored: the configured ``video_gen.model`` is what reaches the provider request."""
import tools.video_generation_tool as vt
import tools.xai_video_tools as xt
home, fal_calls, _ = matrix_env
result = _invoke_tool(
home,
{"video_gen": {"provider": "fal", "model": "pixverse-v6"}},
{"prompt": "a dog", "model": "veo3.1"},
)
assert result["success"] is True
assert result["model"] == "pixverse-v6"
assert fal_calls[0]["endpoint"] == "fal-ai/pixverse/v6/text-to-video"
schemas = [vt._build_dynamic_video_schema(), xt.XAI_VIDEO_EDIT_SCHEMA, xt.XAI_VIDEO_EXTEND_SCHEMA]
assert all("model" not in schema["parameters"]["properties"] for schema in schemas)