Hermes Agent context engine: the main model writes the compaction handoff on its cached prompt prefix
"""Check the plugin capture against a clean Hermes middleware implementation.
Each case uses a fresh process and temporary home. No model or network request runs.
The report contains versions, hashes, check results, and fixed error codes only.
This check does not prove conversation-loop hook order or provider wire behavior.
"""
from __future__ import annotations
import argparse
import copy
import hashlib
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
import tempfile
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
CASES = ("capture_last", "earlier_rewrite", "later_rewrite", "unknown_order", "fork_prompt_prefixes")
WORKER_TIMEOUT = 60
SOURCE_MARKERS = ("hermes_cli/plugins.py", "hermes_cli/middleware.py", "agent/context_engine.py", "run_agent.py")
def plugin_hashes() -> dict[str, str]:
"""Hash the plugin source files. Do not include paths outside the plugin."""
return {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
for path in sorted((ROOT / "warm_compaction").iterdir())
if path.is_file() and path.suffix in (".py", ".yaml")}
def source_state(source: Path) -> dict:
"""Get the exact source commit and reject a dirty checkout."""
def git(*args):
return subprocess.run(["git", "-C", str(source), *args], check=True, capture_output=True,
text=True, timeout=20).stdout.strip()
commit = git("rev-parse", "HEAD")
if len(commit) != 40 or any(char not in "0123456789abcdef" for char in commit):
raise ValueError("invalid_commit")
if git("status", "--porcelain", "--untracked-files=all"):
raise ValueError("dirty_hermes_source")
if not all((source / name).is_file() for name in SOURCE_MARKERS):
raise ValueError("invalid_hermes_source")
return {"commit": commit, "clean": True}
def check_import_origins(source: Path) -> None:
"""Refuse Hermes modules from an installed runtime or another source root."""
for name, module in tuple(sys.modules.items()):
if name == "warm_compaction.capture":
if Path(module.__file__).resolve() != (ROOT / "warm_compaction" / "capture.py").resolve():
raise ValueError("source_mismatch")
continue
if name not in ("hermes_cli", "agent", "hermes_constants", "registration_lifecycle", "utils") and (
not name.startswith(("hermes_cli.", "agent."))):
continue
path = getattr(module, "__file__", None)
if path is None or not Path(path).resolve().is_relative_to(source):
raise ValueError("source_mismatch")
def worker_environment(home: Path) -> dict[str, str]:
"""Keep the runtime paths only. Do not pass credentials or profile settings."""
names = ("PATH", "SYSTEMROOT", "WINDIR", "PATHEXT", "COMSPEC", "SYSTEMDRIVE",
"LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH")
env = {key: value for key, value in os.environ.items() if key.upper() in names}
env.update({key: str(home) for key in ("HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA",
"XDG_CONFIG_HOME", "XDG_CACHE_HOME", "TEMP", "TMP")})
env.update({"HERMES_HOME": str(home), "HERMES_RUNTIME_DIR": str(home / "runtime"),
"HERMES_DISABLE_LAZY_INSTALLS": "1", "HERMES_ENABLE_PROJECT_PLUGINS": "0",
"PYTHONNOUSERSITE": "1", "PYTHONDONTWRITEBYTECODE": "1"})
return env
def fork_prompt_prefix_case(source: Path) -> dict:
"""Check the fork prompt prefixes against the real Hermes prompt texts of this checkout.
The capture filter falls back to the prompt prefixes where the host passes no turn origin.
A reworded prompt would make the filter miss that fork type, so the check reads the prompt
constants from the checked-out source instead of from literals in the test suite.
"""
try:
from agent import background_review, side_question
from warm_compaction.capture import REVIEW_PROMPT_PREFIX, SIDE_QUESTION_PROMPT_PREFIX
except ImportError as error:
raise ValueError("hermes_import_failed") from error
check_import_origins(source)
review_prompts = (background_review._SKILL_REVIEW_PROMPT, background_review._MEMORY_REVIEW_PROMPT,
background_review._COMBINED_REVIEW_PROMPT)
checks = {
"review_prompts_start_with_the_review_prefix": all(
type(prompt) is str and prompt.startswith(REVIEW_PROMPT_PREFIX) for prompt in review_prompts),
"side_question_prompt_starts_with_the_side_prefix": type(side_question._FORK_PROMPT) is str
and side_question._FORK_PROMPT.startswith(SIDE_QUESTION_PROMPT_PREFIX),
}
passed = all(checks.values())
return {"case": "fork_prompt_prefixes", "passed": passed, "checks": checks,
"error": None if passed else "contract_check_failed"}
def worker(source: Path, case: str) -> dict:
"""Use the real Hermes manager, registrations, hooks, and middleware chain."""
# Windows can use a system process for this standard-library value. Cache it before the fence.
platform.uname()
def no_network(event, _args):
if event in ("socket.connect", "socket.connect_ex", "socket.bind", "socket.getaddrinfo",
"subprocess.Popen", "os.system"):
raise RuntimeError("external_operation_blocked")
sys.addaudithook(no_network)
sys.path[:0] = [str(ROOT), str(source)]
if case == "fork_prompt_prefixes":
return fork_prompt_prefix_case(source)
try:
from hermes_cli import plugins
from hermes_cli.middleware import apply_llm_request_middleware, run_llm_execution_middleware
from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest
from warm_compaction.capture import CaptureStore
except ImportError as error:
raise ValueError("hermes_import_failed") from error
check_import_origins(source)
manager = plugins.get_plugin_manager()
if not isinstance(manager, PluginManager):
raise RuntimeError("manager_type_changed")
manager.discover_and_load()
context = PluginContext(PluginManifest(name="wc_compatibility_probe", version="0.0.0"), manager)
store = CaptureStore()
context.register_hook("pre_api_request", store.on_pre_api_request)
context.register_hook("post_api_request", store.on_post_api_request)
def request_rewrite(request, **_kwargs):
return {"request": {**request, "temperature": 0.25}, "source": "synthetic_probe"}
def execution_rewrite(request, next_call, **_kwargs):
changed = copy.deepcopy(request)
changed["extra_body"]["synthetic_marker"] = "execution_rewrite"
return next_call(changed)
def unknown_capture(**kwargs):
# The registered callback cannot be identified as the store callback.
return store.on_llm_execution(**kwargs)
context.register_middleware("llm_request", request_rewrite)
if case == "earlier_rewrite":
context.register_middleware("llm_execution", execution_rewrite)
context.register_middleware("llm_execution", unknown_capture if case == "unknown_order"
else store.on_llm_execution)
if case == "later_rewrite":
context.register_middleware("llm_execution", execution_rewrite)
history = [{"role": "user", "content": "Synthetic probe question."}]
request = {"model": "synthetic-model", "messages": history, "max_tokens": 128,
"extra_body": {"synthetic_marker": "initial"},
"extra_headers": {"x-opencode-session": "synthetic-session-header"}}
runtime = {"api_request_id": "synthetic-request", "session_id": "synthetic-session",
"model": "synthetic-model", "base_url": "https://example.invalid/v1",
"api_mode": "chat_completions"}
applied = apply_llm_request_middleware(request, **runtime)
plugins.invoke_hook("pre_api_request", conversation_history=history, **runtime)
calls = []
response = object()
def terminal(payload):
calls.append(copy.deepcopy(payload))
return response
returned = run_llm_execution_middleware(applied.payload, terminal,
original_request=applied.original_payload, **runtime)
plugins.invoke_hook("post_api_request", finish_reason="stop",
assistant_message=SimpleNamespace(content="Synthetic reply.", tool_calls=[]),
usage={"prompt_tokens": 123}, **runtime)
capture = store.latest(runtime["session_id"]) or {}
expected_refusal = {"later_rewrite": "middleware_after_capture",
"unknown_order": "middleware_order_unknown"}.get(case)
marker = "execution_rewrite" if case in ("earlier_rewrite", "later_rewrite") else "initial"
expected_headers = {"x-opencode-session": "synthetic-session-header"}
expected_body = {"model": "synthetic-model", "messages": history, "max_tokens": 128,
"temperature": 0.25, "synthetic_marker": marker}
expected_kwargs = {"model": "synthetic-model", "messages": history, "max_tokens": 128,
"temperature": 0.25, "extra_body": {"synthetic_marker": marker},
"extra_headers": expected_headers}
checks = {"terminal_once": len(calls) == 1, "response_unchanged": returned is response,
"terminal_payload_exact": bool(calls) and calls[0] == expected_kwargs,
"request_middleware_applied": bool(calls) and calls[0].get("temperature") == 0.25,
"request_header_forwarded": bool(calls) and calls[0].get("extra_headers") == expected_headers,
"reported_prompt_tokens": capture.get("prompt_tokens") == 123,
"refusal_expected": capture.get("refusal") == expected_refusal}
if expected_refusal:
checks["capture_body_refused"] = capture.get("body") is None
checks["capture_headers_refused"] = capture.get("request_headers") == {}
else:
checks["capture_body_exact"] = capture.get("body") == expected_body
checks["capture_headers_exact"] = capture.get("request_headers") == expected_headers
if case in ("earlier_rewrite", "later_rewrite"):
checks["execution_rewrite_reaches_terminal"] = bool(calls) and (
calls[0].get("extra_body", {}).get("synthetic_marker") == "execution_rewrite")
check_import_origins(source)
checks["import_origins_match_source"] = True
return {"case": case, "passed": all(checks.values()), "checks": checks,
"error": None if all(checks.values()) else "contract_check_failed"}
def run(source: Path) -> dict:
"""Run each contract case in a separate process. Discard raw child output."""
report = {"check": "real Hermes middleware capture compatibility", "passed": False,
"python": sys.version.split()[0], "hermes": None, "plugin_sha256": {},
"script_sha256": None, "cases": [], "error": None}
try:
report["script_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
report["hermes"] = source_state(source)
report["plugin_sha256"] = plugin_hashes()
for case in CASES:
with tempfile.TemporaryDirectory(prefix="wc-compatibility-") as folder:
home = Path(folder)
(home / "config.yaml").write_text("plugins:\n enabled: []\n", encoding="utf-8")
result_path = home / "result.json"
completed = subprocess.run(
[sys.executable, "-B", str(Path(__file__).resolve()), "--hermes-source", str(source),
"--worker-case", case, "--report", str(result_path)], cwd=home,
env=worker_environment(home), capture_output=True, timeout=WORKER_TIMEOUT)
result = (json.loads(result_path.read_text(encoding="utf-8")) if result_path.exists()
else {"case": case, "passed": False, "checks": {}, "error": "worker_no_report"})
result["exit_ok"] = completed.returncode == 0
report["cases"].append(result)
report["checks"] = {"hermes_source_unchanged": source_state(source) == report["hermes"],
"plugin_source_unchanged": plugin_hashes() == report["plugin_sha256"],
"script_unchanged": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
== report["script_sha256"]}
report["passed"] = all(report["checks"].values()) and all(
item["passed"] and item["exit_ok"] for item in report["cases"])
if not report["passed"]:
report["error"] = "compatibility_check_failed"
except Exception as error:
code = error.args[0] if type(error) is ValueError and error.args else None
report["error"] = code if code in (
"dirty_hermes_source", "invalid_commit", "invalid_hermes_source") else "probe_setup_failed"
report["error_type"] = type(error).__name__
return report
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--hermes-source", required=True, type=Path)
parser.add_argument("--report", type=Path)
parser.add_argument("--worker-case", choices=CASES, help=argparse.SUPPRESS)
args = parser.parse_args()
if args.worker_case:
try:
report = worker(args.hermes_source.resolve(), args.worker_case)
except Exception as error:
code = error.args[0] if type(error) is ValueError and error.args else None
report = {"case": args.worker_case, "passed": False, "checks": {},
"error": code if code in ("hermes_import_failed", "source_mismatch") else "worker_runtime_failed",
"error_type": type(error.__cause__ or error).__name__}
else:
report = run(args.hermes_source.resolve())
if args.report:
try:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n", encoding="utf-8")
except OSError as error:
print(json.dumps({"passed": False, "error": "report_write_failed", "error_type": type(error).__name__}))
return 1
print(json.dumps({"passed": report["passed"], "error": report["error"]}))
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())