HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.
Python agent-apiagent-frameworkagent-harnessai-agentsai-infrastructurecoding-agentconformance-testingdeepseek-harnessharness-engineeringhermesllmopen-protocolopen-sourceopen-standardpi-coding-agentself-hosteduhpunified-apiunified-harness-protocolunified-interface
"""Tests for the Oh My Pi (OMP) runner backend."""
import json
import pathlib
import sys
import tempfile
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from server import (_same_model, # noqa: E402
_agent_doc_path,
_build_omp,
_pi_to_claude,
_write_skills,
Auth,
BACKENDS,
)
def _flat(chunks):
return [e for evs in chunks for e in evs]
def _run_omp_events(events, model="gpt-5.4"):
state = {"model": model, "final": ""}
out = []
for ev in events:
out.append(_pi_to_claude(ev, state))
return out, state
# ── argv construction and configuration ──────────────────────────────────────────
def test_omp_argv_native_provider():
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-ant-test")
cmd = _build_omp("anthropic", auth, "claude-sonnet-4.6", "hello world", d, env)
assert cmd[:3] == ["omp", "-p", "--mode"]
assert cmd[3] == "json"
assert "--model" in cmd and "claude-sonnet-4.6" in cmd
assert "--auto-approve" in cmd
assert "--no-extensions" in cmd
assert cmd[-1] == "hello world"
assert env.get("ANTHROPIC_API_KEY") == "sk-ant-test"
assert env.get("PI_CODING_AGENT_DIR") == str(pathlib.Path(d) / ".omp" / "agent")
def test_omp_shares_pis_normaliser():
assert BACKENDS["omp"]["normalize"] is _pi_to_claude
def test_an_openai_shape_turn_rides_the_loopback_relay_and_a_custom_endpoint_keeps_its_url():
d = tempfile.mkdtemp()
env = {"HOME": d}
_build_omp("tokenrouter", Auth(api_key="real", base_url="https://api.tokenrouter.com/v1"), "gpt-5.4", "do work", d, env)
cfg = json.loads((pathlib.Path(d) / ".omp" / "agent" / "models.yml").read_text())
assert cfg["providers"]["hr"]["baseUrl"].startswith("http://127.0.0.1:") # the relay, as pi
assert "real" not in (pathlib.Path(d) / ".omp" / "agent" / "models.yml").read_text() # the key stays in the runner
d2 = tempfile.mkdtemp()
_build_omp("openai-api", Auth(api_key="sk-test", base_url="https://relay.example/v1", api_format="openai"), "gpt-5.4", "do work", d2, {"HOME": d2})
cfg2 = json.loads((pathlib.Path(d2) / ".omp" / "agent" / "models.yml").read_text())
assert cfg2["providers"]["hr"]["baseUrl"] == "https://relay.example/v1"
def test_the_served_model_rides_the_result_and_a_substitution_fails_the_turn():
"""omp names the model it ran on every assistant message (measured on 18.1.13). Richard's rule:
the models are honest, no fallback; a turn run on another model fails with the reason."""
ok = [{"type": "session", "id": "s"}, {"type": "message_end", "message": {"role": "assistant", "model": "gemini-3.8-flash", "provider": "hr",
"content": [{"type": "text", "text": "PONG"}], "usage": {"input": 5, "output": 1}, "stopReason": "stop"}}, {"type": "agent_end"}]
chunks, _ = _run_omp_events(ok, model="gemini-3.8-flash")
res = [e for e in _flat(chunks) if e.get("type") == "result"][0]
assert res["is_error"] is False and res["model"] == "gemini-3.8-flash" and res["result"] == "PONG"
swapped = [{"type": "session", "id": "s"}, {"type": "message_end", "message": {"role": "assistant", "model": "gemini-3.5-flash", "provider": "hr",
"content": [{"type": "text", "text": "PONG"}], "usage": {"input": 5, "output": 1}, "stopReason": "stop"}}, {"type": "agent_end"}]
chunks, _ = _run_omp_events(swapped, model="gemini-3.8-flash")
res = [e for e in _flat(chunks) if e.get("type") == "result"][0]
assert res["is_error"] is True and res["result"] == "the CLI ran gemini-3.5-flash instead of gemini-3.8-flash" and res["model"] == "gemini-3.5-flash"
def test_omp_argv_custom_provider_writes_models_json():
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-test", base_url="https://relay.example/v1", api_format="openai")
cmd = _build_omp("openai-api", auth, "gpt-5.4", "do work", d, env)
assert "--model" in cmd
idx = cmd.index("--model")
assert cmd[idx + 1] == "hr/gpt-5.4"
cfg_path = pathlib.Path(d) / ".omp" / "agent" / "models.json"
assert cfg_path.exists()
cfg = json.loads(cfg_path.read_text())
assert "hr" in cfg["providers"]
assert cfg["providers"]["hr"]["baseUrl"] == "https://relay.example/v1"
assert cfg["providers"]["hr"]["api"] == "openai-completions"
assert cfg["providers"]["hr"]["models"][0]["id"] == "gpt-5.4"
def test_omp_resume_skipped_when_session_missing():
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-test")
cmd = _build_omp("openai", auth, "gpt-5.4", "continue", d, env, resume_session_id="missing_sid")
assert "--resume" not in cmd
def test_omp_resume_included_when_session_file_exists():
d = tempfile.mkdtemp()
env = {"HOME": d}
sess_dir = pathlib.Path(d) / ".omp" / "agent" / "sessions" / "proj"
sess_dir.mkdir(parents=True, exist_ok=True)
(sess_dir / "2026-09-05_present_sid_123.jsonl").write_text('{"type":"session"}\n')
auth = Auth(api_key="sk-test")
cmd = _build_omp("openai", auth, "gpt-5.4", "continue", d, env, resume_session_id="present_sid_123")
assert "--resume" in cmd
idx = cmd.index("--resume")
assert cmd[idx + 1] == "present_sid_123"
def test_omp_disabled_tools_generates_tools_flag():
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-test")
cmd = _build_omp("openai", auth, "gpt-5.4", "do task", d, env, tools_disabled=["bash", "browser"])
tools_flags = [x for x in cmd if x.startswith("--tools=")]
assert len(tools_flags) == 1
enabled = tools_flags[0].split("=")[1].split(",")
assert "bash" not in enabled
assert "browser" not in enabled
assert "read" in enabled
assert "edit" in enabled
def test_omp_all_tools_disabled_passes_no_tools():
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-test")
all_tools = ["bash", "read", "write", "edit", "glob", "grep", "lsp", "python", "todo", "task", "browser", "web_search"]
cmd = _build_omp("openai", auth, "gpt-5.4", "do task", d, env, tools_disabled=all_tools)
assert "--no-tools" in cmd
def test_the_omp_tool_list_is_what_the_pinned_build_accepts():
"""Read off omp 18.1.13 by probing each name on --tools (2026-09-08): "python" and "browser"
were in this list and are not tools of that build, so every harness that disabled ANY tool sent
an allowlist omp refused ("Unknown tool in --tools") and every one of its turns died."""
from server import ALL_OMP_TOOLS
assert ALL_OMP_TOOLS == {"bash", "read", "write", "edit", "glob", "grep", "lsp", "todo", "task", "web_search"}
assert "python" not in ALL_OMP_TOOLS and "browser" not in ALL_OMP_TOOLS
def test_a_disable_list_that_names_none_of_omps_tools_sends_no_allowlist():
"""A name from another runtime ("WebSearch", say) removes nothing, so nothing is sent: an
allowlist is only a constraint when it removes something, and sending one for nothing is what
put a refused name in front of omp."""
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-test")
cmd = _build_omp("openai", auth, "gpt-5.4", "do task", d, env, tools_disabled=["WebSearch", "Grep (inherited)"])
assert not [x for x in cmd if x.startswith("--tools=")] or "grep" not in cmd[[i for i, x in enumerate(cmd) if x.startswith("--tools=")][0]]
cmd2 = _build_omp("openai", auth, "gpt-5.4", "do task", d, env, tools_disabled=["WebSearch"])
assert not [x for x in cmd2 if x.startswith("--tools=")] and "--no-tools" not in cmd2
def test_omp_mcp_config_written():
d = tempfile.mkdtemp()
env = {"HOME": d}
auth = Auth(api_key="sk-test")
mcp_servers = [
{"name": "fetcher", "url": "https://mcp.example/sse", "auth": "bearer token123"},
{"name": "db", "url": "https://db.example/mcp", "headers": {"X-Custom": "val"}},
]
_build_omp("openai", auth, "gpt-5.4", "task", d, env, mcp_servers=mcp_servers)
mcp_path = pathlib.Path(d) / ".omp" / "agent" / "mcp.json"
assert mcp_path.exists()
mcp_data = json.loads(mcp_path.read_text())
assert "fetcher" in mcp_data["mcpServers"]
assert mcp_data["mcpServers"]["fetcher"]["url"] == "https://mcp.example/sse"
assert mcp_data["mcpServers"]["fetcher"]["headers"]["Authorization"] == "bearer token123"
assert mcp_data["mcpServers"]["db"]["headers"]["X-Custom"] == "val"
def test_omp_doc_and_skills_paths():
assert _agent_doc_path("/workspace", "omp").name == "AGENTS.md"
d = tempfile.mkdtemp()
skills = [{"name": "test-skill", "content": "# Test Skill"}]
installed = _write_skills(d, skills, backend="omp")
assert len(installed) == 1
assert installed[0]["entry"] == ".harness/home/.omp/agent/skills/test-skill/SKILL.md"
assert (pathlib.Path(d) / ".harness" / "home" / ".omp" / "agent" / "skills" / "test-skill" / "SKILL.md").exists()
# ── event stream normalization ───────────────────────────────────────────────────
def test_omp_normalizer_session_init():
stream = [
{"type": "session", "version": 3, "id": "omp_sess_999", "timestamp": "2026-09-05T08:00:00.000Z", "cwd": "/ws"},
{"type": "agent_end", "messages": [], "isTerminal": True},
]
chunks, _ = _run_omp_events(stream, model="gpt-5.4")
evs = _flat(chunks)
assert evs[0]["type"] == "system"
assert evs[0]["subtype"] == "init"
assert evs[0]["session_id"] == "omp_sess_999"
assert evs[0]["model"] == "gpt-5.4"
def test_omp_normalizer_streaming_text_and_usage():
stream = [
{"type": "session", "version": 3, "id": "s1", "cwd": "/ws"},
{"type": "message_update", "assistantMessageEvent": {"type": "text_delta", "delta": "Hello"}},
{"type": "message_update", "assistantMessageEvent": {"type": "text_delta", "delta": " world!"}},
{"type": "message_end", "message": {
"role": "assistant",
"content": [{"type": "text", "text": "Hello world!"}],
"usage": {"input": 1500, "output": 25, "cacheRead": 300, "cacheWrite": 0},
"stopReason": "stop",
}},
{"type": "agent_end", "messages": [], "isTerminal": True},
]
chunks, state = _run_omp_events(stream)
evs = _flat(chunks)
deltas = [e for e in evs if e.get("type") == "assistant"]
assert len(deltas) == 2
assert deltas[0]["message"]["content"][0]["text"] == "Hello"
assert deltas[1]["message"]["content"][0]["text"] == " world!"
results = [e for e in evs if e.get("type") == "result"]
assert len(results) == 1
assert results[0]["subtype"] == "success"
assert results[0]["is_error"] is False
assert results[0]["result"] == "Hello world!"
assert results[0]["usage"]["input_tokens"] == 1500
assert results[0]["usage"]["output_tokens"] == 25
assert results[0]["usage"]["cache_read_tokens"] == 300
def test_omp_normalizer_tool_calls():
stream = [
{"type": "session", "version": 3, "id": "s2", "cwd": "/ws"},
{"type": "tool_execution_start", "toolCallId": "call_123", "toolName": "bash", "args": {"command": "ls -la"}},
{"type": "tool_execution_end", "toolCallId": "call_123", "toolName": "bash",
"result": {"content": [{"type": "text", "text": "total 0\n"}]}, "isError": False},
{"type": "message_end", "message": {
"role": "assistant", "content": [{"type": "text", "text": "Listed files."}],
"usage": {"input": 50, "output": 10}, "stopReason": "stop"}},
{"type": "agent_end", "messages": [], "isTerminal": True},
]
chunks, _ = _run_omp_events(stream)
evs = _flat(chunks)
tool_use = [e for e in evs if e.get("type") == "assistant" and e["message"]["content"][0].get("type") == "tool_use"]
assert len(tool_use) == 1
assert tool_use[0]["message"]["content"][0]["id"] == "call_123"
assert tool_use[0]["message"]["content"][0]["name"] == "bash"
assert tool_use[0]["message"]["content"][0]["input"] == {"command": "ls -la"}
tool_res = [e for e in evs if e.get("type") == "user" and e["message"]["content"][0].get("type") == "tool_result"]
assert len(tool_res) == 1
assert tool_res[0]["message"]["content"][0]["tool_use_id"] == "call_123"
assert tool_res[0]["message"]["content"][0]["is_error"] is False
assert "total 0" in tool_res[0]["message"]["content"][0]["content"]
def test_omp_normalizer_error_propagation():
stream = [
{"type": "session", "version": 3, "id": "s_err", "cwd": "/ws"},
{"type": "message_end", "message": {
"role": "assistant",
"content": [],
"usage": {"input": 0, "output": 0},
"stopReason": "error",
"errorMessage": "401 Invalid API key for provider",
}},
{"type": "agent_end", "messages": [], "isTerminal": True},
]
chunks, _ = _run_omp_events(stream)
evs = _flat(chunks)
results = [e for e in evs if e.get("type") == "result"]
assert len(results) == 1
assert results[0]["subtype"] == "error"
assert results[0]["is_error"] is True
assert "401 Invalid API key" in results[0]["result"]
# ── live end-to-end turn test with mock LLM (skipped if omp not installed) ─────
import http.server
import shutil
import threading
import time
import pytest
from fastapi.testclient import TestClient
import server
class _MockLLM(http.server.BaseHTTPRequestHandler):
def do_POST(self):
length = int(self.headers.get("content-length", 0))
self.rfile.read(length)
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.end_headers()
chunk1 = {
"id": "chatcmpl-e2e", "object": "chat.completion.chunk", "created": int(time.time()),
"model": "gpt-4o", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "OMP "}, "finish_reason": None}],
}
self.wfile.write(f"data: {json.dumps(chunk1)}\n\n".encode("utf-8"))
self.wfile.flush()
chunk2 = {
"id": "chatcmpl-e2e", "object": "chat.completion.chunk", "created": int(time.time()),
"model": "gpt-4o", "choices": [{"index": 0, "delta": {"content": "works!"}, "finish_reason": None}],
}
self.wfile.write(f"data: {json.dumps(chunk2)}\n\n".encode("utf-8"))
self.wfile.flush()
chunk3 = {
"id": "chatcmpl-e2e", "object": "chat.completion.chunk", "created": int(time.time()),
"model": "gpt-4o", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14},
}
self.wfile.write(f"data: {json.dumps(chunk3)}\n\n".encode("utf-8"))
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
def log_message(self, *args):
pass
@pytest.mark.skipif(shutil.which("omp") is None, reason="omp binary not installed on host")
def test_omp_turn_e2e_with_mock_llm():
mock_server = http.server.HTTPServer(("127.0.0.1", 0), _MockLLM)
port = mock_server.server_address[1]
t = threading.Thread(target=mock_server.serve_forever, daemon=True)
t.start()
client = TestClient(server.app)
d = tempfile.mkdtemp()
req_body = {
"provider": "openai-api",
"backend": "omp",
"model": "gpt-5.4",
"prompt": "Say OMP works",
"cwd": d,
"auth": {
"api_key": "sk-mock",
"base_url": f"http://127.0.0.1:{port}/v1",
"api_format": "openai",
},
}
resp = client.post("/turn", json=req_body)
assert resp.status_code == 200
turn_id = resp.json()["turn_id"]
data = {}
for _ in range(60):
time.sleep(0.3)
t_resp = client.get(f"/turn/{turn_id}")
data = t_resp.json()
if data.get("done"):
break
mock_server.shutdown()
assert data.get("done") is True
assert data.get("status") == "done"
assert "OMP works!" in data.get("result", "")
def test_omp_mcp_entries_carry_the_http_type_omp_requires(tmp_path):
"""omp's docs/mcp-config.md: an http transport entry requires `type: "http"` and `url`, so the
entry carries it. (18.1.13 also infers http from a url when the type is absent; the entry
matches the documented schema rather than the inference.)"""
from server import _omp_write_mcp
ok = _omp_write_mcp(tmp_path, [{"name": "deepwiki", "url": "https://mcp.deepwiki.com/mcp", "auth": "tok"},
{"name": "nourl"}])
assert ok
doc = json.loads((tmp_path / "mcp.json").read_text())
assert doc["mcpServers"] == {"deepwiki": {"type": "http", "url": "https://mcp.deepwiki.com/mcp",
"headers": {"Authorization": "Bearer tok"}}}
def test_the_vendors_own_spelling_of_the_requested_model_is_not_a_substitution():
"""Through an aggregator pi asks for `anthropic/claude-opus-4.8` and Anthropic's stream names
`claude-opus-4-8` (pi stamps the response's model on an Anthropic-shaped message); haiku comes
back as its dated snapshot. Same model; a different model still fails (2026-09-08)."""
def run(requested, served):
evs = [{"type": "session", "id": "s"}, {"type": "message_end", "message": {"role": "assistant", "model": served, "provider": "hr",
"content": [{"type": "text", "text": "PONG"}], "usage": {"input": 5, "output": 1}, "stopReason": "stop"}}, {"type": "agent_end"}]
chunks, _ = _run_omp_events(evs, model=requested)
return [e for e in _flat(chunks) if e.get("type") == "result"][0]
for requested, served in (("anthropic/claude-opus-4.8", "claude-opus-4-8"), ("anthropic/claude-haiku-4.5", "claude-haiku-4-5-20251001"),
("anthropic/claude-fable-5", "claude-fable-5"), ("gpt-5.4", "openai/gpt-5.4")):
res = run(requested, served)
assert res["is_error"] is False and res["result"] == "PONG" and res["model"] == served, (requested, served)
res = run("anthropic/claude-sonnet-5", "claude-sonnet-4-6")
assert res["is_error"] is True and res["result"] == "the CLI ran claude-sonnet-4-6 instead of anthropic/claude-sonnet-5"
assert _same_model("gemini-3.8-flash", "gemini-3-flash-preview") is False
assert _same_model("gpt-5.4", "gpt-5.4-mini") is False