Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

5.5 KB
Raw Blame
"""`uhp-conformance` — run the suite against a server."""
from __future__ import annotations

import argparse
import os
import sys

from . import UHP_VERSION, checks  # noqa: F401 — importing checks populates the registry
from .client import Client
from . import DEFAULT_PLUGIN_MCP_URL
from .context import Context
from .registry import CLASSES, Outcome, checks_for
from .report import render, to_json


def main(argv=None) -> int:
    p = argparse.ArgumentParser(
        prog="uhp-conformance",
        description=f"Run the Unified Harness Protocol {UHP_VERSION} conformance suite "
                    f"against a server.")
    p.add_argument("--base-url", required=True,
                   help="Server root, e.g. https://host or http://127.0.0.1:3000/api/harness")
    p.add_argument("--api-key", default=os.environ.get("UHP_API_KEY", ""),
                   help="Bearer token (or set UHP_API_KEY)")
    p.add_argument("--class", dest="cls", default="core", choices=CLASSES,
                   help="Conformance class to test (cumulative). Default: core")
    p.add_argument("--harness-id", default="", help="Run tasks against this harness id")
    p.add_argument("--model", default="", help="Run tasks with this model")
    p.add_argument("--task-timeout", type=float, default=300.0,
                   help="Seconds to allow one agent task. Default: 300")
    p.add_argument("--json", dest="json_out", default="", help="Write a JSON report to this path")
    p.add_argument("--only", default="", help="Comma-separated check ids to run")
    p.add_argument("--plain", action="store_true", help="No ANSI colour")
    p.add_argument("--header", action="append", default=[], metavar="NAME: VALUE",
                   help="Extra header sent on every request, repeatable. For a target that "
                        "needs something a stock client never sends, e.g. a bring-your-own-key "
                        "host wanting the caller's model key. Never echoed into the report.")
    p.add_argument("--plugin-mcp-url", default=os.environ.get("UHP_PLUGIN_MCP_URL", DEFAULT_PLUGIN_MCP_URL),
                   help="A reachable streamable-HTTP MCP server the plugin checks point a plugin at "
                        "(default: the public copy of uhp-conformance-fixture, or UHP_PLUGIN_MCP_URL). "
                        "Run `uhp-conformance-fixture` to host your own.")
    p.add_argument("--label", default="",
                   help="How the report names the implementation measured, e.g. 'SuperQode 2.3.1'")
    a = p.parse_args(argv)

    extra = {}
    for item in a.header:
        name, sep, value = item.partition(":")
        if not sep or not name.strip():
            print(f"--header expects 'NAME: VALUE', got {item!r}", file=sys.stderr)
            return 2
        extra[name.strip()] = value.strip()

    ctx = Context(client=Client(a.base_url, a.api_key, headers=extra), harness_id=a.harness_id,
                  model=a.model, task_timeout=a.task_timeout, plugin_mcp_url=a.plugin_mcp_url)

    # The harness the tasks run on is resolved BEFORE the first task, and named. A --harness-id
    # that matches nothing (a harness name given for an id, a typo) once fell back silently to
    # the first harness listed, and two runs against different ids produced byte-identical
    # reports (#203). A harness with no default model and no --model runs every task with
    # whatever the server falls back to, which on a self-hosted instance may be an id its
    # provider does not serve: each task then waits out --task-timeout and a full run takes half
    # an hour to say nothing (#199). Both are said here, in one line, before anything runs.
    if a.harness_id:
        r = ctx.client.get(f"/v1/harnesses/{a.harness_id}")
        if r.status != 200 or not isinstance(r.json, dict):
            print(f"no harness with id {a.harness_id!r} on this server (HTTP {r.status}); pass the "
                  "harness's id, not its name, or omit --harness-id to use the first listed",
                  file=sys.stderr)
            return 2
        if not a.model and not str(r.json.get("defaultModel") or "").strip():
            print(f"harness {a.harness_id} has no default model; pass --model <id> so the tasks "
                  "run on a model this server serves", file=sys.stderr)
            return 2
        print(f"tasks run on harness {a.harness_id} ({r.json.get('name') or 'unnamed'}), model "
              f"{a.model or r.json.get('defaultModel')}", file=sys.stderr)

    required = checks_for(a.cls)
    selected = required
    if a.only:
        want = {s.strip().upper() for s in a.only.split(",") if s.strip()}
        selected = [c for c in selected if c.id.upper() in want]
        if not selected:
            print(f"no checks matched --only {a.only!r}", file=sys.stderr)
            return 2

    results = []
    for c in selected:
        results.append(c.run(ctx))
        if not a.plain:
            print(".", end="", flush=True)
    if not a.plain:
        print("\r", end="")

    discovery = ctx.state.get("discovery")
    print(render(results, a.base_url, a.cls, plain=a.plain, discovery=discovery,
                 required_checks=required))

    if a.json_out:
        with open(a.json_out, "w") as f:
            f.write(to_json(results, a.base_url, a.cls, discovery=discovery, label=a.label,
                            required_checks=required) + "\n")
        print(f"  JSON report: {a.json_out}\n")

    bad = sum(1 for r in results if r.outcome in (Outcome.FAIL, Outcome.ERROR))
    return 1 if bad else 0


if __name__ == "__main__":
    raise SystemExit(main())