Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

ci(model-hub): fail an engine change that alters hosted passthrough, unless labeled (#2782)

Call White5bb4cafc1146
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index e23a9c812..c73907d92 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -2,6 +2,8 @@ name: lint
on:
pull_request:
+ # Labeling re-runs the hosted wire gate, which reads the labels it runs with.
+ types: [opened, synchronize, reopened, labeled, unlabeled]
push:
branches:
- master
@@ -332,14 +334,89 @@ jobs:
if-no-files-found: ignore
retention-days: 7
+ model-hub-hosted-wire:
+ needs: changes
+ if: needs.changes.outputs.docs_only == 'false'
+ # A pull request that replaces the engine must not change how it passes
+ # hosted tools through, unless labeled engine-hosted-wire-change
+ # (docs/plans/model-hub-cpa-dependency.md). Any other change passes after
+ # one diff, without installing anything.
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ pull-requests: read
+ env:
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ steps:
+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ # The synthetic PR merge and both parents, base included.
+ fetch-depth: 2
+ - name: Detect an engine manifest change
+ id: manifest
+ run: |
+ if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then
+ # The label that may accept a difference lives on the pull request.
+ echo "Judged on the pull request; nothing to do on $GITHUB_EVENT_NAME."
+ echo "changed=false" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null || git fetch --no-tags --depth=1 origin "$BASE_SHA"
+ if git diff --quiet "$BASE_SHA" HEAD -- vibe/model_hub_runtime/cliproxyapi_manifest.json; then
+ echo "changed=false" >> "$GITHUB_OUTPUT"
+ else
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+ fi
+ - name: Prepare hermetic UI asset placeholder
+ if: steps.manifest.outputs.changed == 'true'
+ run: |
+ mkdir -p ui/dist
+ printf '%s\n' '<!doctype html><html><body></body></html>' > ui/dist/index.html
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ if: steps.manifest.outputs.changed == 'true'
+ with:
+ python-version: "3.12"
+ - name: Install pinned uv
+ if: steps.manifest.outputs.changed == 'true'
+ run: |
+ python -m pip install --disable-pip-version-check --only-binary=:all: --no-deps uv==0.12.10
+ - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
+ if: steps.manifest.outputs.changed == 'true'
+ with:
+ path: ~/.cache/uv
+ key: ${{ runner.os }}-${{ runner.arch }}-ci-uv-0.12.10-py3.12-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+ - name: Install package and test dependencies
+ if: steps.manifest.outputs.changed == 'true'
+ run: uv pip install --system -e . --group dev
+ - name: Compare the hosted passthrough of both engines
+ if: steps.manifest.outputs.changed == 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ run: >-
+ python scripts/model_hub_hosted_wire_gate.py --base "$BASE_SHA"
+ --pull-request "$PR_NUMBER" --work-dir "$RUNNER_TEMP/hosted-wire" --loopback-only
+ - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+ if: always() && steps.manifest.outputs.changed == 'true'
+ with:
+ name: model-hub-hosted-wire-${{ github.sha }}-${{ github.run_attempt }}
+ # Synthetic fixtures on loopback only: per-case wire records and reports.
+ path: |
+ ${{ runner.temp }}/hosted-wire/*/evidence
+ ${{ runner.temp }}/hosted-wire/*/cases.xml
+ if-no-files-found: ignore
+ retention-days: 14
+
unit-tests:
runs-on: ubuntu-latest
# Keep UI validation in the existing required gate while allowing artifact
- # consumers to start as soon as their production bundle is built.
- needs: [changes, unit-test-shards, scenario-evidence, ui-checks, ui-browser-checks]
+ # consumers to start as soon as their production bundle is built. The
+ # hosted wire gate reports through it too, under no new required context.
+ needs: [changes, unit-test-shards, scenario-evidence, ui-checks, ui-browser-checks, model-hub-hosted-wire]
if: always()
steps:
- - name: Verify unit test shards and UI checks
+ - name: Verify unit test shards, UI checks and the hosted_wire gate
env:
SCOPE_RESULT: ${{ needs.changes.result }}
DOCS_ONLY: ${{ needs.changes.outputs.docs_only }}
@@ -347,6 +424,7 @@ jobs:
EVIDENCE_RESULT: ${{ needs['scenario-evidence'].result }}
UI_RESULT: ${{ needs['ui-checks'].result }}
BROWSER_RESULT: ${{ needs['ui-browser-checks'].result }}
+ HOSTED_WIRE_RESULT: ${{ needs['model-hub-hosted-wire'].result }}
run: |
if [ "$SCOPE_RESULT" != "success" ]; then
echo "Change classification did not succeed."
@@ -357,13 +435,13 @@ jobs:
false) expected="success" ;;
*) echo "Missing or invalid change classification."; exit 1 ;;
esac
- for result in "$UNIT_RESULT" "$EVIDENCE_RESULT" "$UI_RESULT" "$BROWSER_RESULT"; do
+ for result in "$UNIT_RESULT" "$EVIDENCE_RESULT" "$UI_RESULT" "$BROWSER_RESULT" "$HOSTED_WIRE_RESULT"; do
if [ "$result" != "$expected" ]; then
- echo "Unit/UI checks finished with result: $result"
+ echo "Unit/UI/hosted_wire checks finished with result: $result"
exit 1
fi
done
- echo "Unit/UI results match the classified scope (docs_only=$DOCS_ONLY)."
+ echo "Unit/UI/hosted_wire results match the classified scope (docs_only=$DOCS_ONLY)."
# Every case a covered scenario row cites as evidence ran and passed in some
# job, or a release workflow runs it (scripts/ci_scenario_evidence.py). It
diff --git a/.github/workflows/model-hub-e2e.yml b/.github/workflows/model-hub-e2e.yml
index d441e781e..a6b04ecc0 100644
--- a/.github/workflows/model-hub-e2e.yml
+++ b/.github/workflows/model-hub-e2e.yml
@@ -78,9 +78,12 @@ jobs:
EOF
echo "VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH=$RUNNER_TEMP/model-hub-engine/offline-manifest.json" >> "$GITHUB_ENV"
+ # Every scenario is hermetic, so it runs where only loopback answers: an
+ # engine or CLI that ignores its loopback upstream fails at once instead
+ # of reaching a real provider. The downloads above stay outside.
- name: Run the Model Hub E2E
run: |
- python -m pytest -m e2e_model_hub tests/e2e -p no:cacheprovider -ra \
+ bash scripts/ci_loopback_only.sh python -m pytest -m e2e_model_hub tests/e2e -p no:cacheprovider -ra \
--basetemp="$RUNNER_TEMP/e2e" 2>&1 | tee model-hub-e2e.log
exit "${PIPESTATUS[0]}"
diff --git a/docs/plans/model-hub-cpa-dependency.md b/docs/plans/model-hub-cpa-dependency.md
index 76899d03f..11af29e5f 100644
--- a/docs/plans/model-hub-cpa-dependency.md
+++ b/docs/plans/model-hub-cpa-dependency.md
@@ -73,6 +73,47 @@ manifest's `build.patches` records; the packaged manifest records the resulting
archive and binary digests. Rolling back means reverting the whole adoption
change, which restores the previous patch directory and manifest together.
+### Hosted passthrough
+
+Hosted tools rely on the engine passing them through unchanged, and they are
+assumed rather than probed, so a passthrough regression in an engine release
+would reach users unnoticed. `lint` guards it as `model-hub-hosted-wire`, which
+reports through the required `unit-tests` check, on every pull request that
+replaces the engine (changes the manifest's `asset_release_tag`):
+
+- `scripts/model_hub_hosted_wire_gate.py` downloads both engines' assets for the
+ runner's platform from their published releases and verifies them against
+ each manifest. It then runs the cases in
+ `tests/e2e/test_model_hub_engine_hosted_passthrough.py` against both and
+ compares each case's upstream request path, query, header set and bytes, and
+ its downstream status and bytes. Only per-run values are masked: the engine's
+ `prompt_cache_key` and `session-id`, and the port in the capturing upstream's
+ `Host`. The new engine must pass every case, where a case marked
+ `xfail(strict=True)` passes by failing as its marker says. The base engine
+ only has to record each case, so a case it fails is compared as recorded.
+ An engine that fixes a known defect turns its strict expected failure into a
+ failure, so that change flips the marker and carries the label below.
+- A byte-identical passthrough passes. A difference fails and names the cases
+ and parts that differ, unless the pull request is labeled
+ `engine-hosted-wire-change`. An intended change, a rollback included, needs
+ that label and a note in the pull request body saying what changed and why.
+ The gate reads the labels when it runs, adding or removing one re-runs
+ `lint`, and labels that cannot be read count as none.
+
+The gate judges pull requests, where the label lives; a push to master does not
+repeat it.
+
+Before dispatching an engine build, run the gate on the local candidate build
+and include its output in the dispatch request; engine sign-off requires it.
+Without a pull request no label applies, so the output shows any difference as
+a failure for review:
+
+```sh
+python scripts/model_hub_hosted_wire_gate.py --base origin/master --head-assets <build output dir>
+```
+
+The pull request's gate stays the enforcement of record.
+
## Acceptance Criteria
- A user who never enables Model Hub receives the CPA target during normal Avibe
diff --git a/scripts/model_hub_hosted_wire_gate.py b/scripts/model_hub_hosted_wire_gate.py
new file mode 100644
index 000000000..d36b66830
--- /dev/null
+++ b/scripts/model_hub_hosted_wire_gate.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""Fail an engine change that alters how the engine passes hosted tools through.
+
+Native hosted tools are assumed rather than probed, so a passthrough regression
+in an engine release would otherwise reach users unnoticed. When a change
+replaces the engine, the hosted passthrough cases run against the base and the
+changed engine and are compared byte for byte. Any difference fails, unless the
+pull request carries the label that says the change is intended. PR CI takes
+both engines from their published releases. Before a candidate is published,
+``--head-assets`` takes it from the local build output instead. See
+docs/plans/model-hub-cpa-dependency.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Mapping
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import urllib.request
+from xml.etree import ElementTree
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(REPO_ROOT))
+
+from scripts import model_hub_engine_release_guard as release_guard # noqa: E402
+
+MANIFEST = "vibe/model_hub_runtime/cliproxyapi_manifest.json"
+CASES = "tests/e2e/test_model_hub_engine_hosted_passthrough.py"
+# Runs a command where only loopback answers; Linux CI only (needs passwordless sudo).
+LOOPBACK_ONLY = "scripts/ci_loopback_only.sh"
+EVIDENCE_DIR_ENV = "AVIBE_HOSTED_WIRE_EVIDENCE_DIR"
+CHANGE_LABEL = "engine-hosted-wire-change"
+# What each case records, in the order a difference is reported.
+WIRE_PARTS = (
+ ("upstream", "path", "upstream request path"),
+ ("upstream", "query", "upstream request query"),
+ ("upstream", "headers", "upstream header set"),
+ ("upstream", "body", "upstream request bytes"),
+ ("downstream", "status", "downstream status"),
+ ("downstream", "body", "downstream bytes"),
+)
+
+
+class GateError(RuntimeError):
+ """Raised when the gate cannot reach a verdict."""
+
+
+def wire_differences(base_cases: Mapping[str, dict], head_cases: Mapping[str, dict]) -> dict[str, list[str]]:
+ """Map every case whose two recordings differ to the wire parts that differ."""
+ differences = {}
+ for case in sorted(base_cases.keys() | head_cases.keys()):
+ if case not in base_cases or case not in head_cases:
+ differences[case] = ["recorded against only one engine"]
+ continue
+ parts = [label for direction, part, label in WIRE_PARTS
+ if base_cases[case][direction][part] != head_cases[case][direction][part]]
+ if parts:
+ differences[case] = parts
+ return differences
+
+
+def wire_change_violation(differences: Mapping[str, list[str]], labels: frozenset[str] | None) -> str | None:
+ """Return why a hosted passthrough difference blocks the change, or None.
+
+ ``labels`` are the pull request's labels, or None when there is no pull
+ request or its labels could not be read; both count as unlabeled.
+ """
+ if not differences or (labels is not None and CHANGE_LABEL in labels):
+ return None
+ summary = "; ".join(f"{case}: {', '.join(parts)}" for case, parts in differences.items())
+ return (f"the engine changes the hosted passthrough ({summary}). If that is intended, label the pull "
+ f"request {CHANGE_LABEL} and say in its body what changed and why")
+
+
+def pull_request_labels(repository: str, number: str, token: str | None) -> frozenset[str] | None:
+ """The pull request's labels as they are now, or None when they cannot be read."""
+ headers = {"Accept": "application/vnd.github+json", "User-Agent": "avibe-model-hub-hosted-wire-gate/1"}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ request = urllib.request.Request(f"https://api.github.com/repos/{repository}/pulls/{number}", headers=headers)
+ try:
+ with urllib.request.urlopen(request, timeout=30) as response:
+ return frozenset(label["name"] for label in json.load(response)["labels"])
+ except (OSError, ValueError, KeyError, TypeError) as exc:
+ print(f"::warning::hosted_wire gate: cannot read the pull request's labels ({type(exc).__name__}); "
+ "judging it as unlabeled")
+ return None
+
+
+def _git_show(revision: str, path: str) -> bytes:
+ try:
+ return subprocess.run(
+ ["git", "-C", str(REPO_ROOT), "show", f"{revision}:{path}"],
+ check=True, capture_output=True, timeout=60,
+ ).stdout
+ except (OSError, subprocess.SubprocessError) as exc:
+ raise GateError(f"cannot read {path} at the base revision {revision}") from exc
+
+
+def _met_its_expectation(testcase: ElementTree.Element) -> bool:
+ """A case passed, or failed as its strict ``xfail`` marker says it must; an unexpected pass fails."""
+ if testcase.find("failure") is not None or testcase.find("error") is not None:
+ return False
+ skipped = testcase.find("skipped")
+ return skipped is None or skipped.get("type") == "pytest.xfail"
+
+
+def _run_cases(
+ side: str, manifest_bytes: bytes, platform: str, work: Path, asset_dir: Path | None = None,
+ *, must_pass: bool, loopback_only: bool = False,
+) -> dict[str, dict]:
+ """Run the case set against one side's engine; return each case's recorded wire.
+
+ Each case records the exchange before asserting on it. The PR engine must pass
+ every case. The base engine only has to record every case: one that fails its
+ assertions there has a different wire, which the comparison reports.
+ """
+ work.mkdir(parents=True)
+ pinned = work / "cliproxyapi_manifest.json"
+ pinned.write_bytes(manifest_bytes)
+ try:
+ spec = release_guard.load_release_spec(pinned)
+ except release_guard.ReleaseGuardError as exc:
+ raise GateError(f"the {side} manifest is invalid: {exc}") from exc
+ archive = next((item for item in spec.archives if item.platform == platform), None)
+ if archive is None:
+ raise GateError(f"the {side} manifest has no {platform} asset")
+ try:
+ if asset_dir is None:
+ local_archive = work / archive.name
+ release_guard._download(archive.owned_url, local_archive, max_bytes=archive.size)
+ release_guard._verify_archive(local_archive, archive)
+ else:
+ # Exactly what publication would upload: every asset, sidecar and the manifest copy.
+ release_guard.verify_release_assets(pinned, asset_dir)
+ local_archive = asset_dir.resolve() / archive.name
+ except release_guard.ReleaseGuardError as exc:
+ raise GateError(f"the {side} engine is not the asset its manifest pins: {exc}") from exc
+ # The E2E harness installs offline from a file URL; the archive was verified above.
+ manifest = json.loads(manifest_bytes)
+ for asset in manifest["assets"]:
+ if asset["platform"] == platform:
+ asset["url"] = local_archive.as_uri()
+ offline_manifest = work / "offline-manifest.json"
+ offline_manifest.write_text(json.dumps(manifest), encoding="utf-8")
+
+ evidence, report = work / "evidence", work / "cases.xml"
+ evidence.mkdir()
+ print(f"hosted_wire gate: running the hosted passthrough cases against the {side} engine "
+ f"{manifest['asset_release_tag']} ({platform})", flush=True)
+ command = [sys.executable, "-m", "pytest", "-m", "e2e_model_hub", "-p", "no:cacheprovider",
+ "-q", "-ra", f"--junitxml={report}", CASES]
+ if loopback_only:
+ # The engine was downloaded above; the cases and the engine they start need only loopback,
+ # so one that ignores its loopback Source fails at once instead of reaching the network.
+ command = ["bash", LOOPBACK_ONLY, *command]
+ result = subprocess.run(
+ command,
+ cwd=REPO_ROOT,
+ env={**os.environ, "VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH": str(offline_manifest),
+ EVIDENCE_DIR_ENV: str(evidence)},
+ check=False,
+ )
+ try:
+ testcases = list(ElementTree.parse(report).getroot().iter("testcase"))
+ except (OSError, ElementTree.ParseError) as exc:
+ raise GateError(f"the {side} engine's case run left no report") from exc
+ names = {testcase.get("name") for testcase in testcases}
+ passed = all(_met_its_expectation(testcase) for testcase in testcases)
+ if not testcases or (must_pass and (result.returncode != 0 or not passed)):
+ raise GateError(f"the hosted passthrough cases did not all pass against the {side} engine")
+ cases = {path.stem: json.loads(path.read_text(encoding="utf-8")) for path in evidence.glob("*.json")}
+ if set(cases) != names:
+ raise GateError(f"the {side} engine's run recorded {sorted(cases)}, expected {sorted(names)}")
+ return cases
+
+
+def hosted_wire_differences(
+ base_bytes: bytes, head_bytes: bytes, work: Path, head_assets: Path | None = None,
+ *, loopback_only: bool = False,
+) -> dict[str, list[str]]:
+ from vibe.model_hub_runtime.installer import EngineRuntimeManager
+
+ platform = EngineRuntimeManager(runtime_dir=work / "probe", offline=True).host_platform()
+ base_cases = _run_cases("base", base_bytes, platform, work / "base",
+ must_pass=False, loopback_only=loopback_only)
+ head_cases = _run_cases("PR", head_bytes, platform, work / "head", head_assets,
+ must_pass=True, loopback_only=loopback_only)
+ return wire_differences(base_cases, head_cases)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base", required=True, help="revision holding the base manifest")
+ parser.add_argument(
+ "--head-assets", type=Path,
+ help="a local candidate's build output directory, instead of its published release",
+ )
+ parser.add_argument("--pull-request", help="number of the pull request whose labels may accept a difference")
+ parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "avibe-bot/avibe"))
+ parser.add_argument("--work-dir", type=Path, help="keep downloads and per-case evidence here")
+ parser.add_argument(
+ "--loopback-only", action="store_true",
+ help=f"run the cases in a network namespace with only loopback ({LOOPBACK_ONLY}; Linux CI)",
+ )
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = _build_parser().parse_args(argv)
+ try:
+ base_bytes = _git_show(args.base, MANIFEST)
+ head_bytes = (REPO_ROOT / MANIFEST).read_bytes()
+ try:
+ base_tag = json.loads(base_bytes)["asset_release_tag"]
+ head_tag = json.loads(head_bytes)["asset_release_tag"]
+ except (ValueError, KeyError, TypeError) as exc:
+ raise GateError(f"cannot read either manifest's asset_release_tag ({type(exc).__name__})") from exc
+ if base_tag == head_tag:
+ print(f"hosted_wire gate: the engine is unchanged ({head_tag}); nothing to compare")
+ return 0
+ with tempfile.TemporaryDirectory(prefix="hosted-wire-") as scratch:
+ differences = hosted_wire_differences(base_bytes, head_bytes, args.work_dir or Path(scratch),
+ args.head_assets, loopback_only=args.loopback_only)
+ except GateError as exc:
+ print(f"::error::hosted_wire gate: {exc}")
+ return 1
+ if not differences:
+ print(f"hosted_wire gate: {base_tag} -> {head_tag}: hosted passthrough byte-identical")
+ return 0
+ print(f"hosted_wire gate: {base_tag} -> {head_tag}: hosted passthrough differs")
+ for case, parts in differences.items():
+ print(f" {case}: {', '.join(parts)}")
+ labels = None
+ if args.pull_request:
+ labels = pull_request_labels(args.repository, args.pull_request, os.environ.get("GH_TOKEN"))
+ else:
+ print("hosted_wire gate: no pull request, so no label can accept the difference")
+ violation = wire_change_violation(differences, labels)
+ if violation:
+ print(f"::error file={MANIFEST}::hosted_wire gate: {violation}")
+ return 1
+ print(f"hosted_wire gate: the pull request is labeled {CHANGE_LABEL}, so the difference is intended")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/e2e/test_model_hub_engine_hosted_passthrough.py b/tests/e2e/test_model_hub_engine_hosted_passthrough.py
new file mode 100644
index 000000000..b99ebbce6
--- /dev/null
+++ b/tests/e2e/test_model_hub_engine_hosted_passthrough.py
@@ -0,0 +1,493 @@
+"""Hosted tools preserve business payloads through the real engine on same-protocol paths.
+
+These engine-boundary scenarios protect same-protocol hosted declarations,
+``include`` values, output items and replayed history against CPA upgrades.
+The existing mocked gateway and consumer tests do not run CPA's translators.
+Each case sends one request to the real engine under Avibe's rendered config;
+a synthetic loopback upstream records request bytes and replays fixed wire
+fixtures. This is not a live provider or a complete Vibey-turn acceptance test.
+With ``AVIBE_HOSTED_WIRE_EVIDENCE_DIR`` set, each case also writes both
+directions there, so ``scripts/model_hub_hosted_wire_gate.py`` can compare two
+engines' runs byte for byte.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import urllib.error
+import urllib.request
+from contextlib import asynccontextmanager
+
+from aiohttp import web
+import pytest
+
+from tests.e2e.test_model_hub_runtime import _isolated_engine_adapter
+from tests.test_model_hub_google import _server
+from tests.scenario_harness.model_hub import MemoryModelHubStore, service_for
+from tests.test_model_hub_l3 import _source
+from tests.test_model_hub_vibey_consumer import _vibey_service
+
+
+pytestmark = pytest.mark.e2e_model_hub
+
+EVIDENCE_DIR_ENV = "AVIBE_HOSTED_WIRE_EVIDENCE_DIR"
+MODEL = "hosted-model"
+# Fernet-shaped, so the engine keeps it with its id (CPA drops other shapes).
+_FERNET = "g" + "A" * 97
+_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+WEB_SEARCH = {"type": "web_search"}
+X_SEARCH = {"type": "x_search"}
+IMAGE_GENERATION = {"type": "image_generation", "partial_images": 2, "output_format": "jpeg", "output_compression": 90}
+FUNCTION = {"type": "function", "name": "read", "description": "fixture",
+ "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}
+INCLUDE = ["web_search_call.action.sources", "reasoning.encrypted_content"]
+
+WEB_SEARCH_CALL = {
+ "id": "ws_1",
+ "type": "web_search_call",
+ "status": "completed",
+ "action": {
+ "type": "search",
+ "query": "CLIProxyAPI 版本 🦊",
+ "queries": ["CLIProxyAPI 版本 🦊"],
+ "sources": [{"type": "url", "url": "https://github.com/router-for-me/CLIProxyAPI/releases"}],
+ },
+}
+X_SEARCH_CALL = {
+ "type": "custom_tool_call",
+ "id": "ctc_1",
+ "call_id": "xs_call-1",
+ "name": "x_keyword_search",
+ "input": json.dumps({"query": "from:OpenAI within_time:7d", "limit": "10", "mode": "Latest"}),
+ "status": "completed",
+}
+X_REASONING = {"type": "reasoning", "id": "tco_1", "summary": [], "encrypted_content": _FERNET}
+# xAI's own ciphertext has no Fernet envelope. Without ``is-compat`` on the Source's models the codex
+# executor drops its ``encrypted_content`` and ``id`` (CPA ``openai_responses_signature.go``); the
+# replay case records what reaches the upstream rather than asserting it.
+XAI_OPAQUE_REASONING = {"type": "reasoning", "id": "tco_2", "summary": [], "encrypted_content": "xai-opaque-ciphertext"}
+
+
+def _xai_ciphertext(seed: str, length: int) -> str:
+ """Opaque like xAI's own ciphertext, which has no Fernet envelope."""
+ stream = b"".join(hashlib.sha256(f"{seed}-{block}".encode()).digest() for block in range(length // 32 + 1))
+ return base64.b64encode(stream).decode()[:length]
+
+
+# One xAI X-search turn in the 2026-10-13 spike's real shapes. The turn's reasoning is ``rs_<turn>``
+# (39 runes). Each server-side tool call ``ctc_<turn>_<call>`` has its own ``tco_<turn>_<call>``
+# reasoning; both of those ids are 84 runes.
+XAI_TURN = "1994eace-959a-9bc5-87ca-e01dac5c7bfe"
+XAI_CALL = "call-9696c08a-e2e1-4204-ab58-9d1d317c128f-2"
+XAI_REASONING = {
+ "id": f"rs_{XAI_TURN}",
+ "summary": [{"type": "summary_text", "text": "I need the most recent post, so I fetch its thread."}],
+ "type": "reasoning",
+ "status": "completed",
+ "encrypted_content": _xai_ciphertext("rs", 3211),
+}
+XAI_THREAD_FETCH = {
+ "call_id": f"xs_{XAI_CALL}",
+ "input": json.dumps({"post_id": "2107909792945832070"}),
+ "name": "x_thread_fetch",
+ "type": "custom_tool_call",
+ "id": f"ctc_{XAI_TURN}_{XAI_CALL}",
+ "status": "completed",
+}
+XAI_CALL_REASONING = {
+ "id": f"tco_{XAI_TURN}_{XAI_CALL}",
+ "summary": [],
+ "type": "reasoning",
+ "status": "completed",
+ "encrypted_content": _xai_ciphertext("tco", 3696),
+}
+IMAGE_CALL = {
+ "id": "ig_1",
+ "type": "image_generation_call",
+ "status": "completed",
+ "result": _PNG_B64,
+ "revised_prompt": "A red fox in snow",
+ "output_format": "jpeg",
+ "size": "1024x1024",
+ "quality": "high",
+ "background": "opaque",
+}
+ANSWER = "The latest release is v8.0.27.[[1]](https://github.com/router-for-me/CLIProxyAPI/releases)"
+ANSWER_ITEM = {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{
+ "type": "output_text",
+ "text": ANSWER,
+ "annotations": [{"type": "url_citation", "url": "https://github.com/router-for-me/CLIProxyAPI/releases",
+ "start_index": 29, "end_index": len(ANSWER), "title": "1"}],
+ }],
+}
+RESPONSES_USAGE = {
+ "input_tokens": 22965,
+ "output_tokens": 1402,
+ "total_tokens": 24367,
+ "input_tokens_details": {"cached_tokens": 0},
+ "output_tokens_details": {"reasoning_tokens": 210},
+ "cost_in_usd_ticks": 277316400,
+ "server_side_tool_usage_details": {"web_search_calls": 1, "x_search_calls": 1},
+}
+
+HIT = {
+ "type": "web_search_result",
+ "title": "Releases · CLIProxyAPI 发布",
+ "url": "https://github.com/router-for-me/CLIProxyAPI/releases",
+ "encrypted_content": "EqoBCioIBxgCIiQ4ZGUxYzRhMS0wYjk0LTQ2NjktYjM1ZS1hYmFhNmRkODQ3OGI",
+ "page_age": "15 days ago",
+}
+LOCATION = {
+ "type": "web_search_result_location",
+ "url": HIT["url"],
+ "title": HIT["title"],
+ "cited_text": "v8.0.24 Latest",
+ "encrypted_index": "Eo8BCioIBxgCIiQ4ZGUxYzRhMS0wYjk0LTQ2NjktYjM1ZS1hYmFhNmRkODQ3OGI",
+}
+ANTHROPIC_WEB_SEARCH = {"type": "web_search_20250305", "name": "web_search", "max_uses": 5}
+ANTHROPIC_TOOL = {"name": "read", "description": "fixture",
+ "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}
+
+
+def _sse(event: dict) -> bytes:
+ return f"event: {event['type']}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()
+
+
+def _responses_stream(items: list[dict], *, partial_images: bool = False) -> bytes:
+ """A Responses stream in the spike's shape: every item added, progress frames, done, then completed."""
+ events: list[dict] = [{"type": "response.created", "response": {
+ "id": "resp_hosted", "object": "response", "status": "in_progress", "model": MODEL, "output": []}}]
+ for index, item in enumerate(items):
+ events.append({"type": "response.output_item.added", "output_index": index,
+ "item": {key: value for key, value in item.items() if key != "result"} | {"status": "in_progress"}})
+ if item["type"] == "web_search_call":
+ for phase in ("in_progress", "searching", "completed"):
+ events.append({"type": f"response.web_search_call.{phase}", "output_index": index, "item_id": item["id"]})
+ if item["type"] == "image_generation_call":
+ events.append({"type": "response.image_generation_call.in_progress", "output_index": index,
+ "item_id": item["id"]})
+ events.append({"type": "response.image_generation_call.generating", "output_index": index,
+ "item_id": item["id"]})
+ if partial_images:
+ for partial in range(2):
+ events.append({"type": "response.image_generation_call.partial_image", "output_index": index,
+ "item_id": item["id"], "partial_image_index": partial,
+ "partial_image_b64": _PNG_B64, "output_format": "jpeg", "size": "1024x1024"})
+ events.append({"type": "response.image_generation_call.completed", "output_index": index,
+ "item_id": item["id"]})
+ if item["type"] == "message":
+ events.append({"type": "response.output_text.delta", "output_index": index, "item_id": item["id"],
+ "content_index": 0, "delta": item["content"][0]["text"]})
+ events.append({"type": "response.output_text.annotation.added", "output_index": index,
+ "item_id": item["id"], "content_index": 0, "annotation_index": 0,
+ "annotation": item["content"][0]["annotations"][0]})
+ events.append({"type": "response.output_item.done", "output_index": index, "item": item})
+ events.append({"type": "response.completed", "response": {
+ "id": "resp_hosted", "object": "response", "status": "completed", "model": MODEL, "output": items,
+ "usage": RESPONSES_USAGE}})
+ return b"".join(_sse({**event, "sequence_number": number}) for number, event in enumerate(events))
+
+
+def _anthropic_stream(blocks: list[list[dict]], *, server_tool_use: int) -> bytes:
+ events: list[dict] = [{"type": "message_start", "message": {
+ "id": "msg_hosted", "type": "message", "role": "assistant", "model": MODEL, "content": [],
+ "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 6, "output_tokens": 0}}}]
+ for index, frames in enumerate(blocks):
+ events.extend({**frame, "index": index} for frame in frames)
+ events.append({"type": "content_block_stop", "index": index})
+ events.append({"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+ "usage": {"output_tokens": 519,
+ "server_tool_use": {"web_search_requests": server_tool_use, "web_fetch_requests": 0}}})
+ events.append({"type": "message_stop"})
+ return b"".join(_sse(event) for event in events)
+
+
+def _anthropic_search(call_id: str, query: str) -> list[list[dict]]:
+ half = len(query) // 2
+ return [
+ [{"type": "content_block_start", "content_block": {"type": "server_tool_use", "id": call_id,
+ "name": "web_search", "input": {}}},
+ {"type": "content_block_delta", "delta": {"type": "input_json_delta",
+ "partial_json": f'{{"query": "{query[:half]}'}},
+ {"type": "content_block_delta", "delta": {"type": "input_json_delta",
+ "partial_json": f'{query[half:]}"}}'}}],
+ [{"type": "content_block_start", "content_block": {"type": "web_search_tool_result", "tool_use_id": call_id,
+ "content": [HIT]}}],
+ [{"type": "content_block_start", "content_block": {"type": "text", "text": ""}},
+ {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "The latest release is v8.0.24."}},
+ {"type": "content_block_delta", "delta": {"type": "citations_delta", "citation": LOCATION}}],
+ ]
+
+
+@asynccontextmanager
+async def _engine(tmp_path, monkeypatch, protocol: str, reply: bytes, *, vendor: str = "custom"):
+ """One Source of ``protocol`` on a capturing upstream; yields (base_url, token, model, captured)."""
+ captured: list[dict] = []
+
+ async def upstream(request):
+ body = await request.read()
+ captured.append({
+ "path": request.path,
+ "query": request.query_string,
+ "headers": [(key.lower(), value) for key, value in request.headers.items()],
+ "body": body,
+ })
+ return web.Response(body=reply, content_type="text/event-stream")
+
+ with _isolated_engine_adapter(tmp_path, monkeypatch) as adapter:
+ async with _server(upstream) as root:
+ source = _source("src_hosted001", "Hosted fixture", vendor=vendor, protocol=protocol, model_id=MODEL)
+ source.base_url = root + ("/v1" if protocol == "openai_responses" else "")
+ source.credential_ref = adapter.state_store.store_api_key(
+ "sk-synthetic-hosted", vendor=vendor, protocol=protocol, base_url=source.base_url,
+ )
+ config = _vibey_service(tmp_path, [source]).store.config
+ service = service_for(tmp_path, MemoryModelHubStore(config), adapter)
+ await service.runtime_start()
+ connection = adapter.supervisor.client_if_running().connection
+ prefix = adapter.state_store.list_sources()[0].prefix
+ yield connection.base_url, connection.gateway_token, f"{prefix}/{MODEL}", captured
+
+
+def _post(base_url: str, token: str, path: str, body: dict) -> tuple[int, bytes]:
+ request = urllib.request.Request(base_url + path, data=json.dumps(body).encode(), method="POST", headers={
+ "Authorization": f"Bearer {token}", "Content-Type": "application/json", "User-Agent": "avibe-hosted-test/1"})
+ try:
+ with urllib.request.urlopen(request, timeout=30) as response:
+ return response.status, response.read()
+ except urllib.error.HTTPError as exc:
+ return exc.code, exc.read()
+
+
+def _events(raw: bytes) -> list[dict]:
+ parsed = []
+ for line in raw.decode("utf-8", errors="replace").splitlines():
+ if line.startswith("data:"):
+ try:
+ parsed.append(json.loads(line[5:].strip()))
+ except json.JSONDecodeError:
+ continue
+ return parsed
+
+
+def _without_cache_control(item: dict) -> dict:
+ """Ignore only engine-added metadata on one tool or content block, never nested business data."""
+ return {key: value for key, value in item.items() if key != "cache_control"}
+
+
+def _anthropic_history(messages: list[dict]) -> list[dict]:
+ """Keep every message/block in order; normalize only text form and shallow cache metadata."""
+ normalized = []
+ for message in messages:
+ content = message["content"]
+ if isinstance(content, str):
+ content = [{"type": "text", "text": content}]
+ normalized.append({**message, "content": [_without_cache_control(block) for block in content]})
+ return normalized
+
+
+def _stable(body: bytes) -> bytes:
+ """The request bytes with only the engine's per-run ``prompt_cache_key`` value masked, in place."""
+ try:
+ payload = json.loads(body)
+ except json.JSONDecodeError:
+ return body
+ if not isinstance(payload, dict) or "prompt_cache_key" not in payload:
+ return body
+ field = b'"prompt_cache_key":' + json.dumps(payload["prompt_cache_key"]).encode()
+ assert body.count(field) == 1, "prompt_cache_key is not one compact field"
+ return body.replace(field, b'"prompt_cache_key":"<per-run>"')
+
+
+def _stable_header(name: str, value: str) -> str:
+ """Mask what differs between two runs of one engine: its per-request session, and the upstream's port."""
+ if name == "session-id":
+ return "<per-run>"
+ if name == "host":
+ return re.sub(r":[0-9]+$", ":<per-run>", value)
+ return value
+
+
+def _record(captured: list[dict], status: int, raw: bytes) -> dict:
+ """Return the one upstream call; with an evidence directory, also write both directions there."""
+ [call] = captured
+ directory = os.environ.get(EVIDENCE_DIR_ENV)
+ if directory:
+ case = os.environ["PYTEST_CURRENT_TEST"].split("::")[-1].split(" ")[0]
+ wire = {
+ "upstream": {
+ "path": call["path"],
+ "query": call["query"],
+ "headers": sorted([name, _stable_header(name, value)] for name, value in call["headers"]),
+ "body": _stable(call["body"]).decode(),
+ },
+ "downstream": {"status": status, "body": raw.decode()},
+ }
+ text = json.dumps(wire, ensure_ascii=False, indent=1, sort_keys=True)
+ (Path(directory) / f"{case}.json").write_text(text, encoding="utf-8")
+ return call
+
+
+async def test_responses_hosted_declarations_and_items_pass_through(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-001: declarations, include values and hosted output items survive CPA."""
+ items = [WEB_SEARCH_CALL, X_SEARCH_CALL, X_REASONING, IMAGE_CALL, ANSWER_ITEM]
+ reply = _responses_stream(items, partial_images=True)
+ body = {"model": None, "stream": True, "input": [
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "search and draw"}]}],
+ "tools": [WEB_SEARCH, X_SEARCH, IMAGE_GENERATION, FUNCTION], "include": INCLUDE,
+ "reasoning": {"effort": "high", "summary": "auto"}, "store": False}
+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply) as (base_url, token, model, captured):
+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})
+ call = _record(captured, status, raw)
+ sent = json.loads(call["body"])
+ assert status == 200, raw[:500]
+ assert sent["tools"] == body["tools"], sent["tools"]
+ # CPA canonicalizes include order; compare the full multiset, not membership.
+ assert sorted(sent["include"]) == sorted(INCLUDE), sent["include"]
+ # Every decoded event must survive, including progress, deltas, citations and usage.
+ assert _events(raw) == _events(reply)
+
+
+async def test_responses_hosted_history_replays_verbatim(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-002: hosted replay keeps content, order and cardinality around messages."""
+ history = [
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "search and draw"}]},
+ WEB_SEARCH_CALL, X_SEARCH_CALL, X_REASONING, XAI_OPAQUE_REASONING, IMAGE_CALL, ANSWER_ITEM,
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "and again"}]},
+ ]
+ reply = _responses_stream([ANSWER_ITEM])
+ body = {"model": None, "stream": True, "input": history, "tools": [WEB_SEARCH, X_SEARCH, IMAGE_GENERATION],
+ "include": INCLUDE, "reasoning": {"effort": "high", "summary": "auto"}, "store": False}
+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply) as (base_url, token, model, captured):
+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})
+ call = _record(captured, status, raw)
+ assert status == 200, raw[:500]
+ replayed = json.loads(call["body"])["input"]
+ # Reasoning items are not hosted items: the engine prefixes their ids (``rs_``) and drops
+ # ciphertext without a Fernet envelope unless the model is ``is-compat``. Recorded, not pinned.
+ # Compare the rest as one sequence, including message anchors, so calls cannot be duplicated,
+ # dropped, reordered, or moved across the assistant answer while still passing the contract.
+ expected_history = [item for item in history if item.get("type") != "reasoning"]
+ replayed_history = [item for item in replayed if item.get("type") != "reasoning"]
+ assert replayed_history == expected_history, replayed
+ assert _events(raw) == _events(reply)
+
+
+async def test_image_generation_is_never_injected_or_answered_by_the_engine(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-003: passthrough mode never injects an undeclared image tool."""
+ reply = _responses_stream([ANSWER_ITEM])
+ body = {"model": None, "stream": True, "input": [
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]}],
+ "tools": [FUNCTION], "store": False}
+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply) as (base_url, token, model, captured):
+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})
+ call = _record(captured, status, raw)
+ assert status == 200, raw[:500]
+ sent = json.loads(call["body"])
+ assert sent["tools"] == body["tools"], sent.get("tools")
+ assert _events(raw) == _events(reply)
+
+
+async def test_anthropic_web_search_and_its_results_pass_through(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-004: Anthropic search declarations, results, citations and usage survive CPA."""
+ reply = _anthropic_stream(_anthropic_search("srvtoolu_01", "CLIProxyAPI 版本 🦊"), server_tool_use=1)
+ body = {"model": None, "stream": True, "max_tokens": 1024,
+ "messages": [{"role": "user", "content": "search the latest CLIProxyAPI release"}],
+ "tools": [ANTHROPIC_WEB_SEARCH, ANTHROPIC_TOOL]}
+ async with _engine(tmp_path, monkeypatch, "anthropic", reply) as (base_url, token, model, captured):
+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/messages", {**body, "model": model})
+ call = _record(captured, status, raw)
+ assert status == 200, raw[:500]
+ sent = json.loads(call["body"])
+ assert [_without_cache_control(tool) for tool in sent["tools"]] == body["tools"], sent["tools"]
+ assert _events(raw) == _events(reply)
+
+
+async def test_anthropic_hosted_history_replays_verbatim(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-005: the entire Anthropic replay keeps messages, tools, results and citations."""
+ use = {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search",
+ "input": {"query": "CLIProxyAPI releases"}}
+ result = {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01", "content": [HIT]}
+ text = {"type": "text", "text": "The latest release is v8.0.24.", "citations": [LOCATION]}
+ reply = _anthropic_stream([[{"type": "content_block_start", "content_block": {"type": "text", "text": ""}},
+ {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "ok"}}]],
+ server_tool_use=0)
+ body = {"model": None, "stream": True, "max_tokens": 1024, "tools": [ANTHROPIC_WEB_SEARCH],
+ "messages": [{"role": "user", "content": "search the latest CLIProxyAPI release"},
+ {"role": "assistant", "content": [use, result, text]},
+ {"role": "user", "content": "and the one before?"}]}
+ async with _engine(tmp_path, monkeypatch, "anthropic", reply) as (base_url, token, model, captured):
+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/messages", {**body, "model": model})
+ call = _record(captured, status, raw)
+ assert status == 200, raw[:500]
+ replayed = json.loads(call["body"])["messages"]
+ assert _anthropic_history(replayed) == _anthropic_history(body["messages"]), replayed
+ assert _events(raw) == _events(reply)
+
+
+XAI_HISTORY = [
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "quote the latest post"}]},
+ XAI_REASONING, XAI_THREAD_FETCH, XAI_CALL_REASONING, ANSWER_ITEM,
+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "and the one before?"}]},
+]
+
+
+async def _replay_xai_turn(tmp_path, monkeypatch) -> tuple[list[dict], bytes, bytes]:
+ """Replay ``XAI_HISTORY`` to an xAI Source as Avibe renders one; return what arrived and the reply sent."""
+ reply = _responses_stream([ANSWER_ITEM])
+ body = {"model": None, "stream": True, "input": XAI_HISTORY, "tools": [X_SEARCH], "include": INCLUDE,
+ "reasoning": {"effort": "high", "summary": "auto"}, "store": False}
+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply, vendor="xai") as (
+ base_url, token, model, captured,
+ ):
+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})
+ call = _record(captured, status, raw)
+ assert status == 200, raw[:500]
+ return json.loads(call["body"])["input"], raw, reply
+
+
+async def test_responses_xai_history_replays_in_real_shapes(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-006: an xAI X-search turn replayed in xAI's real id and ciphertext shapes.
+
+ The 84-rune ids exceed the engine's 64-rune input id limit, which the short ids above never reach.
+ What the engine does with xAI reasoning and the over-limit call id is the next case's subject, so
+ the gate compares those bytes here instead of pinning them. Everything else is pinned: the ordered
+ non-reasoning sequence, the call's own fields, and the decoded response.
+ """
+ replayed, raw, reply = await _replay_xai_turn(tmp_path, monkeypatch)
+
+ def without_call_id(items: list[dict]) -> list[dict]:
+ return [{key: value for key, value in item.items() if not (item["type"] == "custom_tool_call" and key == "id")}
+ for item in items if item.get("type") != "reasoning"]
+
+ assert without_call_id(replayed) == without_call_id(XAI_HISTORY), replayed
+ assert _events(raw) == _events(reply)
+
+
+ strict=True, reason="engine drops/shortens xAI over-64 ids under is-compat; fixed by the id patch in #2780",
+)
+async def test_responses_xai_over_limit_ids_replay_verbatim(tmp_path, monkeypatch) -> None:
+ """MH-ENGINE-HOSTED-007: xAI's 84-rune tool-call reasoning and call ids reach xAI as it issued them.
+
+ Under ``is-compat``, which Avibe renders for xAI, the pinned engine keeps the ``rs_`` reasoning but
+ drops the ``tco_`` reasoning whole and shortens the ``ctc_`` id to 64 runes. An engine that keeps
+ them turns this strict expected failure into a failure, so that change must flip the marker.
+ """
+ replayed, _raw, _reply = await _replay_xai_turn(tmp_path, monkeypatch)
+
+ arrived = [item for item in replayed if item["type"] == "reasoning" and item.get("id") == XAI_CALL_REASONING["id"]]
+ assert [item.get("encrypted_content") for item in arrived] == [XAI_CALL_REASONING["encrypted_content"]], replayed
+ [thread_fetch] = [item for item in replayed if item["type"] == "custom_tool_call"]
+ assert thread_fetch["id"] == XAI_THREAD_FETCH["id"], thread_fetch
diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yaml
index bf42da7b4..7ab83b0d9 100644
--- a/tests/scenarios/INDEX.yaml
+++ b/tests/scenarios/INDEX.yaml
@@ -42,6 +42,7 @@ capabilities:
- tests/e2e/test_codex_request_metadata.py
- tests/e2e/test_codex_request_gateway.py
- tests/e2e/test_model_hub_runtime.py
+ - tests/e2e/test_model_hub_engine_hosted_passthrough.py
- tests/e2e/test_model_hub_sources.py
- tests/e2e/test_model_hub_usage_events.py
- tests/e2e/test_model_hub_engine_wire.py
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 2b1e46c39..6ee9279f1 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -103,6 +103,7 @@ capability:
- tests/e2e/test_codex_request_metadata.py
- tests/e2e/test_codex_request_gateway.py
- tests/e2e/test_model_hub_runtime.py
+ - tests/e2e/test_model_hub_engine_hosted_passthrough.py
- tests/e2e/test_model_hub_sources.py
- tests/e2e/test_model_hub_usage_events.py
- tests/e2e/test_model_hub_engine_wire.py
@@ -2609,6 +2610,48 @@ scenarios:
test: tests/e2e/test_model_hub_engine_wire.py::test_responses_stream_with_reasoning_and_tool_call
related_tests:
- tests/test_model_hub_runtime.py::test_only_an_xai_responses_entry_marks_its_models_is_compat
+ - id: MH-ENGINE-HOSTED-001
+ name: The real CPA Responses path preserves complete tool declarations, include cardinality and the complete decoded response event sequence
+ status: covered
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_hosted_declarations_and_items_pass_through
+ - id: MH-ENGINE-HOSTED-002
+ name: The real CPA Responses path preserves hosted history content, order and cardinality relative to user and assistant messages
+ status: covered
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_hosted_history_replays_verbatim
+ - id: MH-ENGINE-HOSTED-003
+ name: CPA image passthrough mode never injects an image tool absent from the caller request
+ status: covered
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_image_generation_is_never_injected_or_answered_by_the_engine
+ - id: MH-ENGINE-HOSTED-004
+ name: The real CPA Anthropic path preserves complete hosted search declarations and the decoded response event sequence including tool inputs, results, citations and usage
+ status: covered
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_anthropic_web_search_and_its_results_pass_through
+ - id: MH-ENGINE-HOSTED-005
+ name: The real CPA Anthropic path preserves the complete ordered replay message sequence and its tool, result and citation contents
+ status: covered
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_anthropic_hosted_history_replays_verbatim
+ - id: MH-ENGINE-HOSTED-006
+ name: The real CPA Responses path replays an xAI X-search turn in xAI's real 39- and 84-rune id and envelope-less ciphertext shapes, keeping the ordered non-reasoning sequence, the call's own fields and the decoded response
+ status: covered
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_xai_history_replays_in_real_shapes
+ - id: MH-ENGINE-HOSTED-007
+ name: The real CPA Responses path replays xAI's 84-rune tool-call reasoning with its id and ciphertext, and the 84-rune call id, as xAI issued them (expected to fail until the engine keeps over-limit upstream ids under is-compat)
+ status: xfail
+ kind: compatibility
+ layer: e2e
+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_xai_over_limit_ids_replay_verbatim
next_priority:
- MH-PROTOCOL-004
- MH-RUNTIME-001
@@ -2619,3 +2662,4 @@ next_priority:
- B9
- D6
- G3
+ - MH-ENGINE-HOSTED-007
diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py
index ba423cdc9..6d3dda0c2 100644
--- a/tests/test_ci_pipeline.py
+++ b/tests/test_ci_pipeline.py
@@ -500,6 +500,79 @@ def test_distribution_contract_command_fails_on_errors_and_skips(tmp_path, pytes
assert (result.returncode == 0) == (pytest_exit == 0 and "skipped" not in summary), result.stdout + result.stderr
+def test_hosted_wire_gate_judges_a_pull_requests_engine_change_and_blocks_merge(tmp_path: Path) -> None:
+ workflow = yaml.safe_load((ROOT / ".github/workflows/lint.yml").read_text())
+ jobs = workflow["jobs"]
+ job = jobs["model-hub-hosted-wire"]
+ # A failure reaches branch protection through the required aggregate.
+ assert "model-hub-hosted-wire" in jobs["unit-tests"]["needs"]
+ # Labeling re-runs the gate, which reads the labels as they are when it runs.
+ assert {"labeled", "unlabeled"} <= set(workflow[True]["pull_request"]["types"])
+ assert job["permissions"] == {"contents": "read", "pull-requests": "read"}
+ assert job["env"]["BASE_SHA"] == "${{ github.event.pull_request.base.sha }}"
+ steps = job["steps"]
+ detect = next(index for index, step in enumerate(steps) if step.get("id") == "manifest")
+ for step in steps[detect + 1:]:
+ assert "steps.manifest.outputs.changed == 'true'" in step["if"], step
+ gate, = [step for step in steps if "scripts/model_hub_hosted_wire_gate.py" in step.get("run", "")]
+ assert '--base "$BASE_SHA"' in gate["run"] and '--pull-request "$PR_NUMBER"' in gate["run"]
+ # The downloaded engines and the cases run where only loopback answers.
+ assert gate["run"].split().count("--loopback-only") == 1
+ assert gate["env"] == {"GH_TOKEN": "${{ github.token }}", "PR_NUMBER": "${{ github.event.pull_request.number }}"}
+
+ repo = tmp_path / "repo"
+ manifest = repo / "vibe/model_hub_runtime/cliproxyapi_manifest.json"
+ manifest.parent.mkdir(parents=True)
+
+ def git(*args: str) -> str:
+ return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip()
+
+ def commit() -> str:
+ git("add", "-A")
+ git("-c", "user.name=CI Fixture", "-c", "[email protected]", "commit", "-qm", "fixture")
+ return git("rev-parse", "HEAD")
+
+ def changed(base: str, head: str, event: str = "pull_request") -> tuple[int, str]:
+ git("checkout", "-q", head)
+ output = tmp_path / "output"
+ output.write_text("")
+ result = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", steps[detect]["run"]], cwd=repo,
+ env={**os.environ, "BASE_SHA": base, "GITHUB_EVENT_NAME": event, "GITHUB_OUTPUT": str(output)},
+ capture_output=True, text=True,
+ )
+ return result.returncode, output.read_text()
+
+ git("init", "-q")
+ manifest.write_text('{"asset_release_tag": "model-hub-engine-v8.0.23-3"}\n')
+ base = commit()
+ (repo / "README.md").write_text("unrelated\n")
+ unrelated = commit()
+ manifest.write_text('{"asset_release_tag": "model-hub-engine-v8.0.28-1"}\n')
+ engine = commit()
+
+ assert changed(base, unrelated) == (0, "changed=false\n")
+ assert changed(base, engine) == (0, "changed=true\n")
+ # The label lives on the pull request, so a push is not judged again.
+ assert changed("", engine, event="push") == (0, "changed=false\n")
+ # An unreachable or missing base fails the job; it never reads as "unchanged".
+ for missing in ("0" * 40, ""):
+ status, output = changed(missing, engine)
+ assert status != 0 and output == ""
+
+
+def test_every_linux_engine_e2e_runner_has_only_loopback() -> None:
+ """Hosted and engine E2E cases start a real engine; on Linux only the namespace keeps it off the network."""
+ release = yaml.safe_load((ROOT / ".github/workflows/model-hub-e2e.yml").read_text())
+ runs = [step["run"] for job in release["jobs"].values() for step in job["steps"] if "e2e_model_hub" in step.get("run", "")]
+ assert runs, "the release E2E runs no engine scenarios"
+ for run in runs:
+ assert run.lstrip().startswith("bash scripts/ci_loopback_only.sh python -m pytest -m e2e_model_hub"), run
+ gate, = [step for step in _jobs()["model-hub-hosted-wire"]["steps"]
+ if "scripts/model_hub_hosted_wire_gate.py" in step.get("run", "")]
+ assert "--loopback-only" in gate["run"].split()
+
+
@pytest.mark.parametrize("gate_name", ["unit-tests", "install-upgrade-regression", "windows-install-smoke"])
def test_required_gates_accept_skips_only_for_proven_docs_and_fail_closed(tmp_path: Path, gate_name: str) -> None:
gate = _jobs()[gate_name]
@@ -550,7 +623,8 @@ def test_lint_replaces_only_the_same_pr_or_branch_and_runs_no_release_listener()
def test_docs_scope_gates_all_work_without_filtering_away_required_checks() -> None:
workflow = yaml.safe_load((ROOT / ".github/workflows/lint.yml").read_text())
events = workflow[True] # PyYAML YAML 1.1 reads the Actions `on` key as True.
- assert events["pull_request"] is None
+ # Default activity types plus the labeling that re-runs the hosted wire gate.
+ assert events["pull_request"] == {"types": ["opened", "synchronize", "reopened", "labeled", "unlabeled"]}
assert not any(key.startswith("paths") for key in events["push"])
jobs = workflow["jobs"]
required = {"ruff", "unit-tests", "show-router-integration", "memory-insight-contract",
diff --git a/tests/test_model_hub_hosted_wire_gate.py b/tests/test_model_hub_hosted_wire_gate.py
new file mode 100644
index 000000000..00b7116c8
--- /dev/null
+++ b/tests/test_model_hub_hosted_wire_gate.py
@@ -0,0 +1,477 @@
+"""The hosted wire gate and its A/B runner (``scripts/model_hub_hosted_wire_gate.py``)."""
+
+from __future__ import annotations
+
+import io
+import json
+from pathlib import Path
+from types import SimpleNamespace
+import urllib.error
+from urllib.parse import urlparse
+from urllib.request import url2pathname
+
+import pytest
+
+from scripts import model_hub_hosted_wire_gate as gate
+from tests.test_model_hub_engine_release_guard import _fake_download, _manifest
+
+BASE_TAG = "model-hub-engine-v8.0.23-3"
+PR_TAG = "model-hub-engine-v8.0.28-1"
+DIFFERENCES = {
+ "test_responses_xai_history_replays_in_real_shapes": ["upstream header set", "upstream request bytes"],
+ "test_image_generation_is_never_injected_or_answered_by_the_engine": ["downstream bytes"],
+}
+SUMMARY = ("test_responses_xai_history_replays_in_real_shapes: upstream header set, upstream request bytes; "
+ "test_image_generation_is_never_injected_or_answered_by_the_engine: downstream bytes")
+
+
+def test_the_cases_mask_only_per_run_values() -> None:
+ from tests.e2e.test_model_hub_engine_hosted_passthrough import _stable, _stable_header
+
+ assert _stable_header("host", "127.0.0.1:43123") == "127.0.0.1:<per-run>"
+ assert _stable_header("host", "api.openai.com") == "api.openai.com"
+ assert _stable_header("session-id", "019a7f3e-9d1c") == "<per-run>"
+ assert _stable_header("user-agent", "codex-tui/0.154.0") == "codex-tui/0.154.0"
+ body = b'{"model":"m","prompt_cache_key":"019a7f3e","input":[ ]}'
+ assert _stable(body) == b'{"model":"m","prompt_cache_key":"<per-run>","input":[ ]}'
+ assert _stable(b'{"model":"m", "input":[ ]}') == b'{"model":"m", "input":[ ]}'
+ with pytest.raises(AssertionError, match="one compact field"):
+ _stable(b'{"prompt_cache_key": "019a7f3e"}')
+
+
+def _recorded() -> dict:
+ return {
+ "upstream": {
+ "path": "/v1/responses",
+ "query": "",
+ "headers": [["content-type", "application/json"], ["session-id", "<per-run>"]],
+ "body": '{"model":"hosted-model","prompt_cache_key":"<per-run>"}',
+ },
+ "downstream": {"status": 200, "body": "event: response.completed\ndata: {}\n\n"},
+ }
+
+
+def test_identical_recordings_have_no_differences() -> None:
+ assert gate.wire_differences({"case_a": _recorded()}, {"case_a": _recorded()}) == {}
+
+
+ ("direction", "part", "label"),
+ [
+ ("upstream", "path", "upstream request path"),
+ ("upstream", "query", "upstream request query"),
+ ("upstream", "headers", "upstream header set"),
+ ("upstream", "body", "upstream request bytes"),
+ ("downstream", "status", "downstream status"),
+ ("downstream", "body", "downstream bytes"),
+ ],
+)
+def test_each_wire_part_is_compared(direction: str, part: str, label: str) -> None:
+ changed = _recorded()
+ changed[direction][part] = 201 if part == "status" else ["x"] if part == "headers" else "x"
+
+ differences = gate.wire_differences(
+ {"case_a": _recorded(), "case_b": _recorded()}, {"case_a": changed, "case_b": _recorded()},
+ )
+
+ assert differences == {"case_a": [label]}
+
+
+def test_every_differing_part_of_a_case_is_named_in_wire_order() -> None:
+ changed = _recorded()
+ changed["downstream"]["body"] = "x"
+ changed["upstream"]["headers"] = []
+
+ assert gate.wire_differences({"case_a": _recorded()}, {"case_a": changed}) == {
+ "case_a": ["upstream header set", "downstream bytes"],
+ }
+
+
+def test_a_case_recorded_against_one_engine_is_a_difference() -> None:
+ assert gate.wire_differences({"case_a": _recorded()}, {"case_b": _recorded()}) == {
+ "case_a": ["recorded against only one engine"], "case_b": ["recorded against only one engine"],
+ }
+
+
[email protected]("labels", [None, frozenset(), frozenset({gate.CHANGE_LABEL})])
+def test_an_identical_passthrough_passes(labels) -> None:
+ assert gate.wire_change_violation({}, labels) is None
+
+
[email protected]("labels", [frozenset({gate.CHANGE_LABEL}), frozenset({"dependencies", gate.CHANGE_LABEL})])
+def test_a_labeled_difference_passes(labels) -> None:
+ assert gate.wire_change_violation(DIFFERENCES, labels) is None
+
+
+ "labels", [None, frozenset(), frozenset({"dependencies"}), frozenset({gate.CHANGE_LABEL.upper()})],
+)
+def test_an_unlabeled_difference_fails_and_names_what_differs(labels) -> None:
+ assert gate.wire_change_violation(DIFFERENCES, labels) == (
+ f"the engine changes the hosted passthrough ({SUMMARY}). If that is intended, label the pull request "
+ f"{gate.CHANGE_LABEL} and say in its body what changed and why"
+ )
+
+
+def _api(monkeypatch, response: bytes | Exception) -> list:
+ requests = []
+
+ def urlopen(request, *, timeout):
+ requests.append(request)
+ if isinstance(response, Exception):
+ raise response
+ return io.BytesIO(response)
+
+ monkeypatch.setattr(gate.urllib.request, "urlopen", urlopen)
+ return requests
+
+
+def test_labels_are_read_from_the_pull_request_as_it_is_now(monkeypatch) -> None:
+ payload = {"number": 7, "labels": [{"name": gate.CHANGE_LABEL}, {"name": "dependencies"}]}
+ requests = _api(monkeypatch, json.dumps(payload).encode())
+
+ assert gate.pull_request_labels("avibe-bot/avibe", "7", "token-1") == {gate.CHANGE_LABEL, "dependencies"}
+ [request] = requests
+ assert request.full_url == "https://api.github.com/repos/avibe-bot/avibe/pulls/7"
+ assert request.get_header("Authorization") == "Bearer token-1"
+
+ requests.clear()
+ assert gate.pull_request_labels("avibe-bot/avibe", "7", None) == {gate.CHANGE_LABEL, "dependencies"}
+ assert requests[0].get_header("Authorization") is None
+
+
+ "response",
+ [
+ urllib.error.HTTPError("https://api.github.com", 403, "rate limited", {}, None),
+ urllib.error.URLError("unreachable"),
+ TimeoutError("timed out"),
+ b"not json",
+ b'{"number": 7}',
+ b'{"labels": "engine-hosted-wire-change"}',
+ b'{"labels": [{"title": "engine-hosted-wire-change"}]}',
+ b"[]",
+ ],
+)
+def test_labels_that_cannot_be_read_count_as_none(monkeypatch, capsys, response) -> None:
+ _api(monkeypatch, response)
+
+ assert gate.pull_request_labels("avibe-bot/avibe", "7", "token-1") is None
+ assert "cannot read the pull request's labels" in capsys.readouterr().out
+
+
+def _repository(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, base: dict | bytes, pr: dict | bytes) -> None:
+ manifest = tmp_path / "repo" / gate.MANIFEST
+ manifest.parent.mkdir(parents=True)
+ manifest.write_bytes(pr if isinstance(pr, bytes) else json.dumps(pr).encode())
+ monkeypatch.setattr(gate, "REPO_ROOT", tmp_path / "repo")
+ monkeypatch.setattr(gate, "_git_show",
+ lambda revision, path: base if isinstance(base, bytes) else json.dumps(base).encode())
+
+
+def _verdicts(monkeypatch, differences: dict | None, labels: frozenset[str] | None = None) -> dict:
+ """Record the A/B and label reads ``main`` makes; ``differences`` None means it must not compare."""
+ seen = {"ab": [], "labels": []}
+
+ def compare(base, pr, work, assets, *, loopback_only):
+ if differences is None:
+ pytest.fail("ran the A/B")
+ seen["ab"].append((json.loads(base), json.loads(pr), assets, loopback_only))
+ return differences
+
+ def read_labels(repository, number, token):
+ seen["labels"].append((repository, number, token))
+ return labels
+
+ monkeypatch.setattr(gate, "hosted_wire_differences", compare)
+ monkeypatch.setattr(gate, "pull_request_labels", read_labels)
+ return seen
+
+
[email protected]("pr", [{"asset_release_tag": BASE_TAG}, {"asset_release_tag": BASE_TAG, "note": "x"}])
+def test_an_unchanged_engine_has_nothing_to_compare(tmp_path, monkeypatch, capsys, pr) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, pr)
+ seen = _verdicts(monkeypatch, None)
+
+ assert gate.main(["--base", "base", "--pull-request", "7"]) == 0
+ assert seen == {"ab": [], "labels": []}
+ assert "the engine is unchanged" in capsys.readouterr().out
+
+
[email protected]("pr", [b"{", b"[]", b'{"version": "v8.0.28"}'])
+def test_a_manifest_without_an_engine_tag_fails(tmp_path, monkeypatch, capsys, pr) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, pr)
+ _verdicts(monkeypatch, None)
+
+ assert gate.main(["--base", "base"]) == 1
+ assert "::error::hosted_wire gate: cannot read either manifest's asset_release_tag" in capsys.readouterr().out
+
+
+def test_an_identical_passthrough_passes_without_reading_labels(tmp_path, monkeypatch, capsys) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})
+ seen = _verdicts(monkeypatch, {})
+
+ assert gate.main(["--base", "base", "--pull-request", "7", "--head-assets", str(tmp_path / "build")]) == 0
+ assert seen == {"ab": [({"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG}, tmp_path / "build", False)],
+ "labels": []}
+ assert "hosted passthrough byte-identical" in capsys.readouterr().out
+
+
+def test_the_loopback_only_flag_reaches_the_ab(tmp_path, monkeypatch) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})
+ seen = _verdicts(monkeypatch, {})
+
+ assert gate.main(["--base", "base", "--loopback-only"]) == 0
+ assert seen["ab"] == [({"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG}, None, True)]
+
+
+ ("labels", "status"),
+ [(frozenset({gate.CHANGE_LABEL}), 0), (frozenset({"dependencies"}), 1), (frozenset(), 1), (None, 1)],
+)
+def test_a_difference_is_judged_on_the_labels_read_when_the_gate_runs(
+ tmp_path, monkeypatch, capsys, labels, status,
+) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})
+ monkeypatch.setenv("GH_TOKEN", "token-1")
+ seen = _verdicts(monkeypatch, DIFFERENCES, labels)
+
+ assert gate.main(["--base", "base", "--pull-request", "7", "--repository", "avibe-bot/avibe"]) == status
+ assert seen["labels"] == [("avibe-bot/avibe", "7", "token-1")]
+ out = capsys.readouterr().out
+ assert "hosted passthrough differs" in out
+ assert " test_image_generation_is_never_injected_or_answered_by_the_engine: downstream bytes" in out
+ if status:
+ assert f"::error file={gate.MANIFEST}::hosted_wire gate: the engine changes the hosted passthrough ({SUMMARY})" in out
+ else:
+ assert f"labeled {gate.CHANGE_LABEL}, so the difference is intended" in out
+
+
+def test_without_a_pull_request_no_label_can_accept_a_difference(tmp_path, monkeypatch, capsys) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})
+ seen = _verdicts(monkeypatch, DIFFERENCES, frozenset({gate.CHANGE_LABEL}))
+
+ assert gate.main(["--base", "base", "--head-assets", str(tmp_path / "build")]) == 1
+ assert seen["labels"] == []
+ out = capsys.readouterr().out
+ assert "no pull request, so no label can accept the difference" in out
+ assert f"::error file={gate.MANIFEST}::hosted_wire gate: the engine changes the hosted passthrough" in out
+
+
+def test_a_gate_that_cannot_reach_a_verdict_fails(tmp_path, monkeypatch, capsys) -> None:
+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})
+
+ def unreachable(*_args, **_kwargs):
+ raise gate.GateError("the PR engine is not the asset its manifest pins")
+
+ monkeypatch.setattr(gate, "hosted_wire_differences", unreachable)
+ monkeypatch.setattr(gate, "pull_request_labels", lambda *a: frozenset({gate.CHANGE_LABEL}))
+
+ assert gate.main(["--base", "base", "--pull-request", "7"]) == 1
+ assert "::error::hosted_wire gate: the PR engine is not the asset its manifest pins" in capsys.readouterr().out
+
+
+def test_the_ab_takes_base_from_its_release_and_the_pr_engine_from_the_given_build(tmp_path, monkeypatch) -> None:
+ runs = []
+
+ def run_cases(side, manifest_bytes, platform, work, asset_dir=None, *, must_pass, loopback_only):
+ runs.append((side, manifest_bytes, work, asset_dir, must_pass, loopback_only))
+ changed = _recorded()
+ if side == "PR":
+ changed["downstream"]["body"] = "changed"
+ return {"case_a": changed}
+
+ monkeypatch.setattr(gate, "_run_cases", run_cases)
+
+ differences = gate.hosted_wire_differences(b"base", b"pr", tmp_path, tmp_path / "build", loopback_only=True)
+
+ assert differences == {"case_a": ["downstream bytes"]}
+ # Only the engine being adopted must pass; base must record every case. Both run without network.
+ assert runs == [("base", b"base", tmp_path / "base", None, False, True),
+ ("PR", b"pr", tmp_path / "head", tmp_path / "build", True, True)]
+
+
+CASE_NAMES = ("test_one", "test_two")
+
+
+def _fake_pytest(*, returncode: int = 0, outcomes: dict | None = None, recorded: tuple = CASE_NAMES):
+ """Stand in for the case run: check the offline manifest, then write evidence and a JUnit report."""
+ runs = []
+
+ def run(args, *, cwd, env, check):
+ manifest = json.loads(Path(env["VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH"]).read_text(encoding="utf-8"))
+ [url] = [asset["url"] for asset in manifest["assets"] if asset["platform"] == "linux-amd64"]
+ assert urlparse(url).scheme == "file" and Path(url2pathname(urlparse(url).path)).is_file()
+ runs.append(args)
+ for name in recorded:
+ (Path(env[gate.EVIDENCE_DIR_ENV]) / f"{name}.json").write_text(json.dumps(_recorded()), encoding="utf-8")
+ cases = "".join(
+ f'<testcase name="{name}">{(outcomes or {}).get(name, "")}</testcase>' for name in CASE_NAMES
+ )
+ report = next(arg for arg in args if arg.startswith("--junitxml=")).removeprefix("--junitxml=")
+ Path(report).write_text(f"<testsuites><testsuite>{cases}</testsuite></testsuites>", encoding="utf-8")
+ return SimpleNamespace(returncode=returncode)
+
+ return SimpleNamespace(run=run), runs
+
+
+def _release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[bytes, dict, dict]:
+ manifest_path, owned, built = _manifest(tmp_path)
+ monkeypatch.setattr(gate.release_guard, "_download", _fake_download(owned))
+ return manifest_path.read_bytes(), owned, built
+
+
[email protected]("loopback_only", [False, True])
+def test_the_cases_run_in_the_loopback_only_namespace_when_asked(tmp_path, monkeypatch, loopback_only) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ fake, runs = _fake_pytest()
+ monkeypatch.setattr(gate, "subprocess", fake)
+
+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True, loopback_only=loopback_only)
+
+ [args] = runs
+ pytest_at = args.index("pytest") - 2
+ assert args[:pytest_at] == (["bash", gate.LOOPBACK_ONLY] if loopback_only else [])
+ assert (gate.REPO_ROOT / gate.LOOPBACK_ONLY).is_file()
+
+
+def test_a_published_engine_runs_the_cases_from_its_verified_archive(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ fake, runs = _fake_pytest()
+ monkeypatch.setattr(gate, "subprocess", fake)
+
+ cases = gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)
+
+ assert cases == {name: _recorded() for name in CASE_NAMES}
+ [args] = runs
+ assert args[-1] == gate.CASES and "e2e_model_hub" in args
+
+
+ "fake_kwargs",
+ [{"returncode": 1}, {"outcomes": {"test_two": "<skipped/>"}}, {"outcomes": {"test_two": "<failure/>"}},
+ {"outcomes": {"test_two": "<error/>"}}],
+)
+def test_the_pr_engine_must_pass_every_case(tmp_path, monkeypatch, fake_kwargs) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ monkeypatch.setattr(gate, "subprocess", _fake_pytest(**fake_kwargs)[0])
+
+ with pytest.raises(gate.GateError, match="did not all pass against the PR engine"):
+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)
+
+
+def test_a_case_that_fails_as_its_strict_xfail_says_counts_as_passing(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ xfailed = '<skipped type="pytest.xfail" message="known engine defect" />'
+ monkeypatch.setattr(gate, "subprocess", _fake_pytest(outcomes={"test_two": xfailed})[0])
+
+ cases = gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)
+
+ assert cases == {name: _recorded() for name in CASE_NAMES}
+
+
+ "outcome",
+ [
+ # An engine that fixed the defect: strict xfail reports the pass as a failure.
+ '<failure message="[XPASS(strict)] known engine defect">[XPASS(strict)] known engine defect</failure>',
+ '<skipped type="pytest.skip" message="nope" />',
+ "<skipped />",
+ '<skipped type="pytest.xfail" /><error message="teardown" />',
+ ],
+)
+def test_only_an_expected_failure_stands_in_for_a_pass(tmp_path, monkeypatch, outcome) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ monkeypatch.setattr(gate, "subprocess", _fake_pytest(outcomes={"test_two": outcome})[0])
+
+ with pytest.raises(gate.GateError, match="did not all pass against the PR engine"):
+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)
+
+
+def test_a_base_engine_that_fails_a_case_is_compared_as_recorded(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ fake, _runs = _fake_pytest(returncode=1, outcomes={"test_two": "<failure/>"})
+ monkeypatch.setattr(gate, "subprocess", fake)
+
+ cases = gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=False)
+
+ assert cases == {name: _recorded() for name in CASE_NAMES}
+
+
[email protected]("must_pass", [True, False])
+def test_every_case_must_be_recorded(tmp_path, monkeypatch, must_pass) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ # A case skipped or broken before its exchange records nothing.
+ fake, _runs = _fake_pytest(outcomes={"test_two": "<skipped/>"}, recorded=("test_one",))
+ monkeypatch.setattr(gate, "subprocess", fake)
+
+ with pytest.raises(gate.GateError, match=r"did not all pass|recorded \['test_one'\], expected \['test_one', 'test_two'\]"):
+ gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=must_pass)
+
+
+def test_a_case_run_that_collects_nothing_fails(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+
+ def collected_nothing(args, *, cwd, env, check):
+ report = next(arg for arg in args if arg.startswith("--junitxml=")).removeprefix("--junitxml=")
+ Path(report).write_text("<testsuites><testsuite/></testsuites>", encoding="utf-8")
+ return SimpleNamespace(returncode=0)
+
+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=collected_nothing))
+
+ with pytest.raises(gate.GateError, match="did not all pass"):
+ gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=False)
+
+
+def test_a_case_run_without_a_report_fails(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)
+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=0)))
+
+ with pytest.raises(gate.GateError, match="left no report"):
+ gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=False)
+
+
+def test_an_engine_that_is_not_its_manifests_asset_never_runs(tmp_path, monkeypatch) -> None:
+ manifest_bytes, owned, _built = _release(tmp_path, monkeypatch)
+ url = next(url for url in owned if url.endswith("_linux_amd64.tar.gz"))
+ monkeypatch.setattr(gate.release_guard, "_download", _fake_download({**owned, url: b"tampered"}))
+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=lambda *a, **k: pytest.fail("ran the cases")))
+
+ with pytest.raises(gate.GateError, match="the PR engine is not the asset its manifest pins"):
+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)
+ with pytest.raises(gate.GateError, match="the PR manifest has no linux-riscv64 asset"):
+ gate._run_cases("PR", manifest_bytes, "linux-riscv64", tmp_path / "riscv", must_pass=True)
+ with pytest.raises(gate.GateError, match="the PR manifest is invalid"):
+ gate._run_cases("PR", b'{"schema_version": 2}', "linux-amd64", tmp_path / "invalid", must_pass=True)
+
+
+def _build_output(tmp_path: Path, manifest_bytes: bytes, built: dict) -> Path:
+ output = tmp_path / "build"
+ output.mkdir()
+ (output / "model-hub-engine-manifest.json").write_bytes(manifest_bytes)
+ for name, archive in built.items():
+ (output / name).write_bytes(archive)
+ (output / f"{name}.sha256").write_text(f"{gate.release_guard._sha256(output / name)} {name}\n")
+ return output
+
+
+def test_a_local_candidate_runs_from_its_build_output_without_downloading(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, built = _release(tmp_path, monkeypatch)
+ output = _build_output(tmp_path, manifest_bytes, built)
+ monkeypatch.setattr(gate.release_guard, "_download", lambda *a, **k: pytest.fail("downloaded"))
+ fake, runs = _fake_pytest()
+ monkeypatch.setattr(gate, "subprocess", fake)
+
+ assert gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", output, must_pass=True)
+ assert len(runs) == 1
+
+
+def test_a_local_candidate_must_be_exactly_what_its_manifest_pins(tmp_path, monkeypatch) -> None:
+ manifest_bytes, _owned, built = _release(tmp_path, monkeypatch)
+ output = _build_output(tmp_path, manifest_bytes, built)
+ # A build from another manifest: the copy publication would upload differs.
+ (output / "model-hub-engine-manifest.json").write_bytes(manifest_bytes + b"\n")
+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=lambda *a, **k: pytest.fail("ran the cases")))
+
+ with pytest.raises(gate.GateError, match="the PR engine is not the asset its manifest pins"):
+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", output, must_pass=True)