Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
ci(model-hub): fail an engine change that alters hosted passthrough, unless labeled (#2782)
Call White5bb4cafc1146
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.ymlindex e23a9c812..c73907d92 100644--- a/.github/workflows/lint.yml+++ b/.github/workflows/lint.yml@@ -2,6 +2,8 @@ name: linton:pull_request:+ # Labeling re-runs the hosted wire gate, which reads the labels it runs with.+ types: [opened, synchronize, reopened, labeled, unlabeled]push:branches:- master@@ -332,14 +334,89 @@ jobs:if-no-files-found: ignoreretention-days: 7+ model-hub-hosted-wire:+ needs: changes+ if: needs.changes.outputs.docs_only == 'false'+ # A pull request that replaces the engine must not change how it passes+ # hosted tools through, unless labeled engine-hosted-wire-change+ # (docs/plans/model-hub-cpa-dependency.md). Any other change passes after+ # one diff, without installing anything.+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read+ pull-requests: read+ env:+ BASE_SHA: ${{ github.event.pull_request.base.sha }}+ steps:+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ # The synthetic PR merge and both parents, base included.+ fetch-depth: 2+ - name: Detect an engine manifest change+ id: manifest+ run: |+ if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then+ # The label that may accept a difference lives on the pull request.+ echo "Judged on the pull request; nothing to do on $GITHUB_EVENT_NAME."+ echo "changed=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null || git fetch --no-tags --depth=1 origin "$BASE_SHA"+ if git diff --quiet "$BASE_SHA" HEAD -- vibe/model_hub_runtime/cliproxyapi_manifest.json; then+ echo "changed=false" >> "$GITHUB_OUTPUT"+ else+ echo "changed=true" >> "$GITHUB_OUTPUT"+ fi+ - name: Prepare hermetic UI asset placeholder+ if: steps.manifest.outputs.changed == 'true'+ run: |+ mkdir -p ui/dist+ printf '%s\n' '<!doctype html><html><body></body></html>' > ui/dist/index.html+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6+ if: steps.manifest.outputs.changed == 'true'+ with:+ python-version: "3.12"+ - name: Install pinned uv+ if: steps.manifest.outputs.changed == 'true'+ run: |+ python -m pip install --disable-pip-version-check --only-binary=:all: --no-deps uv==0.12.10+ - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5+ if: steps.manifest.outputs.changed == 'true'+ with:+ path: ~/.cache/uv+ key: ${{ runner.os }}-${{ runner.arch }}-ci-uv-0.12.10-py3.12-${{ hashFiles('pyproject.toml', 'uv.lock') }}+ - name: Install package and test dependencies+ if: steps.manifest.outputs.changed == 'true'+ run: uv pip install --system -e . --group dev+ - name: Compare the hosted passthrough of both engines+ if: steps.manifest.outputs.changed == 'true'+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ run: >-+ python scripts/model_hub_hosted_wire_gate.py --base "$BASE_SHA"+ --pull-request "$PR_NUMBER" --work-dir "$RUNNER_TEMP/hosted-wire" --loopback-only+ - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6+ if: always() && steps.manifest.outputs.changed == 'true'+ with:+ name: model-hub-hosted-wire-${{ github.sha }}-${{ github.run_attempt }}+ # Synthetic fixtures on loopback only: per-case wire records and reports.+ path: |+ ${{ runner.temp }}/hosted-wire/*/evidence+ ${{ runner.temp }}/hosted-wire/*/cases.xml+ if-no-files-found: ignore+ retention-days: 14+unit-tests:runs-on: ubuntu-latest# Keep UI validation in the existing required gate while allowing artifact- # consumers to start as soon as their production bundle is built.- needs: [changes, unit-test-shards, scenario-evidence, ui-checks, ui-browser-checks]+ # consumers to start as soon as their production bundle is built. The+ # hosted wire gate reports through it too, under no new required context.+ needs: [changes, unit-test-shards, scenario-evidence, ui-checks, ui-browser-checks, model-hub-hosted-wire]if: always()steps:- - name: Verify unit test shards and UI checks+ - name: Verify unit test shards, UI checks and the hosted_wire gateenv:SCOPE_RESULT: ${{ needs.changes.result }}DOCS_ONLY: ${{ needs.changes.outputs.docs_only }}@@ -347,6 +424,7 @@ jobs:EVIDENCE_RESULT: ${{ needs['scenario-evidence'].result }}UI_RESULT: ${{ needs['ui-checks'].result }}BROWSER_RESULT: ${{ needs['ui-browser-checks'].result }}+ HOSTED_WIRE_RESULT: ${{ needs['model-hub-hosted-wire'].result }}run: |if [ "$SCOPE_RESULT" != "success" ]; thenecho "Change classification did not succeed."@@ -357,13 +435,13 @@ jobs:false) expected="success" ;;*) echo "Missing or invalid change classification."; exit 1 ;;esac- for result in "$UNIT_RESULT" "$EVIDENCE_RESULT" "$UI_RESULT" "$BROWSER_RESULT"; do+ for result in "$UNIT_RESULT" "$EVIDENCE_RESULT" "$UI_RESULT" "$BROWSER_RESULT" "$HOSTED_WIRE_RESULT"; doif [ "$result" != "$expected" ]; then- echo "Unit/UI checks finished with result: $result"+ echo "Unit/UI/hosted_wire checks finished with result: $result"exit 1fidone- echo "Unit/UI results match the classified scope (docs_only=$DOCS_ONLY)."+ echo "Unit/UI/hosted_wire results match the classified scope (docs_only=$DOCS_ONLY)."# Every case a covered scenario row cites as evidence ran and passed in some# job, or a release workflow runs it (scripts/ci_scenario_evidence.py). Itdiff --git a/.github/workflows/model-hub-e2e.yml b/.github/workflows/model-hub-e2e.ymlindex d441e781e..a6b04ecc0 100644--- a/.github/workflows/model-hub-e2e.yml+++ b/.github/workflows/model-hub-e2e.yml@@ -78,9 +78,12 @@ jobs:EOFecho "VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH=$RUNNER_TEMP/model-hub-engine/offline-manifest.json" >> "$GITHUB_ENV"+ # Every scenario is hermetic, so it runs where only loopback answers: an+ # engine or CLI that ignores its loopback upstream fails at once instead+ # of reaching a real provider. The downloads above stay outside.- name: Run the Model Hub E2Erun: |- python -m pytest -m e2e_model_hub tests/e2e -p no:cacheprovider -ra \+ bash scripts/ci_loopback_only.sh python -m pytest -m e2e_model_hub tests/e2e -p no:cacheprovider -ra \--basetemp="$RUNNER_TEMP/e2e" 2>&1 | tee model-hub-e2e.logexit "${PIPESTATUS[0]}"diff --git a/docs/plans/model-hub-cpa-dependency.md b/docs/plans/model-hub-cpa-dependency.mdindex 76899d03f..11af29e5f 100644--- a/docs/plans/model-hub-cpa-dependency.md+++ b/docs/plans/model-hub-cpa-dependency.md@@ -73,6 +73,47 @@ manifest's `build.patches` records; the packaged manifest records the resultingarchive and binary digests. Rolling back means reverting the whole adoptionchange, which restores the previous patch directory and manifest together.+### Hosted passthrough++Hosted tools rely on the engine passing them through unchanged, and they are+assumed rather than probed, so a passthrough regression in an engine release+would reach users unnoticed. `lint` guards it as `model-hub-hosted-wire`, which+reports through the required `unit-tests` check, on every pull request that+replaces the engine (changes the manifest's `asset_release_tag`):++- `scripts/model_hub_hosted_wire_gate.py` downloads both engines' assets for the+ runner's platform from their published releases and verifies them against+ each manifest. It then runs the cases in+ `tests/e2e/test_model_hub_engine_hosted_passthrough.py` against both and+ compares each case's upstream request path, query, header set and bytes, and+ its downstream status and bytes. Only per-run values are masked: the engine's+ `prompt_cache_key` and `session-id`, and the port in the capturing upstream's+ `Host`. The new engine must pass every case, where a case marked+ `xfail(strict=True)` passes by failing as its marker says. The base engine+ only has to record each case, so a case it fails is compared as recorded.+ An engine that fixes a known defect turns its strict expected failure into a+ failure, so that change flips the marker and carries the label below.+- A byte-identical passthrough passes. A difference fails and names the cases+ and parts that differ, unless the pull request is labeled+ `engine-hosted-wire-change`. An intended change, a rollback included, needs+ that label and a note in the pull request body saying what changed and why.+ The gate reads the labels when it runs, adding or removing one re-runs+ `lint`, and labels that cannot be read count as none.++The gate judges pull requests, where the label lives; a push to master does not+repeat it.++Before dispatching an engine build, run the gate on the local candidate build+and include its output in the dispatch request; engine sign-off requires it.+Without a pull request no label applies, so the output shows any difference as+a failure for review:++```sh+python scripts/model_hub_hosted_wire_gate.py --base origin/master --head-assets <build output dir>+```++The pull request's gate stays the enforcement of record.+## Acceptance Criteria- A user who never enables Model Hub receives the CPA target during normal Avibediff --git a/scripts/model_hub_hosted_wire_gate.py b/scripts/model_hub_hosted_wire_gate.pynew file mode 100644index 000000000..d36b66830--- /dev/null+++ b/scripts/model_hub_hosted_wire_gate.py@@ -0,0 +1,254 @@+#!/usr/bin/env python3+"""Fail an engine change that alters how the engine passes hosted tools through.++Native hosted tools are assumed rather than probed, so a passthrough regression+in an engine release would otherwise reach users unnoticed. When a change+replaces the engine, the hosted passthrough cases run against the base and the+changed engine and are compared byte for byte. Any difference fails, unless the+pull request carries the label that says the change is intended. PR CI takes+both engines from their published releases. Before a candidate is published,+``--head-assets`` takes it from the local build output instead. See+docs/plans/model-hub-cpa-dependency.md.+"""++from __future__ import annotations++import argparse+from collections.abc import Mapping+import json+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import urllib.request+from xml.etree import ElementTree++REPO_ROOT = Path(__file__).resolve().parents[1]+if str(REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(REPO_ROOT))++from scripts import model_hub_engine_release_guard as release_guard # noqa: E402++MANIFEST = "vibe/model_hub_runtime/cliproxyapi_manifest.json"+CASES = "tests/e2e/test_model_hub_engine_hosted_passthrough.py"+# Runs a command where only loopback answers; Linux CI only (needs passwordless sudo).+LOOPBACK_ONLY = "scripts/ci_loopback_only.sh"+EVIDENCE_DIR_ENV = "AVIBE_HOSTED_WIRE_EVIDENCE_DIR"+CHANGE_LABEL = "engine-hosted-wire-change"+# What each case records, in the order a difference is reported.+WIRE_PARTS = (+ ("upstream", "path", "upstream request path"),+ ("upstream", "query", "upstream request query"),+ ("upstream", "headers", "upstream header set"),+ ("upstream", "body", "upstream request bytes"),+ ("downstream", "status", "downstream status"),+ ("downstream", "body", "downstream bytes"),+)+++class GateError(RuntimeError):+ """Raised when the gate cannot reach a verdict."""+++def wire_differences(base_cases: Mapping[str, dict], head_cases: Mapping[str, dict]) -> dict[str, list[str]]:+ """Map every case whose two recordings differ to the wire parts that differ."""+ differences = {}+ for case in sorted(base_cases.keys() | head_cases.keys()):+ if case not in base_cases or case not in head_cases:+ differences[case] = ["recorded against only one engine"]+ continue+ parts = [label for direction, part, label in WIRE_PARTS+ if base_cases[case][direction][part] != head_cases[case][direction][part]]+ if parts:+ differences[case] = parts+ return differences+++def wire_change_violation(differences: Mapping[str, list[str]], labels: frozenset[str] | None) -> str | None:+ """Return why a hosted passthrough difference blocks the change, or None.++ ``labels`` are the pull request's labels, or None when there is no pull+ request or its labels could not be read; both count as unlabeled.+ """+ if not differences or (labels is not None and CHANGE_LABEL in labels):+ return None+ summary = "; ".join(f"{case}: {', '.join(parts)}" for case, parts in differences.items())+ return (f"the engine changes the hosted passthrough ({summary}). If that is intended, label the pull "+ f"request {CHANGE_LABEL} and say in its body what changed and why")+++def pull_request_labels(repository: str, number: str, token: str | None) -> frozenset[str] | None:+ """The pull request's labels as they are now, or None when they cannot be read."""+ headers = {"Accept": "application/vnd.github+json", "User-Agent": "avibe-model-hub-hosted-wire-gate/1"}+ if token:+ headers["Authorization"] = f"Bearer {token}"+ request = urllib.request.Request(f"https://api.github.com/repos/{repository}/pulls/{number}", headers=headers)+ try:+ with urllib.request.urlopen(request, timeout=30) as response:+ return frozenset(label["name"] for label in json.load(response)["labels"])+ except (OSError, ValueError, KeyError, TypeError) as exc:+ print(f"::warning::hosted_wire gate: cannot read the pull request's labels ({type(exc).__name__}); "+ "judging it as unlabeled")+ return None+++def _git_show(revision: str, path: str) -> bytes:+ try:+ return subprocess.run(+ ["git", "-C", str(REPO_ROOT), "show", f"{revision}:{path}"],+ check=True, capture_output=True, timeout=60,+ ).stdout+ except (OSError, subprocess.SubprocessError) as exc:+ raise GateError(f"cannot read {path} at the base revision {revision}") from exc+++def _met_its_expectation(testcase: ElementTree.Element) -> bool:+ """A case passed, or failed as its strict ``xfail`` marker says it must; an unexpected pass fails."""+ if testcase.find("failure") is not None or testcase.find("error") is not None:+ return False+ skipped = testcase.find("skipped")+ return skipped is None or skipped.get("type") == "pytest.xfail"+++def _run_cases(+ side: str, manifest_bytes: bytes, platform: str, work: Path, asset_dir: Path | None = None,+ *, must_pass: bool, loopback_only: bool = False,+) -> dict[str, dict]:+ """Run the case set against one side's engine; return each case's recorded wire.++ Each case records the exchange before asserting on it. The PR engine must pass+ every case. The base engine only has to record every case: one that fails its+ assertions there has a different wire, which the comparison reports.+ """+ work.mkdir(parents=True)+ pinned = work / "cliproxyapi_manifest.json"+ pinned.write_bytes(manifest_bytes)+ try:+ spec = release_guard.load_release_spec(pinned)+ except release_guard.ReleaseGuardError as exc:+ raise GateError(f"the {side} manifest is invalid: {exc}") from exc+ archive = next((item for item in spec.archives if item.platform == platform), None)+ if archive is None:+ raise GateError(f"the {side} manifest has no {platform} asset")+ try:+ if asset_dir is None:+ local_archive = work / archive.name+ release_guard._download(archive.owned_url, local_archive, max_bytes=archive.size)+ release_guard._verify_archive(local_archive, archive)+ else:+ # Exactly what publication would upload: every asset, sidecar and the manifest copy.+ release_guard.verify_release_assets(pinned, asset_dir)+ local_archive = asset_dir.resolve() / archive.name+ except release_guard.ReleaseGuardError as exc:+ raise GateError(f"the {side} engine is not the asset its manifest pins: {exc}") from exc+ # The E2E harness installs offline from a file URL; the archive was verified above.+ manifest = json.loads(manifest_bytes)+ for asset in manifest["assets"]:+ if asset["platform"] == platform:+ asset["url"] = local_archive.as_uri()+ offline_manifest = work / "offline-manifest.json"+ offline_manifest.write_text(json.dumps(manifest), encoding="utf-8")++ evidence, report = work / "evidence", work / "cases.xml"+ evidence.mkdir()+ print(f"hosted_wire gate: running the hosted passthrough cases against the {side} engine "+ f"{manifest['asset_release_tag']} ({platform})", flush=True)+ command = [sys.executable, "-m", "pytest", "-m", "e2e_model_hub", "-p", "no:cacheprovider",+ "-q", "-ra", f"--junitxml={report}", CASES]+ if loopback_only:+ # The engine was downloaded above; the cases and the engine they start need only loopback,+ # so one that ignores its loopback Source fails at once instead of reaching the network.+ command = ["bash", LOOPBACK_ONLY, *command]+ result = subprocess.run(+ command,+ cwd=REPO_ROOT,+ env={**os.environ, "VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH": str(offline_manifest),+ EVIDENCE_DIR_ENV: str(evidence)},+ check=False,+ )+ try:+ testcases = list(ElementTree.parse(report).getroot().iter("testcase"))+ except (OSError, ElementTree.ParseError) as exc:+ raise GateError(f"the {side} engine's case run left no report") from exc+ names = {testcase.get("name") for testcase in testcases}+ passed = all(_met_its_expectation(testcase) for testcase in testcases)+ if not testcases or (must_pass and (result.returncode != 0 or not passed)):+ raise GateError(f"the hosted passthrough cases did not all pass against the {side} engine")+ cases = {path.stem: json.loads(path.read_text(encoding="utf-8")) for path in evidence.glob("*.json")}+ if set(cases) != names:+ raise GateError(f"the {side} engine's run recorded {sorted(cases)}, expected {sorted(names)}")+ return cases+++def hosted_wire_differences(+ base_bytes: bytes, head_bytes: bytes, work: Path, head_assets: Path | None = None,+ *, loopback_only: bool = False,+) -> dict[str, list[str]]:+ from vibe.model_hub_runtime.installer import EngineRuntimeManager++ platform = EngineRuntimeManager(runtime_dir=work / "probe", offline=True).host_platform()+ base_cases = _run_cases("base", base_bytes, platform, work / "base",+ must_pass=False, loopback_only=loopback_only)+ head_cases = _run_cases("PR", head_bytes, platform, work / "head", head_assets,+ must_pass=True, loopback_only=loopback_only)+ return wire_differences(base_cases, head_cases)+++def _build_parser() -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--base", required=True, help="revision holding the base manifest")+ parser.add_argument(+ "--head-assets", type=Path,+ help="a local candidate's build output directory, instead of its published release",+ )+ parser.add_argument("--pull-request", help="number of the pull request whose labels may accept a difference")+ parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "avibe-bot/avibe"))+ parser.add_argument("--work-dir", type=Path, help="keep downloads and per-case evidence here")+ parser.add_argument(+ "--loopback-only", action="store_true",+ help=f"run the cases in a network namespace with only loopback ({LOOPBACK_ONLY}; Linux CI)",+ )+ return parser+++def main(argv: list[str] | None = None) -> int:+ args = _build_parser().parse_args(argv)+ try:+ base_bytes = _git_show(args.base, MANIFEST)+ head_bytes = (REPO_ROOT / MANIFEST).read_bytes()+ try:+ base_tag = json.loads(base_bytes)["asset_release_tag"]+ head_tag = json.loads(head_bytes)["asset_release_tag"]+ except (ValueError, KeyError, TypeError) as exc:+ raise GateError(f"cannot read either manifest's asset_release_tag ({type(exc).__name__})") from exc+ if base_tag == head_tag:+ print(f"hosted_wire gate: the engine is unchanged ({head_tag}); nothing to compare")+ return 0+ with tempfile.TemporaryDirectory(prefix="hosted-wire-") as scratch:+ differences = hosted_wire_differences(base_bytes, head_bytes, args.work_dir or Path(scratch),+ args.head_assets, loopback_only=args.loopback_only)+ except GateError as exc:+ print(f"::error::hosted_wire gate: {exc}")+ return 1+ if not differences:+ print(f"hosted_wire gate: {base_tag} -> {head_tag}: hosted passthrough byte-identical")+ return 0+ print(f"hosted_wire gate: {base_tag} -> {head_tag}: hosted passthrough differs")+ for case, parts in differences.items():+ print(f" {case}: {', '.join(parts)}")+ labels = None+ if args.pull_request:+ labels = pull_request_labels(args.repository, args.pull_request, os.environ.get("GH_TOKEN"))+ else:+ print("hosted_wire gate: no pull request, so no label can accept the difference")+ violation = wire_change_violation(differences, labels)+ if violation:+ print(f"::error file={MANIFEST}::hosted_wire gate: {violation}")+ return 1+ print(f"hosted_wire gate: the pull request is labeled {CHANGE_LABEL}, so the difference is intended")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tests/e2e/test_model_hub_engine_hosted_passthrough.py b/tests/e2e/test_model_hub_engine_hosted_passthrough.pynew file mode 100644index 000000000..b99ebbce6--- /dev/null+++ b/tests/e2e/test_model_hub_engine_hosted_passthrough.py@@ -0,0 +1,493 @@+"""Hosted tools preserve business payloads through the real engine on same-protocol paths.++These engine-boundary scenarios protect same-protocol hosted declarations,+``include`` values, output items and replayed history against CPA upgrades.+The existing mocked gateway and consumer tests do not run CPA's translators.+Each case sends one request to the real engine under Avibe's rendered config;+a synthetic loopback upstream records request bytes and replays fixed wire+fixtures. This is not a live provider or a complete Vibey-turn acceptance test.+With ``AVIBE_HOSTED_WIRE_EVIDENCE_DIR`` set, each case also writes both+directions there, so ``scripts/model_hub_hosted_wire_gate.py`` can compare two+engines' runs byte for byte.+"""++from __future__ import annotations++import asyncio+import base64+import hashlib+import json+import os+from pathlib import Path+import re+import urllib.error+import urllib.request+from contextlib import asynccontextmanager++from aiohttp import web+import pytest++from tests.e2e.test_model_hub_runtime import _isolated_engine_adapter+from tests.test_model_hub_google import _server+from tests.scenario_harness.model_hub import MemoryModelHubStore, service_for+from tests.test_model_hub_l3 import _source+from tests.test_model_hub_vibey_consumer import _vibey_service+++pytestmark = pytest.mark.e2e_model_hub++EVIDENCE_DIR_ENV = "AVIBE_HOSTED_WIRE_EVIDENCE_DIR"+MODEL = "hosted-model"+# Fernet-shaped, so the engine keeps it with its id (CPA drops other shapes).+_FERNET = "g" + "A" * 97+_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+WEB_SEARCH = {"type": "web_search"}+X_SEARCH = {"type": "x_search"}+IMAGE_GENERATION = {"type": "image_generation", "partial_images": 2, "output_format": "jpeg", "output_compression": 90}+FUNCTION = {"type": "function", "name": "read", "description": "fixture",+ "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}+INCLUDE = ["web_search_call.action.sources", "reasoning.encrypted_content"]++WEB_SEARCH_CALL = {+ "id": "ws_1",+ "type": "web_search_call",+ "status": "completed",+ "action": {+ "type": "search",+ "query": "CLIProxyAPI 版本 🦊",+ "queries": ["CLIProxyAPI 版本 🦊"],+ "sources": [{"type": "url", "url": "https://github.com/router-for-me/CLIProxyAPI/releases"}],+ },+}+X_SEARCH_CALL = {+ "type": "custom_tool_call",+ "id": "ctc_1",+ "call_id": "xs_call-1",+ "name": "x_keyword_search",+ "input": json.dumps({"query": "from:OpenAI within_time:7d", "limit": "10", "mode": "Latest"}),+ "status": "completed",+}+X_REASONING = {"type": "reasoning", "id": "tco_1", "summary": [], "encrypted_content": _FERNET}+# xAI's own ciphertext has no Fernet envelope. Without ``is-compat`` on the Source's models the codex+# executor drops its ``encrypted_content`` and ``id`` (CPA ``openai_responses_signature.go``); the+# replay case records what reaches the upstream rather than asserting it.+XAI_OPAQUE_REASONING = {"type": "reasoning", "id": "tco_2", "summary": [], "encrypted_content": "xai-opaque-ciphertext"}+++def _xai_ciphertext(seed: str, length: int) -> str:+ """Opaque like xAI's own ciphertext, which has no Fernet envelope."""+ stream = b"".join(hashlib.sha256(f"{seed}-{block}".encode()).digest() for block in range(length // 32 + 1))+ return base64.b64encode(stream).decode()[:length]+++# One xAI X-search turn in the 2026-10-13 spike's real shapes. The turn's reasoning is ``rs_<turn>``+# (39 runes). Each server-side tool call ``ctc_<turn>_<call>`` has its own ``tco_<turn>_<call>``+# reasoning; both of those ids are 84 runes.+XAI_TURN = "1994eace-959a-9bc5-87ca-e01dac5c7bfe"+XAI_CALL = "call-9696c08a-e2e1-4204-ab58-9d1d317c128f-2"+XAI_REASONING = {+ "id": f"rs_{XAI_TURN}",+ "summary": [{"type": "summary_text", "text": "I need the most recent post, so I fetch its thread."}],+ "type": "reasoning",+ "status": "completed",+ "encrypted_content": _xai_ciphertext("rs", 3211),+}+XAI_THREAD_FETCH = {+ "call_id": f"xs_{XAI_CALL}",+ "input": json.dumps({"post_id": "2107909792945832070"}),+ "name": "x_thread_fetch",+ "type": "custom_tool_call",+ "id": f"ctc_{XAI_TURN}_{XAI_CALL}",+ "status": "completed",+}+XAI_CALL_REASONING = {+ "id": f"tco_{XAI_TURN}_{XAI_CALL}",+ "summary": [],+ "type": "reasoning",+ "status": "completed",+ "encrypted_content": _xai_ciphertext("tco", 3696),+}+IMAGE_CALL = {+ "id": "ig_1",+ "type": "image_generation_call",+ "status": "completed",+ "result": _PNG_B64,+ "revised_prompt": "A red fox in snow",+ "output_format": "jpeg",+ "size": "1024x1024",+ "quality": "high",+ "background": "opaque",+}+ANSWER = "The latest release is v8.0.27.[[1]](https://github.com/router-for-me/CLIProxyAPI/releases)"+ANSWER_ITEM = {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "status": "completed",+ "content": [{+ "type": "output_text",+ "text": ANSWER,+ "annotations": [{"type": "url_citation", "url": "https://github.com/router-for-me/CLIProxyAPI/releases",+ "start_index": 29, "end_index": len(ANSWER), "title": "1"}],+ }],+}+RESPONSES_USAGE = {+ "input_tokens": 22965,+ "output_tokens": 1402,+ "total_tokens": 24367,+ "input_tokens_details": {"cached_tokens": 0},+ "output_tokens_details": {"reasoning_tokens": 210},+ "cost_in_usd_ticks": 277316400,+ "server_side_tool_usage_details": {"web_search_calls": 1, "x_search_calls": 1},+}++HIT = {+ "type": "web_search_result",+ "title": "Releases · CLIProxyAPI 发布",+ "url": "https://github.com/router-for-me/CLIProxyAPI/releases",+ "encrypted_content": "EqoBCioIBxgCIiQ4ZGUxYzRhMS0wYjk0LTQ2NjktYjM1ZS1hYmFhNmRkODQ3OGI",+ "page_age": "15 days ago",+}+LOCATION = {+ "type": "web_search_result_location",+ "url": HIT["url"],+ "title": HIT["title"],+ "cited_text": "v8.0.24 Latest",+ "encrypted_index": "Eo8BCioIBxgCIiQ4ZGUxYzRhMS0wYjk0LTQ2NjktYjM1ZS1hYmFhNmRkODQ3OGI",+}+ANTHROPIC_WEB_SEARCH = {"type": "web_search_20250305", "name": "web_search", "max_uses": 5}+ANTHROPIC_TOOL = {"name": "read", "description": "fixture",+ "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}+++def _sse(event: dict) -> bytes:+ return f"event: {event['type']}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()+++def _responses_stream(items: list[dict], *, partial_images: bool = False) -> bytes:+ """A Responses stream in the spike's shape: every item added, progress frames, done, then completed."""+ events: list[dict] = [{"type": "response.created", "response": {+ "id": "resp_hosted", "object": "response", "status": "in_progress", "model": MODEL, "output": []}}]+ for index, item in enumerate(items):+ events.append({"type": "response.output_item.added", "output_index": index,+ "item": {key: value for key, value in item.items() if key != "result"} | {"status": "in_progress"}})+ if item["type"] == "web_search_call":+ for phase in ("in_progress", "searching", "completed"):+ events.append({"type": f"response.web_search_call.{phase}", "output_index": index, "item_id": item["id"]})+ if item["type"] == "image_generation_call":+ events.append({"type": "response.image_generation_call.in_progress", "output_index": index,+ "item_id": item["id"]})+ events.append({"type": "response.image_generation_call.generating", "output_index": index,+ "item_id": item["id"]})+ if partial_images:+ for partial in range(2):+ events.append({"type": "response.image_generation_call.partial_image", "output_index": index,+ "item_id": item["id"], "partial_image_index": partial,+ "partial_image_b64": _PNG_B64, "output_format": "jpeg", "size": "1024x1024"})+ events.append({"type": "response.image_generation_call.completed", "output_index": index,+ "item_id": item["id"]})+ if item["type"] == "message":+ events.append({"type": "response.output_text.delta", "output_index": index, "item_id": item["id"],+ "content_index": 0, "delta": item["content"][0]["text"]})+ events.append({"type": "response.output_text.annotation.added", "output_index": index,+ "item_id": item["id"], "content_index": 0, "annotation_index": 0,+ "annotation": item["content"][0]["annotations"][0]})+ events.append({"type": "response.output_item.done", "output_index": index, "item": item})+ events.append({"type": "response.completed", "response": {+ "id": "resp_hosted", "object": "response", "status": "completed", "model": MODEL, "output": items,+ "usage": RESPONSES_USAGE}})+ return b"".join(_sse({**event, "sequence_number": number}) for number, event in enumerate(events))+++def _anthropic_stream(blocks: list[list[dict]], *, server_tool_use: int) -> bytes:+ events: list[dict] = [{"type": "message_start", "message": {+ "id": "msg_hosted", "type": "message", "role": "assistant", "model": MODEL, "content": [],+ "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 6, "output_tokens": 0}}}]+ for index, frames in enumerate(blocks):+ events.extend({**frame, "index": index} for frame in frames)+ events.append({"type": "content_block_stop", "index": index})+ events.append({"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None},+ "usage": {"output_tokens": 519,+ "server_tool_use": {"web_search_requests": server_tool_use, "web_fetch_requests": 0}}})+ events.append({"type": "message_stop"})+ return b"".join(_sse(event) for event in events)+++def _anthropic_search(call_id: str, query: str) -> list[list[dict]]:+ half = len(query) // 2+ return [+ [{"type": "content_block_start", "content_block": {"type": "server_tool_use", "id": call_id,+ "name": "web_search", "input": {}}},+ {"type": "content_block_delta", "delta": {"type": "input_json_delta",+ "partial_json": f'{{"query": "{query[:half]}'}},+ {"type": "content_block_delta", "delta": {"type": "input_json_delta",+ "partial_json": f'{query[half:]}"}}'}}],+ [{"type": "content_block_start", "content_block": {"type": "web_search_tool_result", "tool_use_id": call_id,+ "content": [HIT]}}],+ [{"type": "content_block_start", "content_block": {"type": "text", "text": ""}},+ {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "The latest release is v8.0.24."}},+ {"type": "content_block_delta", "delta": {"type": "citations_delta", "citation": LOCATION}}],+ ]+++@asynccontextmanager+async def _engine(tmp_path, monkeypatch, protocol: str, reply: bytes, *, vendor: str = "custom"):+ """One Source of ``protocol`` on a capturing upstream; yields (base_url, token, model, captured)."""+ captured: list[dict] = []++ async def upstream(request):+ body = await request.read()+ captured.append({+ "path": request.path,+ "query": request.query_string,+ "headers": [(key.lower(), value) for key, value in request.headers.items()],+ "body": body,+ })+ return web.Response(body=reply, content_type="text/event-stream")++ with _isolated_engine_adapter(tmp_path, monkeypatch) as adapter:+ async with _server(upstream) as root:+ source = _source("src_hosted001", "Hosted fixture", vendor=vendor, protocol=protocol, model_id=MODEL)+ source.base_url = root + ("/v1" if protocol == "openai_responses" else "")+ source.credential_ref = adapter.state_store.store_api_key(+ "sk-synthetic-hosted", vendor=vendor, protocol=protocol, base_url=source.base_url,+ )+ config = _vibey_service(tmp_path, [source]).store.config+ service = service_for(tmp_path, MemoryModelHubStore(config), adapter)+ await service.runtime_start()+ connection = adapter.supervisor.client_if_running().connection+ prefix = adapter.state_store.list_sources()[0].prefix+ yield connection.base_url, connection.gateway_token, f"{prefix}/{MODEL}", captured+++def _post(base_url: str, token: str, path: str, body: dict) -> tuple[int, bytes]:+ request = urllib.request.Request(base_url + path, data=json.dumps(body).encode(), method="POST", headers={+ "Authorization": f"Bearer {token}", "Content-Type": "application/json", "User-Agent": "avibe-hosted-test/1"})+ try:+ with urllib.request.urlopen(request, timeout=30) as response:+ return response.status, response.read()+ except urllib.error.HTTPError as exc:+ return exc.code, exc.read()+++def _events(raw: bytes) -> list[dict]:+ parsed = []+ for line in raw.decode("utf-8", errors="replace").splitlines():+ if line.startswith("data:"):+ try:+ parsed.append(json.loads(line[5:].strip()))+ except json.JSONDecodeError:+ continue+ return parsed+++def _without_cache_control(item: dict) -> dict:+ """Ignore only engine-added metadata on one tool or content block, never nested business data."""+ return {key: value for key, value in item.items() if key != "cache_control"}+++def _anthropic_history(messages: list[dict]) -> list[dict]:+ """Keep every message/block in order; normalize only text form and shallow cache metadata."""+ normalized = []+ for message in messages:+ content = message["content"]+ if isinstance(content, str):+ content = [{"type": "text", "text": content}]+ normalized.append({**message, "content": [_without_cache_control(block) for block in content]})+ return normalized+++def _stable(body: bytes) -> bytes:+ """The request bytes with only the engine's per-run ``prompt_cache_key`` value masked, in place."""+ try:+ payload = json.loads(body)+ except json.JSONDecodeError:+ return body+ if not isinstance(payload, dict) or "prompt_cache_key" not in payload:+ return body+ field = b'"prompt_cache_key":' + json.dumps(payload["prompt_cache_key"]).encode()+ assert body.count(field) == 1, "prompt_cache_key is not one compact field"+ return body.replace(field, b'"prompt_cache_key":"<per-run>"')+++def _stable_header(name: str, value: str) -> str:+ """Mask what differs between two runs of one engine: its per-request session, and the upstream's port."""+ if name == "session-id":+ return "<per-run>"+ if name == "host":+ return re.sub(r":[0-9]+$", ":<per-run>", value)+ return value+++def _record(captured: list[dict], status: int, raw: bytes) -> dict:+ """Return the one upstream call; with an evidence directory, also write both directions there."""+ [call] = captured+ directory = os.environ.get(EVIDENCE_DIR_ENV)+ if directory:+ case = os.environ["PYTEST_CURRENT_TEST"].split("::")[-1].split(" ")[0]+ wire = {+ "upstream": {+ "path": call["path"],+ "query": call["query"],+ "headers": sorted([name, _stable_header(name, value)] for name, value in call["headers"]),+ "body": _stable(call["body"]).decode(),+ },+ "downstream": {"status": status, "body": raw.decode()},+ }+ text = json.dumps(wire, ensure_ascii=False, indent=1, sort_keys=True)+ (Path(directory) / f"{case}.json").write_text(text, encoding="utf-8")+ return call+++async def test_responses_hosted_declarations_and_items_pass_through(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-001: declarations, include values and hosted output items survive CPA."""+ items = [WEB_SEARCH_CALL, X_SEARCH_CALL, X_REASONING, IMAGE_CALL, ANSWER_ITEM]+ reply = _responses_stream(items, partial_images=True)+ body = {"model": None, "stream": True, "input": [+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "search and draw"}]}],+ "tools": [WEB_SEARCH, X_SEARCH, IMAGE_GENERATION, FUNCTION], "include": INCLUDE,+ "reasoning": {"effort": "high", "summary": "auto"}, "store": False}+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply) as (base_url, token, model, captured):+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})+ call = _record(captured, status, raw)+ sent = json.loads(call["body"])+ assert status == 200, raw[:500]+ assert sent["tools"] == body["tools"], sent["tools"]+ # CPA canonicalizes include order; compare the full multiset, not membership.+ assert sorted(sent["include"]) == sorted(INCLUDE), sent["include"]+ # Every decoded event must survive, including progress, deltas, citations and usage.+ assert _events(raw) == _events(reply)+++async def test_responses_hosted_history_replays_verbatim(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-002: hosted replay keeps content, order and cardinality around messages."""+ history = [+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "search and draw"}]},+ WEB_SEARCH_CALL, X_SEARCH_CALL, X_REASONING, XAI_OPAQUE_REASONING, IMAGE_CALL, ANSWER_ITEM,+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "and again"}]},+ ]+ reply = _responses_stream([ANSWER_ITEM])+ body = {"model": None, "stream": True, "input": history, "tools": [WEB_SEARCH, X_SEARCH, IMAGE_GENERATION],+ "include": INCLUDE, "reasoning": {"effort": "high", "summary": "auto"}, "store": False}+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply) as (base_url, token, model, captured):+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})+ call = _record(captured, status, raw)+ assert status == 200, raw[:500]+ replayed = json.loads(call["body"])["input"]+ # Reasoning items are not hosted items: the engine prefixes their ids (``rs_``) and drops+ # ciphertext without a Fernet envelope unless the model is ``is-compat``. Recorded, not pinned.+ # Compare the rest as one sequence, including message anchors, so calls cannot be duplicated,+ # dropped, reordered, or moved across the assistant answer while still passing the contract.+ expected_history = [item for item in history if item.get("type") != "reasoning"]+ replayed_history = [item for item in replayed if item.get("type") != "reasoning"]+ assert replayed_history == expected_history, replayed+ assert _events(raw) == _events(reply)+++async def test_image_generation_is_never_injected_or_answered_by_the_engine(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-003: passthrough mode never injects an undeclared image tool."""+ reply = _responses_stream([ANSWER_ITEM])+ body = {"model": None, "stream": True, "input": [+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hello"}]}],+ "tools": [FUNCTION], "store": False}+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply) as (base_url, token, model, captured):+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})+ call = _record(captured, status, raw)+ assert status == 200, raw[:500]+ sent = json.loads(call["body"])+ assert sent["tools"] == body["tools"], sent.get("tools")+ assert _events(raw) == _events(reply)+++async def test_anthropic_web_search_and_its_results_pass_through(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-004: Anthropic search declarations, results, citations and usage survive CPA."""+ reply = _anthropic_stream(_anthropic_search("srvtoolu_01", "CLIProxyAPI 版本 🦊"), server_tool_use=1)+ body = {"model": None, "stream": True, "max_tokens": 1024,+ "messages": [{"role": "user", "content": "search the latest CLIProxyAPI release"}],+ "tools": [ANTHROPIC_WEB_SEARCH, ANTHROPIC_TOOL]}+ async with _engine(tmp_path, monkeypatch, "anthropic", reply) as (base_url, token, model, captured):+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/messages", {**body, "model": model})+ call = _record(captured, status, raw)+ assert status == 200, raw[:500]+ sent = json.loads(call["body"])+ assert [_without_cache_control(tool) for tool in sent["tools"]] == body["tools"], sent["tools"]+ assert _events(raw) == _events(reply)+++async def test_anthropic_hosted_history_replays_verbatim(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-005: the entire Anthropic replay keeps messages, tools, results and citations."""+ use = {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search",+ "input": {"query": "CLIProxyAPI releases"}}+ result = {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01", "content": [HIT]}+ text = {"type": "text", "text": "The latest release is v8.0.24.", "citations": [LOCATION]}+ reply = _anthropic_stream([[{"type": "content_block_start", "content_block": {"type": "text", "text": ""}},+ {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "ok"}}]],+ server_tool_use=0)+ body = {"model": None, "stream": True, "max_tokens": 1024, "tools": [ANTHROPIC_WEB_SEARCH],+ "messages": [{"role": "user", "content": "search the latest CLIProxyAPI release"},+ {"role": "assistant", "content": [use, result, text]},+ {"role": "user", "content": "and the one before?"}]}+ async with _engine(tmp_path, monkeypatch, "anthropic", reply) as (base_url, token, model, captured):+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/messages", {**body, "model": model})+ call = _record(captured, status, raw)+ assert status == 200, raw[:500]+ replayed = json.loads(call["body"])["messages"]+ assert _anthropic_history(replayed) == _anthropic_history(body["messages"]), replayed+ assert _events(raw) == _events(reply)+++XAI_HISTORY = [+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "quote the latest post"}]},+ XAI_REASONING, XAI_THREAD_FETCH, XAI_CALL_REASONING, ANSWER_ITEM,+ {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "and the one before?"}]},+]+++async def _replay_xai_turn(tmp_path, monkeypatch) -> tuple[list[dict], bytes, bytes]:+ """Replay ``XAI_HISTORY`` to an xAI Source as Avibe renders one; return what arrived and the reply sent."""+ reply = _responses_stream([ANSWER_ITEM])+ body = {"model": None, "stream": True, "input": XAI_HISTORY, "tools": [X_SEARCH], "include": INCLUDE,+ "reasoning": {"effort": "high", "summary": "auto"}, "store": False}+ async with _engine(tmp_path, monkeypatch, "openai_responses", reply, vendor="xai") as (+ base_url, token, model, captured,+ ):+ status, raw = await asyncio.to_thread(_post, base_url, token, "/v1/responses", {**body, "model": model})+ call = _record(captured, status, raw)+ assert status == 200, raw[:500]+ return json.loads(call["body"])["input"], raw, reply+++async def test_responses_xai_history_replays_in_real_shapes(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-006: an xAI X-search turn replayed in xAI's real id and ciphertext shapes.++ The 84-rune ids exceed the engine's 64-rune input id limit, which the short ids above never reach.+ What the engine does with xAI reasoning and the over-limit call id is the next case's subject, so+ the gate compares those bytes here instead of pinning them. Everything else is pinned: the ordered+ non-reasoning sequence, the call's own fields, and the decoded response.+ """+ replayed, raw, reply = await _replay_xai_turn(tmp_path, monkeypatch)++ def without_call_id(items: list[dict]) -> list[dict]:+ return [{key: value for key, value in item.items() if not (item["type"] == "custom_tool_call" and key == "id")}+ for item in items if item.get("type") != "reasoning"]++ assert without_call_id(replayed) == without_call_id(XAI_HISTORY), replayed+ assert _events(raw) == _events(reply)+++ strict=True, reason="engine drops/shortens xAI over-64 ids under is-compat; fixed by the id patch in #2780",+)+async def test_responses_xai_over_limit_ids_replay_verbatim(tmp_path, monkeypatch) -> None:+ """MH-ENGINE-HOSTED-007: xAI's 84-rune tool-call reasoning and call ids reach xAI as it issued them.++ Under ``is-compat``, which Avibe renders for xAI, the pinned engine keeps the ``rs_`` reasoning but+ drops the ``tco_`` reasoning whole and shortens the ``ctc_`` id to 64 runes. An engine that keeps+ them turns this strict expected failure into a failure, so that change must flip the marker.+ """+ replayed, _raw, _reply = await _replay_xai_turn(tmp_path, monkeypatch)++ arrived = [item for item in replayed if item["type"] == "reasoning" and item.get("id") == XAI_CALL_REASONING["id"]]+ assert [item.get("encrypted_content") for item in arrived] == [XAI_CALL_REASONING["encrypted_content"]], replayed+ [thread_fetch] = [item for item in replayed if item["type"] == "custom_tool_call"]+ assert thread_fetch["id"] == XAI_THREAD_FETCH["id"], thread_fetchdiff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yamlindex bf42da7b4..7ab83b0d9 100644--- a/tests/scenarios/INDEX.yaml+++ b/tests/scenarios/INDEX.yaml@@ -42,6 +42,7 @@ capabilities:- tests/e2e/test_codex_request_metadata.py- tests/e2e/test_codex_request_gateway.py- tests/e2e/test_model_hub_runtime.py+ - tests/e2e/test_model_hub_engine_hosted_passthrough.py- tests/e2e/test_model_hub_sources.py- tests/e2e/test_model_hub_usage_events.py- tests/e2e/test_model_hub_engine_wire.pydiff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 2b1e46c39..6ee9279f1 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -103,6 +103,7 @@ capability:- tests/e2e/test_codex_request_metadata.py- tests/e2e/test_codex_request_gateway.py- tests/e2e/test_model_hub_runtime.py+ - tests/e2e/test_model_hub_engine_hosted_passthrough.py- tests/e2e/test_model_hub_sources.py- tests/e2e/test_model_hub_usage_events.py- tests/e2e/test_model_hub_engine_wire.py@@ -2609,6 +2610,48 @@ scenarios:test: tests/e2e/test_model_hub_engine_wire.py::test_responses_stream_with_reasoning_and_tool_callrelated_tests:- tests/test_model_hub_runtime.py::test_only_an_xai_responses_entry_marks_its_models_is_compat+ - id: MH-ENGINE-HOSTED-001+ name: The real CPA Responses path preserves complete tool declarations, include cardinality and the complete decoded response event sequence+ status: covered+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_hosted_declarations_and_items_pass_through+ - id: MH-ENGINE-HOSTED-002+ name: The real CPA Responses path preserves hosted history content, order and cardinality relative to user and assistant messages+ status: covered+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_hosted_history_replays_verbatim+ - id: MH-ENGINE-HOSTED-003+ name: CPA image passthrough mode never injects an image tool absent from the caller request+ status: covered+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_image_generation_is_never_injected_or_answered_by_the_engine+ - id: MH-ENGINE-HOSTED-004+ name: The real CPA Anthropic path preserves complete hosted search declarations and the decoded response event sequence including tool inputs, results, citations and usage+ status: covered+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_anthropic_web_search_and_its_results_pass_through+ - id: MH-ENGINE-HOSTED-005+ name: The real CPA Anthropic path preserves the complete ordered replay message sequence and its tool, result and citation contents+ status: covered+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_anthropic_hosted_history_replays_verbatim+ - id: MH-ENGINE-HOSTED-006+ name: The real CPA Responses path replays an xAI X-search turn in xAI's real 39- and 84-rune id and envelope-less ciphertext shapes, keeping the ordered non-reasoning sequence, the call's own fields and the decoded response+ status: covered+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_xai_history_replays_in_real_shapes+ - id: MH-ENGINE-HOSTED-007+ name: The real CPA Responses path replays xAI's 84-rune tool-call reasoning with its id and ciphertext, and the 84-rune call id, as xAI issued them (expected to fail until the engine keeps over-limit upstream ids under is-compat)+ status: xfail+ kind: compatibility+ layer: e2e+ test: tests/e2e/test_model_hub_engine_hosted_passthrough.py::test_responses_xai_over_limit_ids_replay_verbatimnext_priority:- MH-PROTOCOL-004- MH-RUNTIME-001@@ -2619,3 +2662,4 @@ next_priority:- B9- D6- G3+ - MH-ENGINE-HOSTED-007diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.pyindex ba423cdc9..6d3dda0c2 100644--- a/tests/test_ci_pipeline.py+++ b/tests/test_ci_pipeline.py@@ -500,6 +500,79 @@ def test_distribution_contract_command_fails_on_errors_and_skips(tmp_path, pytesassert (result.returncode == 0) == (pytest_exit == 0 and "skipped" not in summary), result.stdout + result.stderr+def test_hosted_wire_gate_judges_a_pull_requests_engine_change_and_blocks_merge(tmp_path: Path) -> None:+ workflow = yaml.safe_load((ROOT / ".github/workflows/lint.yml").read_text())+ jobs = workflow["jobs"]+ job = jobs["model-hub-hosted-wire"]+ # A failure reaches branch protection through the required aggregate.+ assert "model-hub-hosted-wire" in jobs["unit-tests"]["needs"]+ # Labeling re-runs the gate, which reads the labels as they are when it runs.+ assert {"labeled", "unlabeled"} <= set(workflow[True]["pull_request"]["types"])+ assert job["permissions"] == {"contents": "read", "pull-requests": "read"}+ assert job["env"]["BASE_SHA"] == "${{ github.event.pull_request.base.sha }}"+ steps = job["steps"]+ detect = next(index for index, step in enumerate(steps) if step.get("id") == "manifest")+ for step in steps[detect + 1:]:+ assert "steps.manifest.outputs.changed == 'true'" in step["if"], step+ gate, = [step for step in steps if "scripts/model_hub_hosted_wire_gate.py" in step.get("run", "")]+ assert '--base "$BASE_SHA"' in gate["run"] and '--pull-request "$PR_NUMBER"' in gate["run"]+ # The downloaded engines and the cases run where only loopback answers.+ assert gate["run"].split().count("--loopback-only") == 1+ assert gate["env"] == {"GH_TOKEN": "${{ github.token }}", "PR_NUMBER": "${{ github.event.pull_request.number }}"}++ repo = tmp_path / "repo"+ manifest = repo / "vibe/model_hub_runtime/cliproxyapi_manifest.json"+ manifest.parent.mkdir(parents=True)++ def git(*args: str) -> str:+ return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip()++ def commit() -> str:+ git("add", "-A")+ git("-c", "user.name=CI Fixture", "-c", "[email protected]", "commit", "-qm", "fixture")+ return git("rev-parse", "HEAD")++ def changed(base: str, head: str, event: str = "pull_request") -> tuple[int, str]:+ git("checkout", "-q", head)+ output = tmp_path / "output"+ output.write_text("")+ result = subprocess.run(+ ["bash", "-e", "-o", "pipefail", "-c", steps[detect]["run"]], cwd=repo,+ env={**os.environ, "BASE_SHA": base, "GITHUB_EVENT_NAME": event, "GITHUB_OUTPUT": str(output)},+ capture_output=True, text=True,+ )+ return result.returncode, output.read_text()++ git("init", "-q")+ manifest.write_text('{"asset_release_tag": "model-hub-engine-v8.0.23-3"}\n')+ base = commit()+ (repo / "README.md").write_text("unrelated\n")+ unrelated = commit()+ manifest.write_text('{"asset_release_tag": "model-hub-engine-v8.0.28-1"}\n')+ engine = commit()++ assert changed(base, unrelated) == (0, "changed=false\n")+ assert changed(base, engine) == (0, "changed=true\n")+ # The label lives on the pull request, so a push is not judged again.+ assert changed("", engine, event="push") == (0, "changed=false\n")+ # An unreachable or missing base fails the job; it never reads as "unchanged".+ for missing in ("0" * 40, ""):+ status, output = changed(missing, engine)+ assert status != 0 and output == ""+++def test_every_linux_engine_e2e_runner_has_only_loopback() -> None:+ """Hosted and engine E2E cases start a real engine; on Linux only the namespace keeps it off the network."""+ release = yaml.safe_load((ROOT / ".github/workflows/model-hub-e2e.yml").read_text())+ runs = [step["run"] for job in release["jobs"].values() for step in job["steps"] if "e2e_model_hub" in step.get("run", "")]+ assert runs, "the release E2E runs no engine scenarios"+ for run in runs:+ assert run.lstrip().startswith("bash scripts/ci_loopback_only.sh python -m pytest -m e2e_model_hub"), run+ gate, = [step for step in _jobs()["model-hub-hosted-wire"]["steps"]+ if "scripts/model_hub_hosted_wire_gate.py" in step.get("run", "")]+ assert "--loopback-only" in gate["run"].split()++@pytest.mark.parametrize("gate_name", ["unit-tests", "install-upgrade-regression", "windows-install-smoke"])def test_required_gates_accept_skips_only_for_proven_docs_and_fail_closed(tmp_path: Path, gate_name: str) -> None:gate = _jobs()[gate_name]@@ -550,7 +623,8 @@ def test_lint_replaces_only_the_same_pr_or_branch_and_runs_no_release_listener()def test_docs_scope_gates_all_work_without_filtering_away_required_checks() -> None:workflow = yaml.safe_load((ROOT / ".github/workflows/lint.yml").read_text())events = workflow[True] # PyYAML YAML 1.1 reads the Actions `on` key as True.- assert events["pull_request"] is None+ # Default activity types plus the labeling that re-runs the hosted wire gate.+ assert events["pull_request"] == {"types": ["opened", "synchronize", "reopened", "labeled", "unlabeled"]}assert not any(key.startswith("paths") for key in events["push"])jobs = workflow["jobs"]required = {"ruff", "unit-tests", "show-router-integration", "memory-insight-contract",diff --git a/tests/test_model_hub_hosted_wire_gate.py b/tests/test_model_hub_hosted_wire_gate.pynew file mode 100644index 000000000..00b7116c8--- /dev/null+++ b/tests/test_model_hub_hosted_wire_gate.py@@ -0,0 +1,477 @@+"""The hosted wire gate and its A/B runner (``scripts/model_hub_hosted_wire_gate.py``)."""++from __future__ import annotations++import io+import json+from pathlib import Path+from types import SimpleNamespace+import urllib.error+from urllib.parse import urlparse+from urllib.request import url2pathname++import pytest++from scripts import model_hub_hosted_wire_gate as gate+from tests.test_model_hub_engine_release_guard import _fake_download, _manifest++BASE_TAG = "model-hub-engine-v8.0.23-3"+PR_TAG = "model-hub-engine-v8.0.28-1"+DIFFERENCES = {+ "test_responses_xai_history_replays_in_real_shapes": ["upstream header set", "upstream request bytes"],+ "test_image_generation_is_never_injected_or_answered_by_the_engine": ["downstream bytes"],+}+SUMMARY = ("test_responses_xai_history_replays_in_real_shapes: upstream header set, upstream request bytes; "+ "test_image_generation_is_never_injected_or_answered_by_the_engine: downstream bytes")+++def test_the_cases_mask_only_per_run_values() -> None:+ from tests.e2e.test_model_hub_engine_hosted_passthrough import _stable, _stable_header++ assert _stable_header("host", "127.0.0.1:43123") == "127.0.0.1:<per-run>"+ assert _stable_header("host", "api.openai.com") == "api.openai.com"+ assert _stable_header("session-id", "019a7f3e-9d1c") == "<per-run>"+ assert _stable_header("user-agent", "codex-tui/0.154.0") == "codex-tui/0.154.0"+ body = b'{"model":"m","prompt_cache_key":"019a7f3e","input":[ ]}'+ assert _stable(body) == b'{"model":"m","prompt_cache_key":"<per-run>","input":[ ]}'+ assert _stable(b'{"model":"m", "input":[ ]}') == b'{"model":"m", "input":[ ]}'+ with pytest.raises(AssertionError, match="one compact field"):+ _stable(b'{"prompt_cache_key": "019a7f3e"}')+++def _recorded() -> dict:+ return {+ "upstream": {+ "path": "/v1/responses",+ "query": "",+ "headers": [["content-type", "application/json"], ["session-id", "<per-run>"]],+ "body": '{"model":"hosted-model","prompt_cache_key":"<per-run>"}',+ },+ "downstream": {"status": 200, "body": "event: response.completed\ndata: {}\n\n"},+ }+++def test_identical_recordings_have_no_differences() -> None:+ assert gate.wire_differences({"case_a": _recorded()}, {"case_a": _recorded()}) == {}+++ ("direction", "part", "label"),+ [+ ("upstream", "path", "upstream request path"),+ ("upstream", "query", "upstream request query"),+ ("upstream", "headers", "upstream header set"),+ ("upstream", "body", "upstream request bytes"),+ ("downstream", "status", "downstream status"),+ ("downstream", "body", "downstream bytes"),+ ],+)+def test_each_wire_part_is_compared(direction: str, part: str, label: str) -> None:+ changed = _recorded()+ changed[direction][part] = 201 if part == "status" else ["x"] if part == "headers" else "x"++ differences = gate.wire_differences(+ {"case_a": _recorded(), "case_b": _recorded()}, {"case_a": changed, "case_b": _recorded()},+ )++ assert differences == {"case_a": [label]}+++def test_every_differing_part_of_a_case_is_named_in_wire_order() -> None:+ changed = _recorded()+ changed["downstream"]["body"] = "x"+ changed["upstream"]["headers"] = []++ assert gate.wire_differences({"case_a": _recorded()}, {"case_a": changed}) == {+ "case_a": ["upstream header set", "downstream bytes"],+ }+++def test_a_case_recorded_against_one_engine_is_a_difference() -> None:+ assert gate.wire_differences({"case_a": _recorded()}, {"case_b": _recorded()}) == {+ "case_a": ["recorded against only one engine"], "case_b": ["recorded against only one engine"],+ }++[email protected]("labels", [None, frozenset(), frozenset({gate.CHANGE_LABEL})])+def test_an_identical_passthrough_passes(labels) -> None:+ assert gate.wire_change_violation({}, labels) is None++[email protected]("labels", [frozenset({gate.CHANGE_LABEL}), frozenset({"dependencies", gate.CHANGE_LABEL})])+def test_a_labeled_difference_passes(labels) -> None:+ assert gate.wire_change_violation(DIFFERENCES, labels) is None+++ "labels", [None, frozenset(), frozenset({"dependencies"}), frozenset({gate.CHANGE_LABEL.upper()})],+)+def test_an_unlabeled_difference_fails_and_names_what_differs(labels) -> None:+ assert gate.wire_change_violation(DIFFERENCES, labels) == (+ f"the engine changes the hosted passthrough ({SUMMARY}). If that is intended, label the pull request "+ f"{gate.CHANGE_LABEL} and say in its body what changed and why"+ )+++def _api(monkeypatch, response: bytes | Exception) -> list:+ requests = []++ def urlopen(request, *, timeout):+ requests.append(request)+ if isinstance(response, Exception):+ raise response+ return io.BytesIO(response)++ monkeypatch.setattr(gate.urllib.request, "urlopen", urlopen)+ return requests+++def test_labels_are_read_from_the_pull_request_as_it_is_now(monkeypatch) -> None:+ payload = {"number": 7, "labels": [{"name": gate.CHANGE_LABEL}, {"name": "dependencies"}]}+ requests = _api(monkeypatch, json.dumps(payload).encode())++ assert gate.pull_request_labels("avibe-bot/avibe", "7", "token-1") == {gate.CHANGE_LABEL, "dependencies"}+ [request] = requests+ assert request.full_url == "https://api.github.com/repos/avibe-bot/avibe/pulls/7"+ assert request.get_header("Authorization") == "Bearer token-1"++ requests.clear()+ assert gate.pull_request_labels("avibe-bot/avibe", "7", None) == {gate.CHANGE_LABEL, "dependencies"}+ assert requests[0].get_header("Authorization") is None+++ "response",+ [+ urllib.error.HTTPError("https://api.github.com", 403, "rate limited", {}, None),+ urllib.error.URLError("unreachable"),+ TimeoutError("timed out"),+ b"not json",+ b'{"number": 7}',+ b'{"labels": "engine-hosted-wire-change"}',+ b'{"labels": [{"title": "engine-hosted-wire-change"}]}',+ b"[]",+ ],+)+def test_labels_that_cannot_be_read_count_as_none(monkeypatch, capsys, response) -> None:+ _api(monkeypatch, response)++ assert gate.pull_request_labels("avibe-bot/avibe", "7", "token-1") is None+ assert "cannot read the pull request's labels" in capsys.readouterr().out+++def _repository(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, base: dict | bytes, pr: dict | bytes) -> None:+ manifest = tmp_path / "repo" / gate.MANIFEST+ manifest.parent.mkdir(parents=True)+ manifest.write_bytes(pr if isinstance(pr, bytes) else json.dumps(pr).encode())+ monkeypatch.setattr(gate, "REPO_ROOT", tmp_path / "repo")+ monkeypatch.setattr(gate, "_git_show",+ lambda revision, path: base if isinstance(base, bytes) else json.dumps(base).encode())+++def _verdicts(monkeypatch, differences: dict | None, labels: frozenset[str] | None = None) -> dict:+ """Record the A/B and label reads ``main`` makes; ``differences`` None means it must not compare."""+ seen = {"ab": [], "labels": []}++ def compare(base, pr, work, assets, *, loopback_only):+ if differences is None:+ pytest.fail("ran the A/B")+ seen["ab"].append((json.loads(base), json.loads(pr), assets, loopback_only))+ return differences++ def read_labels(repository, number, token):+ seen["labels"].append((repository, number, token))+ return labels++ monkeypatch.setattr(gate, "hosted_wire_differences", compare)+ monkeypatch.setattr(gate, "pull_request_labels", read_labels)+ return seen++[email protected]("pr", [{"asset_release_tag": BASE_TAG}, {"asset_release_tag": BASE_TAG, "note": "x"}])+def test_an_unchanged_engine_has_nothing_to_compare(tmp_path, monkeypatch, capsys, pr) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, pr)+ seen = _verdicts(monkeypatch, None)++ assert gate.main(["--base", "base", "--pull-request", "7"]) == 0+ assert seen == {"ab": [], "labels": []}+ assert "the engine is unchanged" in capsys.readouterr().out++[email protected]("pr", [b"{", b"[]", b'{"version": "v8.0.28"}'])+def test_a_manifest_without_an_engine_tag_fails(tmp_path, monkeypatch, capsys, pr) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, pr)+ _verdicts(monkeypatch, None)++ assert gate.main(["--base", "base"]) == 1+ assert "::error::hosted_wire gate: cannot read either manifest's asset_release_tag" in capsys.readouterr().out+++def test_an_identical_passthrough_passes_without_reading_labels(tmp_path, monkeypatch, capsys) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})+ seen = _verdicts(monkeypatch, {})++ assert gate.main(["--base", "base", "--pull-request", "7", "--head-assets", str(tmp_path / "build")]) == 0+ assert seen == {"ab": [({"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG}, tmp_path / "build", False)],+ "labels": []}+ assert "hosted passthrough byte-identical" in capsys.readouterr().out+++def test_the_loopback_only_flag_reaches_the_ab(tmp_path, monkeypatch) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})+ seen = _verdicts(monkeypatch, {})++ assert gate.main(["--base", "base", "--loopback-only"]) == 0+ assert seen["ab"] == [({"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG}, None, True)]+++ ("labels", "status"),+ [(frozenset({gate.CHANGE_LABEL}), 0), (frozenset({"dependencies"}), 1), (frozenset(), 1), (None, 1)],+)+def test_a_difference_is_judged_on_the_labels_read_when_the_gate_runs(+ tmp_path, monkeypatch, capsys, labels, status,+) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})+ monkeypatch.setenv("GH_TOKEN", "token-1")+ seen = _verdicts(monkeypatch, DIFFERENCES, labels)++ assert gate.main(["--base", "base", "--pull-request", "7", "--repository", "avibe-bot/avibe"]) == status+ assert seen["labels"] == [("avibe-bot/avibe", "7", "token-1")]+ out = capsys.readouterr().out+ assert "hosted passthrough differs" in out+ assert " test_image_generation_is_never_injected_or_answered_by_the_engine: downstream bytes" in out+ if status:+ assert f"::error file={gate.MANIFEST}::hosted_wire gate: the engine changes the hosted passthrough ({SUMMARY})" in out+ else:+ assert f"labeled {gate.CHANGE_LABEL}, so the difference is intended" in out+++def test_without_a_pull_request_no_label_can_accept_a_difference(tmp_path, monkeypatch, capsys) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})+ seen = _verdicts(monkeypatch, DIFFERENCES, frozenset({gate.CHANGE_LABEL}))++ assert gate.main(["--base", "base", "--head-assets", str(tmp_path / "build")]) == 1+ assert seen["labels"] == []+ out = capsys.readouterr().out+ assert "no pull request, so no label can accept the difference" in out+ assert f"::error file={gate.MANIFEST}::hosted_wire gate: the engine changes the hosted passthrough" in out+++def test_a_gate_that_cannot_reach_a_verdict_fails(tmp_path, monkeypatch, capsys) -> None:+ _repository(tmp_path, monkeypatch, {"asset_release_tag": BASE_TAG}, {"asset_release_tag": PR_TAG})++ def unreachable(*_args, **_kwargs):+ raise gate.GateError("the PR engine is not the asset its manifest pins")++ monkeypatch.setattr(gate, "hosted_wire_differences", unreachable)+ monkeypatch.setattr(gate, "pull_request_labels", lambda *a: frozenset({gate.CHANGE_LABEL}))++ assert gate.main(["--base", "base", "--pull-request", "7"]) == 1+ assert "::error::hosted_wire gate: the PR engine is not the asset its manifest pins" in capsys.readouterr().out+++def test_the_ab_takes_base_from_its_release_and_the_pr_engine_from_the_given_build(tmp_path, monkeypatch) -> None:+ runs = []++ def run_cases(side, manifest_bytes, platform, work, asset_dir=None, *, must_pass, loopback_only):+ runs.append((side, manifest_bytes, work, asset_dir, must_pass, loopback_only))+ changed = _recorded()+ if side == "PR":+ changed["downstream"]["body"] = "changed"+ return {"case_a": changed}++ monkeypatch.setattr(gate, "_run_cases", run_cases)++ differences = gate.hosted_wire_differences(b"base", b"pr", tmp_path, tmp_path / "build", loopback_only=True)++ assert differences == {"case_a": ["downstream bytes"]}+ # Only the engine being adopted must pass; base must record every case. Both run without network.+ assert runs == [("base", b"base", tmp_path / "base", None, False, True),+ ("PR", b"pr", tmp_path / "head", tmp_path / "build", True, True)]+++CASE_NAMES = ("test_one", "test_two")+++def _fake_pytest(*, returncode: int = 0, outcomes: dict | None = None, recorded: tuple = CASE_NAMES):+ """Stand in for the case run: check the offline manifest, then write evidence and a JUnit report."""+ runs = []++ def run(args, *, cwd, env, check):+ manifest = json.loads(Path(env["VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH"]).read_text(encoding="utf-8"))+ [url] = [asset["url"] for asset in manifest["assets"] if asset["platform"] == "linux-amd64"]+ assert urlparse(url).scheme == "file" and Path(url2pathname(urlparse(url).path)).is_file()+ runs.append(args)+ for name in recorded:+ (Path(env[gate.EVIDENCE_DIR_ENV]) / f"{name}.json").write_text(json.dumps(_recorded()), encoding="utf-8")+ cases = "".join(+ f'<testcase name="{name}">{(outcomes or {}).get(name, "")}</testcase>' for name in CASE_NAMES+ )+ report = next(arg for arg in args if arg.startswith("--junitxml=")).removeprefix("--junitxml=")+ Path(report).write_text(f"<testsuites><testsuite>{cases}</testsuite></testsuites>", encoding="utf-8")+ return SimpleNamespace(returncode=returncode)++ return SimpleNamespace(run=run), runs+++def _release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[bytes, dict, dict]:+ manifest_path, owned, built = _manifest(tmp_path)+ monkeypatch.setattr(gate.release_guard, "_download", _fake_download(owned))+ return manifest_path.read_bytes(), owned, built++[email protected]("loopback_only", [False, True])+def test_the_cases_run_in_the_loopback_only_namespace_when_asked(tmp_path, monkeypatch, loopback_only) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ fake, runs = _fake_pytest()+ monkeypatch.setattr(gate, "subprocess", fake)++ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True, loopback_only=loopback_only)++ [args] = runs+ pytest_at = args.index("pytest") - 2+ assert args[:pytest_at] == (["bash", gate.LOOPBACK_ONLY] if loopback_only else [])+ assert (gate.REPO_ROOT / gate.LOOPBACK_ONLY).is_file()+++def test_a_published_engine_runs_the_cases_from_its_verified_archive(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ fake, runs = _fake_pytest()+ monkeypatch.setattr(gate, "subprocess", fake)++ cases = gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)++ assert cases == {name: _recorded() for name in CASE_NAMES}+ [args] = runs+ assert args[-1] == gate.CASES and "e2e_model_hub" in args+++ "fake_kwargs",+ [{"returncode": 1}, {"outcomes": {"test_two": "<skipped/>"}}, {"outcomes": {"test_two": "<failure/>"}},+ {"outcomes": {"test_two": "<error/>"}}],+)+def test_the_pr_engine_must_pass_every_case(tmp_path, monkeypatch, fake_kwargs) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ monkeypatch.setattr(gate, "subprocess", _fake_pytest(**fake_kwargs)[0])++ with pytest.raises(gate.GateError, match="did not all pass against the PR engine"):+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)+++def test_a_case_that_fails_as_its_strict_xfail_says_counts_as_passing(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ xfailed = '<skipped type="pytest.xfail" message="known engine defect" />'+ monkeypatch.setattr(gate, "subprocess", _fake_pytest(outcomes={"test_two": xfailed})[0])++ cases = gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)++ assert cases == {name: _recorded() for name in CASE_NAMES}+++ "outcome",+ [+ # An engine that fixed the defect: strict xfail reports the pass as a failure.+ '<failure message="[XPASS(strict)] known engine defect">[XPASS(strict)] known engine defect</failure>',+ '<skipped type="pytest.skip" message="nope" />',+ "<skipped />",+ '<skipped type="pytest.xfail" /><error message="teardown" />',+ ],+)+def test_only_an_expected_failure_stands_in_for_a_pass(tmp_path, monkeypatch, outcome) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ monkeypatch.setattr(gate, "subprocess", _fake_pytest(outcomes={"test_two": outcome})[0])++ with pytest.raises(gate.GateError, match="did not all pass against the PR engine"):+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)+++def test_a_base_engine_that_fails_a_case_is_compared_as_recorded(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ fake, _runs = _fake_pytest(returncode=1, outcomes={"test_two": "<failure/>"})+ monkeypatch.setattr(gate, "subprocess", fake)++ cases = gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=False)++ assert cases == {name: _recorded() for name in CASE_NAMES}++[email protected]("must_pass", [True, False])+def test_every_case_must_be_recorded(tmp_path, monkeypatch, must_pass) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ # A case skipped or broken before its exchange records nothing.+ fake, _runs = _fake_pytest(outcomes={"test_two": "<skipped/>"}, recorded=("test_one",))+ monkeypatch.setattr(gate, "subprocess", fake)++ with pytest.raises(gate.GateError, match=r"did not all pass|recorded \['test_one'\], expected \['test_one', 'test_two'\]"):+ gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=must_pass)+++def test_a_case_run_that_collects_nothing_fails(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)++ def collected_nothing(args, *, cwd, env, check):+ report = next(arg for arg in args if arg.startswith("--junitxml=")).removeprefix("--junitxml=")+ Path(report).write_text("<testsuites><testsuite/></testsuites>", encoding="utf-8")+ return SimpleNamespace(returncode=0)++ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=collected_nothing))++ with pytest.raises(gate.GateError, match="did not all pass"):+ gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=False)+++def test_a_case_run_without_a_report_fails(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, _built = _release(tmp_path, monkeypatch)+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=0)))++ with pytest.raises(gate.GateError, match="left no report"):+ gate._run_cases("base", manifest_bytes, "linux-amd64", tmp_path / "base", must_pass=False)+++def test_an_engine_that_is_not_its_manifests_asset_never_runs(tmp_path, monkeypatch) -> None:+ manifest_bytes, owned, _built = _release(tmp_path, monkeypatch)+ url = next(url for url in owned if url.endswith("_linux_amd64.tar.gz"))+ monkeypatch.setattr(gate.release_guard, "_download", _fake_download({**owned, url: b"tampered"}))+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=lambda *a, **k: pytest.fail("ran the cases")))++ with pytest.raises(gate.GateError, match="the PR engine is not the asset its manifest pins"):+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", must_pass=True)+ with pytest.raises(gate.GateError, match="the PR manifest has no linux-riscv64 asset"):+ gate._run_cases("PR", manifest_bytes, "linux-riscv64", tmp_path / "riscv", must_pass=True)+ with pytest.raises(gate.GateError, match="the PR manifest is invalid"):+ gate._run_cases("PR", b'{"schema_version": 2}', "linux-amd64", tmp_path / "invalid", must_pass=True)+++def _build_output(tmp_path: Path, manifest_bytes: bytes, built: dict) -> Path:+ output = tmp_path / "build"+ output.mkdir()+ (output / "model-hub-engine-manifest.json").write_bytes(manifest_bytes)+ for name, archive in built.items():+ (output / name).write_bytes(archive)+ (output / f"{name}.sha256").write_text(f"{gate.release_guard._sha256(output / name)} {name}\n")+ return output+++def test_a_local_candidate_runs_from_its_build_output_without_downloading(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, built = _release(tmp_path, monkeypatch)+ output = _build_output(tmp_path, manifest_bytes, built)+ monkeypatch.setattr(gate.release_guard, "_download", lambda *a, **k: pytest.fail("downloaded"))+ fake, runs = _fake_pytest()+ monkeypatch.setattr(gate, "subprocess", fake)++ assert gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", output, must_pass=True)+ assert len(runs) == 1+++def test_a_local_candidate_must_be_exactly_what_its_manifest_pins(tmp_path, monkeypatch) -> None:+ manifest_bytes, _owned, built = _release(tmp_path, monkeypatch)+ output = _build_output(tmp_path, manifest_bytes, built)+ # A build from another manifest: the copy publication would upload differs.+ (output / "model-hub-engine-manifest.json").write_bytes(manifest_bytes + b"\n")+ monkeypatch.setattr(gate, "subprocess", SimpleNamespace(run=lambda *a, **k: pytest.fail("ran the cases")))++ with pytest.raises(gate.GateError, match="the PR engine is not the asset its manifest pins"):+ gate._run_cases("PR", manifest_bytes, "linux-amd64", tmp_path / "head", output, must_pass=True)