Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(ci): fail the release E2E on test failures; hold Responses E2E to the buffered contract (#3041)
Call White41bae267e59b
diff --git a/.github/workflows/model-hub-e2e.yml b/.github/workflows/model-hub-e2e.ymlindex a89000057..89d99cceb 100644--- a/.github/workflows/model-hub-e2e.yml+++ b/.github/workflows/model-hub-e2e.yml@@ -85,8 +85,11 @@ jobs:run: mkdir -p "$RUNNER_TEMP/model-hub-e2e-junit"# Every release E2E case selected below must be present and passing in this- # run's own report; collection or a green process is not enough.+ # run's own report; collection or a green process is not enough. `shell: bash`+ # runs with pipefail: without it the `| tee` pipeline reports tee's status and a+ # failing pytest reads as green.- name: Run the Model Hub E2E and release evidence gate+ shell: bashrun: |bash scripts/ci_loopback_only.sh python -m pytest -m e2e_model_hub tests/e2e -p no:cacheprovider -ra \--basetemp="$RUNNER_TEMP/e2e" \diff --git a/tests/e2e/test_model_hub_engine_wire.py b/tests/e2e/test_model_hub_engine_wire.pyindex 4f43ca051..0eddf7e91 100644--- a/tests/e2e/test_model_hub_engine_wire.py+++ b/tests/e2e/test_model_hub_engine_wire.py@@ -476,7 +476,12 @@ async def test_responses_stream_with_reasoning_and_tool_call(tmp_path, monkeypat@pytest.mark.parametrize("protocol", PROTOCOLS)async def test_consumer_cancel_closes_upstream_and_keeps_the_source(tmp_path, monkeypatch, protocol) -> None:- """A consumer that stops reading cancels the upstream call and leaves the Source routable."""+ """A consumer that stops reading cancels the upstream call and leaves the Source routable.++ An initial Responses stream is held until its upstream terminal (one exact overload replay,+ docs/plans/openai-streamed-overload-retry.md), so its consumer sees no headers while the+ upstream is still writing: it gives up while waiting instead of after the first bytes.+ """upstream_closed: asyncio.Future = asyncio.get_running_loop().create_future()healthy = await _healthy(protocol)calls = 0@@ -503,13 +508,20 @@ async def test_consumer_cancel_closes_upstream_and_keeps_the_source(tmp_path, moasync with _through_engine(tmp_path, monkeypatch, protocol, handler) as harness:url, body = _frontend_request(protocol, harness.hop, stream=True)+ headers = {"Authorization": f'Bearer {harness.hop["token"]}', **harness.hop["request_headers"]}async with aiohttp.ClientSession() as client:- async with client.post(url, headers={"Authorization": f'Bearer {harness.hop["token"]}',- **harness.hop["request_headers"]},- json=body) as response:- assert response.status == 200- await response.content.readuntil(b"partial")+ if protocol == "openai_responses":+ request = asyncio.ensure_future(client.post(url, headers=headers, json=body))+ done, _ = await asyncio.wait({request}, timeout=1.5)+ assert not done, "a held Responses stream committed headers before its upstream terminal"+ request.cancel()+ await asyncio.gather(request, return_exceptions=True)cancelled_at = time.monotonic()+ else:+ async with client.post(url, headers=headers, json=body) as response:+ assert response.status == 200+ await response.content.readuntil(b"partial")+ cancelled_at = time.monotonic()try:lag = round(await asyncio.wait_for(asyncio.shield(upstream_closed), timeout=10) - cancelled_at, 3)except asyncio.TimeoutError:diff --git a/tests/e2e/test_model_hub_routing.py b/tests/e2e/test_model_hub_routing.pyindex 8342e3594..17c6f5b6c 100644--- a/tests/e2e/test_model_hub_routing.py+++ b/tests/e2e/test_model_hub_routing.py@@ -273,10 +273,37 @@ def test_mh_routing_012_persisted_dormant_opencode_override_reaches_exact_sourceasyncio.run(exercise(adapter, first, second))+async def _save_during_incremental_stream(+ client, url, payload, headers, service, menu, next_chain, adapter, old_connection, mock, terminal, disconnect,+):+ """MH-ROUTING-009 for incrementally streamed protocols: Save while the consumer is reading output."""+ async with client.post(url, json=payload, headers=headers) as response:+ assert response.status == 200+ prefix = b""+ while b"mock response" not in prefix:+ prefix += await asyncio.wait_for(response.content.readuntil(b"\n\n"), timeout=10)+ saved = await asyncio.wait_for(service.set_agent_chain("opencode", menu, next_chain), timeout=10)+ assert adapter.supervisor.client_if_running().connection == old_connection+ assert adapter._active_transports == 1+ if disconnect:+ response.close()+ mock.configure(stream="healthy")+ else:+ mock.configure(stream="healthy")+ tail = await asyncio.wait_for(response.read(), timeout=15)+ assert terminal in prefix + tail+ assert saved["chain"]["manual_override"] == next_chain++@pytest.mark.parametrize("protocol", ["openai_responses", "openai_chat", "anthropic"])@pytest.mark.parametrize("disconnect", [False, True], ids=["complete", "client-close"])def test_mh_routing_009_save_during_active_stream_hot_reloads_without_restart_or_truncation(tmp_path, monkeypatch, protocol, disconnect):- """MH-ROUTING-009: Save hot-reloads the live engine during active output; the stream is neither awaited nor truncated."""+ """MH-ROUTING-009: Save hot-reloads the live engine during active output; the stream is neither awaited nor truncated.++ An initial Responses stream is held by the gateway until its upstream terminal (one exact+ overload replay, docs/plans/openai-streamed-overload-retry.md): the consumer has no bytes while+ the upstream is paused, so the Save lands while the held request's engine transport is live.+ """async def exercise(adapter, mock):service, menu, target, sources = _real_route_service(tmp_path, adapter, [mock], protocol, [])await service.runtime_start()@@ -296,31 +323,46 @@ def test_mh_routing_009_save_during_active_stream_hot_reloads_without_restart_ormock.configure(stream="pause_after_first_output")mock.reset_requests()save = None+ next_target = "vendor/next-unlisted-target"+ next_chain = {"hops": [{"source_id": sources[0].id, "model_id": next_target}]}+ headers = {"Authorization": f"Bearer {token}"}try:async with aiohttp.ClientSession(trust_env=False) as client:- async with client.post(f"{base_url}/v1/{path}", json=payload,- headers={"Authorization": f"Bearer {token}"}) as response:- assert response.status == 200- prefix = b""- while b"mock response" not in prefix:- prefix += await asyncio.wait_for(response.content.readuntil(b"\n\n"), timeout=10)- next_target = "vendor/next-unlisted-target"- save = asyncio.create_task(service.set_agent_chain("opencode", menu, {- "hops": [{"source_id": sources[0].id, "model_id": next_target}],- }))- saved = await asyncio.wait_for(save, timeout=10)- assert adapter.supervisor.client_if_running().connection == old_connection- assert adapter._active_transports == 1- if disconnect:- response.close()- mock.configure(stream="healthy")- else:+ if protocol == "openai_responses":+ held = asyncio.ensure_future(client.post(f"{base_url}/v1/{path}", json=payload, headers=headers))+ try:+ # The upstream is paused after its first output while the gateway holds it.+ deadline = time.monotonic() + 10+ while not ([row for row in mock.requests() if row["path"] == f"/v1/{path}"]+ and adapter._active_transports == 1):+ assert time.monotonic() < deadline, "the held Responses stream never reached its upstream"+ await asyncio.sleep(0.05)+ save = asyncio.create_task(service.set_agent_chain("opencode", menu, next_chain))+ saved = await asyncio.wait_for(save, timeout=10)+ assert not held.done()+ assert adapter.supervisor.client_if_running().connection == old_connection+ assert adapter._active_transports == 1mock.configure(stream="healthy")- tail = await asyncio.wait_for(response.read(), timeout=15)- assert terminal in prefix + tail- assert saved["chain"]["manual_override"] == {"hops": [{"source_id": sources[0].id, "model_id": next_target}]}+ if disconnect:+ held.cancel()+ await asyncio.gather(held, return_exceptions=True)+ else:+ async with await asyncio.wait_for(held, timeout=15) as response:+ assert response.status == 200+ body = await asyncio.wait_for(response.read(), timeout=15)+ assert b"mock response" in body and terminal in body+ finally:+ if not held.done():+ held.cancel()+ await asyncio.gather(held, return_exceptions=True)+ assert saved["chain"]["manual_override"] == next_chain+ else:+ await _save_during_incremental_stream(+ client, f"{base_url}/v1/{path}", payload, headers, service, menu, next_chain, adapter,+ old_connection, mock, terminal, disconnect,+ )async with client.post(f"{base_url}/v1/{path}", json={**payload, "stream": False},- headers={"Authorization": f"Bearer {token}"}) as response:+ headers=headers) as response:raw = await asyncio.wait_for(response.read(), timeout=10)assert response.status == 200, rawcaptured = [row["body"]["model"] for row in mock.requests() if row["path"] == f"/v1/{path}"]diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.pyindex 4dcca0386..33fea7b56 100644--- a/tests/test_ci_pipeline.py+++ b/tests/test_ci_pipeline.py@@ -634,6 +634,78 @@ def test_every_linux_engine_e2e_runner_has_only_loopback() -> None:assert "--loopback-only" in gate["run"].split()+# How GitHub runs a step's script on Linux: the default shell has no pipefail;+# `shell: bash` adds it (docs: "Defaults: bash -e {0}" vs "bash --noprofile --norc -eo pipefail {0}").+_GITHUB_LINUX_SHELLS = {+ None: ["bash", "-e"],+ "bash": ["bash", "--noprofile", "--norc", "-eo", "pipefail"],+}+++def _workflow_steps():+ for workflow in sorted((ROOT / ".github/workflows").glob("*.yml")):+ data = yaml.safe_load(workflow.read_text())+ default = ((data.get("defaults") or {}).get("run") or {}).get("shell")+ for job_id, job in (data.get("jobs") or {}).items():+ job_default = ((job.get("defaults") or {}).get("run") or {}).get("shell") or default+ for step in job.get("steps") or []:+ yield workflow.name, job_id, step, step.get("shell") or job_default+++def test_steps_that_tee_a_command_keep_its_exit_status() -> None:+ """A `cmd | tee log` pipeline exits with tee's status unless pipefail is on, hiding cmd's failure."""++ unguarded = [+ f"{workflow} {job_id}: {step.get('name')}"+ for workflow, job_id, step, shell in _workflow_steps()+ if "| tee" in (step.get("run") or "")+ and shell != "bash"+ and "pipefail" not in (step.get("run") or "")+ ]+ assert unguarded == []+++ ("pytest_exit", "evidence_exit", "expected"),+ [(0, 0, 0), (1, 0, 1), (0, 1, 1), (2, 1, 2)],+ ids=["green", "pytest-failed", "evidence-failed", "both-failed"],+)+def test_release_e2e_gate_fails_when_its_tests_or_evidence_fail(+ tmp_path: Path, pytest_exit: int, evidence_exit: int, expected: int,+) -> None:+ """The release E2E step, run as GitHub runs it, exits with pytest's failure, else the gate's.++ Its pytest output goes through `| tee`; before the step ran with pipefail every+ failing release E2E run (three cases from 2026-10-27) still concluded success.+ """++ release = yaml.safe_load((ROOT / ".github/workflows/model-hub-e2e.yml").read_text())+ step, = [step for job in release["jobs"].values() for step in job["steps"]+ if "e2e_model_hub" in step.get("run", "")]+ (tmp_path / "scripts").mkdir()+ (tmp_path / "scripts" / "ci_loopback_only.sh").write_text(+ f'echo "pytest ran: $*"\nexit {pytest_exit}\n', encoding="utf-8",+ )+ (tmp_path / "scripts" / "ci_scenario_evidence.py").write_text(+ f"raise SystemExit({evidence_exit})\n", encoding="utf-8",+ )+ bin_dir = tmp_path / "bin"+ bin_dir.mkdir()+ (bin_dir / "python").symlink_to(sys.executable)+ script = tmp_path / "step.sh"+ script.write_text(step["run"], encoding="utf-8")++ result = subprocess.run(+ [*_GITHUB_LINUX_SHELLS[step.get("shell")], str(script)],+ cwd=tmp_path,+ env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "RUNNER_TEMP": str(tmp_path)},+ capture_output=True, text=True, check=False,+ )++ assert result.returncode == expected, result.stdout + result.stderr+ assert "pytest ran: python -m pytest -m e2e_model_hub" in (tmp_path / "model-hub-e2e.log").read_text()++@pytest.mark.parametrize("gate_name", ["unit-tests", "install-upgrade-regression", "windows-install-smoke"])def test_required_gates_accept_skips_only_for_proven_docs_and_fail_closed(tmp_path: Path, gate_name: str) -> None:gate = _jobs()[gate_name]