Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(ci): fail the release E2E on test failures; hold Responses E2E to the buffered contract (#3041)

Call White41bae267e59b
diff --git a/.github/workflows/model-hub-e2e.yml b/.github/workflows/model-hub-e2e.yml
index a89000057..89d99cceb 100644
--- a/.github/workflows/model-hub-e2e.yml
+++ b/.github/workflows/model-hub-e2e.yml
@@ -85,8 +85,11 @@ jobs:
run: mkdir -p "$RUNNER_TEMP/model-hub-e2e-junit"
# Every release E2E case selected below must be present and passing in this
- # run's own report; collection or a green process is not enough.
+ # run's own report; collection or a green process is not enough. `shell: bash`
+ # runs with pipefail: without it the `| tee` pipeline reports tee's status and a
+ # failing pytest reads as green.
- name: Run the Model Hub E2E and release evidence gate
+ shell: bash
run: |
bash scripts/ci_loopback_only.sh python -m pytest -m e2e_model_hub tests/e2e -p no:cacheprovider -ra \
--basetemp="$RUNNER_TEMP/e2e" \
diff --git a/tests/e2e/test_model_hub_engine_wire.py b/tests/e2e/test_model_hub_engine_wire.py
index 4f43ca051..0eddf7e91 100644
--- a/tests/e2e/test_model_hub_engine_wire.py
+++ b/tests/e2e/test_model_hub_engine_wire.py
@@ -476,7 +476,12 @@ async def test_responses_stream_with_reasoning_and_tool_call(tmp_path, monkeypat
@pytest.mark.parametrize("protocol", PROTOCOLS)
async def test_consumer_cancel_closes_upstream_and_keeps_the_source(tmp_path, monkeypatch, protocol) -> None:
- """A consumer that stops reading cancels the upstream call and leaves the Source routable."""
+ """A consumer that stops reading cancels the upstream call and leaves the Source routable.
+
+ An initial Responses stream is held until its upstream terminal (one exact overload replay,
+ docs/plans/openai-streamed-overload-retry.md), so its consumer sees no headers while the
+ upstream is still writing: it gives up while waiting instead of after the first bytes.
+ """
upstream_closed: asyncio.Future = asyncio.get_running_loop().create_future()
healthy = await _healthy(protocol)
calls = 0
@@ -503,13 +508,20 @@ async def test_consumer_cancel_closes_upstream_and_keeps_the_source(tmp_path, mo
async with _through_engine(tmp_path, monkeypatch, protocol, handler) as harness:
url, body = _frontend_request(protocol, harness.hop, stream=True)
+ headers = {"Authorization": f'Bearer {harness.hop["token"]}', **harness.hop["request_headers"]}
async with aiohttp.ClientSession() as client:
- async with client.post(url, headers={"Authorization": f'Bearer {harness.hop["token"]}',
- **harness.hop["request_headers"]},
- json=body) as response:
- assert response.status == 200
- await response.content.readuntil(b"partial")
+ if protocol == "openai_responses":
+ request = asyncio.ensure_future(client.post(url, headers=headers, json=body))
+ done, _ = await asyncio.wait({request}, timeout=1.5)
+ assert not done, "a held Responses stream committed headers before its upstream terminal"
+ request.cancel()
+ await asyncio.gather(request, return_exceptions=True)
cancelled_at = time.monotonic()
+ else:
+ async with client.post(url, headers=headers, json=body) as response:
+ assert response.status == 200
+ await response.content.readuntil(b"partial")
+ cancelled_at = time.monotonic()
try:
lag = round(await asyncio.wait_for(asyncio.shield(upstream_closed), timeout=10) - cancelled_at, 3)
except asyncio.TimeoutError:
diff --git a/tests/e2e/test_model_hub_routing.py b/tests/e2e/test_model_hub_routing.py
index 8342e3594..17c6f5b6c 100644
--- a/tests/e2e/test_model_hub_routing.py
+++ b/tests/e2e/test_model_hub_routing.py
@@ -273,10 +273,37 @@ def test_mh_routing_012_persisted_dormant_opencode_override_reaches_exact_source
asyncio.run(exercise(adapter, first, second))
+async def _save_during_incremental_stream(
+ client, url, payload, headers, service, menu, next_chain, adapter, old_connection, mock, terminal, disconnect,
+):
+ """MH-ROUTING-009 for incrementally streamed protocols: Save while the consumer is reading output."""
+ async with client.post(url, json=payload, headers=headers) as response:
+ assert response.status == 200
+ prefix = b""
+ while b"mock response" not in prefix:
+ prefix += await asyncio.wait_for(response.content.readuntil(b"\n\n"), timeout=10)
+ saved = await asyncio.wait_for(service.set_agent_chain("opencode", menu, next_chain), timeout=10)
+ assert adapter.supervisor.client_if_running().connection == old_connection
+ assert adapter._active_transports == 1
+ if disconnect:
+ response.close()
+ mock.configure(stream="healthy")
+ else:
+ mock.configure(stream="healthy")
+ tail = await asyncio.wait_for(response.read(), timeout=15)
+ assert terminal in prefix + tail
+ assert saved["chain"]["manual_override"] == next_chain
+
+
@pytest.mark.parametrize("protocol", ["openai_responses", "openai_chat", "anthropic"])
@pytest.mark.parametrize("disconnect", [False, True], ids=["complete", "client-close"])
def test_mh_routing_009_save_during_active_stream_hot_reloads_without_restart_or_truncation(tmp_path, monkeypatch, protocol, disconnect):
- """MH-ROUTING-009: Save hot-reloads the live engine during active output; the stream is neither awaited nor truncated."""
+ """MH-ROUTING-009: Save hot-reloads the live engine during active output; the stream is neither awaited nor truncated.
+
+ An initial Responses stream is held by the gateway until its upstream terminal (one exact
+ overload replay, docs/plans/openai-streamed-overload-retry.md): the consumer has no bytes while
+ the upstream is paused, so the Save lands while the held request's engine transport is live.
+ """
async def exercise(adapter, mock):
service, menu, target, sources = _real_route_service(tmp_path, adapter, [mock], protocol, [])
await service.runtime_start()
@@ -296,31 +323,46 @@ def test_mh_routing_009_save_during_active_stream_hot_reloads_without_restart_or
mock.configure(stream="pause_after_first_output")
mock.reset_requests()
save = None
+ next_target = "vendor/next-unlisted-target"
+ next_chain = {"hops": [{"source_id": sources[0].id, "model_id": next_target}]}
+ headers = {"Authorization": f"Bearer {token}"}
try:
async with aiohttp.ClientSession(trust_env=False) as client:
- async with client.post(f"{base_url}/v1/{path}", json=payload,
- headers={"Authorization": f"Bearer {token}"}) as response:
- assert response.status == 200
- prefix = b""
- while b"mock response" not in prefix:
- prefix += await asyncio.wait_for(response.content.readuntil(b"\n\n"), timeout=10)
- next_target = "vendor/next-unlisted-target"
- save = asyncio.create_task(service.set_agent_chain("opencode", menu, {
- "hops": [{"source_id": sources[0].id, "model_id": next_target}],
- }))
- saved = await asyncio.wait_for(save, timeout=10)
- assert adapter.supervisor.client_if_running().connection == old_connection
- assert adapter._active_transports == 1
- if disconnect:
- response.close()
- mock.configure(stream="healthy")
- else:
+ if protocol == "openai_responses":
+ held = asyncio.ensure_future(client.post(f"{base_url}/v1/{path}", json=payload, headers=headers))
+ try:
+ # The upstream is paused after its first output while the gateway holds it.
+ deadline = time.monotonic() + 10
+ while not ([row for row in mock.requests() if row["path"] == f"/v1/{path}"]
+ and adapter._active_transports == 1):
+ assert time.monotonic() < deadline, "the held Responses stream never reached its upstream"
+ await asyncio.sleep(0.05)
+ save = asyncio.create_task(service.set_agent_chain("opencode", menu, next_chain))
+ saved = await asyncio.wait_for(save, timeout=10)
+ assert not held.done()
+ assert adapter.supervisor.client_if_running().connection == old_connection
+ assert adapter._active_transports == 1
mock.configure(stream="healthy")
- tail = await asyncio.wait_for(response.read(), timeout=15)
- assert terminal in prefix + tail
- assert saved["chain"]["manual_override"] == {"hops": [{"source_id": sources[0].id, "model_id": next_target}]}
+ if disconnect:
+ held.cancel()
+ await asyncio.gather(held, return_exceptions=True)
+ else:
+ async with await asyncio.wait_for(held, timeout=15) as response:
+ assert response.status == 200
+ body = await asyncio.wait_for(response.read(), timeout=15)
+ assert b"mock response" in body and terminal in body
+ finally:
+ if not held.done():
+ held.cancel()
+ await asyncio.gather(held, return_exceptions=True)
+ assert saved["chain"]["manual_override"] == next_chain
+ else:
+ await _save_during_incremental_stream(
+ client, f"{base_url}/v1/{path}", payload, headers, service, menu, next_chain, adapter,
+ old_connection, mock, terminal, disconnect,
+ )
async with client.post(f"{base_url}/v1/{path}", json={**payload, "stream": False},
- headers={"Authorization": f"Bearer {token}"}) as response:
+ headers=headers) as response:
raw = await asyncio.wait_for(response.read(), timeout=10)
assert response.status == 200, raw
captured = [row["body"]["model"] for row in mock.requests() if row["path"] == f"/v1/{path}"]
diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py
index 4dcca0386..33fea7b56 100644
--- a/tests/test_ci_pipeline.py
+++ b/tests/test_ci_pipeline.py
@@ -634,6 +634,78 @@ def test_every_linux_engine_e2e_runner_has_only_loopback() -> None:
assert "--loopback-only" in gate["run"].split()
+# How GitHub runs a step's script on Linux: the default shell has no pipefail;
+# `shell: bash` adds it (docs: "Defaults: bash -e {0}" vs "bash --noprofile --norc -eo pipefail {0}").
+_GITHUB_LINUX_SHELLS = {
+ None: ["bash", "-e"],
+ "bash": ["bash", "--noprofile", "--norc", "-eo", "pipefail"],
+}
+
+
+def _workflow_steps():
+ for workflow in sorted((ROOT / ".github/workflows").glob("*.yml")):
+ data = yaml.safe_load(workflow.read_text())
+ default = ((data.get("defaults") or {}).get("run") or {}).get("shell")
+ for job_id, job in (data.get("jobs") or {}).items():
+ job_default = ((job.get("defaults") or {}).get("run") or {}).get("shell") or default
+ for step in job.get("steps") or []:
+ yield workflow.name, job_id, step, step.get("shell") or job_default
+
+
+def test_steps_that_tee_a_command_keep_its_exit_status() -> None:
+ """A `cmd | tee log` pipeline exits with tee's status unless pipefail is on, hiding cmd's failure."""
+
+ unguarded = [
+ f"{workflow} {job_id}: {step.get('name')}"
+ for workflow, job_id, step, shell in _workflow_steps()
+ if "| tee" in (step.get("run") or "")
+ and shell != "bash"
+ and "pipefail" not in (step.get("run") or "")
+ ]
+ assert unguarded == []
+
+
+ ("pytest_exit", "evidence_exit", "expected"),
+ [(0, 0, 0), (1, 0, 1), (0, 1, 1), (2, 1, 2)],
+ ids=["green", "pytest-failed", "evidence-failed", "both-failed"],
+)
+def test_release_e2e_gate_fails_when_its_tests_or_evidence_fail(
+ tmp_path: Path, pytest_exit: int, evidence_exit: int, expected: int,
+) -> None:
+ """The release E2E step, run as GitHub runs it, exits with pytest's failure, else the gate's.
+
+ Its pytest output goes through `| tee`; before the step ran with pipefail every
+ failing release E2E run (three cases from 2026-10-27) still concluded success.
+ """
+
+ release = yaml.safe_load((ROOT / ".github/workflows/model-hub-e2e.yml").read_text())
+ step, = [step for job in release["jobs"].values() for step in job["steps"]
+ if "e2e_model_hub" in step.get("run", "")]
+ (tmp_path / "scripts").mkdir()
+ (tmp_path / "scripts" / "ci_loopback_only.sh").write_text(
+ f'echo "pytest ran: $*"\nexit {pytest_exit}\n', encoding="utf-8",
+ )
+ (tmp_path / "scripts" / "ci_scenario_evidence.py").write_text(
+ f"raise SystemExit({evidence_exit})\n", encoding="utf-8",
+ )
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ (bin_dir / "python").symlink_to(sys.executable)
+ script = tmp_path / "step.sh"
+ script.write_text(step["run"], encoding="utf-8")
+
+ result = subprocess.run(
+ [*_GITHUB_LINUX_SHELLS[step.get("shell")], str(script)],
+ cwd=tmp_path,
+ env={**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "RUNNER_TEMP": str(tmp_path)},
+ capture_output=True, text=True, check=False,
+ )
+
+ assert result.returncode == expected, result.stdout + result.stderr
+ assert "pytest ran: python -m pytest -m e2e_model_hub" in (tmp_path / "model-hub-e2e.log").read_text()
+
+
@pytest.mark.parametrize("gate_name", ["unit-tests", "install-upgrade-regression", "windows-install-smoke"])
def test_required_gates_accept_skips_only_for_proven_docs_and_fail_closed(tmp_path: Path, gate_name: str) -> None:
gate = _jobs()[gate_name]