ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(portal): a slow gateway answer no longer flashes "Portal is unavailable" mid-build (#8209)
Mike Bradley14b4a4350a39
diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.pyindex 5ed2253d3..a4fe37aa1 100644--- a/ods/extensions/services/dashboard-api/routers/pixel.py+++ b/ods/extensions/services/dashboard-api/routers/pixel.py@@ -726,16 +726,37 @@ async def _current_runtime_identity(edge_url, key):return unknown_runtime_identity()+# The gateway can answer its model list late while it works through a large+# step. On the RTX Spark (2026-10-25) a status poll waited about 10 s while a+# 27 KB page was being written, its edge call failed, and the composer flashed+# "Portal is unavailable" mid-build. Within this grace a transient edge failure+# repeats the last available answer; definitive states (unloaded, switching,+# recovery, a missing model) answer at once and clear it.+_STATUS_GRACE_SECONDS = 20.0+_last_available_status: tuple[float, dict[str, object]] | None = None++@router.get("/status", dependencies=[Depends(verify_api_key)])async def pixel_status(http_response: Response = None) -> dict[str, object]:"""Return a fixed, nonsecret Pixel availability projection."""+ global _last_available_status# The browser-facing response is newly constructed, so upstream no-store# headers do not survive automatically. Never cache a live identity check.if http_response is not None:http_response.headers["Cache-Control"] = "no-store"+ result, transient = await _status_projection()+ remembered = _last_available_status+ if transient and remembered is not None and time.monotonic() - remembered[0] <= _STATUS_GRACE_SECONDS:+ return dict(remembered[1])+ _last_available_status = (time.monotonic(), dict(result)) if result.get("available") is True else None+ return result+++async def _status_projection() -> tuple[dict[str, object], bool]:+ """The status answer, and whether it is only a transient edge failure."""config = _pixel_config()if config is None:- return {"available": False, "model": None, "detail": "Portal is not enabled"}+ return {"available": False, "model": None, "detail": "Portal is not enabled"}, Falsehost_status, (access, access_issue) = await asyncio.gather(_host_model_status(), _current_access_readiness())activation = model_activation_status(host_status)@@ -745,10 +766,10 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:# probing the backend that the coordinator may currently be restarting.if access and access["available"] and access["pending"] and access["reason"] == "model-transition-pending":return {**activation_metadata, "available": False, "model": None, "state": "model_transition_pending",- "detail": _MODEL_HOLD_DETAIL}+ "detail": _MODEL_HOLD_DETAIL}, Falseif access and access["available"] and access["reason"] == "chat-recovery-required":return {**activation_metadata, "available": False, "model": None, "state": "chat_recovery_required",- "detail": _CHAT_RECOVERY_DETAIL, **_recovery_field(_access_recovery(access))}+ "detail": _CHAT_RECOVERY_DETAIL, **_recovery_field(_access_recovery(access))}, Falsereadiness_issue = await _model_readiness_issue_for_status(host_status)if readiness_issue is not None:state, detail = readiness_issue@@ -759,7 +780,7 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:"state": state,"detail": detail,**_recovery_field(_STATE_RECOVERY.get(state)),- }+ }, Falseedge_url, key = configtry:timeout = httpx.Timeout(connect=5.0, read=5.0, write=5.0, pool=5.0)@@ -771,9 +792,9 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:timeout=timeout,) as response:if response.status_code != 200:- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}, Trueif not response.headers.get("content-type", "").lower().startswith("application/json"):- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}, Trueraw = await _bounded_response_bytes(response, _MAX_STATUS_BYTES)payload = json.loads(raw)models = payload.get("data") if isinstance(payload, dict) else None@@ -790,9 +811,12 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:if inference_issue:# A stopped runtime the host owns gets Models' Resume model.stopped = await _owned_runtime_unloaded(_RUNTIME_HINT_SECONDS) is True+ # A health probe that misses its 3 s while the runtime generates+ # a large step (RTX Spark, 2026-10-25) is not proof it stopped:+ # only a runtime the host proves stopped answers at once.return {**activation_metadata, "available": False, "model": None, "state": "model_unavailable","detail": _RUNTIME_STOPPED_DETAIL if stopped else inference_issue,- **_recovery_field("resume-model" if stopped else None)}+ **_recovery_field("resume-model" if stopped else None)}, not stoppedruntime = _active_runtime_projection(host_status)if available and runtime is None:runtime = await _verified_external_host_runtime(host_status)@@ -824,14 +848,14 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:result["detail"] = ("Owner agent available; model transition needs recovery; release readiness is unverified"if readiness["reasonCode"] == "model-transition-recovery-required" else"Owner agent available; access transition is unfinished; release readiness is unverified")- return result+ return result, Falseexcept (httpx.HTTPError, asyncio.TimeoutError) as exc:# Exception text and request objects can contain upstream credentials.# Retain the failure phase/type without logging those sensitive values.logger.warning("Pixel edge status request failed (%s)", type(exc).__name__)- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}, Trueexcept (json.JSONDecodeError, UnicodeDecodeError, ValueError, TypeError):- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}, True@router.get("/ops/{job_id}", dependencies=[Depends(verify_api_key)])diff --git a/ods/extensions/services/dashboard-api/tests/conftest.py b/ods/extensions/services/dashboard-api/tests/conftest.pyindex 8ee628941..1ab6628a0 100644--- a/ods/extensions/services/dashboard-api/tests/conftest.py+++ b/ods/extensions/services/dashboard-api/tests/conftest.py@@ -59,6 +59,14 @@ def mock_edge_read_transport(monkeypatch):monkeypatch.setattr(pixel_edge_read_client, 'get_edge_read_client', get_fake)[email protected](autouse=True)+def _no_remembered_portal_status(monkeypatch):+ """Each test starts without an available Portal status remembered by an earlier one."""+ from routers import pixel++ monkeypatch.setattr(pixel, "_last_available_status", None)++@pytest.fixture(autouse=True)def _no_hugging_face_header_reads(monkeypatch):"""Unit tests never reach Hugging Face; a test that needs a header stubs one."""diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel.py b/ods/extensions/services/dashboard-api/tests/test_pixel.pyindex 29f35fa93..3d60f7de6 100644--- a/ods/extensions/services/dashboard-api/tests/test_pixel.py+++ b/ods/extensions/services/dashboard-api/tests/test_pixel.py@@ -8,7 +8,7 @@ import jsonimport osimport pathlibimport sys-from unittest.mock import ANY, patch+from unittest.mock import ANY, AsyncMock, patchimport pytestfrom fastapi import FastAPI, HTTPException@@ -353,6 +353,69 @@ async def test_status_returns_only_fixed_projection():assert secret not in json.dumps(result)+async def test_status_rides_out_a_transient_edge_failure_but_not_a_definitive_one(monkeypatch):+ # RTX Spark: the gateway answered its model list late while writing a large+ # page, and one failed poll flashed "Portal is unavailable" mid-build.+ listing = FakeResponse(chunks=[json.dumps({"data": [{"id": "portal/default"}]}).encode()])+ clock = [1000.0]+ monkeypatch.setattr(pixel.time, "monotonic", lambda: clock[0])+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ available = await pixel.pixel_status()+ assert available["available"] is True+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(FakeResponse(status=502))):+ clock[0] += pixel._STATUS_GRACE_SECONDS - 1+ assert await pixel.pixel_status() == available+ clock[0] += 2+ expired = await pixel.pixel_status()+ assert expired["available"] is False and expired["detail"] == "Portal service is unavailable"+ # A missing model is an answer, not a transient failure: it shows at once+ # and later failures do not resurrect the earlier available status.+ clock[0] = 2000.0+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ await pixel.pixel_status()+ missing = FakeResponse(chunks=[json.dumps({"data": []}).encode()])+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(missing)):+ assert (await pixel.pixel_status())["available"] is False+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(FakeResponse(status=502))):+ assert (await pixel.pixel_status())["available"] is False+++async def test_status_rides_out_a_slow_runtime_probe_but_not_a_stopped_runtime(monkeypatch):+ # RTX Spark: the host runtime's health probe missed its 3 s while the model+ # wrote a large page, and Portal told the owner to restore the model mid-build.+ listing = FakeResponse(chunks=[json.dumps({"data": [{"id": "portal/default"}]}).encode()])+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ available = await pixel.pixel_status()+ assert available["available"] is True+ issue = "The local model runtime is unavailable. Restore it in Models before sending another task."+ monkeypatch.setattr(pixel, "_local_inference_issue", AsyncMock(return_value=issue))+ monkeypatch.setattr(pixel, "_owned_runtime_unloaded", AsyncMock(return_value=None))+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ assert await pixel.pixel_status() == available+ # A runtime the host proves stopped is an answer: Resume model shows at once.+ monkeypatch.setattr(pixel, "_owned_runtime_unloaded", AsyncMock(return_value=True))+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ stopped = await pixel.pixel_status()+ assert stopped["available"] is False and stopped["state"] == "model_unavailable"+ assert stopped["recovery"] == "resume-model"+++async def test_status_rides_out_an_edge_timeout_within_the_grace(monkeypatch):+ listing = FakeResponse(chunks=[json.dumps({"data": [{"id": "portal/default"}]}).encode()])+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ available = await pixel.pixel_status()++ async def timeout(_self):+ raise pixel.httpx.ReadTimeout("slow gateway")++ monkeypatch.setattr(FakeStreamContext, "__aenter__", timeout)+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):+ assert await pixel.pixel_status() == available++@pytest.mark.asyncioasync def test_status_accepts_real_edge_advertised_model(monkeypatch):# Exercise the production edge listing rather than repeating the dashboarddiff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py b/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.pyindex 74e84d03a..7a09e6c36 100644--- a/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py+++ b/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py@@ -1,6 +1,7 @@"""Read-only edge polls reuse transport, never availability or credentials."""import asyncioimport json+import timefrom unittest.mock import AsyncMockimport httpx@@ -69,7 +70,10 @@ async def test_status_reuses_connection_across_normal_poll_interval(monkeypatch,writer.close()await writer.wait_closed()await server.wait_closed()- # A reused connection is transport only: a stopped edge still fails.+ # A reused connection is transport only: a stopped edge still fails,+ # once the grace for a transient edge failure has run out.+ now = time.monotonic()+ monkeypatch.setattr(pixel.time, 'monotonic', lambda: now + pixel._STATUS_GRACE_SECONDS + 1)assert (await pixel.pixel_status())['available'] is Falsefinally:server.close()