Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(portal): a slow gateway answer no longer flashes "Portal is unavailable" mid-build (#8209)

Mike Bradley14b4a4350a39
diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.py
index 5ed2253d3..a4fe37aa1 100644
--- a/ods/extensions/services/dashboard-api/routers/pixel.py
+++ b/ods/extensions/services/dashboard-api/routers/pixel.py
@@ -726,16 +726,37 @@ async def _current_runtime_identity(edge_url, key):
return unknown_runtime_identity()
+# The gateway can answer its model list late while it works through a large
+# step. On the RTX Spark (2026-10-25) a status poll waited about 10 s while a
+# 27 KB page was being written, its edge call failed, and the composer flashed
+# "Portal is unavailable" mid-build. Within this grace a transient edge failure
+# repeats the last available answer; definitive states (unloaded, switching,
+# recovery, a missing model) answer at once and clear it.
+_STATUS_GRACE_SECONDS = 20.0
+_last_available_status: tuple[float, dict[str, object]] | None = None
+
+
@router.get("/status", dependencies=[Depends(verify_api_key)])
async def pixel_status(http_response: Response = None) -> dict[str, object]:
"""Return a fixed, nonsecret Pixel availability projection."""
+ global _last_available_status
# The browser-facing response is newly constructed, so upstream no-store
# headers do not survive automatically. Never cache a live identity check.
if http_response is not None:
http_response.headers["Cache-Control"] = "no-store"
+ result, transient = await _status_projection()
+ remembered = _last_available_status
+ if transient and remembered is not None and time.monotonic() - remembered[0] <= _STATUS_GRACE_SECONDS:
+ return dict(remembered[1])
+ _last_available_status = (time.monotonic(), dict(result)) if result.get("available") is True else None
+ return result
+
+
+async def _status_projection() -> tuple[dict[str, object], bool]:
+ """The status answer, and whether it is only a transient edge failure."""
config = _pixel_config()
if config is None:
- return {"available": False, "model": None, "detail": "Portal is not enabled"}
+ return {"available": False, "model": None, "detail": "Portal is not enabled"}, False
host_status, (access, access_issue) = await asyncio.gather(
_host_model_status(), _current_access_readiness())
activation = model_activation_status(host_status)
@@ -745,10 +766,10 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
# probing the backend that the coordinator may currently be restarting.
if access and access["available"] and access["pending"] and access["reason"] == "model-transition-pending":
return {**activation_metadata, "available": False, "model": None, "state": "model_transition_pending",
- "detail": _MODEL_HOLD_DETAIL}
+ "detail": _MODEL_HOLD_DETAIL}, False
if access and access["available"] and access["reason"] == "chat-recovery-required":
return {**activation_metadata, "available": False, "model": None, "state": "chat_recovery_required",
- "detail": _CHAT_RECOVERY_DETAIL, **_recovery_field(_access_recovery(access))}
+ "detail": _CHAT_RECOVERY_DETAIL, **_recovery_field(_access_recovery(access))}, False
readiness_issue = await _model_readiness_issue_for_status(host_status)
if readiness_issue is not None:
state, detail = readiness_issue
@@ -759,7 +780,7 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
"state": state,
"detail": detail,
**_recovery_field(_STATE_RECOVERY.get(state)),
- }
+ }, False
edge_url, key = config
try:
timeout = httpx.Timeout(connect=5.0, read=5.0, write=5.0, pool=5.0)
@@ -771,9 +792,9 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
timeout=timeout,
) as response:
if response.status_code != 200:
- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}
+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}, True
if not response.headers.get("content-type", "").lower().startswith("application/json"):
- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}
+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}, True
raw = await _bounded_response_bytes(response, _MAX_STATUS_BYTES)
payload = json.loads(raw)
models = payload.get("data") if isinstance(payload, dict) else None
@@ -790,9 +811,12 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
if inference_issue:
# A stopped runtime the host owns gets Models' Resume model.
stopped = await _owned_runtime_unloaded(_RUNTIME_HINT_SECONDS) is True
+ # A health probe that misses its 3 s while the runtime generates
+ # a large step (RTX Spark, 2026-10-25) is not proof it stopped:
+ # only a runtime the host proves stopped answers at once.
return {**activation_metadata, "available": False, "model": None, "state": "model_unavailable",
"detail": _RUNTIME_STOPPED_DETAIL if stopped else inference_issue,
- **_recovery_field("resume-model" if stopped else None)}
+ **_recovery_field("resume-model" if stopped else None)}, not stopped
runtime = _active_runtime_projection(host_status)
if available and runtime is None:
runtime = await _verified_external_host_runtime(host_status)
@@ -824,14 +848,14 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
result["detail"] = ("Owner agent available; model transition needs recovery; release readiness is unverified"
if readiness["reasonCode"] == "model-transition-recovery-required" else
"Owner agent available; access transition is unfinished; release readiness is unverified")
- return result
+ return result, False
except (httpx.HTTPError, asyncio.TimeoutError) as exc:
# Exception text and request objects can contain upstream credentials.
# Retain the failure phase/type without logging those sensitive values.
logger.warning("Pixel edge status request failed (%s)", type(exc).__name__)
- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}
+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service is unavailable"}, True
except (json.JSONDecodeError, UnicodeDecodeError, ValueError, TypeError):
- return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}
+ return {**activation_metadata, "available": False, "model": None, "detail": "Portal service returned an invalid response"}, True
@router.get("/ops/{job_id}", dependencies=[Depends(verify_api_key)])
diff --git a/ods/extensions/services/dashboard-api/tests/conftest.py b/ods/extensions/services/dashboard-api/tests/conftest.py
index 8ee628941..1ab6628a0 100644
--- a/ods/extensions/services/dashboard-api/tests/conftest.py
+++ b/ods/extensions/services/dashboard-api/tests/conftest.py
@@ -59,6 +59,14 @@ def mock_edge_read_transport(monkeypatch):
monkeypatch.setattr(pixel_edge_read_client, 'get_edge_read_client', get_fake)
[email protected](autouse=True)
+def _no_remembered_portal_status(monkeypatch):
+ """Each test starts without an available Portal status remembered by an earlier one."""
+ from routers import pixel
+
+ monkeypatch.setattr(pixel, "_last_available_status", None)
+
+
@pytest.fixture(autouse=True)
def _no_hugging_face_header_reads(monkeypatch):
"""Unit tests never reach Hugging Face; a test that needs a header stubs one."""
diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel.py b/ods/extensions/services/dashboard-api/tests/test_pixel.py
index 29f35fa93..3d60f7de6 100644
--- a/ods/extensions/services/dashboard-api/tests/test_pixel.py
+++ b/ods/extensions/services/dashboard-api/tests/test_pixel.py
@@ -8,7 +8,7 @@ import json
import os
import pathlib
import sys
-from unittest.mock import ANY, patch
+from unittest.mock import ANY, AsyncMock, patch
import pytest
from fastapi import FastAPI, HTTPException
@@ -353,6 +353,69 @@ async def test_status_returns_only_fixed_projection():
assert secret not in json.dumps(result)
+async def test_status_rides_out_a_transient_edge_failure_but_not_a_definitive_one(monkeypatch):
+ # RTX Spark: the gateway answered its model list late while writing a large
+ # page, and one failed poll flashed "Portal is unavailable" mid-build.
+ listing = FakeResponse(chunks=[json.dumps({"data": [{"id": "portal/default"}]}).encode()])
+ clock = [1000.0]
+ monkeypatch.setattr(pixel.time, "monotonic", lambda: clock[0])
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ available = await pixel.pixel_status()
+ assert available["available"] is True
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(FakeResponse(status=502))):
+ clock[0] += pixel._STATUS_GRACE_SECONDS - 1
+ assert await pixel.pixel_status() == available
+ clock[0] += 2
+ expired = await pixel.pixel_status()
+ assert expired["available"] is False and expired["detail"] == "Portal service is unavailable"
+ # A missing model is an answer, not a transient failure: it shows at once
+ # and later failures do not resurrect the earlier available status.
+ clock[0] = 2000.0
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ await pixel.pixel_status()
+ missing = FakeResponse(chunks=[json.dumps({"data": []}).encode()])
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(missing)):
+ assert (await pixel.pixel_status())["available"] is False
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(FakeResponse(status=502))):
+ assert (await pixel.pixel_status())["available"] is False
+
+
+async def test_status_rides_out_a_slow_runtime_probe_but_not_a_stopped_runtime(monkeypatch):
+ # RTX Spark: the host runtime's health probe missed its 3 s while the model
+ # wrote a large page, and Portal told the owner to restore the model mid-build.
+ listing = FakeResponse(chunks=[json.dumps({"data": [{"id": "portal/default"}]}).encode()])
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ available = await pixel.pixel_status()
+ assert available["available"] is True
+ issue = "The local model runtime is unavailable. Restore it in Models before sending another task."
+ monkeypatch.setattr(pixel, "_local_inference_issue", AsyncMock(return_value=issue))
+ monkeypatch.setattr(pixel, "_owned_runtime_unloaded", AsyncMock(return_value=None))
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ assert await pixel.pixel_status() == available
+ # A runtime the host proves stopped is an answer: Resume model shows at once.
+ monkeypatch.setattr(pixel, "_owned_runtime_unloaded", AsyncMock(return_value=True))
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ stopped = await pixel.pixel_status()
+ assert stopped["available"] is False and stopped["state"] == "model_unavailable"
+ assert stopped["recovery"] == "resume-model"
+
+
+async def test_status_rides_out_an_edge_timeout_within_the_grace(monkeypatch):
+ listing = FakeResponse(chunks=[json.dumps({"data": [{"id": "portal/default"}]}).encode()])
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ available = await pixel.pixel_status()
+
+ async def timeout(_self):
+ raise pixel.httpx.ReadTimeout("slow gateway")
+
+ monkeypatch.setattr(FakeStreamContext, "__aenter__", timeout)
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(listing)):
+ assert await pixel.pixel_status() == available
+
+
@pytest.mark.asyncio
async def test_status_accepts_real_edge_advertised_model(monkeypatch):
# Exercise the production edge listing rather than repeating the dashboard
diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py b/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py
index 74e84d03a..7a09e6c36 100644
--- a/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py
+++ b/ods/extensions/services/dashboard-api/tests/test_pixel_edge_read_client.py
@@ -1,6 +1,7 @@
"""Read-only edge polls reuse transport, never availability or credentials."""
import asyncio
import json
+import time
from unittest.mock import AsyncMock
import httpx
@@ -69,7 +70,10 @@ async def test_status_reuses_connection_across_normal_poll_interval(monkeypatch,
writer.close()
await writer.wait_closed()
await server.wait_closed()
- # A reused connection is transport only: a stopped edge still fails.
+ # A reused connection is transport only: a stopped edge still fails,
+ # once the grace for a transient edge failure has run out.
+ now = time.monotonic()
+ monkeypatch.setattr(pixel.time, 'monotonic', lambda: now + pixel._STATUS_GRACE_SECONDS + 1)
assert (await pixel.pixel_status())['available'] is False
finally:
server.close()