Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(portal): heal a dropped access proof and recover held states in Portal (#8189)

Mike Bradley4731855b6644
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index 2cb1e2949..fedb85c94 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -10295,6 +10295,12 @@ class AgentHandler(BaseHTTPRequestHandler):
json_response(self, status, body)
except Exception:
json_response(self, 503, {"error": "access-service-unavailable"})
+ return
+ if status == 200 and _pixel_access_stale_sandbox_proof(body):
+ # A restart dropped an idle Sandbox's proof. Let the access
+ # monitor re-prove it now with the protected helper, instead of
+ # at its next scheduled check. This read itself changes nothing.
+ _pixel_access_monitor_wake.set()
return
body = read_json_body(self)
if body is None:
@@ -21512,10 +21518,23 @@ def _request_server_shutdown(server, signum=None):
def _reconcile_native_pixel_startup():
- """Re-prove an unchanged native policy, serialized with model operations."""
+ """Re-prove an unchanged native policy, serialized with model operations.
+
+ Returns ``(outcome, detail)`` for the access monitor:
+
+ - ``recheck``: verified, re-proved, in use, or deferred while ODS starts
+ or runs a model operation;
+ - ``attention``: the helper refused before it submitted any change, for
+ example a held transition or an interrupted chat;
+ - ``uncertain``: a change was submitted, or may have been, and is not
+ confirmed;
+ - ``None``: this host has no usable protected helper.
+
+ ``detail`` names the helper's bounded stage and reason tokens.
+ """
helper = Path('/usr/local/libexec/ods-pixel-access/pixel_access_reconcile.py')
if platform.system() not in ('Darwin', 'Linux') or not helper.exists():
- return
+ return None, None
# Execute only the installed root-owned helper, never an owner checkout.
try:
for entry in (helper, *helper.parents):
@@ -21534,10 +21553,10 @@ def _reconcile_native_pixel_startup():
if (len(declarations) != 1 or not isinstance(declarations[0].value, ast.Constant)
or type(declarations[0].value.value) is not int or declarations[0].value.value != 1):
logger.warning('Pixel startup reproof requires a compatible installed helper')
- return
+ return None, None
except (OSError, ValueError, SyntaxError):
logger.warning('Pixel startup reproof refused: helper custody')
- return
+ return None, None
unavailable = None
for attempt in range(12):
acquired, active = _begin_model_lifecycle('pixel_startup_reproof')
@@ -21549,21 +21568,22 @@ def _reconcile_native_pixel_startup():
)
if result.returncode == 0:
logger.debug('Pixel access reproof check completed')
- return True
+ return 'recheck', None
if len(result.stderr) > 8192:
raise ValueError('diagnostic-size')
diagnostic = json.loads(result.stderr)
projection = diagnostic.get('projection', {})
- if (diagnostic.get('stage') == 'unsafe-state'
+ stage = diagnostic.get('stage')
+ if (stage == 'unsafe-state'
and projection.get('scope') == 'owner-host'
and projection.get('available') is True
and projection.get('pending') is False
and projection.get('busy') is True):
- return True
+ return 'recheck', None
# Retry only an unavailable preflight. Never replay an
# uncertain mutation or consume another pending transaction.
- retry = (diagnostic.get('stage') == 'status-transport-unavailable' or (
- diagnostic.get('stage') in ('status-unavailable', 'unsafe-state')
+ retry = (stage == 'status-transport-unavailable' or (
+ stage in ('status-unavailable', 'unsafe-state')
and projection.get('available') is False
and projection.get('pending') is False
and projection.get('busy') is False
@@ -21575,13 +21595,16 @@ def _reconcile_native_pixel_startup():
'edge-container-unavailable', 'host-command-failed')))
if not retry:
# Both values are bounded tokens from the helper's projection.
- logger.warning('Pixel startup reproof requires attention (stage=%s reason=%s)',
- diagnostic.get('stage'), projection.get('reason'))
- return
- unavailable = (diagnostic.get('stage'), projection.get('reason'))
+ detail = 'stage=%s reason=%s' % (stage, projection.get('reason'))
+ # The helper raises these two from its status read, before
+ # it submits a change. Anything else may follow a change.
+ if stage in ('unsafe-state', 'status-unavailable'):
+ return 'attention', detail
+ return 'uncertain', detail
+ unavailable = (stage, projection.get('reason'))
except (OSError, ValueError, TypeError, AttributeError, subprocess.TimeoutExpired):
logger.warning('Pixel startup reproof failed; no automatic mutation retry')
- return
+ return 'uncertain', 'stage=helper-failed'
finally:
_end_model_lifecycle('pixel_startup_reproof')
if attempt < 11:
@@ -21597,14 +21620,108 @@ def _reconcile_native_pixel_startup():
else:
logger.warning('Pixel startup reproof readiness window exhausted '
'(last stage=%s reason=%s)', *unavailable)
- return True
+ return 'recheck', None
+
+
+# The access monitor re-proves a healthy runtime on this interval and backs off
+# to the cap while a check waits on the owner or on another operation.
+_PIXEL_ACCESS_RECHECK_SECONDS = 30
+_PIXEL_ACCESS_BACKOFF_CAP_SECONDS = 300
+# A status read that finds a stale Sandbox proof wakes the monitor. It then
+# waits this long, so a just-restarted gateway settles, and never spins.
+_PIXEL_ACCESS_WAKE_SETTLE_SECONDS = 5
+_pixel_access_monitor_wake = threading.Event()
+
+
+def _pixel_access_stale_sandbox_proof(value) -> bool:
+ """Whether a status is exactly what Verify Sandbox repairs: a restart
+ dropped the runtime proof of an idle, unchanged Sandbox."""
+ return (isinstance(value, dict) and value.get('available') is True
+ and value.get('scope') == 'owner-host' and value.get('configured_mode') == 'sandboxed'
+ and value.get('effective_mode') == 'unknown' and value.get('runtime_verified') is False
+ and value.get('busy') is False and value.get('pending') is False
+ and value.get('reason') == 'runtime-proof-required')
+
+
+def _pixel_model_hold_pending() -> bool:
+ """Whether the host's Portal model transaction is still open.
+
+ Unload and an interrupted switch keep the access coordinator held, so the
+ reproof helper would refuse anyway. Checking here keeps the monitor off the
+ model lifecycle lock while the owner resumes or recovers the model. An
+ unreadable journal is not proof that the transaction finished.
+ """
+ try:
+ return bool(_pixel_model_recovery_status()['pending'])
+ except (OSError, ValueError, RuntimeError, KeyError, TypeError):
+ return True
+
+
+def _pixel_access_verified() -> bool:
+ """Read-only: whether the coordinator reports a verified runtime now."""
+ from pixel_access_client import request_access
+ try:
+ status, value = request_access('status')
+ except (OSError, ValueError, TypeError):
+ return False
+ mode = value.get('configured_mode')
+ return (status == 200 and value.get('available') is True and value.get('scope') == 'owner-host'
+ and mode in ('sandboxed', 'full-access') and value.get('effective_mode') == mode
+ and value.get('runtime_verified') is True and value.get('pending') is False
+ and value.get('reason') is None)
+
+
+def _wait_for_pixel_access_check(delay: float, *, wake: bool) -> None:
+ """Sleep until the next check; a wake shortens it to the settle delay."""
+ if not wake:
+ time.sleep(delay)
+ return
+ if _pixel_access_monitor_wake.wait(delay):
+ _pixel_access_monitor_wake.clear()
+ time.sleep(_PIXEL_ACCESS_WAKE_SETTLE_SECONDS)
def _monitor_native_pixel_access():
- """Recheck healthy/busy instances; stop on uncertain policy mutations."""
- while _reconcile_native_pixel_startup() is True:
- time.sleep(30)
- logger.warning('Pixel access monitor stopped; recovery requires attention')
+ """Keep Portal's access proof current for the life of the host agent.
+
+ A gateway restart drops the runtime proof, so the protected helper re-proves
+ the unchanged mode. The monitor never stops on a state that can resolve
+ later: a held transition, an interrupted chat or a model transaction is
+ checked again with bounded backoff, and the helper itself still refuses
+ to touch it. After a change that was not confirmed, the monitor submits no
+ change by itself. It only reads status until the runtime is verified again
+ (Verify now in Portal, the installer, or that change having completed).
+ """
+ state, attempts = None, 0
+ while True:
+ if state is not None and state[0] == 'uncertain':
+ outcome, detail = ('recheck', None) if _pixel_access_verified() else state
+ elif _pixel_model_hold_pending():
+ outcome, detail = 'hold', None
+ else:
+ outcome, detail = _reconcile_native_pixel_startup()
+ if outcome is None:
+ # Not a Portal host, or its helper was refused (already logged).
+ logger.info('Pixel access monitor stopped: this host has no usable protected access helper')
+ return
+ if (outcome, detail) != state:
+ if outcome == 'attention':
+ logger.warning('Pixel startup reproof requires attention (%s); '
+ 'the access monitor keeps checking', detail)
+ elif outcome == 'uncertain':
+ logger.warning('Pixel access change was not confirmed (%s); no automatic retry. '
+ 'Reading status until the runtime is verified again', detail)
+ elif outcome == 'hold':
+ logger.info('Pixel access check waits while a Portal model transaction is open')
+ elif state is not None and outcome == 'recheck':
+ logger.info('Pixel access monitor resumed automatic reproof')
+ state, attempts = (outcome, detail), 0
+ attempts = min(attempts + 1, 10)
+ delay = (_PIXEL_ACCESS_RECHECK_SECONDS if outcome == 'recheck' else
+ min(_PIXEL_ACCESS_RECHECK_SECONDS * 2 ** (attempts - 1), _PIXEL_ACCESS_BACKOFF_CAP_SECONDS))
+ # A wake can only lead to the helper's own guarded check: never while
+ # a change is unconfirmed, and not while a model transaction is open.
+ _wait_for_pixel_access_check(delay, wake=outcome in ('recheck', 'attention'))
def _unreachable_docker_credential_helpers(config: dict, search_path: str) -> list[str]:
diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.py
index 7defe7ff8..5ed2253d3 100644
--- a/ods/extensions/services/dashboard-api/routers/pixel.py
+++ b/ods/extensions/services/dashboard-api/routers/pixel.py
@@ -72,14 +72,34 @@ _OPS_STATUSES = frozenset(
)
_CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f]")
_MODEL_SWITCH_DETAIL = "Model switch in progress; Portal will be ready when activation completes"
-_MODEL_UNLOADED_DETAIL = "The model is unloaded. Open Models and choose Resume model to use Portal again."
+_MODEL_UNLOADED_DETAIL = "The model is unloaded, so Portal is paused. Choose Resume model to load it again."
+_MODEL_RECOVERY_DETAIL = ("A model switch stopped before it finished, so new messages are held. "
+ "Choose Recover model switch to finish or undo it.")
+_RUNTIME_STOPPED_DETAIL = "The local model runtime is not running, so Portal is paused. Choose Resume model to start it."
# The Windows runtime ownership proof starts host processes and takes seconds.
# Bound it inside the Portal page's 15 s status deadline, and let one answer
# serve its 3 s polls while an unattended hold lasts.
_RUNTIME_PROOF_SECONDS = 10.0
_RUNTIME_PROOF_TTL_SECONDS = 30.0
+# An unavailable runtime is found only after the edge and telemetry reads, so
+# its optional Resume hint gets less of that deadline.
+_RUNTIME_HINT_SECONDS = 6.0
_MODEL_HOLD_DETAIL = "A model transition is holding new messages. Check model update progress. If it has stopped, restore Sandbox in Portal permissions."
-_CHAT_RECOVERY_DETAIL = "Portal stopped during an earlier turn and new messages are held. Open Portal permissions and use Verify Sandbox or Restore Sandbox before continuing."
+_CHAT_RECOVERY_DETAIL = ("Portal stopped during an earlier turn, so new messages are held "
+ "until its permissions are checked again.")
+# Host lifecycle operations that never own or change the Portal model
+# transaction. They take the lifecycle only to serialize with model work (the
+# access monitor's reproof runs every 30 s), so a hold they overlap is still
+# unattended. Mirrors _MODEL_RUNTIME_NEUTRAL_OPERATIONS in ods-host-agent.py.
+_MODEL_NEUTRAL_OPERATIONS = frozenset({
+ "pixel_startup_reproof", "pixel_access_mode", "pixel_open_app",
+ "pixel_providers", "pixel_settings", "model_profile_recheck",
+})
+# The one-click fix Portal offers for a held state ("recovery" in status).
+# Each names an existing API: the Sandbox actions POST /api/pixel/access-mode
+# with {"mode": "sandboxed"} exactly as Portal permissions does; resume-model
+# is Models' Resume model; recover-model is Models' Recover model switch.
+_STATE_RECOVERY = {"model_unloaded": "resume-model", "model_recovery_required": "recover-model"}
_MODEL_CAPABILITY_DETAIL = (
"The active model is recorded as not agent-qualified. Tool-driven tasks "
"may be unreliable; chat and experiments remain available."
@@ -389,6 +409,33 @@ def _access_projection(value):
raise HTTPException(status_code=502, detail="Portal access status could not be verified") from None
+def _access_recovery(access, *, model_hold_possible=True) -> str | None:
+ """The Portal permissions action that recovers this access state, if any.
+
+ Both actions send {"mode": "sandboxed", "confirmed": false}, exactly like
+ Verify Sandbox and Restore Sandbox; the host re-checks the revision and
+ refuses a busy runtime first. Nothing here can select Full Access, and an
+ unverified Full Access is left to the host's own same-mode reproof.
+ """
+ if not access or not access["available"]:
+ return None
+ reason, mode = access["reason"], access["configured_mode"]
+ if reason == "chat-recovery-required":
+ # Settings offers only this while a chat is held: Enable Full Access waits.
+ return {"sandboxed": "verify-sandbox", "full-access": "restore-sandbox"}.get(mode)
+ if not access["pending"] and reason == "runtime-proof-required" and mode == "sandboxed":
+ return "verify-sandbox"
+ # An unfinished access change resumes only by restoring Sandbox. A held
+ # model transaction reads the same way but has its own recovery.
+ if access["pending"] and reason == "transition-recovery-required" and not model_hold_possible:
+ return "restore-sandbox"
+ return None
+
+
+def _recovery_field(action: str | None) -> dict[str, str]:
+ return {"recovery": action} if action else {}
+
+
@router.get("/access-mode", dependencies=[Depends(verify_api_key)])
async def pixel_access_status():
try:
@@ -453,17 +500,25 @@ def _unattended_model_hold(status: object) -> bool:
"""A held Portal model transaction that no host lifecycle operation owns.
A model switch owns the host's model_activation lifecycle until it ends,
- so a hold without any operation will not finish by itself.
+ so a hold without any operation will not finish by itself. Access checks
+ and settings saves also take the lifecycle, but never own a hold.
"""
- return (isinstance(status, dict) and status.get("modelTransactionPending") is True
- and not status.get("lifecycleActive") and not status.get("activeOperation"))
+ if not isinstance(status, dict) or status.get("modelTransactionPending") is not True:
+ return False
+ operation = status.get("activeOperation")
+ return (not status.get("lifecycleActive") and not operation) or operation in _MODEL_NEUTRAL_OPERATIONS
-def _model_readiness_issue_from_status(status: object, *, runtime_stopped: bool = False) -> tuple[str, str] | None:
+def _model_readiness_issue_from_status(status: object, *, runtime_stopped: bool = False,
+ unload_ruled_out: bool = False) -> tuple[str, str] | None:
# Unload model keeps the Portal hold until Resume model. An unattended
# hold on a runtime the host proves stopped is that, not a switch.
if runtime_stopped and _unattended_model_hold(status):
return "model_unloaded", _MODEL_UNLOADED_DETAIL
+ # The host proves it is not an unload either: an interrupted switch that
+ # nothing will finish by itself. Recover model switch completes or undoes it.
+ if unload_ruled_out and _unattended_model_hold(status):
+ return "model_recovery_required", _MODEL_RECOVERY_DETAIL
if isinstance(status, dict) and status.get("modelTransactionPending") is True:
return "model_switching", _MODEL_SWITCH_DETAIL
switching = (
@@ -567,35 +622,62 @@ async def _verified_external_host_runtime(host_status: object) -> dict[str, obje
return _active_runtime_projection({"activeRuntime": runtime})
-_runtime_proof: tuple[float, bool] | None = None
+_runtime_proof: tuple[float, bool | None] | None = None
+
+
+def _runtime_unloaded(value: object) -> bool | None:
+ """Whether a host runtime projection proves the model is unloaded.
+
+ True: the owned runtime is stopped, exactly the proof for which the Models
+ page offers Resume model. False: the host runs the model it owns, or owns
+ no runtime it could have unloaded. Anything else is unknown (None).
+ """
+ if not isinstance(value, dict):
+ return None
+ fields = tuple(value.get(key) for key in ("managed", "canUnload", "running", "canActivate"))
+ if any(type(field) is not bool for field in fields):
+ return None
+ return {(True, True, False, False): True, (True, True, True, True): False,
+ (False, False, False, False): False}.get(fields)
-async def _owned_runtime_stopped() -> bool:
- """Whether the host proves the Windows model runtime it owns is stopped.
+async def _owned_runtime_unloaded(seconds: float | None = None) -> bool | None:
+ """The host's _runtime_unloaded proof for the Windows runtime it owns.
- A failed or invalid proof is unknown, never "stopped". It is remembered
+ A stopped runtime under a switch that did not reach its held phase is not
+ an unload. A failed, slow or invalid proof is unknown. It is remembered
too, so a slow host is not asked again on every poll.
"""
global _runtime_proof
if _runtime_proof is not None and time.monotonic() - _runtime_proof[0] < _RUNTIME_PROOF_TTL_SECONDS:
return _runtime_proof[1]
+ seconds = _RUNTIME_PROOF_SECONDS if seconds is None else seconds
try:
- async with async_timeout(_RUNTIME_PROOF_SECONDS):
- value = await request_agent_json("GET", "/v1/model/management", timeout=_RUNTIME_PROOF_SECONDS)
+ async with async_timeout(seconds):
+ unloaded = _runtime_unloaded(await request_agent_json(
+ "GET", "/v1/model/management", timeout=seconds))
+ if unloaded is True:
+ # Resume model restarts only an unload's held transaction. The
+ # host refuses it for a switch that stopped part-way and asks
+ # for recovery first, so that hold is not an unload.
+ try:
+ journal = await request_agent_json("GET", "/v1/model/recovery", timeout=seconds)
+ except AgentClientError:
+ journal = None
+ if isinstance(journal, dict) and journal.get("pending") is True and journal.get("phase") != "held":
+ unloaded = False
except (asyncio.TimeoutError, AgentClientError):
- value = None
- # Exactly the proof for which the Models page offers Resume model.
- stopped = (isinstance(value, dict) and value.get("managed") is True and value.get("canUnload") is True
- and value.get("running") is False and value.get("canActivate") is False)
- _runtime_proof = (time.monotonic(), stopped)
- return stopped
+ unloaded = None
+ _runtime_proof = (time.monotonic(), unloaded)
+ return unloaded
async def _model_readiness_issue_for_status(status: object) -> tuple[str, str] | None:
# Only an unattended hold needs the runtime proof; a switch in progress
# owns a host lifecycle operation.
- stopped = _unattended_model_hold(status) and await _owned_runtime_stopped()
- return _model_readiness_issue_from_status(status, runtime_stopped=stopped)
+ unloaded = await _owned_runtime_unloaded() if _unattended_model_hold(status) else None
+ return _model_readiness_issue_from_status(status, runtime_stopped=unloaded is True,
+ unload_ruled_out=unloaded is False)
async def _model_activation_in_progress() -> bool:
@@ -666,7 +748,7 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
"detail": _MODEL_HOLD_DETAIL}
if access and access["available"] and access["reason"] == "chat-recovery-required":
return {**activation_metadata, "available": False, "model": None, "state": "chat_recovery_required",
- "detail": _CHAT_RECOVERY_DETAIL}
+ "detail": _CHAT_RECOVERY_DETAIL, **_recovery_field(_access_recovery(access))}
readiness_issue = await _model_readiness_issue_for_status(host_status)
if readiness_issue is not None:
state, detail = readiness_issue
@@ -676,6 +758,7 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
"model": None,
"state": state,
"detail": detail,
+ **_recovery_field(_STATE_RECOVERY.get(state)),
}
edge_url, key = config
try:
@@ -705,7 +788,11 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
if available:
inference_issue = await _local_inference_issue(host_status)
if inference_issue:
- return {**activation_metadata, "available": False, "model": None, "state": "model_unavailable", "detail": inference_issue}
+ # A stopped runtime the host owns gets Models' Resume model.
+ stopped = await _owned_runtime_unloaded(_RUNTIME_HINT_SECONDS) is True
+ return {**activation_metadata, "available": False, "model": None, "state": "model_unavailable",
+ "detail": _RUNTIME_STOPPED_DETAIL if stopped else inference_issue,
+ **_recovery_field("resume-model" if stopped else None)}
runtime = _active_runtime_projection(host_status)
if available and runtime is None:
runtime = await _verified_external_host_runtime(host_status)
@@ -725,6 +812,10 @@ async def pixel_status(http_response: Response = None) -> dict[str, object]:
readiness = project_readiness(available, access, identity, access_issue)
result["readiness"] = readiness
if available:
+ # Only a host that reports no open model transaction lets a held
+ # access transition be offered Restore Sandbox.
+ settled = isinstance(host_status, dict) and host_status.get("modelTransactionPending") is False
+ result.update(_recovery_field(_access_recovery(access, model_hold_possible=not settled)))
result["detail"] = "Owner agent available; " + ("runtime files changed since initialization" if identity["state"] == "mismatch"
else "release identity is not fully verified")
if readiness["accessState"] == "failed":
diff --git a/ods/extensions/services/dashboard-api/tests/test_host_agent_pixel_access_monitor.py b/ods/extensions/services/dashboard-api/tests/test_host_agent_pixel_access_monitor.py
new file mode 100644
index 000000000..f651df46b
--- /dev/null
+++ b/ods/extensions/services/dashboard-api/tests/test_host_agent_pixel_access_monitor.py
@@ -0,0 +1,342 @@
+"""The host agent keeps Portal's access proof current instead of giving up.
+
+On the RTX Spark test laptop (2026-10-25) the agent started at 15:13:06 and,
+at 15:15:31, logged "Pixel startup reproof requires attention
+(stage=unsafe-state reason=transition-recovery-required)" and "Pixel access
+monitor stopped". A model hold held the coordinator at boot. When a later
+gateway restart dropped the runtime proof, nothing re-proved it and Portal
+stayed unverified until someone clicked Verify Sandbox in Settings.
+
+These tests drive the real monitor, its checks and the access-mode status
+handler with the protected helper, the coordinator socket and the clock
+faked. Nothing here starts Docker, bash, WSL or a helper.
+"""
+
+import io
+import json
+import logging
+import stat
+import subprocess
+import threading
+import types
+
+import pytest
+
+# Loading the agent puts ods/bin, where these two modules live, on sys.path.
+from test_host_agent_install_rollback import _mod
+import pixel_access_client
+import pixel_access_relay
+
+ATTENTION = ("attention", "stage=unsafe-state reason=transition-recovery-required")
+STALE_SANDBOX = {"available": True, "surface": "wsl-systemd", "configured_mode": "sandboxed",
+ "effective_mode": "unknown", "runtime_verified": False, "revision": "a" * 64,
+ "busy": False, "pending": False, "reason": "runtime-proof-required", "scope": "owner-host"}
+VERIFIED = {**STALE_SANDBOX, "effective_mode": "sandboxed", "runtime_verified": True, "reason": None}
+
+
+class Stop(Exception):
+ """Ends the otherwise endless monitor loop after the scripted checks."""
+
+
[email protected](autouse=True)
+def _quiet_wake():
+ _mod._pixel_access_monitor_wake.clear()
+ yield
+ _mod._pixel_access_monitor_wake.clear()
+
+
+def _run_monitor(monkeypatch, checks, *, hold=(), verified=()):
+ """Run the real monitor over scripted checks.
+
+ Returns each wait it asked for and, in order, which check ran: the
+ protected helper ("helper"), the read-only status ("status") or the
+ model journal ("journal").
+ """
+ waits: list[tuple] = []
+ ran: list[str] = []
+ check = iter([*checks, Stop()])
+ holds = iter(hold)
+ proofs = iter(verified)
+
+ def reconcile():
+ ran.append("helper")
+ outcome = next(check)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+ def hold_pending():
+ ran.append("journal")
+ return next(holds, False)
+
+ def access_verified():
+ ran.append("status")
+ return next(proofs)
+
+ monkeypatch.setattr(_mod, "_reconcile_native_pixel_startup", reconcile)
+ monkeypatch.setattr(_mod, "_pixel_model_hold_pending", hold_pending)
+ monkeypatch.setattr(_mod, "_pixel_access_verified", access_verified)
+ monkeypatch.setattr(_mod, "_wait_for_pixel_access_check",
+ lambda delay, *, wake: waits.append((delay, wake)))
+ with pytest.raises(Stop):
+ _mod._monitor_native_pixel_access()
+ return waits, ran
+
+
+def test_a_boot_time_hold_no_longer_stops_the_monitor(monkeypatch, caplog):
+ # The laptop's sequence: refused at boot, still refused, then the hold
+ # resolved and the next restart's stale proof was re-proved on schedule.
+ checks = [ATTENTION, ATTENTION, ("recheck", None), ("recheck", None)]
+ with caplog.at_level(logging.INFO, logger="ods-host-agent"):
+ waits, ran = _run_monitor(monkeypatch, checks)
+ assert waits == [(30, True), (60, True), (30, True), (30, True)]
+ assert ran.count("helper") == 5
+ messages = [record.getMessage() for record in caplog.records]
+ assert messages.count("Pixel startup reproof requires attention (stage=unsafe-state "
+ "reason=transition-recovery-required); the access monitor keeps checking") == 1
+ assert "Pixel access monitor resumed automatic reproof" in messages
+ assert not any("stopped" in message for message in messages)
+
+
+def test_backoff_is_bounded_and_resets_when_the_state_changes(monkeypatch):
+ chat = ("attention", "stage=unsafe-state reason=chat-recovery-required")
+ waits, _ran = _run_monitor(monkeypatch, [*[ATTENTION] * 7, chat, chat])
+ assert [delay for delay, _wake in waits] == [30, 60, 120, 240, 300, 300, 300, 30, 60]
+
+
+def test_an_unconfirmed_change_is_never_retried_until_status_is_verified(monkeypatch, caplog):
+ uncertain = ("uncertain", "stage=change-failed reason=None")
+ with caplog.at_level(logging.WARNING, logger="ods-host-agent"):
+ waits, ran = _run_monitor(monkeypatch, [uncertain, ("recheck", None)], verified=[False, False, True])
+ # While latched only the read-only status ran, and no wake could cut a wait.
+ assert ran == ["journal", "helper", "status", "status", "status",
+ "journal", "helper", "journal", "helper"]
+ assert waits == [(30, False), (60, False), (120, False), (30, True), (30, True)]
+ assert [record.getMessage() for record in caplog.records] == [
+ "Pixel access change was not confirmed (stage=change-failed reason=None); no automatic retry. "
+ "Reading status until the runtime is verified again"]
+
+
+def test_an_open_model_transaction_keeps_the_monitor_off_the_lifecycle_lock(monkeypatch):
+ begun: list[tuple] = []
+
+ def begin(*args):
+ begun.append(args)
+ return True, {}
+
+ monkeypatch.setattr(_mod, "_begin_model_lifecycle", begin)
+ waits, ran = _run_monitor(monkeypatch, [("recheck", None)], hold=[True, True, True])
+ assert begun == []
+ assert ran == ["journal", "journal", "journal", "journal", "helper", "journal", "helper"]
+ assert waits == [(30, False), (60, False), (120, False), (30, True)]
+
+
+def test_the_monitor_stops_only_on_a_host_without_a_usable_helper(monkeypatch):
+ monkeypatch.setattr(_mod, "_reconcile_native_pixel_startup", lambda: (None, None))
+ monkeypatch.setattr(_mod, "_pixel_model_hold_pending", lambda: False)
+ monkeypatch.setattr(_mod, "_wait_for_pixel_access_check",
+ lambda *_args, **_kwargs: pytest.fail("waited without a helper"))
+ assert _mod._monitor_native_pixel_access() is None
+
+
+class _Helper:
+ """The installed, root-owned reconcile helper, as the agent inspects it."""
+
+ def __init__(self, _path):
+ self.parents = ()
+
+ def __str__(self):
+ return "/usr/local/libexec/ods-pixel-access/pixel_access_reconcile.py"
+
+ def exists(self):
+ return True
+
+ def lstat(self):
+ return types.SimpleNamespace(st_mode=stat.S_IFREG | 0o644, st_uid=0)
+
+ def stat(self):
+ return types.SimpleNamespace(st_size=100)
+
+ def read_text(self):
+ return "STARTUP_REPROOF_VERSION = 1\n"
+
+
+def helper(monkeypatch):
+ """Answer each protected-helper run with one scripted diagnostic."""
+ answers: list[object] = []
+ runs: list[list[str]] = []
+
+ def run(argv, **_kwargs):
+ runs.append(argv)
+ answer = answers.pop(0)
+ if isinstance(answer, BaseException):
+ raise answer
+ if answer is None:
+ return types.SimpleNamespace(returncode=0, stderr=b"")
+ return types.SimpleNamespace(returncode=1, stderr=json.dumps(answer).encode())
+
+ monkeypatch.setattr(_mod.platform, "system", lambda: "Linux")
+ monkeypatch.setattr(_mod, "Path", _Helper)
+ monkeypatch.setattr(_mod.subprocess, "run", run)
+ monkeypatch.setattr(_mod.time, "sleep", lambda _seconds: None)
+ return types.SimpleNamespace(answers=answers, runs=runs)
+
+
[email protected](("diagnostic", "expected"), [
+ (None, ("recheck", None)),
+ ({"stage": "unsafe-state", "projection": {"scope": "owner-host", "available": True,
+ "busy": True, "pending": False}}, ("recheck", None)),
+ # Read-only refusals: the helper stopped at its status read.
+ ({"stage": "unsafe-state", "projection": {"available": True, "busy": False, "pending": True,
+ "reason": "transition-recovery-required"}}, ATTENTION),
+ ({"stage": "unsafe-state", "projection": {"available": True, "busy": False, "pending": False,
+ "reason": "chat-recovery-required"}},
+ ("attention", "stage=unsafe-state reason=chat-recovery-required")),
+ ({"stage": "status-unavailable", "projection": {"available": True}},
+ ("attention", "stage=status-unavailable reason=None")),
+ # A submitted change that failed, or a helper failure that may follow one.
+ ({"stage": "change-failed", "projection": {"available": True, "busy": True, "pending": False}},
+ ("uncertain", "stage=change-failed reason=None")),
+ ({"stage": "client-exception", "exceptionType": "OSError"},
+ ("uncertain", "stage=client-exception reason=None")),
+ (subprocess.TimeoutExpired("helper", 360), ("uncertain", "stage=helper-failed")),
+], ids=["ready", "busy", "pending", "chat-recovery", "status-unavailable",
+ "change-failed", "client-exception", "timeout"])
+def test_each_helper_answer_maps_to_one_monitor_outcome(helper, diagnostic, expected):
+ helper.answers.append(diagnostic)
+ assert _mod._reconcile_native_pixel_startup() == expected
+ assert helper.runs == [["/usr/bin/python3", "-I",
+ "/usr/local/libexec/ods-pixel-access/pixel_access_reconcile.py", "--startup"]]
+
+
[email protected]("change", [
+ {},
+ {"configured_mode": "full-access"},
+ {"busy": True},
+ {"pending": True},
+ {"reason": "chat-recovery-required"},
+ {"reason": None},
+ {"available": False},
+ {"scope": "guest"},
+ {"effective_mode": "sandboxed", "runtime_verified": True, "reason": None},
+], ids=["stale-sandbox", "full-access", "busy", "pending", "chat-recovery", "no-reason",
+ "unavailable", "scope", "verified"])
+def test_only_an_idle_stale_sandbox_proof_is_eligible_for_a_wake(change):
+ assert _mod._pixel_access_stale_sandbox_proof({**STALE_SANDBOX, **change}) is (change == {})
+ assert _mod._pixel_access_stale_sandbox_proof(["not", "a", "status"]) is False
+
+
+class _Handler:
+ """Minimal stand-in for BaseHTTPRequestHandler."""
+
+ def __init__(self):
+ self.headers = {"Authorization": "Bearer test-key"}
+ self.wfile = io.BytesIO()
+ self.code = None
+
+ def send_response(self, code):
+ self.code = code
+
+ def send_header(self, _name, _value):
+ pass
+
+ def end_headers(self):
+ pass
+
+
[email protected](("status", "body", "wakes"), [
+ (200, STALE_SANDBOX, True),
+ (200, VERIFIED, False),
+ (200, {**STALE_SANDBOX, "configured_mode": "full-access"}, False),
+ (200, {**STALE_SANDBOX, "pending": True, "reason": "transition-recovery-required"}, False),
+ (503, STALE_SANDBOX, False),
+], ids=["stale-sandbox", "verified", "full-access", "pending", "unavailable"])
+def test_a_status_read_of_a_stale_sandbox_wakes_the_monitor_and_changes_nothing(monkeypatch, status, body, wakes):
+ requests: list[tuple] = []
+
+ def relay(operation, request=None, *, config):
+ requests.append((operation, request))
+ return status, body
+
+ monkeypatch.setattr(_mod, "AGENT_API_KEY", "test-key")
+ monkeypatch.setattr(_mod, "load_env", lambda _path: {})
+ monkeypatch.setattr(pixel_access_relay, "request_runtime_access", relay)
+ handler = _Handler()
+ _mod.AgentHandler._handle_pixel_access_mode(handler, False)
+ assert (handler.code, json.loads(handler.wfile.getvalue())) == (status, body)
+ assert requests == [("status", None)]
+ assert _mod._pixel_access_monitor_wake.is_set() is wakes
+
+
+def test_a_wake_ends_the_wait_after_the_settle_delay(monkeypatch):
+ slept: list[float] = []
+ monkeypatch.setattr(_mod.time, "sleep", slept.append)
+ done = threading.Event()
+
+ def wait():
+ _mod._wait_for_pixel_access_check(60, wake=True)
+ done.set()
+
+ waiter = threading.Thread(target=wait)
+ waiter.start()
+ _mod._pixel_access_monitor_wake.set()
+ assert done.wait(5)
+ waiter.join(5)
+ assert slept == [_mod._PIXEL_ACCESS_WAKE_SETTLE_SECONDS]
+ assert not _mod._pixel_access_monitor_wake.is_set()
+
+
+def test_a_wait_without_wakes_ignores_a_pending_wake(monkeypatch):
+ slept: list[float] = []
+ monkeypatch.setattr(_mod.time, "sleep", slept.append)
+ _mod._pixel_access_monitor_wake.set()
+ _mod._wait_for_pixel_access_check(120, wake=False)
+ assert slept == [120]
+ # The wake stays for the first check after the latch clears.
+ assert _mod._pixel_access_monitor_wake.is_set()
+
+
[email protected](("journal", "pending"), [
+ ({"pending": True, "phase": "held", "transactionId": "a" * 64}, True),
+ ({"pending": False, "phase": "completed", "transactionId": "a" * 64}, False),
+ ({"pending": False, "phase": "idle", "transactionId": None}, False),
+ (RuntimeError("Managed model recovery journal is invalid"), True),
+], ids=["held", "completed", "idle", "unreadable"])
+def test_an_unreadable_model_journal_counts_as_an_open_transaction(monkeypatch, journal, pending):
+ def status():
+ if isinstance(journal, BaseException):
+ raise journal
+ return journal
+
+ monkeypatch.setattr(_mod, "_pixel_model_recovery_status", status)
+ assert _mod._pixel_model_hold_pending() is pending
+
+
[email protected](("answer", "verified"), [
+ ((200, VERIFIED), True),
+ ((200, {**VERIFIED, "configured_mode": "full-access", "effective_mode": "full-access"}), True),
+ ((200, STALE_SANDBOX), False),
+ ((200, {**VERIFIED, "pending": True}), False),
+ ((200, {**VERIFIED, "reason": "chat-recovery-required"}), False),
+ ((409, {"error": "transition-busy"}), False),
+ (OSError("socket unavailable"), False),
+ (ValueError("invalid access response"), False),
+], ids=["sandbox", "full-access", "stale", "pending", "chat-recovery", "refused", "transport", "invalid"])
+def test_the_latch_clears_only_on_a_verified_read_only_status(monkeypatch, answer, verified):
+ operations = []
+
+ def request(operation, request=None, **_kwargs):
+ operations.append((operation, request))
+ if isinstance(answer, BaseException):
+ raise answer
+ return answer
+
+ monkeypatch.setattr(pixel_access_client, "request_access", request)
+ assert _mod._pixel_access_verified() is verified
+ assert operations == [("status", None)]
+
+
+def test_the_wake_settle_and_backoff_bounds_are_the_documented_ones():
+ assert (_mod._PIXEL_ACCESS_RECHECK_SECONDS, _mod._PIXEL_ACCESS_BACKOFF_CAP_SECONDS,
+ _mod._PIXEL_ACCESS_WAKE_SETTLE_SECONDS) == (30, 300, 5)
diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel.py b/ods/extensions/services/dashboard-api/tests/test_pixel.py
index 59d2140d1..29f35fa93 100644
--- a/ods/extensions/services/dashboard-api/tests/test_pixel.py
+++ b/ods/extensions/services/dashboard-api/tests/test_pixel.py
@@ -754,7 +754,7 @@ async def test_status_says_the_model_is_unloaded_and_a_send_names_resume(monkeyp
):
status = await pixel.pixel_status()
assert status == {"available": False, "model": None, "state": "model_unloaded",
- "detail": pixel._MODEL_UNLOADED_DETAIL}
+ "detail": pixel._MODEL_UNLOADED_DETAIL, "recovery": "resume-model"}
body = pixel.ChatStreamRequest.model_validate({
"chat_id": "unloaded_model", "messages": [{"role": "user", "content": "hello"}],
})
@@ -782,14 +782,13 @@ async def test_status_keeps_a_real_switch_without_asking_for_the_runtime_proof(m
@pytest.mark.asyncio
@pytest.mark.parametrize("management", [
- {**_STOPPED_RUNTIME, "running": True, "canActivate": True},
- {"managed": False, "canActivate": False, "canUnload": False, "running": False, "vision": False},
{**_STOPPED_RUNTIME, "canActivate": True},
+ {**_STOPPED_RUNTIME, "running": 0, "canActivate": 0},
{"status": "idle"},
pixel.AgentHTTPError(503, "Windows runtime management could not be verified"),
pixel.AgentClientError("unreachable"),
-], ids=["running", "unmanaged", "inconsistent", "malformed", "unverified", "unreachable"])
-async def test_an_unproven_stop_keeps_the_switching_hold(monkeypatch, management):
+], ids=["inconsistent", "untyped", "malformed", "unverified", "unreachable"])
+async def test_an_unproven_runtime_keeps_the_switching_hold(monkeypatch, management):
_host_agent(monkeypatch, _UNATTENDED_HOLD, management)
with patch.object(
pixel.httpx,
@@ -798,6 +797,88 @@ async def test_an_unproven_stop_keeps_the_switching_hold(monkeypatch, management
):
status = await pixel.pixel_status()
assert (status["state"], status["detail"]) == ("model_switching", pixel._MODEL_SWITCH_DETAIL)
+ assert "recovery" not in status
+
+
[email protected]("management", [
+ {**_STOPPED_RUNTIME, "running": True, "canActivate": True},
+ # This installation owns no runtime it could have unloaded.
+ {"managed": False, "canActivate": False, "canUnload": False, "running": False, "vision": False},
+], ids=["running", "unmanaged"])
+async def test_an_unattended_hold_that_cannot_be_an_unload_offers_recover(monkeypatch, management):
+ _host_agent(monkeypatch, _UNATTENDED_HOLD, management)
+ with patch.object(
+ pixel.httpx,
+ "AsyncClient",
+ side_effect=AssertionError("a held model reached Pixel edge"),
+ ):
+ status = await pixel.pixel_status()
+ assert status == {"available": False, "model": None, "state": "model_recovery_required",
+ "detail": pixel._MODEL_RECOVERY_DETAIL, "recovery": "recover-model"}
+ body = pixel.ChatStreamRequest.model_validate({
+ "chat_id": "held_model", "messages": [{"role": "user", "content": "hello"}],
+ })
+ with pytest.raises(HTTPException) as exc:
+ await pixel.pixel_chat_stream(ConnectedRequest(), body) # type: ignore[arg-type] # minimal Request
+ assert (exc.value.status_code, exc.value.detail) == (409, pixel._MODEL_RECOVERY_DETAIL)
+
+
[email protected](("journal", "state"), [
+ ({"pending": True, "phase": "held", "transactionId": "a" * 64}, "model_unloaded"),
+ # The host refuses Resume for a switch that stopped part-way and asks
+ # for recovery first, even though its runtime is stopped too.
+ ({"pending": True, "phase": "applying", "transactionId": "a" * 64}, "model_recovery_required"),
+ ({"pending": True, "phase": "rolling-back", "transactionId": "a" * 64}, "model_recovery_required"),
+ # An unreadable journal keeps the earlier unload reading.
+ (pixel.AgentHTTPError(503, "Model recovery state is unavailable"), "model_unloaded"),
+], ids=["unload", "applying", "rolling-back", "unreadable"])
+async def test_a_stopped_runtime_is_an_unload_only_for_a_held_transaction(monkeypatch, journal, state):
+ paths = _host_agent(monkeypatch, _UNATTENDED_HOLD, _STOPPED_RUNTIME)
+ agent = pixel.request_agent_json
+
+ async def host(method, path, **kwargs):
+ if path == "/v1/model/recovery":
+ paths.append(path)
+ if isinstance(journal, BaseException):
+ raise journal
+ return journal
+ return await agent(method, path, **kwargs)
+
+ monkeypatch.setattr(pixel, "request_agent_json", host)
+ with patch.object(
+ pixel.httpx,
+ "AsyncClient",
+ side_effect=AssertionError("a held model reached Pixel edge"),
+ ):
+ status = await pixel.pixel_status()
+ assert status["state"] == state
+ assert status["recovery"] == pixel._STATE_RECOVERY[state]
+ assert (await pixel.pixel_status())["state"] == state
+ # One proof, with its journal read, serves the following polls.
+ assert (paths.count("/v1/model/management"), paths.count("/v1/model/recovery")) == (1, 1)
+
+
[email protected]("operation", ["pixel_startup_reproof", "pixel_access_mode", "pixel_settings"])
+def test_an_access_check_overlapping_a_hold_does_not_own_it(operation):
+ # The access monitor and Verify Sandbox take the model lifecycle only to
+ # serialize with model work. Portal must not flip to "switching" for them.
+ status = {**_UNATTENDED_HOLD, "lifecycleActive": True, "activeOperation": operation}
+ assert pixel._model_readiness_issue_from_status(status, runtime_stopped=True) == (
+ "model_unloaded", pixel._MODEL_UNLOADED_DETAIL)
+ assert pixel._model_readiness_issue_from_status(status, unload_ruled_out=True) == (
+ "model_recovery_required", pixel._MODEL_RECOVERY_DETAIL)
+
+
+def test_portal_and_the_host_agree_on_which_operations_never_own_a_hold():
+ import ast
+ source = (pathlib.Path(__file__).resolve().parents[4] / "bin" / "ods-host-agent.py").read_text(encoding="utf-8")
+ declaration = next(node for node in ast.parse(source).body if isinstance(node, ast.Assign)
+ and any(getattr(target, "id", None) == "_MODEL_RUNTIME_NEUTRAL_OPERATIONS"
+ for target in node.targets))
+ assert isinstance(declaration.value, ast.Call)
+ assert set(ast.literal_eval(declaration.value.args[0])) == pixel._MODEL_NEUTRAL_OPERATIONS
@pytest.mark.asyncio
diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_inference_readiness.py b/ods/extensions/services/dashboard-api/tests/test_pixel_inference_readiness.py
index 57e1b7fda..1be071edf 100644
--- a/ods/extensions/services/dashboard-api/tests/test_pixel_inference_readiness.py
+++ b/ods/extensions/services/dashboard-api/tests/test_pixel_inference_readiness.py
@@ -1,4 +1,5 @@
"""Pixel discovery must not advertise a stopped host model server as ready."""
+import asyncio
import json
from unittest.mock import patch
@@ -33,6 +34,47 @@ async def test_agent_catalog_does_not_mask_unavailable_inference(monkeypatch, te
assert ('GET', '/v1/llm/status') in calls
[email protected](('management', 'recovery'), [
+ # Exactly the proof for which Models offers Resume model.
+ ({'managed': True, 'canActivate': False, 'canUnload': True, 'running': False, 'vision': False}, 'resume-model'),
+ ({'managed': True, 'canActivate': True, 'canUnload': True, 'running': True, 'vision': False}, None),
+ ({'managed': False, 'canActivate': False, 'canUnload': False, 'running': False, 'vision': False}, None),
+ (pixel.AgentHTTPError(503, 'Windows runtime management could not be verified'), None),
+], ids=['stopped', 'running', 'unmanaged', 'unverified'])
+async def test_a_stopped_owned_runtime_offers_resume_model_in_portal(monkeypatch, management, recovery):
+ async def host(method, path, **kwargs):
+ if path == '/v1/model/management':
+ if isinstance(management, BaseException):
+ raise management
+ return management
+ return {'schema_version': 'ods.host-llm-status.v1', 'health': {'status': 'error'}} if path == '/v1/llm/status' else {'status': 'idle'}
+ monkeypatch.setattr(pixel, 'request_agent_json', host)
+ upstream = FakeResponse(chunks=[json.dumps({'data':[{'id':'portal/default'}]}).encode()])
+ with patch.object(pixel.httpx, 'AsyncClient', return_value=FakeClient(upstream)):
+ result = await pixel.pixel_status()
+ assert (result['available'], result['state']) == (False, 'model_unavailable')
+ assert result.get('recovery') == recovery
+ assert result['detail'] == (pixel._RUNTIME_STOPPED_DETAIL if recovery else
+ 'The local model runtime is unavailable. Restore it in Models before sending another task.')
+
+
+async def test_a_slow_runtime_proof_cannot_hold_the_status_read(monkeypatch):
+ # The hint comes after the edge and telemetry reads, inside the page's deadline.
+ async def host(method, path, **kwargs):
+ if path == '/v1/model/management':
+ await asyncio.sleep(5)
+ return {'managed': True, 'canActivate': False, 'canUnload': True, 'running': False}
+ return {'schema_version': 'ods.host-llm-status.v1', 'health': {'status': 'error'}} if path == '/v1/llm/status' else {'status': 'idle'}
+ monkeypatch.setattr(pixel, 'request_agent_json', host)
+ monkeypatch.setattr(pixel, '_RUNTIME_HINT_SECONDS', 0.05)
+ upstream = FakeResponse(chunks=[json.dumps({'data':[{'id':'portal/default'}]}).encode()])
+ with patch.object(pixel.httpx, 'AsyncClient', return_value=FakeClient(upstream)):
+ result = await asyncio.wait_for(pixel.pixel_status(), timeout=2)
+ assert (result['state'], result.get('recovery')) == ('model_unavailable', None)
+
+
@pytest.mark.asyncio
async def test_live_health_can_recover_without_generating_tokens(monkeypatch):
async def host(method, path, **kwargs):
diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_readiness.py b/ods/extensions/services/dashboard-api/tests/test_pixel_readiness.py
index 43589fd04..eccda97d2 100644
--- a/ods/extensions/services/dashboard-api/tests/test_pixel_readiness.py
+++ b/ods/extensions/services/dashboard-api/tests/test_pixel_readiness.py
@@ -18,7 +18,8 @@ FAILED = dict(ACCESS, available=False, configured_mode="unknown", effective_mode
revision=None, reason="inspection-failed")
-async def status_for(monkeypatch, access=ACCESS, identity=None, *, held=False, recovery=False):
+async def status_for(monkeypatch, access=ACCESS, identity=None, *, held=False, recovery=None, model_status=None):
+ """Portal status for one access projection; ``recovery`` expects the chat gate's fix."""
monkeypatch.setenv("PIXEL_OPENWEBUI_KEY", "e" * 64)
monkeypatch.setenv("PIXEL_EDGE_URL", "http://pixel-edge:9595")
calls = []
@@ -26,7 +27,8 @@ async def status_for(monkeypatch, access=ACCESS, identity=None, *, held=False, r
async def host(method, path, **kwargs):
calls.append((method, path, kwargs))
if path == "/v1/model/status":
- return {"status": "idle", "activeRuntime": {"source": "local-switchboard", "model": "fixture-model", "contextLength": 65536}}
+ return {"status": "idle", "activeRuntime": {"source": "local-switchboard", "model": "fixture-model", "contextLength": 65536},
+ **(model_status or {})}
assert path == "/v1/pixel/access-mode" and method == "GET"
if isinstance(access, Exception):
raise access
@@ -50,8 +52,9 @@ async def status_for(monkeypatch, access=ACCESS, identity=None, *, held=False, r
assert result == {"available": False, "model": None, "state": "model_transition_pending", "detail": pixel._MODEL_HOLD_DETAIL}
assert all(method == "GET" for method, _path, _kwargs in calls)
return result
- if recovery:
- assert result == {"available": False, "model": None, "state": "chat_recovery_required", "detail": pixel._CHAT_RECOVERY_DETAIL}
+ if recovery is not None:
+ expected = {"available": False, "model": None, "state": "chat_recovery_required", "detail": pixel._CHAT_RECOVERY_DETAIL}
+ assert result == ({**expected, "recovery": recovery} if recovery else expected)
assert all(method == "GET" for method, _path, _kwargs in calls)
return result
assert result["available"] is True
@@ -122,9 +125,53 @@ async def test_interrupted_chat_gate_is_held_with_verified_permissions_and_no_ho
projected = pixel._access_projection(access)
assert projected["runtime_verified"] and projected["effective_mode"] == "sandboxed"
assert not projected["pending"]
- await status_for(monkeypatch, access, recovery=True)
+ await status_for(monkeypatch, access, recovery="verify-sandbox")
recovered = await status_for(monkeypatch, {**access, "reason": None})
assert recovered["available"] and recovered["readiness"]["accessState"] == "verified"
+ assert "recovery" not in recovered
+
+
[email protected](("mode", "recovery"), [
+ ("sandboxed", "verify-sandbox"),
+ # Settings offers only Restore Sandbox while a chat is held: Enable Full Access waits.
+ ("full-access", "restore-sandbox"),
+ ("unknown", ""),
+])
+async def test_the_chat_gate_offers_the_one_permissions_action_that_recovers_it(monkeypatch, mode, recovery):
+ verified = mode != "unknown"
+ access = {**ACCESS, "configured_mode": mode, "effective_mode": mode if verified else "unknown",
+ "runtime_verified": verified, "reason": "chat-recovery-required"}
+ await status_for(monkeypatch, access, recovery=recovery)
+
+
[email protected](("access", "model_status", "recovery"), [
+ # A gateway restart dropped an idle Sandbox's proof: Verify now.
+ ({**ACCESS, "runtime_verified": False, "effective_mode": "unknown", "reason": "runtime-proof-required"},
+ None, "verify-sandbox"),
+ # Busy work is refused by the host before anything changes; offer it anyway.
+ ({**ACCESS, "runtime_verified": False, "effective_mode": "unknown", "reason": "runtime-proof-required",
+ "busy": True}, None, "verify-sandbox"),
+ # An unverified Full Access is left to the host's same-mode reproof.
+ ({**ACCESS, "configured_mode": "full-access", "runtime_verified": False, "effective_mode": "unknown",
+ "reason": "runtime-proof-required"}, None, None),
+ # An unfinished access change resumes by restoring Sandbox, but only when
+ # the host proves no model transaction holds the coordinator instead.
+ ({**ACCESS, "pending": True, "runtime_verified": False, "effective_mode": "unknown",
+ "reason": "transition-recovery-required"}, {"modelTransactionPending": False}, "restore-sandbox"),
+ ({**ACCESS, "pending": True, "runtime_verified": False, "effective_mode": "unknown",
+ "reason": "transition-recovery-required"}, None, None),
+ ({**ACCESS, "pending": True, "runtime_verified": False, "effective_mode": "unknown",
+ "reason": "model-transition-recovery-required"}, {"modelTransactionPending": False}, None),
+ (ACCESS, None, None),
+ (FAILED, None, None),
+], ids=["stale-sandbox", "stale-sandbox-busy", "stale-full-access", "unfinished-change",
+ "unfinished-change-model-unknown", "installer-model-journal", "verified", "failed"])
+async def test_an_available_portal_offers_a_fix_only_where_the_host_will_accept_it(
+ monkeypatch, access, model_status, recovery):
+ result = await status_for(monkeypatch, access, model_status=model_status)
+ assert result.get("recovery") == recovery
@pytest.mark.asyncio
diff --git a/ods/extensions/services/dashboard/src/components/PortalModelRecovery.jsx b/ods/extensions/services/dashboard/src/components/PortalModelRecovery.jsx
index 610cf844f..394a1877a 100644
--- a/ods/extensions/services/dashboard/src/components/PortalModelRecovery.jsx
+++ b/ods/extensions/services/dashboard/src/components/PortalModelRecovery.jsx
@@ -2,7 +2,8 @@ import {useEffect,useRef,useState} from 'react'
import HelpLink from './HelpLink'
const phases=new Set(['idle','completed','prepared','held','applying','applied','committing','rolling-back','unavailable'])
-function recovery(value) {
+/** A validated /api/models/recovery receipt, or null. Portal's inline Recover reads it too. */
+export function recoveryReceipt(value) {
if (!value || typeof value.pending!=='boolean' || !phases.has(value.phase)
|| value.pending !== !['idle','completed'].includes(value.phase)) return null
if(value.transactionId!=null && (typeof value.transactionId!=='string'
@@ -31,7 +32,7 @@ export default function PortalModelRecovery({onPendingChange,onBusyChange,onReco
// The API also projects validated pending receipts on 409/503. Other
// failures are not recovery state and must never invent an outcome.
if(!response.ok && ![409,503].includes(response.status))return
- const value=recovery(await response.json())
+ const value=recoveryReceipt(await response.json())
if(!controller.signal.aborted && value){
setState(value);callbacks.current.onPendingChange?.(value.pending)
if(value.pending)pendingTransaction.current=value.transactionId
@@ -53,7 +54,7 @@ export default function PortalModelRecovery({onPendingChange,onBusyChange,onReco
try {
const response=await fetch('/api/models/recovery',{method:'POST',headers:{'Content-Type':'application/json'},
body:release?'{"releaseUnverified":true}':'{}',signal:controller.signal})
- const value=recovery(await response.json())
+ const value=recoveryReceipt(await response.json())
if(!mounted.current || controller.signal.aborted)return
if(value){pendingTransaction.current=value.pending?value.transactionId:null;setState(value);callbacks.current.onPendingChange?.(value.pending)}
if(response.ok && value && !value.pending){setReleasable(false);callbacks.current.onRecovered?.();return}
diff --git a/ods/extensions/services/dashboard/src/components/PortalRecovery.jsx b/ods/extensions/services/dashboard/src/components/PortalRecovery.jsx
new file mode 100644
index 000000000..986465159
--- /dev/null
+++ b/ods/extensions/services/dashboard/src/components/PortalRecovery.jsx
@@ -0,0 +1,200 @@
+import {useEffect, useRef, useState} from 'react'
+import {Loader2} from 'lucide-react'
+import {Link} from 'react-router-dom'
+import HelpLink from './HelpLink'
+import {recoveryReceipt} from './PortalModelRecovery'
+
+// The one-click fixes /api/pixel/status names in "recovery". Each calls the
+// existing API that Portal permissions or Models uses for the same state.
+export const PORTAL_RECOVERY_ACTIONS = new Set(['verify-sandbox', 'restore-sandbox', 'resume-model', 'recover-model'])
+
+// A held state implies its model action. Other actions need the API's hint:
+// only it knows the configured permissions mode, or that the runtime it owns
+// is stopped rather than unreachable.
+const STATE_ACTIONS = {unloaded: 'resume-model', 'model-recovery': 'recover-model'}
+const HINTED_ACTIONS = {
+ available: ['verify-sandbox', 'restore-sandbox'],
+ recovery: ['verify-sandbox', 'restore-sandbox'],
+ unavailable: ['resume-model'],
+}
+
+/** The fix Portal offers for this status and API hint, or null. */
+export function portalRecoveryAction(status, action) {
+ if (Object.hasOwn(STATE_ACTIONS, status)) return STATE_ACTIONS[status]
+ return Object.hasOwn(HINTED_ACTIONS, status) && HINTED_ACTIONS[status].includes(action) ? action : null
+}
+
+const COPY = {
+ 'verify-sandbox': ['Verify now', 'Checking Portal’s sandbox…'],
+ 'restore-sandbox': ['Restore Sandbox', 'Restoring Sandbox and checking the running tools…'],
+ 'resume-model': ['Resume model', 'Loading the model. This can take a few minutes…'],
+ 'recover-model': ['Recover model switch', 'Finishing the model switch…'],
+}
+// Held states show the API's own explanation. An available or offline Portal
+// already shows its status text elsewhere, so it gets these short lines.
+const MESSAGES = {
+ 'verify-sandbox': 'Portal restarted and has not re-checked its sandbox yet. ODS re-checks it automatically; you can also check it now.',
+ 'restore-sandbox': 'An earlier permissions change did not finish, so Portal’s sandbox is not verified.',
+ 'resume-model': 'The model runtime is not running, so Portal is paused.',
+ 'recover-model': 'A model switch stopped before it finished, so new messages are held.',
+}
+const HELD_STATES = new Set(['recovery', 'unloaded', 'model-recovery'])
+const REVISION = /^[a-f0-9]{64}$/
+const verified = value => value?.available === true && value.runtime_verified === true && value.pending === false
+ && value.reason == null && ['sandboxed', 'full-access'].includes(value.effective_mode)
+
+class RecoveryFailure extends Error {
+ constructor(cause, next, {help = true} = {}) {
+ super(cause)
+ this.next = next
+ this.help = help
+ }
+}
+
+// The API's own words for a refusal, as one bounded sentence.
+function apiMessage(body) {
+ const detail = body?.detail
+ const text = (typeof detail === 'string' ? detail
+ : [detail?.message, detail?.error, body?.error].find(value => typeof value === 'string'))?.trim().slice(0, 300)
+ return text ? /[.!?]$/.test(text) ? text : `${text}.` : ''
+}
+
+async function requestJson(url, options, timeout) {
+ const controller = new AbortController()
+ const timer = setTimeout(() => controller.abort(), timeout)
+ try {
+ const response = await fetch(url, {...options, cache: 'no-store', signal: controller.signal})
+ let body = null
+ try { body = await response.json() } catch { body = null }
+ return {ok: response.ok, status: response.status, body}
+ } catch {
+ throw new RecoveryFailure('ODS did not answer, so the result is not confirmed.',
+ 'Portal keeps checking. If this notice stays, choose the button again.')
+ } finally {
+ clearTimeout(timer)
+ }
+}
+
+const post = body => ({method: 'POST', headers: {'Content-Type': 'application/json'}, body})
+
+// Verify Sandbox and Restore Sandbox from Portal permissions: a fresh read for
+// the revision, then {"mode":"sandboxed"}. Never Full Access.
+async function recoverAccess() {
+ const read = await requestJson('/api/pixel/access-mode', {}, 45000)
+ const current = read.body
+ if (!read.ok || current?.available !== true || !REVISION.test(current?.revision ?? '')) {
+ throw new RecoveryFailure('ODS could not read Portal’s permissions, so nothing was changed.',
+ 'Wait a moment and try again.')
+ }
+ if (verified(current)) return
+ if (current.busy) {
+ throw new RecoveryFailure('Portal is still finishing earlier work, so nothing was changed.',
+ 'Try again when that work stops.', {help: false})
+ }
+ const change = await requestJson('/api/pixel/access-mode',
+ post(JSON.stringify({mode: 'sandboxed', revision: current.revision, confirmed: false})), 340000)
+ if (change.status === 409 && change.body?.detail?.code === 'model_lifecycle_busy') {
+ throw new RecoveryFailure('ODS is busy with a model or Portal change, so nothing was changed.',
+ 'Try again in a moment.', {help: false})
+ }
+ if (!change.ok || !verified(change.body)) {
+ throw new RecoveryFailure('Portal could not confirm its sandbox, so messages stay held.',
+ 'Try again. If it fails again, open Portal permissions in Settings to see what is unfinished.')
+ }
+}
+
+// Resume model from Models: start the stopped runtime with the saved model.
+async function resumeModel() {
+ const result = await requestJson('/api/models/runtime/start', post('{}'), 1225000)
+ if (result.ok && result.body?.status === 'started') return
+ const cause = apiMessage(result.body)
+ throw new RecoveryFailure(cause ? `The model did not start: ${cause}` : 'The model did not start.',
+ 'Wait a moment and choose Resume model again. If it keeps failing, open Models to check the model.')
+}
+
+// Recover model switch from Models: finish or undo the existing transaction.
+// It never loads another model; a release without proof is a separate choice.
+async function recoverModel(release) {
+ const result = await requestJson('/api/models/recovery', post(release ? '{"releaseUnverified":true}' : '{}'), 405000)
+ const value = recoveryReceipt(result.body)
+ if (result.ok && value && !value.pending) return 'done'
+ if (value?.reason === 'model-recovery-proof-required' && value.releasable === true && !release) return 'releasable'
+ throw new RecoveryFailure(value?.reason === 'model-recovery-proof-required'
+ ? 'The interrupted switch still needs repair. The saved state has been preserved.'
+ : 'The model switch could not be finished, so messages stay held.',
+ 'Open Models to read its current state and choose another model if needed.')
+}
+
+/** One primary button for a held Portal state, with progress and a helpful failure. */
+export default function PortalRecovery({status, action = null, detail = '', onSettled}) {
+ const chosen = portalRecoveryAction(status, action)
+ // {action, phase: working|failed|releasable|done, failure}. A run stays
+ // visible while Portal's status moves through it (Resume model reads as a
+ // switch), so its outcome is never lost.
+ const [run, setRun] = useState(null)
+ const lock = useRef(false), mounted = useRef(true)
+ const settled = useRef(onSettled)
+ settled.current = onSettled
+ useEffect(() => { mounted.current = true; return () => { mounted.current = false } }, [])
+ // Another held state, or none, retires an earlier outcome.
+ useEffect(() => {
+ setRun(current => current && current.phase !== 'working' && current.action !== chosen ? null : current)
+ }, [chosen])
+
+ async function start(release = false) {
+ const target = run?.phase === 'releasable' ? run.action : chosen
+ if (lock.current || !target) return
+ lock.current = true
+ setRun({action: target, phase: 'working', failure: null})
+ try {
+ let phase = 'done'
+ if (target === 'resume-model') await resumeModel()
+ else if (target === 'recover-model') phase = await recoverModel(release)
+ else await recoverAccess()
+ if (!mounted.current) return
+ setRun({action: target, phase, failure: null})
+ if (phase === 'done') settled.current?.()
+ } catch (error) {
+ if (!mounted.current) return
+ setRun({action: target, phase: 'failed', failure: error instanceof RecoveryFailure ? error
+ : new RecoveryFailure('Portal could not finish this step.', 'Try again.')})
+ } finally {
+ lock.current = false
+ }
+ }
+
+ const shown = run && run.phase !== 'done' ? run.action : chosen
+ // An older API names no access action: keep its link to Portal permissions.
+ if (!shown) {
+ if (status !== 'recovery') return null
+ return <p role="alert" className="shrink-0 border-b border-theme-border px-4 py-2 text-xs text-theme-text-secondary sm:px-6">
+ {detail} <Link to="/settings?section=access" className="underline">Portal permissions</Link>
+ </p>
+ }
+ const working = run?.phase === 'working'
+ const [label, progress] = COPY[shown]
+ const message = HELD_STATES.has(status) && shown === chosen && detail ? detail : MESSAGES[shown]
+ return <section aria-label="Portal recovery"
+ className="shrink-0 border-b border-theme-border px-4 py-2 text-xs text-theme-text-secondary sm:px-6">
+ <div className="flex flex-wrap items-center gap-x-3 gap-y-2">
+ <p role={status === 'available' ? 'status' : 'alert'} className="min-w-0 flex-1">{message}</p>
+ <button type="button" disabled={working} onClick={() => { void start() }}
+ className="inline-flex shrink-0 items-center gap-1.5 rounded-lg bg-theme-accent px-3 py-1.5 font-medium text-white transition hover:bg-theme-accent-hover disabled:cursor-wait disabled:opacity-60">
+ {working && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}
+ {label}
+ </button>
+ </div>
+ {shown === 'restore-sandbox' && <p className="mt-1">If Full Access was on, this turns it off. You can turn it on again in Portal permissions.</p>}
+ {working && <p role="status" className="mt-1">{progress}</p>}
+ {run?.phase === 'done' && shown === run.action && <p role="status" className="mt-1">Done. Checking Portal again…</p>}
+ {run?.phase === 'releasable' && <div className="mt-2">
+ <p>ODS could not confirm the model Portal used before this switch, so it cannot finish the switch the usual way.</p>
+ <p className="mt-1">The switch did not change anything, so ODS can release it and put Portal back on the model settings it had before. If that model was not working, choose another model or rerun the installer afterwards.</p>
+ <button type="button" className="mt-1 underline" onClick={() => { void start(true) }}>Release without checking</button>
+ </div>}
+ {run?.phase === 'failed' && <p role="alert" className="mt-1">
+ {run.failure.message} {run.failure.next}
+ {run.failure.help && <> <HelpLink/></>}
+ </p>}
+ </section>
+}
diff --git a/ods/extensions/services/dashboard/src/components/PortalRecovery.test.jsx b/ods/extensions/services/dashboard/src/components/PortalRecovery.test.jsx
new file mode 100644
index 000000000..274895f61
--- /dev/null
+++ b/ods/extensions/services/dashboard/src/components/PortalRecovery.test.jsx
@@ -0,0 +1,162 @@
+import {act, fireEvent, screen, within} from '@testing-library/react'
+import {render} from '../test/test-utils'
+import PortalRecovery from './PortalRecovery'
+
+const DISCORD = 'https://discord.gg/4ntNp9MAwC'
+const sandbox = {available:true, surface:'wsl-systemd', configured_mode:'sandboxed', effective_mode:'sandboxed',
+ runtime_verified:true, revision:'b'.repeat(64), busy:false, pending:false, reason:null, scope:'owner-host'}
+const stale = {...sandbox, effective_mode:'unknown', runtime_verified:false, revision:'a'.repeat(64), reason:'runtime-proof-required'}
+const reply = (body, status = 200) => ({ok:status < 400, status, json:async () => body})
+const deferred = () => { let resolve; const promise = new Promise(done => { resolve = done }); return {promise, resolve} }
+const banner = () => within(screen.getByRole('region', {name:'Portal recovery'}))
+const posts = url => fetch.mock.calls.filter(([path, options]) => path === url && options?.method === 'POST')
+
+function host(routes) {
+ vi.stubGlobal('fetch', vi.fn(async (url, options = {}) => {
+ const route = routes[`${options.method || 'GET'} ${url}`]
+ if (!route) throw new Error(`unexpected ${options.method || 'GET'} ${url}`)
+ return typeof route === 'function' ? route(options) : route
+ }))
+}
+
+afterEach(() => { vi.unstubAllGlobals() })
+
+it('verifies an idle Sandbox with a fresh revision, exactly as Portal permissions does', async () => {
+ const change = deferred()
+ host({'GET /api/pixel/access-mode': reply(stale), 'POST /api/pixel/access-mode': () => change.promise})
+ const settled = vi.fn()
+ render(<PortalRecovery status="available" action="verify-sandbox" onSettled={settled}/>)
+ expect(banner().getByRole('status')).toHaveTextContent('ODS re-checks it automatically')
+ // Nothing is sent before the owner chooses it.
+ expect(fetch).not.toHaveBeenCalled()
+ fireEvent.click(banner().getByRole('button', {name:'Verify now'}))
+ fireEvent.click(banner().getByRole('button', {name:'Verify now'}))
+ await act(async () => {})
+ expect(banner().getByRole('button', {name:'Verify now'})).toBeDisabled()
+ expect(banner().getByText('Checking Portal’s sandbox…')).toBeVisible()
+ expect(posts('/api/pixel/access-mode')).toHaveLength(1)
+ expect(JSON.parse(posts('/api/pixel/access-mode')[0][1].body)).toEqual({mode:'sandboxed', revision:'a'.repeat(64), confirmed:false})
+ await act(async () => change.resolve(reply(sandbox)))
+ expect(settled).toHaveBeenCalledOnce()
+ expect(banner().getByText('Done. Checking Portal again…')).toBeVisible()
+})
+
+it('restores Sandbox for a held chat when Full Access was configured, and says what that changes', async () => {
+ const fullAccess = {...sandbox, configured_mode:'full-access', effective_mode:'full-access', reason:'chat-recovery-required'}
+ host({'GET /api/pixel/access-mode': reply(fullAccess), 'POST /api/pixel/access-mode': reply(sandbox)})
+ const settled = vi.fn()
+ render(<PortalRecovery status="recovery" action="restore-sandbox" onSettled={settled}
+ detail="Portal stopped during an earlier turn, so new messages are held until its permissions are checked again."/>)
+ expect(banner().getByRole('alert')).toHaveTextContent('Portal stopped during an earlier turn')
+ expect(banner().getByText(/If Full Access was on, this turns it off/)).toBeVisible()
+ fireEvent.click(banner().getByRole('button', {name:'Restore Sandbox'}))
+ await act(async () => {})
+ // Full Access is never requested: only the Sandbox restore Portal permissions offers here.
+ expect(posts('/api/pixel/access-mode').map(([, options]) => JSON.parse(options.body)))
+ .toEqual([{mode:'sandboxed', revision:'b'.repeat(64), confirmed:false}])
+ expect(settled).toHaveBeenCalledOnce()
+})
+
+it('does nothing when the host already re-proved the runtime before the click', async () => {
+ host({'GET /api/pixel/access-mode': reply(sandbox)})
+ const settled = vi.fn()
+ render(<PortalRecovery status="available" action="verify-sandbox" onSettled={settled}/>)
+ fireEvent.click(banner().getByRole('button', {name:'Verify now'}))
+ await act(async () => {})
+ expect(posts('/api/pixel/access-mode')).toHaveLength(0)
+ expect(settled).toHaveBeenCalledOnce()
+})
+
+it.each([
+ ['busy runtime', {'GET /api/pixel/access-mode': reply({...stale, busy:true})},
+ 'Portal is still finishing earlier work, so nothing was changed. Try again when that work stops.', false],
+ ['busy model lifecycle', {'GET /api/pixel/access-mode': reply(stale), 'POST /api/pixel/access-mode':
+ reply({detail:{code:'model_lifecycle_busy', message:'ODS is busy'}}, 409)},
+ 'ODS is busy with a model or Portal change, so nothing was changed. Try again in a moment.', false],
+ ['unconfirmed change', {'GET /api/pixel/access-mode': reply(stale), 'POST /api/pixel/access-mode':
+ reply({detail:'The access change was not verified. Refresh status before retrying or restoring safer mode.'}, 409)},
+ 'Portal could not confirm its sandbox, so messages stay held. Try again. If it fails again, open Portal permissions in Settings to see what is unfinished.', true],
+ ['unreadable permissions', {'GET /api/pixel/access-mode': reply({detail:'Portal access service is unavailable'}, 503)},
+ 'ODS could not read Portal’s permissions, so nothing was changed. Wait a moment and try again.', true],
+ ['no answer', {'GET /api/pixel/access-mode': () => { throw new TypeError('Failed to fetch') }},
+ 'ODS did not answer, so the result is not confirmed. Portal keeps checking. If this notice stays, choose the button again.', true],
+])('explains a %s with its cause and next step', async (_name, routes, text, help) => {
+ host(routes)
+ const settled = vi.fn()
+ render(<PortalRecovery status="available" action="verify-sandbox" onSettled={settled}/>)
+ fireEvent.click(banner().getByRole('button', {name:'Verify now'}))
+ await act(async () => {})
+ const failure = banner().getByRole('alert')
+ expect(failure).toHaveTextContent(text)
+ if (help) expect(within(failure).getByRole('link', {name:'Get help on Discord'})).toHaveAttribute('href', DISCORD)
+ else expect(within(failure).queryByRole('link')).toBeNull()
+ expect(settled).not.toHaveBeenCalled()
+ // The button stays for another attempt.
+ expect(banner().getByRole('button', {name:'Verify now'})).toBeEnabled()
+})
+
+it('resumes an unloaded model and keeps showing a failure while Portal reports the attempt as a switch', async () => {
+ const start = deferred()
+ host({'POST /api/models/runtime/start': () => start.promise})
+ const view = render(<PortalRecovery status="unloaded" detail="The model is unloaded, so Portal is paused. Choose Resume model to load it again."/>)
+ fireEvent.click(banner().getByRole('button', {name:'Resume model'}))
+ await act(async () => {})
+ expect(posts('/api/models/runtime/start').map(([, options]) => options.body)).toEqual(['{}'])
+ // The host owns a model operation while it resumes, so status reads "switching".
+ view.rerender(<PortalRecovery status="switching"/>)
+ expect(banner().getByText('Loading the model. This can take a few minutes…')).toBeVisible()
+ await act(async () => start.resolve(reply({detail:{error:'Windows runtime start is unconfirmed', code:'runtime_control_unconfirmed'}}, 409)))
+ const failure = banner().getByText(/The model did not start/)
+ expect(failure).toHaveAttribute('role', 'alert')
+ expect(failure).toHaveTextContent('The model did not start: Windows runtime start is unconfirmed. Wait a moment and choose Resume model again.')
+ expect(within(failure).getByRole('link', {name:'Get help on Discord'})).toHaveAttribute('href', DISCORD)
+ view.rerender(<PortalRecovery status="unloaded" detail="The model is unloaded, so Portal is paused. Choose Resume model to load it again."/>)
+ expect(banner().getByText(/The model did not start/)).toBeVisible()
+ view.rerender(<PortalRecovery status="available"/>)
+ expect(screen.queryByRole('region', {name:'Portal recovery'})).toBeNull()
+})
+
+it('recovers a stopped switch, and offers the release Models offers when nothing changed', async () => {
+ const held = {pending:true, phase:'held', transactionId:'c'.repeat(64)}
+ const answers = [reply({...held, reason:'model-recovery-proof-required', releasable:true}, 409),
+ reply({pending:false, phase:'completed', transactionId:'c'.repeat(64), outcome:'rollback'})]
+ host({'POST /api/models/recovery': () => answers.shift()})
+ const settled = vi.fn()
+ render(<PortalRecovery status="model-recovery" onSettled={settled}/>)
+ fireEvent.click(banner().getByRole('button', {name:'Recover model switch'}))
+ await act(async () => {})
+ expect(banner().getByText(/cannot finish the switch the usual way/)).toBeVisible()
+ expect(settled).not.toHaveBeenCalled()
+ fireEvent.click(banner().getByRole('button', {name:'Release without checking'}))
+ await act(async () => {})
+ expect(posts('/api/models/recovery').map(([, options]) => options.body)).toEqual(['{}', '{"releaseUnverified":true}'])
+ expect(settled).toHaveBeenCalledOnce()
+})
+
+it('offers Resume model for a stopped runtime only when the API names it', async () => {
+ host({'POST /api/models/runtime/start': reply({status:'started'})})
+ const settled = vi.fn()
+ const view = render(<PortalRecovery status="unavailable" detail="The local model runtime is unavailable."/>)
+ // Unreachable is not stopped: without the host's proof there is no button.
+ expect(screen.queryByRole('region', {name:'Portal recovery'})).toBeNull()
+ view.rerender(<PortalRecovery status="unavailable" action="resume-model" onSettled={settled}
+ detail="The local model runtime is not running, so Portal is paused. Choose Resume model to start it."/>)
+ expect(banner().getByRole('alert')).toHaveTextContent('The model runtime is not running, so Portal is paused.')
+ fireEvent.click(banner().getByRole('button', {name:'Resume model'}))
+ await act(async () => {})
+ expect(posts('/api/models/runtime/start')).toHaveLength(1)
+ expect(settled).toHaveBeenCalledOnce()
+ view.rerender(<PortalRecovery status="unavailable" action="verify-sandbox"/>)
+ expect(screen.queryByRole('region', {name:'Portal recovery'})).toBeNull()
+})
+
+it('shows nothing for a healthy Portal, and keeps the settings link for an older API', () => {
+ host({})
+ const view = render(<PortalRecovery status="available"/>)
+ expect(screen.queryByRole('region', {name:'Portal recovery'})).toBeNull()
+ view.rerender(<PortalRecovery status="available" action="enable-full-access"/>)
+ expect(screen.queryByRole('region', {name:'Portal recovery'})).toBeNull()
+ view.rerender(<PortalRecovery status="recovery" detail="Portal stopped during an earlier turn."/>)
+ expect(screen.getByRole('link', {name:'Portal permissions'})).toHaveAttribute('href', '/settings?section=access')
+ expect(fetch).not.toHaveBeenCalled()
+})
diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.jsx
index bc6023c26..71a722b17 100644
--- a/ods/extensions/services/dashboard/src/pages/Pixel.jsx
+++ b/ods/extensions/services/dashboard/src/pages/Pixel.jsx
@@ -36,6 +36,7 @@ import {compactCommand,CONTEXT_REQUEST_ID,historySnapshot,usePortalContext} from
import PortalModelSelector from '../components/PortalModelSelector'
import PortalRuntimeIdentity from '../components/PortalRuntimeIdentity'
import PortalReadiness from '../components/PortalReadiness'
+import PortalRecovery, { PORTAL_RECOVERY_ACTIONS, portalRecoveryAction } from '../components/PortalRecovery'
import { pixelReadinessView, PIXEL_RECOVERY_REQUIRED } from '../lib/pixelReadiness'
import PortalAgentActivity from '../components/PortalAgentActivity'
import PortalExtensionSetup from '../components/PortalExtensionSetup'
@@ -624,6 +625,8 @@ export default function Pixel({ systemStatus = null }) {
const [status, setStatus] = useState('loading')
const [statusDetail, setStatusDetail] = useState('')
+ // The one-click fix the status API names for a held state, if any.
+ const [statusRecovery, setStatusRecovery] = useState(null)
useEffect(() => {
// Wake an already-open Permissions panel once when admission becomes held.
// The panel still fetches its own authoritative access receipt.
@@ -988,7 +991,10 @@ export default function Pixel({ systemStatus = null }) {
? 'held'
: data.state === 'chat_recovery_required'
? 'recovery'
+ : data.state === 'model_recovery_required'
+ ? 'model-recovery'
: 'unavailable')
+ setStatusRecovery(PORTAL_RECOVERY_ACTIONS.has(data?.recovery) ? data.recovery : null)
setModelActivation(modelActivationStatus(data.modelActivation))
setStatusDetail(typeof data.detail === 'string' ? data.detail : '')
} catch (error) {
@@ -997,6 +1003,7 @@ export default function Pixel({ systemStatus = null }) {
setAgentRuntime(null)
setRuntimeIdentity(null)
setRuntimeReadiness(null)
+ setStatusRecovery(null)
setStatus('unavailable')
setStatusDetail('Could not reach Portal backend')
}
@@ -1850,6 +1857,8 @@ export default function Pixel({ systemStatus = null }) {
? 'Model transition pending'
: status === 'recovery'
? 'Recovery required'
+ : status === 'model-recovery'
+ ? 'Model switch stopped'
: status === 'loading'
? 'Connecting...'
: 'Degraded'
@@ -1951,11 +1960,10 @@ export default function Pixel({ systemStatus = null }) {
</span>
</div>
</header>
- {status === 'available' && <PortalReadiness readiness={runtimeReadiness} />}
+ {status === 'available' && !portalRecoveryAction(status, statusRecovery) && <PortalReadiness readiness={runtimeReadiness} />}
<ModelActivationNotice value={modelActivation} pending={status === 'switching'} available={status === 'available'} portal onRefresh={()=>setModelStatusRefresh(value=>value+1)}/>
{status === 'held' && <p role="status" className="shrink-0 border-b border-theme-border px-4 py-2 text-xs text-theme-text-secondary sm:px-6">{statusDetail}</p>}
- {status === 'recovery' && <p role="alert" className="shrink-0 border-b border-theme-border px-4 py-2 text-xs text-theme-text-secondary sm:px-6">{statusDetail} <Link to="/settings?section=access" className="underline">Portal permissions</Link></p>}
- {status === 'unloaded' && <p role="status" className="shrink-0 border-b border-theme-border px-4 py-2 text-xs text-theme-text-secondary sm:px-6">{statusDetail} <Link to="/models" className="underline">Open Models</Link></p>}
+ <PortalRecovery status={status} action={statusRecovery} detail={statusDetail} onSettled={()=>setModelStatusRefresh(value=>value+1)}/>
{status === 'available' && modelSupport && (
<p role="status" aria-label="Model capability" className="shrink-0 border-b border-theme-border px-4 py-2 text-xs text-theme-text-secondary sm:px-6">
{modelSupport.detail}
diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.model-switch.test.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.model-switch.test.jsx
index 0b8b52820..930825111 100644
--- a/ods/extensions/services/dashboard/src/pages/Pixel.model-switch.test.jsx
+++ b/ods/extensions/services/dashboard/src/pages/Pixel.model-switch.test.jsx
@@ -71,25 +71,51 @@ it('requires an explicit unconfirmed outcome and never enables chat from diagnos
expect(screen.getByLabelText('Model activation')).toHaveTextContent('has not finished')
})
-it('says an unloaded model needs Resume model in Models instead of waiting for a switch',async()=>{
- const detail='The model is unloaded. Open Models and choose Resume model to use Portal again.'
+it('offers Resume model inside Portal for an unloaded model and keeps the draft',async()=>{
+ const detail='The model is unloaded, so Portal is paused. Choose Resume model to load it again.'
render(<Pixel/>); await act(async()=>{})
fireEvent.change(screen.getByRole('textbox'),{target:{value:'Keep this draft intact'}})
- await poll({available:false,model:null,state:'model_unloaded',detail})
+ await poll({available:false,model:null,state:'model_unloaded',detail,recovery:'resume-model'})
expect(screen.getByText('Model unloaded').querySelector('.animate-spin')).toBeNull()
expect(screen.queryByText('Switching model...')).toBeNull()
expect(screen.queryByLabelText('Model activation')).toBeNull()
expect(screen.getByText(detail)).toBeVisible()
- expect(screen.getByRole('link',{name:'Open Models'})).toHaveAttribute('href','/models')
+ // The fix is a button here, not a link to another page.
+ expect(screen.queryByRole('link',{name:'Open Models'})).toBeNull()
+ const resume=within(screen.getByRole('region',{name:'Portal recovery'})).getByRole('button',{name:'Resume model'})
const composer=screen.getByPlaceholderText('Messages are held until the model is resumed')
expect(composer).toBeDisabled()
expect(composer).toHaveValue('Keep this draft intact')
expect(screen.getByTitle('Send')).toBeDisabled()
+ // Nothing is resumed until the owner chooses it.
+ expect(fetch.mock.calls.every(([url,options])=>options?.method!=='POST' || url==='/api/pixel/chat/context')).toBe(true)
+ fetch.mockImplementationOnce(async()=>json({status:'started'}))
+ fireEvent.click(resume)
+ await act(async()=>{})
+ const resumes=fetch.mock.calls.filter(([url])=>url==='/api/models/runtime/start')
+ expect(resumes.map(([,options])=>[options.method,options.body])).toEqual([['POST','{}']])
+ expect(screen.getByText('Done. Checking Portal again…')).toBeVisible()
await poll({available:true})
- expect(screen.queryByRole('link',{name:'Open Models'})).toBeNull()
+ expect(screen.queryByRole('region',{name:'Portal recovery'})).toBeNull()
expect(screen.getByRole('textbox')).toHaveValue('Keep this draft intact')
expect(screen.getByTitle('Send')).toBeEnabled()
- expect(fetch.mock.calls.every(([url,options])=>options?.method!=='POST' || url==='/api/pixel/chat/context')).toBe(true)
+})
+
+it('offers Recover model switch for a switch that stopped and names the next step if it fails',async()=>{
+ const detail='A model switch stopped before it finished, so new messages are held. Choose Recover model switch to finish or undo it.'
+ render(<Pixel/>); await act(async()=>{})
+ await poll({available:false,model:null,state:'model_recovery_required',detail,recovery:'recover-model'})
+ expect(screen.getByText('Model switch stopped')).toBeVisible()
+ expect(screen.queryByText('Switching model...')).toBeNull()
+ expect(screen.getByTitle('Send')).toBeDisabled()
+ fetch.mockImplementationOnce(async()=>({ok:false,status:409,json:async()=>({pending:true,phase:'held',
+ transactionId:'a'.repeat(64),reason:'model-recovery-unavailable'})}))
+ fireEvent.click(within(screen.getByRole('region',{name:'Portal recovery'})).getByRole('button',{name:'Recover model switch'}))
+ await act(async()=>{})
+ const failure=within(screen.getByRole('region',{name:'Portal recovery'})).getByText(/could not be finished/)
+ expect(failure).toHaveTextContent('Open Models to read its current state')
+ expect(within(failure).getByRole('link',{name:'Get help on Discord'})).toHaveAttribute('href','https://discord.gg/4ntNp9MAwC')
+ expect(fetch.mock.calls.filter(([url,options])=>url==='/api/models/recovery'&&options?.method==='POST')).toHaveLength(1)
})
it('ignores malformed diagnostic metadata and removes a stale diagnosis on the next status read',async()=>{
diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.readiness.test.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.readiness.test.jsx
index 73b53aa76..3a291a70d 100644
--- a/ods/extensions/services/dashboard/src/pages/Pixel.readiness.test.jsx
+++ b/ods/extensions/services/dashboard/src/pages/Pixel.readiness.test.jsx
@@ -81,6 +81,59 @@ it('refreshes already-open verified Permissions when Portal discovers recovery w
expect(transport.mock.calls.some(([url])=>url==='/api/pixel/chat/stream')).toBe(false)
})
+it('recovers a held chat from Portal itself with the same Verify Sandbox request',async()=>{
+ const sandbox={available:true,surface:'wsl-systemd',configured_mode:'sandboxed',effective_mode:'sandboxed',runtime_verified:true,revision:'a'.repeat(64),busy:false,pending:false,reason:null,scope:'owner-host'}
+ let held=true
+ const transport=vi.fn(async(url,options)=>{
+ if(url==='/api/pixel/status')return json(held?{available:false,state:'chat_recovery_required',recovery:'verify-sandbox',
+ detail:'Portal stopped during an earlier turn, so new messages are held until its permissions are checked again.'}:{available:true})
+ if(url==='/api/pixel/access-mode'){
+ if(options?.method==='POST'){held=false;return json(sandbox)}
+ return json({...sandbox,reason:'chat-recovery-required'})
+ }
+ if(url==='/api/pixel/chat/context')return json({schemaVersion:1,status:'missing',sessionRevision:null,
+ context:null,model:null,compaction:{status:'idle',count:0},history:{revision:null,acknowledgedMessages:0}})
+ return json({})
+ })
+ vi.stubGlobal('fetch',transport);vi.useFakeTimers()
+ render(<Pixel/>);await act(async()=>{})
+ fireEvent.change(screen.getByPlaceholderText('Messages are held until Portal recovery'),{target:{value:'Keep this draft'}})
+ expect(screen.getByText('Recovery required')).toBeVisible()
+ const recovery=within(screen.getByRole('region',{name:'Portal recovery'}))
+ expect(recovery.getByRole('alert')).toHaveTextContent('new messages are held')
+ expect(screen.queryByRole('link',{name:'Portal permissions'})).toBeNull()
+ // Recovery waits for the owner; polling alone never sends it.
+ await act(async()=>{await vi.advanceTimersByTimeAsync(9000)})
+ expect(transport.mock.calls.some(([url,options])=>options?.method==='POST'&&url!=='/api/pixel/chat/context')).toBe(false)
+ fireEvent.click(recovery.getByRole('button',{name:'Verify now'}));await act(async()=>{})
+ const posts=transport.mock.calls.filter(([url,options])=>url==='/api/pixel/access-mode'&&options?.method==='POST')
+ expect(posts.map(([,options])=>JSON.parse(options.body))).toEqual([{mode:'sandboxed',revision:'a'.repeat(64),confirmed:false}])
+ await act(async()=>{await vi.advanceTimersByTimeAsync(3000)})
+ expect(screen.queryByRole('region',{name:'Portal recovery'})).toBeNull()
+ expect(screen.getByPlaceholderText('Message Portal...')).toHaveValue('Keep this draft')
+ expect(screen.getByPlaceholderText('Message Portal...')).toBeEnabled()
+ expect(transport.mock.calls.some(([url])=>url==='/api/pixel/chat/stream')).toBe(false)
+})
+
+it('offers Verify now for a stale Sandbox proof instead of a settings link',async()=>{
+ const unverified={...failed(),state:'unverified',accessState:'unverified',reasonCode:'access-proof-unverified'}
+ let status={available:true,readiness:unverified,recovery:'verify-sandbox'}
+ transport(()=>status)
+ vi.useFakeTimers()
+ render(<Pixel/>);await act(async()=>{})
+ const recovery=within(screen.getByRole('region',{name:'Portal recovery'}))
+ expect(recovery.getByRole('status')).toHaveTextContent('ODS re-checks it automatically')
+ expect(recovery.getByRole('button',{name:'Verify now'})).toBeEnabled()
+ expect(screen.queryByLabelText('Runtime readiness')).toBeNull()
+ // Chat stays usable meanwhile; the notice is not a hold.
+ expect(screen.getByPlaceholderText('Message Portal...')).toBeEnabled()
+ // The host monitor re-proves it on its own; the notice then goes away.
+ status={available:true,readiness:{...unverified,accessState:'verified',effectiveMode:'sandboxed',reasonCode:'release-binding-unavailable'}}
+ await act(async()=>{await vi.advanceTimersByTimeAsync(3000)})
+ expect(screen.queryByRole('region',{name:'Portal recovery'})).toBeNull()
+ expect(fetch.mock.calls.some(([url])=>url==='/api/pixel/access-mode')).toBe(false)
+})
+
it('retains only a diagnostic cloud label during an available access transition and recovers on polling',async()=>{
const runtime={source:'remote-provider',model:'cloud-model',contextLength:65536,maxTokens:4096,reasoning:false}
let status={available:true,runtime}
diff --git a/ods/installers/macos/PIXEL-NATIVE.md b/ods/installers/macos/PIXEL-NATIVE.md
index 26c5fa8f6..d7790faf8 100644
--- a/ods/installers/macos/PIXEL-NATIVE.md
+++ b/ods/installers/macos/PIXEL-NATIVE.md
@@ -1064,6 +1064,14 @@ the simulated crash. Startup/reconcile tests cover 26 cases and seven
subtests. Cold reboot and interruption during active tools or transactions
remain separate qualification requirements.
+Since 2026-10-25 the monitor no longer stops on a state that can resolve
+later. A pending transition, an interrupted chat or an open Portal model
+transaction is checked again with backoff from 30 seconds to 5 minutes, and
+the helper still refuses to touch it. After a timeout or an unconfirmed
+change the monitor submits nothing further; it reads status until the runtime
+is verified again, then resumes. A status read that shows a stale Sandbox
+proof wakes it early.
+
The local Portal cancellation check on 2026-09-19 observed native inference
active before cancelling the exact chat/request identity. The API acknowledged
cancellation in 0.12 seconds; the stream ended and retained state was
diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txt
index e998cbdda..94ef16fe2 100644
--- a/ods/tests/ci-suite.txt
+++ b/ods/tests/ci-suite.txt
@@ -298,6 +298,7 @@ tests/test_pixel_release_access.py
tests/test_pixel_release_client.py
tests/test_pixel_release_resume.py
tests/test_pixel_source_upgrade_diagnostics.py
+tests/test_pixel_startup_reproof.py
tests/test_pixel_verify_lifecycle.py
tests/test_portal_identity.py
tests/test_preview_inspection.py
diff --git a/ods/tests/test_pixel_startup_reproof.py b/ods/tests/test_pixel_startup_reproof.py
index 217c1d38f..fad04c5cc 100644
--- a/ods/tests/test_pixel_startup_reproof.py
+++ b/ods/tests/test_pixel_startup_reproof.py
@@ -16,6 +16,18 @@ SOURCE = Path(__file__).resolve().parents[1] / 'bin/ods-host-agent.py'
tree = ast.parse(SOURCE.read_text())
function = next(n for n in tree.body if isinstance(n, ast.FunctionDef)
and n.name == '_reconcile_native_pixel_startup')
+monitor = next(n for n in tree.body if isinstance(n, ast.FunctionDef)
+ and n.name == '_monitor_native_pixel_access')
+# The monitor's interval and backoff cap, exactly as the host agent defines them.
+monitor_settings = [n for n in tree.body if isinstance(n, ast.Assign)
+ and any(isinstance(t, ast.Name) and t.id in (
+ '_PIXEL_ACCESS_RECHECK_SECONDS', '_PIXEL_ACCESS_BACKOFF_CAP_SECONDS')
+ for t in n.targets)]
+ATTENTION = ('attention', 'stage=unsafe-state reason=transition-recovery-required')
+
+
+class Stop(Exception):
+ """Ends the otherwise endless monitor loop after the scripted checks."""
class StartupTests(unittest.TestCase):
@@ -42,15 +54,56 @@ class StartupTests(unittest.TestCase):
def call(self):
return self.scope['_reconcile_native_pixel_startup']()
- def test_monitor_continues_only_explicit_read_only_or_success_results(self):
- monitor = next(n for n in tree.body if isinstance(n, ast.FunctionDef)
- and n.name == '_monitor_native_pixel_access')
- exec(compile(ast.Module(body=[monitor], type_ignores=[]), str(SOURCE), 'exec'), self.scope)
- check = Mock(side_effect=[True, True, None])
- self.scope['_reconcile_native_pixel_startup'] = check
- self.scope['_monitor_native_pixel_access']()
+ def run_monitor(self, checks, *, hold=(), verified=()):
+ """Drive the monitor through scripted checks; return each wait it asked for."""
+ waits = []
+ self.scope.update(
+ _reconcile_native_pixel_startup=Mock(side_effect=[*checks, Stop()]),
+ _pixel_model_hold_pending=Mock(side_effect=[*hold, *[False] * 20]),
+ _pixel_access_verified=Mock(side_effect=list(verified)),
+ _wait_for_pixel_access_check=lambda delay, *, wake: waits.append((delay, wake)))
+ exec(compile(ast.Module(body=[*monitor_settings, monitor], type_ignores=[]),
+ str(SOURCE), 'exec'), self.scope)
+ with self.assertRaises(Stop):
+ self.scope['_monitor_native_pixel_access']()
+ return waits
+
+ def test_monitor_keeps_checking_a_state_that_can_resolve_with_bounded_backoff(self):
+ waits = self.run_monitor([('recheck', None), *[ATTENTION] * 7, ('recheck', None)])
+ self.assertEqual([delay for delay, _wake in waits], [30, 30, 60, 120, 240, 300, 300, 300, 30])
+ self.assertTrue(all(wake for _delay, wake in waits))
+
+ def test_monitor_logs_attention_once_and_then_the_recovery(self):
+ with self.assertLogs('test', level='INFO') as logs:
+ self.run_monitor([ATTENTION, ATTENTION, ('recheck', None)])
+ self.assertEqual(logs.output, [
+ 'WARNING:test:Pixel startup reproof requires attention '
+ '(stage=unsafe-state reason=transition-recovery-required); the access monitor keeps checking',
+ 'INFO:test:Pixel access monitor resumed automatic reproof'])
+
+ def test_monitor_never_reruns_the_helper_after_an_uncertain_change_until_verified(self):
+ uncertain = ('uncertain', 'stage=change-failed reason=None')
+ waits = self.run_monitor([uncertain, ('recheck', None)], verified=[False, False, True])
+ check = self.scope['_reconcile_native_pixel_startup']
+ # The helper (the only path that can submit a change) ran once before
+ # and once after the read-only status showed a verified runtime.
self.assertEqual(check.call_count, 3)
- self.assertEqual(self.sleep.call_args_list, [unittest.mock.call(30)] * 2)
+ self.assertEqual(self.scope['_pixel_access_verified'].call_count, 3)
+ self.assertEqual(waits, [(30, False), (60, False), (120, False), (30, True), (30, True)])
+
+ def test_monitor_leaves_an_open_model_transaction_to_resume_or_recover(self):
+ waits = self.run_monitor([('recheck', None)], hold=[True, True])
+ self.assertEqual(self.scope['_reconcile_native_pixel_startup'].call_count, 2)
+ self.assertEqual(waits, [(30, False), (60, False), (30, True)])
+
+ def test_monitor_stops_only_without_a_usable_helper(self):
+ self.scope.update(_reconcile_native_pixel_startup=Mock(return_value=(None, None)),
+ _pixel_model_hold_pending=Mock(return_value=False),
+ _wait_for_pixel_access_check=Mock())
+ exec(compile(ast.Module(body=[*monitor_settings, monitor], type_ignores=[]),
+ str(SOURCE), 'exec'), self.scope)
+ self.scope['_monitor_native_pixel_access']()
+ self.scope['_wait_for_pixel_access_check'].assert_not_called()
def test_host_agent_starts_reproof_monitor_on_native_linux_and_macos(self):
main = next(n for n in tree.body if isinstance(n, ast.FunctionDef)
@@ -64,25 +117,31 @@ class StartupTests(unittest.TestCase):
self.system.return_value = system
self.assertIs(eval(enabled, self.scope), expected)
- def test_busy_preflight_can_be_checked_later_but_pending_cannot(self):
+ def test_busy_preflight_is_checked_again_but_pending_needs_attention(self):
failure = {'stage': 'unsafe-state', 'projection': {'scope': 'owner-host',
'available': True, 'busy': True, 'pending': False}}
self.run.return_value = types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode())
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.run.assert_called_once()
failure['projection']['pending'] = True
self.run.return_value.stderr = json.dumps(failure).encode()
- self.assertIsNone(self.call())
+ self.assertEqual(self.call(), ('attention', 'stage=unsafe-state reason=None'))
- def test_uncertain_change_stops_monitor_even_with_busy_projection(self):
+ def test_uncertain_change_is_uncertain_even_with_busy_projection(self):
failure = {'stage': 'change-failed', 'projection': {'scope': 'owner-host',
'available': True, 'busy': True, 'pending': False}}
self.run.return_value = types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode())
- self.assertIsNone(self.call())
+ self.assertEqual(self.call(), ('uncertain', 'stage=change-failed reason=None'))
+ self.run.assert_called_once()
+
+ def test_helper_exception_after_a_possible_change_is_uncertain(self):
+ failure = {'stage': 'client-exception', 'exceptionType': 'OSError'}
+ self.run.return_value = types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode())
+ self.assertEqual(self.call(), ('uncertain', 'stage=client-exception reason=None'))
self.run.assert_called_once()
def test_ready_executes_protected_helper_under_lifecycle_lock(self):
- self.call()
+ self.assertEqual(self.call(), ('recheck', None))
self.assertEqual(self.run.call_args.args[0], ['/usr/bin/python3', '-I',
'/usr/local/libexec/ods-pixel-access/pixel_access_reconcile.py', '--startup'])
self.end.assert_called_once_with('pixel_startup_reproof')
@@ -90,35 +149,35 @@ class StartupTests(unittest.TestCase):
def test_linux_reuses_the_same_protected_helper_and_lock(self):
self.system.return_value = 'Linux'
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.assertEqual(self.run.call_args.args[0], ['/usr/bin/python3', '-I',
'/usr/local/libexec/ods-pixel-access/pixel_access_reconcile.py', '--startup'])
self.end.assert_called_once_with('pixel_startup_reproof')
def test_other_platform_or_missing_helper_is_read_only(self):
self.system.return_value = 'Windows'
- self.call()
+ self.assertEqual(self.call(), (None, None))
self.system.return_value = 'Darwin'
self.exists.return_value = False
- self.call()
+ self.assertEqual(self.call(), (None, None))
self.begin.assert_not_called()
self.run.assert_not_called()
def test_unsafe_helper_is_never_executed(self):
self.lstat.return_value.st_uid = 501
- self.call()
+ self.assertEqual(self.call(), (None, None))
self.run.assert_not_called()
def test_legacy_helper_without_startup_contract_is_not_executed(self):
for text in ('', 'STARTUP_REPROOF_VERSION = True', 'STARTUP_REPROOF_VERSION = 2',
'STARTUP_REPROOF_VERSION = 1\nSTARTUP_REPROOF_VERSION = 1'):
self.read_text.return_value = text
- self.call()
+ self.assertEqual(self.call(), (None, None))
self.run.assert_not_called()
def test_busy_model_lifecycle_has_bounded_wait_without_execution(self):
self.begin.return_value = (False, {})
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.assertEqual(self.begin.call_count, 12)
self.assertEqual(self.sleep.call_count, 11)
self.run.assert_not_called()
@@ -130,7 +189,7 @@ class StartupTests(unittest.TestCase):
self.begin.return_value = (False, {'operation': 'model_download',
'target': 'model.gguf'})
with self.assertLogs('test', level='INFO') as logs:
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.run.assert_not_called()
self.assertEqual(logs.output, [
'INFO:test:Pixel access reproof deferred while model_download is in progress'])
@@ -141,7 +200,7 @@ class StartupTests(unittest.TestCase):
'pending': False, 'busy': False, 'reason': 'runtime-unavailable-or-busy'}}
self.run.return_value = types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode())
with self.assertLogs('test', level='INFO') as logs:
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.run.assert_called_once()
self.assertEqual(logs.output, [
'WARNING:test:Pixel startup reproof readiness window exhausted '
@@ -152,7 +211,7 @@ class StartupTests(unittest.TestCase):
'pending': False, 'busy': False, 'reason': 'admission-gate-unavailable'}}
self.run.return_value = types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode())
with self.assertLogs('test', level='WARNING') as logs:
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.assertEqual(logs.output, [
'WARNING:test:Pixel startup reproof readiness window exhausted '
'(last stage=unsafe-state reason=admission-gate-unavailable)'])
@@ -161,7 +220,7 @@ class StartupTests(unittest.TestCase):
self.run.reset_mock()
failure['stage'] = 'change-failed'
self.run.return_value.stderr = json.dumps(failure).encode()
- self.call()
+ self.assertEqual(self.call(), ('uncertain', 'stage=change-failed reason=admission-gate-unavailable'))
self.run.assert_called_once()
def test_boot_order_unavailability_retries_until_ready(self):
@@ -179,26 +238,22 @@ class StartupTests(unittest.TestCase):
types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode()),
types.SimpleNamespace(returncode=0, stderr=b''),
]
- self.assertIs(self.call(), True)
+ self.assertEqual(self.call(), ('recheck', None))
self.assertEqual(self.run.call_count, 3)
self.assertEqual(self.end.call_count, 3)
self.assertEqual(self.sleep.call_args_list, [unittest.mock.call(5)] * 2)
self.run.side_effect = None
- def test_attention_warning_names_stage_and_reason(self):
+ def test_attention_names_stage_and_reason(self):
failure = {'stage': 'unsafe-state', 'projection': {'available': True,
'pending': True, 'busy': False, 'reason': 'transition-recovery-required'}}
self.run.return_value = types.SimpleNamespace(returncode=1, stderr=json.dumps(failure).encode())
- with self.assertLogs('test', level='WARNING') as logs:
- self.assertIsNone(self.call())
+ self.assertEqual(self.call(), ATTENTION)
self.run.assert_called_once()
- self.assertEqual(logs.output, [
- 'WARNING:test:Pixel startup reproof requires attention '
- '(stage=unsafe-state reason=transition-recovery-required)'])
def test_timeout_releases_lock_without_retry(self):
self.run.side_effect = subprocess.TimeoutExpired('helper', 360)
- self.call()
+ self.assertEqual(self.call(), ('uncertain', 'stage=helper-failed'))
self.end.assert_called_once()
self.run.assert_called_once()