Skip to content

Self-improving AI Employees for everyone

Harden WebSocket recovery and phone agent lifecycle

trohitgd84e4a87fe75
diff --git a/cli/_common.py b/cli/_common.py
index ef7bdd5b..e22427b5 100644
--- a/cli/_common.py
+++ b/cli/_common.py
@@ -22,6 +22,7 @@ helper that actually needs the dep pulls it in.
from __future__ import annotations
+import os
from pathlib import Path
from typing import TYPE_CHECKING
@@ -65,16 +66,42 @@ def free_all_ports(cfg: Config) -> list[KillResult]:
"""
from cli.ports import kill_port
- return [kill_port(port) for port in cfg.all_ports]
+ grace = backend_shutdown_grace_seconds(cfg)
+ return [
+ kill_port(port, backend_graceful_timeout=grace)
+ for port in cfg.all_ports
+ ]
# Seconds uvicorn waits for open connections / in-flight handler tasks before
# running the lifespan shutdown. uvicorn's default is "forever": a browser
# WebSocket that dies with a TCP reset can leave its handler task lingering,
# and then the backend never reaches the teardown that reaps Temporal, the
-# Node sidecar and the WhatsApp bridge — the supervisor's 5 s grace expired
-# and tree-killed it instead. Same value the desktop shell passes.
+# Node sidecar and the WhatsApp bridge. This is only the connection-drain
+# phase, not the full backend shutdown budget. Same value the desktop shell passes.
UVICORN_GRACEFUL_SHUTDOWN_SECONDS = 5
+BACKEND_CLEANUP_GRACE_SECONDS = 30
+
+
+def backend_shutdown_grace_seconds(cfg: Config) -> float:
+ """Allow connection drain, Temporal teardown, then other cleanup.
+
+ ``preflight``/``load_config`` has already merged the env files into
+ ``os.environ``, matching the backend's Settings without importing its
+ dependencies. The pool, manager worker, and local Temporal runtime
+ currently stop sequentially, each with the configured grace window.
+ Other plugin/resource cleanup gets an additional bounded allowance.
+ """
+ temporal_grace = 0
+ if cfg.temporal_enabled:
+ temporal_grace = int(os.environ["TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS"])
+ if temporal_grace < 1:
+ raise ValueError("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS must be at least 1")
+ return float(
+ UVICORN_GRACEFUL_SHUTDOWN_SECONDS
+ + 3 * temporal_grace
+ + BACKEND_CLEANUP_GRACE_SECONDS
+ )
def build_backend_spec(
@@ -82,11 +109,13 @@ def build_backend_spec(
*,
host: str,
root: Path | None = None,
+ port: int | None = None,
+ env: dict[str, str] | None = None,
) -> "ServiceSpec":
"""The Python backend ``ServiceSpec`` shared by ``start`` and ``dev``.
Invokes the server venv's interpreter directly — the same idiom
- ``company serve`` documents (no resident ``uv run`` parent process,
+ ``company serve`` uses (no resident ``uv run`` parent process,
no runtime dependency on uv being on PATH; ``company build`` owns
sync semantics). ``host`` is the only differentiator (``start``
picks ``127.0.0.1`` on Windows/WSL vs ``0.0.0.0`` elsewhere;
@@ -99,6 +128,7 @@ def build_backend_spec(
from cli.platform_ import server_venv_python
from cli.supervisor import ServiceSpec
+ bind_port = cfg.backend_port if port is None else port
return ServiceSpec(
name="server",
argv=[
@@ -109,14 +139,16 @@ def build_backend_spec(
"--host",
host,
"--port",
- str(cfg.backend_port),
+ str(bind_port),
"--log-level",
"warning",
"--timeout-graceful-shutdown",
str(UVICORN_GRACEFUL_SHUTDOWN_SECONDS),
],
cwd=server_dir(root),
- ready_port=cfg.backend_port,
+ env=env or {},
+ ready_port=bind_port,
+ terminate_grace_seconds=backend_shutdown_grace_seconds(cfg),
)
diff --git a/cli/commands/serve.py b/cli/commands/serve.py
index 2c7a8205..472cc012 100644
--- a/cli/commands/serve.py
+++ b/cli/commands/serve.py
@@ -19,18 +19,16 @@ from __future__ import annotations
import asyncio
import os
-from pathlib import Path
import typer
-from cli._common import UVICORN_GRACEFUL_SHUTDOWN_SECONDS, preflight
+from cli._common import backend_shutdown_grace_seconds, build_backend_spec, preflight
from cli.buildenv import validate_build
from cli.colors import console
-from cli.platform_ import server_dir, server_venv_python
def serve_command(port: int | None = None) -> None:
- from cli.supervisor import Manager, ServiceSpec
+ from cli.supervisor import Manager
cfg, root = preflight()
os.environ.setdefault("PYTHONUTF8", "1")
@@ -43,7 +41,7 @@ def serve_command(port: int | None = None) -> None:
# Free the port we will bind (clears stale orphans; idempotent).
from cli.ports import kill_port
- kill_port(bind_port)
+ kill_port(bind_port, backend_graceful_timeout=backend_shutdown_grace_seconds(cfg))
console.print()
console.print(" [bold]OpenCompany[/] serve (single-port)")
@@ -51,25 +49,12 @@ def serve_command(port: int | None = None) -> None:
console.print()
specs = [
- ServiceSpec(
- name="server",
- argv=[
- str(server_venv_python(root)),
- "-m",
- "uvicorn",
- "main:app",
- "--host",
- "0.0.0.0",
- "--port",
- str(bind_port),
- "--log-level",
- "warning",
- "--timeout-graceful-shutdown",
- str(UVICORN_GRACEFUL_SHUTDOWN_SECONDS),
- ],
- cwd=server_dir(root),
+ build_backend_spec(
+ cfg,
+ host="0.0.0.0",
+ root=root,
+ port=bind_port,
env={"SERVE_STATIC_CLIENT": "1", "PORT": str(bind_port)},
- ready_port=bind_port,
),
]
diff --git a/cli/commands/stop.py b/cli/commands/stop.py
index 55040a49..9441dbd7 100644
--- a/cli/commands/stop.py
+++ b/cli/commands/stop.py
@@ -11,7 +11,7 @@ import time
import typer
-from cli._common import free_all_ports, preflight
+from cli._common import backend_shutdown_grace_seconds, free_all_ports, preflight
from cli.colors import console
from cli.config import load_dev_overrides
from cli.platform_ import platform_name
@@ -55,7 +55,9 @@ def stop_command() -> None:
f"[green]\\[OK][/] Temporal: Killed {len(temporal_pids)} process(es)"
)
- orphaned_pids = kill_orphaned_opencompany_processes(str(root))
+ orphaned_pids = kill_orphaned_opencompany_processes(
+ str(root), backend_graceful_timeout=backend_shutdown_grace_seconds(cfg)
+ )
if orphaned_pids:
time.sleep(0.2) # let DB locks release
console.print(
diff --git a/cli/ports.py b/cli/ports.py
index 58287df5..675182da 100644
--- a/cli/ports.py
+++ b/cli/ports.py
@@ -12,6 +12,7 @@ import sys
import subprocess
import time
from dataclasses import dataclass
+from pathlib import Path
import psutil
@@ -147,7 +148,27 @@ def kill_pid(pid: int, *, graceful_timeout: float = 3.0) -> bool:
return False
-def kill_port(port: int) -> KillResult:
+def _is_backend_process(pid: int, root_dir: str | None = None) -> bool:
+ """Identify this checkout's uvicorn before granting its longer drain.
+
+ A reserved port can belong to an unrelated program. Match both the app
+ invocation and working directory so its generic cleanup grace remains
+ short. Unknown/inaccessible processes retain the generic timeout too.
+ """
+ from cli.platform_ import server_dir
+
+ try:
+ proc = psutil.Process(pid)
+ argv = proc.cmdline()
+ return (
+ any(argv[i : i + 3] == ["-m", "uvicorn", "main:app"] for i in range(len(argv)))
+ and Path(proc.cwd()).resolve() == server_dir(Path(root_dir) if root_dir else None).resolve()
+ )
+ except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
+ return False
+
+
+def kill_port(port: int, *, backend_graceful_timeout: float | None = None) -> KillResult:
"""Kill anything listening on ``port`` and report whether the port is free.
Post-kill recheck sleeps ``_POST_KILL_RECHECK_DELAY`` because Windows
@@ -162,7 +183,10 @@ def kill_port(port: int) -> KillResult:
for pid in find_pids_by_port(port):
if pid == my_pid:
continue
- if kill_pid(pid):
+ grace = 3.0
+ if backend_graceful_timeout is not None and _is_backend_process(pid):
+ grace = backend_graceful_timeout
+ if kill_pid(pid, graceful_timeout=grace):
killed.append(pid)
if killed:
time.sleep(_POST_KILL_RECHECK_DELAY)
@@ -225,7 +249,8 @@ def _names_this_checkout(cmd: str, root_norm: str) -> bool:
def kill_orphaned_opencompany_processes(
- root_dir: str, *, exclude_substring: str | None = None
+ root_dir: str, *, exclude_substring: str | None = None,
+ backend_graceful_timeout: float | None = None,
) -> list[int]:
"""Kill stray python/bun processes whose cmdline references the project root.
@@ -250,7 +275,10 @@ def kill_orphaned_opencompany_processes(
continue
if proc.pid in safe_pids:
continue
- if kill_pid(proc.pid, graceful_timeout=2.0):
+ grace = 2.0
+ if backend_graceful_timeout is not None and _is_backend_process(proc.pid, root_dir):
+ grace = backend_graceful_timeout
+ if kill_pid(proc.pid, graceful_timeout=grace):
killed.append(proc.pid)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
diff --git a/cli/tests/test_backend_shutdown.py b/cli/tests/test_backend_shutdown.py
new file mode 100644
index 00000000..16fe2405
--- /dev/null
+++ b/cli/tests/test_backend_shutdown.py
@@ -0,0 +1,90 @@
+"""Backend shutdown deadlines must cover every sequential teardown phase."""
+
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from cli._common import (
+ BACKEND_CLEANUP_GRACE_SECONDS,
+ UVICORN_GRACEFUL_SHUTDOWN_SECONDS,
+ backend_shutdown_grace_seconds,
+ free_all_ports,
+)
+from cli.commands import dev, serve, start, stop
+from cli.config import load_config
+from cli.ports import KillResult
+
+
[email protected]("temporal_grace", [1, 30, 80])
+def test_start_dev_serve_share_configured_backend_shutdown_budget(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, temporal_grace: int
+):
+ cfg = replace(load_config(), temporal_enabled=True)
+ monkeypatch.setenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", str(temporal_grace))
+ expected = UVICORN_GRACEFUL_SHUTDOWN_SECONDS + 3 * temporal_grace + BACKEND_CLEANUP_GRACE_SECONDS
+ manager = MagicMock()
+ manager.run = AsyncMock(return_value=0)
+ with (
+ patch.object(serve, "preflight", return_value=(cfg, tmp_path)),
+ patch.object(serve, "validate_build"),
+ patch("cli.ports.kill_port") as kill_port,
+ patch("cli.supervisor.Manager", return_value=manager),
+ ):
+ serve.serve_command(port=8090)
+ production_spec = manager.add_all.call_args.args[0][0]
+ specs = [
+ start._build_specs(tmp_path, cfg)[0],
+ next(s for s in dev._build_specs(tmp_path, cfg, daemon=False, use_vite=True) if s.name == "server"),
+ production_spec,
+ ]
+ for spec in specs:
+ assert spec.terminate_grace_seconds == expected
+ drain_index = spec.argv.index("--timeout-graceful-shutdown")
+ assert spec.argv[drain_index + 1] == str(UVICORN_GRACEFUL_SHUTDOWN_SECONDS)
+ assert production_spec.ready_port == 8090
+ assert production_spec.argv[production_spec.argv.index("--port") + 1] == "8090"
+ assert production_spec.env == {"SERVE_STATIC_CLIENT": "1", "PORT": "8090"}
+ kill_port.assert_called_once_with(8090, backend_graceful_timeout=expected)
+ client = next(s for s in dev._build_specs(tmp_path, cfg, daemon=False, use_vite=True) if s.name == "client")
+ assert client.terminate_grace_seconds == 5.0
+
+
+def test_temporal_disabled_does_not_reserve_worker_grace(monkeypatch: pytest.MonkeyPatch):
+ cfg = replace(load_config(), temporal_enabled=False)
+ monkeypatch.delenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", raising=False)
+ assert backend_shutdown_grace_seconds(cfg) == UVICORN_GRACEFUL_SHUTDOWN_SECONDS + BACKEND_CLEANUP_GRACE_SECONDS
+
+
[email protected]("invalid", ["0", "-5", "bad"])
+def test_invalid_temporal_grace_fails_before_launch(monkeypatch: pytest.MonkeyPatch, invalid: str):
+ cfg = replace(load_config(), temporal_enabled=True)
+ monkeypatch.setenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", invalid)
+ with pytest.raises(ValueError):
+ backend_shutdown_grace_seconds(cfg)
+
+
+def test_reserved_port_cleanup_offers_backend_budget(monkeypatch: pytest.MonkeyPatch):
+ cfg = replace(load_config(), temporal_enabled=True)
+ monkeypatch.setenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", "40")
+ grace = backend_shutdown_grace_seconds(cfg)
+ with patch("cli.ports.kill_port", side_effect=lambda port, **kw: KillResult(port, [], True)) as kill_port:
+ results = free_all_ports(cfg)
+ assert [r.port for r in results] == cfg.all_ports
+ assert [c.args[0] for c in kill_port.call_args_list] == cfg.all_ports
+ assert all(c.kwargs == {"backend_graceful_timeout": grace} for c in kill_port.call_args_list)
+
+
+def test_stop_orphan_cleanup_preserves_backend_budget(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
+ cfg = replace(load_config(), temporal_enabled=True)
+ monkeypatch.setenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", "40")
+ with (
+ patch.object(stop, "preflight", return_value=(cfg, tmp_path)),
+ patch.object(stop, "load_dev_overrides"),
+ patch.object(stop, "free_all_ports", return_value=[KillResult(p, [], True) for p in cfg.all_ports]),
+ patch.object(stop, "kill_by_pattern", return_value=[]),
+ patch.object(stop, "kill_orphaned_opencompany_processes", return_value=[]) as kill_orphans,
+ ):
+ stop.stop_command()
+ kill_orphans.assert_called_once_with(str(tmp_path), backend_graceful_timeout=backend_shutdown_grace_seconds(cfg))
diff --git a/cli/tests/test_ports.py b/cli/tests/test_ports.py
index 29bcb7ad..b381a9af 100644
--- a/cli/tests/test_ports.py
+++ b/cli/tests/test_ports.py
@@ -5,6 +5,7 @@ from __future__ import annotations
from unittest.mock import MagicMock, patch
import psutil
+import pytest
from cli import ports
@@ -16,7 +17,7 @@ def test_kill_pid_no_such_process_returns_false():
def test_kill_pid_terminates_then_waits():
proc = MagicMock()
- with patch.object(psutil, "Process", return_value=proc):
+ with patch.object(psutil, "Process", return_value=proc), patch.object(ports.sys, "platform", "linux"):
assert ports.kill_pid(123) is True
proc.terminate.assert_called_once()
proc.wait.assert_called_once()
@@ -25,12 +26,64 @@ def test_kill_pid_terminates_then_waits():
def test_kill_pid_force_kills_on_timeout():
proc = MagicMock()
proc.wait.side_effect = psutil.TimeoutExpired(seconds=1)
- with patch.object(psutil, "Process", return_value=proc):
+ with patch.object(psutil, "Process", return_value=proc), patch.object(ports.sys, "platform", "linux"):
assert ports.kill_pid(123) is True
proc.terminate.assert_called_once()
proc.kill.assert_called_once()
+def test_windows_kill_pid_delivers_graceful_signal_before_waiting():
+ proc = MagicMock()
+ with (
+ patch.object(psutil, "Process", return_value=proc),
+ patch.object(ports.sys, "platform", "win32"),
+ patch.object(ports.signal, "CTRL_BREAK_EVENT", 1, create=True),
+ patch.object(ports.os, "kill") as signal_process,
+ ):
+ assert ports.kill_pid(123, graceful_timeout=125) is True
+ signal_process.assert_called_once_with(123, 1)
+ proc.wait.assert_called_once_with(timeout=125)
+ proc.terminate.assert_not_called()
+ proc.kill.assert_not_called()
+
+
+def test_windows_kill_pid_falls_back_if_control_signal_unavailable():
+ proc = MagicMock()
+ with (
+ patch.object(psutil, "Process", return_value=proc),
+ patch.object(ports.sys, "platform", "win32"),
+ patch.object(ports.signal, "CTRL_BREAK_EVENT", 1, create=True),
+ patch.object(ports.os, "kill", side_effect=OSError("no process group")),
+ ):
+ assert ports.kill_pid(123) is True
+ proc.terminate.assert_called_once()
+ proc.wait.assert_called_once_with(timeout=3.0)
+
+
[email protected]("is_backend,expected", [(True, 125.0), (False, 3.0)])
+def test_port_cleanup_only_grants_long_grace_to_known_backend(is_backend, expected):
+ with (
+ patch.object(ports, "find_pids_by_port", side_effect=[{123}, set()]),
+ patch.object(ports.os, "getpid", return_value=456),
+ patch.object(ports, "_is_backend_process", return_value=is_backend),
+ patch.object(ports, "kill_pid", return_value=True) as kill_pid,
+ patch.object(ports.time, "sleep"),
+ ):
+ result = ports.kill_port(5678, backend_graceful_timeout=125.0)
+ kill_pid.assert_called_once_with(123, graceful_timeout=expected)
+ assert result.port_free
+ assert result.killed_pids == [123]
+
+
[email protected]("app,same_checkout,expected", [("main:app", True, True), ("other:app", True, False), ("main:app", False, False)])
+def test_backend_identification_requires_app_and_checkout(tmp_path, app, same_checkout, expected):
+ proc = MagicMock()
+ proc.cmdline.return_value = ["python", "-m", "uvicorn", app, "--port", "5678"]
+ proc.cwd.return_value = str(tmp_path / ("server" if same_checkout else "other/server"))
+ with patch.object(psutil, "Process", return_value=proc):
+ assert ports._is_backend_process(123, str(tmp_path)) is expected
+
+
def test_find_pids_by_port_counts_only_listeners():
"""A dead server's half-closed sockets must not make a bindable port
look occupied (the 'Port still in use' false positive on stop)."""
diff --git a/client/devProxyDiagnostics.js b/client/devProxyDiagnostics.js
new file mode 100644
index 00000000..46248250
--- /dev/null
+++ b/client/devProxyDiagnostics.js
@@ -0,0 +1,105 @@
+const NOTICE_INTERVAL_MS = 10_000
+const MAX_NOTICE_KEYS = 128
+const PROXY_ERROR = /\b(?:http|ws) proxy (?:socket )?error:/i
+const WS_PROXY_ERROR = /\bws proxy (?:socket )?error:/i
+const TRANSIENT_CODES = /\b(ECONNREFUSED|ECONNABORTED|ECONNRESET)\b/
+
+// Requests can contain credentials in their query string. Only retain a bounded,
+// printable pathname in diagnostics, never headers, query parameters or bodies.
+const safePath = (url) => {
+ try {
+ return new URL(url || '/', 'http://vite.invalid').pathname
+ .replace(/[\u0000-\u001f\u007f-\u009f]/g, '')
+ .slice(0, 200)
+ } catch {
+ return '<invalid path>'
+ }
+}
+
+export const createDevProxyDiagnostics = (logger, { now = Date.now } = {}) => {
+ const logError = logger.error.bind(logger)
+ const logInfo = logger.info.bind(logger)
+ const logWarn = logger.warn.bind(logger)
+ const errorPaths = new WeakMap()
+ const seenErrors = new WeakSet()
+ const notices = new Map()
+
+ const notice = (key, message, log) => {
+ const time = now()
+ const previous = notices.get(key)
+ if (previous && time >= previous.at && time - previous.at < NOTICE_INTERVAL_MS) {
+ previous.suppressed += 1
+ return
+ }
+ const suffix = previous?.suppressed
+ ? ` (${previous.suppressed} similar failures since the last notice)`
+ : ''
+ // Long-lived dev servers must not retain an unlimited set of request paths.
+ if (!previous && notices.size >= MAX_NOTICE_KEYS) {
+ notices.delete(notices.keys().next().value)
+ }
+ notices.set(key, { at: time, suppressed: 0 })
+ log(message + suffix, { timestamp: true })
+ }
+
+ // Vite installs its own proxy listeners after configure(). customLogger is
+ // the supported way to avoid reporting the same socket error twice.
+ // https://vite.dev/config/shared-options.html#customlogger
+ logger.error = (message, options) => {
+ const text = typeof message === 'string' ? message : String(message ?? '')
+ const error = options?.error
+ const code = error?.code || text.match(TRANSIENT_CODES)?.[1]
+ const isRefused = PROXY_ERROR.test(text) && code === 'ECONNREFUSED'
+ const isDisconnect = WS_PROXY_ERROR.test(text)
+ && (code === 'ECONNABORTED' || code === 'ECONNRESET')
+
+ if (!isRefused && !isDisconnect) {
+ logError(message, options)
+ return
+ }
+
+ const hasErrorObject = error !== null && typeof error === 'object'
+ if (hasErrorObject) {
+ if (seenErrors.has(error)) return
+ seenErrors.add(error)
+ }
+
+ if (isRefused) {
+ notice('backend-unavailable',
+ 'backend unavailable (ECONNREFUSED); WebSocket reconnects will retry', logInfo)
+ return
+ }
+
+ const path = (hasErrorObject && errorPaths.get(error)) || '<unknown path>'
+ notice(`transport:${code}:${path}`,
+ `WebSocket proxy transport error for ${path}: ${code}`, logWarn)
+ }
+
+ const rememberErrorPath = (error, path) => {
+ if (error !== null && typeof error === 'object') errorPaths.set(error, path)
+ }
+
+ // Attach before Vite's listeners so its logger receives the request context.
+ // The ClientRequest response event is the non-upgrade HTTP response path;
+ // successful WebSocket handshakes instead emit upgrade.
+ // https://vite.dev/config/server-options.html#server-proxy
+ const configureWebSocketProxy = (proxy) => {
+ proxy.on('error', (error, request) => {
+ rememberErrorPath(error, safePath(request?.url))
+ })
+ proxy.on('proxyReqWs', (proxyRequest, request, socket) => {
+ const path = safePath(request?.url)
+ socket.on('error', (error) => rememberErrorPath(error, path))
+ proxyRequest.once('response', (response) => {
+ if (response.statusCode === 101) return
+ const status = Number.isInteger(response.statusCode)
+ ? response.statusCode
+ : 'unknown'
+ notice(`upgrade:${status}:${path}`,
+ `WebSocket upgrade failed for ${path}: backend returned HTTP ${status}`, logWarn)
+ })
+ })
+ }
+
+ return { logger, configureWebSocketProxy }
+}
diff --git a/client/devProxyDiagnostics.test.js b/client/devProxyDiagnostics.test.js
new file mode 100644
index 00000000..7678b367
--- /dev/null
+++ b/client/devProxyDiagnostics.test.js
@@ -0,0 +1,133 @@
+import assert from 'node:assert/strict'
+import { EventEmitter } from 'node:events'
+import test from 'node:test'
+import { createDevProxyDiagnostics } from './devProxyDiagnostics.js'
+
+const harness = () => {
+ const entries = []
+ let time = 0
+ const logger = Object.fromEntries(['error', 'warn', 'info'].map((level) => [
+ level, (message, options) => entries.push({ level, message, options }),
+ ]))
+ const diagnostics = createDevProxyDiagnostics(logger, { now: () => time })
+ const proxy = new EventEmitter()
+ diagnostics.configureWebSocketProxy(proxy)
+ const request = (url = '/ws') => {
+ const upstream = new EventEmitter()
+ const socket = new EventEmitter()
+ proxy.emit('proxyReqWs', upstream, { url }, socket)
+ return { upstream, socket }
+ }
+ return { ...diagnostics, entries, proxy, request, advance: (ms) => { time += ms } }
+}
+
+test('reports the failed HTTP handshake without leaking credentials or response content', () => {
+ const { request, entries } = harness()
+ request('http://user:password@localhost/ws?token=secret#private').upstream.emit('response', {
+ statusCode: 403,
+ statusMessage: 'sensitive upstream message',
+ headers: { 'set-cookie': 'session=private' },
+ })
+ assert.equal(entries.length, 1)
+ assert.equal(entries[0].level, 'warn')
+ assert.equal(entries[0].message, 'WebSocket upgrade failed for /ws: backend returned HTTP 403')
+})
+
+test('successful upgrades are quiet while distinct failed statuses remain visible', () => {
+ const { request, entries } = harness()
+ request().upstream.emit('upgrade', { statusCode: 101 })
+ request().upstream.emit('response', { statusCode: 101 })
+ assert.equal(entries.length, 0)
+ request().upstream.emit('response', { statusCode: 403 })
+ request().upstream.emit('response', { statusCode: 502 })
+ assert.equal(entries.length, 2)
+ assert.match(entries[0].message, /HTTP 403/)
+ assert.match(entries[1].message, /HTTP 502/)
+})
+
+test('duplicate socket and proxy notifications produce one concise error with a safe path', () => {
+ const { request, proxy, logger, entries } = harness()
+ const { socket } = request('/ws?token=secret')
+ const error = Object.assign(new Error('write ECONNABORTED'), { code: 'ECONNABORTED' })
+ socket.emit('error', error)
+ logger.error(`ws proxy socket error:\n${error.stack}`, { error })
+ proxy.emit('error', error, { url: '/ws?token=secret' })
+ logger.error(`ws proxy error:\n${error.stack}`, { error })
+ assert.equal(entries.length, 1)
+ assert.equal(entries[0].message, 'WebSocket proxy transport error for /ws: ECONNABORTED')
+ assert.equal(entries[0].level, 'warn')
+})
+
+test('repeated transport failures are rate limited and counted without counting duplicate reports', () => {
+ const { request, logger, entries, advance } = harness()
+ const fail = () => {
+ const { socket } = request()
+ const error = Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })
+ socket.emit('error', error)
+ logger.error(`ws proxy socket error:\n${error.stack}`, { error })
+ logger.error(`ws proxy error:\n${error.stack}`, { error })
+ }
+ fail()
+ fail()
+ fail()
+ assert.equal(entries.length, 1)
+ advance(10_000)
+ fail()
+ assert.equal(entries.length, 2)
+ assert.match(entries[1].message, /2 similar failures since the last notice/)
+})
+
+test('an aborted socket cannot hide the upstream rejection or flood repeated handshake notices', () => {
+ const { request, logger, entries, advance } = harness()
+ const { upstream, socket } = request()
+ upstream.emit('response', { statusCode: 401 })
+ const error = Object.assign(new Error('write ECONNRESET'), { code: 'ECONNRESET' })
+ socket.emit('error', error)
+ logger.error(`ws proxy socket error:\n${error.stack}`, { error })
+ request().upstream.emit('response', { statusCode: 401 })
+ assert.equal(entries.length, 2)
+ assert.match(entries[0].message, /HTTP 401/)
+ advance(10_000)
+ request().upstream.emit('response', { statusCode: 401 })
+ assert.equal(entries.length, 3)
+ assert.match(entries[2].message, /HTTP 401 \(1 similar failures since the last notice\)/)
+})
+
+test('backend startup connection refusals are concise and rate limited from the first failure', () => {
+ const { logger, entries, advance } = harness()
+ // Node may report ECONNREFUSED only in an AggregateError stack.
+ logger.error('ws proxy error:\nAggregateError [ECONNREFUSED]', { error: new AggregateError([]) })
+ logger.error('http proxy error: /api?token=secret\nAggregateError [ECONNREFUSED]')
+ assert.equal(entries.length, 1)
+ assert.equal(entries[0].level, 'info')
+ assert.match(entries[0].message, /backend unavailable \(ECONNREFUSED\)/)
+ assert.doesNotMatch(entries[0].message, /secret/)
+ advance(10_000)
+ logger.error('ws proxy error:\nAggregateError [ECONNREFUSED]')
+ assert.equal(entries.length, 2)
+})
+
+test('unexpected errors and HTTP resets retain their full diagnostic evidence', () => {
+ const { logger, entries } = harness()
+ for (const message of [
+ 'ws proxy error:\nError: certificate has expired',
+ 'http proxy error: /api\nError: ECONNRESET',
+ 'unrelated plugin failed with ECONNREFUSED',
+ ]) {
+ const options = { error: new Error(message) }
+ logger.error(message, options)
+ assert.deepEqual(entries.at(-1), { level: 'error', message, options })
+ }
+ assert.equal(entries.length, 3)
+})
+
+test('untrusted malformed and long paths produce bounded, printable diagnostics', () => {
+ const { request, entries } = harness()
+ request('http://[').upstream.emit('response', {})
+ assert.equal(entries[0].message,
+ 'WebSocket upgrade failed for <invalid path>: backend returned HTTP unknown')
+ request('/ws/\u001b[31m' + 'x'.repeat(1000) + '?token=secret')
+ .upstream.emit('response', { statusCode: 404 })
+ assert.ok(entries[1].message.length < 300)
+ assert.doesNotMatch(entries[1].message, /[\u0000-\u001f\u007f-\u009f]|secret/)
+})
diff --git a/client/src/components/mobile/MobileWorkspace.tsx b/client/src/components/mobile/MobileWorkspace.tsx
index 8a05b416..934c1e77 100644
--- a/client/src/components/mobile/MobileWorkspace.tsx
+++ b/client/src/components/mobile/MobileWorkspace.tsx
@@ -5,6 +5,7 @@ import type { PointerEvent } from 'react';
import { ArrowLeft, Home, Play, RotateCw, Smartphone, Square } from 'lucide-react';
import { Button } from '@/components/ui/button';
import { buildApiUrl } from '@/config/api';
+import { useWebSocketActions } from '@/contexts/WebSocketContext';
import { mobilePath, mobilePoint, mobileRequest, type Doctor, type Geometry, type Invocation, type MobileStatus } from './api';
const fieldClass = 'min-w-0 rounded border border-border-default bg-bg-input px-2 py-1.5 text-sm text-fg-default outline-none focus-visible:ring-2 focus-visible:ring-ring';
@@ -15,15 +16,29 @@ const describe = (value: unknown): string => typeof value === 'string' ? value :
export default function MobileWorkspace({ workflowId, nodes, visible = true }: {
workflowId?: string | null; nodes: { node_id: string; label: string }[]; visible?: boolean;
}) {
+ const { addEventListener } = useWebSocketActions();
const [selected, setSelected] = useState('');
+ const [sessionVersion, setSessionVersion] = useState(0);
+ const resetVersion = useRef(0);
+ const isCurrentSession = useCallback(() => resetVersion.current === sessionVersion, [sessionVersion]);
const nodeId = nodes.some((node) => node.node_id === selected) ? selected : nodes[0]?.node_id;
+ useEffect(() => addEventListener('workflow_runtime_reset', (event) => {
+ if (!workflowId || event?.workflow_id !== workflowId) return;
+ // Invalidate pending responses immediately, before React unmounts the old
+ // session. A delayed task submission must not restore its recovery ID.
+ resetVersion.current += 1;
+ for (const node of nodes) {
+ try { sessionStorage.removeItem(`mobile-task:${workflowId}:${node.node_id}`); } catch { /* optional recovery */ }
+ }
+ setSessionVersion(resetVersion.current);
+ }), [addEventListener, workflowId, nodes]);
if (!workflowId || workflowId === 'new') return <Empty message="Save this workflow before opening its mobile workspace." />;
if (!nodeId) return <Empty message="Add a Mobile Agent or Android tool to this workflow to use its phone here." />;
return <div className="flex min-h-0 min-w-0 flex-1 flex-col gap-2">
{nodes.length > 1 && <select className={`${fieldClass} mx-2 mt-2 shrink-0`} aria-label="Mobile agent" value={nodeId} onChange={(event) => setSelected(event.target.value)}>
{nodes.map((node) => <option key={node.node_id} value={node.node_id}>{node.label}</option>)}
</select>}
- <MobileSession key={`${workflowId}:${nodeId}`} workflowId={workflowId} nodeId={nodeId} visible={visible} />
+ <MobileSession key={`${workflowId}:${nodeId}:${sessionVersion}`} workflowId={workflowId} nodeId={nodeId} visible={visible} isCurrentSession={isCurrentSession} />
</div>;
}
@@ -31,7 +46,7 @@ function Empty({ message }: { message: string }) {
return <div className="m-auto flex max-w-80 flex-col items-center gap-3 p-6 text-center text-sm text-fg-muted"><Smartphone aria-hidden className="size-6" />{message}</div>;
}
-function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; nodeId: string; visible: boolean }) {
+function MobileSession({ workflowId, nodeId, visible, isCurrentSession }: { workflowId: string; nodeId: string; visible: boolean; isCurrentSession: () => boolean }) {
const path = mobilePath(workflowId, nodeId);
const [viewerId] = useState(() => crypto.randomUUID());
const [status, setStatus] = useState<MobileStatus | null>(null);
@@ -66,8 +81,8 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
const running = status?.running === true;
const refresh = useCallback(async (signal?: AbortSignal) => {
const next = await mobileRequest<MobileStatus>(`${path}/status`, undefined, signal);
- if (active.current) { setStatus(next); setConnectionError(''); }
- }, [path]);
+ if (active.current && isCurrentSession() && !signal?.aborted) { setStatus(next); setConnectionError(''); }
+ }, [path, isCurrentSession]);
useEffect(() => {
active.current = true;
@@ -81,15 +96,15 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
let timer: ReturnType<typeof setTimeout>;
const poll = async () => {
try { await refresh(controller.signal); }
- catch { if (!controller.signal.aborted && active.current) setConnectionError('Connection to the phone was lost. Retrying automatically…'); }
- if (!controller.signal.aborted) timer = setTimeout(() => void poll(), 2000);
+ catch { if (!controller.signal.aborted && active.current && isCurrentSession()) setConnectionError('Connection to the phone was lost. Retrying automatically…'); }
+ if (!controller.signal.aborted && isCurrentSession()) timer = setTimeout(() => void poll(), 2000);
};
void poll();
void mobileRequest<Doctor>(`${path}/doctor`, undefined, controller.signal).then((value) => {
- if (!controller.signal.aborted) setDoctor(value);
- }).catch((cause) => { if (!controller.signal.aborted) setError(cause.message); });
+ if (!controller.signal.aborted && isCurrentSession()) setDoctor(value);
+ }).catch((cause) => { if (!controller.signal.aborted && isCurrentSession()) setError(cause.message); });
return () => { controller.abort(); clearTimeout(timer); };
- }, [visible, pageVisible, refresh, path]);
+ }, [visible, pageVisible, refresh, path, isCurrentSession]);
useEffect(() => {
if (!submission || !visible || !pageVisible) return;
const controller = new AbortController();
@@ -97,15 +112,15 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
const poll = async () => {
try {
const next = await mobileRequest<Invocation>(`${path}/tasks/${encodeURIComponent(submission)}`, undefined, controller.signal);
- if (!controller.signal.aborted) setTask(next);
- if (!finished.has(taskState(next)) && !controller.signal.aborted) timer = setTimeout(() => void poll(), 1500);
+ if (!controller.signal.aborted && isCurrentSession()) setTask(next);
+ if (!finished.has(taskState(next)) && !controller.signal.aborted && isCurrentSession()) timer = setTimeout(() => void poll(), 1500);
} catch (cause) {
- if (!controller.signal.aborted) { setError(cause instanceof Error ? cause.message : 'Could not read this task.'); timer = setTimeout(() => void poll(), 4000); }
+ if (!controller.signal.aborted && isCurrentSession()) { setError(cause instanceof Error ? cause.message : 'Could not read this task.'); timer = setTimeout(() => void poll(), 4000); }
}
};
void poll();
return () => { controller.abort(); clearTimeout(timer); };
- }, [submission, visible, pageVisible, path]);
+ }, [submission, visible, pageVisible, path, isCurrentSession]);
useEffect(() => {
if (!running || !visible || !pageVisible || !canvas.current) return;
let disposed = false;
@@ -132,16 +147,17 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
return () => {
const releasedEpoch = leaseEpoch.current;
leaseEpoch.current = null;
- if (releasedEpoch !== null) void mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch: releasedEpoch, resume: false }).catch(() => {});
+ // Reset cancels AI work but preserves a person's server-owned lease.
+ if (releasedEpoch !== null && isCurrentSession()) void mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch: releasedEpoch, resume: false }).catch(() => {});
};
- }, [visible, pageVisible, path, viewerId]);
+ }, [visible, pageVisible, path, viewerId, isCurrentSession]);
const perform = async (label: string, action: () => Promise<void>) => {
if (operation.current) return;
operation.current = true; setBusy(label); setError('');
- try { await action(); if (active.current && label !== 'input') await refresh(); }
- catch (cause) { if (active.current) setError(cause instanceof Error ? cause.message : 'That action failed.'); }
- finally { operation.current = false; if (active.current) setBusy(''); }
+ try { await action(); if (active.current && isCurrentSession() && label !== 'input') await refresh(); }
+ catch (cause) { if (active.current && isCurrentSession()) setError(cause instanceof Error ? cause.message : 'That action failed.'); }
+ finally { operation.current = false; if (active.current && isCurrentSession()) setBusy(''); }
};
const input = async (action: string, parameters: Record<string, unknown>, geometry = status?.geometry) => {
if (!held || epoch === null || !geometry) return;
@@ -176,7 +192,7 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
const attempt = pendingSubmission.current?.prompt === clean ? pendingSubmission.current : { id: crypto.randomUUID(), prompt: clean };
pendingSubmission.current = attempt;
const response = await mobileRequest<Invocation>(`${path}/tasks`, { submission_id: attempt.id, prompt: clean });
- if (!active.current) return;
+ if (!active.current || !isCurrentSession()) return;
setSubmission(attempt.id); setTask(response); setPrompt(''); pendingSubmission.current = null;
try { sessionStorage.setItem(`mobile-task:${workflowId}:${nodeId}`, attempt.id); } catch { /* optional recovery */ }
});
@@ -191,13 +207,13 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
return <FullView label="Phone" toolbar={<>
<Smartphone aria-hidden className="size-4 shrink-0 text-fg-muted" />
<span role="status" className="min-w-0 flex-1 truncate text-xs text-fg-muted">{setupActive ? 'Setting up' : status?.starting ? 'Starting phone…' : busy ? `${busy}…` : running ? (held ? 'You’re using the phone' : status.active?.status === 'running' ? 'AI is working' : status.control_state === 'recovering' ? 'Reconnecting…' : 'Ready') : status ? 'Phone is off' : 'Connecting…'}</span>
- {running && status?.active?.run_id != null && <span role="status" className="text-xs text-fg-muted">{String(status.active.phase || status.active.status || 'Working')} · Step {Number(status.active.steps || 0)} / {Number(status.active.max_steps || 0)}</span>}
+ {running && status?.active?.run_id != null && <span role="status" className="text-xs text-fg-muted">{String(status.active.phase || status.active.status || 'Working')} · Engine step {Number(status.active.steps || 0)} / {Number(status.active.max_steps || 0)}</span>}
{!running && powerControl}
{running && (held ? <>
<Button size="sm" onClick={() => void perform('releasing', async () => { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch, resume: true }); leaseEpoch.current = null; setEpoch(null); })} disabled={!!busy}>Let AI continue</Button>
<Button size="sm" variant="outline" onClick={() => void perform('releasing', async () => { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch, resume: false }); leaseEpoch.current = null; setEpoch(null); })} disabled={!!busy}>Finish using phone</Button>
</> : <Button size="sm" variant="outline" disabled={!!busy} onClick={() => void perform('taking control', async () => { const claim = await mobileRequest<{ epoch: number; owner: string }>(`${path}/takeover`, { viewer_id: viewerId });
- if (!active.current || !viewActive.current) { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch: claim.epoch, resume: false }); return; }
+ if (!active.current || !isCurrentSession() || !viewActive.current) { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch: claim.epoch, resume: false }); return; }
leaseEpoch.current = claim.epoch; setEpoch(claim.epoch); setLeaseOwner(claim.owner); })}>Use phone</Button>)}
</>} controls={<>
<div className="flex flex-wrap items-center gap-2">
diff --git a/client/src/components/mobile/PhoneActivity.tsx b/client/src/components/mobile/PhoneActivity.tsx
index e88467b3..50fb819e 100644
--- a/client/src/components/mobile/PhoneActivity.tsx
+++ b/client/src/components/mobile/PhoneActivity.tsx
@@ -1,9 +1,18 @@
import { useEffect, useState } from 'react';
const seconds = (from: unknown, to: number) => typeof from === 'number' ? Math.max(0, Math.floor(to - from)) : 0;
+const duration = (milliseconds: number) => `${(Math.max(0, milliseconds) / 1000).toFixed(1)}s`;
+
+type PlanItem = { id: string; description: string; status: 'pending' | 'not_started' | 'success' | 'failure'; reason?: string };
+type AgentActivity = {
+ at: number; step: number; kind: 'plan' | 'action'; message: string; detail?: string; outcome?: string;
+ state?: 'started' | 'completed' | 'failed' | 'returned'; duration_ms?: number; action_id?: string;
+};
+const planLabels = { pending: 'In progress', not_started: 'Not started', success: 'Done', failure: 'Failed' };
+const actionLabels = { started: 'In progress', completed: 'Completed', failed: 'Failed', returned: 'Unconfirmed' };
export function PhoneActivity({ task }: { task?: Record<string, unknown> | null }) {
- const [now, setNow] = useState(Date.now() / 1000);
+ const [now, setNow] = useState(() => Date.now() / 1000);
const ended = typeof task?.finished_at === 'number';
useEffect(() => {
if (!task?.run_id || ended) return;
@@ -14,16 +23,53 @@ export function PhoneActivity({ task }: { task?: Record<string, unknown> | null
const end = ended ? Number(task.finished_at) : now;
const quiet = seconds(task.updated_at, end);
const events = Array.isArray(task.activity) ? task.activity as { at: number; message: string; step: number }[] : [];
- return <details open className="max-h-44 shrink-0 overflow-y-auto border-t border-border-default text-xs group-data-[full-view=true]/fullview:hidden" aria-label="Android activity">
+ const plan = Array.isArray(task.plan) ? task.plan as PlanItem[] : [];
+ const goal = typeof task.current_goal === 'string' ? task.current_goal : '';
+ const reportedActions = Array.isArray(task.agent_activity) ? task.agent_activity as AgentActivity[] : [];
+ const currentAction = task.current_action && typeof task.current_action === 'object' ? task.current_action as AgentActivity : null;
+ const actions = reportedActions.length ? reportedActions : currentAction ? [currentAction] : [];
+ return <details open className="max-h-64 shrink-0 overflow-y-auto border-t border-border-default text-xs group-data-[full-view=true]/fullview:hidden" aria-label="Android activity">
<summary className="cursor-pointer p-2 font-medium">AI activity · {ended ? String(task.status) : String(task.phase || 'Preparing phone')} · {seconds(task.started_at, end)}s total</summary>
- <div className="space-y-1 px-2 pb-2">
- <p className="m-0 text-fg-muted">{String(task.provider || '')} {String(task.model || '')} · Step {Number(task.steps || 0)} / {Number(task.max_steps || 0)}</p>
+ <div className="space-y-2 px-2 pb-2">
+ {goal && <section aria-label="Current Android subtask" className="rounded border border-border-default bg-bg-panel p-2">
+ <p className="m-0 text-fg-muted">{ended ? 'Last subtask' : 'Current subtask'}</p>
+ <p className="m-0 break-words font-medium">{goal}</p>
+ </section>}
+ <p className="m-0 text-fg-muted" title="An engine step may include several phone actions.">Engine step {Number(task.steps || 0)} / {Number(task.max_steps || 0)}</p>
{!ended && <p className="m-0" role="status">Current phase: {seconds(task.phase_started_at, now)}s · Last activity: {quiet}s ago</p>}
{!ended && quiet >= 30 && <p className="m-0 text-amber-600">No new activity for {quiet}s. The last reported phase is shown above. This may be a slow request; it does not confirm the task is stuck. Use “Use phone” to take control.</p>}
{ended && task.error_code != null && <p className="m-0 text-destructive">Stopped during {String(task.phase || 'startup')} · {String(task.error_code)}</p>}
- <ol className="m-0 list-none space-y-1 p-0" aria-label="Recent Android steps">
- {[...events].reverse().map((item, index) => <li key={`${item.at}-${index}`}><span className="text-fg-muted">+{seconds(task.started_at, item.at)}s · </span>{item.message}</li>)}
- </ol>
+ {!!plan.length && <details aria-label="Android task plan">
+ <summary className="cursor-pointer font-medium">Plan · {plan.filter((item) => item.status === 'success').length} / {plan.length} done</summary>
+ <ol className="m-0 list-none space-y-1 p-0">
+ {plan.map((item) => <li key={item.id} className="break-words">
+ <span className={item.status === 'failure' ? 'text-destructive' : 'text-fg-muted'}>{planLabels[item.status] || item.status} · </span>{item.description}
+ {item.reason && <p className="m-0 text-fg-muted">{item.reason}</p>}
+ </li>)}
+ </ol>
+ </details>}
+ {!!actions.length && <ol className="m-0 list-none space-y-2 p-0" aria-label="Recent Android actions">
+ {[...actions].reverse().map((item, index) => {
+ const elapsed = typeof item.duration_ms === 'number' ? item.duration_ms
+ : item.state === 'started' && !ended ? Math.max(0, end - item.at) * 1000 : null;
+ return <li key={item.action_id || `${item.kind}-${item.at}-${index}`} className="break-words">
+ <p className="m-0 text-fg-muted">+{seconds(task.started_at, item.at)}s · Engine step {item.step}
+ {item.state && <> · <span className={item.state === 'failed' ? 'text-destructive' : undefined}>{ended && item.state === 'started' ? 'Started' : actionLabels[item.state]}</span></>}
+ {elapsed !== null && <> · {duration(elapsed)}</>}
+ </p>
+ <p className="m-0 font-medium">{item.message}</p>
+ {item.detail && <p className="m-0 text-fg-muted">{item.detail}</p>}
+ {item.outcome && <p className={`m-0 ${item.state === 'failed' ? 'text-destructive' : 'text-fg-muted'}`}>Result: {item.outcome}</p>}
+ </li>;
+ })}
+ </ol>}
+ <details aria-label="Android technical activity">
+ <summary className="cursor-pointer text-fg-muted">Technical activity</summary>
+ <p className="my-1 text-fg-muted">{String(task.provider || '')} {String(task.model || '')}</p>
+ <ol className="m-0 list-none space-y-1 p-0" aria-label="Recent Android technical events">
+ {[...events].reverse().map((item, index) => <li key={`${item.at}-${index}`} className="break-words"><span className="text-fg-muted">+{seconds(task.started_at, item.at)}s · Engine step {item.step} · </span>{item.message}</li>)}
+ </ol>
+ </details>
</div>
</details>;
}
diff --git a/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx b/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx
index 5d7c4aeb..306d13e2 100644
--- a/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx
+++ b/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx
@@ -1,20 +1,35 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
-import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import MobileWorkspace from '../MobileWorkspace';
import { mobilePoint, mobileRequest } from '../api';
-const { connectVideo } = vi.hoisted(() => ({ connectVideo: vi.fn(() => () => {}) }));
+const { connectVideo, listeners, addEventListener } = vi.hoisted(() => {
+ const listeners = new Map<string, Set<(data: unknown) => void>>();
+ return {
+ connectVideo: vi.fn(() => () => {}), listeners,
+ addEventListener: (type: string, handler: (data: unknown) => void) => {
+ const handlers = listeners.get(type) ?? new Set();
+ handlers.add(handler); listeners.set(type, handlers);
+ return () => { handlers.delete(handler); };
+ },
+ };
+});
vi.mock('../video', () => ({ connectMobileVideo: connectVideo }));
+vi.mock('@/contexts/WebSocketContext', () => ({ useWebSocketActions: () => ({ addEventListener }) }));
const fetchMock = vi.fn();
const nodes = [{ node_id: 'flow:mobile_agent:1', label: 'Phone assistant' }];
const ready = { supported: true, adb: true, emulator: true, image: true, engine: true, video: true, acceleration: 'WHPX is installed and usable.' };
let snapshot: Record<string, unknown>;
const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } });
+const reset = (workflowId: string) => act(() => {
+ for (const handler of listeners.get('workflow_runtime_reset') ?? []) handler({ workflow_id: workflowId });
+});
beforeEach(() => {
sessionStorage.clear();
connectVideo.mockClear();
+ listeners.clear();
snapshot = { running: false, control_state: 'idle', controller: null, epoch: 0, geometry: { width: 1080, height: 1920, rotation: 0 } };
vi.stubGlobal('fetch', fetchMock);
fetchMock.mockReset().mockImplementation(async (url: string) => json(url.endsWith('/doctor') ? ready : snapshot));
@@ -22,11 +37,115 @@ beforeEach(() => {
afterEach(() => { cleanup(); vi.unstubAllGlobals(); });
describe('Mobile Workspace', () => {
+ it('clears the old run, task recovery IDs and lease after a matching workflow reset', async () => {
+ sessionStorage.setItem('mobile-task:flow:flow:mobile_agent:1', 'old-submission');
+ sessionStorage.setItem('mobile-task:flow:flow:android:2', 'other-phone-submission');
+ sessionStorage.setItem('mobile-task:other:other:mobile_agent:1', 'unrelated-submission');
+ snapshot = { ...snapshot, running: true, active: { run_id: 'old-run', phase: 'Using phone', current_goal: 'Old phone task' } };
+ fetchMock.mockImplementation(async (url: string) => {
+ if (url.endsWith('/takeover')) {
+ snapshot = { ...snapshot, controller: 'viewer:server-hash', control_state: 'human', epoch: 7 };
+ return json({ owner: 'viewer:server-hash', epoch: 7 });
+ }
+ if (url.includes('/tasks/')) return json({ status: 'running' });
+ return json(url.endsWith('/doctor') ? ready : snapshot);
+ });
+ render(<MobileWorkspace workflowId="flow" nodes={[...nodes, { node_id: 'flow:android:2', label: 'Second phone' }]} />);
+ await screen.findByText('Old phone task');
+ await screen.findByText('Working on your request');
+ await userEvent.click(screen.getByRole('button', { name: 'Use phone' }));
+ await screen.findByRole('button', { name: 'Phone Home' });
+ snapshot = { ...snapshot, active: null, last_task: null };
+ reset('flow');
+ await screen.findByText('Ready');
+ expect(screen.queryByText('Old phone task')).toBeNull();
+ expect(screen.queryByText('Working on your request')).toBeNull();
+ expect(screen.queryByRole('button', { name: 'Phone Home' })).toBeNull();
+ expect(fetchMock.mock.calls.some(([url]) => url.endsWith('/release'))).toBe(false);
+ expect(sessionStorage.getItem('mobile-task:flow:flow:mobile_agent:1')).toBeNull();
+ expect(sessionStorage.getItem('mobile-task:flow:flow:android:2')).toBeNull();
+ expect(sessionStorage.getItem('mobile-task:other:other:mobile_agent:1')).toBe('unrelated-submission');
+ await userEvent.click(screen.getByText('Ask AI to use the phone', { selector: 'summary' }));
+ await userEvent.type(screen.getByRole('textbox', { name: 'Ask AI to use the phone' }), 'New task');
+ expect(screen.getByRole('button', { name: 'Run task' })).toBeEnabled();
+ });
+
+ it('aborts old status and task polls and ignores their delayed responses after reset', async () => {
+ sessionStorage.setItem('mobile-task:flow:flow:mobile_agent:1', 'old-submission');
+ let resolveStatus!: (response: Response) => void;
+ let resolveTask!: (response: Response) => void;
+ let oldStatusSignal: AbortSignal | undefined;
+ let oldTaskSignal: AbortSignal | undefined;
+ let statusCalls = 0;
+ snapshot.running = true;
+ fetchMock.mockImplementation((url: string, options: RequestInit) => {
+ if (url.endsWith('/status') && statusCalls++ === 0) {
+ oldStatusSignal = options.signal as AbortSignal;
+ return new Promise<Response>((resolve) => { resolveStatus = resolve; });
+ }
+ if (url.includes('/tasks/old-submission')) {
+ oldTaskSignal = options.signal as AbortSignal;
+ return new Promise<Response>((resolve) => { resolveTask = resolve; });
+ }
+ return Promise.resolve(json(url.endsWith('/doctor') ? ready : snapshot));
+ });
+ render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
+ await waitFor(() => expect(oldTaskSignal).toBeDefined());
+ reset('flow');
+ await screen.findByText('Ready');
+ expect(oldStatusSignal?.aborted).toBe(true);
+ expect(oldTaskSignal?.aborted).toBe(true);
+ await act(async () => {
+ resolveStatus(json({ ...snapshot, active: { run_id: 'stale-run', phase: 'Using phone', current_goal: 'Stale phone activity' } }));
+ resolveTask(json({ status: 'completed', result: 'Stale task result' }));
+ });
+ expect(screen.queryByText('Stale phone activity')).toBeNull();
+ expect(screen.queryByText('Stale task result')).toBeNull();
+ expect(sessionStorage.getItem('mobile-task:flow:flow:mobile_agent:1')).toBeNull();
+ });
+
+ it('ignores resets for another workflow without replacing the phone session', async () => {
+ sessionStorage.setItem('mobile-task:flow:flow:mobile_agent:1', 'existing-submission');
+ snapshot = { ...snapshot, running: true, active: { run_id: 'run', phase: 'Using phone', current_goal: 'Keep this task' } };
+ render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
+ await screen.findByText('Keep this task');
+ await waitFor(() => expect(connectVideo).toHaveBeenCalledOnce());
+ const canvas = screen.getByLabelText('Phone screen, view only').querySelector('canvas');
+ reset('other');
+ expect(screen.getByText('Keep this task')).toBeInTheDocument();
+ expect(screen.getByLabelText('Phone screen, view only').querySelector('canvas')).toBe(canvas);
+ expect(sessionStorage.getItem('mobile-task:flow:flow:mobile_agent:1')).toBe('existing-submission');
+ expect(connectVideo).toHaveBeenCalledOnce();
+ });
+
+ it('does not restore a pre-reset submission when its HTTP response arrives late', async () => {
+ snapshot.running = true;
+ let resolveSubmission!: (response: Response) => void;
+ fetchMock.mockImplementation((url: string, options: RequestInit) => {
+ if (url.endsWith('/tasks') && options.method === 'POST') {
+ return new Promise<Response>((resolve) => { resolveSubmission = resolve; });
+ }
+ return Promise.resolve(json(url.endsWith('/doctor') ? ready : snapshot));
+ });
+ render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
+ await screen.findByText('Ready');
+ await userEvent.click(screen.getByText('Ask AI to use the phone', { selector: 'summary' }));
+ await userEvent.type(screen.getByRole('textbox', { name: 'Ask AI to use the phone' }), 'Old request');
+ await userEvent.click(screen.getByRole('button', { name: 'Run task' }));
+ await waitFor(() => expect(resolveSubmission).toBeDefined());
+ reset('flow');
+ await screen.findByText('Ready');
+ await act(async () => { resolveSubmission(json({ status: 'running', invocation_id: 'old-run' })); });
+ expect(sessionStorage.getItem('mobile-task:flow:flow:mobile_agent:1')).toBeNull();
+ expect(screen.queryByText('Working on your request')).toBeNull();
+ expect(fetchMock.mock.calls.some(([url]) => url.includes('/tasks/'))).toBe(false);
+ });
+
it('shows internal phone progress when invoked by an AI agent tool call', async () => {
snapshot.running = true;
snapshot.active = { run_id: 'tool-run', phase: 'Waiting for model', steps: 7, max_steps: 40 };
render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
- expect(await screen.findByText(/Waiting for model.*Step 7 \/ 40/)).toBeVisible();
+ expect(await screen.findByText(/Waiting for model.*Engine step 7 \/ 40/)).toBeVisible();
});
it('retains a startup failure across status refreshes without claiming the phone is ready', async () => {
snapshot.start_error = 'This phone is already open, but could not be safely reconnected.';
diff --git a/client/src/components/mobile/__tests__/PhoneActivity.test.tsx b/client/src/components/mobile/__tests__/PhoneActivity.test.tsx
index 2ea51049..5766b0a8 100644
--- a/client/src/components/mobile/__tests__/PhoneActivity.test.tsx
+++ b/client/src/components/mobile/__tests__/PhoneActivity.test.tsx
@@ -1,16 +1,27 @@
-import { render, screen } from '@testing-library/react';
-import { describe, expect, it } from 'vitest';
+import { act, cleanup, fireEvent, render, screen, within } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { PhoneActivity } from '../PhoneActivity';
+beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
+});
+afterEach(() => { cleanup(); vi.useRealTimers(); });
+
describe('Android activity', () => {
- it('shows a quiet model wait with timing and actual operations', () => {
+ it('shows a quiet model wait and retains legacy operations in collapsed technical details', () => {
const now = Date.now() / 1000;
render(<PhoneActivity task={{ run_id: 'run', phase: 'Waiting for model', started_at: now - 65,
phase_started_at: now - 40, updated_at: now - 40, model: 'test-model', provider: 'openai',
activity: [{ at: now - 42, message: 'Reading screen and accessibility tree: completed (0.2s)', step: 1 }] }} />);
expect(screen.getByText(/No new activity for 40s/)).toBeInTheDocument();
- expect(screen.getByText(/Reading screen and accessibility tree/)).toBeInTheDocument();
- expect(screen.getByText(/openai test-model/)).toBeInTheDocument();
+ expect(screen.getByText(/Reading screen and accessibility tree/)).not.toBeVisible();
+ expect(screen.getByText(/openai test-model/)).not.toBeVisible();
+ expect(screen.getByLabelText('Android technical activity')).not.toHaveAttribute('open');
+ expect(screen.queryByRole('list', { name: 'Recent Android actions' })).toBeNull();
+ fireEvent.click(screen.getByText('Technical activity'));
+ expect(screen.getByText(/Reading screen and accessibility tree/)).toBeVisible();
+ expect(screen.getByText(/openai test-model/)).toBeVisible();
});
it('retains the failure location after the task ends', () => {
render(<PhoneActivity task={{ run_id: 'run', status: 'failed', phase: 'Waiting for model',
@@ -19,4 +30,109 @@ describe('Android activity', () => {
expect(screen.getByText(/80s total/)).toBeInTheDocument();
expect(screen.queryByText(/No new activity/)).toBeNull();
});
+
+ it('keeps the current subtask visible across model waits without inventing actions', () => {
+ const now = Date.now() / 1000;
+ const task = { run_id: 'run', phase: 'Using phone', started_at: now - 20,
+ phase_started_at: now - 5, updated_at: now, steps: 4, max_steps: 40,
+ current_goal: 'Find the display settings',
+ agent_activity: [{ at: now - 5, step: 4, kind: 'action', message: 'Open Settings',
+ detail: 'Find the display controls', state: 'completed', outcome: 'Settings opened', duration_ms: 1200 }],
+ };
+ const view = render(<PhoneActivity task={task} />);
+ expect(within(screen.getByRole('region', { name: 'Current Android subtask' })).getByText('Find the display settings')).toBeVisible();
+ view.rerender(<PhoneActivity task={{ ...task, phase: 'Waiting for model', phase_started_at: now }} />);
+ act(() => { vi.advanceTimersByTime(5000); });
+ expect(screen.getByText('Find the display settings')).toBeVisible();
+ expect(screen.getByText(/AI activity · Waiting for model/)).toBeVisible();
+ expect(screen.getByText(/Current phase: 5s/)).toBeVisible();
+ expect(within(screen.getByRole('list', { name: 'Recent Android actions' })).getAllByRole('listitem')).toHaveLength(1);
+ expect(screen.getByText('Settings opened', { exact: false })).toBeVisible();
+ expect(screen.getByText(/Engine step 4 \/ 40/)).toBeVisible();
+ });
+
+ it('shows plan statuses and the reason a subtask failed', () => {
+ const now = Date.now() / 1000;
+ render(<PhoneActivity task={{ run_id: 'run', started_at: now, updated_at: now,
+ current_goal: 'Change brightness', plan: [
+ { id: '1', description: 'Open Settings', status: 'success' },
+ { id: '2', description: 'Change brightness', status: 'pending' },
+ { id: '3', description: 'Enable dark mode', status: 'not_started' },
+ { id: '4', description: 'Set a schedule', status: 'failure', reason: 'Scheduling is unavailable on this device' },
+ ],
+ }} />);
+ const details = screen.getByLabelText('Android task plan');
+ expect(details).not.toHaveAttribute('open');
+ expect(screen.getByText('Plan · 1 / 4 done')).toBeVisible();
+ fireEvent.click(screen.getByText('Plan · 1 / 4 done'));
+ const plan = within(details);
+ const rows = plan.getAllByRole('listitem');
+ expect(rows[0]).toHaveTextContent('Done · Open Settings');
+ expect(rows[1]).toHaveTextContent('In progress · Change brightness');
+ expect(rows[2]).toHaveTextContent('Not started · Enable dark mode');
+ expect(rows[3]).toHaveTextContent('Failed · Set a schedule');
+ expect(plan.getByText('Scheduling is unavailable on this device')).toBeVisible();
+ });
+
+ it('shows a failed action with intent, actual result, duration and engine step', () => {
+ const now = Date.now() / 1000;
+ render(<PhoneActivity task={{ run_id: 'run', started_at: now - 20, updated_at: now,
+ agent_activity: [{ action_id: 'action-1', at: now - 5, step: 3, kind: 'action',
+ message: 'Open display settings', detail: 'Find the brightness controls',
+ state: 'failed', outcome: 'Display option was not found', duration_ms: 1350 }],
+ activity: [{ at: now, step: 3, message: 'Accessibility tree: completed (0.2s)' }],
+ }} />);
+ const actions = within(screen.getByRole('list', { name: 'Recent Android actions' }));
+ expect(actions.getByText('Open display settings')).toBeVisible();
+ expect(actions.getByText('Find the brightness controls')).toBeVisible();
+ expect(actions.getByText('Result: Display option was not found')).toBeVisible();
+ expect(actions.getByRole('listitem')).toHaveTextContent('+15s · Engine step 3 · Failed · 1.4s');
+ expect(screen.getByText(/Accessibility tree: completed/)).not.toBeVisible();
+ });
+
+ it('updates an action in place when its result arrives and preserves its start time', () => {
+ const now = Date.now() / 1000;
+ const action = { action_id: 'tap-1', at: now - 2, step: 2, kind: 'action', message: 'Open Wi-Fi settings', state: 'started' };
+ const task = { run_id: 'run', started_at: now - 10, updated_at: now, agent_activity: [action], current_action: action };
+ const view = render(<PhoneActivity task={task} />);
+ const list = screen.getByRole('list', { name: 'Recent Android actions' });
+ expect(within(list).getByRole('listitem')).toHaveTextContent('In progress · 2.0s');
+ const completed = { ...action, state: 'completed', finished_at: now, outcome: 'Wi-Fi settings opened', duration_ms: 2000 };
+ view.rerender(<PhoneActivity task={{ ...task, current_action: completed, agent_activity: [completed] }} />);
+ expect(within(list).getAllByRole('listitem')).toHaveLength(1);
+ expect(within(list).getByRole('listitem')).toHaveTextContent('+8s · Engine step 2 · Completed · 2.0s');
+ expect(within(list).getByText('Result: Wi-Fi settings opened')).toBeVisible();
+ });
+
+ it('shows a current action when a snapshot has no semantic history yet', () => {
+ const now = Date.now() / 1000;
+ render(<PhoneActivity task={{ run_id: 'run', started_at: now, updated_at: now,
+ current_action: { at: now, step: 1, kind: 'action', message: 'Open the requested app', state: 'started' },
+ }} />);
+ expect(screen.getByText('Open the requested app')).toBeVisible();
+ });
+
+ it('marks an interrupted action as unconfirmed rather than claiming it succeeded', () => {
+ const now = Date.now() / 1000;
+ render(<PhoneActivity task={{ run_id: 'run', started_at: now - 2, updated_at: now,
+ current_action: { at: now - 2, step: 1, kind: 'action', message: 'Open the requested app',
+ state: 'returned', outcome: 'Action interrupted before the tool confirmed its outcome.', duration_ms: 2000 },
+ }} />);
+ const actions = within(screen.getByRole('list', { name: 'Recent Android actions' }));
+ expect(actions.getByText('Unconfirmed')).toBeVisible();
+ expect(actions.getByText('Result: Action interrupted before the tool confirmed its outcome.')).toBeVisible();
+ expect(actions.queryByText('Completed')).toBeNull();
+ expect(actions.queryByText('Failed')).toBeNull();
+ });
+
+ it('adds the quiet warning as time passes without replacing the reported goal', () => {
+ const now = Date.now() / 1000;
+ render(<PhoneActivity task={{ run_id: 'run', phase: 'Waiting for model', started_at: now,
+ phase_started_at: now, updated_at: now, current_goal: 'Check whether dark mode is enabled' }} />);
+ expect(screen.queryByText(/No new activity/)).toBeNull();
+ act(() => { vi.advanceTimersByTime(30_000); });
+ expect(screen.getByText(/No new activity for 30s/)).toBeVisible();
+ expect(screen.getByText('Check whether dark mode is enabled')).toBeVisible();
+ expect(screen.queryByRole('list', { name: 'Recent Android actions' })).toBeNull();
+ });
});
diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsx
index f1fd73c4..646459af 100644
--- a/client/src/components/ui/TopToolbar.tsx
+++ b/client/src/components/ui/TopToolbar.tsx
@@ -494,7 +494,7 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
disabled={hasPendingControlMutation}
title={isRetryingReset
? 'Retry the interrupted reset transition'
- : 'Terminate and archive this execution'}
+ : 'Stop active work and reset this workflow'}
>
<RotateCcw className="h-3 w-3" />
{isRetryingReset ? 'Retry Reset' : 'Reset'}
@@ -514,7 +514,7 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
</Badge>
)}
- <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This immediately terminates active work and archives the current generation. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction disabled={hasPendingControlMutation} onClick={onResetWorkflow}>{isRetryingReset ? 'Retry reset' : 'Reset workflow'}</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
+ <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This stops active work, cancels phone tasks, and archives the current generation. Reset waits for phone actions to finish cleaning up. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction disabled={hasPendingControlMutation} onClick={onResetWorkflow}>{isRetryingReset ? 'Retry reset' : 'Reset workflow'}</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
<ActionButton
intent="save"
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index fc473be3..e0fa7cac 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -30,6 +30,7 @@ import {
isAgentCapabilityEvent,
} from '../types/cloudEvents';
import { WS_CLOSE, WS_RECONNECT } from '../lib/connectionConfig';
+import { startWebSocketHeartbeat } from '../lib/webSocketHeartbeat';
import { todoQueryKeyFromEvent } from '../lib/todoQuery';
import { useCanvasDockStore } from '../stores/canvasDockStore';
// Cycle note: lib/nodeSpec imports useWebSocket from this module. Safe in
@@ -82,7 +83,7 @@ export const WORKFLOW_CONTROL_REQUEST_TIMEOUT = 5 * 60 * 1000;
export const CREDENTIAL_PROBE_REQUEST_TIMEOUT = 60 * 1000;
// Maximum queued sends before backpressure kicks in (FIFO eviction of oldest)
-const QUEUE_MAX_SIZE = 200;
+const QUEUE_MAX_SIZE = WS_RECONNECT.MAX_ENQUEUED_MESSAGES;
/**
* How long to let an `execute_node` request stay in flight.
@@ -161,6 +162,10 @@ export interface WorkflowControlStatus {
controller_run_id?: string | null;
state: WorkflowControlState;
revision: number;
+ workspace_epoch?: number;
+ workspace_reset_request_id?: string | null;
+ workspace_resetting?: boolean;
+ workspace_available?: boolean;
active_count: number;
in_flight_count: number;
queued_count: number;
@@ -240,7 +245,24 @@ const WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION: Record<
export const isWorkflowControlMutationConfirmed = (
action: WorkflowControlMutationAction,
status: Pick<WorkflowControlStatus, 'state'>,
-): boolean => status.state === WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION[action];
+): boolean => status.state === WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION[action]
+ || (action === 'reset' && status.state === 'never_started');
+
+/** A phone-only workflow can already be ready before Reset is attempted.
+ * Recover a lost response only when the server proves this Reset completed. */
+export const isWorkflowResetRecoveryConfirmed = (
+ status: WorkflowControlStatus,
+ expectedRevision: number,
+ requestIds: ReadonlySet<string>,
+): boolean => isWorkflowControlMutationConfirmed('reset', status)
+ && status.workspace_available !== false
+ && status.workspace_resetting !== true
+ && (
+ status.revision > expectedRevision
+ || (status.workspace_resetting === false
+ && typeof status.workspace_reset_request_id === 'string'
+ && requestIds.has(status.workspace_reset_request_id))
+ );
export interface TeamTaskTraceEvent {
event_id: string | number;
@@ -606,6 +628,10 @@ export const normalizeWorkflowControlStatus = (value: any, workflowId?: string):
state,
generation: Number(source.generation ?? 0),
revision: Number(source.revision ?? 0),
+ workspace_epoch: Number(source.workspace_epoch ?? 0),
+ workspace_reset_request_id: source.workspace_reset_request_id ?? null,
+ workspace_resetting: source.workspace_resetting === true,
+ workspace_available: source.workspace_available !== false,
active_count: Number(source.active_count ?? source.active_runs ?? 0),
in_flight_count: Number(source.in_flight_count ?? source.active_count ?? source.active_runs ?? 0),
queued_count: Number(source.queued_count || 0),
@@ -824,7 +850,10 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// type tightens to the library's class so any feature-specific calls
// (`shouldReconnect`, `reconnect()`) type-check.
const wsRef = useRef<ReconnectingWebSocket | null>(null);
- const pingIntervalRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+ const isMountedRef = useRef(true);
+ const authenticatedRef = useRef(isAuthenticated);
+ authenticatedRef.current = isAuthenticated;
+ const stopHeartbeatRef = useRef<(() => void) | null>(null);
const pendingRequestsRef = useRef<Map<string, PendingRequest>>(new Map());
// Generic broadcast subscribers ({type -> Set<handler>}) for events
// surfaced via send_custom_event on the backend. Lets new features
@@ -846,6 +875,57 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// do not read from elsewhere.
const previousWorkflowIdForSwitchRef = useRef<string | undefined>(currentWorkflowId);
+ const rejectPendingRequests = useCallback((reason: string) => {
+ for (const pending of pendingRequestsRef.current.values()) {
+ if (pending.timeout) clearTimeout(pending.timeout);
+ pending.reject(new Error(reason));
+ }
+ pendingRequestsRef.current.clear();
+ }, []);
+
+ const disposeConnection = useCallback((reason: string) => {
+ const ws = wsRef.current;
+ // Invalidate ownership before close: stale callbacks must not change a
+ // later connection or restart its heartbeat (including Strict Mode).
+ wsRef.current = null;
+ stopHeartbeatRef.current?.();
+ stopHeartbeatRef.current = null;
+ if (ws) {
+ ws.onopen = ws.onclose = ws.onmessage = ws.onerror = null;
+ ws.close(WS_CLOSE.NORMAL_CLOSURE, reason);
+ }
+ rejectPendingRequests(reason);
+ for (const queued of pendingSendQueueRef.current) {
+ queued.abortController.abort();
+ queued.reject(new Error(reason));
+ }
+ pendingSendQueueRef.current = [];
+ }, [rejectPendingRequests]);
+
+ // One request registry for ordinary calls and reconnect snapshots. A
+ // disconnect rejects sent work; it must never replay an uncertain write.
+ const sendOnSocket = useCallback(<T,>(
+ ws: ReconnectingWebSocket, payload: Record<string, any>, timeoutMs: number,
+ ): Promise<T> => new Promise((resolve, reject) => {
+ if (!isMountedRef.current || !authenticatedRef.current || wsRef.current !== ws || ws.readyState !== ReconnectingWebSocket.OPEN) {
+ reject(new Error('WebSocket is not connected'));
+ return;
+ }
+ const requestId = generateRequestId();
+ const timeout = timeoutMs > 0 ? setTimeout(() => {
+ pendingRequestsRef.current.delete(requestId);
+ reject(new Error(`Request timeout: ${payload.type}`));
+ }, timeoutMs) : null;
+ pendingRequestsRef.current.set(requestId, { resolve, reject, timeout });
+ try {
+ ws.send(JSON.stringify({ ...payload, request_id: requestId }));
+ } catch (error) {
+ if (timeout) clearTimeout(timeout);
+ pendingRequestsRef.current.delete(requestId);
+ reject(error);
+ }
+ }), []);
+
const applyWorkflowControlStatus = useCallback((
workflowId: string,
value: any,
@@ -880,30 +960,13 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// Fetch deployment status for the new workflow (n8n pattern)
// This ensures the deploy button shows correct state when switching workflows
if (wsRef.current?.readyState === ReconnectingWebSocket.OPEN) {
+ const ws = wsRef.current;
const fetchDeploymentStatus = async () => {
try {
- const requestId = generateRequestId();
- const response = await new Promise<any>((resolve, reject) => {
- const timeout = setTimeout(() => reject(new Error('Timeout')), 5000);
-
- const handler = (event: MessageEvent) => {
- try {
- const msg = JSON.parse(event.data);
- if (msg.request_id === requestId) {
- clearTimeout(timeout);
- wsRef.current?.removeEventListener('message', handler);
- resolve(msg);
- }
- } catch { /* swallow message-parse errors — invalid frames are ignored */ }
- };
-
- wsRef.current?.addEventListener('message', handler);
- wsRef.current?.send(JSON.stringify({
- type: 'get_deployment_status',
- request_id: requestId,
- workflow_id: currentWorkflowId
- }));
- });
+ const response = await sendOnSocket<any>(ws, {
+ type: 'get_deployment_status', workflow_id: currentWorkflowId,
+ }, 5000);
+ if (!isMountedRef.current || wsRef.current !== ws || useAppStore.getState().currentWorkflow?.id !== currentWorkflowId) return;
// Update deployment status based on response
const isRunning = response.is_running || false;
@@ -937,12 +1000,12 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// Update at the END of the effect so the next run sees the prior id.
previousWorkflowIdForSwitchRef.current = currentWorkflowId;
- }, [currentWorkflowId]);
+ }, [currentWorkflowId, sendOnSocket]);
// Handle incoming messages
/** Hand a broadcast to every addEventListener(type) subscriber. A case
* that handles a type itself calls this too when listeners may want it. */
- const dispatchToListeners = (type: string, data: any) => {
+ const dispatchToListeners = useCallback((type: string, data: any) => {
const listeners = eventListenersRef.current.get(type);
if (!listeners || listeners.size === 0) return;
for (const handler of listeners) {
@@ -950,7 +1013,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
console.error(`[WebSocket] Listener for '${type}' threw:`, err);
}
}
- };
+ }, []);
const handleMessage = useCallback((event: MessageEvent) => {
try {
@@ -1731,6 +1794,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
setConsoleLogs([]);
setChatMessages([]);
}
+ dispatchToListeners(type, { ...message.data, workflow_id: workflowId });
}
break;
}
@@ -2022,10 +2086,10 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
} catch (error) {
console.error('[WebSocket] Failed to parse message:', error);
}
- }, [applyWorkflowControlStatus]);
+ }, [applyWorkflowControlStatus, dispatchToListeners]);
// Drain queued sends after a successful reconnect. Each queued send gets a
- // fresh request_id and a reset timeout budget; responses correlate via the
+ // fresh request_id and the remaining timeout budget; responses correlate via the
// existing pendingRequestsRef map. Per-call abortController cancels the
// queue-side timeout that was running while the request was waiting in line.
const drainPendingSends = useCallback((ws: ReconnectingWebSocket) => {
@@ -2037,47 +2101,26 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
} catch {
// Ignore — abort is idempotent.
}
- if (ws.readyState !== WebSocket.OPEN) {
- // Defensive: should not happen since we drain inside onopen.
- try {
- queued.reject(new Error('WebSocket closed during drain'));
- } catch {
- // Ignore — caller may have already settled.
- }
+ const remaining = queued.timeoutMs > 0
+ ? queued.timeoutMs - (Date.now() - queued.enqueuedAt) : -1;
+ if (queued.timeoutMs > 0 && remaining <= 0) {
+ queued.reject(new Error(`Request timeout (queued): ${queued.type}`));
continue;
}
- const requestId = generateRequestId();
- let timeout: ReturnType<typeof setTimeout> | null = null;
- if (queued.timeoutMs > 0) {
- // Reset the timeout budget on replay so caller's perspective
- // remains "now"; queue-side timer was already aborted above.
- timeout = setTimeout(() => {
- pendingRequestsRef.current.delete(requestId);
- queued.reject(new Error(`Request timeout: ${queued.type}`));
- }, queued.timeoutMs);
- }
- pendingRequestsRef.current.set(requestId, {
- resolve: queued.resolve,
- reject: queued.reject,
- timeout,
- });
- ws.send(JSON.stringify({
- type: queued.type,
- request_id: requestId,
- ...queued.data,
- }));
+ void sendOnSocket(ws, { type: queued.type, ...queued.data }, remaining)
+ .then(queued.resolve, queued.reject);
}
- }, []);
+ }, [sendOnSocket]);
// Connect to WebSocket. Uses PartySocket's `ReconnectingWebSocket` —
// a native-WebSocket-compatible class with built-in jittered exponential
- // backoff, message replay, and intentional-close (code 1000) handling.
+ // backoff. Explicit close() stops retries; remote close codes do not.
// Replaces the previous flat 3 s `setTimeout(reconnect)` loop.
// Backoff envelope is configured via `WS_RECONNECT` in
// `lib/connectionConfig.ts` so a future tuning pass is a one-file edit.
// Ref: https://docs.partykit.io/reference/partysocket-api/
const connect = useCallback(() => {
- if (wsRef.current?.readyState === ReconnectingWebSocket.OPEN) {
+ if (wsRef.current || !isMountedRef.current || !authenticatedRef.current) {
return;
}
@@ -2089,16 +2132,21 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
maxReconnectionDelay: WS_RECONNECT.MAX_DELAY_MS,
reconnectionDelayGrowFactor: WS_RECONNECT.GROW_FACTOR,
// Reconnect indefinitely while the page is open. Intentional
- // closes via `ws.close(1000, ...)` (logout / unmount) skip the
- // reconnect path because PartySocket inspects the close code.
+ // disposal via ws.close() (logout / unmount) stops retries.
maxRetries: Infinity,
// Send-while-disconnected buffer; replayed automatically on the
// next OPEN. Mirrors the previous `pendingSendQueueRef` cap intent
// for opportunistic out-of-band sends.
maxEnqueuedMessages: WS_RECONNECT.MAX_ENQUEUED_MESSAGES,
});
+ wsRef.current = ws;
+ const ownsConnection = () => isMountedRef.current && authenticatedRef.current && wsRef.current === ws;
+ let openEpoch = 0;
ws.onopen = async () => {
+ if (!ownsConnection()) return;
+ const epoch = ++openEpoch;
+ const isCurrentOpen = () => ownsConnection() && openEpoch === epoch && ws.readyState === ReconnectingWebSocket.OPEN;
setIsConnected(true);
setReconnecting(false);
@@ -2110,12 +2158,8 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// do not invalidate them, so canvas / palette icons no longer
// flash back to fallback on every reconnect.
- // Start ping interval
- pingIntervalRef.current = setInterval(() => {
- if (ws.readyState === ReconnectingWebSocket.OPEN) {
- ws.send(JSON.stringify({ type: 'ping' }));
- }
- }, 30000);
+ stopHeartbeatRef.current?.();
+ stopHeartbeatRef.current = startWebSocketHeartbeat(ws);
// Drain any sends that were queued while the socket was reconnecting.
// Runs BEFORE setIsReady(true) so isReady-gated callers don't race
@@ -2165,34 +2209,15 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// hydrate panels that PersistQueryClient doesn't cache (terminal /
// chat / console history come straight from the server's per-request
// log read, not from a query cache).
- const sendBurstRequest = <T = any>(payload: object, idPrefix: string): Promise<T> =>
- new Promise<T>((resolve, reject) => {
- const requestId = `${idPrefix}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
- const timeout = setTimeout(() => {
- ws.removeEventListener('message', handler);
- reject(new Error('Timeout'));
- }, 5000);
- const handler = (event: MessageEvent) => {
- try {
- const msg = JSON.parse(event.data);
- if (msg.request_id === requestId) {
- clearTimeout(timeout);
- ws.removeEventListener('message', handler);
- resolve(msg);
- }
- } catch { /* swallow message-parse errors — invalid frames are ignored */ }
- };
- ws.addEventListener('message', handler);
- ws.send(JSON.stringify({ ...payload, request_id: requestId }));
- });
+ const sendBurstRequest = <T = any>(payload: Record<string, any>): Promise<T> =>
+ sendOnSocket<T>(ws, payload, 5000);
void (async () => {
try {
const terminalResponse = await sendBurstRequest<any>(
{ type: 'get_terminal_logs' },
- 'terminal_logs',
);
- if (terminalResponse.success && terminalResponse.logs) {
+ if (isCurrentOpen() && terminalResponse.success && terminalResponse.logs) {
const logs: TerminalLogEntry[] = terminalResponse.logs.map((log: any) => ({
timestamp: log.timestamp || new Date().toISOString(),
level: log.level || 'info',
@@ -2212,9 +2237,9 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const workflowId = useAppStore.getState().currentWorkflow?.id || 'default';
const chatResponse = await sendBurstRequest<any>(
{ type: 'get_chat_messages', session_id: workflowId },
- 'chat_messages',
);
- if (chatResponse.success && chatResponse.messages) {
+ if (isCurrentOpen() && (useAppStore.getState().currentWorkflow?.id || 'default') === workflowId
+ && chatResponse.success && chatResponse.messages) {
const messages: ChatMessage[] = chatResponse.messages.map((msg: any) => ({
role: msg.role as 'user' | 'assistant',
message: msg.message,
@@ -2232,9 +2257,9 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const consoleWorkflowId = useAppStore.getState().currentWorkflow?.id;
const consoleResponse = await sendBurstRequest<any>(
{ type: 'get_console_logs', limit: 100, workflow_id: consoleWorkflowId },
- 'console',
);
- if (consoleResponse.success && consoleResponse.logs) {
+ if (isCurrentOpen() && useAppStore.getState().currentWorkflow?.id === consoleWorkflowId
+ && consoleResponse.success && consoleResponse.logs) {
const logs: ConsoleLogEntry[] = consoleResponse.logs.map((log: any) => ({
node_id: log.node_id,
label: log.label,
@@ -2255,69 +2280,35 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
})();
};
- ws.onmessage = handleMessage;
+ ws.onmessage = (event) => {
+ if (ownsConnection()) handleMessage(event);
+ };
ws.onclose = (event) => {
+ if (!ownsConnection()) return;
+ ++openEpoch;
console.log('[WebSocket] Disconnected:', event.code, event.reason);
setIsConnected(false);
setIsReady(false);
- wsRef.current = null;
+ // PartySocket retains ownership across retries, including remote
+ // code 1000 and its synthetic error-close events.
// Reject every in-flight request so TanStack Query's retry +
// any awaiters fail fast on the dead socket instead of waiting
// for the 30s REQUEST_TIMEOUT. Trigger-node waiters that deliberately
// had no timeout are still cleared — the next reconnect will
// re-register them when the user re-runs the trigger.
- if (pendingRequestsRef.current.size > 0) {
- for (const [, pending] of pendingRequestsRef.current) {
- if (pending.timeout) clearTimeout(pending.timeout);
- try {
- pending.reject(new Error('WebSocket closed'));
- } catch {
- // Ignore — caller may have already settled.
- }
- }
- pendingRequestsRef.current.clear();
- }
-
- // Pending-send queue handling: only drop on intentional close
- // (RFC 6455 §7.4.1 Normal Closure, code 1000). Transient closes
- // preserve the queue so the next onopen drain replays them.
- if (event.code === WS_CLOSE.NORMAL_CLOSURE) {
- if (pendingSendQueueRef.current.length > 0) {
- for (const queued of pendingSendQueueRef.current) {
- try {
- queued.abortController.abort();
- queued.reject(new Error('WebSocket closed (intentional)'));
- } catch {
- // Ignore — caller may have already settled.
- }
- }
- pendingSendQueueRef.current = [];
- }
- }
-
- // Clear ping interval
- if (pingIntervalRef.current) {
- clearInterval(pingIntervalRef.current);
- pingIntervalRef.current = null;
- }
-
- // PartySocket performs the reconnect itself when
- // `event.code !== WS_CLOSE.NORMAL_CLOSURE`, honouring the
- // `WS_RECONNECT` envelope passed at construction time. Surface
- // "reconnecting" to the UI for transient closes; intentional
- // closes (logout / unmount) leave the flag false.
- if (event.code !== WS_CLOSE.NORMAL_CLOSURE) {
- setReconnecting(true);
- }
+ rejectPendingRequests('WebSocket closed; request outcome may be unknown');
+ // Only unsent work survives a transient close, under its original
+ // deadline. disposeConnection rejects it on logout/unmount.
+ stopHeartbeatRef.current?.();
+ stopHeartbeatRef.current = null;
+ setReconnecting(true);
};
ws.onerror = (error) => {
- console.error('[WebSocket] Error:', error);
+ if (ownsConnection()) console.error('[WebSocket] Error:', error);
};
-
- wsRef.current = ws;
} catch (error) {
// Construction-time error (e.g. malformed URL). PartySocket has
// not installed its retry loop yet, so just log — there is
@@ -2326,7 +2317,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// jittered backoff.
console.error('[WebSocket] Failed to create connection:', error);
}
- }, [handleMessage, drainPendingSends]);
+ }, [handleMessage, drainPendingSends, rejectPendingRequests, sendOnSocket]);
// Request current status
const requestStatus = useCallback(() => {
@@ -2439,8 +2430,8 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const sessionId = currentWorkflowId || 'default';
const ws = wsRef.current;
- const chatRequestId = `chat_switch_${Date.now()}`;
- const consoleRequestId = `console_switch_${Date.now()}`;
+ const chatRequestId = `chat_switch_${generateRequestId()}`;
+ const consoleRequestId = `console_switch_${generateRequestId()}`;
const handler = (event: MessageEvent) => {
try {
const msg = JSON.parse(event.data);
@@ -2526,26 +2517,17 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
timeoutMs?: number
): Promise<T> => {
return new Promise((resolve, reject) => {
+ if (!isMountedRef.current || !authenticatedRef.current) {
+ reject(new Error('WebSocket unavailable: not authenticated or unmounted'));
+ return;
+ }
const useTimeout = timeoutMs === undefined || timeoutMs >= 0;
const actualTimeout = timeoutMs && timeoutMs > 0 ? timeoutMs : REQUEST_TIMEOUT;
const effectiveTimeout = (timeoutMs === -1 || !useTimeout) ? -1 : actualTimeout;
// FAST PATH: socket open — send immediately.
if (wsRef.current?.readyState === ReconnectingWebSocket.OPEN) {
- const requestId = generateRequestId();
- let timeout: ReturnType<typeof setTimeout> | null = null;
- if (effectiveTimeout > 0) {
- timeout = setTimeout(() => {
- pendingRequestsRef.current.delete(requestId);
- reject(new Error(`Request timeout: ${type}`));
- }, effectiveTimeout);
- }
- pendingRequestsRef.current.set(requestId, { resolve, reject, timeout });
- wsRef.current.send(JSON.stringify({
- type,
- request_id: requestId,
- ...data,
- }));
+ void sendOnSocket<T>(wsRef.current, { type, ...data }, effectiveTimeout).then(resolve, reject);
return;
}
@@ -2593,7 +2575,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
abortController.signal.addEventListener('abort', () => clearTimeout(queueTimeout));
}
});
- }, []);
+ }, [sendOnSocket]);
// =========================================================================
// Chat Message Operations
@@ -2920,6 +2902,17 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
}
const pending = WORKFLOW_CONTROL_PENDING_BY_REQUEST[type];
+ const previousStatus = workflowControlStatusesRef.current[workflowId];
+ const resetRequestIds = new Set<string>();
+ if (type === 'reset_workflow' && previousStatus?.workspace_resetting
+ && typeof previousStatus.workspace_reset_request_id === 'string') {
+ // Retrying an interrupted reset can finish the original server request.
+ resetRequestIds.add(previousStatus.workspace_reset_request_id);
+ }
+ let resetExpectedRevision = Number(data.expected_revision ?? previousStatus?.revision ?? 0);
+ const isReconciled = (status: WorkflowControlStatus) => type === 'reset_workflow'
+ ? isWorkflowResetRecoveryConfirmed(status, resetExpectedRevision, resetRequestIds)
+ : isWorkflowControlMutationConfirmed(pending.action, status);
workflowControlPendingRef.current.set(workflowId, pending);
setWorkflowControlPending((previous) => ({ ...previous, [workflowId]: pending }));
useWorkflowControlStore.getState().setPending(workflowId, pending);
@@ -2927,12 +2920,17 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const sendMutationAttempt = async (
attemptData: Record<string, any>,
): Promise<WorkflowStartResult> => {
+ const idempotencyKey = crypto.randomUUID();
+ if (type === 'reset_workflow') {
+ resetRequestIds.add(idempotencyKey);
+ resetExpectedRevision = Number(attemptData.expected_revision ?? resetExpectedRevision);
+ }
const response = await sendRequest<any>(type, {
...attemptData,
workflow_id: workflowId,
// Every bounded attempt has its own idempotency identity. In
// particular, a conflict retry must not replay the rejected request.
- idempotency_key: crypto.randomUUID(),
+ idempotency_key: idempotencyKey,
}, WORKFLOW_CONTROL_REQUEST_TIMEOUT);
// A success:false response can still be the newest authoritative
@@ -2966,7 +2964,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
let reconciled: WorkflowControlStatus | undefined;
try {
reconciled = await requestWorkflowControlStatus(workflowId);
- if (isWorkflowControlMutationConfirmed(pending.action, reconciled)) {
+ if (isReconciled(reconciled)) {
return reconciled;
}
} catch (resyncError) {
@@ -2992,7 +2990,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// issuing a third mutation attempt.
try {
const retryReconciled = await requestWorkflowControlStatus(workflowId);
- if (isWorkflowControlMutationConfirmed(pending.action, retryReconciled)) {
+ if (isReconciled(retryReconciled)) {
return retryReconciled;
}
} catch (resyncError) {
@@ -3070,8 +3068,9 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
setConsoleLogs([]);
setChatMessages([]);
}
+ dispatchToListeners('workflow_runtime_reset', { workflow_id: workflowId });
return result;
- }, [controlMutation]);
+ }, [controlMutation, dispatchToListeners]);
const getWorkflowControlStatusAsync = useCallback(async (workflowId: string) => {
try {
@@ -3602,9 +3601,6 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
}
}, [sendRequest]);
- // Track if component is mounted to prevent state updates after unmount
- const isMountedRef = useRef(true);
-
// Connect only when authenticated (not during auth loading)
useEffect(() => {
isMountedRef.current = true;
@@ -3614,8 +3610,8 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
return;
}
- // Skip if already connected
- if (wsRef.current?.readyState === ReconnectingWebSocket.OPEN) {
+ // The wrapper owns both the live socket and all reconnect attempts.
+ if (wsRef.current) {
return;
}
@@ -3633,49 +3629,21 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// Handle logout - separate effect to avoid reconnect loops
useEffect(() => {
- if (!isAuthenticated && wsRef.current) {
- // Drain the pending-send queue with a clear auth error before closing.
- if (pendingSendQueueRef.current.length > 0) {
- for (const queued of pendingSendQueueRef.current) {
- try {
- queued.abortController.abort();
- queued.reject(new Error('auth: not authenticated'));
- } catch {
- // Ignore — caller may have already settled.
- }
- }
- pendingSendQueueRef.current = [];
- }
- wsRef.current.close(WS_CLOSE.NORMAL_CLOSURE, 'User logged out');
- wsRef.current = null;
+ if (!isAuthenticated) {
+ disposeConnection('User logged out');
setIsConnected(false);
setIsReady(false);
+ setReconnecting(false);
}
- }, [isAuthenticated]);
+ }, [isAuthenticated, disposeConnection]);
// Cleanup on unmount only
useEffect(() => {
return () => {
isMountedRef.current = false;
- if (pingIntervalRef.current) clearInterval(pingIntervalRef.current);
- // Drain the pending-send queue so any in-flight awaiters fail fast on
- // unmount instead of dangling forever.
- if (pendingSendQueueRef.current.length > 0) {
- for (const queued of pendingSendQueueRef.current) {
- try {
- queued.abortController.abort();
- queued.reject(new Error('WebSocket unmounting'));
- } catch {
- // Ignore — caller may have already settled.
- }
- }
- pendingSendQueueRef.current = [];
- }
- if (wsRef.current?.readyState === ReconnectingWebSocket.OPEN) {
- wsRef.current.close(WS_CLOSE.NORMAL_CLOSURE, 'Component unmounted');
- }
+ disposeConnection('Component unmounted');
};
- }, []);
+ }, [disposeConnection]);
// Memoized provider value: spreading a fresh object each render forces
// every useContext consumer to re-render on any state change. Wrapping
diff --git a/client/src/contexts/__tests__/webSocketLifecycle.test.tsx b/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
new file mode 100644
index 00000000..64e0a5f3
--- /dev/null
+++ b/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
@@ -0,0 +1,338 @@
+import { StrictMode, useLayoutEffect } from 'react';
+import { act, cleanup, render } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { WS_HEARTBEAT, WS_RECONNECT } from '../../lib/connectionConfig';
+
+const auth = vi.hoisted(() => ({ isAuthenticated: true, isLoading: false }));
+vi.mock('../AuthContext', () => ({ useAuth: () => auth }));
+vi.mock('../../services/workflowApi', () => ({ workflowApi: {} }));
+
+import { WebSocketProvider, useWebSocket, useWebSocketActions, type WebSocketActions } from '../WebSocketContext';
+import { useAppStore } from '../../store/useAppStore';
+import { useNodeStatusStore } from '../../stores/nodeStatusStore';
+
+// Exercise the installed PartySocket implementation; only its network peer
+// is simulated. Remote code 1000 and native error events really reconnect.
+class NativeSocket extends EventTarget {
+ static CONNECTING = 0;
+ static OPEN = 1;
+ static CLOSING = 2;
+ static CLOSED = 3;
+ static instances: NativeSocket[] = [];
+ static autoSnapshots = true;
+ readyState = NativeSocket.CONNECTING;
+ sent: Array<Record<string, any>> = [];
+ close = vi.fn((code = 1000, reason = '') => this.remoteClose(code, reason));
+ constructor(public url: string) {
+ super();
+ NativeSocket.instances.push(this);
+ }
+ send(raw: string) {
+ if (this.readyState !== NativeSocket.OPEN) throw new Error('Socket not open');
+ const message = JSON.parse(raw);
+ this.sent.push(message);
+ if (NativeSocket.autoSnapshots && ['get_terminal_logs', 'get_chat_messages', 'get_console_logs'].includes(message.type)) {
+ queueMicrotask(() => this.message({ request_id: message.request_id, success: true, logs: [], messages: [] }));
+ }
+ }
+ open() {
+ this.readyState = NativeSocket.OPEN;
+ this.dispatchEvent(new Event('open'));
+ }
+ message(data: Record<string, unknown>) {
+ this.dispatchEvent(new MessageEvent('message', { data: JSON.stringify(data) }));
+ }
+ remoteClose(code = 1006, reason = '') {
+ this.readyState = NativeSocket.CLOSED;
+ this.dispatchEvent(new CloseEvent('close', { code, reason }));
+ }
+}
+
+let actions: WebSocketActions;
+let context: ReturnType<typeof useWebSocket>;
+function Probe() {
+ const currentActions = useWebSocketActions();
+ const currentContext = useWebSocket();
+ useLayoutEffect(() => {
+ actions = currentActions;
+ context = currentContext;
+ }, [currentActions, currentContext]);
+ return null;
+}
+const tree = () => <WebSocketProvider><Probe /></WebSocketProvider>;
+const advance = async (ms: number) => {
+ await act(async () => { await vi.advanceTimersByTimeAsync(ms); });
+};
+const latest = () => NativeSocket.instances.at(-1)!;
+const open = async () => { await act(async () => latest().open()); };
+const mount = async () => {
+ const view = render(tree());
+ await advance(150);
+ return view;
+};
+
+describe('WebSocket recovery and ownership', () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.stubGlobal('WebSocket', NativeSocket);
+ vi.spyOn(document, 'hidden', 'get').mockReturnValue(false);
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ NativeSocket.instances = [];
+ NativeSocket.autoSnapshots = true;
+ useAppStore.setState({ currentWorkflow: null });
+ auth.isAuthenticated = true;
+ auth.isLoading = false;
+ });
+ afterEach(() => {
+ cleanup();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+ });
+
+ it('does not clear phone UI when a failed reset resync finds an unchanged ready workflow', async () => {
+ await mount();
+ await open();
+ const notify = vi.fn();
+ const clear = vi.spyOn(useNodeStatusStore.getState(), 'clearWorkflow');
+ actions.addEventListener('workflow_runtime_reset', notify);
+ const status = { workflow_id: 'phone', state: 'ready', revision: 3, generation: 1,
+ can_reset: true, workspace_epoch: 2, workspace_reset_request_id: 'previous-reset', workspace_resetting: false };
+ act(() => latest().message({ type: 'workflow_control_status', data: status }));
+ let result!: Promise<unknown>;
+ act(() => { result = actions.resetWorkflow('phone', 3).catch(error => error.message); });
+ const request = latest().sent.find(item => item.type === 'reset_workflow')!;
+ expect(request.idempotency_key).toBeTruthy();
+ await act(async () => {
+ latest().message({ request_id: request.request_id, success: false, error: 'phone_cancel_failed', status });
+ });
+ const resync = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ expect(resync).toBeDefined();
+ await act(async () => { latest().message({ request_id: resync.request_id, success: true, status }); });
+ await expect(result).resolves.toBe('phone_cancel_failed');
+ expect(notify).not.toHaveBeenCalled();
+ expect(clear).not.toHaveBeenCalled();
+ });
+
+ it.each([false, true])('confirms an exact phone reset ID only after cleanup ends (resetting=%s)', async (workspaceResetting) => {
+ await mount();
+ await open();
+ const notify = vi.fn();
+ const clear = vi.spyOn(useNodeStatusStore.getState(), 'clearWorkflow');
+ actions.addEventListener('workflow_runtime_reset', notify);
+ let result!: Promise<unknown>;
+ act(() => { result = actions.resetWorkflow('phone', 0).catch(error => error.message); });
+ const request = latest().sent.find(item => item.type === 'reset_workflow')!;
+ await act(async () => {
+ latest().message({ request_id: request.request_id, success: false, error: 'reset_outcome_unknown' });
+ });
+ const resync = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ const status = { workflow_id: 'phone', state: 'never_started', revision: 0, workspace_epoch: 1,
+ workspace_reset_request_id: request.idempotency_key, workspace_resetting: workspaceResetting };
+ await act(async () => { latest().message({ request_id: resync.request_id, success: true, status }); });
+ if (workspaceResetting) {
+ await expect(result).resolves.toBe('reset_outcome_unknown');
+ expect(notify).not.toHaveBeenCalled();
+ expect(clear).not.toHaveBeenCalled();
+ } else {
+ await expect(result).resolves.toMatchObject(status);
+ expect(notify).toHaveBeenCalledWith({ workflow_id: 'phone' });
+ expect(clear).toHaveBeenCalledWith('phone');
+ }
+ });
+
+ it('recognizes completion of the original pending workspace reset when retrying it', async () => {
+ await mount();
+ await open();
+ const notify = vi.fn();
+ actions.addEventListener('workflow_runtime_reset', notify);
+ const status = { workflow_id: 'phone', state: 'never_started', revision: 0,
+ workspace_epoch: 1, workspace_reset_request_id: 'original-reset', workspace_resetting: true };
+ act(() => latest().message({ type: 'workflow_control_status', data: status }));
+ let result!: Promise<unknown>;
+ act(() => { result = actions.resetWorkflow('phone', 0).catch(error => error.message); });
+ const request = latest().sent.find(item => item.type === 'reset_workflow')!;
+ expect(request.idempotency_key).not.toBe('original-reset');
+ await act(async () => {
+ latest().message({ request_id: request.request_id, success: false, error: 'reset_outcome_unknown' });
+ });
+ const resync = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ await act(async () => {
+ latest().message({ request_id: resync.request_id, success: true, status: { ...status, workspace_resetting: false } });
+ });
+ await expect(result).resolves.toMatchObject({ workspace_reset_request_id: 'original-reset', workspace_resetting: false });
+ expect(notify).toHaveBeenCalledWith({ workflow_id: 'phone' });
+ });
+
+ it.each([1006, 1000])('sends new requests after remote close %s and repeated reconnects', async (code) => {
+ await mount();
+ await open();
+ for (let i = 0; i < 3; i++) {
+ act(() => latest().remoteClose(code));
+ expect(actions.isReady).toBe(false);
+ await advance(WS_RECONNECT.MIN_DELAY_MS * 4);
+ await open();
+ expect(actions.isReady).toBe(true);
+ const response = actions.sendRequest('test_read');
+ const request = latest().sent.filter(m => m.type === 'test_read').at(-1)!;
+ expect(request).toBeDefined();
+ act(() => latest().message({ request_id: request.request_id, success: true }));
+ await expect(response).resolves.toMatchObject({ success: true });
+ }
+ expect(NativeSocket.instances).toHaveLength(4);
+ });
+
+ it('preserves unsent work through PartySocket synthetic code 1000 on native error', async () => {
+ await mount();
+ const response = actions.sendRequest('queued_read');
+ act(() => latest().dispatchEvent(new Event('error')));
+ await advance(WS_RECONNECT.MIN_DELAY_MS * 4);
+ await open();
+ const request = latest().sent.find(m => m.type === 'queued_read')!;
+ expect(request).toBeDefined();
+ act(() => latest().message({ request_id: request.request_id, success: true }));
+ await expect(response).resolves.toMatchObject({ success: true });
+ });
+
+ it('rejects in-flight mutations on disconnect without replaying them', async () => {
+ await mount();
+ await open();
+ const result = actions.sendRequest('test_write').catch(error => error.message);
+ act(() => latest().remoteClose());
+ await expect(result).resolves.toContain('outcome may be unknown');
+ await advance(WS_RECONNECT.MIN_DELAY_MS * 4);
+ await open();
+ expect(latest().sent.some(m => m.type === 'test_write')).toBe(false);
+ });
+
+ it('keeps the original deadline when queued work is sent', async () => {
+ await mount();
+ const result = actions.sendRequest('short_read', {}, 1000).catch(error => error.message);
+ await advance(800);
+ await open();
+ expect(latest().sent.some(m => m.type === 'short_read')).toBe(true);
+ await advance(201);
+ await expect(result).resolves.toContain('Request timeout');
+ });
+
+ it('does not send work that expired while disconnected', async () => {
+ await mount();
+ const result = actions.sendRequest('expired_read', {}, 100).catch(error => error.message);
+ await advance(101);
+ await expect(result).resolves.toContain('Request timeout (queued)');
+ await open();
+ expect(latest().sent.some(m => m.type === 'expired_read')).toBe(false);
+ });
+
+ it('bounds the disconnected queue and cancels all remaining waits on unmount', async () => {
+ const view = await mount();
+ const waits = Array.from({ length: 201 }, () =>
+ actions.sendRequest('queued_read', {}, -1).catch(error => error.message));
+ await expect(waits[0]).resolves.toContain('backpressure');
+ view.unmount();
+ expect((await Promise.all(waits)).slice(1).every(reason => reason === 'Component unmounted')).toBe(true);
+ });
+
+ it('stops a connecting socket and rejects queued work on logout', async () => {
+ const view = await mount();
+ const old = latest();
+ const result = actions.sendRequest('queued_read', {}, -1).catch(error => error.message);
+ auth.isAuthenticated = false;
+ view.rerender(tree());
+ await expect(result).resolves.toBe('User logged out');
+ expect(old.close).toHaveBeenCalledOnce();
+ await advance(60_000);
+ expect(NativeSocket.instances).toHaveLength(1);
+ await expect(actions.sendRequest('after_logout')).rejects.toThrow('not authenticated');
+ auth.isAuthenticated = true;
+ view.rerender(tree());
+ await advance(150);
+ await open();
+ act(() => old.remoteClose());
+ expect(actions.isReady).toBe(true);
+ });
+
+ it.each(['connecting', 'open', 'retrying'])('disposes %s sockets on unmount', async state => {
+ const view = await mount();
+ if (state !== 'connecting') await open();
+ if (state === 'retrying') act(() => latest().remoteClose());
+ const result = actions.sendRequest('pending_read', {}, -1).catch(error => error.message);
+ view.unmount();
+ await expect(result).resolves.toBe('Component unmounted');
+ await advance(60_000);
+ expect(NativeSocket.instances).toHaveLength(1);
+ });
+
+ it('owns only one connection under Strict Mode', async () => {
+ render(<StrictMode>{tree()}</StrictMode>);
+ await advance(150);
+ await open();
+ expect(NativeSocket.instances).toHaveLength(1);
+ expect(actions.isReady).toBe(true);
+ });
+
+ it('ignores late reconnect snapshots after switching workflows in the same tick', async () => {
+ NativeSocket.autoSnapshots = false;
+ useAppStore.setState({ currentWorkflow: { id: 'A' } as any });
+ await mount();
+ await open();
+ const oldRequests = [...latest().sent];
+ const oldCount = latest().sent.length;
+ act(() => useAppStore.setState({ currentWorkflow: { id: 'B' } as any }));
+ const newRequests = latest().sent.slice(oldCount);
+ const respond = async (requests: Array<Record<string, any>>, label: string) => {
+ await act(async () => {
+ for (const request of requests) {
+ if (request.type === 'get_chat_messages') latest().message({
+ request_id: request.request_id, success: true,
+ messages: [{ role: 'assistant', message: label, timestamp: 'now' }],
+ });
+ if (request.type === 'get_console_logs') latest().message({
+ request_id: request.request_id, success: true,
+ logs: [{ node_id: 'n1', label, data: label, timestamp: 'now' }],
+ });
+ }
+ });
+ };
+ await respond(newRequests, 'B');
+ expect(context.chatMessages[0]?.message).toBe('B');
+ expect(context.consoleLogs[0]?.label).toBe('B');
+ await respond(oldRequests, 'A');
+ expect(context.chatMessages[0]?.message).toBe('B');
+ expect(context.consoleLogs[0]?.label).toBe('B');
+ });
+
+ it('reconnects a silent open socket after a missed heartbeat', async () => {
+ await mount();
+ await open();
+ await advance(WS_HEARTBEAT.INTERVAL_MS);
+ expect(latest().sent.some(m => m.type === 'ping')).toBe(true);
+ await advance(WS_HEARTBEAT.TIMEOUT_MS + 1);
+ expect(actions.isReady).toBe(false);
+ expect(NativeSocket.instances).toHaveLength(2);
+ await open();
+ expect(actions.isReady).toBe(true);
+ });
+
+ it('keeps a responsive connection and gives a resumed tab a fresh heartbeat window', async () => {
+ await mount();
+ await open();
+ await advance(WS_HEARTBEAT.INTERVAL_MS);
+ act(() => latest().message({ type: 'pong' }));
+ await advance(WS_HEARTBEAT.TIMEOUT_MS + 1);
+ expect(NativeSocket.instances).toHaveLength(1);
+ vi.spyOn(document, 'hidden', 'get').mockReturnValue(true);
+ act(() => document.dispatchEvent(new Event('visibilitychange')));
+ await advance(120_000);
+ expect(NativeSocket.instances).toHaveLength(1);
+ vi.spyOn(document, 'hidden', 'get').mockReturnValue(false);
+ act(() => document.dispatchEvent(new Event('visibilitychange')));
+ await advance(WS_HEARTBEAT.TIMEOUT_MS - 1);
+ act(() => latest().message({ type: 'pong' }));
+ await advance(2);
+ expect(actions.isReady).toBe(true);
+ expect(NativeSocket.instances).toHaveLength(1);
+ });
+});
diff --git a/client/src/contexts/__tests__/workflowControlStatus.test.ts b/client/src/contexts/__tests__/workflowControlStatus.test.ts
index d4b95639..92c13c93 100644
--- a/client/src/contexts/__tests__/workflowControlStatus.test.ts
+++ b/client/src/contexts/__tests__/workflowControlStatus.test.ts
@@ -4,6 +4,7 @@ import {
assertWorkflowControlMutationSucceeded,
extractWorkflowControlStatusSnapshot,
isWorkflowControlMutationConfirmed,
+ isWorkflowResetRecoveryConfirmed,
mergeWorkflowControlStatus,
normalizeWorkflowControlStatus,
shouldRetryResetWorkflowAfterConflict,
@@ -66,6 +67,9 @@ describe('workflow control status ordering', () => {
workflow_id: 'workflow-1',
generation: '5',
revision: '11',
+ workspace_epoch: '3',
+ workspace_reset_request_id: 'reset-request',
+ workspace_resetting: true,
state: 'paused',
active_runs: '2',
},
@@ -75,6 +79,9 @@ describe('workflow control status ordering', () => {
workflow_id: 'workflow-1',
generation: 5,
revision: 11,
+ workspace_epoch: 3,
+ workspace_reset_request_id: 'reset-request',
+ workspace_resetting: true,
state: 'paused',
active_count: 2,
can_resume: true,
@@ -179,6 +186,40 @@ describe('workflow control mutation responses', () => {
)).toBe(false);
});
+ it('does not confirm a phone-only reset merely because the workflow was already ready', () => {
+ expect(isWorkflowResetRecoveryConfirmed(status({
+ state: 'ready', revision: 3, workspace_epoch: 2,
+ workspace_reset_request_id: 'old-reset', workspace_resetting: false,
+ }), 3, new Set(['new-reset']))).toBe(false);
+ });
+
+ it.each(['ready', 'never_started'] as const)('recovers %s when the exact workspace reset completed', (state) => {
+ expect(isWorkflowResetRecoveryConfirmed(status({
+ state, revision: 3, workspace_epoch: 4,
+ workspace_reset_request_id: 'this-reset', workspace_resetting: false,
+ }), 3, new Set(['this-reset']))).toBe(true);
+ });
+
+ it('requires workspace cleanup to finish even when the request ID or revision matches', () => {
+ expect(isWorkflowResetRecoveryConfirmed(status({
+ state: 'ready', revision: 4, workspace_reset_request_id: 'this-reset', workspace_resetting: true,
+ }), 3, new Set(['this-reset']))).toBe(false);
+ });
+
+ it.each([3, 4])('does not confirm reset with unavailable workspace status at revision %s', (revision) => {
+ const unavailable = normalizeWorkflowControlStatus({
+ state: 'ready', revision, workspace_available: false,
+ workspace_reset_request_id: 'this-reset', workspace_resetting: false,
+ });
+ expect(unavailable.workspace_available).toBe(false);
+ expect(isWorkflowResetRecoveryConfirmed(unavailable, 3, new Set(['this-reset']))).toBe(false);
+ });
+
+ it('preserves recovery of ordinary resets after their lifecycle revision advances', () => {
+ expect(isWorkflowResetRecoveryConfirmed(status({ state: 'ready', revision: 4 }), 3, new Set())).toBe(true);
+ expect(isWorkflowResetRecoveryConfirmed(status({ state: 'resetting', revision: 4 }), 3, new Set())).toBe(false);
+ });
+
it.each([
['start', 'running'],
['pause', 'paused'],
diff --git a/client/src/lib/__tests__/connectionConfig.test.ts b/client/src/lib/__tests__/connectionConfig.test.ts
index 9de55db2..b9200c35 100644
--- a/client/src/lib/__tests__/connectionConfig.test.ts
+++ b/client/src/lib/__tests__/connectionConfig.test.ts
@@ -77,10 +77,8 @@ describe('WS_RECONNECT', () => {
describe('WS_CLOSE', () => {
it('NORMAL_CLOSURE matches RFC 6455 §7.4.1', () => {
- // PartySocket inspects this exact value to skip its reconnect
- // loop. Any drift would either re-trigger reconnect on logout
- // (1000 → !1000) or suppress reconnect on real drops (drift the
- // other way). RFC 6455 §7.4.1 fixes this at 1000.
+ // The code identifies normal closure on the wire. Explicit close(),
+ // rather than this numeric value, tells PartySocket to stop retrying.
expect(WS_CLOSE.NORMAL_CLOSURE).toBe(1000);
});
diff --git a/client/src/lib/connectionConfig.ts b/client/src/lib/connectionConfig.ts
index b87996e0..6af585f7 100644
--- a/client/src/lib/connectionConfig.ts
+++ b/client/src/lib/connectionConfig.ts
@@ -86,13 +86,19 @@ export const WS_RECONNECT = {
MAX_ENQUEUED_MESSAGES: 200,
} as const;
+/** Application heartbeat, independent of the server's protocol-level ping. */
+export const WS_HEARTBEAT = {
+ INTERVAL_MS: 30_000,
+ TIMEOUT_MS: 10_000,
+} as const;
+
/**
* WebSocket close codes per RFC 6455 §7.4.1.
* https://datatracker.ietf.org/doc/html/rfc6455#section-7.4.1
*
- * Browsers expose these via `CloseEvent.code`. PartySocket inspects
- * `1000` to skip its reconnect loop (intentional close); every other
- * code is treated as transient and triggers reconnect.
+ * A remote close (including 1000) may reconnect. Calling the wrapper's
+ * close() explicitly stops retries; the event's code does not express
+ * whether the application deliberately disposed the connection.
*
* Only the subset this app actively sends or branches on is listed.
* Add new entries here rather than inlining the numeric literal.
@@ -104,4 +110,6 @@ export const WS_CLOSE = {
* teardown so PartySocket does not reconnect.
*/
NORMAL_CLOSURE: 1000,
+ /** Application-private code used to replace a nonresponsive connection. */
+ HEARTBEAT_TIMEOUT: 4000,
} as const;
diff --git a/client/src/lib/webSocketHeartbeat.ts b/client/src/lib/webSocketHeartbeat.ts
new file mode 100644
index 00000000..c07b2438
--- /dev/null
+++ b/client/src/lib/webSocketHeartbeat.ts
@@ -0,0 +1,51 @@
+import type ReconnectingWebSocket from 'partysocket/ws';
+import { WS_CLOSE, WS_HEARTBEAT } from './connectionConfig';
+
+/** Monitor one OPEN connection; the owner stops this on close or disposal. */
+export function startWebSocketHeartbeat(ws: ReconnectingWebSocket): () => void {
+ let stopped = false;
+ let deadline: ReturnType<typeof setTimeout> | undefined;
+ const clearDeadline = () => {
+ clearTimeout(deadline);
+ deadline = undefined;
+ };
+ const probe = () => {
+ if (stopped || document.hidden || ws.readyState !== ws.OPEN || deadline !== undefined) return;
+ deadline = setTimeout(() => {
+ deadline = undefined;
+ if (!stopped && !document.hidden && ws.readyState === ws.OPEN) {
+ console.warn('[WebSocket] Heartbeat timed out; reconnecting');
+ ws.reconnect(WS_CLOSE.HEARTBEAT_TIMEOUT, 'Heartbeat timed out');
+ }
+ }, WS_HEARTBEAT.TIMEOUT_MS);
+ try {
+ ws.send(JSON.stringify({ type: 'ping' }));
+ } catch {
+ clearDeadline();
+ ws.reconnect(WS_CLOSE.HEARTBEAT_TIMEOUT, 'Heartbeat send failed');
+ }
+ };
+ const onMessage = (event: MessageEvent) => {
+ try {
+ if (JSON.parse(event.data).type === 'pong') clearDeadline();
+ } catch { /* The normal message handler reports invalid frames. */ }
+ };
+ const onVisibility = () => {
+ // Background throttling/suspend must not make an old deadline kill a
+ // healthy socket. A visible page gets a fresh full response window.
+ clearDeadline();
+ if (!document.hidden) probe();
+ };
+ ws.addEventListener('message', onMessage);
+ document.addEventListener('visibilitychange', onVisibility);
+ window.addEventListener('online', onVisibility);
+ const interval = setInterval(probe, WS_HEARTBEAT.INTERVAL_MS);
+ return () => {
+ stopped = true;
+ clearInterval(interval);
+ clearDeadline();
+ ws.removeEventListener('message', onMessage);
+ document.removeEventListener('visibilitychange', onVisibility);
+ window.removeEventListener('online', onVisibility);
+ };
+}
diff --git a/client/vite.config.js b/client/vite.config.js
index 1ffba389..ea865780 100644
--- a/client/vite.config.js
+++ b/client/vite.config.js
@@ -4,6 +4,7 @@ import tailwindcss from '@tailwindcss/vite'
import { resolve } from 'path'
import { readFileSync } from 'fs'
import { visualizer } from 'rollup-plugin-visualizer'
+import { createDevProxyDiagnostics } from './devProxyDiagnostics.js'
// Read root package.json for app version (one level up from client/)
// Falls back to '0.0.0' in Docker where only client/ is in the build context
@@ -13,46 +14,6 @@ try {
appVersion = rootPkg.version
} catch { /* Docker build - root package.json not available */ }
-// Dev-boot race: Vite is ready in ~1 s while uvicorn is still importing
-// routers, so an already-open browser tab reconnects and its proxied /ws
-// upgrade gets ECONNREFUSED. Vite closes the socket and PartySocket
-// reconnects the moment the backend binds, so nothing is broken — but Vite
-// logs the raw AggregateError stack for every attempt, which buries the real
-// startup output.
-//
-// Vite attaches its own proxy `error` handler AFTER calling `configure()`, so
-// a proxy-level handler cannot replace it. `customLogger` is the documented
-// seam: collapse this one connection-refused case into a single concise line
-// and pass every other message through untouched.
-// Ref: https://vite.dev/config/shared-options.html#customlogger
-const PROXY_BACKEND_UNAVAILABLE = /proxy (error|socket error)/i
-const CONNECTION_REFUSED = /ECONNREFUSED/
-const BACKEND_NOTICE_INTERVAL_MS = 10_000
-
-const createBackendAwareLogger = () => {
- const logger = createLogger()
- const logError = logger.error.bind(logger)
- const logInfo = logger.info.bind(logger)
- let lastNoticeAt = 0
-
- logger.error = (msg, options) => {
- const text = typeof msg === 'string' ? msg : String(msg ?? '')
- if (PROXY_BACKEND_UNAVAILABLE.test(text) && CONNECTION_REFUSED.test(text)) {
- const now = Date.now()
- if (now - lastNoticeAt >= BACKEND_NOTICE_INTERVAL_MS) {
- lastNoticeAt = now
- logInfo('backend not listening yet - proxied /ws and /api will retry', {
- timestamp: true,
- })
- }
- return
- }
- logError(msg, options)
- }
-
- return logger
-}
-
// https://vite.dev/config/
export default defineConfig(({ mode }) => {
// Load env from parent directory (root .env) for local development
@@ -103,8 +64,10 @@ export default defineConfig(({ mode }) => {
},
}
+ const proxyDiagnostics = createDevProxyDiagnostics(createLogger())
+
return {
- customLogger: createBackendAwareLogger(),
+ customLogger: proxyDiagnostics.logger,
plugins: [
tailwindcss(),
react({
@@ -163,6 +126,7 @@ export default defineConfig(({ mode }) => {
target: `http://localhost:${requireEnv('PYTHON_BACKEND_PORT')}`,
changeOrigin: false,
ws: prefix === '/ws',
+ configure: prefix === '/ws' ? proxyDiagnostics.configureWebSocketProxy : undefined,
},
])
),
diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.md
index 4b9aa3a8..3c09a43d 100644
--- a/docs/mobile-workspace.md
+++ b/docs/mobile-workspace.md
@@ -104,6 +104,16 @@ Resuming starts a fresh agent worker against the current device state with a new
Workspace submissions use a client-generated submission UUID and a durable Temporal workflow identity scoped to principal, saved workflow, and node. Repeating a submission returns the existing task; reusing its UUID with a different prompt is rejected. Mobile task activities have automatic retries disabled. This is not a guarantee that arbitrary device actions are globally exactly-once across host failure.
+### Resetting phone tasks
+
+Use the workflow toolbar's **Reset** for phone tasks submitted through **Ask AI to use the phone**, as well as phone work invoked by the workflow. Direct Workspace submissions are tracked by a per-workflow Temporal `WorkspaceTaskControllerWorkflow`, so Reset is available even when the regular workflow has never started.
+
+Reset blocks new Workspace submissions, cancels queued and running tasks for this workflow, and waits for cancellation and phone cleanup before reporting completion. Other workflows' phone tasks are unaffected. If cleanup or Temporal status is unavailable, Reset remains unconfirmed and can be retried; an already-ready workflow alone is not proof that its phone task stopped. The workflow remains stopped afterward.
+
+The phone stays running, with its apps, sign-ins, and existing human-control lease preserved. Reset clears this workflow's task and activity panel and browser submission recovery IDs, then loads a fresh phone status. If you were controlling the phone, choose **Use phone** again to reconnect this view's manual controls. Completed task history remains in Temporal.
+
+After upgrading to this implementation, restart the backend and its Temporal worker so they register `WorkspaceTaskControllerWorkflow` and its cleanup activity. The process-local device broker still requires the backend and mobile activity worker to share the supported single-process deployment described above.
+
## Engine isolation and credentials
OpenCompany installs mobile-use into its own Python 3.12 environment. The main server does not import its agent dependencies. Each task receives a separate worker process; a trusted device process owns the explicitly selected emulator serial. The worker's injected controller talks to a loopback capability broker. It does not choose an ADB serial or receive an arbitrary host-command API.
@@ -191,12 +201,46 @@ Passing a source or fake-process test does not count as passing the correspondin
### Live Android activity
Workspace shows the active phone task's model/provider, step budget, total elapsed time,
-current phase duration, and time since the last reported activity. The expandable activity
-panel retains the latest 40 events: model requests/responses and actual device operations
-(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).
-Completed operations include duration. The last task remains visible after completion,
-failure, or cancellation until another task runs; history is in memory and clears on restart.
+current phase duration, and time since the last reported activity. The main activity panel
+shows the SDK's subtask plan and current subtask, followed by the latest 40 plan/action
+updates. Actions include the selected app or screen target, the SDK tool's explicit intent,
+duration, and confirmed tool outcome. A tool completing an action does not mark a subtask
+complete: subtask status comes from the SDK's planner/orchestrator updates. Interrupted
+actions whose outcomes were not confirmed are shown as unconfirmed.
+
+The counter is labeled **Engine step** because it measures LangGraph execution steps,
+including planning and coordination, rather than taps or completed subtasks. Nested chain
+callbacks and graph branches at the same step do not generate duplicate counter events.
+The current subtask stays visible across model waits and device reads. A resumed worker
+creates a fresh plan from the current screen; earlier activity remains in the timeline.
+
+A separate collapsed **Technical activity** section retains the latest 40 phase/device
+events: model requests/responses, screen/accessibility-tree reads, and device operations.
+Consecutive identical phase messages are coalesced. Completed operations include duration.
+The last task remains visible after completion, failure, or cancellation until another task
+runs; history is in memory and clears on restart. Older runs cannot recover details that
+were not captured at the time.
+
A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last
phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.
-The panel is outside the phone canvas and hidden in full-screen mode. Activity contains
-operational summaries, never private model reasoning, typed text, URLs, or prompt content.
+The panel is outside the phone canvas and hidden in full-screen mode. Detailed summaries
+are bounded and available only through the owner-only mobile status response. They can
+contain task-specific plan descriptions and screen labels. Typed tool payloads and input
+tools' echoed field contents are omitted, and URL strings are redacted. These summaries
+are not sent to diagnostic logs or the generic workflow status broadcast. Raw model
+responses, provider thinking blocks, cumulative agent thoughts, screenshots, and injected
+graph state are never forwarded to the activity panel.
+
+This integration follows the SDK's [Agent configuration](https://www.minitap.ai/docs/mobile-use-sdk/sdk-reference/agent)
+and the pinned source's `AgentConfigBuilder.with_graph_config_callbacks(...)`: graph node
+results expose `subgoal_plan`, and tool callbacks expose intent/arguments plus
+`Command.update.executor_messages` with `ToolMessage.status`. The SDK's internal
+`on_plan_changes` hook is not exposed by this pinned Agent configuration. No additional
+upstream patch or dependency is needed. [Local trace recording](https://www.minitap.ai/docs/mobile-use-sdk/core-concepts/observability)
+is a separate feature that creates screenshots and step artifacts; `record_trace=False`
+remains intentional for the live activity panel.
+
+Regression coverage includes callback filtering, SDK-shaped plans, tool error results,
+input-content filtering, interruption, bounded histories, and preserving subtask context
+during quiet waits. An optional test executes a mock graph with the installed pinned SDK
+and its real executor tool node, without a model request or device connection.
diff --git a/server/nodes/mobile/_node.py b/server/nodes/mobile/_node.py
index 023e8b4d..39230fd4 100644
--- a/server/nodes/mobile/_node.py
+++ b/server/nodes/mobile/_node.py
@@ -147,6 +147,16 @@ class MobileUseAgent(ActionNode):
Params = MobileParams
Output = MobileOutput
+ @classmethod
+ async def reset_execution_state(cls, *, node_id: str, workflow_id: str, execution_id: str,
+ generation: int, graph: dict, database: Any) -> dict:
+ from ._runtime import peek_runtime
+
+ runtime = peek_runtime()
+ if runtime is None:
+ return {"reset": False}
+ return await runtime.reset_execution_state(workflow_id=workflow_id, node_id=node_id, generation=generation)
+
@Operation("execute")
async def execute_op(self, ctx: NodeContext, params: MobileParams) -> dict:
from ._runtime import get_runtime
@@ -164,15 +174,25 @@ class MobileUseAgent(ActionNode):
if directive and directive != config["prompt"]:
config["prompt"] = f"{directive}\n\n{config['prompt']}"
try:
+ broker_url = await runtime.ensure_broker()
+ raw_generation = ctx.raw.get("generation", 0)
+ generation = raw_generation if isinstance(raw_generation, int) and not isinstance(raw_generation, bool) else 0
+ if generation > 0:
+ from services.plugin.deps import get_database
+
+ control = await get_database().get_latest_workflow_control(ctx.workflow_id)
+ if control is None or control.generation != generation or control.status in {"resetting", "reset"}:
+ raise NodeUserError("This mobile task belongs to an inactive workflow generation")
return await runtime.run(
principal=ctx.user_id,
workflow_id=ctx.workflow_id,
node_id=ctx.node_id,
run_id=task_identity(ctx),
execution_id=ctx.execution_id,
+ generation=generation,
params=config,
model=model,
- broker_url=await runtime.ensure_broker(),
+ broker_url=broker_url,
)
except MobileError as exc:
raise NodeUserError(str(exc)) from None
diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.py
index b08c57ec..655285f5 100644
--- a/server/nodes/mobile/_runtime.py
+++ b/server/nodes/mobile/_runtime.py
@@ -15,6 +15,7 @@ from ._paths import mobile_root, runtime_python
from ._process import command, hidden_options
from ._diagnostics import event, recent
from ._emulator import DeviceLock, recover_emulator, boot_error, owned_children, stop_children
+from .runtime.progress import plan_summary, summary
READS = frozenset({"geometry", "observe", "date", "packages", "foreground"})
ACTIONS = frozenset({"tap", "swipe", "touch", "text", "erase", "key", "launch", "terminate", "url", "rotate"})
@@ -42,6 +43,13 @@ class MobileRuntime:
self.active: dict | None = None
self.last_task: dict | None = None
self.queue: list[dict] = []
+ # Execution handles are deliberately separate from public task snapshots.
+ # Completion means run() has drained its worker and admitted device I/O.
+ self._run_tasks: dict[str, tuple[dict, asyncio.Task, asyncio.Event]] = {}
+ self._worker_stop_tasks: set[asyncio.Task] = set()
+ self._reset_runs: set[str] = set()
+ self._draining_runs: set[str] = set()
+ self._retired_generations: dict[tuple[str, str], int] = {}
self.capabilities: dict[str, tuple[str, Lease]] = {}
self.resume_event = asyncio.Event()
self.viewer: str | None = None
@@ -476,22 +484,77 @@ class MobileRuntime:
def matches(entry):
return entry["workflow_id"] == workflow_id and entry["node_id"] == node_id and (run_id is None or entry["run_id"] == run_id)
- for entry in self.queue:
- if matches(entry):
- entry["status"] = "cancelled"
- if self.active and matches(self.active):
- self.active["status"] = "cancelled"
- self.control.revoke("run:" + self.active["run_id"])
- self.capabilities.clear()
- self.resume_event.set()
- await self._stop_worker()
+ await self._cancel_matching(matches)
+
+ async def reset_execution_state(self, *, workflow_id: str, node_id: str, generation: int) -> dict:
+ if generation < 0:
+ raise MobileError("invalid_generation", "Reset generation cannot be negative")
+ # Retire admission before the first await so a late graph activity cannot
+ # enqueue while reset is draining a previously admitted task.
+ if generation > 0:
+ key = (workflow_id, node_id)
+ self._retired_generations[key] = max(generation, self._retired_generations.get(key, 0))
+
+ def matches(entry):
+ return (
+ entry["workflow_id"] == workflow_id and entry["node_id"] == node_id
+ # A phone-only reset must not cancel a graph generation admitted
+ # concurrently by Start. Zero identifies direct Workspace work.
+ and (entry.get("generation", 0) == 0 if generation == 0 else entry.get("generation", 0) <= generation)
+ )
- async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str, execution_id: str | None = None) -> dict:
+ await self._cancel_matching(matches, clear=True)
+ return {"reset": True}
+
+ async def _cancel_matching(self, matches, *, clear: bool = False) -> None:
+ records = [record for record in self._run_tasks.values() if matches(record[0])]
+ entries = [entry for entry in self.queue if matches(entry)]
+ active = self.active if self.active and matches(self.active) else None
+ if active is not None:
+ entries.append(active)
+ run_ids = {entry["run_id"] for entry in entries}
+ for entry in entries:
+ entry["status"] = "cancelled"
+ if clear:
+ self._reset_runs.update(run_ids)
+ if self.last_task and matches(self.last_task):
+ self.last_task = None
+ # Revoke before cancellation/draining. Keep manual and unrelated leases.
+ for run_id in run_ids:
+ self.control.revoke("run:" + run_id)
+ for capability, (run_id, _) in list(self.capabilities.items()):
+ if run_id in run_ids:
+ self.capabilities.pop(capability, None)
+ for entry, task, _ in records:
+ # A second reset must not interrupt a first cancellation's finally.
+ if not task.done() and not task.cancelling() and entry["run_id"] not in self._draining_runs:
+ task.cancel()
+ if records:
+ await asyncio.gather(*(done.wait() for _, _, done in records))
+ if active is not None and self.active is active:
+ raise MobileError("reset_failed", "Mobile task cleanup did not finish; retry reset")
+ elif active is not None:
+ # Also handle state restored/manually owned without a live run handle.
+ await self._stop_worker()
+ async with self.control.lock:
+ pass
+ if self.active is active:
+ self.active = None
+ if clear:
+ if self.last_task and matches(self.last_task):
+ self.last_task = None
+ self._reset_runs.difference_update(run_ids)
+
+ async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str, execution_id: str | None = None, generation: int = 0) -> dict:
+ if generation > 0 and generation <= self._retired_generations.get((workflow_id, node_id), 0):
+ raise MobileError("stale_generation", "This mobile task belongs to a reset workflow generation")
if len(self.queue) >= 32:
raise MobileError("queue_full", "The shared device queue is full")
if any(q["run_id"] == run_id for q in self.queue) or (self.active and self.active["run_id"] == run_id):
raise MobileError("duplicate_run", "This task is already admitted")
- entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "status": "queued", "steps": 0, "max_steps": params["max_steps"]}
+ entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "generation": generation, "status": "queued", "steps": 0, "max_steps": params["max_steps"]}
+ done = asyncio.Event()
+ self._run_tasks[run_id] = (entry, asyncio.current_task(), done)
self.queue.append(entry)
event("task_queued", run_id=run_id, workflow_id=workflow_id, node_id=node_id)
try:
@@ -505,11 +568,11 @@ class MobileRuntime:
raise MobileError("device_offline", "Start the shared device in Workspace first")
self.active = entry
entry.update(started_at=time.time(), provider=model.get("provider"), model=model.get("model"))
- await self._publish_progress("Preparing phone")
active_remaining = params["timeout_s"]
steps_remaining = params["max_steps"]
attempts = 0
try:
+ await self._publish_progress("Preparing phone")
while True:
if entry["status"] == "cancelled":
raise asyncio.CancelledError()
@@ -582,14 +645,18 @@ class MobileRuntime:
event("task_completed", run_id=run_id)
return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}
finally:
- self.capabilities.clear()
+ self._draining_runs.add(run_id)
+ for capability, (owner_run, _) in list(self.capabilities.items()):
+ if owner_run == run_id:
+ self.capabilities.pop(capability, None)
self.control.revoke("run:" + run_id)
await self._stop_worker()
# Wait for any previously admitted broker write before reporting completion.
async with self.control.lock:
pass
entry["finished_at"] = time.time()
- self.last_task = entry
+ if run_id not in self._reset_runs:
+ self.last_task = entry
self.active = None
except asyncio.CancelledError:
entry["status"] = "cancelled"
@@ -603,8 +670,11 @@ class MobileRuntime:
finally:
if entry in self.queue:
self.queue.remove(entry)
+ self._run_tasks.pop(run_id, None)
+ self._draining_runs.discard(run_id)
+ done.set()
- async def _publish_progress(self, phase: str) -> None:
+ async def _publish_progress(self, phase: str, *, record: bool = True) -> None:
if not self.active:
return
entry = self.active
@@ -614,7 +684,8 @@ class MobileRuntime:
entry["phase"] = phase
entry["updated_at"] = now
history = entry.setdefault("activity", [])
- history.append({"at": now, "message": phase, "step": entry.get("steps", 0)})
+ if record and (not history or history[-1]["message"] != phase):
+ history.append({"at": now, "message": phase, "step": entry.get("steps", 0)})
del history[:-40]
try:
from services.status_broadcaster import get_status_broadcaster
@@ -627,6 +698,58 @@ class MobileRuntime:
except Exception as exc:
event("progress_delivery_failed", failed=True, error_type=type(exc).__name__)
+ async def _publish_agent_progress(self, message: dict) -> None:
+ """Detailed summaries stay in the owner-only snapshot, never diagnostic logs."""
+ if not self.active or message.get("kind") not in {"plan", "action"}:
+ return
+ entry = self.active
+ now = time.time()
+ kind = message["kind"]
+ item = {"at": now, "step": entry.get("steps", 0), "kind": kind,
+ "message": summary(message.get("message"), 400), "detail": summary(message.get("detail"))}
+ if not item["message"]:
+ return
+ history = entry.setdefault("agent_activity", [])
+ if kind == "plan":
+ plan = plan_summary(message.get("plan"))
+ if plan is None:
+ return
+ entry["plan"] = plan
+ entry["current_goal"] = next((goal["description"] for goal in plan if goal["status"] == "pending"), None)
+ history.append(item)
+ phase = "Updating task plan"
+ else:
+ action_id = summary(message.get("action_id"), 100)
+ state = message.get("state")
+ if not action_id or state not in {"started", "completed", "failed", "returned"}:
+ return
+ item.update(action_id=action_id, state=state, outcome=summary(message.get("outcome")))
+ duration = message.get("duration_ms")
+ if isinstance(duration, (int, float)) and 0 <= duration <= 86_400_000:
+ item["duration_ms"] = round(duration)
+ previous = next((row for row in reversed(history) if row.get("action_id") == action_id), None)
+ if previous:
+ # Keep one row and the original engine step for each actual tool call.
+ item.update(at=previous["at"], step=previous["step"], finished_at=now)
+ previous.update(item)
+ item = previous
+ else:
+ history.append(item)
+ if state == "started" or not entry.get("current_action") or entry["current_action"].get("action_id") == action_id:
+ entry["current_action"] = item
+ phase = {"started": "Executing phone action", "completed": "Phone action completed",
+ "failed": "Phone action failed", "returned": "Checking action outcome"}[state]
+ del history[:-40]
+ await self._publish_progress(phase, record=False)
+
+ def _finish_pending_actions(self) -> None:
+ if not self.active:
+ return
+ for item in self.active.get("agent_activity", []):
+ if item.get("state") == "started":
+ item.update(state="returned", finished_at=time.time(),
+ outcome="Action interrupted before the tool confirmed its outcome.")
+
async def _run_worker(self, config: dict, timeout: float) -> dict:
entry = Path(__file__).with_name("runtime") / "worker.py"
self.worker = await asyncio.create_subprocess_exec(
@@ -645,6 +768,7 @@ class MobileRuntime:
if config["capability"] not in self.capabilities or not self.active or self.active["status"] != "running":
await self._stop_worker()
return {"success": False, "error": "Task admission revoked"}
+ self.active.update(plan=[], current_goal=None, current_action=None)
proc.stdin.write((json.dumps(config) + "\n").encode())
await proc.stdin.drain()
proc.stdin.close()
@@ -681,10 +805,12 @@ class MobileRuntime:
duration = message.get("duration_ms")
suffix = f" ({duration / 1000:.1f}s)" if isinstance(duration, (int, float)) and duration >= 0 else ""
await self._publish_progress(label if state == "started" else f"{label}: {state}{suffix}")
+ elif message.get("type") == "agent_update":
+ await self._publish_agent_progress(message)
elif message.get("type") == "diagnostic":
event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))
phase = {"model_request": "Waiting for model", "model_response": "Model responded",
- "engine_initialization": "Connecting engine", "task_execution": "Using phone",
+ "engine_initialization": "Connecting engine", "task_execution": "Preparing task plan",
"engine_cleanup": "Finishing engine cleanup"}.get(message.get("stage"))
stage = message.get("stage")
request_id = str(message.get("request_id", "model"))
@@ -714,7 +840,7 @@ class MobileRuntime:
if steps != self.active.get("attempt_steps", 0):
event("task_progress", **scope, steps=steps)
self.active["steps"] = min(self.active.get("max_steps", steps), self.active.get("step_offset", 0) + steps)
- await self._publish_progress("Using phone")
+ await self._publish_progress(self.active.get("phase", "Preparing task plan"), record=False)
self.active["attempt_steps"] = steps
await proc.wait()
event("engine_exited", failed=proc.returncode not in (0, None), **scope, exit_code=proc.returncode)
@@ -725,13 +851,23 @@ class MobileRuntime:
async def _stop_worker(self):
proc, self.worker = self.worker, None
if proc and proc.returncode is None:
- with contextlib.suppress(ProcessLookupError):
- proc.terminate()
- try:
- await asyncio.wait_for(proc.wait(), 5)
- except asyncio.TimeoutError:
- proc.kill()
- await proc.wait()
+ task = asyncio.create_task(self._terminate_worker(proc))
+ self._worker_stop_tasks.add(task)
+ task.add_done_callback(self._worker_stop_tasks.discard)
+ if self._worker_stop_tasks:
+ # Takeover and reset can overlap. Both must await an already-started
+ # worker stop even after self.worker has been detached.
+ await asyncio.shield(asyncio.gather(*self._worker_stop_tasks))
+ self._finish_pending_actions()
+
+ async def _terminate_worker(self, proc):
+ with contextlib.suppress(ProcessLookupError):
+ proc.terminate()
+ try:
+ await asyncio.wait_for(proc.wait(), 5)
+ except asyncio.TimeoutError:
+ proc.kill()
+ await proc.wait()
async def _stop_driver(self):
proc, self.driver = self.driver, None
diff --git a/server/nodes/mobile/_tool.py b/server/nodes/mobile/_tool.py
index 67e7c9de..2984e864 100644
--- a/server/nodes/mobile/_tool.py
+++ b/server/nodes/mobile/_tool.py
@@ -39,6 +39,10 @@ class AndroidTool(ToolNode):
ToolInput = AndroidTaskInput
Output = MobileOutput
+ @classmethod
+ async def reset_execution_state(cls, **kwargs) -> dict:
+ return await MobileUseAgent.reset_execution_state(**kwargs)
+
@classmethod
def interpret_result(cls, result):
# Agent tool invocation flattens failures to {"error": ...}. Do not
diff --git a/server/nodes/mobile/runtime/progress.py b/server/nodes/mobile/runtime/progress.py
new file mode 100644
index 00000000..c0d42313
--- /dev/null
+++ b/server/nodes/mobile/runtime/progress.py
@@ -0,0 +1,179 @@
+"""Bounded, user-facing mobile-use progress; no SDK imports in the server.
+
+The pinned SDK exposes plans through graph updates and action intent/targets through
+tool callbacks. Never serialize graph state, prompts, screenshots, or LLM responses.
+These summaries belong to the owner-only in-memory task status, not diagnostic logs.
+"""
+
+from __future__ import annotations
+
+import re
+import threading
+import time
+
+
+NODES = frozenset({"planner", "orchestrator", "contextor", "cortex", "executor", "executor_tools", "summarizer", "convergence"})
+TOOLS = {
+ "tap": "Tap", "long_press_on": "Long press", "focus_and_input_text": "Type",
+ "focus_and_clear_text": "Clear text", "erase_one_char": "Erase one character",
+ "launch_app": "Open app", "stop_app": "Close app", "back": "Go back",
+ "press_key": "Press", "open_link": "Open link", "wait_for_delay": "Wait",
+ "swipe": "Swipe", "swipe_coordinates": "Swipe", "swipe_percentages": "Swipe",
+}
+
+
+def field(value, key, default=None):
+ return value.get(key, default) if isinstance(value, dict) else getattr(value, key, default)
+
+
+def summary(value, limit=500):
+ if not isinstance(value, str):
+ return ""
+ # URLs can contain login tokens. Render all summaries as plain text in the UI.
+ value = re.sub(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s<>]+", "[link]", value)
+ value = " ".join(value.split())
+ return value if len(value) <= limit else value[:limit - 1] + "…"
+
+
+def plan_summary(value):
+ if not isinstance(value, list):
+ return None
+ result = []
+ for item in value[:40]:
+ description = summary(field(item, "description"), 300)
+ status = field(item, "status")
+ status = str(getattr(status, "value", status)).lower()
+ if description and status in {"not_started", "pending", "success", "failure"}:
+ result.append({"id": summary(field(item, "id"), 100), "description": description,
+ "status": status, "reason": summary(field(item, "completion_reason") or field(item, "reason"), 300)})
+ return result
+
+
+def action_summary(name, args):
+ """Select display fields, excluding typed payloads and injected state."""
+ label = TOOLS.get(name)
+ if not label or not isinstance(args, dict):
+ return None
+ target = args.get("target")
+ target_label = summary(field(target, "text"), 100) or summary(field(target, "resource_id"), 120)
+ if not target_label and target:
+ bounds = field(target, "bounds")
+ coords = [field(bounds, key) for key in ("x", "y", "width", "height")]
+ if all(isinstance(v, (int, float)) for v in coords):
+ target_label = f"area ({coords[0]}, {coords[1]}, {coords[2]} × {coords[3]})"
+ if name in {"tap", "long_press_on", "focus_and_clear_text"}:
+ label += f" · {target_label or 'screen target'}"
+ elif name == "focus_and_input_text":
+ text = args.get("text")
+ if target_label == text:
+ target_label = summary(field(target, "resource_id"), 120)
+ label = f"Type {len(text) if isinstance(text, str) else 0} characters · {target_label or 'text field'}"
+ elif name in {"launch_app", "stop_app"}:
+ label += " · " + (summary(args.get("app_name") or args.get("package_name"), 140) or "current app")
+ elif name == "press_key":
+ key = args.get("key")
+ key = getattr(key, "value", key)
+ label += " · " + (key.title() if isinstance(key, str) and key.lower() in {"home", "back", "enter"} else "navigation key")
+ elif name in {"swipe_coordinates", "swipe_percentages"}:
+ suffix = "_percent" if name == "swipe_percentages" else ""
+ coords = [args.get(key + suffix) for key in ("start_x", "start_y", "end_x", "end_y")]
+ if all(isinstance(v, (int, float)) for v in coords):
+ unit = "%" if suffix else "px"
+ label += f" · ({coords[0]}, {coords[1]}) → ({coords[2]}, {coords[3]}) {unit}"
+ elif name == "wait_for_delay":
+ duration = args.get("time_in_ms")
+ if isinstance(duration, (int, float)):
+ label += f" · {min(max(0, duration), 60000) / 1000:g}s"
+ # agent_thought is the SDK's explicit tool intent, not provider thinking blocks.
+ intent = args.get("agent_thought")
+ if name == "focus_and_input_text" and isinstance(args.get("text"), str) and args["text"]:
+ if isinstance(intent, str):
+ # Replacing a single typed letter would corrupt every word in the intent.
+ intent = intent.replace(args["text"], "[typed text]") if len(args["text"]) > 2 else "Entering text in the selected field."
+ return {"message": summary(label, 240), "detail": summary(intent)}
+
+
+class ProgressCallbacks:
+ """Mixin for the isolated engine's BaseCallbackHandler (callbacks may be threaded)."""
+
+ def __init__(self, emit):
+ self.emit = emit
+ self._lock = threading.RLock()
+ self._nodes = {}
+ self._tools = {}
+ self._step = -1
+ self._plan = None
+
+ def on_chain_start(self, serialized, inputs, *, run_id, metadata=None, name=None, tags=None, **kwargs):
+ role = (metadata or {}).get("langgraph_node")
+ if role not in NODES or name != role or not any(tag.startswith("graph:step:") for tag in tags or []):
+ return
+ with self._lock:
+ self._nodes[str(run_id)] = role
+ step = (metadata or {}).get("langgraph_step")
+ if isinstance(step, int) and step > self._step:
+ self._step = max(0, step)
+ self.emit("progress", steps=self._step)
+
+ def on_chain_end(self, outputs, *, run_id, **kwargs):
+ with self._lock:
+ role = self._nodes.pop(str(run_id), None)
+ if role not in {"planner", "orchestrator"}:
+ return
+ plan = plan_summary(field(outputs, "subgoal_plan"))
+ if plan is None or plan == self._plan:
+ return
+ first = self._plan is None
+ self._plan = plan
+ current = next((item["description"] for item in plan if item["status"] == "pending"), None)
+ completed = sum(item["status"] == "success" for item in plan)
+ message = f"Working on: {current}" if current else (
+ f"Plan created: {len(plan)} subtasks" if first else f"Plan updated: {completed}/{len(plan)} subtasks done"
+ )
+ self.emit("agent_update", kind="plan", message=message, plan=plan, current_goal=current)
+
+ def on_chain_error(self, error, *, run_id, **kwargs):
+ with self._lock:
+ self._nodes.pop(str(run_id), None)
+
+ def on_tool_start(self, serialized, input_str, *, run_id, inputs=None, **kwargs):
+ name = field(serialized, "name")
+ action = action_summary(name, inputs)
+ if not action:
+ return
+ with self._lock:
+ key = str(run_id)
+ self._tools[key] = (name, action, time.monotonic())
+ self.emit("agent_update", kind="action", action_id=key, state="started", **action)
+
+ def on_tool_end(self, output, *, run_id, **kwargs):
+ with self._lock:
+ saved = self._tools.pop(str(run_id), None)
+ if not saved:
+ return
+ name, action, started = saved
+ # SDK tools return Command(update={executor_messages: [ToolMessage]}).
+ update = field(output, "update")
+ messages = field(update, "executor_messages", []) if update is not None else [output]
+ statuses = [field(item, "status") for item in messages if field(item, "status") in {"success", "error"}]
+ state = "failed" if "error" in statuses else "completed" if statuses else "returned"
+ outcome = "Tool returned without a confirmed outcome."
+ if state == "failed":
+ outcome = "The action failed. The agent will reassess the screen."
+ elif state == "completed":
+ outcome = "Action completed; task progress is checked separately by the agent."
+ # Input tools echo field contents in their results: never forward those.
+ if name not in {"focus_and_input_text", "focus_and_clear_text", "open_link"}:
+ result = next((item for item in reversed(messages) if field(item, "status") == ("error" if state == "failed" else "success")), None)
+ outcome = summary(field(result, "content")) or outcome
+ self.emit("agent_update", kind="action", action_id=str(run_id), state=state,
+ duration_ms=round((time.monotonic() - started) * 1000), outcome=outcome, **action)
+
+ def on_tool_error(self, error, *, run_id, **kwargs):
+ with self._lock:
+ saved = self._tools.pop(str(run_id), None)
+ if saved:
+ _, action, started = saved
+ self.emit("agent_update", kind="action", action_id=str(run_id), state="failed",
+ duration_ms=round((time.monotonic() - started) * 1000),
+ outcome="The tool raised an error before confirming the action.", **action)
diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.py
index cd6f0a9d..02fd9a47 100644
--- a/server/nodes/mobile/runtime/worker.py
+++ b/server/nodes/mobile/runtime/worker.py
@@ -15,17 +15,19 @@ from io import BytesIO
try:
from .errors import describe_error
+ from .progress import ProgressCallbacks
except ImportError: # Executed by path in the isolated engine environment.
from errors import describe_error
+ from progress import ProgressCallbacks
STAGE = "bootstrap"
WIRE = sys.stdout
WIRE_LOCK = threading.Lock()
-def emit(kind: str, **data) -> None:
+def emit(event_type: str, **data) -> None:
with WIRE_LOCK:
- WIRE.write(json.dumps({"v": 1, "type": kind, **data}, default=str) + "\n")
+ WIRE.write(json.dumps({"v": 1, "type": event_type, **data}, default=str) + "\n")
WIRE.flush()
@@ -159,12 +161,7 @@ async def main(config: dict) -> None:
# Only this isolated engine process uses LangChain; the server does not.
from langchain_core.callbacks import BaseCallbackHandler
- class Progress(BaseCallbackHandler):
- def on_chain_start(self, serialized, inputs, *, metadata=None, **kwargs):
- step = (metadata or {}).get("langgraph_step")
- if isinstance(step, int):
- emit("progress", steps=max(0, step))
-
+ class Progress(ProgressCallbacks, BaseCallbackHandler):
def on_chat_model_start(self, serialized, messages, **kwargs):
role = (kwargs.get("metadata") or {}).get("langgraph_node")
allowed = {"planner", "orchestrator", "contextor", "cortex", "executor", "outputter", "hopper"}
@@ -198,7 +195,7 @@ async def main(config: dict) -> None:
AgentConfigBuilder()
.with_default_profile(profile)
.for_device(DevicePlatform(config["platform"]), config["serial"])
- .with_graph_config_callbacks([Progress()])
+ .with_graph_config_callbacks([Progress(emit)])
.build()
)
agent_config.local_controller = Controller(config)
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index 2b5b7360..3d7faf68 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -1606,7 +1606,15 @@ async def websocket_status_endpoint(websocket: WebSocket):
websocket.state.user_id = authenticated_user_id
broadcaster = get_status_broadcaster()
- await broadcaster.connect(websocket)
+ try:
+ await broadcaster.connect(websocket)
+ except BaseException:
+ # connect registers the socket before delivering initial snapshots.
+ # A cancellation there must not leave a connection outside the
+ # receive/process cleanup guard below. disconnect is idempotent if
+ # acceptance itself failed before registration.
+ await broadcaster.disconnect(websocket)
+ raise
# Message queue for decoupling receive from processing
message_queue: asyncio.Queue = asyncio.Queue()
@@ -1621,8 +1629,12 @@ async def websocket_status_endpoint(websocket: WebSocket):
while True:
data = await websocket.receive_json()
await message_queue.put(data)
- except WebSocketDisconnect:
- # Don't log here - logging during shutdown can raise KeyboardInterrupt
+ except WebSocketDisconnect as exc:
+ # Routine tab closes and server restarts should not look like
+ # failures, but abnormal closes need enough context to diagnose.
+ log = logger.debug if exc.code in (1000, 1001) else logger.warning
+ reason = "".join(character if character.isprintable() else "?" for character in str(exc.reason or "")[:123])
+ log("[WebSocket] Client disconnected", path="/ws/status", close_code=exc.code, reason=reason)
await message_queue.put(None) # Signal shutdown
except asyncio.CancelledError:
# Task cancelled during shutdown - this is expected
diff --git a/server/services/authz/ws_session.py b/server/services/authz/ws_session.py
index cf837bc4..db441ad8 100644
--- a/server/services/authz/ws_session.py
+++ b/server/services/authz/ws_session.py
@@ -94,11 +94,33 @@ def _origin_refused(websocket: Any, settings: Any) -> bool:
origin = headers.get("origin")
if is_allowed_ws_origin(origin, headers.get("host"), getattr(settings, "cors_origins", None) or ()):
return False
- path = str((getattr(websocket, "scope", None) or {}).get("path") or "")
- logger.warning("[WebSocket] Refused a handshake from a foreign origin", path=path, origin=origin)
return True
+async def _reject_handshake(websocket: Any, *, code: int, reason: str, category: str) -> None:
+ """Record why admission failed without logging request credentials.
+
+ ASGI translates a close before acceptance into HTTP 403; the browser
+ never receives the application close code, so server logs must retain
+ both it and the denial reason to diagnose failed upgrades.
+ """
+ scope = getattr(websocket, "scope", None) or {}
+ route = scope.get("route")
+ path = str(getattr(route, "path", None) or scope.get("path") or "")
+ # Prefer the route template, exclude queries even for non-ASGI callers,
+ # and bound/control-clean the only request-derived diagnostic field.
+ path = path.split("?", 1)[0].split("#", 1)[0]
+ path = "".join(character if character.isprintable() else "?" for character in path[:160])
+ logger.warning(
+ "[WebSocket] Handshake rejected",
+ path=path,
+ reason=category,
+ close_code=code,
+ http_status=403,
+ )
+ await websocket.close(code=code, reason=reason)
+
+
def _auth_disabled(settings: Any) -> bool:
value = getattr(settings, "vite_auth_enabled", None)
return bool(value) and str(value).lower() == "false"
@@ -112,7 +134,7 @@ async def authenticate_ws(websocket: Any, *, settings: Any, user_auth_service: C
every same-origin caller is the owner.
"""
if _origin_refused(websocket, settings):
- await websocket.close(code=4003, reason="Origin not allowed")
+ await _reject_handshake(websocket, code=4003, reason="Origin not allowed", category="origin_not_allowed")
return None
if _auth_disabled(settings):
@@ -124,17 +146,17 @@ async def authenticate_ws(websocket: Any, *, settings: Any, user_auth_service: C
token = get_session_token(websocket.cookies, settings)
if not token:
- await websocket.close(code=4001, reason="Not authenticated")
+ await _reject_handshake(websocket, code=4001, reason="Not authenticated", category="missing_session")
return None
payload = user_auth_service().verify_token(token)
if not payload:
- await websocket.close(code=4001, reason="Invalid or expired session")
+ await _reject_handshake(websocket, code=4001, reason="Invalid or expired session", category="invalid_session")
return None
principal = str(payload.get("sub") or "")
if not principal:
- await websocket.close(code=4001, reason="Invalid session subject")
+ await _reject_handshake(websocket, code=4001, reason="Invalid session subject", category="invalid_session_subject")
return None
return principal
@@ -142,12 +164,10 @@ async def authenticate_ws(websocket: Any, *, settings: Any, user_auth_service: C
async def admit_internal_ws(websocket: Any, *, settings: Any) -> bool:
"""Admit the worker socket, or close it and return ``False``."""
if _origin_refused(websocket, settings):
- await websocket.close(code=4003, reason="Origin not allowed")
+ await _reject_handshake(websocket, code=4003, reason="Origin not allowed", category="origin_not_allowed")
return False
if not is_internal_caller(websocket.headers, settings.secret_key):
- client = websocket.client.host if getattr(websocket, "client", None) else "unknown"
- logger.warning("[WebSocket Internal] Refused a connection without the worker token", client=client)
- await websocket.close(code=4001, reason="Not authenticated")
+ await _reject_handshake(websocket, code=4001, reason="Not authenticated", category="invalid_worker_token")
return False
return True
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index d345bf65..7b933f35 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -994,10 +994,12 @@ async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dic
from core.container import container
status = container.workflow_service().get_deployment_status(workflow_id)
+ workspace = await _workspace_runtime_status(workflow_id)
return {
**payload,
- "active_count": status.get("active_runs", 0),
- "in_flight_count": status.get("active_runs", 0),
+ **{key: value for key, value in workspace.items() if key != "workspace_active_count"},
+ "active_count": status.get("active_runs", 0) + workspace.get("workspace_active_count", 0),
+ "in_flight_count": status.get("active_runs", 0) + workspace.get("workspace_active_count", 0),
"queued_count": (
int(payload.get("queued_count", 0) or 0)
+ int(status.get("queued_events", 0) or 0)
@@ -1005,6 +1007,50 @@ async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dic
}
+async def _workspace_runtime_status(workflow_id: str) -> Dict[str, Any]:
+ from core.container import container
+ from services.node_invocations import controller_status, workspace_task_nodes
+
+ database = container.database()
+ _, nodes = await workspace_task_nodes(database, workflow_id)
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ return {"can_reset": True, "workspace_available": False} if nodes else {}
+ try:
+ state = await controller_status(workflow_id, client=wrapper.client)
+ except Exception as exc:
+ logger.warning("Workspace task status unavailable", workflow_id=workflow_id, error_type=type(exc).__name__)
+ return {"can_reset": True, "workspace_available": False} if nodes else {}
+ if state is None:
+ return {"can_reset": True, "workspace_epoch": 0} if nodes else {}
+ result = {
+ "can_reset": True, "workspace_available": True,
+ "workspace_active_count": state["active_count"], "workspace_epoch": state["epoch"],
+ "workspace_resetting": state["resetting"],
+ "workspace_reset_request_id": state.get("reset_request_id") or state.get("last_reset_request_id"),
+ }
+ if state["resetting"]:
+ result.update(state="resetting", can_start=False, can_pause=False, can_resume=False, can_edit=False)
+ return result
+
+
+async def _reset_workspace_tasks(workflow_id: str, database, control, data: Dict[str, Any]) -> Dict[str, Any]:
+ from services.node_invocations import controller_status, reset, workspace_task_nodes
+ from core.container import container
+
+ _, nodes = await workspace_task_nodes(database, workflow_id, control)
+ wrapper = container.temporal_client()
+ state = None
+ if wrapper is not None and wrapper.client is not None:
+ state = await controller_status(workflow_id, client=wrapper.client)
+ if not nodes and state is None:
+ return {"cancelled": 0, "reset_nodes": [], "present": False}
+ # Phone-only resets do not advance the graph revision. Deriving an ID
+ # from that revision would make every later Reset replay the first one.
+ request_id = str(data.get("idempotency_key") or f"workflow-reset:{uuid.uuid4().hex}")
+ return {**await reset(workflow_id, request_id), "present": True}
+
+
def _close_local_admission(workflow_id: str) -> None:
"""Synchronously gate legacy trigger callbacks before durable cleanup."""
from core.container import container
@@ -2072,23 +2118,29 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
workflow_id = data["workflow_id"]
service = _control_service()
current = await service.database.get_latest_workflow_control(workflow_id)
- if current is None:
- return await _with_runtime_counts(
- await service.get_status(workflow_id),
- workflow_id,
- )
-
- # ``reset`` is a completed cleanup barrier. Re-running generation-wide
- # sweeps here would race a concurrent Start and could terminate resources
- # from the next generation because standalone triggers use the stable
- # application workflow id.
- if current.status == "reset":
- return {
- "success": True,
- "idempotent": True,
- **await _control_payload(current),
- }
+ if current is None or current.status == "reset":
+ # Workspace tasks also exist before Start and after a generation was
+ # reset. Their own Temporal controller supplies the cleanup barrier.
+ revision = current.revision if current else 0
+ if int(data.get("expected_revision", revision)) != revision:
+ if current is not None and current.status == "reset":
+ return {"success": True, "idempotent": True, **await _control_payload(current)}
+ raise ValueError("control_revision_conflict")
+ workspace = await _reset_workspace_tasks(workflow_id, service.database, current, data)
+ if not workspace.get("present", True):
+ payload = await _control_payload(current) if current else await service.get_status(workflow_id)
+ return {"success": True, "idempotent": True, **payload}
+ from services.status_broadcaster import get_status_broadcaster
+ await get_status_broadcaster().broadcast({
+ "type": "workflow_runtime_reset", "workflow_id": workflow_id,
+ "reset_nodes": workspace.get("reset_nodes", []),
+ })
+ payload = await _with_runtime_counts(await service.get_status(workflow_id), workflow_id)
+ await get_status_broadcaster().broadcast({"type": "workflow_control_status", "workflow_id": workflow_id, "data": payload})
+ return {"success": True, "idempotent": not workspace.get("cancelled"), **payload}
+ # The completed-generation branch above only resets direct Workspace
+ # tasks. Generation-wide cleanup below must not sweep a later Start.
if current.status != "resetting":
current = await service.transition(
current, expected_revision=_expected_revision(data, current),
@@ -2144,6 +2196,7 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
# Controller, cron, and legacy local admission paths are now closed. This
# final strict sweep therefore observes a fixed set of generation runs.
terminated = await _terminate_generation_workflows(current, strict=True)
+ workspace = await _reset_workspace_tasks(workflow_id, service.database, current, data)
archived = await service.database.update_workflow_run_data_scope(
current.data_scope_id or current.execution_id,
@@ -2170,6 +2223,7 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
"data_scope_id": current.data_scope_id or current.execution_id,
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
+ "cancelled_workspace_tasks": workspace.get("cancelled", 0),
})
payload = await _broadcast_control(current, extra={
"terminated_executions": terminated,
diff --git a/server/services/node_executor.py b/server/services/node_executor.py
index 61c0d999..d007dd2d 100644
--- a/server/services/node_executor.py
+++ b/server/services/node_executor.py
@@ -207,6 +207,11 @@ class NodeExecutor:
return result
except asyncio.CancelledError:
+ # Workspace engines own subprocesses/device leases. Temporal must
+ # observe cancellation after their finally blocks finish, not an
+ # ordinary failure envelope that can be mistaken for completion.
+ if getattr(get_node_class(node_type), "workspace_task", False):
+ raise
return ExecutionResult(
False, node_id, node_type, error="Cancelled", execution_id=execution_id, execution_time=time.time() - start_time
).to_dict()
diff --git a/server/services/node_invocations.py b/server/services/node_invocations.py
index 6b9d0c92..af180cd0 100644
--- a/server/services/node_invocations.py
+++ b/server/services/node_invocations.py
@@ -1,10 +1,11 @@
"""Authoritative admission for direct Workspace tasks. Never accepts client graphs."""
from __future__ import annotations
+import asyncio
import hashlib
import json
from typing import Any
-from uuid import UUID
+from uuid import UUID, uuid4
def invocation_id(principal: str, workflow_id: str, node_id: str, submission_id: str) -> str:
@@ -23,6 +24,125 @@ def temporal_client() -> Any:
return wrapper.client
+def controller_id(workflow_id: str) -> str:
+ return "workspace-tasks-" + hashlib.sha256(workflow_id.encode()).hexdigest()
+
+
+async def controller_status(workflow_id: str, *, client=None) -> dict | None:
+ from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
+ from temporalio.service import RPCError, RPCStatusCode
+
+ client = client or temporal_client()
+ try:
+ return await client.get_workflow_handle(controller_id(workflow_id)).query(WorkspaceTaskControllerWorkflow.describe)
+ except RPCError as exc:
+ if exc.status == RPCStatusCode.NOT_FOUND:
+ return None
+ raise
+
+
+async def _controller_update(workflow_id: str, update, argument, *, update_id: str):
+ from core.config import Settings
+ from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
+ from temporalio.client import WithStartWorkflowOperation, WorkflowUpdateFailedError
+ from temporalio.common import WorkflowIDConflictPolicy
+ from services.plugin import NodeUserError
+
+ try:
+ return await temporal_client().execute_update_with_start_workflow(
+ update, argument, id=update_id,
+ start_workflow_operation=WithStartWorkflowOperation(
+ WorkspaceTaskControllerWorkflow.run, {"workflow_id": workflow_id},
+ id=controller_id(workflow_id), task_queue=Settings().temporal_task_queue,
+ id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
+ ),
+ )
+ except WorkflowUpdateFailedError as exc:
+ cause = exc.cause
+ messages = {
+ "WorkspaceResetInProgress": "Phone tasks are resetting. Wait for Reset to finish.",
+ "WorkspaceAdmissionChanged": "This submission crossed a Reset. Submit a new task.",
+ "WorkspaceSubmissionConflict": "Submission ID was already used for a different task",
+ "WorkspaceTaskQueueFull": "The Workspace task queue is full",
+ }
+ if getattr(cause, "type", None) in messages:
+ raise NodeUserError(messages[cause.type]) from None
+ raise
+
+
+async def workspace_task_nodes(database, workflow_id: str, control=None) -> tuple[dict, list[dict]]:
+ """Include frozen nodes removed from the editable graph while paused."""
+ from services.node_registry import get_node_class
+
+ saved = await database.get_workflow(workflow_id)
+ graph = (saved.data if hasattr(saved, "data") else saved.get("data", saved)) if saved else {}
+ # Filter before merging: replacing an old phone node with a non-device
+ # node in the editable graph must not hide its frozen cleanup hook.
+ candidates = [*(getattr(control, "graph_snapshot", None) or {}).get("nodes", []), *graph.get("nodes", [])]
+ nodes = {(node["id"], node["type"]): node for node in candidates
+ if node.get("id") and getattr(get_node_class(node.get("type", "")), "workspace_task", False)}
+ return graph, list(nodes.values())
+
+
+async def reset(workflow_id: str, request_id: str) -> dict:
+ """Durably cancel admitted tasks and await plugin cleanup before acknowledging."""
+ from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
+
+ state = await controller_status(workflow_id)
+ # A failed cleanup keeps its admission fence. Resume that operation even
+ # when the user's Retry Reset has a new transport request identity.
+ request_id = (state or {}).get("reset_request_id") or request_id
+ return await _controller_update(
+ workflow_id, WorkspaceTaskControllerWorkflow.reset, request_id,
+ update_id="reset:" + uuid4().hex,
+ )
+
+
+async def cancel_legacy_invocations(workflow_id: str) -> int:
+ """Migration path for pre-controller roots; Visibility is not new-task admission."""
+ from services.temporal.node_invocation import NodeInvocationWorkflow
+ from temporalio.service import RPCError, RPCStatusCode
+
+ client = temporal_client()
+ targets = []
+ async for execution in client.list_workflows(query="WorkflowType='NodeInvocationWorkflow' AND ExecutionStatus='Running'"):
+ # New children are cancelled by the durable controller, not a visibility scan.
+ if getattr(execution, "parent_id", None):
+ continue
+ handle = client.get_workflow_handle(execution.id, run_id=execution.run_id)
+ try:
+ metadata = await handle.query(NodeInvocationWorkflow.describe)
+ if metadata.get("workflow_id") == workflow_id:
+ targets.append(handle)
+ except RPCError as exc:
+ if exc.status != RPCStatusCode.NOT_FOUND:
+ raise
+ await asyncio.gather(*(_cancel_and_wait(handle) for handle in targets))
+ return len(targets)
+
+
+async def _cancel_and_wait(handle) -> None:
+ from temporalio.client import WorkflowFailureError
+ from temporalio.exceptions import is_cancelled_exception
+ from temporalio.service import RPCError, RPCStatusCode
+
+ try:
+ await handle.cancel()
+ try:
+ await asyncio.wait_for(handle.result(), timeout=180)
+ except WorkflowFailureError as exc:
+ # Any already-terminal failure has closed the activity's workflow.
+ # Unexpected RPC/timeouts still propagate; they are not cleanup receipts.
+ if not is_cancelled_exception(exc.cause):
+ description = await handle.describe()
+ from temporalio.client import WorkflowExecutionStatus
+ if description.status == WorkflowExecutionStatus.RUNNING:
+ raise
+ except RPCError as exc:
+ if exc.status != RPCStatusCode.NOT_FOUND:
+ raise
+
+
async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, submission_id: str) -> dict:
from core.config import Settings
from services.authz.workflow_node import resolve_workflow_node
@@ -30,9 +150,12 @@ async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, su
from services.plugin import NodeUserError
from services.plugin.deps import get_database
from services.temporal.node_invocation import NodeInvocationWorkflow
- from temporalio.common import WorkflowIDReusePolicy
- from temporalio.exceptions import WorkflowAlreadyStartedError
+ from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
+ # Establish the admission epoch before any saved-graph/authorization await.
+ # Reads are harmless; authorization still completes before any mutation.
+ # A Reset during graph resolution must reject this original request.
+ admission = await controller_status(workflow_id)
saved, graph, node = await resolve_workflow_node(principal, workflow_id, node_id)
cls = get_node_class(node["type"])
if cls is None or not getattr(cls, "workspace_task", False):
@@ -41,6 +164,10 @@ async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, su
raise NodeUserError("Enable the node before submitting a task")
if not prompt.strip() or len(prompt) > 20000:
raise NodeUserError("Task must contain between 1 and 20000 characters")
+ database = get_database()
+ control = await database.get_latest_workflow_control(workflow_id)
+ if (admission or {}).get("resetting") or (control and control.status == "resetting"):
+ raise NodeUserError("Phone tasks are resetting. Wait for Reset to finish.")
run_id = invocation_id(principal, workflow_id, node_id, submission_id)
fingerprint = hashlib.sha256(prompt.encode()).hexdigest()
params = {**(await get_database().get_node_parameters(node_id) or {}), "prompt": prompt}
@@ -69,20 +196,33 @@ async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, su
"task_queue": cls.task_queue if settings.temporal_worker_pool_enabled else settings.temporal_task_queue,
"timeout_s": int(cls.start_to_close_timeout.total_seconds()),
"context": context,
+ "admission_epoch": (admission or {}).get("epoch", 0),
}
client = temporal_client()
- try:
- await client.start_workflow(
- NodeInvocationWorkflow.run,
- payload,
- id=run_id,
- task_queue=settings.temporal_task_queue,
- id_reuse_policy=WorkflowIDReusePolicy.REJECT_DUPLICATE,
- )
- except WorkflowAlreadyStartedError:
+ # The controller owns submission idempotence. A new Update ID permits a
+ # retry after queue-full/reset rejection; a completed failed Update would
+ # otherwise permanently cache that transient admission result.
+ admitted = await _controller_update(
+ workflow_id, WorkspaceTaskControllerWorkflow.submit, payload,
+ update_id=f"submit:{run_id}:{uuid4().hex}",
+ )
+ if admitted.get("status") == "duplicate":
existing = await client.get_workflow_handle(run_id).query(NodeInvocationWorkflow.describe)
if existing["fingerprint"] != fingerprint:
raise NodeUserError("Submission ID was already used for a different task") from None
+ from services.deployment.handlers import _with_runtime_counts
+ from services.deployment.control import serialize_control
+ from services.status_broadcaster import get_status_broadcaster
+ try:
+ await get_status_broadcaster().broadcast({
+ "type": "workflow_control_status", "workflow_id": workflow_id,
+ "data": await _with_runtime_counts(serialize_control(control), workflow_id),
+ })
+ except Exception:
+ # Admission is already committed. Poll/reconnect can recover status;
+ # a best-effort UI notification must not report submission failure.
+ from core.logging import get_logger
+ get_logger(__name__).warning("Workspace admission notification failed", workflow_id=workflow_id, exc_info=True)
return {"run_id": run_id, "submission_id": submission_id, "status": "accepted"}
@@ -96,7 +236,7 @@ async def status(principal: str, workflow_id: str, node_id: str, submission_id:
handle = temporal_client().get_workflow_handle(invocation_id(principal, workflow_id, node_id, submission_id))
try:
if cancel:
- await handle.cancel()
+ await _cancel_and_wait(handle)
result = await handle.query(NodeInvocationWorkflow.describe)
except RPCError as exc:
if exc.status == RPCStatusCode.NOT_FOUND:
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index f6c30f66..ae3e1330 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -121,7 +121,7 @@ async def _execute_plugin_tool_activity(
# The caller's authenticated principal is not a model argument. Keep
# it on the leaf context so owner-only plugins cannot inherit the
# legacy default principal accidentally.
- tool_payload = {**tool_payload, "user_id": str(context.get("user_id") or "owner")}
+ tool_payload = {**tool_payload, **_inherited_scope(context), "user_id": str(context.get("user_id") or "owner")}
return await workflow.execute_activity(
activity_name,
args=[tool_payload],
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 74f520f0..2d97c778 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -54,9 +54,11 @@ def _framework_workflows() -> list:
from services.temporal.workflow_control_workflow import WorkflowControlWorkflow
from services.temporal.node_invocation import NodeInvocationWorkflow
+ from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
return [
NodeInvocationWorkflow,
+ WorkspaceTaskControllerWorkflow,
MachinaWorkflow,
AgentWorkflow,
DelegatedTaskWorkflow,
@@ -223,6 +225,7 @@ class TemporalWorkerManager:
collect_polling_activities,
)
from services.temporal.agent_activities import collect_agent_activities
+ from services.temporal.workspace_task_activities import reset_workspace_task_runtime
from services.temporal.activities import (
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
@@ -278,6 +281,7 @@ class TemporalWorkerManager:
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
+ reset_workspace_task_runtime,
store_node_output_activity,
*per_type,
*agent_activities,
@@ -736,6 +740,7 @@ async def run_standalone_worker(
store_node_output_activity,
)
from services.temporal.agent_activities import collect_agent_activities
+ from services.temporal.workspace_task_activities import reset_workspace_task_runtime
from services.temporal.agent_workflow import (
AgentWorkflow,
DelegatedTaskWorkflow,
@@ -761,6 +766,7 @@ async def run_standalone_worker(
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
+ reset_workspace_task_runtime,
store_node_output_activity,
*registered_plugin_activities,
*registered_agent_activities,
@@ -817,6 +823,7 @@ async def create_worker(
store_node_output_activity,
)
from services.temporal.agent_activities import collect_agent_activities
+ from services.temporal.workspace_task_activities import reset_workspace_task_runtime
from services.temporal.agent_workflow import (
AgentWorkflow,
DelegatedTaskWorkflow,
@@ -841,6 +848,7 @@ async def create_worker(
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
+ reset_workspace_task_runtime,
store_node_output_activity,
*registered_plugin_activities,
*registered_agent_activities,
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index 16d56707..8f4dfbe1 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -662,6 +662,12 @@ class MachinaWorkflow:
heartbeat_timeout=timedelta(minutes=2),
retry_policy=activity_retry_policy,
)
+ if getattr(node_cls, "workspace_task", False) and workflow.patched("workspace-node-cancellation-v1"):
+ start_kwargs.update(
+ start_to_close_timeout=node_cls.start_to_close_timeout,
+ heartbeat_timeout=node_cls.heartbeat_timeout,
+ cancellation_type=workflow.ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
+ )
if dispatch.get("queue") is not None:
start_kwargs["task_queue"] = dispatch["queue"]
diff --git a/server/services/temporal/workspace_task_activities.py b/server/services/temporal/workspace_task_activities.py
new file mode 100644
index 00000000..41b90696
--- /dev/null
+++ b/server/services/temporal/workspace_task_activities.py
@@ -0,0 +1,41 @@
+"""Plugin-owned Workspace cleanup, acknowledged by the durable task controller."""
+
+import asyncio
+from temporalio import activity
+
+
[email protected](name="workspace_tasks.reset_runtime")
+async def reset_workspace_task_runtime(payload: dict) -> dict:
+ from services.plugin.deps import get_database
+ from services.node_registry import get_node_class
+ from services.node_invocations import cancel_legacy_invocations, workspace_task_nodes
+
+ workflow_id = payload["workflow_id"]
+
+ async def heartbeat():
+ while True:
+ activity.heartbeat("Waiting for Workspace task cleanup")
+ await asyncio.sleep(5)
+
+ beat = asyncio.create_task(heartbeat())
+ try:
+ # Upgrade compatibility: old standalone roots were not children of
+ # this controller. New admissions are fenced while this activity runs.
+ legacy = await cancel_legacy_invocations(workflow_id)
+ database = get_database()
+ control = await database.get_latest_workflow_control(workflow_id)
+ graph, nodes = await workspace_task_nodes(database, workflow_id, control)
+ generation = int(control.generation) if control and control.status == "resetting" else 0
+ reset_nodes = []
+ for node in nodes:
+ result = await get_node_class(node["type"]).reset_execution_state(
+ node_id=node["id"], workflow_id=workflow_id,
+ execution_id=control.execution_id if control else "",
+ generation=generation, graph=graph, database=database,
+ )
+ if result.get("reset"):
+ reset_nodes.append(node["id"])
+ return {"reset_nodes": reset_nodes, "legacy_cancelled": legacy}
+ finally:
+ beat.cancel()
+ await asyncio.gather(beat, return_exceptions=True)
diff --git a/server/services/temporal/workspace_tasks_workflow.py b/server/services/temporal/workspace_tasks_workflow.py
new file mode 100644
index 00000000..a6f97221
--- /dev/null
+++ b/server/services/temporal/workspace_tasks_workflow.py
@@ -0,0 +1,201 @@
+"""Durable admission and cooperative Reset for one workflow's Workspace tasks.
+
+Child-start acknowledgement is part of admission. Reset fences new submissions
+before waiting for that acknowledgement, then waits for every admitted child to
+close. No process-local registry or visibility query is the source of truth.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import timedelta
+from typing import Any
+
+from temporalio import workflow
+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
+from temporalio.exceptions import ApplicationError, WorkflowAlreadyStartedError
+
+from services.temporal.node_invocation import NodeInvocationWorkflow
+
+MAX_ACTIVE_TASKS = 32
+MAX_REMEMBERED_SUBMISSIONS = 128
+MAX_REMEMBERED_RESETS = 128
+_OPERATIONS_PER_RUN = 256
+_HISTORY_SOFT_CAP = 10_000
+
+
+def _reject(message: str, error_type: str) -> None:
+ raise ApplicationError(message, type=error_type, non_retryable=True)
+
+
+def _bounded_remember(records: dict, key: str, value: Any, limit: int) -> None:
+ records[key] = value
+ while len(records) > limit:
+ del records[next(iter(records))]
+
+
[email protected](sandboxed=False)
+class WorkspaceTaskControllerWorkflow:
+ @workflow.init
+ def __init__(self, payload: dict) -> None:
+ # Update-With-Start can invoke a handler before run(), so its input
+ # must already be available here.
+ self._workflow_id = str(payload["workflow_id"])
+ self._epoch = int(payload.get("epoch", 0))
+ self._submissions: dict[str, str] = dict(list((payload.get("submissions") or {}).items())[-MAX_REMEMBERED_SUBMISSIONS:])
+ self._resets: dict[str, dict] = dict(list((payload.get("resets") or {}).items())[-MAX_REMEMBERED_RESETS:])
+ self._active: dict[str, tuple[str, Any]] = {}
+ self._starting: str | None = None
+ self._resetting: str | None = None
+ self._reset_running = False
+ self._reset_cancelled: set[str] = set()
+ self._admission_lock = asyncio.Lock()
+ self._operations = 0
+
+ @workflow.query
+ def describe(self) -> dict:
+ return {
+ "epoch": self._epoch,
+ "resetting": self._resetting is not None,
+ "reset_request_id": self._resetting,
+ "last_reset_request_id": next(reversed(self._resets), None),
+ "active_count": len(self._active) + int(self._starting is not None),
+ }
+
+ def _check_admission(self, payload: dict) -> tuple[str, str]:
+ if self._resetting is not None:
+ _reject("Workspace tasks are resetting", "WorkspaceResetInProgress")
+ epoch = payload.get("admission_epoch", 0)
+ if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch != self._epoch:
+ _reject("Workspace admission changed; submit a fresh task", "WorkspaceAdmissionChanged")
+ context = payload.get("context") or {}
+ if payload.get("workflow_id") != self._workflow_id or context.get("workflow_id") != self._workflow_id:
+ _reject("Workspace task belongs to a different workflow", "WorkspaceScopeMismatch")
+ run_id = context.get("execution_id")
+ fingerprint = payload.get("fingerprint")
+ if not isinstance(run_id, str) or not run_id or len(run_id) > 256:
+ _reject("Workspace task needs a bounded execution ID", "WorkspaceInvalidSubmission")
+ if not isinstance(fingerprint, str) or not fingerprint or len(fingerprint) > 128:
+ _reject("Workspace task needs a bounded fingerprint", "WorkspaceInvalidSubmission")
+ return run_id, fingerprint
+
+ @workflow.update
+ async def submit(self, payload: dict) -> dict:
+ self._operations += 1
+ self._check_admission(payload)
+ async with self._admission_lock:
+ # Reset can begin while this handler waits behind another start.
+ run_id, fingerprint = self._check_admission(payload)
+ known = self._active.get(run_id)
+ known_fingerprint = known[0] if known else self._submissions.get(run_id)
+ if known_fingerprint is not None:
+ if known_fingerprint != fingerprint:
+ _reject("Submission ID was already used for a different task", "WorkspaceSubmissionConflict")
+ return {"run_id": run_id, "status": "accepted"}
+ if len(self._active) >= MAX_ACTIVE_TASKS:
+ _reject("The Workspace task queue is full", "WorkspaceTaskQueueFull")
+
+ self._starting = run_id
+ try:
+ child = await workflow.start_child_workflow(
+ NodeInvocationWorkflow.run,
+ payload,
+ id=run_id,
+ id_reuse_policy=WorkflowIDReusePolicy.REJECT_DUPLICATE,
+ cancellation_type=workflow.ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED,
+ parent_close_policy=workflow.ParentClosePolicy.REQUEST_CANCEL,
+ )
+ except WorkflowAlreadyStartedError:
+ # This can be a pre-controller root, or an evicted completed
+ # record. Only the service can query that execution to verify
+ # its fingerprint; never trust/cache the newly supplied one.
+ return {"run_id": run_id, "status": "duplicate"}
+ finally:
+ self._starting = None
+ self._active[run_id] = (fingerprint, child)
+ child.add_done_callback(lambda completed: self._child_closed(run_id, fingerprint, completed))
+ return {"run_id": run_id, "status": "accepted"}
+
+ def _child_closed(self, run_id: str, fingerprint: str, child: Any) -> None:
+ # Consume failures even when the caller only waited for admission.
+ # Completion, failure, and cancellation all release the same slot.
+ try:
+ child.exception()
+ except asyncio.CancelledError:
+ pass
+ if self._active.get(run_id, (None, None))[1] is child:
+ del self._active[run_id]
+ _bounded_remember(self._submissions, run_id, fingerprint, MAX_REMEMBERED_SUBMISSIONS)
+
+ @workflow.update
+ async def reset(self, request_id: str) -> dict:
+ self._operations += 1
+ if not isinstance(request_id, str) or not request_id or len(request_id) > 256:
+ _reject("Reset needs a bounded request ID", "WorkspaceInvalidReset")
+ while True:
+ if request_id in self._resets:
+ return dict(self._resets[request_id])
+ if self._resetting is not None and self._resetting != request_id:
+ _reject("Workspace tasks are already resetting", "WorkspaceResetInProgress")
+ if not self._reset_running:
+ break
+ await workflow.wait_condition(lambda: not self._reset_running)
+
+ if self._resetting is None:
+ # Fence late network requests immediately, including while a previous
+ # submission is awaiting ChildWorkflowExecutionStarted under the lock.
+ self._resetting = request_id
+ self._epoch += 1
+ self._reset_cancelled.clear()
+ # A failed cleanup leaves the fence latched. The same request can retry
+ # without advancing the epoch or forgetting the children it cancelled.
+ self._reset_running = True
+ try:
+ async with self._admission_lock:
+ children = [(run_id, child) for run_id, (_, child) in self._active.items() if not child.done()]
+ for run_id, child in children:
+ self._reset_cancelled.add(run_id)
+ child.cancel()
+ # WAIT_CANCELLATION_COMPLETED on each handle means this waits
+ # for activity cleanup and child close, not just cancel ack.
+ await asyncio.gather(*(child for _, child in children), return_exceptions=True)
+ # Local plugin runtimes may predate this controller, including
+ # work in a phone-only graph with no deployment control row.
+ runtime_result = await workflow.execute_activity(
+ "workspace_tasks.reset_runtime",
+ {"workflow_id": self._workflow_id},
+ start_to_close_timeout=timedelta(seconds=120),
+ heartbeat_timeout=timedelta(seconds=30),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ cancellation_type=workflow.ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
+ )
+ result = {"cancelled": len(self._reset_cancelled)}
+ if isinstance(runtime_result, dict) and "reset_nodes" in runtime_result:
+ result["reset_nodes"] = runtime_result["reset_nodes"]
+ _bounded_remember(self._resets, request_id, result, MAX_REMEMBERED_RESETS)
+ self._resetting = None
+ return dict(result)
+ finally:
+ self._reset_running = False
+
+ def _ready_to_roll_over(self) -> bool:
+ if self._active or self._starting or self._resetting or self._admission_lock.locked():
+ return False
+ if not workflow.all_handlers_finished():
+ return False
+ info = workflow.info()
+ return (
+ self._operations >= _OPERATIONS_PER_RUN
+ or info.is_continue_as_new_suggested()
+ or info.get_current_history_length() >= _HISTORY_SOFT_CAP
+ )
+
+ @workflow.run
+ async def run(self, payload: dict) -> None:
+ await workflow.wait_condition(self._ready_to_roll_over)
+ workflow.continue_as_new({
+ "workflow_id": self._workflow_id,
+ "epoch": self._epoch,
+ "submissions": self._submissions,
+ "resets": self._resets,
+ })
diff --git a/server/tests/services/test_workflow_control_handlers.py b/server/tests/services/test_workflow_control_handlers.py
index f1e0fcdf..7c73244c 100644
--- a/server/tests/services/test_workflow_control_handlers.py
+++ b/server/tests/services/test_workflow_control_handlers.py
@@ -8,6 +8,14 @@ import pytest
from services.deployment import handlers
[email protected](autouse=True)
+def no_workspace_tasks(monkeypatch):
+ # These lifecycle contracts use ordinary graph nodes. Workspace-specific
+ # admission and cleanup are covered in test_workspace_task_reset.py.
+ monkeypatch.setattr(handlers, "_workspace_runtime_status", AsyncMock(return_value={}))
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", AsyncMock(return_value={"present": False}))
+
+
def _control(
status: str,
revision: int,
diff --git a/server/tests/services/test_workspace_task_reset.py b/server/tests/services/test_workspace_task_reset.py
new file mode 100644
index 00000000..1c7eff1d
--- /dev/null
+++ b/server/tests/services/test_workspace_task_reset.py
@@ -0,0 +1,268 @@
+"""Workflow Reset must acknowledge standalone Workspace cleanup before success."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from services.deployment import handlers
+
+
+def control(status="reset", revision=4):
+ return SimpleNamespace(
+ id="control-wf-2", workflow_id="wf", generation=2, execution_id="execution-2",
+ root_execution_id="execution-2", data_scope_id="scope-2", status=status, revision=revision,
+ )
+
+
+def broadcaster(monkeypatch, order=None):
+ from services import status_broadcaster
+
+ async def broadcast(message):
+ if order is not None:
+ order.append(message["type"])
+
+ value = SimpleNamespace(broadcast=AsyncMock(side_effect=broadcast))
+ monkeypatch.setattr(status_broadcaster, "get_status_broadcaster", lambda: value)
+ return value
+
+
[email protected]("current", [None, control()])
+async def test_direct_reset_waits_for_workspace_barrier_before_any_success_broadcast(monkeypatch, current):
+ entered, release = asyncio.Event(), asyncio.Event()
+ order = []
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=current))
+ service = SimpleNamespace(database=database, get_status=AsyncMock(return_value={"state": "ready"}), transition=AsyncMock())
+
+ async def reset_workspace(*_args):
+ order.append("workspace:start")
+ entered.set()
+ await release.wait()
+ order.append("workspace:done")
+ return {"cancelled": 1, "reset_nodes": ["phone"], "present": True}
+
+ workspace = AsyncMock(side_effect=reset_workspace)
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", workspace)
+ monkeypatch.setattr(handlers, "_with_runtime_counts", AsyncMock(return_value={"state": "ready", "can_reset": True}))
+ notices = broadcaster(monkeypatch, order)
+ data = {"workflow_id": "wf", "expected_revision": current.revision if current else 0, "idempotency_key": "reset-phone-1"}
+ task = asyncio.create_task(handlers.handle_reset_workflow(data, None))
+ await asyncio.wait_for(entered.wait(), 1)
+ assert not task.done()
+ notices.broadcast.assert_not_awaited()
+ service.get_status.assert_not_awaited()
+ service.transition.assert_not_awaited()
+ release.set()
+ result = await asyncio.wait_for(task, 1)
+ assert result == {"success": True, "idempotent": False, "state": "ready", "can_reset": True}
+ workspace.assert_awaited_once_with("wf", database, current, data)
+ assert order == ["workspace:start", "workspace:done", "workflow_runtime_reset", "workflow_control_status"]
+ assert notices.broadcast.await_args_list[0].args[0]["reset_nodes"] == ["phone"]
+
+
[email protected]("current", [None, control()])
+async def test_direct_reset_barrier_failure_does_not_publish_success(monkeypatch, current):
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=current))
+ service = SimpleNamespace(database=database, get_status=AsyncMock(), transition=AsyncMock())
+ workspace = AsyncMock(side_effect=RuntimeError("workspace_cleanup_failed"))
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", workspace)
+ notices = broadcaster(monkeypatch)
+ result = await handlers.handle_reset_workflow({"workflow_id": "wf"}, None)
+ assert result == {"success": False, "error": "workspace_cleanup_failed"}
+ notices.broadcast.assert_not_awaited()
+ service.get_status.assert_not_awaited()
+ service.transition.assert_not_awaited()
+
+
[email protected]("current", [None, control()])
+async def test_ordinary_idempotent_reset_has_no_workspace_reset_broadcast(monkeypatch, current):
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=current))
+ payload = {"state": "ready", "can_reset": False}
+ service = SimpleNamespace(database=database, get_status=AsyncMock(return_value=payload))
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", AsyncMock(return_value={"present": False}))
+ monkeypatch.setattr(handlers, "_control_payload", AsyncMock(return_value=payload))
+ notices = broadcaster(monkeypatch)
+ assert await handlers.handle_reset_workflow({"workflow_id": "wf"}, None) == {
+ "success": True, "idempotent": True, **payload,
+ }
+ notices.broadcast.assert_not_awaited()
+
+
[email protected]("initial_status", ["running", "resetting"])
[email protected]("workspace_fails", [False, True])
+async def test_generation_reset_waits_for_workspace_before_archive_and_final_reset(monkeypatch, initial_status, workspace_fails):
+ from services.deployment import runtime_state
+
+ initial = control(initial_status, 3 if initial_status == "running" else 4)
+ resetting, reset = control("resetting", 4), control("reset", 5)
+ entered, release = asyncio.Event(), asyncio.Event()
+ order = []
+
+ async def transition(_current, **kwargs):
+ order.append("db:" + kwargs["status"])
+ return resetting if kwargs["status"] == "resetting" else reset
+
+ async def archive_scope(*_args, **_kwargs):
+ order.append("archive:scope")
+ return True
+
+ async def archive_nodes(*_args, **_kwargs):
+ order.append("archive:nodes")
+ return {"archived_nodes": 1, "reset_nodes": ["phone"]}
+
+ async def terminate(*_args, **_kwargs):
+ order.append("terminate:graph")
+ return 1
+
+ async def reset_workspace(*_args):
+ order.append("workspace:start")
+ entered.set()
+ await release.wait()
+ if workspace_fails:
+ raise RuntimeError("workspace_cleanup_failed")
+ order.append("workspace:done")
+ return {"cancelled": 2, "reset_nodes": ["phone"], "present": True}
+
+ async def broadcast_control(current, **_kwargs):
+ order.append("control:" + current.status)
+ return {"state": "ready" if current.status == "reset" else current.status}
+
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=initial),
+ update_workflow_run_data_scope=AsyncMock(side_effect=archive_scope),
+ )
+ service = SimpleNamespace(database=database, transition=AsyncMock(side_effect=transition))
+ archive = AsyncMock(side_effect=archive_nodes)
+ workspace = AsyncMock(side_effect=reset_workspace)
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_close_local_admission", Mock())
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(side_effect=broadcast_control))
+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
+ monkeypatch.setattr(handlers, "_delete_cron_schedules", AsyncMock(return_value=1))
+ monkeypatch.setattr(handlers, "handle_cancel_deployment", AsyncMock(return_value={"success": True}))
+ monkeypatch.setattr(handlers, "_terminate_generation_workflows", AsyncMock(side_effect=terminate))
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", workspace)
+ monkeypatch.setattr(runtime_state, "archive_and_reset_node_state", archive)
+ notices = broadcaster(monkeypatch, order)
+ data = {"workflow_id": "wf", "expected_revision": initial.revision, "idempotency_key": "reset-generation"}
+ task = asyncio.create_task(handlers.handle_reset_workflow(data, None))
+ await asyncio.wait_for(entered.wait(), 1)
+ assert not task.done()
+ database.update_workflow_run_data_scope.assert_not_awaited()
+ archive.assert_not_awaited()
+ notices.broadcast.assert_not_awaited()
+ assert all(call.kwargs["status"] != "reset" for call in service.transition.await_args_list)
+ release.set()
+ result = await asyncio.wait_for(task, 1)
+ workspace.assert_awaited_once_with("wf", database, resetting if initial_status == "running" else initial, data)
+ if workspace_fails:
+ assert result == {"success": False, "error": "workspace_cleanup_failed"}
+ database.update_workflow_run_data_scope.assert_not_awaited()
+ archive.assert_not_awaited()
+ notices.broadcast.assert_not_awaited()
+ assert "db:reset" not in order and "control:reset" not in order
+ else:
+ assert result["success"] is True and result["state"] == "ready"
+ assert order[-7:] == ["workspace:start", "workspace:done", "archive:scope", "archive:nodes",
+ "db:reset", "workflow_runtime_reset", "control:reset"]
+ assert order.index("terminate:graph") < order.index("workspace:start")
+ assert notices.broadcast.await_args.args[0]["cancelled_workspace_tasks"] == 2
+
+
+def invocation_services(monkeypatch, *, nodes, actor=None, available=True):
+ import core.container as container_module
+ from services import node_invocations
+
+ database, client = object(), object()
+ container = SimpleNamespace(database=lambda: database,
+ temporal_client=lambda: SimpleNamespace(client=client) if available else None)
+ monkeypatch.setattr(container_module, "container", container)
+ find_nodes = AsyncMock(return_value=({"nodes": nodes}, nodes))
+ query = AsyncMock(return_value=actor)
+ reset = AsyncMock(return_value={"cancelled": 1, "reset_nodes": ["phone"]})
+ monkeypatch.setattr(node_invocations, "workspace_task_nodes", find_nodes)
+ monkeypatch.setattr(node_invocations, "controller_status", query)
+ monkeypatch.setattr(node_invocations, "reset", reset)
+ return SimpleNamespace(database=database, client=client, container=container, find_nodes=find_nodes, query=query, reset=reset)
+
+
[email protected]("has_nodes,has_actor", [(False, False), (True, False), (False, True), (True, True)])
+async def test_reset_helper_requires_saved_workspace_nodes_or_existing_actor(monkeypatch, has_nodes, has_actor):
+ nodes = [{"id": "phone", "type": "mobile_use_agent"}] if has_nodes else []
+ mocks = invocation_services(monkeypatch, nodes=nodes, actor={"active_count": 1} if has_actor else None)
+ current = control()
+ result = await handlers._reset_workspace_tasks("wf", mocks.database, current, {"idempotency_key": "client-reset-id"})
+ mocks.find_nodes.assert_awaited_once_with(mocks.database, "wf", current)
+ mocks.query.assert_awaited_once_with("wf", client=mocks.client)
+ if has_nodes or has_actor:
+ mocks.reset.assert_awaited_once_with("wf", "client-reset-id")
+ assert result == {"cancelled": 1, "reset_nodes": ["phone"], "present": True}
+ else:
+ mocks.reset.assert_not_awaited()
+ assert result == {"cancelled": 0, "reset_nodes": [], "present": False}
+
+
[email protected]("current,revision", [(None, 0), (control(), 4)])
+async def test_reset_helper_generates_fresh_request_id_when_client_omits_one(monkeypatch, current, revision):
+ mocks = invocation_services(monkeypatch, nodes=[{"id": "phone"}])
+ await handlers._reset_workspace_tasks("wf", mocks.database, current, {"expected_revision": revision})
+ await handlers._reset_workspace_tasks("wf", mocks.database, current, {"expected_revision": revision})
+ assert mocks.reset.await_count == 2
+ calls = mocks.reset.await_args_list
+ assert all(call.args[0] == "wf" for call in calls)
+ first_id, second_id = (call.args[1] for call in calls)
+ assert isinstance(first_id, str) and first_id
+ assert isinstance(second_id, str) and second_id
+ assert first_id != second_id
+
+
+async def test_reset_helper_does_not_treat_unavailable_engine_as_completed_cleanup(monkeypatch):
+ mocks = invocation_services(monkeypatch, nodes=[{"id": "phone"}], available=False)
+ mocks.reset.side_effect = RuntimeError("temporal_unavailable")
+ with pytest.raises(RuntimeError, match="temporal_unavailable"):
+ await handlers._reset_workspace_tasks("wf", mocks.database, None, {"idempotency_key": "reset-id"})
+ mocks.query.assert_not_awaited()
+ mocks.reset.assert_awaited_once_with("wf", "reset-id")
+
+
[email protected]("available", [False, True])
[email protected]("has_nodes", [False, True])
+async def test_status_exposes_reset_for_direct_workspace_nodes_before_any_actor(monkeypatch, available, has_nodes):
+ mocks = invocation_services(monkeypatch, nodes=[{"id": "phone"}] if has_nodes else [], available=available)
+ result = await handlers._workspace_runtime_status("wf")
+ assert result.get("can_reset", False) is has_nodes
+ if not available:
+ mocks.query.assert_not_awaited()
+ if has_nodes:
+ assert result["workspace_available"] is False
+ elif has_nodes:
+ assert result["workspace_epoch"] == 0
+
+
+async def test_status_preserves_reset_affordance_when_workspace_query_fails(monkeypatch):
+ mocks = invocation_services(monkeypatch, nodes=[{"id": "phone"}])
+ mocks.query.side_effect = RuntimeError("engine temporarily unavailable")
+ assert await handlers._workspace_runtime_status("wf") == {"can_reset": True, "workspace_available": False}
+
+
+async def test_runtime_counts_enable_reset_for_direct_nodes_without_deployment(monkeypatch):
+ mocks = invocation_services(monkeypatch, nodes=[{"id": "phone"}])
+ mocks.container.workflow_service = lambda: SimpleNamespace(get_deployment_status=lambda _wf: {})
+ result = await handlers._with_runtime_counts({"state": "ready", "can_reset": False}, "wf")
+ assert result["can_reset"] is True
+ assert result["active_count"] == result["in_flight_count"] == 0
+
+
+async def test_workspace_resetting_actor_disables_start_and_preserves_reset_request(monkeypatch):
+ invocation_services(monkeypatch, nodes=[], actor={"active_count": 2, "epoch": 3, "resetting": True,
+ "reset_request_id": "reset-pending"})
+ result = await handlers._workspace_runtime_status("wf")
+ assert result["state"] == "resetting"
+ assert result["can_reset"] is True
+ assert result["can_start"] is result["can_pause"] is result["can_resume"] is result["can_edit"] is False
+ assert result["workspace_active_count"] == 2
+ assert result["workspace_reset_request_id"] == "reset-pending"
diff --git a/server/tests/temporal/test_workspace_task_activities.py b/server/tests/temporal/test_workspace_task_activities.py
new file mode 100644
index 00000000..acef50bc
--- /dev/null
+++ b/server/tests/temporal/test_workspace_task_activities.py
@@ -0,0 +1,46 @@
+"""Cleanup activity scope and failure behavior; no device or Temporal server."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from temporalio.testing import ActivityEnvironment
+
+from services.temporal.workspace_task_activities import reset_workspace_task_runtime
+
+
[email protected]("status,generation", [(None, 0), ("resetting", 5), ("running", 0), ("reset", 0)])
+async def test_cleanup_waits_for_legacy_roots_and_passes_retired_scope(monkeypatch, status, generation):
+ from services import node_invocations, node_registry
+ from services.plugin import deps
+
+ order = []
+ async def cancel(_):
+ order.append("legacy_closed")
+ return 2
+ async def clear(**kwargs):
+ assert order == ["legacy_closed"]
+ assert kwargs["generation"] == generation
+ assert kwargs["workflow_id"] == "wf"
+ order.append("runtime_cleared")
+ return {"reset": True}
+ control = SimpleNamespace(status=status, generation=5, execution_id="exec") if status else None
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=control))
+ monkeypatch.setattr(deps, "get_database", lambda: database)
+ monkeypatch.setattr(node_invocations, "cancel_legacy_invocations", cancel)
+ monkeypatch.setattr(node_invocations, "workspace_task_nodes", AsyncMock(return_value=({}, [{"id": "phone", "type": "device"}])))
+ monkeypatch.setattr(node_registry, "get_node_class", lambda _: SimpleNamespace(reset_execution_state=clear))
+ result = await ActivityEnvironment().run(reset_workspace_task_runtime, {"workflow_id": "wf"})
+ assert result == {"reset_nodes": ["phone"], "legacy_cancelled": 2}
+ assert order == ["legacy_closed", "runtime_cleared"]
+
+
+async def test_legacy_cleanup_failure_does_not_acknowledge_runtime_reset(monkeypatch):
+ from services import node_invocations
+
+ monkeypatch.setattr(node_invocations, "cancel_legacy_invocations", AsyncMock(side_effect=RuntimeError("Temporal unavailable")))
+ nodes = AsyncMock()
+ monkeypatch.setattr(node_invocations, "workspace_task_nodes", nodes)
+ with pytest.raises(RuntimeError, match="unavailable"):
+ await ActivityEnvironment().run(reset_workspace_task_runtime, {"workflow_id": "wf"})
+ nodes.assert_not_awaited()
diff --git a/server/tests/temporal/test_workspace_task_activity_policy.py b/server/tests/temporal/test_workspace_task_activity_policy.py
index d5f1680c..46beb65e 100644
--- a/server/tests/temporal/test_workspace_task_activity_policy.py
+++ b/server/tests/temporal/test_workspace_task_activity_policy.py
@@ -26,7 +26,8 @@ async def test_workspace_task_honors_plugin_policy_and_authenticated_principal(m
monkeypatch.setattr(module.workflow, "execute_activity", execute)
tool_payload = {"node_type": "embedded_task", "node_id": "child", "user_id": "untrusted", "nodes": [{"id": "model"}]}
result = await module._execute_plugin_tool_activity(
- "node.embedded_task.v1", "call-1", tool_payload, {"user_id": "authenticated-owner", "temporal_worker_pool_enabled": pool}
+ "node.embedded_task.v1", "call-1", tool_payload,
+ {"user_id": "authenticated-owner", "temporal_worker_pool_enabled": pool, "generation": 7, "workflow_id": "wf"}
)
patched.assert_called_once_with("workspace-task-activity-policy-v1")
assert result == {"success": True}
@@ -37,6 +38,7 @@ async def test_workspace_task_honors_plugin_policy_and_authenticated_principal(m
assert kwargs["cancellation_type"] == ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
assert kwargs.get("task_queue") == ("device-pool" if pool else None)
assert kwargs["args"][0]["user_id"] == "authenticated-owner"
+ assert kwargs["args"][0]["generation"] == 7
assert kwargs["args"][0]["nodes"] == [{"id": "model"}]
assert tool_payload["user_id"] == "untrusted" # caller input is not mutated
diff --git a/server/tests/temporal/test_workspace_tasks_integration.py b/server/tests/temporal/test_workspace_tasks_integration.py
new file mode 100644
index 00000000..f6df6d93
--- /dev/null
+++ b/server/tests/temporal/test_workspace_tasks_integration.py
@@ -0,0 +1,237 @@
+"""Real Temporal admission/Reset protocol using only installed local binaries.
+
+Set TEMPORAL_TEST_CLI to a native Temporal CLI binary when it is not on PATH
+or installed by the project's temporal-server npm package. This test never
+downloads a server or connects to the application's running Temporal server.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+import subprocess
+import sys
+from contextlib import suppress
+from datetime import timedelta
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+
+
+def _existing_cli() -> Path | None:
+ configured = os.environ.get("TEMPORAL_TEST_CLI")
+ if configured:
+ path = Path(configured)
+ return path if path.is_file() else None
+ executable = "temporal.exe" if sys.platform == "win32" else "temporal"
+ candidates = [Path(__file__).parents[3] / "node_modules" / "temporal-server" / "bin" / executable]
+ on_path = shutil.which(executable) or shutil.which("temporal")
+ if on_path:
+ path = Path(on_path)
+ # The npm entry point is a launcher with application database/port
+ # defaults; start_local needs its native binary instead.
+ candidates.append(path.parent / "node_modules" / "temporal-server" / "bin" / executable)
+ if path.suffix.lower() not in {".cmd", ".ps1", ".bat"}:
+ candidates.append(path)
+ if os.environ.get("APPDATA"):
+ candidates.append(Path(os.environ["APPDATA"]) / "npm" / "node_modules" / "temporal-server" / "bin" / executable)
+ return next((path for path in candidates if path.is_file()), None)
+
+
+async def _run_gate(cli: str) -> None:
+ from temporalio import activity
+ from temporalio.api.enums.v1 import EventType
+ from temporalio.client import (
+ WithStartWorkflowOperation,
+ WorkflowExecutionStatus,
+ WorkflowUpdateFailedError,
+ )
+ from temporalio.common import WorkflowIDConflictPolicy
+ from temporalio.testing import WorkflowEnvironment
+ from temporalio.worker import Replayer, Worker
+
+ from services.temporal.node_invocation import NodeInvocationWorkflow
+ from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
+
+ token = uuid4().hex
+ task_queue = f"workspace-reset-test-{token}"
+ controller_id = f"workspace-controller-{token}"
+ workflow_id = f"workspace-graph-{token}"
+ first_id, second_id = f"device-first-{token}", f"device-second-{token}"
+ devices = {
+ run_id: SimpleNamespace(
+ started=asyncio.Event(), finish=asyncio.Event(), cancelled=asyncio.Event(),
+ release_cleanup=asyncio.Event(), cleaned=asyncio.Event(),
+ )
+ for run_id in (first_id, second_id)
+ }
+
+ class FakeActivities:
+ def __init__(self):
+ self.cleanup_started = asyncio.Event()
+ self.release_cleanup = asyncio.Event()
+ self.cleanup_calls = 0
+ self.summaries = set()
+
+ @activity.defn(name="test.workspace.device")
+ async def device(self, context: dict) -> dict:
+ state = devices[context["execution_id"]]
+ state.started.set()
+ try:
+ while not state.finish.is_set():
+ activity.heartbeat("fake device is running")
+ try:
+ await asyncio.wait_for(state.finish.wait(), 0.05)
+ except TimeoutError:
+ pass
+ return {"success": True}
+ except asyncio.CancelledError:
+ state.cancelled.set()
+ await state.release_cleanup.wait()
+ state.cleaned.set()
+ raise
+
+ @activity.defn(name="workflow_runs.record_completion")
+ async def record(self, payload: dict) -> dict:
+ self.summaries.add(payload["run_id"].split(":", 1)[0])
+ return {"recorded": True}
+
+ @activity.defn(name="workspace_tasks.reset_runtime")
+ async def reset_runtime(self, payload: dict) -> dict:
+ assert payload == {"workflow_id": workflow_id}
+ assert devices[first_id].cleaned.is_set()
+ assert first_id in self.summaries
+ self.cleanup_calls += 1
+ self.cleanup_started.set()
+ while not self.release_cleanup.is_set():
+ activity.heartbeat("fake plugin cleanup is running")
+ try:
+ await asyncio.wait_for(self.release_cleanup.wait(), 0.05)
+ except TimeoutError:
+ pass
+ return {"reset_nodes": ["phone-one"]}
+
+ activities = FakeActivities()
+
+ def submission(run_id: str, epoch: int) -> dict:
+ return {
+ "workflow_id": workflow_id, "node_id": "phone-one", "principal": "test-owner",
+ "fingerprint": run_id, "admission_epoch": epoch,
+ "activity": "test.workspace.device", "task_queue": task_queue, "timeout_s": 60,
+ "context": {"workflow_id": workflow_id, "execution_id": run_id},
+ }
+
+ # An explicit native binary disables SDK downloads. The omitted database
+ # filename selects in-memory SQLite, and port=None selects a free port.
+ async with await WorkflowEnvironment.start_local(
+ dev_server_existing_path=cli, ip="127.0.0.1", port=None,
+ ui=False, dev_server_log_level="error",
+ ) as environment:
+ client = environment.client
+
+ async def update(method, argument):
+ return await client.execute_update_with_start_workflow(
+ method, argument, id=uuid4().hex,
+ start_workflow_operation=WithStartWorkflowOperation(
+ WorkspaceTaskControllerWorkflow.run, {"workflow_id": workflow_id},
+ id=controller_id, task_queue=task_queue,
+ id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
+ ),
+ )
+
+ async def refused(request, error_type):
+ with pytest.raises(WorkflowUpdateFailedError) as rejected:
+ await update(WorkspaceTaskControllerWorkflow.submit, request)
+ assert rejected.value.cause.type == error_type
+
+ async with Worker(
+ client, task_queue=task_queue,
+ workflows=[WorkspaceTaskControllerWorkflow, NodeInvocationWorkflow],
+ activities=[activities.device, activities.record, activities.reset_runtime],
+ max_heartbeat_throttle_interval=timedelta(milliseconds=50),
+ default_heartbeat_throttle_interval=timedelta(milliseconds=50),
+ ):
+ controller = client.get_workflow_handle(controller_id)
+ reset_task = None
+ try:
+ assert await update(WorkspaceTaskControllerWorkflow.submit, submission(first_id, 0)) == {
+ "run_id": first_id, "status": "accepted",
+ }
+ await asyncio.wait_for(devices[first_id].started.wait(), 10)
+ assert (await controller.query(WorkspaceTaskControllerWorkflow.describe))["active_count"] == 1
+
+ reset_task = asyncio.create_task(update(WorkspaceTaskControllerWorkflow.reset, "reset-one"))
+ await asyncio.wait_for(devices[first_id].cancelled.wait(), 10)
+ assert not reset_task.done()
+ assert not activities.cleanup_started.is_set()
+ assert (await controller.query(WorkspaceTaskControllerWorkflow.describe))["reset_request_id"] == "reset-one"
+ await refused(submission(second_id, 1), "WorkspaceResetInProgress")
+
+ devices[first_id].release_cleanup.set()
+ await asyncio.wait_for(activities.cleanup_started.wait(), 10)
+ assert not reset_task.done()
+ state = await controller.query(WorkspaceTaskControllerWorkflow.describe)
+ assert state["active_count"] == 0 and state["resetting"] and state["epoch"] == 1
+ assert (await client.get_workflow_handle(first_id).describe()).status == WorkflowExecutionStatus.CANCELED
+ await refused(submission(second_id, 1), "WorkspaceResetInProgress")
+
+ activities.release_cleanup.set()
+ reset_result = {"cancelled": 1, "reset_nodes": ["phone-one"]}
+ assert await asyncio.wait_for(reset_task, 10) == reset_result
+ await refused(submission(second_id, 0), "WorkspaceAdmissionChanged")
+ assert (await update(WorkspaceTaskControllerWorkflow.submit, submission(second_id, 1)))["status"] == "accepted"
+ await asyncio.wait_for(devices[second_id].started.wait(), 10)
+ # Fresh RPC update ID, same logical reset: never cancel work
+ # admitted after that reset or rerun runtime cleanup.
+ assert await update(WorkspaceTaskControllerWorkflow.reset, "reset-one") == reset_result
+ assert not devices[second_id].cancelled.is_set()
+ assert activities.cleanup_calls == 1
+ devices[second_id].finish.set()
+ assert await asyncio.wait_for(client.get_workflow_handle(second_id).result(), 10) == {"success": True}
+
+ async def registry_empty():
+ while (await controller.query(WorkspaceTaskControllerWorkflow.describe))["active_count"]:
+ await asyncio.sleep(0.05)
+
+ await asyncio.wait_for(registry_empty(), 10)
+ history = await controller.fetch_history()
+ child_closed = next(event.event_id for event in history.events if
+ event.event_type == EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED)
+ cleanup_scheduled = next(event.event_id for event in history.events if
+ event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED
+ and event.activity_task_scheduled_event_attributes.activity_type.name == "workspace_tasks.reset_runtime")
+ assert child_closed < cleanup_scheduled
+ replay = await Replayer(workflows=[WorkspaceTaskControllerWorkflow]).replay_workflow(history)
+ assert replay.replay_failure is None
+ finally:
+ for device in devices.values():
+ device.finish.set()
+ device.release_cleanup.set()
+ activities.release_cleanup.set()
+ # Only this in-memory test controller is terminated. Release
+ # every fake cleanup gate even after a failed assertion so
+ # Worker/server context managers can always stop promptly.
+ with suppress(Exception):
+ await controller.terminate("isolated integration test finished")
+ if reset_task is not None:
+ with suppress(TimeoutError):
+ await asyncio.wait_for(asyncio.gather(reset_task, return_exceptions=True), 5)
+
+
+def test_real_temporal_admission_reset_waits_for_child_and_plugin_cleanup():
+ cli = _existing_cli()
+ if cli is None:
+ pytest.skip("Native Temporal CLI unavailable; set TEMPORAL_TEST_CLI (test never downloads it)")
+ completed = subprocess.run(
+ [sys.executable, "-X", "utf8", "-m", "tests.temporal.test_workspace_tasks_integration", str(cli)],
+ cwd=Path(__file__).parents[2], stdin=subprocess.DEVNULL,
+ capture_output=True, text=True, encoding="utf-8", timeout=120, check=False,
+ )
+ assert completed.returncode == 0, completed.stdout + completed.stderr
+
+
+if __name__ == "__main__":
+ asyncio.run(_run_gate(sys.argv[1]))
diff --git a/server/tests/temporal/test_workspace_tasks_workflow.py b/server/tests/temporal/test_workspace_tasks_workflow.py
new file mode 100644
index 00000000..82cdb688
--- /dev/null
+++ b/server/tests/temporal/test_workspace_tasks_workflow.py
@@ -0,0 +1,425 @@
+"""Workspace admission/Reset races using cooperative child handles, no server."""
+
+import asyncio
+from datetime import timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from temporalio.common import WorkflowIDReusePolicy
+from temporalio.exceptions import ApplicationError, WorkflowAlreadyStartedError
+
+from services.temporal import workspace_tasks_workflow as module
+
+
+def payload(run_id="node-invoke-one", *, epoch=0, fingerprint="fingerprint-one"):
+ return {
+ "workflow_id": "workflow-one",
+ "node_id": "phone-one",
+ "principal": "owner",
+ "fingerprint": fingerprint,
+ "activity": "node.mobile_use_agent.v1",
+ "task_queue": "android",
+ "timeout_s": 10800,
+ "admission_epoch": epoch,
+ "context": {"workflow_id": "workflow-one", "execution_id": run_id},
+ }
+
+
+async def settle():
+ for _ in range(5):
+ await asyncio.sleep(0)
+
+
+async def harness(monkeypatch):
+ children = []
+
+ async def wait_condition(predicate):
+ while not predicate():
+ await asyncio.sleep(0)
+
+ monkeypatch.setattr(module.workflow, "wait_condition", wait_condition)
+ monkeypatch.setattr(module.workflow, "all_handlers_finished", lambda: True)
+ monkeypatch.setattr(module.workflow, "info", lambda: SimpleNamespace(
+ is_continue_as_new_suggested=lambda: False,
+ get_current_history_length=lambda: 0,
+ ))
+ cleanup_activity = AsyncMock(return_value={"reset": True})
+ monkeypatch.setattr(module.workflow, "execute_activity", cleanup_activity)
+
+ async def new_child(*, fail=False):
+ child = SimpleNamespace(
+ started=asyncio.Event(), finish=asyncio.Event(),
+ cancel_received=asyncio.Event(), cleanup_finished=asyncio.Event(),
+ )
+
+ async def execute():
+ child.started.set()
+ try:
+ await child.finish.wait()
+ if fail:
+ raise RuntimeError("phone task failed")
+ return {"success": True}
+ except asyncio.CancelledError:
+ child.cancel_received.set()
+ # Mirrors WAIT_CANCELLATION_COMPLETED: requesting cancel does
+ # not complete the handle while the child is cleaning up.
+ await child.cleanup_finished.wait()
+ raise
+
+ child.task = asyncio.create_task(execute())
+ children.append(child)
+ await child.started.wait()
+ return child
+
+ controller = module.WorkspaceTaskControllerWorkflow({"workflow_id": "workflow-one"})
+ yield SimpleNamespace(
+ controller=controller, new_child=new_child, children=children,
+ cleanup_activity=cleanup_activity,
+ )
+ for child in children:
+ child.finish.set()
+ child.cleanup_finished.set()
+ await asyncio.gather(*(child.task for child in children), return_exceptions=True)
+
+
+async def test_submission_returns_after_start_and_releases_completed_or_failed_slots(harness, monkeypatch):
+ controller = harness.controller
+ for fail in (False, True):
+ child = await harness.new_child(fail=fail)
+ start = AsyncMock(return_value=child.task)
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ request = payload(f"node-invoke-{fail}")
+
+ assert await controller.submit(request) == {"run_id": request["context"]["execution_id"], "status": "accepted"}
+ assert not child.task.done()
+ assert controller.describe() == {
+ "epoch": 0, "resetting": False, "active_count": 1,
+ "reset_request_id": None, "last_reset_request_id": None,
+ }
+ options = start.await_args.kwargs
+ assert options["id_reuse_policy"] == WorkflowIDReusePolicy.REJECT_DUPLICATE
+ assert options["cancellation_type"] == module.workflow.ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED
+ assert options["parent_close_policy"] == module.workflow.ParentClosePolicy.REQUEST_CANCEL
+ assert "search_attributes" not in options
+ assert start.await_args.args == (module.NodeInvocationWorkflow.run, request)
+
+ child.finish.set()
+ await settle()
+ assert controller.describe()["active_count"] == 0
+ assert await controller.submit(request) == {"run_id": request["context"]["execution_id"], "status": "accepted"}
+ start.assert_awaited_once()
+
+
+async def test_reset_waits_for_every_child_cleanup_and_blocks_admission(harness, monkeypatch):
+ controller = harness.controller
+ first, second = await harness.new_child(), await harness.new_child()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", AsyncMock(side_effect=[first.task, second.task]))
+ await controller.submit(payload("first"))
+ await controller.submit(payload("second"))
+
+ reset = asyncio.create_task(controller.reset("reset-one"))
+ await asyncio.wait_for(first.cancel_received.wait(), 1)
+ await asyncio.wait_for(second.cancel_received.wait(), 1)
+ assert controller.describe() == {
+ "epoch": 1, "resetting": True, "active_count": 2,
+ "reset_request_id": "reset-one", "last_reset_request_id": None,
+ }
+ assert not reset.done()
+ harness.cleanup_activity.assert_not_awaited()
+ with pytest.raises(ApplicationError) as refused:
+ await controller.submit(payload("late", epoch=1))
+ assert refused.value.type == "WorkspaceResetInProgress"
+
+ duplicate_reset = asyncio.create_task(controller.reset("reset-one"))
+ with pytest.raises(ApplicationError) as concurrent:
+ await controller.reset("different-reset")
+ assert concurrent.value.type == "WorkspaceResetInProgress"
+ first.cleanup_finished.set()
+ await settle()
+ assert not reset.done()
+ assert not duplicate_reset.done()
+ harness.cleanup_activity.assert_not_awaited()
+ second.cleanup_finished.set()
+ assert await asyncio.wait_for(reset, 1) == {"cancelled": 2}
+ assert await asyncio.wait_for(duplicate_reset, 1) == {"cancelled": 2}
+ assert controller.describe() == {
+ "epoch": 1, "resetting": False, "active_count": 0,
+ "reset_request_id": None, "last_reset_request_id": "reset-one",
+ }
+ harness.cleanup_activity.assert_awaited_once()
+ assert harness.cleanup_activity.await_args.args == (
+ "workspace_tasks.reset_runtime", {"workflow_id": "workflow-one"},
+ )
+ options = harness.cleanup_activity.await_args.kwargs
+ assert options["start_to_close_timeout"] == timedelta(seconds=120)
+ assert options["heartbeat_timeout"] == timedelta(seconds=30)
+ assert options["retry_policy"].maximum_attempts == 3
+ assert options["cancellation_type"] == module.workflow.ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+
+
+async def test_reset_fences_no_child_workflow_until_local_runtime_cleanup_finishes(harness, monkeypatch):
+ controller = harness.controller
+ cleanup_started, finish_cleanup = asyncio.Event(), asyncio.Event()
+
+ async def cleanup(*args, **kwargs):
+ cleanup_started.set()
+ await finish_cleanup.wait()
+
+ harness.cleanup_activity.side_effect = cleanup
+ start = AsyncMock()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ reset = asyncio.create_task(controller.reset("local-runtime-reset"))
+ await asyncio.wait_for(cleanup_started.wait(), 1)
+ assert controller.describe() == {
+ "epoch": 1, "resetting": True, "active_count": 0,
+ "reset_request_id": "local-runtime-reset", "last_reset_request_id": None,
+ }
+ assert not controller._ready_to_roll_over()
+ with pytest.raises(ApplicationError) as refused:
+ await controller.submit(payload(epoch=1))
+ assert refused.value.type == "WorkspaceResetInProgress"
+ start.assert_not_awaited()
+ assert not reset.done()
+ finish_cleanup.set()
+ assert await asyncio.wait_for(reset, 1) == {"cancelled": 0}
+ assert not controller.describe()["resetting"]
+
+
+async def test_cleanup_failure_keeps_fence_and_same_reset_can_retry_without_new_epoch(harness, monkeypatch):
+ controller = harness.controller
+ child = await harness.new_child()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", AsyncMock(return_value=child.task))
+ await controller.submit(payload())
+ harness.cleanup_activity.side_effect = [RuntimeError("runtime cleanup failed"), {"reset": True}]
+ reset = asyncio.create_task(controller.reset("retry-cleanup"))
+ await asyncio.wait_for(child.cancel_received.wait(), 1)
+ child.cleanup_finished.set()
+ with pytest.raises(RuntimeError, match="runtime cleanup failed"):
+ await asyncio.wait_for(reset, 1)
+
+ assert controller.describe() == {
+ "epoch": 1, "resetting": True, "active_count": 0,
+ "reset_request_id": "retry-cleanup", "last_reset_request_id": None,
+ }
+ assert "retry-cleanup" not in controller._resets
+ assert not controller._admission_lock.locked()
+ assert not controller._ready_to_roll_over()
+ with pytest.raises(ApplicationError) as refused:
+ await controller.submit(payload("new-request", epoch=1))
+ assert refused.value.type == "WorkspaceResetInProgress"
+ with pytest.raises(ApplicationError) as other_reset:
+ await controller.reset("different-reset")
+ assert other_reset.value.type == "WorkspaceResetInProgress"
+
+ assert await controller.reset("retry-cleanup") == {"cancelled": 1}
+ assert controller.describe() == {
+ "epoch": 1, "resetting": False, "active_count": 0,
+ "reset_request_id": None, "last_reset_request_id": "retry-cleanup",
+ }
+ assert harness.cleanup_activity.await_count == 2
+ assert await controller.reset("retry-cleanup") == {"cancelled": 1}
+ assert harness.cleanup_activity.await_count == 2
+ next_child = await harness.new_child()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", AsyncMock(return_value=next_child.task))
+ assert (await controller.submit(payload("new-request", epoch=1)))["status"] == "accepted"
+
+
+async def test_reset_relays_runtime_reset_nodes_and_remembers_them_for_retries(harness):
+ harness.cleanup_activity.return_value = {"reset_nodes": ["phone-one"], "legacy_cancelled": 2}
+ expected = {"cancelled": 0, "reset_nodes": ["phone-one"]}
+ assert await harness.controller.reset("reset-with-nodes") == expected
+ assert await harness.controller.reset("reset-with-nodes") == expected
+ harness.cleanup_activity.assert_awaited_once()
+
+
+async def test_reset_includes_child_whose_start_acknowledgement_is_pending(harness, monkeypatch):
+ controller = harness.controller
+ child = await harness.new_child()
+ starting, acknowledge = asyncio.Event(), asyncio.Event()
+
+ async def start(*args, **kwargs):
+ starting.set()
+ await acknowledge.wait()
+ return child.task
+
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ submission = asyncio.create_task(controller.submit(payload()))
+ await asyncio.wait_for(starting.wait(), 1)
+ assert controller.describe()["active_count"] == 1
+ reset = asyncio.create_task(controller.reset("reset-during-start"))
+ await settle()
+ assert controller.describe()["epoch"] == 1
+ assert not reset.done()
+ acknowledge.set()
+ assert (await asyncio.wait_for(submission, 1))["status"] == "accepted"
+ await asyncio.wait_for(child.cancel_received.wait(), 1)
+ assert not reset.done()
+ child.cleanup_finished.set()
+ assert await asyncio.wait_for(reset, 1) == {"cancelled": 1}
+ assert controller.describe()["active_count"] == 0
+
+
+async def test_queued_submission_rechecks_reset_fence_after_waiting_for_start_lock(harness, monkeypatch):
+ controller = harness.controller
+ child = await harness.new_child()
+ starting, acknowledge = asyncio.Event(), asyncio.Event()
+
+ async def start(*args, **kwargs):
+ starting.set()
+ await acknowledge.wait()
+ return child.task
+
+ start_mock = AsyncMock(side_effect=start)
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start_mock)
+ first = asyncio.create_task(controller.submit(payload("first")))
+ await asyncio.wait_for(starting.wait(), 1)
+ queued = asyncio.create_task(controller.submit(payload("queued")))
+ await settle()
+ reset = asyncio.create_task(controller.reset("reset-queued"))
+ await settle()
+ acknowledge.set()
+ await first
+ with pytest.raises(ApplicationError) as refused:
+ await queued
+ assert refused.value.type == "WorkspaceResetInProgress"
+ await asyncio.wait_for(child.cancel_received.wait(), 1)
+ child.cleanup_finished.set()
+ await asyncio.wait_for(reset, 1)
+ start_mock.assert_awaited_once()
+
+
+async def test_repeated_reset_preserves_new_work_and_late_network_admission_is_rejected(harness, monkeypatch):
+ controller = harness.controller
+ assert await controller.reset("first-reset") == {"cancelled": 0}
+ child = await harness.new_child()
+ start = AsyncMock(return_value=child.task)
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+
+ with pytest.raises(ApplicationError) as stale:
+ await controller.submit(payload("late-old-request"))
+ assert stale.value.type == "WorkspaceAdmissionChanged"
+ start.assert_not_awaited()
+ await controller.submit(payload("new-request", epoch=1))
+ assert await controller.reset("first-reset") == {"cancelled": 0}
+ assert not child.cancel_received.is_set()
+ assert controller.describe() == {
+ "epoch": 1, "resetting": False, "active_count": 1,
+ "reset_request_id": None, "last_reset_request_id": "first-reset",
+ }
+
+
+async def test_duplicate_submission_checks_fingerprint_without_starting_twice(harness, monkeypatch):
+ child = await harness.new_child()
+ start = AsyncMock(return_value=child.task)
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ await harness.controller.submit(payload())
+ assert (await harness.controller.submit(payload()))["status"] == "accepted"
+ with pytest.raises(ApplicationError) as changed:
+ await harness.controller.submit(payload(fingerprint="different-task"))
+ assert changed.value.type == "WorkspaceSubmissionConflict"
+ start.assert_awaited_once()
+
+
+async def test_historical_duplicate_requires_service_verification_and_is_not_cached(harness, monkeypatch):
+ start = AsyncMock(side_effect=WorkflowAlreadyStartedError("node-invoke-one", "NodeInvocationWorkflow"))
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ assert await harness.controller.submit(payload()) == {"run_id": "node-invoke-one", "status": "duplicate"}
+ assert (await harness.controller.submit(payload(fingerprint="unverified-different")))["status"] == "duplicate"
+ assert harness.controller.describe()["active_count"] == 0
+ assert harness.controller._submissions == {}
+ assert start.await_count == 2
+
+
+async def test_failed_start_does_not_leak_an_admission_slot(harness, monkeypatch):
+ monkeypatch.setattr(module.workflow, "start_child_workflow", AsyncMock(side_effect=RuntimeError("start failed")))
+ with pytest.raises(RuntimeError, match="start failed"):
+ await harness.controller.submit(payload())
+ assert harness.controller.describe()["active_count"] == 0
+ assert not harness.controller._admission_lock.locked()
+ assert harness.controller._submissions == {}
+
+
+async def test_active_queue_is_bounded_and_completed_bookkeeping_is_bounded(harness, monkeypatch):
+ controller = harness.controller
+ children = [await harness.new_child() for _ in range(module.MAX_ACTIVE_TASKS)]
+ start = AsyncMock(side_effect=[child.task for child in children])
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ for index in range(module.MAX_ACTIVE_TASKS):
+ await controller.submit(payload(f"active-{index}"))
+ with pytest.raises(ApplicationError) as full:
+ await controller.submit(payload("overflow"))
+ assert full.value.type == "WorkspaceTaskQueueFull"
+ assert start.await_count == module.MAX_ACTIVE_TASKS
+ for child in children:
+ child.finish.set()
+ await settle()
+
+ for index in range(module.MAX_REMEMBERED_SUBMISSIONS + 2):
+ child = await harness.new_child()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", AsyncMock(return_value=child.task))
+ await controller.submit(payload(f"completed-{index}"))
+ child.finish.set()
+ await settle()
+ assert not controller._active
+ assert len(controller._submissions) == module.MAX_REMEMBERED_SUBMISSIONS
+ assert "completed-0" not in controller._submissions
+
+
+async def test_continue_as_new_waits_for_children_and_handlers_and_carries_reset_fence(harness, monkeypatch):
+ controller = harness.controller
+ for index in range(module.MAX_REMEMBERED_RESETS + 2):
+ await controller.reset(f"reset-{index}")
+ assert len(controller._resets) == module.MAX_REMEMBERED_RESETS
+ child = await harness.new_child()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", AsyncMock(return_value=child.task))
+ await controller.submit(payload(epoch=controller.describe()["epoch"]))
+ monkeypatch.setattr(module.workflow, "info", lambda: SimpleNamespace(
+ is_continue_as_new_suggested=lambda: True,
+ get_current_history_length=lambda: 10001,
+ ))
+ assert not controller._ready_to_roll_over()
+ child.finish.set()
+ await settle()
+ monkeypatch.setattr(module.workflow, "all_handlers_finished", lambda: False)
+ assert not controller._ready_to_roll_over()
+ monkeypatch.setattr(module.workflow, "all_handlers_finished", lambda: True)
+ carried = {}
+
+ class Continued(BaseException):
+ pass
+
+ def continue_as_new(value):
+ carried.update(value)
+ raise Continued()
+
+ monkeypatch.setattr(module.workflow, "continue_as_new", continue_as_new)
+ with pytest.raises(Continued):
+ await controller.run({"workflow_id": "workflow-one"})
+ assert carried["epoch"] == module.MAX_REMEMBERED_RESETS + 2
+ assert len(carried["resets"]) == module.MAX_REMEMBERED_RESETS
+ replacement = module.WorkspaceTaskControllerWorkflow(carried)
+ assert await replacement.reset(f"reset-{module.MAX_REMEMBERED_RESETS + 1}") == {"cancelled": 0}
+ assert replacement.describe()["epoch"] == carried["epoch"]
+ assert (await replacement.submit(payload(epoch=carried["epoch"])))["status"] == "accepted"
+ assert replacement.describe()["active_count"] == 0 # completed dedup survives rollover
+
+
[email protected]("change", [
+ {"workflow_id": "other-workflow"},
+ {"context": {"workflow_id": "other-workflow", "execution_id": "other"}},
+])
+async def test_controller_rejects_cross_workflow_submission(harness, monkeypatch, change):
+ start = AsyncMock()
+ monkeypatch.setattr(module.workflow, "start_child_workflow", start)
+ with pytest.raises(ApplicationError) as error:
+ await harness.controller.submit({**payload(), **change})
+ assert error.value.type == "WorkspaceScopeMismatch"
+ start.assert_not_awaited()
+
+
+def test_framework_registers_workspace_task_controller():
+ from services.temporal.worker import _framework_workflows
+
+ assert _framework_workflows().count(module.WorkspaceTaskControllerWorkflow) == 1
diff --git a/server/tests/test_mobile_progress.py b/server/tests/test_mobile_progress.py
new file mode 100644
index 00000000..9ceb59f0
--- /dev/null
+++ b/server/tests/test_mobile_progress.py
@@ -0,0 +1,135 @@
+"""SDK callback contract without installing mobile-use in the main server."""
+
+from enum import Enum
+from types import SimpleNamespace
+
+import pytest
+
+from nodes.mobile.runtime.progress import ProgressCallbacks, action_summary, plan_summary
+
+
+def progress():
+ events = []
+ handler = ProgressCallbacks(lambda event_type, **data: events.append({"type": event_type, **data}))
+ return handler, events
+
+
+def start_node(handler, role, step, run_id="node"):
+ handler.on_chain_start(None, {"screenshot": "private"}, run_id=run_id, name=role,
+ metadata={"langgraph_node": role, "langgraph_step": step}, tags=[f"graph:step:{step}"])
+
+
+def test_worker_wire_preserves_event_type_and_semantic_kind(monkeypatch):
+ import json
+ from io import StringIO
+ from nodes.mobile.runtime import worker
+
+ wire = StringIO()
+ monkeypatch.setattr(worker, "WIRE", wire)
+ handler = ProgressCallbacks(worker.emit)
+ handler.on_tool_start({"name": "tap"}, "", run_id="tap", inputs={"target": {"text": "Settings"}})
+ event = json.loads(wire.getvalue())
+ assert event["v"] == 1
+ assert event["type"] == "agent_update"
+ assert event["kind"] == "action"
+ assert event["message"] == "Tap · Settings"
+
+
+def test_nested_callbacks_and_parallel_nodes_do_not_duplicate_steps(progress):
+ handler, events = progress
+ start_node(handler, "cortex", 5)
+ handler.on_chain_start(None, {}, run_id="nested", name="RunnableSequence",
+ metadata={"langgraph_node": "cortex", "langgraph_step": 5}, tags=["seq:step:1"])
+ start_node(handler, "orchestrator", 5, "parallel")
+ start_node(handler, "executor", 6, "later")
+ start_node(handler, "orchestrator", 5, "late-callback")
+ assert events == [{"type": "progress", "steps": 5}, {"type": "progress", "steps": 6}]
+
+
+def test_plan_reads_sdk_objects_and_replanning_without_cumulative_thoughts(progress):
+ class Status(Enum):
+ PENDING = "PENDING"
+ SUCCESS = "SUCCESS"
+
+ handler, events = progress
+ goal = SimpleNamespace(id="g1", description="Open Settings", status=Status.PENDING, completion_reason=None)
+ start_node(handler, "planner", 1)
+ handler.on_chain_end({"subgoal_plan": [goal], "agents_thoughts": ["PRIVATE REASONING"],
+ "latest_screenshot": "private"}, run_id="node")
+ assert events[-1]["current_goal"] == "Open Settings"
+ assert events[-1]["plan"][0]["status"] == "pending"
+ start_node(handler, "orchestrator", 2)
+ handler.on_chain_end({"subgoal_plan": [goal]}, run_id="node")
+ assert len([e for e in events if e["type"] == "agent_update"]) == 1
+ goal.status, goal.completion_reason = Status.SUCCESS, "Settings screen is visible."
+ start_node(handler, "orchestrator", 3)
+ handler.on_chain_end({"subgoal_plan": [goal]}, run_id="node")
+ assert events[-1]["current_goal"] is None
+ assert events[-1]["plan"][0]["reason"] == "Settings screen is visible."
+ assert "private" not in str(events).lower()
+ # The parent validates the normalized wire plan again without losing reasons.
+ assert plan_summary(events[-1]["plan"]) == events[-1]["plan"]
+
+
+def test_actual_tool_error_result_is_not_reported_as_success(progress):
+ handler, events = progress
+ handler.on_tool_start({"name": "tap"}, "NOT JSON", run_id="tap1",
+ inputs={"target": {"text": "Search"}, "agent_thought": "Open search to find the app.",
+ "state": {"screenshot": "PRIVATE"}})
+ handler.on_tool_end(SimpleNamespace(update={"executor_messages": [SimpleNamespace(status="error", content="Element not found")]}), run_id="tap1")
+ assert events[0]["message"] == "Tap · Search"
+ assert events[0]["detail"] == "Open search to find the app."
+ assert events[1]["state"] == "failed"
+ assert events[1]["outcome"] == "Element not found"
+ assert events[1]["duration_ms"] >= 0
+ assert "PRIVATE" not in str(events)
+
+
+def test_typed_payload_and_echoed_field_contents_never_leave_callbacks(progress):
+ handler, events = progress
+ handler.on_tool_start({"name": "focus_and_input_text"}, "private", run_id="type1", inputs={
+ "text": "my-secret", "target": {"resource_id": "search_field"}, "agent_thought": "Enter my-secret in the field."})
+ handler.on_tool_end(SimpleNamespace(update={"executor_messages": [SimpleNamespace(status="success", content="Typed my-secret; field content: private")] }), run_id="type1")
+ assert events[0]["message"] == "Type 9 characters · search_field"
+ assert "my-secret" not in str(events)
+ assert "private" not in str(events)
+ assert "[typed text]" in events[0]["detail"]
+ assert events[-1]["state"] == "completed"
+
+
+def test_unknown_result_is_unconfirmed_and_exception_does_not_echo_error(progress):
+ handler, events = progress
+ handler.on_tool_start({"name": "back"}, "", run_id="back", inputs={})
+ handler.on_tool_end("arbitrary output is not evidence of success", run_id="back")
+ assert events[-1]["state"] == "returned"
+ handler.on_tool_start({"name": "back"}, "", run_id="back", inputs={})
+ handler.on_tool_error(RuntimeError("token=private"), run_id="back")
+ assert events[-1]["state"] == "failed"
+ assert "token=" not in str(events)
+ handler.on_tool_start({"name": "arbitrary-secret-tool"}, "private", run_id="other", inputs={})
+ handler.on_tool_end("private", run_id="other")
+ assert len(events) == 4
+
+
[email protected](("name", "args", "expected"), [
+ ("launch_app", {"app_name": "Settings"}, "Open app · Settings"),
+ ("stop_app", {"package_name": "com.android.settings"}, "Close app · com.android.settings"),
+ ("tap", {"target": {"bounds": {"x": 10, "y": 20, "width": 80, "height": 40}}}, "Tap · area (10, 20, 80 × 40)"),
+ ("swipe_percentages", {"start_x_percent": 50, "start_y_percent": 80, "end_x_percent": 50, "end_y_percent": 20}, "Swipe · (50, 80) → (50, 20) %"),
+ ("press_key", {"key": "home"}, "Press · Home"),
+ ("wait_for_delay", {"time_in_ms": 1500}, "Wait · 1.5s"),
+])
+def test_concrete_action_labels(name, args, expected):
+ assert action_summary(name, args)["message"] == expected
+
+
+def test_bounds_and_links_are_filtered():
+ assert action_summary("open_link", {"url": "https://private.test/?token=secret", "agent_thought": "Open https://private.test/?token=secret now"}) == {
+ "message": "Open link", "detail": "Open [link] now"}
+ goals = [{"id": "a" * 200, "description": "x" * 1000, "status": "NOT_STARTED"}] * 100
+ plan = plan_summary(goals)
+ assert len(plan) == 40
+ assert len(plan[0]["description"]) == 300
+ assert len(plan[0]["id"]) == 100
+ assert plan_summary({"unexpected": "private"}) is None
diff --git a/server/tests/test_mobile_progress_sdk.py b/server/tests/test_mobile_progress_sdk.py
new file mode 100644
index 00000000..7e08e41a
--- /dev/null
+++ b/server/tests/test_mobile_progress_sdk.py
@@ -0,0 +1,228 @@
+"""Optional contract test against the installed, isolated mobile-use SDK.
+
+Runs a local graph with fake actions, not a phone or model. No SDK installation
+or downloads are performed; point MOBILE_USE_UPSTREAM_SOURCE at an existing
+checkout with its own .venv to exercise another pinned installation.
+"""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import textwrap
+
+import pytest
+
+
+_SERVER = Path(__file__).resolve().parents[1]
+_RESULT_PREFIX = "MOBILE_PROGRESS_SDK_RESULT="
+
+# Keep SDK imports in its interpreter: the ordinary server must not acquire
+# langchain/mobile-use dependencies just to collect this optional test.
+_GRAPH_SCRIPT = textwrap.dedent(r'''
+ import asyncio
+ import importlib.metadata
+ import importlib.util
+ import json
+ import socket
+ import sys
+ import types
+ from typing import Annotated
+
+ sys.path.insert(0, sys.argv[2])
+ if any(importlib.util.find_spec(name) is None for name in ("langgraph", "langchain_core", "pydantic")):
+ raise SystemExit(77)
+
+ # The real ExecutorToolNode imports SDK logging/telemetry. They are not
+ # under test; isolate them so the probe cannot send telemetry or write logs.
+ telemetry_module = types.ModuleType("minitap.mobile_use.services.telemetry")
+ telemetry_module.telemetry = None
+ sys.modules[telemetry_module.__name__] = telemetry_module
+ logger_module = types.ModuleType("minitap.mobile_use.utils.logger")
+ logger_module.get_logger = lambda _: types.SimpleNamespace(info=lambda *_args, **_kwargs: None)
+ sys.modules[logger_module.__name__] = logger_module
+
+ from pydantic import BaseModel
+ from langchain_core.callbacks import BaseCallbackHandler
+ from langchain_core.messages import AIMessage, AnyMessage, ToolMessage
+ from langchain_core.runnables import RunnableLambda
+ from langchain_core.tools import tool
+ from langchain_core.tools.base import InjectedToolCallId
+ from langgraph.graph import START, END, StateGraph, add_messages
+ from langgraph.prebuilt import InjectedState
+ from langgraph.types import Command
+ from minitap.mobile_use.agents.executor.tool_node import ExecutorToolNode
+ from minitap.mobile_use.agents.planner.types import Subgoal, SubgoalStatus
+
+ spec = importlib.util.spec_from_file_location("mobile_progress_under_test", sys.argv[1])
+ progress_module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(progress_module)
+
+ class Progress(progress_module.ProgressCallbacks, BaseCallbackHandler):
+ # Callback errors must fail this test rather than being logged and ignored.
+ raise_error = True
+
+ events = []
+ callback = Progress(lambda event, **fields: events.append({"type": event, **fields}))
+ screenshot = "SCREENSHOT_BASE64_MUST_NOT_BE_PUBLISHED"
+ typed_text = "private-field-value-12345"
+ raw_input = "RAW_INPUT_MUST_NOT_BE_PUBLISHED"
+
+ class State(BaseModel):
+ subgoal_plan: list[Subgoal]
+ executor_messages: Annotated[list[AnyMessage], add_messages]
+ latest_screenshot: str
+
+ @tool
+ async def tap(
+ agent_thought: str,
+ target: dict,
+ extra_input: str,
+ tool_call_id: Annotated[str, InjectedToolCallId],
+ state: Annotated[State, InjectedState],
+ ) -> Command:
+ """Fake successful tap; no phone interaction."""
+ assert state.latest_screenshot == screenshot
+ return Command(update={"executor_messages": [ToolMessage(
+ tool_call_id=tool_call_id, content="Tapped Settings.", status="success",
+ )]})
+
+ @tool
+ async def focus_and_input_text(
+ agent_thought: str,
+ text: str,
+ target: dict,
+ tool_call_id: Annotated[str, InjectedToolCallId],
+ state: Annotated[State, InjectedState],
+ ) -> Command:
+ """Fake failed text entry; outcome deliberately echoes private text."""
+ return Command(update={"executor_messages": [ToolMessage(
+ tool_call_id=tool_call_id, content="Failed to type " + text, status="error",
+ )]})
+
+ async def nested_chain():
+ # Nested chains inherit langgraph_node. Their result must not be
+ # mistaken for a real planner/orchestrator update or phase change.
+ await RunnableLambda(lambda _: {"subgoal_plan": [Subgoal(
+ id="nested", description="NOT_A_GRAPH_PLAN", status=SubgoalStatus.PENDING,
+ )]}, name="nested_parser").ainvoke({"private_input": raw_input})
+
+ class Planner:
+ async def __call__(self, state):
+ await nested_chain()
+ return {"subgoal_plan": [
+ Subgoal(id="open", description="Open Settings", status=SubgoalStatus.NOT_STARTED),
+ Subgoal(id="name", description="Update display name", status=SubgoalStatus.NOT_STARTED),
+ ]}
+
+ async def orchestrator(state):
+ await nested_chain()
+ return {"subgoal_plan": [
+ state.subgoal_plan[0].model_copy(update={"status": SubgoalStatus.PENDING}),
+ state.subgoal_plan[1],
+ ]}
+
+ async def executor(state):
+ return {"executor_messages": [AIMessage(content="", tool_calls=[
+ {"name": "tap", "id": "tap-call", "args": {
+ "agent_thought": "Open the app settings", "target": {"text": "Settings"},
+ "extra_input": raw_input,
+ }},
+ {"name": "focus_and_input_text", "id": "type-call", "args": {
+ "agent_thought": "Enter " + typed_text + " into the display-name field",
+ "text": typed_text, "target": {"text": "Display name"},
+ }},
+ {"name": "tap", "id": "skipped-call", "args": {
+ "agent_thought": "This action must be skipped after failure",
+ "target": {"text": "Save"}, "extra_input": raw_input,
+ }},
+ ])]}
+
+ def post_executor_gate(state):
+ return "invoke_tools"
+
+ async def main():
+ # Install the guard after asyncio has created its internal wakeup
+ # socket, so any graph/model/device network attempt fails immediately.
+ def refuse_network(*args, **kwargs):
+ raise AssertionError("The SDK callback probe must not use the network")
+ socket.create_connection = socket.socket.connect = socket.socket.connect_ex = refuse_network
+
+ graph = StateGraph(State)
+ graph.add_node("planner", Planner())
+ graph.add_node("orchestrator", orchestrator)
+ graph.add_node("executor", executor)
+ graph.add_node("executor_tools", ExecutorToolNode(
+ [tap, focus_and_input_text], messages_key="executor_messages",
+ ))
+ graph.add_edge(START, "planner")
+ graph.add_edge("planner", "orchestrator")
+ graph.add_edge("orchestrator", "executor")
+ graph.add_conditional_edges("executor", post_executor_gate, {"invoke_tools": "executor_tools"})
+ graph.add_edge("executor_tools", END)
+ async for _ in graph.compile().astream(
+ {"subgoal_plan": [], "executor_messages": [], "latest_screenshot": screenshot},
+ config={"callbacks": [callback]},
+ stream_mode=["messages", "custom", "updates", "values"],
+ ):
+ pass
+
+ asyncio.run(main())
+ print("MOBILE_PROGRESS_SDK_RESULT=" + json.dumps(events, ensure_ascii=False))
+''')
+
+
+def test_installed_sdk_emits_plans_and_semantic_actions_without_nested_duplicates_or_raw_state(tmp_path):
+ default = _SERVER / ".opencompany" / "mobile" / "engine"
+ engine = Path(os.environ.get("MOBILE_USE_UPSTREAM_SOURCE", default)).expanduser().resolve()
+ python = engine / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+ if not python.is_file() or not (engine / "minitap/mobile_use/agents/executor/tool_node.py").is_file():
+ pytest.skip("Requires the installed mobile engine, or MOBILE_USE_UPSTREAM_SOURCE with its isolated .venv")
+
+ result = subprocess.run(
+ [str(python), "-X", "utf8", "-c", _GRAPH_SCRIPT, str(_SERVER / "nodes/mobile/runtime/progress.py"), str(engine)],
+ cwd=tmp_path,
+ env={**os.environ, "PYTHONUTF8": "1", "MOBILE_USE_TELEMETRY_ENABLED": "false",
+ "LANGCHAIN_TRACING_V2": "false", "LANGSMITH_TRACING": "false"},
+ capture_output=True,
+ text=True,
+ encoding="utf-8",
+ timeout=30,
+ check=False,
+ )
+ if result.returncode == 77:
+ pytest.skip("The isolated mobile SDK dependencies are not installed")
+ assert result.returncode == 0, result.stdout + result.stderr
+ line = next((line for line in result.stdout.splitlines() if line.startswith(_RESULT_PREFIX)), None)
+ assert line is not None, result.stdout + result.stderr
+ events = json.loads(line.removeprefix(_RESULT_PREFIX))
+
+ assert [event["steps"] for event in events if event["type"] == "progress"] == [1, 2, 3, 4]
+ plans = [event for event in events if event.get("kind") == "plan"]
+ assert len(plans) == 2
+ assert plans[0]["plan"] == [
+ {"id": "open", "description": "Open Settings", "status": "not_started", "reason": ""},
+ {"id": "name", "description": "Update display name", "status": "not_started", "reason": ""},
+ ]
+ assert plans[0]["current_goal"] is None
+ assert plans[1]["current_goal"] == "Open Settings"
+ assert plans[1]["plan"][0]["status"] == "pending"
+
+ actions = [event for event in events if event.get("kind") == "action"]
+ assert [event["state"] for event in actions] == ["started", "completed", "started", "failed"]
+ assert actions[0]["message"] == "Tap · Settings"
+ assert actions[0]["detail"] == "Open the app settings"
+ assert actions[1]["action_id"] == actions[0]["action_id"]
+ assert actions[1]["outcome"] == "Tapped Settings."
+ assert actions[2]["message"] == "Type 25 characters · Display name"
+ assert actions[2]["detail"] == "Enter [typed text] into the display-name field"
+ assert actions[3]["action_id"] == actions[2]["action_id"]
+ assert actions[2]["action_id"] != actions[0]["action_id"]
+ assert actions[3]["duration_ms"] >= 0
+
+ serialized = json.dumps(events)
+ for forbidden in (
+ "NOT_A_GRAPH_PLAN", "SCREENSHOT_BASE64_MUST_NOT_BE_PUBLISHED", "RAW_INPUT_MUST_NOT_BE_PUBLISHED",
+ "private-field-value-12345", "latest_screenshot", "executor_messages", "extra_input", "skipped-call",
+ ):
+ assert forbidden not in serialized
diff --git a/server/tests/test_mobile_reset.py b/server/tests/test_mobile_reset.py
new file mode 100644
index 00000000..192cb0e5
--- /dev/null
+++ b/server/tests/test_mobile_reset.py
@@ -0,0 +1,372 @@
+"""Reset acknowledges only after phone tasks and admitted writes have drained."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from nodes.mobile._control import MobileError
+from nodes.mobile._node import MobileParams, MobileUseAgent
+from nodes.mobile._runtime import MobileRuntime
+from nodes.mobile._tool import AndroidTool
+from services.plugin import NodeContext, NodeUserError
+
+
+def runtime(monkeypatch):
+ import nodes.mobile._runtime as module
+
+ value = MobileRuntime()
+ value.serial = "emulator-test"
+ value._setup_loaded = True
+ monkeypatch.setattr(module, "event", Mock())
+ monkeypatch.setattr(value, "driver_call", AsyncMock(return_value={"width": 1080, "height": 1920}))
+ monkeypatch.setattr(value, "_publish_progress", AsyncMock())
+ return value
+
+
+def args(**updates):
+ return {
+ "principal": "owner", "workflow_id": "wf", "node_id": "phone", "run_id": "run",
+ "generation": 4, "params": {"prompt": "Open settings", "max_steps": 5, "timeout_s": 30},
+ "model": {}, "broker_url": "http://127.0.0.1/broker", **updates,
+ }
+
+
+async def cancelled(task):
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, 1)
+
+
+async def reset(runtime, **updates):
+ await runtime.reset_execution_state(**{"workflow_id": "wf", "node_id": "phone", "generation": 4, **updates})
+
+
+async def test_reset_cancels_queued_coroutine_without_acquiring_device(runtime, monkeypatch):
+ worker = AsyncMock()
+ monkeypatch.setattr(runtime, "_run_worker", worker)
+ await runtime.task_lock.acquire()
+ task = asyncio.create_task(runtime.run(**args()))
+ other = asyncio.create_task(runtime.run(**args(run_id="other", workflow_id="other-wf")))
+ await asyncio.sleep(0)
+ await asyncio.wait_for(reset(runtime), 1)
+ await cancelled(task)
+ assert [entry["run_id"] for entry in runtime.queue] == ["other"]
+ assert not other.done()
+ assert runtime.last_task is None
+ assert "run" not in runtime._run_tasks
+ worker.assert_not_awaited()
+ await runtime.cancel("other-wf", "phone")
+ await cancelled(other)
+ runtime.task_lock.release()
+
+
+class WaitingWorker:
+ def __init__(self):
+ self.returncode = None
+ self.terminated = asyncio.Event()
+
+ def terminate(self):
+ self.returncode = -15
+ self.terminated.set()
+
+ def kill(self):
+ self.terminate()
+
+ async def wait(self):
+ await self.terminated.wait()
+ return self.returncode
+
+
+async def test_reset_stops_model_wait_and_clears_activity_before_ack(runtime, monkeypatch):
+ entered = asyncio.Event()
+ process = WaitingWorker()
+
+ async def worker(*_args):
+ runtime.worker = process
+ runtime.active.update(phase="Waiting for model", activity=[{"message": "old"}], plan=[{"description": "old"}])
+ entered.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(runtime, "_run_worker", worker)
+ task = asyncio.create_task(runtime.run(**args()))
+ await entered.wait()
+ assert runtime.capabilities
+ await asyncio.wait_for(reset(runtime), 1)
+ await cancelled(task)
+ assert process.terminated.is_set()
+ assert runtime.worker is None
+ assert not runtime.capabilities
+ assert not runtime._run_tasks
+ assert runtime.active is None
+ assert runtime.last_task is None
+ assert runtime.serial == "emulator-test"
+ await reset(runtime) # The two reset paths may call a hook twice.
+ assert runtime.last_task is None
+
+
+async def test_reset_paused_task_preserves_human_lease(runtime):
+ lease = await runtime.takeover("viewer")
+ task = asyncio.create_task(runtime.run(**args()))
+ await asyncio.sleep(0)
+ assert runtime.active["status"] == "awaiting_user"
+ await asyncio.wait_for(reset(runtime), 1)
+ await cancelled(task)
+ assert runtime.control.owner == lease.owner
+ assert runtime.control.epoch == lease.epoch
+ assert runtime.viewer == "viewer"
+ assert runtime.last_task is None
+
+
+async def test_reset_cleans_task_cancelled_during_initial_progress(runtime, monkeypatch):
+ entered = asyncio.Event()
+
+ async def publish(_phase):
+ entered.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(runtime, "_publish_progress", publish)
+ task = asyncio.create_task(runtime.run(**args()))
+ await entered.wait()
+ await asyncio.wait_for(reset(runtime), 1)
+ await cancelled(task)
+ assert runtime.active is None
+ assert runtime.last_task is None
+ assert not runtime._run_tasks
+
+
+async def test_reset_waits_for_admitted_mutation_and_rejects_late_generation(runtime, monkeypatch):
+ entered, mutation_entered, finish = asyncio.Event(), asyncio.Event(), asyncio.Event()
+
+ async def worker(*_args):
+ entered.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(runtime, "_run_worker", worker)
+ task = asyncio.create_task(runtime.run(**args()))
+ await entered.wait()
+ lease = next(iter(runtime.capabilities.values()))[1]
+
+ async def mutation():
+ mutation_entered.set()
+ await finish.wait()
+
+ write = asyncio.create_task(runtime.control.perform(lease, "write", mutation))
+ await mutation_entered.wait()
+ resetting = asyncio.create_task(reset(runtime))
+ await asyncio.sleep(0)
+ assert not resetting.done()
+ assert runtime.control.owner is None
+ assert not runtime.capabilities
+ with pytest.raises(MobileError, match="reset workflow generation"):
+ await runtime.run(**args(run_id="late"))
+ finish.set()
+ await write
+ await asyncio.wait_for(resetting, 1)
+ await cancelled(task)
+ assert runtime.active is None
+ assert runtime.last_task is None
+ monkeypatch.setattr(runtime, "_run_worker", AsyncMock(return_value={"success": True, "result": "new"}))
+ assert (await runtime.run(**args(run_id="new", generation=5)))["response"] == "new"
+
+
[email protected]("reset_generation,other", [
+ (4, {"workflow_id": "other"}), (4, {"node_id": "other"}), (4, {"generation": 5}),
+ (0, {"generation": 1}), (0, {"generation": 5}),
+])
+async def test_reset_preserves_unrelated_active_task_and_capability(runtime, monkeypatch, reset_generation, other):
+ entered, finish = asyncio.Event(), asyncio.Event()
+
+ async def worker(*_args):
+ entered.set()
+ await finish.wait()
+ return {"success": True, "result": "untouched"}
+
+ monkeypatch.setattr(runtime, "_run_worker", worker)
+ task = asyncio.create_task(runtime.run(**args(**other)))
+ await entered.wait()
+ before = dict(runtime.capabilities)
+ owner, epoch = runtime.control.owner, runtime.control.epoch
+ await reset(runtime, generation=reset_generation)
+ assert runtime.capabilities == before
+ assert (runtime.control.owner, runtime.control.epoch) == (owner, epoch)
+ assert not task.done()
+ finish.set()
+ assert (await task)["response"] == "untouched"
+ assert runtime.last_task["status"] == "completed"
+
+
+async def test_cancel_run_id_waits_for_only_selected_queue_entry(runtime):
+ await runtime.task_lock.acquire()
+ first = asyncio.create_task(runtime.run(**args(run_id="first")))
+ second = asyncio.create_task(runtime.run(**args(run_id="second")))
+ await asyncio.sleep(0)
+ await runtime.cancel("wf", "phone", run_id="first")
+ await cancelled(first)
+ assert [entry["run_id"] for entry in runtime.queue] == ["second"]
+ assert not second.done()
+ await runtime.cancel("wf", "phone", run_id="second")
+ await cancelled(second)
+ runtime.task_lock.release()
+
+
+async def test_phone_only_reset_cancels_direct_queue_but_preserves_graph_and_other_node(runtime):
+ await runtime.task_lock.acquire()
+ direct = asyncio.create_task(runtime.run(**args(run_id="direct", generation=0)))
+ graph = asyncio.create_task(runtime.run(**args(run_id="graph", generation=5)))
+ other = asyncio.create_task(runtime.run(**args(run_id="other", node_id="other", generation=0)))
+ await asyncio.sleep(0)
+ await reset(runtime, generation=0)
+ await cancelled(direct)
+ assert [entry["run_id"] for entry in runtime.queue] == ["graph", "other"]
+ assert not graph.done() and not other.done()
+ assert runtime._retired_generations == {}
+ await runtime.cancel("wf", "phone")
+ await runtime.cancel("wf", "other")
+ await cancelled(graph)
+ await cancelled(other)
+ runtime.task_lock.release()
+
+
+async def test_negative_reset_generation_cannot_expand_reset_scope(runtime):
+ await runtime.task_lock.acquire()
+ task = asyncio.create_task(runtime.run(**args()))
+ await asyncio.sleep(0)
+ prior = {"workflow_id": "wf", "node_id": "phone", "run_id": "prior", "generation": 4}
+ runtime.last_task = prior
+ with pytest.raises(MobileError, match="cannot be negative"):
+ await reset(runtime, generation=-1)
+ assert runtime.last_task is prior
+ assert runtime.queue[0]["status"] == "queued"
+ assert not task.done()
+ assert runtime._retired_generations == {}
+ await runtime.cancel("wf", "phone")
+ await cancelled(task)
+ runtime.task_lock.release()
+
+
+async def test_reset_during_natural_cleanup_does_not_interrupt_drain(runtime, monkeypatch):
+ worker_entered, finish_worker = asyncio.Event(), asyncio.Event()
+ mutation_entered, finish_mutation = asyncio.Event(), asyncio.Event()
+
+ async def worker(*_args):
+ worker_entered.set()
+ await finish_worker.wait()
+ return {"success": True, "result": "completed"}
+
+ async def mutation():
+ mutation_entered.set()
+ await finish_mutation.wait()
+
+ monkeypatch.setattr(runtime, "_run_worker", worker)
+ task = asyncio.create_task(runtime.run(**args()))
+ await worker_entered.wait()
+ lease = next(iter(runtime.capabilities.values()))[1]
+ write = asyncio.create_task(runtime.control.perform(lease, "write", mutation))
+ await mutation_entered.wait()
+ finish_worker.set()
+ await asyncio.sleep(0)
+ assert "run" in runtime._draining_runs
+ first = asyncio.create_task(reset(runtime))
+ second = asyncio.create_task(reset(runtime))
+ await asyncio.sleep(0)
+ assert not first.done() and not second.done()
+ assert not task.cancelling()
+ finish_mutation.set()
+ await write
+ await asyncio.wait_for(asyncio.gather(task, first, second), 1)
+ assert runtime.active is None
+ assert runtime.last_task is None
+ assert not runtime._run_tasks
+
+
[email protected]("reset_generation,generation,cleared", [
+ (4, 0, True), (4, 4, True), (4, 5, False),
+ (0, 0, True), (0, 4, False), (0, 5, False), (0, -1, False),
+])
+async def test_reset_clears_only_matching_finished_activity(runtime, reset_generation, generation, cleared):
+ entry = {"workflow_id": "wf", "node_id": "phone", "run_id": "old", "generation": generation,
+ "status": "completed", "activity": [{"message": "old activity"}]}
+ runtime.last_task = entry
+ await reset(runtime, generation=reset_generation)
+ assert runtime.last_task is (None if cleared else entry)
+
+
[email protected]("node_cls", [MobileUseAgent, AndroidTool])
+async def test_plugin_reset_does_not_instantiate_runtime(monkeypatch, node_cls):
+ import nodes.mobile._runtime as module
+
+ monkeypatch.setattr(module, "_runtime", None)
+ constructor = Mock(side_effect=AssertionError("Reset must not initialize a phone"))
+ monkeypatch.setattr(module, "MobileRuntime", constructor)
+ result = await node_cls.reset_execution_state(node_id="phone", workflow_id="wf", execution_id="exec",
+ generation=4, graph={}, database=None)
+ assert result == {"reset": False}
+ constructor.assert_not_called()
+
+
[email protected]("node_cls", [MobileUseAgent, AndroidTool])
+async def test_plugin_reset_delegates_existing_runtime(monkeypatch, node_cls):
+ import nodes.mobile._runtime as module
+
+ hook = AsyncMock(return_value={"reset": True})
+ monkeypatch.setattr(module, "_runtime", SimpleNamespace(reset_execution_state=hook))
+ assert await node_cls.reset_execution_state(node_id="phone", workflow_id="wf", execution_id="exec",
+ generation=4, graph={}, database=None) == {"reset": True}
+ hook.assert_awaited_once_with(workflow_id="wf", node_id="phone", generation=4)
+
+
[email protected]("generation,status,rejected", [(4, "resetting", True), (4, "reset", True),
+ (5, "active", True), (4, "active", False)])
+async def test_graph_admission_reads_latest_control_immediately_before_run(monkeypatch, generation, status, rejected):
+ import nodes.mobile._node as node
+ import nodes.mobile._runtime as module
+ import services.plugin.deps as deps
+
+ order = []
+
+ async def broker():
+ order.append("broker")
+ return "http://local"
+
+ async def latest(_workflow):
+ order.append("control")
+ return SimpleNamespace(generation=generation, status=status)
+
+ async def run(**kwargs):
+ order.append("run")
+ assert kwargs["generation"] == 4
+ return {"outcome": "completed"}
+
+ value = SimpleNamespace(ensure_broker=broker, run=AsyncMock(side_effect=run))
+ monkeypatch.setattr(module, "get_runtime", lambda: value)
+ monkeypatch.setattr(node, "require_mobile_owner", AsyncMock())
+ monkeypatch.setattr(node, "resolve_model", AsyncMock(return_value={}))
+ monkeypatch.setattr(deps, "get_database", lambda: SimpleNamespace(get_latest_workflow_control=latest))
+ ctx = NodeContext(node_id="phone", node_type="mobile_use_agent", workflow_id="wf", execution_id="exec", raw={"generation": 4})
+ if rejected:
+ with pytest.raises(NodeUserError, match="inactive workflow generation"):
+ await MobileUseAgent().execute_op(ctx, MobileParams(prompt="Open settings"))
+ value.run.assert_not_awaited()
+ assert order == ["broker", "control"]
+ else:
+ await MobileUseAgent().execute_op(ctx, MobileParams(prompt="Open settings"))
+ assert order == ["broker", "control", "run"]
+
+
+async def test_workspace_generation_zero_does_not_query_graph_control(monkeypatch):
+ import nodes.mobile._node as node
+ import nodes.mobile._runtime as module
+ import services.plugin.deps as deps
+
+ value = SimpleNamespace(ensure_broker=AsyncMock(return_value="http://local"), run=AsyncMock(return_value={}))
+ monkeypatch.setattr(module, "get_runtime", lambda: value)
+ monkeypatch.setattr(node, "require_mobile_owner", AsyncMock())
+ monkeypatch.setattr(node, "resolve_model", AsyncMock(return_value={}))
+ database = Mock(side_effect=AssertionError("Direct invocations use their Temporal barrier"))
+ monkeypatch.setattr(deps, "get_database", database)
+ ctx = NodeContext(node_id="phone", node_type="mobile_use_agent", workflow_id="wf", execution_id="node-invoke-direct")
+ await MobileUseAgent().execute_op(ctx, MobileParams(prompt="Open settings"))
+ assert value.run.await_args.kwargs["generation"] == 0
+ database.assert_not_called()
diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.py
index da2065fd..df521b82 100644
--- a/server/tests/test_mobile_runtime.py
+++ b/server/tests/test_mobile_runtime.py
@@ -209,7 +209,7 @@ async def test_queued_cancel_is_scoped_to_run_and_never_starts_worker(runtime, m
await asyncio.sleep(0)
assert len(runtime.queue) == 2
await runtime.cancel("workflow", "agent", run_id="run")
- assert [entry["status"] for entry in runtime.queue] == ["cancelled", "queued"]
+ assert [(entry["run_id"], entry["status"]) for entry in runtime.queue] == [("other-run", "queued")]
await runtime.cancel("workflow", "agent", run_id="other-run")
runtime.serial = None # Cancellation must win over offline validation.
runtime.task_lock.release()
@@ -348,7 +348,93 @@ async def test_activity_is_bounded_and_phase_clock_is_stable(runtime, monkeypatc
for _ in range(50):
await runtime._publish_progress("Waiting for model")
assert runtime.active["phase_started_at"] == started
- assert len(runtime.active["activity"]) == 40
+ assert len(runtime.active["activity"]) == 1
assert runtime.active["updated_at"] >= started
+ for index in range(50):
+ await runtime._publish_progress(f"Operation {index}")
+ assert len(runtime.active["activity"]) == 40
await runtime._publish_progress("Tapping screen")
assert runtime.snapshot()["active"]["activity"][-1]["message"] == "Tapping screen"
+
+
+async def test_worker_semantic_progress_survives_device_noise_and_model_wait(runtime, monkeypatch):
+ from types import SimpleNamespace
+ from unittest.mock import AsyncMock, Mock
+ import nodes.mobile._runtime as module
+ import services.status_broadcaster as status
+
+ broadcast = AsyncMock()
+ monkeypatch.setattr(status, "get_status_broadcaster", lambda: SimpleNamespace(update_node_status=broadcast))
+ log = Mock()
+ monkeypatch.setattr(module, "event", log)
+ proc = FakeProcess()
+ messages = [
+ {"type": "progress", "steps": 1},
+ {"type": "agent_update", "kind": "plan", "message": "Working on: Find private test app",
+ "plan": [{"id": "g1", "description": "Find private test app", "status": "pending", "reason": "Search is visible"}]},
+ {"type": "agent_update", "kind": "action", "action_id": "tap1", "state": "started", "message": "Tap · Search", "detail": "Find private test app"},
+ *[{"type": "activity", "operation": "observe", "state": "completed", "duration_ms": i} for i in range(50)],
+ {"type": "progress", "steps": 2},
+ {"type": "agent_update", "kind": "action", "action_id": "tap1", "state": "failed", "message": "Tap · Search",
+ "detail": "Find private test app", "duration_ms": 1050, "outcome": "Element not found"},
+ {"type": "diagnostic", "stage": "model_request", "role": "cortex"},
+ {"type": "progress", "steps": 3},
+ {"type": "completed", "result": "done"},
+ ]
+ for message in messages:
+ proc.stdout.feed_data((json.dumps(message) + "\n").encode())
+ proc.returncode = 0
+ proc.stdout.feed_eof()
+ proc.terminated.set()
+
+ async def spawn(*_args, **_kwargs):
+ return proc
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)
+ await runtime.run(**run_args())
+ task = runtime.last_task
+ assert task["current_goal"] == "Find private test app"
+ assert task["plan"][0]["reason"] == "Search is visible"
+ assert len(task["agent_activity"]) == 2 # start/end share one action row
+ action = task["current_action"]
+ assert action["step"] == 1
+ assert action["state"] == "failed"
+ assert action["outcome"] == "Element not found"
+ assert action["duration_ms"] == 1050
+ assert task["phase"] == "Waiting for model: choosing an action"
+ assert task["steps"] == 3
+ assert all(item["message"] != "Using phone" for item in task["activity"])
+ assert "Find private test app" not in str(log.call_args_list)
+ assert "Find private test app" not in str(broadcast.call_args_list)
+
+
+async def test_semantic_history_is_bounded_and_interruption_is_not_success(runtime, monkeypatch):
+ from types import SimpleNamespace
+ from unittest.mock import AsyncMock
+ import services.status_broadcaster as status
+ monkeypatch.setattr(status, "get_status_broadcaster", lambda: SimpleNamespace(update_node_status=AsyncMock()))
+ runtime.active = {"node_id": "phone", "workflow_id": "wf", "run_id": "run", "status": "running"}
+ for index in range(45):
+ await runtime._publish_agent_progress({"kind": "action", "action_id": str(index), "state": "started",
+ "message": "Tap · Search", "detail": "a" * 1000})
+ assert len(runtime.active["agent_activity"]) == 40
+ assert len(runtime.active["current_action"]["detail"]) == 500
+ await runtime._stop_worker()
+ assert runtime.active["current_action"]["state"] == "returned"
+ assert "interrupted" in runtime.active["current_action"]["outcome"]
+ assert all(item["state"] == "returned" for item in runtime.active["agent_activity"])
+
+
+async def test_plan_completion_clears_current_goal_and_invalid_updates_are_ignored(runtime, monkeypatch):
+ from types import SimpleNamespace
+ from unittest.mock import AsyncMock
+ import services.status_broadcaster as status
+ monkeypatch.setattr(status, "get_status_broadcaster", lambda: SimpleNamespace(update_node_status=AsyncMock()))
+ runtime.active = {"node_id": "phone", "workflow_id": "wf", "run_id": "run", "status": "running", "current_goal": "Open app"}
+ await runtime._publish_agent_progress({"kind": "plan", "message": "Plan updated", "plan": [
+ {"id": "g", "description": "Open app", "status": "success", "reason": "Home screen visible"}]})
+ assert runtime.active["current_goal"] is None
+ assert runtime.active["plan"][0]["reason"] == "Home screen visible"
+ await runtime._publish_agent_progress({"kind": "arbitrary", "message": "private"})
+ await runtime._publish_agent_progress({"kind": "action", "action_id": "a", "message": "private", "state": "success"})
+ assert len(runtime.active["agent_activity"]) == 1
diff --git a/server/tests/test_workspace_invocations.py b/server/tests/test_workspace_invocations.py
new file mode 100644
index 00000000..aa6da919
--- /dev/null
+++ b/server/tests/test_workspace_invocations.py
@@ -0,0 +1,163 @@
+"""Workspace admission and cancellation boundaries without device access."""
+
+import asyncio
+from datetime import timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from services import node_invocations as service
+from services.plugin import NodeUserError
+
+SUBMISSION = "63cb635f-c758-42bc-a9b6-673655a72fa0"
+
+
+def admission(monkeypatch):
+ from core import config
+ from services import node_registry
+ from services.authz import workflow_node
+ from services.plugin import deps
+ from services.deployment import handlers
+ from services import status_broadcaster
+
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=None),
+ get_node_parameters=AsyncMock(return_value={"max_steps": 40}))
+ plugin = SimpleNamespace(workspace_task=True, type="phone", version=1, task_queue="device",
+ start_to_close_timeout=timedelta(hours=3))
+ monkeypatch.setattr(config, "Settings", lambda: SimpleNamespace(temporal_worker_pool_enabled=False,
+ temporal_task_queue="main"))
+ monkeypatch.setattr(node_registry, "get_node_class", lambda _: plugin)
+ monkeypatch.setattr(deps, "get_database", lambda: database)
+ graph = {"nodes": [{"id": "phone", "type": "phone"}], "edges": []}
+ monkeypatch.setattr(workflow_node, "resolve_workflow_node", AsyncMock(return_value=(SimpleNamespace(name="saved"), graph, graph["nodes"][0])))
+ status = AsyncMock(return_value={"epoch": 3, "resetting": False})
+ update = AsyncMock(return_value={"status": "accepted"})
+ monkeypatch.setattr(service, "controller_status", status)
+ monkeypatch.setattr(service, "_controller_update", update)
+ client = SimpleNamespace(get_workflow_handle=Mock())
+ monkeypatch.setattr(service, "temporal_client", lambda: client)
+ monkeypatch.setattr(handlers, "_with_runtime_counts", AsyncMock(return_value={}))
+ broadcast = AsyncMock()
+ monkeypatch.setattr(status_broadcaster, "get_status_broadcaster", lambda: SimpleNamespace(broadcast=broadcast))
+ return SimpleNamespace(database=database, status=status, update=update, broadcast=broadcast, client=client)
+
+
+async def test_submit_uses_saved_context_and_admission_epoch(admission):
+ result = await service.submit("owner", "wf", "phone", "Read the screen", SUBMISSION)
+ payload = admission.update.await_args.args[2]
+ assert payload["admission_epoch"] == 3
+ assert payload["context"]["generation"] == 0
+ assert payload["context"]["user_id"] == "owner"
+ assert payload["context"]["node_data"]["prompt"] == "Read the screen"
+ assert payload["task_queue"] == "main"
+ assert result["run_id"] == service.invocation_id("owner", "wf", "phone", SUBMISSION)
+
+
[email protected]("source", ["controller", "generation"])
+async def test_reset_blocks_admission_from_either_durable_source(admission, source):
+ if source == "controller":
+ admission.status.return_value["resetting"] = True
+ else:
+ admission.database.get_latest_workflow_control.return_value = SimpleNamespace(status="resetting")
+ with pytest.raises(NodeUserError, match="resetting"):
+ await service.submit("owner", "wf", "phone", "Read", SUBMISSION)
+ admission.update.assert_not_awaited()
+
+
+async def test_retry_uses_same_child_but_new_update_identity(admission):
+ for _ in range(2):
+ await service.submit("owner", "wf", "phone", "Read", SUBMISSION)
+ first, second = admission.update.await_args_list
+ assert first.args[2]["context"]["execution_id"] == second.args[2]["context"]["execution_id"]
+ assert first.kwargs["update_id"] != second.kwargs["update_id"]
+
+
+async def test_notification_failure_does_not_undo_committed_admission(admission):
+ admission.broadcast.side_effect = RuntimeError("socket unavailable")
+ assert (await service.submit("owner", "wf", "phone", "Read", SUBMISSION))["status"] == "accepted"
+
+
+async def test_reset_retry_resumes_latched_request_with_fresh_rpc_id(admission):
+ admission.status.return_value = {"reset_request_id": "original-reset", "resetting": True}
+ await service.reset("wf", "new-reset")
+ await service.reset("wf", "retry-reset")
+ first, second = admission.update.await_args_list
+ assert first.args[2] == second.args[2] == "original-reset"
+ assert first.kwargs["update_id"] != second.kwargs["update_id"]
+
+
+async def test_reset_during_saved_graph_resolution_preserves_old_epoch(admission, monkeypatch):
+ from services.authz import workflow_node
+
+ resolver = workflow_node.resolve_workflow_node
+ async def resolve(*args):
+ saved = await resolver(*args)
+ # Simulate Reset completing while the database resolves the graph.
+ admission.status.return_value = {"epoch": 4, "resetting": False}
+ return saved
+ monkeypatch.setattr(workflow_node, "resolve_workflow_node", resolve)
+ await service.submit("owner", "wf", "phone", "Read", SUBMISSION)
+ assert admission.update.await_args.args[2]["admission_epoch"] == 3
+
+
+async def test_cancel_waits_for_workflow_close():
+ gate = asyncio.Event()
+ handle = SimpleNamespace(cancel=AsyncMock(), result=AsyncMock(side_effect=gate.wait))
+ pending = asyncio.create_task(service._cancel_and_wait(handle))
+ await asyncio.sleep(0)
+ handle.cancel.assert_awaited_once()
+ assert not pending.done()
+ gate.set()
+ await pending
+
+
+async def test_confirmed_cancellation_does_not_require_another_rpc():
+ from temporalio.client import WorkflowFailureError
+ from temporalio.exceptions import CancelledError
+
+ handle = SimpleNamespace(cancel=AsyncMock(),
+ result=AsyncMock(side_effect=WorkflowFailureError(cause=CancelledError())),
+ describe=AsyncMock(side_effect=RuntimeError("connection dropped")))
+ await service._cancel_and_wait(handle)
+ handle.describe.assert_not_awaited()
+
+
+async def test_frozen_phone_node_is_not_hidden_by_editable_replacement(monkeypatch):
+ from services import node_registry
+
+ monkeypatch.setattr(node_registry, "get_node_class", lambda kind: SimpleNamespace(workspace_task=kind == "phone"))
+ database = SimpleNamespace(get_workflow=AsyncMock(return_value={"nodes": [{"id": "same", "type": "ordinary"}]}))
+ control = SimpleNamespace(graph_snapshot={"nodes": [{"id": "same", "type": "phone"}]})
+ _, nodes = await service.workspace_task_nodes(database, "wf", control)
+ assert nodes == [{"id": "same", "type": "phone"}]
+
+
+async def test_legacy_reset_cancels_only_matching_roots(monkeypatch):
+ matching = SimpleNamespace(query=AsyncMock(return_value={"workflow_id": "wf"}))
+ other = SimpleNamespace(query=AsyncMock(return_value={"workflow_id": "other"}))
+ async def executions(**_):
+ for key, parent in [("mine", None), ("other", None), ("child", "controller")]:
+ yield SimpleNamespace(id=key, run_id=key + "-run", parent_id=parent)
+ client = SimpleNamespace(list_workflows=executions, get_workflow_handle=Mock(side_effect=lambda key, **_: {"mine": matching, "other": other}[key]))
+ monkeypatch.setattr(service, "temporal_client", lambda: client)
+ cancel = AsyncMock()
+ monkeypatch.setattr(service, "_cancel_and_wait", cancel)
+ assert await service.cancel_legacy_invocations("wf") == 1
+ cancel.assert_awaited_once_with(matching)
+
+
[email protected]("workspace", [True, False])
+async def test_node_executor_preserves_workspace_cancellation(monkeypatch, workspace):
+ from services import node_executor as module
+
+ executor = module.NodeExecutor.__new__(module.NodeExecutor)
+ executor._prepare_parameters = AsyncMock(return_value={})
+ executor._dispatch = AsyncMock(side_effect=asyncio.CancelledError())
+ monkeypatch.setattr(module, "get_node_class", lambda _: SimpleNamespace(workspace_task=workspace))
+ if workspace:
+ with pytest.raises(asyncio.CancelledError):
+ await executor.execute("phone", "device", {}, {})
+ else:
+ assert (await executor.execute("phone", "ordinary", {}, {}))["error"] == "Cancelled"
diff --git a/server/tests/test_ws_diagnostics.py b/server/tests/test_ws_diagnostics.py
new file mode 100644
index 00000000..bd9b5e21
--- /dev/null
+++ b/server/tests/test_ws_diagnostics.py
@@ -0,0 +1,190 @@
+"""Failed upgrades and dropped connections leave useful, bounded diagnostics."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from starlette.websockets import WebSocketDisconnect
+
+
+class _Socket:
+ def __init__(self, *, path="/ws/status", headers=None, cookies=None):
+ self.scope = {"path": path, "query_string": b"token=query-secret"}
+ self.headers = headers or {"host": "localhost:5678"}
+ self.cookies = cookies or {}
+ self.close = AsyncMock()
+ self.state = SimpleNamespace()
+
+
+ ("origin", "token", "payload", "category", "close_code"),
+ [
+ ("http://foreign.example/?token=origin-secret", "cookie-secret", {"sub": "owner"}, "origin_not_allowed", 4003),
+ (None, None, None, "missing_session", 4001),
+ (None, "cookie-secret", None, "invalid_session", 4001),
+ (None, "cookie-secret", {"sub": ""}, "invalid_session_subject", 4001),
+ ],
+)
+async def test_browser_rejection_explains_http_denial_without_request_secrets(
+ monkeypatch, origin, token, payload, category, close_code
+):
+ import services.authz.ws_session as session
+
+ logger = Mock()
+ monkeypatch.setattr(session, "logger", logger)
+ headers = {"host": "localhost:5678", "authorization": "Bearer authorization-secret"}
+ if origin:
+ headers["origin"] = origin
+ socket = _Socket(
+ path="/ws/status?token=path-query-secret",
+ headers=headers,
+ cookies={"opencompany_token": token} if token else {},
+ )
+ settings = SimpleNamespace(cors_origins=[], vite_auth_enabled="true", jwt_cookie_name="opencompany_token")
+ auth = Mock()
+ auth.verify_token.return_value = payload
+
+ assert await session.authenticate_ws(socket, settings=settings, user_auth_service=lambda: auth) is None
+
+ logger.warning.assert_called_once_with(
+ "[WebSocket] Handshake rejected",
+ path="/ws/status",
+ reason=category,
+ close_code=close_code,
+ http_status=403,
+ )
+ assert socket.close.await_args.kwargs["code"] == close_code
+ assert "secret" not in repr(logger.mock_calls)
+
+
[email protected]("origin", [None, "http://foreign.example/?token=origin-secret"])
+async def test_internal_rejection_uses_the_same_safe_diagnostics(monkeypatch, origin):
+ import services.authz.ws_session as session
+
+ logger = Mock()
+ monkeypatch.setattr(session, "logger", logger)
+ headers = {"host": "localhost:5678", "x-opencompany-internal-token": "worker-secret"}
+ if origin:
+ headers["origin"] = origin
+ socket = _Socket(path="/ws/internal", headers=headers)
+
+ assert not await session.admit_internal_ws(socket, settings=SimpleNamespace(cors_origins=[], secret_key="server-secret"))
+
+ logger.warning.assert_called_once_with(
+ "[WebSocket] Handshake rejected",
+ path="/ws/internal",
+ reason="origin_not_allowed" if origin else "invalid_worker_token",
+ close_code=4003 if origin else 4001,
+ http_status=403,
+ )
+ assert "secret" not in repr(logger.mock_calls)
+
+
+async def test_rejection_prefers_route_template_and_bounds_fallback_path(monkeypatch):
+ import services.authz.ws_session as session
+
+ logger = Mock()
+ monkeypatch.setattr(session, "logger", logger)
+ settings = SimpleNamespace(cors_origins=[], vite_auth_enabled="true", jwt_cookie_name="opencompany_token")
+ socket = _Socket(path="/ws/browser/private-path-secret")
+ socket.scope["route"] = SimpleNamespace(path="/ws/browser/{session_id}")
+
+ await session.authenticate_ws(socket, settings=settings, user_auth_service=lambda: None)
+
+ assert logger.warning.call_args.kwargs["path"] == "/ws/browser/{session_id}"
+ assert "secret" not in repr(logger.mock_calls)
+ socket.scope = {"path": "/ws/\r\n" + "a" * 1000 + "?token=query-secret"}
+
+ await session.authenticate_ws(socket, settings=settings, user_auth_service=lambda: None)
+
+ path = logger.warning.call_args.kwargs["path"]
+ assert len(path) == 160
+ assert path.isprintable()
+
+
[email protected]("auth_enabled", ["true", "false"])
+async def test_accepted_browser_handshake_does_not_emit_a_rejection(monkeypatch, auth_enabled):
+ import services.authz.ws_session as session
+
+ logger = Mock()
+ monkeypatch.setattr(session, "logger", logger)
+ socket = _Socket(cookies={"opencompany_token": "cookie-secret"})
+ settings = SimpleNamespace(cors_origins=[], vite_auth_enabled=auth_enabled, jwt_cookie_name="opencompany_token")
+ auth = Mock()
+ auth.verify_token.return_value = {"sub": "owner"}
+
+ assert await session.authenticate_ws(socket, settings=settings, user_auth_service=lambda: auth) == "owner"
+
+ logger.warning.assert_not_called()
+ socket.close.assert_not_awaited()
+
+
+ ("code", "reason", "level"),
+ [(1000, "tab closed", "debug"), (1001, "server restarting", "debug"), (1006, "", "warning"), (1011, "keepalive timeout", "warning")],
+)
+async def test_status_disconnect_reports_close_details_and_cleans_up(monkeypatch, code, reason, level):
+ import routers.websocket as router
+
+ socket = _Socket()
+ socket.receive_json = AsyncMock(side_effect=WebSocketDisconnect(code=code, reason=reason))
+ logger = Mock()
+ broadcaster = SimpleNamespace(connect=AsyncMock(), disconnect=AsyncMock())
+ monkeypatch.setattr(router, "logger", logger)
+ monkeypatch.setattr(router, "authenticate_ws", AsyncMock(return_value="owner"))
+ monkeypatch.setattr(router, "get_status_broadcaster", lambda: broadcaster)
+
+ await router.websocket_status_endpoint(socket)
+
+ getattr(logger, level).assert_called_once_with(
+ "[WebSocket] Client disconnected", path="/ws/status", close_code=code, reason=reason
+ )
+ broadcaster.disconnect.assert_awaited_once_with(socket)
+ assert socket not in router._handler_tasks
+
+
+async def test_status_disconnect_bounds_and_control_cleans_peer_reason(monkeypatch):
+ import routers.websocket as router
+
+ socket = _Socket()
+ socket.receive_json = AsyncMock(side_effect=WebSocketDisconnect(code=1011, reason="line\n" + "x" * 1000))
+ logger = Mock()
+ monkeypatch.setattr(router, "logger", logger)
+ monkeypatch.setattr(router, "authenticate_ws", AsyncMock(return_value="owner"))
+ monkeypatch.setattr(router, "get_status_broadcaster", lambda: SimpleNamespace(connect=AsyncMock(), disconnect=AsyncMock()))
+
+ await router.websocket_status_endpoint(socket)
+
+ reason = logger.warning.call_args.kwargs["reason"]
+ assert reason.startswith("line?")
+ assert len(reason) == 123
+
+
[email protected]("cancel_during", ["accept", "initial_snapshot"])
+async def test_cancelled_connection_setup_does_not_leave_a_registered_socket(monkeypatch, cancel_during):
+ import routers.websocket as router
+ from services.status_broadcaster import StatusBroadcaster
+
+ broadcaster = StatusBroadcaster()
+ socket = _Socket()
+ setup_blocked = asyncio.Event()
+
+ async def block_setup(*_args):
+ setup_blocked.set()
+ await asyncio.Event().wait()
+
+ socket.accept = AsyncMock(side_effect=block_setup if cancel_during == "accept" else None)
+ socket.send_json = AsyncMock(side_effect=block_setup)
+ monkeypatch.setattr(router, "authenticate_ws", AsyncMock(return_value="owner"))
+ monkeypatch.setattr(router, "get_status_broadcaster", lambda: broadcaster)
+ task = asyncio.create_task(router.websocket_status_endpoint(socket))
+ await asyncio.wait_for(setup_blocked.wait(), timeout=1)
+ assert (socket in broadcaster._connections) is (cancel_during == "initial_snapshot")
+
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ assert socket not in broadcaster._connections
+ assert socket not in router._handler_tasks