Skip to content

Self-improving AI Employees for everyone

Show timed Android model and device activity with retained failure history

trohitga0db9754d10b
diff --git a/client/src/components/mobile/MobileWorkspace.tsx b/client/src/components/mobile/MobileWorkspace.tsx
index 8719f5b7..8a05b416 100644
--- a/client/src/components/mobile/MobileWorkspace.tsx
+++ b/client/src/components/mobile/MobileWorkspace.tsx
@@ -1,3 +1,4 @@
+import { PhoneActivity } from './PhoneActivity';
import { FullView } from '../workspace/FullView';
import { useCallback, useEffect, useRef, useState } from 'react';
import type { PointerEvent } from 'react';
@@ -259,6 +260,7 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no
<Button size="sm" variant="outline" disabled={!text || !!busy} onClick={() => void perform('input', async () => { await input('text', { text }); setText(''); })}>Type</Button>
</div>}
</>}
+ <PhoneActivity task={status?.active?.run_id ? status.active : status?.last_task} />
<details aria-label="Phone AI task" className="max-h-[45%] shrink-0 overflow-y-auto border-t border-border-default group-data-[full-view=true]/fullview:hidden">
<summary className="cursor-pointer px-2 py-2 text-xs font-medium outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring">Ask AI to use the phone
{task && <span role="status" className="ml-2 font-normal text-fg-muted">{({queued: 'Waiting for the phone', running: 'Working on your request', completed: 'Done', failed: 'Could not finish', cancelled: 'Cancelled', awaiting_user: 'Waiting for you'})[taskState(task)] || 'Request received'}</span>}
diff --git a/client/src/components/mobile/PhoneActivity.tsx b/client/src/components/mobile/PhoneActivity.tsx
new file mode 100644
index 00000000..e88467b3
--- /dev/null
+++ b/client/src/components/mobile/PhoneActivity.tsx
@@ -0,0 +1,29 @@
+import { useEffect, useState } from 'react';
+
+const seconds = (from: unknown, to: number) => typeof from === 'number' ? Math.max(0, Math.floor(to - from)) : 0;
+
+export function PhoneActivity({ task }: { task?: Record<string, unknown> | null }) {
+ const [now, setNow] = useState(Date.now() / 1000);
+ const ended = typeof task?.finished_at === 'number';
+ useEffect(() => {
+ if (!task?.run_id || ended) return;
+ const timer = setInterval(() => setNow(Date.now() / 1000), 1000);
+ return () => clearInterval(timer);
+ }, [task?.run_id, ended]);
+ if (!task?.run_id) return null;
+ const end = ended ? Number(task.finished_at) : now;
+ const quiet = seconds(task.updated_at, end);
+ const events = Array.isArray(task.activity) ? task.activity as { at: number; message: string; step: number }[] : [];
+ return <details open className="max-h-44 shrink-0 overflow-y-auto border-t border-border-default text-xs group-data-[full-view=true]/fullview:hidden" aria-label="Android activity">
+ <summary className="cursor-pointer p-2 font-medium">AI activity · {ended ? String(task.status) : String(task.phase || 'Preparing phone')} · {seconds(task.started_at, end)}s total</summary>
+ <div className="space-y-1 px-2 pb-2">
+ <p className="m-0 text-fg-muted">{String(task.provider || '')} {String(task.model || '')} · Step {Number(task.steps || 0)} / {Number(task.max_steps || 0)}</p>
+ {!ended && <p className="m-0" role="status">Current phase: {seconds(task.phase_started_at, now)}s · Last activity: {quiet}s ago</p>}
+ {!ended && quiet >= 30 && <p className="m-0 text-amber-600">No new activity for {quiet}s. The last reported phase is shown above. This may be a slow request; it does not confirm the task is stuck. Use “Use phone” to take control.</p>}
+ {ended && task.error_code != null && <p className="m-0 text-destructive">Stopped during {String(task.phase || 'startup')} · {String(task.error_code)}</p>}
+ <ol className="m-0 list-none space-y-1 p-0" aria-label="Recent Android steps">
+ {[...events].reverse().map((item, index) => <li key={`${item.at}-${index}`}><span className="text-fg-muted">+{seconds(task.started_at, item.at)}s · </span>{item.message}</li>)}
+ </ol>
+ </div>
+ </details>;
+}
diff --git a/client/src/components/mobile/__tests__/PhoneActivity.test.tsx b/client/src/components/mobile/__tests__/PhoneActivity.test.tsx
new file mode 100644
index 00000000..2ea51049
--- /dev/null
+++ b/client/src/components/mobile/__tests__/PhoneActivity.test.tsx
@@ -0,0 +1,22 @@
+import { render, screen } from '@testing-library/react';
+import { describe, expect, it } from 'vitest';
+import { PhoneActivity } from '../PhoneActivity';
+
+describe('Android activity', () => {
+ it('shows a quiet model wait with timing and actual operations', () => {
+ const now = Date.now() / 1000;
+ render(<PhoneActivity task={{ run_id: 'run', phase: 'Waiting for model', started_at: now - 65,
+ phase_started_at: now - 40, updated_at: now - 40, model: 'test-model', provider: 'openai',
+ activity: [{ at: now - 42, message: 'Reading screen and accessibility tree: completed (0.2s)', step: 1 }] }} />);
+ expect(screen.getByText(/No new activity for 40s/)).toBeInTheDocument();
+ expect(screen.getByText(/Reading screen and accessibility tree/)).toBeInTheDocument();
+ expect(screen.getByText(/openai test-model/)).toBeInTheDocument();
+ });
+ it('retains the failure location after the task ends', () => {
+ render(<PhoneActivity task={{ run_id: 'run', status: 'failed', phase: 'Waiting for model',
+ started_at: 100, finished_at: 180, error_code: 'timeout' }} />);
+ expect(screen.getByText(/Stopped during Waiting for model · timeout/)).toBeInTheDocument();
+ expect(screen.getByText(/80s total/)).toBeInTheDocument();
+ expect(screen.queryByText(/No new activity/)).toBeNull();
+ });
+});
diff --git a/client/src/components/mobile/api.ts b/client/src/components/mobile/api.ts
index 9e0890ee..706449a6 100644
--- a/client/src/components/mobile/api.ts
+++ b/client/src/components/mobile/api.ts
@@ -13,6 +13,7 @@ export interface MobileStatus {
controller?: string | null;
epoch: number;
control_state: string;
+ last_task?: Record<string, unknown> | null;
active?: Record<string, unknown> | null;
queue?: unknown[];
}
diff --git a/docs-internal/node-logic-flows/mobile/android_tool.md b/docs-internal/node-logic-flows/mobile/android_tool.md
index f57f8964..0de4449a 100644
--- a/docs-internal/node-logic-flows/mobile/android_tool.md
+++ b/docs-internal/node-logic-flows/mobile/android_tool.md
@@ -59,3 +59,17 @@ Windows x64 managed Android only; one shared phone and one video viewer. No iOS,
- [Mobile Agent](mobile_use_agent.md)
- [Mobile Workspace setup, controls, logging and limitations](../../../docs/mobile-workspace.md)
+
+
+### Live Android activity
+
+Workspace shows the active phone task's model/provider, step budget, total elapsed time,
+current phase duration, and time since the last reported activity. The expandable activity
+panel retains the latest 40 events: model requests/responses and actual device operations
+(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).
+Completed operations include duration. The last task remains visible after completion,
+failure, or cancellation until another task runs; history is in memory and clears on restart.
+A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last
+phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.
+The panel is outside the phone canvas and hidden in full-screen mode. Activity contains
+operational summaries, never private model reasoning, typed text, URLs, or prompt content.
diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.md
index 480d6b6e..4b9aa3a8 100644
--- a/docs/mobile-workspace.md
+++ b/docs/mobile-workspace.md
@@ -186,3 +186,17 @@ This is an implementation plan, not currently supported functionality. Keep the
| iOS | Not implemented | Same-Mac lifecycle, IDB adapter, simulator app install, preview, gestures, fenced takeover |
Passing a source or fake-process test does not count as passing the corresponding device acceptance check.
+
+
+### Live Android activity
+
+Workspace shows the active phone task's model/provider, step budget, total elapsed time,
+current phase duration, and time since the last reported activity. The expandable activity
+panel retains the latest 40 events: model requests/responses and actual device operations
+(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).
+Completed operations include duration. The last task remains visible after completion,
+failure, or cancellation until another task runs; history is in memory and clears on restart.
+A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last
+phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.
+The panel is outside the phone canvas and hidden in full-screen mode. Activity contains
+operational summaries, never private model reasoning, typed text, URLs, or prompt content.
diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.py
index a97a254b..b08c57ec 100644
--- a/server/nodes/mobile/_runtime.py
+++ b/server/nodes/mobile/_runtime.py
@@ -40,6 +40,7 @@ class MobileRuntime:
self._setup_loaded = False
self._setup_saved = 0.0
self.active: dict | None = None
+ self.last_task: dict | None = None
self.queue: list[dict] = []
self.capabilities: dict[str, tuple[str, Lease]] = {}
self.resume_event = asyncio.Event()
@@ -119,6 +120,7 @@ class MobileRuntime:
"setup_progress": self.setup_progress,
"diagnostics": recent(),
"active": {k: v for k, v in (self.active or {}).items() if k not in {"prompt"}},
+ "last_task": self.last_task,
"queue": [{k: v for k, v in q.items() if k != "prompt"} for q in self.queue],
**self.control.snapshot(),
}
@@ -502,6 +504,8 @@ class MobileRuntime:
if not self.serial:
raise MobileError("device_offline", "Start the shared device in Workspace first")
self.active = entry
+ entry.update(started_at=time.time(), provider=model.get("provider"), model=model.get("model"))
+ await self._publish_progress("Preparing phone")
active_remaining = params["timeout_s"]
steps_remaining = params["max_steps"]
attempts = 0
@@ -512,6 +516,7 @@ class MobileRuntime:
if self.control.owner and self.control.owner.startswith("viewer:"):
entry["status"] = "awaiting_user"
if entry["status"] == "awaiting_user":
+ await self._publish_progress("Waiting for you")
await asyncio.wait_for(self.resume_event.wait(), 1800)
self.resume_event.clear()
if entry["status"] == "cancelled":
@@ -573,6 +578,7 @@ class MobileRuntime:
continue
if not result.get("success"):
raise MobileError(result.get("code", "agent_failed"), result.get("error", "Mobile agent failed"))
+ entry["status"] = "completed"
event("task_completed", run_id=run_id)
return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}
finally:
@@ -582,11 +588,16 @@ class MobileRuntime:
# Wait for any previously admitted broker write before reporting completion.
async with self.control.lock:
pass
+ entry["finished_at"] = time.time()
+ self.last_task = entry
self.active = None
except asyncio.CancelledError:
+ entry["status"] = "cancelled"
event("task_cancelled", run_id=run_id)
raise
except Exception as exc:
+ entry["status"] = "failed"
+ entry["error_code"] = getattr(exc, "code", "timeout" if isinstance(exc, asyncio.TimeoutError) else "task_failed")
event("task_failed", failed=True, run_id=run_id, error_type=type(exc).__name__, code=getattr(exc, "code", "task_failed"))
raise
finally:
@@ -597,7 +608,14 @@ class MobileRuntime:
if not self.active:
return
entry = self.active
+ now = time.time()
+ if entry.get("phase") != phase:
+ entry["phase_started_at"] = now
entry["phase"] = phase
+ entry["updated_at"] = now
+ history = entry.setdefault("activity", [])
+ history.append({"at": now, "message": phase, "step": entry.get("steps", 0)})
+ del history[:-40]
try:
from services.status_broadcaster import get_status_broadcaster
await get_status_broadcaster().update_node_status(
@@ -639,6 +657,8 @@ class MobileRuntime:
event("engine_starting", **scope)
await self._publish_progress("Starting Android task")
+ pending_models: dict[str, float] = {}
+
async def read():
result = {"success": False, "error": "Mobile worker exited before completing the task"}
while line := await proc.stdout.readline():
@@ -649,11 +669,41 @@ class MobileRuntime:
result = {"success": False, "error": message.get("error", "Worker failed"), "code": message.get("code", "engine_failed")}
event("engine_failed", failed=True, **scope, engine_trace=message.get("trace", []),
**{key: message.get(key) for key in ("error_type", "code", "stage", "http_status", "provider_status")})
+ elif message.get("type") == "activity":
+ labels = {"observe": "Reading screen and accessibility tree", "geometry": "Checking screen size",
+ "tap": "Tapping screen", "swipe": "Scrolling", "text": "Typing text",
+ "key": "Pressing navigation key", "launch": "Opening app", "terminate": "Closing app",
+ "url": "Opening link", "erase": "Erasing text", "packages": "Listing apps",
+ "foreground": "Checking current app", "date": "Checking device time"}
+ label = labels.get(message.get("operation"))
+ state = message.get("state")
+ if label and state in {"started", "completed", "failed"}:
+ duration = message.get("duration_ms")
+ suffix = f" ({duration / 1000:.1f}s)" if isinstance(duration, (int, float)) and duration >= 0 else ""
+ await self._publish_progress(label if state == "started" else f"{label}: {state}{suffix}")
elif message.get("type") == "diagnostic":
event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))
phase = {"model_request": "Waiting for model", "model_response": "Model responded",
- "engine_initialization": "Connecting engine", "task_execution": "Using phone"}.get(message.get("stage"))
- if phase and self.active and self.active.get("phase") != phase:
+ "engine_initialization": "Connecting engine", "task_execution": "Using phone",
+ "engine_cleanup": "Finishing engine cleanup"}.get(message.get("stage"))
+ stage = message.get("stage")
+ request_id = str(message.get("request_id", "model"))
+ if stage == "model_request":
+ pending_models[request_id] = time.monotonic()
+ role = {"planner": "planning", "orchestrator": "coordinating next step",
+ "contextor": "understanding the screen", "cortex": "choosing an action",
+ "executor": "preparing an action", "outputter": "preparing the result",
+ "hopper": "selecting an app"}.get(message.get("role"))
+ if role:
+ phase = f"Waiting for model: {role}"
+ elif stage in {"model_response", "model_error"}:
+ started = pending_models.pop(request_id, None)
+ phase = "Model request failed" if stage == "model_error" else "Model responded"
+ if started is not None:
+ phase += f" ({time.monotonic() - started:.1f}s)"
+ if pending_models:
+ phase += f"; waiting for {len(pending_models)} model request(s)"
+ if phase and self.active:
await self._publish_progress(phase)
elif message.get("type") == "ready":
event("engine_ready", **scope)
diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.py
index 4383fcd7..cd6f0a9d 100644
--- a/server/nodes/mobile/runtime/worker.py
+++ b/server/nodes/mobile/runtime/worker.py
@@ -9,6 +9,8 @@ import re
import sys
import urllib.request
import uuid
+import time
+import threading
from io import BytesIO
try:
@@ -18,11 +20,13 @@ except ImportError: # Executed by path in the isolated engine environment.
STAGE = "bootstrap"
WIRE = sys.stdout
+WIRE_LOCK = threading.Lock()
def emit(kind: str, **data) -> None:
- WIRE.write(json.dumps({"v": 1, "type": kind, **data}, default=str) + "\n")
- WIRE.flush()
+ with WIRE_LOCK:
+ WIRE.write(json.dumps({"v": 1, "type": kind, **data}, default=str) + "\n")
+ WIRE.flush()
class Controller:
@@ -43,10 +47,17 @@ class Controller:
json.dumps(payload).encode(),
{"Content-Type": "application/json", "Authorization": f"Bearer {self.config['capability']}"},
)
- with urllib.request.urlopen(req, timeout=35) as response:
- result = json.load(response)
- if not result.get("success"):
- raise RuntimeError(result.get("error", "Device command failed"))
+ started = time.monotonic()
+ emit("activity", operation=operation, state="started")
+ try:
+ with urllib.request.urlopen(req, timeout=35) as response:
+ result = json.load(response)
+ if not result.get("success"):
+ raise RuntimeError(result.get("error", "Device command failed"))
+ except Exception:
+ emit("activity", operation=operation, state="failed", duration_ms=round((time.monotonic() - started) * 1000))
+ raise
+ emit("activity", operation=operation, state="completed", duration_ms=round((time.monotonic() - started) * 1000))
return result["result"]
async def read(self, operation: str, **params):
@@ -155,10 +166,16 @@ async def main(config: dict) -> None:
emit("progress", steps=max(0, step))
def on_chat_model_start(self, serialized, messages, **kwargs):
- emit("diagnostic", stage="model_request")
+ role = (kwargs.get("metadata") or {}).get("langgraph_node")
+ allowed = {"planner", "orchestrator", "contextor", "cortex", "executor", "outputter", "hopper"}
+ emit("diagnostic", stage="model_request", request_id=str(kwargs.get("run_id", "model")),
+ role=role if role in allowed else None)
+
+ def on_llm_error(self, error, **kwargs):
+ emit("diagnostic", stage="model_error", request_id=str(kwargs.get("run_id", "model")))
def on_llm_end(self, response, **kwargs):
- emit("diagnostic", stage="model_response")
+ emit("diagnostic", stage="model_response", request_id=str(kwargs.get("run_id", "model")))
output = response.llm_output or {}
usage = output.get("token_usage") or output.get("usage") or {}
if usage:
@@ -211,6 +228,8 @@ async def main(config: dict) -> None:
raise
finally:
try:
+ if not task_failed:
+ emit("diagnostic", stage="engine_cleanup")
await agent.clean()
except Exception as cleanup_error:
if not task_failed:
diff --git a/server/skills/android_agent/android-phone-skill/SKILL.md b/server/skills/android_agent/android-phone-skill/SKILL.md
index 9c03e55a..0c74c796 100644
--- a/server/skills/android_agent/android-phone-skill/SKILL.md
+++ b/server/skills/android_agent/android-phone-skill/SKILL.md
@@ -22,3 +22,17 @@ Call `android` with a single `prompt` describing one clear goal, the relevant ap
- Terminal source `mobile` shows the execution ID, provider, model backend, model wait, and device timing. Report the error and relevant phase without asking the owner to reveal API keys.
Example: `{"prompt":"Open Settings, find the Android version, and report it without changing settings."}`
+
+
+### Live Android activity
+
+Workspace shows the active phone task's model/provider, step budget, total elapsed time,
+current phase duration, and time since the last reported activity. The expandable activity
+panel retains the latest 40 events: model requests/responses and actual device operations
+(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).
+Completed operations include duration. The last task remains visible after completion,
+failure, or cancellation until another task runs; history is in memory and clears on restart.
+A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last
+phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.
+The panel is outside the phone canvas and hidden in full-screen mode. Activity contains
+operational summaries, never private model reasoning, typed text, URLs, or prompt content.
diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.py
index 52563b17..da2065fd 100644
--- a/server/tests/test_mobile_runtime.py
+++ b/server/tests/test_mobile_runtime.py
@@ -303,6 +303,9 @@ async def test_worker_failure_reaches_diagnostics_and_keeps_provider_code(runtim
from nodes.mobile._control import MobileError
proc = FakeProcess()
+ proc.stdout.feed_data(b'{"type":"activity","operation":"observe","state":"started"}\n')
+ proc.stdout.feed_data(b'{"type":"activity","operation":"observe","state":"completed","duration_ms":123}\n')
+ proc.stdout.feed_data(b'{"type":"activity","operation":"secret-url","state":"started"}\n')
proc.stdout.feed_data(b'{"type":"diagnostic","stage":"model_request"}\n')
proc.stdout.feed_data(b'{"type":"failed","error":"Model not found (HTTP 404)","code":"model_not_found","http_status":404,"error_type":"ClientError","stage":"task_execution"}\n')
proc.returncode = 1
@@ -324,3 +327,28 @@ async def test_worker_failure_reaches_diagnostics_and_keeps_provider_code(runtim
assert recorded[0]["execution_id"] == "execution-2"
assert recorded[0]["node_id"] == "agent"
assert recorded[0]["model"] == "selected-model"
+ assert runtime.active is None
+ assert runtime.last_task["status"] == "failed"
+ assert runtime.last_task["error_code"] == "model_not_found"
+ assert runtime.last_task["activity"][-1]["message"] == "Waiting for model"
+ assert runtime.last_task["finished_at"] >= runtime.last_task["started_at"]
+ messages = [item["message"] for item in runtime.last_task["activity"]]
+ assert "Reading screen and accessibility tree: completed (0.1s)" in messages
+ assert not any("secret-url" in item for item in messages)
+
+
+async def test_activity_is_bounded_and_phase_clock_is_stable(runtime, monkeypatch):
+ from unittest.mock import AsyncMock
+ from types import SimpleNamespace
+ import services.status_broadcaster as status
+ monkeypatch.setattr(status, "get_status_broadcaster", lambda: SimpleNamespace(update_node_status=AsyncMock()))
+ runtime.active = {"node_id": "phone", "workflow_id": "wf", "run_id": "run", "status": "running"}
+ await runtime._publish_progress("Waiting for model")
+ started = runtime.active["phase_started_at"]
+ for _ in range(50):
+ await runtime._publish_progress("Waiting for model")
+ assert runtime.active["phase_started_at"] == started
+ assert len(runtime.active["activity"]) == 40
+ assert runtime.active["updated_at"] >= started
+ await runtime._publish_progress("Tapping screen")
+ assert runtime.snapshot()["active"]["activity"][-1]["message"] == "Tapping screen"