Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
Show timed Android model and device activity with retained failure history
trohitga0db9754d10b
diff --git a/client/src/components/mobile/MobileWorkspace.tsx b/client/src/components/mobile/MobileWorkspace.tsxindex 8719f5b7..8a05b416 100644--- a/client/src/components/mobile/MobileWorkspace.tsx+++ b/client/src/components/mobile/MobileWorkspace.tsx@@ -1,3 +1,4 @@+import { PhoneActivity } from './PhoneActivity';import { FullView } from '../workspace/FullView';import { useCallback, useEffect, useRef, useState } from 'react';import type { PointerEvent } from 'react';@@ -259,6 +260,7 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; no<Button size="sm" variant="outline" disabled={!text || !!busy} onClick={() => void perform('input', async () => { await input('text', { text }); setText(''); })}>Type</Button></div>}</>}+ <PhoneActivity task={status?.active?.run_id ? status.active : status?.last_task} /><details aria-label="Phone AI task" className="max-h-[45%] shrink-0 overflow-y-auto border-t border-border-default group-data-[full-view=true]/fullview:hidden"><summary className="cursor-pointer px-2 py-2 text-xs font-medium outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring">Ask AI to use the phone{task && <span role="status" className="ml-2 font-normal text-fg-muted">{({queued: 'Waiting for the phone', running: 'Working on your request', completed: 'Done', failed: 'Could not finish', cancelled: 'Cancelled', awaiting_user: 'Waiting for you'})[taskState(task)] || 'Request received'}</span>}diff --git a/client/src/components/mobile/PhoneActivity.tsx b/client/src/components/mobile/PhoneActivity.tsxnew file mode 100644index 00000000..e88467b3--- /dev/null+++ b/client/src/components/mobile/PhoneActivity.tsx@@ -0,0 +1,29 @@+import { useEffect, useState } from 'react';++const seconds = (from: unknown, to: number) => typeof from === 'number' ? Math.max(0, Math.floor(to - from)) : 0;++export function PhoneActivity({ task }: { task?: Record<string, unknown> | null }) {+ const [now, setNow] = useState(Date.now() / 1000);+ const ended = typeof task?.finished_at === 'number';+ useEffect(() => {+ if (!task?.run_id || ended) return;+ const timer = setInterval(() => setNow(Date.now() / 1000), 1000);+ return () => clearInterval(timer);+ }, [task?.run_id, ended]);+ if (!task?.run_id) return null;+ const end = ended ? Number(task.finished_at) : now;+ const quiet = seconds(task.updated_at, end);+ const events = Array.isArray(task.activity) ? task.activity as { at: number; message: string; step: number }[] : [];+ return <details open className="max-h-44 shrink-0 overflow-y-auto border-t border-border-default text-xs group-data-[full-view=true]/fullview:hidden" aria-label="Android activity">+ <summary className="cursor-pointer p-2 font-medium">AI activity · {ended ? String(task.status) : String(task.phase || 'Preparing phone')} · {seconds(task.started_at, end)}s total</summary>+ <div className="space-y-1 px-2 pb-2">+ <p className="m-0 text-fg-muted">{String(task.provider || '')} {String(task.model || '')} · Step {Number(task.steps || 0)} / {Number(task.max_steps || 0)}</p>+ {!ended && <p className="m-0" role="status">Current phase: {seconds(task.phase_started_at, now)}s · Last activity: {quiet}s ago</p>}+ {!ended && quiet >= 30 && <p className="m-0 text-amber-600">No new activity for {quiet}s. The last reported phase is shown above. This may be a slow request; it does not confirm the task is stuck. Use “Use phone” to take control.</p>}+ {ended && task.error_code != null && <p className="m-0 text-destructive">Stopped during {String(task.phase || 'startup')} · {String(task.error_code)}</p>}+ <ol className="m-0 list-none space-y-1 p-0" aria-label="Recent Android steps">+ {[...events].reverse().map((item, index) => <li key={`${item.at}-${index}`}><span className="text-fg-muted">+{seconds(task.started_at, item.at)}s · </span>{item.message}</li>)}+ </ol>+ </div>+ </details>;+}diff --git a/client/src/components/mobile/__tests__/PhoneActivity.test.tsx b/client/src/components/mobile/__tests__/PhoneActivity.test.tsxnew file mode 100644index 00000000..2ea51049--- /dev/null+++ b/client/src/components/mobile/__tests__/PhoneActivity.test.tsx@@ -0,0 +1,22 @@+import { render, screen } from '@testing-library/react';+import { describe, expect, it } from 'vitest';+import { PhoneActivity } from '../PhoneActivity';++describe('Android activity', () => {+ it('shows a quiet model wait with timing and actual operations', () => {+ const now = Date.now() / 1000;+ render(<PhoneActivity task={{ run_id: 'run', phase: 'Waiting for model', started_at: now - 65,+ phase_started_at: now - 40, updated_at: now - 40, model: 'test-model', provider: 'openai',+ activity: [{ at: now - 42, message: 'Reading screen and accessibility tree: completed (0.2s)', step: 1 }] }} />);+ expect(screen.getByText(/No new activity for 40s/)).toBeInTheDocument();+ expect(screen.getByText(/Reading screen and accessibility tree/)).toBeInTheDocument();+ expect(screen.getByText(/openai test-model/)).toBeInTheDocument();+ });+ it('retains the failure location after the task ends', () => {+ render(<PhoneActivity task={{ run_id: 'run', status: 'failed', phase: 'Waiting for model',+ started_at: 100, finished_at: 180, error_code: 'timeout' }} />);+ expect(screen.getByText(/Stopped during Waiting for model · timeout/)).toBeInTheDocument();+ expect(screen.getByText(/80s total/)).toBeInTheDocument();+ expect(screen.queryByText(/No new activity/)).toBeNull();+ });+});diff --git a/client/src/components/mobile/api.ts b/client/src/components/mobile/api.tsindex 9e0890ee..706449a6 100644--- a/client/src/components/mobile/api.ts+++ b/client/src/components/mobile/api.ts@@ -13,6 +13,7 @@ export interface MobileStatus {controller?: string | null;epoch: number;control_state: string;+ last_task?: Record<string, unknown> | null;active?: Record<string, unknown> | null;queue?: unknown[];}diff --git a/docs-internal/node-logic-flows/mobile/android_tool.md b/docs-internal/node-logic-flows/mobile/android_tool.mdindex f57f8964..0de4449a 100644--- a/docs-internal/node-logic-flows/mobile/android_tool.md+++ b/docs-internal/node-logic-flows/mobile/android_tool.md@@ -59,3 +59,17 @@ Windows x64 managed Android only; one shared phone and one video viewer. No iOS,- [Mobile Agent](mobile_use_agent.md)- [Mobile Workspace setup, controls, logging and limitations](../../../docs/mobile-workspace.md)+++### Live Android activity++Workspace shows the active phone task's model/provider, step budget, total elapsed time,+current phase duration, and time since the last reported activity. The expandable activity+panel retains the latest 40 events: model requests/responses and actual device operations+(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).+Completed operations include duration. The last task remains visible after completion,+failure, or cancellation until another task runs; history is in memory and clears on restart.+A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last+phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.+The panel is outside the phone canvas and hidden in full-screen mode. Activity contains+operational summaries, never private model reasoning, typed text, URLs, or prompt content.diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.mdindex 480d6b6e..4b9aa3a8 100644--- a/docs/mobile-workspace.md+++ b/docs/mobile-workspace.md@@ -186,3 +186,17 @@ This is an implementation plan, not currently supported functionality. Keep the| iOS | Not implemented | Same-Mac lifecycle, IDB adapter, simulator app install, preview, gestures, fenced takeover |Passing a source or fake-process test does not count as passing the corresponding device acceptance check.+++### Live Android activity++Workspace shows the active phone task's model/provider, step budget, total elapsed time,+current phase duration, and time since the last reported activity. The expandable activity+panel retains the latest 40 events: model requests/responses and actual device operations+(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).+Completed operations include duration. The last task remains visible after completion,+failure, or cancellation until another task runs; history is in memory and clears on restart.+A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last+phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.+The panel is outside the phone canvas and hidden in full-screen mode. Activity contains+operational summaries, never private model reasoning, typed text, URLs, or prompt content.diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.pyindex a97a254b..b08c57ec 100644--- a/server/nodes/mobile/_runtime.py+++ b/server/nodes/mobile/_runtime.py@@ -40,6 +40,7 @@ class MobileRuntime:self._setup_loaded = Falseself._setup_saved = 0.0self.active: dict | None = None+ self.last_task: dict | None = Noneself.queue: list[dict] = []self.capabilities: dict[str, tuple[str, Lease]] = {}self.resume_event = asyncio.Event()@@ -119,6 +120,7 @@ class MobileRuntime:"setup_progress": self.setup_progress,"diagnostics": recent(),"active": {k: v for k, v in (self.active or {}).items() if k not in {"prompt"}},+ "last_task": self.last_task,"queue": [{k: v for k, v in q.items() if k != "prompt"} for q in self.queue],**self.control.snapshot(),}@@ -502,6 +504,8 @@ class MobileRuntime:if not self.serial:raise MobileError("device_offline", "Start the shared device in Workspace first")self.active = entry+ entry.update(started_at=time.time(), provider=model.get("provider"), model=model.get("model"))+ await self._publish_progress("Preparing phone")active_remaining = params["timeout_s"]steps_remaining = params["max_steps"]attempts = 0@@ -512,6 +516,7 @@ class MobileRuntime:if self.control.owner and self.control.owner.startswith("viewer:"):entry["status"] = "awaiting_user"if entry["status"] == "awaiting_user":+ await self._publish_progress("Waiting for you")await asyncio.wait_for(self.resume_event.wait(), 1800)self.resume_event.clear()if entry["status"] == "cancelled":@@ -573,6 +578,7 @@ class MobileRuntime:continueif not result.get("success"):raise MobileError(result.get("code", "agent_failed"), result.get("error", "Mobile agent failed"))+ entry["status"] = "completed"event("task_completed", run_id=run_id)return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}finally:@@ -582,11 +588,16 @@ class MobileRuntime:# Wait for any previously admitted broker write before reporting completion.async with self.control.lock:pass+ entry["finished_at"] = time.time()+ self.last_task = entryself.active = Noneexcept asyncio.CancelledError:+ entry["status"] = "cancelled"event("task_cancelled", run_id=run_id)raiseexcept Exception as exc:+ entry["status"] = "failed"+ entry["error_code"] = getattr(exc, "code", "timeout" if isinstance(exc, asyncio.TimeoutError) else "task_failed")event("task_failed", failed=True, run_id=run_id, error_type=type(exc).__name__, code=getattr(exc, "code", "task_failed"))raisefinally:@@ -597,7 +608,14 @@ class MobileRuntime:if not self.active:returnentry = self.active+ now = time.time()+ if entry.get("phase") != phase:+ entry["phase_started_at"] = nowentry["phase"] = phase+ entry["updated_at"] = now+ history = entry.setdefault("activity", [])+ history.append({"at": now, "message": phase, "step": entry.get("steps", 0)})+ del history[:-40]try:from services.status_broadcaster import get_status_broadcasterawait get_status_broadcaster().update_node_status(@@ -639,6 +657,8 @@ class MobileRuntime:event("engine_starting", **scope)await self._publish_progress("Starting Android task")+ pending_models: dict[str, float] = {}+async def read():result = {"success": False, "error": "Mobile worker exited before completing the task"}while line := await proc.stdout.readline():@@ -649,11 +669,41 @@ class MobileRuntime:result = {"success": False, "error": message.get("error", "Worker failed"), "code": message.get("code", "engine_failed")}event("engine_failed", failed=True, **scope, engine_trace=message.get("trace", []),**{key: message.get(key) for key in ("error_type", "code", "stage", "http_status", "provider_status")})+ elif message.get("type") == "activity":+ labels = {"observe": "Reading screen and accessibility tree", "geometry": "Checking screen size",+ "tap": "Tapping screen", "swipe": "Scrolling", "text": "Typing text",+ "key": "Pressing navigation key", "launch": "Opening app", "terminate": "Closing app",+ "url": "Opening link", "erase": "Erasing text", "packages": "Listing apps",+ "foreground": "Checking current app", "date": "Checking device time"}+ label = labels.get(message.get("operation"))+ state = message.get("state")+ if label and state in {"started", "completed", "failed"}:+ duration = message.get("duration_ms")+ suffix = f" ({duration / 1000:.1f}s)" if isinstance(duration, (int, float)) and duration >= 0 else ""+ await self._publish_progress(label if state == "started" else f"{label}: {state}{suffix}")elif message.get("type") == "diagnostic":event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))phase = {"model_request": "Waiting for model", "model_response": "Model responded",- "engine_initialization": "Connecting engine", "task_execution": "Using phone"}.get(message.get("stage"))- if phase and self.active and self.active.get("phase") != phase:+ "engine_initialization": "Connecting engine", "task_execution": "Using phone",+ "engine_cleanup": "Finishing engine cleanup"}.get(message.get("stage"))+ stage = message.get("stage")+ request_id = str(message.get("request_id", "model"))+ if stage == "model_request":+ pending_models[request_id] = time.monotonic()+ role = {"planner": "planning", "orchestrator": "coordinating next step",+ "contextor": "understanding the screen", "cortex": "choosing an action",+ "executor": "preparing an action", "outputter": "preparing the result",+ "hopper": "selecting an app"}.get(message.get("role"))+ if role:+ phase = f"Waiting for model: {role}"+ elif stage in {"model_response", "model_error"}:+ started = pending_models.pop(request_id, None)+ phase = "Model request failed" if stage == "model_error" else "Model responded"+ if started is not None:+ phase += f" ({time.monotonic() - started:.1f}s)"+ if pending_models:+ phase += f"; waiting for {len(pending_models)} model request(s)"+ if phase and self.active:await self._publish_progress(phase)elif message.get("type") == "ready":event("engine_ready", **scope)diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.pyindex 4383fcd7..cd6f0a9d 100644--- a/server/nodes/mobile/runtime/worker.py+++ b/server/nodes/mobile/runtime/worker.py@@ -9,6 +9,8 @@ import reimport sysimport urllib.requestimport uuid+import time+import threadingfrom io import BytesIOtry:@@ -18,11 +20,13 @@ except ImportError: # Executed by path in the isolated engine environment.STAGE = "bootstrap"WIRE = sys.stdout+WIRE_LOCK = threading.Lock()def emit(kind: str, **data) -> None:- WIRE.write(json.dumps({"v": 1, "type": kind, **data}, default=str) + "\n")- WIRE.flush()+ with WIRE_LOCK:+ WIRE.write(json.dumps({"v": 1, "type": kind, **data}, default=str) + "\n")+ WIRE.flush()class Controller:@@ -43,10 +47,17 @@ class Controller:json.dumps(payload).encode(),{"Content-Type": "application/json", "Authorization": f"Bearer {self.config['capability']}"},)- with urllib.request.urlopen(req, timeout=35) as response:- result = json.load(response)- if not result.get("success"):- raise RuntimeError(result.get("error", "Device command failed"))+ started = time.monotonic()+ emit("activity", operation=operation, state="started")+ try:+ with urllib.request.urlopen(req, timeout=35) as response:+ result = json.load(response)+ if not result.get("success"):+ raise RuntimeError(result.get("error", "Device command failed"))+ except Exception:+ emit("activity", operation=operation, state="failed", duration_ms=round((time.monotonic() - started) * 1000))+ raise+ emit("activity", operation=operation, state="completed", duration_ms=round((time.monotonic() - started) * 1000))return result["result"]async def read(self, operation: str, **params):@@ -155,10 +166,16 @@ async def main(config: dict) -> None:emit("progress", steps=max(0, step))def on_chat_model_start(self, serialized, messages, **kwargs):- emit("diagnostic", stage="model_request")+ role = (kwargs.get("metadata") or {}).get("langgraph_node")+ allowed = {"planner", "orchestrator", "contextor", "cortex", "executor", "outputter", "hopper"}+ emit("diagnostic", stage="model_request", request_id=str(kwargs.get("run_id", "model")),+ role=role if role in allowed else None)++ def on_llm_error(self, error, **kwargs):+ emit("diagnostic", stage="model_error", request_id=str(kwargs.get("run_id", "model")))def on_llm_end(self, response, **kwargs):- emit("diagnostic", stage="model_response")+ emit("diagnostic", stage="model_response", request_id=str(kwargs.get("run_id", "model")))output = response.llm_output or {}usage = output.get("token_usage") or output.get("usage") or {}if usage:@@ -211,6 +228,8 @@ async def main(config: dict) -> None:raisefinally:try:+ if not task_failed:+ emit("diagnostic", stage="engine_cleanup")await agent.clean()except Exception as cleanup_error:if not task_failed:diff --git a/server/skills/android_agent/android-phone-skill/SKILL.md b/server/skills/android_agent/android-phone-skill/SKILL.mdindex 9c03e55a..0c74c796 100644--- a/server/skills/android_agent/android-phone-skill/SKILL.md+++ b/server/skills/android_agent/android-phone-skill/SKILL.md@@ -22,3 +22,17 @@ Call `android` with a single `prompt` describing one clear goal, the relevant ap- Terminal source `mobile` shows the execution ID, provider, model backend, model wait, and device timing. Report the error and relevant phase without asking the owner to reveal API keys.Example: `{"prompt":"Open Settings, find the Android version, and report it without changing settings."}`+++### Live Android activity++Workspace shows the active phone task's model/provider, step budget, total elapsed time,+current phase duration, and time since the last reported activity. The expandable activity+panel retains the latest 40 events: model requests/responses and actual device operations+(screen plus accessibility-tree reads, taps, scrolling, typing, navigation, and app actions).+Completed operations include duration. The last task remains visible after completion,+failure, or cancellation until another task runs; history is in memory and clears on restart.+A 30-second quiet period shows a notice, not a claim that the engine is stuck. Use the last+phase to distinguish a model wait from device I/O; take manual control using Use phone if needed.+The panel is outside the phone canvas and hidden in full-screen mode. Activity contains+operational summaries, never private model reasoning, typed text, URLs, or prompt content.diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.pyindex 52563b17..da2065fd 100644--- a/server/tests/test_mobile_runtime.py+++ b/server/tests/test_mobile_runtime.py@@ -303,6 +303,9 @@ async def test_worker_failure_reaches_diagnostics_and_keeps_provider_code(runtimfrom nodes.mobile._control import MobileErrorproc = FakeProcess()+ proc.stdout.feed_data(b'{"type":"activity","operation":"observe","state":"started"}\n')+ proc.stdout.feed_data(b'{"type":"activity","operation":"observe","state":"completed","duration_ms":123}\n')+ proc.stdout.feed_data(b'{"type":"activity","operation":"secret-url","state":"started"}\n')proc.stdout.feed_data(b'{"type":"diagnostic","stage":"model_request"}\n')proc.stdout.feed_data(b'{"type":"failed","error":"Model not found (HTTP 404)","code":"model_not_found","http_status":404,"error_type":"ClientError","stage":"task_execution"}\n')proc.returncode = 1@@ -324,3 +327,28 @@ async def test_worker_failure_reaches_diagnostics_and_keeps_provider_code(runtimassert recorded[0]["execution_id"] == "execution-2"assert recorded[0]["node_id"] == "agent"assert recorded[0]["model"] == "selected-model"+ assert runtime.active is None+ assert runtime.last_task["status"] == "failed"+ assert runtime.last_task["error_code"] == "model_not_found"+ assert runtime.last_task["activity"][-1]["message"] == "Waiting for model"+ assert runtime.last_task["finished_at"] >= runtime.last_task["started_at"]+ messages = [item["message"] for item in runtime.last_task["activity"]]+ assert "Reading screen and accessibility tree: completed (0.1s)" in messages+ assert not any("secret-url" in item for item in messages)+++async def test_activity_is_bounded_and_phase_clock_is_stable(runtime, monkeypatch):+ from unittest.mock import AsyncMock+ from types import SimpleNamespace+ import services.status_broadcaster as status+ monkeypatch.setattr(status, "get_status_broadcaster", lambda: SimpleNamespace(update_node_status=AsyncMock()))+ runtime.active = {"node_id": "phone", "workflow_id": "wf", "run_id": "run", "status": "running"}+ await runtime._publish_progress("Waiting for model")+ started = runtime.active["phase_started_at"]+ for _ in range(50):+ await runtime._publish_progress("Waiting for model")+ assert runtime.active["phase_started_at"] == started+ assert len(runtime.active["activity"]) == 40+ assert runtime.active["updated_at"] >= started+ await runtime._publish_progress("Tapping screen")+ assert runtime.snapshot()["active"]["activity"][-1]["message"] == "Tapping screen"