Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(mobile): show tool progress and reduce control latency
trohitge8edd099972c
diff --git a/client/src/components/SquareNode.tsx b/client/src/components/SquareNode.tsxindex 891c403a..388f6631 100644--- a/client/src/components/SquareNode.tsx+++ b/client/src/components/SquareNode.tsx@@ -315,6 +315,10 @@ const SquareNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnectabcursor: 'pointer',} as NodeStyle}>+ {isExecuting && typeof nodeStatus?.data?.iteration === 'number' && <span role="status" className="text-xs text-fg-muted">+ Step {nodeStatus.data.iteration} / {Number(nodeStatus.data.max_iterations || 0)}+ {typeof nodeStatus.data.phase === 'string' ? ` · ${nodeStatus.data.phase}` : ''}+ </span>}{/* Main Square Node */}<divclassName="sq-node-box"diff --git a/client/src/components/mobile/MobileWorkspace.tsx b/client/src/components/mobile/MobileWorkspace.tsxindex 7dcfabd7..8719f5b7 100644--- a/client/src/components/mobile/MobileWorkspace.tsx+++ b/client/src/components/mobile/MobileWorkspace.tsx@@ -138,7 +138,7 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; noconst perform = async (label: string, action: () => Promise<void>) => {if (operation.current) return;operation.current = true; setBusy(label); setError('');- try { await action(); if (active.current) await refresh(); }+ try { await action(); if (active.current && label !== 'input') await refresh(); }catch (cause) { if (active.current) setError(cause instanceof Error ? cause.message : 'That action failed.'); }finally { operation.current = false; if (active.current) setBusy(''); }};@@ -190,6 +190,7 @@ function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; noreturn <FullView label="Phone" toolbar={<><Smartphone aria-hidden className="size-4 shrink-0 text-fg-muted" /><span role="status" className="min-w-0 flex-1 truncate text-xs text-fg-muted">{setupActive ? 'Setting up' : status?.starting ? 'Starting phone…' : busy ? `${busy}…` : running ? (held ? 'You’re using the phone' : status.active?.status === 'running' ? 'AI is working' : status.control_state === 'recovering' ? 'Reconnecting…' : 'Ready') : status ? 'Phone is off' : 'Connecting…'}</span>+ {running && status?.active?.run_id != null && <span role="status" className="text-xs text-fg-muted">{String(status.active.phase || status.active.status || 'Working')} · Step {Number(status.active.steps || 0)} / {Number(status.active.max_steps || 0)}</span>}{!running && powerControl}{running && (held ? <><Button size="sm" onClick={() => void perform('releasing', async () => { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch, resume: true }); leaseEpoch.current = null; setEpoch(null); })} disabled={!!busy}>Let AI continue</Button>diff --git a/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx b/client/src/components/mobile/__tests__/MobileWorkspace.test.tsxindex deaffb1b..5d7c4aeb 100644--- a/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx+++ b/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx@@ -22,6 +22,12 @@ beforeEach(() => {afterEach(() => { cleanup(); vi.unstubAllGlobals(); });describe('Mobile Workspace', () => {+ it('shows internal phone progress when invoked by an AI agent tool call', async () => {+ snapshot.running = true;+ snapshot.active = { run_id: 'tool-run', phase: 'Waiting for model', steps: 7, max_steps: 40 };+ render(<MobileWorkspace workflowId="flow" nodes={nodes} />);+ expect(await screen.findByText(/Waiting for model.*Step 7 \/ 40/)).toBeVisible();+ });it('retains a startup failure across status refreshes without claiming the phone is ready', async () => {snapshot.start_error = 'This phone is already open, but could not be safely reconnected.';render(<MobileWorkspace workflowId="flow" nodes={nodes} />);diff --git a/docs-internal/node-logic-flows/mobile/android_tool.md b/docs-internal/node-logic-flows/mobile/android_tool.mdindex dd6c44ed..f57f8964 100644--- a/docs-internal/node-logic-flows/mobile/android_tool.md+++ b/docs-internal/node-logic-flows/mobile/android_tool.md@@ -11,6 +11,8 @@Give an existing AI agent a natural-language tool for the shared local Android phone. Setup and startup happen explicitly in Workspace → Mobile. This is separate from the legacy Android relay services.+The paired `android-phone-skill` is registered in `nodes/visuals.json` for the auto-add-skill policy. Internal phone steps publish `iteration`, `max_iterations`, `phase`, execution ID and run ID through `update_node_status`, scoped to the Android node and workflow. The square tool node and Workspace display those steps while the parent AI agent remains in its current tool-call turn. Manual takeover changes the phase to “Waiting for you”; resumed tasks retain their consumed step budget.+## Inputs (handles)| Handle | Required | Purpose |diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.mdindex 0bf36450..480d6b6e 100644--- a/docs/mobile-workspace.md+++ b/docs/mobile-workspace.md@@ -139,6 +139,10 @@ Minitap's [platform](https://www.minitap.ai/platform#slack-cli-mcp) and [miniTes## Validation and limitations+Android tool calls publish live step counts and phases to the normal canvas node-status channel and Workspace's phone header, including calls made by another AI agent. These are internal phone steps; the parent agent advances its own turn when the tool finishes. Step counts retain the already-consumed budget after manual takeover/resume. Model waits are labelled separately from device work. The Android tool's registered `android-phone-skill` teaches goal-based calls, manual handoff, and error handling; existing workflows can add it through their Master Skill node, while new tool connections use the normal auto-add-skill policy.++Manual control avoids an extra geometry/status round trip after every input. Device geometry uses the automation server's current display dimensions, falling back to the SDK size query when missing; taps and swipes still validate current size and rotation before executing. Non-coordinate commands avoid unnecessary geometry reads, and status polling does not queue behind a busy driver. Device read/action timings remain in Terminal to distinguish device latency from model inference time. These changes do not eliminate the phone engine's multiple model calls per task.+Android lifecycle and task events appear in the application's **Terminal** under source `mobile`, prefixed `Android:`. They are also retained in the bounded `mobile.log` and Workspace's recent phone activity. For task debugging, look for `engine_starting`, `engine_stage`, `engine_failed`, and `engine_exited`; correlate them by execution ID, run ID, workflow, and node. Model events include the selected provider/model and failures include the HTTP status and engine stage. HTTP 400 means the provider rejected the request, 401/403 indicates credentials or permissions, 404 indicates a missing model, 429 indicates quota/rate limits, and 5xx indicates a provider service failure. Invalid-key responses reported as HTTP 400 are classified as authentication failures.Raw engine stderr and provider response bodies are not forwarded because they can contain prompts, phone screenshots, or API keys. The structured worker protocol supplies diagnostic metadata instead. Earlier generic `ClientError` failures cannot be reconstructed from logs that discarded this metadata; restart the backend after updating, then retry the task to capture a diagnostic event. This logging does not include a continuous Android OS `logcat` stream.diff --git a/server/nodes/mobile/_router.py b/server/nodes/mobile/_router.pyindex c1d519fe..35b173be 100644--- a/server/nodes/mobile/_router.py+++ b/server/nodes/mobile/_router.py@@ -96,7 +96,7 @@ async def status(principal: str = Depends(authorize)):runtime = get_runtime()# During startup/shutdown return lifecycle status without touching the# driver. The serial is assigned before Android and automation are ready.- if runtime.serial and not runtime.lifecycle_lock.locked():+ if runtime.serial and not runtime.lifecycle_lock.locked() and not runtime.driver_lock.locked():try:await runtime.driver_call("geometry")except Exception:diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.pyindex 9343abe2..a97a254b 100644--- a/server/nodes/mobile/_runtime.py+++ b/server/nodes/mobile/_runtime.py@@ -418,6 +418,8 @@ class MobileRuntime:raise MobileError(result.get("code", "device_action_failed"), result.get("error", "Device command failed"))if operation not in READS:event("device_action_completed", operation=operation, duration_ms=round((time.monotonic() - started) * 1000))+ elif operation != "geometry":+ event("device_read_completed", operation=operation, duration_ms=round((time.monotonic() - started) * 1000))if operation == "geometry":self.geometry = result["result"]elif operation == "observe":@@ -457,6 +459,7 @@ class MobileRuntime:lease = await self.control.claim("viewer:" + viewer, takeover=True)self.viewer = viewerevent("manual_control_started")+ await self._publish_progress("Waiting for you")return leasedef release(self, viewer: str, *, resume: bool = False):@@ -486,7 +489,7 @@ class MobileRuntime:raise MobileError("queue_full", "The shared device queue is full")if any(q["run_id"] == run_id for q in self.queue) or (self.active and self.active["run_id"] == run_id):raise MobileError("duplicate_run", "This task is already admitted")- entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "status": "queued"}+ entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "status": "queued", "steps": 0, "max_steps": params["max_steps"]}self.queue.append(entry)event("task_queued", run_id=run_id, workflow_id=workflow_id, node_id=node_id)try:@@ -548,6 +551,7 @@ class MobileRuntime:}attempts += 1entry["attempt_steps"] = 0+ entry["step_offset"] = params["max_steps"] - steps_remainingstarted = time.monotonic()try:try:@@ -589,6 +593,22 @@ class MobileRuntime:if entry in self.queue:self.queue.remove(entry)+ async def _publish_progress(self, phase: str) -> None:+ if not self.active:+ return+ entry = self.active+ entry["phase"] = phase+ try:+ from services.status_broadcaster import get_status_broadcaster+ await get_status_broadcaster().update_node_status(+ entry["node_id"], "waiting" if entry["status"] == "awaiting_user" else "executing",+ {"iteration": entry.get("steps", 0), "max_iterations": entry.get("max_steps", 0),+ "phase": phase, "execution_id": entry.get("execution_id"), "run_id": entry["run_id"]},+ workflow_id=entry["workflow_id"],+ )+ except Exception as exc:+ event("progress_delivery_failed", failed=True, error_type=type(exc).__name__)+async def _run_worker(self, config: dict, timeout: float) -> dict:entry = Path(__file__).with_name("runtime") / "worker.py"self.worker = await asyncio.create_subprocess_exec(@@ -617,6 +637,7 @@ class MobileRuntime:if config.get("provider") == "google":scope["model_backend"] = "vertex_express" if config.get("model_env", {}).get("GOOGLE_GENAI_USE_VERTEXAI") == "true" else "gemini_developer"event("engine_starting", **scope)+ await self._publish_progress("Starting Android task")async def read():result = {"success": False, "error": "Mobile worker exited before completing the task"}@@ -630,6 +651,10 @@ class MobileRuntime:**{key: message.get(key) for key in ("error_type", "code", "stage", "http_status", "provider_status")})elif message.get("type") == "diagnostic":event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))+ phase = {"model_request": "Waiting for model", "model_response": "Model responded",+ "engine_initialization": "Connecting engine", "task_execution": "Using phone"}.get(message.get("stage"))+ if phase and self.active and self.active.get("phase") != phase:+ await self._publish_progress(phase)elif message.get("type") == "ready":event("engine_ready", **scope)elif message.get("type") == "usage" and self.active:@@ -638,6 +663,8 @@ class MobileRuntime:steps = max(self.active.get("attempt_steps", 0), int(message.get("steps", 0)))if steps != self.active.get("attempt_steps", 0):event("task_progress", **scope, steps=steps)+ self.active["steps"] = min(self.active.get("max_steps", steps), self.active.get("step_offset", 0) + steps)+ await self._publish_progress("Using phone")self.active["attempt_steps"] = stepsawait proc.wait()event("engine_exited", failed=proc.returncode not in (0, None), **scope, exit_code=proc.returncode)diff --git a/server/nodes/mobile/runtime/device_server.py b/server/nodes/mobile/runtime/device_server.pyindex 1f3ff66e..853ede8a 100644--- a/server/nodes/mobile/runtime/device_server.py+++ b/server/nodes/mobile/runtime/device_server.py@@ -24,12 +24,14 @@ class AndroidDriver:def geometry(self):info = self.device.info- width, height = self.device.window_size()+ width, height = info.get("displayWidth"), info.get("displayHeight")+ if not isinstance(width, int) or not isinstance(height, int) or width <= 0 or height <= 0:+ width, height = self.device.window_size()return {"width": width, "height": height, "rotation": info.get("displayRotation", 0)}def call(self, op: str, p: dict):d = self.device- geometry = self.geometry()+ geometry = self.geometry() if op in {"geometry", "observe", "tap", "swipe", "touch"} else Noneif op in {"tap", "swipe", "touch"}:if p.pop("geometry", None) != geometry:raise ValueError("Screen changed; refresh before interacting")diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.pyindex 1cca5941..4383fcd7 100644--- a/server/nodes/mobile/runtime/worker.py+++ b/server/nodes/mobile/runtime/worker.py@@ -152,7 +152,7 @@ async def main(config: dict) -> None:def on_chain_start(self, serialized, inputs, *, metadata=None, **kwargs):step = (metadata or {}).get("langgraph_step")if isinstance(step, int):- emit("progress", steps=step + 1)+ emit("progress", steps=max(0, step))def on_chat_model_start(self, serialized, messages, **kwargs):emit("diagnostic", stage="model_request")diff --git a/server/nodes/visuals.json b/server/nodes/visuals.jsonindex cd6101c4..d45753d6 100644--- a/server/nodes/visuals.json+++ b/server/nodes/visuals.json@@ -15,6 +15,9 @@"android_agent": {"icon": "📱"},+ "android_tool": {+ "skill": "android-phone-skill"+ },"anthropicChatModel": {"icon": "lobehub:claude"},diff --git a/server/skills/android_agent/android-phone-skill/SKILL.md b/server/skills/android_agent/android-phone-skill/SKILL.mdnew file mode 100644index 00000000..9c03e55a--- /dev/null+++ b/server/skills/android_agent/android-phone-skill/SKILL.md@@ -0,0 +1,24 @@+---+name: android-phone-skill+description: Use the shared local Android phone through the Android tool, with visible progress and safe manual handoff.+allowed-tools: android+metadata:+ author: opencompany+ version: "1.0"+ category: android+---++# Shared Android phone++Call `android` with a single `prompt` describing one clear goal, the relevant app, and what result to verify. The tool controls the phone visible in Workspace > Mobile. It returns the result after its internal phone steps finish.++- Use one call for a coherent task; avoid one model-driven call per tap. Each call starts a phone agent and has model latency.+- Do not invent low-level operations or pass model, credential, timeout, or step-budget arguments. These come from the saved phone settings.+- Global model selection is the default. An optional connected model overrides it. OpenAI, Claude and Gemini are supported; Gemini Express credentials automatically route to Vertex.+- A phone task is one tool call in the parent agent's turn. Its internal steps appear on the Android node and in Workspace; they are not extra parent-agent turns. Do not claim completion while the tool is still running.+- If the owner takes control, wait for them to resume the task. Ask them to enter passwords and handle interactive login themselves. Never request passwords in chat.+- If the phone is off, direct the owner to Workspace > Mobile > Start phone. Do not install another emulator or attempt raw ADB access.+- Read returned errors. Authentication, quota, unsupported-model, and connection failures need their stated remedy; do not repeat failed calls in a loop or treat an error payload as success.+- Terminal source `mobile` shows the execution ID, provider, model backend, model wait, and device timing. Report the error and relevant phase without asking the owner to reveal API keys.++Example: `{"prompt":"Open Settings, find the Android version, and report it without changing settings."}`diff --git a/server/skills/android_agent/personality/SKILL.md b/server/skills/android_agent/personality/SKILL.mdindex 48eda077..805ced8b 100644--- a/server/skills/android_agent/personality/SKILL.md+++ b/server/skills/android_agent/personality/SKILL.md@@ -3,7 +3,7 @@ name: android-personalitydescription: Android device assistant personality. Use when the user wants to interact with their Android phone or tablet in a natural, conversational way.metadata:author: opencompany- version: "1.0"+ version: "1.1"category: assistanticon: "🤖"color: "#3DDC84"@@ -12,6 +12,8 @@ metadata:# Android Device Assistant+When the connected tool is `android`, use the `android-phone-skill` instructions: submit one plain-language goal for the shared Workspace phone and wait for its result. Do not invent battery, Wi-Fi, or other low-level tool names unless those separate tools are actually available. Internal phone steps are shown on the Android tool node and in Workspace; they are not additional turns of the parent agent. A model wait is not a disconnected phone. If the owner takes control, wait for resume instead of submitting duplicate tasks.+You are an Android device assistant. Your role is to help users interact with their Android phone or tablet naturally and efficiently.## Core Principlesdiff --git a/server/tests/test_mobile_driver_latency.py b/server/tests/test_mobile_driver_latency.pynew file mode 100644index 00000000..aa5e56d9--- /dev/null+++ b/server/tests/test_mobile_driver_latency.py@@ -0,0 +1,32 @@+from types import SimpleNamespace+from unittest.mock import Mock++from nodes.mobile.runtime.device_server import AndroidDriver+++def test_geometry_uses_one_device_query_and_preserves_rotation():+ driver = AndroidDriver.__new__(AndroidDriver)+ driver.device = SimpleNamespace(info={"displayWidth": 2400, "displayHeight": 1080, "displayRotation": 1},+ window_size=Mock(side_effect=AssertionError("redundant query")))+ assert driver.geometry() == {"width": 2400, "height": 1080, "rotation": 1}+++def test_key_input_does_not_wait_for_geometry():+ driver = AndroidDriver.__new__(AndroidDriver)+ driver.device = SimpleNamespace(press=Mock())+ driver.geometry = Mock(side_effect=AssertionError("unneeded query"))+ assert driver.call("key", {"key": "home"}) is True+ driver.device.press.assert_called_once_with("home")+++def test_pointer_still_rejects_stale_screen():+ import pytest+ driver = AndroidDriver.__new__(AndroidDriver)+ driver.device = SimpleNamespace(info={"displayWidth": 2400, "displayHeight": 1080, "displayRotation": 1})+ with pytest.raises(ValueError, match="Screen changed"):+ driver.call("tap", {"x": 10, "y": 10, "geometry": {"width": 1080, "height": 2400, "rotation": 0}})+++def test_android_skill_is_registered_for_auto_attachment():+ from nodes._visuals import get_skill+ assert get_skill("android_tool") == "android-phone-skill"diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.pyindex 546a6128..52563b17 100644--- a/server/tests/test_mobile_runtime.py+++ b/server/tests/test_mobile_runtime.py@@ -60,6 +60,32 @@ async def test_status_during_startup_does_not_query_driver(monkeypatch, tmp_pathquery.assert_not_awaited()+async def test_phone_progress_uses_normal_node_status_channel(runtime, monkeypatch):+ from unittest.mock import AsyncMock+ from types import SimpleNamespace+ import services.status_broadcaster as status+ update = AsyncMock()+ monkeypatch.setattr(status, "get_status_broadcaster", lambda: SimpleNamespace(update_node_status=update))+ runtime.active = {"node_id": "phone", "workflow_id": "wf", "execution_id": "exec", "run_id": "run",+ "status": "running", "steps": 7, "max_steps": 40}+ await runtime._publish_progress("Waiting for model")+ update.assert_awaited_once_with("phone", "executing", {"iteration": 7, "max_iterations": 40,+ "phase": "Waiting for model", "execution_id": "exec", "run_id": "run"}, workflow_id="wf")+ assert runtime.snapshot()["active"]["phase"] == "Waiting for model"+++async def test_status_poll_does_not_queue_behind_input(runtime, monkeypatch):+ from unittest.mock import AsyncMock+ import nodes.mobile._runtime as module+ from nodes.mobile._router import status+ monkeypatch.setattr(module, "get_runtime", lambda: runtime)+ query = AsyncMock()+ monkeypatch.setattr(runtime, "driver_call", query)+ async with runtime.driver_lock:+ await status(principal="owner")+ query.assert_not_awaited()++class FakeStdin:def __init__(self):self.data = b""