Skip to content

Self-improving AI Employees for everyone

fix(mobile): expose Android diagnostics and actionable provider errors

trohitgf0a0db164a19
diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.md
index c1a29af3..3ebfa29f 100644
--- a/docs/mobile-workspace.md
+++ b/docs/mobile-workspace.md
@@ -135,6 +135,10 @@ Minitap's [platform](https://www.minitap.ai/platform#slack-cli-mcp) and [miniTes
## Validation and limitations
+Android lifecycle and task events appear in the application's **Terminal** under source `mobile`, prefixed `Android:`. They are also retained in the bounded `mobile.log` and Workspace's recent phone activity. For task debugging, look for `engine_starting`, `engine_stage`, `engine_failed`, and `engine_exited`; correlate them by execution ID, run ID, workflow, and node. Model events include the selected provider/model and failures include the HTTP status and engine stage. HTTP 400 means the provider rejected the request, 401/403 indicates credentials or permissions, 404 indicates a missing model, 429 indicates quota/rate limits, and 5xx indicates a provider service failure. Invalid-key responses reported as HTTP 400 are classified as authentication failures.
+
+Raw engine stderr and provider response bodies are not forwarded because they can contain prompts, phone screenshots, or API keys. The structured worker protocol supplies diagnostic metadata instead. Earlier generic `ClientError` failures cannot be reconstructed from logs that discarded this metadata; restart the backend after updating, then retry the task to capture a diagnostic event. This logging does not include a continuous Android OS `logcat` stream.
+
Phone startup first checks for an existing emulator. Recovery requires the console-reported AVD path to match this installation and the process to belong to its Android SDK. A process-wide device lock prevents two updated backends from owning the same writable phone. A disconnected automation driver can be restarted without launching a second emulator. Stop waits for the emulator and cleans up its owned child processes; it does not wipe phone data or stop unrelated emulators.
If startup fails, the message remains in Workspace after a refresh. A duplicate-AVD error means another emulator still holds the phone; if verified reconnection is unavailable, close that OpenCompany emulator and retry. Do not delete AVD lock files or use `-read-only` to bypass ownership. `emulator.log` retains launcher output and rotates to `emulator.previous.log` before a new launch when it exceeds 2 MiB. Error classification reads only the current attempt; `mobile.log` records recovery and boot-probe timeout events without model prompts or credentials.
diff --git a/server/nodes/mobile/_diagnostics.py b/server/nodes/mobile/_diagnostics.py
index d947f8ac..9dd6b05f 100644
--- a/server/nodes/mobile/_diagnostics.py
+++ b/server/nodes/mobile/_diagnostics.py
@@ -14,7 +14,8 @@ _recent: deque = deque(maxlen=40)
_logger = logging.Logger("opencompany.mobile.diagnostics")
_logger.setLevel(logging.INFO)
_logger.propagate = False
-_fields = {"operation", "duration_ms", "error_type", "code", "serial", "run_id", "node_id", "workflow_id", "state"}
+_fields = {"operation", "duration_ms", "error_type", "code", "serial", "run_id", "node_id", "workflow_id", "state",
+ "execution_id", "provider", "model", "stage", "http_status", "provider_status", "steps", "exit_code", "engine_trace"}
def event(name: str, *, failed: bool = False, **fields) -> None:
@@ -25,6 +26,16 @@ def event(name: str, *, failed: bool = False, **fields) -> None:
record["trace"] = [f"{Path(frame.filename).name}:{frame.lineno}:{frame.name}"
for frame in traceback.extract_tb(sys.exc_info()[2])[-12:]]
_recent.append(record)
+ # Use the application's configured pipeline as well as the bounded mobile
+ # file. This feeds both the supervisor console and WebSocket Terminal.
+ try:
+ from core.logging import get_logger
+ logger = get_logger("nodes.mobile")
+ getattr(logger, "error" if failed else "info")(
+ "Android: " + name, **{k: v for k, v in record.items() if k not in {"event", "level", "at"}}
+ )
+ except Exception:
+ pass # Logging cannot break device operations during startup/shutdown.
try:
path = mobile_root() / "mobile.log"
if not _logger.handlers or _logger.handlers[0].baseFilename != str(path.resolve()):
diff --git a/server/nodes/mobile/_node.py b/server/nodes/mobile/_node.py
index 11160707..0e3b3854 100644
--- a/server/nodes/mobile/_node.py
+++ b/server/nodes/mobile/_node.py
@@ -154,6 +154,7 @@ class MobileUseAgent(ActionNode):
workflow_id=ctx.workflow_id,
node_id=ctx.node_id,
run_id=task_identity(ctx),
+ execution_id=ctx.execution_id,
params=config,
model=model,
broker_url=await runtime.ensure_broker(),
diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.py
index 92dfccaf..fd942b9f 100644
--- a/server/nodes/mobile/_runtime.py
+++ b/server/nodes/mobile/_runtime.py
@@ -481,12 +481,12 @@ class MobileRuntime:
self.resume_event.set()
await self._stop_worker()
- async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str) -> dict:
+ async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str, execution_id: str | None = None) -> dict:
if len(self.queue) >= 32:
raise MobileError("queue_full", "The shared device queue is full")
if any(q["run_id"] == run_id for q in self.queue) or (self.active and self.active["run_id"] == run_id):
raise MobileError("duplicate_run", "This task is already admitted")
- entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "status": "queued"}
+ entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "status": "queued"}
self.queue.append(entry)
event("task_queued", run_id=run_id, workflow_id=workflow_id, node_id=node_id)
try:
@@ -568,7 +568,7 @@ class MobileRuntime:
raise MobileError("budget_exhausted", "Task exhausted its time or step budget")
continue
if not result.get("success"):
- raise MobileError("agent_failed", result.get("error", "Mobile agent failed"))
+ raise MobileError(result.get("code", "agent_failed"), result.get("error", "Mobile agent failed"))
event("task_completed", run_id=run_id)
return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}
finally:
@@ -611,19 +611,34 @@ class MobileRuntime:
await proc.stdin.drain()
proc.stdin.close()
+ scope = {"run_id": (self.active or {}).get("run_id"), "workflow_id": (self.active or {}).get("workflow_id"),
+ "node_id": (self.active or {}).get("node_id"), "execution_id": (self.active or {}).get("execution_id"),
+ "provider": config.get("provider"), "model": config.get("model")}
+ event("engine_starting", **scope)
+
async def read():
result = {"success": False, "error": "Mobile worker exited before completing the task"}
while line := await proc.stdout.readline():
- event = json.loads(line)
- if event.get("type") == "completed":
- result = {"success": True, "result": event.get("result")}
- elif event.get("type") == "failed":
- result = {"success": False, "error": event.get("error", "Worker failed")}
- elif event.get("type") == "usage" and self.active:
- self.active["usage"] = event.get("usage")
- elif event.get("type") == "progress" and self.active:
- self.active["attempt_steps"] = max(self.active.get("attempt_steps", 0), int(event.get("steps", 0)))
+ message = json.loads(line)
+ if message.get("type") == "completed":
+ result = {"success": True, "result": message.get("result")}
+ elif message.get("type") == "failed":
+ result = {"success": False, "error": message.get("error", "Worker failed"), "code": message.get("code", "engine_failed")}
+ event("engine_failed", failed=True, **scope, engine_trace=message.get("trace", []),
+ **{key: message.get(key) for key in ("error_type", "code", "stage", "http_status", "provider_status")})
+ elif message.get("type") == "diagnostic":
+ event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))
+ elif message.get("type") == "ready":
+ event("engine_ready", **scope)
+ elif message.get("type") == "usage" and self.active:
+ self.active["usage"] = message.get("usage")
+ elif message.get("type") == "progress" and self.active:
+ steps = max(self.active.get("attempt_steps", 0), int(message.get("steps", 0)))
+ if steps != self.active.get("attempt_steps", 0):
+ event("task_progress", **scope, steps=steps)
+ self.active["attempt_steps"] = steps
await proc.wait()
+ event("engine_exited", failed=proc.returncode not in (0, None), **scope, exit_code=proc.returncode)
return result
return await asyncio.wait_for(read(), max(1, timeout) + 15)
diff --git a/server/nodes/mobile/runtime/errors.py b/server/nodes/mobile/runtime/errors.py
new file mode 100644
index 00000000..7499eb0c
--- /dev/null
+++ b/server/nodes/mobile/runtime/errors.py
@@ -0,0 +1,42 @@
+"""Safe error metadata shared with the isolated engine (standard library only)."""
+
+import traceback
+from pathlib import Path
+
+
+def describe_error(exc: BaseException, stage: str) -> dict:
+ kind = type(exc).__name__
+ status = getattr(exc, "status_code", None) or getattr(exc, "code", None)
+ if not isinstance(status, int) or isinstance(status, bool) or not 100 <= status <= 599:
+ status = None
+ provider_status = getattr(exc, "status", None)
+ if provider_status not in {"INVALID_ARGUMENT", "NOT_FOUND", "UNAUTHENTICATED", "PERMISSION_DENIED", "RESOURCE_EXHAUSTED", "UNAVAILABLE", "INTERNAL", "FAILED_PRECONDITION"}:
+ provider_status = None
+ code, message = "engine_failed", f"Android engine failed during {stage} ({kind}). See Android events in Terminal."
+ if kind == "HTTPError":
+ code, message = "device_connection_failed", "Android device control changed or disconnected. Check the phone connection in Workspace."
+ elif status == 401 or kind == "AuthenticationError":
+ code, message = "model_authentication_failed", "The model provider rejected its credentials. Reconnect the provider in Settings."
+ elif status == 403:
+ code, message = "model_permission_denied", "The model provider denied access. Check that this API key/project can use the selected model."
+ elif status == 404:
+ code, message = "model_not_found", "The selected model was not found by the provider. Choose an available model in the global selector or phone settings."
+ elif status == 429 or kind == "RateLimitError":
+ code, message = "model_rate_limited", "The model provider's quota or rate limit was reached. Check billing/quota or try again later."
+ elif status == 400 and any(term in str(exc).lower() for term in ("api key not valid", "api_key_invalid", "api_key_expired")):
+ code, message = "model_authentication_failed", "The model provider rejected its API key. Reconnect the provider in Settings."
+ elif status == 400:
+ code, message = "model_request_rejected", "The model provider rejected the request. Check that the selected model supports images and tool/structured responses."
+ elif status and status >= 500:
+ code, message = "model_service_unavailable", "The model provider is unavailable. Try again later."
+ elif kind in {"TimeoutError", "APITimeoutError", "ReadTimeout"}:
+ code, message = "task_timeout", "The Android task or model request timed out. Check connectivity and the task time budget."
+ elif kind == "GraphRecursionError":
+ code, message = "step_budget_exhausted", "The Android task exhausted its step budget."
+ elif kind in {"APIConnectionError", "ConnectError"}:
+ code, message = "model_connection_failed", "Could not reach the model provider. Check network and proxy settings."
+ # Never serialize str(exc), response bodies, request headers or locals:
+ # provider exceptions can contain API keys, prompts and screenshots.
+ return {"error": message + (f" (HTTP {status})" if status else ""), "code": code,
+ "error_type": kind, "stage": stage, "http_status": status, "provider_status": provider_status,
+ "trace": [f"{Path(frame.filename).name}:{frame.lineno}:{frame.name}" for frame in traceback.extract_tb(exc.__traceback__)[-8:]]}
diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.py
index 5864837a..1cca5941 100644
--- a/server/nodes/mobile/runtime/worker.py
+++ b/server/nodes/mobile/runtime/worker.py
@@ -11,6 +11,12 @@ import urllib.request
import uuid
from io import BytesIO
+try:
+ from .errors import describe_error
+except ImportError: # Executed by path in the isolated engine environment.
+ from errors import describe_error
+
+STAGE = "bootstrap"
WIRE = sys.stdout
@@ -127,6 +133,8 @@ class Controller:
async def main(config: dict) -> None:
+ global STAGE
+ emit("diagnostic", stage=STAGE)
sys.stdout = sys.stderr # upstream diagnostics can never corrupt the protocol
os.environ.update(config.pop("model_env"))
os.environ["MOBILE_USE_TELEMETRY_ENABLED"] = "false"
@@ -146,7 +154,11 @@ async def main(config: dict) -> None:
if isinstance(step, int):
emit("progress", steps=step + 1)
+ def on_chat_model_start(self, serialized, messages, **kwargs):
+ emit("diagnostic", stage="model_request")
+
def on_llm_end(self, response, **kwargs):
+ emit("diagnostic", stage="model_response")
output = response.llm_output or {}
usage = output.get("token_usage") or output.get("usage") or {}
if usage:
@@ -181,31 +193,36 @@ async def main(config: dict) -> None:
device_height=config["height"],
)
agent = Agent(config=agent_config)
- await agent.init()
- emit("ready")
+ task_failed = False
try:
+ STAGE = "engine_initialization"
+ emit("diagnostic", stage=STAGE)
+ await agent.init()
+ emit("ready")
+ STAGE = "task_execution"
+ emit("diagnostic", stage=STAGE)
result = await asyncio.wait_for(
agent.run_task(request=TaskRequest(goal=config["prompt"], max_steps=config["max_steps"], record_trace=False)),
config["timeout_s"],
)
emit("completed", result=result)
+ except BaseException:
+ task_failed = True
+ raise
finally:
- await agent.clean()
+ try:
+ await agent.clean()
+ except Exception as cleanup_error:
+ if not task_failed:
+ STAGE = "engine_cleanup"
+ raise
+ # Preserve the original provider/device failure if cleanup fails.
+ emit("diagnostic", stage="cleanup_failed", error_type=type(cleanup_error).__name__)
if __name__ == "__main__":
try:
asyncio.run(main(json.loads(sys.stdin.readline())))
except BaseException as exc:
- # Provider errors may echo credentials; expose only an actionable category.
- # Prompts/screenshots in upstream stderr are not persisted.
- kind = type(exc).__name__
- message = {
- "AuthenticationError": "The selected model rejected its credentials.",
- "RateLimitError": "The selected model is rate limited. Try again later.",
- "TimeoutError": "The mobile task exceeded its active time budget.",
- "GraphRecursionError": "The mobile task exhausted its step budget.",
- "HTTPError": "Device control changed or the device connection failed.",
- }.get(kind, f"Mobile engine failed ({kind}).")
- emit("failed", error=message)
+ emit("failed", **describe_error(exc, STAGE))
sys.exit(1)
diff --git a/server/tests/test_mobile_diagnostics.py b/server/tests/test_mobile_diagnostics.py
new file mode 100644
index 00000000..6c19c487
--- /dev/null
+++ b/server/tests/test_mobile_diagnostics.py
@@ -0,0 +1,69 @@
+"""Actionable provider failures without leaking request/response contents."""
+
+from unittest.mock import Mock
+
+import pytest
+
+from nodes.mobile.runtime.errors import describe_error
+from nodes.mobile import _diagnostics
+
+
[email protected]("status,code", [(400, "model_request_rejected"), (401, "model_authentication_failed"),
+ (403, "model_permission_denied"), (404, "model_not_found"),
+ (429, "model_rate_limited"), (503, "model_service_unavailable")])
+def test_client_error_retains_status_without_content(status, code):
+ class ClientError(Exception):
+ pass
+ error = ClientError("secret-key private prompt base64-screenshot")
+ error.code = status
+ error.status = "NOT_FOUND" if status == 404 else "unsafe response content"
+ result = describe_error(error, "task_execution")
+ assert result["code"] == code
+ assert result["http_status"] == status
+ assert result["error_type"] == "ClientError"
+ assert "secret" not in str(result)
+ assert "prompt" not in str(result)
+ assert "screenshot" not in str(result)
+ assert "unsafe" not in str(result)
+
+
+def test_invalid_google_key_is_not_reported_as_model_capability_error():
+ error = RuntimeError("API key not valid. secret-key")
+ error.code = 400
+ assert describe_error(error, "engine_initialization")["code"] == "model_authentication_failed"
+
+
+def test_mobile_events_reach_terminal_pipeline_with_only_safe_fields(monkeypatch, tmp_path):
+ import core.logging
+ logger = Mock()
+ monkeypatch.setattr(core.logging, "get_logger", lambda name: logger)
+ monkeypatch.setattr(_diagnostics, "mobile_root", lambda: tmp_path)
+ try:
+ _diagnostics.event("engine_failed", failed=True, http_status=404, model="selected-model",
+ execution_id="execution-2", prompt="private", api_key="secret")
+ logger.error.assert_called_once()
+ args, fields = logger.error.call_args
+ assert args == ("Android: engine_failed",)
+ assert fields["http_status"] == 404
+ assert fields["execution_id"] == "execution-2"
+ assert "prompt" not in fields and "api_key" not in fields
+ assert '"http_status": 404' in (tmp_path / "mobile.log").read_text()
+ finally:
+ for handler in list(_diagnostics._logger.handlers):
+ handler.close()
+ _diagnostics._logger.removeHandler(handler)
+
+
+def test_terminal_logger_failure_does_not_lose_file_diagnostics(monkeypatch, tmp_path):
+ import core.logging
+ logger = Mock()
+ logger.info.side_effect = RuntimeError("logger unavailable")
+ monkeypatch.setattr(core.logging, "get_logger", lambda name: logger)
+ monkeypatch.setattr(_diagnostics, "mobile_root", lambda: tmp_path)
+ try:
+ _diagnostics.event("phone_ready")
+ assert "phone_ready" in (tmp_path / "mobile.log").read_text()
+ finally:
+ for handler in list(_diagnostics._logger.handlers):
+ handler.close()
+ _diagnostics._logger.removeHandler(handler)
diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.py
index 5827beec..546a6128 100644
--- a/server/tests/test_mobile_runtime.py
+++ b/server/tests/test_mobile_runtime.py
@@ -140,6 +140,7 @@ async def test_takeover_stops_worker_then_resumes_with_fresh_capability(runtime,
human = await runtime.takeover("browser")
assert first.terminated.is_set()
assert not runtime.capabilities
+
assert human.owner == "viewer:browser"
assert runtime.active["status"] == "awaiting_user"
runtime.release("browser", resume=True)
@@ -268,3 +269,32 @@ async def test_stop_preserves_epoch_monotonicity_and_clears_viewer(runtime):
assert not runtime.resume_event.is_set()
next_lease = await runtime.control.claim("viewer:browser")
assert next_lease.epoch > lease.epoch
+
+
+async def test_worker_failure_reaches_diagnostics_and_keeps_provider_code(runtime, monkeypatch):
+ import nodes.mobile._runtime as module
+ from unittest.mock import Mock
+ from nodes.mobile._control import MobileError
+
+ proc = FakeProcess()
+ proc.stdout.feed_data(b'{"type":"diagnostic","stage":"model_request"}\n')
+ proc.stdout.feed_data(b'{"type":"failed","error":"Model not found (HTTP 404)","code":"model_not_found","http_status":404,"error_type":"ClientError","stage":"task_execution"}\n')
+ proc.returncode = 1
+ proc.stdout.feed_eof()
+ proc.terminated.set()
+ async def spawn(*_args, **_kwargs):
+ return proc
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)
+ log = Mock()
+ monkeypatch.setattr(module, "event", log)
+ args = run_args()
+ args.update(execution_id="execution-2", model={"provider": "google", "model": "selected-model"})
+ with pytest.raises(MobileError) as failure:
+ await runtime.run(**args)
+ assert failure.value.code == "model_not_found"
+ recorded = [call.kwargs for call in log.call_args_list if call.args == ("engine_failed",)]
+ assert len(recorded) == 1
+ assert recorded[0]["http_status"] == 404
+ assert recorded[0]["execution_id"] == "execution-2"
+ assert recorded[0]["node_id"] == "agent"
+ assert recorded[0]["model"] == "selected-model"