Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(mobile): expose Android diagnostics and actionable provider errors
trohitgf0a0db164a19
diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.mdindex c1a29af3..3ebfa29f 100644--- a/docs/mobile-workspace.md+++ b/docs/mobile-workspace.md@@ -135,6 +135,10 @@ Minitap's [platform](https://www.minitap.ai/platform#slack-cli-mcp) and [miniTes## Validation and limitations+Android lifecycle and task events appear in the application's **Terminal** under source `mobile`, prefixed `Android:`. They are also retained in the bounded `mobile.log` and Workspace's recent phone activity. For task debugging, look for `engine_starting`, `engine_stage`, `engine_failed`, and `engine_exited`; correlate them by execution ID, run ID, workflow, and node. Model events include the selected provider/model and failures include the HTTP status and engine stage. HTTP 400 means the provider rejected the request, 401/403 indicates credentials or permissions, 404 indicates a missing model, 429 indicates quota/rate limits, and 5xx indicates a provider service failure. Invalid-key responses reported as HTTP 400 are classified as authentication failures.++Raw engine stderr and provider response bodies are not forwarded because they can contain prompts, phone screenshots, or API keys. The structured worker protocol supplies diagnostic metadata instead. Earlier generic `ClientError` failures cannot be reconstructed from logs that discarded this metadata; restart the backend after updating, then retry the task to capture a diagnostic event. This logging does not include a continuous Android OS `logcat` stream.+Phone startup first checks for an existing emulator. Recovery requires the console-reported AVD path to match this installation and the process to belong to its Android SDK. A process-wide device lock prevents two updated backends from owning the same writable phone. A disconnected automation driver can be restarted without launching a second emulator. Stop waits for the emulator and cleans up its owned child processes; it does not wipe phone data or stop unrelated emulators.If startup fails, the message remains in Workspace after a refresh. A duplicate-AVD error means another emulator still holds the phone; if verified reconnection is unavailable, close that OpenCompany emulator and retry. Do not delete AVD lock files or use `-read-only` to bypass ownership. `emulator.log` retains launcher output and rotates to `emulator.previous.log` before a new launch when it exceeds 2 MiB. Error classification reads only the current attempt; `mobile.log` records recovery and boot-probe timeout events without model prompts or credentials.diff --git a/server/nodes/mobile/_diagnostics.py b/server/nodes/mobile/_diagnostics.pyindex d947f8ac..9dd6b05f 100644--- a/server/nodes/mobile/_diagnostics.py+++ b/server/nodes/mobile/_diagnostics.py@@ -14,7 +14,8 @@ _recent: deque = deque(maxlen=40)_logger = logging.Logger("opencompany.mobile.diagnostics")_logger.setLevel(logging.INFO)_logger.propagate = False-_fields = {"operation", "duration_ms", "error_type", "code", "serial", "run_id", "node_id", "workflow_id", "state"}+_fields = {"operation", "duration_ms", "error_type", "code", "serial", "run_id", "node_id", "workflow_id", "state",+ "execution_id", "provider", "model", "stage", "http_status", "provider_status", "steps", "exit_code", "engine_trace"}def event(name: str, *, failed: bool = False, **fields) -> None:@@ -25,6 +26,16 @@ def event(name: str, *, failed: bool = False, **fields) -> None:record["trace"] = [f"{Path(frame.filename).name}:{frame.lineno}:{frame.name}"for frame in traceback.extract_tb(sys.exc_info()[2])[-12:]]_recent.append(record)+ # Use the application's configured pipeline as well as the bounded mobile+ # file. This feeds both the supervisor console and WebSocket Terminal.+ try:+ from core.logging import get_logger+ logger = get_logger("nodes.mobile")+ getattr(logger, "error" if failed else "info")(+ "Android: " + name, **{k: v for k, v in record.items() if k not in {"event", "level", "at"}}+ )+ except Exception:+ pass # Logging cannot break device operations during startup/shutdown.try:path = mobile_root() / "mobile.log"if not _logger.handlers or _logger.handlers[0].baseFilename != str(path.resolve()):diff --git a/server/nodes/mobile/_node.py b/server/nodes/mobile/_node.pyindex 11160707..0e3b3854 100644--- a/server/nodes/mobile/_node.py+++ b/server/nodes/mobile/_node.py@@ -154,6 +154,7 @@ class MobileUseAgent(ActionNode):workflow_id=ctx.workflow_id,node_id=ctx.node_id,run_id=task_identity(ctx),+ execution_id=ctx.execution_id,params=config,model=model,broker_url=await runtime.ensure_broker(),diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.pyindex 92dfccaf..fd942b9f 100644--- a/server/nodes/mobile/_runtime.py+++ b/server/nodes/mobile/_runtime.py@@ -481,12 +481,12 @@ class MobileRuntime:self.resume_event.set()await self._stop_worker()- async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str) -> dict:+ async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str, execution_id: str | None = None) -> dict:if len(self.queue) >= 32:raise MobileError("queue_full", "The shared device queue is full")if any(q["run_id"] == run_id for q in self.queue) or (self.active and self.active["run_id"] == run_id):raise MobileError("duplicate_run", "This task is already admitted")- entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "status": "queued"}+ entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "status": "queued"}self.queue.append(entry)event("task_queued", run_id=run_id, workflow_id=workflow_id, node_id=node_id)try:@@ -568,7 +568,7 @@ class MobileRuntime:raise MobileError("budget_exhausted", "Task exhausted its time or step budget")continueif not result.get("success"):- raise MobileError("agent_failed", result.get("error", "Mobile agent failed"))+ raise MobileError(result.get("code", "agent_failed"), result.get("error", "Mobile agent failed"))event("task_completed", run_id=run_id)return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}finally:@@ -611,19 +611,34 @@ class MobileRuntime:await proc.stdin.drain()proc.stdin.close()+ scope = {"run_id": (self.active or {}).get("run_id"), "workflow_id": (self.active or {}).get("workflow_id"),+ "node_id": (self.active or {}).get("node_id"), "execution_id": (self.active or {}).get("execution_id"),+ "provider": config.get("provider"), "model": config.get("model")}+ event("engine_starting", **scope)+async def read():result = {"success": False, "error": "Mobile worker exited before completing the task"}while line := await proc.stdout.readline():- event = json.loads(line)- if event.get("type") == "completed":- result = {"success": True, "result": event.get("result")}- elif event.get("type") == "failed":- result = {"success": False, "error": event.get("error", "Worker failed")}- elif event.get("type") == "usage" and self.active:- self.active["usage"] = event.get("usage")- elif event.get("type") == "progress" and self.active:- self.active["attempt_steps"] = max(self.active.get("attempt_steps", 0), int(event.get("steps", 0)))+ message = json.loads(line)+ if message.get("type") == "completed":+ result = {"success": True, "result": message.get("result")}+ elif message.get("type") == "failed":+ result = {"success": False, "error": message.get("error", "Worker failed"), "code": message.get("code", "engine_failed")}+ event("engine_failed", failed=True, **scope, engine_trace=message.get("trace", []),+ **{key: message.get(key) for key in ("error_type", "code", "stage", "http_status", "provider_status")})+ elif message.get("type") == "diagnostic":+ event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))+ elif message.get("type") == "ready":+ event("engine_ready", **scope)+ elif message.get("type") == "usage" and self.active:+ self.active["usage"] = message.get("usage")+ elif message.get("type") == "progress" and self.active:+ steps = max(self.active.get("attempt_steps", 0), int(message.get("steps", 0)))+ if steps != self.active.get("attempt_steps", 0):+ event("task_progress", **scope, steps=steps)+ self.active["attempt_steps"] = stepsawait proc.wait()+ event("engine_exited", failed=proc.returncode not in (0, None), **scope, exit_code=proc.returncode)return resultreturn await asyncio.wait_for(read(), max(1, timeout) + 15)diff --git a/server/nodes/mobile/runtime/errors.py b/server/nodes/mobile/runtime/errors.pynew file mode 100644index 00000000..7499eb0c--- /dev/null+++ b/server/nodes/mobile/runtime/errors.py@@ -0,0 +1,42 @@+"""Safe error metadata shared with the isolated engine (standard library only)."""++import traceback+from pathlib import Path+++def describe_error(exc: BaseException, stage: str) -> dict:+ kind = type(exc).__name__+ status = getattr(exc, "status_code", None) or getattr(exc, "code", None)+ if not isinstance(status, int) or isinstance(status, bool) or not 100 <= status <= 599:+ status = None+ provider_status = getattr(exc, "status", None)+ if provider_status not in {"INVALID_ARGUMENT", "NOT_FOUND", "UNAUTHENTICATED", "PERMISSION_DENIED", "RESOURCE_EXHAUSTED", "UNAVAILABLE", "INTERNAL", "FAILED_PRECONDITION"}:+ provider_status = None+ code, message = "engine_failed", f"Android engine failed during {stage} ({kind}). See Android events in Terminal."+ if kind == "HTTPError":+ code, message = "device_connection_failed", "Android device control changed or disconnected. Check the phone connection in Workspace."+ elif status == 401 or kind == "AuthenticationError":+ code, message = "model_authentication_failed", "The model provider rejected its credentials. Reconnect the provider in Settings."+ elif status == 403:+ code, message = "model_permission_denied", "The model provider denied access. Check that this API key/project can use the selected model."+ elif status == 404:+ code, message = "model_not_found", "The selected model was not found by the provider. Choose an available model in the global selector or phone settings."+ elif status == 429 or kind == "RateLimitError":+ code, message = "model_rate_limited", "The model provider's quota or rate limit was reached. Check billing/quota or try again later."+ elif status == 400 and any(term in str(exc).lower() for term in ("api key not valid", "api_key_invalid", "api_key_expired")):+ code, message = "model_authentication_failed", "The model provider rejected its API key. Reconnect the provider in Settings."+ elif status == 400:+ code, message = "model_request_rejected", "The model provider rejected the request. Check that the selected model supports images and tool/structured responses."+ elif status and status >= 500:+ code, message = "model_service_unavailable", "The model provider is unavailable. Try again later."+ elif kind in {"TimeoutError", "APITimeoutError", "ReadTimeout"}:+ code, message = "task_timeout", "The Android task or model request timed out. Check connectivity and the task time budget."+ elif kind == "GraphRecursionError":+ code, message = "step_budget_exhausted", "The Android task exhausted its step budget."+ elif kind in {"APIConnectionError", "ConnectError"}:+ code, message = "model_connection_failed", "Could not reach the model provider. Check network and proxy settings."+ # Never serialize str(exc), response bodies, request headers or locals:+ # provider exceptions can contain API keys, prompts and screenshots.+ return {"error": message + (f" (HTTP {status})" if status else ""), "code": code,+ "error_type": kind, "stage": stage, "http_status": status, "provider_status": provider_status,+ "trace": [f"{Path(frame.filename).name}:{frame.lineno}:{frame.name}" for frame in traceback.extract_tb(exc.__traceback__)[-8:]]}diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.pyindex 5864837a..1cca5941 100644--- a/server/nodes/mobile/runtime/worker.py+++ b/server/nodes/mobile/runtime/worker.py@@ -11,6 +11,12 @@ import urllib.requestimport uuidfrom io import BytesIO+try:+ from .errors import describe_error+except ImportError: # Executed by path in the isolated engine environment.+ from errors import describe_error++STAGE = "bootstrap"WIRE = sys.stdout@@ -127,6 +133,8 @@ class Controller:async def main(config: dict) -> None:+ global STAGE+ emit("diagnostic", stage=STAGE)sys.stdout = sys.stderr # upstream diagnostics can never corrupt the protocolos.environ.update(config.pop("model_env"))os.environ["MOBILE_USE_TELEMETRY_ENABLED"] = "false"@@ -146,7 +154,11 @@ async def main(config: dict) -> None:if isinstance(step, int):emit("progress", steps=step + 1)+ def on_chat_model_start(self, serialized, messages, **kwargs):+ emit("diagnostic", stage="model_request")+def on_llm_end(self, response, **kwargs):+ emit("diagnostic", stage="model_response")output = response.llm_output or {}usage = output.get("token_usage") or output.get("usage") or {}if usage:@@ -181,31 +193,36 @@ async def main(config: dict) -> None:device_height=config["height"],)agent = Agent(config=agent_config)- await agent.init()- emit("ready")+ task_failed = Falsetry:+ STAGE = "engine_initialization"+ emit("diagnostic", stage=STAGE)+ await agent.init()+ emit("ready")+ STAGE = "task_execution"+ emit("diagnostic", stage=STAGE)result = await asyncio.wait_for(agent.run_task(request=TaskRequest(goal=config["prompt"], max_steps=config["max_steps"], record_trace=False)),config["timeout_s"],)emit("completed", result=result)+ except BaseException:+ task_failed = True+ raisefinally:- await agent.clean()+ try:+ await agent.clean()+ except Exception as cleanup_error:+ if not task_failed:+ STAGE = "engine_cleanup"+ raise+ # Preserve the original provider/device failure if cleanup fails.+ emit("diagnostic", stage="cleanup_failed", error_type=type(cleanup_error).__name__)if __name__ == "__main__":try:asyncio.run(main(json.loads(sys.stdin.readline())))except BaseException as exc:- # Provider errors may echo credentials; expose only an actionable category.- # Prompts/screenshots in upstream stderr are not persisted.- kind = type(exc).__name__- message = {- "AuthenticationError": "The selected model rejected its credentials.",- "RateLimitError": "The selected model is rate limited. Try again later.",- "TimeoutError": "The mobile task exceeded its active time budget.",- "GraphRecursionError": "The mobile task exhausted its step budget.",- "HTTPError": "Device control changed or the device connection failed.",- }.get(kind, f"Mobile engine failed ({kind}).")- emit("failed", error=message)+ emit("failed", **describe_error(exc, STAGE))sys.exit(1)diff --git a/server/tests/test_mobile_diagnostics.py b/server/tests/test_mobile_diagnostics.pynew file mode 100644index 00000000..6c19c487--- /dev/null+++ b/server/tests/test_mobile_diagnostics.py@@ -0,0 +1,69 @@+"""Actionable provider failures without leaking request/response contents."""++from unittest.mock import Mock++import pytest++from nodes.mobile.runtime.errors import describe_error+from nodes.mobile import _diagnostics++[email protected]("status,code", [(400, "model_request_rejected"), (401, "model_authentication_failed"),+ (403, "model_permission_denied"), (404, "model_not_found"),+ (429, "model_rate_limited"), (503, "model_service_unavailable")])+def test_client_error_retains_status_without_content(status, code):+ class ClientError(Exception):+ pass+ error = ClientError("secret-key private prompt base64-screenshot")+ error.code = status+ error.status = "NOT_FOUND" if status == 404 else "unsafe response content"+ result = describe_error(error, "task_execution")+ assert result["code"] == code+ assert result["http_status"] == status+ assert result["error_type"] == "ClientError"+ assert "secret" not in str(result)+ assert "prompt" not in str(result)+ assert "screenshot" not in str(result)+ assert "unsafe" not in str(result)+++def test_invalid_google_key_is_not_reported_as_model_capability_error():+ error = RuntimeError("API key not valid. secret-key")+ error.code = 400+ assert describe_error(error, "engine_initialization")["code"] == "model_authentication_failed"+++def test_mobile_events_reach_terminal_pipeline_with_only_safe_fields(monkeypatch, tmp_path):+ import core.logging+ logger = Mock()+ monkeypatch.setattr(core.logging, "get_logger", lambda name: logger)+ monkeypatch.setattr(_diagnostics, "mobile_root", lambda: tmp_path)+ try:+ _diagnostics.event("engine_failed", failed=True, http_status=404, model="selected-model",+ execution_id="execution-2", prompt="private", api_key="secret")+ logger.error.assert_called_once()+ args, fields = logger.error.call_args+ assert args == ("Android: engine_failed",)+ assert fields["http_status"] == 404+ assert fields["execution_id"] == "execution-2"+ assert "prompt" not in fields and "api_key" not in fields+ assert '"http_status": 404' in (tmp_path / "mobile.log").read_text()+ finally:+ for handler in list(_diagnostics._logger.handlers):+ handler.close()+ _diagnostics._logger.removeHandler(handler)+++def test_terminal_logger_failure_does_not_lose_file_diagnostics(monkeypatch, tmp_path):+ import core.logging+ logger = Mock()+ logger.info.side_effect = RuntimeError("logger unavailable")+ monkeypatch.setattr(core.logging, "get_logger", lambda name: logger)+ monkeypatch.setattr(_diagnostics, "mobile_root", lambda: tmp_path)+ try:+ _diagnostics.event("phone_ready")+ assert "phone_ready" in (tmp_path / "mobile.log").read_text()+ finally:+ for handler in list(_diagnostics._logger.handlers):+ handler.close()+ _diagnostics._logger.removeHandler(handler)diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.pyindex 5827beec..546a6128 100644--- a/server/tests/test_mobile_runtime.py+++ b/server/tests/test_mobile_runtime.py@@ -140,6 +140,7 @@ async def test_takeover_stops_worker_then_resumes_with_fresh_capability(runtime,human = await runtime.takeover("browser")assert first.terminated.is_set()assert not runtime.capabilities+assert human.owner == "viewer:browser"assert runtime.active["status"] == "awaiting_user"runtime.release("browser", resume=True)@@ -268,3 +269,32 @@ async def test_stop_preserves_epoch_monotonicity_and_clears_viewer(runtime):assert not runtime.resume_event.is_set()next_lease = await runtime.control.claim("viewer:browser")assert next_lease.epoch > lease.epoch+++async def test_worker_failure_reaches_diagnostics_and_keeps_provider_code(runtime, monkeypatch):+ import nodes.mobile._runtime as module+ from unittest.mock import Mock+ from nodes.mobile._control import MobileError++ proc = FakeProcess()+ proc.stdout.feed_data(b'{"type":"diagnostic","stage":"model_request"}\n')+ proc.stdout.feed_data(b'{"type":"failed","error":"Model not found (HTTP 404)","code":"model_not_found","http_status":404,"error_type":"ClientError","stage":"task_execution"}\n')+ proc.returncode = 1+ proc.stdout.feed_eof()+ proc.terminated.set()+ async def spawn(*_args, **_kwargs):+ return proc+ monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)+ log = Mock()+ monkeypatch.setattr(module, "event", log)+ args = run_args()+ args.update(execution_id="execution-2", model={"provider": "google", "model": "selected-model"})+ with pytest.raises(MobileError) as failure:+ await runtime.run(**args)+ assert failure.value.code == "model_not_found"+ recorded = [call.kwargs for call in log.call_args_list if call.args == ("engine_failed",)]+ assert len(recorded) == 1+ assert recorded[0]["http_status"] == 404+ assert recorded[0]["execution_id"] == "execution-2"+ assert recorded[0]["node_id"] == "agent"+ assert recorded[0]["model"] == "selected-model"