Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Lazy mobile broker. Owns one persistent emulator and serializes all device I/O."""
from __future__ import annotations
import asyncio
import contextlib
import json
import os
import secrets
import socket
import time
from pathlib import Path
from services.process_environment import without_onepassword_environment
from ._control import DeviceControl, Lease, MobileError
from ._install import AVD_NAME, DEVICE_NAME, VIDEO_MAX_FPS, install_engine, create_device, engine_ready, sdk_tool, sdk_root
from ._paths import mobile_root, runtime_python
from ._process import command, hidden_options
from ._diagnostics import event, recent
from ._emulator import DeviceLock, recover_emulator, boot_error, owned_children, stop_children
from .runtime.progress import plan_summary, summary
READS = frozenset({"geometry", "observe", "screenshot", "date", "packages", "foreground"})
#: What the Workspace says about the phone, above its frame.
DEVICE_LINE = f"{DEVICE_NAME} · local emulator · up to {VIDEO_MAX_FPS} fps"
ACTIONS = frozenset({"tap", "swipe", "touch", "text", "erase", "key", "launch", "terminate", "url", "rotate"})
#: The Workspace's step log (services/workspace_steps.py): what a phone task
#: did, never what it typed. ``touch`` (a finger's down, move, up) is part of
#: a gesture, not a step of its own.
TASK_STEPS = {
"tap": "Tapped the screen",
"swipe": "Swiped",
"text": "Typed on the phone",
"erase": "Deleted text",
"launch": "Opened an app",
"terminate": "Closed an app",
"url": "Opened a link",
"rotate": "Turned the phone",
}
KEY_STEPS = {
"home": "Went to the home screen",
"back": "Went back",
"enter": "Pressed Enter",
"recent": "Opened recent apps",
"power": "Pressed the power button",
}
class MobileRuntime:
def __init__(self):
self.control = DeviceControl()
self.process: asyncio.subprocess.Process | None = None
self.driver: asyncio.subprocess.Process | None = None
self.worker: asyncio.subprocess.Process | None = None
self.serial: str | None = None
self.geometry: dict | None = None
self.device_lock = None
self.start_error = ""
self.driver_lock = asyncio.Lock()
self.lifecycle_lock = asyncio.Lock()
self.task_lock = asyncio.Lock()
self.setup_task: asyncio.Task | None = None
self.setup_state = "idle"
self.setup_error = ""
self.setup_progress: dict = {}
self._setup_loaded = False
self._setup_saved = 0.0
self.active: dict | None = None
self.last_task: dict | None = None
self.queue: list[dict] = []
# Execution handles are deliberately separate from public task snapshots.
# Completion means run() has drained its worker and admitted device I/O.
self._run_tasks: dict[str, tuple[dict, asyncio.Task, asyncio.Event]] = {}
self._worker_stop_tasks: set[asyncio.Task] = set()
self._reset_runs: set[str] = set()
self._draining_runs: set[str] = set()
self._retired_generations: dict[tuple[str, str], int] = {}
self.capabilities: dict[str, tuple[str, Lease]] = {}
self.resume_event = asyncio.Event()
self.viewer: str | None = None
self.video_viewer: str | None = None
self.stopping = False
self.broker_runner = None
self.broker_url: str | None = None
self.broker_lock = asyncio.Lock()
async def ensure_broker(self) -> str:
"""Private engine IPC. Capabilities never appear in URLs or public responses."""
async with self.broker_lock:
if self.broker_url:
return self.broker_url
from aiohttp import web
async def dispatch(request):
try:
capability = request.headers.get("Authorization", "").removeprefix("Bearer ")
grant = self.capabilities.get(capability)
if not grant:
return web.json_response({"success": False, "error": "Task capability expired"}, status=403)
run_id, lease = grant
data = await request.json()
if not isinstance(data, dict) or data.get("epoch") != lease.epoch:
raise MobileError("stale_lease", "Task control changed")
operation = data.get("operation")
parameters = data.get("parameters", {})
if not isinstance(parameters, dict):
raise MobileError("invalid_action", "Expected command parameters")
if operation in READS:
# Reads are fenced too: a paused task cannot keep observing passwords.
result = await self.control.perform(
lease, data.get("operation_id"), lambda: self.driver_call(operation, parameters), cache_result=False
)
else:
result = await self.input(lease, operation, parameters, data.get("operation_id"), data.get("geometry"))
await self._record_task_step(run_id, operation, parameters)
return web.json_response({"success": True, "result": result})
except MobileError as exc:
return web.json_response({"success": False, "error": str(exc), "code": exc.code}, status=409)
except (ValueError, TypeError):
return web.json_response({"success": False, "error": "Invalid command"}, status=400)
except Exception:
return web.json_response({"success": False, "error": "Device connection failed"}, status=503)
app = web.Application(client_max_size=65536)
app.router.add_post("/control", dispatch)
runner = web.AppRunner(app, access_log=None, shutdown_timeout=2)
await runner.setup()
sock = socket.socket()
try:
sock.bind(("127.0.0.1", 0))
sock.setblocking(False)
port = sock.getsockname()[1]
await web.SockSite(runner, sock).start()
except BaseException:
sock.close()
await runner.cleanup()
raise
self.broker_runner = runner
self.broker_url = f"http://127.0.0.1:{port}/control"
return self.broker_url
def snapshot(self) -> dict:
self._load_setup()
return {
"running": self.process is not None and self.process.returncode is None
and self.driver is not None and self.driver.returncode is None and self.geometry is not None,
"starting": self.lifecycle_lock.locked() and self.geometry is None and not self.stopping,
"serial": self.serial,
"geometry": self.geometry,
"device": DEVICE_LINE,
"setup": self.setup_state,
"setup_error": self.setup_error,
"start_error": self.start_error,
"setup_progress": self.setup_progress,
"diagnostics": recent(),
"active": {k: v for k, v in (self.active or {}).items() if k not in {"prompt"}},
"last_task": self.last_task,
"queue": [{k: v for k, v in q.items() if k != "prompt"} for q in self.queue],
**self.control.snapshot(),
}
def _load_setup(self) -> None:
if self._setup_loaded:
return
self._setup_loaded = True
try:
record = json.loads((mobile_root() / "setup-status.json").read_text(encoding="utf-8"))
if not isinstance(record, dict):
return
if record.get("state") not in {"installing_engine", "installing_device", "ready", "error", "interrupted"}:
return
if not isinstance(record.get("progress"), dict):
return
progress = record["progress"]
events = progress.get("events", [])
if not isinstance(events, list) or any(
not isinstance(event, dict) or not isinstance(event.get("message"), str)
or not isinstance(event.get("at"), (int, float)) for event in events
):
return
progress["events"] = events[-80:]
self.setup_state = record["state"]
self.setup_error = str(record.get("error", ""))[:1500]
self.setup_progress = record["progress"]
if self.setup_state.startswith("installing_"):
self.setup_state = "interrupted"
self.setup_error = "Setup was interrupted by a backend restart. Retry setup to reuse completed components."
self.setup_progress["finished_at"] = time.time()
self._setup_update(self.setup_error, force=True)
except (OSError, ValueError, TypeError):
pass
def _setup_update(self, message: str, *, force: bool = False) -> None:
now = time.time()
message = message.strip()[-1500:]
if not message:
return
self.setup_progress.update(message=message, updated_at=now)
events = self.setup_progress.setdefault("events", [])
if not events or events[-1]["message"] != message:
events.append({"at": now, "message": message})
del events[:-80]
if not force and time.monotonic() - self._setup_saved < 1:
return
try:
root = mobile_root()
root.mkdir(parents=True, exist_ok=True)
temp = root / "setup-status.json.tmp"
temp.write_text(json.dumps({"state": self.setup_state, "error": self.setup_error, "progress": self.setup_progress}), encoding="utf-8")
temp.replace(root / "setup-status.json")
self._setup_saved = time.monotonic()
except OSError:
# Disk failures must not erase the live status or abort installation.
pass
def setup(self, accepted: bool) -> None:
if not accepted:
raise MobileError("license_required", "Read and accept the SDK license before setup")
if self.setup_task and not self.setup_task.done():
return
if self.stopping or self.lifecycle_lock.locked() or self.active or self.serial:
raise MobileError("device_busy", "Stop the device before changing its runtime")
self._setup_loaded = True
self.setup_error = ""
self.setup_state = "installing_engine"
self.setup_progress = {"started_at": time.time(), "events": []}
self._setup_update("Checking the mobile agent runtime", force=True)
event("setup_started")
async def run():
try:
await install_engine(progress=self._setup_update)
self.setup_state = "installing_device"
self._setup_update("Preparing Android tools and the shared phone", force=True)
await create_device(licenses_accepted=True, progress=self._setup_update)
self.setup_state = "ready"
self.setup_progress["finished_at"] = time.time()
self._setup_update("Setup complete. Start the phone to connect it.", force=True)
event("setup_completed")
except asyncio.CancelledError:
self.setup_state = "interrupted"
self.setup_error = "Setup stopped because the backend shut down. Retry setup to reuse completed components."
self.setup_progress["finished_at"] = time.time()
self._setup_update(self.setup_error, force=True)
event("setup_interrupted", failed=True)
raise
except Exception as exc:
self.setup_state, self.setup_error = "error", str(exc)[-1500:] or f"Setup failed ({type(exc).__name__}). Retry setup."
self.setup_progress["finished_at"] = time.time()
self._setup_update(self.setup_error, force=True)
event("setup_failed", failed=True, error_type=type(exc).__name__)
self.setup_task = asyncio.create_task(run())
async def start(self) -> dict:
async with self.lifecycle_lock:
if self.stopping or (self.setup_task and not self.setup_task.done()):
raise MobileError("device_busy", "Wait for setup or shutdown to finish")
self.start_error = ""
try:
return await self._start_locked()
except BaseException as exc:
self.start_error = str(exc) if isinstance(exc, MobileError) else "Phone startup was interrupted. Try Start phone again."
event("phone_start_failed", failed=True, error_type=type(exc).__name__, code=getattr(exc, "code", "startup_failed"))
await self._stop_owned()
raise
async def _start_locked(self) -> dict:
if self.stopping or (self.setup_task and not self.setup_task.done()):
raise MobileError("device_busy", "Wait for setup or shutdown to finish")
if self.process and self.process.returncode is None:
if not self.driver or self.driver.returncode is not None or self.geometry is None:
await self._stop_driver()
await self._start_driver()
self.control.revoke()
self.control.state = "idle"
return self.snapshot()
if self.process is not None:
await self._stop_driver()
self.process = None
self.serial = None
emulator, adb = sdk_tool("emulator"), sdk_tool("adb")
if not emulator or not adb or not engine_ready() or not (mobile_root() / "resource.json").is_file():
raise MobileError("setup_required", "Complete Mobile setup before starting the device")
root = mobile_root()
root.mkdir(parents=True, exist_ok=True)
if self.device_lock is None:
self.device_lock = DeviceLock(root)
recovered = await recover_emulator(adb, emulator, root / "avd" / f"{AVD_NAME}.avd")
if recovered:
self.process, self.serial = recovered
event("phone_reconnected", serial=self.serial)
await self._start_driver()
self.control.revoke()
self.control.state = "idle"
return self.snapshot()
port = None
for candidate in range(5560, 5680, 2):
held = []
try:
for n in (candidate, candidate + 1):
test = socket.socket()
held.append(test)
test.bind(("127.0.0.1", n))
port = candidate
break
except OSError:
pass
finally:
for test in held:
test.close()
if port is None:
raise MobileError("ports_busy", "No emulator port pair is free")
root = mobile_root()
event("phone_starting")
root.mkdir(parents=True, exist_ok=True)
log_path = root / "emulator.log"
if log_path.exists() and log_path.stat().st_size > 2 * 1024 * 1024:
log_path.replace(root / "emulator.previous.log")
log = log_path.open("ab")
log_offset = log.tell()
env = {**os.environ, "ANDROID_AVD_HOME": str(root / "avd"), "ANDROID_SDK_ROOT": str(sdk_root())}
try:
self.process = await asyncio.create_subprocess_exec(
str(emulator),
"-avd",
AVD_NAME,
"-port",
str(port),
"-no-window",
"-no-audio",
stdout=log,
stderr=log,
env=without_onepassword_environment(env),
**hidden_options(),
)
finally:
log.close()
self.serial = f"emulator-{port}"
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
if self.process.returncode is not None:
raise boot_error(log_path, log_offset, self.process.returncode)
try:
boot = await command([str(adb), "-s", self.serial, "shell", "getprop", "sys.boot_completed"], check=False, timeout=10)
except TimeoutError:
event("phone_boot_probe_timeout", serial=self.serial)
continue
if boot.strip() == "1":
break
await asyncio.sleep(1)
else:
raise MobileError("boot_timeout", "Android did not finish booting within three minutes")
await self._start_driver()
event("phone_ready", serial=self.serial)
self.control.revoke()
self.control.state = "idle"
return self.snapshot()
def driver_env(self) -> dict:
allowed = {
"PATH",
"SYSTEMROOT",
"WINDIR",
"COMSPEC",
"TEMP",
"TMP",
"TMPDIR",
"APPDATA",
"LOCALAPPDATA",
"PROGRAMFILES",
"PROGRAMFILES(X86)",
"PROGRAMDATA",
"USERPROFILE",
"HOME",
"LANG",
"LC_ALL",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
"REQUESTS_CA_BUNDLE",
"HTTPS_PROXY",
"HTTP_PROXY",
"NO_PROXY",
}
env = {k: v for k, v in os.environ.items() if k.upper() in allowed}
env.update(
MOBILE_USE_TELEMETRY_ENABLED="false",
PYTHON_DOTENV_DISABLED="1",
PYTHONUTF8="1",
LANGCHAIN_TRACING_V2="false",
LANGSMITH_TRACING="false",
PATH=str(sdk_root() / "platform-tools") + os.pathsep + env.get("PATH", ""),
)
for name in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"):
env.pop(name, None)
return without_onepassword_environment(env)
async def _start_driver(self):
event("driver_starting")
# Initialization consumes the first protocol reply. It must own the
# same lock as commands so polling cannot read that reply concurrently.
async with self.driver_lock:
try:
await self._initialize_driver()
event("driver_ready")
except BaseException:
await self._stop_driver()
raise
async def _initialize_driver(self):
entry = Path(__file__).with_name("runtime") / "device_server.py"
self.driver = await asyncio.create_subprocess_exec(
str(runtime_python()),
"-u",
str(entry),
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
limit=8 * 1024 * 1024,
env=self.driver_env(),
cwd=str(mobile_root()),
**hidden_options(),
)
self.driver.stdin.write((json.dumps({"serial": self.serial, "source": str(mobile_root() / "engine")}) + "\n").encode())
await self.driver.stdin.drain()
data = await asyncio.wait_for(self.driver.stdout.readline(), 60)
if not data:
raise MobileError("driver_failed", "Android automation did not initialize")
reply = json.loads(data)
if not reply.get("success"):
raise MobileError("driver_failed", reply.get("error", "Android automation did not initialize"))
self.geometry = reply["result"]
async def driver_call(self, operation: str, parameters: dict | None = None):
async with self.driver_lock:
if not self.driver or self.driver.returncode is not None:
raise MobileError("device_offline", "Start the device before using it")
started = time.monotonic()
try:
self.driver.stdin.write((json.dumps({"operation": operation, "parameters": parameters or {}}) + "\n").encode())
await self.driver.stdin.drain()
line = await asyncio.wait_for(self.driver.stdout.readline(), 30)
if not line:
raise RuntimeError("Device driver disconnected")
result = json.loads(line)
except BaseException as exc:
event("driver_connection_failed", failed=True, operation=operation, error_type=type(exc).__name__)
self.control.state = "recovering"
await self._stop_driver()
raise
if not result.get("success"):
event("device_action_failed", failed=True, operation=operation, code=result.get("code", "device_action_failed"))
raise MobileError(result.get("code", "device_action_failed"), result.get("error", "Device command failed"))
if operation not in READS:
event("device_action_completed", operation=operation, duration_ms=round((time.monotonic() - started) * 1000))
elif operation != "geometry":
event("device_read_completed", operation=operation, duration_ms=round((time.monotonic() - started) * 1000))
if operation == "geometry":
self.geometry = result["result"]
elif operation == "observe":
self.geometry = result["result"]["geometry"]
return result["result"]
async def input(self, lease: Lease, operation: str, parameters: dict, operation_id: str, geometry: dict | None = None):
if operation not in ACTIONS:
raise MobileError("unsupported_action", "Unsupported device command")
if operation in {"tap", "swipe", "touch"}:
parameters = {**parameters, "geometry": geometry}
return await self.control.perform(lease, operation_id, lambda: self.driver_call(operation, parameters))
async def install_apk(self, path: Path, lease: Lease, operation_id: str):
if not lease.owner.startswith("viewer:"):
raise MobileError("not_controller", "Take manual control before installing an APK")
if not self.serial or not self.process or self.process.returncode is not None:
raise MobileError("device_offline", "Start the shared device first")
uploads = (mobile_root() / "uploads").resolve()
path = path.resolve()
if not path.is_relative_to(uploads) or path.suffix.lower() != ".apk" or not path.is_file():
raise MobileError("invalid_action", "Expected an uploaded APK")
async def install():
return await command([str(sdk_tool("adb")), "-s", self.serial, "install", "-r", str(path)], timeout=180)
return await self.control.perform(lease, operation_id, install)
async def takeover(self, viewer: str) -> Lease:
if self.stopping or self.lifecycle_lock.locked():
raise MobileError("device_busy", "Wait for the device to finish starting or stopping")
self.control.revoke()
self.capabilities.clear()
if self.active:
self.active["status"] = "awaiting_user"
await self._stop_worker()
lease = await self.control.claim("viewer:" + viewer, takeover=True)
self.viewer = viewer
event("manual_control_started")
await self._publish_progress("Waiting for you")
return lease
def release(self, viewer: str, *, resume: bool = False):
if self.viewer != viewer:
raise MobileError("not_controller", "Another viewer controls the device")
self.control.revoke("viewer:" + viewer)
self.viewer = None
if resume:
self.resume_event.set()
async def _record_task_step(self, run_id: str, operation: str, parameters: dict) -> None:
"""A phone task's finished action, in its workflow's step log."""
entry = self.active
if not entry or entry.get("run_id") != run_id:
return
text = KEY_STEPS.get(str(parameters.get("key"))) if operation == "key" else TASK_STEPS.get(operation)
if text:
await record_phone_step(entry.get("workflow_id"), entry.get("node_id"), text)
def resume_waiting(self, workflow_id: str) -> None:
"""Let ``workflow_id``'s AI task that waits for the owner go on,
without holding the phone: Hand back after the view that held it went
away. Refused while someone uses the phone. Nothing happens when no
task of that workflow waits: the phone is shared by every employee."""
if self.viewer is not None:
raise MobileError("not_controller", "Someone is using the phone. Finish using it first.")
if self.active and self.active.get("status") == "awaiting_user" and self.active.get("workflow_id") == workflow_id:
self.resume_event.set()
async def cancel(self, workflow_id: str, node_id: str, *, run_id: str | None = None):
def matches(entry):
return entry["workflow_id"] == workflow_id and entry["node_id"] == node_id and (run_id is None or entry["run_id"] == run_id)
await self._cancel_matching(matches)
async def reset_execution_state(self, *, workflow_id: str, node_id: str, generation: int) -> dict:
if generation < 0:
raise MobileError("invalid_generation", "Reset generation cannot be negative")
# Retire admission before the first await so a late graph activity cannot
# enqueue while reset is draining a previously admitted task.
if generation > 0:
key = (workflow_id, node_id)
self._retired_generations[key] = max(generation, self._retired_generations.get(key, 0))
def matches(entry):
return (
entry["workflow_id"] == workflow_id and entry["node_id"] == node_id
# A phone-only reset must not cancel a graph generation admitted
# concurrently by Start. Zero identifies direct Workspace work.
and (entry.get("generation", 0) == 0 if generation == 0 else entry.get("generation", 0) <= generation)
)
await self._cancel_matching(matches, clear=True)
return {"reset": True}
async def _cancel_matching(self, matches, *, clear: bool = False) -> None:
records = [record for record in self._run_tasks.values() if matches(record[0])]
entries = [entry for entry in self.queue if matches(entry)]
active = self.active if self.active and matches(self.active) else None
if active is not None:
entries.append(active)
run_ids = {entry["run_id"] for entry in entries}
for entry in entries:
entry["status"] = "cancelled"
if clear:
self._reset_runs.update(run_ids)
if self.last_task and matches(self.last_task):
self.last_task = None
# Revoke before cancellation/draining. Keep manual and unrelated leases.
for run_id in run_ids:
self.control.revoke("run:" + run_id)
for capability, (run_id, _) in list(self.capabilities.items()):
if run_id in run_ids:
self.capabilities.pop(capability, None)
for entry, task, _ in records:
# A second reset must not interrupt a first cancellation's finally.
if not task.done() and not task.cancelling() and entry["run_id"] not in self._draining_runs:
task.cancel()
if records:
await asyncio.gather(*(done.wait() for _, _, done in records))
if active is not None and self.active is active:
raise MobileError("reset_failed", "Mobile task cleanup did not finish; retry reset")
elif active is not None:
# Also handle state restored/manually owned without a live run handle.
await self._stop_worker()
async with self.control.lock:
pass
if self.active is active:
self.active = None
if clear:
if self.last_task and matches(self.last_task):
self.last_task = None
self._reset_runs.difference_update(run_ids)
async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str, execution_id: str | None = None, generation: int = 0) -> dict:
if generation > 0 and generation <= self._retired_generations.get((workflow_id, node_id), 0):
raise MobileError("stale_generation", "This mobile task belongs to a reset workflow generation")
if len(self.queue) >= 32:
raise MobileError("queue_full", "The shared device queue is full")
if any(q["run_id"] == run_id for q in self.queue) or (self.active and self.active["run_id"] == run_id):
raise MobileError("duplicate_run", "This task is already admitted")
entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "execution_id": execution_id, "generation": generation, "status": "queued", "steps": 0, "max_steps": params["max_steps"]}
done = asyncio.Event()
self._run_tasks[run_id] = (entry, asyncio.current_task(), done)
self.queue.append(entry)
event("task_queued", run_id=run_id, workflow_id=workflow_id, node_id=node_id)
try:
async with self.task_lock:
self.queue.remove(entry)
if entry["status"] == "cancelled":
raise asyncio.CancelledError()
if self.stopping:
raise MobileError("stopping", "Mobile runtime is shutting down")
if not self.serial:
raise MobileError("device_offline", "Start the shared device in Workspace first")
self.active = entry
entry.update(started_at=time.time(), provider=model.get("provider"), model=model.get("model"))
active_remaining = params["timeout_s"]
steps_remaining = params["max_steps"]
attempts = 0
try:
await self._publish_progress("Preparing phone")
while True:
if entry["status"] == "cancelled":
raise asyncio.CancelledError()
if self.control.owner and self.control.owner.startswith("viewer:"):
entry["status"] = "awaiting_user"
if entry["status"] == "awaiting_user":
await self._publish_progress("Waiting for you")
await asyncio.wait_for(self.resume_event.wait(), 1800)
self.resume_event.clear()
if entry["status"] == "cancelled":
raise asyncio.CancelledError()
lease = await self.control.claim("run:" + run_id)
entry["status"] = "starting"
self.resume_event.clear()
capability = secrets.token_urlsafe(32)
self.capabilities[capability] = (run_id, lease)
geometry = await self.driver_call("geometry")
if entry["status"] == "cancelled":
raise asyncio.CancelledError()
if entry["status"] == "awaiting_user" or lease.owner != self.control.owner or lease.epoch != self.control.epoch:
self.capabilities.pop(capability, None)
entry["status"] = "awaiting_user"
continue
entry["status"] = "running"
event("task_running", run_id=run_id)
config = {
**model,
"source": str(mobile_root() / "engine"),
"serial": self.serial,
"platform": "android",
"width": geometry["width"],
"height": geometry["height"],
"epoch": lease.epoch,
"capability": capability,
"broker_url": broker_url,
"prompt": params["prompt"]
+ (
"\nYou are resuming after the user took manual control. Inspect the current screen first. Continue the goal from the current state; do not repeat actions already completed."
if attempts
else ""
),
"max_steps": steps_remaining,
"timeout_s": active_remaining,
}
attempts += 1
entry["attempt_steps"] = 0
entry["step_offset"] = params["max_steps"] - steps_remaining
started = time.monotonic()
try:
try:
result = await self._run_worker(config, active_remaining)
except Exception:
if entry["status"] not in {"awaiting_user", "cancelled"}:
raise
result = {"success": False}
finally:
active_remaining -= time.monotonic() - started
steps_remaining -= max(1, entry.get("attempt_steps", 0))
self.capabilities.pop(capability, None)
await self._stop_worker()
if entry["status"] == "cancelled":
raise asyncio.CancelledError()
if entry["status"] == "awaiting_user":
if active_remaining <= 0 or steps_remaining <= 0:
raise MobileError("budget_exhausted", "Task exhausted its time or step budget")
continue
if not result.get("success"):
raise MobileError(result.get("code", "agent_failed"), result.get("error", "Mobile agent failed"))
entry["status"] = "completed"
event("task_completed", run_id=run_id)
return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}
finally:
self._draining_runs.add(run_id)
for capability, (owner_run, _) in list(self.capabilities.items()):
if owner_run == run_id:
self.capabilities.pop(capability, None)
self.control.revoke("run:" + run_id)
await self._stop_worker()
# Wait for any previously admitted broker write before reporting completion.
async with self.control.lock:
pass
entry["finished_at"] = time.time()
if run_id not in self._reset_runs:
self.last_task = entry
self.active = None
except asyncio.CancelledError:
entry["status"] = "cancelled"
event("task_cancelled", run_id=run_id)
raise
except Exception as exc:
entry["status"] = "failed"
entry["error_code"] = getattr(exc, "code", "timeout" if isinstance(exc, asyncio.TimeoutError) else "task_failed")
event("task_failed", failed=True, run_id=run_id, error_type=type(exc).__name__, code=getattr(exc, "code", "task_failed"))
raise
finally:
if entry in self.queue:
self.queue.remove(entry)
self._run_tasks.pop(run_id, None)
self._draining_runs.discard(run_id)
done.set()
async def _publish_progress(self, phase: str, *, record: bool = True) -> None:
if not self.active:
return
entry = self.active
now = time.time()
if entry.get("phase") != phase:
entry["phase_started_at"] = now
entry["phase"] = phase
entry["updated_at"] = now
history = entry.setdefault("activity", [])
if record and (not history or history[-1]["message"] != phase):
history.append({"at": now, "message": phase, "step": entry.get("steps", 0)})
del history[:-40]
try:
from services.status_broadcaster import get_status_broadcaster
await get_status_broadcaster().update_node_status(
entry["node_id"], "waiting" if entry["status"] == "awaiting_user" else "executing",
{"iteration": entry.get("steps", 0), "max_iterations": entry.get("max_steps", 0),
"phase": phase, "execution_id": entry.get("execution_id"), "run_id": entry["run_id"]},
workflow_id=entry["workflow_id"],
)
except Exception as exc:
event("progress_delivery_failed", failed=True, error_type=type(exc).__name__)
async def _publish_agent_progress(self, message: dict) -> None:
"""Detailed summaries stay in the owner-only snapshot, never diagnostic logs."""
if not self.active or message.get("kind") not in {"plan", "action"}:
return
entry = self.active
now = time.time()
kind = message["kind"]
item = {"at": now, "step": entry.get("steps", 0), "kind": kind,
"message": summary(message.get("message"), 400), "detail": summary(message.get("detail"))}
if not item["message"]:
return
history = entry.setdefault("agent_activity", [])
if kind == "plan":
plan = plan_summary(message.get("plan"))
if plan is None:
return
entry["plan"] = plan
entry["current_goal"] = next((goal["description"] for goal in plan if goal["status"] == "pending"), None)
history.append(item)
phase = "Updating task plan"
else:
action_id = summary(message.get("action_id"), 100)
state = message.get("state")
if not action_id or state not in {"started", "completed", "failed", "returned"}:
return
item.update(action_id=action_id, state=state, outcome=summary(message.get("outcome")))
duration = message.get("duration_ms")
if isinstance(duration, (int, float)) and 0 <= duration <= 86_400_000:
item["duration_ms"] = round(duration)
previous = next((row for row in reversed(history) if row.get("action_id") == action_id), None)
if previous:
# Keep one row and the original engine step for each actual tool call.
item.update(at=previous["at"], step=previous["step"], finished_at=now)
previous.update(item)
item = previous
else:
history.append(item)
if state == "started" or not entry.get("current_action") or entry["current_action"].get("action_id") == action_id:
entry["current_action"] = item
phase = {"started": "Executing phone action", "completed": "Phone action completed",
"failed": "Phone action failed", "returned": "Checking action outcome"}[state]
del history[:-40]
await self._publish_progress(phase, record=False)
def _finish_pending_actions(self) -> None:
if not self.active:
return
for item in self.active.get("agent_activity", []):
if item.get("state") == "started":
item.update(state="returned", finished_at=time.time(),
outcome="Action interrupted before the tool confirmed its outcome.")
async def _run_worker(self, config: dict, timeout: float) -> dict:
entry = Path(__file__).with_name("runtime") / "worker.py"
self.worker = await asyncio.create_subprocess_exec(
str(runtime_python()),
"-u",
str(entry),
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
env=self.driver_env(),
cwd=str(mobile_root()),
limit=2 * 1024 * 1024,
**hidden_options(),
)
proc = self.worker
if config["capability"] not in self.capabilities or not self.active or self.active["status"] != "running":
await self._stop_worker()
return {"success": False, "error": "Task admission revoked"}
self.active.update(plan=[], current_goal=None, current_action=None)
proc.stdin.write((json.dumps(config) + "\n").encode())
await proc.stdin.drain()
proc.stdin.close()
scope = {"run_id": (self.active or {}).get("run_id"), "workflow_id": (self.active or {}).get("workflow_id"),
"node_id": (self.active or {}).get("node_id"), "execution_id": (self.active or {}).get("execution_id"),
"provider": config.get("provider"), "model": config.get("model")}
if config.get("provider") == "google":
scope["model_backend"] = "vertex_express" if config.get("model_env", {}).get("GOOGLE_GENAI_USE_VERTEXAI") == "true" else "gemini_developer"
event("engine_starting", **scope)
await self._publish_progress("Starting Android task")
pending_models: dict[str, float] = {}
async def read():
result = {"success": False, "error": "Mobile worker exited before completing the task"}
while line := await proc.stdout.readline():
message = json.loads(line)
if message.get("type") == "completed":
result = {"success": True, "result": message.get("result")}
elif message.get("type") == "failed":
result = {"success": False, "error": message.get("error", "Worker failed"), "code": message.get("code", "engine_failed")}
event("engine_failed", failed=True, **scope, engine_trace=message.get("trace", []),
**{key: message.get(key) for key in ("error_type", "code", "stage", "http_status", "provider_status")})
elif message.get("type") == "activity":
labels = {"observe": "Reading screen and accessibility tree", "geometry": "Checking screen size",
"tap": "Tapping screen", "swipe": "Scrolling", "text": "Typing text",
"key": "Pressing navigation key", "launch": "Opening app", "terminate": "Closing app",
"url": "Opening link", "erase": "Erasing text", "packages": "Listing apps",
"foreground": "Checking current app", "date": "Checking device time"}
label = labels.get(message.get("operation"))
state = message.get("state")
if label and state in {"started", "completed", "failed"}:
duration = message.get("duration_ms")
suffix = f" ({duration / 1000:.1f}s)" if isinstance(duration, (int, float)) and duration >= 0 else ""
await self._publish_progress(label if state == "started" else f"{label}: {state}{suffix}")
elif message.get("type") == "agent_update":
await self._publish_agent_progress(message)
elif message.get("type") == "diagnostic":
event("engine_stage", **scope, stage=message.get("stage"), error_type=message.get("error_type"))
phase = {"model_request": "Waiting for model", "model_response": "Model responded",
"engine_initialization": "Connecting engine", "task_execution": "Preparing task plan",
"engine_cleanup": "Finishing engine cleanup"}.get(message.get("stage"))
stage = message.get("stage")
request_id = str(message.get("request_id", "model"))
if stage == "model_request":
pending_models[request_id] = time.monotonic()
role = {"planner": "planning", "orchestrator": "coordinating next step",
"contextor": "understanding the screen", "cortex": "choosing an action",
"executor": "preparing an action", "outputter": "preparing the result",
"hopper": "selecting an app"}.get(message.get("role"))
if role:
phase = f"Waiting for model: {role}"
elif stage in {"model_response", "model_error"}:
started = pending_models.pop(request_id, None)
phase = "Model request failed" if stage == "model_error" else "Model responded"
if started is not None:
phase += f" ({time.monotonic() - started:.1f}s)"
if pending_models:
phase += f"; waiting for {len(pending_models)} model request(s)"
if phase and self.active:
await self._publish_progress(phase)
elif message.get("type") == "ready":
event("engine_ready", **scope)
elif message.get("type") == "usage" and self.active:
self.active["usage"] = message.get("usage")
elif message.get("type") == "progress" and self.active:
steps = max(self.active.get("attempt_steps", 0), int(message.get("steps", 0)))
if steps != self.active.get("attempt_steps", 0):
event("task_progress", **scope, steps=steps)
self.active["steps"] = min(self.active.get("max_steps", steps), self.active.get("step_offset", 0) + steps)
await self._publish_progress(self.active.get("phase", "Preparing task plan"), record=False)
self.active["attempt_steps"] = steps
await proc.wait()
event("engine_exited", failed=proc.returncode not in (0, None), **scope, exit_code=proc.returncode)
return result
return await asyncio.wait_for(read(), max(1, timeout) + 15)
async def _stop_worker(self):
proc, self.worker = self.worker, None
if proc and proc.returncode is None:
task = asyncio.create_task(self._terminate_worker(proc))
self._worker_stop_tasks.add(task)
task.add_done_callback(self._worker_stop_tasks.discard)
if self._worker_stop_tasks:
# Takeover and reset can overlap. Both must await an already-started
# worker stop even after self.worker has been detached.
await asyncio.shield(asyncio.gather(*self._worker_stop_tasks))
self._finish_pending_actions()
async def _terminate_worker(self, proc):
with contextlib.suppress(ProcessLookupError):
proc.terminate()
try:
await asyncio.wait_for(proc.wait(), 5)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
async def _stop_driver(self):
proc, self.driver = self.driver, None
self.geometry = None
if proc and proc.returncode is None:
proc.kill()
await proc.wait()
async def _stop_owned(self):
event("phone_stopping", serial=self.serial)
self.control.revoke()
self.capabilities.clear()
await self._stop_worker()
await self._stop_driver()
proc = self.process
children = owned_children(getattr(proc, "pid", None)) if proc else []
if proc and proc.returncode is None:
if self.serial:
with contextlib.suppress(Exception):
await command([str(sdk_tool("adb")), "-s", self.serial, "emu", "kill"], timeout=5)
try:
await asyncio.wait_for(proc.wait(), 10)
except asyncio.TimeoutError:
proc.kill()
await asyncio.wait_for(proc.wait(), 5)
await stop_children(children)
self.process = None
self.serial, self.geometry = None, None
if self.device_lock is not None:
self.device_lock.close()
self.device_lock = None
async with self.control.lock:
self.control.state = "idle"
self.control.results.clear()
self.viewer = None
self.resume_event.clear()
event("phone_stopped")
async def stop(self):
if self.active:
raise MobileError("device_busy", "Cancel the active task before stopping the device")
async with self.lifecycle_lock:
await self._stop_owned()
self.start_error = ""
async def shutdown(self):
self.stopping = True
if self.setup_task and not self.setup_task.done():
self.setup_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self.setup_task
if self.active:
self.active["status"] = "cancelled"
self.resume_event.set()
async with self.lifecycle_lock:
await self._stop_owned()
if self.broker_runner:
await self.broker_runner.cleanup()
self.broker_runner, self.broker_url = None, None
_runtime: MobileRuntime | None = None
def get_runtime() -> MobileRuntime:
global _runtime
if _runtime is None:
_runtime = MobileRuntime()
return _runtime
def peek_runtime() -> MobileRuntime | None:
return _runtime
async def record_phone_step(workflow_id: str | None, node_id: str | None, text: str) -> None:
"""One line in a workflow's Workspace step log (services/workspace_steps.py).
Best effort: the phone's work stands whether or not its step is kept."""
try:
from services.plugin.deps import get_database
from services.workspace_steps import record_step
await record_step(get_database(), workflow_id=workflow_id, surface="mobile", text=text, node_id=node_id)
except Exception:
event("workspace_step_failed", failed=True)