Skip to content

Self-improving AI Employees for everyone

feat(mobile): add local Android workspace and mobile-use agent

trohitg5c14991667e8
diff --git a/bun.lock b/bun.lock
index 6eba3b9b..e81bf262 100644
--- a/bun.lock
+++ b/bun.lock
@@ -31,6 +31,9 @@
"@types/react-dom": "^19.2.3",
"@uiw/react-json-view": "2.0.0-alpha.41",
"@vitejs/plugin-react": "^5.2.0",
+ "@yume-chan/scrcpy": "3.0.0-beta.3",
+ "@yume-chan/scrcpy-decoder-webcodecs": "3.0.0-beta.2",
+ "@yume-chan/stream-extra": "3.0.0-beta.3",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.1.1",
@@ -960,6 +963,24 @@
"@vitest/utils": ["@vitest/[email protected]", "", { "dependencies": { "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ=="],
+ "@yume-chan/async": ["@yume-chan/[email protected]", "", {}, "sha512-0vzhNJMkWUPyjKzUK4rqHEeCU6YQtF78RsB1kFRB6Y2BLupmEQNxcSb0mjKabPL9jZpCCiLa5KL8oTOJClUVaw=="],
+
+ "@yume-chan/event": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3" } }, "sha512-d9nQ0su78cV7OAV9Y1Ue/W2ZyF9JaWbFSU6kjQC/IoPCkov127Zv4m/5x7OC35MSAGcJvG5jkIcicNBD5mwr7A=="],
+
+ "@yume-chan/media-codec": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/no-data-view": "3.0.0-beta.2" } }, "sha512-SY8Vk0BQk79O1LvXBjX83mDygEB+gVjKByEz6DbEF630Ak5Ks5WNAhGpe39PH0dXrod4nrNA47LJzvl/PsJa+A=="],
+
+ "@yume-chan/no-data-view": ["@yume-chan/[email protected]", "", {}, "sha512-VozbKnaUjnL8zF+TkioqHGRhP5w1/dAeRTytroO9VJQJat9acSoc9yXvyABb90hS0ZS+qgN2suyE8Xf+2KfVnA=="],
+
+ "@yume-chan/scrcpy": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/event": "3.0.0-beta.3", "@yume-chan/no-data-view": "3.0.0-beta.3", "@yume-chan/stream-extra": "3.0.0-beta.3", "@yume-chan/struct": "3.0.0-beta.3" } }, "sha512-YcXffFvcWJW/FoogoeELdvzB1PtJ7aUDgb5OK6q0ivVfY/cqjpKm+15dADooHMJ9oLH5NUXI/ttHUkIhBsWZPQ=="],
+
+ "@yume-chan/scrcpy-decoder-shared": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/event": "3.0.0-beta.2", "@yume-chan/scrcpy": "3.0.0-beta.2", "@yume-chan/stream-extra": "3.0.0-beta.2" } }, "sha512-YH22RYdxxi5qKJF8UE+llfquhGjqSFFvf7iVOn92JSQOZU2yEeSG/45ma/vAxqhBOrSj+2aqX47mdhxjHgQ7aQ=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/event": "3.0.0-beta.2", "@yume-chan/media-codec": "3.0.0-beta.2", "@yume-chan/no-data-view": "3.0.0-beta.2", "@yume-chan/scrcpy": "3.0.0-beta.2", "@yume-chan/scrcpy-decoder-shared": "3.0.0-beta.2", "@yume-chan/stream-extra": "3.0.0-beta.2" } }, "sha512-uBPpg5K79+2eUGzrtg17KELoOf6dH0I69FEChk8sKS1ZXQIcKYYequpoU7vMLEDNfHE3dhgVbUYQsheS+bUyDg=="],
+
+ "@yume-chan/stream-extra": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/struct": "3.0.0-beta.3" } }, "sha512-7Jkm8UYZmWloRqfVunPtDFOF3oO1WkzTuv9h2gUNr1T6o5vAYKfDlsc59bvv+rX4BWsiPkD8DHn2U6DUYbS29g=="],
+
+ "@yume-chan/struct": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/no-data-view": "3.0.0-beta.3" } }, "sha512-cACtkzk1+czRVM/uWUAmr/rdUFQ/dISa7ocXfRU6tpV44pBKe9jP08lDnZiSohqR1fkPSJtPERfPViN5oxCuag=="],
+
"accepts": ["[email protected]", "", { "dependencies": { "mime-types": "3.0.2", "negotiator": "1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
"acorn": ["[email protected]", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="],
@@ -2776,6 +2797,22 @@
"@typescript-eslint/utils/@eslint-community/eslint-utils": ["@eslint-community/[email protected]", "", { "dependencies": { "eslint-visitor-keys": "3.4.3" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg=="],
+ "@yume-chan/media-codec/@yume-chan/no-data-view": ["@yume-chan/[email protected]", "", {}, "sha512-+YaNDEhEiW6JM4DTeIkr7twls+bLjZlRqi9KFR17y9hASAnRNGVUVwYntvozCw5cfCDxjRgFdGYRRSThGq3x+A=="],
+
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/event": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3" } }, "sha512-Lihf7J1lNQNgyx/FFl7u55gVVmvSXv+Z2heYZjz0gFq8zGN6WNOKBtXnaJVGuXnumTWmcm9seLqcWc15RaOlPA=="],
+
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/scrcpy": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/event": "3.0.0-beta.2", "@yume-chan/no-data-view": "3.0.0-beta.2", "@yume-chan/stream-extra": "3.0.0-beta.2", "@yume-chan/struct": "3.0.0-beta.2" } }, "sha512-9iDyPF/K7r6jjRNvQ//Tz2VY3m9vNNz3d2hqFLpN5BI7uwrpdScqXH848u9MXukplgr+25dWXGOeqTbByVzscg=="],
+
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/stream-extra": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/struct": "3.0.0-beta.2" } }, "sha512-94YT3d1+DlrGrKIlwB9o9YTPs7yf2+C+tjPi9cAWviANGixAv+K8V27iDIC4FKO6HUaAvRNdHai0MuXufiW8Zg=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs/@yume-chan/event": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3" } }, "sha512-Lihf7J1lNQNgyx/FFl7u55gVVmvSXv+Z2heYZjz0gFq8zGN6WNOKBtXnaJVGuXnumTWmcm9seLqcWc15RaOlPA=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs/@yume-chan/no-data-view": ["@yume-chan/[email protected]", "", {}, "sha512-+YaNDEhEiW6JM4DTeIkr7twls+bLjZlRqi9KFR17y9hASAnRNGVUVwYntvozCw5cfCDxjRgFdGYRRSThGq3x+A=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs/@yume-chan/scrcpy": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/event": "3.0.0-beta.2", "@yume-chan/no-data-view": "3.0.0-beta.2", "@yume-chan/stream-extra": "3.0.0-beta.2", "@yume-chan/struct": "3.0.0-beta.2" } }, "sha512-9iDyPF/K7r6jjRNvQ//Tz2VY3m9vNNz3d2hqFLpN5BI7uwrpdScqXH848u9MXukplgr+25dWXGOeqTbByVzscg=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs/@yume-chan/stream-extra": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/struct": "3.0.0-beta.2" } }, "sha512-94YT3d1+DlrGrKIlwB9o9YTPs7yf2+C+tjPi9cAWviANGixAv+K8V27iDIC4FKO6HUaAvRNdHai0MuXufiW8Zg=="],
+
"antd/@babel/runtime": ["@babel/[email protected]", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="],
"antd/dayjs": ["[email protected]", "", {}, "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA=="],
@@ -3010,6 +3047,16 @@
"@typescript-eslint/utils/@eslint-community/eslint-utils/eslint-visitor-keys": ["[email protected]", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="],
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/scrcpy/@yume-chan/no-data-view": ["@yume-chan/[email protected]", "", {}, "sha512-+YaNDEhEiW6JM4DTeIkr7twls+bLjZlRqi9KFR17y9hASAnRNGVUVwYntvozCw5cfCDxjRgFdGYRRSThGq3x+A=="],
+
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/scrcpy/@yume-chan/struct": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/no-data-view": "3.0.0-beta.2" } }, "sha512-6wyW1i3IGlRurdFt2jzfE1FSq+lsgp7ccbzBWUW/7O1+GtRRFjMUljnoeqcbcmerRhCb6rHI6GRxk2EOKrUNLw=="],
+
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/stream-extra/@yume-chan/struct": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/no-data-view": "3.0.0-beta.2" } }, "sha512-6wyW1i3IGlRurdFt2jzfE1FSq+lsgp7ccbzBWUW/7O1+GtRRFjMUljnoeqcbcmerRhCb6rHI6GRxk2EOKrUNLw=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs/@yume-chan/scrcpy/@yume-chan/struct": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/no-data-view": "3.0.0-beta.2" } }, "sha512-6wyW1i3IGlRurdFt2jzfE1FSq+lsgp7ccbzBWUW/7O1+GtRRFjMUljnoeqcbcmerRhCb6rHI6GRxk2EOKrUNLw=="],
+
+ "@yume-chan/scrcpy-decoder-webcodecs/@yume-chan/stream-extra/@yume-chan/struct": ["@yume-chan/[email protected]", "", { "dependencies": { "@yume-chan/async": "^4.1.3", "@yume-chan/no-data-view": "3.0.0-beta.2" } }, "sha512-6wyW1i3IGlRurdFt2jzfE1FSq+lsgp7ccbzBWUW/7O1+GtRRFjMUljnoeqcbcmerRhCb6rHI6GRxk2EOKrUNLw=="],
+
"cytoscape-fcose/cose-base/layout-base": ["[email protected]", "", {}, "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="],
"d3-sankey/d3-array/internmap": ["[email protected]", "", {}, "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw=="],
@@ -3028,6 +3075,8 @@
"shadcn/@babel/parser/@babel/types": ["@babel/[email protected]", "", { "dependencies": { "@babel/helper-string-parser": "7.29.7", "@babel/helper-validator-identifier": "7.29.7" } }, "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg=="],
+ "@yume-chan/scrcpy-decoder-shared/@yume-chan/stream-extra/@yume-chan/struct/@yume-chan/no-data-view": ["@yume-chan/[email protected]", "", {}, "sha512-+YaNDEhEiW6JM4DTeIkr7twls+bLjZlRqi9KFR17y9hASAnRNGVUVwYntvozCw5cfCDxjRgFdGYRRSThGq3x+A=="],
+
"pkg-up/find-up/locate-path/p-locate": ["[email protected]", "", { "dependencies": { "p-limit": "2.3.0" } }, "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ=="],
"pkg-up/find-up/locate-path/path-exists": ["[email protected]", "", {}, "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ=="],
diff --git a/client/package.json b/client/package.json
index fac02513..3b6c78e0 100644
--- a/client/package.json
+++ b/client/package.json
@@ -33,6 +33,9 @@
"@types/react-dom": "^19.2.3",
"@uiw/react-json-view": "2.0.0-alpha.41",
"@vitejs/plugin-react": "^5.2.0",
+ "@yume-chan/scrcpy": "3.0.0-beta.3",
+ "@yume-chan/scrcpy-decoder-webcodecs": "3.0.0-beta.2",
+ "@yume-chan/stream-extra": "3.0.0-beta.3",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.1.1",
diff --git a/client/src/components/mobile/MobileWorkspace.tsx b/client/src/components/mobile/MobileWorkspace.tsx
new file mode 100644
index 00000000..e0482fdb
--- /dev/null
+++ b/client/src/components/mobile/MobileWorkspace.tsx
@@ -0,0 +1,256 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import type { PointerEvent } from 'react';
+import { ArrowLeft, Home, Play, RotateCw, Smartphone, Square } from 'lucide-react';
+import { Button } from '@/components/ui/button';
+import { buildApiUrl } from '@/config/api';
+import { mobilePath, mobilePoint, mobileRequest, type Doctor, type Geometry, type Invocation, type MobileStatus } from './api';
+
+const fieldClass = 'min-w-0 rounded border border-border-default bg-bg-input px-2 py-1.5 text-sm text-fg-default outline-none focus-visible:ring-2 focus-visible:ring-ring';
+const finished = new Set(['completed', 'success', 'succeeded', 'failed', 'cancelled', 'canceled']);
+const taskState = (task: Invocation | null) => task?.status ?? task?.state ?? '';
+const describe = (value: unknown): string => typeof value === 'string' ? value : value == null ? '' : JSON.stringify(value);
+
+export default function MobileWorkspace({ workflowId, nodes, visible = true }: {
+ workflowId?: string | null; nodes: { node_id: string; label: string }[]; visible?: boolean;
+}) {
+ const [selected, setSelected] = useState('');
+ const nodeId = nodes.some((node) => node.node_id === selected) ? selected : nodes[0]?.node_id;
+ if (!workflowId || workflowId === 'new') return <Empty message="Save this workflow before opening its mobile workspace." />;
+ if (!nodeId) return <Empty message="Add a Mobile Agent node to this workflow to use its phone here." />;
+ return <div className="flex min-h-0 flex-1 flex-col gap-2">
+ {nodes.length > 1 && <select className={fieldClass} aria-label="Mobile agent" value={nodeId} onChange={(event) => setSelected(event.target.value)}>
+ {nodes.map((node) => <option key={node.node_id} value={node.node_id}>{node.label}</option>)}
+ </select>}
+ <MobileSession key={`${workflowId}:${nodeId}`} workflowId={workflowId} nodeId={nodeId} visible={visible} />
+ </div>;
+}
+
+function Empty({ message }: { message: string }) {
+ return <div className="m-auto flex max-w-80 flex-col items-center gap-3 p-6 text-center text-sm text-fg-muted"><Smartphone aria-hidden className="size-6" />{message}</div>;
+}
+
+function MobileSession({ workflowId, nodeId, visible }: { workflowId: string; nodeId: string; visible: boolean }) {
+ const path = mobilePath(workflowId, nodeId);
+ const [viewerId] = useState(() => crypto.randomUUID());
+ const [status, setStatus] = useState<MobileStatus | null>(null);
+ const [doctor, setDoctor] = useState<Doctor | null>(null);
+ const [error, setError] = useState('');
+ const [busy, setBusy] = useState('');
+ const [accepted, setAccepted] = useState(false);
+ const [prompt, setPrompt] = useState('');
+ const [text, setText] = useState('');
+ const [task, setTask] = useState<Invocation | null>(null);
+ const [submission, setSubmission] = useState<string | null>(() => {
+ try { return sessionStorage.getItem(`mobile-task:${workflowId}:${nodeId}`); } catch { return null; }
+ });
+ const [epoch, setEpoch] = useState<number | null>(null);
+ const [leaseOwner, setLeaseOwner] = useState<string | null>(null);
+ const [videoError, setVideoError] = useState('');
+ const [videoAttempt, setVideoAttempt] = useState(0);
+ const [live, setLive] = useState(false);
+ const [pageVisible, setPageVisible] = useState(!document.hidden);
+ const canvas = useRef<HTMLCanvasElement>(null);
+ const surface = useRef<HTMLDivElement>(null);
+ const active = useRef(true);
+ const operation = useRef(false);
+ const leaseEpoch = useRef<number | null>(null);
+ const viewActive = useRef(visible && pageVisible);
+ viewActive.current = visible && pageVisible;
+ const pendingSubmission = useRef<{ id: string; prompt: string } | null>(null);
+ const pointer = useRef<{ x: number; y: number; at: number; geometry: Geometry } | null>(null);
+ const held = status?.control_state === 'human' && leaseOwner !== null && status.controller === leaseOwner && epoch === status.epoch;
+ const hasTask = !!submission && !finished.has(taskState(task));
+ const running = status?.running === true;
+ const refresh = useCallback(async (signal?: AbortSignal) => {
+ const next = await mobileRequest<MobileStatus>(`${path}/status`, undefined, signal);
+ if (active.current) setStatus(next);
+ }, [path]);
+
+ useEffect(() => {
+ active.current = true;
+ const changed = () => setPageVisible(!document.hidden);
+ document.addEventListener('visibilitychange', changed);
+ return () => { active.current = false; document.removeEventListener('visibilitychange', changed); };
+ }, []);
+ useEffect(() => {
+ if (!visible || !pageVisible) return;
+ const controller = new AbortController();
+ let timer: ReturnType<typeof setTimeout>;
+ const poll = async () => {
+ try { await refresh(controller.signal); }
+ catch (cause) { if (!controller.signal.aborted && active.current) setError(describe(cause instanceof Error ? cause.message : cause)); }
+ if (!controller.signal.aborted) timer = setTimeout(() => void poll(), 2000);
+ };
+ void poll();
+ void mobileRequest<Doctor>(`${path}/doctor`, undefined, controller.signal).then((value) => {
+ if (!controller.signal.aborted) setDoctor(value);
+ }).catch((cause) => { if (!controller.signal.aborted) setError(cause.message); });
+ return () => { controller.abort(); clearTimeout(timer); };
+ }, [visible, pageVisible, refresh, path]);
+ useEffect(() => {
+ if (!submission || !visible || !pageVisible) return;
+ const controller = new AbortController();
+ let timer: ReturnType<typeof setTimeout>;
+ const poll = async () => {
+ try {
+ const next = await mobileRequest<Invocation>(`${path}/tasks/${encodeURIComponent(submission)}`, undefined, controller.signal);
+ if (!controller.signal.aborted) setTask(next);
+ if (!finished.has(taskState(next)) && !controller.signal.aborted) timer = setTimeout(() => void poll(), 1500);
+ } catch (cause) {
+ if (!controller.signal.aborted) { setError(cause instanceof Error ? cause.message : 'Could not read this task.'); timer = setTimeout(() => void poll(), 4000); }
+ }
+ };
+ void poll();
+ return () => { controller.abort(); clearTimeout(timer); };
+ }, [submission, visible, pageVisible, path]);
+ useEffect(() => {
+ if (!running || !visible || !pageVisible || !canvas.current) return;
+ let disposed = false;
+ let disconnect: (() => void) | undefined;
+ setVideoError(''); setLive(false);
+ const target = canvas.current;
+ void import('./video').then(({ connectMobileVideo }) => {
+ if (!disposed) disconnect = connectMobileVideo({ workflowId, nodeId, viewerId, canvas: target,
+ onError: setVideoError, onLive: () => setLive(true) });
+ }).catch(() => { if (!disposed) setVideoError('Could not load the video player. Reconnect to try again.'); });
+ return () => { disposed = true; disconnect?.(); };
+ }, [running, visible, pageVisible, workflowId, nodeId, viewerId, videoAttempt]);
+ useEffect(() => {
+ if (status?.setup !== 'ready') return;
+ const controller = new AbortController();
+ void mobileRequest<Doctor>(`${path}/doctor`, undefined, controller.signal).then((value) => {
+ if (!controller.signal.aborted) setDoctor(value);
+ }).catch(() => {});
+ return () => controller.abort();
+ }, [status?.setup, path]);
+ // Hiding the view relinquishes only this grant, never a later control session.
+ useEffect(() => {
+ if (!visible || !pageVisible) return;
+ return () => {
+ const releasedEpoch = leaseEpoch.current;
+ leaseEpoch.current = null;
+ if (releasedEpoch !== null) void mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch: releasedEpoch, resume: false }).catch(() => {});
+ };
+ }, [visible, pageVisible, path, viewerId]);
+
+ const perform = async (label: string, action: () => Promise<void>) => {
+ if (operation.current) return;
+ operation.current = true; setBusy(label); setError('');
+ try { await action(); if (active.current) await refresh(); }
+ catch (cause) { if (active.current) setError(cause instanceof Error ? cause.message : 'That action failed.'); }
+ finally { operation.current = false; if (active.current) setBusy(''); }
+ };
+ const input = async (action: string, parameters: Record<string, unknown>, geometry = status?.geometry) => {
+ if (!held || epoch === null || !geometry) return;
+ await mobileRequest(`${path}/input`, { viewer_id: viewerId, epoch, operation_id: crypto.randomUUID(), operation: action, parameters, geometry });
+ };
+ const point = (event: PointerEvent<HTMLDivElement>) => {
+ const rect = surface.current?.getBoundingClientRect();
+ const geometry = status?.geometry;
+ if (!rect || !geometry) return null;
+ return mobilePoint(event.clientX - rect.left, event.clientY - rect.top, rect.width, rect.height, geometry);
+ };
+ const down = (event: PointerEvent<HTMLDivElement>) => {
+ if (!held || busy || !live || videoError) return;
+ const start = point(event);
+ if (!start || !status?.geometry) return;
+ pointer.current = { ...start, at: performance.now(), geometry: { ...status.geometry } };
+ event.currentTarget.setPointerCapture(event.pointerId);
+ };
+ const up = (event: PointerEvent<HTMLDivElement>) => {
+ const start = pointer.current; pointer.current = null;
+ const end = point(event);
+ if (!start || !end || !held || !live || videoError) return;
+ const duration = Math.max(100, Math.min(2000, Math.round(performance.now() - start.at)));
+ void perform('input', () => Math.hypot(end.x - start.x, end.y - start.y) < 12
+ ? input('tap', { x: end.x, y: end.y }, start.geometry)
+ : input('swipe', { x: start.x, y: start.y, end_x: end.x, end_y: end.y, duration }, start.geometry));
+ };
+ const submit = () => perform('task', async () => {
+ const clean = prompt.trim();
+ if (!clean || hasTask) return;
+ // A lost response retries the same task; changing the text makes a new submission.
+ const attempt = pendingSubmission.current?.prompt === clean ? pendingSubmission.current : { id: crypto.randomUUID(), prompt: clean };
+ pendingSubmission.current = attempt;
+ const response = await mobileRequest<Invocation>(`${path}/tasks`, { submission_id: attempt.id, prompt: clean });
+ if (!active.current) return;
+ setSubmission(attempt.id); setTask(response); setPrompt(''); pendingSubmission.current = null;
+ try { sessionStorage.setItem(`mobile-task:${workflowId}:${nodeId}`, attempt.id); } catch { /* optional recovery */ }
+ });
+ const installed = doctor?.adb && doctor?.emulator && doctor?.image && doctor?.engine && doctor?.video;
+ const setupActive = typeof status?.setup === 'string' && status.setup.startsWith('installing_');
+ return <div className="flex min-h-0 flex-1 flex-col gap-2 overflow-y-auto">
+ <div className="flex flex-wrap items-center gap-2">
+ <Smartphone aria-hidden className="size-4" />
+ <span className="flex-1 text-sm font-medium">Shared phone</span>
+ <span role="status" className="text-xs text-fg-muted">{busy ? `${busy}…` : running ? status.control_state : status ? 'Stopped' : 'Connecting…'}</span>
+ <Button size="icon-sm" variant="ghost" aria-label="Refresh mobile status" onClick={() => void perform('refresh', async () => { setDoctor(await mobileRequest(`${path}/doctor`)); })}><RotateCw /></Button>
+ </div>
+ {error && <p role="alert" className="m-0 rounded border border-destructive/30 bg-destructive/10 p-2 text-sm text-destructive">{error}</p>}
+ {status?.setup_error && <p role="alert" className="m-0 text-sm text-destructive">{status.setup_error}</p>}
+ {doctor?.supported === false && <p className="m-0 text-sm text-fg-muted">This host cannot run the mobile runtime. Android setup currently requires Windows x64 with virtualization available.</p>}
+ {doctor && !installed && !running && <section aria-label="Mobile setup" className="space-y-2 rounded border border-border-default bg-bg-panel p-3">
+ <h3 className="m-0 text-sm font-semibold">Set up your shared Android phone</h3>
+ <p className="m-0 text-xs text-fg-muted">Download the Android tools, emulator image and mobile agent runtime. Apps and sign-ins stay on this phone between tasks.</p>
+ <ul className="m-0 list-none space-y-1 p-0 text-xs text-fg-muted">{(['adb', 'emulator', 'image', 'engine', 'video'] as const).map((key) => <li key={key}>{key === 'adb' ? 'Android tools' : key === 'engine' ? 'Agent runtime' : key === 'video' ? 'Live video' : key === 'image' ? 'Android image' : 'Emulator'}: {doctor[key] ? 'Ready' : 'Needed'}</li>)}</ul>
+ <label className="flex items-start gap-2 text-xs"><input type="checkbox" checked={accepted} onChange={(event) => setAccepted(event.target.checked)} aria-label="Accept Android SDK license terms" />
+ <span>I accept the <a className="underline" href="https://developer.android.com/studio/terms" target="_blank" rel="noreferrer">Android SDK license terms</a> and want to download the required components.</span></label>
+ <Button size="sm" disabled={!accepted || !!busy || setupActive || doctor.supported === false} onClick={() => void perform('setup', async () => {
+ await mobileRequest(`${path}/setup`, { licenses_accepted: true }); setDoctor(await mobileRequest(`${path}/doctor`));
+ })}>Set up phone</Button>
+ {setupActive && <p role="status" className="m-0 break-words text-xs text-fg-muted">{status?.setup === 'installing_engine' ? 'Installing the agent runtime…' : 'Installing Android tools and preparing your phone…'}</p>}
+ </section>}
+ {doctor?.acceleration && <details className="rounded border border-border-default px-3 py-2 text-xs text-fg-muted">
+ <summary className="cursor-pointer font-medium">Host readiness: hardware acceleration</summary>
+ <p className="mb-0 mt-2 whitespace-pre-wrap break-words">{doctor.acceleration}</p>
+ <p className="mb-0 mt-2">If acceleration is unavailable on Windows, enable virtualization in your computer�s firmware and Windows Hypervisor Platform in Windows Features, then restart Windows and refresh mobile status.</p>
+ </details>}
+ <div className="flex flex-wrap gap-2">
+ <Button size="sm" variant="outline" disabled={!!busy || setupActive || !status || doctor?.supported === false || (!running && !installed)} onClick={() => void perform(running ? 'stopping' : 'starting', async () => { await mobileRequest(`${path}/${running ? 'stop' : 'start'}`, {}); setEpoch(null); })}>
+ {running ? <Square className="size-3.5" /> : <Play className="size-3.5" />}{running ? 'Stop phone' : 'Start phone'}
+ </Button>
+ {running && (held ? <>
+ <Button size="sm" onClick={() => void perform('releasing', async () => { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch, resume: true }); leaseEpoch.current = null; setEpoch(null); })} disabled={!!busy}>Hand back & resume</Button>
+ <Button size="sm" variant="outline" onClick={() => void perform('releasing', async () => { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch, resume: false }); leaseEpoch.current = null; setEpoch(null); })} disabled={!!busy}>Release control</Button>
+ </> : <Button size="sm" variant="outline" disabled={!!busy} onClick={() => void perform('taking control', async () => { const claim = await mobileRequest<{ epoch: number; owner: string }>(`${path}/takeover`, { viewer_id: viewerId });
+ if (!active.current || !viewActive.current) { await mobileRequest(`${path}/release`, { viewer_id: viewerId, epoch: claim.epoch, resume: false }); return; }
+ leaseEpoch.current = claim.epoch; setEpoch(claim.epoch); setLeaseOwner(claim.owner); })}>Take control</Button>)}
+ </div>
+ {running && <>
+ <div ref={surface} className={`relative flex min-h-64 flex-1 touch-none items-center justify-center overflow-hidden rounded border border-border-default bg-bg-canvas ${held ? 'cursor-crosshair' : ''}`} onPointerDown={down} onPointerUp={up} onPointerCancel={() => { pointer.current = null; }} aria-label={held ? 'Phone screen: tap or drag to interact' : 'Phone screen, view only'}>
+ <canvas ref={canvas} className="absolute h-full w-full object-contain" />
+ {!live && !videoError && <p className="pointer-events-none z-10 text-sm text-fg-muted">Connecting live view…</p>}
+ </div>
+ {videoError && <div className="flex items-center gap-2"><p role="alert" className="m-0 flex-1 text-xs text-fg-muted">{videoError}</p><Button size="sm" variant="outline" onClick={() => setVideoAttempt((value) => value + 1)}>Reconnect</Button></div>}
+ {held && <div className="flex flex-wrap gap-1">
+ <Button size="icon-sm" variant="outline" aria-label="Phone Back" disabled={!!busy} onClick={() => void perform('input', () => input('key', { key: 'back' }))}><ArrowLeft /></Button>
+ <Button size="icon-sm" variant="outline" aria-label="Phone Home" disabled={!!busy} onClick={() => void perform('input', () => input('key', { key: 'home' }))}><Home /></Button>
+ <input className={`${fieldClass} flex-1`} aria-label="Text to type on phone" value={text} onChange={(event) => setText(event.target.value)} placeholder="Type on phone" />
+ <Button size="sm" variant="outline" disabled={!text || !!busy} onClick={() => void perform('input', async () => { await input('text', { text }); setText(''); })}>Type</Button>
+ </div>}
+ {held && <label className="flex items-center gap-2 text-xs text-fg-muted">Install APK (up to 256 MB)
+ <input type="file" accept=".apk" aria-label="Install APK" disabled={!!busy} className="min-w-0 flex-1 text-xs" onChange={(event) => {
+ const file = event.target.files?.[0]; event.target.value = '';
+ if (!file) return;
+ if (!file.name.toLowerCase().endsWith('.apk') || file.size > 256 * 1024 * 1024) { setError('Choose an APK file no larger than 256 MB.'); return; }
+ void perform('installing APK', async () => {
+ const query = new URLSearchParams({ viewer_id: viewerId, epoch: String(epoch), operation_id: crypto.randomUUID(), filename: file.name });
+ const response = await fetch(buildApiUrl(`${path}/apk?${query}`), { method: 'POST', credentials: 'include', headers: { 'Content-Type': 'application/vnd.android.package-archive' }, body: file });
+ if (!response.ok) { const body = await response.json().catch(() => null); throw new Error(typeof body?.detail === 'string' ? body.detail : 'APK installation failed.'); }
+ });
+ }} />
+ </label>}
+ </>}
+ <form className="mt-auto flex shrink-0 flex-col gap-2 border-t border-border-default pt-3" onSubmit={(event) => { event.preventDefault(); void submit(); }}>
+ <label htmlFor={`task-${nodeId}`} className="text-sm font-medium">Give this mobile agent a task</label>
+ <textarea id={`task-${nodeId}`} className={`${fieldClass} resize-y`} rows={2} maxLength={12000} value={prompt} onChange={(event) => setPrompt(event.target.value)} placeholder="What should it do on the phone?" />
+ <p className="m-0 text-xs text-fg-muted">Runs in this workflow. The phone is shared; tasks wait while another employee or person controls it.</p>
+ <div className="flex items-center gap-2"><Button size="sm" type="submit" disabled={!prompt.trim() || !!busy || hasTask}>Run task</Button>
+ {hasTask && <Button size="sm" type="button" variant="outline" disabled={!!busy} onClick={() => void perform('cancelling', async () => { setTask(await mobileRequest(`${path}/tasks/${encodeURIComponent(submission!)}/cancel`, {})); })}>Cancel task</Button>}
+ {task && <span role="status" className="text-xs text-fg-muted">{taskState(task) || 'Submitted'}</span>}
+ </div>
+ {task?.error != null && <p role="alert" className="m-0 break-words text-sm text-destructive">{describe(task.error)}</p>}
+ {task?.result != null && <pre className="m-0 max-h-48 overflow-auto whitespace-pre-wrap rounded bg-bg-panel p-2 text-xs">{describe(task.result)}</pre>}
+ </form>
+ </div>;
+}
diff --git a/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx b/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx
new file mode 100644
index 00000000..d52977b8
--- /dev/null
+++ b/client/src/components/mobile/__tests__/MobileWorkspace.test.tsx
@@ -0,0 +1,109 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import MobileWorkspace from '../MobileWorkspace';
+import { mobilePoint, mobileRequest } from '../api';
+
+vi.mock('../video', () => ({ connectMobileVideo: () => () => {} }));
+const fetchMock = vi.fn();
+const nodes = [{ node_id: 'flow:mobile_agent:1', label: 'Phone assistant' }];
+const ready = { supported: true, adb: true, emulator: true, image: true, engine: true, video: true, acceleration: 'WHPX is installed and usable.' };
+let snapshot: Record<string, unknown>;
+const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } });
+
+beforeEach(() => {
+ sessionStorage.clear();
+ snapshot = { running: false, control_state: 'idle', controller: null, epoch: 0, geometry: { width: 1080, height: 1920, rotation: 0 } };
+ vi.stubGlobal('fetch', fetchMock);
+ fetchMock.mockReset().mockImplementation(async (url: string) => json(url.endsWith('/doctor') ? ready : snapshot));
+});
+afterEach(() => { cleanup(); vi.unstubAllGlobals(); });
+
+describe('Mobile Workspace', () => {
+ it('requires a saved workflow and an explicitly discovered Mobile node', () => {
+ const view = render(<MobileWorkspace nodes={nodes} />);
+ expect(screen.getByText(/Save this workflow/)).toBeInTheDocument();
+ view.rerender(<MobileWorkspace workflowId="flow" nodes={[]} />);
+ expect(screen.getByText(/Add a Mobile Agent/)).toBeInTheDocument();
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('only reads status on open; never starts or installs automatically', async () => {
+ render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
+ await screen.findByRole('button', { name: 'Start phone' });
+ await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
+ expect(fetchMock.mock.calls.every(([, options]) => options.method === 'GET')).toBe(true);
+ expect(screen.getByText('Host readiness: hardware acceleration')).toBeInTheDocument();
+ expect(screen.getByText('WHPX is installed and usable.')).toBeInTheDocument();
+ });
+
+ it('requires explicit license acceptance before setup', async () => {
+ fetchMock.mockImplementation(async (url: string) => json(url.endsWith('/doctor') ? { supported: true } : snapshot));
+ render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
+ const setup = await screen.findByRole('button', { name: 'Set up phone' });
+ expect(setup).toBeDisabled();
+ await userEvent.click(screen.getByRole('checkbox', { name: 'Accept Android SDK license terms' }));
+ await userEvent.click(setup);
+ await waitFor(() => expect(fetchMock.mock.calls.some(([url, options]) => url.endsWith('/setup') && JSON.parse(options.body).licenses_accepted === true)).toBe(true));
+ });
+
+ it('submits only to the displayed workflow and retries a lost response with the same id', async () => {
+ let attempts = 0;
+ fetchMock.mockImplementation(async (url: string, options: RequestInit) => {
+ if (url.endsWith('/tasks') && options.method === 'POST') {
+ attempts++;
+ if (attempts === 1) throw new Error('Connection dropped');
+ return json({ status: 'queued', invocation_id: 'inv-1' });
+ }
+ if (url.includes('/tasks/')) return json({ status: 'completed', result: 'Done' });
+ return json(url.endsWith('/doctor') ? ready : snapshot);
+ });
+ render(<MobileWorkspace workflowId="home-flow" nodes={nodes} />);
+ await userEvent.type(screen.getByRole('textbox', { name: 'Give this mobile agent a task' }), 'Open settings');
+ await userEvent.click(screen.getByRole('button', { name: 'Run task' }));
+ await screen.findByText('Connection dropped');
+ await userEvent.click(screen.getByRole('button', { name: 'Run task' }));
+ await screen.findByText('Done');
+ const requests = fetchMock.mock.calls.filter(([url, options]) => url.endsWith('/tasks') && options.method === 'POST');
+ expect(requests).toHaveLength(2);
+ expect(requests[0][0]).toBe('/api/mobile/home-flow/flow%3Amobile_agent%3A1/tasks');
+ expect(JSON.parse(requests[0][1].body)).toEqual(JSON.parse(requests[1][1].body));
+ expect(JSON.parse(requests[0][1].body)).toMatchObject({ prompt: 'Open settings' });
+ });
+
+ it('unlocks input only after an opaque lease grant matches status and sends epoch', async () => {
+ snapshot.running = true;
+ fetchMock.mockImplementation(async (url: string) => {
+ if (url.endsWith('/takeover')) {
+ snapshot = { ...snapshot, controller: 'viewer:server-hash', control_state: 'human', epoch: 7 };
+ return json({ owner: 'viewer:server-hash', epoch: 7 });
+ }
+ return json(url.endsWith('/doctor') ? ready : snapshot);
+ });
+ const view = render(<MobileWorkspace workflowId="flow" nodes={nodes} />);
+ expect(screen.queryByRole('button', { name: 'Phone Home' })).toBeNull();
+ await userEvent.click(await screen.findByRole('button', { name: 'Take control' }));
+ await userEvent.click(await screen.findByRole('button', { name: 'Phone Home' }));
+ await waitFor(() => expect(fetchMock.mock.calls.some(([url]) => url.endsWith('/input'))).toBe(true));
+ const request = fetchMock.mock.calls.find(([url]) => url.endsWith('/input'))!;
+ expect(JSON.parse(request[1].body)).toMatchObject({ epoch: 7, operation: 'key', parameters: { key: 'home' } });
+ expect(screen.getByLabelText('Install APK')).toBeInTheDocument();
+ fireEvent.change(screen.getByLabelText('Install APK'), { target: { files: [new File(['bad'], 'not-an-app.txt')] } });
+ expect(screen.getByText('Choose an APK file no larger than 256 MB.')).toBeInTheDocument();
+ view.rerender(<MobileWorkspace workflowId="flow" nodes={nodes} visible={false} />);
+ await waitFor(() => expect(fetchMock.mock.calls.some(([url, options]) => url.endsWith('/release') && JSON.parse(options.body).epoch === 7)).toBe(true));
+ });
+});
+
+describe('mobile transport boundaries', () => {
+ it('maps aspect-fit coordinates and refuses letterbox input', () => {
+ const geometry = { width: 100, height: 200 };
+ expect(mobilePoint(100, 100, 200, 200, geometry)).toEqual({ x: 50, y: 100 });
+ expect(mobilePoint(5, 100, 200, 200, geometry)).toBeNull();
+ expect(mobilePoint(0, 0, 0, 0, geometry)).toBeNull();
+ });
+ it('surfaces structured device errors', async () => {
+ fetchMock.mockResolvedValue(json({ detail: { code: 'stale_lease', message: 'Device control changed.' } }, 409));
+ await expect(mobileRequest('/api/mobile/test')).rejects.toThrow('Device control changed.');
+ });
+});
diff --git a/client/src/components/mobile/api.ts b/client/src/components/mobile/api.ts
new file mode 100644
index 00000000..d700a337
--- /dev/null
+++ b/client/src/components/mobile/api.ts
@@ -0,0 +1,46 @@
+import { buildApiUrl } from '@/config/api';
+
+export interface Geometry { width: number; height: number; rotation?: number; revision?: number; [key: string]: unknown }
+export interface MobileStatus {
+ running: boolean;
+ geometry?: Geometry | null;
+ setup?: string | Record<string, unknown> | null;
+ setup_error?: string | null;
+ controller?: string | null;
+ epoch: number;
+ control_state: string;
+ active?: Record<string, unknown> | null;
+ queue?: unknown[];
+}
+export interface Doctor {
+ supported?: boolean; platform?: string; sdk_tools?: boolean; adb?: boolean; emulator?: boolean;
+ engine?: boolean; video?: boolean; acceleration?: string; image?: boolean;
+ license_url?: string; [key: string]: unknown;
+}
+export interface Invocation { status?: string; state?: string; error?: unknown; result?: unknown; invocation_id?: string }
+
+export function mobilePath(workflowId: string, nodeId: string): string {
+ return `/api/mobile/${encodeURIComponent(workflowId)}/${encodeURIComponent(nodeId)}`;
+}
+
+export async function mobileRequest<T>(path: string, body?: unknown, signal?: AbortSignal): Promise<T> {
+ const response = await fetch(buildApiUrl(path), {
+ method: body === undefined ? 'GET' : 'POST', credentials: 'include', signal,
+ ...(body === undefined ? {} : { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }),
+ });
+ const data = await response.json().catch(() => null);
+ if (!response.ok || data?.success === false) {
+ const detail = data?.detail ?? data?.error;
+ throw new Error(typeof detail === 'string' ? detail : typeof detail?.message === 'string' ? detail.message : `Mobile request failed (${response.status})`);
+ }
+ return data as T;
+}
+
+export function mobilePoint(x: number, y: number, width: number, height: number, geometry: Geometry) {
+ if (![width, height, geometry.width, geometry.height].every((n) => Number.isFinite(n) && n > 0)) return null;
+ const scale = Math.min(width / geometry.width, height / geometry.height);
+ const px = (x - (width - geometry.width * scale) / 2) / scale;
+ const py = (y - (height - geometry.height * scale) / 2) / scale;
+ if (px < 0 || py < 0 || px >= geometry.width || py >= geometry.height) return null;
+ return { x: Math.floor(px), y: Math.floor(py) };
+}
diff --git a/client/src/components/mobile/video.ts b/client/src/components/mobile/video.ts
new file mode 100644
index 00000000..e5a38ac0
--- /dev/null
+++ b/client/src/components/mobile/video.ts
@@ -0,0 +1,66 @@
+import { ScrcpyOptions4_1 } from '@yume-chan/scrcpy';
+import { BitmapVideoFrameRenderer, WebCodecsVideoDecoder } from '@yume-chan/scrcpy-decoder-webcodecs';
+import { ReadableStream } from '@yume-chan/stream-extra';
+import { buildApiUrl } from '@/config/api';
+
+/** A dedicated, bounded stream: encoded video never enters React or the main RPC socket. */
+export function connectMobileVideo({ workflowId, nodeId, viewerId, canvas, onError, onLive }: {
+ workflowId: string; nodeId: string; viewerId: string; canvas: HTMLCanvasElement;
+ onError: (message: string) => void; onLive: () => void;
+}): () => void {
+ if (!WebCodecsVideoDecoder.isSupported) {
+ onError('Live video needs a browser with WebCodecs on localhost or HTTPS. Open this app in a current Chrome or Edge browser.');
+ return () => {};
+ }
+ const url = new URL(buildApiUrl(`/ws/mobile/${encodeURIComponent(workflowId)}/${encodeURIComponent(nodeId)}`), window.location.href);
+ url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+ url.searchParams.set('viewer_id', viewerId);
+ const socket = new WebSocket(url);
+ socket.binaryType = 'arraybuffer';
+ let closed = false;
+ let decoder: WebCodecsVideoDecoder | undefined;
+ let streamController: ReadableStreamDefaultController<Uint8Array>;
+ const abort = new AbortController();
+ const stream = new ReadableStream<Uint8Array>({ start(controller) { streamController = controller; } }, {
+ highWaterMark: 8 * 1024 * 1024, size: (chunk) => chunk.byteLength,
+ });
+ const close = () => {
+ if (closed) return;
+ closed = true;
+ abort.abort();
+ socket.close();
+ try { streamController.error(new Error('View closed')); } catch { /* already closed */ }
+ decoder?.dispose();
+ };
+ const fail = (message: string) => {
+ if (closed) return;
+ onError(message);
+ close();
+ };
+ socket.onmessage = (event) => {
+ if (closed) return;
+ if (typeof event.data === 'string') {
+ try {
+ const metadata = JSON.parse(event.data);
+ if (metadata.type === 'error') fail(metadata.message || metadata.error || 'Mobile stream failed.');
+ } catch { fail('Invalid mobile stream metadata.'); }
+ return;
+ }
+ if (!(event.data instanceof ArrayBuffer)) return;
+ if ((streamController.desiredSize ?? 0) <= 0) { fail('Video fell behind. Reconnect the live view.'); return; }
+ try { streamController.enqueue(new Uint8Array(event.data)); } catch { /* stream disposed */ }
+ };
+ socket.onerror = () => fail('Could not connect to the mobile video stream.');
+ socket.onclose = () => {
+ fail('Live view disconnected. Reconnect to watch again.');
+ };
+ void (async () => {
+ const options = new ScrcpyOptions4_1({ videoCodec: 'h264', sendDeviceMeta: true, sendStreamMeta: true, sendFrameMeta: true });
+ const video = await options.parseVideoStreamMetadata(stream);
+ if (closed) return;
+ decoder = new WebCodecsVideoDecoder({ codec: video.metadata.codec, renderer: new BitmapVideoFrameRenderer({ canvas }), optimizeForLatency: true });
+ decoder.sizeChanged(() => { if (!closed) onLive(); });
+ await video.stream.pipeThrough(options.createMediaStreamTransformer()).pipeTo(decoder.writable, { signal: abort.signal });
+ })().catch((error) => { if (!closed && !abort.signal.aborted) fail(error instanceof Error ? error.message : 'Could not decode mobile video.'); });
+ return close;
+}
diff --git a/client/src/components/ui/CanvasDock.tsx b/client/src/components/ui/CanvasDock.tsx
index 28b95225..55d81206 100644
--- a/client/src/components/ui/CanvasDock.tsx
+++ b/client/src/components/ui/CanvasDock.tsx
@@ -38,6 +38,8 @@ import {
import CanvasContent from '../parameterPanel/canvas/CanvasContent';
import BrowserWorkspace from '../browser/BrowserWorkspace';
import { WorkspaceTabs } from '../workspace/WorkspaceTabs';
+import { workspaceNodes } from '../workspace/descriptors';
+import MobileWorkspace from '../mobile/MobileWorkspace';
import { queryClient } from '../../lib/queryClient';
interface CanvasDockProps {
@@ -76,7 +78,7 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
}),
() => nodes.map((node) => {
const hints = resolveNodeDescription(node.type || '')?.uiHints;
- return `${node.id}:${!!hints?.isCanvasPanel}:${!!hints?.isBrowserPanel}`;
+ return `${node.id}:${!!hints?.isCanvasPanel}:${!!hints?.isBrowserPanel}:${hints?.workspace?.kind ?? ''}`;
}).join('|'),
);
// eslint-disable-next-line react-hooks/exhaustive-deps -- hintsVersion tracks external schema hydration.
@@ -87,6 +89,7 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
// eslint-disable-next-line react-hooks/exhaustive-deps -- hintsVersion tracks external schema hydration.
[nodes, hintsVersion]);
const shownWidth = wide ? Math.min(1100, window.innerWidth - 160) : Math.min(widthPx, window.innerWidth - 40);
+ const mobileNodes = workspaceNodes(nodes).filter((node) => node.kind === 'mobile');
// A stale selection (workflow switch, node deleted) falls back to the
// first Canvas node rather than a dead board.
@@ -164,6 +167,7 @@ const CanvasDock: React.FC<CanvasDockProps> = ({ nodes }) => {
tab={tab}
onTabChange={setTab}
browser={<BrowserWorkspace key={workflowId ?? 'unsaved'} workflowId={workflowId} nodes={browserNodes} visible={open && tab === 'browser'} />}
+ android={<MobileWorkspace key={workflowId ?? 'unsaved'} workflowId={workflowId} nodes={mobileNodes} visible={open && tab === 'android'} />}
board={<>
<div className="mb-2 flex shrink-0 items-center gap-2">
{mode === 'ephemeral' ? (
diff --git a/client/src/components/workspace/WorkspaceTabs.tsx b/client/src/components/workspace/WorkspaceTabs.tsx
index 9e524ff8..16fd6121 100644
--- a/client/src/components/workspace/WorkspaceTabs.tsx
+++ b/client/src/components/workspace/WorkspaceTabs.tsx
@@ -18,7 +18,7 @@ export function WorkspaceTabs({ tab, onTabChange, browser, board, android }: {
{([
['browser', 'Browser', Globe],
['board', 'Canvas', PanelsTopLeft],
- ['android', 'Android', Smartphone],
+ ['android', 'Mobile', Smartphone],
] as const).map(([value, label, Icon]) => (
<TabsPrimitive.Trigger key={value} value={value} className="flex h-10 items-center gap-1.75 whitespace-nowrap border-b-2 border-transparent px-2.5 text-sm font-medium text-fg-muted outline-none transition-colors hover:text-fg-default focus-visible:ring-3 focus-visible:ring-ring/50 data-[state=active]:border-node-agent data-[state=active]:text-fg-default">
<Icon aria-hidden className="size-3.75" strokeWidth={1.75} />{label}
diff --git a/client/src/components/workspace/descriptors.ts b/client/src/components/workspace/descriptors.ts
new file mode 100644
index 00000000..6807d159
--- /dev/null
+++ b/client/src/components/workspace/descriptors.ts
@@ -0,0 +1,13 @@
+import type { Node } from 'reactflow';
+import { resolveNodeDescription } from '@/lib/nodeSpec';
+
+export interface WorkspaceNode { kind: string; node_id: string; label: string }
+
+/** NodeSpec owns discovery; the client only maps known renderer kinds. */
+export function workspaceNodes(nodes: Node[]): WorkspaceNode[] {
+ return nodes.flatMap((node) => {
+ const definition = resolveNodeDescription(node.type || '');
+ const kind = definition?.uiHints?.workspace?.kind;
+ return kind ? [{ kind, node_id: node.id, label: String(node.data?.label || definition?.displayName || kind) }] : [];
+ });
+}
diff --git a/client/src/features/home/__tests__/workspace.test.tsx b/client/src/features/home/__tests__/workspace.test.tsx
index 162be63c..ba6f65a4 100644
--- a/client/src/features/home/__tests__/workspace.test.tsx
+++ b/client/src/features/home/__tests__/workspace.test.tsx
@@ -161,14 +161,14 @@ describe('WorkspaceDock', () => {
expect(screen.getByText('Leo’s workspace')).toBeInTheDocument();
});
- it('mounts the selected employee’s browser nodes and keeps the Android panel available', async () => {
+ it('mounts the selected employee’s browser nodes and preserves the Mobile tab preference', async () => {
useHomeStore.setState({ workspaceOpen: true, workspaceTab: 'browser' });
renderWith([employee({ browser_nodes: [{ node_id: 'w1:browser:1', label: 'Research browser' }] })]);
expect(screen.getByText('Research browser')).toBeInTheDocument();
expect(screen.getByTestId('browser-workspace')).toHaveAttribute('data-workflow', 'w1');
expect(screen.getByTestId('browser-workspace')).toHaveAttribute('data-visible', 'true');
- await userEvent.click(screen.getByRole('tab', { name: 'Android' }));
- expect(screen.getByText('The Android mirror isn’t available yet')).toBeInTheDocument();
+ await userEvent.click(screen.getByRole('tab', { name: 'Mobile' }));
+ expect(screen.getByText('Add a Mobile Agent node to this workflow to use its phone here.')).toBeInTheDocument();
expect(useHomeStore.getState().workspaceTab).toBe('android');
expect(screen.queryByTestId('browser-workspace')).toBeNull();
});
diff --git a/client/src/features/home/data/schemas.ts b/client/src/features/home/data/schemas.ts
index a8cfd58f..6723f6e5 100644
--- a/client/src/features/home/data/schemas.ts
+++ b/client/src/features/home/data/schemas.ts
@@ -55,6 +55,7 @@ export const employeeSummarySchema = z.object({
canvas_node_id: z.string().nullable().catch(null),
/** Saved Browser nodes available in this employee's workspace. */
browser_nodes: z.array(z.object({ node_id: z.string().min(1), label: z.string() })).catch([]),
+ workspace_nodes: z.array(z.object({ kind: z.string().min(1), node_id: z.string().min(1), label: z.string() })).catch([]),
/** A browser waiting for the owner (the agent asked for help). The agent's message is in the live view only. */
browser_request: z
.object({ node_id: z.string().min(1), reason: z.string(), since: z.string().nullable().catch(null) })
diff --git a/client/src/features/home/workspace/WorkspaceDock.tsx b/client/src/features/home/workspace/WorkspaceDock.tsx
index 59747112..e704c69c 100644
--- a/client/src/features/home/workspace/WorkspaceDock.tsx
+++ b/client/src/features/home/workspace/WorkspaceDock.tsx
@@ -17,6 +17,7 @@ import type { LucideIcon } from 'lucide-react';
import { Code, Maximize2, Minimize2, Monitor, PanelsTopLeft, X } from 'lucide-react';
import BrowserWorkspace from '@/components/browser/BrowserWorkspace';
import { WorkspaceTabs } from '@/components/workspace/WorkspaceTabs';
+import MobileWorkspace from '@/components/mobile/MobileWorkspace';
import { Suspense, lazy, useCallback, useLayoutEffect, useRef, useState, type ReactNode } from 'react';
import { Button } from '@/components/ui/button';
import { Skeleton } from '@/components/ui/skeleton';
@@ -120,6 +121,7 @@ function DockTabs({ employee, visible }: { employee: EmployeeSummary; visible: b
tab={tab}
onTabChange={setTab}
browser={<BrowserWorkspace key={employee.workflow_id} workflowId={employee.workflow_id} nodes={employee.browser_nodes} visible={visible && tab === 'browser'} />}
+ android={<MobileWorkspace key={employee.workflow_id} workflowId={employee.workflow_id} nodes={employee.workspace_nodes.filter((node) => node.kind === 'mobile')} visible={visible && tab === 'android'} />}
board={<CanvasTab employee={employee} />}
/>
);
diff --git a/client/src/types/INodeProperties.ts b/client/src/types/INodeProperties.ts
index ec1fae90..e2d013bd 100644
--- a/client/src/types/INodeProperties.ts
+++ b/client/src/types/INodeProperties.ts
@@ -246,6 +246,8 @@ export interface INodeUIHints {
isCanvasPanel?: boolean;
/** Live browser view backed by the plugin's browser-session transport. */
isBrowserPanel?: boolean;
+ /** Plugin-owned Workspace renderer capability; never inferred from node type. */
+ workspace?: { kind: string };
/** Special-case panel for gmaps_create with map preview. */
showLocationPanel?: boolean;
/** ConsolePanel: this node is a chat-message target. */
diff --git a/docs/mobile-workspace.md b/docs/mobile-workspace.md
new file mode 100644
index 00000000..88f08488
--- /dev/null
+++ b/docs/mobile-workspace.md
@@ -0,0 +1,99 @@
+# Mobile Workspace
+
+Mobile Workspace runs the open-source **mobile-use** agent against one persistent, local Android emulator. The Mobile Agent is a plugin node with a Workspace capability; its task execution uses the saved workflow, node configuration, and authenticated principal. The emulator belongs to the OpenCompany installation and is shared across Mobile nodes and workflows. Tasks queue for this device.
+
+## Supported deployment
+
+The initial implementation supports **Android on a Windows host**. The OpenCompany backend, Android SDK, emulator, device broker, and mobile worker must run on the same host. A remote browser can display the Workspace, but does not supply the emulator. Only the installation owner can access Mobile Workspace.
+
+iOS is not enabled. Future iOS support requires a same-host macOS backend and Apple's simulator tooling; installing this plugin on Windows does not provide an iOS simulator. Physical devices, remote device farms, multiple concurrent emulators, and distributed mobile workers are outside this implementation.
+
+The runtime singleton and its leases are process-local. Run one backend process with its local Temporal activity worker; multiple API processes or an independently deployed Android activity worker do not share this device broker. Merely placing separate processes on the same machine does not make their runtime state shared. A future multiprocess deployment needs a dedicated broker service and explicit authenticated routing to it.
+
+## Setup and use
+
+1. Install `uv` for the optional Python environment and enable the host virtualization support required by Android Emulator. Setup downloads pinned Android command-line tools and a private Temurin Java runtime when needed. An existing SDK can be selected with `ANDROID_SDK_ROOT` or `ANDROID_HOME`; the standard Windows Android SDK location is also detected.
+2. Add a Mobile Agent node, save the workflow, and open its Mobile Workspace. The setup diagnostics inspect SDK tools, emulator acceleration, the system image, engine, and video server.
+3. Review and accept the Android SDK license in setup. Setup installs the optional engine and Android packages and creates the persistent `OpenCompany` AVD using a Pixel 7 device profile. Network access and sufficient disk space are required for these downloads.
+4. Start the shared device. The emulator runs without a separate desktop window; the Workspace shows its video. Device storage persists across stopping and starting the emulator.
+5. Take manual control to sign in to the Play Store, install applications, or upload an APK. APK uploads are limited to 256 MB and require the current manual-control lease. Play Store availability comes from the selected Google Play system image; account sign-in is performed on the device.
+6. Connect a provider in OpenCompany Settings, then connect exactly one AI model node to the Mobile Agent's Model input and save the workflow. The model selected on that connector is used for every mobile-use stage. OpenAI, Anthropic, and Gemini credentials are supported. Choose a model with image input and tool use, then submit a task from Workspace or delegate from an employee.
+
+The setup is optional. Ordinary application startup does not install the mobile engine or boot an emulator. Runtime data lives under the configured OpenCompany data directory's `mobile` subdirectory, including the engine environment, AVD, and device resource metadata. Stopping the emulator does not reset its apps or account sessions.
+
+## Control and task lifecycle
+
+Video is a read-only scrcpy transport. Mouse, touch, text, navigation, rotation, APK installation, and agent mutations go through the server's device-control broker rather than the video socket. The initial preview supports one browser tab at a time.
+
+Taking control revokes the agent's lease and capability before stopping its worker, then waits for an already admitted device mutation to drain. A monotonically increasing epoch rejects delayed inputs from an old controller. Operation identifiers deduplicate successful mutations within the current lease. An uncertain device failure fences control and requires recovery rather than replaying the mutation automatically.
+
+Resuming starts a fresh agent worker against the current device state with a new capability. This is not continuation of an in-memory model conversation. The remaining active time and step budgets are carried forward. A task can wait for human control for up to 30 minutes. Cancelling a queued task prevents it from starting; cancelling a task while a person controls the device preserves that person's lease.
+
+Workspace submissions use a client-generated submission UUID and a durable Temporal workflow identity scoped to principal, saved workflow, and node. Repeating a submission returns the existing task; reusing its UUID with a different prompt is rejected. Mobile task activities have automatic retries disabled. This is not a guarantee that arbitrary device actions are globally exactly-once across host failure.
+
+## Engine isolation and credentials
+
+OpenCompany installs mobile-use into its own Python 3.12 environment. The main server does not import its agent dependencies. Each task receives a separate worker process; a trusted device process owns the explicitly selected emulator serial. The worker's injected controller talks to a loopback capability broker. It does not choose an ADB serial or receive an arbitrary host-command API.
+
+A maintained source patch injects the controller before mobile-use's normal device initialization and redirects screenshot and platform-helper calls. It blocks SDK-side app installation and disables dotenv loading. The patch fails closed when its expected source structure changes. The upstream SDK's cancellation method alone is not used as proof that device work has stopped.
+
+The node resolves the selected provider's existing default-account credentials on the server and passes only that provider credential in the worker configuration. Credentials are not supplied by the browser and are not included in task status. Device processes receive a filtered environment. Agent telemetry and trace recording are disabled by this integration. Task screenshots and prompts still go to the configured model provider as part of inference. Custom provider base URLs are not currently exposed by the Mobile Agent configuration.
+
+## Dependency pins and attribution
+
+The optional installer verifies downloaded artifacts before extraction or execution:
+
+| Component | Pin |
+| --- | --- |
+| mobile-use source | `62913c933e21b27da353a89316a09f60525af496` |
+| Source archive SHA-256 | `e2bac1f1d47e895d8c9ec840b0e896391ffd664ed2d4149a1ff1744252ca8384` |
+| Python dependencies | Upstream lockfile, installed with `uv sync --frozen --no-dev` |
+| scrcpy server | `4.1` |
+| scrcpy artifact SHA-256 | `deacb991ed2509715160ffdc7907e47b4160eb30d1566217e9047fd5b8850cae` |
+| Android command-line tools | Windows revision `15859902` |
+| Command-line tools SHA-256 | `90ae805d20434428bffcb699c290860f19bb5f66a67e6b330067e3de801fb04a` |
+| Private Java runtime | Temurin JRE Windows x64 `21.0.12.1+1` |
+| Java archive SHA-256 | `d35f31e712f0fcf6ac5a093edc90204fbff22f720ba3950bd09d331d5e621636` |
+| Android system image | `system-images;android-36;google_apis_playstore;x86_64` |
+
+The Android package identifier selects an SDK package, not an immutable revision checksum. Updating the engine pin requires reviewing the patch, its bypass coverage, and the upstream lockfile together. Source archive extraction preserves upstream legal files; mobile-use is Apache-2.0 licensed, and its attribution and NOTICE must remain with redistribution.
+
+## Local integration versus Minitap services
+
+[mobile-use](https://github.com/minitap-ai/mobile-use) is the open-source Python agent and CLI reused here. The embedded preview, lease broker, takeover behavior, optional installation, and Workspace task routing are OpenCompany integration code.
+
+Minitap's [platform](https://www.minitap.ai/platform#slack-cli-mcp) and [miniTest documentation](https://www.minitap.ai/docs/minitest) describe separate hosted testing products, including managed runs and integrations. This local plugin does not automatically provide their device fleet, hosted test reports, Slack integration, or hosted MCP service, and does not require a Minitap account for local engine execution.
+
+## Validation and limitations
+
+Automated tests exercise fencing, successful-operation deduplication, cancellation while draining, takeover and resume with fake subprocesses, queued cancellation, source-patch compatibility, and Python syntax compilation of the patched upstream package. Additional tests use the actual loopback HTTP broker with a fake driver, reject unsafe archives and checksum mismatches, and verify that SDK tools run through Java arguments without a command shell. Patch tests require `.tmp/mobile-use-upstream` or `MOBILE_USE_UPSTREAM_SOURCE` pointing to the pinned checkout; they skip when that fixture is absent.
+
+A local smoke test executed `worker.main` profile construction, SDK initialization, a fake task, and cleanup using the actual pinned dependency environment in `.tmp/engine-smoke` and a fake controller. It made no model request and connected to no emulator.
+
+These tests do not validate a real emulator, Play Store login, APK execution, model inference, hardware acceleration, or browser video decoding. Validate those on a supported Windows host before treating this as production device automation. Current task results contain a response and run identity; the artifact list is empty, and hosted-style videos or reports are not generated.
+
+## Planned iOS implementation
+
+This is an implementation plan, not currently supported functionality. Keep the Workspace capability, task submission, ownership checks, and lease protocol shared, and add platform-specific lifecycle, driver, installation, and video adapters.
+
+1. Add a macOS-only setup diagnostic for Xcode selection, accepted Apple licenses, installed simulator runtimes, and the chosen simulator device type. Setup must ask for the required Apple license acceptance and must not promise unattended Xcode installation.
+2. Create or select one managed simulator by persisted UDID using `xcrun simctl`; explicitly boot it, wait for readiness, and shut down only the simulator owned by this installation. Never infer the target from whichever simulator happens to be booted. Preserve simulator data across normal restarts.
+3. Implement an iOS driver in a separate trusted process on the same Mac. Evaluate and pin the upstream simulator controller's IDB companion requirements. Route its screenshots, hierarchy reads, gestures, text, app lifecycle, and platform helpers through the same lease checks as Android. Do not allow the agent worker to discover devices or spawn its own companion.
+4. Add a simulator video adapter with a defined framing and geometry contract. Android scrcpy is not an iOS transport. Validate encoding, browser decoding, rotation, coordinate scaling, disconnect cleanup, and backpressure on the supported macOS and browser versions before enabling live preview.
+5. Accept simulator-compatible application builds and install them through a lease-protected `simctl` operation. Do not describe arbitrary physical-device IPA files or iOS App Store installation as supported simulator workflows. Display platform-specific app prerequisites and controls.
+6. Extend the pinned upstream patch and smoke tests for the iOS context, then execute the acceptance matrix below on a real Mac. Keep the iOS Workspace entry disabled until the lifecycle, automation, and takeover checks pass together.
+
+## Acceptance matrix
+
+| Area | Automated coverage now | Required device acceptance |
+| --- | --- | --- |
+| Setup | Checksums, archive safety, license refusal, Java argument boundary | Clean Windows x64 setup, acceleration diagnostics, SDK download failure and retry |
+| Agent adapter | Pinned-source patch compilation and real-dependency fake-task smoke | Image-capable model completes a task through the broker |
+| Device lifecycle | Runtime tests use fake processes | Start, stop, crash, restart, persistent apps and session state |
+| Ownership | Lease fencing, drain, deduplication, queue cancellation, scoped identities | Two browser tabs, delayed input, takeover during a real device mutation |
+| Human resume | Fake subprocess termination and new capability | Agent resumes from the manually changed screen within remaining budgets |
+| Video and input | Transport code is separate from mutation broker | H.264 decoding, rotation, scaled pointer coordinates, typing, disconnect cleanup |
+| Android apps | APK admission and installation are broker-owned | Install a simulator-compatible APK; manually authenticate and install from Play Store |
+| iOS | Not implemented | Same-Mac lifecycle, IDB adapter, simulator app install, preview, gestures, fenced takeover |
+
+Passing a source or fake-process test does not count as passing the corresponding device acceptance check.
diff --git a/server/nodes/mobile/__init__.py b/server/nodes/mobile/__init__.py
new file mode 100644
index 00000000..c617b766
--- /dev/null
+++ b/server/nodes/mobile/__init__.py
@@ -0,0 +1,27 @@
+"""Optional Mobile plugin. Import registers capabilities; starts no runtime."""
+
+from services.ws_handler_registry import register_router
+from services.plugin.shutdown_hooks import register_shutdown_hook
+from services.node_output_schemas import register_output_schema
+from ._node import MobileUseAgent, MobileOutput
+from ._router import router
+
+__all__ = ["MobileUseAgent", "MobileOutput"]
+
+
+async def shutdown() -> None:
+ import asyncio
+
+ from ._video import shutdown_video
+ from ._runtime import peek_runtime
+
+ runtime = peek_runtime()
+ if runtime is not None:
+ await asyncio.gather(shutdown_video(), runtime.shutdown())
+ else:
+ await shutdown_video()
+
+
+register_router(router, name="mobile")
+register_shutdown_hook("mobile", shutdown)
+register_output_schema("mobile_use_agent", MobileOutput)
diff --git a/server/nodes/mobile/_control.py b/server/nodes/mobile/_control.py
new file mode 100644
index 00000000..59202b55
--- /dev/null
+++ b/server/nodes/mobile/_control.py
@@ -0,0 +1,104 @@
+"""Device-wide arbitration, independent of transport and automation engine."""
+
+from __future__ import annotations
+
+import asyncio
+import secrets
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable
+
+
+class MobileError(ValueError):
+ def __init__(self, code: str, message: str):
+ self.code = code
+ super().__init__(message)
+
+
+@dataclass(frozen=True)
+class Lease:
+ owner: str
+ epoch: int
+
+
+class DeviceControl:
+ """Revocation happens before draining. Cancelled callers cannot release an active write."""
+
+ def __init__(self) -> None:
+ # A delayed browser request from an earlier server lifetime must not
+ # regain a valid epoch after restart. Stay within JS safe integers.
+ self.epoch = secrets.randbits(48)
+ self.owner: str | None = None
+ self.state = "idle"
+ self.lock = asyncio.Lock()
+ self.results: OrderedDict[tuple[int, str], Any] = OrderedDict()
+
+ def snapshot(self) -> dict:
+ return {"controller": self.owner, "epoch": self.epoch, "control_state": self.state}
+
+ async def claim(self, owner: str, *, takeover: bool = False) -> Lease:
+ if self.state == "recovering":
+ raise MobileError("recovering", "Restart the device connection before taking control.")
+ if self.owner == owner:
+ return Lease(owner, self.epoch)
+ if self.owner is not None and not takeover:
+ raise MobileError("device_busy", "The shared device is in use.")
+ self.epoch += 1
+ epoch = self.epoch
+ self.owner = None
+ self.state = "draining"
+ async with self.lock:
+ if epoch != self.epoch:
+ raise MobileError("superseded", "Another control request superseded this request.")
+ if self.state == "recovering":
+ raise MobileError("recovering", "A device action has an uncertain outcome.")
+ self.owner, self.state = owner, "human" if owner.startswith("viewer:") else "agent"
+ return Lease(owner, epoch)
+
+ def revoke(self, owner: str | None = None) -> None:
+ if owner is None or self.owner == owner:
+ self.epoch += 1
+ self.owner = None
+ if self.state != "recovering":
+ self.state = "idle"
+
+ async def perform(self, lease: Lease, operation_id: str, action: Callable[[], Awaitable[Any]], *, cache_result: bool = True) -> Any:
+ if not operation_id or len(operation_id) > 128:
+ raise MobileError("invalid_operation", "A bounded operation ID is required.")
+ async with self.lock:
+ if self.state == "recovering":
+ raise MobileError("recovering", "Reconnect the device before continuing.")
+ if lease.owner != self.owner or lease.epoch != self.epoch:
+ raise MobileError("stale_lease", "Device control changed. Refresh before acting.")
+ key = (lease.epoch, operation_id)
+ if cache_result and key in self.results:
+ return self.results[key]
+ task = asyncio.create_task(action())
+ cancelled = False
+ try:
+ # The lock outlives caller cancellation, including HTTP disconnects.
+ while True:
+ try:
+ result = await asyncio.shield(task)
+ break
+ except asyncio.CancelledError:
+ cancelled = True
+ if task.done():
+ result = task.result()
+ break
+ except MobileError as exc:
+ if exc.code not in {"invalid_action", "screen_changed", "unsupported_action"}:
+ self.state = "recovering"
+ self.revoke()
+ raise
+ except BaseException:
+ self.state = "recovering"
+ self.revoke()
+ raise
+ if cache_result:
+ self.results[key] = result
+ while len(self.results) > 256:
+ self.results.popitem(last=False)
+ if cancelled:
+ raise asyncio.CancelledError()
+ return result
diff --git a/server/nodes/mobile/_install.py b/server/nodes/mobile/_install.py
new file mode 100644
index 00000000..f24e550f
--- /dev/null
+++ b/server/nodes/mobile/_install.py
@@ -0,0 +1,234 @@
+"""Explicit, optional installation. Download pins are verified before extraction/execution."""
+
+from __future__ import annotations
+import asyncio
+import hashlib
+import json
+import os
+import platform
+import shutil
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+from ._paths import mobile_root, runtime_python
+from ._process import command
+
+UPSTREAM_SHA = "62913c933e21b27da353a89316a09f60525af496"
+SOURCE_SHA256 = "e2bac1f1d47e895d8c9ec840b0e896391ffd664ed2d4149a1ff1744252ca8384"
+SCRCPY_VERSION = "4.1"
+SCRCPY_SHA256 = "deacb991ed2509715160ffdc7907e47b4160eb30d1566217e9047fd5b8850cae"
+IMAGE = "system-images;android-36;google_apis_playstore;x86_64"
+AVD_NAME = "OpenCompany"
+COMMAND_TOOLS_REVISION = "15859902"
+COMMAND_TOOLS_SHA256 = "90ae805d20434428bffcb699c290860f19bb5f66a67e6b330067e3de801fb04a"
+JAVA_URL = (
+ "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.12.1%2B1/OpenJDK21U-jre_x64_windows_hotspot_21.0.12.1_1.zip"
+)
+JAVA_SHA256 = "d35f31e712f0fcf6ac5a093edc90204fbff22f720ba3950bd09d331d5e621636"
+
+
+def engine_ready() -> bool:
+ from .runtime.patch_source import PATCH_VERSION
+
+ try:
+ stamp = json.loads((mobile_root() / "engine-ready.json").read_text(encoding="utf-8"))
+ return runtime_python().is_file() and stamp == {"source": UPSTREAM_SHA, "patch": PATCH_VERSION}
+ except (OSError, ValueError):
+ return False
+
+
+def sdk_root() -> Path:
+ for value in (os.environ.get("ANDROID_SDK_ROOT"), os.environ.get("ANDROID_HOME")):
+ if value and Path(value).is_dir():
+ return Path(value).resolve()
+ default = Path(os.environ.get("LOCALAPPDATA", "")) / "Android" / "Sdk"
+ return default if os.name == "nt" and default.is_dir() else mobile_root() / "sdk"
+
+
+def sdk_tool(name: str) -> Path | None:
+ suffix = ".exe" if os.name == "nt" else ""
+ if name in {"adb", "emulator"}:
+ path = sdk_root() / ("platform-tools" if name == "adb" else "emulator") / (name + suffix)
+ return path if path.is_file() else None
+ suffix = ".bat" if os.name == "nt" else ""
+ candidates = list((sdk_root() / "cmdline-tools").glob(f"*/bin/{name}{suffix}"))
+ return sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True)[0] if candidates else None
+
+
+def sdk_command(name: str, *args: str) -> list[str]:
+ path = sdk_tool(name)
+ if path is None:
+ raise ValueError(f"Android {name} is missing. Install Android SDK command-line tools first.")
+ if os.name == "nt":
+ # Launch the documented Java entry points directly. Never feed paths or
+ # SDK package names through cmd.exe / a batch file.
+ java = next((mobile_root() / "java").glob("*/bin/java.exe"), None)
+ if java is None:
+ raise ValueError("Complete Mobile setup to install its private Java runtime")
+ main = {"sdkmanager": "com.android.sdklib.tool.sdkmanager.SdkManagerCli", "avdmanager": "com.android.sdklib.tool.AvdManagerCli"}[
+ name
+ ]
+ tools = path.parent.parent
+ return [str(java), f"-Dcom.android.sdkmanager.toolsdir={tools}", "-classpath", str(tools / "lib" / "*"), main, *args]
+ return [str(path), *args]
+
+
+def download(url: str, target: Path, digest: str, *, limit: int = 512 * 1024 * 1024) -> None:
+ target.parent.mkdir(parents=True, exist_ok=True)
+ temp = target.with_suffix(target.suffix + ".part")
+ hasher = hashlib.sha256()
+ try:
+ with urllib.request.urlopen(url, timeout=60) as response, temp.open("wb") as output:
+ size = 0
+ while data := response.read(256 * 1024):
+ size += len(data)
+ if size > limit:
+ raise ValueError("Download exceeds expected size")
+ hasher.update(data)
+ output.write(data)
+ if hasher.hexdigest() != digest:
+ raise ValueError("Runtime download checksum mismatch")
+ temp.replace(target)
+ finally:
+ temp.unlink(missing_ok=True)
+
+
+def extract_zip(archive: Path, destination: Path) -> None:
+ destination = destination.resolve()
+ with zipfile.ZipFile(archive) as source:
+ if sum(entry.file_size for entry in source.infolist()) > 2 * 1024**3:
+ raise ValueError("Runtime archive is too large")
+ for entry in source.infolist():
+ target = (destination / entry.filename).resolve()
+ if not target.is_relative_to(destination) or (entry.external_attr >> 16) & 0o170000 == 0o120000:
+ raise ValueError("Unsafe runtime archive")
+ source.extractall(destination)
+
+
+async def install_engine() -> None:
+ from .runtime.patch_source import patch
+
+ root = mobile_root()
+ source = root / "engine"
+ marker = source / ".opencompany-patch"
+ from .runtime.patch_source import PATCH_VERSION
+
+ if marker.is_file() and marker.read_text(encoding="ascii") != str(PATCH_VERSION):
+ raise ValueError(
+ "The installed mobile engine uses a different patch version. Stop the server and remove only DATA_DIR/mobile/engine, then run setup again. Device apps and sign-ins remain in DATA_DIR/mobile/avd."
+ )
+ ready = engine_ready() and marker.is_file()
+ server = root / "scrcpy-server"
+ if ready and server.is_file() and hashlib.sha256(server.read_bytes()).hexdigest() == SCRCPY_SHA256:
+ return
+ uv = os.environ.get("OPENCOMPANY_UV_BIN") or shutil.which("uv")
+ if not uv:
+ raise ValueError("uv is required for optional mobile runtime setup")
+ if not marker.is_file():
+ archive = root / "downloads" / f"mobile-use-{UPSTREAM_SHA}.zip"
+ await asyncio.to_thread(download, f"https://codeload.github.com/minitap-ai/mobile-use/zip/{UPSTREAM_SHA}", archive, SOURCE_SHA256)
+ staging = root / "source"
+ await asyncio.to_thread(extract_zip, archive, staging)
+ extracted = staging / f"mobile-use-{UPSTREAM_SHA}"
+ patch(extracted)
+ if source.exists():
+ raise ValueError("Incomplete mobile runtime exists; use Repair setup")
+ extracted.rename(source)
+ if not ready:
+ env = {**os.environ, "UV_PROJECT_ENVIRONMENT": str(source / ".venv"), "UV_NO_CONFIG": "1"}
+ env.pop("VIRTUAL_ENV", None)
+ await command([uv, "sync", "--frozen", "--no-dev", "--python", "3.12", "--project", str(source)], timeout=900, env=env)
+ stamp = root / "engine-ready.json.tmp"
+ stamp.write_text(json.dumps({"source": UPSTREAM_SHA, "patch": PATCH_VERSION}), encoding="utf-8")
+ stamp.replace(root / "engine-ready.json")
+ await asyncio.to_thread(
+ download,
+ f"https://github.com/Genymobile/scrcpy/releases/download/v{SCRCPY_VERSION}/scrcpy-server-v{SCRCPY_VERSION}",
+ server,
+ SCRCPY_SHA256,
+ )
+
+
+async def doctor() -> dict:
+ sdk = sdk_root()
+ emulator = sdk_tool("emulator")
+ try:
+ acceleration = (
+ await command([str(emulator), "-accel-check"], timeout=15, check=False) if emulator else "Android Emulator is not installed"
+ )
+ except (OSError, asyncio.TimeoutError):
+ acceleration = "Unable to check acceleration. Enable Windows Hypervisor Platform and restart Windows."
+ return {
+ "supported": sys.platform == "win32" and platform.machine().lower() in {"amd64", "x86_64"},
+ "platform": sys.platform,
+ "sdk_root": str(sdk),
+ "sdk_tools": sdk_tool("sdkmanager") is not None,
+ "adb": sdk_tool("adb") is not None,
+ "emulator": emulator is not None,
+ "engine": engine_ready(),
+ "video": (mobile_root() / "scrcpy-server").is_file(),
+ "acceleration": acceleration[-1500:],
+ "image": (sdk / Path(*IMAGE.split(";"))).is_dir() and (mobile_root() / "avd" / f"{AVD_NAME}.ini").is_file(),
+ "image_id": IMAGE,
+ "license_url": "https://developer.android.com/studio/terms",
+ }
+
+
+async def install_sdk_tools() -> None:
+ root = mobile_root()
+ if not next((root / "java").glob("*/bin/java.exe"), None):
+ archive = root / "downloads" / "temurin-jre.zip"
+ await asyncio.to_thread(download, JAVA_URL, archive, JAVA_SHA256)
+ await asyncio.to_thread(extract_zip, archive, root / "java")
+ if not sdk_tool("sdkmanager"):
+ archive = root / "downloads" / f"commandlinetools-win-{COMMAND_TOOLS_REVISION}.zip"
+ await asyncio.to_thread(
+ download,
+ f"https://dl.google.com/android/repository/commandlinetools-win-{COMMAND_TOOLS_REVISION}_latest.zip",
+ archive,
+ COMMAND_TOOLS_SHA256,
+ )
+ staging = root / "sdk-staging"
+ await asyncio.to_thread(extract_zip, archive, staging)
+ target = sdk_root() / "cmdline-tools" / COMMAND_TOOLS_REVISION
+ target.parent.mkdir(parents=True, exist_ok=True)
+ if target.exists():
+ raise ValueError("Incomplete SDK tools exist; remove that incomplete revision before retrying setup")
+ (staging / "cmdline-tools").rename(target)
+
+
+async def create_device(*, licenses_accepted: bool) -> None:
+ if sys.platform != "win32" or platform.machine().lower() not in {"amd64", "x86_64"}:
+ raise ValueError("This release supports managed Android setup on Windows x64 only")
+ if not licenses_accepted:
+ raise ValueError("Accept the Android SDK license in setup before installing packages")
+ await install_sdk_tools()
+ args = [f"--sdk_root={sdk_root()}"]
+ # Explicit user acceptance above is the only path allowed to answer these prompts.
+ # Accept only the packages selected below, not every unrelated pending SDK
+ # license in a shared Android Studio installation.
+ await command(sdk_command("sdkmanager", *args, "platform-tools", "emulator", IMAGE), timeout=1800, input_text="y\n" * 100)
+ avd_home = mobile_root() / "avd"
+ avd_home.mkdir(parents=True, exist_ok=True)
+ env = {**os.environ, "ANDROID_AVD_HOME": str(avd_home), "ANDROID_SDK_ROOT": str(sdk_root())}
+ if not (avd_home / f"{AVD_NAME}.ini").is_file():
+ await command(
+ sdk_command("avdmanager", "create", "avd", "--name", AVD_NAME, "--package", IMAGE, "--device", "pixel_7"),
+ timeout=120,
+ env=env,
+ input_text="no\n",
+ )
+ (mobile_root() / "resource.json").write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "name": AVD_NAME,
+ "platform": "android",
+ "image": IMAGE,
+ "sdk_root": str(sdk_root()),
+ "upstream_sha": UPSTREAM_SHA,
+ }
+ ),
+ encoding="utf-8",
+ )
diff --git a/server/nodes/mobile/_node.py b/server/nodes/mobile/_node.py
new file mode 100644
index 00000000..c9cd8ff6
--- /dev/null
+++ b/server/nodes/mobile/_node.py
@@ -0,0 +1,136 @@
+"""Mobile-use agent: one connector and one broker-owned Android device."""
+
+from datetime import timedelta
+import hashlib
+import json
+from typing import Any
+from pydantic import BaseModel, Field
+from services.plugin import ActionNode, NodeContext, Operation, NodeUserError
+from services.plugin.scaling import RetryPolicy, TaskQueue
+
+
+class MobileParams(BaseModel):
+ prompt: str = Field(default="", max_length=20000, json_schema_extra={"rows": 4})
+ max_steps: int = Field(default=40, ge=1, le=200)
+ timeout_s: int = Field(default=900, ge=30, le=3600)
+
+
+class MobileOutput(BaseModel):
+ response: Any = None
+ outcome: str = "completed"
+ run_id: str
+ artifacts: list = Field(default_factory=list)
+
+
+def task_identity(ctx: NodeContext) -> str:
+ # Direct Workspace invocations already carry a principal/workflow/node UUID
+ # hash used by the status/cancel endpoints. Graph and delegated executions
+ # share their parent execution ID, so include the node and tool call too.
+ if (ctx.execution_id or "").startswith("node-invoke-"):
+ return ctx.execution_id
+ identity = [ctx.user_id, ctx.workflow_id, ctx.execution_id, ctx.node_id, ctx.raw.get("tool_call_id")]
+ return "mobile-task-" + hashlib.sha256(json.dumps(identity, separators=(",", ":")).encode()).hexdigest()
+
+
+async def require_mobile_owner(principal: str) -> None:
+ from core.container import container
+ from constants import OWNER_PRINCIPAL_ID
+
+ settings = container.settings()
+ if settings.deployment_mode != "local":
+ raise NodeUserError("Mobile Workspace requires a local installation with an embedded worker")
+ if str(settings.vite_auth_enabled).lower() == "false" and principal == OWNER_PRINCIPAL_ID:
+ return
+ try:
+ user = await container.user_auth_service().get_user_by_id(int(principal))
+ except (ValueError, TypeError):
+ user = None
+ if user is None or not user.is_active or not user.is_owner:
+ raise NodeUserError("The shared mobile device is available to the installation owner only")
+
+
+async def resolve_model(ctx: NodeContext) -> dict:
+ from services.plugin.deps import get_ai_service, get_database
+ from services.llm.config import get_default_model_async
+ from services.node_registry import get_node_class
+ from services.plugin.edge_walker import edge_target_handle
+ from constants import detect_ai_provider
+
+ connections = [edge for edge in ctx.edges if edge.get("target") == ctx.node_id and edge_target_handle(edge) == "input-model"]
+ if len(connections) != 1:
+ raise NodeUserError("Connect exactly one AI model to Mobile Agent's Model input")
+ source = next((node for node in ctx.nodes if node.get("id") == connections[0].get("source")), None)
+ cls = get_node_class(source.get("type", "")) if source else None
+ if cls is None or cls.component_kind != "model" or (source.get("data") or {}).get("disabled"):
+ raise NodeUserError("Connect an enabled AI model node to Mobile Agent")
+ parameters = await get_database().get_node_parameters(source["id"]) or {}
+ provider_ref = detect_ai_provider(source["type"], parameters)
+ adapters = {
+ "openai": ("openai", "OPENAI_API_KEY"),
+ "anthropic": ("anthropic", "ANTHROPIC_API_KEY"),
+ "gemini": ("google", "GOOGLE_API_KEY"),
+ }
+ if provider_ref not in adapters:
+ raise NodeUserError("Mobile currently supports OpenAI, Anthropic, or Gemini model connectors")
+ auth = get_ai_service().auth
+ key = await auth.get_api_key(provider_ref, "default")
+ if not key:
+ raise NodeUserError("Connect the selected model provider in Settings before running Mobile")
+ models = await auth.get_stored_models(provider_ref, "default")
+ model = parameters.get("model") or (models[0] if models else await get_default_model_async(provider_ref, get_database()))
+ provider, variable = adapters[provider_ref]
+ return {"provider": provider, "model": model, "model_env": {variable: key}}
+
+
+class MobileUseAgent(ActionNode):
+ type = "mobile_use_agent"
+ display_name = "Mobile Agent"
+ description = "Run tasks on the shared Android device with mobile-use"
+ group = ("agent",)
+ component_kind = "agent"
+ supports_delegation = True
+ requires_context = True
+ needs_canvas = True
+ handles = (
+ {"name": "input-main", "kind": "input", "position": "left", "label": "Task", "role": "main"},
+ {"name": "input-model", "kind": "input", "position": "bottom", "label": "Model", "role": "model"},
+ {"name": "input-context", "kind": "input", "position": "left", "offset": "65%", "label": "Context", "role": "context"},
+ {"name": "output-main", "kind": "output", "position": "right", "label": "Output", "role": "main"},
+ {"name": "output-top", "kind": "output", "position": "top", "label": "Delegate", "role": "main"},
+ )
+ workspace_task = True
+ ui_hints = {"workspace": {"kind": "mobile"}, "width": 300, "height": 200}
+ task_queue = TaskQueue.ANDROID
+ retry_policy = RetryPolicy(maximum_attempts=1)
+ start_to_close_timeout = timedelta(hours=3)
+ Params = MobileParams
+ Output = MobileOutput
+
+ @Operation("execute")
+ async def execute_op(self, ctx: NodeContext, params: MobileParams) -> dict:
+ from ._runtime import get_runtime
+ from ._control import MobileError
+
+ await require_mobile_owner(ctx.user_id)
+ if not ctx.workflow_id or not ctx.execution_id:
+ raise NodeUserError("Mobile tasks require a saved workflow and execution identity")
+ if not params.prompt.strip():
+ raise NodeUserError("Enter a task for the mobile agent")
+ runtime = get_runtime()
+ model = await resolve_model(ctx)
+ config = params.model_dump()
+ directive = (ctx.raw.get("_raw_parameters") or {}).get("system_message")
+ if directive and directive != config["prompt"]:
+ config["prompt"] = f"{directive}\n\n{config['prompt']}"
+ try:
+ return await runtime.run(
+ principal=ctx.user_id,
+ workflow_id=ctx.workflow_id,
+ node_id=ctx.node_id,
+ run_id=task_identity(ctx),
+ params=config,
+ model=model,
+ broker_url=await runtime.ensure_broker(),
+ )
+ except MobileError as exc:
+ raise NodeUserError(str(exc)) from None
diff --git a/server/nodes/mobile/_paths.py b/server/nodes/mobile/_paths.py
new file mode 100644
index 00000000..6bc3cbf9
--- /dev/null
+++ b/server/nodes/mobile/_paths.py
@@ -0,0 +1,19 @@
+"""All optional runtime state belongs to DATA_DIR, never the source bundle."""
+
+from pathlib import Path
+
+
+def mobile_root() -> Path:
+ from core.config import Settings
+
+ return Path(Settings().data_dir).expanduser().resolve() / "mobile"
+
+
+def runtime_python() -> Path:
+ import os
+
+ return mobile_root() / "engine" / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+
+
+def resource_manifest() -> Path:
+ return mobile_root() / "resource.json"
diff --git a/server/nodes/mobile/_process.py b/server/nodes/mobile/_process.py
new file mode 100644
index 00000000..06888be1
--- /dev/null
+++ b/server/nodes/mobile/_process.py
@@ -0,0 +1,41 @@
+"""Bounded, shell-free optional runtime subprocesses."""
+
+from __future__ import annotations
+import asyncio
+import os
+import subprocess
+
+
+def hidden_options() -> dict:
+ return {"creationflags": subprocess.CREATE_NO_WINDOW} if os.name == "nt" else {}
+
+
+async def command(
+ argv: list[str],
+ *,
+ timeout: float = 30,
+ env: dict | None = None,
+ cwd: str | None = None,
+ input_text: str | None = None,
+ check: bool = True,
+) -> str:
+ proc = await asyncio.create_subprocess_exec(
+ *map(str, argv),
+ stdin=asyncio.subprocess.PIPE if input_text is not None else asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.STDOUT,
+ env=env,
+ cwd=cwd,
+ **hidden_options(),
+ )
+ try:
+ data, _ = await asyncio.wait_for(proc.communicate(input_text.encode() if input_text is not None else None), timeout)
+ except BaseException:
+ if proc.returncode is None:
+ proc.kill()
+ await proc.wait()
+ raise
+ text = data.decode("utf-8", "replace")
+ if check and proc.returncode:
+ raise RuntimeError(f"{os.path.basename(str(argv[0]))} failed ({proc.returncode}): {text[-1500:]}")
+ return text
diff --git a/server/nodes/mobile/_router.py b/server/nodes/mobile/_router.py
new file mode 100644
index 00000000..690279e4
--- /dev/null
+++ b/server/nodes/mobile/_router.py
@@ -0,0 +1,276 @@
+"""Owner-only, workflow-scoped Mobile Workspace routes."""
+
+from __future__ import annotations
+import hashlib
+import hmac
+from typing import Literal
+from uuid import UUID
+from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, Query
+from pydantic import BaseModel, ConfigDict, Field
+from services.plugin import NodeUserError
+from ._control import Lease, MobileError
+from ._node import require_mobile_owner
+
+router = APIRouter()
+BASE = "/api/mobile/{workflow_id}/{node_id}"
+
+
+class StrictBody(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class Setup(StrictBody):
+ licenses_accepted: bool = False
+
+
+class Viewer(StrictBody):
+ viewer_id: UUID
+
+
+class Release(Viewer):
+ epoch: int = Field(ge=0)
+ resume: bool = False
+
+
+class DeviceInput(Viewer):
+ epoch: int = Field(ge=0)
+ operation_id: UUID
+ operation: Literal["tap", "swipe", "text", "key", "rotate"]
+ parameters: dict = Field(default_factory=dict)
+ geometry: dict | None = None
+
+
+class Task(StrictBody):
+ prompt: str = Field(min_length=1, max_length=20000)
+ submission_id: UUID
+
+
+def viewer_identity(principal: str, viewer_id: UUID, cookies: dict) -> str:
+ """Tab identity is bound to this authenticated browser session, never a claimed owner."""
+ from core.container import container
+ from core.auth_cookies import get_session_token
+
+ settings = container.settings()
+ session = get_session_token(cookies, settings) or "local-owner"
+ message = f"mobile-viewer:{principal}:{session}:{viewer_id}".encode()
+ return hmac.new(settings.secret_key.encode(), message, hashlib.sha256).hexdigest()
+
+
+async def authorize(request: Request, workflow_id: str, node_id: str) -> str:
+ from core.container import container
+ from core.auth_cookies import get_session_token
+ from constants import OWNER_PRINCIPAL_ID
+ from services.authz.workflow_node import resolve_workflow_node
+ from services.authz.ws_session import is_allowed_ws_origin
+
+ settings = container.settings()
+ if not is_allowed_ws_origin(request.headers.get("origin"), request.headers.get("host"), settings.cors_origins or ()):
+ raise HTTPException(403, "Origin not allowed")
+ if str(settings.vite_auth_enabled).lower() == "false":
+ principal = OWNER_PRINCIPAL_ID
+ else:
+ token = get_session_token(request.cookies, settings)
+ user = await container.user_auth_service().get_current_user(token) if token else None
+ if user is None:
+ raise HTTPException(401, "Not authenticated")
+ principal = str(user.id)
+ try:
+ await require_mobile_owner(principal)
+ await resolve_workflow_node(principal, workflow_id, node_id, workspace_kind="mobile")
+ except NodeUserError as exc:
+ raise HTTPException(403, str(exc)) from None
+ return principal
+
+
+async def call(awaitable):
+ try:
+ return await awaitable
+ except (MobileError, NodeUserError) as exc:
+ raise HTTPException(409, {"code": getattr(exc, "code", "mobile_error"), "message": str(exc)}) from None
+
+
[email protected](BASE + "/status")
+async def status(principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ runtime = get_runtime()
+ if runtime.serial:
+ await call(runtime.driver_call("geometry"))
+ return runtime.snapshot()
+
+
[email protected](BASE + "/doctor")
+async def doctor(principal: str = Depends(authorize)):
+ from ._install import doctor as inspect
+
+ return await inspect()
+
+
[email protected](BASE + "/setup")
+async def setup(body: Setup, principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ runtime = get_runtime()
+ try:
+ runtime.setup(body.licenses_accepted)
+ except MobileError as exc:
+ raise HTTPException(409, str(exc)) from None
+ return runtime.snapshot()
+
+
[email protected](BASE + "/start")
+async def start(principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ return await call(get_runtime().start())
+
+
[email protected](BASE + "/stop")
+async def stop(principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ await call(get_runtime().stop())
+ return get_runtime().snapshot()
+
+
[email protected](BASE + "/takeover")
+async def takeover(body: Viewer, request: Request, principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ lease = await call(get_runtime().takeover(viewer_identity(principal, body.viewer_id, request.cookies)))
+ return {"owner": lease.owner, "epoch": lease.epoch}
+
+
[email protected](BASE + "/release")
+async def release(body: Release, request: Request, principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ runtime = get_runtime()
+ try:
+ if runtime.control.epoch != body.epoch:
+ raise MobileError("stale_lease", "Device control changed. Refresh before releasing it.")
+ runtime.release(viewer_identity(principal, body.viewer_id, request.cookies), resume=body.resume)
+ except MobileError as exc:
+ raise HTTPException(409, str(exc)) from None
+ return runtime.snapshot()
+
+
+def validate_input(body: DeviceInput) -> dict:
+ """Only explicit device operations; no shell, serial, URL, or host path from clients."""
+ import math
+
+ p = body.parameters
+ allowed = {
+ "tap": {"x", "y", "duration"},
+ "swipe": {"x", "y", "end_x", "end_y", "duration"},
+ "text": {"text"},
+ "key": {"key"},
+ "rotate": {"orientation"},
+ }[body.operation]
+ if set(p) - allowed:
+ raise HTTPException(422, "Unexpected device input fields")
+ if body.operation in {"tap", "swipe"}:
+ required = {"x", "y"} | ({"end_x", "end_y"} if body.operation == "swipe" else set())
+ if not required <= p.keys() or not body.geometry:
+ raise HTTPException(422, "Coordinates and current geometry are required")
+ if any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) for v in p.values()):
+ raise HTTPException(422, "Coordinates must be finite numbers")
+ if "duration" in p and not 0 <= p["duration"] <= 3000:
+ raise HTTPException(422, "Duration is out of range")
+ elif body.operation == "text":
+ if not isinstance(p.get("text"), str) or len(p["text"]) > 10000:
+ raise HTTPException(422, "Text must be at most 10000 characters")
+ elif body.operation == "key" and p.get("key") not in {"home", "back", "enter", "recent", "power"}:
+ raise HTTPException(422, "Unsupported key")
+ elif body.operation == "rotate" and p.get("orientation") not in {"natural", "left", "right"}:
+ raise HTTPException(422, "Unsupported orientation")
+ return p
+
+
[email protected](BASE + "/input")
+async def device_input(body: DeviceInput, request: Request, principal: str = Depends(authorize)):
+ from ._runtime import get_runtime
+
+ parameters = validate_input(body)
+ viewer = viewer_identity(principal, body.viewer_id, request.cookies)
+ result = await call(
+ get_runtime().input(Lease("viewer:" + viewer, body.epoch), body.operation, parameters, str(body.operation_id), body.geometry)
+ )
+ return {"success": True, "result": result}
+
+
[email protected](BASE + "/tasks")
+async def submit(body: Task, workflow_id: str, node_id: str, principal: str = Depends(authorize)):
+ from services.node_invocations import submit as invoke
+
+ return await call(invoke(principal, workflow_id, node_id, body.prompt, str(body.submission_id)))
+
+
[email protected](BASE + "/apk")
+async def install_apk(
+ request: Request,
+ viewer_id: UUID,
+ operation_id: UUID,
+ epoch: int = Query(..., ge=0),
+ filename: str = Query(..., max_length=255),
+ principal: str = Depends(authorize),
+):
+ from uuid import uuid4
+ from ._paths import mobile_root
+ from ._runtime import get_runtime
+
+ if not filename.lower().endswith(".apk"):
+ raise HTTPException(422, "Choose an Android APK file")
+ root = mobile_root() / "uploads"
+ root.mkdir(parents=True, exist_ok=True)
+ path = root / (uuid4().hex + ".apk")
+ try:
+ size = 0
+ with path.open("xb") as output:
+ async for chunk in request.stream():
+ size += len(chunk)
+ if size > 256 * 1024 * 1024:
+ raise HTTPException(413, "APK files must be at most 256 MiB")
+ output.write(chunk)
+ if not size:
+ raise HTTPException(422, "APK is empty")
+ viewer = viewer_identity(principal, viewer_id, request.cookies)
+ await call(get_runtime().install_apk(path, Lease("viewer:" + viewer, epoch), str(operation_id)))
+ return {"success": True}
+ finally:
+ path.unlink(missing_ok=True)
+
+
[email protected](BASE + "/tasks/{submission_id}")
+async def task_status(submission_id: UUID, workflow_id: str, node_id: str, principal: str = Depends(authorize)):
+ from services.node_invocations import status as inspect
+
+ return await call(inspect(principal, workflow_id, node_id, str(submission_id)))
+
+
[email protected](BASE + "/tasks/{submission_id}/cancel")
+async def cancel(submission_id: UUID, workflow_id: str, node_id: str, principal: str = Depends(authorize)):
+ from services.node_invocations import status as inspect, invocation_id
+ from ._runtime import get_runtime
+
+ await get_runtime().cancel(workflow_id, node_id, run_id=invocation_id(principal, workflow_id, node_id, str(submission_id)))
+ return await call(inspect(principal, workflow_id, node_id, str(submission_id), cancel=True))
+
+
[email protected]("/ws/mobile/{workflow_id}/{node_id}")
+async def video(websocket: WebSocket, workflow_id: str, node_id: str, viewer_id: UUID):
+ from core.container import container
+ from services.authz.ws_session import authenticate_ws
+ from services.authz.workflow_node import resolve_workflow_node
+ from ._video import stream_video
+
+ principal = await authenticate_ws(websocket, settings=container.settings(), user_auth_service=container.user_auth_service)
+ if principal is None:
+ return
+ try:
+ await require_mobile_owner(principal)
+ await resolve_workflow_node(principal, workflow_id, node_id, workspace_kind="mobile")
+ except NodeUserError:
+ await websocket.close(code=4003, reason="Mobile workspace access denied")
+ return
+ await stream_video(websocket, viewer_identity(principal, viewer_id, websocket.cookies))
diff --git a/server/nodes/mobile/_runtime.py b/server/nodes/mobile/_runtime.py
new file mode 100644
index 00000000..d7d926cf
--- /dev/null
+++ b/server/nodes/mobile/_runtime.py
@@ -0,0 +1,554 @@
+"""Lazy mobile broker. Owns one persistent emulator and serializes all device I/O."""
+
+from __future__ import annotations
+import asyncio
+import contextlib
+import json
+import os
+import secrets
+import socket
+import time
+from pathlib import Path
+from ._control import DeviceControl, Lease, MobileError
+from ._install import AVD_NAME, install_engine, create_device, engine_ready, sdk_tool, sdk_root
+from ._paths import mobile_root, runtime_python
+from ._process import command, hidden_options
+
+READS = frozenset({"geometry", "observe", "date", "packages", "foreground"})
+ACTIONS = frozenset({"tap", "swipe", "touch", "text", "erase", "key", "launch", "terminate", "url", "rotate"})
+
+
+class MobileRuntime:
+ def __init__(self):
+ self.control = DeviceControl()
+ self.process: asyncio.subprocess.Process | None = None
+ self.driver: asyncio.subprocess.Process | None = None
+ self.worker: asyncio.subprocess.Process | None = None
+ self.serial: str | None = None
+ self.geometry: dict | None = None
+ self.driver_lock = asyncio.Lock()
+ self.lifecycle_lock = asyncio.Lock()
+ self.task_lock = asyncio.Lock()
+ self.setup_task: asyncio.Task | None = None
+ self.setup_state = "idle"
+ self.setup_error = ""
+ self.active: dict | None = None
+ self.queue: list[dict] = []
+ self.capabilities: dict[str, tuple[str, Lease]] = {}
+ self.resume_event = asyncio.Event()
+ self.viewer: str | None = None
+ self.video_viewer: str | None = None
+ self.stopping = False
+ self.broker_runner = None
+ self.broker_url: str | None = None
+ self.broker_lock = asyncio.Lock()
+
+ async def ensure_broker(self) -> str:
+ """Private engine IPC. Capabilities never appear in URLs or public responses."""
+ async with self.broker_lock:
+ if self.broker_url:
+ return self.broker_url
+ from aiohttp import web
+
+ async def dispatch(request):
+ try:
+ capability = request.headers.get("Authorization", "").removeprefix("Bearer ")
+ grant = self.capabilities.get(capability)
+ if not grant:
+ return web.json_response({"success": False, "error": "Task capability expired"}, status=403)
+ _, lease = grant
+ data = await request.json()
+ if not isinstance(data, dict) or data.get("epoch") != lease.epoch:
+ raise MobileError("stale_lease", "Task control changed")
+ operation = data.get("operation")
+ parameters = data.get("parameters", {})
+ if not isinstance(parameters, dict):
+ raise MobileError("invalid_action", "Expected command parameters")
+ if operation in READS:
+ # Reads are fenced too: a paused task cannot keep observing passwords.
+ result = await self.control.perform(
+ lease, data.get("operation_id"), lambda: self.driver_call(operation, parameters), cache_result=False
+ )
+ else:
+ result = await self.input(lease, operation, parameters, data.get("operation_id"), data.get("geometry"))
+ return web.json_response({"success": True, "result": result})
+ except MobileError as exc:
+ return web.json_response({"success": False, "error": str(exc), "code": exc.code}, status=409)
+ except (ValueError, TypeError):
+ return web.json_response({"success": False, "error": "Invalid command"}, status=400)
+ except Exception:
+ return web.json_response({"success": False, "error": "Device connection failed"}, status=503)
+
+ app = web.Application(client_max_size=65536)
+ app.router.add_post("/control", dispatch)
+ runner = web.AppRunner(app, access_log=None, shutdown_timeout=2)
+ await runner.setup()
+ sock = socket.socket()
+ try:
+ sock.bind(("127.0.0.1", 0))
+ sock.setblocking(False)
+ port = sock.getsockname()[1]
+ await web.SockSite(runner, sock).start()
+ except BaseException:
+ sock.close()
+ await runner.cleanup()
+ raise
+ self.broker_runner = runner
+ self.broker_url = f"http://127.0.0.1:{port}/control"
+ return self.broker_url
+
+ def snapshot(self) -> dict:
+ return {
+ "running": self.process is not None and self.process.returncode is None,
+ "serial": self.serial,
+ "geometry": self.geometry,
+ "device": "Shared Android device",
+ "setup": self.setup_state,
+ "setup_error": self.setup_error,
+ "active": {k: v for k, v in (self.active or {}).items() if k not in {"prompt"}},
+ "queue": [{k: v for k, v in q.items() if k != "prompt"} for q in self.queue],
+ **self.control.snapshot(),
+ }
+
+ def setup(self, accepted: bool) -> None:
+ if not accepted:
+ raise MobileError("license_required", "Read and accept the SDK license before setup")
+ if self.setup_task and not self.setup_task.done():
+ return
+ if self.stopping or self.lifecycle_lock.locked() or self.active or self.serial:
+ raise MobileError("device_busy", "Stop the device before changing its runtime")
+
+ async def run():
+ self.setup_error = ""
+ try:
+ self.setup_state = "installing_engine"
+ await install_engine()
+ self.setup_state = "installing_device"
+ await create_device(licenses_accepted=True)
+ self.setup_state = "ready"
+ except Exception as exc:
+ self.setup_state, self.setup_error = "error", str(exc)[-1500:]
+
+ self.setup_task = asyncio.create_task(run())
+
+ async def start(self) -> dict:
+ async with self.lifecycle_lock:
+ if self.stopping or (self.setup_task and not self.setup_task.done()):
+ raise MobileError("device_busy", "Wait for setup or shutdown to finish")
+ if self.process and self.process.returncode is None:
+ return self.snapshot()
+ if self.process is not None:
+ await self._stop_driver()
+ self.process = None
+ self.serial = None
+ emulator, adb = sdk_tool("emulator"), sdk_tool("adb")
+ if not emulator or not adb or not engine_ready() or not (mobile_root() / "resource.json").is_file():
+ raise MobileError("setup_required", "Complete Mobile setup before starting the device")
+ port = None
+ for candidate in range(5560, 5680, 2):
+ held = []
+ try:
+ for n in (candidate, candidate + 1):
+ test = socket.socket()
+ held.append(test)
+ test.bind(("127.0.0.1", n))
+ port = candidate
+ break
+ except OSError:
+ pass
+ finally:
+ for test in held:
+ test.close()
+ if port is None:
+ raise MobileError("ports_busy", "No emulator port pair is free")
+ root = mobile_root()
+ root.mkdir(parents=True, exist_ok=True)
+ log = (root / "emulator.log").open("ab")
+ env = {**os.environ, "ANDROID_AVD_HOME": str(root / "avd"), "ANDROID_SDK_ROOT": str(sdk_root())}
+ try:
+ self.process = await asyncio.create_subprocess_exec(
+ str(emulator),
+ "-avd",
+ AVD_NAME,
+ "-port",
+ str(port),
+ "-no-window",
+ "-no-audio",
+ stdout=log,
+ stderr=log,
+ env=env,
+ **hidden_options(),
+ )
+ finally:
+ log.close()
+ self.serial = f"emulator-{port}"
+ try:
+ deadline = time.monotonic() + 180
+ while time.monotonic() < deadline:
+ if self.process.returncode is not None:
+ raise MobileError("boot_failed", "Emulator exited; inspect the Mobile emulator log")
+ boot = await command([str(adb), "-s", self.serial, "shell", "getprop", "sys.boot_completed"], check=False, timeout=10)
+ if boot.strip() == "1":
+ break
+ await asyncio.sleep(1)
+ else:
+ raise MobileError("boot_timeout", "Android did not finish booting within three minutes")
+ await self._start_driver()
+ self.control.revoke()
+ self.control.state = "idle"
+ except BaseException:
+ await self._stop_owned()
+ raise
+ return self.snapshot()
+
+ def driver_env(self) -> dict:
+ allowed = {
+ "PATH",
+ "SYSTEMROOT",
+ "WINDIR",
+ "COMSPEC",
+ "TEMP",
+ "TMP",
+ "TMPDIR",
+ "APPDATA",
+ "LOCALAPPDATA",
+ "PROGRAMFILES",
+ "PROGRAMFILES(X86)",
+ "PROGRAMDATA",
+ "USERPROFILE",
+ "HOME",
+ "LANG",
+ "LC_ALL",
+ "SSL_CERT_FILE",
+ "SSL_CERT_DIR",
+ "REQUESTS_CA_BUNDLE",
+ "HTTPS_PROXY",
+ "HTTP_PROXY",
+ "NO_PROXY",
+ }
+ env = {k: v for k, v in os.environ.items() if k.upper() in allowed}
+ env.update(
+ MOBILE_USE_TELEMETRY_ENABLED="false",
+ PYTHON_DOTENV_DISABLED="1",
+ PYTHONUTF8="1",
+ LANGCHAIN_TRACING_V2="false",
+ LANGSMITH_TRACING="false",
+ PATH=str(sdk_root() / "platform-tools") + os.pathsep + env.get("PATH", ""),
+ )
+ for name in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"):
+ env.pop(name, None)
+ return env
+
+ async def _start_driver(self):
+ entry = Path(__file__).with_name("runtime") / "device_server.py"
+ self.driver = await asyncio.create_subprocess_exec(
+ str(runtime_python()),
+ "-u",
+ str(entry),
+ stdin=asyncio.subprocess.PIPE,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.DEVNULL,
+ limit=8 * 1024 * 1024,
+ env=self.driver_env(),
+ cwd=str(mobile_root()),
+ **hidden_options(),
+ )
+ self.driver.stdin.write((json.dumps({"serial": self.serial, "source": str(mobile_root() / "engine")}) + "\n").encode())
+ await self.driver.stdin.drain()
+ data = await asyncio.wait_for(self.driver.stdout.readline(), 60)
+ if not data:
+ raise MobileError("driver_failed", "Android automation did not initialize")
+ reply = json.loads(data)
+ if not reply.get("success"):
+ raise MobileError("driver_failed", reply.get("error", "Android automation did not initialize"))
+ self.geometry = reply["result"]
+
+ async def driver_call(self, operation: str, parameters: dict | None = None):
+ async with self.driver_lock:
+ if not self.driver or self.driver.returncode is not None:
+ raise MobileError("device_offline", "Start the device before using it")
+ self.driver.stdin.write((json.dumps({"operation": operation, "parameters": parameters or {}}) + "\n").encode())
+ await self.driver.stdin.drain()
+ try:
+ line = await asyncio.wait_for(self.driver.stdout.readline(), 30)
+ if not line:
+ raise RuntimeError("Device driver disconnected")
+ result = json.loads(line)
+ except BaseException:
+ self.control.state = "recovering"
+ await self._stop_driver()
+ raise
+ if not result.get("success"):
+ raise MobileError(result.get("code", "device_action_failed"), result.get("error", "Device command failed"))
+ if operation == "geometry":
+ self.geometry = result["result"]
+ elif operation == "observe":
+ self.geometry = result["result"]["geometry"]
+ return result["result"]
+
+ async def input(self, lease: Lease, operation: str, parameters: dict, operation_id: str, geometry: dict | None = None):
+ if operation not in ACTIONS:
+ raise MobileError("unsupported_action", "Unsupported device command")
+ if operation in {"tap", "swipe", "touch"}:
+ parameters = {**parameters, "geometry": geometry}
+ return await self.control.perform(lease, operation_id, lambda: self.driver_call(operation, parameters))
+
+ async def install_apk(self, path: Path, lease: Lease, operation_id: str):
+ if not lease.owner.startswith("viewer:"):
+ raise MobileError("not_controller", "Take manual control before installing an APK")
+ if not self.serial or not self.process or self.process.returncode is not None:
+ raise MobileError("device_offline", "Start the shared device first")
+ uploads = (mobile_root() / "uploads").resolve()
+ path = path.resolve()
+ if not path.is_relative_to(uploads) or path.suffix.lower() != ".apk" or not path.is_file():
+ raise MobileError("invalid_action", "Expected an uploaded APK")
+
+ async def install():
+ return await command([str(sdk_tool("adb")), "-s", self.serial, "install", "-r", str(path)], timeout=180)
+
+ return await self.control.perform(lease, operation_id, install)
+
+ async def takeover(self, viewer: str) -> Lease:
+ if self.stopping or self.lifecycle_lock.locked():
+ raise MobileError("device_busy", "Wait for the device to finish starting or stopping")
+ self.control.revoke()
+ self.capabilities.clear()
+ if self.active:
+ self.active["status"] = "awaiting_user"
+ await self._stop_worker()
+ lease = await self.control.claim("viewer:" + viewer, takeover=True)
+ self.viewer = viewer
+ return lease
+
+ def release(self, viewer: str, *, resume: bool = False):
+ if self.viewer != viewer:
+ raise MobileError("not_controller", "Another viewer controls the device")
+ self.control.revoke("viewer:" + viewer)
+ self.viewer = None
+ if resume:
+ self.resume_event.set()
+
+ async def cancel(self, workflow_id: str, node_id: str, *, run_id: str | None = None):
+ def matches(entry):
+ return entry["workflow_id"] == workflow_id and entry["node_id"] == node_id and (run_id is None or entry["run_id"] == run_id)
+
+ for entry in self.queue:
+ if matches(entry):
+ entry["status"] = "cancelled"
+ if self.active and matches(self.active):
+ self.active["status"] = "cancelled"
+ self.control.revoke("run:" + self.active["run_id"])
+ self.capabilities.clear()
+ self.resume_event.set()
+ await self._stop_worker()
+
+ async def run(self, *, principal: str, workflow_id: str, node_id: str, run_id: str, params: dict, model: dict, broker_url: str) -> dict:
+ if len(self.queue) >= 32:
+ raise MobileError("queue_full", "The shared device queue is full")
+ if any(q["run_id"] == run_id for q in self.queue) or (self.active and self.active["run_id"] == run_id):
+ raise MobileError("duplicate_run", "This task is already admitted")
+ entry = {"run_id": run_id, "workflow_id": workflow_id, "node_id": node_id, "status": "queued"}
+ self.queue.append(entry)
+ try:
+ async with self.task_lock:
+ self.queue.remove(entry)
+ if entry["status"] == "cancelled":
+ raise asyncio.CancelledError()
+ if self.stopping:
+ raise MobileError("stopping", "Mobile runtime is shutting down")
+ if not self.serial:
+ raise MobileError("device_offline", "Start the shared device in Workspace first")
+ self.active = entry
+ active_remaining = params["timeout_s"]
+ steps_remaining = params["max_steps"]
+ attempts = 0
+ try:
+ while True:
+ if entry["status"] == "cancelled":
+ raise asyncio.CancelledError()
+ if self.control.owner and self.control.owner.startswith("viewer:"):
+ entry["status"] = "awaiting_user"
+ if entry["status"] == "awaiting_user":
+ await asyncio.wait_for(self.resume_event.wait(), 1800)
+ self.resume_event.clear()
+ if entry["status"] == "cancelled":
+ raise asyncio.CancelledError()
+ lease = await self.control.claim("run:" + run_id)
+ entry["status"] = "starting"
+ self.resume_event.clear()
+ capability = secrets.token_urlsafe(32)
+ self.capabilities[capability] = (run_id, lease)
+ geometry = await self.driver_call("geometry")
+ if entry["status"] == "cancelled":
+ raise asyncio.CancelledError()
+ if entry["status"] == "awaiting_user" or lease.owner != self.control.owner or lease.epoch != self.control.epoch:
+ self.capabilities.pop(capability, None)
+ entry["status"] = "awaiting_user"
+ continue
+ entry["status"] = "running"
+ config = {
+ **model,
+ "source": str(mobile_root() / "engine"),
+ "serial": self.serial,
+ "platform": "android",
+ "width": geometry["width"],
+ "height": geometry["height"],
+ "epoch": lease.epoch,
+ "capability": capability,
+ "broker_url": broker_url,
+ "prompt": params["prompt"]
+ + (
+ "\nYou are resuming after the user took manual control. Inspect the current screen first. Continue the goal from the current state; do not repeat actions already completed."
+ if attempts
+ else ""
+ ),
+ "max_steps": steps_remaining,
+ "timeout_s": active_remaining,
+ }
+ attempts += 1
+ entry["attempt_steps"] = 0
+ started = time.monotonic()
+ try:
+ try:
+ result = await self._run_worker(config, active_remaining)
+ except Exception:
+ if entry["status"] not in {"awaiting_user", "cancelled"}:
+ raise
+ result = {"success": False}
+ finally:
+ active_remaining -= time.monotonic() - started
+ steps_remaining -= max(1, entry.get("attempt_steps", 0))
+ self.capabilities.pop(capability, None)
+ await self._stop_worker()
+ if entry["status"] == "cancelled":
+ raise asyncio.CancelledError()
+ if entry["status"] == "awaiting_user":
+ if active_remaining <= 0 or steps_remaining <= 0:
+ raise MobileError("budget_exhausted", "Task exhausted its time or step budget")
+ continue
+ if not result.get("success"):
+ raise MobileError("agent_failed", result.get("error", "Mobile agent failed"))
+ return {"response": result.get("result"), "outcome": "completed", "run_id": run_id, "artifacts": []}
+ finally:
+ self.capabilities.clear()
+ self.control.revoke("run:" + run_id)
+ await self._stop_worker()
+ # Wait for any previously admitted broker write before reporting completion.
+ async with self.control.lock:
+ pass
+ self.active = None
+ finally:
+ if entry in self.queue:
+ self.queue.remove(entry)
+
+ async def _run_worker(self, config: dict, timeout: float) -> dict:
+ entry = Path(__file__).with_name("runtime") / "worker.py"
+ self.worker = await asyncio.create_subprocess_exec(
+ str(runtime_python()),
+ "-u",
+ str(entry),
+ stdin=asyncio.subprocess.PIPE,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.DEVNULL,
+ env=self.driver_env(),
+ cwd=str(mobile_root()),
+ limit=2 * 1024 * 1024,
+ **hidden_options(),
+ )
+ proc = self.worker
+ if config["capability"] not in self.capabilities or not self.active or self.active["status"] != "running":
+ await self._stop_worker()
+ return {"success": False, "error": "Task admission revoked"}
+ proc.stdin.write((json.dumps(config) + "\n").encode())
+ await proc.stdin.drain()
+ proc.stdin.close()
+
+ async def read():
+ result = {"success": False, "error": "Mobile worker exited before completing the task"}
+ while line := await proc.stdout.readline():
+ event = json.loads(line)
+ if event.get("type") == "completed":
+ result = {"success": True, "result": event.get("result")}
+ elif event.get("type") == "failed":
+ result = {"success": False, "error": event.get("error", "Worker failed")}
+ elif event.get("type") == "usage" and self.active:
+ self.active["usage"] = event.get("usage")
+ elif event.get("type") == "progress" and self.active:
+ self.active["attempt_steps"] = max(self.active.get("attempt_steps", 0), int(event.get("steps", 0)))
+ await proc.wait()
+ return result
+
+ return await asyncio.wait_for(read(), max(1, timeout) + 15)
+
+ async def _stop_worker(self):
+ proc, self.worker = self.worker, None
+ if proc and proc.returncode is None:
+ with contextlib.suppress(ProcessLookupError):
+ proc.terminate()
+ try:
+ await asyncio.wait_for(proc.wait(), 5)
+ except asyncio.TimeoutError:
+ proc.kill()
+ await proc.wait()
+
+ async def _stop_driver(self):
+ proc, self.driver = self.driver, None
+ if proc and proc.returncode is None:
+ proc.kill()
+ await proc.wait()
+
+ async def _stop_owned(self):
+ self.control.revoke()
+ self.capabilities.clear()
+ await self._stop_worker()
+ await self._stop_driver()
+ proc, self.process = self.process, None
+ if proc and proc.returncode is None:
+ if self.serial:
+ with contextlib.suppress(Exception):
+ await command([str(sdk_tool("adb")), "-s", self.serial, "emu", "kill"], timeout=2)
+ try:
+ await asyncio.wait_for(proc.wait(), 2)
+ except asyncio.TimeoutError:
+ proc.kill()
+ await proc.wait()
+ self.serial, self.geometry = None, None
+ async with self.control.lock:
+ self.control.state = "idle"
+ self.control.results.clear()
+ self.viewer = None
+ self.resume_event.clear()
+
+ async def stop(self):
+ if self.active:
+ raise MobileError("device_busy", "Cancel the active task before stopping the device")
+ async with self.lifecycle_lock:
+ await self._stop_owned()
+
+ async def shutdown(self):
+ self.stopping = True
+ if self.setup_task and not self.setup_task.done():
+ self.setup_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self.setup_task
+ if self.active:
+ self.active["status"] = "cancelled"
+ self.resume_event.set()
+ await self._stop_owned()
+ if self.broker_runner:
+ await self.broker_runner.cleanup()
+ self.broker_runner, self.broker_url = None, None
+
+
+_runtime: MobileRuntime | None = None
+
+
+def get_runtime() -> MobileRuntime:
+ global _runtime
+ if _runtime is None:
+ _runtime = MobileRuntime()
+ return _runtime
+
+
+def peek_runtime() -> MobileRuntime | None:
+ return _runtime
diff --git a/server/nodes/mobile/_video.py b/server/nodes/mobile/_video.py
new file mode 100644
index 00000000..1fdda856
--- /dev/null
+++ b/server/nodes/mobile/_video.py
@@ -0,0 +1,161 @@
+"""Pinned scrcpy video-only transport. Device control never enters this socket.
+
+The client uses the matching maintained Tango parser. We remove only scrcpy's
+forward-tunnel sentinel; metadata and encoded packet bytes pass through intact.
+"""
+
+from __future__ import annotations
+import asyncio
+import contextlib
+import hashlib
+import secrets
+from fastapi import WebSocket, WebSocketDisconnect
+from ._control import MobileError
+from ._install import SCRCPY_SHA256, SCRCPY_VERSION, sdk_tool
+from ._paths import mobile_root
+from ._process import command, hidden_options
+
+_streams: set[asyncio.Task] = set()
+
+
+async def shutdown_video() -> None:
+ tasks = list(_streams)
+ for task in tasks:
+ task.cancel()
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+
+def server_arguments(serial: str, scid: str) -> list[str]:
+ return [
+ str(sdk_tool("adb")),
+ "-s",
+ serial,
+ "shell",
+ "CLASSPATH=/data/local/tmp/opencompany-scrcpy.jar",
+ "app_process",
+ "/",
+ "com.genymobile.scrcpy.Server",
+ SCRCPY_VERSION,
+ f"scid={scid}",
+ "tunnel_forward=true",
+ "audio=false",
+ "control=false",
+ "cleanup=false",
+ "power_on=false",
+ "video_codec=h264",
+ "max_size=1280",
+ "max_fps=30",
+ "video_bit_rate=4000000",
+ "send_dummy_byte=true",
+ "send_device_meta=true",
+ "send_stream_meta=true",
+ "send_frame_meta=true",
+ ]
+
+
+async def stream_video(websocket: WebSocket, viewer: str) -> None:
+ from ._runtime import get_runtime
+
+ runtime = get_runtime()
+ if runtime.video_viewer is not None:
+ await websocket.close(code=4009, reason="The shared preview is open in another tab")
+ return
+ if not runtime.serial or not sdk_tool("adb"):
+ await websocket.close(code=4004, reason="Start the device first")
+ return
+ runtime.video_viewer = viewer
+ current = asyncio.current_task()
+ _streams.add(current)
+ proc = writer = None
+ jobs: list[asyncio.Task] = []
+ port = None
+ serial = runtime.serial
+ adb = str(sdk_tool("adb"))
+ try:
+ server = mobile_root() / "scrcpy-server"
+ if not server.is_file() or await asyncio.to_thread(lambda: hashlib.sha256(server.read_bytes()).hexdigest()) != SCRCPY_SHA256:
+ raise MobileError("video_setup_required", "Repair Mobile setup to restore the pinned video server")
+ await command([adb, "-s", serial, "push", str(server), "/data/local/tmp/opencompany-scrcpy.jar"], timeout=30)
+ scid = f"{secrets.randbelow(2**31):08x}"
+ raw_port = await command([adb, "-s", serial, "forward", "tcp:0", f"localabstract:scrcpy_{scid}"])
+ port = int(raw_port.strip())
+ if not 1 <= port <= 65535:
+ raise MobileError("video_tunnel_failed", "Invalid video tunnel port")
+ proc = await asyncio.create_subprocess_exec(
+ *server_arguments(serial, scid),
+ stdin=asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.DEVNULL,
+ stderr=asyncio.subprocess.DEVNULL,
+ **hidden_options(),
+ )
+ reader = None
+ for _ in range(60):
+ if proc.returncode is not None:
+ break
+ try:
+ reader, writer = await asyncio.wait_for(asyncio.open_connection("127.0.0.1", port, limit=256 * 1024), 1)
+ sentinel = await asyncio.wait_for(reader.readexactly(1), 1)
+ if sentinel != b"\x00":
+ raise MobileError("video_protocol_error", "Unexpected scrcpy handshake")
+ break
+ except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError):
+ if writer is not None:
+ writer.close()
+ with contextlib.suppress(Exception):
+ await writer.wait_closed()
+ writer = None
+ reader = None
+ await asyncio.sleep(0.1)
+ if reader is None:
+ raise MobileError("video_failed", "The device video stream did not start")
+ await websocket.accept()
+ await websocket.send_json({"type": "video", "version": SCRCPY_VERSION, "codec": "h264"})
+
+ async def relay():
+ while chunk := await reader.read(64 * 1024):
+ # TCP backpressure and bounded send time: never build an unbounded
+ # browser queue or discard H.264 delta packets. Reconnect starts
+ # a fresh encoder/config/keyframe when a viewer falls behind.
+ await asyncio.wait_for(websocket.send_bytes(chunk), 2)
+
+ async def disconnected():
+ while True:
+ message = await websocket.receive()
+ if message["type"] == "websocket.disconnect":
+ return
+ # This is receive-only video. Even a valid owner must use the
+ # HTTP broker routes and lease for commands.
+ await websocket.close(code=1008, reason="Video socket accepts no commands")
+ return
+
+ jobs = [asyncio.create_task(relay()), asyncio.create_task(disconnected())]
+ done, _ = await asyncio.wait(jobs, return_when=asyncio.FIRST_COMPLETED)
+ for job in done:
+ job.result()
+ except (MobileError, OSError, ValueError, asyncio.TimeoutError, WebSocketDisconnect) as exc:
+ with contextlib.suppress(Exception):
+ await websocket.close(code=1011, reason=str(exc)[:110])
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close(code=1000)
+ for job in jobs:
+ job.cancel()
+ if jobs:
+ await asyncio.gather(*jobs, return_exceptions=True)
+ if writer is not None:
+ writer.close()
+ with contextlib.suppress(Exception):
+ await writer.wait_closed()
+ if proc is not None and proc.returncode is None:
+ proc.kill()
+ await proc.wait()
+ if port is not None:
+ with contextlib.suppress(Exception):
+ await command([adb, "-s", serial, "forward", "--remove", f"tcp:{port}"], timeout=5)
+ if runtime.video_viewer == viewer:
+ runtime.video_viewer = None
+ # Video reconnects independently of manual control. The UI releases
+ # its exact epoch through the broker; an old stream must never revoke
+ # a newer lease held by the same tab.
+ _streams.discard(current)
diff --git a/server/nodes/mobile/meta.json b/server/nodes/mobile/meta.json
new file mode 100644
index 00000000..42a60471
--- /dev/null
+++ b/server/nodes/mobile/meta.json
@@ -0,0 +1,4 @@
+{
+ "color": "#22c55e",
+ "icon": "lucide:Smartphone"
+}
diff --git a/server/nodes/mobile/runtime/device_server.py b/server/nodes/mobile/runtime/device_server.py
new file mode 100644
index 00000000..1f3ff66e
--- /dev/null
+++ b/server/nodes/mobile/runtime/device_server.py
@@ -0,0 +1,139 @@
+"""Persistent optional driver process. Stdio is private to the device broker."""
+
+from __future__ import annotations
+import base64
+import json
+import re
+import sys
+from io import BytesIO
+from urllib.parse import urlsplit
+
+WIRE = sys.stdout
+
+
+class AndroidDriver:
+ def __init__(self, serial: str, source: str):
+ if not re.fullmatch(r"emulator-\d+", serial):
+ raise ValueError("Only the explicitly managed emulator is supported")
+ sys.path.insert(0, source)
+ import uiautomator2 as u2
+
+ self.device = u2.connect(serial)
+ if "dev.mobile.maestro" in self.device.app_list():
+ raise ValueError("Maestro's helper conflicts with mobile automation. Remove it manually before connecting.")
+
+ def geometry(self):
+ info = self.device.info
+ width, height = self.device.window_size()
+ return {"width": width, "height": height, "rotation": info.get("displayRotation", 0)}
+
+ def call(self, op: str, p: dict):
+ d = self.device
+ geometry = self.geometry()
+ if op in {"tap", "swipe", "touch"}:
+ if p.pop("geometry", None) != geometry:
+ raise ValueError("Screen changed; refresh before interacting")
+ for key, limit in (
+ ("x", geometry["width"]),
+ ("y", geometry["height"]),
+ ("end_x", geometry["width"]),
+ ("end_y", geometry["height"]),
+ ):
+ if key in p and not 0 <= float(p[key]) < limit:
+ raise ValueError("Coordinates outside device screen")
+ if op == "geometry":
+ return geometry
+ if op == "observe":
+ from minitap.mobile_use.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
+
+ output = BytesIO()
+ d.screenshot().save(output, "PNG")
+ elements = _parse_hierarchy_xml_to_elements(d.dump_hierarchy())
+ return {
+ "base64": base64.b64encode(output.getvalue()).decode(),
+ "elements": elements,
+ "width": geometry["width"],
+ "height": geometry["height"],
+ "platform": "android",
+ "geometry": geometry,
+ }
+ if op == "date":
+ return d.shell(["date"]).output
+ if op == "packages":
+ return "\n".join(sorted(d.app_list()))
+ if op == "foreground":
+ return d.app_current().get("package", "")
+ if op == "tap":
+ duration = min(max(float(p.get("duration", 0)), 0), 3000)
+ if duration:
+ d.long_click(p["x"], p["y"], duration / 1000)
+ else:
+ d.click(p["x"], p["y"])
+ elif op == "swipe":
+ d.swipe(p["x"], p["y"], p["end_x"], p["end_y"], min(max(float(p.get("duration", 400)), 50), 3000) / 1000)
+ elif op == "touch":
+ action = p.get("action")
+ if action not in {"down", "move", "up"}:
+ raise ValueError("Invalid touch action")
+ getattr(d.touch, action)(p["x"], p["y"])
+ elif op == "text":
+ text = str(p.get("text", ""))
+ if len(text) > 10000:
+ raise ValueError("Text too long")
+ d.send_keys(text, clear=False)
+ elif op == "erase":
+ count = min(max(int(p.get("count", 50)), 1), 1000)
+ d.shell(["input", "keyevent", *(["KEYCODE_DEL"] * count)])
+ elif op == "key":
+ key = p.get("key")
+ if key not in {"home", "back", "enter", "recent", "power"}:
+ raise ValueError("Unsupported key")
+ d.press(key)
+ elif op in {"launch", "terminate"}:
+ package = p.get("package") or d.app_current().get("package")
+ if not package or not re.fullmatch(r"[A-Za-z][A-Za-z0-9_.]+", package):
+ raise ValueError("Invalid package")
+ if op == "launch":
+ d.app_start(package)
+ else:
+ d.app_stop(package)
+ elif op == "url":
+ url = str(p.get("url", ""))
+ if urlsplit(url).scheme not in {"https", "http"} or len(url) > 4096:
+ raise ValueError("Unsupported URL")
+ d.shell(["am", "start", "-a", "android.intent.action.VIEW", "-d", url])
+ elif op == "rotate":
+ orientation = p.get("orientation")
+ if orientation not in {"natural", "left", "right"}:
+ raise ValueError("Invalid orientation")
+ d.set_orientation(orientation)
+ else:
+ raise ValueError("Unsupported device operation")
+ return True
+
+
+def main():
+ sys.stdout = sys.stderr
+ config = json.loads(sys.stdin.readline())
+ try:
+ driver = AndroidDriver(config["serial"], config["source"])
+ print(json.dumps({"success": True, "result": driver.geometry()}), file=WIRE, flush=True)
+ except Exception as exc:
+ print(json.dumps({"success": False, "error": str(exc)[:500]}), file=WIRE, flush=True)
+ return
+ for line in sys.stdin:
+ try:
+ request = json.loads(line)
+ result = driver.call(request["operation"], request.get("parameters", {}))
+ reply = {"success": True, "result": result}
+ except Exception as exc:
+ reply = {
+ "success": False,
+ "error": str(exc)[:500],
+ "code": "invalid_action" if isinstance(exc, (ValueError, KeyError, TypeError)) else "device_action_failed",
+ }
+ print(json.dumps(reply), file=WIRE, flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server/nodes/mobile/runtime/patch_source.py b/server/nodes/mobile/runtime/patch_source.py
new file mode 100644
index 00000000..441242bc
--- /dev/null
+++ b/server/nodes/mobile/runtime/patch_source.py
@@ -0,0 +1,109 @@
+"""Auditable source patch for the pinned mobile-use archive; not runtime monkeypatching."""
+
+from __future__ import annotations
+import ast
+from pathlib import Path
+
+PATCH_VERSION = 1
+
+
+def patch(root: Path) -> None:
+ base = root / "minitap" / "mobile_use"
+
+ def replace(file: str, old: str, new: str) -> None:
+ path = base / file
+ source = path.read_text(encoding="utf-8")
+ if source.count(old) != 1:
+ raise ValueError(f"Upstream compatibility mismatch: {file}: {old[:60]}")
+ path.write_text(source.replace(old, new), encoding="utf-8")
+
+ def prepend(file: str, name: str, body: str) -> None:
+ path = base / file
+ source = path.read_text(encoding="utf-8")
+ tree = ast.parse(source)
+ matches = [n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name]
+ if len(matches) != 1:
+ raise ValueError(f"Expected one {name} in {file}")
+ statements = matches[0].body
+ first = (
+ statements[1]
+ if isinstance(statements[0], ast.Expr)
+ and isinstance(statements[0].value, ast.Constant)
+ and isinstance(statements[0].value.value, str)
+ else statements[0]
+ )
+ lines = source.splitlines(keepends=True)
+ indent = " " * first.col_offset
+ lines.insert(first.lineno - 1, "".join(indent + line + "\n" for line in body.splitlines()))
+ path.write_text("".join(lines), encoding="utf-8")
+
+ replace("sdk/types/agent.py", "from typing import Literal", "from typing import Any, Literal")
+ replace(
+ "sdk/types/agent.py",
+ " agent_profiles: dict[str, AgentProfile]",
+ " local_controller: Any = None\n local_device_context: Any = None\n agent_profiles: dict[str, AgentProfile]",
+ )
+ replace("context.py", "from typing import Literal", "from typing import Any, Literal")
+ replace("context.py", " trace_id: str", " local_controller: Any = None\n trace_id: str")
+ replace("context.py", 'Literal["WINDOWS", "LINUX"]', 'Literal["WINDOWS", "LINUX", "MACOS"]')
+ prepend(
+ "sdk/agent.py",
+ "_init_internal",
+ "if self._config.local_controller is not None:\n self._device_context = self._config.local_device_context\n self._adb_client = self._ui_adb_client = self._ios_client = None\n self._cloud_controller = None\n self._initialized = True\n return True",
+ )
+ replace(
+ "sdk/agent.py",
+ " context = MobileUseContext(\n",
+ " context = MobileUseContext(\n local_controller=self._config.local_controller,\n",
+ )
+ prepend(
+ "controllers/controller_factory.py",
+ "create_device_controller",
+ "if ctx.local_controller is not None:\n return ctx.local_controller",
+ )
+ for name, operation, asynchronous in [
+ ("get_device_date", "date", False),
+ ("list_packages", "packages", False),
+ ("list_packages_async", "packages", True),
+ ("get_current_foreground_package", "foreground", False),
+ ("get_current_foreground_package_async", "foreground", True),
+ ]:
+ call = f'await ctx.local_controller.read("{operation}")' if asynchronous else f'ctx.local_controller.read_sync("{operation}")'
+ prepend("controllers/platform_specific_commands_controller.py", name, f"if ctx.local_controller is not None:\n return {call}")
+ prepend(
+ "sdk/agent.py",
+ "_install_apk_internal",
+ "if self._config.local_controller is not None:\n raise RuntimeError('Install applications through OpenCompany Workspace, not an agent task')",
+ )
+ prepend(
+ "sdk/agent.py",
+ "_install_ios_app",
+ "if self._config.local_controller is not None:\n raise RuntimeError('Install applications through OpenCompany Workspace')",
+ )
+ prepend(
+ "sdk/agent.py",
+ "get_screenshot",
+ "if self._config.local_controller is not None:\n import base64\n return Image.open(BytesIO(base64.b64decode(await self._config.local_controller.screenshot())))",
+ )
+ prepend(
+ "sdk/agent.py",
+ "clean",
+ "if self._config.local_controller is not None:\n await self._config.local_controller.cleanup()\n self._initialized = False\n return",
+ )
+ prepend(
+ "clients/ui_automator_client.py",
+ "_ensure_maestro_not_installed",
+ "if _is_package_installed(device_id, MAESTRO_PACKAGE):\n raise RuntimeError('Maestro conflicts with this automation driver. Remove its helper manually or use another device.')\nreturn",
+ )
+ replace("config.py", '"env_file": ".env"', '"env_file": None')
+ # Both SDK and config call load_dotenv; remove discovery even if dependency
+ # versions change the meaning of PYTHON_DOTENV_DISABLED.
+ for name in ("config.py", "sdk/agent.py"):
+ path = base / name
+ text = path.read_text(encoding="utf-8")
+ text = text.replace("load_dotenv(verbose=True)", "# OpenCompany supplies explicit process configuration.").replace(
+ "load_dotenv()", "# OpenCompany supplies explicit process configuration."
+ )
+ ast.parse(text)
+ path.write_text(text, encoding="utf-8")
+ (root / ".opencompany-patch").write_text(str(PATCH_VERSION), encoding="ascii")
diff --git a/server/nodes/mobile/runtime/worker.py b/server/nodes/mobile/runtime/worker.py
new file mode 100644
index 00000000..99e62aa1
--- /dev/null
+++ b/server/nodes/mobile/runtime/worker.py
@@ -0,0 +1,210 @@
+"""One trusted mobile-use task. No device clients: every action crosses the broker."""
+
+from __future__ import annotations
+import asyncio
+import base64
+import json
+import os
+import re
+import sys
+import urllib.request
+import uuid
+from io import BytesIO
+
+WIRE = sys.stdout
+
+
+def emit(kind: str, **data) -> None:
+ WIRE.write(json.dumps({"v": 1, "type": kind, **data}, default=str) + "\n")
+ WIRE.flush()
+
+
+class Controller:
+ def __init__(self, config: dict):
+ self.config = config
+ self.geometry = None
+
+ def read_sync(self, operation: str, **params):
+ payload = {
+ "operation": operation,
+ "parameters": params,
+ "epoch": self.config["epoch"],
+ "operation_id": str(uuid.uuid4()),
+ "geometry": self.geometry,
+ }
+ req = urllib.request.Request(
+ self.config["broker_url"],
+ json.dumps(payload).encode(),
+ {"Content-Type": "application/json", "Authorization": f"Bearer {self.config['capability']}"},
+ )
+ with urllib.request.urlopen(req, timeout=35) as response:
+ result = json.load(response)
+ if not result.get("success"):
+ raise RuntimeError(result.get("error", "Device command failed"))
+ return result["result"]
+
+ async def read(self, operation: str, **params):
+ return await asyncio.to_thread(self.read_sync, operation, **params)
+
+ async def get_screen_data(self):
+ from minitap.mobile_use.controllers.device_controller import ScreenDataResponse
+
+ result = await self.read("observe")
+ self.geometry = result.pop("geometry")
+ return ScreenDataResponse(**result)
+
+ async def screenshot(self):
+ return (await self.get_screen_data()).base64
+
+ async def get_ui_hierarchy(self):
+ return (await self.get_screen_data()).elements
+
+ async def tap(self, coords, long_press=False, long_press_duration=1000):
+ from minitap.mobile_use.controllers.types import TapOutput
+
+ await self.read("tap", x=coords.x, y=coords.y, duration=long_press_duration if long_press else 0)
+ return TapOutput(error=None)
+
+ async def swipe(self, start, end, duration=400):
+ await self.read("swipe", x=start.x, y=start.y, end_x=end.x, end_y=end.y, duration=duration)
+ return None
+
+ async def input_text(self, text):
+ return await self.read("text", text=text)
+
+ async def launch_app(self, package_or_bundle_id):
+ return await self.read("launch", package=package_or_bundle_id)
+
+ async def terminate_app(self, package_or_bundle_id):
+ return await self.read("terminate", package=package_or_bundle_id)
+
+ async def open_url(self, url):
+ return await self.read("url", url=url)
+
+ async def press_back(self):
+ return await self.read("key", key="back")
+
+ async def press_home(self):
+ return await self.read("key", key="home")
+
+ async def press_enter(self):
+ return await self.read("key", key="enter")
+
+ async def erase_text(self, nb_chars=None):
+ return await self.read("erase", count=nb_chars or 50)
+
+ async def cleanup(self):
+ pass # broker owns the persistent device
+
+ def find_element(self, ui_hierarchy, resource_id=None, text=None, index=0):
+ from minitap.mobile_use.controllers.types import Bounds
+
+ matches = [
+ e
+ for e in ui_hierarchy
+ if (resource_id and e.get("resource-id") == resource_id)
+ or (text and (e.get("text") == text or e.get("accessibilityText") == text))
+ ]
+ if index < 0 or index >= len(matches):
+ return None, None, "Element not found"
+ element = matches[index]
+ points = re.fullmatch(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", element.get("bounds", ""))
+ bounds = Bounds(**dict(zip(("x1", "y1", "x2", "y2"), map(int, points.groups())))) if points else None
+ return element, bounds, None
+
+ def get_compressed_b64_screenshot(self, image_base64, quality=50):
+ from PIL import Image
+
+ output = BytesIO()
+ Image.open(BytesIO(base64.b64decode(image_base64))).convert("RGB").save(output, "JPEG", quality=quality)
+ return base64.b64encode(output.getvalue()).decode()
+
+ async def start_video_recording(self, max_duration_seconds=900):
+ raise RuntimeError("Recording is not enabled; use Workspace preview")
+
+ async def stop_video_recording(self):
+ raise RuntimeError("Recording is not enabled")
+
+
+async def main(config: dict) -> None:
+ sys.stdout = sys.stderr # upstream diagnostics can never corrupt the protocol
+ os.environ.update(config.pop("model_env"))
+ os.environ["MOBILE_USE_TELEMETRY_ENABLED"] = "false"
+ os.environ["PYTHON_DOTENV_DISABLED"] = "1"
+ sys.path.insert(0, config["source"])
+ from minitap.mobile_use.config import LLM, LLMWithFallback, LLMConfig, LLMConfigUtils
+ from minitap.mobile_use.context import DeviceContext, DevicePlatform
+ from minitap.mobile_use.sdk.agent import Agent
+ from minitap.mobile_use.sdk.builders.agent_config_builder import AgentConfigBuilder
+ from minitap.mobile_use.sdk.types.task import AgentProfile, TaskRequest
+ from langchain_core.callbacks import BaseCallbackHandler
+
+ class Progress(BaseCallbackHandler):
+ def on_chain_start(self, serialized, inputs, *, metadata=None, **kwargs):
+ step = (metadata or {}).get("langgraph_step")
+ if isinstance(step, int):
+ emit("progress", steps=step + 1)
+
+ def on_llm_end(self, response, **kwargs):
+ output = response.llm_output or {}
+ usage = output.get("token_usage") or output.get("usage") or {}
+ if usage:
+ emit("usage", usage=usage)
+
+ model = LLM(provider=config["provider"], model=config["model"])
+ stage = LLMWithFallback(provider=model.provider, model=model.model, fallback=model)
+ profile = AgentProfile(
+ name="opencompany",
+ llm_config=LLMConfig(
+ planner=stage,
+ orchestrator=stage,
+ contextor=stage,
+ cortex=stage,
+ executor=stage,
+ utils=LLMConfigUtils(outputter=stage, hopper=stage),
+ ),
+ )
+ agent_config = (
+ AgentConfigBuilder()
+ .with_default_profile(profile)
+ .for_device(DevicePlatform(config["platform"]), config["serial"])
+ .with_graph_config_callbacks([Progress()])
+ .build()
+ )
+ agent_config.local_controller = Controller(config)
+ agent_config.local_device_context = DeviceContext(
+ host_platform="WINDOWS" if sys.platform == "win32" else "MACOS" if sys.platform == "darwin" else "LINUX",
+ mobile_platform=DevicePlatform(config["platform"]),
+ device_id=config["serial"],
+ device_width=config["width"],
+ device_height=config["height"],
+ )
+ agent = Agent(config=agent_config)
+ await agent.init()
+ emit("ready")
+ try:
+ result = await asyncio.wait_for(
+ agent.run_task(request=TaskRequest(goal=config["prompt"], max_steps=config["max_steps"], record_trace=False)),
+ config["timeout_s"],
+ )
+ emit("completed", result=result)
+ finally:
+ await agent.clean()
+
+
+if __name__ == "__main__":
+ try:
+ asyncio.run(main(json.loads(sys.stdin.readline())))
+ except BaseException as exc:
+ # Provider errors may echo credentials; expose only an actionable category.
+ # Prompts/screenshots in upstream stderr are not persisted.
+ kind = type(exc).__name__
+ message = {
+ "AuthenticationError": "The selected model rejected its credentials.",
+ "RateLimitError": "The selected model is rate limited. Try again later.",
+ "TimeoutError": "The mobile task exceeded its active time budget.",
+ "GraphRecursionError": "The mobile task exhausted its step budget.",
+ "HTTPError": "Device control changed or the device connection failed.",
+ }.get(kind, f"Mobile engine failed ({kind}).")
+ emit("failed", error=message)
+ sys.exit(1)
diff --git a/server/services/ai.py b/server/services/ai.py
index 591aef1e..88b03556 100644
--- a/server/services/ai.py
+++ b/server/services/ai.py
@@ -2431,9 +2431,9 @@ class AIService:
# For AI Agent nodes, enhance description with child agent's tool capabilities
# This allows parent agent to know what the child agent can do
- from constants import AI_AGENT_TYPES
+ from services.workspace_capabilities import is_registered_agent
- if node_type in AI_AGENT_TYPES:
+ if is_registered_agent(node_type):
child_tools = tool_info.get("child_tools", [])
if child_tools:
# Build capability description from child's connected tools
diff --git a/server/services/authz/workflow_node.py b/server/services/authz/workflow_node.py
new file mode 100644
index 00000000..547aca60
--- /dev/null
+++ b/server/services/authz/workflow_node.py
@@ -0,0 +1,27 @@
+"""Server-owned workflow and capability resolution for Workspace plugins."""
+
+from typing import Any
+
+
+async def resolve_workflow_node(
+ principal: str, workflow_id: str, node_id: str, *, workspace_kind: str | None = None
+) -> tuple[Any, dict, dict]:
+ from services.plugin import NodeUserError
+ from services.plugin.deps import get_database
+ from services.workspace_capabilities import workspace_nodes
+
+ if not principal or not workflow_id or not node_id:
+ raise NodeUserError("Authenticated workflow and node identity required")
+ saved = await get_database().get_workflow(workflow_id)
+ if saved is None:
+ raise NodeUserError("Workflow not found")
+ graph = saved.data if hasattr(saved, "data") else saved.get("data", saved)
+ owner = str(graph.get("owner_id") or "")
+ if owner and owner != principal:
+ raise NodeUserError("Workflow access denied")
+ matches = [n for n in graph.get("nodes", []) if n.get("id") == node_id]
+ if len(matches) != 1:
+ raise NodeUserError("Node does not belong to workflow")
+ if workspace_kind and not any(n["node_id"] == node_id and n["kind"] == workspace_kind for n in workspace_nodes(graph)):
+ raise NodeUserError("Node does not provide this Workspace capability")
+ return saved, graph, matches[0]
diff --git a/server/services/employees/graph_index.py b/server/services/employees/graph_index.py
index 8edd89a6..51f16df2 100644
--- a/server/services/employees/graph_index.py
+++ b/server/services/employees/graph_index.py
@@ -14,7 +14,8 @@ from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Dict, List, Mapping, Optional, Tuple
-from constants import AI_AGENT_TYPES, WORKFLOW_TRIGGER_TYPES
+from constants import WORKFLOW_TRIGGER_TYPES
+from services.workspace_capabilities import is_registered_agent
from services.employees.apps import AppSpec, app_for_node_type
from services.node_registry import get_node_class
@@ -88,7 +89,7 @@ def index_graph(graph: Optional[Mapping[str, Any]]) -> GraphIndex:
browsers.append(node_id)
if isinstance(data, Mapping) and data.get("disabled"):
continue
- if node_type in AI_AGENT_TYPES:
+ if is_registered_agent(node_type):
agents.append(node_id)
if node_type in WORKFLOW_TRIGGER_TYPES:
triggers.append(node_id)
diff --git a/server/services/employees/summaries.py b/server/services/employees/summaries.py
index 0cae4aed..d9d00001 100644
--- a/server/services/employees/summaries.py
+++ b/server/services/employees/summaries.py
@@ -229,6 +229,8 @@ async def _summary(
pending: int,
done_today: int,
) -> Dict[str, Any]:
+ from services.workspace_capabilities import workspace_nodes
+
graph = index_graph(getattr(workflow, "data", None))
control = serialize_control(control_row)
control.setdefault("workflow_id", workflow.id)
@@ -270,6 +272,7 @@ async def _summary(
"control": control,
"watch_node_ids": watch,
"canvas_node_id": canvas_node_id,
+ "workspace_nodes": workspace_nodes(getattr(workflow, "data", None) or {}),
"browser_nodes": [{"node_id": node_id, "label": graph.labels.get(node_id) or "Browser"} for node_id in graph.browser_ids],
#: A browser node waiting for the owner ({node_id, reason, since}),
#: else None. Read live, so it changes without a new revision.
diff --git a/server/services/handlers/tools.py b/server/services/handlers/tools.py
index e416697d..a0db6201 100644
--- a/server/services/handlers/tools.py
+++ b/server/services/handlers/tools.py
@@ -11,7 +11,7 @@ import hashlib
from typing import Dict, Any, Optional, List, Tuple, TYPE_CHECKING
from core.logging import get_logger
-from constants import AI_AGENT_TYPES, ANDROID_SERVICE_NODE_TYPES
+from constants import ANDROID_SERVICE_NODE_TYPES
if TYPE_CHECKING:
pass
@@ -200,7 +200,9 @@ async def _dispatch_tool(tool_name: str, tool_args: Dict[str, Any], config: Dict
# ----------------------------------------------------------------
from services.node_registry import get_node_class
- plugin_cls = None if node_type in AI_AGENT_TYPES else get_node_class(node_type)
+ from services.workspace_capabilities import is_registered_agent
+
+ plugin_cls = None if is_registered_agent(node_type) else get_node_class(node_type)
if plugin_cls is not None:
from services.plugin import NodeContext
@@ -254,7 +256,7 @@ async def _dispatch_tool(tool_name: str, tool_args: Dict[str, Any], config: Dict
# ``AI_AGENT_TYPES`` (imported above from ``constants``) is the
# canonical 18-entry frozenset; a new agent type added there picks
# up delegation support without touching this dispatcher.
- if node_type in AI_AGENT_TYPES:
+ if is_registered_agent(node_type):
return await _execute_delegated_agent(tool_args, config)
# Generic fallback for unknown node types
diff --git a/server/services/node_invocations.py b/server/services/node_invocations.py
new file mode 100644
index 00000000..6b9d0c92
--- /dev/null
+++ b/server/services/node_invocations.py
@@ -0,0 +1,110 @@
+"""Authoritative admission for direct Workspace tasks. Never accepts client graphs."""
+
+from __future__ import annotations
+import hashlib
+import json
+from typing import Any
+from uuid import UUID
+
+
+def invocation_id(principal: str, workflow_id: str, node_id: str, submission_id: str) -> str:
+ submission_id = str(UUID(submission_id))
+ identity = json.dumps([principal, workflow_id, node_id, submission_id], separators=(",", ":"))
+ return "node-invoke-" + hashlib.sha256(identity.encode()).hexdigest()
+
+
+def temporal_client() -> Any:
+ from core.container import container
+ from services.plugin import NodeUserError
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ raise NodeUserError("Workflow engine is not ready. Try again after startup completes.")
+ return wrapper.client
+
+
+async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, submission_id: str) -> dict:
+ from core.config import Settings
+ from services.authz.workflow_node import resolve_workflow_node
+ from services.node_registry import get_node_class
+ from services.plugin import NodeUserError
+ from services.plugin.deps import get_database
+ from services.temporal.node_invocation import NodeInvocationWorkflow
+ from temporalio.common import WorkflowIDReusePolicy
+ from temporalio.exceptions import WorkflowAlreadyStartedError
+
+ saved, graph, node = await resolve_workflow_node(principal, workflow_id, node_id)
+ cls = get_node_class(node["type"])
+ if cls is None or not getattr(cls, "workspace_task", False):
+ raise NodeUserError("This node does not accept direct Workspace tasks")
+ if (node.get("data") or {}).get("disabled"):
+ raise NodeUserError("Enable the node before submitting a task")
+ if not prompt.strip() or len(prompt) > 20000:
+ raise NodeUserError("Task must contain between 1 and 20000 characters")
+ run_id = invocation_id(principal, workflow_id, node_id, submission_id)
+ fingerprint = hashlib.sha256(prompt.encode()).hexdigest()
+ params = {**(await get_database().get_node_parameters(node_id) or {}), "prompt": prompt}
+ settings = Settings()
+ context = {
+ "node_id": node_id,
+ "node_type": node["type"],
+ "node_data": params,
+ "workflow_id": workflow_id,
+ "workflow_slug": getattr(saved, "name", "workflow"),
+ "execution_id": run_id,
+ "session_id": run_id,
+ "user_id": principal,
+ "nodes": graph.get("nodes", []),
+ "edges": graph.get("edges", []),
+ "generation": 0,
+ "graphVersion": graph.get("graphVersion", 1),
+ "inputs": {},
+ }
+ payload = {
+ "principal": principal,
+ "workflow_id": workflow_id,
+ "node_id": node_id,
+ "fingerprint": fingerprint,
+ "activity": f"node.{cls.type}.v{cls.version}",
+ "task_queue": cls.task_queue if settings.temporal_worker_pool_enabled else settings.temporal_task_queue,
+ "timeout_s": int(cls.start_to_close_timeout.total_seconds()),
+ "context": context,
+ }
+ client = temporal_client()
+ try:
+ await client.start_workflow(
+ NodeInvocationWorkflow.run,
+ payload,
+ id=run_id,
+ task_queue=settings.temporal_task_queue,
+ id_reuse_policy=WorkflowIDReusePolicy.REJECT_DUPLICATE,
+ )
+ except WorkflowAlreadyStartedError:
+ existing = await client.get_workflow_handle(run_id).query(NodeInvocationWorkflow.describe)
+ if existing["fingerprint"] != fingerprint:
+ raise NodeUserError("Submission ID was already used for a different task") from None
+ return {"run_id": run_id, "submission_id": submission_id, "status": "accepted"}
+
+
+async def status(principal: str, workflow_id: str, node_id: str, submission_id: str, *, cancel: bool = False) -> dict:
+ from services.authz.workflow_node import resolve_workflow_node
+ from services.temporal.node_invocation import NodeInvocationWorkflow
+ from services.plugin import NodeUserError
+ from temporalio.service import RPCError, RPCStatusCode
+
+ await resolve_workflow_node(principal, workflow_id, node_id)
+ handle = temporal_client().get_workflow_handle(invocation_id(principal, workflow_id, node_id, submission_id))
+ try:
+ if cancel:
+ await handle.cancel()
+ result = await handle.query(NodeInvocationWorkflow.describe)
+ except RPCError as exc:
+ if exc.status == RPCStatusCode.NOT_FOUND:
+ raise NodeUserError("This Workspace task was not found") from None
+ raise NodeUserError("Workflow engine is temporarily unavailable") from None
+ if result.get("status") in ("completed", "failed"):
+ try:
+ result["result"] = await handle.result()
+ except Exception:
+ result["status"] = "failed"
+ return result
diff --git a/server/services/plugin/edge_walker.py b/server/services/plugin/edge_walker.py
index acc90de2..69d46d4d 100644
--- a/server/services/plugin/edge_walker.py
+++ b/server/services/plugin/edge_walker.py
@@ -38,7 +38,7 @@ import re
from collections import Counter
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Tuple
-from constants import AI_AGENT_TYPES
+from services.workspace_capabilities import is_registered_agent
from core.logging import get_logger
if TYPE_CHECKING:
@@ -75,7 +75,7 @@ def build_teammate_descriptors(node_id: str, context: Dict[str, Any]) -> List[Di
if edge.get("target") != node_id or edge_target_handle(edge) != TEAMMATE_HANDLE:
continue
source = node_by_id.get(edge.get("source"))
- if not source or source.get("type") not in AI_AGENT_TYPES:
+ if not source or not is_registered_agent(source.get("type")):
continue
node_type = source.get("type", "")
raw_label = (source.get("data") or {}).get("label")
@@ -460,7 +460,7 @@ async def _append_tool_entry(
"label": source_node.get("data", {}).get("label", tool_type),
}
- if tool_type in AI_AGENT_TYPES:
+ if is_registered_agent(tool_type):
child_tools: List[Dict[str, Any]] = []
child_incoming_edges = [e for e in edges if e.get("target") == source_node_id]
child_tool_edges = [e for e in child_incoming_edges if edge_target_handle(e) == "input-tools"]
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index 5d3011f7..18ed8bcd 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -91,6 +91,44 @@ TOOL_STEP_TIMEOUT = timedelta(minutes=10)
TOOL_HEARTBEAT_TIMEOUT = timedelta(minutes=2)
LLM_STEP_HEARTBEAT_TIMEOUT = timedelta(minutes=1)
+
+async def _execute_plugin_tool_activity(
+ activity_name: str,
+ activity_id: str,
+ tool_payload: Dict[str, Any],
+ context: Dict[str, Any],
+) -> Any:
+ """Preserve legacy commands; honor embedded task engines' execution policy.
+
+ These plugins can perform irreversible device actions. Temporal's implicit
+ retry defaults must not override their explicitly declared attempt limit.
+ Routing comes from frozen workflow input, never live process settings.
+ """
+ options: Dict[str, Any] = {
+ "start_to_close_timeout": TOOL_STEP_TIMEOUT,
+ "heartbeat_timeout": TOOL_HEARTBEAT_TIMEOUT,
+ }
+ cls = get_node_class(tool_payload["node_type"])
+ if getattr(cls, "workspace_task", False) and workflow.patched("workspace-task-activity-policy-v1"):
+ options.update(
+ start_to_close_timeout=cls.start_to_close_timeout,
+ heartbeat_timeout=cls.heartbeat_timeout,
+ retry_policy=cls.retry_policy.to_temporal(),
+ cancellation_type=ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
+ )
+ if context.get("temporal_worker_pool_enabled") is True:
+ options["task_queue"] = cls.task_queue
+ # The caller's authenticated principal is not a model argument. Keep
+ # it on the leaf context so owner-only plugins cannot inherit the
+ # legacy default principal accidentally.
+ tool_payload = {**tool_payload, "user_id": str(context.get("user_id") or "owner")}
+ return await workflow.execute_activity(
+ activity_name,
+ args=[tool_payload],
+ activity_id=activity_id,
+ **options,
+ )
+
# Bounded loop count to defend against a runaway LLM. Plugin classes
# override via ``payload["max_iterations"]`` (set by
# ``prepare_agent_payload`` from Settings.agent_recursion_limit). This
@@ -1846,12 +1884,11 @@ class AgentWorkflow:
raise tool_result
else:
await self._wait_until_resumed()
- tool_result = await workflow.execute_activity(
+ tool_result = await _execute_plugin_tool_activity(
tool_activity_name,
- args=[tool_payload],
- activity_id=tool_activity_id,
- start_to_close_timeout=TOOL_STEP_TIMEOUT,
- heartbeat_timeout=TOOL_HEARTBEAT_TIMEOUT,
+ tool_activity_id,
+ tool_payload,
+ context,
)
if (
tool_info["node_type"] == "taskManager"
diff --git a/server/services/temporal/node_invocation.py b/server/services/temporal/node_invocation.py
new file mode 100644
index 00000000..3388d938
--- /dev/null
+++ b/server/services/temporal/node_invocation.py
@@ -0,0 +1,67 @@
+"""Durable single-node execution with the complete saved graph as context."""
+
+from __future__ import annotations
+import asyncio
+from datetime import timedelta
+from typing import Any
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+
+
+# The parent Temporal package registers services on import. Match the other
+# framework workflows: orchestration remains deterministic, without re-importing
+# that package and its logging/DI dependencies inside the workflow sandbox.
[email protected](sandboxed=False)
+class NodeInvocationWorkflow:
+ def __init__(self) -> None:
+ self.metadata: dict[str, Any] = {}
+ self.status = "queued"
+
+ @workflow.query
+ def describe(self) -> dict:
+ return {**self.metadata, "status": self.status}
+
+ @workflow.run
+ async def run(self, payload: dict) -> dict:
+ self.metadata = {k: payload[k] for k in ("principal", "workflow_id", "node_id", "fingerprint")}
+ self.status = "running"
+ result: dict = {}
+ try:
+ result = await workflow.execute_activity(
+ payload["activity"],
+ payload["context"],
+ task_queue=payload["task_queue"],
+ start_to_close_timeout=timedelta(seconds=payload["timeout_s"]),
+ heartbeat_timeout=timedelta(minutes=2),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ cancellation_type=workflow.ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
+ )
+ self.status = "completed" if result.get("success") else "failed"
+ return result
+ except asyncio.CancelledError:
+ self.status = "cancelled"
+ raise
+ except Exception:
+ self.status = "failed"
+ raise
+ finally:
+
+ async def record() -> None:
+ await workflow.execute_activity(
+ "workflow_runs.record_completion",
+ {
+ "workflow_id": payload["workflow_id"],
+ "run_id": f"{workflow.info().workflow_id}:{workflow.info().run_id}",
+ "generation": payload["context"].get("generation", 0),
+ "status": "success" if self.status == "completed" else "failed",
+ },
+ start_to_close_timeout=timedelta(seconds=30),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+
+ try:
+ await asyncio.shield(asyncio.create_task(record()))
+ except Exception as exc:
+ # A summary outage must not convert a completed device task into
+ # a failure that a user may replay. Temporal remains authoritative.
+ workflow.logger.warning("Workspace run summary failed: %s", type(exc).__name__)
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 4f701b4b..74f520f0 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -53,7 +53,10 @@ def _framework_workflows() -> list:
from services.temporal.polling_trigger_workflow import PollingTriggerWorkflow
from services.temporal.workflow_control_workflow import WorkflowControlWorkflow
+ from services.temporal.node_invocation import NodeInvocationWorkflow
+
return [
+ NodeInvocationWorkflow,
MachinaWorkflow,
AgentWorkflow,
DelegatedTaskWorkflow,
diff --git a/server/services/workspace_capabilities.py b/server/services/workspace_capabilities.py
new file mode 100644
index 00000000..9fd93cdf
--- /dev/null
+++ b/server/services/workspace_capabilities.py
@@ -0,0 +1,29 @@
+"""Generic, declarative Workspace discovery. Plugins own runtime and renderers."""
+
+from typing import Any
+
+
+def workspace_nodes(graph: dict[str, Any]) -> list[dict[str, str]]:
+ from services.node_registry import get_node_class
+
+ found = []
+ for node in graph.get("nodes", []):
+ cls = get_node_class(node.get("type", ""))
+ hints = getattr(cls, "ui_hints", {}) or {}
+ descriptor = hints.get("workspace") or ({"kind": "browser"} if hints.get("isBrowserPanel") else {})
+ kind = descriptor.get("kind")
+ if kind and node.get("id"):
+ found.append(
+ {"kind": kind, "node_id": node["id"], "label": (node.get("data") or {}).get("label") or getattr(cls, "display_name", kind)}
+ )
+ return found
+
+
+def is_registered_agent(node_type: str) -> bool:
+ from constants import AI_AGENT_TYPES
+ from services.node_registry import get_node_class
+
+ cls = get_node_class(node_type)
+ # component_kind is a renderer choice (socialSend/socialReceive also use
+ # the agent card); delegation must be an explicit execution capability.
+ return node_type in AI_AGENT_TYPES or bool(getattr(cls, "supports_delegation", False))
diff --git a/server/tests/temporal/test_workspace_task_activity_policy.py b/server/tests/temporal/test_workspace_task_activity_policy.py
new file mode 100644
index 00000000..d5f1680c
--- /dev/null
+++ b/server/tests/temporal/test_workspace_task_activity_policy.py
@@ -0,0 +1,78 @@
+"""Actual native-agent leaf scheduling, without a Temporal server."""
+
+from datetime import timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from services.plugin.scaling import RetryPolicy
+from services.temporal import agent_workflow as module
+from temporalio.workflow import ActivityCancellationType
+
+
[email protected]("pool", [False, True])
+async def test_workspace_task_honors_plugin_policy_and_authenticated_principal(monkeypatch, pool):
+ plugin = SimpleNamespace(
+ workspace_task=True,
+ start_to_close_timeout=timedelta(hours=3),
+ heartbeat_timeout=timedelta(seconds=75),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ task_queue="device-pool",
+ )
+ monkeypatch.setattr(module, "get_node_class", lambda _: plugin)
+ patched = Mock(return_value=True)
+ execute = AsyncMock(return_value={"success": True})
+ monkeypatch.setattr(module.workflow, "patched", patched)
+ monkeypatch.setattr(module.workflow, "execute_activity", execute)
+ tool_payload = {"node_type": "embedded_task", "node_id": "child", "user_id": "untrusted", "nodes": [{"id": "model"}]}
+ result = await module._execute_plugin_tool_activity(
+ "node.embedded_task.v1", "call-1", tool_payload, {"user_id": "authenticated-owner", "temporal_worker_pool_enabled": pool}
+ )
+ patched.assert_called_once_with("workspace-task-activity-policy-v1")
+ assert result == {"success": True}
+ kwargs = execute.await_args.kwargs
+ assert kwargs["start_to_close_timeout"] == timedelta(hours=3)
+ assert kwargs["heartbeat_timeout"] == timedelta(seconds=75)
+ assert kwargs["retry_policy"].maximum_attempts == 1
+ assert kwargs["cancellation_type"] == ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+ assert kwargs.get("task_queue") == ("device-pool" if pool else None)
+ assert kwargs["args"][0]["user_id"] == "authenticated-owner"
+ assert kwargs["args"][0]["nodes"] == [{"id": "model"}]
+ assert tool_payload["user_id"] == "untrusted" # caller input is not mutated
+
+
+async def test_prepatch_workspace_history_emits_identical_legacy_command(monkeypatch):
+ monkeypatch.setattr(module, "get_node_class", lambda _: SimpleNamespace(workspace_task=True))
+ monkeypatch.setattr(module.workflow, "patched", Mock(return_value=False))
+ execute = AsyncMock()
+ monkeypatch.setattr(module.workflow, "execute_activity", execute)
+ payload = {"node_type": "embedded_task", "node_id": "child"}
+ await module._execute_plugin_tool_activity(
+ "node.embedded_task.v1", "old-id", payload, {"user_id": "42", "temporal_worker_pool_enabled": True}
+ )
+ execute.assert_awaited_once_with(
+ "node.embedded_task.v1",
+ args=[payload],
+ activity_id="old-id",
+ start_to_close_timeout=module.TOOL_STEP_TIMEOUT,
+ heartbeat_timeout=module.TOOL_HEARTBEAT_TIMEOUT,
+ )
+ assert "user_id" not in payload
+
+
+async def test_ordinary_tools_do_not_add_patch_markers_or_change_policy(monkeypatch):
+ monkeypatch.setattr(module, "get_node_class", lambda _: SimpleNamespace(workspace_task=False))
+ patched = Mock(side_effect=AssertionError("Ordinary tool must not change history"))
+ monkeypatch.setattr(module.workflow, "patched", patched)
+ execute = AsyncMock()
+ monkeypatch.setattr(module.workflow, "execute_activity", execute)
+ payload = {"node_type": "ordinary_tool"}
+ await module._execute_plugin_tool_activity("node.ordinary_tool.v1", "ordinary-id", payload, {})
+ patched.assert_not_called()
+ execute.assert_awaited_once_with(
+ "node.ordinary_tool.v1",
+ args=[payload],
+ activity_id="ordinary-id",
+ start_to_close_timeout=module.TOOL_STEP_TIMEOUT,
+ heartbeat_timeout=module.TOOL_HEARTBEAT_TIMEOUT,
+ )
diff --git a/server/tests/test_mobile_broker.py b/server/tests/test_mobile_broker.py
new file mode 100644
index 00000000..f44f780f
--- /dev/null
+++ b/server/tests/test_mobile_broker.py
@@ -0,0 +1,79 @@
+"""Exercise the actual loopback HTTP broker with a fake device driver."""
+
+import json
+from unittest.mock import AsyncMock
+
+import aiohttp
+
+from nodes.mobile._runtime import MobileRuntime
+
+
+async def test_broker_expired_stale_and_revoked_capabilities(monkeypatch):
+ runtime = MobileRuntime()
+ driver = AsyncMock(return_value={"ok": True})
+ monkeypatch.setattr(runtime, "driver_call", driver)
+ lease = await runtime.control.claim("run:task")
+ runtime.capabilities["private-capability"] = ("task", lease)
+ url = await runtime.ensure_broker()
+ assert url.startswith("http://127.0.0.1:")
+ assert await runtime.ensure_broker() == url
+ try:
+ async with aiohttp.ClientSession() as client:
+
+ async def post(capability, operation="observe", epoch=lease.epoch, operation_id="one"):
+ async with client.post(
+ url,
+ headers={"Authorization": "Bearer " + capability},
+ json={"epoch": epoch, "operation": operation, "parameters": {}, "operation_id": operation_id},
+ ) as reply:
+ return reply.status, await reply.json()
+
+ assert (await post("expired"))[0] == 403
+ assert (await post("private-capability", epoch=lease.epoch - 1))[0] == 409
+ driver.assert_not_awaited()
+ assert (await post("private-capability"))[0] == 200
+ assert runtime.control.results == {} # never retain screenshot history
+ assert (await post("private-capability", "key", operation_id="two"))[0] == 200
+ assert driver.await_count == 2
+ runtime.control.revoke()
+ for operation in ("observe", "key"):
+ code, body = await post("private-capability", operation, operation_id="revoked-" + operation)
+ assert code == 409
+ assert body["success"] is False
+ assert driver.await_count == 2
+ finally:
+ await runtime.broker_runner.cleanup()
+
+
+def test_public_snapshot_excludes_private_capabilities_and_task_prompt():
+ runtime = MobileRuntime()
+ runtime.capabilities["secret-capability-value"] = ("run", None)
+ runtime.active = {"run_id": "run", "status": "running", "prompt": "private task text"}
+ runtime.queue = [{"run_id": "next", "prompt": "queued private text"}]
+ snapshot = json.dumps(runtime.snapshot())
+ assert "secret-capability-value" not in snapshot
+ assert "private task text" not in snapshot
+ assert "queued private text" not in snapshot
+
+
+def test_driver_environment_is_allowlisted_and_disables_tracing(monkeypatch):
+ for key in (
+ "OPENAI_API_KEY",
+ "ANTHROPIC_API_KEY",
+ "DATABASE_URL",
+ "AWS_ACCESS_KEY_ID",
+ "UNRELATED_SETTING",
+ "PYTHONPATH",
+ "PYTHONHOME",
+ "VIRTUAL_ENV",
+ ):
+ monkeypatch.setenv(key, "must-not-inherit")
+ monkeypatch.setenv("LANGSMITH_TRACING", "true")
+ monkeypatch.setenv("TEMP", "permitted-temp-directory")
+ env = MobileRuntime().driver_env()
+ assert "must-not-inherit" not in env.values()
+ assert env["TEMP"] == "permitted-temp-directory"
+ assert env["MOBILE_USE_TELEMETRY_ENABLED"] == "false"
+ assert env["LANGSMITH_TRACING"] == "false"
+ assert env["LANGCHAIN_TRACING_V2"] == "false"
+ assert env["PYTHON_DOTENV_DISABLED"] == "1"
diff --git a/server/tests/test_mobile_control.py b/server/tests/test_mobile_control.py
new file mode 100644
index 00000000..046e4685
--- /dev/null
+++ b/server/tests/test_mobile_control.py
@@ -0,0 +1,116 @@
+"""Behavioral checks for the shared device's ownership boundary."""
+
+import asyncio
+
+import pytest
+
+from nodes.mobile._control import DeviceControl, MobileError
+
+
+async def test_takeover_fences_queued_actions_before_draining():
+ control = DeviceControl()
+ lease = await control.claim("run:first")
+ entered, finish = asyncio.Event(), asyncio.Event()
+ calls = []
+
+ async def action():
+ entered.set()
+ await finish.wait()
+ calls.append("first")
+ return "done"
+
+ first = asyncio.create_task(control.perform(lease, "first", action))
+ await entered.wait()
+ queued = asyncio.create_task(control.perform(lease, "second", action))
+ await asyncio.sleep(0)
+ takeover = asyncio.create_task(control.claim("viewer:human", takeover=True))
+ await asyncio.sleep(0)
+ assert control.owner is None
+ assert not takeover.done()
+ finish.set()
+ assert await first == "done"
+ with pytest.raises(MobileError, match="control changed"):
+ await queued
+ human = await takeover
+ assert human.owner == "viewer:human"
+ assert calls == ["first"]
+
+
+async def test_retries_deduplicate_only_inside_the_same_lease():
+ control = DeviceControl()
+ lease = await control.claim("run:first")
+ calls = 0
+
+ async def action():
+ nonlocal calls
+ calls += 1
+ return {"count": calls}
+
+ assert await control.perform(lease, "id", action) == {"count": 1}
+ assert await control.perform(lease, "id", action) == {"count": 1}
+ control.revoke()
+ replacement = await control.claim("run:first")
+ assert await control.perform(replacement, "id", action) == {"count": 2}
+
+
+async def test_cancelled_caller_keeps_lock_and_success_receipt():
+ control = DeviceControl()
+ lease = await control.claim("run:first")
+ entered, finish = asyncio.Event(), asyncio.Event()
+ calls = 0
+
+ async def action():
+ nonlocal calls
+ calls += 1
+ entered.set()
+ await finish.wait()
+ return "committed"
+
+ caller = asyncio.create_task(control.perform(lease, "mutation", action))
+ await entered.wait()
+ caller.cancel()
+ await asyncio.sleep(0)
+ assert control.lock.locked()
+ finish.set()
+ with pytest.raises(asyncio.CancelledError):
+ await caller
+ assert await control.perform(lease, "mutation", action) == "committed"
+ assert calls == 1
+
+
+async def test_uncertain_failure_after_cancellation_fences_the_owner():
+ control = DeviceControl()
+ lease = await control.claim("run:first")
+ entered, finish = asyncio.Event(), asyncio.Event()
+
+ async def broken():
+ entered.set()
+ await finish.wait()
+ raise OSError("transport lost")
+
+ caller = asyncio.create_task(control.perform(lease, "mutation", broken))
+ await entered.wait()
+ caller.cancel()
+ await asyncio.sleep(0)
+ finish.set()
+ # Either cancellation or the late transport failure may be surfaced;
+ # ownership must be fenced in both cases.
+ with pytest.raises((asyncio.CancelledError, OSError)):
+ await caller
+ assert control.state == "recovering"
+ assert control.owner is None
+ with pytest.raises(MobileError):
+ await control.claim("viewer:human", takeover=True)
+
+
+async def test_failure_fences_lease_and_requires_recovery():
+ control = DeviceControl()
+ lease = await control.claim("run:first")
+
+ async def broken():
+ raise OSError("unknown device outcome")
+
+ with pytest.raises(OSError):
+ await control.perform(lease, "mutation", broken)
+ assert control.state == "recovering"
+ assert control.owner is None
diff --git a/server/tests/test_mobile_install.py b/server/tests/test_mobile_install.py
new file mode 100644
index 00000000..cbf5cd9d
--- /dev/null
+++ b/server/tests/test_mobile_install.py
@@ -0,0 +1,83 @@
+"""Optional installation boundaries; no network or SDK installation."""
+
+import hashlib
+import io
+import zipfile
+from unittest.mock import AsyncMock
+
+import pytest
+
+from nodes.mobile import _install as install
+from nodes.mobile._control import MobileError
+from nodes.mobile._runtime import MobileRuntime
+
+
+def test_bad_download_checksum_preserves_previous_target(monkeypatch, tmp_path):
+ target = tmp_path / "artifact.zip"
+ target.write_bytes(b"previous verified artifact")
+ monkeypatch.setattr(install.urllib.request, "urlopen", lambda *_args, **_kwargs: io.BytesIO(b"corrupt"))
+ with pytest.raises(ValueError, match="checksum"):
+ install.download("https://example.invalid/artifact", target, hashlib.sha256(b"expected").hexdigest())
+ assert target.read_bytes() == b"previous verified artifact"
+ assert not target.with_suffix(".zip.part").exists()
+
+
[email protected]("filename,mode", [("../outside", 0o100644), ("safe/link", 0o120777)])
+def test_archive_rejects_traversal_and_symlinks(tmp_path, filename, mode):
+ archive = tmp_path / "source.zip"
+ with zipfile.ZipFile(archive, "w") as source:
+ source.writestr("ordinary", "must not extract before validation")
+ entry = zipfile.ZipInfo(filename)
+ entry.create_system = 3
+ entry.external_attr = mode << 16
+ source.writestr(entry, "target")
+ destination = tmp_path / "unpacked"
+ with pytest.raises(ValueError, match="Unsafe"):
+ install.extract_zip(archive, destination)
+ assert not (destination / "ordinary").exists()
+ assert not (tmp_path / "outside").exists()
+
+
+async def test_declining_license_never_installs(monkeypatch):
+ tools = AsyncMock()
+ monkeypatch.setattr(install.sys, "platform", "win32")
+ monkeypatch.setattr(install.platform, "machine", lambda: "AMD64")
+ monkeypatch.setattr(install, "install_sdk_tools", tools)
+ with pytest.raises(ValueError, match="license"):
+ await install.create_device(licenses_accepted=False)
+ tools.assert_not_awaited()
+ runtime = MobileRuntime()
+ with pytest.raises(MobileError, match="license"):
+ runtime.setup(False)
+ assert runtime.setup_task is None
+
+
[email protected](install.os.name != "nt", reason="Windows Java launcher contract")
+def test_sdk_launch_uses_java_argv_without_command_shell(monkeypatch, tmp_path):
+ java = tmp_path / "java" / "jdk-test" / "bin" / "java.exe"
+ java.parent.mkdir(parents=True)
+ java.touch()
+ script = tmp_path / "sdk with spaces & symbols" / "bin" / "sdkmanager.bat"
+ monkeypatch.setattr(install, "mobile_root", lambda: tmp_path)
+ monkeypatch.setattr(install, "sdk_tool", lambda _name: script)
+ args = install.sdk_command("sdkmanager", "--sdk_root=C:\\SDK with spaces", install.IMAGE)
+ assert args[0] == str(java)
+ assert "com.android.sdklib.tool.sdkmanager.SdkManagerCli" in args
+ assert args[-1] == install.IMAGE
+ assert not any(arg.lower().endswith(("cmd.exe", ".bat")) for arg in args)
+
+
+def test_incomplete_environment_is_not_ready(monkeypatch, tmp_path):
+ from nodes.mobile import _install
+ from nodes.mobile.runtime.patch_source import PATCH_VERSION
+ import json
+
+ executable = tmp_path / "python.exe"
+ executable.touch()
+ monkeypatch.setattr(_install, "mobile_root", lambda: tmp_path)
+ monkeypatch.setattr(_install, "runtime_python", lambda: executable)
+ assert not _install.engine_ready()
+ (tmp_path / "engine-ready.json").write_text(json.dumps({"source": _install.UPSTREAM_SHA, "patch": PATCH_VERSION}))
+ assert _install.engine_ready()
+ (tmp_path / "engine-ready.json").write_text(json.dumps({"source": "old", "patch": PATCH_VERSION}))
+ assert not _install.engine_ready()
diff --git a/server/tests/test_mobile_integration.py b/server/tests/test_mobile_integration.py
new file mode 100644
index 00000000..02e7c9e8
--- /dev/null
+++ b/server/tests/test_mobile_integration.py
@@ -0,0 +1,233 @@
+"""Mobile boundary contracts, using no emulator, credentials, or network."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+from uuid import uuid4
+import pytest
+from fastapi import HTTPException
+from services.plugin import NodeContext, NodeUserError
+from nodes.mobile import _node, _router, _video
+
+
+def context():
+ return NodeContext(
+ node_id="mobile",
+ node_type="mobile_use_agent",
+ workflow_id="wf",
+ execution_id="run",
+ nodes=[{"id": "model", "type": "test_model", "data": {}}],
+ edges=[{"source": "model", "target": "mobile", "targetHandle": "input-model"}],
+ )
+
+
+async def test_connector_uses_saved_model_and_only_selected_key(monkeypatch):
+ import services.plugin.deps as deps
+ import services.node_registry as registry
+ import constants
+
+ database = SimpleNamespace(get_node_parameters=AsyncMock(return_value={"model": "chosen-model", "api_key": "untrusted-inline"}))
+ auth = SimpleNamespace(get_api_key=AsyncMock(return_value="resolved-secret"), get_stored_models=AsyncMock(return_value=[]))
+ monkeypatch.setattr(deps, "get_database", lambda: database)
+ monkeypatch.setattr(deps, "get_ai_service", lambda: SimpleNamespace(auth=auth))
+ monkeypatch.setattr(registry, "get_node_class", lambda _: SimpleNamespace(component_kind="model"))
+ monkeypatch.setattr(constants, "detect_ai_provider", lambda *_: "gemini")
+ config = await _node.resolve_model(context())
+ assert config == {"provider": "google", "model": "chosen-model", "model_env": {"GOOGLE_API_KEY": "resolved-secret"}}
+ auth.get_api_key.assert_awaited_once_with("gemini", "default")
+
+
+async def test_connector_requires_exactly_one_model():
+ ctx = context()
+ ctx.edges = []
+ with pytest.raises(NodeUserError, match="exactly one"):
+ await _node.resolve_model(ctx)
+ ctx.edges = [{"target": "mobile", "targetHandle": "input-model"}] * 2
+ with pytest.raises(NodeUserError, match="exactly one"):
+ await _node.resolve_model(ctx)
+
+
+async def test_node_passes_execution_identity_and_broker(monkeypatch):
+ from nodes.mobile import _runtime
+
+ runtime = SimpleNamespace(
+ run=AsyncMock(return_value={"outcome": "completed"}), ensure_broker=AsyncMock(return_value="http://127.0.0.1/private")
+ )
+ monkeypatch.setattr(_runtime, "get_runtime", lambda: runtime)
+ monkeypatch.setattr(_node, "require_mobile_owner", AsyncMock())
+ monkeypatch.setattr(_node, "resolve_model", AsyncMock(return_value={"provider": "openai"}))
+ await _node.MobileUseAgent().execute_op(context(), _node.MobileParams(prompt="Open Settings"))
+ args = runtime.run.await_args.kwargs
+ assert (args["principal"], args["workflow_id"], args["node_id"]) == ("owner", "wf", "mobile")
+ assert args["run_id"] == _node.task_identity(context())
+ assert args["broker_url"] == "http://127.0.0.1/private"
+
+
+ "operation,parameters",
+ [
+ ("tap", {"x": float("nan"), "y": 1}),
+ ("swipe", {"x": 1, "y": 2}),
+ ("key", {"key": "shell"}),
+ ("text", {"text": "ok", "serial": "other-device"}),
+ ],
+)
+def test_manual_input_rejects_malformed_commands(operation, parameters):
+ body = _router.DeviceInput(
+ viewer_id=uuid4(),
+ operation_id=uuid4(),
+ epoch=1,
+ operation=operation,
+ parameters=parameters,
+ geometry={"width": 10, "height": 10, "rotation": 0},
+ )
+ with pytest.raises(HTTPException) as exc:
+ _router.validate_input(body)
+ assert exc.value.status_code == 422
+
+
+async def test_input_passes_bound_lease_not_claimed_owner(monkeypatch):
+ from nodes.mobile import _runtime
+
+ runtime = SimpleNamespace(input=AsyncMock(return_value=True))
+ monkeypatch.setattr(_runtime, "get_runtime", lambda: runtime)
+ monkeypatch.setattr(_router, "viewer_identity", lambda principal, *_: "bound-" + principal)
+ body = _router.DeviceInput(viewer_id=uuid4(), operation_id=uuid4(), epoch=7, operation="key", parameters={"key": "home"})
+ await _router.device_input(body, SimpleNamespace(cookies={}), "user123")
+ lease = runtime.input.await_args.args[0]
+ assert lease.owner == "viewer:bound-user123" and lease.epoch == 7
+
+
+async def test_cancel_targets_only_matching_submission(monkeypatch):
+ from nodes.mobile import _runtime
+ import services.node_invocations as invocations
+
+ runtime = SimpleNamespace(cancel=AsyncMock())
+ monkeypatch.setattr(_runtime, "get_runtime", lambda: runtime)
+ monkeypatch.setattr(invocations, "status", AsyncMock(return_value={"status": "cancelled"}))
+ submission = uuid4()
+ await _router.cancel(submission, "wf", "node", "user")
+ runtime.cancel.assert_awaited_once_with("wf", "node", run_id=invocations.invocation_id("user", "wf", "node", str(submission)))
+
+
+def test_video_server_is_read_only_and_exact_version(monkeypatch):
+ monkeypatch.setattr(_video, "sdk_tool", lambda _: "adb")
+ args = _video.server_arguments("emulator-5560", "00000001")
+ assert "control=false" in args and "audio=false" in args
+ assert "4.1" in args and "video_codec=h264" in args and "max_fps=30" in args
+ assert "send_frame_meta=true" in args and "send_device_meta=true" in args
+
+
+async def test_shared_device_denies_nonowner_and_remote_installation(monkeypatch):
+ from core.container import container
+
+ settings = SimpleNamespace(vite_auth_enabled="true", deployment_mode="local")
+ monkeypatch.setattr(container, "settings", lambda: settings)
+ monkeypatch.setattr(
+ container,
+ "user_auth_service",
+ lambda: SimpleNamespace(get_user_by_id=AsyncMock(return_value=SimpleNamespace(is_active=True, is_owner=False))),
+ )
+ with pytest.raises(NodeUserError, match="owner only"):
+ await _node.require_mobile_owner("12")
+ settings.deployment_mode = "cloud"
+ with pytest.raises(NodeUserError, match="local installation"):
+ await _node.require_mobile_owner("12")
+
+
+async def test_http_authorization_rejects_foreign_origin_and_resolves_saved_node(monkeypatch):
+ import sys
+ from types import ModuleType
+ from core.container import container
+ import services.authz.workflow_node as resolver
+
+ cookies = ModuleType("core.auth_cookies")
+ cookies.get_session_token = lambda *_: "cookie-token"
+ monkeypatch.setitem(sys.modules, "core.auth_cookies", cookies)
+ settings = SimpleNamespace(vite_auth_enabled="true", cors_origins=[])
+ auth = SimpleNamespace(get_current_user=AsyncMock(return_value=SimpleNamespace(id=14)))
+ monkeypatch.setattr(container, "settings", lambda: settings)
+ monkeypatch.setattr(container, "user_auth_service", lambda: auth)
+ owner_check = AsyncMock()
+ saved_node = AsyncMock()
+ monkeypatch.setattr(_router, "require_mobile_owner", owner_check)
+ monkeypatch.setattr(resolver, "resolve_workflow_node", saved_node)
+ request = SimpleNamespace(headers={"origin": "https://foreign.example", "host": "localhost:8000"}, cookies={})
+ with pytest.raises(HTTPException) as exc:
+ await _router.authorize(request, "saved-workflow", "saved-node")
+ assert exc.value.status_code == 403
+ auth.get_current_user.assert_not_awaited()
+ request.headers["origin"] = "http://localhost:8000"
+ assert await _router.authorize(request, "saved-workflow", "saved-node") == "14"
+ owner_check.assert_awaited_once_with("14")
+ saved_node.assert_awaited_once_with("14", "saved-workflow", "saved-node", workspace_kind="mobile")
+
+
+def test_agent_discovery_requires_execution_capability(monkeypatch):
+ from services.workspace_capabilities import is_registered_agent
+ import services.node_registry as registry
+
+ monkeypatch.setattr(registry, "get_node_class", lambda _: SimpleNamespace(component_kind="agent"))
+ assert not is_registered_agent("renderer_only_agent")
+ monkeypatch.setattr(registry, "get_node_class", lambda _: _node.MobileUseAgent)
+ assert is_registered_agent("mobile_use_agent")
+ assert _node.MobileUseAgent.needs_canvas and _node.MobileUseAgent.requires_context
+ assert {"input-context", "input-model"} <= {handle["name"] for handle in _node.MobileUseAgent.handles}
+
+
+async def test_video_strips_only_forward_sentinel_and_cleans_tunnel(monkeypatch):
+ import asyncio
+ import hashlib
+ from nodes.mobile import _runtime
+
+ runtime = SimpleNamespace(serial="emulator-5560", video_viewer=None, viewer=None)
+ monkeypatch.setattr(_runtime, "get_runtime", lambda: runtime)
+ monkeypatch.setattr(_video, "sdk_tool", lambda _: "adb")
+
+ class Server:
+ def __truediv__(self, _):
+ return self
+
+ def is_file(self):
+ return True
+
+ def read_bytes(self):
+ return b"pinned"
+
+ def __str__(self):
+ return "scrcpy-server"
+
+ monkeypatch.setattr(_video, "mobile_root", Server)
+ monkeypatch.setattr(_video, "SCRCPY_SHA256", hashlib.sha256(b"pinned").hexdigest())
+ command = AsyncMock(side_effect=["", "12345", ""])
+ monkeypatch.setattr(_video, "command", command)
+ proc = SimpleNamespace(returncode=None, kill=lambda: None, wait=AsyncMock())
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", AsyncMock(return_value=proc))
+ # Raw payload deliberately includes a metadata-looking zero prefix; it
+ # must not be trimmed or repacked by the backend.
+ raw = b"\x00device-name-and-scrcpy-packets"
+ reader = SimpleNamespace(readexactly=AsyncMock(return_value=b"\x00"), read=AsyncMock(side_effect=[raw, b""]))
+ writer = SimpleNamespace(close=lambda: None, wait_closed=AsyncMock())
+ monkeypatch.setattr(asyncio, "open_connection", AsyncMock(return_value=(reader, writer)))
+
+ async def receive():
+ await asyncio.Event().wait()
+
+ websocket = SimpleNamespace(accept=AsyncMock(), send_json=AsyncMock(), send_bytes=AsyncMock(), receive=receive, close=AsyncMock())
+ await _video.stream_video(websocket, "viewer1")
+ reader.readexactly.assert_awaited_once_with(1)
+ websocket.send_bytes.assert_awaited_once_with(raw)
+ websocket.send_json.assert_awaited_once_with({"type": "video", "version": "4.1", "codec": "h264"})
+ assert command.await_args_list[-1].args[0][-2:] == ["--remove", "tcp:12345"]
+ assert runtime.video_viewer is None
+ proc.wait.assert_awaited_once()
+
+
+def test_task_identity_separates_nodes_and_tool_calls():
+ ctx = context()
+ first = _node.task_identity(ctx)
+ ctx.node_id = "another-mobile"
+ second = _node.task_identity(ctx)
+ ctx.raw["tool_call_id"] = "second-call"
+ assert len({first, second, _node.task_identity(ctx)}) == 3
+ ctx.execution_id = "node-invoke-already-scoped"
+ assert _node.task_identity(ctx) == ctx.execution_id
diff --git a/server/tests/test_mobile_invocation_workflow.py b/server/tests/test_mobile_invocation_workflow.py
new file mode 100644
index 00000000..377b74a3
--- /dev/null
+++ b/server/tests/test_mobile_invocation_workflow.py
@@ -0,0 +1,65 @@
+"""Durable admission orchestration without a live Temporal server."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from services.temporal.node_invocation import NodeInvocationWorkflow
+
+
+def payload():
+ return {
+ "principal": "owner",
+ "workflow_id": "saved",
+ "node_id": "mobile",
+ "fingerprint": "hash",
+ "activity": "node.mobile_use_agent.v1",
+ "task_queue": "android",
+ "timeout_s": 900,
+ "context": {"generation": 2, "nodes": [{"id": "mobile"}], "edges": []},
+ }
+
+
+async def test_result_and_retry_policy_survive_summary_failure(monkeypatch, payload):
+ from services.temporal.node_invocation import workflow
+
+ execute = AsyncMock(side_effect=[{"success": True, "result": "done"}, RuntimeError("summary unavailable")])
+ monkeypatch.setattr(workflow, "execute_activity", execute)
+ monkeypatch.setattr(workflow, "info", lambda: SimpleNamespace(workflow_id="invocation", run_id="attempt"))
+ monkeypatch.setattr(workflow, "logger", SimpleNamespace(warning=lambda *_args: None))
+ run = NodeInvocationWorkflow()
+ assert await run.run(payload) == {"success": True, "result": "done"}
+ assert run.describe()["status"] == "completed"
+ first, record = execute.await_args_list
+ assert first.args == (payload["activity"], payload["context"])
+ assert first.kwargs["retry_policy"].maximum_attempts == 1
+ assert first.kwargs["cancellation_type"] == workflow.ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+ assert record.args[1]["status"] == "success"
+
+
+async def test_cancellation_records_terminal_status_and_propagates(monkeypatch, payload):
+ from services.temporal.node_invocation import workflow
+
+ entered = asyncio.Event()
+ records = []
+
+ async def execute(name, data, **kwargs):
+ if name == payload["activity"]:
+ entered.set()
+ await asyncio.Event().wait()
+ else:
+ records.append(data)
+
+ monkeypatch.setattr(workflow, "execute_activity", execute)
+ monkeypatch.setattr(workflow, "info", lambda: SimpleNamespace(workflow_id="invocation", run_id="attempt"))
+ run = NodeInvocationWorkflow()
+ task = asyncio.create_task(run.run(payload))
+ await entered.wait()
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert run.describe()["status"] == "cancelled"
+ assert records[0]["run_id"] == "invocation:attempt"
diff --git a/server/tests/test_mobile_invocations.py b/server/tests/test_mobile_invocations.py
new file mode 100644
index 00000000..0f788693
--- /dev/null
+++ b/server/tests/test_mobile_invocations.py
@@ -0,0 +1,63 @@
+"""Workspace task identities are stable, scoped, and server-authorized."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from services.node_invocations import invocation_id
+from services.authz.workflow_node import resolve_workflow_node
+from services.plugin import NodeUserError
+
+
+SUBMISSION = "63cb635f-c758-42bc-a9b6-673655a72fa0"
+
+
+def test_invocation_identity_is_stable_and_scoped():
+ identity = invocation_id("owner", "workflow", "node", SUBMISSION)
+ assert identity == invocation_id("owner", "workflow", "node", SUBMISSION)
+ assert (
+ len(
+ {
+ identity,
+ invocation_id("other", "workflow", "node", SUBMISSION),
+ invocation_id("owner", "other", "node", SUBMISSION),
+ invocation_id("owner", "workflow", "other", SUBMISSION),
+ }
+ )
+ == 4
+ )
+
+
+def test_invocation_requires_a_uuid():
+ with pytest.raises(ValueError):
+ invocation_id("owner", "workflow", "node", "arbitrary-client-id")
+
+
+def saved_graph(monkeypatch):
+ import services.plugin.deps as deps
+
+ graph = {"owner_id": "owner", "nodes": [{"id": "node", "type": "mobile_use_agent"}], "edges": []}
+ database = SimpleNamespace(get_workflow=AsyncMock(return_value=SimpleNamespace(data=graph)))
+ monkeypatch.setattr(deps, "get_database", lambda: database)
+ return graph
+
+
+async def test_workflow_resolution_rejects_other_owner(saved_graph):
+ with pytest.raises(NodeUserError, match="Workflow access denied"):
+ await resolve_workflow_node("other", "workflow", "node")
+
+
+async def test_workflow_resolution_rejects_absent_and_ambiguous_nodes(saved_graph):
+ with pytest.raises(NodeUserError, match="Node does not belong"):
+ await resolve_workflow_node("owner", "workflow", "missing")
+ saved_graph["nodes"].append(dict(saved_graph["nodes"][0]))
+ with pytest.raises(NodeUserError, match="Node does not belong"):
+ await resolve_workflow_node("owner", "workflow", "node")
+
+
+async def test_workflow_resolution_returns_saved_graph(saved_graph):
+ _, graph, node = await resolve_workflow_node("owner", "workflow", "node")
+ assert graph is saved_graph
+ assert node is saved_graph["nodes"][0]
diff --git a/server/tests/test_mobile_patch.py b/server/tests/test_mobile_patch.py
new file mode 100644
index 00000000..c1007ab4
--- /dev/null
+++ b/server/tests/test_mobile_patch.py
@@ -0,0 +1,59 @@
+"""Optional real-upstream compatibility check; no downloads during tests."""
+
+import ast
+import os
+import shutil
+from pathlib import Path
+
+import pytest
+
+from nodes.mobile.runtime.patch_source import PATCH_VERSION, patch
+
+
+def patched_source(tmp_path):
+ default = Path(__file__).resolve().parents[2] / ".tmp" / "mobile-use-upstream"
+ upstream = Path(os.environ.get("MOBILE_USE_UPSTREAM_SOURCE", default))
+ if not (upstream / "minitap/mobile_use/sdk/agent.py").is_file():
+ pytest.skip("Set MOBILE_USE_UPSTREAM_SOURCE to the pinned upstream checkout")
+ shutil.copytree(upstream / "minitap", tmp_path / "minitap")
+ patch(tmp_path)
+ return tmp_path
+
+
+def test_patch_applies_to_upstream_and_every_python_file_compiles(patched_source):
+ assert (patched_source / ".opencompany-patch").read_text() == str(PATCH_VERSION)
+ for source in (patched_source / "minitap").rglob("*.py"):
+ compile(source.read_text(encoding="utf-8"), str(source), "exec")
+
+
+def test_injected_branches_cover_native_bypasses(patched_source):
+ base = patched_source / "minitap/mobile_use"
+ expected = {
+ "sdk/agent.py": ["_init_internal", "_install_apk_internal", "_install_ios_app", "get_screenshot", "clean"],
+ "controllers/controller_factory.py": ["create_device_controller"],
+ "controllers/platform_specific_commands_controller.py": [
+ "get_device_date",
+ "list_packages",
+ "list_packages_async",
+ "get_current_foreground_package",
+ "get_current_foreground_package_async",
+ ],
+ }
+ for file, names in expected.items():
+ tree = ast.parse((base / file).read_text(encoding="utf-8"))
+ for name in names:
+ function = next(n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == name)
+ body = function.body
+ if isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
+ body = body[1:]
+ assert isinstance(body[0], ast.If), (file, name)
+ assert "local_controller" in ast.unparse(body[0].test), (file, name)
+ for file in ("config.py", "sdk/agent.py"):
+ tree = ast.parse((base / file).read_text(encoding="utf-8"))
+ assert not any(isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id == "load_dotenv" for n in ast.walk(tree))
+
+
+def test_patch_refuses_second_application(patched_source):
+ with pytest.raises(ValueError, match="compatibility mismatch"):
+ patch(patched_source)
diff --git a/server/tests/test_mobile_runtime.py b/server/tests/test_mobile_runtime.py
new file mode 100644
index 00000000..6ff11118
--- /dev/null
+++ b/server/tests/test_mobile_runtime.py
@@ -0,0 +1,218 @@
+"""Runtime orchestration without an emulator, SDK, or model request."""
+
+import asyncio
+import json
+
+import pytest
+
+from nodes.mobile._runtime import MobileRuntime
+
+
+class FakeStdin:
+ def __init__(self):
+ self.data = b""
+
+ def write(self, data):
+ self.data += data
+
+ async def drain(self):
+ pass
+
+ def close(self):
+ pass
+
+
+class FakeProcess:
+ def __init__(self):
+ self.stdin = FakeStdin()
+ self.stdout = asyncio.StreamReader()
+ self.returncode = None
+ self.terminated = asyncio.Event()
+
+ def terminate(self):
+ self.returncode = -15
+ self.stdout.feed_eof()
+ self.terminated.set()
+
+ def kill(self):
+ self.terminate()
+
+ async def wait(self):
+ await self.terminated.wait()
+ return self.returncode
+
+
+def runtime(monkeypatch, tmp_path):
+ import nodes.mobile._runtime as module
+
+ value = MobileRuntime()
+ value.serial = "emulator-5560"
+ monkeypatch.setattr(module, "mobile_root", lambda: tmp_path)
+ monkeypatch.setattr(value, "driver_env", lambda: {})
+
+ async def geometry(*_args, **_kwargs):
+ return {"width": 1080, "height": 1920}
+
+ monkeypatch.setattr(value, "driver_call", geometry)
+ return value
+
+
+def run_args():
+ return dict(
+ principal="owner",
+ workflow_id="workflow",
+ node_id="agent",
+ run_id="run",
+ params={"prompt": "test", "max_steps": 3, "timeout_s": 30},
+ model={},
+ broker_url="http://127.0.0.1/broker",
+ )
+
+
+async def test_takeover_stops_worker_then_resumes_with_fresh_capability(runtime, monkeypatch):
+ processes = []
+ spawned = asyncio.Queue()
+
+ async def spawn(*_args, **_kwargs):
+ proc = FakeProcess()
+ processes.append(proc)
+ spawned.put_nowait(proc)
+ return proc
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)
+ task = asyncio.create_task(runtime.run(**run_args()))
+ first = await asyncio.wait_for(spawned.get(), 1)
+ await asyncio.sleep(0)
+ first_config = json.loads(first.stdin.data)
+ human = await runtime.takeover("browser")
+ assert first.terminated.is_set()
+ assert not runtime.capabilities
+ assert human.owner == "viewer:browser"
+ assert runtime.active["status"] == "awaiting_user"
+ runtime.release("browser", resume=True)
+ second = await asyncio.wait_for(spawned.get(), 1)
+ await asyncio.sleep(0)
+ second_config = json.loads(second.stdin.data)
+ assert second_config["epoch"] > first_config["epoch"]
+ assert second_config["capability"] != first_config["capability"]
+ second.stdout.feed_data(b'{"type":"completed","result":"done"}\n')
+ second.returncode = 0
+ second.stdout.feed_eof()
+ second.terminated.set()
+ assert (await asyncio.wait_for(task, 1))["response"] == "done"
+ assert runtime.active is None
+ assert not runtime.capabilities
+
+
+async def test_cancel_while_waiting_for_human_releases_task(runtime):
+ await runtime.takeover("browser")
+ task = asyncio.create_task(runtime.run(**run_args()))
+ await asyncio.sleep(0)
+ assert runtime.active["status"] == "awaiting_user"
+ await runtime.cancel("workflow", "agent")
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, 1)
+ assert runtime.active is None
+ assert runtime.queue == []
+ assert runtime.control.owner == "viewer:browser"
+
+
+async def test_queued_cancel_is_scoped_to_run_and_never_starts_worker(runtime, monkeypatch):
+ async def unexpected_spawn(*_args, **_kwargs):
+ pytest.fail("A cancelled queued task must never spawn a worker")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", unexpected_spawn)
+ await runtime.task_lock.acquire()
+ first = asyncio.create_task(runtime.run(**run_args()))
+ second_args = {**run_args(), "run_id": "other-run"}
+ second = asyncio.create_task(runtime.run(**second_args))
+ await asyncio.sleep(0)
+ assert len(runtime.queue) == 2
+ await runtime.cancel("workflow", "agent", run_id="run")
+ assert [entry["status"] for entry in runtime.queue] == ["cancelled", "queued"]
+ await runtime.cancel("workflow", "agent", run_id="other-run")
+ runtime.serial = None # Cancellation must win over offline validation.
+ runtime.task_lock.release()
+ for task in (first, second):
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, 1)
+ assert runtime.queue == []
+ assert runtime.active is None
+
+
+async def test_takeover_waits_for_already_accepted_mutation(runtime):
+ lease = await runtime.control.claim("run:run")
+ entered, finish = asyncio.Event(), asyncio.Event()
+
+ async def mutation():
+ entered.set()
+ await finish.wait()
+
+ action = asyncio.create_task(runtime.control.perform(lease, "write", mutation))
+ await entered.wait()
+ takeover = asyncio.create_task(runtime.takeover("browser"))
+ await asyncio.sleep(0)
+ assert not takeover.done()
+ assert runtime.control.owner is None
+ finish.set()
+ await action
+ assert (await takeover).owner == "viewer:browser"
+
+
+async def test_cancel_during_geometry_no_spawn(runtime, monkeypatch):
+ entered, finish = asyncio.Event(), asyncio.Event()
+
+ async def geometry(*_args, **_kwargs):
+ entered.set()
+ await finish.wait()
+ return {"width": 1080, "height": 1920}
+
+ async def unexpected_spawn(*_args, **_kwargs):
+ pytest.fail("Cancelled task must not launch after geometry resolves")
+
+ monkeypatch.setattr(runtime, "driver_call", geometry)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", unexpected_spawn)
+ task = asyncio.create_task(runtime.run(**run_args()))
+ await asyncio.wait_for(entered.wait(), 1)
+ await runtime.cancel("workflow", "agent")
+ finish.set()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, 1)
+ assert runtime.active is None
+
+
+async def test_takeover_during_spawn_does_not_send_config(runtime, monkeypatch):
+ entered, finish = asyncio.Event(), asyncio.Event()
+ process = FakeProcess()
+
+ async def spawn(*_args, **_kwargs):
+ entered.set()
+ await finish.wait()
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)
+ task = asyncio.create_task(runtime.run(**run_args()))
+ await asyncio.wait_for(entered.wait(), 1)
+ await runtime.takeover("browser")
+ finish.set()
+ try:
+ await asyncio.wait_for(process.terminated.wait(), 1)
+ assert process.stdin.data == b""
+ assert runtime.control.owner == "viewer:browser"
+ finally:
+ await runtime.cancel("workflow", "agent")
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, 1)
+
+
+async def test_stop_preserves_epoch_monotonicity_and_clears_viewer(runtime):
+ lease = await runtime.takeover("browser")
+ runtime.resume_event.set()
+ await runtime.stop()
+ assert runtime.control.owner is None
+ assert runtime.control.epoch > lease.epoch
+ assert runtime.viewer is None
+ assert not runtime.resume_event.is_set()
+ next_lease = await runtime.control.claim("viewer:browser")
+ assert next_lease.epoch > lease.epoch
diff --git a/server/tests/test_node_spec.py b/server/tests/test_node_spec.py
index 95751265..8832e74e 100644
--- a/server/tests/test_node_spec.py
+++ b/server/tests/test_node_spec.py
@@ -17,6 +17,13 @@ from services.node_input_schemas import (
from services.node_spec import get_node_spec, list_node_types_with_spec
+def _standalone_workspace_agent(node_type: str) -> bool:
+ """Embedded engines accept a delegated task; their tools live inside that engine."""
+ from services.node_registry import get_node_class
+ cls = get_node_class(node_type)
+ return bool(getattr(cls, "workspace_task", False) and getattr(cls, "supports_delegation", False))
+
+
def _numeric_constraints(prop: dict) -> dict:
"""Pydantic emits ``Optional[float] = Field(ge=..., le=...)`` as
``{anyOf: [{number, minimum, maximum}, {null}]}``. This helper digs
@@ -944,6 +951,7 @@ class TestNodeSpecContractInvariants:
# browser: live view + take-over above the node's parameters (the
# Dev side panel and the Home Workspace find browser nodes by it).
"isBrowserPanel",
+ "workspace",
# Auto-derived on every node from BaseNode.start_to_close_timeout:
# how long it may legitimately run. Lets the client size its
# request budget instead of keeping its own list of slow types.
@@ -1053,6 +1061,8 @@ class TestPluginContractInvariants:
# inputs — detect those by group.
if "social" in (spec.get("group") or []):
continue
+ if _standalone_workspace_agent(t):
+ continue
names = {h.get("name") for h in spec.get("handles") or []}
for required in ("input-skill", "input-tools", "input-context", "input-task"):
assert required in names, f"{t}: componentKind=agent missing handle {required!r}; got {sorted(names)}"
@@ -1122,6 +1132,8 @@ class TestWave10GContractInvariants:
continue
if t in self.AGENT_EXEMPT:
continue
+ if _standalone_workspace_agent(t):
+ continue
hints = spec.get("uiHints") or {}
assert hints.get("hasSkills") is True, f"{t}: componentKind=agent must declare uiHints.hasSkills=True"