A desktop coding agent with free access to frontier models.
TypeScript
fix: restart the desktop worker and follow own retitles
ogsamratd9c1aa98a126
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjsindex 0b66574c..4e46f341 100644--- a/apps/desktop/electron/computer-use.mjs+++ b/apps/desktop/electron/computer-use.mjs@@ -168,8 +168,8 @@ async function startHost() {if (!executable) throw new Error("The Computer Use helper is unavailable. Rebuild or reinstall OmniRush.ai.");created = await createComputerUseHost({ executable, ...callbacks, shared: { portable: hostModule, policy: policyModule, contract: contractModule, image: imageModule } });} else {- const native = hostModule.createNativeWorker({ nodePath: process.execPath, nodeEnv: { ELECTRON_RUN_AS_NODE: "1" }, koffiPath: require.resolve("koffi") });- created = await hostModule.createPortableHost({ native, ...callbacks });+ const nativeFactory = () => hostModule.createNativeWorker({ nodePath: process.execPath, nodeEnv: { ELECTRON_RUN_AS_NODE: "1" }, koffiPath: require.resolve("koffi") });+ created = await hostModule.createPortableHost({ native: nativeFactory(), nativeFactory, ...callbacks });preview = createComputerPreview();preview.bind(created);}diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex b6e8ef54..e8af1fbe 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -24,7 +24,7 @@ function fakeNative() {const listeners = new Set<(v: Record<string, unknown>) => void>();const calls: Array<{ method: string; params: Record<string, unknown> }> = [];let generation = 0;- const state = { occluded: false, synthetic: false };+ const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null } = { occluded: false, synthetic: false, retitle: null };const frame = (w: Window): Frame => {const width = Math.min(w.bounds.width, 640), height = Math.round((width * w.bounds.height) / w.bounds.width);const data = new Uint8Array(width * height * 4).fill(shade);@@ -40,6 +40,7 @@ function fakeNative() {windows,setShade(v: number) { shade = v; },personInput() { generation += 1; for (const l of listeners) l({ event: "person-input", generation }); },+ die() { for (const l of listeners) l({ event: "unavailable" }); },onEvent(listener: (v: Record<string, unknown>) => void) { listeners.add(listener); return () => listeners.delete(listener); },async call(method: string, params: Record<string, unknown> = {}) {calls.push({ method, params });@@ -52,7 +53,11 @@ function fakeNative() {if (method === "start") return { generation };if (method === "hit") return { inside: (params.points as unknown[]).map(() => !state.occluded) };if (method === "input-source") return { synthetic: state.synthetic, at: Date.now(), kind: "pointer" };- if (method === "act") { if (params.generation !== generation) throw Object.assign(new Error("Your input paused computer use."), { code: "paused_by_person" }); return { generation }; }+ if (method === "act") {+ if (params.generation !== generation) throw Object.assign(new Error("Your input paused computer use."), { code: "paused_by_person" });+ if (state.retitle) windows[0]!.title = state.retitle(windows[0]!.title);+ return { generation };+ }return {};},async close() {},@@ -282,6 +287,57 @@ describe("portable host: observe and act", () => {});});+describe("portable host: titles that follow the agent's own typing, and a worker that died", () => {+ test("Normal: the agent's own act retitling the window does not ask; a change between acts does", async () => {+ const r = rig("normal");+ const { host, c, session, obs } = await opened(r);+ r.native.state.retitle = () => "first line - Notepad";+ const a = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type the first line", action: { type: "type", text: "first line" } });+ expect(a.state).toMatchObject({ ok: true, approval: { asked: false } });+ r.native.state.retitle = null;+ const b = await call(c, "computer_act", { session_id: session, observation_id: (a.state.observation as { id: string }).id, request_id: "b", intent: "Type more", action: { type: "type", text: " more" } });+ expect(b.state).toMatchObject({ ok: true, approval: { asked: false } });+ expect(r.acts.length).toBe(0);+ // The window shows another document now, and not because of the agent's last act.+ r.native.windows[0]!.title = "secrets.txt - Notepad";+ const next = await call(c, "computer_observe", { session_id: session });+ const d = await call(c, "computer_act", { session_id: session, observation_id: next.state.observation?.id, request_id: "d", intent: "Type more", action: { type: "type", text: "x" } });+ expect(d.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["document_changed"] } });+ await host.close();+ });++ test("a dead worker: sessions stop; the next open starts a new one (noted on the event); at most 3 in 10 minutes", async () => {+ const workers: Array<ReturnType<typeof fakeNative>> = [];+ const first = fakeNative();+ const events: Array<Record<string, unknown>> = [];+ const host = await createPortableHost({+ native: first,+ nativeFactory: () => { const w = fakeNative(); workers.push(w); return w; },+ settings: () => ({ enabled: true, real_screen: true, mode: "auto" }),+ context: () => ({ chat: "ses_r" }),+ approve: { open: async (request: OpenRequest) => ({ decision: "once", window_id: request.window_id }), act: async () => ({ allow: true }) },+ onEvent: (event: Record<string, unknown>) => events.push(event),+ });+ const c = host.connect();+ const open = await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "x" });+ first.die();+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(events.at(-1)).toMatchObject({ event: "stopped", by: "limit", reason: "native_unavailable" });+ expect((await call(c, "computer_observe", { session_id: open.state.session_id })).state.code).toBe("session_unavailable");+ expect((await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "again" })).state).toMatchObject({ ok: true });+ expect(workers.length).toBe(1);+ expect(events.find((e) => e.event === "opened" && e.reason === "worker_restarted")).toBeTruthy();+ for (let i = 0; i < 3; i += 1) {+ workers.at(-1)!.die();+ await new Promise((resolve) => setTimeout(resolve, 10));+ await call(c, "computer_discover", {});+ }+ expect(workers.length).toBe(3);+ expect((await call(c, "computer_discover", {})).state.code).toBe("native_unavailable");+ await host.close();+ });+});+describe("portable host on Windows: the agent's ancestors come from the worker", () => {test("an IDE the agent runs in is denied as an ancestor when the app passed none", async () => {const native = fakeNative();diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index fc344f32..f1c04537 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -9,5 +9,5 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs98ffcc70c69299917c9804c32b786e58c1677edf7ea0732a3fe12b9f6781ba29 media.mjsc12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mtsa9d1df37870802ffdca408a29735d1ce2a629e93435bc46ff36a0c5f8108869a native-worker.mjs-dd4dc647994d6f52724d06856deff846cd545dc7c477b20348c14273073aa205 portable-host.mjs-3adf92d8ab9fe7a1a8b595c090f6ba4f99ed10d5146c318cfbdf7b674aec9657 portable-host.d.mts+52d7c84fbb0ee09ff74bcf2f1b19c85745211b883140efaff69ca80880735631 portable-host.mjs+444f1cdfd00eedc49626f7ce62a64fa3ca75b7c389abbe0b22a3bb3bb822711c portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mtsindex 6bb5546d..01a993de 100644--- a/apps/server/src/computer-use/shared/portable-host.d.mts+++ b/apps/server/src/computer-use/shared/portable-host.d.mts@@ -44,6 +44,7 @@ export type SessionEvent = {approval?: "once" | "chat";screen?: { id: string; width: number; height: number; scale: number };os: string;+ /** e.g. native_unavailable, action_limit, window_unavailable, worker_restarted (on "opened"). */reason?: string;reused?: boolean;};@@ -71,7 +72,7 @@ export type HostSessionState = {};export type McpConnection = { id: string; handle(message: unknown): Promise<Record<string, unknown> | null>; close(): Promise<void> };export type PortableHost = {- availability: { supported: boolean; code?: string; error?: string };+ readonly availability: { supported: boolean; code?: string; error?: string };os: string;connect(): McpConnection;listen(dir: string): Promise<{ discoveryPath: string; close(): Promise<void> }>;@@ -84,6 +85,8 @@ export type PortableHost = {};export function createPortableHost(options: {native: NativeWorker;+ /** Starts a new worker after this one died (at most 3 times in 10 minutes, on the next call). */+ nativeFactory?: (() => NativeWorker) | null;spool?: MediaSpool | null;settings: () => HostSettings;/** May be async (read once per discover / open-session). */diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 24e17e75..5ca3bff3 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -135,8 +135,12 @@ function describeAction(a) {* act(request) → { allow: boolean } }* - onEvent(event): `computer.session` events; onState(states); onFrame(frame)*/-export async function createPortableHost({ native, spool = null, settings, context = () => ({}), approve, onEvent = () => {}, onState = () => {}, onFrame = () => {}, platform = process.platform }) {- const availability = await native.ready;+export async function createPortableHost({ native: firstNative, nativeFactory = null, spool = null, settings, context = () => ({}), approve, onEvent = () => {}, onState = () => {}, onFrame = () => {}, platform = process.platform }) {+ let native = firstNative;+ let availability = await native.ready;+ let nativeDown = false;+ const restarts = [];+ let restartedNote = false;const os = osName(platform);const sessions = new Map();const grants = new Map();@@ -177,11 +181,36 @@ export async function createPortableHost({ native, spool = null, settings, conteemit(s, "paused", by);update();}- native.onEvent((value) => {+ const attach = (worker) => worker.onEvent((value) => {+ if (worker !== native) return;// The person's own answer to a question (a click or key in the app) is not a takeover.if (value.event === "person-input") for (const s of sessions.values()) if (s.phase === "working" && s.mode === "control" && !s.waiting) pause(s, "person", "Your input paused computer use. Choose Continue when you are ready.");- if (value.event === "unavailable") for (const s of [...sessions.values()]) void stop(s, "stopped", "limit", { reason: "native_unavailable" });+ if (value.event === "unavailable") {+ nativeDown = true;+ for (const s of [...sessions.values()]) void stop(s, "stopped", "limit", { reason: "native_unavailable" });+ }});+ attach(native);+ /**+ * After the native worker died (its held input already released by createNativeWorker), a new one+ * is started on the next call, at most 3 times in 10 minutes. Whether the host can work now.+ */+ async function ensureNative() {+ if (!nativeDown || !nativeFactory || closed) return !nativeDown;+ const now = Date.now();+ while (restarts.length && now - restarts[0] > 600_000) restarts.shift();+ if (restarts.length >= 3) return false;+ restarts.push(now);+ try {+ const next = nativeFactory();+ const ready = await next.ready;+ native = next; availability = ready; nativeDown = !ready.supported; restartedNote = ready.supported;+ attach(next);+ } catch {+ return false;+ }+ return !nativeDown;+ }const ticker = setInterval(() => {const l = limits();for (const s of [...sessions.values()]) {@@ -370,6 +399,9 @@ export async function createPortableHost({ native, spool = null, settings, conteconst w = await native.call("window", { id: s.window.id });const after = await observation(s, w);s.window = w; s.observation = { ...after, generation: s.generation };+ // The agent's own act may retitle the window (an editor's title follows its text): the next+ // "document changed" check compares with the title right after this act, not the approval-time one.+ s.approved = { ...s.approved, title: w.title };return finish({ ...done, frame_after: { sha256: after.sha }, observation: after.ref, window: windowRef(w), next: "computer_act" }, { data: after.bytes, mime: after.mime });} catch (error) {return finish({ ...done, next: "computer_observe", message: `The action was done, but the window could not be observed: ${error.message}` });@@ -465,7 +497,8 @@ export async function createPortableHost({ native, spool = null, settings, contes.phase = "working"; s.lastOperation = Date.now();s.expiresAt = Date.now() + l.session_seconds * 1000;s.approved = { window_id: String(s.window.id), app_id: s.window.appId, title: s.window.title };- emit(s, "opened", "agent");+ emit(s, "opened", "agent", restartedNote ? { reason: "worker_restarted" } : {});+ restartedNote = false;emit(s, "approved", answer.reused ? "agent" : "person", answer.reused ? { reused: true } : {});update();return mcpResult({ ok: true, session_id: s.id, state: "active", mode: s.mode, approval: s.approval, window: windowRef(s.window), screen: s.screen, os, expires_in_seconds: l.session_seconds, actions_left: l.actions_per_session });@@ -473,6 +506,7 @@ export async function createPortableHost({ native, spool = null, settings, conteasync function call(connection, name, rawArgs) {if (!TOOLS.includes(name)) return mcpResult(failure("unknown_tool", "Unknown computer use tool."));+ if (nativeDown && !(await ensureNative())) return mcpResult(failure("native_unavailable", "Desktop access stopped. Try again in a few minutes, or restart the app."));if (!availability.supported) return mcpResult(failure(availability.code ?? "desktop_unavailable", availability.error ?? "Computer use is unavailable on this desktop."));const valid = validateCall(name, rawArgs);if (!valid.ok) return mcpResult(failure(valid.code, valid.message));@@ -608,5 +642,5 @@ export async function createPortableHost({ native, spool = null, settings, conte}update();- return { availability, os, connect, listen, control, stopAll, syntheticInput, watchInput, state: publicState, close };+ return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, watchInput, state: publicState, close };}