Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: restart the desktop worker and follow own retitles

ogsamratd9c1aa98a126
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index 0b66574c..4e46f341 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -168,8 +168,8 @@ async function startHost() {
if (!executable) throw new Error("The Computer Use helper is unavailable. Rebuild or reinstall OmniRush.ai.");
created = await createComputerUseHost({ executable, ...callbacks, shared: { portable: hostModule, policy: policyModule, contract: contractModule, image: imageModule } });
} else {
- const native = hostModule.createNativeWorker({ nodePath: process.execPath, nodeEnv: { ELECTRON_RUN_AS_NODE: "1" }, koffiPath: require.resolve("koffi") });
- created = await hostModule.createPortableHost({ native, ...callbacks });
+ const nativeFactory = () => hostModule.createNativeWorker({ nodePath: process.execPath, nodeEnv: { ELECTRON_RUN_AS_NODE: "1" }, koffiPath: require.resolve("koffi") });
+ created = await hostModule.createPortableHost({ native: nativeFactory(), nativeFactory, ...callbacks });
preview = createComputerPreview();
preview.bind(created);
}
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index b6e8ef54..e8af1fbe 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -24,7 +24,7 @@ function fakeNative() {
const listeners = new Set<(v: Record<string, unknown>) => void>();
const calls: Array<{ method: string; params: Record<string, unknown> }> = [];
let generation = 0;
- const state = { occluded: false, synthetic: false };
+ const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null } = { occluded: false, synthetic: false, retitle: null };
const frame = (w: Window): Frame => {
const width = Math.min(w.bounds.width, 640), height = Math.round((width * w.bounds.height) / w.bounds.width);
const data = new Uint8Array(width * height * 4).fill(shade);
@@ -40,6 +40,7 @@ function fakeNative() {
windows,
setShade(v: number) { shade = v; },
personInput() { generation += 1; for (const l of listeners) l({ event: "person-input", generation }); },
+ die() { for (const l of listeners) l({ event: "unavailable" }); },
onEvent(listener: (v: Record<string, unknown>) => void) { listeners.add(listener); return () => listeners.delete(listener); },
async call(method: string, params: Record<string, unknown> = {}) {
calls.push({ method, params });
@@ -52,7 +53,11 @@ function fakeNative() {
if (method === "start") return { generation };
if (method === "hit") return { inside: (params.points as unknown[]).map(() => !state.occluded) };
if (method === "input-source") return { synthetic: state.synthetic, at: Date.now(), kind: "pointer" };
- if (method === "act") { if (params.generation !== generation) throw Object.assign(new Error("Your input paused computer use."), { code: "paused_by_person" }); return { generation }; }
+ if (method === "act") {
+ if (params.generation !== generation) throw Object.assign(new Error("Your input paused computer use."), { code: "paused_by_person" });
+ if (state.retitle) windows[0]!.title = state.retitle(windows[0]!.title);
+ return { generation };
+ }
return {};
},
async close() {},
@@ -282,6 +287,57 @@ describe("portable host: observe and act", () => {
});
});
+describe("portable host: titles that follow the agent's own typing, and a worker that died", () => {
+ test("Normal: the agent's own act retitling the window does not ask; a change between acts does", async () => {
+ const r = rig("normal");
+ const { host, c, session, obs } = await opened(r);
+ r.native.state.retitle = () => "first line - Notepad";
+ const a = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type the first line", action: { type: "type", text: "first line" } });
+ expect(a.state).toMatchObject({ ok: true, approval: { asked: false } });
+ r.native.state.retitle = null;
+ const b = await call(c, "computer_act", { session_id: session, observation_id: (a.state.observation as { id: string }).id, request_id: "b", intent: "Type more", action: { type: "type", text: " more" } });
+ expect(b.state).toMatchObject({ ok: true, approval: { asked: false } });
+ expect(r.acts.length).toBe(0);
+ // The window shows another document now, and not because of the agent's last act.
+ r.native.windows[0]!.title = "secrets.txt - Notepad";
+ const next = await call(c, "computer_observe", { session_id: session });
+ const d = await call(c, "computer_act", { session_id: session, observation_id: next.state.observation?.id, request_id: "d", intent: "Type more", action: { type: "type", text: "x" } });
+ expect(d.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["document_changed"] } });
+ await host.close();
+ });
+
+ test("a dead worker: sessions stop; the next open starts a new one (noted on the event); at most 3 in 10 minutes", async () => {
+ const workers: Array<ReturnType<typeof fakeNative>> = [];
+ const first = fakeNative();
+ const events: Array<Record<string, unknown>> = [];
+ const host = await createPortableHost({
+ native: first,
+ nativeFactory: () => { const w = fakeNative(); workers.push(w); return w; },
+ settings: () => ({ enabled: true, real_screen: true, mode: "auto" }),
+ context: () => ({ chat: "ses_r" }),
+ approve: { open: async (request: OpenRequest) => ({ decision: "once", window_id: request.window_id }), act: async () => ({ allow: true }) },
+ onEvent: (event: Record<string, unknown>) => events.push(event),
+ });
+ const c = host.connect();
+ const open = await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "x" });
+ first.die();
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ expect(events.at(-1)).toMatchObject({ event: "stopped", by: "limit", reason: "native_unavailable" });
+ expect((await call(c, "computer_observe", { session_id: open.state.session_id })).state.code).toBe("session_unavailable");
+ expect((await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "again" })).state).toMatchObject({ ok: true });
+ expect(workers.length).toBe(1);
+ expect(events.find((e) => e.event === "opened" && e.reason === "worker_restarted")).toBeTruthy();
+ for (let i = 0; i < 3; i += 1) {
+ workers.at(-1)!.die();
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ await call(c, "computer_discover", {});
+ }
+ expect(workers.length).toBe(3);
+ expect((await call(c, "computer_discover", {})).state.code).toBe("native_unavailable");
+ await host.close();
+ });
+});
+
describe("portable host on Windows: the agent's ancestors come from the worker", () => {
test("an IDE the agent runs in is denied as an ancestor when the app passed none", async () => {
const native = fakeNative();
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index fc344f32..f1c04537 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -9,5 +9,5 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
98ffcc70c69299917c9804c32b786e58c1677edf7ea0732a3fe12b9f6781ba29 media.mjs
c12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts
a9d1df37870802ffdca408a29735d1ce2a629e93435bc46ff36a0c5f8108869a native-worker.mjs
-dd4dc647994d6f52724d06856deff846cd545dc7c477b20348c14273073aa205 portable-host.mjs
-3adf92d8ab9fe7a1a8b595c090f6ba4f99ed10d5146c318cfbdf7b674aec9657 portable-host.d.mts
+52d7c84fbb0ee09ff74bcf2f1b19c85745211b883140efaff69ca80880735631 portable-host.mjs
+444f1cdfd00eedc49626f7ce62a64fa3ca75b7c389abbe0b22a3bb3bb822711c portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mts
index 6bb5546d..01a993de 100644
--- a/apps/server/src/computer-use/shared/portable-host.d.mts
+++ b/apps/server/src/computer-use/shared/portable-host.d.mts
@@ -44,6 +44,7 @@ export type SessionEvent = {
approval?: "once" | "chat";
screen?: { id: string; width: number; height: number; scale: number };
os: string;
+ /** e.g. native_unavailable, action_limit, window_unavailable, worker_restarted (on "opened"). */
reason?: string;
reused?: boolean;
};
@@ -71,7 +72,7 @@ export type HostSessionState = {
};
export type McpConnection = { id: string; handle(message: unknown): Promise<Record<string, unknown> | null>; close(): Promise<void> };
export type PortableHost = {
- availability: { supported: boolean; code?: string; error?: string };
+ readonly availability: { supported: boolean; code?: string; error?: string };
os: string;
connect(): McpConnection;
listen(dir: string): Promise<{ discoveryPath: string; close(): Promise<void> }>;
@@ -84,6 +85,8 @@ export type PortableHost = {
};
export function createPortableHost(options: {
native: NativeWorker;
+ /** Starts a new worker after this one died (at most 3 times in 10 minutes, on the next call). */
+ nativeFactory?: (() => NativeWorker) | null;
spool?: MediaSpool | null;
settings: () => HostSettings;
/** May be async (read once per discover / open-session). */
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index 24e17e75..5ca3bff3 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -135,8 +135,12 @@ function describeAction(a) {
* act(request) → { allow: boolean } }
* - onEvent(event): `computer.session` events; onState(states); onFrame(frame)
*/
-export async function createPortableHost({ native, spool = null, settings, context = () => ({}), approve, onEvent = () => {}, onState = () => {}, onFrame = () => {}, platform = process.platform }) {
- const availability = await native.ready;
+export async function createPortableHost({ native: firstNative, nativeFactory = null, spool = null, settings, context = () => ({}), approve, onEvent = () => {}, onState = () => {}, onFrame = () => {}, platform = process.platform }) {
+ let native = firstNative;
+ let availability = await native.ready;
+ let nativeDown = false;
+ const restarts = [];
+ let restartedNote = false;
const os = osName(platform);
const sessions = new Map();
const grants = new Map();
@@ -177,11 +181,36 @@ export async function createPortableHost({ native, spool = null, settings, conte
emit(s, "paused", by);
update();
}
- native.onEvent((value) => {
+ const attach = (worker) => worker.onEvent((value) => {
+ if (worker !== native) return;
// The person's own answer to a question (a click or key in the app) is not a takeover.
if (value.event === "person-input") for (const s of sessions.values()) if (s.phase === "working" && s.mode === "control" && !s.waiting) pause(s, "person", "Your input paused computer use. Choose Continue when you are ready.");
- if (value.event === "unavailable") for (const s of [...sessions.values()]) void stop(s, "stopped", "limit", { reason: "native_unavailable" });
+ if (value.event === "unavailable") {
+ nativeDown = true;
+ for (const s of [...sessions.values()]) void stop(s, "stopped", "limit", { reason: "native_unavailable" });
+ }
});
+ attach(native);
+ /**
+ * After the native worker died (its held input already released by createNativeWorker), a new one
+ * is started on the next call, at most 3 times in 10 minutes. Whether the host can work now.
+ */
+ async function ensureNative() {
+ if (!nativeDown || !nativeFactory || closed) return !nativeDown;
+ const now = Date.now();
+ while (restarts.length && now - restarts[0] > 600_000) restarts.shift();
+ if (restarts.length >= 3) return false;
+ restarts.push(now);
+ try {
+ const next = nativeFactory();
+ const ready = await next.ready;
+ native = next; availability = ready; nativeDown = !ready.supported; restartedNote = ready.supported;
+ attach(next);
+ } catch {
+ return false;
+ }
+ return !nativeDown;
+ }
const ticker = setInterval(() => {
const l = limits();
for (const s of [...sessions.values()]) {
@@ -370,6 +399,9 @@ export async function createPortableHost({ native, spool = null, settings, conte
const w = await native.call("window", { id: s.window.id });
const after = await observation(s, w);
s.window = w; s.observation = { ...after, generation: s.generation };
+ // The agent's own act may retitle the window (an editor's title follows its text): the next
+ // "document changed" check compares with the title right after this act, not the approval-time one.
+ s.approved = { ...s.approved, title: w.title };
return finish({ ...done, frame_after: { sha256: after.sha }, observation: after.ref, window: windowRef(w), next: "computer_act" }, { data: after.bytes, mime: after.mime });
} catch (error) {
return finish({ ...done, next: "computer_observe", message: `The action was done, but the window could not be observed: ${error.message}` });
@@ -465,7 +497,8 @@ export async function createPortableHost({ native, spool = null, settings, conte
s.phase = "working"; s.lastOperation = Date.now();
s.expiresAt = Date.now() + l.session_seconds * 1000;
s.approved = { window_id: String(s.window.id), app_id: s.window.appId, title: s.window.title };
- emit(s, "opened", "agent");
+ emit(s, "opened", "agent", restartedNote ? { reason: "worker_restarted" } : {});
+ restartedNote = false;
emit(s, "approved", answer.reused ? "agent" : "person", answer.reused ? { reused: true } : {});
update();
return mcpResult({ ok: true, session_id: s.id, state: "active", mode: s.mode, approval: s.approval, window: windowRef(s.window), screen: s.screen, os, expires_in_seconds: l.session_seconds, actions_left: l.actions_per_session });
@@ -473,6 +506,7 @@ export async function createPortableHost({ native, spool = null, settings, conte
async function call(connection, name, rawArgs) {
if (!TOOLS.includes(name)) return mcpResult(failure("unknown_tool", "Unknown computer use tool."));
+ if (nativeDown && !(await ensureNative())) return mcpResult(failure("native_unavailable", "Desktop access stopped. Try again in a few minutes, or restart the app."));
if (!availability.supported) return mcpResult(failure(availability.code ?? "desktop_unavailable", availability.error ?? "Computer use is unavailable on this desktop."));
const valid = validateCall(name, rawArgs);
if (!valid.ok) return mcpResult(failure(valid.code, valid.message));
@@ -608,5 +642,5 @@ export async function createPortableHost({ native, spool = null, settings, conte
}
update();
- return { availability, os, connect, listen, control, stopAll, syntheticInput, watchInput, state: publicState, close };
+ return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, watchInput, state: publicState, close };
}