Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat: computer use settings and chat tip

ogsamratd711ebd66a5e
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/setup.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/setup.tsx
new file mode 100644
index 00000000..2ab7e72b
--- /dev/null
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/setup.tsx
@@ -0,0 +1,158 @@
+/** @jsxImportSource react */
+// Computer use, findable: the Settings card (status, "Use my own screen", the account's yes) and the hint above
+// the composer when a draft asks for something on a screen. The agent's own display needs nothing from the
+// person; their own screen needs this computer's switch AND the account's yes, which only the web console sets.
+import { useState } from "react";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { ExternalLink, MousePointer2, X } from "lucide-react";
+
+import type { ComputerUseRealScreen } from "@omnirush/types/desktop-ipc";
+
+import { desktopBridge, openDesktopUrl } from "@/app/lib/desktop";
+import { Button } from "@/components/ui/button";
+import { Switch } from "@/components/ui/switch";
+
+import { useComputerUseStatus } from "./status-store";
+
+/** Mirrors ACCOUNT_REAL_SCREEN_URL in apps/server/src/computer-use/shared/policy.mjs. */
+export const ACCOUNT_REAL_SCREEN_URL = "https://omnirush.ai/console/account#computer-use";
+/** Mirrors SCREEN_INTENT in apps/server/src/computer-use/shared/policy.mjs (setup.test.ts compares them). */
+export const SCREEN_INTENT = /(?:^|[.,;!]\s*|\b(?:and|then|please|to)\s+)(?:double[- ])?click\b|\b(?:my (?:browser|screen|desktop|mouse|computer)|(?:use|open|launch|start) (?:the |a |my )?(?:browser|chrome|firefox|edge|safari|app|application|program)|screenshot|computer use|take over my|fill (?:in|out) (?:the |this )?form)\b/i;
+export const screenIntent = (text: string) => SCREEN_INTENT.test(text);
+
+export const COMPUTER_USE_TITLE = "Computer use";
+export const COMPUTER_USE_ON = "On. The agent works on its own display, with its own cursor and browser, and never asks for approval there.";
+export const OWN_SCREEN_HELP = "Off by default. When on, the agent can also click and type in the apps on your screen, without asking. Stop it any time with Stop.";
+export const ACCOUNT_NEEDED = "Your account must allow it too. The app can't turn that on: it opens your account page, where you switch on “Let the agent use my real screen”, then come back here.";
+
+const REAL_SCREEN_KEY = ["computer-use", "real-screen"];
+
+/** The status, this computer's own-screen switch and the account's yes, and the one-click way to turn both on. */
+export function useComputerUseSetup(active = true) {
+ const status = useComputerUseStatus();
+ const queryClient = useQueryClient();
+ const enabled = status?.enabled === true;
+ const realScreen = useQuery({
+ queryKey: REAL_SCREEN_KEY,
+ queryFn: async (): Promise<ComputerUseRealScreen> => desktopBridge.getComputerUseRealScreen(),
+ enabled: active && enabled,
+ retry: false,
+ staleTime: 30_000,
+ // Back from the account page: read the account's yes again.
+ refetchOnWindowFocus: true,
+ });
+ const setOn = useMutation({
+ mutationFn: async (on: boolean) => desktopBridge.setComputerUseRealScreen(on),
+ onSuccess: (next) => queryClient.setQueryData(REAL_SCREEN_KEY, next),
+ });
+ /** This computer's switch on, then the account page when the account has not said yes. */
+ const turnOn = async () => {
+ let now = realScreen.data ?? (await desktopBridge.getComputerUseRealScreen());
+ if (!now.on && !now.forced) now = await setOn.mutateAsync(true);
+ if (!now.allowed) await openDesktopUrl(ACCOUNT_REAL_SCREEN_URL);
+ };
+ const ownScreenReady = realScreen.data?.on === true && realScreen.data.allowed;
+ return { status, enabled, realScreen, setOn, turnOn, ownScreenReady };
+}
+
+function statusLine(status: ReturnType<typeof useComputerUseStatus>) {
+ if (!status) return "Checking…";
+ if (status.enabled) return COMPUTER_USE_ON;
+ return status.message ?? "Computer use is not turned on for your account yet.";
+}
+
+function modelsLine(models: string[] | null | undefined) {
+ return models?.length ? `Works with ${models.join(", ")}. Pick one of them in the chat.` : null;
+}
+
+/** Settings > General > Computer use. */
+export function ComputerUseSettingsCard() {
+ const setup = useComputerUseSetup();
+ const data = setup.realScreen.data;
+ const [error, setError] = useState<string | null>(null);
+ const run = (action: () => Promise<unknown>) => { setError(null); action().catch((cause: unknown) => setError(cause instanceof Error ? cause.message : "That did not work. Try again.")); };
+ return (
+ <div className="space-y-3" data-computer-use-settings>
+ <div className="text-[11px] font-semibold uppercase tracking-[0.15em] text-dls-secondary">{COMPUTER_USE_TITLE}</div>
+ <div className="space-y-3 rounded-2xl border border-dls-border bg-dls-surface p-4">
+ <div className="flex items-center gap-4">
+ <div className="flex h-10 w-10 shrink-0 items-center justify-center rounded-xl border border-dls-border bg-dls-hover">
+ <MousePointer2 size={17} className="text-dls-secondary" />
+ </div>
+ <div className="min-w-0 flex-1">
+ <div className="text-[13px] font-medium text-dls-text">{COMPUTER_USE_TITLE}</div>
+ <div className="text-[11px] text-dls-secondary" role="status" data-computer-use-status>{statusLine(setup.status)}</div>
+ {setup.enabled && modelsLine(setup.status?.models) ? <div className="text-[11px] text-dls-secondary">{modelsLine(setup.status?.models)}</div> : null}
+ </div>
+ </div>
+ {setup.enabled ? (
+ <div className="flex items-start gap-4 border-t border-dls-border pt-3">
+ <div className="min-w-0 flex-1">
+ <div className="text-[13px] font-medium text-dls-text">Use my own screen</div>
+ <div id="computer-use-own-screen-help" className="text-[11px] text-dls-secondary">
+ {data?.forced ? "Set by the OMNIRUSH_COMPUTER_USE_REAL_SCREEN environment variable." : OWN_SCREEN_HELP}
+ </div>
+ {data && !data.allowed ? (
+ <div className="mt-2 space-y-2">
+ <div className="text-[11px] text-dls-secondary" data-computer-use-account>{ACCOUNT_NEEDED}</div>
+ <Button type="button" variant="outline" size="xs" onClick={() => run(setup.turnOn)} data-computer-use-allow>
+ <ExternalLink data-icon="inline-start" />
+ Allow in my account
+ </Button>
+ </div>
+ ) : null}
+ {error ? <div className="mt-1 text-[11px] text-red-11" role="alert">{error}</div> : null}
+ </div>
+ <Switch
+ aria-label="Use my own screen"
+ aria-describedby="computer-use-own-screen-help"
+ checked={data?.on === true}
+ disabled={!data || data.forced || setup.setOn.isPending}
+ onCheckedChange={(on: boolean) => run(() => setup.setOn.mutateAsync(on))}
+ data-computer-use-real-screen
+ />
+ </div>
+ ) : null}
+ </div>
+ </div>
+ );
+}
+
+// "Not now": hidden for the rest of this launch.
+let hintDismissed = false;
+
+/** Above the composer, while the draft asks for something on a screen and the person's own screen is not ready. */
+export function useComputerUseHint(draft: string) {
+ const intent = !hintDismissed && screenIntent(draft);
+ const setup = useComputerUseSetup(intent);
+ const [, rerender] = useState(0);
+ if (!intent || !setup.status || setup.ownScreenReady) return null;
+ if (setup.enabled && !setup.realScreen.data) return null;
+ const dismiss = () => { hintDismissed = true; rerender((n) => n + 1); };
+ const data = setup.realScreen.data;
+ const message = !setup.enabled
+ ? statusLine(setup.status)
+ : "The agent can already do this on its own display, with its own browser and cursor. To let it use your screen and apps, turn on your own screen.";
+ const action = setup.enabled ? (data?.allowed ? "Use my own screen" : "Allow in my account") : null;
+ return (
+ <div className="mb-2 flex items-start gap-3 rounded-xl border border-dls-border bg-dls-surface px-3 py-2" role="status" data-computer-use-hint>
+ <MousePointer2 size={15} className="mt-0.5 shrink-0 text-dls-secondary" />
+ <div className="min-w-0 flex-1 text-[12px] text-dls-text">
+ <div className="font-medium">{COMPUTER_USE_TITLE}</div>
+ <div className="text-dls-secondary">{message}</div>
+ {action && !data?.allowed ? <div className="mt-0.5 text-[11px] text-dls-secondary">Opens your account page; come back here when it's on.</div> : null}
+ </div>
+ <div className="flex shrink-0 items-center gap-1">
+ {action ? (
+ <Button type="button" variant="outline" size="xs" onClick={() => void setup.turnOn().catch(() => undefined)} data-computer-use-hint-action>
+ {data?.allowed ? null : <ExternalLink data-icon="inline-start" />}
+ {action}
+ </Button>
+ ) : null}
+ <Button type="button" variant="ghost" size="icon-xs" aria-label="Not now" onClick={dismiss}>
+ <X />
+ </Button>
+ </div>
+ </div>
+ );
+}
diff --git a/apps/app/src/react-app/domains/session/surface/session-surface.tsx b/apps/app/src/react-app/domains/session/surface/session-surface.tsx
index 935560d5..b82403b7 100644
--- a/apps/app/src/react-app/domains/session/surface/session-surface.tsx
+++ b/apps/app/src/react-app/domains/session/surface/session-surface.tsx
@@ -101,6 +101,7 @@ import { getSessionActivityStatusLabel, useSessionActivityStore, type SessionAct
import { PermissionApprovalPanel } from "@/react-app/domains/session/chat/permission-approval-modal";
import { QuestionPanel } from "@/react-app/domains/session/modals/question-modal";
import { ComputerUseInline, useComputerUseForChat } from "@/react-app/domains/session/surface/computer-use/inline";
+import { useComputerUseHint } from "@/react-app/domains/session/surface/computer-use/setup";
import { QueuedMessagesPanel } from "@/react-app/domains/session/modals/queued-messages-panel";
import { GoodSessionChecklistBar, noteUserStop } from "@/react-app/domains/quality/good-session";
import { PlacementTurnFooter } from "@/react-app/domains/placements/placements";
@@ -1196,6 +1197,7 @@ export function SessionSurface(props: SessionSurfaceProps) {
);
const draft = useComposerStateStore((state) => getComposerDraft(state, props.sessionId));
const attachments = useComposerStateStore((state) => getComposerAttachments(state, props.sessionId));
+ const computerUseHint = useComputerUseHint(draft);
// True while a send with attachments is in flight (compression + inbox
// upload + prompt POST); drives the uploading overlay on composer chips.
const [attachmentsUploading, setAttachmentsUploading] = useState(false);
@@ -3746,9 +3748,9 @@ export function SessionSurface(props: SessionSurfaceProps) {
isRemoteWorkspace={props.isRemoteWorkspace}
isSandboxWorkspace={props.isSandboxWorkspace}
onUploadInboxFiles={props.onUploadInboxFiles ?? handleUploadInboxFiles}
- compactTopSpacing={Boolean(showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 || hasComputerUse)}
+ compactTopSpacing={Boolean(showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 || hasComputerUse || computerUseHint !== null)}
topAccessory={
- showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 || hasComputerUse ? (
+ showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 || hasComputerUse || computerUseHint !== null ? (
<div>
{showCompactionProgress ? <ContextCompactionProgress /> : null}
{queuedItems.length > 0 ? (
@@ -3775,6 +3777,7 @@ export function SessionSurface(props: SessionSurfaceProps) {
<TodoPanel todos={props.todos ?? []} />
) : null}
{hasComputerUse ? <ComputerUseInline sessions={computerUse} /> : null}
+ {computerUseHint}
{props.activePermission ? (
<PermissionApprovalPanel
permission={props.activePermission}
diff --git a/apps/app/src/react-app/domains/settings/pages/general-view.tsx b/apps/app/src/react-app/domains/settings/pages/general-view.tsx
index dca6fbc8..f4b68c59 100644
--- a/apps/app/src/react-app/domains/settings/pages/general-view.tsx
+++ b/apps/app/src/react-app/domains/settings/pages/general-view.tsx
@@ -51,6 +51,7 @@ import type { OmniRushAgentAssist, OmniRushAgentAssistFeature } from "../../../.
import { useLinuxSandbox, type LinuxSandboxClient } from "../linux-sandbox";
import { LinuxSandboxCard } from "../linux-sandbox-card";
import { BuildathonTrackCard } from "../../buildathon/buildathon";
+import { ComputerUseSettingsCard } from "../../session/surface/computer-use/setup";
export type GeneralSettingsViewProps = {
onNavigateTab: (tab: SettingsTab) => void;
@@ -580,6 +581,8 @@ export function GeneralSettingsView(props: GeneralSettingsViewProps) {
/>
) : null}
+ {isElectronRuntime() ? <ComputerUseSettingsCard /> : null}
+
{props.buildathonProject ? (
<BuildathonTrackCard projectPath={props.buildathonProject.path} projectName={props.buildathonProject.name} />
) : null}
diff --git a/apps/app/src/react-app/domains/whats-new/PARITY.sha256 b/apps/app/src/react-app/domains/whats-new/PARITY.sha256
index 646868f4..00e854f9 100644
--- a/apps/app/src/react-app/domains/whats-new/PARITY.sha256
+++ b/apps/app/src/react-app/domains/whats-new/PARITY.sha256
@@ -1,3 +1,3 @@
# whats-new.json is the same file in omnirush-cli (src/whats-new/) and omnirush-gui (apps/app/src/react-app/domains/whats-new/).
# A change to one must be ported to the other and this list updated in both.
-20520da3f4ac8c2b3d2bd7244afc79bc16e45b04c54e74c388b7078e9a89159a whats-new.json
+45097a0df296ac81336af2e24f54a8e6e0e1709ee5497ecb63350478ed20ee85 whats-new.json
diff --git a/apps/app/src/react-app/domains/whats-new/whats-new.json b/apps/app/src/react-app/domains/whats-new/whats-new.json
index 9b57df2b..e031c5c7 100644
--- a/apps/app/src/react-app/domains/whats-new/whats-new.json
+++ b/apps/app/src/react-app/domains/whats-new/whats-new.json
@@ -2,6 +2,15 @@
"$comment": "What's new in OmniRush: release notes written for users, newest first, shown in the CLI (/changelog, omnirush changelog) and the desktop app (Help > What's new). Same file in omnirush-cli (src/whats-new/) and omnirush-gui (apps/app/src/react-app/domains/whats-new/); keep the copies byte-identical (PARITY.sha256). Plain language, no PR numbers, nothing internal. Write the entry for every release before tagging.",
"schema": 1,
"cli": [
+ {
+ "version": "2.7.0",
+ "date": "2026-10-24",
+ "new": [
+ "/computer shows whether computer use is on and why not, and turns on the agent's use of your own screen, with a link to the account setting it also needs. When you ask for something on a screen (use my browser, click, open an app), a tip says what to turn on."
+ ],
+ "improved": [],
+ "fixed": []
+ },
{
"version": "2.6.9",
"date": "2026-10-23",
@@ -257,6 +266,15 @@
}
],
"desktop": [
+ {
+ "version": "3.7.5",
+ "date": "2026-10-24",
+ "new": [
+ "Computer use in Settings > General: see whether it is on (and why not, such as on macOS), and turn on “Use my own screen”. When you ask for something on a screen (use my browser, click, open an app), a tip above the message box turns it on in one click and opens the account setting it also needs."
+ ],
+ "improved": [],
+ "fixed": []
+ },
{
"version": "3.7.4",
"date": "2026-10-23",
diff --git a/apps/app/tests/computer-use-setup.test.ts b/apps/app/tests/computer-use-setup.test.ts
new file mode 100644
index 00000000..ea5b6c14
--- /dev/null
+++ b/apps/app/tests/computer-use-setup.test.ts
@@ -0,0 +1,19 @@
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+import { ACCOUNT_REAL_SCREEN_URL, SCREEN_INTENT, screenIntent } from "../src/react-app/domains/session/surface/computer-use/setup";
+
+const POLICY = readFileSync(path.join(import.meta.dir, "../../server/src/computer-use/shared/policy.mjs"), "utf8");
+
+describe("computer use setup", () => {
+ test("mirrors the shared policy's screen words and account link", () => {
+ expect(POLICY).toContain(`const SCREEN_INTENT = ${SCREEN_INTENT.toString()};`);
+ expect(POLICY).toContain(`export const ACCOUNT_REAL_SCREEN_URL = "${ACCOUNT_REAL_SCREEN_URL}";`);
+ });
+
+ test("a draft that asks for the screen shows the hint; code words do not", () => {
+ for (const text of ["use my browser to book a table", "click Submit", "open the app and log in", "then click Save"]) expect(screenIntent(text)).toBe(true);
+ for (const text of ["fix the failing test", "add a click handler to the button", "clickhouse migration"]) expect(screenIntent(text)).toBe(false);
+ });
+});
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index 6bc9ee6e..57fc6e40 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -123,7 +123,9 @@ function statusNow() {
enabled: server?.enabled === true,
// The agent's own display works without desktop access (no helper, Wayland), so only "off" is unavailable.
supported: supported(),
- message: server?.enabled ? (supported() ? null : "Computer use is unavailable on this desktop.") : server?.setting?.message ?? (server ? "Computer use is turned off." : null),
+ message: server?.enabled ? (supported() ? null : "Computer use is unavailable on this desktop.") : server?.unavailable ?? server?.setting?.message ?? (server ? "Computer use is turned off." : null),
+ // The models that can use it (null: any).
+ models: server?.setting?.models ?? null,
mode: server?.mode ?? "normal",
sessions,
};
diff --git a/apps/server/src/computer-use/routes.ts b/apps/server/src/computer-use/routes.ts
index c26c4c40..c83ee048 100644
--- a/apps/server/src/computer-use/routes.ts
+++ b/apps/server/src/computer-use/routes.ts
@@ -3,7 +3,7 @@
* loopback): the switch and permission mode it applies, the host file it
* writes when its host listens, and the `computer.session` events it reports.
*
- * GET /computer-use/state { enabled, setting, mode, dir, spool, chat }
+ * GET /computer-use/state { enabled, unavailable, setting, mode, dir, spool, chat }
* POST /computer-use/refresh reads /collect/capabilities again (≤ 60 s old), then the state; with `read`
* true, `setting` is the answer just read (held in memory, never re-read from the
* state folder, which the agent's shell can write), so the person's real screen
@@ -13,7 +13,7 @@
* POST /computer-use/event a computer.session event → the chat's trace
*/
import { addRoute, type Route } from "../routes/registry.js";
-import { computerUseSetting } from "./shared/policy.mjs";
+import { computerUseSetting, unavailableMessage } from "./shared/policy.mjs";
import {
computerUseDir,
computerUseSettingFor,
@@ -66,6 +66,8 @@ export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {
const chat = deps.chats.only(await deps.runningSessions().catch(() => null));
return {
enabled: setting.enabled,
+ // Why it is off here, in plain words (macOS, this app, the account), for the app to show.
+ unavailable: unavailableMessage(setting),
setting: { enabled: setting.enabled, real_screen: setting.real_screen, real_screen_opt_in: setting.real_screen_opt_in, virtual_display: setting.virtual_display, limits: setting.limits, deny_apps: setting.deny_apps, models: setting.models, message: setting.message, ask_send_delete: setting.ask_send_delete === true, reason: setting.reason, source: setting.source },
mode,
dir,
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index 8164d14b..67e15d3e 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -2,8 +2,8 @@
# A change to one must be ported to the other and this list updated in both.
da1cc6c26f3bf6758d3e2ce70e87bce383f68ee0ec6c85bbdc33b1f9a6c407fa contract.mjs
30cfcead85a5b2b54005ba6a0a67a43d9950de944b4615861ce9342bf043d4a4 contract.d.mts
-4e8ade9f4ab4bc0efef8c30ac97f8fcd04eac16b5467e26880fe87c0b6e28c20 policy.mjs
-2dbfd8c20996603445f8dc6ff30c86dddd77426d4c123cc27d99361dc2f7c50a policy.d.mts
+4d82d0f4510a5b09d2ee0272fd43ac85c4f1e1062d828ca50af33400649283e2 policy.mjs
+8f15c565a8d80054cf41f68bb37b1801cb410d260928801d11e3f042a7858c52 policy.d.mts
da515baa8aec2b533685d20c2be7ae007acc6b116db0386f8180e6779cb80f5c image.mjs
7c1a3224b7efd215656a94c38cf3152fe7a2444f25a6162988528978acf5458d image.d.mts
e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mts
index c39b3718..a3c5ded7 100644
--- a/apps/server/src/computer-use/shared/policy.d.mts
+++ b/apps/server/src/computer-use/shared/policy.d.mts
@@ -25,11 +25,16 @@ export type ComputerUseBlock = {
export type ComputerUseSetting = ComputerUseBlock & {
source: "env" | "server" | "default";
reason: null | "env" | "no_switch" | "off" | "client" | "os" | "no_display";
+ /** This device's OS (macos, windows, linux) when computer use is off for it. */
+ os_name?: string;
};
export function normalizeComputerUse(block: unknown): ComputerUseBlock;
export function computerUseSetting(options?: { env?: Record<string, string | undefined>; block?: unknown; client?: "cli" | "desktop"; platform?: string }): ComputerUseSetting;
export function modelAllowed(setting: ComputerUseSetting | ComputerUseBlock | null | undefined, modelId: unknown): boolean;
+export const ACCOUNT_REAL_SCREEN_URL: string;
+export function unavailableMessage(setting: { enabled?: boolean; reason?: string | null; os_name?: string; message?: string | null } | null | undefined): string | null;
+export function screenIntent(text: unknown): boolean;
export function offMessage(setting: { message?: string | null } | null | undefined): string;
export const REAL_SCREEN_DEFAULT: boolean;
diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjs
index 00d60026..b8c03063 100644
--- a/apps/server/src/computer-use/shared/policy.mjs
+++ b/apps/server/src/computer-use/shared/policy.mjs
@@ -93,9 +93,10 @@ export function computerUseSetting({ env = process.env, block, client, platform
// The environment turns computer use on (tests, support), never the person's real screen: that stays the server's.
if (ON.includes(asked)) return { ...server, enabled: true, virtual_display: true, source: "env", reason: null };
if (!isRecord(block)) return { ...server, enabled: false, source: "default", reason: "no_switch" };
- if (!server.enabled) return { ...server, source: "server", reason: "off" };
+ // Off: say why when the block shows it (this OS or this app not in the rollout), so people get a plain reason.
+ if (!server.enabled) return { ...server, source: "server", reason: server.os && !server.os.includes(osName(platform)) ? "os" : server.clients && client && !server.clients.includes(client) ? "client" : "off", os_name: osName(platform) };
if (server.clients && client && !server.clients.includes(client)) return { ...server, enabled: false, source: "server", reason: "client" };
- if (server.os && !server.os.includes(osName(platform))) return { ...server, enabled: false, source: "server", reason: "os" };
+ if (server.os && !server.os.includes(osName(platform))) return { ...server, enabled: false, source: "server", reason: "os", os_name: osName(platform) };
if (!server.real_screen && !server.virtual_display) return { ...server, enabled: false, source: "server", reason: "no_display" };
return { ...server, source: "server", reason: null };
}
@@ -106,6 +107,29 @@ export function modelAllowed(setting, modelId) {
return !Array.isArray(setting.models) || setting.models.includes(String(modelId ?? ""));
}
+const OS_LABELS = { macos: "macOS", windows: "Windows", linux: "Linux" };
+
+/** Where the person turns their own screen on for the account (the web console only; the apps cannot). */
+export const ACCOUNT_REAL_SCREEN_URL = "https://omnirush.ai/console/account#computer-use";
+
+/** Why computer use is not available here, in plain words for people; null while it is on. */
+export function unavailableMessage(setting) {
+ if (!setting || setting.enabled) return null;
+ if (setting.reason === "os") return `Computer use is not available on ${OS_LABELS[setting.os_name] ?? "this system"} yet.`;
+ if (setting.reason === "client") return "Computer use is not available in this app yet.";
+ if (setting.reason === "env") return "Computer use is off on this computer (OMNIRUSH_COMPUTER_USE=0).";
+ if (setting.reason === "no_switch") return "Computer use is not ready yet: sign in and stay online once so the app can check your account.";
+ return setting.message ?? "Computer use is not turned on for your account yet.";
+}
+
+// Words that ask for something on a screen: a browser, an app, clicks, typing into a window.
+const SCREEN_INTENT = /(?:^|[.,;!]\s*|\b(?:and|then|please|to)\s+)(?:double[- ])?click\b|\b(?:my (?:browser|screen|desktop|mouse|computer)|(?:use|open|launch|start) (?:the |a |my )?(?:browser|chrome|firefox|edge|safari|app|application|program)|screenshot|computer use|take over my|fill (?:in|out) (?:the |this )?form)\b/i;
+
+/** Whether a prompt asks for something done on a screen (use my browser, click, open an app). */
+export function screenIntent(text) {
+ return typeof text === "string" && SCREEN_INTENT.test(text);
+}
+
/** The message people see when computer use is off. */
export function offMessage(setting) {
return setting?.message ?? OFF_MESSAGE;
diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.ts
index 9c3efc49..9b789375 100644
--- a/packages/types/src/desktop-ipc.ts
+++ b/packages/types/src/desktop-ipc.ts
@@ -923,6 +923,8 @@ export type ComputerUseStatus = {
supported: boolean;
/** Why it is off or unavailable (people-facing), when it is. */
message: string | null;
+ /** The models that can use computer use (null: any). */
+ models?: string[] | null;
mode: "careful" | "normal" | "auto";
sessions: ComputerUseSessionState[];
};