Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Close the Devin Review findings on computer use

Fei2411189f88ef9aa8
diff --git a/app/src/computer-indicator-preload.cjs b/app/src/computer-indicator-preload.cjs
new file mode 100644
index 00000000..fb62b30c
--- /dev/null
+++ b/app/src/computer-indicator-preload.cjs
@@ -0,0 +1,8 @@
+// The floating computer-use indicator's only window API: send a button's
+// action to the main process. The main process accepts the channel from this
+// window alone.
+const { contextBridge, ipcRenderer } = require('electron')
+
+contextBridge.exposeInMainWorld('computerIndicator', {
+ act: (kind, botId) => ipcRenderer.send('ds-bot:computer-indicator:act', kind, botId),
+})
diff --git a/app/src/computer-indicator.mjs b/app/src/computer-indicator.mjs
index 25cfadc8..c6aa6987 100644
--- a/app/src/computer-indicator.mjs
+++ b/app/src/computer-indicator.mjs
@@ -8,12 +8,14 @@
* BrowserWindow: new (options: object) => object,
* screen: { getPrimaryDisplay: () => { workAreaSize: { width: number, height: number }, workArea: { x: number, y: number } } },
* html: (entries: object[]) => string,
+ * ipcMain?: { on: (channel: string, listener: (event: { sender: object }, ...args: any[]) => void) => void },
+ * preload?: string,
* onAction?: (action: string, botId: string | null) => void,
* log?: Pick<Console, 'warn'>,
* }} options injected Electron surface and a renderer for the bar
* @returns {{ update: (entries: object[]) => void, dispose: () => void }}
*/
-export function createComputerIndicator({ BrowserWindow, screen, html, onAction = () => {}, log = console }) {
+export function createComputerIndicator({ BrowserWindow, screen, html, ipcMain, preload, onAction = () => {}, log = console }) {
let win = null
let entries = []
let labels = {}
@@ -36,19 +38,18 @@ export function createComputerIndicator({ BrowserWindow, screen, html, onAction
skipTaskbar: true,
hasShadow: false,
show: false,
- webPreferences: { sandbox: true },
+ webPreferences: { sandbox: true, preload },
})
// The bar must never appear inside a screenshot the provider hands a Bot.
try { win.setContentProtection(true) } catch (error) { log.warn(`DS Bot: the indicator could not hide from screenshots: ${error instanceof Error ? error.message : String(error)}`) }
try { win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }) } catch (error) { log.warn(`DS Bot: the indicator could not join every workspace: ${error instanceof Error ? error.message : String(error)}`) }
win.once('ready-to-show', () => { win?.showInactive() })
win.on('closed', () => { win = null })
- // Button clicks in the bar page post a message back; the main process turns
- // them into client actions. The page is data: only, no preload.
- win.webContents?.on?.('console-message', (_event, _level, message) => {
- if (typeof message !== 'string' || !message.startsWith('dsbot-computer:')) return
- const [, action, botId] = message.split(':')
- onAction(action, botId === '' || botId === undefined ? null : botId)
+ // Button clicks reach the main process over a dedicated channel through the
+ // indicator's own preload; nothing else may send on it.
+ ipcMain?.on?.('ds-bot:computer-indicator:act', (event, action, botId) => {
+ if (event?.sender !== win?.webContents) return
+ onAction(String(action ?? ''), botId === '' || botId === undefined || botId === null ? null : String(botId))
})
const page = html(entries, labels)
win.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(page)}`).catch((error) => {
@@ -93,7 +94,7 @@ export function indicatorHtml(entries, labels = {}) {
const rows = entries.map(entry => entry.waiting ? waiting(entry) : row(entry)).join('')
const stopAll = entries.filter(entry => !entry.waiting).length > 1
? `<button data-act="stopall:" style="border:0;background:none;color:#2563eb;cursor:pointer;font:inherit;white-space:nowrap">${escapeHtml(labels.stopAll ?? 'Stop all')}</button>` : ''
- return `<!doctype html><html><body style="margin:0;display:flex;align-items:center;gap:8px;height:40px;padding:0 12px;border-radius:20px;background:rgba(28,28,30,.92);color:#fff;font:12.5px -apple-system,sans-serif;-webkit-app-region:no-drag;user-select:none" onclick="const a=event.target.dataset&&event.target.dataset.act;if(a)console.log('dsbot-computer:'+a)">${rows}${stopAll}</body></html>`
+ return `<!doctype html><html><body style="margin:0;display:flex;align-items:center;gap:8px;height:40px;padding:0 12px;border-radius:20px;background:rgba(28,28,30,.92);color:#fff;font:12.5px -apple-system,sans-serif;-webkit-app-region:no-drag;user-select:none" onclick="const a=event.target.dataset&&event.target.dataset.act;if(a&&window.computerIndicator){const i=a.indexOf(':');computerIndicator.act(a.slice(0,i),a.slice(i+1)||null)}">${rows}${stopAll}</body></html>`
}
function escapeHtml(value) {
diff --git a/app/src/main.mjs b/app/src/main.mjs
index 80aa1f7a..c8c98b94 100644
--- a/app/src/main.mjs
+++ b/app/src/main.mjs
@@ -726,7 +726,8 @@ function handleIpc() {
const fixed = Boolean(process.env.DS_BOT_HOME)
// The team view has mounted (ds-bot calls dsBotApp.ready()): the start window hands over.
const computerIndicator = createComputerIndicator({
- BrowserWindow, screen, html: indicatorHtml,
+ BrowserWindow, screen, html: indicatorHtml, ipcMain,
+ preload: join(here, 'computer-indicator-preload.cjs'),
onAction: (kind, botId) => { mainWindow?.webContents.send(IPC.computerAction, { kind, botId }) },
log: logs,
})
diff --git a/app/test/computer-indicator.test.mjs b/app/test/computer-indicator.test.mjs
index 6cf46018..31d4d029 100644
--- a/app/test/computer-indicator.test.mjs
+++ b/app/test/computer-indicator.test.mjs
@@ -49,13 +49,15 @@ test('the indicator window is frameless, unfocusable, protected and centered', (
test('empty entries hide the window; button clicks reach onAction', async () => {
const { BrowserWindow, screen, windows } = fakeElectron()
const actions = []
- const indicator = createComputerIndicator({ BrowserWindow, screen, html: indicatorHtml, onAction: (kind, botId) => actions.push([kind, botId]), log: { warn: () => {} } })
+ const listeners = {}
+ const ipcMain = { on: (channel, fn) => { listeners[channel] = fn } }
+ const indicator = createComputerIndicator({ BrowserWindow, screen, html: indicatorHtml, ipcMain, onAction: (kind, botId) => actions.push([kind, botId]), log: { warn: () => {} } })
indicator.update({ rows: [{ label: 'River · Notes', botId: 'w1' }], labels: {} })
const [win] = windows
- win.web['console-message'](null, 3, 'dsbot-computer:stop:w1')
- win.web['console-message'](null, 3, 'dsbot-computer:stopall:')
- win.web['console-message'](null, 3, 'dsbot-computer:letgo:m1')
- win.web['console-message'](null, 3, 'console noise')
+ const act = listeners['ds-bot:computer-indicator:act']
+ act({ sender: win.webContents }, 'stop', 'w1')
+ act({ sender: win.webContents }, 'stopall', '')
+ act({ sender: win.webContents }, 'letgo', 'm1')
assert.deepEqual(actions, [['stop', 'w1'], ['stopall', null], ['letgo', 'm1']])
indicator.update({ rows: [], labels: {} })
assert.equal(win.closed, true)
@@ -78,3 +80,30 @@ test('the page lists every entry, escapes markup and uses the passed labels', ()
const zh = indicatorHtml([{ label: 'River · Notes', botId: 'w1' }], { stop: '停止' })
assert.match(zh, />停止</)
})
+
+test('button actions go over the dedicated channel and only from the indicator', () => {
+ const { windows, BrowserWindow } = fakeElectron()
+ const listeners = {}
+ const ipcMain = { on: (channel, fn) => { listeners[channel] = fn } }
+ const calls = []
+ const indicator = createComputerIndicator({
+ BrowserWindow,
+ screen: { getPrimaryDisplay: () => ({ workAreaSize: { width: 800, height: 600 }, workArea: { x: 0, y: 0 } }) },
+ html: () => '<html></html>',
+ ipcMain,
+ preload: '/tmp/preload.cjs',
+ onAction: (kind, botId) => calls.push([kind, botId]),
+ log: { warn: () => {} },
+ })
+ indicator.update({ rows: [{ label: 'Chief · Notes', botId: 'w1' }], labels: {} })
+ const win = windows[0]
+ assert.equal(win.options.webPreferences.preload, '/tmp/preload.cjs')
+ const handler = listeners['ds-bot:computer-indicator:act']
+ assert.equal(typeof handler, 'function')
+ // another window's webContents may not send
+ handler({ sender: {} }, 'stop', 'w1')
+ assert.deepEqual(calls, [])
+ handler({ sender: win.webContents }, 'stop', 'w1')
+ assert.deepEqual(calls, [['stop', 'w1']])
+ indicator.dispose()
+})
diff --git a/ds-bot/client.js b/ds-bot/client.js
index 707004e7..c65f3ec2 100644
--- a/ds-bot/client.js
+++ b/ds-bot/client.js
@@ -5372,20 +5372,12 @@ var actionName = (method) => {
const words = String(method ?? "").replace(/^cua_driver_native__/, "").replace(/([a-z])([A-Z])/g, "$1 $2").replace(/_/g, " ").trim();
return words === "" ? "Computer" : words[0].toUpperCase() + words.slice(1);
};
-var callArgs = (call2) => {
- const args = call2?.arguments;
- if (args !== null && typeof args === "object") return args;
- try {
- return JSON.parse(args ?? "{}");
- } catch {
- return {};
- }
-};
function ComputerCallRow({ sessionId, call: call2, roster, actions, hooks }) {
- const args = callArgs(call2);
- const method = args.method ?? String(call2?.name ?? "").replace(/^cua_driver_native__/, "");
+ const { value: argValue } = toolArgs(call2);
+ const method = argValue("method") ?? String(call2?.name ?? "").replace(/^cua_driver_native__/, "");
const stepApp = hooks?.useComputer?.((map) => (map.bots?.[roster.byId[sessionId]?.id]?.steps ?? []).find((step) => step.callId === call2?.callId)?.app);
- const app = stepApp ?? (method === "getApp" && typeof args.params?.selector === "string" && args.params.selector !== "" ? args.params.selector : void 0);
+ const selector = argValue("params")?.selector;
+ const app = stepApp ?? (method === "getApp" && typeof selector === "string" && selector !== "" ? selector : void 0);
const shot = (call2?.content ?? []).find((block) => (block?.type === "image" || block?.type === "file") && block.attachment !== void 0)?.attachment;
const [url, setUrl] = (0, import_react10.useState)(null);
const [open, setOpen] = (0, import_react10.useState)(false);
@@ -5442,7 +5434,7 @@ function PendingComputerHandoff({ sessionId, bot, hold, actions, inline }) {
const [closed, setClosed] = (0, import_react10.useState)(false);
const busyRef = (0, import_react10.useRef)(false);
const titleId = (0, import_react10.useId)();
- const keep = useDockHold(sessionId, "handoff", hold.callId, actions, inline);
+ const keep = useDockHold(sessionId, "handoff", hold.callId ?? hold.id, actions, inline);
const taken = hold.by === "user";
const run2 = (make, close) => {
if (busyRef.current) return;
@@ -5456,7 +5448,7 @@ function PendingComputerHandoff({ sessionId, bot, hold, actions, inline }) {
busyRef.current = false;
});
};
- const answer = (outcome) => run2(() => actions.computerHoldAnswer(bot.id, { callId: hold.callId, outcome }), true);
+ const answer = (outcome) => run2(() => actions.computerHoldAnswer(bot.id, { callId: hold.callId ?? hold.id, outcome }), true);
const takeOver = () => run2(async () => {
await actions.computerTakeover(bot.id);
actions.openDetails?.(sessionId, "computer");
@@ -5508,10 +5500,11 @@ function ComputerHandoffCall({ sessionId, callId, roster, actions, heldCards, ho
function ComputerHandoffDock({ sessionId, useRoster, useCards, useHeldCards, useComputer, actions }) {
const bot = useRoster((value) => value.byId[sessionId]);
const hold = useComputer?.((map) => bot ? map.holds?.[bot.id] : void 0);
- const drawn = useCards?.((state) => state[sessionId]?.[hold?.callId] > 0) === true;
- const held = useHeldCards?.((state) => state[sessionId]?.[heldCardKey("handoff", hold?.callId)] > 0) === true;
- if (!bot || hold?.callId === void 0 || !actions.computerHoldAnswer || drawn && !held) return null;
- return (0, import_react10.createElement)(PendingComputerHandoff, { key: JSON.stringify([sessionId, hold.callId]), sessionId, bot, hold, actions, inline: false });
+ const holdKey = hold?.callId ?? hold?.id;
+ const drawn = useCards?.((state) => state[sessionId]?.[holdKey] > 0) === true;
+ const held = useHeldCards?.((state) => state[sessionId]?.[heldCardKey("handoff", holdKey)] > 0) === true;
+ if (!bot || holdKey === void 0 || !actions.computerHoldAnswer || drawn && !held) return null;
+ return (0, import_react10.createElement)(PendingComputerHandoff, { key: JSON.stringify([sessionId, holdKey]), sessionId, bot, hold, actions, inline: false });
}
function SettledComputerHandoff({ handoff }) {
const skipped = handoff.outcome === "skipped";
@@ -10399,7 +10392,7 @@ function ToolCell({ node, sessionId, useRoster, useUi, useWork, useHeldCards, us
const running = root.kind === "tool-result" ? void 0 : calls.findLast((call2) => call2.kind !== "tool-result");
const note = running ? toolActivity(toolName(running), toolArgs(running).text, findBot, roster.byId[sessionId]) : null;
useLiveNote(actions, sessionId, `tool:${root.callId ?? node.key}`, note);
- useShownCards(actions, sessionId, drawn.filter((call2) => CARDS.has(toolName(call2)) && call2.callId).map((call2) => call2.callId));
+ useShownCards(actions, sessionId, drawn.filter((call2) => (CARDS.has(toolName(call2)) || isComputerCall(toolName(call2))) && call2.callId).map((call2) => call2.callId));
if (drawn.length === 0) return (0, import_react21.createElement)(CellEmpty);
const env = { sessionId, roster, actions, useUi, useWork, findBot, heldCards, hooks: { useBrowserHolds, useBrowserTabs, useBrowserPages, useComputer } };
return (0, import_react21.createElement)(import_react21.Fragment, null, drawn.map((call2, index) => (0, import_react21.createElement)(import_react21.Fragment, { key: call2.callId ?? index }, renderCall(call2, env))));
@@ -21368,11 +21361,11 @@ function SettingsDialog({ page, id, roster, actions, useUpdate, leaving = false
back ? (0, import_react50.createElement)("button", { type: "button", className: "bt-icon-btn", "aria-label": back, title: back, onClick: () => actions.openSettings(shown) }, (0, import_react50.createElement)(ChevronLeftIcon)) : null,
(0, import_react50.createElement)("h2", { key: bot?.id ?? room2?.id ?? shown }, bot?.name ?? room2?.name ?? label)
),
- shown === "computer" ? (0, import_react50.createElement)(ComputerPage, { roster, actions, run: run2, Section: SetSection, Row: SetRow, Switch }) : null,
shown === "usage" ? null : (0, import_react50.createElement)(
"div",
{ key: pageKey, className: "bt-settings-scroll bt-settings-page", "data-move": last.current.move },
errorLine,
+ shown === "computer" ? (0, import_react50.createElement)(ComputerPage, { roster, actions, run: run2, Section: SetSection, Row: SetRow, Switch }) : null,
shown === "general" ? (0, import_react50.createElement)(GeneralPage, { roster, actions, run: run2 }) : null,
shown === "models" ? (0, import_react50.createElement)(ProvidersPage, { actions, run: run2, Section: SetSection }) : null,
bot ? (0, import_react50.createElement)("div", { className: "bt-settings-form" }, (0, import_react50.createElement)(BotSettingsForm, { key: bot.id, bot, roster, actions, run: run2 })) : null,
@@ -25600,11 +25593,8 @@ function apply(ctx) {
() => shadow("conversation.input.dock", { id: "bot-computer-app", order: -83, inject: face }, AppAccessDock),
() => shadow("conversation.input.dock", { id: "bot-learning", order: 850, priority: BOT_PRIORITY, inject: face }, LearningDock)
];
- ctx.effect(
- () => installCompiledStyles(RETURN_CSS, "return"),
- () => installCompiledStyles(COMPUTER_CSS, "computer"),
- "ds-bot: return styles"
- );
+ ctx.effect(() => installCompiledStyles(RETURN_CSS, "return"), "ds-bot: return styles");
+ ctx.effect(() => installCompiledStyles(COMPUTER_CSS, "computer"), "ds-bot: computer styles");
shadow("sidebar.footer.action", { id: "bot-return", order: -300, inject: face }, BotReturn);
ctx.effect(() => {
let previous;
diff --git a/ds-bot/src/client/cells.js b/ds-bot/src/client/cells.js
index 27f4664f..5831bada 100644
--- a/ds-bot/src/client/cells.js
+++ b/ds-bot/src/client/cells.js
@@ -1154,7 +1154,7 @@ export function ToolCell({ node, sessionId, useRoster, useUi, useWork, useHeldCa
const note = running ? toolActivity(toolName(running), toolArgs(running).text, findBot, roster.byId[sessionId]) : null
// A running tool shows only as the activity row's phrase.
useLiveNote(actions, sessionId, `tool:${root.callId ?? node.key}`, note)
- useShownCards(actions, sessionId, drawn.filter(call => CARDS.has(toolName(call)) && call.callId).map(call => call.callId))
+ useShownCards(actions, sessionId, drawn.filter(call => (CARDS.has(toolName(call)) || isComputerCall(toolName(call))) && call.callId).map(call => call.callId))
if (drawn.length === 0) return h(CellEmpty)
const env = { sessionId, roster, actions, useUi, useWork, findBot, heldCards, hooks: { useBrowserHolds, useBrowserTabs, useBrowserPages, useComputer } }
return h(Fragment, null, drawn.map((call, index) => h(Fragment, { key: call.callId ?? index }, renderCall(call, env))))
diff --git a/ds-bot/src/client/computer.js b/ds-bot/src/client/computer.js
index 2c54046e..97191d5c 100644
--- a/ds-bot/src/client/computer.js
+++ b/ds-bot/src/client/computer.js
@@ -4,6 +4,7 @@ import { useDockHold } from './card-hold.js'
import { heldCardKey } from './sources.js'
import { CellEmpty } from './cell-empty.js'
import { t } from './i18n.js'
+import { toolArgs } from './text.js'
// -------------------------------------------------------------------------
// Computer use: the switch page, a Bot's Control card, its Computer drawer page,
@@ -158,17 +159,13 @@ const actionName = (method) => {
const words = String(method ?? '').replace(/^cua_driver_native__/, '').replace(/([a-z])([A-Z])/g, '$1 $2').replace(/_/g, ' ').trim()
return words === '' ? 'Computer' : words[0].toUpperCase() + words.slice(1)
}
-const callArgs = (call) => {
- const args = call?.arguments
- if (args !== null && typeof args === 'object') return args
- try { return JSON.parse(args ?? '{}') } catch { return {} }
-}
export function ComputerCallRow({ sessionId, call, roster, actions, hooks }) {
- const args = callArgs(call)
- const method = args.method ?? String(call?.name ?? '').replace(/^cua_driver_native__/, '')
+ const { value: argValue } = toolArgs(call)
+ const method = argValue('method') ?? String(call?.name ?? '').replace(/^cua_driver_native__/, '')
const stepApp = hooks?.useComputer?.(map => (map.bots?.[roster.byId[sessionId]?.id]?.steps ?? []).find(step => step.callId === call?.callId)?.app)
+ const selector = argValue('params')?.selector
// getApp names its target in the selector; a step record names the app later calls act on.
- const app = stepApp ?? (method === 'getApp' && typeof args.params?.selector === 'string' && args.params.selector !== '' ? args.params.selector : undefined)
+ const app = stepApp ?? (method === 'getApp' && typeof selector === 'string' && selector !== '' ? selector : undefined)
const shot = (call?.content ?? []).find(block => (block?.type === 'image' || block?.type === 'file') && block.attachment !== undefined)?.attachment
const [url, setUrl] = useState(null)
const [open, setOpen] = useState(false)
@@ -219,7 +216,7 @@ export function PendingComputerHandoff({ sessionId, bot, hold, actions, inline }
const [closed, setClosed] = useState(false)
const busyRef = useRef(false)
const titleId = useId()
- const keep = useDockHold(sessionId, 'handoff', hold.callId, actions, inline)
+ const keep = useDockHold(sessionId, 'handoff', hold.callId ?? hold.id, actions, inline)
const taken = hold.by === 'user'
const run = (make, close) => {
if (busyRef.current) return
@@ -231,7 +228,7 @@ export function PendingComputerHandoff({ sessionId, bot, hold, actions, inline }
actions.reportError?.(error, 'handoff')
}).finally(() => { busyRef.current = false })
}
- const answer = outcome => run(() => actions.computerHoldAnswer(bot.id, { callId: hold.callId, outcome }), true)
+ const answer = outcome => run(() => actions.computerHoldAnswer(bot.id, { callId: hold.callId ?? hold.id, outcome }), true)
const takeOver = () => run(async () => {
await actions.computerTakeover(bot.id)
actions.openDetails?.(sessionId, 'computer')
@@ -267,10 +264,11 @@ export function ComputerHandoffCall({ sessionId, callId, roster, actions, heldCa
export function ComputerHandoffDock({ sessionId, useRoster, useCards, useHeldCards, useComputer, actions }) {
const bot = useRoster(value => value.byId[sessionId])
const hold = useComputer?.(map => (bot ? map.holds?.[bot.id] : undefined))
- const drawn = useCards?.(state => state[sessionId]?.[hold?.callId] > 0) === true
- const held = useHeldCards?.(state => state[sessionId]?.[heldCardKey('handoff', hold?.callId)] > 0) === true
- if (!bot || hold?.callId === undefined || !actions.computerHoldAnswer || (drawn && !held)) return null
- return h(PendingComputerHandoff, { key: JSON.stringify([sessionId, hold.callId]), sessionId, bot, hold, actions, inline: false })
+ const holdKey = hold?.callId ?? hold?.id
+ const drawn = useCards?.(state => state[sessionId]?.[holdKey] > 0) === true
+ const held = useHeldCards?.(state => state[sessionId]?.[heldCardKey('handoff', holdKey)] > 0) === true
+ if (!bot || holdKey === undefined || !actions.computerHoldAnswer || (drawn && !held)) return null
+ return h(PendingComputerHandoff, { key: JSON.stringify([sessionId, holdKey]), sessionId, bot, hold, actions, inline: false })
}
// The settled card, for a persisted computer handoff reply.
diff --git a/ds-bot/src/client/index.js b/ds-bot/src/client/index.js
index d67957ce..24df7d0d 100644
--- a/ds-bot/src/client/index.js
+++ b/ds-bot/src/client/index.js
@@ -1063,8 +1063,8 @@ export function apply(ctx) {
() => shadow('conversation.input.dock', { id: 'bot-learning', order: 850, priority: BOT_PRIORITY, inject: face }, LearningDock),
]
// The return button belongs to neither group: it renders only in Agent mode.
- ctx.effect(() => installCompiledStyles(RETURN_CSS, 'return'),
- () => installCompiledStyles(COMPUTER_CSS, 'computer'), 'ds-bot: return styles')
+ ctx.effect(() => installCompiledStyles(RETURN_CSS, 'return'), 'ds-bot: return styles')
+ ctx.effect(() => installCompiledStyles(COMPUTER_CSS, 'computer'), 'ds-bot: computer styles')
shadow('sidebar.footer.action', { id: 'bot-return', order: -300, inject: face }, BotReturn)
// The main view decides `plain`: a DSH Agent Session keeps the Bot shell but
diff --git a/ds-bot/src/client/settings.js b/ds-bot/src/client/settings.js
index 7a62f9a7..e3be9e3a 100644
--- a/ds-bot/src/client/settings.js
+++ b/ds-bot/src/client/settings.js
@@ -290,9 +290,9 @@ export function SettingsDialog({ page, id, roster, actions, useUpdate, leaving =
h('div', { className: 'bt-settings-head' },
back ? h('button', { type: 'button', className: 'bt-icon-btn', 'aria-label': back, title: back, onClick: () => actions.openSettings(shown) }, h(ChevronLeftIcon)) : null,
h('h2', { key: bot?.id ?? room?.id ?? shown }, bot?.name ?? room?.name ?? label)),
- shown === 'computer' ? h(ComputerPage, { roster, actions, run, Section: SetSection, Row: SetRow, Switch }) : null,
shown === 'usage' ? null : h('div', { key: pageKey, className: 'bt-settings-scroll bt-settings-page', 'data-move': last.current.move },
errorLine,
+ shown === 'computer' ? h(ComputerPage, { roster, actions, run, Section: SetSection, Row: SetRow, Switch }) : null,
shown === 'general' ? h(GeneralPage, { roster, actions, run }) : null,
shown === 'models' ? h(ProvidersPage, { actions, run, Section: SetSection }) : null,
bot ? h('div', { className: 'bt-settings-form' }, h(BotSettingsForm, { key: bot.id, bot, roster, actions, run })) : null,
diff --git a/ds-bot/src/host/computer/catalog.js b/ds-bot/src/host/computer/catalog.js
index 57419390..c812f3cd 100644
--- a/ds-bot/src/host/computer/catalog.js
+++ b/ds-bot/src/host/computer/catalog.js
@@ -63,6 +63,17 @@ export function classify(name, args = {}) {
// Whether a tool name belongs to a computer-use provider at all.
export const isComputerTool = name => classify(name, {}) !== undefined
+// Driver calls these without an app target: lists and the agent's own
+// session/config. Everything else must resolve the app it acts on; a call that
+// cannot is refused before it can reach the desktop.
+export const DRIVER_GLOBAL = new Set([
+ 'list_apps', 'list_windows', 'get_screen_size', 'get_cursor_position',
+ 'get_config', 'set_config', 'check_permissions', 'health_report',
+ 'get_session', 'list_sessions', 'get_session_state', 'get_recording_state',
+ 'get_agent_cursor_state', 'get_browser_state', 'install_ffmpeg',
+ 'start_session', 'end_session',
+])
+
// The kind a level admits. 'off' sees nothing.
export const LEVELS = ['off', 'view', 'background', 'foreground']
export function levelAllows(level, kind) {
diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.js
index 1732eaca..6ae0de3b 100644
--- a/ds-bot/src/host/computer/index.js
+++ b/ds-bot/src/host/computer/index.js
@@ -6,7 +6,7 @@ import { randomUUID } from 'node:crypto'
import { createUserMessage } from '@deepseek-ai/dsh-llm'
import { defineTool } from '@deepseek-ai/dsh-tools'
import { LEGACY_EXECUTABLES, LEGACY_NAME } from '../legacy.mjs'
-import { classify, isComputerTool, levelAllows, LEVELS } from './catalog.js'
+import { classify, isComputerTool, levelAllows, LEVELS, DRIVER_GLOBAL } from './catalog.js'
import { createLeases } from './leases.js'
import { REPLY_KIND } from '../constants.js'
import { output } from '../tools.js'
@@ -74,6 +74,7 @@ export function install(rt) {
/** @type {Map<string, Map<string, 'once'|'deny'>>} `${botId}:${sessionId}` -> appKey -> this-session decision */
const sessionDecisions = new Map()
const shotMimes = new Map()
+ const SHOT_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif'])
/** @type {Map<string, {by:string, reason:string, note:string, callId?:string, surface:'computer'}>} */
const holds = new Map()
/** @type {Map<string, {botId:string, sessionId:string, appKey?:string, abort:AbortController}>} callId -> in-flight call */
@@ -82,6 +83,13 @@ export function install(rt) {
const applied = new Map()
let userActive
let providerStatus = { state: 'off' }
+ // The provider status is volatile, but it rides in the view payload: the poll
+ // only re-sends the view after a new revision, so a status change must save.
+ const setProviderStatus = (next) => {
+ if (next.state === providerStatus.state && next.name === providerStatus.name && next.message === providerStatus.message) return
+ providerStatus = next
+ void rt.save().catch(error => rt.warn('ds-bot: computer-use status not saved: %s', error))
+ }
let mountedByUs = false
let registryFiber
let providerFiber
@@ -129,15 +137,15 @@ export function install(rt) {
await rt.load()
if (!enabled()) {
await unmountProvider()
- providerStatus = { state: 'off' }
+ setProviderStatus({ state: 'off' })
return
}
const name = PROVIDER_PACKAGES[process.platform]
if (name === undefined) {
- providerStatus = { state: 'unsupported' }
+ setProviderStatus({ state: 'unsupported' })
return
}
- providerStatus = { state: 'loading' }
+ setProviderStatus({ state: 'loading' })
try {
if (service('computerUse') === undefined) {
const registry = await importPackage(REGISTRY_PACKAGE).catch(error => { throw missing(error, REGISTRY_PACKAGE) })
@@ -147,7 +155,7 @@ export function install(rt) {
if (stopped || !enabled()) return
const existing = service('computerUse').providerName
if (existing !== undefined && !mountedByUs) {
- providerStatus = { state: 'occupied', name: String(existing) }
+ setProviderStatus({ state: 'occupied', name: String(existing) })
return
}
const provider = await importPackage(name).catch(error => { throw missing(error, name) })
@@ -157,13 +165,13 @@ export function install(rt) {
await providerFiber.await?.()
if (stopped || !enabled()) return
mountedByUs = true
- providerStatus = { state: 'ready', name: service('computerUse').providerName ?? 'dsh-cua' }
+ setProviderStatus({ state: 'ready', name: service('computerUse').providerName ?? 'dsh-cua' })
refreshComputer()
} catch (error) {
await unmountProvider().catch(() => {})
- providerStatus = error?.code === 'bot/missing'
+ setProviderStatus(error?.code === 'bot/missing'
? { state: 'missing', message: error.message }
- : { state: 'error', message: rt.redactSecrets?.(String(error?.message ?? error)) ?? String(error) }
+ : { state: 'error', message: rt.redactSecrets?.(String(error?.message ?? error)) ?? String(error) })
}
})().finally(() => { ensuring = undefined })
return ensuring
@@ -294,9 +302,24 @@ export function install(rt) {
}
const args = exec.arguments ?? {}
const pid = [args.pid, args.appPid, args.windowPid, args.target_pid].find(value => Number.isInteger(value))
- if (pid === undefined) return undefined
- const known = appsOfSession(sessionId).get(`pid:${pid}`)
- return known ?? { key: `pid:${pid}`, name: `pid ${pid}`, pid, unknown: true }
+ if (pid !== undefined) {
+ const known = appsOfSession(sessionId).get(`pid:${pid}`)
+ return known ?? { key: `pid:${pid}`, name: `pid ${pid}`, pid, unknown: true }
+ }
+ // A window_id resolves to the app that owns the window, recorded from a
+ // list_windows/get_window_state result.
+ const windowId = [args.window_id, args.windowId, args.wid, args.window].find(value => value !== undefined && value !== null)
+ if (windowId !== undefined) {
+ const known = appsOfSession(sessionId).get(`window:${windowId}`)
+ return known ?? { key: `window:${windowId}`, name: `window ${windowId}`, unknown: true }
+ }
+ // An element token resolves to the window it was read from.
+ const token = [args.target, args.element, args.element_id, args.elementId, args.token, args.ref].find(value => value !== undefined && value !== null)
+ if (token !== undefined) {
+ const known = appsOfSession(sessionId).get(`token:${token}`)
+ return known ?? { key: `token:${token}`, name: 'an element', unknown: true }
+ }
+ return undefined
}
// What a result tells DS Bot about apps: getApp returns the handle plus the app's
// identity; driver lists name pids.
@@ -319,19 +342,43 @@ export function install(rt) {
return
}
if (call.provider !== 'cua-driver') return
+ const apps = appsOfSession(sessionId)
+ const recordPid = (item) => {
+ const pid = [item?.pid, item?.appPid, item?.app_pid, item?.processId].find(value => Number.isInteger(value))
+ if (pid === undefined) return undefined
+ const app = {
+ key: appKeyOf({ id: item.bundleId ?? item.id, path: item.path ?? item.executablePath, name: item.name ?? item.title }),
+ name: String(item.name ?? item.title ?? `pid ${pid}`),
+ id: item.bundleId ?? item.id,
+ path: item.path ?? item.executablePath,
+ pid,
+ }
+ apps.set(`pid:${pid}`, app)
+ return app
+ }
const lists = [result.apps, result.windows, result.items, result]
for (const list of lists) {
if (!Array.isArray(list)) continue
for (const item of list) {
- const pid = [item?.pid, item?.appPid, item?.processId].find(value => Number.isInteger(value))
- if (pid === undefined) continue
- appsOfSession(sessionId).set(`pid:${pid}`, {
- key: appKeyOf({ id: item.bundleId ?? item.id, path: item.path ?? item.executablePath, name: item.name ?? item.title }),
- name: String(item.name ?? item.title ?? `pid ${pid}`),
- id: item.bundleId ?? item.id,
- path: item.path ?? item.executablePath,
- pid,
- })
+ const app = recordPid(item) ?? (typeof item?.title === 'string' ? { key: `title:${normName(item.title)}`, name: item.title } : undefined)
+ if (app === undefined) continue
+ const windowId = [item?.window_id, item?.windowId, item?.wid].find(value => value !== undefined && value !== null)
+ if (windowId !== undefined) apps.set(`window:${windowId}`, app)
+ }
+ }
+ // get_window_state reports one window and the element tokens inside it;
+ // record both so a later call on either resolves to the window's app.
+ const stateWindow = [result.window_id, result.windowId, result.wid, result.window?.window_id, result.window?.id].find(value => value !== undefined && value !== null)
+ const statePid = [result.pid, result.appPid, result.app_pid, result.window?.pid, result.window?.app_pid].find(value => Number.isInteger(value))
+ let stateApp = statePid !== undefined ? recordPid(result) ?? apps.get(`pid:${statePid}`) : undefined
+ if (stateApp === undefined && stateWindow !== undefined) stateApp = apps.get(`window:${stateWindow}`)
+ if (stateApp === undefined && result?.window && typeof result.window === 'object') stateApp = recordPid(result.window)
+ if (stateWindow !== undefined && stateApp !== undefined) apps.set(`window:${stateWindow}`, stateApp)
+ if (stateApp !== undefined) {
+ const elements = [result.elements, result.interactive_elements, result.targets, result.items].find(list => Array.isArray(list)) ?? []
+ for (const element of elements) {
+ const token = [element?.token, element?.element_id, element?.elementId, element?.id, element?.ref, element?.handle].find(value => value !== undefined && value !== null)
+ if (token !== undefined) apps.set(`token:${token}`, stateApp)
}
}
}
@@ -355,8 +402,12 @@ export function install(rt) {
const grants = grantsOf(bot)
const byKey = grants[app.key]
if (byKey !== undefined) return byKey
+ if (app.id !== undefined) return Object.values(grants).find(grant => grant.id === app.id)
+ // The name decides only when no id or path can: the macOS consent question
+ // carries a name alone. A same-named app with its own identity asks again.
+ if (app.path !== undefined) return undefined
const wanted = normName(app.name)
- return Object.values(grants).find(grant => normName(grant.name) === wanted || (app.id !== undefined && grant.id === app.id))
+ return Object.values(grants).find(grant => normName(grant.name) === wanted)
}
const recordGrant = async (bot, app) => {
bot.computer ??= {}
@@ -433,7 +484,6 @@ export function install(rt) {
return 'allow'
}
decisions.set(app.key, answer === 'once' ? 'once' : 'deny')
- if (normName(app.name) !== app.key) decisions.set(normName(app.name), answer === 'once' ? 'once' : 'deny')
return answer === 'once' ? 'allow' : `The user did not allow ${bot.name} to use ${app.name}.`
}
@@ -483,12 +533,17 @@ export function install(rt) {
if (verdict !== 'allow') return { kind: 'deny', reason: verdict }
}
}
- if (call.provider === 'cua-driver' && app !== undefined) {
- if (app.unknown === true) {
- return { kind: 'deny', reason: `${bot.name} does not know which application pid ${app.pid ?? ''} is. List the windows or apps first.` }
+ if (call.provider === 'cua-driver') {
+ if (app === undefined) {
+ if (!DRIVER_GLOBAL.has(call.method) && !call.method.startsWith('browser_')) {
+ return { kind: 'deny', reason: 'Name the window from list_windows first.' }
+ }
+ } else if (app.unknown === true) {
+ return { kind: 'deny', reason: `${bot.name} does not know ${app.name}. List the windows or apps first.` }
+ } else {
+ const verdict = await driverConsent(exec, bot, sessionId, app)
+ if (verdict !== 'allow') return { kind: 'deny', reason: verdict }
}
- const verdict = await driverConsent(exec, bot, sessionId, app)
- if (verdict !== 'allow') return { kind: 'deny', reason: verdict }
}
if (call.kind !== 'observe' && app !== undefined) {
const held = leases.holderOfApp(app.key)
@@ -573,7 +628,7 @@ export function install(rt) {
const handoffReplyText = (hold, outcome) => JSON.stringify({
kind: 'answer_to_pending_handoff',
tool: HANDOFF_TOOL,
- callId: hold.callId,
+ callId: hold.callId ?? hold.id,
reason: hold.reason,
...(hold.note ? { note: hold.note } : {}),
outcome,
@@ -587,7 +642,7 @@ export function install(rt) {
holds.delete(botId)
controlOf(botId).waitingFor = undefined
settleControl(botId)
- if (hold.callId === undefined || outcome === 'takeover') return true
+ if (outcome === 'takeover') return true
const settled = outcome === 'done' ? 'done' : 'skipped'
try {
const sessionId = rt.chatOf(botId)
@@ -595,7 +650,7 @@ export function install(rt) {
const agent = await rt.resolveAgent(sessionId)
agent.followup(createUserMessage({
content: [{ type: 'text', text: handoffReplyText(hold, settled) }],
- source: { kind: REPLY_KIND, callId: hold.callId, outcome: 'answered' },
+ source: { kind: REPLY_KIND, callId: hold.callId ?? hold.id, outcome: 'answered' },
}))
await ctx.sessions.flush(agent.session)
} catch (error) {
@@ -611,7 +666,7 @@ export function install(rt) {
ctl.state = 'user'
ctl.waitingFor = undefined
const hold = holds.get(botId)
- holds.set(botId, { by: 'user', surface: 'computer', reason: hold?.reason ?? 'takeover', note: hold?.note ?? 'The user is operating the computer.', ...(hold?.callId !== undefined ? { callId: hold.callId } : {}) })
+ holds.set(botId, { id: hold?.id ?? randomUUID(), by: 'user', surface: 'computer', reason: hold?.reason ?? 'takeover', note: hold?.note ?? 'The user is operating the computer.', ...(hold?.callId !== undefined ? { callId: hold.callId } : {}) })
return true
}
const handBack = async (botId) => {
@@ -646,6 +701,7 @@ export function install(rt) {
if (!enabled() || levelOf(bot) === 'off') return 'Computer use is off for you.'
const callId = rt.chatOwners.get(sessionId) === bot.id && typeof exec.callId === 'string' ? exec.callId : undefined
holds.set(bot.id, {
+ id: randomUUID(),
by: 'bot', surface: 'computer',
reason: HANDOFF_REASONS.includes(args.reason) ? args.reason : 'other',
note: String(args.note ?? '').slice(0, 500),
@@ -693,10 +749,10 @@ export function install(rt) {
}
if (next) {
void ensureProvider()
- providerStatus = providerStatus.state === 'off' ? { state: 'loading' } : providerStatus
+ setProviderStatus(providerStatus.state === 'off' ? { state: 'loading' } : providerStatus)
} else {
await teardownProvider()
- providerStatus = { state: 'off' }
+ setProviderStatus({ state: 'off' })
}
refreshComputer()
return rt.computerView()
@@ -743,11 +799,17 @@ export function install(rt) {
const recorded = [...sessionApps.values()].flatMap(map => [...map.values()])
.find(app => app.key === appKey || app.id === appKey || normName(app.name) === normName(grant?.name ?? appKey))
const wanted = normName(grant?.name ?? recorded?.name ?? appKey)
+ const aliases = new Set([appKey])
+ for (const map of sessionApps.values()) {
+ for (const [key, app] of map) {
+ if (app.key === appKey || app.id === appKey || normName(app.name) === wanted) aliases.add(key)
+ }
+ }
// A "This time" must not outlive the revoke, in any of this Bot's Sessions.
for (const [key, decisions] of sessionDecisions) {
if (!key.startsWith(`${botId}:`)) continue
for (const entry of [...decisions.keys()]) {
- if (entry === appKey || normName(entry) === wanted) decisions.delete(entry)
+ if (aliases.has(entry) || normName(entry) === wanted) decisions.delete(entry)
}
}
const knownId = grant?.id ?? recorded?.id
@@ -778,7 +840,7 @@ export function install(rt) {
case 'revoke-app':
return revokeApp(String(body.botId ?? ''), String(body.appKey ?? ''))
case 'provider-retry':
- providerStatus = { state: 'loading' }
+ setProviderStatus({ state: 'loading' })
await ensureProvider()
return rt.computerView()
case 'system-status': {
@@ -833,10 +895,12 @@ export function install(rt) {
case 'shot': {
const attachments = service('attachments')
const id = String(body.attachmentId ?? '')
- if (attachments === undefined || id === '') return null
+ // Only screenshots this module recorded are served — an arbitrary
+ // attachment id is not readable through this endpoint.
+ if (attachments === undefined || !shotMimes.has(id)) return null
const chunks = []
for await (const chunk of attachments.readFileStream({ attachmentId: id }, AbortSignal.timeout(5000))) chunks.push(chunk)
- const mime = shotMimes.get(id) ?? 'image/png'
+ const mime = SHOT_MIMES.has(shotMimes.get(id)) ? shotMimes.get(id) : 'image/png'
return { dataUrl: `data:${mime};base64,${Buffer.concat(chunks).toString('base64')}` }
}
case 'app-answer':
diff --git a/ds-bot/test/computer-ui.test.js b/ds-bot/test/computer-ui.test.js
index b496acec..5ff52c94 100644
--- a/ds-bot/test/computer-ui.test.js
+++ b/ds-bot/test/computer-ui.test.js
@@ -133,15 +133,31 @@ test('a computer call row names the app and the action', () => {
hooks: { useComputer: map => map(botsLive) },
})
// arguments may arrive as a JSON string; the app comes from the step record
- const row = render('ComputerCallRow', props({ name: 'cua', callId: 'c1', arguments: JSON.stringify({ method: 'click', params: { appHandle: 'h1' } }), content: [] }))
+ const row = render('ComputerCallRow', props({ name: 'cua', callId: 'c1', argsRaw: '{"method":"click","params":{"appHandle":"h1"}}', content: [] }))
assert.deepEqual(texts(row), ['Notes · Click'])
// getApp names its target by the selector, before any step record exists
- const getApp = render('ComputerCallRow', props({ name: 'cua', callId: 'c2', arguments: { method: 'getApp', params: { selector: 'TextEdit' } }, content: [] }, { bots: {} }))
+ const getApp = render('ComputerCallRow', props({ name: 'cua', callId: 'c2', argsRaw: '{"method":"getApp","params":{"selector":"TextEdit"}}', content: [] }, { bots: {} }))
assert.deepEqual(texts(getApp), ['TextEdit · Get App'])
// no app yet: just the action
- const caps = render('ComputerCallRow', props({ name: 'cua', callId: 'c3', arguments: { method: 'getCapabilities', params: {} }, content: [] }, { bots: {} }))
+ const caps = render('ComputerCallRow', props({ name: 'cua', callId: 'c3', argsRaw: '{"method":"getCapabilities","params":{}}', content: [] }, { bots: {} }))
assert.deepEqual(texts(caps), ['Capabilities'])
// cua-driver: the tool suffix is the action
- const driver = render('ComputerCallRow', props({ name: 'cua_driver_native__screenshot', callId: 'c1', arguments: {}, content: [] }))
+ const driver = render('ComputerCallRow', props({ name: 'cua_driver_native__screenshot', callId: 'c1', argsRaw: '{}', content: [] }))
assert.deepEqual(texts(driver), ['Notes · Screenshot'])
})
+
+// The settings scroll container wraps the computer page like every other page.
+test('the computer settings page sits inside the scroll container', () => {
+ const src = new URL('../src/client/settings.js', import.meta.url)
+ const code = String((process.getBuiltinModule('node:fs')).readFileSync(src))
+ const scroll = code.indexOf('bt-settings-scroll')
+ assert.ok(scroll > 0)
+ assert.ok(code.indexOf('h(ComputerPage') > scroll)
+})
+
+// The computer styles register in their own effect, like the other style packs.
+test('the computer styles register in their own effect call', () => {
+ const src = new URL('../src/client/index.js', import.meta.url)
+ const code = String((process.getBuiltinModule('node:fs')).readFileSync(src))
+ assert.match(code, /ctx\.effect\(\(\) => installCompiledStyles\(COMPUTER_CSS, 'computer'\)/)
+})
diff --git a/ds-bot/test/computer.test.js b/ds-bot/test/computer.test.js
index 5c6ee3fd..c7d5c840 100644
--- a/ds-bot/test/computer.test.js
+++ b/ds-bot/test/computer.test.js
@@ -656,3 +656,179 @@ test("the macOS consent answerer is prepended ahead of DSH answerer", async (t)
await rt.load()
assert.ok(ctx.calls.includes('on user-questions/request (prepend)'))
})
+
+test('a provider status change saves, so the view reaches the next poll', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ const services = { loader: { import: async name => ({ name }) }, agents: { list: () => [] } }
+ ctx.get = name => services[name]
+ withProvider(ctx, services)
+ await rt.load()
+ const saves = []
+ const save = rt.save
+ rt.save = async () => { saves.push(1); return save() }
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ const before = saves.length
+ await rt.computerEndpoint('provider-retry', {})
+ await new Promise(resolve => setImmediate(resolve))
+ assert.equal(rt.computerView().provider.state, 'ready')
+ assert.ok(saves.length > before, 'the status change did not advance the revision')
+})
+
+test('a group computer_handoff docks by its hold id and the answer reaches the Bot chat', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ const services = { loader: { import: async name => ({ name }) }, agents: { list: () => [] } }
+ ctx.get = name => services[name]
+ withProvider(ctx, services)
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('provider-retry', {})
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'foreground' })
+ const agent = ctx.scopeAgent('w1')
+ await ctx.emitAsync('agent/created', { agent })
+ const handoff = ctx.registeredTools.get('computer_handoff')
+ const group = 'w1-group'
+ const selfOf = rt.selfOf
+ rt.selfOf = id => (id === group ? 'w1' : selfOf(id))
+ const isBotSession = rt.isBotSession
+ rt.isBotSession = id => (id === group ? true : isBotSession(id))
+ const followups = []
+ ctx.sessions = { flush: async () => {} }
+ rt.resolveAgent = async () => ({ followup: message => followups.push(message), session: { id: 'w1' } })
+ rt.chatOf = () => 'w1'
+ // the group call has no callId for the Bot's own chat
+ await handoff.execute({ reason: 'login' }, { agent: { id: group }, callId: 'call-group' })
+ const [, hold] = Object.entries(rt.computerLive().holds ?? {})[0]
+ assert.equal(typeof hold.id, 'string')
+ assert.equal(hold.callId, undefined)
+ await rt.computerEndpoint('hold-answer', { botId: 'w1', outcome: 'skip' })
+ const reply = JSON.parse(followups.at(-1).content[0].text)
+ assert.equal(reply.kind, 'answer_to_pending_handoff')
+ assert.equal(reply.callId, hold.id)
+ assert.equal(reply.outcome, 'skipped')
+ assert.equal(rt.computerLive().holds?.w1, undefined)
+})
+
+test('cua-driver resolves a window_id or an element token to the app, and refuses what it cannot', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ const allow = async () => 'allowed'
+ const call = (name, args = {}) => gate({ name, arguments: args, agent: { id: 'w1' }, callId: `c-${name}` }, allow)
+ // a targeted call with nothing to resolve against is refused before consent
+ const blind = await call('cua_driver_native__click', {})
+ assert.equal(blind.kind, 'deny')
+ assert.match(blind.reason, /list_windows/)
+ // untargeted capture and desktop-state reads are refused at every level
+ assert.equal((await call('cua_driver_native__get_desktop_state', {})).kind, 'deny')
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'view' })
+ assert.equal((await call('cua_driver_native__zoom', {})).kind, 'deny')
+ // list_windows stays allowed metadata
+ assert.equal(await call('cua_driver_native__list_windows', {}), 'allowed')
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ // get_window_state records the window and its element tokens
+ await track({ name: 'cua_driver_native__get_window_state', arguments: { window_id: 7 }, agent: { id: 'w1' }, callId: 'state-1' },
+ async () => ({ window_id: 7, pid: 42, name: 'Notes', path: '/Applications/Notes.app', elements: [{ token: 'el-9' }] }))
+ // a window_id call resolves to Notes and asks once, then runs
+ const pending = call('cua_driver_native__click', { window_id: 7 })
+ await new Promise(resolve => setImmediate(resolve))
+ const [id, ask] = Object.entries(rt.computerLive().asks)[0]
+ assert.equal(ask.appName, 'Notes')
+ await rt.computerEndpoint('app-answer', { id, answer: 'once' })
+ assert.equal(await pending, 'allowed')
+ // an element token from the same window resolves to the same app
+ assert.equal(await call('cua_driver_native__click', { target: 'el-9' }), 'allowed')
+ // an unknown window is refused
+ const unknown = await call('cua_driver_native__click', { window_id: 99 })
+ assert.equal(unknown.kind, 'deny')
+})
+
+test('revoke clears a This time under every key, so the next call asks again', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ const allow = async () => 'allowed'
+ const call = (args = {}) => gate({ name: 'cua_driver_native__click', arguments: args, agent: { id: 'w1' }, callId: 'c-revoke' }, allow)
+ await track({ name: 'cua_driver_native__list_windows', arguments: {}, agent: { id: 'w1' }, callId: 'list-1' }, async () => ({ windows: [{ pid: 42, name: 'Notes', path: '/Applications/Notes.app' }] }))
+ // Always grant, then a This time decision in the same Session
+ const grantAsk = call({ pid: 42 })
+ await new Promise(resolve => setImmediate(resolve))
+ await rt.computerEndpoint('app-answer', { id: Object.keys(rt.computerLive().asks)[0], answer: 'always' })
+ assert.equal(await grantAsk, 'allowed')
+ // revoke the grant and record a Session decision under a different key, then revoke again
+ await rt.computerEndpoint('revoke-app', { botId: 'w1', appKey: '/Applications/Notes.app' })
+ const onceAsk = call({ pid: 42 })
+ await new Promise(resolve => setImmediate(resolve))
+ await rt.computerEndpoint('app-answer', { id: Object.keys(rt.computerLive().asks)[0], answer: 'once' })
+ assert.equal(await onceAsk, 'allowed')
+ await rt.computerEndpoint('revoke-app', { botId: 'w1', appKey: '/Applications/Notes.app' })
+ // the This time is gone with it: the next call opens the card again
+ const again = call({ pid: 42 })
+ await new Promise(resolve => setImmediate(resolve))
+ assert.equal(Object.keys(rt.computerLive().asks).length, 1)
+ await rt.computerEndpoint('app-answer', { id: Object.keys(rt.computerLive().asks)[0], answer: 'deny' })
+ assert.equal((await again).kind, 'deny')
+})
+
+test('computer-shot serves only screenshots the module recorded, with a safe MIME', async (t) => {
+ const services = { attachments: { readFileStream: async function* () { yield Buffer.from('img') } } }
+ const { ctx, home, rt } = setup({ services })
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ // an arbitrary attachment id is not readable
+ assert.equal(await rt.computerEndpoint('shot', { attachmentId: 'any-1' }), null)
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ // a screenshot result with a hostile MIME is served as png
+ await track({ name: 'cua', arguments: { method: 'screenshot', params: {} }, agent: { id: 'w1' }, callId: 'shot-1' },
+ async () => ({ attachment: { attachmentId: 'rec-1', mimeType: 'text/html' } }))
+ const shot = await rt.computerEndpoint('shot', { attachmentId: 'rec-1' })
+ assert.match(shot.dataUrl, /^data:image\/png;base64,/)
+ // a real image MIME survives
+ await track({ name: 'cua', arguments: { method: 'screenshot', params: {} }, agent: { id: 'w1' }, callId: 'shot-2' },
+ async () => ({ attachment: { attachmentId: 'rec-2', mimeType: 'image/jpeg' } }))
+ assert.match((await rt.computerEndpoint('shot', { attachmentId: 'rec-2' })).dataUrl, /^data:image\/jpeg;base64,/)
+})
+
+test('same-named apps do not share an approval; a name-only consent still matches', async (t) => {
+ const { ctx, home, rt } = setup()
+ t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+ await rt.load()
+ await rt.computerEndpoint('set-enabled', { enabled: true })
+ await rt.computerEndpoint('set-level', { botId: 'w1', level: 'background' })
+ const gate = ctx.listeners.get('tools/pre-execute').find(fn => fn.name === 'computerGate')
+ const track = ctx.listeners.get('tools/execute').find(fn => fn.name === 'computerTrack')
+ const allow = async () => 'allowed'
+ const call = (args = {}) => gate({ name: 'cua_driver_native__click', arguments: args, agent: { id: 'w1' }, callId: `c-${args.pid}` }, allow)
+ // app A at path X gets an Always grant
+ await track({ name: 'cua_driver_native__list_windows', arguments: {}, agent: { id: 'w1' }, callId: 'list-a' }, async () => ({ windows: [{ pid: 42, name: 'Notes', path: '/a/Notes.app' }] }))
+ const askA = call({ pid: 42 })
+ await new Promise(resolve => setImmediate(resolve))
+ await rt.computerEndpoint('app-answer', { id: Object.keys(rt.computerLive().asks)[0], answer: 'always' })
+ assert.equal(await askA, 'allowed')
+ // app B: same name, another path — it asks again
+ await track({ name: 'cua_driver_native__list_windows', arguments: {}, agent: { id: 'w1' }, callId: 'list-b' }, async () => ({ windows: [{ pid: 43, name: 'Notes', path: '/b/Notes.app' }] }))
+ const askB = call({ pid: 43 })
+ await new Promise(resolve => setImmediate(resolve))
+ assert.equal(Object.keys(rt.computerLive().asks).length, 1)
+ await rt.computerEndpoint('app-answer', { id: Object.keys(rt.computerLive().asks)[0], answer: 'deny' })
+ assert.equal((await askB).kind, 'deny')
+ // the macOS consent question carries only a name; that path stays name-matched
+ const consent = ctx.listeners.get('user-questions/request').find(fn => fn.name === 'computerConsent')
+ const w1 = rt.botOf('w1')
+ const question = { questions: [{ id: 'cua-app-access', intent: { subject: { name: 'Notes' } } }], agent: { id: 'w1', session: { id: 'w1' } }, signal: new AbortController().signal }
+ // drop the path-keyed grants and keep a name-only Always grant
+ w1.computer = { apps: { notes: { name: 'Notes', grantedAt: 0 } } }
+ const answer = await consent(question, async () => 'next')
+ assert.deepEqual(answer.answers[0].selected, ['Allow for this session'])
+})