Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

perf(computer): 合并动作后观察并优化 CUA 0.37 兼容性

Dash4ee691e50599
diff --git a/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts b/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts
index 2e89a03383..9185dd8e90 100644
--- a/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts
+++ b/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts
@@ -2,6 +2,7 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
const {
callToolMock,
+ listToolsMock,
clientCloseMock,
clientConnectMock,
clientCtorMock,
@@ -9,6 +10,7 @@ const {
transportCtorMock,
} = vi.hoisted(() => ({
callToolMock: vi.fn(),
+ listToolsMock: vi.fn(),
clientCloseMock: vi.fn(),
clientConnectMock: vi.fn(),
clientCtorMock: vi.fn(),
@@ -42,6 +44,7 @@ vi.mock('@modelcontextprotocol/sdk/client/index.js', () => ({
clientCtorMock(...args);
return {
callTool: callToolMock,
+ listTools: listToolsMock,
close: clientCloseMock,
connect: clientConnectMock,
};
@@ -102,6 +105,10 @@ beforeEach(async () => {
existsSyncMock.mockReturnValue(true);
clientConnectMock.mockResolvedValue(undefined);
clientCloseMock.mockResolvedValue(undefined);
+ listToolsMock.mockResolvedValue({ tools: [{ name: 'get_window_state', inputSchema: {
+ properties: { pid: {}, window_id: {}, include_screenshot: {}, tree_format: {}, query: {}, max_elements: {}, max_depth: {}, session: {} },
+ additionalProperties: false,
+ } }] });
mockSuccessfulLocate();
});
@@ -110,6 +117,23 @@ afterAll(() => {
});
describe('persistent switch locator connection', () => {
+ // CUA 0.35+ no longer returns elements by default. Verify the actual locator
+ // requests frames for the switch, while old schemas do not receive tree_format.
+ it.each([true, false])('locates switches using the advertised observation format (modern=%s)', async (modern) => {
+ listToolsMock.mockResolvedValue({ tools: [{ name: 'get_window_state', inputSchema: {
+ properties: { pid: {}, window_id: {}, include_screenshot: {}, ...(modern ? { tree_format: {} } : {}), query: {}, max_elements: {}, max_depth: {}, session: {} },
+ additionalProperties: false,
+ } }] });
+ callToolMock.mockImplementation(async ({ name, arguments: args }) => name === 'list_windows' ? windowsResult
+ : modern && args.tree_format !== 'elements' ? { structuredContent: { tree_markdown: 'CuaDriver', element_count: 2 } } : stateResult);
+ await expect(locateComputerUseSwitchTarget()).resolves.toMatchObject({ status: 'found' });
+ await expect(locateComputerUseSwitchTarget()).resolves.toMatchObject({ status: 'found' });
+ expect(listToolsMock).toHaveBeenCalledTimes(1);
+ const args = callToolMock.mock.calls.find(([request]) => request.name === 'get_window_state')?.[0].arguments;
+ if (modern) expect(args).toHaveProperty('tree_format', 'elements');
+ else expect(args).not.toHaveProperty('tree_format');
+ });
+
it('does not connect during isolated module import and connects on first locate', async () => {
vi.resetModules();
diff --git a/apps/desktop/src/main/computer-permission-guide/switch-target.ts b/apps/desktop/src/main/computer-permission-guide/switch-target.ts
index fb06ae21ec..7f5ccdcc65 100644
--- a/apps/desktop/src/main/computer-permission-guide/switch-target.ts
+++ b/apps/desktop/src/main/computer-permission-guide/switch-target.ts
@@ -5,6 +5,7 @@ import os from 'node:os';
import path from 'node:path';
import { findSystemSettingsWindowBounds } from './placement.js';
+import { adaptComputerDriverArgs, type ComputerDriverToolSchema } from '../mcp-integrations/computer-contract.js';
const SYSTEM_CUA_DRIVER_APP_EXECUTABLE = path.join(
'/Applications',
@@ -55,6 +56,7 @@ interface SwitchLocatorConnection {
client: Client;
transport: StdioClientTransport;
session: string;
+ toolSchemas: Map<string, ComputerDriverToolSchema>;
}
let switchLocatorConnection: SwitchLocatorConnection | null = null;
@@ -281,9 +283,12 @@ async function connectSwitchLocator(executable: string): Promise<SwitchLocatorCo
client,
transport,
session: `xdt-permission-switch-${Date.now()}`,
+ toolSchemas: new Map<string, ComputerDriverToolSchema>(),
};
try {
await withTimeout(client.connect(transport), CONNECT_TIMEOUT_MS, 'switch locator startup');
+ const { tools } = await withTimeout(client.listTools(), TOOL_TIMEOUT_MS, 'switch locator tools');
+ connection.toolSchemas = new Map(tools.map((tool) => [tool.name, tool.inputSchema as ComputerDriverToolSchema]));
switchLocatorConnection = connection;
return connection;
} catch (error) {
@@ -324,17 +329,19 @@ async function locateComputerUseSwitchTargetSerialized(): Promise<ComputerUseSwi
const stateResult = parseSuccessfulMcpResult(await withTimeout(
connection.client.callTool({
name: 'get_window_state',
- arguments: {
+ arguments: adaptComputerDriverArgs('get_window_state', {
pid,
window_id: windowId,
include_screenshot: false,
+ // The locator consumes frames; 0.35+ defaults to Markdown without them.
+ tree_format: 'elements',
// Match the stable app row instead of the localized permission-pane
// title, while avoiding a walk of the full 400-500 element tree.
max_elements: 180,
max_depth: 6,
query: 'CuaDriver',
session: connection.session,
- },
+ }, connection.toolSchemas),
}),
TOOL_TIMEOUT_MS,
'Computer Use switch lookup',
diff --git a/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts b/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts
index 201d6dac07..3a74054e92 100644
--- a/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts
+++ b/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts
@@ -6,6 +6,17 @@ const schemas = new Map(
fixture.tools.map((tool) => [tool.name, tool.inputSchema as ComputerDriverToolSchema]),
);
describe('installed Computer Use contract', () => {
+ // Production argument adapter, unit tier: new incremental evidence requests
+ // must still dispatch with the existing old-driver fixture (previously untested).
+ it('preserves Markdown evidence diffs only for drivers advertising since', () => {
+ const input = { pid: 1, window_id: 2, include_screenshot: false, tree_format: 'markdown', since: 'latest' };
+ const modern = new Map<string, ComputerDriverToolSchema>([['get_window_state', {
+ properties: { pid: {}, window_id: {}, include_screenshot: {}, tree_format: {}, since: {} },
+ additionalProperties: false,
+ }]]);
+ expect(adaptComputerDriverArgs('get_window_state', input, modern)).toEqual(input);
+ expect(adaptComputerDriverArgs('get_window_state', input, schemas)).not.toHaveProperty('since');
+ });
it('preserves driver credentials and rejects missing observation provenance', () => {
const args = {
pid: 1,
diff --git a/apps/desktop/src/main/mcp-integrations/computer-contract.ts b/apps/desktop/src/main/mcp-integrations/computer-contract.ts
index 37525b89ea..d25c14199f 100644
--- a/apps/desktop/src/main/mcp-integrations/computer-contract.ts
+++ b/apps/desktop/src/main/mcp-integrations/computer-contract.ts
@@ -48,6 +48,8 @@ export function adaptComputerDriverArgs(
// elements to register tokens and frames, and trims them from the model view.
if ('tree_format' in props) args.tree_format ??= 'both';
else delete args.tree_format;
+ // Evidence reads may use compact diffs; older drivers simply return full state.
+ if (!('since' in props)) delete args.since;
}
if (args.element_index !== undefined && !('element_index' in props) && 'element_token' in props) {
// Newer drivers address an observed row only as `<snapshot_id>:<row>`.
diff --git a/packages/lizi-mcps/src/computer/README.md b/packages/lizi-mcps/src/computer/README.md
new file mode 100644
index 0000000000..b78e1da6e7
--- /dev/null
+++ b/packages/lizi-mcps/src/computer/README.md
@@ -0,0 +1,98 @@
+# Computer Use observations
+
+Discover only the schemas needed for the next operation, for example
+`list_tools({"names":["list_windows","get_window_state","click"]})`.
+An empty argument object still returns the complete catalogue. MCP argument
+objects are not CLI flags; `list_apps` accepts no `running_only` filter.
+
+After selecting the actual `pid` and `window_id`, read the window once. A
+subsequent action can request the next usable observation in the same call:
+
+```json
+{
+ "name": "click",
+ "args": {
+ "pid": 123,
+ "window_id": 456,
+ "element_token": "<token from the latest observation>",
+ "observe": {}
+ }
+}
+```
+
+`observe: {}` returns text; `observe: {"include_screenshot": true}` requests
+visual evidence through the normal host screenshot path. Omitted `query`,
+`max_elements` and `max_depth` inherit the latest successful observation of
+that session/window. Override them only when the target is outside the current
+scope. The returned `observation` uses the ordinary read result envelope and
+registers fresh element tokens and secure-field frames. It is mutually exclusive
+with `postcondition`, which checks an explicit assertion instead.
+
+Delivery and observation have separate outcomes: inspect `observation.ok` and
+the returned state before continuing. Failed reads do not undo delivery, and
+unknown delivery is not proof that the goal succeeded. Input is never replayed
+automatically. Refused actions do not trigger the optional observation.
+
+Cindy MCP supplies the task's stable driver session. When using the native CLI
+instead, pass the **same non-default `session` on every read and action**, then
+call `end_session` for that session. Independent invocations without a session
+do not share the screenshot lease. Reuse discovered process/window identities;
+do not guess bundle IDs. Foreground input already activates its target, so a
+preceding `bring_to_front` is usually redundant.
+
+## CUA 0.37 compatibility
+
+On 2026-10-22, a read-only native check reproduced `typed_output_mismatch`
+(`returned_element_count does not match elements`) with `tree_format: "both"`
+and `since: "latest"`. Full actionable observations therefore keep structured
+elements and do not request a diff. Automatic evidence, which cannot supply
+tokens for later actions, uses Markdown diffs with the previous traversal scope.
+The adapter drops `since` for older drivers that do not advertise it. The
+permission-switch locator also discovers the installed schema once per connection
+and explicitly requests structured elements when supported.
+
+## Validation and cost, 2026-10-22
+
+Measured through the production MCP server and an in-memory driver, against
+base `0a9987b416`:
+
+| Measurement | Before | After |
+| --- | ---: | ---: |
+| Advertised MCP catalogue, JSON bytes | 1,540 | 2,023 |
+| Complete discovery response, UTF-8 bytes | 42,269 | 50,028 |
+| Discovery for `click` alone, UTF-8 bytes | 42,269 (full list required) | 5,455 |
+| Driver calls after an initial read, for unknown click delivery followed by a usable read | 3 | 2 |
+| Model tool calls for that action and usable read | 2 | 1 |
+
+The full response grows because actions advertise the optional observation
+schema; focused discovery is about 87% smaller than the previous full response.
+Repeated MCP catalogues were byte-identical within each version. Tool names,
+system-prompt assembly and usage accounting are unchanged; this is not a
+measurement of vendor cache-hit rates or live model latency.
+
+Regression coverage runs through real MCP dispatch for token freshness, session
+and scope retention, separate action/read outcomes, refusal and password-field
+protection. Desktop tests exercise schema adaptation and the switch locator with
+both older and modern driver contracts. GUI mutations use mock drivers in these
+tests; the native compatibility check above only read a window.
+
+A separate native macOS benchmark used the production MCP server and actual
+CUA 0.37.0 against a dedicated AppKit window with 80 static rows. Each click
+incremented a counter, verified through the next AX observation. Twelve
+alternating before/after pairs per mode produced 48 measured actions, plus
+eight excluded warmups; all 56 postconditions passed.
+
+| Median per action and usable observation | Before | After | Reduction |
+| --- | ---: | ---: | ---: |
+| Text observation | 782.5 ms | 713.6 ms | 8.8% |
+| Screenshot observation | 882.1 ms | 774.0 ms | 12.3% |
+| Text response JSON | 11,591 B | 6,076 B | 47.6% |
+| Screenshot response JSON | 12,007 B | 6,492 B | 45.9% |
+
+Both versions used the same driver, window and traversal bounds (200 nodes,
+depth 15). Timing starts after the initial observation and ends with the next
+usable snapshot. Actual clicks returned `unverifiable`, so the old path made
+three driver calls and the new path made two. This measures the production MCP
+server and native driver, not the full Electron host or model/network latency.
+PNG bytes are excluded from JSON sizes. The small controlled sample does not
+establish an improvement in complex-task success rates or overall token costs.
diff --git a/packages/lizi-mcps/src/computer/computerMcpServer.test.ts b/packages/lizi-mcps/src/computer/computerMcpServer.test.ts
index 53a3333b7c..307404c695 100644
--- a/packages/lizi-mcps/src/computer/computerMcpServer.test.ts
+++ b/packages/lizi-mcps/src/computer/computerMcpServer.test.ts
@@ -75,6 +75,20 @@ async function makeHarness(
}
describe('createComputerMcpServer', () => {
+ // Audit regression: discovery returned every schema even when only one was
+ // needed. Exercise the real MCP list entry with no driver/process side effects.
+ it('returns only requested schemas for focused discovery', async () => {
+ const deps: ComputerMcpDeps = { getStatus: vi.fn(), callTool: vi.fn() };
+ const h = await makeHarness(deps);
+ try {
+ const payload = textPayload(await h.client.callTool({ name: 'list_tools', arguments: { names: ['click'] } })) as {
+ tools: Array<{ name: string; inputSchema: { properties: Record<string, unknown> } }>;
+ };
+ expect(payload.tools.map((tool) => tool.name)).toEqual(['click']);
+ expect(payload.tools[0]?.inputSchema.properties).toHaveProperty('observe');
+ expect(deps.callTool).not.toHaveBeenCalled();
+ } finally { await h.cleanup(); }
+ });
it('advertises a first-call goal and keeps host session routing independent of it', async () => {
const deps: ComputerMcpDeps = { getStatus: vi.fn(), callTool: vi.fn(async () => ({ windows: [] })) };
const h = await makeHarness(deps, { sessionId: 'stable-task' });
diff --git a/packages/lizi-mcps/src/computer/recovery.test.ts b/packages/lizi-mcps/src/computer/recovery.test.ts
index 1692dd0ea1..963fb3b226 100644
--- a/packages/lizi-mcps/src/computer/recovery.test.ts
+++ b/packages/lizi-mcps/src/computer/recovery.test.ts
@@ -33,6 +33,87 @@ async function harness(dispatch: ComputerMcpDeps["callTool"]) {
}
describe("bounded Computer Use recovery", () => {
+ // Production MCP entry, unit tier: the 0.37 audit exposed redundant reads and
+ // lost CLI leases. Existing recovery tests deliberately forbid token reuse;
+ // explicit observe must instead keep the session and return usable new tokens.
+ it('returns one usable observation after an action, preserving the read scope and session', async () => {
+ let reads = 0;
+ const dispatch = vi.fn(async (name: string) => name === 'get_window_state'
+ ? {
+ snapshot_id: `s${++reads}`,
+ tree_markdown: '- [0] AXButton Next',
+ elements: [{ element_index: 0, element_token: `s${reads}:0`, frame: { x: 1, y: 1, w: 20, h: 20 } }],
+ }
+ : { effect: 'unverifiable' });
+ const call = await harness(dispatch);
+ await call('get_window_state', { pid: 1, window_id: 2, query: 'Next', max_elements: 70, max_depth: 6 });
+ const result = await call('click', { pid: 1, element_token: 's1:0', observe: {} });
+ expect(result).toMatchObject({
+ ok: true, outcome: { status: 'unknown' },
+ observation: { ok: true, snapshot_id: expect.any(String), data: { snapshot_id: 's2' } },
+ });
+ expect(result).not.toHaveProperty('postcheck');
+ expect(result.observation.data).not.toHaveProperty('elements');
+ expect(dispatch).toHaveBeenLastCalledWith('get_window_state', {
+ pid: 1, window_id: 2, session: 'recovery-test', query: 'Next', max_elements: 70, max_depth: 6, include_screenshot: false,
+ }, { sessionId: 'recovery-test', signal: expect.any(AbortSignal) });
+ expect(dispatch.mock.calls.map(([name]) => name)).toEqual(['get_window_state', 'click', 'get_window_state']);
+ expect(await call('click', { pid: 1, element_token: 's1:0' })).toMatchObject({ errorCode: 'STALE_SNAPSHOT' });
+ expect(await call('click', { pid: 1, element_token: 's2:0', observe: { include_screenshot: true } }))
+ .toMatchObject({ ok: true, observation: { data: { snapshot_id: 's3' } } });
+ expect(dispatch).toHaveBeenLastCalledWith('get_window_state', expect.objectContaining({ include_screenshot: true }), expect.any(Object));
+ });
+
+ it.each([
+ { pid: 1, x: 1, y: 2, observe: {} },
+ { pid: 1, window_id: 2, observe: {}, postcondition: [{ window: { exists: true } }] },
+ { pid: 1, window_id: 2, observe: { pid: 3 } },
+ ])('rejects invalid follow-up observations before dispatch: %j', async (args) => {
+ const dispatch = vi.fn();
+ const call = await harness(dispatch);
+ expect(await call('click', args)).toMatchObject({ ok: false, errorCode: 'INVALID_ARGS' });
+ expect(dispatch).not.toHaveBeenCalled();
+ });
+
+ it('preserves action delivery when its explicit observation fails, without retrying input', async () => {
+ const dispatch = vi.fn(async (name: string) => {
+ if (name === 'get_window_state') throw new Error('Capture unavailable');
+ return { effect: 'confirmed' };
+ });
+ const call = await harness(dispatch);
+ expect(await call('click', { pid: 1, window_id: 2, x: 1, y: 2, observe: {} })).toMatchObject({
+ ok: true, outcome: { status: 'confirmed' },
+ observation: { ok: false, errorCode: 'COMPUTER_DRIVER_ERROR' },
+ });
+ expect(dispatch.mock.calls.map(([name]) => name)).toEqual(['click', 'get_window_state']);
+ });
+
+ it('does not follow a refused action with an explicit observation', async () => {
+ const dispatch = vi.fn(async () => ({ effect: 'refused', code: 'background_unavailable' }));
+ const call = await harness(dispatch);
+ expect(await call('click', { pid: 1, window_id: 2, observe: {} })).toMatchObject({ ok: false });
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ });
+
+ it('uses a Markdown diff with the prior scope for automatic evidence, never reusing its tokens', async () => {
+ let reads = 0;
+ const dispatch = vi.fn(async (name: string) => name === 'click'
+ ? { effect: 'unverifiable' }
+ : ++reads === 1
+ ? { snapshot_id: 's1', elements: [{ element_index: 0, element_token: 's1:0' }] }
+ : { snapshot_id: 's2', tree_diff: 'no change since s1', since_status: 'no_change' });
+ const call = await harness(dispatch);
+ await call('get_window_state', { pid: 1, window_id: 2, query: 'Next', max_elements: 70, max_depth: 6 });
+ expect(await call('click', { pid: 1, element_token: 's1:0' })).toMatchObject({
+ postcheck: { ok: true, reusable_snapshot: false, data: { since_status: 'no_change' } },
+ });
+ expect(dispatch).toHaveBeenLastCalledWith('get_window_state', {
+ pid: 1, window_id: 2, session: 'recovery-test', query: 'Next', max_elements: 70, max_depth: 6,
+ include_screenshot: false, tree_format: 'markdown', since: 'latest',
+ }, expect.objectContaining({ observationPurpose: 'recovery' }));
+ expect(await call('click', { pid: 1, element_token: 's2:0' })).toMatchObject({ errorCode: 'STALE_SNAPSHOT' });
+ });
+
it("preserves interrupted typing progress without replaying input", async () => {
const inputProgress = { completed_chars: 400, attempted_chars: 0, remaining_chars: 450 };
const dispatch = vi.fn(async (name: string) => {
diff --git a/packages/lizi-mcps/src/computer/recovery.ts b/packages/lizi-mcps/src/computer/recovery.ts
index 694899f021..83a0d2165a 100644
--- a/packages/lizi-mcps/src/computer/recovery.ts
+++ b/packages/lizi-mcps/src/computer/recovery.ts
@@ -68,6 +68,7 @@ export async function readForRecovery(
: tool === "get_window_state"
? Array.isArray(object.elements) ||
typeof object.tree_markdown === "string" ||
+ typeof object.tree_diff === "string" ||
typeof object.screenshot_frame_valid === "boolean"
: true);
return {
diff --git a/packages/lizi-mcps/src/computer/server.ts b/packages/lizi-mcps/src/computer/server.ts
index 1e8c8a5035..1bde3ab5a4 100644
--- a/packages/lizi-mcps/src/computer/server.ts
+++ b/packages/lizi-mcps/src/computer/server.ts
@@ -41,7 +41,7 @@ export interface ComputerMcpServerOptions {
const DESCRIPTION_LIST =
'Discover local desktop computer-use tools. These tools operate on the user desktop through the installed driver. ' +
- 'Use read-only status/get_accessibility_tree/list_apps/list_windows/get_window_state before click/type_text/press_key/hotkey.';
+ 'Pass names to retrieve only the needed schemas; reuse them instead of guessing CLI flags. Use read-only status/list_windows/get_window_state before input.';
const DESCRIPTION_CALL =
'Invoke a local desktop computer-use tool. After status/permission checks, include session_goal in the first call\'s args as a short English name for the overall objective (not the current step); omit it thereafter. Arguments are validated before dispatching to the host driver.';
@@ -405,17 +405,19 @@ export function createComputerMcpServer(
// 单实例跨 session 共享,所以 tracker 内部按 sessionId 分键。
const snapshotTracker = new WindowSnapshotTracker();
- server.tool('list_tools', DESCRIPTION_LIST, {}, async () =>
+ server.tool('list_tools', DESCRIPTION_LIST, {
+ names: z.array(z.enum(COMPUTER_TOOL_NAMES)).min(1).max(COMPUTER_TOOL_NAMES.length).optional(),
+ }, async ({ names }) =>
textResult({
ok: true,
- tools: COMPUTER_TOOLS.map((tool) => ({
+ tools: COMPUTER_TOOLS.filter((tool) => !names || names.includes(tool.name)).map((tool) => ({
name: tool.name,
description: tool.description,
readOnly: tool.readOnly === true,
inputSchema: z.toJSONSchema(z.object(tool.inputShape).strict()),
})),
workflow:
- 'Start with status and check_permissions. Use get_accessibility_tree/list_windows, optionally narrow list_windows with query/workspace_root/process_name (for example, {"process_name":"Simulator"}), inspect a target with get_window_state, perform one action, and call get_window_state again to verify. Targeted actions such as click/type_text require pid; include window_id whenever the target window is known, and always for coordinates. Use get_window_state with include_screenshot:false for bounded text observations; request an image for visual grounding and normally omit screenshot_out_file. Prefer the exact element_token returned by the latest observation, or pass snapshot_id with element_index. Re-observe on STALE_SNAPSHOT. After an action, verify its intended postcondition with verify_state or a fresh observation; a delivered or unverifiable action does not prove completion. Never replay a mutation automatically after cancellation or a connection failure. Use start_recording/stop_recording/replay_trajectory only when the user explicitly asks for recording or replay.',
+ 'Start with status and check_permissions. Reuse these schemas; CLI flags are not MCP arguments. Find the exact pid/window_id with list_windows, optionally query/workspace_root/process_name (for example, {"process_name":"Simulator"}); reuse known identities instead of guessing app names. Inspect with get_window_state (include_screenshot:false for text), then use an action with observe:{} to return the next usable text snapshot in the same call (observe:{include_screenshot:true} for visual evidence), or postcondition for an explicit assertion. Keep query/max_elements/max_depth stable; narrow with query and broaden only if needed. Targeted actions require pid; include window_id whenever known, always for coordinates. Prefer the latest element_token or snapshot_id plus element_index. Re-observe on STALE_SNAPSHOT. A delivered/unverifiable action is not proof of success: assess the observation before continuing. This MCP manages a stable session automatically. If using the native CLI instead, repeat the same non-default session on EVERY observation and action, then end_session; CLI invocations without it are separate leases. Prefer element actions; foreground input already activates its target, so do not also bring_to_front unless it must stay frontmost between calls. Never replay input after cancellation or uncertain delivery. Record/replay only when explicitly requested.',
}),
);
@@ -446,7 +448,7 @@ export function createComputerMcpServer(
args: Record<string, unknown> | undefined,
signal?: AbortSignal,
dispatchOptions?: ComputerDispatchOptions,
- ) {
+ ): Promise<ReturnType<typeof textResult>> {
const def = getComputerTool(name);
if (!def) {
return textResult({ ok: false, errorCode: 'UNKNOWN_TOOL', data: { requested: name } }, true);
@@ -495,20 +497,28 @@ export function createComputerMcpServer(
const secureFieldResult = await checkSecureField(name as ComputerMcpToolName, parsedData, sessionId);
if (secureFieldResult) return secureFieldResult;
const postcondition = parsedData.postcondition;
+ const observe = parsedData.observe as Record<string, unknown> | undefined;
delete parsedData.postcondition; // Cindy-owned verification, never part of driver action args.
- if (postcondition && typeof parsedData.window_id !== "number") {
+ delete parsedData.observe;
+ if (observe && postcondition) {
+ return textResult({ ok: false, errorCode: 'INVALID_ARGS', data: {
+ message: 'Use observe for a fresh snapshot or postcondition for an assertion, not both.',
+ } }, true);
+ }
+ if ((postcondition || observe) && typeof parsedData.window_id !== "number") {
return textResult(
{
ok: false,
errorCode: "INVALID_ARGS",
data: {
message:
- "postcondition requires an exact window_id or a fresh element snapshot identifying the window.",
+ "postcondition/observe requires an exact window_id or a fresh element snapshot identifying the window.",
},
},
true,
);
}
+ const observationScope = snapshotTracker.observationScope(sessionId, parsedData.pid as number, parsedData.window_id as number);
if (typeof parsedData.snapshot_id === "string") {
const driverId = snapshotTracker.driverSnapshotId(
sessionId,
@@ -590,6 +600,7 @@ export function createComputerMcpServer(
: undefined;
const postcheck =
!recovery &&
+ !observe &&
POSTCHECK_ACTION_TOOLS.has(name as ComputerMcpToolName) &&
outcome.ok &&
(postcondition || outcome.outcome?.status === "unknown")
@@ -598,6 +609,19 @@ export function createComputerMcpServer(
signal,
})
: undefined;
+ // Use the normal explicit-read path: host session ownership, secure-field
+ // frames, output handling and fresh element registration remain identical.
+ // A returned observation is evidence, never a reason to replay the input.
+ const observation = observe && !recovery && outcome.ok && !signal?.aborted
+ ? JSON.parse((await handleCallTool('get_window_state', {
+ ...observationScope,
+ ...observe,
+ pid: parsedData.pid,
+ window_id: parsedData.window_id,
+ include_screenshot: observe.include_screenshot ?? false,
+ ...(typeof parsedData.session === 'string' ? { session: parsedData.session } : {}),
+ }, signal, dispatchOptions)).content[0].text) as Record<string, unknown>
+ : undefined;
if (postcondition && postcheck) {
const checked = computerResultOutcome("verify_state", postcheck.data);
// An interrupted multi-chunk input is not a completed action, even if a
@@ -629,6 +653,7 @@ export function createComputerMcpServer(
data: withoutDuplicateElements(name, data),
...(recovery ? { recovery } : {}),
...(postcheck ? { postcheck } : {}),
+ ...(observation ? { observation } : {}),
},
!outcome.ok,
);
@@ -849,6 +874,7 @@ export function createComputerMcpServer(
context,
);
}
+ const scope = snapshotTracker.observationScope(context.sessionId, args.pid, args.window_id);
snapshotTracker.invalidate(context.sessionId, args.pid, args.window_id);
const tool = postcondition ? "verify_state" : "get_window_state";
const readArgs = {
@@ -863,7 +889,14 @@ export function createComputerMcpServer(
}
// Automatic evidence must not create unmanaged screenshots of user windows.
// It is never reused for element actions, so the compact tree is enough.
- : { include_screenshot: false, tree_format: "markdown" }),
+ : {
+ ...scope,
+ include_screenshot: false,
+ tree_format: "markdown",
+ // CUA 0.37's structured diff fails its own output validation. Only
+ // non-reusable Markdown evidence uses since; actionable reads stay full.
+ ...(scope ? { since: 'latest' } : {}),
+ }),
};
const read = await readForRecovery(
deps,
@@ -1567,7 +1600,11 @@ export function createComputerMcpServer(
if (!data || typeof data !== 'object' || Array.isArray(data) || !computerResultOutcome(name, data).ok) {
return null;
}
- const snapshotId = snapshotTracker.record(sessionId, parsedData.pid, parsedData.window_id);
+ const snapshotId = snapshotTracker.record(sessionId, parsedData.pid, parsedData.window_id, {
+ ...(typeof parsedData.query === 'string' ? { query: parsedData.query } : {}),
+ ...(typeof parsedData.max_elements === 'number' ? { max_elements: parsedData.max_elements } : {}),
+ ...(typeof parsedData.max_depth === 'number' ? { max_depth: parsedData.max_depth } : {}),
+ });
const driverSnapshotId = readDriverSnapshotId(data);
if (driverSnapshotId) {
snapshotTracker.registerAlias(snapshotId, driverSnapshotId);
diff --git a/packages/lizi-mcps/src/computer/snapshot-tracker.ts b/packages/lizi-mcps/src/computer/snapshot-tracker.ts
index 7aea1e4d52..ec6bf69b96 100644
--- a/packages/lizi-mcps/src/computer/snapshot-tracker.ts
+++ b/packages/lizi-mcps/src/computer/snapshot-tracker.ts
@@ -42,6 +42,13 @@ interface SnapshotMeta {
elements?: Map<string, number>;
/** Screen frames by element_index, to ask the OS what an action would land on. */
frames?: Map<number, ElementFrame>;
+ scope?: WindowObservationScope;
+}
+
+export interface WindowObservationScope {
+ query?: string;
+ max_elements?: number;
+ max_depth?: number;
}
/** 保留的窗口数与历史快照数上限(FIFO 淘汰)。 */
@@ -66,7 +73,7 @@ export class WindowSnapshotTracker {
}
/** get_window_state 成功后调用:登记新一代快照并返回其 id。 */
- record(sessionId: string | undefined, pid: number, windowId: number): string {
+ record(sessionId: string | undefined, pid: number, windowId: number, scope?: WindowObservationScope): string {
const sessionKey = sessionId ?? '';
const windowKey = this.windowKey(sessionKey, pid, windowId);
if (!this.latestByWindow.has(windowKey) && this.latestByWindow.size >= MAX_TRACKED_WINDOWS) {
@@ -84,10 +91,16 @@ export class WindowSnapshotTracker {
this.seq += 1;
const id = `ws-${this.seq.toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
this.latestByWindow.set(windowKey, id);
- this.metaById.set(id, { windowKey, sessionKey, pid, windowId });
+ this.metaById.set(id, { windowKey, sessionKey, pid, windowId, scope });
return id;
}
+ /** Only the latest successful explicit read supplies a follow-up's traversal scope. */
+ observationScope(sessionId: string | undefined, pid: number, windowId: number): WindowObservationScope | undefined {
+ const id = this.latestByWindow.get(this.windowKey(sessionId ?? '', pid, windowId));
+ return id ? this.metaById.get(id)?.scope : undefined;
+ }
+
/** A failed observation invalidates earlier IDs and aliases without issuing a usable ID. */
invalidate(
sessionId: string | undefined,
diff --git a/packages/lizi-mcps/src/computer/snapshotGuard.test.ts b/packages/lizi-mcps/src/computer/snapshotGuard.test.ts
index 381820e58b..a507ce2cc5 100644
--- a/packages/lizi-mcps/src/computer/snapshotGuard.test.ts
+++ b/packages/lizi-mcps/src/computer/snapshotGuard.test.ts
@@ -770,7 +770,9 @@ describe('computer snapshot guard', () => {
});
describe('password fields', () => {
- it('refuses agent typing into a password field and points at 代输入', async () => {
+ // MCP entry, unit tier: follow-up observations must retain OS hit-test frames;
+ // ordinary explicit reads alone did not cover the action + observation path.
+ it.each([false, true])('refuses typing into passwords, including action observations (observe=%s)', async (observe) => {
const dispatch = vi.fn(async (name: string) => name === 'get_window_state'
? {
elements: [
@@ -785,7 +787,9 @@ describe('password fields', () => {
point ? point.y === 70 : focusedSecure);
const h = await makeHarness({ getStatus: vi.fn(), callTool: dispatch, isSecureTarget }, { sessionId: 'secure' });
try {
- const state = await h.call('get_window_state', { pid: 1, window_id: 2 });
+ const state = observe
+ ? (await h.call('click', { pid: 1, window_id: 2, x: 1, y: 1, observe: {} })).observation as Record<string, unknown>
+ : await h.call('get_window_state', { pid: 1, window_id: 2 });
for (const [tool, extra] of [['type_text', { text: 'synthetic' }], ['set_value', { value: 'synthetic' }]] as const) {
expect(await h.call(tool, { pid: 1, window_id: 2, element_token: 'w:1', ...extra })).toMatchObject({
ok: false,
@@ -796,7 +800,7 @@ describe('password fields', () => {
.toMatchObject({ errorCode: 'SECURE_FIELD' });
}
expect(isSecureTarget).toHaveBeenCalledWith({ pid: 1, point: { x: 60, y: 70 } });
- expect(dispatch).toHaveBeenCalledTimes(1);
+ expect(dispatch).toHaveBeenCalledTimes(observe ? 2 : 1);
expect(await h.call('type_text', { pid: 1, window_id: 2, element_token: 'w:0', text: '[email protected]' }))
.toMatchObject({ ok: true });
// Plain typing lands in the focused element.
diff --git a/packages/lizi-mcps/src/computer/tools.ts b/packages/lizi-mcps/src/computer/tools.ts
index 8b75d9c48e..ab0bb8a5ce 100644
--- a/packages/lizi-mcps/src/computer/tools.ts
+++ b/packages/lizi-mcps/src/computer/tools.ts
@@ -82,6 +82,13 @@ export const POSTCHECK_ACTION_TOOLS = new Set<ComputerMcpToolName>([
"scroll",
]);
+/** Keep an action's follow-up read in the same bounded window scope. */
+export const WINDOW_READ_SCOPE = {
+ query: z.string().optional(),
+ max_elements: z.number().int().positive().max(2000).optional().describe('Maximum AX nodes to walk (host default 200). Keep this and query/max_depth stable across rereads; increase only when truncated.'),
+ max_depth: z.number().int().positive().optional().describe('Maximum AX tree depth (host default 15).'),
+};
+
const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [
{
name: 'status',
@@ -121,7 +128,7 @@ const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [
},
{
name: 'list_apps',
- description: 'List installed and running desktop applications.',
+ description: 'List installed and running desktop applications. Takes no filters; do not pass running_only. Reuse a known pid from list_windows rather than guessing an app bundle ID.',
readOnly: true,
inputShape: {},
},
@@ -148,11 +155,9 @@ const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [
window_id: z.number().int().nonnegative(),
capture_mode: z.enum(['som', 'vision', 'ax']).optional(),
include_screenshot: z.boolean().optional(),
- query: z.string().optional(),
+ ...WINDOW_READ_SCOPE,
screenshot_out_file: z.string().optional(),
session: z.string().optional(),
- max_elements: z.number().int().positive().max(2000).optional().describe("Maximum number of accessibility tree elements to return (up to 2000; host default 200). Narrow with query for larger windows."),
- max_depth: z.number().int().positive().optional().describe("Maximum depth of the accessibility tree to traverse. Use to limit tree depth for complex windows."),
},
},
{
@@ -404,6 +409,12 @@ export const COMPUTER_TOOLS: readonly ComputerToolDef[] =
postcondition: POSTCONDITION.optional().describe(
"Optional expected final state, using verify_state predicates. Cindy checks it once after delivery (requires an exact window). Without it, unknown delivery gets one fresh observation but is never declared successful. Actions are never replayed automatically.",
),
+ observe: z.object({
+ ...WINDOW_READ_SCOPE,
+ include_screenshot: z.boolean().optional(),
+ }).strict().optional().describe(
+ 'Read the same window once after delivery, returning observation with a fresh usable snapshot. Use {} for text; include_screenshot:true for visual evidence. Omitted query/max_elements/max_depth inherit the last explicit read. Mutually exclusive with postcondition. Does not replay actions or prove the goal succeeded.',
+ ),
}
: {}),
session_goal: z