Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
perf(computer): 合并动作后观察并优化 CUA 0.37 兼容性
Dash4ee691e50599
diff --git a/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts b/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.tsindex 2e89a03383..9185dd8e90 100644--- a/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts+++ b/apps/desktop/src/main/computer-permission-guide/__tests__/switch-target.test.ts@@ -2,6 +2,7 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';const {callToolMock,+ listToolsMock,clientCloseMock,clientConnectMock,clientCtorMock,@@ -9,6 +10,7 @@ const {transportCtorMock,} = vi.hoisted(() => ({callToolMock: vi.fn(),+ listToolsMock: vi.fn(),clientCloseMock: vi.fn(),clientConnectMock: vi.fn(),clientCtorMock: vi.fn(),@@ -42,6 +44,7 @@ vi.mock('@modelcontextprotocol/sdk/client/index.js', () => ({clientCtorMock(...args);return {callTool: callToolMock,+ listTools: listToolsMock,close: clientCloseMock,connect: clientConnectMock,};@@ -102,6 +105,10 @@ beforeEach(async () => {existsSyncMock.mockReturnValue(true);clientConnectMock.mockResolvedValue(undefined);clientCloseMock.mockResolvedValue(undefined);+ listToolsMock.mockResolvedValue({ tools: [{ name: 'get_window_state', inputSchema: {+ properties: { pid: {}, window_id: {}, include_screenshot: {}, tree_format: {}, query: {}, max_elements: {}, max_depth: {}, session: {} },+ additionalProperties: false,+ } }] });mockSuccessfulLocate();});@@ -110,6 +117,23 @@ afterAll(() => {});describe('persistent switch locator connection', () => {+ // CUA 0.35+ no longer returns elements by default. Verify the actual locator+ // requests frames for the switch, while old schemas do not receive tree_format.+ it.each([true, false])('locates switches using the advertised observation format (modern=%s)', async (modern) => {+ listToolsMock.mockResolvedValue({ tools: [{ name: 'get_window_state', inputSchema: {+ properties: { pid: {}, window_id: {}, include_screenshot: {}, ...(modern ? { tree_format: {} } : {}), query: {}, max_elements: {}, max_depth: {}, session: {} },+ additionalProperties: false,+ } }] });+ callToolMock.mockImplementation(async ({ name, arguments: args }) => name === 'list_windows' ? windowsResult+ : modern && args.tree_format !== 'elements' ? { structuredContent: { tree_markdown: 'CuaDriver', element_count: 2 } } : stateResult);+ await expect(locateComputerUseSwitchTarget()).resolves.toMatchObject({ status: 'found' });+ await expect(locateComputerUseSwitchTarget()).resolves.toMatchObject({ status: 'found' });+ expect(listToolsMock).toHaveBeenCalledTimes(1);+ const args = callToolMock.mock.calls.find(([request]) => request.name === 'get_window_state')?.[0].arguments;+ if (modern) expect(args).toHaveProperty('tree_format', 'elements');+ else expect(args).not.toHaveProperty('tree_format');+ });+it('does not connect during isolated module import and connects on first locate', async () => {vi.resetModules();diff --git a/apps/desktop/src/main/computer-permission-guide/switch-target.ts b/apps/desktop/src/main/computer-permission-guide/switch-target.tsindex fb06ae21ec..7f5ccdcc65 100644--- a/apps/desktop/src/main/computer-permission-guide/switch-target.ts+++ b/apps/desktop/src/main/computer-permission-guide/switch-target.ts@@ -5,6 +5,7 @@ import os from 'node:os';import path from 'node:path';import { findSystemSettingsWindowBounds } from './placement.js';+import { adaptComputerDriverArgs, type ComputerDriverToolSchema } from '../mcp-integrations/computer-contract.js';const SYSTEM_CUA_DRIVER_APP_EXECUTABLE = path.join('/Applications',@@ -55,6 +56,7 @@ interface SwitchLocatorConnection {client: Client;transport: StdioClientTransport;session: string;+ toolSchemas: Map<string, ComputerDriverToolSchema>;}let switchLocatorConnection: SwitchLocatorConnection | null = null;@@ -281,9 +283,12 @@ async function connectSwitchLocator(executable: string): Promise<SwitchLocatorCoclient,transport,session: `xdt-permission-switch-${Date.now()}`,+ toolSchemas: new Map<string, ComputerDriverToolSchema>(),};try {await withTimeout(client.connect(transport), CONNECT_TIMEOUT_MS, 'switch locator startup');+ const { tools } = await withTimeout(client.listTools(), TOOL_TIMEOUT_MS, 'switch locator tools');+ connection.toolSchemas = new Map(tools.map((tool) => [tool.name, tool.inputSchema as ComputerDriverToolSchema]));switchLocatorConnection = connection;return connection;} catch (error) {@@ -324,17 +329,19 @@ async function locateComputerUseSwitchTargetSerialized(): Promise<ComputerUseSwiconst stateResult = parseSuccessfulMcpResult(await withTimeout(connection.client.callTool({name: 'get_window_state',- arguments: {+ arguments: adaptComputerDriverArgs('get_window_state', {pid,window_id: windowId,include_screenshot: false,+ // The locator consumes frames; 0.35+ defaults to Markdown without them.+ tree_format: 'elements',// Match the stable app row instead of the localized permission-pane// title, while avoiding a walk of the full 400-500 element tree.max_elements: 180,max_depth: 6,query: 'CuaDriver',session: connection.session,- },+ }, connection.toolSchemas),}),TOOL_TIMEOUT_MS,'Computer Use switch lookup',diff --git a/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts b/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.tsindex 201d6dac07..3a74054e92 100644--- a/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts+++ b/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts@@ -6,6 +6,17 @@ const schemas = new Map(fixture.tools.map((tool) => [tool.name, tool.inputSchema as ComputerDriverToolSchema]),);describe('installed Computer Use contract', () => {+ // Production argument adapter, unit tier: new incremental evidence requests+ // must still dispatch with the existing old-driver fixture (previously untested).+ it('preserves Markdown evidence diffs only for drivers advertising since', () => {+ const input = { pid: 1, window_id: 2, include_screenshot: false, tree_format: 'markdown', since: 'latest' };+ const modern = new Map<string, ComputerDriverToolSchema>([['get_window_state', {+ properties: { pid: {}, window_id: {}, include_screenshot: {}, tree_format: {}, since: {} },+ additionalProperties: false,+ }]]);+ expect(adaptComputerDriverArgs('get_window_state', input, modern)).toEqual(input);+ expect(adaptComputerDriverArgs('get_window_state', input, schemas)).not.toHaveProperty('since');+ });it('preserves driver credentials and rejects missing observation provenance', () => {const args = {pid: 1,diff --git a/apps/desktop/src/main/mcp-integrations/computer-contract.ts b/apps/desktop/src/main/mcp-integrations/computer-contract.tsindex 37525b89ea..d25c14199f 100644--- a/apps/desktop/src/main/mcp-integrations/computer-contract.ts+++ b/apps/desktop/src/main/mcp-integrations/computer-contract.ts@@ -48,6 +48,8 @@ export function adaptComputerDriverArgs(// elements to register tokens and frames, and trims them from the model view.if ('tree_format' in props) args.tree_format ??= 'both';else delete args.tree_format;+ // Evidence reads may use compact diffs; older drivers simply return full state.+ if (!('since' in props)) delete args.since;}if (args.element_index !== undefined && !('element_index' in props) && 'element_token' in props) {// Newer drivers address an observed row only as `<snapshot_id>:<row>`.diff --git a/packages/lizi-mcps/src/computer/README.md b/packages/lizi-mcps/src/computer/README.mdnew file mode 100644index 0000000000..b78e1da6e7--- /dev/null+++ b/packages/lizi-mcps/src/computer/README.md@@ -0,0 +1,98 @@+# Computer Use observations++Discover only the schemas needed for the next operation, for example+`list_tools({"names":["list_windows","get_window_state","click"]})`.+An empty argument object still returns the complete catalogue. MCP argument+objects are not CLI flags; `list_apps` accepts no `running_only` filter.++After selecting the actual `pid` and `window_id`, read the window once. A+subsequent action can request the next usable observation in the same call:++```json+{+ "name": "click",+ "args": {+ "pid": 123,+ "window_id": 456,+ "element_token": "<token from the latest observation>",+ "observe": {}+ }+}+```++`observe: {}` returns text; `observe: {"include_screenshot": true}` requests+visual evidence through the normal host screenshot path. Omitted `query`,+`max_elements` and `max_depth` inherit the latest successful observation of+that session/window. Override them only when the target is outside the current+scope. The returned `observation` uses the ordinary read result envelope and+registers fresh element tokens and secure-field frames. It is mutually exclusive+with `postcondition`, which checks an explicit assertion instead.++Delivery and observation have separate outcomes: inspect `observation.ok` and+the returned state before continuing. Failed reads do not undo delivery, and+unknown delivery is not proof that the goal succeeded. Input is never replayed+automatically. Refused actions do not trigger the optional observation.++Cindy MCP supplies the task's stable driver session. When using the native CLI+instead, pass the **same non-default `session` on every read and action**, then+call `end_session` for that session. Independent invocations without a session+do not share the screenshot lease. Reuse discovered process/window identities;+do not guess bundle IDs. Foreground input already activates its target, so a+preceding `bring_to_front` is usually redundant.++## CUA 0.37 compatibility++On 2026-10-22, a read-only native check reproduced `typed_output_mismatch`+(`returned_element_count does not match elements`) with `tree_format: "both"`+and `since: "latest"`. Full actionable observations therefore keep structured+elements and do not request a diff. Automatic evidence, which cannot supply+tokens for later actions, uses Markdown diffs with the previous traversal scope.+The adapter drops `since` for older drivers that do not advertise it. The+permission-switch locator also discovers the installed schema once per connection+and explicitly requests structured elements when supported.++## Validation and cost, 2026-10-22++Measured through the production MCP server and an in-memory driver, against+base `0a9987b416`:++| Measurement | Before | After |+| --- | ---: | ---: |+| Advertised MCP catalogue, JSON bytes | 1,540 | 2,023 |+| Complete discovery response, UTF-8 bytes | 42,269 | 50,028 |+| Discovery for `click` alone, UTF-8 bytes | 42,269 (full list required) | 5,455 |+| Driver calls after an initial read, for unknown click delivery followed by a usable read | 3 | 2 |+| Model tool calls for that action and usable read | 2 | 1 |++The full response grows because actions advertise the optional observation+schema; focused discovery is about 87% smaller than the previous full response.+Repeated MCP catalogues were byte-identical within each version. Tool names,+system-prompt assembly and usage accounting are unchanged; this is not a+measurement of vendor cache-hit rates or live model latency.++Regression coverage runs through real MCP dispatch for token freshness, session+and scope retention, separate action/read outcomes, refusal and password-field+protection. Desktop tests exercise schema adaptation and the switch locator with+both older and modern driver contracts. GUI mutations use mock drivers in these+tests; the native compatibility check above only read a window.++A separate native macOS benchmark used the production MCP server and actual+CUA 0.37.0 against a dedicated AppKit window with 80 static rows. Each click+incremented a counter, verified through the next AX observation. Twelve+alternating before/after pairs per mode produced 48 measured actions, plus+eight excluded warmups; all 56 postconditions passed.++| Median per action and usable observation | Before | After | Reduction |+| --- | ---: | ---: | ---: |+| Text observation | 782.5 ms | 713.6 ms | 8.8% |+| Screenshot observation | 882.1 ms | 774.0 ms | 12.3% |+| Text response JSON | 11,591 B | 6,076 B | 47.6% |+| Screenshot response JSON | 12,007 B | 6,492 B | 45.9% |++Both versions used the same driver, window and traversal bounds (200 nodes,+depth 15). Timing starts after the initial observation and ends with the next+usable snapshot. Actual clicks returned `unverifiable`, so the old path made+three driver calls and the new path made two. This measures the production MCP+server and native driver, not the full Electron host or model/network latency.+PNG bytes are excluded from JSON sizes. The small controlled sample does not+establish an improvement in complex-task success rates or overall token costs.diff --git a/packages/lizi-mcps/src/computer/computerMcpServer.test.ts b/packages/lizi-mcps/src/computer/computerMcpServer.test.tsindex 53a3333b7c..307404c695 100644--- a/packages/lizi-mcps/src/computer/computerMcpServer.test.ts+++ b/packages/lizi-mcps/src/computer/computerMcpServer.test.ts@@ -75,6 +75,20 @@ async function makeHarness(}describe('createComputerMcpServer', () => {+ // Audit regression: discovery returned every schema even when only one was+ // needed. Exercise the real MCP list entry with no driver/process side effects.+ it('returns only requested schemas for focused discovery', async () => {+ const deps: ComputerMcpDeps = { getStatus: vi.fn(), callTool: vi.fn() };+ const h = await makeHarness(deps);+ try {+ const payload = textPayload(await h.client.callTool({ name: 'list_tools', arguments: { names: ['click'] } })) as {+ tools: Array<{ name: string; inputSchema: { properties: Record<string, unknown> } }>;+ };+ expect(payload.tools.map((tool) => tool.name)).toEqual(['click']);+ expect(payload.tools[0]?.inputSchema.properties).toHaveProperty('observe');+ expect(deps.callTool).not.toHaveBeenCalled();+ } finally { await h.cleanup(); }+ });it('advertises a first-call goal and keeps host session routing independent of it', async () => {const deps: ComputerMcpDeps = { getStatus: vi.fn(), callTool: vi.fn(async () => ({ windows: [] })) };const h = await makeHarness(deps, { sessionId: 'stable-task' });diff --git a/packages/lizi-mcps/src/computer/recovery.test.ts b/packages/lizi-mcps/src/computer/recovery.test.tsindex 1692dd0ea1..963fb3b226 100644--- a/packages/lizi-mcps/src/computer/recovery.test.ts+++ b/packages/lizi-mcps/src/computer/recovery.test.ts@@ -33,6 +33,87 @@ async function harness(dispatch: ComputerMcpDeps["callTool"]) {}describe("bounded Computer Use recovery", () => {+ // Production MCP entry, unit tier: the 0.37 audit exposed redundant reads and+ // lost CLI leases. Existing recovery tests deliberately forbid token reuse;+ // explicit observe must instead keep the session and return usable new tokens.+ it('returns one usable observation after an action, preserving the read scope and session', async () => {+ let reads = 0;+ const dispatch = vi.fn(async (name: string) => name === 'get_window_state'+ ? {+ snapshot_id: `s${++reads}`,+ tree_markdown: '- [0] AXButton Next',+ elements: [{ element_index: 0, element_token: `s${reads}:0`, frame: { x: 1, y: 1, w: 20, h: 20 } }],+ }+ : { effect: 'unverifiable' });+ const call = await harness(dispatch);+ await call('get_window_state', { pid: 1, window_id: 2, query: 'Next', max_elements: 70, max_depth: 6 });+ const result = await call('click', { pid: 1, element_token: 's1:0', observe: {} });+ expect(result).toMatchObject({+ ok: true, outcome: { status: 'unknown' },+ observation: { ok: true, snapshot_id: expect.any(String), data: { snapshot_id: 's2' } },+ });+ expect(result).not.toHaveProperty('postcheck');+ expect(result.observation.data).not.toHaveProperty('elements');+ expect(dispatch).toHaveBeenLastCalledWith('get_window_state', {+ pid: 1, window_id: 2, session: 'recovery-test', query: 'Next', max_elements: 70, max_depth: 6, include_screenshot: false,+ }, { sessionId: 'recovery-test', signal: expect.any(AbortSignal) });+ expect(dispatch.mock.calls.map(([name]) => name)).toEqual(['get_window_state', 'click', 'get_window_state']);+ expect(await call('click', { pid: 1, element_token: 's1:0' })).toMatchObject({ errorCode: 'STALE_SNAPSHOT' });+ expect(await call('click', { pid: 1, element_token: 's2:0', observe: { include_screenshot: true } }))+ .toMatchObject({ ok: true, observation: { data: { snapshot_id: 's3' } } });+ expect(dispatch).toHaveBeenLastCalledWith('get_window_state', expect.objectContaining({ include_screenshot: true }), expect.any(Object));+ });++ it.each([+ { pid: 1, x: 1, y: 2, observe: {} },+ { pid: 1, window_id: 2, observe: {}, postcondition: [{ window: { exists: true } }] },+ { pid: 1, window_id: 2, observe: { pid: 3 } },+ ])('rejects invalid follow-up observations before dispatch: %j', async (args) => {+ const dispatch = vi.fn();+ const call = await harness(dispatch);+ expect(await call('click', args)).toMatchObject({ ok: false, errorCode: 'INVALID_ARGS' });+ expect(dispatch).not.toHaveBeenCalled();+ });++ it('preserves action delivery when its explicit observation fails, without retrying input', async () => {+ const dispatch = vi.fn(async (name: string) => {+ if (name === 'get_window_state') throw new Error('Capture unavailable');+ return { effect: 'confirmed' };+ });+ const call = await harness(dispatch);+ expect(await call('click', { pid: 1, window_id: 2, x: 1, y: 2, observe: {} })).toMatchObject({+ ok: true, outcome: { status: 'confirmed' },+ observation: { ok: false, errorCode: 'COMPUTER_DRIVER_ERROR' },+ });+ expect(dispatch.mock.calls.map(([name]) => name)).toEqual(['click', 'get_window_state']);+ });++ it('does not follow a refused action with an explicit observation', async () => {+ const dispatch = vi.fn(async () => ({ effect: 'refused', code: 'background_unavailable' }));+ const call = await harness(dispatch);+ expect(await call('click', { pid: 1, window_id: 2, observe: {} })).toMatchObject({ ok: false });+ expect(dispatch).toHaveBeenCalledTimes(1);+ });++ it('uses a Markdown diff with the prior scope for automatic evidence, never reusing its tokens', async () => {+ let reads = 0;+ const dispatch = vi.fn(async (name: string) => name === 'click'+ ? { effect: 'unverifiable' }+ : ++reads === 1+ ? { snapshot_id: 's1', elements: [{ element_index: 0, element_token: 's1:0' }] }+ : { snapshot_id: 's2', tree_diff: 'no change since s1', since_status: 'no_change' });+ const call = await harness(dispatch);+ await call('get_window_state', { pid: 1, window_id: 2, query: 'Next', max_elements: 70, max_depth: 6 });+ expect(await call('click', { pid: 1, element_token: 's1:0' })).toMatchObject({+ postcheck: { ok: true, reusable_snapshot: false, data: { since_status: 'no_change' } },+ });+ expect(dispatch).toHaveBeenLastCalledWith('get_window_state', {+ pid: 1, window_id: 2, session: 'recovery-test', query: 'Next', max_elements: 70, max_depth: 6,+ include_screenshot: false, tree_format: 'markdown', since: 'latest',+ }, expect.objectContaining({ observationPurpose: 'recovery' }));+ expect(await call('click', { pid: 1, element_token: 's2:0' })).toMatchObject({ errorCode: 'STALE_SNAPSHOT' });+ });+it("preserves interrupted typing progress without replaying input", async () => {const inputProgress = { completed_chars: 400, attempted_chars: 0, remaining_chars: 450 };const dispatch = vi.fn(async (name: string) => {diff --git a/packages/lizi-mcps/src/computer/recovery.ts b/packages/lizi-mcps/src/computer/recovery.tsindex 694899f021..83a0d2165a 100644--- a/packages/lizi-mcps/src/computer/recovery.ts+++ b/packages/lizi-mcps/src/computer/recovery.ts@@ -68,6 +68,7 @@ export async function readForRecovery(: tool === "get_window_state"? Array.isArray(object.elements) ||typeof object.tree_markdown === "string" ||+ typeof object.tree_diff === "string" ||typeof object.screenshot_frame_valid === "boolean": true);return {diff --git a/packages/lizi-mcps/src/computer/server.ts b/packages/lizi-mcps/src/computer/server.tsindex 1e8c8a5035..1bde3ab5a4 100644--- a/packages/lizi-mcps/src/computer/server.ts+++ b/packages/lizi-mcps/src/computer/server.ts@@ -41,7 +41,7 @@ export interface ComputerMcpServerOptions {const DESCRIPTION_LIST ='Discover local desktop computer-use tools. These tools operate on the user desktop through the installed driver. ' +- 'Use read-only status/get_accessibility_tree/list_apps/list_windows/get_window_state before click/type_text/press_key/hotkey.';+ 'Pass names to retrieve only the needed schemas; reuse them instead of guessing CLI flags. Use read-only status/list_windows/get_window_state before input.';const DESCRIPTION_CALL ='Invoke a local desktop computer-use tool. After status/permission checks, include session_goal in the first call\'s args as a short English name for the overall objective (not the current step); omit it thereafter. Arguments are validated before dispatching to the host driver.';@@ -405,17 +405,19 @@ export function createComputerMcpServer(// 单实例跨 session 共享,所以 tracker 内部按 sessionId 分键。const snapshotTracker = new WindowSnapshotTracker();- server.tool('list_tools', DESCRIPTION_LIST, {}, async () =>+ server.tool('list_tools', DESCRIPTION_LIST, {+ names: z.array(z.enum(COMPUTER_TOOL_NAMES)).min(1).max(COMPUTER_TOOL_NAMES.length).optional(),+ }, async ({ names }) =>textResult({ok: true,- tools: COMPUTER_TOOLS.map((tool) => ({+ tools: COMPUTER_TOOLS.filter((tool) => !names || names.includes(tool.name)).map((tool) => ({name: tool.name,description: tool.description,readOnly: tool.readOnly === true,inputSchema: z.toJSONSchema(z.object(tool.inputShape).strict()),})),workflow:- 'Start with status and check_permissions. Use get_accessibility_tree/list_windows, optionally narrow list_windows with query/workspace_root/process_name (for example, {"process_name":"Simulator"}), inspect a target with get_window_state, perform one action, and call get_window_state again to verify. Targeted actions such as click/type_text require pid; include window_id whenever the target window is known, and always for coordinates. Use get_window_state with include_screenshot:false for bounded text observations; request an image for visual grounding and normally omit screenshot_out_file. Prefer the exact element_token returned by the latest observation, or pass snapshot_id with element_index. Re-observe on STALE_SNAPSHOT. After an action, verify its intended postcondition with verify_state or a fresh observation; a delivered or unverifiable action does not prove completion. Never replay a mutation automatically after cancellation or a connection failure. Use start_recording/stop_recording/replay_trajectory only when the user explicitly asks for recording or replay.',+ 'Start with status and check_permissions. Reuse these schemas; CLI flags are not MCP arguments. Find the exact pid/window_id with list_windows, optionally query/workspace_root/process_name (for example, {"process_name":"Simulator"}); reuse known identities instead of guessing app names. Inspect with get_window_state (include_screenshot:false for text), then use an action with observe:{} to return the next usable text snapshot in the same call (observe:{include_screenshot:true} for visual evidence), or postcondition for an explicit assertion. Keep query/max_elements/max_depth stable; narrow with query and broaden only if needed. Targeted actions require pid; include window_id whenever known, always for coordinates. Prefer the latest element_token or snapshot_id plus element_index. Re-observe on STALE_SNAPSHOT. A delivered/unverifiable action is not proof of success: assess the observation before continuing. This MCP manages a stable session automatically. If using the native CLI instead, repeat the same non-default session on EVERY observation and action, then end_session; CLI invocations without it are separate leases. Prefer element actions; foreground input already activates its target, so do not also bring_to_front unless it must stay frontmost between calls. Never replay input after cancellation or uncertain delivery. Record/replay only when explicitly requested.',}),);@@ -446,7 +448,7 @@ export function createComputerMcpServer(args: Record<string, unknown> | undefined,signal?: AbortSignal,dispatchOptions?: ComputerDispatchOptions,- ) {+ ): Promise<ReturnType<typeof textResult>> {const def = getComputerTool(name);if (!def) {return textResult({ ok: false, errorCode: 'UNKNOWN_TOOL', data: { requested: name } }, true);@@ -495,20 +497,28 @@ export function createComputerMcpServer(const secureFieldResult = await checkSecureField(name as ComputerMcpToolName, parsedData, sessionId);if (secureFieldResult) return secureFieldResult;const postcondition = parsedData.postcondition;+ const observe = parsedData.observe as Record<string, unknown> | undefined;delete parsedData.postcondition; // Cindy-owned verification, never part of driver action args.- if (postcondition && typeof parsedData.window_id !== "number") {+ delete parsedData.observe;+ if (observe && postcondition) {+ return textResult({ ok: false, errorCode: 'INVALID_ARGS', data: {+ message: 'Use observe for a fresh snapshot or postcondition for an assertion, not both.',+ } }, true);+ }+ if ((postcondition || observe) && typeof parsedData.window_id !== "number") {return textResult({ok: false,errorCode: "INVALID_ARGS",data: {message:- "postcondition requires an exact window_id or a fresh element snapshot identifying the window.",+ "postcondition/observe requires an exact window_id or a fresh element snapshot identifying the window.",},},true,);}+ const observationScope = snapshotTracker.observationScope(sessionId, parsedData.pid as number, parsedData.window_id as number);if (typeof parsedData.snapshot_id === "string") {const driverId = snapshotTracker.driverSnapshotId(sessionId,@@ -590,6 +600,7 @@ export function createComputerMcpServer(: undefined;const postcheck =!recovery &&+ !observe &&POSTCHECK_ACTION_TOOLS.has(name as ComputerMcpToolName) &&outcome.ok &&(postcondition || outcome.outcome?.status === "unknown")@@ -598,6 +609,19 @@ export function createComputerMcpServer(signal,}): undefined;+ // Use the normal explicit-read path: host session ownership, secure-field+ // frames, output handling and fresh element registration remain identical.+ // A returned observation is evidence, never a reason to replay the input.+ const observation = observe && !recovery && outcome.ok && !signal?.aborted+ ? JSON.parse((await handleCallTool('get_window_state', {+ ...observationScope,+ ...observe,+ pid: parsedData.pid,+ window_id: parsedData.window_id,+ include_screenshot: observe.include_screenshot ?? false,+ ...(typeof parsedData.session === 'string' ? { session: parsedData.session } : {}),+ }, signal, dispatchOptions)).content[0].text) as Record<string, unknown>+ : undefined;if (postcondition && postcheck) {const checked = computerResultOutcome("verify_state", postcheck.data);// An interrupted multi-chunk input is not a completed action, even if a@@ -629,6 +653,7 @@ export function createComputerMcpServer(data: withoutDuplicateElements(name, data),...(recovery ? { recovery } : {}),...(postcheck ? { postcheck } : {}),+ ...(observation ? { observation } : {}),},!outcome.ok,);@@ -849,6 +874,7 @@ export function createComputerMcpServer(context,);}+ const scope = snapshotTracker.observationScope(context.sessionId, args.pid, args.window_id);snapshotTracker.invalidate(context.sessionId, args.pid, args.window_id);const tool = postcondition ? "verify_state" : "get_window_state";const readArgs = {@@ -863,7 +889,14 @@ export function createComputerMcpServer(}// Automatic evidence must not create unmanaged screenshots of user windows.// It is never reused for element actions, so the compact tree is enough.- : { include_screenshot: false, tree_format: "markdown" }),+ : {+ ...scope,+ include_screenshot: false,+ tree_format: "markdown",+ // CUA 0.37's structured diff fails its own output validation. Only+ // non-reusable Markdown evidence uses since; actionable reads stay full.+ ...(scope ? { since: 'latest' } : {}),+ }),};const read = await readForRecovery(deps,@@ -1567,7 +1600,11 @@ export function createComputerMcpServer(if (!data || typeof data !== 'object' || Array.isArray(data) || !computerResultOutcome(name, data).ok) {return null;}- const snapshotId = snapshotTracker.record(sessionId, parsedData.pid, parsedData.window_id);+ const snapshotId = snapshotTracker.record(sessionId, parsedData.pid, parsedData.window_id, {+ ...(typeof parsedData.query === 'string' ? { query: parsedData.query } : {}),+ ...(typeof parsedData.max_elements === 'number' ? { max_elements: parsedData.max_elements } : {}),+ ...(typeof parsedData.max_depth === 'number' ? { max_depth: parsedData.max_depth } : {}),+ });const driverSnapshotId = readDriverSnapshotId(data);if (driverSnapshotId) {snapshotTracker.registerAlias(snapshotId, driverSnapshotId);diff --git a/packages/lizi-mcps/src/computer/snapshot-tracker.ts b/packages/lizi-mcps/src/computer/snapshot-tracker.tsindex 7aea1e4d52..ec6bf69b96 100644--- a/packages/lizi-mcps/src/computer/snapshot-tracker.ts+++ b/packages/lizi-mcps/src/computer/snapshot-tracker.ts@@ -42,6 +42,13 @@ interface SnapshotMeta {elements?: Map<string, number>;/** Screen frames by element_index, to ask the OS what an action would land on. */frames?: Map<number, ElementFrame>;+ scope?: WindowObservationScope;+}++export interface WindowObservationScope {+ query?: string;+ max_elements?: number;+ max_depth?: number;}/** 保留的窗口数与历史快照数上限(FIFO 淘汰)。 */@@ -66,7 +73,7 @@ export class WindowSnapshotTracker {}/** get_window_state 成功后调用:登记新一代快照并返回其 id。 */- record(sessionId: string | undefined, pid: number, windowId: number): string {+ record(sessionId: string | undefined, pid: number, windowId: number, scope?: WindowObservationScope): string {const sessionKey = sessionId ?? '';const windowKey = this.windowKey(sessionKey, pid, windowId);if (!this.latestByWindow.has(windowKey) && this.latestByWindow.size >= MAX_TRACKED_WINDOWS) {@@ -84,10 +91,16 @@ export class WindowSnapshotTracker {this.seq += 1;const id = `ws-${this.seq.toString(36)}-${Math.random().toString(36).slice(2, 8)}`;this.latestByWindow.set(windowKey, id);- this.metaById.set(id, { windowKey, sessionKey, pid, windowId });+ this.metaById.set(id, { windowKey, sessionKey, pid, windowId, scope });return id;}+ /** Only the latest successful explicit read supplies a follow-up's traversal scope. */+ observationScope(sessionId: string | undefined, pid: number, windowId: number): WindowObservationScope | undefined {+ const id = this.latestByWindow.get(this.windowKey(sessionId ?? '', pid, windowId));+ return id ? this.metaById.get(id)?.scope : undefined;+ }+/** A failed observation invalidates earlier IDs and aliases without issuing a usable ID. */invalidate(sessionId: string | undefined,diff --git a/packages/lizi-mcps/src/computer/snapshotGuard.test.ts b/packages/lizi-mcps/src/computer/snapshotGuard.test.tsindex 381820e58b..a507ce2cc5 100644--- a/packages/lizi-mcps/src/computer/snapshotGuard.test.ts+++ b/packages/lizi-mcps/src/computer/snapshotGuard.test.ts@@ -770,7 +770,9 @@ describe('computer snapshot guard', () => {});describe('password fields', () => {- it('refuses agent typing into a password field and points at 代输入', async () => {+ // MCP entry, unit tier: follow-up observations must retain OS hit-test frames;+ // ordinary explicit reads alone did not cover the action + observation path.+ it.each([false, true])('refuses typing into passwords, including action observations (observe=%s)', async (observe) => {const dispatch = vi.fn(async (name: string) => name === 'get_window_state'? {elements: [@@ -785,7 +787,9 @@ describe('password fields', () => {point ? point.y === 70 : focusedSecure);const h = await makeHarness({ getStatus: vi.fn(), callTool: dispatch, isSecureTarget }, { sessionId: 'secure' });try {- const state = await h.call('get_window_state', { pid: 1, window_id: 2 });+ const state = observe+ ? (await h.call('click', { pid: 1, window_id: 2, x: 1, y: 1, observe: {} })).observation as Record<string, unknown>+ : await h.call('get_window_state', { pid: 1, window_id: 2 });for (const [tool, extra] of [['type_text', { text: 'synthetic' }], ['set_value', { value: 'synthetic' }]] as const) {expect(await h.call(tool, { pid: 1, window_id: 2, element_token: 'w:1', ...extra })).toMatchObject({ok: false,@@ -796,7 +800,7 @@ describe('password fields', () => {.toMatchObject({ errorCode: 'SECURE_FIELD' });}expect(isSecureTarget).toHaveBeenCalledWith({ pid: 1, point: { x: 60, y: 70 } });- expect(dispatch).toHaveBeenCalledTimes(1);+ expect(dispatch).toHaveBeenCalledTimes(observe ? 2 : 1);expect(await h.call('type_text', { pid: 1, window_id: 2, element_token: 'w:0', text: '[email protected]' })).toMatchObject({ ok: true });// Plain typing lands in the focused element.diff --git a/packages/lizi-mcps/src/computer/tools.ts b/packages/lizi-mcps/src/computer/tools.tsindex 8b75d9c48e..ab0bb8a5ce 100644--- a/packages/lizi-mcps/src/computer/tools.ts+++ b/packages/lizi-mcps/src/computer/tools.ts@@ -82,6 +82,13 @@ export const POSTCHECK_ACTION_TOOLS = new Set<ComputerMcpToolName>(["scroll",]);+/** Keep an action's follow-up read in the same bounded window scope. */+export const WINDOW_READ_SCOPE = {+ query: z.string().optional(),+ max_elements: z.number().int().positive().max(2000).optional().describe('Maximum AX nodes to walk (host default 200). Keep this and query/max_depth stable across rereads; increase only when truncated.'),+ max_depth: z.number().int().positive().optional().describe('Maximum AX tree depth (host default 15).'),+};+const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [{name: 'status',@@ -121,7 +128,7 @@ const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [},{name: 'list_apps',- description: 'List installed and running desktop applications.',+ description: 'List installed and running desktop applications. Takes no filters; do not pass running_only. Reuse a known pid from list_windows rather than guessing an app bundle ID.',readOnly: true,inputShape: {},},@@ -148,11 +155,9 @@ const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [window_id: z.number().int().nonnegative(),capture_mode: z.enum(['som', 'vision', 'ax']).optional(),include_screenshot: z.boolean().optional(),- query: z.string().optional(),+ ...WINDOW_READ_SCOPE,screenshot_out_file: z.string().optional(),session: z.string().optional(),- max_elements: z.number().int().positive().max(2000).optional().describe("Maximum number of accessibility tree elements to return (up to 2000; host default 200). Narrow with query for larger windows."),- max_depth: z.number().int().positive().optional().describe("Maximum depth of the accessibility tree to traverse. Use to limit tree depth for complex windows."),},},{@@ -404,6 +409,12 @@ export const COMPUTER_TOOLS: readonly ComputerToolDef[] =postcondition: POSTCONDITION.optional().describe("Optional expected final state, using verify_state predicates. Cindy checks it once after delivery (requires an exact window). Without it, unknown delivery gets one fresh observation but is never declared successful. Actions are never replayed automatically.",),+ observe: z.object({+ ...WINDOW_READ_SCOPE,+ include_screenshot: z.boolean().optional(),+ }).strict().optional().describe(+ 'Read the same window once after delivery, returning observation with a fresh usable snapshot. Use {} for text; include_screenshot:true for visual evidence. Omitted query/max_elements/max_depth inherit the last explicit read. Mutually exclusive with postcondition. Does not replay actions or prove the goal succeeded.',+ ),}: {}),session_goal: z