Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(computer): address elements by token on cua-driver 0.35+

Dash014d0bcba488
diff --git a/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts b/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts
index ab2a8845b4..201d6dac07 100644
--- a/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts
+++ b/apps/desktop/src/main/mcp-integrations/__tests__/computer-contract.test.ts
@@ -47,6 +47,32 @@ describe('installed Computer Use contract', () => {
adaptComputerDriverArgs('get_window_state', { include_screenshot: false }, legacy),
).toEqual({ capture_mode: 'ax' });
});
+ it('addresses token-only drivers with the observed row and asks for structured elements', () => {
+ // cua-driver 0.35+: Markdown-only reads by default, actions take only element_token.
+ const tokenOnly = new Map<string, ComputerDriverToolSchema>([
+ ['click', { properties: { pid: {}, window_id: {}, element_token: {} }, additionalProperties: false }],
+ ['get_window_state', {
+ properties: { pid: {}, window_id: {}, include_screenshot: {}, max_elements: {}, max_depth: {}, tree_format: {} },
+ required: ['pid', 'window_id'],
+ additionalProperties: false,
+ }],
+ ]);
+ expect(adaptComputerDriverArgs('click', {
+ pid: 1, window_id: 2, element_index: 5, snapshot_id: 's0000002d',
+ }, tokenOnly)).toEqual({ pid: 1, window_id: 2, element_token: 's0000002d:5' });
+ expect(adaptComputerDriverArgs('click', {
+ pid: 1, window_id: 2, element_index: 5, element_token: 's0000002d:5', snapshot_id: 's0000002d',
+ }, tokenOnly)).toEqual({ pid: 1, window_id: 2, element_token: 's0000002d:5' });
+ expect(() => adaptComputerDriverArgs('click', { pid: 1, window_id: 2, element_index: 5 }, tokenOnly))
+ .toThrow('fresh snapshot_id or element_token');
+ expect(adaptComputerDriverArgs('get_window_state', { pid: 1, window_id: 2 }, tokenOnly))
+ .toMatchObject({ tree_format: 'both' });
+ expect(adaptComputerDriverArgs('get_window_state', { pid: 1, window_id: 2, tree_format: 'markdown' }, tokenOnly))
+ .toMatchObject({ tree_format: 'markdown' });
+ // Older drivers have no tree_format; Cindy's internal reads must still dispatch.
+ expect(adaptComputerDriverArgs('get_window_state', { pid: 1, window_id: 2, tree_format: 'markdown' }, schemas))
+ .not.toHaveProperty('tree_format');
+ });
it('rejects unavailable tools and incompatible arguments before dispatch', () => {
expect(() => adaptComputerDriverArgs('missing', {}, schemas)).toThrow('does not advertise');
const olderClick = new Map([['click', {
diff --git a/apps/desktop/src/main/mcp-integrations/computer-contract.ts b/apps/desktop/src/main/mcp-integrations/computer-contract.ts
index 17751d3c34..37525b89ea 100644
--- a/apps/desktop/src/main/mcp-integrations/computer-contract.ts
+++ b/apps/desktop/src/main/mcp-integrations/computer-contract.ts
@@ -44,7 +44,24 @@ export function adaptComputerDriverArgs(
// Bound the actual AX walk, not a lossy re-indexing of its response.
if ('max_elements' in props) args.max_elements ??= 200;
if ('max_depth' in props) args.max_depth ??= 15;
+ // Drivers since 0.35 default to Markdown only; Cindy needs the structured
+ // elements to register tokens and frames, and trims them from the model view.
+ if ('tree_format' in props) args.tree_format ??= 'both';
+ else delete args.tree_format;
}
+ if (args.element_index !== undefined && !('element_index' in props) && 'element_token' in props) {
+ // Newer drivers address an observed row only as `<snapshot_id>:<row>`.
+ if (args.element_token === undefined) {
+ if (typeof args.snapshot_id !== 'string')
+ throw new ComputerContractError(
+ 'Element actions require a fresh snapshot_id or element_token. Call get_window_state first.',
+ );
+ args.element_token = `${args.snapshot_id}:${args.element_index}`;
+ }
+ delete args.element_index;
+ }
+ // Cindy has already checked freshness; drivers without the field never see it.
+ if (!('snapshot_id' in props)) delete args.snapshot_id;
for (const key of Object.keys(args)) {
if (args[key] === undefined) delete args[key];
else if (schema.additionalProperties === false && !(key in props)) {
diff --git a/packages/lizi-mcps/src/computer/recovery.test.ts b/packages/lizi-mcps/src/computer/recovery.test.ts
index 4e761ebe01..1692dd0ea1 100644
--- a/packages/lizi-mcps/src/computer/recovery.test.ts
+++ b/packages/lizi-mcps/src/computer/recovery.test.ts
@@ -83,6 +83,7 @@ describe("bounded Computer Use recovery", () => {
window_id: 2,
session: "recovery-test",
include_screenshot: false,
+ tree_format: "markdown",
},
expect.objectContaining({
sessionId: "recovery-test",
@@ -107,7 +108,7 @@ describe("bounded Computer Use recovery", () => {
else expect(result.outcome.status).toBe("unknown");
expect(dispatch).toHaveBeenLastCalledWith(
"get_window_state",
- { pid: 1, window_id: 2, session: "recovery-test", include_screenshot: false },
+ { pid: 1, window_id: 2, session: "recovery-test", include_screenshot: false, tree_format: "markdown" },
expect.objectContaining({ observationPurpose: "recovery" }),
);
expect(dispatch).toHaveBeenCalledTimes(2);
diff --git a/packages/lizi-mcps/src/computer/server.ts b/packages/lizi-mcps/src/computer/server.ts
index 503f1cb84c..cf59553c17 100644
--- a/packages/lizi-mcps/src/computer/server.ts
+++ b/packages/lizi-mcps/src/computer/server.ts
@@ -626,7 +626,7 @@ export function createComputerMcpServer(
...outcome,
tool: name,
...(snapshotId ? { snapshot_id: snapshotId } : {}),
- data,
+ data: withoutDuplicateElements(name, data),
...(recovery ? { recovery } : {}),
...(postcheck ? { postcheck } : {}),
},
@@ -862,7 +862,8 @@ export function createComputerMcpServer(
include_screenshot: false,
}
// Automatic evidence must not create unmanaged screenshots of user windows.
- : { include_screenshot: false }),
+ // It is never reused for element actions, so the compact tree is enough.
+ : { include_screenshot: false, tree_format: "markdown" }),
};
const read = await readForRecovery(
deps,
@@ -1579,6 +1580,18 @@ export function createComputerMcpServer(
}
+/**
+ * Cindy asks drivers for both tree formats to register element tokens and frames.
+ * The model reads the compact Markdown rows; the structured copy stays host-side.
+ */
+function withoutDuplicateElements(name: string, data: unknown): unknown {
+ if (name !== 'get_window_state' || !data || typeof data !== 'object' || Array.isArray(data)) return data;
+ if (typeof (data as { tree_markdown?: unknown }).tree_markdown !== 'string') return data;
+ if (!('elements' in data)) return data;
+ const { elements: _elements, ...rest } = data as Record<string, unknown>;
+ return rest;
+}
+
function readDriverSnapshotId(data: unknown): string | null {
if (!data || typeof data !== 'object' || Array.isArray(data)) return null;
const direct = (data as { snapshot_id?: unknown }).snapshot_id;
diff --git a/packages/lizi-mcps/src/computer/snapshotGuard.test.ts b/packages/lizi-mcps/src/computer/snapshotGuard.test.ts
index 8405106112..c65394b957 100644
--- a/packages/lizi-mcps/src/computer/snapshotGuard.test.ts
+++ b/packages/lizi-mcps/src/computer/snapshotGuard.test.ts
@@ -401,6 +401,40 @@ describe('computer snapshot guard', () => {
await h.cleanup();
});
+ it('registers tokens from a Markdown-plus-elements read and keeps only Markdown for the model', async () => {
+ const callTool = vi.fn(async (name: string) => {
+ if (name === 'get_window_state') {
+ return {
+ ok: true,
+ snapshot_id: 's0000002d',
+ tree_markdown: '- [0] AXWindow "Calc"\n - [1] AXButton (7)',
+ elements: [
+ { element_index: 0, element_token: 's0000002d:0' },
+ { element_index: 1, element_token: 's0000002d:1' },
+ ],
+ };
+ }
+ return { ok: true };
+ });
+ const h = await makeHarness({ getStatus: vi.fn(), callTool }, { sessionId: 'sess-1' });
+
+ const observed = await h.call('get_window_state', { pid: 100, window_id: 1 });
+ expect(observed.data).toEqual({
+ ok: true,
+ snapshot_id: 's0000002d',
+ tree_markdown: '- [0] AXWindow "Calc"\n - [1] AXButton (7)',
+ });
+
+ expect((await h.call('click', { pid: 100, window_id: 1, element_token: 's0000002d:1' })).ok).toBe(true);
+ expect(callTool).toHaveBeenLastCalledWith('click', {
+ pid: 100,
+ window_id: 1,
+ element_token: 's0000002d:1',
+ session: 'sess-1',
+ }, { signal: expect.any(AbortSignal), sessionId: 'sess-1' });
+ await h.cleanup();
+ });
+
it('accepts a driver data.snapshot_id as a snapshot_id alias', async () => {
const callTool = vi.fn(async (name: string) => {
if (name === 'get_window_state') {
@@ -700,7 +734,9 @@ describe('computer snapshot guard', () => {
});
expect(observed.ok).toBe(ok);
expect(Boolean(observed.snapshot_id)).toBe(ok);
- expect(observed.data).toEqual(state);
+ // The model reads Markdown rows; the structured duplicate stays host-side.
+ const { elements: _elements, ...markdownOnly } = state as Record<string, unknown>;
+ expect(observed.data).toEqual(ok && "tree_markdown" in state ? markdownOnly : state);
// A failed AX observation may discover windows once, but never replays
// the observation, switches the target, or restarts the driver.
const rediscovered =
diff --git a/packages/lizi-mcps/src/computer/tools.ts b/packages/lizi-mcps/src/computer/tools.ts
index c770f226f7..8b75d9c48e 100644
--- a/packages/lizi-mcps/src/computer/tools.ts
+++ b/packages/lizi-mcps/src/computer/tools.ts
@@ -21,7 +21,7 @@ const SNAPSHOT_ID_ARG = z
);
const ELEMENT_TOKEN_ARG = z.string().min(1).optional().describe(
- 'Opaque element_token from the latest get_window_state. Prefer this over element_index; never invent or reuse it after another observation.',
+ 'element_token from the latest get_window_state: `<data.snapshot_id>:<row>` for a tree_markdown row [row], or elements[].element_token when listed. Never invent or reuse it after another observation.',
);
const POSTCONDITION = z
@@ -141,7 +141,7 @@ const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [
{
name: 'get_window_state',
description:
- 'Inspect one exact window before acting. Prefer elements[].element_token for actions, or pass snapshot_id with element_index. include_screenshot:false returns only the accessibility tree; request an image when visual grounding is needed. query, max_elements and max_depth bound the observation. Omit screenshot_out_file for a host-managed temporary image.',
+ 'Inspect one exact window before acting. A tree_markdown row [N] is element_index N: act on it with the top-level snapshot_id plus element_index, or with its element_token. include_screenshot:false returns only the accessibility tree; request an image when visual grounding is needed. query, max_elements and max_depth bound the observation. Omit screenshot_out_file for a host-managed temporary image.',
readOnly: true,
inputShape: {
pid: z.number().int().positive(),