Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(computer): make the real-driver smoke read Markdown rows
Dashf7012975aa9a
diff --git a/apps/desktop/src/main/smoke/__tests__/windowStateElements.test.ts b/apps/desktop/src/main/smoke/__tests__/windowStateElements.test.tsnew file mode 100644index 0000000000..7ac0edbf51--- /dev/null+++ b/apps/desktop/src/main/smoke/__tests__/windowStateElements.test.ts@@ -0,0 +1,34 @@+import { describe, expect, it } from 'vitest';+import { windowStateElements } from '../windowStateElements.js';++// Rows copied from a real cua-driver 0.37.0 read of the smoke fixture window.+const REAL_MARKDOWN = [+ '- [0] AXWindow "Cindy Computer Use Smoke" [actions=[raise]]',+ ' - [1] AXWebArea "Cindy Computer Use Smoke"',+ ' - [4] AXStaticText = "Smoke message"',+ ' - [5] AXTextArea (Smoke message) [actions=[press]]',+ ' - [6] AXButton "Increment smoke count" [actions=[press]]',+ ' - [8] AXButton [actions=[press]]',+].join('\n');++describe('windowStateElements', () => {+ it('addresses the fixture controls by <snapshot_id>:row, as the model reads them', () => {+ const elements = windowStateElements({ snapshot_id: 's00000036', tree_markdown: REAL_MARKDOWN });+ expect(elements.find((e) => e.role === 'AXTextArea' && e.label === 'Smoke message')).toEqual({+ element_token: 's00000036:5',+ role: 'AXTextArea',+ label: 'Smoke message',+ });+ expect(elements.find((e) => e.role === 'AXButton' && e.label === 'Increment smoke count')?.element_token).toBe(+ 's00000036:6',+ );+ expect(elements.find((e) => e.element_token === 's00000036:8')).toMatchObject({ role: 'AXButton', label: undefined });+ });++ it('falls back to elements when the Markdown tree is empty or absent', () => {+ const listed = [{ element_token: 'a:1', role: 'AXButton', label: 'OK' }];+ expect(windowStateElements({ snapshot_id: 's1', tree_markdown: '', elements: listed })).toEqual(listed);+ expect(windowStateElements({ elements: listed })).toEqual(listed);+ expect(windowStateElements(undefined)).toEqual([]);+ });+});diff --git a/apps/desktop/src/main/smoke/computerUseSmoke.ts b/apps/desktop/src/main/smoke/computerUseSmoke.tsindex ecb8431cb6..db3c16551d 100644--- a/apps/desktop/src/main/smoke/computerUseSmoke.ts+++ b/apps/desktop/src/main/smoke/computerUseSmoke.ts@@ -10,6 +10,7 @@ import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';import { createComputerMcpServer } from '@cindy/mcps/computer';import { cleanupComputerDriverSession, getComputerMcpDeps } from '../mcp-integrations/computer.js';import { createLogger } from '../logger.js';+import { windowStateElements } from './windowStateElements.js';const logger = createLogger('computer-use-smoke');let started = false;@@ -152,9 +153,8 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {for (let attempt = 0;attempt < 10 &&- !state.data?.elements?.some(- (item: { role?: string; label?: string }) =>- item.role === 'AXTextArea' && item.label === 'Smoke message',+ !windowStateElements(state.data).some(+ (item) => item.role === 'AXTextArea' && item.label === 'Smoke message',);attempt += 1) {@@ -168,25 +168,21 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {!state.data?.screenshot_out_file,'text observation excludes screenshot',);- const elements = state.data?.elements as Array<{- element_token?: string;- role?: string;- label?: string;- }>;+ const elements = windowStateElements(state.data);report.observationSummary = {keys: Object.keys(state.data ?? {}),element_count: state.data?.element_count,degraded_reason: state.data?.degraded_reason,- roles: elements?.slice(0, 12).map((item) => item.role),+ roles: elements.slice(0, 12).map((item) => item.role),window_id: target.window_id,};- report.fixtureElements = elements?.filter(+ report.fixtureElements = elements.filter((item) => item.label === 'Smoke message' || item.label === 'Increment smoke count',);- const input = elements?.find(+ const input = elements.find((item) => item.role === 'AXTextArea' && item.label === 'Smoke message',);- let button = elements?.find(+ let button = elements.find((item) => item.role === 'AXButton' && item.label === 'Increment smoke count',);check(input?.element_token && button?.element_token, 'opaque element tokens available');@@ -212,9 +208,8 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {// An unknown input now triggers a bounded observation and invalidates the// previous credentials. Obtain an explicit fresh snapshot before clicking.state = await call('get_window_state', { ...targetArgs, include_screenshot: false });- button = state.data?.elements?.find(- (item: { role?: string; label?: string }) =>- item.role === 'AXButton' && item.label === 'Increment smoke count',+ button = windowStateElements(state.data).find(+ (item) => item.role === 'AXButton' && item.label === 'Increment smoke count',);check(state.ok && button?.element_token, 'fresh button token obtained after input');const click = await call('click', {@@ -290,15 +285,14 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {);await window.webContents.executeJavaScript('document.getElementById("message").value = ""');state = await call('get_window_state', { ...targetArgs, include_screenshot: false });- const freshInput = state.data?.elements?.find(- (item: { role?: string; label?: string }) =>- item.role === 'AXTextArea' && item.label === 'Smoke message',+ const freshInput = windowStateElements(state.data).find(+ (item) => item.role === 'AXTextArea' && item.label === 'Smoke message',);check(freshInput?.element_token, 'fresh input token available for native typing');const typedValue = '原生输入测试🙂\n第二行内容';const typed = await call('type_text', {...targetArgs,- element_token: freshInput.element_token,+ element_token: freshInput!.element_token,text: typedValue,delivery_mode: 'foreground',postcondition: [diff --git a/apps/desktop/src/main/smoke/windowStateElements.ts b/apps/desktop/src/main/smoke/windowStateElements.tsnew file mode 100644index 0000000000..5af74c6d04--- /dev/null+++ b/apps/desktop/src/main/smoke/windowStateElements.ts@@ -0,0 +1,32 @@+export interface SmokeElement {+ element_token?: string;+ role?: string;+ label?: string;+}++// `[N] AXRole (label)`, `[N] AXRole "label"` or `[N] AXRole = "value"`.+const MARKDOWN_ROW = /^\s*- \[(\d+)\] (AX\w+)(?:\s+(?:\(([^)]*)\)|"([^"]*)"|= "([^"]*)"))?/;++/**+ * Elements of a Computer Use `get_window_state` observation, addressed the way the+ * model does: Cindy returns only `tree_markdown` once it is non-empty, and a row `[N]`+ * is `<data.snapshot_id>:N`. Drivers or degraded reads that return only `elements`+ * keep working through that list.+ */+export function windowStateElements(data: unknown): SmokeElement[] {+ const state = data as { elements?: unknown; tree_markdown?: unknown; snapshot_id?: unknown } | null | undefined;+ if (typeof state?.tree_markdown === 'string' && typeof state.snapshot_id === 'string') {+ const rows: SmokeElement[] = [];+ for (const line of state.tree_markdown.split('\n')) {+ const match = MARKDOWN_ROW.exec(line);+ if (!match) continue;+ rows.push({+ element_token: `${state.snapshot_id}:${match[1]}`,+ role: match[2],+ label: match[3] ?? match[4] ?? match[5],+ });+ }+ if (rows.length > 0) return rows;+ }+ return Array.isArray(state?.elements) ? (state.elements as SmokeElement[]) : [];+}