Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(computer): make the real-driver smoke read Markdown rows

Dashf7012975aa9a
diff --git a/apps/desktop/src/main/smoke/__tests__/windowStateElements.test.ts b/apps/desktop/src/main/smoke/__tests__/windowStateElements.test.ts
new file mode 100644
index 0000000000..7ac0edbf51
--- /dev/null
+++ b/apps/desktop/src/main/smoke/__tests__/windowStateElements.test.ts
@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest';
+import { windowStateElements } from '../windowStateElements.js';
+
+// Rows copied from a real cua-driver 0.37.0 read of the smoke fixture window.
+const REAL_MARKDOWN = [
+ '- [0] AXWindow "Cindy Computer Use Smoke" [actions=[raise]]',
+ ' - [1] AXWebArea "Cindy Computer Use Smoke"',
+ ' - [4] AXStaticText = "Smoke message"',
+ ' - [5] AXTextArea (Smoke message) [actions=[press]]',
+ ' - [6] AXButton "Increment smoke count" [actions=[press]]',
+ ' - [8] AXButton [actions=[press]]',
+].join('\n');
+
+describe('windowStateElements', () => {
+ it('addresses the fixture controls by <snapshot_id>:row, as the model reads them', () => {
+ const elements = windowStateElements({ snapshot_id: 's00000036', tree_markdown: REAL_MARKDOWN });
+ expect(elements.find((e) => e.role === 'AXTextArea' && e.label === 'Smoke message')).toEqual({
+ element_token: 's00000036:5',
+ role: 'AXTextArea',
+ label: 'Smoke message',
+ });
+ expect(elements.find((e) => e.role === 'AXButton' && e.label === 'Increment smoke count')?.element_token).toBe(
+ 's00000036:6',
+ );
+ expect(elements.find((e) => e.element_token === 's00000036:8')).toMatchObject({ role: 'AXButton', label: undefined });
+ });
+
+ it('falls back to elements when the Markdown tree is empty or absent', () => {
+ const listed = [{ element_token: 'a:1', role: 'AXButton', label: 'OK' }];
+ expect(windowStateElements({ snapshot_id: 's1', tree_markdown: '', elements: listed })).toEqual(listed);
+ expect(windowStateElements({ elements: listed })).toEqual(listed);
+ expect(windowStateElements(undefined)).toEqual([]);
+ });
+});
diff --git a/apps/desktop/src/main/smoke/computerUseSmoke.ts b/apps/desktop/src/main/smoke/computerUseSmoke.ts
index ecb8431cb6..db3c16551d 100644
--- a/apps/desktop/src/main/smoke/computerUseSmoke.ts
+++ b/apps/desktop/src/main/smoke/computerUseSmoke.ts
@@ -10,6 +10,7 @@ import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';
import { createComputerMcpServer } from '@cindy/mcps/computer';
import { cleanupComputerDriverSession, getComputerMcpDeps } from '../mcp-integrations/computer.js';
import { createLogger } from '../logger.js';
+import { windowStateElements } from './windowStateElements.js';
const logger = createLogger('computer-use-smoke');
let started = false;
@@ -152,9 +153,8 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {
for (
let attempt = 0;
attempt < 10 &&
- !state.data?.elements?.some(
- (item: { role?: string; label?: string }) =>
- item.role === 'AXTextArea' && item.label === 'Smoke message',
+ !windowStateElements(state.data).some(
+ (item) => item.role === 'AXTextArea' && item.label === 'Smoke message',
);
attempt += 1
) {
@@ -168,25 +168,21 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {
!state.data?.screenshot_out_file,
'text observation excludes screenshot',
);
- const elements = state.data?.elements as Array<{
- element_token?: string;
- role?: string;
- label?: string;
- }>;
+ const elements = windowStateElements(state.data);
report.observationSummary = {
keys: Object.keys(state.data ?? {}),
element_count: state.data?.element_count,
degraded_reason: state.data?.degraded_reason,
- roles: elements?.slice(0, 12).map((item) => item.role),
+ roles: elements.slice(0, 12).map((item) => item.role),
window_id: target.window_id,
};
- report.fixtureElements = elements?.filter(
+ report.fixtureElements = elements.filter(
(item) => item.label === 'Smoke message' || item.label === 'Increment smoke count',
);
- const input = elements?.find(
+ const input = elements.find(
(item) => item.role === 'AXTextArea' && item.label === 'Smoke message',
);
- let button = elements?.find(
+ let button = elements.find(
(item) => item.role === 'AXButton' && item.label === 'Increment smoke count',
);
check(input?.element_token && button?.element_token, 'opaque element tokens available');
@@ -212,9 +208,8 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {
// An unknown input now triggers a bounded observation and invalidates the
// previous credentials. Obtain an explicit fresh snapshot before clicking.
state = await call('get_window_state', { ...targetArgs, include_screenshot: false });
- button = state.data?.elements?.find(
- (item: { role?: string; label?: string }) =>
- item.role === 'AXButton' && item.label === 'Increment smoke count',
+ button = windowStateElements(state.data).find(
+ (item) => item.role === 'AXButton' && item.label === 'Increment smoke count',
);
check(state.ok && button?.element_token, 'fresh button token obtained after input');
const click = await call('click', {
@@ -290,15 +285,14 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {
);
await window.webContents.executeJavaScript('document.getElementById("message").value = ""');
state = await call('get_window_state', { ...targetArgs, include_screenshot: false });
- const freshInput = state.data?.elements?.find(
- (item: { role?: string; label?: string }) =>
- item.role === 'AXTextArea' && item.label === 'Smoke message',
+ const freshInput = windowStateElements(state.data).find(
+ (item) => item.role === 'AXTextArea' && item.label === 'Smoke message',
);
check(freshInput?.element_token, 'fresh input token available for native typing');
const typedValue = '原生输入测试🙂\n第二行内容';
const typed = await call('type_text', {
...targetArgs,
- element_token: freshInput.element_token,
+ element_token: freshInput!.element_token,
text: typedValue,
delivery_mode: 'foreground',
postcondition: [
diff --git a/apps/desktop/src/main/smoke/windowStateElements.ts b/apps/desktop/src/main/smoke/windowStateElements.ts
new file mode 100644
index 0000000000..5af74c6d04
--- /dev/null
+++ b/apps/desktop/src/main/smoke/windowStateElements.ts
@@ -0,0 +1,32 @@
+export interface SmokeElement {
+ element_token?: string;
+ role?: string;
+ label?: string;
+}
+
+// `[N] AXRole (label)`, `[N] AXRole "label"` or `[N] AXRole = "value"`.
+const MARKDOWN_ROW = /^\s*- \[(\d+)\] (AX\w+)(?:\s+(?:\(([^)]*)\)|"([^"]*)"|= "([^"]*)"))?/;
+
+/**
+ * Elements of a Computer Use `get_window_state` observation, addressed the way the
+ * model does: Cindy returns only `tree_markdown` once it is non-empty, and a row `[N]`
+ * is `<data.snapshot_id>:N`. Drivers or degraded reads that return only `elements`
+ * keep working through that list.
+ */
+export function windowStateElements(data: unknown): SmokeElement[] {
+ const state = data as { elements?: unknown; tree_markdown?: unknown; snapshot_id?: unknown } | null | undefined;
+ if (typeof state?.tree_markdown === 'string' && typeof state.snapshot_id === 'string') {
+ const rows: SmokeElement[] = [];
+ for (const line of state.tree_markdown.split('\n')) {
+ const match = MARKDOWN_ROW.exec(line);
+ if (!match) continue;
+ rows.push({
+ element_token: `${state.snapshot_id}:${match[1]}`,
+ role: match[2],
+ label: match[3] ?? match[4] ?? match[5],
+ });
+ }
+ if (rows.length > 0) return rows;
+ }
+ return Array.isArray(state?.elements) ? (state.elements as SmokeElement[]) : [];
+}