Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat(computer): 按本轮目标命名光标 session (#4473)

Dash02ec275a8aad
diff --git a/apps/desktop/src/main/mcp-integrations/__tests__/computer.test.ts b/apps/desktop/src/main/mcp-integrations/__tests__/computer.test.ts
index 9355f5a527..7bf4b9bff5 100644
--- a/apps/desktop/src/main/mcp-integrations/__tests__/computer.test.ts
+++ b/apps/desktop/src/main/mcp-integrations/__tests__/computer.test.ts
@@ -1066,6 +1066,63 @@ describe('computer mcp integration', () => {
expect(mcpConnectMock).toHaveBeenCalledTimes(1);
});
+ it('names the run after its first goal and ignores later step descriptions', async () => {
+ mcpCallToolMock.mockResolvedValue({ content: [{ type: 'text', text: '{"ok":true}' }] });
+ await callComputerDriverTool('list_windows', { session_goal: '提交报销申请' }, { sessionId: 'goal-a' });
+ await callComputerDriverTool('type_text', { pid: 1, text: 'x'.repeat(850), session_goal: '输入文字' }, { sessionId: 'goal-a' });
+ const calls = mcpCallToolMock.mock.calls.map(([call]) => call)
+ .filter((call) => ['list_windows', 'type_text'].includes(call.name));
+ expect(calls).toHaveLength(4);
+ const driverSession = calls[0].arguments.session;
+ expect(driverSession).toMatch(/^提交报销申请 · \d+-cua-[a-f0-9]{12}-0$/);
+ expect(calls.every((call) => call.arguments.session === driverSession)).toBe(true);
+ expect(calls.every((call) => !('session_goal' in call.arguments))).toBe(true);
+ await cleanupComputerDriverSession('goal-a');
+ expect(mcpCallToolMock.mock.lastCall![0]).toEqual({ name: 'end_session', arguments: { session: driverSession } });
+ await callComputerDriverTool('list_windows', { session_goal: '调整通知设置' }, { sessionId: 'goal-a' });
+ expect(mcpCallToolMock.mock.lastCall![0].arguments.session).toMatch(/^调整通知设置 · \d+-cua-[a-f0-9]{12}-1$/);
+ });
+
+ it('claims the first goal before asynchronous startup and isolates identical goals', async () => {
+ const startup = createDeferred<void>();
+ mcpConnectMock.mockReturnValueOnce(startup.promise);
+ mcpCallToolMock.mockResolvedValue({ content: [{ type: 'text', text: '{"ok":true}' }] });
+ const first = callComputerDriverTool('list_windows', { session_goal: 'Configure notifications' }, { sessionId: 'goal-a' });
+ const second = callComputerDriverTool('list_windows', { session_goal: 'Click button' }, { sessionId: 'goal-a' });
+ startup.resolve();
+ await Promise.all([first, second]);
+ await callComputerDriverTool('list_windows', { session_goal: 'Configure notifications' }, { sessionId: 'goal-b' });
+ const ids = mcpCallToolMock.mock.calls.map(([call]) => call.arguments.session);
+ expect(ids[0]).toBe(ids[1]);
+ expect(ids[0]).toMatch(/^Configure notifications · /);
+ expect(ids[2]).toMatch(/^Configure notifications · /);
+ expect(ids[2]).not.toBe(ids[0]);
+ });
+
+ it('preserves the goal through transport recovery and only rotates the generation', async () => {
+ mcpCallToolMock.mockImplementation(async (call) => {
+ if (call.name === 'list_windows' && call.arguments.session.endsWith('-0')) {
+ return { content: [{ type: 'text', text: '"session ended; tool call ignored"' }] };
+ }
+ return { content: [{ type: 'text', text: '{"ok":true}' }] };
+ });
+ await callComputerDriverTool('list_windows', { session_goal: 'Submit report' }, { sessionId: 'goal-retry' });
+ const ids = mcpCallToolMock.mock.calls.map(([call]) => call)
+ .filter((call) => call.name === 'list_windows').map((call) => call.arguments.session);
+ expect(ids).toHaveLength(2);
+ expect(ids[0]).toMatch(/^Submit report · /);
+ expect(ids[1]).toBe(ids[0].replace(/-0$/, '-1'));
+ });
+
+ it('keeps legacy identity when the first call omits a goal and strips host metadata on old drivers', async () => {
+ mcpCallToolMock.mockResolvedValue({ content: [{ type: 'text', text: '{"ok":true}' }] });
+ await callComputerDriverTool('list_windows', {}, { sessionId: 'legacy' });
+ await callComputerDriverTool('list_windows', { session_goal: 'Late name' }, { sessionId: 'legacy' });
+ const calls = mcpCallToolMock.mock.calls.map(([call]) => call.arguments);
+ expectDriverSessionGenerations(calls.map((args) => args.session), 'legacy', [0, 0]);
+ expect(calls.every((args) => !('session_goal' in args))).toBe(true);
+ });
+
it('retries Cindy cursor styling after a transient styling failure', async () => {
mcpCallToolMock
.mockRejectedValueOnce(new Error('motion unavailable'))
diff --git a/apps/desktop/src/main/mcp-integrations/computer.ts b/apps/desktop/src/main/mcp-integrations/computer.ts
index fc06f351c9..4810d3eb81 100644
--- a/apps/desktop/src/main/mcp-integrations/computer.ts
+++ b/apps/desktop/src/main/mcp-integrations/computer.ts
@@ -527,6 +527,9 @@ const cuaMcpSessionCleanups = new Map<string, Promise<void>>();
// applied/pending 仍保留在 entry 内,因为新 generation 必须重新应用成功的设置。
const cuaMcpSessionCursorCapabilities = new Map<string, CuaMcpSessionCursorCapabilities>();
const cuaDriverSessionGenerations = new Map<string, number>();
+// Retained across transport retries; cleared only when the owning run closes.
+const cuaDriverSessionNames = new Map<string, string>();
+let cuaDriverNamedRunSequence = 0;
const cuaMcpSessionCloseVersions = new Map<string, number>();
function getInstallCommand(): string {
@@ -1941,7 +1944,8 @@ function parseMcpToolResult(result: unknown): unknown {
function getDriverSessionId(sessionId: string): string {
const generation = cuaDriverSessionGenerations.get(sessionId) ?? 0;
- return `${sessionId}-cua-${CUA_DRIVER_SESSION_PROCESS_NONCE}-${generation}`;
+ const name = cuaDriverSessionNames.get(sessionId) ?? sessionId;
+ return `${name}-cua-${CUA_DRIVER_SESSION_PROCESS_NONCE}-${generation}`;
}
function rotateDriverSessionId(sessionId: string, failedDriverSessionId: string): void {
@@ -1958,9 +1962,10 @@ function applyDriverSessionArgs(
): Record<string, unknown> {
if (!DRIVER_SESSION_ARG_TOOL_NAMES.has(name)) return args;
const result = { ...args };
- const keys = name === 'get_agent_cursor_state' ? ['cursor_id']
- : name === 'move_cursor' ? ['cursor_id', 'session'] : ['session'];
const schema = schemas.get(name);
+ const keys = name === 'get_agent_cursor_state'
+ ? ['cursor_id', ...(schema?.properties && 'session' in schema.properties ? ['session'] : [])]
+ : name === 'move_cursor' ? ['cursor_id', 'session'] : ['session'];
for (const key of keys) {
// Host-owned routing metadata is injected only when the installed tool accepts it.
if (schema?.additionalProperties === false && !(key in (schema.properties ?? {}))) delete result[key];
@@ -3384,7 +3389,16 @@ async function callComputerDriverToolImpl(
}
};
const rawArgs = args ?? {};
- const driverInputArgs = stripLocalListWindowsArgs(name, rawArgs);
+ const { session_goal: sessionGoal, ...operationArgs } = rawArgs;
+ // Claim synchronously before shared startup, so concurrent first calls cannot
+ // rename an in-flight cursor. Even callers without a goal lock their old name.
+ if (!cuaDriverSessionNames.has(sessionId)) {
+ const goal = typeof sessionGoal === 'string'
+ ? Array.from(sessionGoal.replace(/[\p{Cc}\p{Cf}]/gu, ' ').replace(/\s+/gu, ' ').trim()).slice(0, 40).join('')
+ : '';
+ cuaDriverSessionNames.set(sessionId, goal ? `${goal} · ${++cuaDriverNamedRunSequence}` : sessionId);
+ }
+ const driverInputArgs = stripLocalListWindowsArgs(name, operationArgs);
const normalizedArgs = normalizeToolArgsForDriver(name, driverInputArgs);
const entry = await getCuaMcpSession(sessionId, signal);
const driverArgs = applyDriverSessionArgs(name, normalizedArgs, entry.driverSessionId, entry.toolSchemas);
@@ -3487,6 +3501,7 @@ export function cleanupComputerDriverSession(sessionId: string): Promise<void> {
if (entry) {
rotateDriverSessionId(sessionId, entry.driverSessionId);
}
+ cuaDriverSessionNames.delete(sessionId);
return cleanupComputerDriverSessionInternal(sessionId, { resetGeneration: false });
}
@@ -3525,6 +3540,7 @@ export async function cleanupAllComputerDriverSessions(): Promise<void> {
await cleanupActiveComputerDriverSessions();
cuaMcpSessionCursorCapabilities.clear();
cuaDriverSessionGenerations.clear();
+ cuaDriverSessionNames.clear();
cuaMcpSessionCloseVersions.clear();
}
diff --git a/apps/desktop/src/main/smoke/computerUseSmoke.ts b/apps/desktop/src/main/smoke/computerUseSmoke.ts
index 5d699d582b..07ba83bc56 100644
--- a/apps/desktop/src/main/smoke/computerUseSmoke.ts
+++ b/apps/desktop/src/main/smoke/computerUseSmoke.ts
@@ -13,8 +13,11 @@ import { createLogger } from '../logger.js';
const logger = createLogger('computer-use-smoke');
let started = false;
+const cursorGoalDemo = process.env.CINDY_CUA_SMOKE === 'cursor-goal';
const requested =
- !app.isPackaged && process.env.XDT_ISOLATED === '1' && process.env.CINDY_CUA_SMOKE === '1';
+ !app.isPackaged &&
+ process.env.XDT_ISOLATED === '1' &&
+ (process.env.CINDY_CUA_SMOKE === '1' || cursorGoalDemo);
/** Explicit, isolated, real-driver smoke. It operates only on its own disposable window. */
export async function runComputerUseSmokeIfRequested(): Promise<void> {
@@ -100,7 +103,46 @@ export async function runComputerUseSmokeIfRequested(): Promise<void> {
if (name === 'verify_state') report.verification = payload.data;
return payload;
};
- const catalog = await call('list_windows', { pid: process.pid });
+ if (cursorGoalDemo) {
+ // Exercise the real MCP -> host -> native driver path, without clicks or typing.
+ // Closing the disposable fixture ends the demo; otherwise it lasts two minutes.
+ let previousSession: unknown;
+ for (let step = 0; step < 12 && !window.isDestroyed(); step += 1) {
+ const bounds = window.getContentBounds();
+ const moved = await call('move_cursor', {
+ x: bounds.x + 160 + (step % 3) * 140,
+ y: bounds.y + 220,
+ ...(step === 0 ? { session_goal: 'Test cursor goal' } : {}),
+ });
+ check(moved.ok, `cursor demo move ${step + 1} succeeded`);
+ const state = await call('get_agent_cursor_state', {});
+ check(state.ok, `cursor demo state ${step + 1} available`);
+ report.cursorState = state.data;
+ if (step === 0) {
+ previousSession = state.data?.session;
+ check(
+ typeof previousSession === 'string' && previousSession.startsWith('Test cursor goal'),
+ 'native cursor session starts with the run goal',
+ );
+ } else check(state.data?.session === previousSession, 'cursor session remains stable');
+ await new Promise<void>((resolve) => {
+ const done = () => {
+ clearTimeout(timer);
+ window?.removeListener('closed', done);
+ resolve();
+ };
+ const timer = setTimeout(done, 10_000);
+ window!.once('closed', done);
+ if (window!.isDestroyed()) done();
+ });
+ }
+ report.ok = true;
+ return;
+ }
+ const catalog = await call('list_windows', {
+ pid: process.pid,
+ session_goal: 'Test computer controls',
+ });
const target = catalog.data?.windows?.find(
(item: { title?: string }) => item.title === 'Cindy Computer Use Smoke',
);
diff --git a/packages/lizi-mcps/src/computer/computerMcpServer.test.ts b/packages/lizi-mcps/src/computer/computerMcpServer.test.ts
index 5281a47194..fe19224842 100644
--- a/packages/lizi-mcps/src/computer/computerMcpServer.test.ts
+++ b/packages/lizi-mcps/src/computer/computerMcpServer.test.ts
@@ -69,6 +69,22 @@ async function makeHarness(
}
describe('createComputerMcpServer', () => {
+ it('advertises a first-call goal and keeps host session routing independent of it', async () => {
+ const deps: ComputerMcpDeps = { getStatus: vi.fn(), callTool: vi.fn(async () => ({ windows: [] })) };
+ const h = await makeHarness(deps, { sessionId: 'stable-task' });
+ try {
+ const listed = textPayload(await h.client.callTool({ name: 'list_tools', arguments: {} })) as {
+ tools: Array<{ name: string; inputSchema: { properties: Record<string, unknown> } }>;
+ };
+ expect(listed.tools.find((tool) => tool.name === 'list_windows')?.inputSchema.properties).toHaveProperty('session_goal');
+ await h.client.callTool({ name: 'call_tool', arguments: {
+ name: 'list_windows', args: { session_goal: '提交报销申请' },
+ } });
+ expect(deps.callTool).toHaveBeenLastCalledWith('list_windows', {
+ session: 'stable-task', session_goal: '提交报销申请',
+ }, { sessionId: 'stable-task', signal: expect.any(AbortSignal) });
+ } finally { await h.cleanup(); }
+ });
it('halts replay after an unknown action effect even with stop_on_error false', async () => {
const root = await makeWorkingDir();
const h = await makeHarness({ getStatus: vi.fn(), callTool: vi.fn(async () => ({ effect: 'unverifiable' })) }, {
diff --git a/packages/lizi-mcps/src/computer/server.ts b/packages/lizi-mcps/src/computer/server.ts
index c98349aa0e..7d7ebddf17 100644
--- a/packages/lizi-mcps/src/computer/server.ts
+++ b/packages/lizi-mcps/src/computer/server.ts
@@ -30,7 +30,7 @@ const DESCRIPTION_LIST =
'Use read-only status/get_accessibility_tree/list_apps/list_windows/get_window_state before click/type_text/press_key/hotkey.';
const DESCRIPTION_CALL =
- 'Invoke a local desktop computer-use tool. Arguments are validated before dispatching to the host driver.';
+ 'Invoke a local desktop computer-use tool. After status/permission checks, include session_goal in the first call\'s args as a short English name for the overall objective (not the current step); omit it thereafter. Arguments are validated before dispatching to the host driver.';
function textResult(value: unknown, isError?: boolean) {
return {
diff --git a/packages/lizi-mcps/src/computer/tools.ts b/packages/lizi-mcps/src/computer/tools.ts
index 8ed8f39f8b..6afc00081d 100644
--- a/packages/lizi-mcps/src/computer/tools.ts
+++ b/packages/lizi-mcps/src/computer/tools.ts
@@ -24,7 +24,7 @@ const ELEMENT_TOKEN_ARG = z.string().min(1).optional().describe(
'Opaque element_token from the latest get_window_state. Prefer this over element_index; never invent or reuse it after another observation.',
);
-export const COMPUTER_TOOLS: readonly ComputerToolDef[] = [
+const COMPUTER_TOOL_DEFINITIONS: readonly ComputerToolDef[] = [
{
name: 'status',
description: 'Check whether the local computer-use driver is installed and callable.',
@@ -343,6 +343,19 @@ export const COMPUTER_TOOLS: readonly ComputerToolDef[] = [
},
];
+/** Name the whole computer-use objective once; the host owns stable routing. */
+export const COMPUTER_TOOLS: readonly ComputerToolDef[] = COMPUTER_TOOL_DEFINITIONS.map((tool) => ({
+ ...tool,
+ inputShape: !['status', 'check_permissions', 'replay_trajectory'].includes(tool.name)
+ ? {
+ ...tool.inputShape,
+ session_goal: z.string().trim().min(1).max(80).optional().describe(
+ 'On your FIRST computer-use call, provide a short English name for the overall goal of this run (e.g. "Submit expense report" or "Configure notifications"), not the current click or typing step. Use English because the native cursor font does not support CJK text. The host sets the driver session name once and keeps it for the run; omit this field on later calls. Never include credentials or internal IDs.',
+ ),
+ }
+ : tool.inputShape,
+}));
+
export const COMPUTER_TOOL_NAMES = COMPUTER_TOOLS.map((tool) => tool.name) as [
ComputerMcpToolName,
...ComputerMcpToolName[],
diff --git a/scripts/__tests__/restart-desktop-remote.test.mjs b/scripts/__tests__/restart-desktop-remote.test.mjs
index 8e69615cac..987ee23932 100644
--- a/scripts/__tests__/restart-desktop-remote.test.mjs
+++ b/scripts/__tests__/restart-desktop-remote.test.mjs
@@ -935,7 +935,7 @@ test("devEnvPrefix passes harness envs through on Windows cmd with quote strippi
});
test("devEnvPrefix overrides a stale Computer Use smoke flag in the target shell", () => {
- for (const value of [undefined, "", "0", "1"]) {
+ for (const value of [undefined, "", "0", "1", "cursor-goal", "invalid"]) {
const env = value === undefined ? {} : { CINDY_CUA_SMOKE: value };
const node = process.platform === "win32" ? "%CINDY_TEST_NODE%" : "$CINDY_TEST_NODE";
const result = spawnSync(
@@ -948,7 +948,7 @@ test("devEnvPrefix overrides a stale Computer Use smoke flag in the target shell
},
);
assert.equal(result.status, 0, result.stderr);
- assert.equal(result.stdout.trim(), value === "1" ? "1" : "0", `caller value: ${value}`);
+ assert.equal(result.stdout.trim(), ["1", "cursor-goal"].includes(value) ? value : "0", `caller value: ${value}`);
}
});
diff --git a/scripts/restart-desktop-remote.mjs b/scripts/restart-desktop-remote.mjs
index d68d5bf054..5cd266d0bb 100644
--- a/scripts/restart-desktop-remote.mjs
+++ b/scripts/restart-desktop-remote.mjs
@@ -879,7 +879,7 @@ export function devEnvPrefix(env = process.env, platform = process.platform) {
['XDT_CDP_PORT', env.XDT_CDP_PORT],
// A long-lived Terminal can retain a previous smoke run's environment.
// Override its value even when this invocation did not request the smoke.
- ['CINDY_CUA_SMOKE', env.CINDY_CUA_SMOKE === '1' ? '1' : '0'],
+ ['CINDY_CUA_SMOKE', ['1', 'cursor-goal'].includes(env.CINDY_CUA_SMOKE) ? env.CINDY_CUA_SMOKE : '0'],
// 一次性 Grok wire 归因探针(dev-only;正常环境不设置,不产生额外日志)。
['XDT_WIRE_DIAGNOSTICS', env.XDT_WIRE_DIAGNOSTICS],
// 一次性 Grok strict tool spike(dev-only;必须与 wire probe 一起显式开启)。