Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

browser: make the real-task lessons hold for every browser choice

Dash4053935be409
diff --git a/apps/desktop/src/main/mcp-integrations/browser-backend/__tests__/user-browser-backend.test.ts b/apps/desktop/src/main/mcp-integrations/browser-backend/__tests__/user-browser-backend.test.ts
index 415010cdd8..19750cc9a4 100644
--- a/apps/desktop/src/main/mcp-integrations/browser-backend/__tests__/user-browser-backend.test.ts
+++ b/apps/desktop/src/main/mcp-integrations/browser-backend/__tests__/user-browser-backend.test.ts
@@ -709,16 +709,12 @@ describe('UserBrowserBackend · 读取与动作', () => {
]);
});
- it('打开页面时提示已开着的同站点页面;任务停下时清掉它操作过的页面上的标记', async () => {
+ it('任务停下时清掉它操作过的页面上的标记', async () => {
const fake = fakeControl({ version: '0.4.43' });
fake.responses.set('open', (payload) => ({ ok: true, tabId: 42, url: payload.url, title: 'New' }));
const b = backend(fake.control);
await call(b, { action: 'tabs', __mcpSessionId: 's1' });
- const opened = await call(b, { action: 'open', url: 'https://example.test/other', __mcpSessionId: 's1' });
- expect(opened).toMatchObject({
- ok: true,
- data: { sameSiteTabs: [{ targetId: 't1', url: 'https://example.test/a' }], note: expect.stringContaining('复用') },
- });
+ expect(await call(b, { action: 'open', url: 'https://example.test/other', __mcpSessionId: 's1' })).toMatchObject({ ok: true });
await call(b, { action: 'act', targetId: 't1', request: { kind: 'click', ref: 'r' }, __mcpSessionId: 's1' });
await b.clearSessionMarks('s1');
const unmarked = fake.jobs.filter(([action]) => action === 'unmark').map(([, payload]) => payload.tabId as number).sort((x, y) => x - y);
diff --git a/apps/desktop/src/main/mcp-integrations/browser-backend/user-browser-backend.ts b/apps/desktop/src/main/mcp-integrations/browser-backend/user-browser-backend.ts
index fd7f9578ea..da4774a64a 100644
--- a/apps/desktop/src/main/mcp-integrations/browser-backend/user-browser-backend.ts
+++ b/apps/desktop/src/main/mcp-integrations/browser-backend/user-browser-backend.ts
@@ -700,22 +700,6 @@ export class UserBrowserBackend implements BrowserBackend {
if (!picked.ok) return picked.result;
const label = typeof req.label === 'string' ? req.label : undefined;
if (this.parity(gate.control, picked.clientId)) {
- // 已知的同站点标签页(用户或之前的任务开着的,常常已在正确账号下):提示模型优先复用。
- let host = '';
- try {
- host = new URL(url).hostname;
- } catch {
- // validated by the bridge below
- }
- const sameSite = existing
- ? []
- : [...this.targets.values()].filter((r) => {
- try {
- return r.clientId === picked.clientId && new URL(r.url).hostname === host;
- } catch {
- return false;
- }
- });
// navigate 带 targetId:在原标签页跳转;否则新页开在 Cindy 自己的后台窗口,不碰用户的窗口。
const job =
existing && existing.clientId === picked.clientId
@@ -733,21 +717,7 @@ export class UserBrowserBackend implements BrowserBackend {
},
label,
);
- const others = sameSite.filter((r) => r.targetId !== record.targetId).slice(0, 5);
- return actionOk(req.action, {
- backend: this.kind,
- ...this.describe(record),
- ...(others.length
- ? {
- sameSiteTabs: others.map((r) => ({
- targetId: r.label ?? r.alias,
- url: r.url,
- ...(r.title ? { title: r.title } : {}),
- })),
- note: '已有同一网站的页面开着(往往已经在正确的账号和位置):优先复用它们,不要从入口重新找;新开的这页不需要时可以 close。',
- }
- : {}),
- });
+ return actionOk(req.action, { backend: this.kind, ...this.describe(record) });
}
const navigated = await gate.control.bridge.act('navigate', { url, browser: picked.clientId });
if (!navigated.ok) return this.failureResult(req.action, navigated);
diff --git a/packages/lizi-mcps/src/browser/browserMcpServer.test.ts b/packages/lizi-mcps/src/browser/browserMcpServer.test.ts
index 127140a0da..0eb4360174 100644
--- a/packages/lizi-mcps/src/browser/browserMcpServer.test.ts
+++ b/packages/lizi-mcps/src/browser/browserMcpServer.test.ts
@@ -1742,6 +1742,29 @@ describe('browser · backend capability gating (user browser via extension)', ()
} finally { await h.cleanup(); }
});
+ it('points out open tabs on the same site after open, and bounds snapshots by maxChars, for any backend', async () => {
+ const h = await makeHarness({}, (request) => {
+ if (request.action === 'open') return { targetId: 'new', url: 'https://play.example.test/console' };
+ if (request.action === 'tabs') return { tabs: [
+ { targetId: 'new', url: 'https://play.example.test/console' },
+ { targetId: 'old', suggestedTargetId: 'play', url: 'https://play.example.test/console/u/1/app', title: 'Release overview' },
+ { targetId: 'other', url: 'https://elsewhere.test/' },
+ ] };
+ if (request.action === 'snapshot') return { targetId: 'old', snapshot: Array.from({ length: 400 }, (_, i) => `line ${i} [ref=e${i}]`).join('\n'), refs: { e1: {}, e399: {} } };
+ return undefined;
+ });
+ try {
+ const opened = await h.client.callTool({ name: 'call_tool', arguments: { name: 'browser', args: { action: 'open', url: 'https://play.example.test/console' } } });
+ const body = JSON.parse((opened.content as Array<{ text: string }>)[0].text);
+ expect(body.sameSiteTabs).toEqual([{ targetId: 'play', url: 'https://play.example.test/console/u/1/app', title: 'Release overview' }]);
+ const snap = await h.client.callTool({ name: 'call_tool', arguments: { name: 'browser', args: { action: 'snapshot', targetId: 'play', maxChars: 500 } } });
+ const data = JSON.parse((snap.content as Array<{ text: string }>)[0].text).data;
+ expect(data.snapshot.length).toBeLessThanOrEqual(500);
+ expect(data.truncated).toBe(true);
+ expect(Object.keys(data.refs)).toEqual(['e1']);
+ } finally { await h.cleanup(); }
+ });
+
it('ignores a browser name passed as target instead of failing schema validation', async () => {
const h = await makeHarness({ backendCapabilities: () => ({ ...caps(), pointer: true }) });
try {
diff --git a/packages/lizi-mcps/src/browser/prompts/rules/browser-workflow.md b/packages/lizi-mcps/src/browser/prompts/rules/browser-workflow.md
index 2d3dd15cf5..60bc242687 100644
--- a/packages/lizi-mcps/src/browser/prompts/rules/browser-workflow.md
+++ b/packages/lizi-mcps/src/browser/prompts/rules/browser-workflow.md
@@ -76,6 +76,10 @@
大结果返回结构摘要和 `fullResult`,不是完整事实;用 `readResult(resultId,offset,maxChars)` 补读(`nextOffset:null` 表示结束),不猜省略内容或 ref。原文只留当前工具会话最近四份、每份最多 200 万字符;更大结果沿用明确截断,需缩小读取范围。`advisory` 是代码发现的可疑输出,不自动认定业务错误。
+#### 先复用已打开的页面(三种浏览器通用)
+
+开新页之前先 `tabs`:目标网站已有打开的页面时(用户开的或之前任务留下的),它往往已在正确的账号和位置,直接在它上面 `navigate` / 操作,不要另开新页从入口重新找账号。`open` 结果带 `sameSiteTabs` 就是在提醒这一点;新开的页面用不上时 `close`。需要用户亲手完成登录、验证码时,对那一页 `focus` 再告诉用户,不要自己尝试。
+
#### 三种浏览器目标(先认环境,再谈登录)
宿主用三种后端之一承接本工具。专用浏览器与侧边栏**顶层 action 集合一致**(都支持 snapshot / act 各 kind / requests / upload / dialog 等),差别在少数参数级能力与"页面呈现在哪里";用户浏览器(user-browser)与专用浏览器能力基本一致,`status.data.capabilities` 列出不支持项。判别依据是 `action: "status"` 返回结果里的 **`data.backend`**(工具结果的外层是 `{ ok, action, status, data }`,`status` 是数字状态码——**别去读 `status.backend`**,那是 undefined):
@@ -85,8 +89,7 @@
- **`data.backend === "external"` = 独立外置浏览器**:一个专属持久自动化浏览器窗口(profile 显示为 "Cindy")。`act:saveResource` 与语义 `query` 在此模式不可用,元素定位改用 snapshot 的 `ref` 或 `selector`。旧宿主可能省略该字段,不要把缺少模式信息当成已使用内置浏览器。
- 此模式的 `status.data.proxy` 报告当前启动路由:`direct`、`proxied`(带脱敏 `server`)或无法确认继承进程路由时的 `unknown`。需要指定代理时必须重新调用 `start` 并检查结果,不要从已有 tab 猜测出口。代理模式下只有 `proxyAllowedHostnames` 列出的公网域名可导航,内网地址不在其中。
- **`data.backend === "user-browser"` = 用户正在使用的浏览器(经 Cindy 扩展)**:页面活在用户自己的 Chrome / Edge 里,带着用户本人的登录态;Cindy 不启停它。`open` 的新页面开在 Cindy 自己的后台窗口里,不抢焦点;用户自己的标签页只在任务需要时操作,不要关闭、重排或切换它们。
- - **先 `tabs`,能复用就复用**:用户或之前任务已经开着目标网站的页面时,它往往已在正确的账号和位置,直接在它上面 `navigate` / 操作,不要另开新页从入口重新找账号;`open` 结果里的 `sameSiteTabs` 就是在提醒这一点。
- - **需要用户亲手操作时**(登录、验证码、确认弹窗):对那一页调用 `focus`。页面在 Cindy 窗口里时会把窗口带到用户面前(结果带 `broughtToFront`),然后告诉用户去完成;用户的窗口不会被抢焦点。截图、`evaluate`、`requests` / `responseBody`、上传、原生弹窗、`focus` / `close`、坐标点击、拖拽、`resize`、`pdf` 与配方都可用,但有这些差别:
+ - 需要用户亲手操作(登录、验证码)时对那一页 `focus`:页面在 Cindy 窗口里时会把窗口带到用户面前(结果带 `broughtToFront`);用户自己的窗口不会被抢焦点。截图、`evaluate`、`requests` / `responseBody`、上传、原生弹窗、`focus` / `close`、坐标点击、拖拽、`resize`、`pdf` 与配方都可用,但有这些差别:
- 有些动作借用 Chrome 的调试功能,需要用户允许一次:`pdf`、整页 / 元素截图、截取用户窗口里的后台标签页、读 Cookie 或抓网络请求的配方。`pdf` 与配方第一次需要时报 `DEBUGGER_NOT_ALLOWED`,扩展同时在用户的 Chrome 里打开说明页;请用户在那里点「开启」,确认后再重试,用户拒绝就不要重试。使用期间 Chrome 顶部会短暂显示调试提示。未允许时截图退回可视区域并在 `note` 里说明。`resize` 只作用于 Cindy 自己的窗口。
- 点击、坐标与拖拽是合成事件(非可信输入),个别要求可信输入的站点会忽略;页面 CSP 禁止脚本时 `evaluate` 报 `EVAL_BLOCKED_BY_PAGE`,改用 snapshot / extract。
- 控制台从第一次 `console` 调用开始捕获;`requests` 来自页面资源计时(无方法与请求头);`responseBody` 以页面身份重新请求该地址;`dialog` 预先应答页面下一个 alert / confirm / prompt,已经弹出的原生弹窗需要用户自己处理。
diff --git a/packages/lizi-mcps/src/browser/tools.ts b/packages/lizi-mcps/src/browser/tools.ts
index 78dab1cc74..132130b788 100644
--- a/packages/lizi-mcps/src/browser/tools.ts
+++ b/packages/lizi-mcps/src/browser/tools.ts
@@ -239,6 +239,36 @@ async function detectLoginWall(runtime: BrowserControlRuntime, profile: unknown,
}
}
+/** Other open tabs on the same site as a freshly opened page (any backend's tabs shape). */
+async function listSameSiteTabs(
+ runtime: BrowserControlRuntime,
+ profile: string | undefined,
+ opened: BrowserControlResult,
+): Promise<Array<{ targetId: string; url: string; title?: string }>> {
+ const data = opened.data as { url?: unknown; targetId?: unknown } | undefined;
+ if (typeof data?.url !== 'string') return [];
+ const host = new URL(data.url).hostname;
+ const listed = await Promise.race([
+ runtime.call({ action: 'tabs', ...(profile ? { profile } : {}) } as BrowserControlRequest),
+ new Promise<null>((resolve) => setTimeout(() => resolve(null), 3000)),
+ ]);
+ const tabs = (listed?.ok ? (listed.data as { tabs?: unknown } | undefined)?.tabs : undefined) ?? [];
+ const out: Array<{ targetId: string; url: string; title?: string }> = [];
+ for (const tab of Array.isArray(tabs) ? (tabs as Array<Record<string, unknown>>) : []) {
+ const url = typeof tab.url === 'string' ? tab.url : '';
+ const id = typeof tab.suggestedTargetId === 'string' ? tab.suggestedTargetId : typeof tab.targetId === 'string' ? tab.targetId : '';
+ if (!url || !id || tab.targetId === data.targetId) continue;
+ try {
+ if (new URL(url).hostname !== host) continue;
+ } catch {
+ continue;
+ }
+ out.push({ targetId: id, url, ...(typeof tab.title === 'string' ? { title: tab.title.slice(0, 120) } : {}) });
+ if (out.length >= 5) break;
+ }
+ return out;
+}
+
function readTargetId(result: { data?: unknown }): string | undefined {
const id = (result.data as { targetId?: unknown } | undefined)?.targetId;
return typeof id === 'string' && id ? id : undefined;
@@ -1177,6 +1207,22 @@ export function registerBrowserTools(registry: BrowserToolRegistry, deps: Browse
if (args.action === 'screenshot') {
return await screenshotToolResult(result, deps);
}
+ // Every backend: an explicit maxChars bounds the whole snapshot (some runtime snapshots exceed it).
+ if (args.action === 'snapshot' && result.ok && typeof args.maxChars === 'number') {
+ result = limitAfterSnapshot(result, args.maxChars);
+ }
+ // Every backend: a new page on a site that already has open tabs is usually a detour — those tabs are
+ // often already signed in to the right account and place. Point them out (best effort, never fails open).
+ if (args.action === 'open' && result.ok) {
+ const sameSiteTabs = await listSameSiteTabs(runtime, args.profile, result).catch(() => []);
+ if (sameSiteTabs.length) {
+ result = {
+ ...result,
+ sameSiteTabs,
+ sameSiteHint: '这个网站已有打开的页面(往往已在正确的账号和位置):优先在它们上面继续,不要从入口重新找;新开的页面不需要时可以 close。',
+ } as typeof result;
+ }
+ }
if (args.action === 'snapshot' && result.ok) {
const loginWall = await detectLoginWall(runtime, args.profile, readTargetId(result) ?? args.targetId);
// First key, so it survives truncation of a large snapshot.