Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

Merge upstream/main into browser screenshot fix

elfcvb96cee729ee5
diff --cc apps/desktop/src/main/mcp-integrations/browser.ts
index 02f259edb2,d33a588fe8..694ac625e7
--- a/apps/desktop/src/main/mcp-integrations/browser.ts
+++ b/apps/desktop/src/main/mcp-integrations/browser.ts
@@@ -11,14 -11,15 +11,17 @@@ import nodePath from 'node:path'
import { app, ipcMain } from 'electron';
import {
createBrowserControlRuntime,
+ setBrowserControlRuntimeConfig,
+ type BrowserProxyRoute,
+ type BrowserControlRuntime,
} from '@cindy/browser-control-runtime';
+
+import type { BrowserMcpDeps } from '@cindy/mcps';
+
import { createLogger } from '../logger.js';
import { extractBrowserAvailability, type BrowserAvailability } from './browser-availability.js';
- import { loadUserBrowserRecipes } from '../browser-recipes/loader.js';
- import { writeUserRecipe } from '../browser-recipes/writer.js';
- import { stopRuntimeForQuitIfUsed, trackBrowserRuntimeUsage } from './browser-dispose.js';
+ import { loadUserBrowserRecipes, type UserRecipesResult } from '../browser-recipes/loader.js';
+ import { writeUserRecipe, type WriteUserRecipeResult } from '../browser-recipes/writer.js';
import {
BrowserBackendController,
BrowserBackendHealthService,
@@@ -38,11 -42,33 +44,35 @@@ import
setActiveRsbSessionId,
} from '../rsb-browser-bridge/active-session.js';
import { requireObject, optionalNullableString } from '../utils/ipcValidate.js';
- import { buildManagedConfig, MANAGED_PROFILE } from './browser-managed-config.js';
+ import {
+ buildManagedConfig,
+ MANAGED_CDP_PORT,
+ MANAGED_PROFILE,
+ managedBrowserGuardIdentity,
+ type ManagedBrowserConfigOptions,
+ } from './browser-managed-config.js';
+ import {
+ assertManagedBrowserStopped,
+ cleanupCopiedLoginsThen,
+ RealProfileError,
+ createBrowserProfileLifecycleQueue,
+ managedConfigPatchBeforeStop,
+ FOREIGN_AGENT_BROWSER_ERROR,
+ probeOsSourceProfileReadAccess,
+ readCopiedLoginsCdpPort,
+ wrapRuntimeWithRealProfile,
+ wrapRuntimeWithProfileLifecycleQueue,
+ } from './browser-real-profile/index.js';
import { assertTrustedAppRendererEvent } from '../security/trustedAppRenderer.js';
import { createBrowserBackendIpcHandlers } from './browser-backend/settings-ipc.js';
+import { resolveBrowserScreenshot } from './browserScreenshotResolver.js';
+import { compressInlineImage } from './inlineImageCompressor.js';
+ import { raiseAgentBrowserWindow } from './raise-agent-browser-window.js';
+ import { hasSecretPages, isSecretPage } from '../input-handoff/secretTaint.js';
+ import {
+ BrowserOpenForLoginError,
+ browserOpenForLoginErrorCodeFromData,
+ } from '../../shared/browserBackend.js';
export { extractBrowserAvailability, type BrowserAvailability } from './browser-availability.js';
@@@ -251,8 -447,21 +451,23 @@@ export function setBrowserUseRealProfil
* BrowserControlRuntime contract so the desktop host does not depend on an
* upstream product API or product-facing name.
*/
- export function getBrowserMcpDeps(): BrowserMcpDeps {
+ export function getBrowserMcpDeps(): {
+ getRuntime(): BrowserControlRuntime;
++ resolveScreenshot: NonNullable<BrowserMcpDeps['resolveScreenshot']>;
++ compressInlineImage: NonNullable<BrowserMcpDeps['compressInlineImage']>;
+ setBackend(kind: BackendKind): Promise<BackendKind>;
+ supportsResourceDownloads(): boolean;
+ supportsSemanticQueries(): boolean;
+ logger: typeof logger;
+ getUserRecipes(): Promise<UserRecipesResult>;
+ saveUserRecipe(input: Parameters<typeof writeUserRecipe>[0]): Promise<WriteUserRecipeResult>;
+ pageReadsRestricted(target: { profile?: string; targetId?: string }): Promise<boolean>;
+ } {
return {
+ setBackend: async (kind) => {
+ await setActiveBrowserBackendKind(kind);
+ return backendController.getCurrentBackendKind();
+ },
// L2 user-recipe layer (userData/browser-recipes); merged over the bundled
// L1 catalog inside the MCP. Empty/missing dir → bundled-only (== before).
getUserRecipes: () => loadUserBrowserRecipes(),
@@@ -262,13 -471,19 +477,24 @@@
// the backend split. Swapping the active backend (Phase 5) is invisible from
// @cindy/mcps' perspective.
getRuntime: () => backendController,
+ resolveScreenshot: (result) =>
+ resolveBrowserScreenshot(result, {
+ runtimeDir: process.env.XDT_BROWSER_RUNTIME_DIR,
+ }),
+ compressInlineImage,
supportsResourceDownloads: () => backendController.kind === 'rsb-webview',
supportsSemanticQueries: () => backendController.kind === 'rsb-webview',
+ // 代输入: pause page-state reads on a tab still showing a page where the owner typed a secret.
+ pageReadsRestricted: async ({ profile, targetId }) => {
+ if (!hasSecretPages()) return false;
+ const listed = await backendController.call({ action: 'tabs', ...(profile ? { profile } : {}) });
+ if (!listed.ok) return true;
+ const tabs = (listed.data as { tabs?: Array<{ targetId?: unknown; url?: unknown }> } | undefined)?.tabs ?? [];
+ // Without a target the runtime acts on its last tab; check every tab conservatively.
+ return tabs
+ .filter((tab) => !targetId || tab.targetId === targetId)
+ .some((tab) => typeof tab.url === 'string' && isSecretPage(tab.url));
+ },
logger,
};
}
diff --cc packages/lizi-mcps/src/browser/tools.ts
index 74ee1d974d,b568711035..947308d87d
--- a/packages/lizi-mcps/src/browser/tools.ts
+++ b/packages/lizi-mcps/src/browser/tools.ts
@@@ -1,7 -1,7 +1,8 @@@
import {
isPublicHttpResourceUrl,
+ parseBrowserProxyServer,
type BrowserControlRequest,
+ type BrowserControlResult,
type BrowserControlRuntime,
} from '@cindy/browser-control-runtime';
import { z } from 'zod';
@@@ -777,9 -738,11 +901,14 @@@ export function registerBrowserTools(re
} else {
result = await runtime.call(toRuntimeRequest(args));
}
+ if (args.action === 'screenshot') {
+ return await screenshotToolResult(result, deps);
+ }
+ if (args.action === 'snapshot' && result.ok) {
+ const loginWall = await detectLoginWall(runtime, args.profile, readTargetId(result) ?? args.targetId);
+ // First key, so it survives truncation of a large snapshot.
+ if (loginWall) result = { loginWall, ...result } as typeof result;
+ }
// For extract, the runtime call may succeed (evaluate ran) while the
// in-page extract reports ok:false (invalid selector); surface that as an
// error so the agent sees + acts on the embedded hint.
diff --cc packages/lizi-mcps/src/types.ts
index 6be6cd4ed0,5e2277799f..9fe2b9e23b
--- a/packages/lizi-mcps/src/types.ts
+++ b/packages/lizi-mcps/src/types.ts
@@@ -507,16 -634,8 +642,18 @@@ export interface ResolvedBrowserScreens
/** Browser automation MCP host deps. Core browser execution is injected by host. */
export interface BrowserMcpDeps {
getRuntime(): BrowserControlRuntime;
+ /**
+ * Resolve a successful screenshot runtime result into verified image bytes.
+ * The host owns file-system access so this package never receives a generic
+ * arbitrary-path reader; inline-only hosts may validate and decode in memory.
+ */
+ resolveScreenshot?(
+ result: BrowserControlResult,
+ ): Promise<ResolvedBrowserScreenshot>;
+ /** Shrink the LLM-facing copy to the shared stream-safe byte budget. */
+ compressInlineImage?: CompressInlineImageFn;
+ /** Switch the host-wide, persisted automation target; returns the actual mode. */
+ setBackend?(backend: 'external' | 'rsb-webview'): Promise<'external' | 'rsb-webview'>;
/** Whether the active backend accepts managed resource downloads. */
supportsResourceDownloads?(): boolean;
/** Whether the active backend accepts semantic element queries. */