🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
fix: type script browser cdp connection (#3009)
Puzhen Zhanga5e15daa0181
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.pyindex e367c3320..870d83262 100644--- a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py@@ -674,12 +674,29 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):"total_tabs": 0,}- async def browser_type(self, *, ref: str, text: str) -> Dict[str, Any]:- r"""Types text into an input element on the page.+ async def browser_type(+ self,+ *,+ ref: Optional[str] = None,+ text: Optional[str] = None,+ inputs: Optional[List[Dict[str, str]]] = None,+ ) -> Dict[str, Any]:+ r"""Types text into one or more input elements on the page.++ This method supports two modes:+ 1. Single input mode (backward compatible): Provide 'ref' and 'text'+ 2. Multiple inputs mode: Provide 'inputs' as a list of dictionaries+ with 'ref' and 'text' keysArgs:- ref (str): The `ref` ID of the input element, from a snapshot.- text (str): The text to type into the element.+ ref (Optional[str]): The `ref` ID of the input element, from a+ snapshot. Required when using single input mode.+ text (Optional[str]): The text to type into the element. Required+ when using single input mode.+ inputs (Optional[List[Dict[str, str]]]): List of dictionaries,+ each containing 'ref' and 'text' keys for typing into multiple+ elements. Example: [{'ref': '1', 'text': 'username'},+ {'ref': '2', 'text': 'password'}]Returns:Dict[str, Any]: A dictionary with the result of the action:@@ -689,10 +706,23 @@ class HybridBrowserToolkit(BaseToolkit, RegisteredAgentToolkit):- "tabs" (List[Dict]): Information about all open tabs.- "current_tab" (int): Index of the active tab.- "total_tabs" (int): Total number of open tabs.+ - "details" (Dict[str, Any]): When using multiple inputs,+ contains success/error status for each ref."""try:ws_wrapper = await self._get_ws_wrapper()- result = await ws_wrapper.type(ref, text)++ # Handle single input mode (backward compatibility)+ if ref is not None and text is not None:+ result = await ws_wrapper.type(ref, text)+ # Handle multiple inputs mode+ elif inputs is not None:+ result = await ws_wrapper.type_multiple(inputs)+ else:+ raise ValueError(+ "Either provide 'ref' and 'text' for single input, "+ "or 'inputs' for multiple inputs"+ )# Add tab informationtab_info = await ws_wrapper.get_tab_info()diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.tsindex 1496b312b..62d720232 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts@@ -72,21 +72,19 @@ export class HybridBrowserSession {this.context = await this.browser.newContext(contextOptions);}- // Handle existing pagesconst pages = this.context.pages();if (pages.length > 0) {- // Map existing pages - for CDP, only use pages with about:blank URL+ // Map existing pages - for CDP, find ONE available blank pagelet availablePageFound = false;for (const page of pages) {const pageUrl = page.url();- // In CDP mode, only consider pages with about:blank as available- if (pageUrl === 'about:blank') {+ if (this.isBlankPageUrl(pageUrl)) {const tabId = this.generateTabId();this.registerNewPage(tabId, page);- if (!this.currentTabId) {- this.currentTabId = tabId;- availablePageFound = true;- }+ this.currentTabId = tabId;+ availablePageFound = true;+ console.log(`[CDP] Registered blank page as initial tab: ${tabId}, URL: ${pageUrl}`);+ break; // Only register ONE page initially}}@@ -157,8 +155,45 @@ export class HybridBrowserSession {return `${browserConfig.tabIdPrefix}${String(++this.tabCounter).padStart(browserConfig.tabCounterPadding, '0')}`;}+ private isBlankPageUrl(url: string): boolean {+ // Unified blank page detection logic used across the codebase+ const browserConfig = this.configLoader.getBrowserConfig();+ return (+ // Standard about:blank variations (prefix match for query params)+ url === 'about:blank' ||+ url.startsWith('about:blank?') ||+ // Configured blank page URLs (exact match for compatibility)+ browserConfig.blankPageUrls.includes(url) ||+ // Empty URL+ url === '' ||+ // Data URLs (often used for blank pages)+ url.startsWith(browserConfig.dataUrlPrefix || 'data:')+ );+ }+async getCurrentPage(): Promise<Page> {if (!this.currentTabId || !this.pages.has(this.currentTabId)) {+ // In CDP mode, try to create a new page if none exists+ const browserConfig = this.configLoader.getBrowserConfig();+ if (browserConfig.connectOverCdp && this.context) {+ console.log('[CDP] No active page found, attempting to create new page...');+ try {+ const newPage = await this.context.newPage();+ const newTabId = this.generateTabId();+ this.registerNewPage(newTabId, newPage);+ this.currentTabId = newTabId;++ // Set page timeouts+ newPage.setDefaultNavigationTimeout(browserConfig.navigationTimeout);+ newPage.setDefaultTimeout(browserConfig.navigationTimeout);++ console.log(`[CDP] Created new page with tab ID: ${newTabId}`);+ return newPage;+ } catch (error) {+ console.error('[CDP] Failed to create new page:', error);+ throw new Error('No active page available and failed to create new page in CDP mode');+ }+ }throw new Error('No active page available');}return this.pages.get(this.currentTabId)!;@@ -416,25 +451,67 @@ export class HybridBrowserSession {/*** Simplified type implementation using Playwright's aria-ref selector+ * Supports both single and multiple input operations*/- private async performType(page: Page, ref: string, text: string): Promise<{ success: boolean; error?: string }> {+ private async performType(page: Page, ref: string | undefined, text: string | undefined, inputs?: Array<{ ref: string; text: string }>): Promise<{ success: boolean; error?: string; details?: Record<string, any> }> {try {// Ensure we have the latest snapshotawait (page as any)._snapshotForAI();- // Use Playwright's aria-ref selector- const selector = `aria-ref=${ref}`;- const element = await page.locator(selector).first();-- const exists = await element.count() > 0;- if (!exists) {- return { success: false, error: `Element with ref ${ref} not found` };+ // Handle multiple inputs if provided+ if (inputs && inputs.length > 0) {+ const results: Record<string, { success: boolean; error?: string }> = {};++ for (const input of inputs) {+ const selector = `aria-ref=${input.ref}`;+ const element = await page.locator(selector).first();++ const exists = await element.count() > 0;+ if (!exists) {+ results[input.ref] = { success: false, error: `Element with ref ${input.ref} not found` };+ continue;+ }++ try {+ // Type text using Playwright's built-in fill method+ await element.fill(input.text);+ results[input.ref] = { success: true };+ } catch (error) {+ results[input.ref] = { success: false, error: `Type failed: ${error}` };+ }+ }++ // Check if all inputs were successful+ const allSuccess = Object.values(results).every(r => r.success);+ const errors = Object.entries(results)+ .filter(([_, r]) => !r.success)+ .map(([ref, r]) => `${ref}: ${r.error}`)+ .join('; ');++ return {+ success: allSuccess,+ error: allSuccess ? undefined : `Some inputs failed: ${errors}`,+ details: results+ };}- // Type text using Playwright's built-in fill method- await element.fill(text);+ // Handle single input (backward compatibility)+ if (ref && text !== undefined) {+ const selector = `aria-ref=${ref}`;+ const element = await page.locator(selector).first();++ const exists = await element.count() > 0;+ if (!exists) {+ return { success: false, error: `Element with ref ${ref} not found` };+ }++ // Type text using Playwright's built-in fill method+ await element.fill(text);++ return { success: true };+ }- return { success: true };+ return { success: false, error: 'No valid input provided' };} catch (error) {return { success: false, error: `Type failed: ${error}` };}@@ -572,6 +649,8 @@ export class HybridBrowserSession {// No need to pre-fetch snapshot - each action method handles thislet newTabId: string | undefined;+ let customMessage: string | undefined;+ let actionDetails: Record<string, any> | undefined;switch (action.type) {case 'click': {@@ -596,12 +675,20 @@ export class HybridBrowserSession {elementSearchTime = Date.now() - elementSearchStart;const typeStart = Date.now();- const typeResult = await this.performType(page, action.ref, action.text);+ const typeResult = await this.performType(page, action.ref, action.text, action.inputs);if (!typeResult.success) {throw new Error(`Type failed: ${typeResult.error}`);}+ // Set custom message and details if multiple inputs were used+ if (typeResult.details) {+ const successCount = Object.values(typeResult.details).filter((r: any) => r.success).length;+ const totalCount = Object.keys(typeResult.details).length;+ customMessage = `Typed text into ${successCount}/${totalCount} elements`;+ actionDetails = typeResult.details;+ }+actionExecutionTime = Date.now() - typeStart;break;}@@ -689,7 +776,7 @@ export class HybridBrowserSession {return {success: true,- message: `Action ${action.type} executed successfully`,+ message: customMessage || `Action ${action.type} executed successfully`,timing: {total_time_ms: totalTime,element_search_time_ms: elementSearchTime,@@ -699,6 +786,7 @@ export class HybridBrowserSession {network_idle_time_ms: stabilityResult.networkIdleTime,},...(newTabId && { newTabId }), // Include new tab ID if present+ ...(actionDetails && { details: actionDetails }), // Include action details if present};} catch (error) {const totalTime = Date.now() - startTime;@@ -740,16 +828,23 @@ export class HybridBrowserSession {try {// Get current page to check if it's blank- const currentPage = await this.getCurrentPage();- const currentUrl = currentPage.url();+ let currentPage: Page;+ let currentUrl: string;++ try {+ currentPage = await this.getCurrentPage();+ currentUrl = currentPage.url();+ } catch (error: any) {+ // If no active page is available, getCurrentPage() will create one in CDP mode+ console.log('[visitPage] Failed to get current page:', error);+ throw new Error(`No active page available: ${error?.message || error}`);+ }// Check if current page is blank or if this is the first navigationconst browserConfig = this.configLoader.getBrowserConfig();- const isBlankPage = (- browserConfig.blankPageUrls.includes(currentUrl) ||- currentUrl === browserConfig.defaultStartUrl ||- currentUrl.startsWith(browserConfig.dataUrlPrefix) // data URLs are often used for blank pages- );++ // Use unified blank page detection+ const isBlankPage = this.isBlankPageUrl(currentUrl) || currentUrl === browserConfig.defaultStartUrl;const shouldUseCurrentTab = isBlankPage || !this.hasNavigatedBefore;@@ -804,7 +899,7 @@ export class HybridBrowserSession {const pageUrl = page.url();// Check if this page is not already tracked and is blankconst isTracked = Array.from(this.pages.values()).includes(page);- if (!isTracked && pageUrl === 'about:blank') {+ if (!isTracked && this.isBlankPageUrl(pageUrl)) {newPage = page;newTabId = this.generateTabId();this.registerNewPage(newTabId, newPage);diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.tsindex 428863a77..f92d120d1 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts@@ -117,7 +117,7 @@ function getDefaultBrowserConfig(): BrowserConfig {consoleLogLimit: 1000,scrollPositionScale: 0.1,navigationDelay: 100,- blankPageUrls: ['about:blank', ''],+ blankPageUrls: [],dataUrlPrefix: 'data:',domContentLoadedState: 'domcontentloaded',networkIdleState: 'networkidle',diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.tsindex 38587733c..4c627b359 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts@@ -69,35 +69,52 @@ export class HybridBrowserToolkit {}async visitPage(url: string): Promise<any> {- const result = await this.session.visitPage(url);-- // Format response for Python layer compatibility- const response: any = {- result: result.message,- snapshot: '',- };-- if (result.success) {- const snapshotStart = Date.now();- response.snapshot = await this.getPageSnapshot(this.viewportLimit);- const snapshotTime = Date.now() - snapshotStart;+ try {+ // Ensure browser is initialized before visiting page+ await this.session.ensureBrowser();++ const result = await this.session.visitPage(url);++ // Format response for Python layer compatibility+ const response: any = {+ result: result.message,+ snapshot: '',+ };++ if (result.success) {+ const snapshotStart = Date.now();+ response.snapshot = await this.getPageSnapshot(this.viewportLimit);+ const snapshotTime = Date.now() - snapshotStart;++ if (result.timing) {+ result.timing.snapshot_time_ms = snapshotTime;+ }+ }+ // Include timing if availableif (result.timing) {- result.timing.snapshot_time_ms = snapshotTime;+ response.timing = result.timing;}++ // Include newTabId if present+ if (result.newTabId) {+ response.newTabId = result.newTabId;+ }++ return response;+ } catch (error) {+ console.error('[visitPage] Error:', error);+ return {+ result: `Navigation to ${url} failed: ${error}`,+ snapshot: '',+ timing: {+ total_time_ms: 0,+ navigation_time_ms: 0,+ dom_content_loaded_time_ms: 0,+ network_idle_time_ms: 0,+ }+ };}-- // Include timing if available- if (result.timing) {- response.timing = result.timing;- }-- // Include newTabId if present- if (result.newTabId) {- response.newTabId = result.newTabId;- }-- return response;}async getPageSnapshot(viewportLimit: boolean = false): Promise<string> {@@ -179,7 +196,40 @@ export class HybridBrowserToolkit {// Use sharp for image processingconst sharp = require('sharp');const page = await this.session.getCurrentPage();- const viewport = page.viewportSize() || { width: 1280, height: 720 };+ let viewport = page.viewportSize();++ // In CDP mode, viewportSize might be null, get it from window dimensions+ if (!viewport) {+ const windowSize = await page.evaluate(() => ({+ width: window.innerWidth,+ height: window.innerHeight+ }));+ viewport = windowSize;+ }++ // Get device pixel ratio to handle high DPI screens+ const dpr = await page.evaluate(() => window.devicePixelRatio) || 1;++ // Get actual screenshot dimensions+ const metadata = await sharp(screenshotBuffer).metadata();+ const screenshotWidth = metadata.width || viewport.width;+ const screenshotHeight = metadata.height || viewport.height;++ // Calculate scaling factor between CSS pixels and screenshot pixels+ const scaleX = screenshotWidth / viewport.width;+ const scaleY = screenshotHeight / viewport.height;++ // Debug logging for CDP mode+ if (process.env.HYBRID_BROWSER_DEBUG === '1') {+ console.log('[CDP Debug] Viewport size:', viewport);+ console.log('[CDP Debug] Device pixel ratio:', dpr);+ console.log('[CDP Debug] Screenshot dimensions:', { width: screenshotWidth, height: screenshotHeight });+ console.log('[CDP Debug] Scale factors:', { scaleX, scaleY });+ console.log('[CDP Debug] Elements with coordinates:', elementsWithCoords.length);+ elementsWithCoords.slice(0, 3).forEach(([ref, element]) => {+ console.log(`[CDP Debug] Element ${ref}:`, element.coordinates);+ });+ }// Filter elements visible in viewportconst visibleElements = elementsWithCoords.filter(([ref, element]) => {@@ -198,18 +248,19 @@ export class HybridBrowserToolkit {const coords = element.coordinates!;const isClickable = clickableElements.has(ref);- // Use original coordinates for elements within viewport- // Clamp only to prevent marks from extending beyond screenshot bounds- const x = Math.max(0, coords.x);- const y = Math.max(0, coords.y);- const maxWidth = viewport.width - x;- const maxHeight = viewport.height - y;- const width = Math.min(coords.width, maxWidth);- const height = Math.min(coords.height, maxHeight);+ // Scale coordinates from CSS pixels to screenshot pixels+ const x = Math.max(0, coords.x * scaleX);+ const y = Math.max(0, coords.y * scaleY);+ const width = coords.width * scaleX;+ const height = coords.height * scaleY;++ // Clamp to screenshot bounds+ const clampedWidth = Math.min(width, screenshotWidth - x);+ const clampedHeight = Math.min(height, screenshotHeight - y);// Position text to be visible even if element is partially cut off- const textX = Math.max(2, Math.min(x + 2, viewport.width - 40));- const textY = Math.max(14, Math.min(y + 14, viewport.height - 4));+ const textX = Math.max(2, Math.min(x + 2, screenshotWidth - 40));+ const textY = Math.max(14, Math.min(y + 14, screenshotHeight - 4));// Different colors for clickable vs non-clickable elementsconst colors = isClickable ? {@@ -223,7 +274,7 @@ export class HybridBrowserToolkit {};return `- <rect x="${x}" y="${y}" width="${width}" height="${height}"+ <rect x="${x}" y="${y}" width="${clampedWidth}" height="${clampedHeight}"fill="${colors.fill}" stroke="${colors.stroke}" stroke-width="2" rx="2"/><text x="${textX}" y="${textY}" font-family="Arial, sans-serif"font-size="12" fill="${colors.textFill}" font-weight="bold">${ref}</text>@@ -231,7 +282,7 @@ export class HybridBrowserToolkit {}).join('');const svgOverlay = `- <svg width="${viewport.width}" height="${viewport.height}" xmlns="http://www.w3.org/2000/svg">+ <svg width="${screenshotWidth}" height="${screenshotHeight}" xmlns="http://www.w3.org/2000/svg">${marks}</svg>`;@@ -363,8 +414,20 @@ export class HybridBrowserToolkit {return this.executeActionWithSnapshot(action);}- async type(ref: string, text: string): Promise<any> {- const action: BrowserAction = { type: 'type', ref, text };+ async type(refOrInputs: string | Array<{ ref: string; text: string }>, text?: string): Promise<any> {+ let action: BrowserAction;++ if (typeof refOrInputs === 'string') {+ // Single input mode (backward compatibility)+ if (text === undefined) {+ throw new Error('Text parameter is required when ref is a string');+ }+ action = { type: 'type', ref: refOrInputs, text };+ } else {+ // Multiple inputs mode+ action = { type: 'type', inputs: refOrInputs };+ }+return this.executeActionWithSnapshot(action);}diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.tsindex 5ec940b9c..180173e48 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts@@ -81,8 +81,9 @@ export interface ClickAction {export interface TypeAction {type: 'type';- ref: string;- text: string;+ ref?: string; // Optional for backward compatibility+ text?: string; // Optional for backward compatibility+ inputs?: Array<{ ref: string; text: string }>; // New field for multiple inputs}export interface SelectAction {diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.jsindex a4541a3c9..327338f8e 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js+++ b/camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js@@ -160,7 +160,14 @@ class WebSocketBrowserServer {case 'type':if (!this.toolkit) throw new Error('Toolkit not initialized');- return await this.toolkit.type(params.ref, params.text);+ // Handle both single input and multiple inputs+ if (params.inputs) {+ // Multiple inputs mode - pass inputs array directly+ return await this.toolkit.type(params.inputs);+ } else {+ // Single input mode - pass ref and text+ return await this.toolkit.type(params.ref, params.text);+ }case 'select':if (!this.toolkit) throw new Error('Toolkit not initialized');diff --git a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.pyindex 1d4e1e9aa..266b09cef 100644--- a/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py+++ b/camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py@@ -396,6 +396,9 @@ class WebSocketBrowserWrapper:"""Send a command to the WebSocket server and get response."""await self._ensure_connection()+ # Process params to ensure refs have 'e' prefix+ params = self._process_refs_in_params(params)+message_id = str(uuid.uuid4())message = {'id': message_id, 'command': command, 'params': params}@@ -503,6 +506,50 @@ class WebSocketBrowserWrapper:return ToolResult(text=response['text'], images=response['images'])+ def _ensure_ref_prefix(self, ref: str) -> str:+ """Ensure ref has 'e' prefix."""+ if ref and not ref.startswith('e'):+ return f'e{ref}'+ return ref++ def _process_refs_in_params(+ self, params: Dict[str, Any]+ ) -> Dict[str, Any]:+ """Process parameters to ensure all refs have 'e' prefix."""+ if not params:+ return params++ # Create a copy to avoid modifying the original+ processed = params.copy()++ # Handle direct ref parameters+ if 'ref' in processed:+ processed['ref'] = self._ensure_ref_prefix(processed['ref'])++ # Handle from_ref and to_ref for drag operations+ if 'from_ref' in processed:+ processed['from_ref'] = self._ensure_ref_prefix(+ processed['from_ref']+ )+ if 'to_ref' in processed:+ processed['to_ref'] = self._ensure_ref_prefix(processed['to_ref'])++ # Handle inputs array for type_multiple+ if 'inputs' in processed and isinstance(processed['inputs'], list):+ processed_inputs = []+ for input_item in processed['inputs']:+ if isinstance(input_item, dict) and 'ref' in input_item:+ processed_input = input_item.copy()+ processed_input['ref'] = self._ensure_ref_prefix(+ input_item['ref']+ )+ processed_inputs.append(processed_input)+ else:+ processed_inputs.append(input_item)+ processed['inputs'] = processed_inputs++ return processed+@action_loggerasync def click(self, ref: str) -> Dict[str, Any]:"""Click an element."""@@ -515,6 +562,14 @@ class WebSocketBrowserWrapper:response = await self._send_command('type', {'ref': ref, 'text': text})return response+ @action_logger+ async def type_multiple(+ self, inputs: List[Dict[str, str]]+ ) -> Dict[str, Any]:+ """Type text into multiple elements."""+ response = await self._send_command('type', {'inputs': inputs})+ return response+@action_loggerasync def select(self, ref: str, value: str) -> Dict[str, Any]:"""Select an option."""