🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
fix: return updated combobox ref in click diffSnapshot (#3491)
Ishneet Sukhvinder Singh6bc92cdbb3b6
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.tsindex d59c8db4f..8827628a0 100644--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts@@ -579,8 +579,24 @@ export class HybridBrowserSession {const shouldCheckDiff = isCombobox || isTextbox;let snapshotBefore: string | null = null;+ let comboboxAriaLabel: string | null = null;if (shouldCheckDiff) {snapshotBefore = await (page as any)._snapshotForAI();+ // Capture aria-label for combobox to find it again after click (ref may change)+ if (isCombobox) {+ comboboxAriaLabel = await element.getAttribute('aria-label');+ if (!comboboxAriaLabel) {+ // Try to get accessible name from aria-labelledby or inner text+ comboboxAriaLabel = await element.evaluate(el => {+ const labelledBy = el.getAttribute('aria-labelledby');+ if (labelledBy) {+ const labelEl = document.getElementById(labelledBy);+ if (labelEl) return labelEl.textContent?.trim() || null;+ }+ return el.textContent?.trim() || null;+ });+ }+ }}// Check element properties@@ -658,7 +674,19 @@ export class HybridBrowserSession {if (shouldCheckDiff && snapshotBefore) {await page.waitForTimeout(300);const snapshotAfter = await (page as any)._snapshotForAI();- const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotAfter, ['option', 'menuitem']);+ let diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotAfter, ['option', 'menuitem']);++ // For combobox, find the new ref based on aria-label and prepend to diffSnapshot+ if (isCombobox && comboboxAriaLabel) {+ const newComboboxRef = this.findComboboxRefByAriaLabel(snapshotAfter, comboboxAriaLabel);+ if (newComboboxRef) {+ // Find the full line for this combobox in the snapshot+ const comboboxLine = this.findSnapshotLineByRef(snapshotAfter, newComboboxRef);+ if (comboboxLine) {+ diffSnapshot = comboboxLine + (diffSnapshot ? '\n' + diffSnapshot : '');+ }+ }+ }if (diffSnapshot && diffSnapshot.trim() !== '') {return { success: true, method: 'playwright-aria-ref', diffSnapshot };@@ -709,6 +737,57 @@ export class HybridBrowserSession {}}+ /**+ * Find a combobox ref in the snapshot by its aria-label or expanded state+ */+ private findComboboxRefByAriaLabel(snapshot: string, ariaLabel: string): string | null {+ const lines = snapshot.split('\n');+ // Escape special regex characters in ariaLabel+ const escapedLabel = ariaLabel.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');++ // First, try to find by aria-label+ for (const line of lines) {+ const isCombobox = /\bcombobox\b/i.test(line);+ const hasLabel = new RegExp(`["']${escapedLabel}["']`, 'i').test(line) ||+ line.includes(ariaLabel);++ if (isCombobox && hasLabel) {+ const refMatch = line.match(/\[ref=([^\]]+)\]/);+ if (refMatch) {+ return refMatch[1];+ }+ }+ }++ // Fallback: find the expanded combobox (since we just clicked it)+ for (const line of lines) {+ const isCombobox = /\bcombobox\b/i.test(line);+ const isExpanded = /\[expanded\]/i.test(line);++ if (isCombobox && isExpanded) {+ const refMatch = line.match(/\[ref=([^\]]+)\]/);+ if (refMatch) {+ return refMatch[1];+ }+ }+ }++ return null;+ }++ /**+ * Find the full snapshot line for a given ref+ */+ private findSnapshotLineByRef(snapshot: string, ref: string): string | null {+ const lines = snapshot.split('\n');+ for (const line of lines) {+ if (line.includes(`[ref=${ref}]`)) {+ return line.trim();+ }+ }+ return null;+ }+/*** Simplified type implementation using Playwright's aria-ref selector* Supports both single and multiple input operationsdiff --git a/docs/mintlify/docs.json b/docs/mintlify/docs.jsonindex e09e23ca5..374c34f0b 100644--- a/docs/mintlify/docs.json+++ b/docs/mintlify/docs.json@@ -766,4 +766,4 @@"measurementId": "G-XV5VXFS2RG"}}-}\ No newline at end of file+}diff --git a/docs/mintlify/reference/camel.agents.chat_agent.mdx b/docs/mintlify/reference/camel.agents.chat_agent.mdxindex 5805742fd..3fecd4dfe 100644--- a/docs/mintlify/reference/camel.agents.chat_agent.mdx+++ b/docs/mintlify/reference/camel.agents.chat_agent.mdx@@ -1453,7 +1453,7 @@ intermediate responses as they are generated.- **input_message** (Union[BaseMessage, str]): The input message for the agent.- **response_format** (Optional[Type[BaseModel]], optional): A Pydantic model defining the expected structure of the response.-- **Yields**:+- **Yields**:- **ChatAgentResponse**: Intermediate responses containing partial content, tool calls, and other information as they become available.<a id="camel.agents.chat_agent.ChatAgent._get_token_count"></a>diff --git a/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx b/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdxindex f3ab33002..5a0d0c047 100644--- a/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx+++ b/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx@@ -14,7 +14,7 @@ Firecrawl allows you to turn entire websites into LLM-ready markdown.- **api_key** (Optional[str]): API key for authenticating with the Firecrawl API.- **api_url** (Optional[str]): Base URL for the Firecrawl API.-- **References**:+- **References**:- **https**: //docs.firecrawl.dev/introduction<a id="camel.loaders.firecrawl_reader.Firecrawl.__init__"></a>diff --git a/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx b/docs/mintlify/reference/camel.loaders.jina_url_reader.mdxindex bcb5868eb..4a726e233 100644--- a/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx+++ b/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx@@ -18,7 +18,7 @@ replace the UnstructuredIO URL Reader in the pipeline.- **return_format** (ReturnFormat, optional): The level of detail of the returned content, which is optimized for LLMs. For now screenshots are not supported. Defaults to ReturnFormat.DEFAULT.- **json_response** (bool, optional): Whether to return the response in JSON format. Defaults to False.- **timeout** (int, optional): The maximum time in seconds to wait for the page to be rendered. Defaults to 30. **kwargs (Any): Additional keyword arguments, including proxies, cookies, etc. It should align with the HTTP Header field and value pairs listed in the reference.-- **References**:+- **References**:- **https**: //jina.ai/reader<a id="camel.loaders.jina_url_reader.JinaURLReader.__init__"></a>diff --git a/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx b/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdxindex 5834f25db..39bbb7ce2 100644--- a/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx+++ b/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx@@ -14,7 +14,7 @@ searching.**Parameters:**- **api_key** (Optional[str]): API key for authenticating with the ScrapeGraphAI API.-- **References**:+- **References**:- **https**: //scrapegraph.ai/<a id="camel.loaders.scrapegraph_reader.ScrapeGraphAI.__init__"></a>diff --git a/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx b/docs/mintlify/reference/camel.models.aws_bedrock_model.mdxindex 66c018a8f..c03b3e431 100644--- a/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx+++ b/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx@@ -20,7 +20,7 @@ AWS Bedrock API in a unified OpenAICompatibleModel interface.- **token_counter** (BaseTokenCounter, optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter( ModelType.GPT_4O_MINI)` will be used. (default: :obj:`None`)- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.-- **References**:+- **References**:- **https**: //docs.aws.amazon.com/bedrock/latest/APIReference/welcome.html<a id="camel.models.aws_bedrock_model.AWSBedrockModel.__init__"></a>diff --git a/docs/mintlify/reference/camel.models.azure_openai_model.mdx b/docs/mintlify/reference/camel.models.azure_openai_model.mdxindex cb881f1b6..33c48bc45 100644--- a/docs/mintlify/reference/camel.models.azure_openai_model.mdx+++ b/docs/mintlify/reference/camel.models.azure_openai_model.mdx@@ -25,7 +25,7 @@ Azure OpenAI API in a unified BaseModelBackend interface.- **client** (Optional[Any], optional): A custom synchronous AzureOpenAI client instance. If provided, this client will be used instead of creating a new one. Useful for RL frameworks like AReaL or rLLM that provide Azure OpenAI-compatible clients. The client should implement the AzureOpenAI client interface with `.chat.completions.create()` and `.beta.chat.completions.parse()` methods. (default: :obj:`None`)- **async_client** (Optional[Any], optional): A custom asynchronous AzureOpenAI client instance. If provided, this client will be used instead of creating a new one. The client should implement the AsyncAzureOpenAI client interface. (default: :obj:`None`)- **azure_deployment_name** (Optional[str], optional): **Deprecated**. Use `model_type` parameter instead. This parameter is kept for backward compatibility and will be removed in a future version. (default: :obj:`None`) **kwargs (Any): Additional arguments to pass to the client initialization. Ignored if custom clients are provided.-- **References**:+- **References**:- **https**: //learn.microsoft.com/en-us/azure/ai-services/openai/<a id="camel.models.azure_openai_model.AzureOpenAIModel.__init__"></a>diff --git a/docs/mintlify/reference/camel.models.deepseek_model.mdx b/docs/mintlify/reference/camel.models.deepseek_model.mdxindex 515c27f71..3ddde463d 100644--- a/docs/mintlify/reference/camel.models.deepseek_model.mdx+++ b/docs/mintlify/reference/camel.models.deepseek_model.mdx@@ -19,7 +19,7 @@ DeepSeek API in a unified OpenAICompatibleModel interface.- **token_counter** (Optional[BaseTokenCounter], optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter` will be used. (default: :obj:`None`)- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.-- **References**:+- **References**:- **https**: //api-docs.deepseek.com/<a id="camel.models.deepseek_model.DeepSeekModel.__init__"></a>diff --git a/docs/mintlify/reference/camel.models.ollama_model.mdx b/docs/mintlify/reference/camel.models.ollama_model.mdxindex 998f1203b..120869374 100644--- a/docs/mintlify/reference/camel.models.ollama_model.mdx+++ b/docs/mintlify/reference/camel.models.ollama_model.mdx@@ -20,7 +20,7 @@ Ollama service interface.- **token_counter** (Optional[BaseTokenCounter], optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter( ModelType.GPT_4O_MINI)` will be used. (default: :obj:`None`)- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.-- **References**:+- **References**:- **https**: //github.com/ollama/ollama/blob/main/docs/openai.md<a id="camel.models.ollama_model.OllamaModel.__init__"></a>diff --git a/docs/mintlify/reference/camel.models.vllm_model.mdx b/docs/mintlify/reference/camel.models.vllm_model.mdxindex 94f9d9b80..e094621b1 100644--- a/docs/mintlify/reference/camel.models.vllm_model.mdx+++ b/docs/mintlify/reference/camel.models.vllm_model.mdx@@ -19,7 +19,7 @@ vLLM service interface.- **token_counter** (Optional[BaseTokenCounter], optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter( ModelType.GPT_4O_MINI)` will be used. (default: :obj:`None`)- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.-- **References**:+- **References**:- **https**: //docs.vllm.ai/en/latest/serving/openai_compatible_server.html<a id="camel.models.vllm_model.VLLMModel.__init__"></a>diff --git a/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx b/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdxindex 80674920e..e0d782cd4 100644--- a/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx+++ b/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx@@ -19,7 +19,7 @@ frequency of the query terms.- **bm25** (BM25Okapi): An instance of the BM25Okapi class used for calculating document scores.- **content_input_path** (str): The path to the content that has been processed and stored.- **unstructured_modules** (UnstructuredIO): A module for parsing files and URLs and chunking content based on specified parameters.-- **References**:+- **References**:- **https**: //github.com/dorianbrown/rank_bm25<a id="camel.retrievers.bm25_retriever.BM25Retriever.__init__"></a>diff --git a/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx b/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdxindex 692844e38..f04ee8cd1 100644--- a/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx+++ b/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx@@ -15,7 +15,7 @@ model.- **model_name** (str): The model name to use for re-ranking.- **api_key** (Optional[str]): The API key for authenticating with the Cohere service.-- **References**:+- **References**:- **https**: //txt.cohere.com/rerank/<a id="camel.retrievers.cohere_rerank_retriever.CohereRerankRetriever.__init__"></a>diff --git a/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx b/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdxindex 00282663c..f0a6d7043 100644--- a/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx+++ b/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx@@ -18,7 +18,7 @@ search, and manage text with context.- **api_key** (str, optional): The API key for authentication. If not provided, will try to get from environment variable MEM0_API_KEY (default: :obj:`None`).- **user_id** (str, optional): Default user ID to associate memories with (default: :obj:`None`).- **metadata** (Dict[str, Any], optional): Default metadata to include with all memories (default: :obj:`None`).-- **References**:+- **References**:- **https**: //docs.mem0.ai<a id="camel.storages.key_value_storages.mem0_cloud.Mem0Storage.__init__"></a>diff --git a/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx b/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdxindex 1e77af5cc..06b48cfaa 100644--- a/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx+++ b/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx@@ -22,7 +22,7 @@ logged in with AWS CLI).- **access_key_id** (Optional[str], optional): The AWS access key ID. Defaults to None.- **secret_access_key** (Optional[str], optional): The AWS secret access key. Defaults to None.- **anonymous** (bool, optional): Whether to use anonymous access. Defaults to False.-- **References**:+- **References**:- **https**: //aws.amazon.com/pm/serv-s3/- **https**: //aws.amazon.com/cli/diff --git a/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx b/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdxindex d785edb61..ea16da013 100644--- a/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx+++ b/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx@@ -16,7 +16,7 @@ container in the storage account.- **storage_account_name** (str): The name of the storage account.- **container_name** (str): The name of the container.- **access_key** (Optional[str], optional): The access key of the storage account. Defaults to None.-- **References**:+- **References**:- **https**: //azure.microsoft.com/en-us/products/storage/blobs<a id="camel.storages.object_storages.azure_blob.AzureBlobStorage.__init__"></a>diff --git a/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx b/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdxindex eee8cf59d..fc5014378 100644--- a/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx+++ b/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx@@ -20,7 +20,7 @@ line tool and save the credentials first.- **bucket_name** (str): The name of the bucket.- **create_if_not_exists** (bool, optional): Whether to create the bucket if it does not exist. Defaults to True.- **anonymous** (bool, optional): Whether to use anonymous access. Defaults to False.-- **References**:+- **References**:- **https**: //cloud.google.com/storage- **https**: //cloud.google.com/docs/authentication/api-keysdiff --git a/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx b/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdxindex c2e53b60c..429c2da03 100644--- a/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx+++ b/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx@@ -15,7 +15,7 @@ def _get_wechat_access_token():**Raises:**- **ValueError**: If credentials are missing or token retrieval fails.-- **References**:+- **References**:- **https**: //developers.weixin.qq.com/doc/offiaccount/Basic_Information/Get_access_token.html<a id="camel.toolkits.wechat_official_toolkit._make_wechat_request"></a>