Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

fix: return updated combobox ref in click diffSnapshot (#3491)

Ishneet Sukhvinder Singh6bc92cdbb3b6
diff --git a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
index d59c8db4f..8827628a0 100644
--- a/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
+++ b/camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts
@@ -579,8 +579,24 @@ export class HybridBrowserSession {
const shouldCheckDiff = isCombobox || isTextbox;
let snapshotBefore: string | null = null;
+ let comboboxAriaLabel: string | null = null;
if (shouldCheckDiff) {
snapshotBefore = await (page as any)._snapshotForAI();
+ // Capture aria-label for combobox to find it again after click (ref may change)
+ if (isCombobox) {
+ comboboxAriaLabel = await element.getAttribute('aria-label');
+ if (!comboboxAriaLabel) {
+ // Try to get accessible name from aria-labelledby or inner text
+ comboboxAriaLabel = await element.evaluate(el => {
+ const labelledBy = el.getAttribute('aria-labelledby');
+ if (labelledBy) {
+ const labelEl = document.getElementById(labelledBy);
+ if (labelEl) return labelEl.textContent?.trim() || null;
+ }
+ return el.textContent?.trim() || null;
+ });
+ }
+ }
}
// Check element properties
@@ -658,7 +674,19 @@ export class HybridBrowserSession {
if (shouldCheckDiff && snapshotBefore) {
await page.waitForTimeout(300);
const snapshotAfter = await (page as any)._snapshotForAI();
- const diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotAfter, ['option', 'menuitem']);
+ let diffSnapshot = this.getSnapshotDiff(snapshotBefore, snapshotAfter, ['option', 'menuitem']);
+
+ // For combobox, find the new ref based on aria-label and prepend to diffSnapshot
+ if (isCombobox && comboboxAriaLabel) {
+ const newComboboxRef = this.findComboboxRefByAriaLabel(snapshotAfter, comboboxAriaLabel);
+ if (newComboboxRef) {
+ // Find the full line for this combobox in the snapshot
+ const comboboxLine = this.findSnapshotLineByRef(snapshotAfter, newComboboxRef);
+ if (comboboxLine) {
+ diffSnapshot = comboboxLine + (diffSnapshot ? '\n' + diffSnapshot : '');
+ }
+ }
+ }
if (diffSnapshot && diffSnapshot.trim() !== '') {
return { success: true, method: 'playwright-aria-ref', diffSnapshot };
@@ -709,6 +737,57 @@ export class HybridBrowserSession {
}
}
+ /**
+ * Find a combobox ref in the snapshot by its aria-label or expanded state
+ */
+ private findComboboxRefByAriaLabel(snapshot: string, ariaLabel: string): string | null {
+ const lines = snapshot.split('\n');
+ // Escape special regex characters in ariaLabel
+ const escapedLabel = ariaLabel.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+ // First, try to find by aria-label
+ for (const line of lines) {
+ const isCombobox = /\bcombobox\b/i.test(line);
+ const hasLabel = new RegExp(`["']${escapedLabel}["']`, 'i').test(line) ||
+ line.includes(ariaLabel);
+
+ if (isCombobox && hasLabel) {
+ const refMatch = line.match(/\[ref=([^\]]+)\]/);
+ if (refMatch) {
+ return refMatch[1];
+ }
+ }
+ }
+
+ // Fallback: find the expanded combobox (since we just clicked it)
+ for (const line of lines) {
+ const isCombobox = /\bcombobox\b/i.test(line);
+ const isExpanded = /\[expanded\]/i.test(line);
+
+ if (isCombobox && isExpanded) {
+ const refMatch = line.match(/\[ref=([^\]]+)\]/);
+ if (refMatch) {
+ return refMatch[1];
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Find the full snapshot line for a given ref
+ */
+ private findSnapshotLineByRef(snapshot: string, ref: string): string | null {
+ const lines = snapshot.split('\n');
+ for (const line of lines) {
+ if (line.includes(`[ref=${ref}]`)) {
+ return line.trim();
+ }
+ }
+ return null;
+ }
+
/**
* Simplified type implementation using Playwright's aria-ref selector
* Supports both single and multiple input operations
diff --git a/docs/mintlify/docs.json b/docs/mintlify/docs.json
index e09e23ca5..374c34f0b 100644
--- a/docs/mintlify/docs.json
+++ b/docs/mintlify/docs.json
@@ -766,4 +766,4 @@
"measurementId": "G-XV5VXFS2RG"
}
}
-}
\ No newline at end of file
+}
diff --git a/docs/mintlify/reference/camel.agents.chat_agent.mdx b/docs/mintlify/reference/camel.agents.chat_agent.mdx
index 5805742fd..3fecd4dfe 100644
--- a/docs/mintlify/reference/camel.agents.chat_agent.mdx
+++ b/docs/mintlify/reference/camel.agents.chat_agent.mdx
@@ -1453,7 +1453,7 @@ intermediate responses as they are generated.
- **input_message** (Union[BaseMessage, str]): The input message for the agent.
- **response_format** (Optional[Type[BaseModel]], optional): A Pydantic model defining the expected structure of the response.
-- **Yields**:
+- **Yields**:
- **ChatAgentResponse**: Intermediate responses containing partial content, tool calls, and other information as they become available.
<a id="camel.agents.chat_agent.ChatAgent._get_token_count"></a>
diff --git a/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx b/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx
index f3ab33002..5a0d0c047 100644
--- a/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx
+++ b/docs/mintlify/reference/camel.loaders.firecrawl_reader.mdx
@@ -14,7 +14,7 @@ Firecrawl allows you to turn entire websites into LLM-ready markdown.
- **api_key** (Optional[str]): API key for authenticating with the Firecrawl API.
- **api_url** (Optional[str]): Base URL for the Firecrawl API.
-- **References**:
+- **References**:
- **https**: //docs.firecrawl.dev/introduction
<a id="camel.loaders.firecrawl_reader.Firecrawl.__init__"></a>
diff --git a/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx b/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx
index bcb5868eb..4a726e233 100644
--- a/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx
+++ b/docs/mintlify/reference/camel.loaders.jina_url_reader.mdx
@@ -18,7 +18,7 @@ replace the UnstructuredIO URL Reader in the pipeline.
- **return_format** (ReturnFormat, optional): The level of detail of the returned content, which is optimized for LLMs. For now screenshots are not supported. Defaults to ReturnFormat.DEFAULT.
- **json_response** (bool, optional): Whether to return the response in JSON format. Defaults to False.
- **timeout** (int, optional): The maximum time in seconds to wait for the page to be rendered. Defaults to 30. **kwargs (Any): Additional keyword arguments, including proxies, cookies, etc. It should align with the HTTP Header field and value pairs listed in the reference.
-- **References**:
+- **References**:
- **https**: //jina.ai/reader
<a id="camel.loaders.jina_url_reader.JinaURLReader.__init__"></a>
diff --git a/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx b/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx
index 5834f25db..39bbb7ce2 100644
--- a/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx
+++ b/docs/mintlify/reference/camel.loaders.scrapegraph_reader.mdx
@@ -14,7 +14,7 @@ searching.
**Parameters:**
- **api_key** (Optional[str]): API key for authenticating with the ScrapeGraphAI API.
-- **References**:
+- **References**:
- **https**: //scrapegraph.ai/
<a id="camel.loaders.scrapegraph_reader.ScrapeGraphAI.__init__"></a>
diff --git a/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx b/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx
index 66c018a8f..c03b3e431 100644
--- a/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx
+++ b/docs/mintlify/reference/camel.models.aws_bedrock_model.mdx
@@ -20,7 +20,7 @@ AWS Bedrock API in a unified OpenAICompatibleModel interface.
- **token_counter** (BaseTokenCounter, optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter( ModelType.GPT_4O_MINI)` will be used. (default: :obj:`None`)
- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)
- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.
-- **References**:
+- **References**:
- **https**: //docs.aws.amazon.com/bedrock/latest/APIReference/welcome.html
<a id="camel.models.aws_bedrock_model.AWSBedrockModel.__init__"></a>
diff --git a/docs/mintlify/reference/camel.models.azure_openai_model.mdx b/docs/mintlify/reference/camel.models.azure_openai_model.mdx
index cb881f1b6..33c48bc45 100644
--- a/docs/mintlify/reference/camel.models.azure_openai_model.mdx
+++ b/docs/mintlify/reference/camel.models.azure_openai_model.mdx
@@ -25,7 +25,7 @@ Azure OpenAI API in a unified BaseModelBackend interface.
- **client** (Optional[Any], optional): A custom synchronous AzureOpenAI client instance. If provided, this client will be used instead of creating a new one. Useful for RL frameworks like AReaL or rLLM that provide Azure OpenAI-compatible clients. The client should implement the AzureOpenAI client interface with `.chat.completions.create()` and `.beta.chat.completions.parse()` methods. (default: :obj:`None`)
- **async_client** (Optional[Any], optional): A custom asynchronous AzureOpenAI client instance. If provided, this client will be used instead of creating a new one. The client should implement the AsyncAzureOpenAI client interface. (default: :obj:`None`)
- **azure_deployment_name** (Optional[str], optional): **Deprecated**. Use `model_type` parameter instead. This parameter is kept for backward compatibility and will be removed in a future version. (default: :obj:`None`) **kwargs (Any): Additional arguments to pass to the client initialization. Ignored if custom clients are provided.
-- **References**:
+- **References**:
- **https**: //learn.microsoft.com/en-us/azure/ai-services/openai/
<a id="camel.models.azure_openai_model.AzureOpenAIModel.__init__"></a>
diff --git a/docs/mintlify/reference/camel.models.deepseek_model.mdx b/docs/mintlify/reference/camel.models.deepseek_model.mdx
index 515c27f71..3ddde463d 100644
--- a/docs/mintlify/reference/camel.models.deepseek_model.mdx
+++ b/docs/mintlify/reference/camel.models.deepseek_model.mdx
@@ -19,7 +19,7 @@ DeepSeek API in a unified OpenAICompatibleModel interface.
- **token_counter** (Optional[BaseTokenCounter], optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter` will be used. (default: :obj:`None`)
- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)
- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.
-- **References**:
+- **References**:
- **https**: //api-docs.deepseek.com/
<a id="camel.models.deepseek_model.DeepSeekModel.__init__"></a>
diff --git a/docs/mintlify/reference/camel.models.ollama_model.mdx b/docs/mintlify/reference/camel.models.ollama_model.mdx
index 998f1203b..120869374 100644
--- a/docs/mintlify/reference/camel.models.ollama_model.mdx
+++ b/docs/mintlify/reference/camel.models.ollama_model.mdx
@@ -20,7 +20,7 @@ Ollama service interface.
- **token_counter** (Optional[BaseTokenCounter], optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter( ModelType.GPT_4O_MINI)` will be used. (default: :obj:`None`)
- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)
- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.
-- **References**:
+- **References**:
- **https**: //github.com/ollama/ollama/blob/main/docs/openai.md
<a id="camel.models.ollama_model.OllamaModel.__init__"></a>
diff --git a/docs/mintlify/reference/camel.models.vllm_model.mdx b/docs/mintlify/reference/camel.models.vllm_model.mdx
index 94f9d9b80..e094621b1 100644
--- a/docs/mintlify/reference/camel.models.vllm_model.mdx
+++ b/docs/mintlify/reference/camel.models.vllm_model.mdx
@@ -19,7 +19,7 @@ vLLM service interface.
- **token_counter** (Optional[BaseTokenCounter], optional): Token counter to use for the model. If not provided, :obj:`OpenAITokenCounter( ModelType.GPT_4O_MINI)` will be used. (default: :obj:`None`)
- **timeout** (Optional[float], optional): The timeout value in seconds for API calls. If not provided, will fall back to the MODEL_TIMEOUT environment variable or default to 180 seconds. (default: :obj:`None`)
- **max_retries** (int, optional): Maximum number of retries for API calls. (default: :obj:`3`) **kwargs (Any): Additional arguments to pass to the client initialization.
-- **References**:
+- **References**:
- **https**: //docs.vllm.ai/en/latest/serving/openai_compatible_server.html
<a id="camel.models.vllm_model.VLLMModel.__init__"></a>
diff --git a/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx b/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx
index 80674920e..e0d782cd4 100644
--- a/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx
+++ b/docs/mintlify/reference/camel.retrievers.bm25_retriever.mdx
@@ -19,7 +19,7 @@ frequency of the query terms.
- **bm25** (BM25Okapi): An instance of the BM25Okapi class used for calculating document scores.
- **content_input_path** (str): The path to the content that has been processed and stored.
- **unstructured_modules** (UnstructuredIO): A module for parsing files and URLs and chunking content based on specified parameters.
-- **References**:
+- **References**:
- **https**: //github.com/dorianbrown/rank_bm25
<a id="camel.retrievers.bm25_retriever.BM25Retriever.__init__"></a>
diff --git a/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx b/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx
index 692844e38..f04ee8cd1 100644
--- a/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx
+++ b/docs/mintlify/reference/camel.retrievers.cohere_rerank_retriever.mdx
@@ -15,7 +15,7 @@ model.
- **model_name** (str): The model name to use for re-ranking.
- **api_key** (Optional[str]): The API key for authenticating with the Cohere service.
-- **References**:
+- **References**:
- **https**: //txt.cohere.com/rerank/
<a id="camel.retrievers.cohere_rerank_retriever.CohereRerankRetriever.__init__"></a>
diff --git a/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx b/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx
index 00282663c..f0a6d7043 100644
--- a/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx
+++ b/docs/mintlify/reference/camel.storages.key_value_storages.mem0_cloud.mdx
@@ -18,7 +18,7 @@ search, and manage text with context.
- **api_key** (str, optional): The API key for authentication. If not provided, will try to get from environment variable MEM0_API_KEY (default: :obj:`None`).
- **user_id** (str, optional): Default user ID to associate memories with (default: :obj:`None`).
- **metadata** (Dict[str, Any], optional): Default metadata to include with all memories (default: :obj:`None`).
-- **References**:
+- **References**:
- **https**: //docs.mem0.ai
<a id="camel.storages.key_value_storages.mem0_cloud.Mem0Storage.__init__"></a>
diff --git a/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx b/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx
index 1e77af5cc..06b48cfaa 100644
--- a/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx
+++ b/docs/mintlify/reference/camel.storages.object_storages.amazon_s3.mdx
@@ -22,7 +22,7 @@ logged in with AWS CLI).
- **access_key_id** (Optional[str], optional): The AWS access key ID. Defaults to None.
- **secret_access_key** (Optional[str], optional): The AWS secret access key. Defaults to None.
- **anonymous** (bool, optional): Whether to use anonymous access. Defaults to False.
-- **References**:
+- **References**:
- **https**: //aws.amazon.com/pm/serv-s3/
- **https**: //aws.amazon.com/cli/
diff --git a/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx b/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx
index d785edb61..ea16da013 100644
--- a/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx
+++ b/docs/mintlify/reference/camel.storages.object_storages.azure_blob.mdx
@@ -16,7 +16,7 @@ container in the storage account.
- **storage_account_name** (str): The name of the storage account.
- **container_name** (str): The name of the container.
- **access_key** (Optional[str], optional): The access key of the storage account. Defaults to None.
-- **References**:
+- **References**:
- **https**: //azure.microsoft.com/en-us/products/storage/blobs
<a id="camel.storages.object_storages.azure_blob.AzureBlobStorage.__init__"></a>
diff --git a/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx b/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx
index eee8cf59d..fc5014378 100644
--- a/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx
+++ b/docs/mintlify/reference/camel.storages.object_storages.google_cloud.mdx
@@ -20,7 +20,7 @@ line tool and save the credentials first.
- **bucket_name** (str): The name of the bucket.
- **create_if_not_exists** (bool, optional): Whether to create the bucket if it does not exist. Defaults to True.
- **anonymous** (bool, optional): Whether to use anonymous access. Defaults to False.
-- **References**:
+- **References**:
- **https**: //cloud.google.com/storage
- **https**: //cloud.google.com/docs/authentication/api-keys
diff --git a/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx b/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx
index c2e53b60c..429c2da03 100644
--- a/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx
+++ b/docs/mintlify/reference/camel.toolkits.wechat_official_toolkit.mdx
@@ -15,7 +15,7 @@ def _get_wechat_access_token():
**Raises:**
- **ValueError**: If credentials are missing or token retrieval fails.
-- **References**:
+- **References**:
- **https**: //developers.weixin.qq.com/doc/offiaccount/Basic_Information/Get_access_token.html
<a id="camel.toolkits.wechat_official_toolkit._make_wechat_request"></a>