Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: integrate visual tool for non vision browser (#2743)

Puzhen Zhang227ddeea3f75
diff --git a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py b/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py
index a4d1b9129..58454ef54 100644
--- a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py
+++ b/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py
@@ -80,7 +80,28 @@ class BrowserNonVisualToolkit(BaseToolkit):
return
if loop.is_closed():
- # Event loop already closed → cannot run async cleanup
+ # The default loop is closed, create a *temporary* loop just
+ # for cleanup so that Playwright / asyncio transports are
+ # gracefully shut down. This avoids noisy warnings such as
+ # "RuntimeError: Event loop is closed" when the program
+ # exits.
+ try:
+ tmp_loop = asyncio.new_event_loop()
+ try:
+ asyncio.set_event_loop(tmp_loop)
+ tmp_loop.run_until_complete(self.close_browser())
+ finally:
+ # Best-effort shutdown of async generators and loop
+ # itself (Python ≥3.6).
+ if hasattr(tmp_loop, "shutdown_asyncgens"):
+ tmp_loop.run_until_complete(
+ tmp_loop.shutdown_asyncgens()
+ )
+ tmp_loop.close()
+ finally:
+ # Ensure no subsequent get_event_loop() picks up a now
+ # closed temporary loop.
+ asyncio.set_event_loop(None)
return
if loop.is_running():
diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.js b/camel/toolkits/non_visual_browser_toolkit/snapshot.js
index 844d1432b..ae42bdc7f 100644
--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.js
+++ b/camel/toolkits/non_visual_browser_toolkit/snapshot.js
@@ -1,188 +1,268 @@
(() => {
- // Store each element as {text, priority, depth}
- const elements = [];
-
- // Maximum lines allowed before we start dropping lower-priority nodes
- const MAX_LINES = 400;
-
- // Priority helper – lower number = higher priority
- function getPriority(tag, role, text) {
- // 1. Interactive elements
- if (["input", "button", "a", "select", "textarea"].includes(tag)) return 1;
- if (["checkbox", "radio"].includes(role)) return 1;
-
- // 2. Labels / descriptive adjacent text (label elements)
- if (tag === "label") return 2;
-
- // 3. General visible text
- if (text) return 3;
-
- // 4. Low-value structural nodes
- return 4;
- }
+ // Playwright's snapshot logic focuses on semantics and visibility, not arbitrary limits.
+ // We will first build a semantic tree in memory, then render it.
function isVisible(node) {
- const rect = node.getBoundingClientRect();
- if (rect.width === 0 || rect.height === 0) return false;
-
+ if (node.nodeType !== Node.ELEMENT_NODE) return true;
const style = window.getComputedStyle(node);
- if (style.display === 'none' || style.visibility === 'hidden') return false;
-
- return true;
+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
+ return false;
+ // An element with `display: contents` is not rendered itself, but its children are.
+ if (style.display === 'contents')
+ return true;
+ const rect = node.getBoundingClientRect();
+ return rect.width > 0 && rect.height > 0;
}
function getRole(node) {
- const tag = node.tagName.toLowerCase();
- const type = node.getAttribute('type');
-
- if (node.getAttribute('role')) return node.getAttribute('role');
+ const role = node.getAttribute('role');
+ if (role) return role;
- if (tag === 'input') {
- if (type === 'checkbox') return 'checkbox';
- if (type === 'radio') return 'radio';
- return 'input';
+ const tagName = node.tagName.toLowerCase();
+ if (tagName === 'a') return 'link';
+ if (tagName === 'button') return 'button';
+ if (tagName === 'input') {
+ const type = node.getAttribute('type')?.toLowerCase();
+ if (['button', 'checkbox', 'radio', 'reset', 'submit'].includes(type)) return type;
+ return 'textbox';
}
-
- if (tag === 'button') return 'button';
- if (tag === 'a') return 'link';
- if (tag === 'select') return 'select';
- if (tag === 'textarea') return 'textarea';
- if (tag === 'p') return 'paragraph';
- if (tag === 'span') return 'text';
-
+ if (['select', 'textarea'].includes(tagName)) return tagName;
+ if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tagName)) return 'heading';
return 'generic';
}
function getAccessibleName(node) {
- if (node.hasAttribute('aria-label')) {
- return node.getAttribute('aria-label');
- }
+ if (node.hasAttribute('aria-label')) return node.getAttribute('aria-label') || '';
if (node.hasAttribute('aria-labelledby')) {
const id = node.getAttribute('aria-labelledby');
const labelEl = document.getElementById(id);
- if (labelEl) return labelEl.textContent.trim();
+ if (labelEl) return labelEl.textContent || '';
}
- if (node.hasAttribute('title')) {
- return node.getAttribute('title');
+ // This is the new, visibility-aware text extraction logic.
+ const text = getVisibleTextContent(node);
+
+ // Add a heuristic to ignore code-like text that might be in the DOM
+ if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';
+ return text;
+ }
+
+ const textCache = new Map();
+ function getVisibleTextContent(_node) {
+ if (textCache.has(_node)) return textCache.get(_node);
+
+ if (_node.nodeType === Node.TEXT_NODE) {
+ // For a text node, its content is visible if its parent is.
+ // The isVisible check on the parent happens before this recursion.
+ return _node.nodeValue || '';
}
- const tagName = node.tagName?.toLowerCase();
- if (['style', 'script', 'meta', 'noscript', 'svg'].includes(tagName)) {
+ if (_node.nodeType !== Node.ELEMENT_NODE || !isVisible(_node) || ['SCRIPT', 'STYLE', 'NOSCRIPT', 'META', 'HEAD'].includes(_node.tagName)) {
return '';
}
- const text = node.textContent?.trim() || '';
-
- // Ignore styles, tokens, or long CSS-like expressions
- if (/^[.#]?[a-zA-Z0-9\-_]+\s*\{[^}]*\}/.test(text)) return '';
- if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';
+ let result = '';
+ for (const child of _node.childNodes) {
+ result += getVisibleTextContent(child);
+ }
- return text.replace(/[^\w\u4e00-\u9fa5\s\-.,?!'"()()]/g, '').trim();
+ // Caching the result for performance.
+ textCache.set(_node, result);
+ return result;
}
let refCounter = 1;
+ function generateRef() {
+ return `e${refCounter++}`;
+ }
- function traverse(node, depth) {
- if (node.nodeType !== Node.ELEMENT_NODE) return;
- if (!isVisible(node)) return;
+ /**
+ * Phase 1: Build an in-memory representation of the accessibility tree.
+ */
+ function buildAriaTree(rootElement) {
+ const visited = new Set();
- const tagName = node.tagName.toLowerCase();
- const text = getAccessibleName(node).slice(0, 50);
+ function toAriaNode(element) {
+ // Only consider visible elements
+ if (!isVisible(element)) return null;
- // Skip unlabeled links (anchors without any accessible name)
- if (tagName === 'a' && !text) {
- // Skip unlabeled links; process children if any
- for (const child of node.children) {
- traverse(child, depth + 1);
- }
- return;
- }
+ const role = getRole(element);
+ // 'presentation' and 'none' roles are ignored, but their children are processed.
+ if (['presentation', 'none'].includes(role)) return null;
- const hasRoleOrText = ['button', 'a', 'input', 'select', 'textarea', 'p', 'span'].includes(tagName) ||
- node.getAttribute('role') || text;
+ const name = getAccessibleName(element);
- if (hasRoleOrText) {
- const role = getRole(node);
- const ref = `e${refCounter++}`;
- const label = text ? `"${text}"` : '';
+ // Create the node
+ const node = {
+ role,
+ name,
+ children: [],
+ element: element,
+ ref: generateRef(),
+ };
- // Raw line (without indent) – we will apply indentation later once we know
- // which ancestor lines survive filtering so that indentation always reflects
- // the visible hierarchy.
- const lineText = `- ${role}${label ? ` ${label}` : ''} [ref=${ref}]`;
- const priority = getPriority(tagName, role, text);
+ // Add states for interactive elements, similar to Playwright
+ if (element.hasAttribute('disabled')) node.disabled = true;
+ if (element.hasAttribute('aria-checked')) node.checked = element.getAttribute('aria-checked');
+ if (element.hasAttribute('aria-expanded')) node.expanded = element.getAttribute('aria-expanded');
- elements.push({ text: lineText, priority, depth });
+ // Tag element with a ref for later lookup
+ element.setAttribute('aria-ref', node.ref);
- // Always inject ref so Playwright can still locate the element even if line is later filtered out.
- node.setAttribute('aria-ref', ref);
+ return node;
}
- for (const child of node.children) {
- traverse(child, depth + 1);
+ function traverse(element, parentNode) {
+ if (visited.has(element)) return;
+ visited.add(element);
+
+ // FIX: Completely skip script and style tags and their children.
+ const tagName = element.tagName.toLowerCase();
+ if (['script', 'style', 'meta', 'noscript'].includes(tagName))
+ return;
+
+ // Check if element is explicitly hidden by CSS - if so, skip entirely including children
+ const style = window.getComputedStyle(element);
+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
+ return;
+ }
+
+ const ariaNode = toAriaNode(element);
+ // If the element is not rendered or is presentational, its children
+ // are attached directly to the parent.
+ const newParent = ariaNode || parentNode;
+ if (ariaNode) parentNode.children.push(ariaNode);
+
+ for (const child of element.childNodes) {
+ if (child.nodeType === Node.ELEMENT_NODE) {
+ traverse(child, newParent);
+ } else if (child.nodeType === Node.TEXT_NODE) {
+ const text = (child.textContent || '').trim();
+ if (text) newParent.children.push(text);
+ }
+ }
+
+ // Also traverse into shadow DOM if it exists
+ if (element.shadowRoot) {
+ for (const child of element.shadowRoot.childNodes) {
+ if (child.nodeType === Node.ELEMENT_NODE) {
+ traverse(child, newParent);
+ } else if (child.nodeType === Node.TEXT_NODE) {
+ const text = (child.textContent || '').trim();
+ if (text) newParent.children.push(text);
+ }
+ }
+ }
+
+ // FIX: If an element's name is the same as its only text child, remove the redundant child.
+ if (ariaNode && ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {
+ ariaNode.children = [];
+ }
}
+
+ const root = { role: 'Root', name: '', children: [], element: rootElement };
+ traverse(rootElement, root);
+ return root;
}
- function processDocument(doc, depth = 0) {
- try {
- traverse(doc.body, depth);
- } catch (e) {
- // Handle docs without body (e.g., about:blank)
- }
+ /**
+ * Phase 2: Normalize the tree by removing redundant generic wrappers.
+ * This is a key optimization in Playwright to simplify the structure.
+ */
+ function normalizeTree(node) {
+ if (typeof node === 'string') return [node];
- const frames = doc.querySelectorAll('iframe');
- for (const frame of frames) {
- try {
- if (frame.contentDocument) {
- processDocument(frame.contentDocument, depth + 1);
- }
- } catch (e) {
- // Skip cross-origin iframes
+ const newChildren = [];
+ for (const child of node.children) {
+ newChildren.push(...normalizeTree(child));
+ }
+ node.children = newChildren;
+
+ // Remove child elements that have the same name as their parent
+ if (node.children.length === 1 && typeof node.children[0] !== 'string') {
+ const child = node.children[0];
+ if (child.name && node.name && child.name.trim() === node.name.trim()) {
+ // Merge child's children into parent and remove the redundant child
+ node.children = child.children || [];
}
}
+
+ // A 'generic' role that just wraps a single other element is redundant.
+ // We lift its child up to replace it, simplifying the hierarchy.
+ const isRedundantWrapper = node.role === 'generic' && node.children.length === 1 && typeof node.children[0] !== 'string';
+ if (isRedundantWrapper) {
+ return node.children;
+ }
+ return [node];
}
- processDocument(document);
- // Always drop priority-4 nodes (low-value structural or invisible)
- let finalElements = elements.filter(el => el.priority <= 3);
+ /**
+ * Phase 3: Render the normalized tree into the final string format.
+ */
+ function renderTree(node, indent = '') {
+ const lines = [];
+ let meaningfulProps = '';
+ if (node.disabled) meaningfulProps += ' disabled';
+ if (node.checked !== undefined) meaningfulProps += ` checked=${node.checked}`;
+ if (node.expanded !== undefined) meaningfulProps += ` expanded=${node.expanded}`;
+
+ const ref = node.ref ? ` [ref=${node.ref}]` : '';
+ const name = (node.name || '').replace(/\s+/g, ' ').trim();
- // Additional size condensation when still exceeding MAX_LINES
- if (finalElements.length > MAX_LINES) {
- const filterBy = (maxPriority) => finalElements.filter(el => el.priority <= maxPriority);
+ // Skip elements with empty names and no meaningful props (ref is not considered meaningful)
+ if (!name && !meaningfulProps) {
+ // If element has no name and no meaningful props, render its children directly at current level
+ for (const child of node.children) {
+ if (typeof child === 'string') {
+ const childText = child.replace(/\s+/g, ' ').trim();
+ if (childText) { // Only add non-empty text
+ lines.push(`${indent}- text "${childText}"`);
+ }
+ } else {
+ lines.push(...renderTree(child, indent));
+ }
+ }
+ return lines;
+ }
+
+ lines.push(`${indent}- ${node.role}${name ? ` "${name}"` : ''}${meaningfulProps}${ref}`);
- // Progressively tighten: keep 1-3, then 1-2, finally only 1
- for (const limit of [3, 2, 1]) {
- const candidate = filterBy(limit);
- if (candidate.length <= MAX_LINES || limit === 1) {
- finalElements = candidate;
- break;
+ for (const child of node.children) {
+ if (typeof child === 'string') {
+ const childText = child.replace(/\s+/g, ' ').trim();
+ if (childText) { // Only add non-empty text
+ lines.push(`${indent} - text "${childText}"`);
+ }
+ } else {
+ lines.push(...renderTree(child, indent + ' '));
}
}
+ return lines;
}
- // ------------------------------------------------------------------
- // Re-apply indentation so that it matches the *visible* hierarchy only.
- // Whenever an ancestor element is removed due to priority rules, its
- // children will be re-indented one level up so the structure remains
- // intuitive.
- // ------------------------------------------------------------------
- const outputLines = [];
- const depthStack = []; // keeps track of kept original depths
-
- for (const el of finalElements) {
- // Pop depths that are not ancestors of current element
- while (depthStack.length && depthStack[depthStack.length - 1] >= el.depth) {
- depthStack.pop();
- }
+ function processDocument(doc) {
+ if (!doc.body) return [];
+
+ // Clear cache for each new document processing.
+ textCache.clear();
+ let tree = buildAriaTree(doc.body);
+ [tree] = normalizeTree(tree);
- // Push the current depth so future descendants know their ancestor chain
- depthStack.push(el.depth);
+ const lines = renderTree(tree).slice(1); // Skip the root node line
- const compressedDepth = depthStack.length - 1; // root level has zero indent
- const indent = '\t'.repeat(compressedDepth);
- outputLines.push(indent + el.text);
+ const frames = doc.querySelectorAll('iframe');
+ for (const frame of frames) {
+ try {
+ if (frame.contentDocument) {
+ lines.push(...processDocument(frame.contentDocument));
+ }
+ } catch (e) {
+ // Skip cross-origin iframes
+ }
+ }
+ return lines;
}
+ const outputLines = processDocument(document);
return outputLines.join('\n');
})();
diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.py b/camel/toolkits/non_visual_browser_toolkit/snapshot.py
index 1c6fb0301..46c756da1 100644
--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.py
+++ b/camel/toolkits/non_visual_browser_toolkit/snapshot.py
@@ -48,14 +48,15 @@ class PageSnapshot:
try:
current_url = self.page.url
- # Serve cached copy (unless diff requested)
- if (
- not force_refresh
- and current_url == self._last_url
- and self.snapshot_data
- and not diff_only
- ):
- return self.snapshot_data
+ # Previously we skipped regeneration when the URL had not changed
+ # and no explicit refresh was requested. This prevented the agent
+ # from seeing DOM updates that occur without a navigation (e.g.
+ # single-page apps, dynamic games such as Wordle). The early-exit
+ # logic has been removed so that we always capture a *fresh* DOM
+ # snapshot. If the snapshot happens to be byte-for-byte identical
+ # to the previous one we simply return it after the standard
+ # comparison step below; otherwise callers receive the updated
+ # snapshot even when the URL did not change.
# ensure DOM stability
await self.page.wait_for_load_state(
diff --git a/examples/toolkits/browser_non_visual_toolkit_example.py b/examples/toolkits/browser_non_visual_toolkit_example.py
index 2e4093b04..ecee5b084 100644
--- a/examples/toolkits/browser_non_visual_toolkit_example.py
+++ b/examples/toolkits/browser_non_visual_toolkit_example.py
@@ -23,15 +23,17 @@ logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
- logging.StreamHandler(), # Console output
- logging.FileHandler('browser_dom_debug.log'), # File output
+ logging.StreamHandler(),
],
)
logging.getLogger('camel.agents').setLevel(logging.INFO)
logging.getLogger('camel.models').setLevel(logging.INFO)
-
-USER_DATA_DIR = r"D:/User_Data"
+logging.getLogger('camel.models').setLevel(logging.INFO)
+logging.getLogger('camel.toolkits.non_visual_browser_toolkit').setLevel(
+ logging.DEBUG
+)
+USER_DATA_DIR = "User_Data"
model_backend = ModelFactory.create(
model_platform=ModelPlatformType.OPENAI,
@@ -40,7 +42,7 @@ model_backend = ModelFactory.create(
)
web_toolkit = BrowserNonVisualToolkit(
- headless=False,
+ headless=False, user_data_dir=USER_DATA_DIR
)
agent = ChatAgent(
@@ -49,25 +51,29 @@ agent = ChatAgent(
max_iteration=10,
)
-TASK_PROMPT = """Click "Sign in to Google Account" on Google.
-If a CAPTCHA appears requiring user input, call the wait_user function —
-wait 5 seconds each time and keep waiting until you detect that the snapshot
-no longer contains content requiring user action.
-Make sure not to help the user fill in their account or CAPTCHA information.
-Before the task is completed, you must call the tool in every session and
-use the wait_user function whenever needed.
+TASK_PROMPT = """Play this game:
+https://www.nytimes.com/games/wordle/index.html
+You need to enter meaningful five-letter words, such as "Apple".
+Before completing the task, do not do anything outside of using the call tool.
+If the game tells you that something is wrong, you must resolve it yourself.
+Also, you must attempt all six tries.
+If the registration window can be closed, please close it.
"""
async def main() -> None:
response = await agent.astep(TASK_PROMPT)
-
print("Task:", TASK_PROMPT)
print(f"Using user data directory: {USER_DATA_DIR}\n")
print("Response from agent:")
print(response.msgs[0].content if response.msgs else "<no response>")
+ try:
+ await web_toolkit.close_browser()
+ except Exception as err:
+ logging.warning("Failed to close browser session explicitly: %s", err)
+
if __name__ == "__main__":
asyncio.run(main())