🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: integrate visual tool for non vision browser (#2743)
Puzhen Zhang227ddeea3f75
diff --git a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py b/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.pyindex a4d1b9129..58454ef54 100644--- a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py+++ b/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py@@ -80,7 +80,28 @@ class BrowserNonVisualToolkit(BaseToolkit):returnif loop.is_closed():- # Event loop already closed → cannot run async cleanup+ # The default loop is closed, create a *temporary* loop just+ # for cleanup so that Playwright / asyncio transports are+ # gracefully shut down. This avoids noisy warnings such as+ # "RuntimeError: Event loop is closed" when the program+ # exits.+ try:+ tmp_loop = asyncio.new_event_loop()+ try:+ asyncio.set_event_loop(tmp_loop)+ tmp_loop.run_until_complete(self.close_browser())+ finally:+ # Best-effort shutdown of async generators and loop+ # itself (Python ≥3.6).+ if hasattr(tmp_loop, "shutdown_asyncgens"):+ tmp_loop.run_until_complete(+ tmp_loop.shutdown_asyncgens()+ )+ tmp_loop.close()+ finally:+ # Ensure no subsequent get_event_loop() picks up a now+ # closed temporary loop.+ asyncio.set_event_loop(None)returnif loop.is_running():diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.js b/camel/toolkits/non_visual_browser_toolkit/snapshot.jsindex 844d1432b..ae42bdc7f 100644--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.js+++ b/camel/toolkits/non_visual_browser_toolkit/snapshot.js@@ -1,188 +1,268 @@(() => {- // Store each element as {text, priority, depth}- const elements = [];-- // Maximum lines allowed before we start dropping lower-priority nodes- const MAX_LINES = 400;-- // Priority helper – lower number = higher priority- function getPriority(tag, role, text) {- // 1. Interactive elements- if (["input", "button", "a", "select", "textarea"].includes(tag)) return 1;- if (["checkbox", "radio"].includes(role)) return 1;-- // 2. Labels / descriptive adjacent text (label elements)- if (tag === "label") return 2;-- // 3. General visible text- if (text) return 3;-- // 4. Low-value structural nodes- return 4;- }+ // Playwright's snapshot logic focuses on semantics and visibility, not arbitrary limits.+ // We will first build a semantic tree in memory, then render it.function isVisible(node) {- const rect = node.getBoundingClientRect();- if (rect.width === 0 || rect.height === 0) return false;-+ if (node.nodeType !== Node.ELEMENT_NODE) return true;const style = window.getComputedStyle(node);- if (style.display === 'none' || style.visibility === 'hidden') return false;-- return true;+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')+ return false;+ // An element with `display: contents` is not rendered itself, but its children are.+ if (style.display === 'contents')+ return true;+ const rect = node.getBoundingClientRect();+ return rect.width > 0 && rect.height > 0;}function getRole(node) {- const tag = node.tagName.toLowerCase();- const type = node.getAttribute('type');-- if (node.getAttribute('role')) return node.getAttribute('role');+ const role = node.getAttribute('role');+ if (role) return role;- if (tag === 'input') {- if (type === 'checkbox') return 'checkbox';- if (type === 'radio') return 'radio';- return 'input';+ const tagName = node.tagName.toLowerCase();+ if (tagName === 'a') return 'link';+ if (tagName === 'button') return 'button';+ if (tagName === 'input') {+ const type = node.getAttribute('type')?.toLowerCase();+ if (['button', 'checkbox', 'radio', 'reset', 'submit'].includes(type)) return type;+ return 'textbox';}-- if (tag === 'button') return 'button';- if (tag === 'a') return 'link';- if (tag === 'select') return 'select';- if (tag === 'textarea') return 'textarea';- if (tag === 'p') return 'paragraph';- if (tag === 'span') return 'text';-+ if (['select', 'textarea'].includes(tagName)) return tagName;+ if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tagName)) return 'heading';return 'generic';}function getAccessibleName(node) {- if (node.hasAttribute('aria-label')) {- return node.getAttribute('aria-label');- }+ if (node.hasAttribute('aria-label')) return node.getAttribute('aria-label') || '';if (node.hasAttribute('aria-labelledby')) {const id = node.getAttribute('aria-labelledby');const labelEl = document.getElementById(id);- if (labelEl) return labelEl.textContent.trim();+ if (labelEl) return labelEl.textContent || '';}- if (node.hasAttribute('title')) {- return node.getAttribute('title');+ // This is the new, visibility-aware text extraction logic.+ const text = getVisibleTextContent(node);++ // Add a heuristic to ignore code-like text that might be in the DOM+ if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';+ return text;+ }++ const textCache = new Map();+ function getVisibleTextContent(_node) {+ if (textCache.has(_node)) return textCache.get(_node);++ if (_node.nodeType === Node.TEXT_NODE) {+ // For a text node, its content is visible if its parent is.+ // The isVisible check on the parent happens before this recursion.+ return _node.nodeValue || '';}- const tagName = node.tagName?.toLowerCase();- if (['style', 'script', 'meta', 'noscript', 'svg'].includes(tagName)) {+ if (_node.nodeType !== Node.ELEMENT_NODE || !isVisible(_node) || ['SCRIPT', 'STYLE', 'NOSCRIPT', 'META', 'HEAD'].includes(_node.tagName)) {return '';}- const text = node.textContent?.trim() || '';-- // Ignore styles, tokens, or long CSS-like expressions- if (/^[.#]?[a-zA-Z0-9\-_]+\s*\{[^}]*\}/.test(text)) return '';- if ((text.match(/[;:{}]/g)?.length || 0) > 2) return '';+ let result = '';+ for (const child of _node.childNodes) {+ result += getVisibleTextContent(child);+ }- return text.replace(/[^\w\u4e00-\u9fa5\s\-.,?!'"()()]/g, '').trim();+ // Caching the result for performance.+ textCache.set(_node, result);+ return result;}let refCounter = 1;+ function generateRef() {+ return `e${refCounter++}`;+ }- function traverse(node, depth) {- if (node.nodeType !== Node.ELEMENT_NODE) return;- if (!isVisible(node)) return;+ /**+ * Phase 1: Build an in-memory representation of the accessibility tree.+ */+ function buildAriaTree(rootElement) {+ const visited = new Set();- const tagName = node.tagName.toLowerCase();- const text = getAccessibleName(node).slice(0, 50);+ function toAriaNode(element) {+ // Only consider visible elements+ if (!isVisible(element)) return null;- // Skip unlabeled links (anchors without any accessible name)- if (tagName === 'a' && !text) {- // Skip unlabeled links; process children if any- for (const child of node.children) {- traverse(child, depth + 1);- }- return;- }+ const role = getRole(element);+ // 'presentation' and 'none' roles are ignored, but their children are processed.+ if (['presentation', 'none'].includes(role)) return null;- const hasRoleOrText = ['button', 'a', 'input', 'select', 'textarea', 'p', 'span'].includes(tagName) ||- node.getAttribute('role') || text;+ const name = getAccessibleName(element);- if (hasRoleOrText) {- const role = getRole(node);- const ref = `e${refCounter++}`;- const label = text ? `"${text}"` : '';+ // Create the node+ const node = {+ role,+ name,+ children: [],+ element: element,+ ref: generateRef(),+ };- // Raw line (without indent) – we will apply indentation later once we know- // which ancestor lines survive filtering so that indentation always reflects- // the visible hierarchy.- const lineText = `- ${role}${label ? ` ${label}` : ''} [ref=${ref}]`;- const priority = getPriority(tagName, role, text);+ // Add states for interactive elements, similar to Playwright+ if (element.hasAttribute('disabled')) node.disabled = true;+ if (element.hasAttribute('aria-checked')) node.checked = element.getAttribute('aria-checked');+ if (element.hasAttribute('aria-expanded')) node.expanded = element.getAttribute('aria-expanded');- elements.push({ text: lineText, priority, depth });+ // Tag element with a ref for later lookup+ element.setAttribute('aria-ref', node.ref);- // Always inject ref so Playwright can still locate the element even if line is later filtered out.- node.setAttribute('aria-ref', ref);+ return node;}- for (const child of node.children) {- traverse(child, depth + 1);+ function traverse(element, parentNode) {+ if (visited.has(element)) return;+ visited.add(element);++ // FIX: Completely skip script and style tags and their children.+ const tagName = element.tagName.toLowerCase();+ if (['script', 'style', 'meta', 'noscript'].includes(tagName))+ return;++ // Check if element is explicitly hidden by CSS - if so, skip entirely including children+ const style = window.getComputedStyle(element);+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {+ return;+ }++ const ariaNode = toAriaNode(element);+ // If the element is not rendered or is presentational, its children+ // are attached directly to the parent.+ const newParent = ariaNode || parentNode;+ if (ariaNode) parentNode.children.push(ariaNode);++ for (const child of element.childNodes) {+ if (child.nodeType === Node.ELEMENT_NODE) {+ traverse(child, newParent);+ } else if (child.nodeType === Node.TEXT_NODE) {+ const text = (child.textContent || '').trim();+ if (text) newParent.children.push(text);+ }+ }++ // Also traverse into shadow DOM if it exists+ if (element.shadowRoot) {+ for (const child of element.shadowRoot.childNodes) {+ if (child.nodeType === Node.ELEMENT_NODE) {+ traverse(child, newParent);+ } else if (child.nodeType === Node.TEXT_NODE) {+ const text = (child.textContent || '').trim();+ if (text) newParent.children.push(text);+ }+ }+ }++ // FIX: If an element's name is the same as its only text child, remove the redundant child.+ if (ariaNode && ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {+ ariaNode.children = [];+ }}++ const root = { role: 'Root', name: '', children: [], element: rootElement };+ traverse(rootElement, root);+ return root;}- function processDocument(doc, depth = 0) {- try {- traverse(doc.body, depth);- } catch (e) {- // Handle docs without body (e.g., about:blank)- }+ /**+ * Phase 2: Normalize the tree by removing redundant generic wrappers.+ * This is a key optimization in Playwright to simplify the structure.+ */+ function normalizeTree(node) {+ if (typeof node === 'string') return [node];- const frames = doc.querySelectorAll('iframe');- for (const frame of frames) {- try {- if (frame.contentDocument) {- processDocument(frame.contentDocument, depth + 1);- }- } catch (e) {- // Skip cross-origin iframes+ const newChildren = [];+ for (const child of node.children) {+ newChildren.push(...normalizeTree(child));+ }+ node.children = newChildren;++ // Remove child elements that have the same name as their parent+ if (node.children.length === 1 && typeof node.children[0] !== 'string') {+ const child = node.children[0];+ if (child.name && node.name && child.name.trim() === node.name.trim()) {+ // Merge child's children into parent and remove the redundant child+ node.children = child.children || [];}}++ // A 'generic' role that just wraps a single other element is redundant.+ // We lift its child up to replace it, simplifying the hierarchy.+ const isRedundantWrapper = node.role === 'generic' && node.children.length === 1 && typeof node.children[0] !== 'string';+ if (isRedundantWrapper) {+ return node.children;+ }+ return [node];}- processDocument(document);- // Always drop priority-4 nodes (low-value structural or invisible)- let finalElements = elements.filter(el => el.priority <= 3);+ /**+ * Phase 3: Render the normalized tree into the final string format.+ */+ function renderTree(node, indent = '') {+ const lines = [];+ let meaningfulProps = '';+ if (node.disabled) meaningfulProps += ' disabled';+ if (node.checked !== undefined) meaningfulProps += ` checked=${node.checked}`;+ if (node.expanded !== undefined) meaningfulProps += ` expanded=${node.expanded}`;++ const ref = node.ref ? ` [ref=${node.ref}]` : '';+ const name = (node.name || '').replace(/\s+/g, ' ').trim();- // Additional size condensation when still exceeding MAX_LINES- if (finalElements.length > MAX_LINES) {- const filterBy = (maxPriority) => finalElements.filter(el => el.priority <= maxPriority);+ // Skip elements with empty names and no meaningful props (ref is not considered meaningful)+ if (!name && !meaningfulProps) {+ // If element has no name and no meaningful props, render its children directly at current level+ for (const child of node.children) {+ if (typeof child === 'string') {+ const childText = child.replace(/\s+/g, ' ').trim();+ if (childText) { // Only add non-empty text+ lines.push(`${indent}- text "${childText}"`);+ }+ } else {+ lines.push(...renderTree(child, indent));+ }+ }+ return lines;+ }++ lines.push(`${indent}- ${node.role}${name ? ` "${name}"` : ''}${meaningfulProps}${ref}`);- // Progressively tighten: keep 1-3, then 1-2, finally only 1- for (const limit of [3, 2, 1]) {- const candidate = filterBy(limit);- if (candidate.length <= MAX_LINES || limit === 1) {- finalElements = candidate;- break;+ for (const child of node.children) {+ if (typeof child === 'string') {+ const childText = child.replace(/\s+/g, ' ').trim();+ if (childText) { // Only add non-empty text+ lines.push(`${indent} - text "${childText}"`);+ }+ } else {+ lines.push(...renderTree(child, indent + ' '));}}+ return lines;}- // ------------------------------------------------------------------- // Re-apply indentation so that it matches the *visible* hierarchy only.- // Whenever an ancestor element is removed due to priority rules, its- // children will be re-indented one level up so the structure remains- // intuitive.- // ------------------------------------------------------------------- const outputLines = [];- const depthStack = []; // keeps track of kept original depths-- for (const el of finalElements) {- // Pop depths that are not ancestors of current element- while (depthStack.length && depthStack[depthStack.length - 1] >= el.depth) {- depthStack.pop();- }+ function processDocument(doc) {+ if (!doc.body) return [];++ // Clear cache for each new document processing.+ textCache.clear();+ let tree = buildAriaTree(doc.body);+ [tree] = normalizeTree(tree);- // Push the current depth so future descendants know their ancestor chain- depthStack.push(el.depth);+ const lines = renderTree(tree).slice(1); // Skip the root node line- const compressedDepth = depthStack.length - 1; // root level has zero indent- const indent = '\t'.repeat(compressedDepth);- outputLines.push(indent + el.text);+ const frames = doc.querySelectorAll('iframe');+ for (const frame of frames) {+ try {+ if (frame.contentDocument) {+ lines.push(...processDocument(frame.contentDocument));+ }+ } catch (e) {+ // Skip cross-origin iframes+ }+ }+ return lines;}+ const outputLines = processDocument(document);return outputLines.join('\n');})();diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.py b/camel/toolkits/non_visual_browser_toolkit/snapshot.pyindex 1c6fb0301..46c756da1 100644--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.py+++ b/camel/toolkits/non_visual_browser_toolkit/snapshot.py@@ -48,14 +48,15 @@ class PageSnapshot:try:current_url = self.page.url- # Serve cached copy (unless diff requested)- if (- not force_refresh- and current_url == self._last_url- and self.snapshot_data- and not diff_only- ):- return self.snapshot_data+ # Previously we skipped regeneration when the URL had not changed+ # and no explicit refresh was requested. This prevented the agent+ # from seeing DOM updates that occur without a navigation (e.g.+ # single-page apps, dynamic games such as Wordle). The early-exit+ # logic has been removed so that we always capture a *fresh* DOM+ # snapshot. If the snapshot happens to be byte-for-byte identical+ # to the previous one we simply return it after the standard+ # comparison step below; otherwise callers receive the updated+ # snapshot even when the URL did not change.# ensure DOM stabilityawait self.page.wait_for_load_state(diff --git a/examples/toolkits/browser_non_visual_toolkit_example.py b/examples/toolkits/browser_non_visual_toolkit_example.pyindex 2e4093b04..ecee5b084 100644--- a/examples/toolkits/browser_non_visual_toolkit_example.py+++ b/examples/toolkits/browser_non_visual_toolkit_example.py@@ -23,15 +23,17 @@ logging.basicConfig(level=logging.DEBUG,format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',handlers=[- logging.StreamHandler(), # Console output- logging.FileHandler('browser_dom_debug.log'), # File output+ logging.StreamHandler(),],)logging.getLogger('camel.agents').setLevel(logging.INFO)logging.getLogger('camel.models').setLevel(logging.INFO)--USER_DATA_DIR = r"D:/User_Data"+logging.getLogger('camel.models').setLevel(logging.INFO)+logging.getLogger('camel.toolkits.non_visual_browser_toolkit').setLevel(+ logging.DEBUG+)+USER_DATA_DIR = "User_Data"model_backend = ModelFactory.create(model_platform=ModelPlatformType.OPENAI,@@ -40,7 +42,7 @@ model_backend = ModelFactory.create()web_toolkit = BrowserNonVisualToolkit(- headless=False,+ headless=False, user_data_dir=USER_DATA_DIR)agent = ChatAgent(@@ -49,25 +51,29 @@ agent = ChatAgent(max_iteration=10,)-TASK_PROMPT = """Click "Sign in to Google Account" on Google.-If a CAPTCHA appears requiring user input, call the wait_user function —-wait 5 seconds each time and keep waiting until you detect that the snapshot-no longer contains content requiring user action.-Make sure not to help the user fill in their account or CAPTCHA information.-Before the task is completed, you must call the tool in every session and-use the wait_user function whenever needed.+TASK_PROMPT = """Play this game:+https://www.nytimes.com/games/wordle/index.html+You need to enter meaningful five-letter words, such as "Apple".+Before completing the task, do not do anything outside of using the call tool.+If the game tells you that something is wrong, you must resolve it yourself.+Also, you must attempt all six tries.+If the registration window can be closed, please close it."""async def main() -> None:response = await agent.astep(TASK_PROMPT)-print("Task:", TASK_PROMPT)print(f"Using user data directory: {USER_DATA_DIR}\n")print("Response from agent:")print(response.msgs[0].content if response.msgs else "<no response>")+ try:+ await web_toolkit.close_browser()+ except Exception as err:+ logging.warning("Failed to close browser session explicitly: %s", err)+if __name__ == "__main__":asyncio.run(main())