A programming framework for agentic AI
WebSurfer: print viewport text (#5329)
afourney877796ded12a
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.pyindex f90dc01cd..748f61e42 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py@@ -264,8 +264,6 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):TOOL_SLEEP,TOOL_HOVER,]- # Number of lines of text to extract from the page in the absence of OCR- self.n_lines_page_text = 50self.did_lazy_init = False # flag to check if we have initialized the browserasync def _lazy_init(@@ -743,7 +741,7 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):ocr_text = (await self._get_ocr_text(new_screenshot, cancellation_token=cancellation_token)if self.use_ocr is True- else await self._playwright_controller.get_webpage_text(self._page, n_lines=self.n_lines_page_text)+ else await self._playwright_controller.get_visible_text(self._page))# Return the complete observation@@ -752,7 +750,7 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):if self.use_ocr:message_content += f"Automatic OCR of the page screenshot has detected the following text:\n\n{ocr_text}"else:- message_content += f"The first {self.n_lines_page_text} lines of the page text is:\n\n{ocr_text}"+ message_content += f"The following text is visible in the viewport:\n\n{ocr_text}"return [message_content,diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/page_script.js b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/page_script.jsindex 95b32d5b9..1363e83db 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/page_script.js+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/page_script.js@@ -367,10 +367,63 @@ var MultimodalWebSurfer = MultimodalWebSurfer || (function() {return results;};++ let getVisibleText = function() {+ // Get the window’s current viewport boundaries+ const viewportHeight = window.innerHeight || document.documentElement.clientHeight;+ const viewportWidth = window.innerWidth || document.documentElement.clientWidth;++ let textInView = "";+ const walker = document.createTreeWalker(+ document.body,+ NodeFilter.SHOW_TEXT,+ null,+ false+ );++ while (walker.nextNode()) {+ const textNode = walker.currentNode;+ // Create a range to retrieve bounding rectangles of the current text node+ const range = document.createRange();+ range.selectNodeContents(textNode);++ const rects = range.getClientRects();++ // Check if any rect is inside (or partially inside) the viewport+ for (const rect of rects) {+ const isVisible =+ rect.width > 0 &&+ rect.height > 0 &&+ rect.bottom >= 0 &&+ rect.right >= 0 &&+ rect.top <= viewportHeight &&+ rect.left <= viewportWidth;++ if (isVisible) {+ textInView += textNode.nodeValue.replace(/\s+/g, " ");+ // Is the parent a block element?+ if (textNode.parentNode) {+ const parent = textNode.parentNode;+ const style = window.getComputedStyle(parent);+ if (["inline", "hidden", "none"].indexOf(style.display) === -1) {+ textInView += "\n";+ }+ }+ break; // No need to check other rects once found visible+ }+ }+ }++ // Remove blank lines from textInView+ textInView = textInView.replace(/^\s*\n/gm, "").trim().replace(/\n+/g, "\n");+ return textInView;+ };+return {getInteractiveRects: getInteractiveRects,getVisualViewport: getVisualViewport,getFocusedElementId: getFocusedElementId,getPageMetadata: getPageMetadata,+ getVisibleText: getVisibleText,};})();diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.pyindex d691826a4..412bc07db 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py@@ -527,6 +527,25 @@ class PlaywrightController:except Exception:return ""+ async def get_visible_text(self, page: Page) -> str:+ """+ Retrieve the text content of the browser viewport (approximately).++ Args:+ page (Page): The Playwright page object.++ Returns:+ str: The text content of the page.+ """+ assert page is not None+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = await page.evaluate("MultimodalWebSurfer.getVisibleText();")+ assert isinstance(result, str)+ return result+async def get_page_markdown(self, page: Page) -> str:"""Retrieve the markdown content of the web page.