A programming framework for agentic AI
Various web surfer fixes. (#5393)
afourneycf798aef3f19
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.pyindex 90ddbdb2b..73c0799ff 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py@@ -53,9 +53,9 @@ from ._tool_definitions import (TOOL_CLICK,TOOL_HISTORY_BACK,TOOL_HOVER,- TOOL_PAGE_DOWN,- TOOL_PAGE_UP,TOOL_READ_PAGE_AND_ANSWER,+ TOOL_SCROLL_DOWN,+ TOOL_SCROLL_UP,TOOL_SLEEP,TOOL_SUMMARIZE_PAGE,TOOL_TYPE,@@ -466,11 +466,11 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):# We can scroll upif viewport["pageTop"] > 5:- tools.append(TOOL_PAGE_UP)+ tools.append(TOOL_SCROLL_UP)# Can scroll downif (viewport["pageTop"] + viewport["height"] + 5) < viewport["scrollHeight"]:- tools.append(TOOL_PAGE_DOWN)+ tools.append(TOOL_SCROLL_DOWN)# Focus hintfocused = await self._playwright_controller.get_focused_rect_id(self._page)@@ -479,6 +479,8 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):name = self._target_name(focused, rects)if name:name = f"(and name '{name}') "+ else:+ name = ""role = "control"try:@@ -613,10 +615,10 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):self._last_download = Noneif reset_prior_metadata and self._prior_metadata_hash is not None:self._prior_metadata_hash = None- elif name == "page_up":+ elif name == "scroll_up":action_description = "I scrolled up one page in the browser."await self._playwright_controller.page_up(self._page)- elif name == "page_down":+ elif name == "scroll_down":action_description = "I scrolled down one page in the browser."await self._playwright_controller.page_down(self._page)diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.pyindex 050b267e0..59a0a7c95 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py@@ -8,7 +8,7 @@ You are to respond to the most recent request by selecting an appropriate tool f{tool_names}When deciding between tools, consider if the request can be best addressed by:- - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)+ - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text, or hovering over an element might be most appropriate)- contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)- on some other website entirely (in which case actions like performing a new web search might be the best option)"""@@ -29,7 +29,7 @@ You are to respond to the most recent request by selecting an appropriate tool f{tool_names}When deciding between tools, consider if the request can be best addressed by:- - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)+ - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text, or hovering over an element might be most appropriate)- contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)- on some other website entirely (in which case actions like performing a new web search might be the best option)"""diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.pyindex fd2928248..c80f6de31 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py@@ -87,11 +87,11 @@ TOOL_HISTORY_BACK: ToolSchema = _load_tool(})-TOOL_PAGE_UP: ToolSchema = _load_tool(+TOOL_SCROLL_UP: ToolSchema = _load_tool({"type": "function","function": {- "name": "page_up",+ "name": "scroll_up","description": "Scrolls the entire browser viewport one page UP towards the beginning.","parameters": {"type": "object",@@ -107,11 +107,11 @@ TOOL_PAGE_UP: ToolSchema = _load_tool(})-TOOL_PAGE_DOWN: ToolSchema = _load_tool(+TOOL_SCROLL_DOWN: ToolSchema = _load_tool({"type": "function","function": {- "name": "page_down",+ "name": "scroll_down","description": "Scrolls the entire browser viewport one page DOWN towards the end.","parameters": {"type": "object",diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.pyindex 412bc07db..b99fbc49a 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py@@ -122,7 +122,7 @@ class PlaywrightController:passreturn visualviewport_from_dict(await page.evaluate("MultimodalWebSurfer.getVisualViewport();"))- async def get_focused_rect_id(self, page: Page) -> str:+ async def get_focused_rect_id(self, page: Page) -> str | None:"""Retrieve the ID of the currently focused element.@@ -130,7 +130,7 @@ class PlaywrightController:page (Page): The Playwright page object.Returns:- str: The ID of the focused element.+ str: The ID of the focused element or None if no control has focus."""assert page is not Nonetry:@@ -138,7 +138,7 @@ class PlaywrightController:except Exception:passresult = await page.evaluate("MultimodalWebSurfer.getFocusedElementId();")- return str(result)+ return None if result is None else str(result)async def get_page_metadata(self, page: Page) -> Dict[str, Any]:"""