Skip to content

A programming framework for agentic AI

Various web surfer fixes. (#5393)

afourneycf798aef3f19
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py
index 90ddbdb2b..73c0799ff 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py
@@ -53,9 +53,9 @@ from ._tool_definitions import (
TOOL_CLICK,
TOOL_HISTORY_BACK,
TOOL_HOVER,
- TOOL_PAGE_DOWN,
- TOOL_PAGE_UP,
TOOL_READ_PAGE_AND_ANSWER,
+ TOOL_SCROLL_DOWN,
+ TOOL_SCROLL_UP,
TOOL_SLEEP,
TOOL_SUMMARIZE_PAGE,
TOOL_TYPE,
@@ -466,11 +466,11 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):
# We can scroll up
if viewport["pageTop"] > 5:
- tools.append(TOOL_PAGE_UP)
+ tools.append(TOOL_SCROLL_UP)
# Can scroll down
if (viewport["pageTop"] + viewport["height"] + 5) < viewport["scrollHeight"]:
- tools.append(TOOL_PAGE_DOWN)
+ tools.append(TOOL_SCROLL_DOWN)
# Focus hint
focused = await self._playwright_controller.get_focused_rect_id(self._page)
@@ -479,6 +479,8 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):
name = self._target_name(focused, rects)
if name:
name = f"(and name '{name}') "
+ else:
+ name = ""
role = "control"
try:
@@ -613,10 +615,10 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):
self._last_download = None
if reset_prior_metadata and self._prior_metadata_hash is not None:
self._prior_metadata_hash = None
- elif name == "page_up":
+ elif name == "scroll_up":
action_description = "I scrolled up one page in the browser."
await self._playwright_controller.page_up(self._page)
- elif name == "page_down":
+ elif name == "scroll_down":
action_description = "I scrolled down one page in the browser."
await self._playwright_controller.page_down(self._page)
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py
index 050b267e0..59a0a7c95 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py
@@ -8,7 +8,7 @@ You are to respond to the most recent request by selecting an appropriate tool f
{tool_names}
When deciding between tools, consider if the request can be best addressed by:
- - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)
+ - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text, or hovering over an element might be most appropriate)
- contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)
- on some other website entirely (in which case actions like performing a new web search might be the best option)
"""
@@ -29,7 +29,7 @@ You are to respond to the most recent request by selecting an appropriate tool f
{tool_names}
When deciding between tools, consider if the request can be best addressed by:
- - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)
+ - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text, or hovering over an element might be most appropriate)
- contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)
- on some other website entirely (in which case actions like performing a new web search might be the best option)
"""
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py
index fd2928248..c80f6de31 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py
@@ -87,11 +87,11 @@ TOOL_HISTORY_BACK: ToolSchema = _load_tool(
}
)
-TOOL_PAGE_UP: ToolSchema = _load_tool(
+TOOL_SCROLL_UP: ToolSchema = _load_tool(
{
"type": "function",
"function": {
- "name": "page_up",
+ "name": "scroll_up",
"description": "Scrolls the entire browser viewport one page UP towards the beginning.",
"parameters": {
"type": "object",
@@ -107,11 +107,11 @@ TOOL_PAGE_UP: ToolSchema = _load_tool(
}
)
-TOOL_PAGE_DOWN: ToolSchema = _load_tool(
+TOOL_SCROLL_DOWN: ToolSchema = _load_tool(
{
"type": "function",
"function": {
- "name": "page_down",
+ "name": "scroll_down",
"description": "Scrolls the entire browser viewport one page DOWN towards the end.",
"parameters": {
"type": "object",
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py
index 412bc07db..b99fbc49a 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/playwright_controller.py
@@ -122,7 +122,7 @@ class PlaywrightController:
pass
return visualviewport_from_dict(await page.evaluate("MultimodalWebSurfer.getVisualViewport();"))
- async def get_focused_rect_id(self, page: Page) -> str:
+ async def get_focused_rect_id(self, page: Page) -> str | None:
"""
Retrieve the ID of the currently focused element.
@@ -130,7 +130,7 @@ class PlaywrightController:
page (Page): The Playwright page object.
Returns:
- str: The ID of the focused element.
+ str: The ID of the focused element or None if no control has focus.
"""
assert page is not None
try:
@@ -138,7 +138,7 @@ class PlaywrightController:
except Exception:
pass
result = await page.evaluate("MultimodalWebSurfer.getFocusedElementId();")
- return str(result)
+ return None if result is None else str(result)
async def get_page_metadata(self, page: Page) -> Dict[str, Any]:
"""