A programming framework for agentic AI
Websurfer Refactor (#4165)
Hussein Mozannar317d5d03ec50
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.pyindex 60611cd7d..ae2906a77 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py@@ -15,7 +15,6 @@ from typing import (Optional,Sequence,Tuple,- Union,cast,)@@ -38,15 +37,17 @@ from autogen_core.components.models import (SystemMessage,UserMessage,)-from playwright._impl._errors import Error as PlaywrightError-from playwright._impl._errors import TimeoutError+from PIL import Imagefrom playwright.async_api import BrowserContext, Download, Page, Playwright, async_playwrightfrom ._events import WebSurferEvent+from ._playwright_controller import PlaywrightController+from ._prompts import WEB_SURFER_OCR_PROMPT, WEB_SURFER_QA_PROMPT, WEB_SURFER_QA_SYSTEM_MESSAGE, WEB_SURFER_TOOL_PROMPTfrom ._set_of_mark import add_set_of_markfrom ._tool_definitions import (TOOL_CLICK,TOOL_HISTORY_BACK,+ TOOL_HOVER,TOOL_PAGE_DOWN,TOOL_PAGE_UP,TOOL_READ_PAGE_AND_ANSWER,@@ -58,13 +59,7 @@ from ._tool_definitions import (TOOL_VISIT_URL,TOOL_WEB_SEARCH,)-from ._types import (- InteractiveRegion,- UserContent,- VisualViewport,- interactiveregion_from_dict,- visualviewport_from_dict,-)+from ._types import InteractiveRegion, UserContentfrom ._utils import message_content_to_str# Viewport dimensions@@ -98,6 +93,11 @@ class MultimodalWebSurfer(BaseChatAgent):downloads_folder: str | None = None,debug_dir: str | None = os.getcwd(),to_save_screenshots: bool = False,+ animate_actions: bool = False,+ use_ocr: bool = True,+ to_resize_viewport: bool = True,+ playwright: Playwright | None = None,+ context: BrowserContext | None = None,):"""Initialize the MultimodalWebSurfer.@@ -113,6 +113,11 @@ class MultimodalWebSurfer(BaseChatAgent):downloads_folder (str | None): The folder to save downloads. Defaults to None.debug_dir (str | None): The directory to save debug information. Defaults to the current working directory.to_save_screenshots (bool): Whether to save screenshots. Defaults to False.+ animate_actions (bool): Whether to animate actions. Defaults to False.+ use_ocr (bool): Whether to use OCR to extract text from screenshots, otherwise extract text from page. Defaults to True.+ to_resize_viewport (bool): Whether to resize the viewport. Defaults to True.+ playwright (Playwright | None): The playwright instance to use. Defaults to None and creates a new one.+ context (BrowserContext | None): The browser context to use. Defaults to None and creates a new one."""super().__init__(name, description)self._model_client = model_client@@ -124,21 +129,18 @@ class MultimodalWebSurfer(BaseChatAgent):self.downloads_folder = downloads_folderself.debug_dir = debug_dirself.to_save_screenshots = to_save_screenshots+ self.use_ocr = use_ocr+ self.to_resize_viewport = to_resize_viewport+ self.animate_actions = animate_actions- self._chat_history: List[LLMMessage] = []-- # Call init to set these- self._playwright: Playwright | None = None- self._context: BrowserContext | None = None+ # Call init to set these in case not set+ self._playwright: Playwright | None = playwright+ self._context: BrowserContext | None = contextself._page: Page | None = Noneself._last_download: Download | None = Noneself._prior_metadata_hash: str | None = Noneself.logger = logging.getLogger(EVENT_LOGGER_NAME + f".{self.name}.MultimodalWebSurfer")-- # Read page_script- self._page_script: str = ""- with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"), "rt") as fh:- self._page_script = fh.read()+ self._chat_history: List[LLMMessage] = []# Define the download handlerdef _download_handler(download: Download) -> None:@@ -146,6 +148,27 @@ class MultimodalWebSurfer(BaseChatAgent):self._download_handler = _download_handler+ # Define the Playwright controller that handles the browser interactions+ self._playwright_controller = PlaywrightController(+ animate_actions=self.animate_actions,+ downloads_folder=self.downloads_folder,+ viewport_width=VIEWPORT_WIDTH,+ viewport_height=VIEWPORT_HEIGHT,+ _download_handler=self._download_handler,+ to_resize_viewport=self.to_resize_viewport,+ )+ self.default_tools = [+ TOOL_VISIT_URL,+ TOOL_HISTORY_BACK,+ TOOL_CLICK,+ TOOL_TYPE,+ TOOL_READ_PAGE_AND_ANSWER,+ TOOL_SUMMARIZE_PAGE,+ TOOL_SLEEP,+ TOOL_HOVER,+ ]+ self.did_lazy_init = False+@propertydef produced_message_types(self) -> List[type[ChatMessage]]:return [MultiModalMessage]@@ -169,9 +192,18 @@ class MultimodalWebSurfer(BaseChatAgent):)async def on_reset(self, cancellation_token: CancellationToken) -> None:+ if not self.did_lazy_init:+ returnassert self._page is not None+self._chat_history.clear()- await self._visit_page(self.start_page)+ reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(+ self._page, self.start_page+ )+ if reset_last_download and self._last_download is not None:+ self._last_download = None+ if reset_prior_metadata and self._prior_metadata_hash is not None:+ self._prior_metadata_hash = Noneif self.to_save_screenshots:current_timestamp = "_" + int(time.time()).__str__()screenshot_png_name = "screenshot" + current_timestamp + ".png"@@ -202,18 +234,20 @@ class MultimodalWebSurfer(BaseChatAgent):launch_args: Dict[str, Any] = {"headless": self.headless}if self.browser_channel is not None:launch_args["channel"] = self.browser_channel- self._playwright = await async_playwright().start()+ if self._playwright is None:+ self._playwright = await async_playwright().start()# Create the context -- are we launching persistent?- if self.browser_data_dir is None:- browser = await self._playwright.chromium.launch(**launch_args)- self._context = await browser.new_context(- user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"- )- else:- self._context = await self._playwright.chromium.launch_persistent_context(- self.browser_data_dir, **launch_args- )+ if self._context is None:+ if self.browser_data_dir is None:+ browser = await self._playwright.chromium.launch(**launch_args)+ self._context = await browser.new_context(+ user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"+ )+ else:+ self._context = await self._playwright.chromium.launch_persistent_context(+ self.browser_data_dir, **launch_args+ )# Create the pageself._context.set_default_timeout(60000) # One minute@@ -221,7 +255,8 @@ class MultimodalWebSurfer(BaseChatAgent):assert self._page is not None# self._page.route(lambda x: True, self._route_handler)self._page.on("download", self._download_handler)- await self._page.set_viewport_size({"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT})+ if self.to_resize_viewport:+ await self._page.set_viewport_size({"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT})await self._page.add_init_script(path=os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"))@@ -231,10 +266,6 @@ class MultimodalWebSurfer(BaseChatAgent):# Prepare the debug directory -- which stores the screenshots generated throughout the processawait self._set_debug_dir(self.debug_dir)- async def _sleep(self, duration: Union[int, float]) -> None:- assert self._page is not None- await self._page.wait_for_timeout(duration * 1000)-async def _set_debug_dir(self, debug_dir: str | None) -> None:assert self._page is not Noneself.debug_dir = debug_dir@@ -275,7 +306,7 @@ class MultimodalWebSurfer(BaseChatAgent):aria_name = re.sub(r"[\n\r]+", " ", rects[r].get("aria_name", "")).strip()# What are the actions?- actions = ['"click"']+ actions = ['"click", "hover"']if rects[r]["role"] in ["textbox", "searchbox", "search"]:actions = ['"input_text"']actions_str = "[" + ",".join(actions) + "]"@@ -289,7 +320,6 @@ class MultimodalWebSurfer(BaseChatAgent):message: List[FunctionCall],rects: Dict[str, InteractiveRegion],tool_names: str,- use_ocr: bool = True,cancellation_token: Optional[CancellationToken] = None,) -> Tuple[bool, UserContent]:name = message[0].name@@ -311,30 +341,43 @@ class MultimodalWebSurfer(BaseChatAgent):action_description = f"I typed '{url}' into the browser address bar."# Check if the argument starts with a known protocolif url.startswith(("https://", "http://", "file://", "about:")):- await self._visit_page(url)+ reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(+ self._page, url+ )# If the argument contains a space, treat it as a search queryelif " " in url:- await self._visit_page(f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH")+ reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(+ self._page, f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH"+ )# Otherwise, prefix with https://else:- await self._visit_page("https://" + url)-+ reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(+ self._page, "https://" + url+ )+ if reset_last_download and self._last_download is not None:+ self._last_download = None+ if reset_prior_metadata and self._prior_metadata_hash is not None:+ self._prior_metadata_hash = Noneelif name == "history_back":action_description = "I clicked the browser back button."- await self._back()+ await self._playwright_controller.back(self._page)elif name == "web_search":query = args.get("query")action_description = f"I typed '{query}' into the browser search bar."- await self._visit_page(f"https://www.bing.com/search?q={quote_plus(query)}&FORM=QBLH")-+ reset_prior_metadata, reset_last_download = await self._playwright_controller.visit_page(+ self._page, f"https://www.bing.com/search?q={quote_plus(query)}&FORM=QBLH"+ )+ if reset_last_download and self._last_download is not None:+ self._last_download = None+ if reset_prior_metadata and self._prior_metadata_hash is not None:+ self._prior_metadata_hash = Noneelif name == "page_up":action_description = "I scrolled up one page in the browser."- await self._page_up()-+ await self._playwright_controller.page_up(self._page)elif name == "page_down":action_description = "I scrolled down one page in the browser."- await self._page_down()+ await self._playwright_controller.page_down(self._page)elif name == "click":target_id = str(args.get("target_id"))@@ -343,8 +386,17 @@ class MultimodalWebSurfer(BaseChatAgent):action_description = f"I clicked '{target_name}'."else:action_description = "I clicked the control."- await self._click_id(target_id)-+ new_page_tentative = await self._playwright_controller.click_id(self._page, target_id)+ if new_page_tentative is not None:+ self._page = new_page_tentative+ self._prior_metadata_hash = None+ self.logger.info(+ WebSurferEvent(+ source=self.name,+ url=self._page.url,+ message="New tab or window.",+ )+ )elif name == "input_text":input_field_id = str(args.get("input_field_id"))text_value = str(args.get("text_value"))@@ -353,7 +405,7 @@ class MultimodalWebSurfer(BaseChatAgent):action_description = f"I typed '{text_value}' into '{input_field_name}'."else:action_description = f"I input '{text_value}'."- await self._fill_id(input_field_id, text_value)+ await self._playwright_controller.fill_id(self._page, input_field_id, text_value)elif name == "scroll_element_up":target_id = str(args.get("target_id"))@@ -364,7 +416,7 @@ class MultimodalWebSurfer(BaseChatAgent):else:action_description = "I scrolled the control up."- await self._scroll_id(target_id, "up")+ await self._playwright_controller.scroll_id(self._page, target_id, "up")elif name == "scroll_element_down":target_id = str(args.get("target_id"))@@ -375,17 +427,36 @@ class MultimodalWebSurfer(BaseChatAgent):else:action_description = "I scrolled the control down."- await self._scroll_id(target_id, "down")+ await self._playwright_controller.scroll_id(self._page, target_id, "down")++ elif name == "answer_question":+ question = str(args.get("question"))+ action_description = f"I answered the following question '{question}' based on the web page."+ # Do Q&A on the DOM. No need to take further action. Browser state does not change.+ return False, await self._summarize_page(question=question, cancellation_token=cancellation_token)+ elif name == "summarize_page":+ # Summarize the DOM. No need to take further action. Browser state does not change.+ action_description = "I summarized the current web page"+ return False, await self._summarize_page(cancellation_token=cancellation_token)++ elif name == "hover":+ target_id = str(args.get("target_id"))+ target_name = self._target_name(target_id, rects)+ if target_name:+ action_description = f"I hovered over '{target_name}'."+ else:+ action_description = "I hovered over the control."+ await self._playwright_controller.hover_id(self._page, target_id)elif name == "sleep":action_description = "I am waiting a short period of time before taking further action."- await self._sleep(3) # There's a 2s sleep below too+ await self._playwright_controller.sleep(self._page, 3) # There's a 2s sleep below tooelse:raise ValueError(f"Unknown tool '{name}'. Please choose from:\n\n{tool_names}")await self._page.wait_for_load_state()- await self._sleep(3)+ await self._playwright_controller.sleep(self._page, 3) # There's a 2s sleep below too# Handle downloadsif self._last_download is not None and self.downloads_folder is not None:@@ -399,7 +470,7 @@ class MultimodalWebSurfer(BaseChatAgent):await self._page.wait_for_load_state()# Handle metadata- page_metadata = json.dumps(await self._get_page_metadata(), indent=4)+ page_metadata = json.dumps(await self._playwright_controller.get_page_metadata(self._page), indent=4)metadata_hash = hashlib.md5(page_metadata.encode("utf-8")).hexdigest()if metadata_hash != self._prior_metadata_hash:page_metadata = (@@ -410,7 +481,7 @@ class MultimodalWebSurfer(BaseChatAgent):self._prior_metadata_hash = metadata_hash# Describe the viewport of the new page in words- viewport = await self._get_visual_viewport()+ viewport = await self._playwright_controller.get_visual_viewport(self._page)percent_visible = int(viewport["height"] * 100 / viewport["scrollHeight"])percent_scrolled = int(viewport["pageTop"] * 100 / viewport["scrollHeight"])if percent_scrolled < 1: # Allow some rounding error@@ -435,7 +506,9 @@ class MultimodalWebSurfer(BaseChatAgent):)ocr_text = (- await self._get_ocr_text(new_screenshot, cancellation_token=cancellation_token) if use_ocr is True else ""+ await self._get_ocr_text(new_screenshot, cancellation_token=cancellation_token)+ if self.use_ocr is True+ else await self._playwright_controller.get_webpage_text(self._page))# Return the complete observation@@ -451,7 +524,7 @@ class MultimodalWebSurfer(BaseChatAgent):"""Generates the actual reply. First calls the LLM to figure out which tool to use, then executes the tool."""# Lazy init- if self._playwright is None:+ if not self.did_lazy_init:await self._lazy_init()assert self._page is not None@@ -471,8 +544,8 @@ class MultimodalWebSurfer(BaseChatAgent):history.append(SystemMessage(content=content))# Ask the page for interactive elements, then prepare the state-of-mark screenshot- rects = await self._get_interactive_rects()- viewport = await self._get_visual_viewport()+ rects = await self._playwright_controller.get_interactive_rects(self._page)+ viewport = await self._playwright_controller.get_visual_viewport(self._page)screenshot = await self._page.screenshot()som_screenshot, visible_rects, rects_above, rects_below = add_set_of_mark(screenshot, rects)@@ -488,18 +561,8 @@ class MultimodalWebSurfer(BaseChatAgent):))# What tools are available?- tools = [- TOOL_VISIT_URL,- TOOL_HISTORY_BACK,- TOOL_CLICK,- TOOL_TYPE,- TOOL_SUMMARIZE_PAGE,- TOOL_READ_PAGE_AND_ANSWER,- TOOL_SLEEP,- ]+ tools = self.default_tools.copy()- # Can we reach Bing to search?- # if self._navigation_allow_list("https://www.bing.com/"):tools.append(TOOL_WEB_SEARCH)# We can scroll up@@ -511,7 +574,7 @@ class MultimodalWebSurfer(BaseChatAgent):tools.append(TOOL_PAGE_DOWN)# Focus hint- focused = await self._get_focused_rect_id()+ focused = await self._playwright_controller.get_focused_rect_id(self._page)focused_hint = ""if focused:name = self._target_name(focused, rects)@@ -549,18 +612,13 @@ class MultimodalWebSurfer(BaseChatAgent):tool_names = "\n".join([t["name"] for t in tools])- text_prompt = f"""-Consider the following screenshot of a web browser, which is open to the page '{self._page.url}'. In this screenshot, interactive elements are outlined in bounding boxes of different colors. Each bounding box has a numeric ID label in the same color. Additional information about each visible label is listed below:--{visible_targets}{other_targets_str}{focused_hint}You are to respond to the user's most recent request by selecting an appropriate tool the following set, or by answering the question directly if possible:--{tool_names}--When deciding between tools, consider if the request can be best addressed by:- - the contents of the current viewport (in which case actions like clicking links, clicking buttons, or inputting text might be most appropriate)- - contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)- - on some other website entirely (in which case actions like performing a new web search might be the best option)-""".strip()+ text_prompt = WEB_SURFER_TOOL_PROMPT.format(+ url=self._page.url,+ visible_targets=visible_targets,+ other_targets_str=other_targets_str,+ focused_hint=focused_hint,+ tool_names=tool_names,+ ).strip()# Scale the screenshot for the MLM, and close the originalscaled_screenshot = som_screenshot.resize((MLM_WIDTH, MLM_HEIGHT))@@ -574,7 +632,6 @@ When deciding between tools, consider if the request can be best addressed by:history, tools=tools, extra_create_args={"tool_choice": "auto"}, cancellation_token=cancellation_token) # , "parallel_tool_calls": False})message = response.content-self._last_download = Noneif isinstance(message, str):@@ -587,181 +644,6 @@ When deciding between tools, consider if the request can be best addressed by:# Not sure what happened hereraise AssertionError(f"Unknown response format '{message}'")- async def _get_interactive_rects(self) -> Dict[str, InteractiveRegion]:- assert self._page is not None-- # Read the regions from the DOM- try:- await self._page.evaluate(self._page_script)- except Exception:- pass- result = cast(- Dict[str, Dict[str, Any]], await self._page.evaluate("MultimodalWebSurfer.getInteractiveRects();")- )-- # Convert the results into appropriate types- assert isinstance(result, dict)- typed_results: Dict[str, InteractiveRegion] = {}- for k in result:- assert isinstance(k, str)- typed_results[k] = interactiveregion_from_dict(result[k])-- return typed_results-- async def _get_visual_viewport(self) -> VisualViewport:- assert self._page is not None- try:- await self._page.evaluate(self._page_script)- except Exception:- pass- return visualviewport_from_dict(await self._page.evaluate("MultimodalWebSurfer.getVisualViewport();"))-- async def _get_focused_rect_id(self) -> str:- assert self._page is not None- try:- await self._page.evaluate(self._page_script)- except Exception:- pass- result = await self._page.evaluate("MultimodalWebSurfer.getFocusedElementId();")- return str(result)-- async def _get_page_metadata(self) -> Dict[str, Any]:- assert self._page is not None- try:- await self._page.evaluate(self._page_script)- except Exception:- pass- result = await self._page.evaluate("MultimodalWebSurfer.getPageMetadata();")- assert isinstance(result, dict)- return cast(Dict[str, Any], result)-- async def _on_new_page(self, page: Page) -> None:- self._page = page- assert self._page is not None- # self._page.route(lambda x: True, self._route_handler)- self._page.on("download", self._download_handler)- await self._page.set_viewport_size({"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT})- await self._sleep(0.2)- self._prior_metadata_hash = None- await self._page.add_init_script(- path=os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js")- )- await self._page.wait_for_load_state()-- async def _back(self) -> None:- assert self._page is not None- await self._page.go_back()-- async def _visit_page(self, url: str) -> None:- assert self._page is not None- try:- # Regular webpage- await self._page.goto(url)- await self._page.wait_for_load_state()- self._prior_metadata_hash = None- except Exception as e_outer:- # Downloaded file- if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):- async with self._page.expect_download() as download_info:- try:- await self._page.goto(url)- except Exception as e_inner:- if "net::ERR_ABORTED" in str(e_inner):- pass- else:- raise e_inner- download = await download_info.value- fname = os.path.join(self.downloads_folder, download.suggested_filename)- await download.save_as(fname)- message = f"<body style=\"margin: 20px;\"><h1>Successfully downloaded '{download.suggested_filename}' to local path:<br><br>{fname}</h1></body>"- await self._page.goto(- "data:text/html;base64," + base64.b64encode(message.encode("utf-8")).decode("utf-8")- )- self._last_download = None # Since we already handled it- else:- raise e_outer-- async def _page_down(self) -> None:- assert self._page is not None- await self._page.evaluate(f"window.scrollBy(0, {VIEWPORT_HEIGHT-50});")-- async def _page_up(self) -> None:- assert self._page is not None- await self._page.evaluate(f"window.scrollBy(0, -{VIEWPORT_HEIGHT-50});")-- async def _click_id(self, identifier: str) -> None:- assert self._page is not None- target = self._page.locator(f"[__elementId='{identifier}']")-- # See if it exists- try:- await target.wait_for(timeout=100)- except TimeoutError:- raise ValueError("No such element.") from None-- # Click it- await target.scroll_into_view_if_needed()- box = cast(Dict[str, Union[int, float]], await target.bounding_box())- try:- # Give it a chance to open a new page- # TODO: Having trouble with these types- async with self._page.expect_event("popup", timeout=1000) as page_info: # type: ignore- await self._page.mouse.click(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2, delay=10)- # If we got this far without error, than a popup or new tab opened. Handle it.-- new_page = await page_info.value # type: ignore-- assert isinstance(new_page, Page)- await self._on_new_page(new_page)-- self.logger.info(- WebSurferEvent(- source=self.name,- url=self._page.url,- message="New tab or window.",- )- )-- except TimeoutError:- pass-- async def _fill_id(self, identifier: str, value: str) -> None:- assert self._page is not None- target = self._page.locator(f"[__elementId='{identifier}']")-- # See if it exists- try:- await target.wait_for(timeout=100)- except TimeoutError:- raise ValueError("No such element.") from None-- # Fill it- await target.scroll_into_view_if_needed()- await target.focus()- try:- await target.fill(value)- except PlaywrightError:- await target.press_sequentially(value)- await target.press("Enter")-- async def _scroll_id(self, identifier: str, direction: str) -> None:- assert self._page is not None- await self._page.evaluate(- f"""- (function() {{- let elm = document.querySelector("[__elementId='{identifier}']");- if (elm) {{- if ("{direction}" == "up") {{- elm.scrollTop = Math.max(0, elm.scrollTop - elm.clientHeight);- }}- else {{- elm.scrollTop = Math.min(elm.scrollHeight - elm.clientHeight, elm.scrollTop + elm.clientHeight);- }}- }}- }})();- """- )-async def _get_ocr_text(self, image: bytes | io.BufferedIOBase | PIL.Image.Image, cancellation_token: Optional[CancellationToken] = None) -> str:@@ -783,7 +665,7 @@ When deciding between tools, consider if the request can be best addressed by:messages.append(UserMessage(content=[- "Please transcribe all visible text on this page, including both main content and the labels of UI elements.",+ WEB_SURFER_OCR_PROMPT,AGImage.from_pil(scaled_screenshot),],source=self.name,@@ -793,3 +675,68 @@ When deciding between tools, consider if the request can be best addressed by:scaled_screenshot.close()assert isinstance(response.content, str)return response.content++ async def _summarize_page(+ self,+ question: str | None = None,+ cancellation_token: Optional[CancellationToken] = None,+ ) -> str:+ assert self._page is not None++ page_markdown: str = await self._playwright_controller.get_page_markdown(self._page)++ title: str = self._page.url+ try:+ title = await self._page.title()+ except Exception:+ pass++ # Take a screenshot and scale it+ screenshot = Image.open(io.BytesIO(await self._page.screenshot()))+ scaled_screenshot = screenshot.resize((MLM_WIDTH, MLM_HEIGHT))+ screenshot.close()+ ag_image = AGImage.from_pil(scaled_screenshot)++ # Prepare the system prompt+ messages: List[LLMMessage] = []+ messages.append(SystemMessage(content=WEB_SURFER_QA_SYSTEM_MESSAGE))+ prompt = WEB_SURFER_QA_PROMPT(title, question)+ # Grow the buffer (which is added to the prompt) until we overflow the context window or run out of lines+ buffer = ""+ # for line in re.split(r"([\r\n]+)", page_markdown):+ for line in page_markdown.splitlines():+ message = UserMessage(+ # content=[+ prompt + buffer + line,+ # ag_image,+ # ],+ source=self.name,+ )++ remaining = self._model_client.remaining_tokens(messages + [message])+ if remaining > SCREENSHOT_TOKENS:+ buffer += line+ else:+ break++ # Nothing to do+ buffer = buffer.strip()+ if len(buffer) == 0:+ return "Nothing to summarize."++ # Append the message+ messages.append(+ UserMessage(+ content=[+ prompt + buffer,+ ag_image,+ ],+ source=self.name,+ )+ )++ # Generate the response+ response = await self._model_client.create(messages, cancellation_token=cancellation_token)+ scaled_screenshot.close()+ assert isinstance(response.content, str)+ return response.contentdiff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_playwright_controller.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_playwright_controller.pynew file mode 100644index 000000000..162980af1--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_playwright_controller.py@@ -0,0 +1,380 @@+import asyncio+import base64+import os+import random+from typing import Any, Callable, Dict, Optional, Tuple, Union, cast++from playwright._impl._errors import Error as PlaywrightError+from playwright._impl._errors import TimeoutError+from playwright.async_api import Download, Page++from ._types import (+ InteractiveRegion,+ VisualViewport,+ interactiveregion_from_dict,+ visualviewport_from_dict,+)+++class PlaywrightController:+ def __init__(+ self,+ animate_actions: bool = False,+ downloads_folder: Optional[str] = None,+ viewport_width: int = 1440,+ viewport_height: int = 900,+ _download_handler: Optional[Callable[[Download], None]] = None,+ to_resize_viewport: bool = True,+ ) -> None:+ """+ A controller for Playwright to interact with web pages.+ animate_actions: If True, actions will be animated.+ downloads_folder: The folder to save downloads to.+ viewport_width: The width of the viewport.+ viewport_height: The height of the viewport.+ _download_handler: A handler for downloads.+ to_resize_viewport: If True, the viewport will be resized.+ """+ self.animate_actions = animate_actions+ self.downloads_folder = downloads_folder+ self.viewport_width = viewport_width+ self.viewport_height = viewport_height+ self._download_handler = _download_handler+ self.to_resize_viewport = to_resize_viewport+ self._page_script: str = ""+ self.last_cursor_position: Tuple[float, float] = (0.0, 0.0)++ # Read page_script+ with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"), "rt") as fh:+ self._page_script = fh.read()++ async def sleep(self, page: Page, duration: Union[int, float]) -> None:+ assert page is not None+ await page.wait_for_timeout(duration * 1000)++ async def get_interactive_rects(self, page: Page) -> Dict[str, InteractiveRegion]:+ assert page is not None+ # Read the regions from the DOM+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = cast(Dict[str, Dict[str, Any]], await page.evaluate("MultimodalWebSurfer.getInteractiveRects();"))++ # Convert the results into appropriate types+ assert isinstance(result, dict)+ typed_results: Dict[str, InteractiveRegion] = {}+ for k in result:+ assert isinstance(k, str)+ typed_results[k] = interactiveregion_from_dict(result[k])++ return typed_results++ async def get_visual_viewport(self, page: Page) -> VisualViewport:+ assert page is not None+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ return visualviewport_from_dict(await page.evaluate("MultimodalWebSurfer.getVisualViewport();"))++ async def get_focused_rect_id(self, page: Page) -> str:+ assert page is not None+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = await page.evaluate("MultimodalWebSurfer.getFocusedElementId();")+ return str(result)++ async def get_page_metadata(self, page: Page) -> Dict[str, Any]:+ assert page is not None+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = await page.evaluate("MultimodalWebSurfer.getPageMetadata();")+ assert isinstance(result, dict)+ return cast(Dict[str, Any], result)++ async def on_new_page(self, page: Page) -> None:+ assert page is not None+ page.on("download", self._download_handler) # type: ignore+ if self.to_resize_viewport and self.viewport_width and self.viewport_height:+ await page.set_viewport_size({"width": self.viewport_width, "height": self.viewport_height})+ await self.sleep(page, 0.2)+ await page.add_init_script(path=os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"))+ await page.wait_for_load_state()++ async def back(self, page: Page) -> None:+ assert page is not None+ await page.go_back()++ async def visit_page(self, page: Page, url: str) -> Tuple[bool, bool]:+ assert page is not None+ reset_prior_metadata_hash = False+ reset_last_download = False+ try:+ # Regular webpage+ await page.goto(url)+ await page.wait_for_load_state()+ reset_prior_metadata_hash = True+ except Exception as e_outer:+ # Downloaded file+ if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):+ async with page.expect_download() as download_info:+ try:+ await page.goto(url)+ except Exception as e_inner:+ if "net::ERR_ABORTED" in str(e_inner):+ pass+ else:+ raise e_inner+ download = await download_info.value+ fname = os.path.join(self.downloads_folder, download.suggested_filename)+ await download.save_as(fname)+ message = f"<body style=\"margin: 20px;\"><h1>Successfully downloaded '{download.suggested_filename}' to local path:<br><br>{fname}</h1></body>"+ await page.goto(+ "data:text/html;base64," + base64.b64encode(message.encode("utf-8")).decode("utf-8")+ )+ reset_last_download = True+ else:+ raise e_outer+ return reset_prior_metadata_hash, reset_last_download++ async def page_down(self, page: Page) -> None:+ assert page is not None+ await page.evaluate(f"window.scrollBy(0, {self.viewport_height-50});")++ async def page_up(self, page: Page) -> None:+ assert page is not None+ await page.evaluate(f"window.scrollBy(0, -{self.viewport_height-50});")++ async def gradual_cursor_animation(+ self, page: Page, start_x: float, start_y: float, end_x: float, end_y: float+ ) -> None:+ # animation helper+ steps = 20+ for step in range(steps):+ x = start_x + (end_x - start_x) * (step / steps)+ y = start_y + (end_y - start_y) * (step / steps)+ # await page.mouse.move(x, y, steps=1)+ await page.evaluate(f"""+ (function() {{+ let cursor = document.getElementById('red-cursor');+ cursor.style.left = '{x}px';+ cursor.style.top = '{y}px';+ }})();+ """)+ await asyncio.sleep(0.05)++ self.last_cursor_position = (end_x, end_y)++ async def add_cursor_box(self, page: Page, identifier: str) -> None:+ # animation helper+ await page.evaluate(f"""+ (function() {{+ let elm = document.querySelector("[__elementId='{identifier}']");+ if (elm) {{+ elm.style.transition = 'border 0.3s ease-in-out';+ elm.style.border = '2px solid red';+ }}+ }})();+ """)+ await asyncio.sleep(0.3)++ # Create a red cursor+ await page.evaluate("""+ (function() {+ let cursor = document.createElement('div');+ cursor.id = 'red-cursor';+ cursor.style.width = '10px';+ cursor.style.height = '10px';+ cursor.style.backgroundColor = 'red';+ cursor.style.position = 'absolute';+ cursor.style.borderRadius = '50%';+ cursor.style.zIndex = '10000';+ document.body.appendChild(cursor);+ })();+ """)++ async def remove_cursor_box(self, page: Page, identifier: str) -> None:+ # Remove the highlight and cursor+ await page.evaluate(f"""+ (function() {{+ let elm = document.querySelector("[__elementId='{identifier}']");+ if (elm) {{+ elm.style.border = '';+ }}+ let cursor = document.getElementById('red-cursor');+ if (cursor) {{+ cursor.remove();+ }}+ }})();+ """)++ async def click_id(self, page: Page, identifier: str) -> Page | None:+ """+ Returns new page if a new page is opened, otherwise None.+ """+ new_page: Page | None = None+ assert page is not None+ target = page.locator(f"[__elementId='{identifier}']")++ # See if it exists+ try:+ await target.wait_for(timeout=5000)+ except TimeoutError:+ raise ValueError("No such element.") from None++ # Click it+ await target.scroll_into_view_if_needed()+ await asyncio.sleep(0.3)++ box = cast(Dict[str, Union[int, float]], await target.bounding_box())++ if self.animate_actions:+ await self.add_cursor_box(page, identifier)+ # Move cursor to the box slowly+ start_x, start_y = self.last_cursor_position+ end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2+ await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)+ await asyncio.sleep(0.1)++ try:+ # Give it a chance to open a new page+ async with page.expect_event("popup", timeout=1000) as page_info: # type: ignore+ await page.mouse.click(end_x, end_y, delay=10)+ new_page = await page_info.value # type: ignore+ assert isinstance(new_page, Page)+ await self.on_new_page(new_page)+ except TimeoutError:+ pass+ await self.remove_cursor_box(page, identifier)++ else:+ try:+ # Give it a chance to open a new page+ async with page.expect_event("popup", timeout=1000) as page_info: # type: ignore+ await page.mouse.click(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2, delay=10)+ new_page = await page_info.value # type: ignore+ assert isinstance(new_page, Page)+ await self.on_new_page(new_page)+ except TimeoutError:+ pass+ return new_page # type: ignore++ async def hover_id(self, page: Page, identifier: str) -> None:+ """+ Hovers the mouse over the target with the given id.+ """+ assert page is not None+ target = page.locator(f"[__elementId='{identifier}']")++ # See if it exists+ try:+ await target.wait_for(timeout=5000)+ except TimeoutError:+ raise ValueError("No such element.") from None++ # Hover over it+ await target.scroll_into_view_if_needed()+ await asyncio.sleep(0.3)++ box = cast(Dict[str, Union[int, float]], await target.bounding_box())++ if self.animate_actions:+ await self.add_cursor_box(page, identifier)+ # Move cursor to the box slowly+ start_x, start_y = self.last_cursor_position+ end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2+ await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)+ await asyncio.sleep(0.1)+ await page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)++ await self.remove_cursor_box(page, identifier)+ else:+ await page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)++ async def fill_id(self, page: Page, identifier: str, value: str) -> None:+ assert page is not None+ target = page.locator(f"[__elementId='{identifier}']")++ # See if it exists+ try:+ await target.wait_for(timeout=5000)+ except TimeoutError:+ raise ValueError("No such element.") from None++ # Fill it+ await target.scroll_into_view_if_needed()+ box = cast(Dict[str, Union[int, float]], await target.bounding_box())++ if self.animate_actions:+ await self.add_cursor_box(page, identifier)+ # Move cursor to the box slowly+ start_x, start_y = self.last_cursor_position+ end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2+ await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)+ await asyncio.sleep(0.1)++ # Focus on the element+ await target.focus()+ if self.animate_actions:+ # fill char by char to mimic human speed for short text and type fast for long text+ if len(value) < 100:+ delay_typing_speed = 50 + 100 * random.random()+ else:+ delay_typing_speed = 10+ await target.press_sequentially(value, delay=delay_typing_speed)+ else:+ try:+ await target.fill(value)+ except PlaywrightError:+ await target.press_sequentially(value)+ await target.press("Enter")++ if self.animate_actions:+ await self.remove_cursor_box(page, identifier)++ async def scroll_id(self, page: Page, identifier: str, direction: str) -> None:+ assert page is not None+ await page.evaluate(+ f"""+ (function() {{+ let elm = document.querySelector("[__elementId='{identifier}']");+ if (elm) {{+ if ("{direction}" == "up") {{+ elm.scrollTop = Math.max(0, elm.scrollTop - elm.clientHeight);+ }}+ else {{+ elm.scrollTop = Math.min(elm.scrollHeight - elm.clientHeight, elm.scrollTop + elm.clientHeight);+ }}+ }}+ }})();+ """+ )++ async def get_webpage_text(self, page: Page, n_lines: int = 100) -> str:+ """+ page: playwright page object+ n_lines: number of lines to return from the page innertext+ return: text in the first n_lines of the page+ """+ assert page is not None+ try:+ text_in_viewport = await page.evaluate("""() => {+ return document.body.innerText;+ }""")+ text_in_viewport = "\n".join(text_in_viewport.split("\n")[:n_lines])+ # remove empty lines+ text_in_viewport = "\n".join([line for line in text_in_viewport.split("\n") if line.strip()])+ assert isinstance(text_in_viewport, str)+ return text_in_viewport+ except Exception:+ return ""++ async def get_page_markdown(self, page: Page) -> str:+ # TODO: replace with mdconvert+ assert page is not None+ return await self.get_webpage_text(page, n_lines=1000)diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.pynew file mode 100644index 000000000..2292a1fba--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py@@ -0,0 +1,32 @@+WEB_SURFER_TOOL_PROMPT = """+Consider the following screenshot of a web browser, which is open to the page '{url}'. In this screenshot, interactive elements are outlined in bounding boxes of different colors. Each bounding box has a numeric ID label in the same color. Additional information about each visible label is listed below:++{visible_targets}{other_targets_str}{focused_hint}++You are to respond to the most recent request by selecting an appropriate tool from the following set, or by answering the question directly if possible without tools:++{tool_names}++When deciding between tools, consider if the request can be best addressed by:+ - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)+ - contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)+ - on some other website entirely (in which case actions like performing a new web search might be the best option)+"""++WEB_SURFER_OCR_PROMPT = """+Please transcribe all visible text on this page, including both main content and the labels of UI elements.+"""++WEB_SURFER_QA_SYSTEM_MESSAGE = """+You are a helpful assistant that can summarize long documents to answer question.+"""+++def WEB_SURFER_QA_PROMPT(title: str, question: str | None = None) -> str:+ base_prompt = f"We are visiting the webpage '{title}'. Its full-text content are pasted below, along with a screenshot of the page's current viewport."+ if question is not None:+ return (+ f"{base_prompt} Please summarize the webpage into one or two paragraphs with respect to '{question}':\n\n"+ )+ else:+ return f"{base_prompt} Please summarize the webpage into one or two paragraphs:\n\n"diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.pyindex b662f4101..5d54d5ff1 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_tool_definitions.py@@ -1,6 +1,5 @@from typing import Any, Dict-# TODO Why does pylance fail if I import from autogen_core.components.tools instead?from autogen_core.components.tools._base import ParametersSchema, ToolSchema@@ -16,6 +15,10 @@ def _load_tool(tooldef: Dict[str, Any]) -> ToolSchema:)+REASONING_TOOL_PROMPT = (+ "A short description of the action to be performed and reason for doing so, do not mention the user."+)+TOOL_VISIT_URL: ToolSchema = _load_tool({"type": "function",@@ -27,7 +30,7 @@ TOOL_VISIT_URL: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"url": {"type": "string",@@ -51,7 +54,7 @@ TOOL_WEB_SEARCH: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"query": {"type": "string",@@ -75,7 +78,7 @@ TOOL_HISTORY_BACK: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},},"required": ["reasoning"],@@ -95,7 +98,7 @@ TOOL_PAGE_UP: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},},"required": ["reasoning"],@@ -115,7 +118,7 @@ TOOL_PAGE_DOWN: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},},"required": ["reasoning"],@@ -135,7 +138,7 @@ TOOL_CLICK: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"target_id": {"type": "integer",@@ -159,7 +162,7 @@ TOOL_TYPE: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"input_field_id": {"type": "integer",@@ -187,7 +190,7 @@ TOOL_SCROLL_ELEMENT_DOWN: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"target_id": {"type": "integer",@@ -211,7 +214,7 @@ TOOL_SCROLL_ELEMENT_UP: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"target_id": {"type": "integer",@@ -224,6 +227,31 @@ TOOL_SCROLL_ELEMENT_UP: ToolSchema = _load_tool(})+TOOL_HOVER: ToolSchema = _load_tool(+ {+ "type": "function",+ "function": {+ "name": "hover",+ "description": "Hovers the mouse over the target with the given id.",+ "parameters": {+ "type": "object",+ "properties": {+ "reasoning": {+ "type": "string",+ "description": REASONING_TOOL_PROMPT,+ },+ "target_id": {+ "type": "integer",+ "description": "The numeric id of the target to hover over.",+ },+ },+ "required": ["reasoning", "target_id"],+ },+ },+ }+)++TOOL_READ_PAGE_AND_ANSWER: ToolSchema = _load_tool({"type": "function",@@ -235,7 +263,7 @@ TOOL_READ_PAGE_AND_ANSWER: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},"question": {"type": "string",@@ -259,7 +287,7 @@ TOOL_SUMMARIZE_PAGE: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},},"required": ["reasoning"],@@ -279,7 +307,7 @@ TOOL_SLEEP: ToolSchema = _load_tool("properties": {"reasoning": {"type": "string",- "description": "A short explanation of the reasoning for calling this tool and taking this action.",+ "description": REASONING_TOOL_PROMPT,},},"required": ["reasoning"],