A programming framework for agentic AI
Initial work moving mdconvert, and enabling browser Q&A (#211)
afourney524a5a78f99e
diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py b/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.pyindex cff1fac97..a801193cb 100644--- a/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py+++ b/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py@@ -12,10 +12,11 @@ from agnext.components.models import (# from agnext.components.tools import FunctionToolfrom agnext.core import CancellationToken-# from typing_extensions import Annotated+from ...markdown_browser import RequestsMarkdownBrowserfrom ..base_agent import BaseAgent++# from typing_extensions import Annotatedfrom ._tools import TOOL_FIND_NEXT, TOOL_FIND_ON_PAGE_CTRL_F, TOOL_OPEN_LOCAL_FILE, TOOL_PAGE_DOWN, TOOL_PAGE_UP-from .requests_markdown_browser import RequestsMarkdownBrowser# async def read_local_file(file_path: Annotated[str, "relative or absolute path of file to read"]) -> str:# """Read contents of a file."""diff --git a/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py b/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.pyindex be0104af2..6391f3b02 100644--- a/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py+++ b/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py@@ -30,17 +30,24 @@ from team_one.agents.base_agent import BaseAgentfrom team_one.messages import UserContentfrom team_one.utils import SentinelMeta, message_content_to_str+# TODO: Fix mdconvert+from ...markdown_browser import ( # type: ignore+ DocumentConverterResult, # type: ignore+ FileConversionException, # type: ignore+ MarkdownConverter, # type: ignore+ UnsupportedFormatException, # type: ignore+)from .set_of_mark import add_set_of_markfrom .tool_definitions import (TOOL_CLICK,TOOL_HISTORY_BACK,TOOL_PAGE_DOWN,TOOL_PAGE_UP,- # TOOL_READ_PAGE_AND_ANSWER,- # TOOL_SCROLL_ELEMENT_DOWN,- # TOOL_SCROLL_ELEMENT_UP,+ TOOL_READ_PAGE_AND_ANSWER,+ # TOOL_SCROLL_ELEMENT_DOWN,+ # TOOL_SCROLL_ELEMENT_UP,TOOL_SLEEP,- # TOOL_SUMMARIZE_PAGE,+ TOOL_SUMMARIZE_PAGE,TOOL_TYPE,TOOL_VISIT_URL,TOOL_WEB_SEARCH,@@ -102,7 +109,7 @@ class MultimodalWebSurfer(BaseAgent):downloads_folder: str | None = None,debug_dir: str | None = os.getcwd(),# navigation_allow_list=lambda url: True,- # markdown_converter: Optional[Union[MarkdownConverter, None]] = None,+ markdown_converter: Any | None = None, # TODO: Fixme) -> None:self._model_client = model_clientself.start_page = start_page or self.DEFAULT_START_PAGE@@ -117,10 +124,10 @@ class MultimodalWebSurfer(BaseAgent):self._prior_metadata_hash = None## Create or use the provided MarkdownConverter- # if markdown_converter is None:- # self._markdown_converter = MarkdownConverter()- # else:- # self._markdown_converter = markdown_converter+ if markdown_converter is None:+ self._markdown_converter = MarkdownConverter() # type: ignore+ else:+ self._markdown_converter = markdown_converter # type: ignore# Create the playwright selflaunch_args: Dict[str, Any] = {"headless": headless}@@ -266,8 +273,8 @@ setInterval(function() {{TOOL_HISTORY_BACK,TOOL_CLICK,TOOL_TYPE,- # TOOL_SUMMARIZE_PAGE,- # TOOL_READ_PAGE_AND_ANSWER,+ TOOL_SUMMARIZE_PAGE,+ TOOL_READ_PAGE_AND_ANSWER,TOOL_SLEEP,]@@ -354,7 +361,13 @@ When deciding between tools, consider if the request can be best addressed by:# self._last_download = None# try:if True:- if isinstance(message, list):+ if isinstance(message, str):+ # Answer directly+ return False, message++ elif isinstance(message, list):+ # Take an action+name = message[0].nameargs = json.loads(message[0].arguments)@@ -429,12 +442,14 @@ When deciding between tools, consider if the request can be best addressed by:await self._scroll_id(target_id, "down")- # elif name == "answer_question":- # question = str(args.get("question"))- # action_description = self._summarize_page(question=question)- #- # elif name == "summarize_page":- # action_description = self._summarize_page()+ elif name == "answer_question":+ question = str(args.get("question"))+ # Do Q&A on the DOM. No need to take further action. Browser state does not change.+ return False, await self._summarize_page(question=question)++ elif name == "summarize_page":+ # Summarize the DOM. No need to take further action. Browser state does not change.+ return False, await self._summarize_page()elif name == "sleep":action_description = "I am waiting a short period of time before taking further action."@@ -442,6 +457,9 @@ When deciding between tools, consider if the request can be best addressed by:else:raise ValueError(f"Unknown tool '{name}'. Please choose from:\n\n{tool_names}")+ else:+ # Not sure what happened here+ raise AssertionError(f"Unknown response format '{message}'")# except ValueError as e:# return True, str(e)@@ -543,11 +561,12 @@ When deciding between tools, consider if the request can be best addressed by:assert isinstance(result, dict)return cast(Dict[str, Any], result)- # async def _get_page_markdown(self):- # assert self._page is not None- # html = self._page.evaluate("document.documentElement.outerHTML;")- # res = self._markdown_converter.convert_stream(io.StringIO(html), file_extension=".html", url=self._page.url)- # return res.text_content+ async def _get_page_markdown(self) -> str:+ assert self._page is not None+ html = await self._page.evaluate("document.documentElement.outerHTML;")+ # TODO: fix types+ res = self._markdown_converter.convert_stream(io.StringIO(html), file_extension=".html", url=self._page.url) # type: ignore+ return res.text_content # type: ignoreasync def _on_new_page(self, page: Page) -> None:self._page = page@@ -665,57 +684,62 @@ When deciding between tools, consider if the request can be best addressed by:""")- # def _summarize_page(self, question=None, token_limit=100000):- # page_markdown = self._get_page_markdown()- #- # buffer = ""- # for line in re.split(r"([\r\n]+)", page_markdown):- # tokens = count_token(buffer + line)- # if tokens + 1024 > token_limit: # Leave room for our summary- # break- # buffer += line- #- # buffer = buffer.strip()- # if len(buffer) == 0:- # return "Nothing to summarize."- #- # title = self._page.url- # try:- # title = self._page.title()- # except:- # pass- #- # # Take a screenshot and scale it- # screenshot = self._page.screenshot()- # if not isinstance(screenshot, io.BufferedIOBase):- # screenshot = io.BytesIO(screenshot)- # screenshot = Image.open(screenshot)- # scaled_screenshot = screenshot.resize((MLM_WIDTH, MLM_HEIGHT))- # screenshot.close()- #- # messages = [- # {- # "role": "system",- # "content": "You are a helpful assistant that can summarize long documents to answer question.",- # }- # ]- #- # prompt = f"We are visiting the webpage '{title}'. Its full-text contents are pasted below, along with a screenshot of the page's current viewport."- # if question is not None:- # prompt += (- # f" Please summarize the webpage into one or two paragraphs with respect to '{question}':\n\n{buffer}"- # )- # else:- # prompt += f" Please summarize the webpage into one or two paragraphs:\n\n{buffer}"- #- # messages.append(- # self._make_mm_message(prompt, scaled_screenshot),- # )- # scaled_screenshot.close()- #- # response = self.client.create(context=None, messages=messages)- # extracted_response = self.client.extract_text_or_completion_object(response)[0]- # return str(extracted_response)+ async def _summarize_page(self, question: str | None = None, token_limit: int = 100000) -> str:+ assert self._page is not None++ page_markdown: str = await self._get_page_markdown()++ # TODO: Get token count working++ buffer = page_markdown+ # buffer: str = ""+ # for line in re.split(r"([\r\n]+)", page_markdown):+ # tokens = count_token(buffer + line)+ # if tokens + 1024 > token_limit: # Leave room for our summary+ # break+ # buffer += line++ buffer = buffer.strip()+ if len(buffer) == 0:+ return "Nothing to summarize."++ title: str = self._page.url+ try:+ title = await self._page.title()+ except Exception:+ pass++ # Take a screenshot and scale it+ screenshot = Image.open(io.BytesIO(await self._page.screenshot()))+ scaled_screenshot = screenshot.resize((MLM_WIDTH, MLM_HEIGHT))+ screenshot.close()++ prompt = f"We are visiting the webpage '{title}'. Its full-text contents are pasted below, along with a screenshot of the page's current viewport."+ if question is not None:+ prompt += (+ f" Please summarize the webpage into one or two paragraphs with respect to '{question}':\n\n{buffer}"+ )+ else:+ prompt += f" Please summarize the webpage into one or two paragraphs:\n\n{buffer}"++ # Add the multimodal message and make the request+ messages: List[LLMMessage] = []+ messages.append(+ SystemMessage(content="You are a helpful assistant that can summarize long documents to answer question.")+ )+ messages.append(+ UserMessage(+ content=[+ prompt,+ AGImage.from_pil(scaled_screenshot),+ ],+ source=self.metadata["name"],+ )+ )+ response = await self._model_client.create(messages)+ scaled_screenshot.close()+ assert isinstance(response.content, str)+ return response.contentasync def _get_ocr_text(self, image: bytes | io.BufferedIOBase | Image.Image) -> str:scaled_screenshot = Nonediff --git a/python/teams/team-one/src/team_one/markdown_browser/__init__.py b/python/teams/team-one/src/team_one/markdown_browser/__init__.pynew file mode 100644index 000000000..5b6880eba--- /dev/null+++ b/python/teams/team-one/src/team_one/markdown_browser/__init__.py@@ -0,0 +1,22 @@+from .abstract_markdown_browser import AbstractMarkdownBrowser+from .markdown_search import AbstractMarkdownSearch, BingMarkdownSearch++# TODO: Fix mdconvert+from .mdconvert import ( # type: ignore+ DocumentConverterResult,+ FileConversionException,+ MarkdownConverter,+ UnsupportedFormatException,+)+from .requests_markdown_browser import RequestsMarkdownBrowser++__all__ = (+ "AbstractMarkdownBrowser",+ "RequestsMarkdownBrowser",+ "AbstractMarkdownSearch",+ "BingMarkdownSearch",+ "MarkdownConverter",+ "UnsupportedFormatException",+ "FileConversionException",+ "DocumentConverterResult",+)diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.py b/python/teams/team-one/src/team_one/markdown_browser/abstract_markdown_browser.pysimilarity index 100%rename from python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.pyrename to python/teams/team-one/src/team_one/markdown_browser/abstract_markdown_browser.pydiff --git a/python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.py b/python/teams/team-one/src/team_one/markdown_browser/markdown_search.pysimilarity index 100%rename from python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.pyrename to python/teams/team-one/src/team_one/markdown_browser/markdown_search.pydiff --git a/python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.py b/python/teams/team-one/src/team_one/markdown_browser/mdconvert.pysimilarity index 100%rename from python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.pyrename to python/teams/team-one/src/team_one/markdown_browser/mdconvert.pydiff --git a/python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.py b/python/teams/team-one/src/team_one/markdown_browser/requests_markdown_browser.pysimilarity index 100%rename from python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.pyrename to python/teams/team-one/src/team_one/markdown_browser/requests_markdown_browser.py