Skip to content

A programming framework for agentic AI

Initial work moving mdconvert, and enabling browser Q&A (#211)

afourney524a5a78f99e
diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py b/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py
index cff1fac97..a801193cb 100644
--- a/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py
+++ b/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py
@@ -12,10 +12,11 @@ from agnext.components.models import (
# from agnext.components.tools import FunctionTool
from agnext.core import CancellationToken
-# from typing_extensions import Annotated
+from ...markdown_browser import RequestsMarkdownBrowser
from ..base_agent import BaseAgent
+
+# from typing_extensions import Annotated
from ._tools import TOOL_FIND_NEXT, TOOL_FIND_ON_PAGE_CTRL_F, TOOL_OPEN_LOCAL_FILE, TOOL_PAGE_DOWN, TOOL_PAGE_UP
-from .requests_markdown_browser import RequestsMarkdownBrowser
# async def read_local_file(file_path: Annotated[str, "relative or absolute path of file to read"]) -> str:
# """Read contents of a file."""
diff --git a/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py b/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py
index be0104af2..6391f3b02 100644
--- a/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py
+++ b/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py
@@ -30,17 +30,24 @@ from team_one.agents.base_agent import BaseAgent
from team_one.messages import UserContent
from team_one.utils import SentinelMeta, message_content_to_str
+# TODO: Fix mdconvert
+from ...markdown_browser import ( # type: ignore
+ DocumentConverterResult, # type: ignore
+ FileConversionException, # type: ignore
+ MarkdownConverter, # type: ignore
+ UnsupportedFormatException, # type: ignore
+)
from .set_of_mark import add_set_of_mark
from .tool_definitions import (
TOOL_CLICK,
TOOL_HISTORY_BACK,
TOOL_PAGE_DOWN,
TOOL_PAGE_UP,
- # TOOL_READ_PAGE_AND_ANSWER,
- # TOOL_SCROLL_ELEMENT_DOWN,
- # TOOL_SCROLL_ELEMENT_UP,
+ TOOL_READ_PAGE_AND_ANSWER,
+ # TOOL_SCROLL_ELEMENT_DOWN,
+ # TOOL_SCROLL_ELEMENT_UP,
TOOL_SLEEP,
- # TOOL_SUMMARIZE_PAGE,
+ TOOL_SUMMARIZE_PAGE,
TOOL_TYPE,
TOOL_VISIT_URL,
TOOL_WEB_SEARCH,
@@ -102,7 +109,7 @@ class MultimodalWebSurfer(BaseAgent):
downloads_folder: str | None = None,
debug_dir: str | None = os.getcwd(),
# navigation_allow_list=lambda url: True,
- # markdown_converter: Optional[Union[MarkdownConverter, None]] = None,
+ markdown_converter: Any | None = None, # TODO: Fixme
) -> None:
self._model_client = model_client
self.start_page = start_page or self.DEFAULT_START_PAGE
@@ -117,10 +124,10 @@ class MultimodalWebSurfer(BaseAgent):
self._prior_metadata_hash = None
## Create or use the provided MarkdownConverter
- # if markdown_converter is None:
- # self._markdown_converter = MarkdownConverter()
- # else:
- # self._markdown_converter = markdown_converter
+ if markdown_converter is None:
+ self._markdown_converter = MarkdownConverter() # type: ignore
+ else:
+ self._markdown_converter = markdown_converter # type: ignore
# Create the playwright self
launch_args: Dict[str, Any] = {"headless": headless}
@@ -266,8 +273,8 @@ setInterval(function() {{
TOOL_HISTORY_BACK,
TOOL_CLICK,
TOOL_TYPE,
- # TOOL_SUMMARIZE_PAGE,
- # TOOL_READ_PAGE_AND_ANSWER,
+ TOOL_SUMMARIZE_PAGE,
+ TOOL_READ_PAGE_AND_ANSWER,
TOOL_SLEEP,
]
@@ -354,7 +361,13 @@ When deciding between tools, consider if the request can be best addressed by:
# self._last_download = None
# try:
if True:
- if isinstance(message, list):
+ if isinstance(message, str):
+ # Answer directly
+ return False, message
+
+ elif isinstance(message, list):
+ # Take an action
+
name = message[0].name
args = json.loads(message[0].arguments)
@@ -429,12 +442,14 @@ When deciding between tools, consider if the request can be best addressed by:
await self._scroll_id(target_id, "down")
- # elif name == "answer_question":
- # question = str(args.get("question"))
- # action_description = self._summarize_page(question=question)
- #
- # elif name == "summarize_page":
- # action_description = self._summarize_page()
+ elif name == "answer_question":
+ question = str(args.get("question"))
+ # Do Q&A on the DOM. No need to take further action. Browser state does not change.
+ return False, await self._summarize_page(question=question)
+
+ elif name == "summarize_page":
+ # Summarize the DOM. No need to take further action. Browser state does not change.
+ return False, await self._summarize_page()
elif name == "sleep":
action_description = "I am waiting a short period of time before taking further action."
@@ -442,6 +457,9 @@ When deciding between tools, consider if the request can be best addressed by:
else:
raise ValueError(f"Unknown tool '{name}'. Please choose from:\n\n{tool_names}")
+ else:
+ # Not sure what happened here
+ raise AssertionError(f"Unknown response format '{message}'")
# except ValueError as e:
# return True, str(e)
@@ -543,11 +561,12 @@ When deciding between tools, consider if the request can be best addressed by:
assert isinstance(result, dict)
return cast(Dict[str, Any], result)
- # async def _get_page_markdown(self):
- # assert self._page is not None
- # html = self._page.evaluate("document.documentElement.outerHTML;")
- # res = self._markdown_converter.convert_stream(io.StringIO(html), file_extension=".html", url=self._page.url)
- # return res.text_content
+ async def _get_page_markdown(self) -> str:
+ assert self._page is not None
+ html = await self._page.evaluate("document.documentElement.outerHTML;")
+ # TODO: fix types
+ res = self._markdown_converter.convert_stream(io.StringIO(html), file_extension=".html", url=self._page.url) # type: ignore
+ return res.text_content # type: ignore
async def _on_new_page(self, page: Page) -> None:
self._page = page
@@ -665,57 +684,62 @@ When deciding between tools, consider if the request can be best addressed by:
"""
)
- # def _summarize_page(self, question=None, token_limit=100000):
- # page_markdown = self._get_page_markdown()
- #
- # buffer = ""
- # for line in re.split(r"([\r\n]+)", page_markdown):
- # tokens = count_token(buffer + line)
- # if tokens + 1024 > token_limit: # Leave room for our summary
- # break
- # buffer += line
- #
- # buffer = buffer.strip()
- # if len(buffer) == 0:
- # return "Nothing to summarize."
- #
- # title = self._page.url
- # try:
- # title = self._page.title()
- # except:
- # pass
- #
- # # Take a screenshot and scale it
- # screenshot = self._page.screenshot()
- # if not isinstance(screenshot, io.BufferedIOBase):
- # screenshot = io.BytesIO(screenshot)
- # screenshot = Image.open(screenshot)
- # scaled_screenshot = screenshot.resize((MLM_WIDTH, MLM_HEIGHT))
- # screenshot.close()
- #
- # messages = [
- # {
- # "role": "system",
- # "content": "You are a helpful assistant that can summarize long documents to answer question.",
- # }
- # ]
- #
- # prompt = f"We are visiting the webpage '{title}'. Its full-text contents are pasted below, along with a screenshot of the page's current viewport."
- # if question is not None:
- # prompt += (
- # f" Please summarize the webpage into one or two paragraphs with respect to '{question}':\n\n{buffer}"
- # )
- # else:
- # prompt += f" Please summarize the webpage into one or two paragraphs:\n\n{buffer}"
- #
- # messages.append(
- # self._make_mm_message(prompt, scaled_screenshot),
- # )
- # scaled_screenshot.close()
- #
- # response = self.client.create(context=None, messages=messages)
- # extracted_response = self.client.extract_text_or_completion_object(response)[0]
- # return str(extracted_response)
+ async def _summarize_page(self, question: str | None = None, token_limit: int = 100000) -> str:
+ assert self._page is not None
+
+ page_markdown: str = await self._get_page_markdown()
+
+ # TODO: Get token count working
+
+ buffer = page_markdown
+ # buffer: str = ""
+ # for line in re.split(r"([\r\n]+)", page_markdown):
+ # tokens = count_token(buffer + line)
+ # if tokens + 1024 > token_limit: # Leave room for our summary
+ # break
+ # buffer += line
+
+ buffer = buffer.strip()
+ if len(buffer) == 0:
+ return "Nothing to summarize."
+
+ title: str = self._page.url
+ try:
+ title = await self._page.title()
+ except Exception:
+ pass
+
+ # Take a screenshot and scale it
+ screenshot = Image.open(io.BytesIO(await self._page.screenshot()))
+ scaled_screenshot = screenshot.resize((MLM_WIDTH, MLM_HEIGHT))
+ screenshot.close()
+
+ prompt = f"We are visiting the webpage '{title}'. Its full-text contents are pasted below, along with a screenshot of the page's current viewport."
+ if question is not None:
+ prompt += (
+ f" Please summarize the webpage into one or two paragraphs with respect to '{question}':\n\n{buffer}"
+ )
+ else:
+ prompt += f" Please summarize the webpage into one or two paragraphs:\n\n{buffer}"
+
+ # Add the multimodal message and make the request
+ messages: List[LLMMessage] = []
+ messages.append(
+ SystemMessage(content="You are a helpful assistant that can summarize long documents to answer question.")
+ )
+ messages.append(
+ UserMessage(
+ content=[
+ prompt,
+ AGImage.from_pil(scaled_screenshot),
+ ],
+ source=self.metadata["name"],
+ )
+ )
+ response = await self._model_client.create(messages)
+ scaled_screenshot.close()
+ assert isinstance(response.content, str)
+ return response.content
async def _get_ocr_text(self, image: bytes | io.BufferedIOBase | Image.Image) -> str:
scaled_screenshot = None
diff --git a/python/teams/team-one/src/team_one/markdown_browser/__init__.py b/python/teams/team-one/src/team_one/markdown_browser/__init__.py
new file mode 100644
index 000000000..5b6880eba
--- /dev/null
+++ b/python/teams/team-one/src/team_one/markdown_browser/__init__.py
@@ -0,0 +1,22 @@
+from .abstract_markdown_browser import AbstractMarkdownBrowser
+from .markdown_search import AbstractMarkdownSearch, BingMarkdownSearch
+
+# TODO: Fix mdconvert
+from .mdconvert import ( # type: ignore
+ DocumentConverterResult,
+ FileConversionException,
+ MarkdownConverter,
+ UnsupportedFormatException,
+)
+from .requests_markdown_browser import RequestsMarkdownBrowser
+
+__all__ = (
+ "AbstractMarkdownBrowser",
+ "RequestsMarkdownBrowser",
+ "AbstractMarkdownSearch",
+ "BingMarkdownSearch",
+ "MarkdownConverter",
+ "UnsupportedFormatException",
+ "FileConversionException",
+ "DocumentConverterResult",
+)
diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.py b/python/teams/team-one/src/team_one/markdown_browser/abstract_markdown_browser.py
similarity index 100%
rename from python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.py
rename to python/teams/team-one/src/team_one/markdown_browser/abstract_markdown_browser.py
diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.py b/python/teams/team-one/src/team_one/markdown_browser/markdown_search.py
similarity index 100%
rename from python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.py
rename to python/teams/team-one/src/team_one/markdown_browser/markdown_search.py
diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.py b/python/teams/team-one/src/team_one/markdown_browser/mdconvert.py
similarity index 100%
rename from python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.py
rename to python/teams/team-one/src/team_one/markdown_browser/mdconvert.py
diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.py b/python/teams/team-one/src/team_one/markdown_browser/requests_markdown_browser.py
similarity index 100%
rename from python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.py
rename to python/teams/team-one/src/team_one/markdown_browser/requests_markdown_browser.py