A programming framework for agentic AI
Improve file surfer (#204)
gagb4c87ed71ebb6
diff --git a/python/teams/team-one/pyproject.toml b/python/teams/team-one/pyproject.tomlindex 49820fc40..4623a33b5 100644--- a/python/teams/team-one/pyproject.toml+++ b/python/teams/team-one/pyproject.toml@@ -18,6 +18,20 @@ classifiers = [dependencies = ["agnext@{root:parent:parent:uri}","aiofiles",+ "requests",+ "mammoth",+ "markdownify",+ "numpy",+ "python-pptx",+ "pandas",+ "pdfminer.six",+ "easyocr",+ "puremagic",+ "binaryornot",+ "pydub",+ "youtube-transcript-api",+ "SpeechRecognition",+ "pathvalidate","playwright"]@@ -30,6 +44,7 @@ dependencies = ["pytest","aiofiles","types-aiofiles",+ "types-requests","azure-identity"]diff --git a/python/teams/team-one/src/team_one/agents/file_surfer.py b/python/teams/team-one/src/team_one/agents/file_surfer.pydeleted file mode 100644index d1a300a1d..000000000--- a/python/teams/team-one/src/team_one/agents/file_surfer.py+++ /dev/null@@ -1,122 +0,0 @@-import asyncio-import json-from pathlib import Path-from typing import List, Tuple--import aiofiles-from agnext.components import FunctionCall, message_handler-from agnext.components.models import (- ChatCompletionClient,- FunctionExecutionResult,- SystemMessage,-)-from agnext.components.tools import FunctionTool-from agnext.core import CancellationToken-from typing_extensions import Annotated--from ..messages import UserContent-from .base_agent import BaseAgent---async def read_local_file(file_path: Annotated[str, "relative or absolute path of file to read"]) -> str:- """Read contents of a file."""- try:- async with aiofiles.open(file_path, mode="r") as file:- file_contents = str(await file.read())- return f"""-Here are the contents of the file at path: {file_path}-```-{file_contents}-```-"""-- except FileNotFoundError:- return f"File not found: {file_path}"---def list_files_and_dirs_like_tree(dir_path: str) -> str:- """List files and directories in a directory in a format similar to 'tree' command with level 1."""- path = Path(dir_path)- if not path.is_dir():- return f"{dir_path} is not a valid directory."-- items = [f"{dir_path}"]- for item in path.iterdir():- if item.is_dir():- items.append(f"├── {item.name}/") # Indicate directories with a trailing slash- else:- items.append(f"├── {item.name}") # List files as is- return "\n".join(items)---class FileSurfer(BaseAgent):- """An agent that uses tools to read and navigate local files."""-- DEFAULT_DESCRIPTION = "An agent that can handle local files."-- DEFAULT_SYSTEM_MESSAGES = [- SystemMessage("""- You are a helpful AI Assistant.- When given a user query, use available functions to help the user with their request."""),- ]-- def __init__(- self,- model_client: ChatCompletionClient,- description: str = DEFAULT_DESCRIPTION,- system_messages: List[SystemMessage] = DEFAULT_SYSTEM_MESSAGES,- ) -> None:- super().__init__(description)- self._model_client = model_client- self._system_messages = system_messages- self._tools = [- FunctionTool(- read_local_file,- description="Use this function to read the contents of a local file whose relative or absolute path is given.",- name="read_local_file",- ),- FunctionTool(- list_files_and_dirs_like_tree,- description="List files and directories in a directory in a format similar to 'tree' command with level 1",- name="list_files_and_dirs_like_tree",- ),- ]-- async def _generate_reply(self, cancellation_token: CancellationToken) -> Tuple[bool, UserContent]:- response = await self._model_client.create(self._system_messages + self._chat_history, tools=self._tools)-- if isinstance(response.content, str):- final_result = response.content-- elif isinstance(response.content, list) and all(isinstance(item, FunctionCall) for item in response.content):- results = await asyncio.gather(*[self.send_message(call, self.id) for call in response.content])- for result in results:- assert isinstance(result, FunctionExecutionResult)- final_result = "\n".join(result.content for result in results)- else:- raise ValueError(f"Unexpected response type: {response.content}")-- assert isinstance(final_result, str)-- return "TERMINATE" in final_result, final_result-- @message_handler- async def handle_tool_call(- self, message: FunctionCall, cancellation_token: CancellationToken- ) -> FunctionExecutionResult:- """Handle a tool execution task. This method executes the tool and publishes the result."""- function_call = message- # Find the tool- tool = next((tool for tool in self._tools if tool.name == function_call.name), None)- if tool is None:- result_as_str = f"Error: Tool not found: {function_call.name}"- else:- try:- arguments = json.loads(function_call.arguments)- result = await tool.run_json(args=arguments, cancellation_token=cancellation_token)- result_as_str = tool.return_value_as_string(result)- except json.JSONDecodeError:- result_as_str = f"Error: Invalid arguments: {function_call.arguments}"- except Exception as e:- result_as_str = f"Error: {e}"- return FunctionExecutionResult(content=result_as_str, call_id=function_call.id)diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/__init__.py b/python/teams/team-one/src/team_one/agents/file_surfer/__init__.pynew file mode 100644index 000000000..e09fdb7f1--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/__init__.py@@ -0,0 +1,3 @@+from .file_surfer import FileSurfer++__all__ = ["FileSurfer"]diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/_tools.py b/python/teams/team-one/src/team_one/agents/file_surfer/_tools.pynew file mode 100644index 000000000..d2dafe5cb--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/_tools.py@@ -0,0 +1,114 @@+from agnext.components.tools import ParametersSchema, ToolSchema++TOOL_OPEN_LOCAL_FILE = ToolSchema(+ name="open_local_file",+ description="Open a local file at a path in the text-based browser and return current viewport content.",+ parameters=ParametersSchema(+ type="object",+ properties={+ "path": {+ "type": "string",+ "description": "The relative or absolute path of a local file to visit.",+ },+ },+ required=["path"],+ ),+)+++TOOL_PAGE_UP = ToolSchema(+ name="page_up",+ description="Scroll the viewport UP one page-length in the current file and return the new viewport content.",+)+++TOOL_PAGE_DOWN = ToolSchema(+ name="page_down",+ description="Scroll the viewport DOWN one page-length in the current file and return the new viewport content.",+)+++TOOL_FIND_ON_PAGE_CTRL_F = ToolSchema(+ name="find_on_page_ctrl_f",+ description="Scroll the viewport to the first occurrence of the search string. This is equivalent to Ctrl+F.",+ parameters=ParametersSchema(+ type="object",+ properties={+ "search_string": {+ "type": "string",+ "description": "The string to search for on the page. This search string supports wildcards like '*'",+ },+ },+ required=["search_string"],+ ),+)+++TOOL_FIND_NEXT = ToolSchema(+ name="find_next",+ description="Scroll the viewport to next occurrence of the search string.",+)++# TOOL_OPEN_LOCAL_FILE = {+# "type": "function",+# "function": {+# "name": "open_local_file",+# "description": "Open a local file at a path in the text-based browser and return current viewport content.",+# "parameters": {+# "type": "object",+# "properties": {+# "path": {+# "type": "string",+# "description": "The relative or absolute path of a local file to visit.",+# },+# },+# "required": ["path"],+# },+# },+# }+++# TOOL_PAGE_UP = {+# "type": "function",+# "function": {+# "name": "page_up",+# "description": "Scroll the viewport UP one page-length in the current file and return the new viewport content.",+# },+# }+++# TOOL_PAGE_DOWN = {+# "type": "function",+# "function": {+# "name": "page_down",+# "description": "Scroll the viewport DOWN one page-length in the current file and return the new viewport content.",+# },+# }+++# TOOL_FIND_ON_PAGE_CTRL_F = {+# "type": "function",+# "function": {+# "name": "find_on_page_ctrl_f",+# "description": "Scroll the viewport to the first occurrence of the search string. This is equivalent to Ctrl+F.",+# "parameters": {+# "type": "object",+# "properties": {+# "search_string": {+# "type": "string",+# "description": "The string to search for on the page. This search string supports wildcards like '*'",+# },+# },+# "required": ["search_string"],+# },+# },+# }+++# TOOL_FIND_NEXT = {+# "type": "function",+# "function": {+# "name": "find_next",+# "description": "Scroll the viewport to next occurrence of the search string.",+# },+# }diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.py b/python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.pynew file mode 100644index 000000000..70417a6ad--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/abstract_markdown_browser.py@@ -0,0 +1,64 @@+from abc import ABC, abstractmethod+from typing import Union+++class AbstractMarkdownBrowser(ABC):+ """+ An abstract class for a Markdown web browser.++ All MarkdownBrowers work by:++ (1) fetching a web page by URL (via requests, Selenium, Playwright, etc.)+ (2) converting the page's HTML or DOM to Markdown+ (3) operating on the Markdown++ Such browsers are simple, and suitable for read-only agentic use.+ They cannot be used to interact with complex web applications.+ """++ @abstractmethod+ def __init__(self) -> None:+ pass++ @property+ @abstractmethod+ def address(self) -> str:+ pass++ @abstractmethod+ def set_address(self, uri_or_path: str) -> None:+ pass++ @property+ @abstractmethod+ def viewport(self) -> str:+ pass++ @property+ @abstractmethod+ def page_content(self) -> str:+ pass++ @abstractmethod+ def page_down(self) -> None:+ pass++ @abstractmethod+ def page_up(self) -> None:+ pass++ @abstractmethod+ def visit_page(self, path_or_uri: str) -> str:+ pass++ @abstractmethod+ def open_local_file(self, local_path: str) -> str:+ pass++ @abstractmethod+ def find_on_page(self, query: str) -> Union[str, None]:+ pass++ @abstractmethod+ def find_next(self) -> Union[str, None]:+ passdiff --git a/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py b/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.pynew file mode 100644index 000000000..cff1fac97--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/file_surfer.py@@ -0,0 +1,210 @@+import json+import time+from typing import List, Optional, Tuple++from agnext.components import FunctionCall+from agnext.components.models import (+ ChatCompletionClient,+ SystemMessage,+ UserMessage,+)++# from agnext.components.tools import FunctionTool+from agnext.core import CancellationToken++# from typing_extensions import Annotated+from ..base_agent import BaseAgent+from ._tools import TOOL_FIND_NEXT, TOOL_FIND_ON_PAGE_CTRL_F, TOOL_OPEN_LOCAL_FILE, TOOL_PAGE_DOWN, TOOL_PAGE_UP+from .requests_markdown_browser import RequestsMarkdownBrowser++# async def read_local_file(file_path: Annotated[str, "relative or absolute path of file to read"]) -> str:+# """Read contents of a file."""+# try:+# async with aiofiles.open(file_path, mode="r") as file:+# file_contents = str(await file.read())+# return f"""+# Here are the contents of the file at path: {file_path}+# ```+# {file_contents}+# ```+# """++# except FileNotFoundError:+# return f"File not found: {file_path}"+++# def list_files_and_dirs_like_tree(dir_path: str) -> str:+# """List files and directories in a directory in a format similar to 'tree' command with level 1."""+# path = Path(dir_path)+# if not path.is_dir():+# return f"{dir_path} is not a valid directory."++# items = [f"{dir_path}"]+# for item in path.iterdir():+# if item.is_dir():+# items.append(f"├── {item.name}/") # Indicate directories with a trailing slash+# else:+# items.append(f"├── {item.name}") # List files as is+# return "\n".join(items)+++class FileSurfer(BaseAgent):+ """An agent that uses tools to read and navigate local files."""++ DEFAULT_DESCRIPTION = "An agent that can handle local files."++ DEFAULT_SYSTEM_MESSAGES = [+ SystemMessage("""+ You are a helpful AI Assistant.+ When given a user query, use available functions to help the user with their request."""),+ ]++ def __init__(+ self,+ model_client: ChatCompletionClient,+ description: str = DEFAULT_DESCRIPTION,+ system_messages: List[SystemMessage] = DEFAULT_SYSTEM_MESSAGES,+ browser: Optional[RequestsMarkdownBrowser] = None,+ ) -> None:+ super().__init__(description)+ self._model_client = model_client+ self._system_messages = system_messages+ self._browser = browser+ self._tools = [TOOL_OPEN_LOCAL_FILE, TOOL_PAGE_UP, TOOL_PAGE_DOWN, TOOL_FIND_ON_PAGE_CTRL_F, TOOL_FIND_NEXT]+ # self._tools = [+ # FunctionTool(+ # read_local_file,+ # description="Use this function to read the contents of a local file whose relative or absolute path is given.",+ # name="read_local_file",+ # ),+ # FunctionTool(+ # list_files_and_dirs_like_tree,+ # description="List files and directories in a directory in a format similar to 'tree' command with level 1",+ # name="list_files_and_dirs_like_tree",+ # ),+ # ]++ def _get_browser_state(self) -> Tuple[str, str]:+ """+ Get the current state of the browser, including the header and content.+ """+ if self._browser is None:+ self._browser = RequestsMarkdownBrowser(viewport_size=1024 * 5, downloads_folder="coding")++ header = f"Address: {self._browser.address}\n"++ if self._browser.page_title is not None:+ header += f"Title: {self._browser.page_title}\n"++ current_page = self._browser.viewport_current_page+ total_pages = len(self._browser.viewport_pages)++ address = self._browser.address+ for i in range(len(self._browser.history) - 2, -1, -1): # Start from the second last+ if self._browser.history[i][0] == address:+ header += f"You previously visited this page {round(time.time() - self._browser.history[i][1])} seconds ago.\n"+ break++ header += f"Viewport position: Showing page {current_page+1} of {total_pages}.\n"++ return (header, self._browser.viewport)++ # async def _generate_reply(self, cancellation_token: CancellationToken) -> Tuple[bool, UserContent]:++ # if self._browser is None:+ # self._browse = RequestsMarkdownBrowser(viewport_size=1024 * 5, downloads_folder="coding")++ # response = await self._model_client.create(self._system_messages + self._chat_history, tools=self._tools)++ # if isinstance(response.content, str):+ # final_result = response.content++ # elif isinstance(response.content, list) and all(isinstance(item, FunctionCall) for item in response.content):+ # results = await asyncio.gather(*[self.send_message(call, self.id) for call in response.content])+ # for result in results:+ # assert isinstance(result, FunctionExecutionResult)+ # final_result = "\n".join(result.content for result in results)+ # else:+ # raise ValueError(f"Unexpected response type: {response.content}")++ # assert isinstance(final_result, str)++ # return "TERMINATE" in final_result, final_result++ # @message_handler+ # async def handle_tool_call(+ # self, message: FunctionCall, cancellation_token: CancellationToken+ # ) -> FunctionExecutionResult:+ # """Handle a tool execution task. This method executes the tool and publishes the result."""+ # function_call = message+ # # Find the tool+ # tool = next((tool for tool in self._tools if tool.name == function_call.name), None)+ # if tool is None:+ # result_as_str = f"Error: Tool not found: {function_call.name}"+ # else:+ # try:+ # arguments = json.loads(function_call.arguments)+ # result = await tool.run_json(args=arguments, cancellation_token=cancellation_token)+ # result_as_str = tool.return_value_as_string(result)+ # except json.JSONDecodeError:+ # result_as_str = f"Error: Invalid arguments: {function_call.arguments}"+ # except Exception as e:+ # result_as_str = f"Error: {e}"+ # return FunctionExecutionResult(content=result_as_str, call_id=function_call.id)++ async def _generate_reply(self, cancellation_token: CancellationToken) -> Tuple[bool, str]:+ if self._browser is None:+ self._browser = RequestsMarkdownBrowser(viewport_size=1024 * 5, downloads_folder="coding")++ history = self._chat_history[0:-1]+ last_message = self._chat_history[-1]+ assert isinstance(last_message, UserMessage)++ task_content = last_message.content # the last message from the sender is the task++ assert self._browser is not None+ # assert self._browser.page_title is not None++ context_message = UserMessage(+ source="user",+ content=f"Your browser is currently open to the page '{self._browser.page_title}' at the address '{self._browser.address}'.",+ )++ task_message = UserMessage(+ source="user",+ content=task_content,+ )++ create_result = await self._model_client.create(+ messages=history + [context_message, task_message], tools=self._tools+ )++ response = create_result.content++ if isinstance(response, str):+ return False, response++ elif isinstance(response, list) and all(isinstance(item, FunctionCall) for item in response):+ function_calls = response+ for function_call in function_calls:+ tool_name = function_call.name+ arguments = json.loads(function_call.arguments)++ if tool_name == "open_local_file":+ path = arguments["path"]+ self._browser.open_local_file(path)+ elif tool_name == "page_up":+ self._browser.page_up()+ elif tool_name == "page_down":+ self._browser.page_down()+ elif tool_name == "find_on_page_ctrl_f":+ search_string = arguments["search_string"]+ self._browser.find_on_page(search_string)+ elif tool_name == "find_next":+ self._browser.find_next()+ header, content = self._get_browser_state()+ final_response = header.strip() + "\n=======================\n" + content+ return False, final_response++ final_response = "TERMINATE"+ return False, final_responsediff --git a/python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.py b/python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.pynew file mode 100644index 000000000..afc504e3a--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/markdown_search.py@@ -0,0 +1,282 @@+import logging+import os+import re+from abc import ABC, abstractmethod+from typing import Any, Dict, List, Optional, cast+from urllib.parse import quote, quote_plus, unquote, urlparse, urlunparse++import requests++# TODO: Fix these types+from .mdconvert import MarkdownConverter # type: ignore++logger = logging.getLogger(__name__)+++class AbstractMarkdownSearch(ABC):+ """+ An abstract class for providing search capabilities to a Markdown browser.+ """++ @abstractmethod+ def search(self, query: str) -> str:+ pass+++class BingMarkdownSearch(AbstractMarkdownSearch):+ """+ Provides Bing web search capabilities to Markdown browsers.+ """++ def __init__(self, bing_api_key: Optional[str] = None, interleave_results: bool = True):+ """+ Perform a Bing web search, and return the results formatted in Markdown.++ Args:+ bing_api_key: key for the Bing search API. If omitted, an attempt is made to read the key from the BING_API_KEY environment variable. If no key is found, BingMarkdownSearch will print a warning, and will fall back to visiting and scraping the live Bing results page. Scraping is objectively worse than using the API, and thus is not recommended.+ interleave_results: When using the Bing API, results are returned based on category (web, news, videos, etc.), along with instructions for how they should be interleaved on the page. When `interleave` is set to True, these interleaving instructions are followed, and a single results list is returned by BingMarkdownSearch. When `interleave` is set to false, results are separated by category, and no interleaving is done.+ """+ self._mdconvert = MarkdownConverter()+ self._interleave_results = interleave_results++ if bing_api_key is None or bing_api_key.strip() == "":+ self._bing_api_key = os.environ.get("BING_API_KEY")+ else:+ self._bing_api_key = bing_api_key++ if self._bing_api_key is None:+ if not self._interleave_results:+ raise ValueError(+ "No Bing API key was provided. This is incompatible with setting `interleave_results` to False. Please provide a key, or set `interleave_results` to True."+ )++ logger.warning(+ "Warning: No Bing API key provided. BingMarkdownSearch will submit an HTTP request to the Bing landing page, but results may be missing or low quality. To resolve this warning, provide a Bing API key by setting the BING_API_KEY environment variable, or using the 'bing_api_key' parameter in by BingMarkdownSearch's constructor. Bing API keys can be obtained via https://www.microsoft.com/en-us/bing/apis/bing-web-search-api\n"+ )++ def search(self, query: str) -> str:+ """Search Bing and return the results formatted in Markdown. If a Bing API key is available, the API is used to perform the search. If no API key is available, the search is performed by submitting an HTTPs GET request directly to Bing. Searches performed with the API are much higher quality, and are more reliable.++ Args:+ query: The search query to issue++ Returns:+ A Markdown rendering of the search results.+ """++ if self._bing_api_key is None:+ return self._fallback_search(query)+ else:+ return self._api_search(query)++ def _api_search(self, query: str) -> str:+ """Search Bing using the API, and return the results formatted in Markdown.++ Args:+ query: The search query to issue++ Returns:+ A Markdown rendering of the search results.+ """+ results = self._bing_api_call(query)++ snippets: Dict[str, List[str]] = dict()++ def _processFacts(elm: List[Dict[str, Any]]) -> str:+ facts: List[str] = list()+ for e in elm:+ k = e["label"]["text"]+ v = " ".join(item["text"] for item in e["items"])+ facts.append(f"{k}: {v}")+ return "\n".join(facts)++ # Web pages+ # __POS__ is a placeholder for the final ranking position, added at the end+ web_snippets: List[str] = list()+ if "webPages" in results:+ for page in results["webPages"]["value"]:+ snippet = f"__POS__. {self._markdown_link(page['name'], page['url'])}\n{page['snippet']}"++ if "richFacts" in page:+ snippet += "\n" + _processFacts(page["richFacts"])++ if "mentions" in page:+ snippet += "\nMentions: " + ", ".join(e["name"] for e in page["mentions"])++ if page["id"] not in snippets:+ snippets[page["id"]] = list()+ snippets[page["id"]].append(snippet)+ web_snippets.append(snippet)++ if "deepLinks" in page:+ for dl in page["deepLinks"]:+ deep_snippet = f"__POS__. {self._markdown_link(dl['name'], dl['url'])}\n{dl['snippet'] if 'snippet' in dl else ''}"+ snippets[page["id"]].append(deep_snippet)+ web_snippets.append(deep_snippet)++ # News results+ news_snippets: List[str] = list()+ if "news" in results:+ for page in results["news"]["value"]:+ snippet = (+ f"__POS__. {self._markdown_link(page['name'], page['url'])}\n{page.get('description', '')}".strip()+ )++ if "datePublished" in page:+ snippet += "\nDate published: " + page["datePublished"].split("T")[0]++ if "richFacts" in page:+ snippet += "\n" + _processFacts(page["richFacts"])++ if "mentions" in page:+ snippet += "\nMentions: " + ", ".join(e["name"] for e in page["mentions"])++ news_snippets.append(snippet)++ if len(news_snippets) > 0:+ snippets[results["news"]["id"]] = news_snippets++ # Videos+ video_snippets: List[str] = list()+ if "videos" in results:+ for page in results["videos"]["value"]:+ if not page["contentUrl"].startswith("https://www.youtube.com/watch?v="):+ continue++ snippet = f"__POS__. {self._markdown_link(page['name'], page['contentUrl'])}\n{page.get('description', '')}".strip()++ if "datePublished" in page:+ snippet += "\nDate published: " + page["datePublished"].split("T")[0]++ if "richFacts" in page:+ snippet += "\n" + _processFacts(page["richFacts"])++ if "mentions" in page:+ snippet += "\nMentions: " + ", ".join(e["name"] for e in page["mentions"])++ video_snippets.append(snippet)++ if len(video_snippets) > 0:+ snippets[results["videos"]["id"]] = video_snippets++ # Related searches+ related_searches = ""+ if "relatedSearches" in results:+ related_searches = "## Related Searches:\n"+ for s in results["relatedSearches"]["value"]:+ related_searches += "- " + s["text"] + "\n"+ snippets[results["relatedSearches"]["id"]] = [related_searches.strip()]++ idx = 0+ content = ""+ if self._interleave_results:+ # Interleaved+ for item in results["rankingResponse"]["mainline"]["items"]:+ _id = item["value"]["id"]+ if _id in snippets:+ for s in snippets[_id]:+ if "__POS__" in s:+ idx += 1+ content += s.replace("__POS__", str(idx)) + "\n\n"+ else:+ content += s + "\n\n"+ else:+ # Categorized+ if len(web_snippets) > 0:+ content += "## Web Results\n\n"+ for s in web_snippets:+ if "__POS__" in s:+ idx += 1+ content += s.replace("__POS__", str(idx)) + "\n\n"+ else:+ content += s + "\n\n"+ if len(news_snippets) > 0:+ content += "## News Results\n\n"+ for s in news_snippets:+ if "__POS__" in s:+ idx += 1+ content += s.replace("__POS__", str(idx)) + "\n\n"+ else:+ content += s + "\n\n"+ if len(video_snippets) > 0:+ content += "## Video Results\n\n"+ for s in video_snippets:+ if "__POS__" in s:+ idx += 1+ content += s.replace("__POS__", str(idx)) + "\n\n"+ else:+ content += s + "\n\n"+ if len(related_searches) > 0:+ content += related_searches++ return f"## A Bing search for '{query}' found {idx} results:\n\n" + content.strip()++ def _bing_api_call(self, query: str) -> Dict[str, Any]:+ """Make a Bing API call, and return a Python representation of the JSON response."++ Args:+ query: The search query to issue++ Returns:+ A Python representation of the Bing API's JSON response (as parsed by `json.loads()`).+ """+ # Make sure the key was set+ if not self._bing_api_key:+ raise ValueError("Missing Bing API key.")++ # Prepare the request parameters+ request_kwargs: Dict[str, Any] = {}+ request_kwargs["headers"] = {}+ request_kwargs["headers"]["Ocp-Apim-Subscription-Key"] = self._bing_api_key++ request_kwargs["params"] = {}+ request_kwargs["params"]["q"] = query+ request_kwargs["params"]["textDecorations"] = False+ request_kwargs["params"]["textFormat"] = "raw"++ request_kwargs["stream"] = False++ # Make the request+ response = requests.get("https://api.bing.microsoft.com/v7.0/search", **request_kwargs)+ response.raise_for_status()+ results = response.json()++ return cast(Dict[str, Any], results)++ def _fallback_search(self, query: str) -> str:+ """When no Bing API key is provided, we issue a simple HTTPs GET call to the Bing landing page and convert it to Markdown.++ Args:+ query: The search query to issue++ Returns:+ The Bing search results page, converted to Markdown.+ """+ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"+ headers = {"User-Agent": user_agent}++ url = f"https://www.bing.com/search?q={quote_plus(query)}&FORM=QBLH"+ response = requests.get(url, headers=headers)+ response.raise_for_status()+ # TODO: Fix the types+ return self._mdconvert.convert_response(response).text_content # type: ignore++ def _markdown_link(self, anchor: str, href: str) -> str:+ """Create a Markdown hyperlink, escaping the URLs as appropriate.++ Args:+ anchor: The anchor text of the hyperlink+ href: The href destination of the hyperlink++ Returns:+ A correctly-formatted Markdown hyperlink+ """+ try:+ parsed_url = urlparse(href)+ # URLs provided by Bing are sometimes only partially quoted, leaving in characters+ # the conflict with Markdown. We unquote the URL, and then re-quote more completely+ href = urlunparse(parsed_url._replace(path=quote(unquote(parsed_url.path))))+ anchor = re.sub(r"[\[\]]", " ", anchor)+ return f"[{anchor}]({href})"+ except ValueError: # It's not clear if this ever gets thrown+ return f"[{anchor}]({href})"diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.py b/python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.pynew file mode 100644index 000000000..1b68e4bda--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/mdconvert.py@@ -0,0 +1,1055 @@+# type: ignore+import base64+import binascii+import copy+import html+import json+import mimetypes+import os+import re+import shutil+import subprocess+import sys+import tempfile+from typing import Any, Dict, List, Optional, Union+from urllib.parse import parse_qs, quote, unquote, urlparse, urlunparse++import mammoth+import markdownify+import pandas as pd+import pdfminer+import pdfminer.high_level+import pptx++# File-format detection+import puremagic+import requests+from binaryornot.check import is_binary # type: ignore+from bs4 import BeautifulSoup++# Optional OCR support+try:+ import easyocr+ import numpy as np+ import PIL++ IS_OCR_CAPABLE = True+except ModuleNotFoundError:+ pass++# Optional Transcription support+try:+ import pydub+ import speech_recognition as sr++ IS_AUDIO_TRANSCRIPTION_CAPABLE = True+except ModuleNotFoundError:+ pass++# Optional YouTube transcription support+try:+ from youtube_transcript_api import YouTubeTranscriptApi++ IS_YOUTUBE_TRANSCRIPT_CAPABLE = True+except ModuleNotFoundError:+ pass+++class _CustomMarkdownify(markdownify.MarkdownConverter):+ """+ A custom version of markdownify's MarkdownConverter. Changes include:++ - Altering the default heading style to use '#', '##', etc.+ - Removing javascript hyperlinks.+ - Truncating images with large data:uri sources.+ - Ensuring URIs are properly escaped, and do not conflict with Markdown syntax+ """++ def __init__(self, **options: Any):+ options["heading_style"] = options.get("heading_style", markdownify.ATX)+ # Explicitly cast options to the expected type if necessary+ super().__init__(**options)++ def convert_hn(self, n: int, el: Any, text: str, convert_as_inline: bool) -> str:+ """Same as usual, but be sure to start with a new line"""+ if not convert_as_inline:+ if not re.search(r"^\n", text):+ return "\n" + super().convert_hn(n, el, text, convert_as_inline) # type: ignore++ return super().convert_hn(n, el, text, convert_as_inline) # type: ignore++ def convert_a(self, el: Any, text: str, convert_as_inline: bool):+ """Same as usual converter, but removes Javascript links and escapes URIs."""+ prefix, suffix, text = markdownify.chomp(text) # type: ignore+ if not text:+ return ""+ href = el.get("href")+ title = el.get("title")++ # Escape URIs and skip non-http or file schemes+ if href:+ try:+ parsed_url = urlparse(href) # type: ignore+ if parsed_url.scheme and parsed_url.scheme.lower() not in ["http", "https", "file"]: # type: ignore+ return "%s%s%s" % (prefix, text, suffix)+ href = urlunparse(parsed_url._replace(path=quote(unquote(parsed_url.path)))) # type: ignore+ except ValueError: # It's not clear if this ever gets thrown+ return "%s%s%s" % (prefix, text, suffix)++ # For the replacement see #29: text nodes underscores are escaped+ if (+ self.options["autolinks"]+ and text.replace(r"\_", "_") == href+ and not title+ and not self.options["default_title"]+ ):+ # Shortcut syntax+ return "<%s>" % href+ if self.options["default_title"] and not title:+ title = href+ title_part = ' "%s"' % title.replace('"', r"\"") if title else ""+ return "%s[%s](%s%s)%s" % (prefix, text, href, title_part, suffix) if href else text++ def convert_img(self, el: Any, text: str, convert_as_inline: bool) -> str:+ """Same as usual converter, but removes data URIs"""++ alt = el.attrs.get("alt", None) or ""+ src = el.attrs.get("src", None) or ""+ title = el.attrs.get("title", None) or ""+ title_part = ' "%s"' % title.replace('"', r"\"") if title else ""+ if convert_as_inline and el.parent.name not in self.options["keep_inline_images_in"]:+ return alt++ # Remove dataURIs+ if src.startswith("data:"):+ src = src.split(",")[0] + "..."++ return "" % (alt, src, title_part)++ def convert_soup(self, soup: Any) -> str:+ return super().convert_soup(soup) # type: ignore+++class DocumentConverterResult:+ """The result of converting a document to text."""++ def __init__(self, title: Union[str, None] = None, text_content: str = ""):+ self.title: Union[str, None] = title+ self.text_content: str = text_content+++class DocumentConverter:+ """Abstract superclass of all DocumentConverters."""++ def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:+ raise NotImplementedError()+++class PlainTextConverter(DocumentConverter):+ """Anything with content type text/plain"""++ def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:+ # Guess the content type from any file extension that might be around+ content_type, _ = mimetypes.guess_type("__placeholder" + kwargs.get("file_extension", ""))++ if content_type is None:+ # No content type, so peek at the file and see if it's binary+ if is_binary(local_path):+ return None+ elif "text/" not in content_type.lower():+ return None++ text_content = ""+ with open(local_path, "rt") as fh:+ text_content = fh.read()++ return DocumentConverterResult(+ title=None,+ text_content=text_content,+ )+++class HtmlConverter(DocumentConverter):+ """Anything with content type text/html"""++ def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:+ # Bail if not html+ extension = kwargs.get("file_extension", "")+ if extension.lower() not in [".html", ".htm"]:+ return None++ result = None+ with open(local_path, "rt") as fh:+ result = self._convert(fh.read())++ return result++ def _convert(self, html_content: str) -> Union[None, DocumentConverterResult]:+ """Helper function that converts and HTML string."""++ # Parse the string+ soup = BeautifulSoup(html_content, "html.parser")++ # Remove javascript and style blocks+ for script in soup(["script", "style"]):+ script.extract()++ # Print only the main content+ body_elm = soup.find("body")+ webpage_text = ""+ if body_elm:+ webpage_text = _CustomMarkdownify().convert_soup(body_elm)+ else:+ webpage_text = _CustomMarkdownify().convert_soup(soup)++ assert isinstance(webpage_text, str)++ return DocumentConverterResult(+ title=None if soup.title is None else soup.title.string, text_content=webpage_text+ )+++class WikipediaConverter(DocumentConverter):+ """Handle Wikipedia pages separately, focusing only on the main document content."""++ def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:+ # Bail if not Wikipedia+ extension = kwargs.get("file_extension", "")+ if extension.lower() not in [".html", ".htm"]:+ return None+ url = kwargs.get("url", "")+ if not re.search(r"^https?:\/\/[a-zA-Z]{2,3}\.wikipedia.org\/", url):+ return None++ # Parse the file+ soup = None+ with open(local_path, "rt") as fh:+ soup = BeautifulSoup(fh.read(), "html.parser")++ # Remove javascript and style blocks+ for script in soup(["script", "style"]):+ script.extract()++ # Print only the main content+ body_elm = soup.find("div", {"id": "mw-content-text"})+ title_elm = soup.find("span", {"class": "mw-page-title-main"})++ webpage_text = ""+ main_title = None if soup.title is None else soup.title.string++ if body_elm:+ # What's the title+ if title_elm and len(title_elm) > 0:+ main_title = title_elm.string # type: ignore+ assert isinstance(main_title, str)++ # Convert the page+ webpage_text = f"# {main_title}\n\n" + _CustomMarkdownify().convert_soup(body_elm)+ else:+ webpage_text = _CustomMarkdownify().convert_soup(soup)++ return DocumentConverterResult(+ title=main_title,+ text_content=webpage_text,+ )+++class YouTubeConverter(DocumentConverter):+ """Handle YouTube specially, focusing on the video title, description, and transcript."""++ def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:+ # Bail if not YouTube+ extension = kwargs.get("file_extension", "")+ if extension.lower() not in [".html", ".htm"]:+ return None+ url = kwargs.get("url", "")+ if not url.startswith("https://www.youtube.com/watch?"):+ return None++ # Parse the file+ soup = None+ with open(local_path, "rt") as fh:+ soup = BeautifulSoup(fh.read(), "html.parser")++ # Read the meta tags+ assert soup.title is not None and soup.title.string is not None+ metadata: Dict[str, str] = {"title": soup.title.string}+ for meta in soup(["meta"]):+ for a in meta.attrs:+ if a in ["itemprop", "property", "name"]:+ metadata[meta[a]] = meta.get("content", "")+ break++ # We can also try to read the full description. This is more prone to breaking, since it reaches into the page implementation+ try:+ for script in soup(["script"]):+ content = script.text+ if "ytInitialData" in content:+ lines = re.split(r"\r?\n", content)+ obj_start = lines[0].find("{")+ obj_end = lines[0].rfind("}")+ if obj_start >= 0 and obj_end >= 0:+ data = json.loads(lines[0][obj_start : obj_end + 1])+ attrdesc = self._findKey(data, "attributedDescriptionBodyText") # type: ignore+ if attrdesc:+ metadata["description"] = str(attrdesc["content"])+ break+ except Exception:+ pass++ # Start preparing the page+ webpage_text = "# YouTube\n"++ title = self._get(metadata, ["title", "og:title", "name"]) # type: ignore+ assert isinstance(title, str)++ if title:+ webpage_text += f"\n## {title}\n"++ stats = ""+ views = self._get(metadata, ["interactionCount"]) # type: ignore+ if views:+ stats += f"- **Views:** {views}\n"++ keywords = self._get(metadata, ["keywords"]) # type: ignore+ if keywords:+ stats += f"- **Keywords:** {keywords}\n"++ runtime = self._get(metadata, ["duration"]) # type: ignore+ if runtime:+ stats += f"- **Runtime:** {runtime}\n"++ if len(stats) > 0:+ webpage_text += f"\n### Video Metadata\n{stats}\n"++ description = self._get(metadata, ["description", "og:description"]) # type: ignore+ if description:+ webpage_text += f"\n### Description\n{description}\n"++ if IS_YOUTUBE_TRANSCRIPT_CAPABLE:+ transcript_text = ""+ parsed_url = urlparse(url) # type: ignore+ params = parse_qs(parsed_url.query) # type: ignore+ if "v" in params:+ assert isinstance(params["v"][0], str)+ video_id = str(params["v"][0])+ try:+ # Must be a single transcript.+ transcript = YouTubeTranscriptApi.get_transcript(video_id) # type: ignore+ transcript_text = " ".join([part["text"] for part in transcript]) # type: ignore+ # Alternative formatting:+ # formatter = TextFormatter()+ # formatter.format_transcript(transcript)+ except Exception:+ pass+ if transcript_text:+ webpage_text += f"\n### Transcript\n{transcript_text}\n"++ title = title if title else soup.title.string+ assert isinstance(title, str)++ return DocumentConverterResult(+ title=title,+ text_content=webpage_text,+ )++ def _get(self, metadata: Dict[str, str], keys: List[str], default: Union[str, None] = None) -> Union[str, None]:+ for k in keys:+ if k in metadata:+ return metadata[k]+ return default++ def _findKey(self, json: Any, key: str) -> Union[str, None]: # TODO: Fix json type+ if isinstance(json, list):+ for elm in json:+ ret = self._findKey(elm, key)+ if ret is not None:+ return ret+ elif isinstance(json, dict):+ for k in json:+ if k == key:+ return json[k]+ else:+ ret = self._findKey(json[k], key)+ if ret is not None:+ return ret+ return None+++class BingSerpConverter(DocumentConverter):+ """+ Handle Bing results pages (only the organic search results).+ NOTE: It is better to use the Bing API+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a Bing SERP+ extension = kwargs.get("file_extension", "")+ if extension.lower() not in [".html", ".htm"]:+ return None+ url = kwargs.get("url", "")+ if not re.search(r"^https://www\.bing\.com/search\?q=", url):+ return None++ # Parse the query parameters+ parsed_params = parse_qs(urlparse(url).query)+ query = parsed_params.get("q", [""])[0]++ # Parse the file+ soup = None+ with open(local_path, "rt") as fh:+ soup = BeautifulSoup(fh.read(), "html.parser")++ # Clean up some formatting+ for tptt in soup.find_all(class_="tptt"):+ if hasattr(tptt, "string") and tptt.string:+ tptt.string += " "+ for slug in soup.find_all(class_="algoSlug_icon"):+ slug.extract()++ # Parse the algorithmic results+ _markdownify = _CustomMarkdownify()+ results = list()+ for result in soup.find_all(class_="b_algo"):+ # Rewrite redirect urls+ for a in result.find_all("a", href=True):+ parsed_href = urlparse(a["href"])+ qs = parse_qs(parsed_href.query)++ # The destination is contained in the u parameter,+ # but appears to be base64 encoded, with some prefix+ if "u" in qs:+ u = qs["u"][0][2:].strip() + "==" # Python 3 doesn't care about extra padding++ try:+ # RFC 4648 / Base64URL" variant, which uses "-" and "_"+ a["href"] = base64.b64decode(u, altchars="-_").decode("utf-8")+ except UnicodeDecodeError:+ pass+ except binascii.Error:+ pass++ # Convert to markdown+ md_result = _markdownify.convert_soup(result).strip()+ lines = [line.strip() for line in re.split(r"\n+", md_result)]+ results.append("\n".join([line for line in lines if len(line) > 0]))++ webpage_text = f"## A Bing search for '{query}' found the following results:\n\n" + "\n\n".join(results)++ return DocumentConverterResult(+ title=None if soup.title is None else soup.title.string,+ text_content=webpage_text,+ )+++class PdfConverter(DocumentConverter):+ """+ Converts PDFs to Markdown. Most style information is ignored, so the results are essentially plain-text.+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a PDF+ extension = kwargs.get("file_extension", "")+ if extension.lower() != ".pdf":+ return None++ return DocumentConverterResult(+ title=None,+ text_content=pdfminer.high_level.extract_text(local_path),+ )+++class DocxConverter(HtmlConverter):+ """+ Converts DOCX files to Markdown. Style information (e.g.m headings) and tables are preserved where possible.+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a DOCX+ extension = kwargs.get("file_extension", "")+ if extension.lower() != ".docx":+ return None++ result = None+ with open(local_path, "rb") as docx_file:+ result = mammoth.convert_to_html(docx_file)+ html_content = result.value+ result = self._convert(html_content)++ return result+++class XlsxConverter(HtmlConverter):+ """+ Converts XLSX files to Markdown, with each sheet presented as a separate Markdown table.+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a XLSX+ extension = kwargs.get("file_extension", "")+ if extension.lower() != ".xlsx":+ return None++ sheets = pd.read_excel(local_path, sheet_name=None)+ md_content = ""+ for s in sheets:+ md_content += f"## {s}\n"+ html_content = sheets[s].to_html(index=False)+ md_content += self._convert(html_content).text_content.strip() + "\n\n"++ return DocumentConverterResult(+ title=None,+ text_content=md_content.strip(),+ )+++class PptxConverter(HtmlConverter):+ """+ Converts PPTX files to Markdown. Supports heading, tables and images with alt text.+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a PPTX+ extension = kwargs.get("file_extension", "")+ if extension.lower() != ".pptx":+ return None++ md_content = ""++ presentation = pptx.Presentation(local_path)+ slide_num = 0+ for slide in presentation.slides:+ slide_num += 1++ md_content += f"\n\n<!-- Slide number: {slide_num} -->\n"++ title = slide.shapes.title+ for shape in slide.shapes:+ # Pictures+ if self._is_picture(shape):+ # https://github.com/scanny/python-pptx/pull/512#issuecomment-1713100069+ alt_text = ""+ try:+ alt_text = shape._element._nvXxPr.cNvPr.attrib.get("descr", "")+ except Exception:+ pass++ # A placeholder name+ filename = re.sub(r"\W", "", shape.name) + ".jpg"+ md_content += "\n\n"++ # Tables+ if self._is_table(shape):+ html_table = "<html><body><table>"+ first_row = True+ for row in shape.table.rows:+ html_table += "<tr>"+ for cell in row.cells:+ if first_row:+ html_table += "<th>" + html.escape(cell.text) + "</th>"+ else:+ html_table += "<td>" + html.escape(cell.text) + "</td>"+ html_table += "</tr>"+ first_row = False+ html_table += "</table></body></html>"+ md_content += "\n" + self._convert(html_table).text_content.strip() + "\n"++ # Text areas+ elif shape.has_text_frame:+ if shape == title:+ md_content += "# " + shape.text.lstrip() + "\n"+ else:+ md_content += shape.text + "\n"++ md_content = md_content.strip()++ if slide.has_notes_slide:+ md_content += "\n\n### Notes:\n"+ notes_frame = slide.notes_slide.notes_text_frame+ if notes_frame is not None:+ md_content += notes_frame.text+ md_content = md_content.strip()++ return DocumentConverterResult(+ title=None,+ text_content=md_content.strip(),+ )++ def _is_picture(self, shape):+ if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE:+ return True+ if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PLACEHOLDER:+ if hasattr(shape, "image"):+ return True+ return False++ def _is_table(self, shape):+ if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.TABLE:+ return True+ return False+++class MediaConverter(DocumentConverter):+ """+ Abstract class for multi-modal media (e.g., images and audio)+ """++ def _get_metadata(self, local_path):+ exiftool = shutil.which("exiftool")+ if not exiftool:+ return None+ else:+ try:+ result = subprocess.run([exiftool, "-json", local_path], capture_output=True, text=True).stdout+ return json.loads(result)[0]+ except Exception:+ return None+++class WavConverter(MediaConverter):+ """+ Converts WAV files to markdown via extraction of metadata (if `exiftool` is installed), and speech transcription (if `speech_recognition` is installed).+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a XLSX+ extension = kwargs.get("file_extension", "")+ if extension.lower() != ".wav":+ return None++ md_content = ""++ # Add metadata+ metadata = self._get_metadata(local_path)+ if metadata:+ for f in [+ "Title",+ "Artist",+ "Author",+ "Band",+ "Album",+ "Genre",+ "Track",+ "DateTimeOriginal",+ "CreateDate",+ "Duration",+ ]:+ if f in metadata:+ md_content += f"{f}: {metadata[f]}\n"++ # Transcribe+ if IS_AUDIO_TRANSCRIPTION_CAPABLE:+ try:+ transcript = self._transcribe_audio(local_path)+ md_content += "\n\n### Audio Transcript:\n" + (+ "[No speech detected]" if transcript == "" else transcript+ )+ except Exception:+ md_content += "\n\n### Audio Transcript:\nError. Could not transcribe this audio."++ return DocumentConverterResult(+ title=None,+ text_content=md_content.strip(),+ )++ def _transcribe_audio(self, local_path) -> str:+ recognizer = sr.Recognizer()+ with sr.AudioFile(local_path) as source:+ audio = recognizer.record(source)+ return recognizer.recognize_google(audio).strip()+++class Mp3Converter(WavConverter):+ """+ Converts MP3 files to markdown via extraction of metadata (if `exiftool` is installed), and speech transcription (if `speech_recognition` AND `pydub` are installed).+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a MP3+ extension = kwargs.get("file_extension", "")+ if extension.lower() != ".mp3":+ return None++ md_content = ""++ # Add metadata+ metadata = self._get_metadata(local_path)+ if metadata:+ for f in [+ "Title",+ "Artist",+ "Author",+ "Band",+ "Album",+ "Genre",+ "Track",+ "DateTimeOriginal",+ "CreateDate",+ "Duration",+ ]:+ if f in metadata:+ md_content += f"{f}: {metadata[f]}\n"++ # Transcribe+ if IS_AUDIO_TRANSCRIPTION_CAPABLE:+ handle, temp_path = tempfile.mkstemp(suffix=".wav")+ os.close(handle)+ try:+ sound = pydub.AudioSegment.from_mp3(local_path)+ sound.export(temp_path, format="wav")++ _args = dict()+ _args.update(kwargs)+ _args["file_extension"] = ".wav"++ try:+ transcript = super()._transcribe_audio(temp_path).strip()+ md_content += "\n\n### Audio Transcript:\n" + (+ "[No speech detected]" if transcript == "" else transcript+ )+ except Exception:+ md_content += "\n\n### Audio Transcript:\nError. Could not transcribe this audio."++ finally:+ os.unlink(temp_path)++ # Return the result+ return DocumentConverterResult(+ title=None,+ text_content=md_content.strip(),+ )+++class ImageConverter(MediaConverter):+ """+ Converts images to markdown via extraction of metadata (if `exiftool` is installed), OCR (if `easyocr` is installed), and description via a multimodal LLM (if an mlm_client is configured).+ """++ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:+ # Bail if not a XLSX+ extension = kwargs.get("file_extension", "")+ if extension.lower() not in [".jpg", ".jpeg", ".png"]:+ return None++ ocr_min_confidence = kwargs.get("ocr_min_confidence", 0.25)++ md_content = ""++ # Add metadata+ metadata = self._get_metadata(local_path)+ if metadata:+ for f in [+ "ImageSize",+ "Title",+ "Caption",+ "Description",+ "Keywords",+ "Artist",+ "Author",+ "DateTimeOriginal",+ "CreateDate",+ "GPSPosition",+ ]:+ if f in metadata:+ md_content += f"{f}: {metadata[f]}\n"++ # Try describing the image with GPTV+ mlm_client = kwargs.get("mlm_client")+ if mlm_client is not None:+ md_content += (+ "\n# Description:\n"+ + self._get_mlm_description(local_path, extension, mlm_client, prompt=kwargs.get("mlm_prompt")).strip()+ + "\n"+ )++ if IS_OCR_CAPABLE:+ image = PIL.Image.open(local_path)+ # Remove transparency+ if image.mode in ("RGBA", "P"):+ image = image.convert("RGB")++ reader = easyocr.Reader(["en"]) # specify the language(s)+ output = reader.readtext(np.array(image)) # local_path)+ # The output is a list of tuples, each containing the coordinates of the text and the text itself.+ # We join all the text pieces together to get the final text.+ ocr_text = " "+ for item in output:+ if item[2] >= ocr_min_confidence:+ ocr_text += item[1] + " "+ ocr_text = ocr_text.strip()++ if len(ocr_text) > 0:+ md_content += "\n# Text detected by OCR:\n" + ocr_text++ return DocumentConverterResult(+ title=None,+ text_content=md_content,+ )++ def _get_mlm_description(self, local_path, extension, client, prompt=None):+ if prompt is None or prompt.strip() == "":+ prompt = "Write a detailed caption for this image."++ sys.stderr.write(f"MLM Prompt:\n{prompt}\n")++ data_uri = ""+ with open(local_path, "rb") as image_file:+ content_type, encoding = mimetypes.guess_type("_dummy" + extension)+ if content_type is None:+ content_type = "image/jpeg"+ image_base64 = base64.b64encode(image_file.read()).decode("utf-8")+ data_uri = f"data:{content_type};base64,{image_base64}"++ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": prompt},+ {+ "type": "image_url",+ "image_url": {+ "url": data_uri,+ },+ },+ ],+ }+ ]++ response = client.create(messages=messages)+ return client.extract_text_or_completion_object(response)[0]+++class FileConversionException(BaseException):+ pass+++class UnsupportedFormatException(BaseException):+ pass+++class MarkdownConverter:+ """(In preview) An extremely simple text-based document reader, suitable for LLM use.+ This reader will convert common file-types or webpages to Markdown."""++ def __init__(+ self,+ requests_session: Optional[requests.Session] = None,+ mlm_client: Optional[Any] = None,+ ):+ if requests_session is None:+ self._requests_session = requests.Session()+ else:+ self._requests_session = requests_session++ self._mlm_client = mlm_client++ self._page_converters: List[DocumentConverter] = []++ # Register converters for successful browsing operations+ # Later registrations are tried first / take higher priority than earlier registrations+ # To this end, the most specific converters should appear below the most generic converters+ self.register_page_converter(PlainTextConverter())+ self.register_page_converter(HtmlConverter())+ self.register_page_converter(WikipediaConverter())+ self.register_page_converter(YouTubeConverter())+ self.register_page_converter(BingSerpConverter())+ self.register_page_converter(DocxConverter())+ self.register_page_converter(XlsxConverter())+ self.register_page_converter(PptxConverter())+ self.register_page_converter(WavConverter())+ self.register_page_converter(Mp3Converter())+ self.register_page_converter(ImageConverter())+ self.register_page_converter(PdfConverter())++ def convert(+ self, source: Union[str, requests.Response], **kwargs: Any+ ) -> DocumentConverterResult: # TODO: deal with kwargs+ """+ Args:+ - source: can be a string representing a path or url, or a requests.response object+ - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.)+ """++ # Local path or url+ if isinstance(source, str):+ if source.startswith("http://") or source.startswith("https://") or source.startswith("file://"):+ return self.convert_url(source, **kwargs)+ else:+ return self.convert_local(source, **kwargs)+ # Request response+ elif isinstance(source, requests.Response):+ return self.convert_response(source, **kwargs)++ def convert_local(self, path: str, **kwargs: Any) -> DocumentConverterResult: # TODO: deal with kwargs+ # Prepare a list of extensions to try (in order of priority)+ ext = kwargs.get("file_extension")+ extensions = [ext] if ext is not None else []++ # Get extension alternatives from the path and puremagic+ base, ext = os.path.splitext(path)+ self._append_ext(extensions, ext)+ self._append_ext(extensions, self._guess_ext_magic(path))++ # Convert+ return self._convert(path, extensions, **kwargs)++ def convert_stream(self, stream, **kwargs: Any) -> DocumentConverterResult: # TODO: deal with kwargs+ # Prepare a list of extensions to try (in order of priority)+ ext = kwargs.get("file_extension")+ extensions = [ext] if ext is not None else []++ # Save the file locally to a temporary file. It will be deleted before this method exits+ handle, temp_path = tempfile.mkstemp()+ fh = os.fdopen(handle, "wb")+ result = None+ try:+ # Write to the temporary file+ content = stream.read()+ if isinstance(content, str):+ fh.write(content.encode("utf-8"))+ else:+ fh.write(content)+ fh.close()++ # Use puremagic to check for more extension options+ self._append_ext(extensions, self._guess_ext_magic(temp_path))++ # Convert+ result = self._convert(temp_path, extensions, **kwargs)+ # Clean up+ finally:+ try:+ fh.close()+ except Exception:+ pass+ os.unlink(temp_path)++ return result++ def convert_url(self, url: str, **kwargs: Any) -> DocumentConverterResult: # TODO: fix kwargs type+ # Send a HTTP request to the URL+ response = self._requests_session.get(url, stream=True)+ response.raise_for_status()+ return self.convert_response(response, **kwargs)++ def convert_response(+ self, response: requests.Response, **kwargs: Any+ ) -> DocumentConverterResult: # TODO fix kwargs type+ # Prepare a list of extensions to try (in order of priority)+ ext = kwargs.get("file_extension")+ extensions = [ext] if ext is not None else []++ # Guess from the mimetype+ content_type = response.headers.get("content-type", "").split(";")[0]+ self._append_ext(extensions, mimetypes.guess_extension(content_type))++ # Read the content disposition if there is one+ content_disposition = response.headers.get("content-disposition", "")+ m = re.search(r"filename=([^;]+)", content_disposition)+ if m:+ base, ext = os.path.splitext(m.group(1).strip("\"'"))+ self._append_ext(extensions, ext)++ # Read from the extension from the path+ base, ext = os.path.splitext(urlparse(response.url).path)+ self._append_ext(extensions, ext)++ # Save the file locally to a temporary file. It will be deleted before this method exits+ handle, temp_path = tempfile.mkstemp()+ fh = os.fdopen(handle, "wb")+ result = None+ try:+ # Download the file+ for chunk in response.iter_content(chunk_size=512):+ fh.write(chunk)+ fh.close()++ # Use puremagic to check for more extension options+ self._append_ext(extensions, self._guess_ext_magic(temp_path))++ # Convert+ result = self._convert(temp_path, extensions, url=response.url)++ # Clean up+ finally:+ try:+ fh.close()+ except Exception:+ pass+ os.unlink(temp_path)++ return result++ def _convert(self, local_path: str, extensions: List[Union[str, None]], **kwargs) -> DocumentConverterResult:+ error_trace = ""+ for ext in extensions + [None]: # Try last with no extension+ for converter in self._page_converters:+ _kwargs = copy.deepcopy(kwargs)++ # Overwrite file_extension appropriately+ if ext is None:+ if "file_extension" in _kwargs:+ del _kwargs["file_extension"]+ else:+ _kwargs.update({"file_extension": ext})++ # Copy any additional global options+ if "mlm_client" not in _kwargs and self._mlm_client is not None:+ _kwargs["mlm_client"] = self._mlm_client++ # If we hit an error log it and keep trying+ # try:+ res = converter.convert(local_path, **_kwargs)+ # except Exception:+ # error_trace = ("\n\n" + traceback.format_exc()).strip()++ if res is not None:+ # Normalize the content+ res.text_content = "\n".join([line.rstrip() for line in re.split(r"\r?\n", res.text_content)])+ res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)++ # Todo+ return res++ # If we got this far without success, report any exceptions+ if len(error_trace) > 0:+ raise FileConversionException(+ f"Could not convert '{local_path}' to Markdown. File type was recognized as {extensions}. While converting the file, the following error was encountered:\n\n{error_trace}"+ )++ # Nothing can handle it!+ raise UnsupportedFormatException(+ f"Could not convert '{local_path}' to Markdown. The formats {extensions} are not supported."+ )++ def _append_ext(self, extensions, ext):+ """Append a unique non-None, non-empty extension to a list of extensions."""+ if ext is None:+ return+ ext = ext.strip()+ if ext == "":+ return+ # if ext not in extensions:+ if True:+ extensions.append(ext)++ def _guess_ext_magic(self, path):+ """Use puremagic (a Python implementation of libmagic) to guess a file's extension based on the first few bytes."""+ # Use puremagic to guess+ try:+ guesses = puremagic.magic_file(path)+ if len(guesses) > 0:+ ext = guesses[0].extension.strip()+ if len(ext) > 0:+ return ext+ except FileNotFoundError:+ pass+ except IsADirectoryError:+ pass+ except PermissionError:+ pass+ return None++ def register_page_converter(self, converter: DocumentConverter) -> None:+ """Register a page text converter."""+ self._page_converters.insert(0, converter)diff --git a/python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.py b/python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.pynew file mode 100644index 000000000..9bf0a45c4--- /dev/null+++ b/python/teams/team-one/src/team_one/agents/file_surfer/requests_markdown_browser.py@@ -0,0 +1,437 @@+# ruff: noqa: E722+import datetime+import html+import io+import mimetypes+import os+import pathlib+import re+import time+import traceback+import uuid+from typing import Any, Dict, List, Optional, Tuple, Union+from urllib.parse import unquote, urljoin, urlparse++import pathvalidate+import requests++from .abstract_markdown_browser import AbstractMarkdownBrowser+from .markdown_search import AbstractMarkdownSearch, BingMarkdownSearch++# TODO: Fix unfollowed import+from .mdconvert import FileConversionException, MarkdownConverter, UnsupportedFormatException # type: ignore+++class RequestsMarkdownBrowser(AbstractMarkdownBrowser):+ """+ (In preview) An extremely simple Python requests-powered Markdown web browser.+ This browser cannot run JavaScript, compute CSS, etc. It simply fetches the HTML document, and converts it to Markdown.+ See AbstractMarkdownBrowser for more details.+ """++ # TODO: Fix unfollowed import+ def __init__( # type: ignore+ self,+ start_page: Union[str, None] = None,+ viewport_size: Union[int, None] = 1024 * 8,+ downloads_folder: Union[str, None] = None,+ search_engine: Union[AbstractMarkdownSearch, None] = None,+ markdown_converter: Union[MarkdownConverter, None] = None,+ requests_session: Union[requests.Session, None] = None,+ requests_get_kwargs: Union[Dict[str, Any], None] = None,+ ):+ """+ Instantiate a new RequestsMarkdownBrowser.++ Arguments:+ start_page: The page on which the browser starts (default: "about:blank")+ viewport_size: Approximately how many *characters* fit in the viewport. Viewport dimensions are adjusted dynamically to avoid cutting off words (default: 8192).+ downloads_folder: Path to where downloads are saved. If None, downloads are disabled. (default: None)+ search_engine: An instance of MarkdownSearch, which handles web searches performed by this browser (default: a new `BingMarkdownSearch()` with default parameters)+ markdown_converted: An instance of a MarkdownConverter used to convert HTML pages and downloads to Markdown (default: a new `MarkdownConerter()` with default parameters)+ request_session: The session from which to issue requests (default: a new `requests.Session()` instance with default parameters)+ request_get_kwargs: Extra parameters passed to evert `.get()` call made to requests.+ """+ self.start_page: str = start_page if start_page else "about:blank"+ self.viewport_size = viewport_size # Applies only to the standard uri types+ self.downloads_folder = downloads_folder+ self.history: List[Tuple[str, float]] = list()+ self.page_title: Optional[str] = None+ self.viewport_current_page = 0+ self.viewport_pages: List[Tuple[int, int]] = list()+ self.set_address(self.start_page)+ self._page_content: str = ""++ if search_engine is None:+ self._search_engine: AbstractMarkdownSearch = BingMarkdownSearch()+ else:+ self._search_engine = search_engine++ if markdown_converter is None:+ self._markdown_converter = MarkdownConverter()+ else:+ self._markdown_converter = markdown_converter++ if requests_session is None:+ self._requests_session = requests.Session()+ else:+ self._requests_session = requests_session++ if requests_get_kwargs is None:+ self._requests_get_kwargs = {}+ else:+ self._requests_get_kwargs = requests_get_kwargs++ self._find_on_page_query: Union[str, None] = None+ self._find_on_page_last_result: Union[int, None] = None # Location of the last result++ @property+ def address(self) -> str:+ """Return the address of the current page."""+ return self.history[-1][0]++ def set_address(self, uri_or_path: str) -> None:+ """Sets the address of the current page.+ This will result in the page being fetched via the underlying requests session.++ Arguments:+ uri_or_path: The fully-qualified URI to fetch, or the path to fetch from the current location. If the URI protocol is `search:`, the remainder of the URI is interpreted as a search query, and a web search is performed. If the URI protocol is `file://`, the remainder of the URI is interpreted as a local absolute file path.+ """+ # TODO: Handle anchors+ self.history.append((uri_or_path, time.time()))++ # Handle special URIs+ if uri_or_path == "about:blank":+ self._set_page_content("")+ elif uri_or_path.startswith("search:"):+ query = uri_or_path[len("search:") :].strip()+ results = self._search_engine.search(query)+ self.page_title = f"{query} - Search"+ self._set_page_content(results, split_pages=False)+ else:+ if (+ not uri_or_path.startswith("http:")+ and not uri_or_path.startswith("https:")+ and not uri_or_path.startswith("file:")+ ):+ if len(self.history) > 1:+ prior_address = self.history[-2][0]+ uri_or_path = urljoin(prior_address, uri_or_path)+ # Update the address with the fully-qualified path+ self.history[-1] = (uri_or_path, self.history[-1][1])+ self._fetch_page(uri_or_path)++ self.viewport_current_page = 0+ self.find_on_page_query = None+ self.find_on_page_viewport = None++ @property+ def viewport(self) -> str:+ """Return the content of the current viewport."""+ bounds = self.viewport_pages[self.viewport_current_page]+ return self.page_content[bounds[0] : bounds[1]]++ @property+ def page_content(self) -> str:+ """Return the full contents of the current page."""+ return self._page_content++ def _set_page_content(self, content: str, split_pages: bool = True) -> None:+ """Sets the text content of the current page."""+ self._page_content = content++ if split_pages:+ self._split_pages()+ else:+ self.viewport_pages = [(0, len(self._page_content))]++ if self.viewport_current_page >= len(self.viewport_pages):+ self.viewport_current_page = len(self.viewport_pages) - 1++ def page_down(self) -> None:+ """Move the viewport down one page, if possible."""+ self.viewport_current_page = min(self.viewport_current_page + 1, len(self.viewport_pages) - 1)++ def page_up(self) -> None:+ """Move the viewport up one page, if possible."""+ self.viewport_current_page = max(self.viewport_current_page - 1, 0)++ def find_on_page(self, query: str) -> Union[str, None]:+ """Searches for the query from the current viewport forward, looping back to the start if necessary."""++ # Did we get here via a previous find_on_page search with the same query?+ # If so, map to find_next+ if query == self._find_on_page_query and self.viewport_current_page == self._find_on_page_last_result:+ return self.find_next()++ # Ok it's a new search start from the current viewport+ self._find_on_page_query = query+ viewport_match = self._find_next_viewport(query, self.viewport_current_page)+ if viewport_match is None:+ self._find_on_page_last_result = None+ return None+ else:+ self.viewport_current_page = viewport_match+ self._find_on_page_last_result = viewport_match+ return self.viewport++ def find_next(self) -> Union[str, None]:+ """Scroll to the next viewport that matches the query"""++ if self._find_on_page_query is None:+ return None++ starting_viewport = self._find_on_page_last_result+ if starting_viewport is None:+ starting_viewport = 0+ else:+ starting_viewport += 1+ if starting_viewport >= len(self.viewport_pages):+ starting_viewport = 0++ viewport_match = self._find_next_viewport(self._find_on_page_query, starting_viewport)+ if viewport_match is None:+ self._find_on_page_last_result = None+ return None+ else:+ self.viewport_current_page = viewport_match+ self._find_on_page_last_result = viewport_match+ return self.viewport++ def _find_next_viewport(self, query: Optional[str], starting_viewport: int) -> Union[int, None]:+ """Search for matches between the starting viewport looping when reaching the end."""++ if query is None:+ return None++ # Normalize the query, and convert to a regular expression+ nquery = re.sub(r"\*", "__STAR__", query)+ nquery = " " + (" ".join(re.split(r"\W+", nquery))).strip() + " "+ nquery = nquery.replace(" __STAR__ ", "__STAR__ ") # Merge isolated stars with prior word+ nquery = nquery.replace("__STAR__", ".*").lower()++ if nquery.strip() == "":+ return None++ idxs: List[int] = list()+ idxs.extend(range(starting_viewport, len(self.viewport_pages)))+ idxs.extend(range(0, starting_viewport))++ for i in idxs:+ bounds = self.viewport_pages[i]+ content = self.page_content[bounds[0] : bounds[1]]++ # TODO: Remove markdown links and images+ ncontent = " " + (" ".join(re.split(r"\W+", content))).strip().lower() + " "+ if re.search(nquery, ncontent):+ return i++ return None++ def visit_page(self, path_or_uri: str) -> str:+ """Update the address, visit the page, and return the content of the viewport."""+ self.set_address(path_or_uri)+ return self.viewport++ def open_local_file(self, local_path: str) -> str:+ """Convert a local file path to a file:/// URI, update the address, visit the page, and return the contents of the viewport."""+ full_path = os.path.abspath(os.path.expanduser(local_path))+ self.set_address(pathlib.Path(full_path).as_uri())+ return self.viewport++ def _split_pages(self) -> None:+ """Split the page contents into pages that are approximately the viewport size. Small deviations are permitted to ensure words are not broken."""+ # Handle empty pages+ if len(self._page_content) == 0:+ self.viewport_pages = [(0, 0)]+ return++ # Break the viewport into pages+ self.viewport_pages = []+ start_idx = 0+ while start_idx < len(self._page_content):+ end_idx = min(start_idx + self.viewport_size, len(self._page_content)) # type: ignore[operator]+ # Adjust to end on a space+ while end_idx < len(self._page_content) and self._page_content[end_idx - 1] not in [" ", "\t", "\r", "\n"]:+ end_idx += 1+ self.viewport_pages.append((start_idx, end_idx))+ start_idx = end_idx++ def _fetch_page(+ self,+ url: str,+ session: Optional[requests.Session] = None,+ requests_get_kwargs: Union[Dict[str, Any], None] = None,+ ) -> None:+ """Fetch a page using the requests library. Then convert it to Markdown, and set `page_content` (which splits the content into pages as necessary.++ Arguments:+ url: The fully-qualified URL to fetch.+ session: Used to override the session used for this request. If None, use `self._requests_session` as usual.+ requests_get_kwargs: Extra arguments passes to `requests.Session.get`.+ """+ download_path: str = ""+ response: Union[requests.Response, None] = None+ try:+ if url.startswith("file://"):+ download_path = os.path.normcase(os.path.normpath(unquote(url[7:])))+ if os.path.isdir(download_path): # TODO: Fix markdown_converter types+ res = self._markdown_converter.convert_stream( # type: ignore+ io.StringIO(self._fetch_local_dir(download_path)), file_extension=".html"+ )+ self.page_title = res.title+ self._set_page_content(+ res.text_content, split_pages=False+ ) # Like search results, don't split directory listings+ else:+ res = self._markdown_converter.convert_local(download_path)+ self.page_title = res.title+ self._set_page_content(res.text_content)+ else:+ # Send a HTTP request to the URL+ if session is None:+ session = self._requests_session++ _get_kwargs: Dict[str, Any] = {} # TODO: Deal with kwargs+ _get_kwargs.update(self._requests_get_kwargs)+ if requests_get_kwargs is not None:+ _get_kwargs.update(requests_get_kwargs)+ _get_kwargs["stream"] = True++ response = session.get(url, **_get_kwargs)+ response.raise_for_status()++ # If the HTTP request was successful+ content_type = response.headers.get("content-type", "")++ # Text or HTML+ if "text/" in content_type.lower():+ res = self._markdown_converter.convert_response(response)+ self.page_title = res.title+ self._set_page_content(res.text_content)+ # A download+ else:+ # Was a downloads folder configured?+ if self.downloads_folder is None:+ self.page_title = "Error 400"+ self._set_page_content("## Error 400\n\nClient does not support downloads")+ return++ assert self.downloads_folder is not None++ # Try producing a safe filename+ fname: str = ""+ try:+ fname = pathvalidate.sanitize_filename(os.path.basename(urlparse(url).path)).strip()+ download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))++ suffix = 0+ while os.path.exists(download_path) and suffix < 1000:+ suffix += 1+ base, ext = os.path.splitext(fname)+ new_fname = f"{base}__{suffix}{ext}"+ download_path = os.path.abspath(os.path.join(self.downloads_folder, new_fname))++ except NameError:+ pass++ # No suitable name, so make one+ if fname == "":+ extension = mimetypes.guess_extension(content_type)+ if extension is None:+ extension = ".download"+ fname = str(uuid.uuid4()) + extension+ download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))++ # Open a file for writing+ with open(download_path, "wb") as fh:+ for chunk in response.iter_content(chunk_size=512):+ fh.write(chunk)++ # Render it+ local_uri = pathlib.Path(download_path).as_uri()+ self.set_address(local_uri)++ except UnsupportedFormatException:+ self.page_title = "Download complete."+ self._set_page_content(f"# Download complete\n\nSaved file to '{download_path}'")+ except FileConversionException:+ self.page_title = "Download complete."+ self._set_page_content(f"# Download complete\n\nSaved file to '{download_path}'")+ except FileNotFoundError:+ self.page_title = "Error 404"+ self._set_page_content(f"## Error 404\n\nFile not found: {download_path}")+ except requests.exceptions.RequestException:+ if response is None:+ self.page_title = "Request Exception"+ self._set_page_content("## Unhandled Request Exception:\n\n" + traceback.format_exc())+ else:+ self.page_title = f"Error {response.status_code}"++ # If the error was rendered in HTML we might as well render it+ content_type = response.headers.get("content-type", "")+ if "text/html" in content_type.lower():+ res = self._markdown_converter.convert(response)+ self.page_title = f"Error {response.status_code}"+ self._set_page_content(f"## Error {response.status_code}\n\n{res.text_content}")+ else:+ text = ""+ for chunk in response.iter_content(chunk_size=512, decode_unicode=True):+ text += chunk+ self.page_title = f"Error {response.status_code}"+ self._set_page_content(f"## Error {response.status_code}\n\n{text}")++ def _fetch_local_dir(self, local_path: str) -> str:+ """Render a local directory listing in HTML to assist with local file browsing via the "file://" protocol.+ Through rendered in HTML, later parts of the pipeline will convert the listing to Markdown.++ Arguments:+ local_path: A path to the local directory whose contents are to be listed.++ Returns:+ A directory listing, rendered in HTML.+ """+ pardir = os.path.normpath(os.path.join(local_path, os.pardir))+ pardir_uri = pathlib.Path(pardir).as_uri()+ listing = f"""+<!DOCTYPE html>+<html>+ <head>+ <title>Index of {html.escape(local_path)}</title>+ </head>+ <body>+ <h1>Index of {html.escape(local_path)}</h1>++ <a href="{html.escape(pardir_uri, quote=True)}">.. (parent directory)</a>++ <table>+ <tr>+ <th>Name</th><th>Size</th><th>Date modified</th>+ </tr>+"""++ for entry in os.listdir(local_path):+ full_path = os.path.normpath(os.path.join(local_path, entry))+ full_path_uri = pathlib.Path(full_path).as_uri()+ size = ""+ mtime = datetime.datetime.fromtimestamp(os.path.getmtime(full_path)).strftime("%Y-%m-%d %H:%M")++ if os.path.isdir(full_path):+ entry = entry + os.path.sep+ else:+ size = str(os.path.getsize(full_path))++ listing += (+ "<tr>\n"+ + f'<td><a href="{html.escape(full_path_uri, quote=True)}">{html.escape(entry)}</a></td>'+ + f"<td>{html.escape(size)}</td>"+ + f"<td>{html.escape(mtime)}</td>"+ + "</tr>"+ )++ listing += """+ </table>+ </body>+</html>+"""+ return listingdiff --git a/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py b/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.pyindex f80ea35aa..be0104af2 100644--- a/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py+++ b/python/teams/team-one/src/team_one/agents/multimodal_web_surfer/multimodal_web_surfer.py@@ -1,10 +1,10 @@import base64import hashlibimport io-import reimport jsonimport osimport pathlib+import reimport tracebackfrom typing import Any, BinaryIO, Dict, List, Tuple, Union, cast # Any, Callable, Dict, List, Literal, Tuplefrom urllib.parse import quote_plus # parse_qs, quote, unquote, urlparse, urlunparse