Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Add log streaming
Tomas Valenta959266098fe5
diff --git a/api-service/codegen/agent/base.py b/api-service/codegen/agent/base.pyindex 3daaf8a23..f7dfc4623 100644--- a/api-service/codegen/agent/base.py+++ b/api-service/codegen/agent/base.py@@ -3,8 +3,8 @@ from html import unescapeimport xml.etree.ElementTree as ETimport reimport traceback-import sys-+from lxml import etree+import cchardetfrom langchain.agents.tools import InvalidToolfrom langchain.agents import AgentExecutor@@ -31,10 +31,15 @@ def xml_escape(text: str) -> str:return text-def actions_from_xml_string(xml_string: str) -> str:+def actions_from_xml_string(xml_string: str):"""The `xml_string` must be escaped"""+# Parse the escaped string as an XML tree.- root = ET.fromstring(f"<root>{xml_string}</root>")+ root = ET.fromstring(+ f"<root>{xml_string}</root>",+ # Use Parser with C binding for libxml2+libxslt, with recovery mode that allows to parse incomplete XML.+ etree.XMLParser(recover=True),+ )actions = root.findall("action")# Because we XML-escaped action element's body (= tool input) so we could XML-parse it,@@ -66,9 +71,13 @@ def parse_action_string(action_string: str):# We will use regex to match content after the `<action tool="name">` part.for idx, part in enumerate(parts):if part:- action_name, action_content = re.search(+ match = re.search(r"<action tool=\"(.*)\">(.*)$", part- ).groups()+ )+ if not match:+ return []++ action_name, action_content = match.groups()escaped_content = (xml_escape(action_content) if action_content else action_content)diff --git a/api-service/codegen/callbacks/logs.py b/api-service/codegen/callbacks/logs.pyindex 2e7be716a..5b0a88031 100644--- a/api-service/codegen/callbacks/logs.py+++ b/api-service/codegen/callbacks/logs.py@@ -1,7 +1,8 @@from typing import Dict, Any, List, Union, Optionalimport datetimeimport uuid-from pprint import pprint+from codegen.agent.base import parse_action_string+from pydantic import PrivateAttrfrom langchain.callbacks.base import BaseCallbackHandler@@ -11,12 +12,14 @@ from database import Databaseclass LogsCallbackHandler(BaseCallbackHandler):- _logs: List[Dict[str, str]] = []- _raw_logs: str = ""- _current_action_id: Optional[str] = None_database: Database = PrivateAttr()_run_id: str = PrivateAttr()- _file = open("out.txt", "w+")++ _raw_logs: str = ""+ _token_buffer: str = ""+ _logs: List[Dict[str, str]] = []+ _active_logs: List[Dict[str, str]] = []+ _active_action_outputs: List[Dict[str, str]] = []def __init__(self, database: Database, run_id: str, **kwargs: Any):super().__init__(**kwargs)@@ -25,7 +28,45 @@ class LogsCallbackHandler(BaseCallbackHandler):def _push_raw_logs(self) -> None:if self._raw_logs:- self._database.push_raw_logs(self._run_id, self._raw_logs)+ pass+ # self._database.push_raw_logs(self._run_id, self._raw_logs)++ def _parse_token_buffer(self, token: str):+ self._token_buffer += token+ thought, *action_string = self._token_buffer.split("```")++ actions = (+ parse_action_string(action_string[0]) if len(action_string) > 0 else []+ )++ action_logs = [+ {"type": "tool", "name": action.attrib["tool"], "input": action.text or ""}+ for action in actions+ ]++ for i in range(len(self._active_action_outputs)):+ action_logs[i] = {+ **action_logs[i],+ **self._active_action_outputs[i],+ }++ self._active_logs = [+ {+ "type": "thought",+ "content": thought.removeprefix("Thought:")+ .replace("Action:", "")+ .strip(),+ },+ *action_logs,+ ]++ self._database.push_logs(+ run_id=self._run_id,+ logs=[+ *self._logs,+ *self._active_logs,+ ],+ )def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any@@ -35,6 +76,8 @@ class LogsCallbackHandler(BaseCallbackHandler):def on_llm_new_token(self, token: str, **kwargs: Any) -> None:"""Run on new LLM token. Only available when streaming is enabled."""+ self._parse_token_buffer(token)+self._raw_logs += tokenself._push_raw_logs()@@ -79,22 +122,23 @@ class LogsCallbackHandler(BaseCallbackHandler):self._raw_logs += f"\n{output}\n"self._push_raw_logs()- # Update the correct action in the list with the new output- action = next(a for a in self._logs if a["id"] == self._current_action_id)- action["finish_at"] = str(datetime.datetime.now())- action["output"] = output- self._database.push_logs(- run_id=self._run_id,- logs=self._logs,+ self._active_action_outputs.append(+ {+ "finish_at": str(datetime.datetime.now()),+ "output": output,+ })- def on_tool_error(- self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any- ) -> None:- """Run when tool errors."""- print("Tool error", error)- self._raw_logs += f"Tool error:\n{error}\n"- self._push_raw_logs()+ active_tools_logs = [log for log in self._active_logs if log["type"] == "tool"]++ if len(self._active_action_outputs) == len(active_tools_logs):+ self._parse_token_buffer("")++ self._logs.extend(self._active_logs)++ self._token_buffer = ""+ self._active_logs = []+ self._active_action_outputs = []def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:"""Run on agent action."""@@ -109,37 +153,46 @@ class LogsCallbackHandler(BaseCallbackHandler):# $INPUT# </action># ```- thought, _ = action.log.split("Action:")- thought = thought.removeprefix("Thought:")- thought = thought.strip()- self._logs.append(- {- "id": str(uuid.uuid4()),- "type": "thought",- "content": thought,- "created_at": str(datetime.datetime.now()),- }- )-- # We save the UUID for this action so we can later- # update this action with its output.- self._current_action_id = str(uuid.uuid4())- self._logs.append(- {- "id": self._current_action_id,- "type": "tool",- "name": action.tool,- "input": action.tool_input,- "start_at": str(datetime.datetime.now()),- "finish_at": "",- "output": "",- }- )+ pass+ # thought, _ = action.log.split("Action:")+ # thought = thought.removeprefix("Thought:")+ # thought = thought.strip()+ # self._logs.append(+ # {+ # "id": str(uuid.uuid4()),+ # "type": "thought",+ # "content": thought,+ # "created_at": str(datetime.datetime.now()),+ # }+ # )++ # # We save the UUID for this action so we can later+ # # update this action with its output.+ # self._current_action_id = str(uuid.uuid4())+ # self._logs.append(+ # {+ # "id": self._current_action_id,+ # "type": "tool",+ # "name": action.tool,+ # "input": action.tool_input,+ # "start_at": str(datetime.datetime.now()),+ # "finish_at": "",+ # "output": "",+ # }+ # )++ # self._database.push_logs(+ # run_id=self._run_id,+ # logs=self._logs,+ # )- self._database.push_logs(- run_id=self._run_id,- logs=self._logs,- )+ def on_tool_error(+ self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any+ ) -> None:+ """Run when tool errors."""+ print("Tool error", error)+ self._raw_logs += f"Tool error:\n{error}\n"+ self._push_raw_logs()def on_text(self, text: str, **kwargs: Any) -> None:"""Run on arbitrary text."""diff --git a/api-service/poetry.lock b/api-service/poetry.lockindex eb17317bc..1baaa8171 100644--- a/api-service/poetry.lock+++ b/api-service/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.+# This file is automatically @generated by Poetry and should not be changed by hand.[[package]]name = "aenum"@@ -204,6 +204,25 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-tests = ["attrs[tests-no-zope]", "zope.interface"]tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"]+[[package]]+name = "beautifulsoup4"+version = "4.12.0"+description = "Screen-scraping library"+category = "main"+optional = false+python-versions = ">=3.6.0"+files = [+ {file = "beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"},+ {file = "beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"},+]++[package.dependencies]+soupsieve = ">1.2"++[package.extras]+html5lib = ["html5lib"]+lxml = ["lxml"]+[[package]]name = "bleach"version = "6.0.0"@@ -223,6 +242,45 @@ webencodings = "*"[package.extras]css = ["tinycss2 (>=1.1.0,<1.2)"]+[[package]]+name = "cchardet"+version = "2.1.7"+description = "cChardet is high speed universal character encoding detector."+category = "main"+optional = false+python-versions = "*"+files = [+ {file = "cchardet-2.1.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c6f70139aaf47ffb94d89db603af849b82efdf756f187cdd3e566e30976c519f"},+ {file = "cchardet-2.1.7-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5a25f9577e9bebe1a085eec2d6fdd72b7a9dd680811bba652ea6090fb2ff472f"},+ {file = "cchardet-2.1.7-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6b6397d8a32b976a333bdae060febd39ad5479817fabf489e5596a588ad05133"},+ {file = "cchardet-2.1.7-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:228d2533987c450f39acf7548f474dd6814c446e9d6bd228e8f1d9a2d210f10b"},+ {file = "cchardet-2.1.7-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:54341e7e1ba9dc0add4c9d23b48d3a94e2733065c13920e85895f944596f6150"},+ {file = "cchardet-2.1.7-cp36-cp36m-win32.whl", hash = "sha256:eee4f5403dc3a37a1ca9ab87db32b48dc7e190ef84601068f45397144427cc5e"},+ {file = "cchardet-2.1.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f86e0566cb61dc4397297696a4a1b30f6391b50bc52b4f073507a48466b6255a"},+ {file = "cchardet-2.1.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:302aa443ae2526755d412c9631136bdcd1374acd08e34f527447f06f3c2ddb98"},+ {file = "cchardet-2.1.7-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:70eeae8aaf61192e9b247cf28969faef00578becd2602526ecd8ae7600d25e0e"},+ {file = "cchardet-2.1.7-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a39526c1c526843965cec589a6f6b7c2ab07e3e56dc09a7f77a2be6a6afa4636"},+ {file = "cchardet-2.1.7-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:b154effa12886e9c18555dfc41a110f601f08d69a71809c8d908be4b1ab7314f"},+ {file = "cchardet-2.1.7-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ec3eb5a9c475208cf52423524dcaf713c394393e18902e861f983c38eeb77f18"},+ {file = "cchardet-2.1.7-cp37-cp37m-win32.whl", hash = "sha256:50ad671e8d6c886496db62c3bd68b8d55060688c655873aa4ce25ca6105409a1"},+ {file = "cchardet-2.1.7-cp37-cp37m-win_amd64.whl", hash = "sha256:54d0b26fd0cd4099f08fb9c167600f3e83619abefeaa68ad823cc8ac1f7bcc0c"},+ {file = "cchardet-2.1.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b59ddc615883835e03c26f81d5fc3671fab2d32035c87f50862de0da7d7db535"},+ {file = "cchardet-2.1.7-cp38-cp38-manylinux1_i686.whl", hash = "sha256:27a9ba87c9f99e0618e1d3081189b1217a7d110e5c5597b0b7b7c3fedd1c340a"},+ {file = "cchardet-2.1.7-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:90086e5645f8a1801350f4cc6cb5d5bf12d3fa943811bb08667744ec1ecc9ccd"},+ {file = "cchardet-2.1.7-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:45456c59ec349b29628a3c6bfb86d818ec3a6fbb7eb72de4ff3bd4713681c0e3"},+ {file = "cchardet-2.1.7-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:f16517f3697569822c6d09671217fdeab61dfebc7acb5068634d6b0728b86c0b"},+ {file = "cchardet-2.1.7-cp38-cp38-win32.whl", hash = "sha256:0b859069bbb9d27c78a2c9eb997e6f4b738db2d7039a03f8792b4058d61d1109"},+ {file = "cchardet-2.1.7-cp38-cp38-win_amd64.whl", hash = "sha256:273699c4e5cd75377776501b72a7b291a988c6eec259c29505094553ee505597"},+ {file = "cchardet-2.1.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:48ba829badef61441e08805cfa474ccd2774be2ff44b34898f5854168c596d4d"},+ {file = "cchardet-2.1.7-cp39-cp39-manylinux1_i686.whl", hash = "sha256:bd7f262f41fd9caf5a5f09207a55861a67af6ad5c66612043ed0f81c58cdf376"},+ {file = "cchardet-2.1.7-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:fdac1e4366d0579fff056d1280b8dc6348be964fda8ebb627c0269e097ab37fa"},+ {file = "cchardet-2.1.7-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:80e6faae75ecb9be04a7b258dc4750d459529debb6b8dee024745b7b5a949a34"},+ {file = "cchardet-2.1.7-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c96aee9ebd1147400e608a3eff97c44f49811f8904e5a43069d55603ac4d8c97"},+ {file = "cchardet-2.1.7-cp39-cp39-win32.whl", hash = "sha256:2309ff8fc652b0fc3c0cff5dbb172530c7abb92fe9ba2417c9c0bcf688463c1c"},+ {file = "cchardet-2.1.7-cp39-cp39-win_amd64.whl", hash = "sha256:24974b3e40fee9e7557bb352be625c39ec6f50bc2053f44a3d1191db70b51675"},+ {file = "cchardet-2.1.7.tar.gz", hash = "sha256:c428b6336545053c2589f6caf24ea32276c6664cb86db817e03a94c60afa0eaf"},+]+[[package]]name = "certifi"version = "2022.12.7"@@ -1046,6 +1104,99 @@ tenacity = ">=8.1.0,<9.0.0"all = ["aleph-alpha-client (>=2.15.0,<3.0.0)", "anthropic (>=0.2.2,<0.3.0)", "beautifulsoup4 (>=4,<5)", "cohere (>=3,<4)", "deeplake (>=3.2.9,<4.0.0)", "elasticsearch (>=8,<9)", "faiss-cpu (>=1,<2)", "google-api-python-client (==2.70.0)", "google-search-results (>=2,<3)", "huggingface_hub (>=0,<1)", "jinja2 (>=3,<4)", "manifest-ml (>=0.0.1,<0.0.2)", "networkx (>=2.6.3,<3.0.0)", "nlpcloud (>=1,<2)", "nltk (>=3,<4)", "nomic (>=1.0.43,<2.0.0)", "openai (>=0,<1)", "opensearch-py (>=2.0.0,<3.0.0)", "pgvector (>=0.1.6,<0.2.0)", "pinecone-client (>=2,<3)", "psycopg2-binary (>=2.9.5,<3.0.0)", "pypdf (>=3.4.0,<4.0.0)", "qdrant-client (>=1.0.4,<2.0.0)", "redis (>=4,<5)", "sentence-transformers (>=2,<3)", "spacy (>=3,<4)", "tensorflow-text (>=2.11.0,<3.0.0)", "tiktoken (>=0,<1)", "torch (>=1,<2)", "transformers (>=4,<5)", "weaviate-client (>=3,<4)", "wikipedia (>=1,<2)", "wolframalpha (==5.0.0)"]llms = ["anthropic (>=0.2.2,<0.3.0)", "cohere (>=3,<4)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "torch (>=1,<2)", "transformers (>=4,<5)"]+[[package]]+name = "lxml"+version = "4.9.2"+description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."+category = "main"+optional = false+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*"+files = [+ {file = "lxml-4.9.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2"},+ {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892"},+ {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a"},+ {file = "lxml-4.9.2-cp27-cp27m-win32.whl", hash = "sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de"},+ {file = "lxml-4.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3"},+ {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50"},+ {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975"},+ {file = "lxml-4.9.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c"},+ {file = "lxml-4.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a"},+ {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4"},+ {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4"},+ {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7"},+ {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184"},+ {file = "lxml-4.9.2-cp310-cp310-win32.whl", hash = "sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda"},+ {file = "lxml-4.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab"},+ {file = "lxml-4.9.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9"},+ {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf"},+ {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380"},+ {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92"},+ {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1"},+ {file = "lxml-4.9.2-cp311-cp311-win32.whl", hash = "sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33"},+ {file = "lxml-4.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd"},+ {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"},+ {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"},+ {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"},+ {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"},+ {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"},+ {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"},+ {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"},+ {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"},+ {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1"},+ {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e"},+ {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"},+ {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"},+ {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"},+ {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"},+ {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"},+ {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"},+ {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"},+ {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b"},+ {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894"},+ {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45"},+ {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e"},+ {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b"},+ {file = "lxml-4.9.2-cp37-cp37m-win32.whl", hash = "sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe"},+ {file = "lxml-4.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9"},+ {file = "lxml-4.9.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8"},+ {file = "lxml-4.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24"},+ {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889"},+ {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f"},+ {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03"},+ {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c"},+ {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f"},+ {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457"},+ {file = "lxml-4.9.2-cp38-cp38-win32.whl", hash = "sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b"},+ {file = "lxml-4.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7"},+ {file = "lxml-4.9.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1"},+ {file = "lxml-4.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140"},+ {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4"},+ {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf"},+ {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947"},+ {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5"},+ {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5"},+ {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2"},+ {file = "lxml-4.9.2-cp39-cp39-win32.whl", hash = "sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1"},+ {file = "lxml-4.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f"},+ {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c"},+ {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a"},+ {file = "lxml-4.9.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419"},+ {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05"},+ {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f"},+ {file = "lxml-4.9.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9"},+ {file = "lxml-4.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5"},+ {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746"},+ {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7"},+ {file = "lxml-4.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409"},+ {file = "lxml-4.9.2.tar.gz", hash = "sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67"},+]++[package.extras]+cssselect = ["cssselect (>=0.7)"]+html5 = ["html5lib"]+htmlsoup = ["BeautifulSoup4"]+source = ["Cython (>=0.29.7)"]+[[package]]name = "markupsafe"version = "2.1.2"@@ -1753,6 +1904,18 @@ files = [{file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},]+[[package]]+name = "soupsieve"+version = "2.4"+description = "A modern CSS selector implementation for Beautiful Soup."+category = "main"+optional = false+python-versions = ">=3.7"+files = [+ {file = "soupsieve-2.4-py3-none-any.whl", hash = "sha256:49e5368c2cda80ee7e84da9dbe3e110b70a4575f196efb74e51b94549d921955"},+ {file = "soupsieve-2.4.tar.gz", hash = "sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a"},+]+[[package]]name = "sqlalchemy"version = "1.4.46"@@ -1805,7 +1968,7 @@ files = [][package.dependencies]-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""}+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"}[package.extras]aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]@@ -2246,4 +2409,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more[metadata]lock-version = "2.0"python-versions = ">=3.8.1,<4.0"-content-hash = "cd7de79f14ab9d21609c65df3e0be437c65747048d134ab6696d56eab8d2edf6"+content-hash = "614cd15bcf0371ad5925b4f597a294007cef31dcf22d58bb8b06a07ddadf829c"diff --git a/api-service/pyproject.toml b/api-service/pyproject.tomlindex 66f772aa2..ae26d3576 100644--- a/api-service/pyproject.toml+++ b/api-service/pyproject.toml@@ -18,6 +18,9 @@ aenum = "^3.1.11"python-dotenv = "^1.0.0"openai = "^0.27.0"realtime = "^1.0.0"+beautifulsoup4 = "^4.12.0"+lxml = "^4.9.2"+cchardet = "^2.1.7"[build-system]requires = ["poetry-core>=1.3.0"]diff --git a/components/Editor/RightSidebar/Logs/LogEntry.tsx b/components/Editor/RightSidebar/Logs/LogEntry.tsxindex e988fc88c..9b67a54aa 100644--- a/components/Editor/RightSidebar/Logs/LogEntry.tsx+++ b/components/Editor/RightSidebar/Logs/LogEntry.tsx@@ -1,5 +1,11 @@import { Log } from 'db/types'-import { Code2, Package, TerminalSquare, Wrench } from 'lucide-react'+import {+ Code2,+ Package,+ TerminalSquare,+ Wrench,+ Loader,+} from 'lucide-react'import { ReactMarkdown } from 'react-markdown/lib/react-markdown'function getToolIcon(toolName: string) {@@ -40,14 +46,23 @@ function LogEntry({ log }: Props) {{log.type === 'tool' &&<><div className="+ flex+ justify-between+ items-center+ ">+ <div className="flexitems-centerspace-x-2">- {getToolIcon(log.name)}- <div className="font-medium">- {log.name}+ {getToolIcon(log.name)}+ <div className="font-medium">+ {log.name}+ </div></div>+ {log.output === undefined &&+ <Loader size="16px" className="text-slate-400 animate-spin" />+ }</div>{log.input.trim() &&<div className="@@ -56,9 +71,11 @@ function LogEntry({ log }: Props) {pt-2">- <ReactMarkdown>- {'```' + log.input + '```'}- </ReactMarkdown>+ <pre>+ <ReactMarkdown>+ {log.input}+ </ReactMarkdown>+ </pre></div>}{log.output?.trim() &&diff --git a/db/types.ts b/db/types.tsindex 19b42970d..f68123de3 100644--- a/db/types.ts+++ b/db/types.ts@@ -2,18 +2,14 @@ export interface ThoughtLog {id: stringtype: 'thought'content: string- created_at: Date}export interface ToolLog {- id: stringtype: 'tool'name: stringinput: string- start_at: Datefinish_at?: Dateoutput?: string}export type Log = ThoughtLog | ToolLog-