Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Cleanup
Tomas Valenta5600992a4787
diff --git a/api-service/codegen/agent/base.py b/api-service/codegen/agent/base.pyindex 7c551021f..1195fde0e 100644--- a/api-service/codegen/agent/base.py+++ b/api-service/codegen/agent/base.py@@ -1,86 +1,18 @@-import xml.etree.ElementTree as ET-import reimport traceback-from typing_extensions import NotRequired-from typing import List, TypedDict-from typing import Literal, Optional, Tuple, List, Dict, Union, cast-from lxml import etree++from typing import Optional, Tuple, List, Dict, Union, castfrom langchain.agents.tools import InvalidToolfrom langchain.agents import AgentExecutorfrom langchain.agents.chat.base import ChatAgentfrom langchain.schema import AgentAction, AgentFinishfrom langchain.tools.base import BaseTool+from codegen.agent.parsing import parse_text, ToolLogFINAL_ANSWER_ACTION = "Final Answer:"ACTIONS_QUEUE = "action_queue"MALFORMED_ANSWER = "malformed_answer"-escape_table = str.maketrans({- "<": "<",- ">": ">",- "&": "&",- "'": "'",- '"': """,-})--def xml_escape(txt: str):- return txt.translate(escape_table)--class Log(TypedDict):- id: NotRequired[str]- type: Literal["thought"] | Literal["tool"]- created_at: NotRequired[str]--class ThoughtLog(Log):- content: str--class ToolLog(Log):- tool_name: str- tool_input: str- tool_output: NotRequired[str]- finish_at: NotRequired[str]--def merge_dict(t: TypedDict, o: TypedDict):- t.update(o)--action_tag_open = "<action(\\s+tool=\".+?\")?\\s*\\/?>?"-action_tag_close = "</\\s*action\\s*>?"--action_tag_check_pattern = re.compile(f"{action_tag_open}|{action_tag_close}")-action_tag_split_pattern = re.compile(f"({action_tag_open}.*?{action_tag_close})|(.+)")--def parse_text(text: str):- escaped = "".join(- # If the text part is not action tag escape it.- part if action_tag_check_pattern.match(part) else xml_escape(part)- for part- # Split the text by action opening and closing tags.- in re.split(action_tag_split_pattern, text)- if part- )-- root = ET.fromstring(- # Wrapping the XML in a fully formed root tag while some inner tags can be incomplete doesn't make sense.- f"<root>{escaped}",- # Use Parser with C binding for libxml2+libxslt, with recovery mode that allows to parse incomplete XML.- etree.XMLParser(recover=True),- )-- leading_thought = root.text.strip() if root.text else None- if leading_thought:- yield ThoughtLog(type="thought", content=leading_thought)-- for action in root.findall("action"):- tool_name = action.attrib.get("tool", "")- if tool_name:- yield ToolLog(type="tool", tool_name=tool_name, tool_input=action.text or "")-- # Create thoughts from text between and after actions.- trailing_thought = action.tail.strip() if action.tail else None- if trailing_thought:- yield ThoughtLog(type="thought", content=trailing_thought)-class CodegenAgent(ChatAgent):def _extract_tool_and_input(self, text: str) -> Optional[Tuple[str, str | None]]:if FINAL_ANSWER_ACTION in text:@@ -101,37 +33,6 @@ class CodegenAgent(ChatAgent):class CodegenAgentExecutor(AgentExecutor):- def _run_tool(- self,- tool_name: str,- tool_input: str,- color_mapping: Dict[str, str],- name_to_tool_map: Dict[str, BaseTool],- ) -> str:- # Otherwise we lookup the tool- if tool_name in name_to_tool_map:- tool = name_to_tool_map[tool_name]- return_direct = tool.return_direct- color = color_mapping[tool_name]- llm_prefix = "" if return_direct else self.agent.llm_prefix- # We then call the tool on the tool input to get an observation- observation = tool.run(- tool_input,- verbose=self.verbose,- color=color,- llm_prefix=llm_prefix,- observation_prefix=self.agent.observation_prefix,- )- else:- observation = InvalidTool().run(- tool_name,- verbose=self.verbose,- color=None,- llm_prefix="",- observation_prefix=self.agent.observation_prefix,- )- return observation-async def _arun_tool(self,tool_name: str,@@ -154,7 +55,7 @@ class CodegenAgentExecutor(AgentExecutor):observation_prefix=self.agent.observation_prefix,)else:- observation = await InvalidTool().arun(+ observation = await InvalidTool().arun( # type: ignoretool_name,verbose=self.verbose,color=None,diff --git a/api-service/codegen/agent/parsing.py b/api-service/codegen/agent/parsing.pynew file mode 100644index 000000000..afe639bed--- /dev/null+++ b/api-service/codegen/agent/parsing.py@@ -0,0 +1,74 @@+import xml.etree.ElementTree as ET+import re+from typing_extensions import NotRequired+from typing import TypedDict+from typing import Literal+from lxml import etree+++escape_table = str.maketrans({+ "<": "<",+ ">": ">",+ "&": "&",+ "'": "'",+ '"': """,+})++def xml_escape(txt: str):+ return txt.translate(escape_table)+++class Log(TypedDict):+ id: NotRequired[str]+ type: Literal["thought"] | Literal["tool"]+ created_at: NotRequired[str]++class ThoughtLog(Log):+ content: str++class ToolLog(Log):+ tool_name: str+ tool_input: str+ tool_output: NotRequired[str]+ finish_at: NotRequired[str]++def merge_logs(t: TypedDict, o: TypedDict):+ t.update(o)+++action_tag_open = "<action(\\s+tool=\".+?\")?\\s*\\/?>?"+action_tag_close = "</\\s*action\\s*>?"++action_tag_check_pattern = re.compile(f"{action_tag_open}|{action_tag_close}")+action_tag_split_pattern = re.compile(f"({action_tag_open}.*?{action_tag_close})|(.+)")++def parse_text(text: str):+ escaped = "".join(+ # If the text part is not action tag escape it.+ part if action_tag_check_pattern.match(part) else xml_escape(part)+ for part+ # Split the text by action opening and closing tags.+ in re.split(action_tag_split_pattern, text)+ if part+ )++ root = ET.fromstring(+ # Wrapping the XML in a fully formed root tag while some inner tags can be incomplete doesn't make sense.+ f"<root>{escaped}",+ # Use Parser with C binding for libxml2+libxslt, with recovery mode that allows to parse incomplete XML.+ etree.XMLParser(recover=True),+ )++ leading_thought = root.text.strip() if root.text else None+ if leading_thought:+ yield ThoughtLog(type="thought", content=leading_thought)++ for action in root.findall("action"):+ tool_name = action.attrib.get("tool", "")+ if tool_name:+ yield ToolLog(type="tool", tool_name=tool_name, tool_input=action.text or "")++ # Create thoughts from text between and after actions.+ trailing_thought = action.tail.strip() if action.tail else None+ if trailing_thought:+ yield ThoughtLog(type="thought", content=trailing_thought)diff --git a/api-service/codegen/callbacks/log_parser.py b/api-service/codegen/callbacks/log_parser.pyindex d3c81a4de..87b746983 100644--- a/api-service/codegen/callbacks/log_parser.py+++ b/api-service/codegen/callbacks/log_parser.py@@ -3,7 +3,7 @@ import uuidfrom typing import List, TypedDict-from codegen.agent.base import ThoughtLog, ToolLog, merge_dict, parse_text+from codegen.agent.parsing import ThoughtLog, ToolLog, merge_logs, parse_textclass ToolOutput(TypedDict):finish_at: str@@ -93,14 +93,14 @@ class LogStreamParser:# Assign the stable ids and timestamps to logsfor log, meta in zip(self._logs_buffer, self._logs_meta_buffer):- merge_dict(log, meta)+ merge_logs(log, meta)# Update tools' logs with the information from ingested tools' outputs.for tool_log, output in zip((tool_log for tool_log in self._logs_buffer if tool_log["type"] == "tool"),self._tools_output_buffer):- merge_dict(tool_log, output)+ merge_logs(tool_log, output)def ingest_token(self, token: str):"""Ingest token and update the logs."""diff --git a/api-service/codegen/callbacks/logs.py b/api-service/codegen/callbacks/logs.pyindex 24aff336d..d4ad3b718 100644--- a/api-service/codegen/callbacks/logs.py+++ b/api-service/codegen/callbacks/logs.py@@ -1,10 +1,10 @@from typing import Dict, Any, List, Union-from codegen.agent.base import ThoughtLog, ToolLog-from codegen.callbacks.log_queue import LogQueuefrom langchain.callbacks.base import AsyncCallbackHandlerfrom langchain.schema import AgentAction, AgentFinish, LLMResultfrom pydantic import PrivateAttr+from codegen.callbacks.log_queue import LogQueue+from codegen.agent.parsing import ToolLog, ThoughtLogfrom codegen.callbacks.log_parser import LogStreamParserfrom database import Database