Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Fix parsing of multiple actions from LLM in a single response

Vasek Mlejnskyd440016e1ab0
diff --git a/api-service/.gitignore b/api-service/.gitignore
index 727c50088..671894279 100644
--- a/api-service/.gitignore
+++ b/api-service/.gitignore
@@ -1,4 +1,3 @@
__pycache__
dist
.venv
-out.txt
diff --git a/api-service/codegen/agent/base.py b/api-service/codegen/agent/base.py
index 4f374f330..d3a0f358c 100644
--- a/api-service/codegen/agent/base.py
+++ b/api-service/codegen/agent/base.py
@@ -27,7 +27,46 @@ def xml_escape(text: str) -> str:
return text
-# def xml_unescape(xml: str) -> str:
+def parse_action_string(action_string: str):
+ # The action is in the XML format, we need to try to a XML escape
+ # the body of an XML element. We'll use a bit of heuristics here.
+ # We assume the action looks like this:
+ # <action tool="...">
+ # action_text_content
+ # </action>
+ # In this case, we escape all but the first and the last line.
+ # However!
+ # What happens sometimes is that the model outputs multiple actions
+ # in sequence like this:
+ # <action tool="...">
+ # action_text_content
+ # </action>
+ # <action tool="...">
+ # action_text_content
+ # </action>
+ # So we should really escape everyline that DOES NOT start with either
+ # <action or with </action.
+ # Split the action into a list of lines.
+
+ lines = action_string.strip().splitlines()
+ # Escape all but the lines the start or end the XML element.
+ for idx, line in enumerate(lines):
+ if line.startswith("<action") or line.startswith("</action"):
+ continue
+ lines[idx] = xml_escape(line)
+
+ # Join the lines back into a single string.
+ escaped = "\n".join(lines)
+
+ # Parse the escaped string as a XML tree.
+ root = ET.fromstring(f"<root>{escaped}</root>")
+ actions = root.findall("action")
+
+ # Because we XML-escaped action element's body (= tool input) so we could XML parse it,
+ # we need to unescape it now to get the actual tool input.
+ for a in actions:
+ a.text = unescape(a.text)
+ return actions
class CodegenAgent(ChatAgent):
@@ -63,49 +102,15 @@ class CodegenAgent(ChatAgent):
if FINAL_ANSWER_ACTION in text:
return "Final Answer", text.split(FINAL_ANSWER_ACTION)[-1].strip()
try:
- _, action, _ = text.split("```")
-
- # The action is in the XML format, we need to try to a XML escape
- # the body of an XML element. We'll use a bit of heuristics here.
- # We assume the action looks like this:
- # <action tool="...">
- # action_text_content
- # </action>
- # In this case, we escape all but the first and the last line.
- # However!
- # What happens sometimes is that the model outputs multiple actions
- # in sequence like this:
- # <action tool="...">
- # action_text_content
- # </action>
- # <action tool="...">
- # action_text_content
- # </action>
- # So we should really escape everyline that DOES NOT start with either
- # <action or with </action.
-
- # Split the action into a list of lines.
- lines = action.strip().splitlines()
- # Escape all but the lines the start or end the XML element.
- for idx, line in enumerate(lines):
- if line.startswith("<action") or line.startswith("</action"):
- continue
- lines[idx] = xml_escape(line)
-
- # Join the lines back into a single string.
- escaped = "\n".join(lines)
-
- root = ET.fromstring(f"<root>{escaped}</root>")
- actions = root.findall("action")
-
- # Because we XML-escaped action element's body (= tool input) so we could XML parse it,
- # we need to unescape it now to get the actual tool input.
- for a in actions:
- a.text = unescape(a.text)
+ _, action_string, _ = text.split("```")
+
+ # A list of XML elements that represents actions.
+ actions = parse_action_string(action_string)
# We handle the case when the LLM starts specifying multiple actions in a single response.
+ # Return a special `ACTIONS_QUEUE` identifier and the raw action string.
if len(actions) > 1:
- return ACTIONS_QUEUE, actions.strip()
+ return ACTIONS_QUEUE, action_string
else:
return actions[0].attrib["tool"], actions[0].text
except Exception as e:
@@ -197,11 +202,12 @@ class CodegenAgentExecutor(AgentExecutor):
# We handle this case by running each action one by one.
if output.tool == ACTIONS_QUEUE:
observation = ""
- # The `output.tool_input` is a XML tree with multiple actions
+
+ # The `output.tool_input` is a raw action string
+ actions = parse_action_string(output.tool_input)
+
# Go through each action and run it.
# Collect outputs from each action.
- root = ET.fromstring(f"<root>{output.tool_input.strip()}</root>")
- actions = root.findall("action")
for action in actions:
tool_name = action.attrib["tool"]
tool_input = action.text
diff --git a/api-service/codegen/callbacks/logs.py b/api-service/codegen/callbacks/logs.py
index 6a76a3e5f..02759854e 100644
--- a/api-service/codegen/callbacks/logs.py
+++ b/api-service/codegen/callbacks/logs.py
@@ -68,6 +68,8 @@ class LogsCallbackHandler(BaseCallbackHandler):
) -> None:
"""Run when tool starts running."""
print("Starting tool")
+ self._raw_logs += "Starting tool..."
+ self._push_raw_logs()
def on_tool_end(self, output: str, **kwargs: Any) -> None:
"""Run when tool ends running."""
@@ -90,6 +92,8 @@ class LogsCallbackHandler(BaseCallbackHandler):
) -> None:
"""Run when tool errors."""
print("Tool error", error)
+ self._raw_logs += f"Tool error:\n{error}\n"
+ self._push_raw_logs()
def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
"""Run on agent action."""
diff --git a/api-service/out.txt b/api-service/out.txt
new file mode 100644
index 000000000..e69de29bb