Skip to content

Agent Zero AI framework

vision prototype, terminal sessions

frdel8a9de4a2ee33
diff --git a/agent.py b/agent.py
index 415b4a20e..d6bc73973 100644
--- a/agent.py
+++ b/agent.py
@@ -10,7 +10,12 @@ import models
from langchain_core.prompt_values import ChatPromptValue
from python.helpers import extract_tools, rate_limiter, files, errors, history, tokens
from python.helpers.print_style import PrintStyle
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, StringPromptTemplate
+from langchain_core.prompts import (
+ ChatPromptTemplate,
+ MessagesPlaceholder,
+ HumanMessagePromptTemplate,
+ StringPromptTemplate,
+)
from langchain_core.prompts.image import ImagePromptTemplate
from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage
from langchain_core.language_models.chat_models import BaseChatModel
@@ -91,7 +96,7 @@ class AgentContext:
else:
current_agent = self.agent0
- self.task =self.run_task(current_agent.monologue)
+ self.task = self.run_task(current_agent.monologue)
return self.task
def communicate(self, msg: "UserMessage", broadcast_level: int = 1):
@@ -130,9 +135,9 @@ class AgentContext:
async def _process_chain(self, agent: "Agent", msg: "UserMessage|str", user=True):
try:
msg_template = (
- await agent.hist_add_user_message(msg) # type: ignore
+ agent.hist_add_user_message(msg) # type: ignore
if user
- else await agent.hist_add_tool_result(
+ else agent.hist_add_tool_result(
tool_name="call_subordinate", tool_result=msg # type: ignore
)
)
@@ -283,9 +288,6 @@ class Agent:
printer.stream(chunk)
self.log_from_stream(full, log)
- # store as last context window content
- self.set_data(Agent.DATA_NAME_CTX_WINDOW, prompt.format())
-
agent_response = await self.call_chat_model(
prompt, callback=stream_callback
)
@@ -296,10 +298,10 @@ class Agent:
self.loop_data.last_response == agent_response
): # if assistant_response is the same as last message in history, let him know
# Append the assistant's response to the history
- await self.hist_add_ai_response(agent_response)
+ self.hist_add_ai_response(agent_response)
# Append warning message to the history
warning_msg = self.read_prompt("fw.msg_repeat.md")
- await self.hist_add_warning(message=warning_msg)
+ self.hist_add_warning(message=warning_msg)
PrintStyle(font_color="orange", padding=True).print(
warning_msg
)
@@ -307,7 +309,7 @@ class Agent:
else: # otherwise proceed with tool
# Append the assistant's response to the history
- await self.hist_add_ai_response(agent_response)
+ self.hist_add_ai_response(agent_response)
# process tools requested in agent message
tools_result = await self.process_tools(agent_response)
if tools_result: # final response of message loop available
@@ -319,7 +321,7 @@ class Agent:
except RepairableException as e:
# Forward repairable errors to the LLM, maybe it can fix them
error_message = errors.format_error(e)
- await self.hist_add_warning(error_message)
+ self.hist_add_warning(error_message)
PrintStyle(font_color="red", padding=True).print(error_message)
self.context.log.log(type="error", content=error_message)
except Exception as e:
@@ -358,23 +360,31 @@ class Agent:
extras += history.Message(False, content=extra).output()
loop_data.extras_temporary.clear()
- # combine history and extras
- history_combined: list[OutputMessage] = history.group_outputs_abab(loop_data.history_output + extras)
-
- # convert history to LLM format
- history_langchain: list[BaseMessage] = history.output_langchain(history_combined)
-
- PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(
- f"History Langchain: {history_langchain}"
+ # convert history + extras to LLM format
+ history_langchain: list[BaseMessage] = history.output_langchain(
+ loop_data.history_output + extras
)
# build chain from system prompt, message history and model
+ system_text = "\n\n".join(loop_data.system)
prompt = ChatPromptTemplate.from_messages(
[
- SystemMessage(content="\n\n".join(loop_data.system)),
+ SystemMessage(content=system_text),
*history_langchain,
]
)
+
+ # store as last context window content
+ self.set_data(
+ Agent.DATA_NAME_CTX_WINDOW,
+ {
+ "text": prompt.format(),
+ "tokens": self.history.get_tokens()
+ + tokens.approximate_tokens(system_text)
+ + tokens.approximate_tokens(history.output_text(extras)),
+ },
+ )
+
return prompt
def handle_critical_exception(self, exception: Exception):
@@ -441,12 +451,12 @@ class Agent:
def set_data(self, field: str, value):
self.data[field] = value
- def hist_add_message(self, ai: bool, content: history.MessageContent):
- return self.history.add_message(ai=ai, content=content)
-
- async def hist_add_user_message(
- self, message: UserMessage, intervention: bool = False
+ def hist_add_message(
+ self, ai: bool, content: history.MessageContent, tokens: int = 0
):
+ return self.history.add_message(ai=ai, content=content, tokens=tokens)
+
+ def hist_add_user_message(self, message: UserMessage, intervention: bool = False):
self.history.new_topic() # user message starts a new topic in history
# load message template based on intervention
@@ -476,19 +486,18 @@ class Agent:
self.last_user_message = msg
return msg
- async def hist_add_ai_response(self, message: str):
+ def hist_add_ai_response(self, message: str):
self.loop_data.last_response = message
content = self.parse_prompt("fw.ai_response.md", message=message)
return self.hist_add_message(True, content=content)
- async def hist_add_warning(self, message: history.MessageContent):
+ def hist_add_warning(self, message: history.MessageContent):
content = self.parse_prompt("fw.warning.md", message=message)
return self.hist_add_message(False, content=content)
- async def hist_add_tool_result(self, tool_name: str, tool_result: str, attachments: list[str] = []):
- attachments_str = json.dumps(attachments).replace("\n", "")
+ def hist_add_tool_result(self, tool_name: str, tool_result: str):
content = self.parse_prompt(
- "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result, attachments=attachments_str
+ "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result
)
return self.hist_add_message(False, content=content)
@@ -620,9 +629,9 @@ class Agent:
msg = self.intervention
self.intervention = None # reset the intervention message
if progress.strip():
- await self.hist_add_ai_response(progress)
+ self.hist_add_ai_response(progress)
# append the intervention message
- await self.hist_add_user_message(msg, intervention=True)
+ self.hist_add_user_message(msg, intervention=True)
raise InterventionException(msg)
async def wait_if_paused(self):
@@ -649,7 +658,7 @@ class Agent:
return response.message
else:
msg = self.read_prompt("fw.msg_misformat.md")
- await self.hist_add_warning(msg)
+ self.hist_add_warning(msg)
PrintStyle(font_color="red", padding=True).print(msg)
self.context.log.log(
type="error", content=f"{self.agent_name}: Message misformat"
diff --git a/prompts/default/agent.system.instruments.md b/prompts/default/agent.system.instruments.md
index 677907270..ece095724 100644
--- a/prompts/default/agent.system.instruments.md
+++ b/prompts/default/agent.system.instruments.md
@@ -1,4 +1,5 @@
# Instruments
-- following are instruments at disposal:
+- following are instruments at disposal
+- do not overly rely on them they might not be relevant
-{{instruments}}
\ No newline at end of file
+{{instruments}}
diff --git a/prompts/default/agent.system.memories.md b/prompts/default/agent.system.memories.md
index f7ab13fa5..ae32bf361 100644
--- a/prompts/default/agent.system.memories.md
+++ b/prompts/default/agent.system.memories.md
@@ -1,4 +1,5 @@
# Memories on the topic
-- following are your memories about current topic:
+- following are memories about current topic
+- do not overly rely on them they might not be relevant
{{memories}}
\ No newline at end of file
diff --git a/prompts/default/agent.system.solutions.md b/prompts/default/agent.system.solutions.md
index 0d8743d15..926f35b51 100644
--- a/prompts/default/agent.system.solutions.md
+++ b/prompts/default/agent.system.solutions.md
@@ -1,4 +1,5 @@
# Solutions from the past
-- following are your memories about successful solutions of related problems:
+- following are memories about successful solutions of related problems
+- do not overly rely on them they might not be relevant
{{solutions}}
\ No newline at end of file
diff --git a/prompts/default/agent.system.tool.code_exe.md b/prompts/default/agent.system.tool.code_exe.md
index 9be20508f..cd7f2a8e1 100644
--- a/prompts/default/agent.system.tool.code_exe.md
+++ b/prompts/default/agent.system.tool.code_exe.md
@@ -3,10 +3,9 @@
execute terminal commands python nodejs code for computation or software tasks
place code in "code" arg; escape carefully and indent properly
select "runtime" arg: "terminal" "python" "nodejs" "output" "reset"
-for dialogues (Y/N etc.), use "terminal" runtime next step, send answer
+select "session" number, 0 default, others for multitasking
if code runs long, use "output" to wait, "reset" to kill process
use "pip" "npm" "apt-get" in "terminal" to install packages
-important: never use implicit print/output—it doesn't work!
to output, use print() or console.log()
if tool outputs error, adjust code before retrying; knowledge_tool can help
important: check code for placeholders or demo data; replace with real variables; don't reuse snippets
@@ -26,6 +25,7 @@ usage:
"tool_name": "code_execution_tool",
"tool_args": {
"runtime": "python",
+ "session": 0,
"code": "import os\nprint(os.getcwd())",
}
}
@@ -41,6 +41,7 @@ usage:
"tool_name": "code_execution_tool",
"tool_args": {
"runtime": "terminal",
+ "session": 0,
"code": "apt-get install zip",
}
}
@@ -55,6 +56,7 @@ usage:
"tool_name": "code_execution_tool",
"tool_args": {
"runtime": "output",
+ "session": 0,
}
}
~~~
@@ -68,6 +70,7 @@ usage:
"tool_name": "code_execution_tool",
"tool_args": {
"runtime": "reset",
+ "session": 0,
}
}
~~~
\ No newline at end of file
diff --git a/prompts/default/agent.system.tools_vision.md b/prompts/default/agent.system.tools_vision.md
index 0af7e7527..dd04e8b26 100644
--- a/prompts/default/agent.system.tools_vision.md
+++ b/prompts/default/agent.system.tools_vision.md
@@ -1,3 +1,18 @@
## "Multimodal (Vision) Agent Tools" available:
-None yet. In future, this section will contain vision-only tools
+### vision_load:
+load image data to LLM
+use paths arg for attachments
+
+**Example usage**:
+```json
+{
+ "thoughts": [
+ "I need to see the image...",
+ ],
+ "tool_name": "vision_load",
+ "tool_args": {
+ "paths": ["/path/to/image.png"],
+ }
+}
+```
\ No newline at end of file
diff --git a/prompts/default/fw.tool_result.md b/prompts/default/fw.tool_result.md
index b45d456e8..ef41f23bf 100644
--- a/prompts/default/fw.tool_result.md
+++ b/prompts/default/fw.tool_result.md
@@ -1,7 +1,6 @@
-~~~json
+```json
{
"tool_name": {{tool_name}},
- "tool_result": {{tool_result}},
- "attachments": {{attachments}}
+ "tool_result": {{tool_result}}
}
-~~~
+```
diff --git a/python/api/ctx_window_get.py b/python/api/ctx_window_get.py
index a8e20db94..16a4438b7 100644
--- a/python/api/ctx_window_get.py
+++ b/python/api/ctx_window_get.py
@@ -9,6 +9,10 @@ class GetCtxWindow(ApiHandler):
context = self.get_context(ctxid)
agent = context.streaming_agent or context.agent0
window = agent.get_data(agent.DATA_NAME_CTX_WINDOW)
- size = tokens.approximate_tokens(window)
+ if not window or not isinstance(window, dict):
+ return {"content": "", "tokens": 0}
- return {"content": window, "tokens": size}
+ text = window["text"]
+ tokens = window["tokens"]
+
+ return {"content": text, "tokens": tokens}
diff --git a/python/api/history_get.py b/python/api/history_get.py
index 579a32890..9b1359c49 100644
--- a/python/api/history_get.py
+++ b/python/api/history_get.py
@@ -8,8 +8,8 @@ class GetHistory(ApiHandler):
ctxid = input.get("context", [])
context = self.get_context(ctxid)
agent = context.streaming_agent or context.agent0
- history = agent.history.output()
- size = tokens.approximate_tokens(agent.history.output_text())
+ history = agent.history.output_text()
+ size = agent.history.get_tokens()
return {
"history": history,
diff --git a/python/helpers/files.py b/python/helpers/files.py
index 91bdd0566..bc0e97eb9 100644
--- a/python/helpers/files.py
+++ b/python/helpers/files.py
@@ -1,6 +1,7 @@
from fnmatch import fnmatch
import json
import os, re
+import base64
import re
import shutil
@@ -45,6 +46,32 @@ def read_file(_relative_path, _backup_dirs=None, _encoding="utf-8", **kwargs):
return content
+def read_file_bin(_relative_path, _backup_dirs=None):
+ # init backup dirs
+ if _backup_dirs is None:
+ _backup_dirs = []
+
+ # get absolute path
+ absolute_path = find_file_in_dirs(_relative_path, _backup_dirs)
+
+ # read binary content
+ with open(absolute_path, "rb") as f:
+ return f.read()
+
+
+def read_file_base64(_relative_path, _backup_dirs=None):
+ # init backup dirs
+ if _backup_dirs is None:
+ _backup_dirs = []
+
+ # get absolute path
+ absolute_path = find_file_in_dirs(_relative_path, _backup_dirs)
+
+ # read binary content and encode to base64
+ with open(absolute_path, "rb") as f:
+ return base64.b64encode(f.read()).decode('utf-8')
+
+
def replace_placeholders_text(_content: str, **kwargs):
# Replace placeholders with values from kwargs
for key, value in kwargs.items():
@@ -175,6 +202,15 @@ def write_file_bin(relative_path: str, content: bytes):
f.write(content)
+def write_file_base64(relative_path: str, content: str):
+ # decode base64 string to bytes
+ data = base64.b64decode(content)
+ abs_path = get_abs_path(relative_path)
+ os.makedirs(os.path.dirname(abs_path), exist_ok=True)
+ with open(abs_path, "wb") as f:
+ f.write(data)
+
+
def delete_file(relative_path: str):
abs_path = get_abs_path(relative_path)
if os.path.exists(abs_path):
diff --git a/python/helpers/history.py b/python/helpers/history.py
index 3fcb02426..c09ef61a5 100644
--- a/python/helpers/history.py
+++ b/python/helpers/history.py
@@ -1,16 +1,14 @@
from abc import abstractmethod
import asyncio
from collections import OrderedDict
+from collections.abc import Mapping
import json
import math
-import os
-from typing import Coroutine, Literal, TypedDict, cast
+from typing import Coroutine, Literal, TypedDict, cast, Union, Dict, List, Any, override
from python.helpers import messages, tokens, settings, call_llm
from enum import Enum
-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage
-from python.helpers.print_style import PrintStyle
-from langchain_core.prompts import HumanMessagePromptTemplate
-from typing import Any
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AIMessage
+
BULK_MERGE_COUNT = 3
TOPICS_KEEP_COUNT = 3
CURRENT_TOPIC_RATIO = 0.5
@@ -18,14 +16,22 @@ HISTORY_TOPIC_RATIO = 0.3
HISTORY_BULK_RATIO = 0.2
TOPIC_COMPRESS_RATIO = 0.65
LARGE_MESSAGE_TO_TOPIC_RATIO = 0.25
+RAW_MESSAGE_OUTPUT_TEXT_TRIM = 100
+
+
+class RawMessage(TypedDict):
+ raw_content: "MessageContent"
+ preview: str | None
+
-MessageContent = (
- list["MessageContent"]
- | OrderedDict[str, "MessageContent"]
- | list[OrderedDict[str, "MessageContent"]]
- | str
- | list[str]
-)
+MessageContent = Union[
+ List["MessageContent"],
+ Dict[str, "MessageContent"],
+ List[Dict[str, "MessageContent"]],
+ str,
+ List[str],
+ RawMessage,
+]
class OutputMessage(TypedDict):
@@ -37,9 +43,9 @@ class Record:
def __init__(self):
pass
+ @abstractmethod
def get_tokens(self) -> int:
- out = self.output_text()
- return tokens.approximate_tokens(out)
+ pass
@abstractmethod
async def compress(self) -> bool:
@@ -70,10 +76,25 @@ class Record:
class Message(Record):
- def __init__(self, ai: bool, content: MessageContent):
+ def __init__(self, ai: bool, content: MessageContent, tokens: int = 0):
self.ai = ai
self.content = content
- self.summary: MessageContent = ""
+ self.summary: str = ""
+ self.tokens: int = tokens or self.calculate_tokens()
+
+ @override
+ def get_tokens(self) -> int:
+ if not self.tokens:
+ self.tokens = self.calculate_tokens()
+ return self.tokens
+
+ def calculate_tokens(self):
+ text = self.output_text()
+ return tokens.approximate_tokens(text)
+
+ def set_summary(self, summary: str):
+ self.summary = summary
+ self.tokens = self.calculate_tokens()
async def compress(self):
return False
@@ -93,12 +114,15 @@ class Message(Record):
"ai": self.ai,
"content": self.content,
"summary": self.summary,
+ "tokens": self.tokens,
}
@staticmethod
def from_dict(data: dict, history: "History"):
- msg = Message(ai=data["ai"], content=data.get("content", "Content lost"))
+ content = data.get("content", "Content lost")
+ msg = Message(ai=data["ai"], content=content)
msg.summary = data.get("summary", "")
+ msg.tokens = data.get("tokens", 0)
return msg
@@ -108,8 +132,16 @@ class Topic(Record):
self.summary: str = ""
self.messages: list[Message] = []
- def add_message(self, ai: bool, content: MessageContent):
- msg = Message(ai=ai, content=content)
+ def get_tokens(self):
+ if self.summary:
+ return tokens.approximate_tokens(self.summary)
+ else:
+ return sum(msg.get_tokens() for msg in self.messages)
+
+ def add_message(
+ self, ai: bool, content: MessageContent, tokens: int = 0
+ ) -> Message:
+ msg = Message(ai=ai, content=content, tokens=tokens)
self.messages.append(msg)
return msg
@@ -118,7 +150,7 @@ class Topic(Record):
return [OutputMessage(ai=False, content=self.summary)]
else:
msgs = [m for r in self.messages for m in r.output()]
- return group_outputs_abab(msgs)
+ return msgs
async def summarize(self):
self.summary = await self.summarize_messages(self.messages)
@@ -134,22 +166,31 @@ class Topic(Record):
)
large_msgs = []
for m in (m for m in self.messages if not m.summary):
+ # TODO refactor this
out = m.output()
text = output_text(out)
- tok = tokens.approximate_tokens(text)
+ tok = m.get_tokens()
leng = len(text)
if tok > msg_max_size:
large_msgs.append((m, tok, leng, out))
large_msgs.sort(key=lambda x: x[1], reverse=True)
for msg, tok, leng, out in large_msgs:
trim_to_chars = leng * (msg_max_size / tok)
- trunc = messages.truncate_dict_by_ratio(
- self.history.agent,
- out[0]["content"],
- trim_to_chars * 1.15,
- trim_to_chars * 0.85,
- )
- msg.summary = trunc
+ # raw messages will be replaced as a whole, they would become invalid when truncated
+ if _is_raw_message(out[0]["content"]):
+ msg.set_summary(
+ "Message content replaced to save space in context window"
+ )
+
+ # regular messages will be truncated
+ else:
+ trunc = messages.truncate_dict_by_ratio(
+ self.history.agent,
+ out[0]["content"],
+ trim_to_chars * 1.15,
+ trim_to_chars * 0.85,
+ )
+ msg.set_summary(_json_dumps(trunc))
return True
return False
@@ -175,6 +216,7 @@ class Topic(Record):
return False
async def summarize_messages(self, messages: list[Message]):
+ # FIXME: vision bytes are sent to utility LLM, send summary instead
msg_txt = [m.output_text() for m in messages]
summary = await self.history.agent.call_utility_model(
system=self.history.agent.read_prompt("fw.topic_summary.sys.md"),
@@ -194,9 +236,9 @@ class Topic(Record):
@staticmethod
def from_dict(data: dict, history: "History"):
topic = Topic(history=history)
- topic.summary = data["summary"]
+ topic.summary = data.get("summary", "")
topic.messages = [
- Message.from_dict(m, history=history) for m in data["messages"]
+ Message.from_dict(m, history=history) for m in data.get("messages", [])
]
return topic
@@ -214,7 +256,7 @@ class Bulk(Record):
return [OutputMessage(ai=False, content=self.summary)]
else:
msgs = [m for r in self.records for m in r.output()]
- return group_outputs_abab(msgs)
+ return msgs
async def compress(self):
return False
@@ -253,8 +295,15 @@ class History(Record):
self.current = Topic(history=self)
self.agent: Agent = agent
+ def get_tokens(self) -> int:
+ return (
+ self.get_bulks_tokens()
+ + self.get_topics_tokens()
+ + self.get_current_topic_tokens()
+ )
+
def is_over_limit(self):
- limit = get_ctx_size_for_history()
+ limit = _get_ctx_size_for_history()
total = self.get_tokens()
return total > limit
@@ -267,15 +316,10 @@ class History(Record):
def get_current_topic_tokens(self) -> int:
return self.current.get_tokens()
- def get_tokens(self) -> int:
- return (
- self.get_bulks_tokens()
- + self.get_topics_tokens()
- + self.get_current_topic_tokens()
- )
-
- def add_message(self, ai: bool, content: MessageContent):
- return self.current.add_message(ai, content=content)
+ def add_message(
+ self, ai: bool, content: MessageContent, tokens: int = 0
+ ) -> Message:
+ return self.current.add_message(ai, content=content, tokens=tokens)
def new_topic(self):
if self.current.messages:
@@ -287,7 +331,6 @@ class History(Record):
result += [m for b in self.bulks for m in b.output()]
result += [m for t in self.topics for m in t.output()]
result += self.current.output()
- result = group_outputs_abab(result)
return result
@staticmethod
@@ -307,7 +350,7 @@ class History(Record):
def serialize(self):
data = self.to_dict()
- return json.dumps(data)
+ return _json_dumps(data)
async def compress(self):
compressed = False
@@ -317,7 +360,7 @@ class History(Record):
self.get_topics_tokens(),
self.get_bulks_tokens(),
)
- total = get_ctx_size_for_history()
+ total = _get_ctx_size_for_history()
ratios = [
(curr, CURRENT_TOPIC_RATIO, "current_topic"),
(hist, HISTORY_TOPIC_RATIO, "history_topic"),
@@ -392,25 +435,46 @@ class History(Record):
def deserialize_history(json_data: str, agent) -> History:
history = History(agent=agent)
if json_data:
- data = json.loads(json_data)
+ data = _json_loads(json_data)
history = History.from_dict(data, history=history)
return history
-def get_ctx_size_for_history() -> int:
+def _get_ctx_size_for_history() -> int:
set = settings.get_settings()
return int(set["chat_model_ctx_length"] * set["chat_model_ctx_history"])
-def serialize_output(output: OutputMessage, ai_label="ai", human_label="human"):
- return f'{ai_label if output["ai"] else human_label}: {serialize_content(output["content"])}'
+def _stringify_output(output: OutputMessage, ai_label="ai", human_label="human"):
+ return f'{ai_label if output["ai"] else human_label}: {_stringify_content(output["content"])}'
-def serialize_content(content: MessageContent) -> str:
+def _stringify_content(content: MessageContent) -> str:
+ # already a string
if isinstance(content, str):
return content
+
+ # raw messages return preview or trimmed json
+ if _is_raw_message(content):
+ preview: str = content.get("preview", "") # type: ignore
+ if preview:
+ return preview
+ text = _json_dumps(content)
+ if len(text) > RAW_MESSAGE_OUTPUT_TEXT_TRIM:
+ return text[:RAW_MESSAGE_OUTPUT_TEXT_TRIM] + "... TRIMMED"
+ return text
+
+ # regular messages of non-string are dumped as json
+ return _json_dumps(content)
+
+
+def _output_content_langchain(content: MessageContent):
+ if isinstance(content, str):
+ return content
+ if _is_raw_message(content):
+ return content["raw_content"] # type: ignore
try:
- return json.dumps(content)
+ return _json_dumps(content)
except Exception as e:
raise e
@@ -421,98 +485,73 @@ def group_outputs_abab(outputs: list[OutputMessage]) -> list[OutputMessage]:
if result and result[-1]["ai"] == out["ai"]:
result[-1] = OutputMessage(
ai=result[-1]["ai"],
- content=merge_outputs(result[-1]["content"], out["content"]),
+ content=_merge_outputs(result[-1]["content"], out["content"]),
)
else:
result.append(out)
return result
-def output_langchain(messages: list[OutputMessage]) -> list[BaseMessage]:
+def group_messages_abab(messages: list[BaseMessage]) -> list[BaseMessage]:
result = []
- for m in messages:
- if m["ai"]:
- result.append(AIMessage(content=serialize_content(m["content"])))
+ for msg in messages:
+ if result and isinstance(result[-1], type(msg)):
+ # create new instance of the same type with merged content
+ result[-1] = type(result[-1])(content=_merge_outputs(result[-1].content, msg.content)) # type: ignore
else:
- contents = m["content"]
-
- # sometimes content is a list sometimes not
- if not isinstance(contents, list):
- contents = [contents]
+ result.append(msg)
+ return result
- PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(
- f"Contents: {json.dumps(contents, indent=2)}"
- )
- template: list[dict[str, str]] = [] # type: ignore
- message = ""
- images = {}
- for _, content in enumerate(contents):
- if message:
- # the first message is the user message, then the memory and solutions
- message += "\n\n--- Memory & Solutions Section: ---\n\n"
-
- message += serialize_content(content)
-
- if isinstance(content, dict) and "attachments" in content:
- attachments: list[str] = cast(list[str], content["attachments"])
- for attachment in attachments:
- if not os.path.exists(str(attachment)):
- continue
- if attachment not in images:
- import base64
- from mimetypes import guess_type
- mime_type, _ = guess_type(str(attachment))
- if mime_type.startswith("image/"):
- # Read and encode the image file
- with open(str(attachment), "rb") as image_file:
- base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')
- # Construct the data URL
- images[attachment] = f"data:{mime_type};base64,{base64_encoded_data}"
-
- if message:
- template.append({"type": "text", "text": message})
- if images:
- for _, image in images.items():
- template.append({"type": "image_url", "image_url": image})
- if template:
- # only jinja2 is safe for json, both mustache({{...}}) and f-string({...}) are not
- result.append(HumanMessagePromptTemplate.from_template(template=template, partial_variables={}, template_format="jinja2")) # type: ignore
-
- PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(
- f"Result: {result}"
- )
+def output_langchain(messages: list[OutputMessage]):
+ result = []
+ for m in messages:
+ if m["ai"]:
+ # result.append(AIMessage(content=serialize_content(m["content"])))
+ result.append(AIMessage(_output_content_langchain(content=m["content"]))) # type: ignore
+ else:
+ # result.append(HumanMessage(content=serialize_content(m["content"])))
+ result.append(HumanMessage(_output_content_langchain(content=m["content"]))) # type: ignore
+ # ensure message type alternation
+ result = group_messages_abab(result)
return result
def output_text(messages: list[OutputMessage], ai_label="ai", human_label="human"):
- return "\n".join(serialize_output(o, ai_label, human_label) for o in messages)
+ return "\n".join(_stringify_output(o, ai_label, human_label) for o in messages)
+
+def _merge_outputs(a: MessageContent, b: MessageContent) -> MessageContent:
+ if isinstance(a, str) and isinstance(b, str):
+ return a + b
-def merge_outputs(a: MessageContent, b: MessageContent) -> MessageContent:
if not isinstance(a, list):
a = [a]
if not isinstance(b, list):
b = [b]
- return a + b # type: ignore
- # return merge_properties(a, b)
+
+ return cast(MessageContent, a + b)
-def merge_properties(a: MessageContent, b: MessageContent) -> MessageContent:
- if isinstance(a, list):
- if isinstance(b, list):
- return a + b # type: ignore
+def _merge_properties(
+ a: Dict[str, MessageContent], b: Dict[str, MessageContent]
+) -> Dict[str, MessageContent]:
+ result = a.copy()
+ for k, v in b.items():
+ if k in result:
+ result[k] = _merge_outputs(result[k], v)
else:
- return a + [b]
- elif isinstance(b, list):
- return [a] + b # type: ignore
- elif isinstance(a, dict) and isinstance(b, dict):
- for key, value in b.items():
- if key in a:
- a[key] = merge_properties(a[key], value)
- else:
- a[key] = value
- return a
- elif isinstance(a, str) and isinstance(b, str):
- return a + b
- raise ValueError(f"Cannot merge {a} and {b}")
+ result[k] = v
+ return result
+
+
+def _is_raw_message(obj: object) -> bool:
+ return isinstance(obj, Mapping) and "raw_content" in obj
+
+
+def _json_dumps(obj):
+ return json.dumps(obj, ensure_ascii=False)
+
+
+def _json_loads(obj):
+ return json.loads(obj)
diff --git a/python/helpers/images.py b/python/helpers/images.py
new file mode 100644
index 000000000..9377f1e9c
--- /dev/null
+++ b/python/helpers/images.py
@@ -0,0 +1,35 @@
+from PIL import Image
+import io
+import math
+
+
+def compress_image(image_data: bytes, *, max_pixels: int = 256_000, quality: int = 50) -> bytes:
+ """Compress an image by scaling it down and converting to JPEG with quality settings.
+
+ Args:
+ image_data: Raw image bytes
+ max_pixels: Maximum number of pixels in the output image (width * height)
+ quality: JPEG quality setting (1-100)
+
+ Returns:
+ Compressed image as bytes
+ """
+ # load image from bytes
+ img = Image.open(io.BytesIO(image_data))
+
+ # calculate scaling factor to get to max_pixels
+ current_pixels = img.width * img.height
+ if current_pixels > max_pixels:
+ scale = math.sqrt(max_pixels / current_pixels)
+ new_width = int(img.width * scale)
+ new_height = int(img.height * scale)
+ img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
+
+ # convert to RGB if needed (for JPEG)
+ if img.mode in ('RGBA', 'P'):
+ img = img.convert('RGB')
+
+ # save as JPEG with compression
+ output = io.BytesIO()
+ img.save(output, format='JPEG', quality=quality, optimize=True)
+ return output.getvalue()
diff --git a/python/helpers/runtime.py b/python/helpers/runtime.py
index 6709c6fdf..189d45288 100644
--- a/python/helpers/runtime.py
+++ b/python/helpers/runtime.py
@@ -2,6 +2,9 @@ import argparse
import inspect
from typing import TypeVar, Callable, Awaitable, Union, overload, cast
from python.helpers import dotenv, rfc, settings
+import asyncio
+import threading
+import queue
T = TypeVar('T')
R = TypeVar('R')
@@ -102,3 +105,22 @@ def _get_rfc_url() -> str:
url = url+":"+str(set["rfc_port_http"])
url += "/rfc"
return url
+
+
+def call_development_function_sync(func: Union[Callable[..., T], Callable[..., Awaitable[T]]], *args, **kwargs) -> T:
+ # run async function in sync manner
+ result_queue = queue.Queue()
+
+ def run_in_thread():
+ result = asyncio.run(call_development_function(func, *args, **kwargs))
+ result_queue.put(result)
+
+ thread = threading.Thread(target=run_in_thread)
+ thread.start()
+ thread.join(timeout=30) # wait for thread with timeout
+
+ if thread.is_alive():
+ raise TimeoutError("Function call timed out after 30 seconds")
+
+ result = result_queue.get_nowait()
+ return cast(T, result)
diff --git a/python/helpers/tool.py b/python/helpers/tool.py
index fcfb51070..fef3058ac 100644
--- a/python/helpers/tool.py
+++ b/python/helpers/tool.py
@@ -1,5 +1,6 @@
from abc import abstractmethod
-from dataclasses import dataclass, field
+from dataclasses import dataclass
+
from agent import Agent
from python.helpers.print_style import PrintStyle
@@ -8,8 +9,6 @@ from python.helpers.print_style import PrintStyle
class Response:
message:str
break_loop: bool
- attachments: list[str] = field(default_factory=list[str])
-
class Tool:
@@ -34,7 +33,7 @@ class Tool:
async def after_execution(self, response: Response, **kwargs):
text = response.message.strip()
- await self.agent.hist_add_tool_result(self.name, text, response.attachments)
+ self.agent.hist_add_tool_result(self.name, text)
PrintStyle(font_color="#1B4F72", background_color="white", padding=True, bold=True).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")
PrintStyle(font_color="#85C1E9").print(response.message)
self.log.update(content=response.message)
diff --git a/python/helpers/whisper.py b/python/helpers/whisper.py
index f92e4c8b5..0f644487d 100644
--- a/python/helpers/whisper.py
+++ b/python/helpers/whisper.py
@@ -30,7 +30,7 @@ async def _preload(model_name:str):
is_updating_model = True
if not _model or _model_name != model_name:
PrintStyle.standard(f"Loading Whisper model: {model_name}")
- _model = whisper.load_model(model_name)
+ _model = whisper.load_model(name=model_name) # type: ignore
_model_name = model_name
finally:
is_updating_model = False
diff --git a/python/tools/call_subordinate.py b/python/tools/call_subordinate.py
index 73eb9b574..3a50ffdc4 100644
--- a/python/tools/call_subordinate.py
+++ b/python/tools/call_subordinate.py
@@ -18,7 +18,7 @@ class Delegation(Tool):
# add user message to subordinate agent
subordinate: Agent = self.agent.get_data(Agent.DATA_NAME_SUBORDINATE)
- await subordinate.hist_add_user_message(UserMessage(message=message, attachments=[]))
+ subordinate.hist_add_user_message(UserMessage(message=message, attachments=[]))
# run subordinate monologue
result = await subordinate.monologue()
# result
diff --git a/python/tools/code_execution_tool.py b/python/tools/code_execution_tool.py
index 6a0686875..593e734fa 100644
--- a/python/tools/code_execution_tool.py
+++ b/python/tools/code_execution_tool.py
@@ -12,7 +12,7 @@ from python.helpers.docker import DockerContainerManager
@dataclass
class State:
- shell: LocalInteractiveSession | SSHInteractiveSession
+ shells: dict[int, LocalInteractiveSession | SSHInteractiveSession]
docker: DockerContainerManager | None
@@ -27,19 +27,26 @@ class CodeExecution(Tool):
# os.chdir(files.get_abs_path("./work_dir")) #change CWD to work_dir
runtime = self.args.get("runtime", "").lower().strip()
+ session = int(self.args.get("session", 0))
if runtime == "python":
- response = await self.execute_python_code(self.args["code"])
+ response = await self.execute_python_code(
+ code=self.args["code"], session=session
+ )
elif runtime == "nodejs":
- response = await self.execute_nodejs_code(self.args["code"])
+ response = await self.execute_nodejs_code(
+ code=self.args["code"], session=session
+ )
elif runtime == "terminal":
- response = await self.execute_terminal_command(self.args["code"])
+ response = await self.execute_terminal_command(
+ command=self.args["code"], session=session
+ )
elif runtime == "output":
response = await self.get_terminal_output(
- wait_with_output=5, wait_without_output=60
+ session=session, wait_with_output=5, wait_without_output=60
)
elif runtime == "reset":
- response = await self.reset_terminal()
+ response = await self.reset_terminal(session=session)
else:
response = self.agent.read_prompt(
"fw.code_runtime_wrong.md", runtime=runtime
@@ -72,11 +79,15 @@ class CodeExecution(Tool):
# PrintStyle().print()
def get_log_object(self):
- return self.agent.context.log.log(type="code_exe", heading=f"{self.agent.agent_name}: Using tool '{self.name}'", content="", kvps=self.args)
-
+ return self.agent.context.log.log(
+ type="code_exe",
+ heading=f"{self.agent.agent_name}: Using tool '{self.name}'",
+ content="",
+ kvps=self.args,
+ )
async def after_execution(self, response, **kwargs):
- await self.agent.hist_add_tool_result(self.name, response.message)
+ self.agent.hist_add_tool_result(self.name, response.message)
async def prepare_state(self, reset=False):
self.state = self.agent.get_data("_cot_state")
@@ -97,7 +108,11 @@ class CodeExecution(Tool):
# initialize local or remote interactive shell insterface
if self.agent.config.code_exec_ssh_enabled:
- pswd = self.agent.config.code_exec_ssh_pass if self.agent.config.code_exec_ssh_pass else await rfc_exchange.get_root_password()
+ pswd = (
+ self.agent.config.code_exec_ssh_pass
+ if self.agent.config.code_exec_ssh_pass
+ else await rfc_exchange.get_root_password()
+ )
shell = SSHInteractiveSession(
self.agent.context.log,
self.agent.config.code_exec_ssh_addr,
@@ -108,42 +123,63 @@ class CodeExecution(Tool):
else:
shell = LocalInteractiveSession()
- self.state = State(shell=shell, docker=docker)
+ self.state = State(shells={0: shell}, docker=docker)
await shell.connect()
self.agent.set_data("_cot_state", self.state)
- async def execute_python_code(self, code: str, reset: bool = False):
+ async def execute_python_code(self, session: int, code: str, reset: bool = False):
escaped_code = shlex.quote(code)
command = f"ipython -c {escaped_code}"
- return await self.terminal_session(command, reset)
+ return await self.terminal_session(session, command, reset)
- async def execute_nodejs_code(self, code: str, reset: bool = False):
+ async def execute_nodejs_code(self, session: int, code: str, reset: bool = False):
escaped_code = shlex.quote(code)
command = f"node /exe/node_eval.js {escaped_code}"
- return await self.terminal_session(command, reset)
+ return await self.terminal_session(session, command, reset)
- async def execute_terminal_command(self, command: str, reset: bool = False):
- return await self.terminal_session(command, reset)
+ async def execute_terminal_command(
+ self, session: int, command: str, reset: bool = False
+ ):
+ return await self.terminal_session(session, command, reset)
- async def terminal_session(self, command: str, reset: bool = False):
+ async def terminal_session(self, session: int, command: str, reset: bool = False):
await self.agent.handle_intervention() # wait for intervention and handle it, if paused
# try again on lost connection
for i in range(2):
try:
-
+
if reset:
await self.reset_terminal()
- self.state.shell.send_command(command)
-
- PrintStyle(background_color="white", font_color="#1B4F72", bold=True).print(
- f"{self.agent.agent_name} code execution output"
- )
- return await self.get_terminal_output()
+ if session not in self.state.shells:
+ if self.agent.config.code_exec_ssh_enabled:
+ pswd = (
+ self.agent.config.code_exec_ssh_pass
+ if self.agent.config.code_exec_ssh_pass
+ else await rfc_exchange.get_root_password()
+ )
+ shell = SSHInteractiveSession(
+ self.agent.context.log,
+ self.agent.config.code_exec_ssh_addr,
+ self.agent.config.code_exec_ssh_port,
+ self.agent.config.code_exec_ssh_user,
+ pswd,
+ )
+ else:
+ shell = LocalInteractiveSession()
+ self.state.shells[session] = shell
+ await shell.connect()
+
+ self.state.shells[session].send_command(command)
+
+ PrintStyle(
+ background_color="white", font_color="#1B4F72", bold=True
+ ).print(f"{self.agent.agent_name} code execution output")
+ return await self.get_terminal_output(session)
except Exception as e:
- if i==1:
+ if i == 1:
# try again on lost connection
PrintStyle.error(str(e))
await self.prepare_state(reset=True)
@@ -153,6 +189,7 @@ class CodeExecution(Tool):
async def get_terminal_output(
self,
+ session=0,
reset_full_output=True,
wait_with_output=3,
wait_without_output=10,
@@ -165,10 +202,10 @@ class CodeExecution(Tool):
while max_exec_time <= 0 or time.time() - start_time < max_exec_time:
await asyncio.sleep(SLEEP_TIME) # Wait for some output to be generated
- full_output, partial_output = await self.state.shell.read_output(
+ full_output, partial_output = await self.state.shells[session].read_output(
timeout=max_exec_time, reset_full_output=reset_full_output
)
- reset_full_output = False # only reset once
+ reset_full_output = False # only reset once
await self.agent.handle_intervention() # wait for intervention and handle it, if paused
@@ -184,8 +221,10 @@ class CodeExecution(Tool):
break
return full_output
- async def reset_terminal(self):
- self.state.shell.close()
+ async def reset_terminal(self, session=0):
+ if session in self.state.shells:
+ self.state.shells[session].close()
+ del self.state.shells[session]
await self.prepare_state(reset=True)
response = self.agent.read_prompt("fw.code_reset.md")
self.log.update(content=response)
diff --git a/python/tools/input.py b/python/tools/input.py
index 6e5a812f8..b23c9adee 100644
--- a/python/tools/input.py
+++ b/python/tools/input.py
@@ -20,4 +20,4 @@ class Input(Tool):
return self.agent.context.log.log(type="code_exe", heading=f"{self.agent.agent_name}: Using tool '{self.name}'", content="", kvps=self.args)
async def after_execution(self, response, **kwargs):
- await self.agent.hist_add_tool_result(self.name, response.message)
\ No newline at end of file
+ self.agent.hist_add_tool_result(self.name, response.message)
\ No newline at end of file
diff --git a/python/tools/vision_load.py b/python/tools/vision_load.py
new file mode 100644
index 000000000..e1c455fa5
--- /dev/null
+++ b/python/tools/vision_load.py
@@ -0,0 +1,79 @@
+import base64
+from python.helpers.print_style import PrintStyle
+from python.helpers.tool import Tool, Response
+from python.helpers import runtime, files, images
+from mimetypes import guess_type
+from python.helpers import history
+
+# image optimization and token estimation for context window
+MAX_PIXELS = 768_000
+QUALITY = 75
+TOKENS_ESTIMATE = 1500
+
+
+class VisionLoad(Tool):
+ async def execute(self, paths: list[str] = [], **kwargs) -> Response:
+
+ self.images_dict = {}
+ template: list[dict[str, str]] = [] # type: ignore
+
+ for path in paths:
+ if not await runtime.call_development_function(files.exists, str(path)):
+ continue
+
+ if path not in self.images_dict:
+ mime_type, _ = guess_type(str(path))
+ if mime_type and mime_type.startswith("image/"):
+ # Read binary file
+ file_content = await runtime.call_development_function(
+ files.read_file_base64, str(path)
+ )
+ file_content = base64.b64decode(file_content)
+ # Compress and convert to JPEG
+ compressed = images.compress_image(
+ file_content, max_pixels=MAX_PIXELS, quality=QUALITY
+ )
+ # Encode as base64
+ file_content_b64 = base64.b64encode(compressed).decode("utf-8")
+
+ # DEBUG: Save compressed image
+ # await runtime.call_development_function(
+ # files.write_file_base64, str(path), file_content_b64
+ # )
+
+ # Construct the data URL (always JPEG after compression)
+ self.images_dict[path] = file_content_b64
+
+ return Response(message="dummy", break_loop=False)
+
+ async def after_execution(self, response: Response, **kwargs):
+
+ # build image data messages for LLMs, or error message
+ content = []
+ if self.images_dict:
+ for _, image in self.images_dict.items():
+ content.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{image}"},
+ }
+ )
+ # append as raw message content for LLMs with vision tokens estimate
+ msg = history.RawMessage(raw_content=content, preview="<Base64 encoded image data>")
+ self.agent.hist_add_message(
+ False, content=msg, tokens=TOKENS_ESTIMATE * len(content)
+ )
+ else:
+ self.agent.hist_add_tool_result(self.name, "No images processed")
+
+ # print and log short version
+ message = (
+ "No images processed"
+ if not self.images_dict
+ else f"{len(self.images_dict)} images processed"
+ )
+ PrintStyle(
+ font_color="#1B4F72", background_color="white", padding=True, bold=True
+ ).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")
+ PrintStyle(font_color="#85C1E9").print(message)
+ self.log.update(result=message)
diff --git a/webui/js/history.js b/webui/js/history.js
index e62fc615e..040de72fe 100644
--- a/webui/js/history.js
+++ b/webui/js/history.js
@@ -3,9 +3,10 @@ import { getContext } from "../index.js";
export async function openHistoryModal() {
try {
const hist = await window.sendJsonData("/history_get", { context: getContext() });
- const data = JSON.stringify(hist.history, null, 4);
+ // const data = JSON.stringify(hist.history, null, 4);
+ const data = hist.history
const size = hist.tokens
- await showEditorModal(data, "json", `History ~${size} tokens`, "Conversation history visible to the LLM. History is compressed to fit into the context window over time.");
+ await showEditorModal(data, "markdown", `History ~${size} tokens`, "Conversation history visible to the LLM. History is compressed to fit into the context window over time.");
} catch (e) {
window.toastFetchError("Error fetching history", e)
return