Agent Zero AI framework
vision prototype, terminal sessions
frdel8a9de4a2ee33
diff --git a/agent.py b/agent.pyindex 415b4a20e..d6bc73973 100644--- a/agent.py+++ b/agent.py@@ -10,7 +10,12 @@ import modelsfrom langchain_core.prompt_values import ChatPromptValuefrom python.helpers import extract_tools, rate_limiter, files, errors, history, tokensfrom python.helpers.print_style import PrintStyle-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, StringPromptTemplate+from langchain_core.prompts import (+ ChatPromptTemplate,+ MessagesPlaceholder,+ HumanMessagePromptTemplate,+ StringPromptTemplate,+)from langchain_core.prompts.image import ImagePromptTemplatefrom langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessagefrom langchain_core.language_models.chat_models import BaseChatModel@@ -91,7 +96,7 @@ class AgentContext:else:current_agent = self.agent0- self.task =self.run_task(current_agent.monologue)+ self.task = self.run_task(current_agent.monologue)return self.taskdef communicate(self, msg: "UserMessage", broadcast_level: int = 1):@@ -130,9 +135,9 @@ class AgentContext:async def _process_chain(self, agent: "Agent", msg: "UserMessage|str", user=True):try:msg_template = (- await agent.hist_add_user_message(msg) # type: ignore+ agent.hist_add_user_message(msg) # type: ignoreif user- else await agent.hist_add_tool_result(+ else agent.hist_add_tool_result(tool_name="call_subordinate", tool_result=msg # type: ignore))@@ -283,9 +288,6 @@ class Agent:printer.stream(chunk)self.log_from_stream(full, log)- # store as last context window content- self.set_data(Agent.DATA_NAME_CTX_WINDOW, prompt.format())-agent_response = await self.call_chat_model(prompt, callback=stream_callback)@@ -296,10 +298,10 @@ class Agent:self.loop_data.last_response == agent_response): # if assistant_response is the same as last message in history, let him know# Append the assistant's response to the history- await self.hist_add_ai_response(agent_response)+ self.hist_add_ai_response(agent_response)# Append warning message to the historywarning_msg = self.read_prompt("fw.msg_repeat.md")- await self.hist_add_warning(message=warning_msg)+ self.hist_add_warning(message=warning_msg)PrintStyle(font_color="orange", padding=True).print(warning_msg)@@ -307,7 +309,7 @@ class Agent:else: # otherwise proceed with tool# Append the assistant's response to the history- await self.hist_add_ai_response(agent_response)+ self.hist_add_ai_response(agent_response)# process tools requested in agent messagetools_result = await self.process_tools(agent_response)if tools_result: # final response of message loop available@@ -319,7 +321,7 @@ class Agent:except RepairableException as e:# Forward repairable errors to the LLM, maybe it can fix themerror_message = errors.format_error(e)- await self.hist_add_warning(error_message)+ self.hist_add_warning(error_message)PrintStyle(font_color="red", padding=True).print(error_message)self.context.log.log(type="error", content=error_message)except Exception as e:@@ -358,23 +360,31 @@ class Agent:extras += history.Message(False, content=extra).output()loop_data.extras_temporary.clear()- # combine history and extras- history_combined: list[OutputMessage] = history.group_outputs_abab(loop_data.history_output + extras)-- # convert history to LLM format- history_langchain: list[BaseMessage] = history.output_langchain(history_combined)-- PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(- f"History Langchain: {history_langchain}"+ # convert history + extras to LLM format+ history_langchain: list[BaseMessage] = history.output_langchain(+ loop_data.history_output + extras)# build chain from system prompt, message history and model+ system_text = "\n\n".join(loop_data.system)prompt = ChatPromptTemplate.from_messages([- SystemMessage(content="\n\n".join(loop_data.system)),+ SystemMessage(content=system_text),*history_langchain,])++ # store as last context window content+ self.set_data(+ Agent.DATA_NAME_CTX_WINDOW,+ {+ "text": prompt.format(),+ "tokens": self.history.get_tokens()+ + tokens.approximate_tokens(system_text)+ + tokens.approximate_tokens(history.output_text(extras)),+ },+ )+return promptdef handle_critical_exception(self, exception: Exception):@@ -441,12 +451,12 @@ class Agent:def set_data(self, field: str, value):self.data[field] = value- def hist_add_message(self, ai: bool, content: history.MessageContent):- return self.history.add_message(ai=ai, content=content)-- async def hist_add_user_message(- self, message: UserMessage, intervention: bool = False+ def hist_add_message(+ self, ai: bool, content: history.MessageContent, tokens: int = 0):+ return self.history.add_message(ai=ai, content=content, tokens=tokens)++ def hist_add_user_message(self, message: UserMessage, intervention: bool = False):self.history.new_topic() # user message starts a new topic in history# load message template based on intervention@@ -476,19 +486,18 @@ class Agent:self.last_user_message = msgreturn msg- async def hist_add_ai_response(self, message: str):+ def hist_add_ai_response(self, message: str):self.loop_data.last_response = messagecontent = self.parse_prompt("fw.ai_response.md", message=message)return self.hist_add_message(True, content=content)- async def hist_add_warning(self, message: history.MessageContent):+ def hist_add_warning(self, message: history.MessageContent):content = self.parse_prompt("fw.warning.md", message=message)return self.hist_add_message(False, content=content)- async def hist_add_tool_result(self, tool_name: str, tool_result: str, attachments: list[str] = []):- attachments_str = json.dumps(attachments).replace("\n", "")+ def hist_add_tool_result(self, tool_name: str, tool_result: str):content = self.parse_prompt(- "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result, attachments=attachments_str+ "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result)return self.hist_add_message(False, content=content)@@ -620,9 +629,9 @@ class Agent:msg = self.interventionself.intervention = None # reset the intervention messageif progress.strip():- await self.hist_add_ai_response(progress)+ self.hist_add_ai_response(progress)# append the intervention message- await self.hist_add_user_message(msg, intervention=True)+ self.hist_add_user_message(msg, intervention=True)raise InterventionException(msg)async def wait_if_paused(self):@@ -649,7 +658,7 @@ class Agent:return response.messageelse:msg = self.read_prompt("fw.msg_misformat.md")- await self.hist_add_warning(msg)+ self.hist_add_warning(msg)PrintStyle(font_color="red", padding=True).print(msg)self.context.log.log(type="error", content=f"{self.agent_name}: Message misformat"diff --git a/prompts/default/agent.system.instruments.md b/prompts/default/agent.system.instruments.mdindex 677907270..ece095724 100644--- a/prompts/default/agent.system.instruments.md+++ b/prompts/default/agent.system.instruments.md@@ -1,4 +1,5 @@# Instruments-- following are instruments at disposal:+- following are instruments at disposal+- do not overly rely on them they might not be relevant-{{instruments}}\ No newline at end of file+{{instruments}}diff --git a/prompts/default/agent.system.memories.md b/prompts/default/agent.system.memories.mdindex f7ab13fa5..ae32bf361 100644--- a/prompts/default/agent.system.memories.md+++ b/prompts/default/agent.system.memories.md@@ -1,4 +1,5 @@# Memories on the topic-- following are your memories about current topic:+- following are memories about current topic+- do not overly rely on them they might not be relevant{{memories}}\ No newline at end of filediff --git a/prompts/default/agent.system.solutions.md b/prompts/default/agent.system.solutions.mdindex 0d8743d15..926f35b51 100644--- a/prompts/default/agent.system.solutions.md+++ b/prompts/default/agent.system.solutions.md@@ -1,4 +1,5 @@# Solutions from the past-- following are your memories about successful solutions of related problems:+- following are memories about successful solutions of related problems+- do not overly rely on them they might not be relevant{{solutions}}\ No newline at end of filediff --git a/prompts/default/agent.system.tool.code_exe.md b/prompts/default/agent.system.tool.code_exe.mdindex 9be20508f..cd7f2a8e1 100644--- a/prompts/default/agent.system.tool.code_exe.md+++ b/prompts/default/agent.system.tool.code_exe.md@@ -3,10 +3,9 @@execute terminal commands python nodejs code for computation or software tasksplace code in "code" arg; escape carefully and indent properlyselect "runtime" arg: "terminal" "python" "nodejs" "output" "reset"-for dialogues (Y/N etc.), use "terminal" runtime next step, send answer+select "session" number, 0 default, others for multitaskingif code runs long, use "output" to wait, "reset" to kill processuse "pip" "npm" "apt-get" in "terminal" to install packages-important: never use implicit print/output—it doesn't work!to output, use print() or console.log()if tool outputs error, adjust code before retrying; knowledge_tool can helpimportant: check code for placeholders or demo data; replace with real variables; don't reuse snippets@@ -26,6 +25,7 @@ usage:"tool_name": "code_execution_tool","tool_args": {"runtime": "python",+ "session": 0,"code": "import os\nprint(os.getcwd())",}}@@ -41,6 +41,7 @@ usage:"tool_name": "code_execution_tool","tool_args": {"runtime": "terminal",+ "session": 0,"code": "apt-get install zip",}}@@ -55,6 +56,7 @@ usage:"tool_name": "code_execution_tool","tool_args": {"runtime": "output",+ "session": 0,}}~~~@@ -68,6 +70,7 @@ usage:"tool_name": "code_execution_tool","tool_args": {"runtime": "reset",+ "session": 0,}}~~~\ No newline at end of filediff --git a/prompts/default/agent.system.tools_vision.md b/prompts/default/agent.system.tools_vision.mdindex 0af7e7527..dd04e8b26 100644--- a/prompts/default/agent.system.tools_vision.md+++ b/prompts/default/agent.system.tools_vision.md@@ -1,3 +1,18 @@## "Multimodal (Vision) Agent Tools" available:-None yet. In future, this section will contain vision-only tools+### vision_load:+load image data to LLM+use paths arg for attachments++**Example usage**:+```json+{+ "thoughts": [+ "I need to see the image...",+ ],+ "tool_name": "vision_load",+ "tool_args": {+ "paths": ["/path/to/image.png"],+ }+}+```\ No newline at end of filediff --git a/prompts/default/fw.tool_result.md b/prompts/default/fw.tool_result.mdindex b45d456e8..ef41f23bf 100644--- a/prompts/default/fw.tool_result.md+++ b/prompts/default/fw.tool_result.md@@ -1,7 +1,6 @@-~~~json+```json{"tool_name": {{tool_name}},- "tool_result": {{tool_result}},- "attachments": {{attachments}}+ "tool_result": {{tool_result}}}-~~~+```diff --git a/python/api/ctx_window_get.py b/python/api/ctx_window_get.pyindex a8e20db94..16a4438b7 100644--- a/python/api/ctx_window_get.py+++ b/python/api/ctx_window_get.py@@ -9,6 +9,10 @@ class GetCtxWindow(ApiHandler):context = self.get_context(ctxid)agent = context.streaming_agent or context.agent0window = agent.get_data(agent.DATA_NAME_CTX_WINDOW)- size = tokens.approximate_tokens(window)+ if not window or not isinstance(window, dict):+ return {"content": "", "tokens": 0}- return {"content": window, "tokens": size}+ text = window["text"]+ tokens = window["tokens"]++ return {"content": text, "tokens": tokens}diff --git a/python/api/history_get.py b/python/api/history_get.pyindex 579a32890..9b1359c49 100644--- a/python/api/history_get.py+++ b/python/api/history_get.py@@ -8,8 +8,8 @@ class GetHistory(ApiHandler):ctxid = input.get("context", [])context = self.get_context(ctxid)agent = context.streaming_agent or context.agent0- history = agent.history.output()- size = tokens.approximate_tokens(agent.history.output_text())+ history = agent.history.output_text()+ size = agent.history.get_tokens()return {"history": history,diff --git a/python/helpers/files.py b/python/helpers/files.pyindex 91bdd0566..bc0e97eb9 100644--- a/python/helpers/files.py+++ b/python/helpers/files.py@@ -1,6 +1,7 @@from fnmatch import fnmatchimport jsonimport os, re+import base64import reimport shutil@@ -45,6 +46,32 @@ def read_file(_relative_path, _backup_dirs=None, _encoding="utf-8", **kwargs):return content+def read_file_bin(_relative_path, _backup_dirs=None):+ # init backup dirs+ if _backup_dirs is None:+ _backup_dirs = []++ # get absolute path+ absolute_path = find_file_in_dirs(_relative_path, _backup_dirs)++ # read binary content+ with open(absolute_path, "rb") as f:+ return f.read()+++def read_file_base64(_relative_path, _backup_dirs=None):+ # init backup dirs+ if _backup_dirs is None:+ _backup_dirs = []++ # get absolute path+ absolute_path = find_file_in_dirs(_relative_path, _backup_dirs)++ # read binary content and encode to base64+ with open(absolute_path, "rb") as f:+ return base64.b64encode(f.read()).decode('utf-8')++def replace_placeholders_text(_content: str, **kwargs):# Replace placeholders with values from kwargsfor key, value in kwargs.items():@@ -175,6 +202,15 @@ def write_file_bin(relative_path: str, content: bytes):f.write(content)+def write_file_base64(relative_path: str, content: str):+ # decode base64 string to bytes+ data = base64.b64decode(content)+ abs_path = get_abs_path(relative_path)+ os.makedirs(os.path.dirname(abs_path), exist_ok=True)+ with open(abs_path, "wb") as f:+ f.write(data)++def delete_file(relative_path: str):abs_path = get_abs_path(relative_path)if os.path.exists(abs_path):diff --git a/python/helpers/history.py b/python/helpers/history.pyindex 3fcb02426..c09ef61a5 100644--- a/python/helpers/history.py+++ b/python/helpers/history.py@@ -1,16 +1,14 @@from abc import abstractmethodimport asynciofrom collections import OrderedDict+from collections.abc import Mappingimport jsonimport math-import os-from typing import Coroutine, Literal, TypedDict, cast+from typing import Coroutine, Literal, TypedDict, cast, Union, Dict, List, Any, overridefrom python.helpers import messages, tokens, settings, call_llmfrom enum import Enum-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage-from python.helpers.print_style import PrintStyle-from langchain_core.prompts import HumanMessagePromptTemplate-from typing import Any+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AIMessage+BULK_MERGE_COUNT = 3TOPICS_KEEP_COUNT = 3CURRENT_TOPIC_RATIO = 0.5@@ -18,14 +16,22 @@ HISTORY_TOPIC_RATIO = 0.3HISTORY_BULK_RATIO = 0.2TOPIC_COMPRESS_RATIO = 0.65LARGE_MESSAGE_TO_TOPIC_RATIO = 0.25+RAW_MESSAGE_OUTPUT_TEXT_TRIM = 100+++class RawMessage(TypedDict):+ raw_content: "MessageContent"+ preview: str | None+-MessageContent = (- list["MessageContent"]- | OrderedDict[str, "MessageContent"]- | list[OrderedDict[str, "MessageContent"]]- | str- | list[str]-)+MessageContent = Union[+ List["MessageContent"],+ Dict[str, "MessageContent"],+ List[Dict[str, "MessageContent"]],+ str,+ List[str],+ RawMessage,+]class OutputMessage(TypedDict):@@ -37,9 +43,9 @@ class Record:def __init__(self):pass+ @abstractmethoddef get_tokens(self) -> int:- out = self.output_text()- return tokens.approximate_tokens(out)+ pass@abstractmethodasync def compress(self) -> bool:@@ -70,10 +76,25 @@ class Record:class Message(Record):- def __init__(self, ai: bool, content: MessageContent):+ def __init__(self, ai: bool, content: MessageContent, tokens: int = 0):self.ai = aiself.content = content- self.summary: MessageContent = ""+ self.summary: str = ""+ self.tokens: int = tokens or self.calculate_tokens()++ @override+ def get_tokens(self) -> int:+ if not self.tokens:+ self.tokens = self.calculate_tokens()+ return self.tokens++ def calculate_tokens(self):+ text = self.output_text()+ return tokens.approximate_tokens(text)++ def set_summary(self, summary: str):+ self.summary = summary+ self.tokens = self.calculate_tokens()async def compress(self):return False@@ -93,12 +114,15 @@ class Message(Record):"ai": self.ai,"content": self.content,"summary": self.summary,+ "tokens": self.tokens,}@staticmethoddef from_dict(data: dict, history: "History"):- msg = Message(ai=data["ai"], content=data.get("content", "Content lost"))+ content = data.get("content", "Content lost")+ msg = Message(ai=data["ai"], content=content)msg.summary = data.get("summary", "")+ msg.tokens = data.get("tokens", 0)return msg@@ -108,8 +132,16 @@ class Topic(Record):self.summary: str = ""self.messages: list[Message] = []- def add_message(self, ai: bool, content: MessageContent):- msg = Message(ai=ai, content=content)+ def get_tokens(self):+ if self.summary:+ return tokens.approximate_tokens(self.summary)+ else:+ return sum(msg.get_tokens() for msg in self.messages)++ def add_message(+ self, ai: bool, content: MessageContent, tokens: int = 0+ ) -> Message:+ msg = Message(ai=ai, content=content, tokens=tokens)self.messages.append(msg)return msg@@ -118,7 +150,7 @@ class Topic(Record):return [OutputMessage(ai=False, content=self.summary)]else:msgs = [m for r in self.messages for m in r.output()]- return group_outputs_abab(msgs)+ return msgsasync def summarize(self):self.summary = await self.summarize_messages(self.messages)@@ -134,22 +166,31 @@ class Topic(Record):)large_msgs = []for m in (m for m in self.messages if not m.summary):+ # TODO refactor thisout = m.output()text = output_text(out)- tok = tokens.approximate_tokens(text)+ tok = m.get_tokens()leng = len(text)if tok > msg_max_size:large_msgs.append((m, tok, leng, out))large_msgs.sort(key=lambda x: x[1], reverse=True)for msg, tok, leng, out in large_msgs:trim_to_chars = leng * (msg_max_size / tok)- trunc = messages.truncate_dict_by_ratio(- self.history.agent,- out[0]["content"],- trim_to_chars * 1.15,- trim_to_chars * 0.85,- )- msg.summary = trunc+ # raw messages will be replaced as a whole, they would become invalid when truncated+ if _is_raw_message(out[0]["content"]):+ msg.set_summary(+ "Message content replaced to save space in context window"+ )++ # regular messages will be truncated+ else:+ trunc = messages.truncate_dict_by_ratio(+ self.history.agent,+ out[0]["content"],+ trim_to_chars * 1.15,+ trim_to_chars * 0.85,+ )+ msg.set_summary(_json_dumps(trunc))return Truereturn False@@ -175,6 +216,7 @@ class Topic(Record):return Falseasync def summarize_messages(self, messages: list[Message]):+ # FIXME: vision bytes are sent to utility LLM, send summary insteadmsg_txt = [m.output_text() for m in messages]summary = await self.history.agent.call_utility_model(system=self.history.agent.read_prompt("fw.topic_summary.sys.md"),@@ -194,9 +236,9 @@ class Topic(Record):@staticmethoddef from_dict(data: dict, history: "History"):topic = Topic(history=history)- topic.summary = data["summary"]+ topic.summary = data.get("summary", "")topic.messages = [- Message.from_dict(m, history=history) for m in data["messages"]+ Message.from_dict(m, history=history) for m in data.get("messages", [])]return topic@@ -214,7 +256,7 @@ class Bulk(Record):return [OutputMessage(ai=False, content=self.summary)]else:msgs = [m for r in self.records for m in r.output()]- return group_outputs_abab(msgs)+ return msgsasync def compress(self):return False@@ -253,8 +295,15 @@ class History(Record):self.current = Topic(history=self)self.agent: Agent = agent+ def get_tokens(self) -> int:+ return (+ self.get_bulks_tokens()+ + self.get_topics_tokens()+ + self.get_current_topic_tokens()+ )+def is_over_limit(self):- limit = get_ctx_size_for_history()+ limit = _get_ctx_size_for_history()total = self.get_tokens()return total > limit@@ -267,15 +316,10 @@ class History(Record):def get_current_topic_tokens(self) -> int:return self.current.get_tokens()- def get_tokens(self) -> int:- return (- self.get_bulks_tokens()- + self.get_topics_tokens()- + self.get_current_topic_tokens()- )-- def add_message(self, ai: bool, content: MessageContent):- return self.current.add_message(ai, content=content)+ def add_message(+ self, ai: bool, content: MessageContent, tokens: int = 0+ ) -> Message:+ return self.current.add_message(ai, content=content, tokens=tokens)def new_topic(self):if self.current.messages:@@ -287,7 +331,6 @@ class History(Record):result += [m for b in self.bulks for m in b.output()]result += [m for t in self.topics for m in t.output()]result += self.current.output()- result = group_outputs_abab(result)return result@staticmethod@@ -307,7 +350,7 @@ class History(Record):def serialize(self):data = self.to_dict()- return json.dumps(data)+ return _json_dumps(data)async def compress(self):compressed = False@@ -317,7 +360,7 @@ class History(Record):self.get_topics_tokens(),self.get_bulks_tokens(),)- total = get_ctx_size_for_history()+ total = _get_ctx_size_for_history()ratios = [(curr, CURRENT_TOPIC_RATIO, "current_topic"),(hist, HISTORY_TOPIC_RATIO, "history_topic"),@@ -392,25 +435,46 @@ class History(Record):def deserialize_history(json_data: str, agent) -> History:history = History(agent=agent)if json_data:- data = json.loads(json_data)+ data = _json_loads(json_data)history = History.from_dict(data, history=history)return history-def get_ctx_size_for_history() -> int:+def _get_ctx_size_for_history() -> int:set = settings.get_settings()return int(set["chat_model_ctx_length"] * set["chat_model_ctx_history"])-def serialize_output(output: OutputMessage, ai_label="ai", human_label="human"):- return f'{ai_label if output["ai"] else human_label}: {serialize_content(output["content"])}'+def _stringify_output(output: OutputMessage, ai_label="ai", human_label="human"):+ return f'{ai_label if output["ai"] else human_label}: {_stringify_content(output["content"])}'-def serialize_content(content: MessageContent) -> str:+def _stringify_content(content: MessageContent) -> str:+ # already a stringif isinstance(content, str):return content++ # raw messages return preview or trimmed json+ if _is_raw_message(content):+ preview: str = content.get("preview", "") # type: ignore+ if preview:+ return preview+ text = _json_dumps(content)+ if len(text) > RAW_MESSAGE_OUTPUT_TEXT_TRIM:+ return text[:RAW_MESSAGE_OUTPUT_TEXT_TRIM] + "... TRIMMED"+ return text++ # regular messages of non-string are dumped as json+ return _json_dumps(content)+++def _output_content_langchain(content: MessageContent):+ if isinstance(content, str):+ return content+ if _is_raw_message(content):+ return content["raw_content"] # type: ignoretry:- return json.dumps(content)+ return _json_dumps(content)except Exception as e:raise e@@ -421,98 +485,73 @@ def group_outputs_abab(outputs: list[OutputMessage]) -> list[OutputMessage]:if result and result[-1]["ai"] == out["ai"]:result[-1] = OutputMessage(ai=result[-1]["ai"],- content=merge_outputs(result[-1]["content"], out["content"]),+ content=_merge_outputs(result[-1]["content"], out["content"]),)else:result.append(out)return result-def output_langchain(messages: list[OutputMessage]) -> list[BaseMessage]:+def group_messages_abab(messages: list[BaseMessage]) -> list[BaseMessage]:result = []- for m in messages:- if m["ai"]:- result.append(AIMessage(content=serialize_content(m["content"])))+ for msg in messages:+ if result and isinstance(result[-1], type(msg)):+ # create new instance of the same type with merged content+ result[-1] = type(result[-1])(content=_merge_outputs(result[-1].content, msg.content)) # type: ignoreelse:- contents = m["content"]-- # sometimes content is a list sometimes not- if not isinstance(contents, list):- contents = [contents]+ result.append(msg)+ return result- PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(- f"Contents: {json.dumps(contents, indent=2)}"- )- template: list[dict[str, str]] = [] # type: ignore- message = ""- images = {}- for _, content in enumerate(contents):- if message:- # the first message is the user message, then the memory and solutions- message += "\n\n--- Memory & Solutions Section: ---\n\n"-- message += serialize_content(content)-- if isinstance(content, dict) and "attachments" in content:- attachments: list[str] = cast(list[str], content["attachments"])- for attachment in attachments:- if not os.path.exists(str(attachment)):- continue- if attachment not in images:- import base64- from mimetypes import guess_type- mime_type, _ = guess_type(str(attachment))- if mime_type.startswith("image/"):- # Read and encode the image file- with open(str(attachment), "rb") as image_file:- base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')- # Construct the data URL- images[attachment] = f"data:{mime_type};base64,{base64_encoded_data}"-- if message:- template.append({"type": "text", "text": message})- if images:- for _, image in images.items():- template.append({"type": "image_url", "image_url": image})- if template:- # only jinja2 is safe for json, both mustache({{...}}) and f-string({...}) are not- result.append(HumanMessagePromptTemplate.from_template(template=template, partial_variables={}, template_format="jinja2")) # type: ignore-- PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(- f"Result: {result}"- )+def output_langchain(messages: list[OutputMessage]):+ result = []+ for m in messages:+ if m["ai"]:+ # result.append(AIMessage(content=serialize_content(m["content"])))+ result.append(AIMessage(_output_content_langchain(content=m["content"]))) # type: ignore+ else:+ # result.append(HumanMessage(content=serialize_content(m["content"])))+ result.append(HumanMessage(_output_content_langchain(content=m["content"]))) # type: ignore+ # ensure message type alternation+ result = group_messages_abab(result)return resultdef output_text(messages: list[OutputMessage], ai_label="ai", human_label="human"):- return "\n".join(serialize_output(o, ai_label, human_label) for o in messages)+ return "\n".join(_stringify_output(o, ai_label, human_label) for o in messages)++def _merge_outputs(a: MessageContent, b: MessageContent) -> MessageContent:+ if isinstance(a, str) and isinstance(b, str):+ return a + b-def merge_outputs(a: MessageContent, b: MessageContent) -> MessageContent:if not isinstance(a, list):a = [a]if not isinstance(b, list):b = [b]- return a + b # type: ignore- # return merge_properties(a, b)++ return cast(MessageContent, a + b)-def merge_properties(a: MessageContent, b: MessageContent) -> MessageContent:- if isinstance(a, list):- if isinstance(b, list):- return a + b # type: ignore+def _merge_properties(+ a: Dict[str, MessageContent], b: Dict[str, MessageContent]+) -> Dict[str, MessageContent]:+ result = a.copy()+ for k, v in b.items():+ if k in result:+ result[k] = _merge_outputs(result[k], v)else:- return a + [b]- elif isinstance(b, list):- return [a] + b # type: ignore- elif isinstance(a, dict) and isinstance(b, dict):- for key, value in b.items():- if key in a:- a[key] = merge_properties(a[key], value)- else:- a[key] = value- return a- elif isinstance(a, str) and isinstance(b, str):- return a + b- raise ValueError(f"Cannot merge {a} and {b}")+ result[k] = v+ return result+++def _is_raw_message(obj: object) -> bool:+ return isinstance(obj, Mapping) and "raw_content" in obj+++def _json_dumps(obj):+ return json.dumps(obj, ensure_ascii=False)+++def _json_loads(obj):+ return json.loads(obj)diff --git a/python/helpers/images.py b/python/helpers/images.pynew file mode 100644index 000000000..9377f1e9c--- /dev/null+++ b/python/helpers/images.py@@ -0,0 +1,35 @@+from PIL import Image+import io+import math+++def compress_image(image_data: bytes, *, max_pixels: int = 256_000, quality: int = 50) -> bytes:+ """Compress an image by scaling it down and converting to JPEG with quality settings.++ Args:+ image_data: Raw image bytes+ max_pixels: Maximum number of pixels in the output image (width * height)+ quality: JPEG quality setting (1-100)++ Returns:+ Compressed image as bytes+ """+ # load image from bytes+ img = Image.open(io.BytesIO(image_data))++ # calculate scaling factor to get to max_pixels+ current_pixels = img.width * img.height+ if current_pixels > max_pixels:+ scale = math.sqrt(max_pixels / current_pixels)+ new_width = int(img.width * scale)+ new_height = int(img.height * scale)+ img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)++ # convert to RGB if needed (for JPEG)+ if img.mode in ('RGBA', 'P'):+ img = img.convert('RGB')++ # save as JPEG with compression+ output = io.BytesIO()+ img.save(output, format='JPEG', quality=quality, optimize=True)+ return output.getvalue()diff --git a/python/helpers/runtime.py b/python/helpers/runtime.pyindex 6709c6fdf..189d45288 100644--- a/python/helpers/runtime.py+++ b/python/helpers/runtime.py@@ -2,6 +2,9 @@ import argparseimport inspectfrom typing import TypeVar, Callable, Awaitable, Union, overload, castfrom python.helpers import dotenv, rfc, settings+import asyncio+import threading+import queueT = TypeVar('T')R = TypeVar('R')@@ -102,3 +105,22 @@ def _get_rfc_url() -> str:url = url+":"+str(set["rfc_port_http"])url += "/rfc"return url+++def call_development_function_sync(func: Union[Callable[..., T], Callable[..., Awaitable[T]]], *args, **kwargs) -> T:+ # run async function in sync manner+ result_queue = queue.Queue()++ def run_in_thread():+ result = asyncio.run(call_development_function(func, *args, **kwargs))+ result_queue.put(result)++ thread = threading.Thread(target=run_in_thread)+ thread.start()+ thread.join(timeout=30) # wait for thread with timeout++ if thread.is_alive():+ raise TimeoutError("Function call timed out after 30 seconds")++ result = result_queue.get_nowait()+ return cast(T, result)diff --git a/python/helpers/tool.py b/python/helpers/tool.pyindex fcfb51070..fef3058ac 100644--- a/python/helpers/tool.py+++ b/python/helpers/tool.py@@ -1,5 +1,6 @@from abc import abstractmethod-from dataclasses import dataclass, field+from dataclasses import dataclass+from agent import Agentfrom python.helpers.print_style import PrintStyle@@ -8,8 +9,6 @@ from python.helpers.print_style import PrintStyleclass Response:message:strbreak_loop: bool- attachments: list[str] = field(default_factory=list[str])-class Tool:@@ -34,7 +33,7 @@ class Tool:async def after_execution(self, response: Response, **kwargs):text = response.message.strip()- await self.agent.hist_add_tool_result(self.name, text, response.attachments)+ self.agent.hist_add_tool_result(self.name, text)PrintStyle(font_color="#1B4F72", background_color="white", padding=True, bold=True).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")PrintStyle(font_color="#85C1E9").print(response.message)self.log.update(content=response.message)diff --git a/python/helpers/whisper.py b/python/helpers/whisper.pyindex f92e4c8b5..0f644487d 100644--- a/python/helpers/whisper.py+++ b/python/helpers/whisper.py@@ -30,7 +30,7 @@ async def _preload(model_name:str):is_updating_model = Trueif not _model or _model_name != model_name:PrintStyle.standard(f"Loading Whisper model: {model_name}")- _model = whisper.load_model(model_name)+ _model = whisper.load_model(name=model_name) # type: ignore_model_name = model_namefinally:is_updating_model = Falsediff --git a/python/tools/call_subordinate.py b/python/tools/call_subordinate.pyindex 73eb9b574..3a50ffdc4 100644--- a/python/tools/call_subordinate.py+++ b/python/tools/call_subordinate.py@@ -18,7 +18,7 @@ class Delegation(Tool):# add user message to subordinate agentsubordinate: Agent = self.agent.get_data(Agent.DATA_NAME_SUBORDINATE)- await subordinate.hist_add_user_message(UserMessage(message=message, attachments=[]))+ subordinate.hist_add_user_message(UserMessage(message=message, attachments=[]))# run subordinate monologueresult = await subordinate.monologue()# resultdiff --git a/python/tools/code_execution_tool.py b/python/tools/code_execution_tool.pyindex 6a0686875..593e734fa 100644--- a/python/tools/code_execution_tool.py+++ b/python/tools/code_execution_tool.py@@ -12,7 +12,7 @@ from python.helpers.docker import DockerContainerManager@dataclassclass State:- shell: LocalInteractiveSession | SSHInteractiveSession+ shells: dict[int, LocalInteractiveSession | SSHInteractiveSession]docker: DockerContainerManager | None@@ -27,19 +27,26 @@ class CodeExecution(Tool):# os.chdir(files.get_abs_path("./work_dir")) #change CWD to work_dirruntime = self.args.get("runtime", "").lower().strip()+ session = int(self.args.get("session", 0))if runtime == "python":- response = await self.execute_python_code(self.args["code"])+ response = await self.execute_python_code(+ code=self.args["code"], session=session+ )elif runtime == "nodejs":- response = await self.execute_nodejs_code(self.args["code"])+ response = await self.execute_nodejs_code(+ code=self.args["code"], session=session+ )elif runtime == "terminal":- response = await self.execute_terminal_command(self.args["code"])+ response = await self.execute_terminal_command(+ command=self.args["code"], session=session+ )elif runtime == "output":response = await self.get_terminal_output(- wait_with_output=5, wait_without_output=60+ session=session, wait_with_output=5, wait_without_output=60)elif runtime == "reset":- response = await self.reset_terminal()+ response = await self.reset_terminal(session=session)else:response = self.agent.read_prompt("fw.code_runtime_wrong.md", runtime=runtime@@ -72,11 +79,15 @@ class CodeExecution(Tool):# PrintStyle().print()def get_log_object(self):- return self.agent.context.log.log(type="code_exe", heading=f"{self.agent.agent_name}: Using tool '{self.name}'", content="", kvps=self.args)-+ return self.agent.context.log.log(+ type="code_exe",+ heading=f"{self.agent.agent_name}: Using tool '{self.name}'",+ content="",+ kvps=self.args,+ )async def after_execution(self, response, **kwargs):- await self.agent.hist_add_tool_result(self.name, response.message)+ self.agent.hist_add_tool_result(self.name, response.message)async def prepare_state(self, reset=False):self.state = self.agent.get_data("_cot_state")@@ -97,7 +108,11 @@ class CodeExecution(Tool):# initialize local or remote interactive shell insterfaceif self.agent.config.code_exec_ssh_enabled:- pswd = self.agent.config.code_exec_ssh_pass if self.agent.config.code_exec_ssh_pass else await rfc_exchange.get_root_password()+ pswd = (+ self.agent.config.code_exec_ssh_pass+ if self.agent.config.code_exec_ssh_pass+ else await rfc_exchange.get_root_password()+ )shell = SSHInteractiveSession(self.agent.context.log,self.agent.config.code_exec_ssh_addr,@@ -108,42 +123,63 @@ class CodeExecution(Tool):else:shell = LocalInteractiveSession()- self.state = State(shell=shell, docker=docker)+ self.state = State(shells={0: shell}, docker=docker)await shell.connect()self.agent.set_data("_cot_state", self.state)- async def execute_python_code(self, code: str, reset: bool = False):+ async def execute_python_code(self, session: int, code: str, reset: bool = False):escaped_code = shlex.quote(code)command = f"ipython -c {escaped_code}"- return await self.terminal_session(command, reset)+ return await self.terminal_session(session, command, reset)- async def execute_nodejs_code(self, code: str, reset: bool = False):+ async def execute_nodejs_code(self, session: int, code: str, reset: bool = False):escaped_code = shlex.quote(code)command = f"node /exe/node_eval.js {escaped_code}"- return await self.terminal_session(command, reset)+ return await self.terminal_session(session, command, reset)- async def execute_terminal_command(self, command: str, reset: bool = False):- return await self.terminal_session(command, reset)+ async def execute_terminal_command(+ self, session: int, command: str, reset: bool = False+ ):+ return await self.terminal_session(session, command, reset)- async def terminal_session(self, command: str, reset: bool = False):+ async def terminal_session(self, session: int, command: str, reset: bool = False):await self.agent.handle_intervention() # wait for intervention and handle it, if paused# try again on lost connectionfor i in range(2):try:-+if reset:await self.reset_terminal()- self.state.shell.send_command(command)-- PrintStyle(background_color="white", font_color="#1B4F72", bold=True).print(- f"{self.agent.agent_name} code execution output"- )- return await self.get_terminal_output()+ if session not in self.state.shells:+ if self.agent.config.code_exec_ssh_enabled:+ pswd = (+ self.agent.config.code_exec_ssh_pass+ if self.agent.config.code_exec_ssh_pass+ else await rfc_exchange.get_root_password()+ )+ shell = SSHInteractiveSession(+ self.agent.context.log,+ self.agent.config.code_exec_ssh_addr,+ self.agent.config.code_exec_ssh_port,+ self.agent.config.code_exec_ssh_user,+ pswd,+ )+ else:+ shell = LocalInteractiveSession()+ self.state.shells[session] = shell+ await shell.connect()++ self.state.shells[session].send_command(command)++ PrintStyle(+ background_color="white", font_color="#1B4F72", bold=True+ ).print(f"{self.agent.agent_name} code execution output")+ return await self.get_terminal_output(session)except Exception as e:- if i==1:+ if i == 1:# try again on lost connectionPrintStyle.error(str(e))await self.prepare_state(reset=True)@@ -153,6 +189,7 @@ class CodeExecution(Tool):async def get_terminal_output(self,+ session=0,reset_full_output=True,wait_with_output=3,wait_without_output=10,@@ -165,10 +202,10 @@ class CodeExecution(Tool):while max_exec_time <= 0 or time.time() - start_time < max_exec_time:await asyncio.sleep(SLEEP_TIME) # Wait for some output to be generated- full_output, partial_output = await self.state.shell.read_output(+ full_output, partial_output = await self.state.shells[session].read_output(timeout=max_exec_time, reset_full_output=reset_full_output)- reset_full_output = False # only reset once+ reset_full_output = False # only reset onceawait self.agent.handle_intervention() # wait for intervention and handle it, if paused@@ -184,8 +221,10 @@ class CodeExecution(Tool):breakreturn full_output- async def reset_terminal(self):- self.state.shell.close()+ async def reset_terminal(self, session=0):+ if session in self.state.shells:+ self.state.shells[session].close()+ del self.state.shells[session]await self.prepare_state(reset=True)response = self.agent.read_prompt("fw.code_reset.md")self.log.update(content=response)diff --git a/python/tools/input.py b/python/tools/input.pyindex 6e5a812f8..b23c9adee 100644--- a/python/tools/input.py+++ b/python/tools/input.py@@ -20,4 +20,4 @@ class Input(Tool):return self.agent.context.log.log(type="code_exe", heading=f"{self.agent.agent_name}: Using tool '{self.name}'", content="", kvps=self.args)async def after_execution(self, response, **kwargs):- await self.agent.hist_add_tool_result(self.name, response.message)\ No newline at end of file+ self.agent.hist_add_tool_result(self.name, response.message)\ No newline at end of filediff --git a/python/tools/vision_load.py b/python/tools/vision_load.pynew file mode 100644index 000000000..e1c455fa5--- /dev/null+++ b/python/tools/vision_load.py@@ -0,0 +1,79 @@+import base64+from python.helpers.print_style import PrintStyle+from python.helpers.tool import Tool, Response+from python.helpers import runtime, files, images+from mimetypes import guess_type+from python.helpers import history++# image optimization and token estimation for context window+MAX_PIXELS = 768_000+QUALITY = 75+TOKENS_ESTIMATE = 1500+++class VisionLoad(Tool):+ async def execute(self, paths: list[str] = [], **kwargs) -> Response:++ self.images_dict = {}+ template: list[dict[str, str]] = [] # type: ignore++ for path in paths:+ if not await runtime.call_development_function(files.exists, str(path)):+ continue++ if path not in self.images_dict:+ mime_type, _ = guess_type(str(path))+ if mime_type and mime_type.startswith("image/"):+ # Read binary file+ file_content = await runtime.call_development_function(+ files.read_file_base64, str(path)+ )+ file_content = base64.b64decode(file_content)+ # Compress and convert to JPEG+ compressed = images.compress_image(+ file_content, max_pixels=MAX_PIXELS, quality=QUALITY+ )+ # Encode as base64+ file_content_b64 = base64.b64encode(compressed).decode("utf-8")++ # DEBUG: Save compressed image+ # await runtime.call_development_function(+ # files.write_file_base64, str(path), file_content_b64+ # )++ # Construct the data URL (always JPEG after compression)+ self.images_dict[path] = file_content_b64++ return Response(message="dummy", break_loop=False)++ async def after_execution(self, response: Response, **kwargs):++ # build image data messages for LLMs, or error message+ content = []+ if self.images_dict:+ for _, image in self.images_dict.items():+ content.append(+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{image}"},+ }+ )+ # append as raw message content for LLMs with vision tokens estimate+ msg = history.RawMessage(raw_content=content, preview="<Base64 encoded image data>")+ self.agent.hist_add_message(+ False, content=msg, tokens=TOKENS_ESTIMATE * len(content)+ )+ else:+ self.agent.hist_add_tool_result(self.name, "No images processed")++ # print and log short version+ message = (+ "No images processed"+ if not self.images_dict+ else f"{len(self.images_dict)} images processed"+ )+ PrintStyle(+ font_color="#1B4F72", background_color="white", padding=True, bold=True+ ).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")+ PrintStyle(font_color="#85C1E9").print(message)+ self.log.update(result=message)diff --git a/webui/js/history.js b/webui/js/history.jsindex e62fc615e..040de72fe 100644--- a/webui/js/history.js+++ b/webui/js/history.js@@ -3,9 +3,10 @@ import { getContext } from "../index.js";export async function openHistoryModal() {try {const hist = await window.sendJsonData("/history_get", { context: getContext() });- const data = JSON.stringify(hist.history, null, 4);+ // const data = JSON.stringify(hist.history, null, 4);+ const data = hist.historyconst size = hist.tokens- await showEditorModal(data, "json", `History ~${size} tokens`, "Conversation history visible to the LLM. History is compressed to fit into the context window over time.");+ await showEditorModal(data, "markdown", `History ~${size} tokens`, "Conversation history visible to the LLM. History is compressed to fit into the context window over time.");} catch (e) {window.toastFetchError("Error fetching history", e)return