Agent Zero AI framework
caching and ctx window optimizations
frdel3dceaca64ec4
diff --git a/agent.py b/agent.pyindex 6cb96e3a7..cffacf6e1 100644--- a/agent.py+++ b/agent.py@@ -798,6 +798,7 @@ class Agent:response_callback: Callable[[str, str], Awaitable[None]] | None = None,reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,background: bool = False,+ explicit_caching: bool = True,):response = ""@@ -812,6 +813,7 @@ class Agent:rate_limiter_callback=(self.rate_limiter_callback if not background else None),+ explicit_caching=explicit_caching,)return response, reasoningdiff --git a/models.py b/models.pyindex c8c3af9fc..bf584f763 100644--- a/models.py+++ b/models.py@@ -316,7 +316,7 @@ class LiteLLMChatWrapper(SimpleChatModel):def _llm_type(self) -> str:return "litellm-chat"- def _convert_messages(self, messages: List[BaseMessage]) -> List[dict]:+ def _convert_messages(self, messages: List[BaseMessage], explicit_caching: bool = False) -> List[dict]:result = []# Map LangChain message types to LiteLLM rolesrole_mapping = {@@ -362,6 +362,15 @@ class LiteLLMChatWrapper(SimpleChatModel):message_dict["tool_call_id"] = tool_call_idresult.append(message_dict)++ if explicit_caching and result:+ if result[0]["role"] == "system":+ result[0]["cache_control"] = {"type": "ephemeral"}+ for i in range(len(result) - 1, -1, -1):+ if result[i]["role"] == "assistant":+ result[i]["cache_control"] = {"type": "ephemeral"}+ break+return resultdef _call(@@ -464,6 +473,7 @@ class LiteLLMChatWrapper(SimpleChatModel):rate_limiter_callback: (Callable[[str, str, int, int], Awaitable[bool]] | None) = None,+ explicit_caching: bool = False,**kwargs: Any,) -> Tuple[str, str]:@@ -478,7 +488,7 @@ class LiteLLMChatWrapper(SimpleChatModel):messages.append(HumanMessage(content=user_message))# convert to litellm format- msgs_conv = self._convert_messages(messages)+ msgs_conv = self._convert_messages(messages, explicit_caching=explicit_caching)# Apply rate limiting if configuredlimiter = await apply_rate_limiter(diff --git a/prompts/fw.topic_summary.sys.md b/prompts/fw.topic_summary.sys.mdindex 46004b15b..b8d676647 100644--- a/prompts/fw.topic_summary.sys.md+++ b/prompts/fw.topic_summary.sys.md@@ -6,7 +6,8 @@ You must return a single summary of all records# Expected outputYour output will be a text of the summary-Length of the text should be one paragraph, approximately 100 words+Summary must be shorter than original messages+Length of the text should be maximum one paragraph, approximately 100 words, shorter if original is shorterNo introNo conclusionNo formattingdiff --git a/python/extensions/message_loop_prompts_after/_50_recall_memories.py b/python/extensions/message_loop_prompts_after/_50_recall_memories.pyindex 9e2beb9f8..e7fd509f8 100644--- a/python/extensions/message_loop_prompts_after/_50_recall_memories.py+++ b/python/extensions/message_loop_prompts_after/_50_recall_memories.py@@ -8,6 +8,7 @@ from python.helpers import dirty_json, errors, settings, logDATA_NAME_TASK = "_recall_memories_task"DATA_NAME_ITER = "_recall_memories_iter"+SEARCH_TIMEOUT = 30class RecallMemories(Extension):@@ -38,7 +39,10 @@ class RecallMemories(Extension):)task = asyncio.create_task(- self.search_memories(loop_data=loop_data, log_item=log_item, **kwargs)+ asyncio.wait_for(+ self.search_memories(loop_data=loop_data, log_item=log_item, **kwargs),+ timeout=SEARCH_TIMEOUT,+ ))else:task = Nonediff --git a/python/helpers/document_query.py b/python/helpers/document_query.pyindex ff5575ec0..4ba82b159 100644--- a/python/helpers/document_query.py+++ b/python/helpers/document_query.py@@ -433,7 +433,8 @@ class DocumentQueryHelper:messages=[SystemMessage(content=qa_system_message),HumanMessage(content=qa_user_message),- ]+ ],+ explicit_caching=False,)self.progress_callback(f"Q&A process completed")diff --git a/python/helpers/history.py b/python/helpers/history.pyindex d8b7d6c5f..0604bb4d9 100644--- a/python/helpers/history.py+++ b/python/helpers/history.py@@ -10,13 +10,16 @@ from enum import Enumfrom langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AIMessageBULK_MERGE_COUNT = 3-TOPICS_KEEP_COUNT = 3+TOPICS_MERGE_COUNT = 3CURRENT_TOPIC_RATIO = 0.5HISTORY_TOPIC_RATIO = 0.3HISTORY_BULK_RATIO = 0.2-TOPIC_COMPRESS_RATIO = 0.65-LARGE_MESSAGE_TO_TOPIC_RATIO = 0.5+CURRENT_TOPIC_ATTENTION_COMPRESSION = 0.65 # compress current topic's attention window to 65% of size+HISTORY_TOPIC_ATTENTION_COMPRESSION = 0 # compress history topic's attention window to 0% of size - only request and response remain intact+LARGE_MESSAGE_TO_CURRENT_TOPIC_RATIO = 0.5+LARGE_MESSAGE_TO_HISTORY_TOPIC_RATIO = 0.2RAW_MESSAGE_OUTPUT_TEXT_TRIM = 100+COMPRESSION_TARGET_RATIO = 0.8class RawMessage(TypedDict):@@ -155,13 +158,12 @@ class Topic(Record):self.summary = await self.summarize_messages(self.messages)return self.summary- async def compress_large_messages(self) -> bool:+ def compress_large_messages(self, message_ratio: float = CURRENT_TOPIC_RATIO * LARGE_MESSAGE_TO_CURRENT_TOPIC_RATIO) -> bool:set = settings.get_settings()msg_max_size = (set["chat_model_ctx_length"]* set["chat_model_ctx_history"]- * CURRENT_TOPIC_RATIO- * LARGE_MESSAGE_TO_TOPIC_RATIO+ * message_ratio)large_msgs = []for m in (m for m in self.messages if not m.summary):@@ -195,27 +197,29 @@ class Topic(Record):return Falseasync def compress(self) -> bool:- compress = await self.compress_large_messages()+ compress = self.compress_large_messages()if not compress:compress = await self.compress_attention()return compress- async def compress_attention(self) -> bool:+ async def compress_attention(self, ratio: float = CURRENT_TOPIC_ATTENTION_COMPRESSION) -> bool:- if len(self.messages) > 2:- cnt_to_sum = math.ceil((len(self.messages) - 2) * TOPIC_COMPRESS_RATIO)- msg_to_sum = self.messages[1 : cnt_to_sum + 1]- summary = await self.summarize_messages(msg_to_sum)- sum_msg_content = self.history.agent.parse_prompt(- "fw.msg_summary.md", summary=summary- )- sum_msg = Message(False, sum_msg_content)- self.messages[1 : cnt_to_sum + 1] = [sum_msg]- return True- return False+ middle = len(self.messages) - 2+ if middle < 2:+ return False+ cnt_to_sum = middle - math.floor(middle * ratio)+ if cnt_to_sum < 1:+ return False+ msg_to_sum = self.messages[1 : cnt_to_sum + 1]+ summary = await self.summarize_messages(msg_to_sum)+ sum_msg_content = self.history.agent.parse_prompt(+ "fw.msg_summary.md", summary=summary+ )+ sum_msg = Message(False, sum_msg_content)+ self.messages[1 : cnt_to_sum + 1] = [sum_msg]+ return Trueasync def summarize_messages(self, messages: list[Message]):- # FIXME: vision bytes are sent to utility LLM, send summary insteadmsg_txt = [m.output_text() for m in messages]summary = await self.history.agent.call_utility_model(system=self.history.agent.read_prompt("fw.topic_summary.sys.md"),@@ -363,22 +367,38 @@ class History(Record):async def compress(self):compressed = False+ total = _get_ctx_size_for_history()+ curr, hist, bulk = (+ self.get_current_topic_tokens(),+ self.get_topics_tokens(),+ self.get_bulks_tokens(),+ )+ if (curr + hist + bulk) <= total:+ return False++ target = total * COMPRESSION_TARGET_RATIO+ prev_total = curr + hist + bulkwhile True:curr, hist, bulk = (self.get_current_topic_tokens(),self.get_topics_tokens(),self.get_bulks_tokens(),)- total = _get_ctx_size_for_history()++ # safeguard against infinite loop in case LLM bloats the summary for some reason+ if (curr + hist + bulk) >= prev_total:+ break+ prev_total = curr + hist + bulk+ratios = [(curr, CURRENT_TOPIC_RATIO, "current_topic"),(hist, HISTORY_TOPIC_RATIO, "history_topic"),(bulk, HISTORY_BULK_RATIO, "history_bulk"),]- ratios = sorted(ratios, key=lambda x: (x[0] / total) / x[1], reverse=True)+ ratios = sorted(ratios, key=lambda x: (x[0] / target) / x[1], reverse=True)compressed_part = Falsefor ratio in ratios:- if ratio[0] > ratio[1] * total:+ if ratio[0] > ratio[1] * target:over_part = ratio[2]if over_part == "current_topic":compressed_part = await self.current.compress()@@ -394,24 +414,29 @@ class History(Record):continueelse:return compressed+ return compressedasync def compress_topics(self) -> bool:- # summarize topics one by one++ # 1. first identify large messages and compress them cheaplyfor topic in self.topics:- if not topic.summary:- await topic.summarize()+ if topic.compress_large_messages(HISTORY_TOPIC_RATIO*LARGE_MESSAGE_TO_HISTORY_TOPIC_RATIO):return True- # move oldest topic to bulks and summarize+ # 2. summarize topics attention window one by onefor topic in self.topics:+ if await topic.compress_attention(HISTORY_TOPIC_ATTENTION_COMPRESSION):+ return True++ # 3. move oldest topics to bulks in chunks+ if self.topics:+ count = TOPICS_MERGE_COUNT if len(self.topics) >= TOPICS_MERGE_COUNT else 1+ chunk = self.topics[:count]bulk = Bulk(history=self)- bulk.records.append(topic)- if topic.summary:- bulk.summary = topic.summary- else:- await bulk.summarize()+ bulk.records.extend(chunk)+ await bulk.summarize()self.bulks.append(bulk)- self.topics.remove(topic)+ self.topics[:count] = []return Truereturn Falsediff --git a/python/helpers/memory_consolidation.py b/python/helpers/memory_consolidation.pyindex 244ebbc8d..86a5cf598 100644--- a/python/helpers/memory_consolidation.py+++ b/python/helpers/memory_consolidation.py@@ -82,7 +82,7 @@ class MemoryConsolidator:Args:new_memory: The new memory content to process- area: Memory area (MAIN, FRAGMENTS, SOLUTIONS, INSTRUMENTS)+ area: Memory area (MAIN, FRAGMENTS, SOLUTIONS)metadata: Initial metadata for the memorylog_item: Optional log item for progress tracking