Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Make get context window much more efficient (#3292)
Matthew Zhou80b3356945f9
diff --git a/letta/server/server.py b/letta/server/server.pyindex 8a2c38fef..af996ffc2 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -1342,9 +1342,6 @@ class SyncServer(Server):new_passage_size = await self.agent_manager.passage_size_async(actor=actor, agent_id=agent_id)assert new_passage_size >= curr_passage_size # in case empty files are added- # rebuild system prompt and force- agent_state = await self.agent_manager.rebuild_system_prompt_async(agent_id=agent_id, actor=actor, force=True)-# update job statusjob.status = JobStatus.completedjob.metadata["num_passages"] = num_passagesdiff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 953be77b8..b641041b6 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -1540,29 +1540,42 @@ class AgentManager:else:return agent_state- @trace_method+ # TODO: This is probably one of the worst pieces of code I've ever written please rip up as you see wish@enforce_types+ @trace_methodasync def rebuild_system_prompt_async(- self, agent_id: str, actor: PydanticUser, force=False, update_timestamp=True, tool_rules_solver: Optional[ToolRulesSolver] = None- ) -> PydanticAgentState:+ self,+ agent_id: str,+ actor: PydanticUser,+ force=False,+ update_timestamp=True,+ tool_rules_solver: Optional[ToolRulesSolver] = None,+ dry_run: bool = False,+ ) -> Tuple[PydanticAgentState, Optional[PydanticMessage], int, int]:"""Rebuilds the system message with the latest memory object and any shared memory block updatesUpdates to core memory blocks should trigger a "rebuild", which itself will create a new message objectUpdates to the memory header should *not* trigger a rebuild, since that will simply flood recall storage with excess messages"""- # Get the current agent state- agent_state = await self.get_agent_by_id_async(agent_id=agent_id, include_relationships=["memory", "sources"], actor=actor)+ num_messages_task = self.message_manager.size_async(actor=actor, agent_id=agent_id)+ num_archival_memories_task = self.passage_manager.agent_passage_size_async(actor=actor, agent_id=agent_id)+ agent_state_task = self.get_agent_by_id_async(agent_id=agent_id, include_relationships=["memory", "sources", "tools"], actor=actor)++ num_messages, num_archival_memories, agent_state = await asyncio.gather(+ num_messages_task,+ num_archival_memories_task,+ agent_state_task,+ )+if not tool_rules_solver:tool_rules_solver = ToolRulesSolver(agent_state.tool_rules)- curr_system_message = await self.get_system_message_async(- agent_id=agent_id, actor=actor- ) # this is the system + memory bank, not just the system prompt+ curr_system_message = await self.message_manager.get_message_by_id_async(message_id=agent_state.message_ids[0], actor=actor)if curr_system_message is None:logger.warning(f"No system message found for agent {agent_state.id} and user {actor}")- return agent_state+ return agent_state, curr_system_message, num_messages, num_archival_memoriescurr_system_message_openai = curr_system_message.to_openai_dict()@@ -1576,7 +1589,7 @@ class AgentManager:logger.debug(f"Memory hasn't changed for agent id={agent_id} and actor=({actor.id}, {actor.name}), skipping system prompt rebuild")- return agent_state+ return agent_state, curr_system_message, num_messages, num_archival_memories# If the memory didn't update, we probably don't want to update the timestamp inside# For example, if we're doing a system prompt swap, this should probably be False@@ -1586,9 +1599,6 @@ class AgentManager:# NOTE: a bit of a hack - we pull the timestamp from the message created_bymemory_edit_timestamp = curr_system_message.created_at- num_messages = await self.message_manager.size_async(actor=actor, agent_id=agent_id)- num_archival_memories = await self.passage_manager.agent_passage_size_async(actor=actor, agent_id=agent_id)-# update memory (TODO: potentially update recall/archival stats separately)new_system_message_str = compile_system_message(@@ -1607,19 +1617,23 @@ class AgentManager:logger.debug(f"Rebuilding system with new memory...\nDiff:\n{diff}")# Swap the system message out (only if there is a diff)- message = PydanticMessage.dict_to_message(+ temp_message = PydanticMessage.dict_to_message(agent_id=agent_id,model=agent_state.llm_config.model,openai_message_dict={"role": "system", "content": new_system_message_str},)- message = await self.message_manager.update_message_by_id_async(- message_id=curr_system_message.id,- message_update=MessageUpdate(**message.model_dump()),- actor=actor,- )- return await self.set_in_context_messages_async(agent_id=agent_id, message_ids=agent_state.message_ids, actor=actor)- else:- return agent_state+ temp_message.id = curr_system_message.id++ if not dry_run:+ await self.message_manager.update_message_by_id_async(+ message_id=curr_system_message.id,+ message_update=MessageUpdate(**temp_message.model_dump()),+ actor=actor,+ )+ else:+ curr_system_message = temp_message++ return agent_state, curr_system_message, num_messages, num_archival_memories@trace_method@enforce_types@@ -1781,7 +1795,7 @@ class AgentManager:# NOTE: don't do this since re-buildin the memory is handled at the start of the step# rebuild memory - this records the last edited timestamp of the memory# TODO: pass in update timestamp from block edit time- agent_state = await self.rebuild_system_prompt_async(agent_id=agent_id, actor=actor)+ await self.rebuild_system_prompt_async(agent_id=agent_id, actor=actor)return agent_state@@ -1845,12 +1859,8 @@ class AgentManager:)# Commit the changes- await agent.update_async(session, actor=actor)-- # Force rebuild of system prompt so that the agent is updated with passage count- pydantic_agent = await self.rebuild_system_prompt_async(agent_id=agent_id, actor=actor, force=True)-- return pydantic_agent+ agent = await agent.update_async(session, actor=actor)+ return await agent.to_pydantic_async()@trace_method@enforce_types@@ -2761,8 +2771,11 @@ class AgentManager:results = [row[0] for row in result.all()]return results+ @trace_methodasync def get_context_window(self, agent_id: str, actor: PydanticUser) -> ContextWindowOverview:- agent_state = await self.rebuild_system_prompt_async(agent_id=agent_id, actor=actor, force=True)+ agent_state, system_message, num_messages, num_archival_memories = await self.rebuild_system_prompt_async(+ agent_id=agent_id, actor=actor, force=True, dry_run=True+ )calculator = ContextWindowCalculator()if os.getenv("LETTA_ENVIRONMENT") == "PRODUCTION" or agent_state.llm_config.model_endpoint_type == "anthropic":@@ -2778,5 +2791,7 @@ class AgentManager:actor=actor,token_counter=token_counter,message_manager=self.message_manager,- passage_manager=self.passage_manager,+ system_message_compiled=system_message,+ num_archival_memories=num_archival_memories,+ num_messages=num_messages,)diff --git a/letta/services/context_window_calculator/context_window_calculator.py b/letta/services/context_window_calculator/context_window_calculator.pyindex b3a890280..c405d289f 100644--- a/letta/services/context_window_calculator/context_window_calculator.py+++ b/letta/services/context_window_calculator/context_window_calculator.py@@ -8,8 +8,10 @@ from letta.schemas.agent import AgentStatefrom letta.schemas.enums import MessageRolefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.memory import ContextWindowOverview+from letta.schemas.message import Messagefrom letta.schemas.user import User as PydanticUserfrom letta.services.context_window_calculator.token_counter import TokenCounter+from letta.services.message_manager import MessageManagerlogger = get_logger(__name__)@@ -57,16 +59,18 @@ class ContextWindowCalculator:return None, 1async def calculate_context_window(- self, agent_state: AgentState, actor: PydanticUser, token_counter: TokenCounter, message_manager: Any, passage_manager: Any+ self,+ agent_state: AgentState,+ actor: PydanticUser,+ token_counter: TokenCounter,+ message_manager: MessageManager,+ system_message_compiled: Message,+ num_archival_memories: int,+ num_messages: int,) -> ContextWindowOverview:"""Calculate context window information using the provided token counter"""-- # Fetch data concurrently- (in_context_messages, passage_manager_size, message_manager_size) = await asyncio.gather(- message_manager.get_messages_by_ids_async(message_ids=agent_state.message_ids, actor=actor),- passage_manager.agent_passage_size_async(actor=actor, agent_id=agent_state.id),- message_manager.size_async(actor=actor, agent_id=agent_state.id),- )+ messages = await message_manager.get_messages_by_ids_async(message_ids=agent_state.message_ids[1:], actor=actor)+ in_context_messages = [system_message_compiled] + messages# Convert messages to appropriate formatconverted_messages = token_counter.convert_messages(in_context_messages)@@ -129,8 +133,8 @@ class ContextWindowCalculator:return ContextWindowOverview(# context window breakdown (in messages)num_messages=len(in_context_messages),- num_archival_memory=passage_manager_size,- num_recall_memory=message_manager_size,+ num_archival_memory=num_archival_memories,+ num_recall_memory=num_messages,num_tokens_external_memory_summary=num_tokens_external_memory_summary,external_memory_summary=external_memory_summary,# top-level informationdiff --git a/letta/services/context_window_calculator/token_counter.py b/letta/services/context_window_calculator/token_counter.pyindex 1ec4a3fc9..52e432444 100644--- a/letta/services/context_window_calculator/token_counter.py+++ b/letta/services/context_window_calculator/token_counter.py@@ -5,6 +5,7 @@ from typing import Any, Dict, Listfrom letta.helpers.decorators import async_redis_cachefrom letta.llm_api.anthropic_client import AnthropicClient+from letta.otel.tracing import trace_methodfrom letta.schemas.openai.chat_completion_request import Tool as OpenAIToolfrom letta.utils import count_tokens@@ -36,6 +37,7 @@ class AnthropicTokenCounter(TokenCounter):self.client = anthropic_clientself.model = model+ @trace_method@async_redis_cache(key_func=lambda self, text: f"anthropic_text_tokens:{self.model}:{hashlib.sha256(text.encode()).hexdigest()[:16]}",prefix="token_counter",@@ -46,6 +48,7 @@ class AnthropicTokenCounter(TokenCounter):return 0return await self.client.count_tokens(model=self.model, messages=[{"role": "user", "content": text}])+ @trace_method@async_redis_cache(key_func=lambda self, messages: f"anthropic_message_tokens:{self.model}:{hashlib.sha256(json.dumps(messages, sort_keys=True).encode()).hexdigest()[:16]}",prefix="token_counter",@@ -56,6 +59,7 @@ class AnthropicTokenCounter(TokenCounter):return 0return await self.client.count_tokens(model=self.model, messages=messages)+ @trace_method@async_redis_cache(key_func=lambda self, tools: f"anthropic_tool_tokens:{self.model}:{hashlib.sha256(json.dumps([t.model_dump() for t in tools], sort_keys=True).encode()).hexdigest()[:16]}",prefix="token_counter",@@ -76,6 +80,7 @@ class TiktokenCounter(TokenCounter):def __init__(self, model: str):self.model = model+ @trace_method@async_redis_cache(key_func=lambda self, text: f"tiktoken_text_tokens:{self.model}:{hashlib.sha256(text.encode()).hexdigest()[:16]}",prefix="token_counter",@@ -86,6 +91,7 @@ class TiktokenCounter(TokenCounter):return 0return count_tokens(text)+ @trace_method@async_redis_cache(key_func=lambda self, messages: f"tiktoken_message_tokens:{self.model}:{hashlib.sha256(json.dumps(messages, sort_keys=True).encode()).hexdigest()[:16]}",prefix="token_counter",@@ -98,6 +104,7 @@ class TiktokenCounter(TokenCounter):return num_tokens_from_messages(messages=messages, model=self.model)+ @trace_method@async_redis_cache(key_func=lambda self, tools: f"tiktoken_tool_tokens:{self.model}:{hashlib.sha256(json.dumps([t.model_dump() for t in tools], sort_keys=True).encode()).hexdigest()[:16]}",prefix="token_counter",diff --git a/tests/helpers/utils.py b/tests/helpers/utils.pyindex 82abb3a73..950e44d0c 100644--- a/tests/helpers/utils.py+++ b/tests/helpers/utils.py@@ -237,7 +237,7 @@ def validate_context_window_overview(# 16. Check attached file is visibleif attached_file:- assert attached_file.visible_content in overview.core_memory+ assert attached_file.visible_content in overview.core_memory, "File must be attached in core memory"assert '<file status="open"' in overview.core_memoryassert "</file>" in overview.core_memory