Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: thread billing_context through summarization pipeline (#10089)
cthomasbe91c2f94c46
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 1e85ba076..9b9babffb 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -344,6 +344,7 @@ class LettaAgentV3(LettaAgentV2):include_return_message_types=include_return_message_types,request_start_timestamp_ns=request_start_timestamp_ns,include_compaction_messages=include_compaction_messages,+ billing_context=billing_context,)input_messages_to_persist = [] # clear after first step@@ -582,6 +583,7 @@ class LettaAgentV3(LettaAgentV2):include_return_message_types=include_return_message_types,request_start_timestamp_ns=request_start_timestamp_ns,include_compaction_messages=include_compaction_messages,+ billing_context=billing_context,)input_messages_to_persist = [] # clear after first stepasync for chunk in response:@@ -897,6 +899,7 @@ class LettaAgentV3(LettaAgentV2):dry_run: bool = False,enforce_run_id_set: bool = True,include_compaction_messages: bool = False,+ billing_context: Optional["BillingContext"] = None,) -> AsyncGenerator[LettaMessage | dict, None]:"""Execute a single agent step (one LLM call and tool execution).@@ -1221,6 +1224,7 @@ class LettaAgentV3(LettaAgentV2):trigger="context_window_exceeded",context_tokens_before=context_tokens_before,messages_count_before=messages_count_before,+ billing_context=billing_context,)# Recompile the persisted system prompt after compaction so subsequent@@ -1443,6 +1447,7 @@ class LettaAgentV3(LettaAgentV2):trigger="post_step_context_check",context_tokens_before=context_tokens_before,messages_count_before=messages_count_before,+ billing_context=billing_context,)# Recompile the persisted system prompt after compaction so subsequent@@ -2057,6 +2062,7 @@ class LettaAgentV3(LettaAgentV2):trigger: Optional[str] = None,context_tokens_before: Optional[int] = None,messages_count_before: Optional[int] = None,+ billing_context: Optional["BillingContext"] = None,) -> tuple[Message, list[Message], str]:"""Compact the current in-context messages for this agent.@@ -2096,6 +2102,7 @@ class LettaAgentV3(LettaAgentV2):trigger=trigger,context_tokens_before=context_tokens_before,messages_count_before=messages_count_before,+ billing_context=billing_context,)# Update the agent's context token estimatediff --git a/letta/services/summarizer/compact.py b/letta/services/summarizer/compact.pyindex 95d54d4f1..606d16f63 100644--- a/letta/services/summarizer/compact.py+++ b/letta/services/summarizer/compact.py@@ -13,6 +13,7 @@ from letta.schemas.enums import MessageRolefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message, MessageCreate+from letta.schemas.provider_trace import BillingContextfrom letta.schemas.user import Userfrom letta.services.summarizer.self_summarizer import self_summarize_all, self_summarize_sliding_windowfrom letta.services.summarizer.summarizer_all import summarize_all@@ -171,6 +172,7 @@ async def compact_messages(trigger: Optional[str] = None,context_tokens_before: Optional[int] = None,messages_count_before: Optional[int] = None,+ billing_context: Optional[BillingContext] = None,) -> CompactResult:"""Compact in-context messages using summarization.@@ -224,6 +226,7 @@ async def compact_messages(timezone=timezone,agent_tags=agent_tags,tools=tools,+ billing_context=billing_context,)except Exception as e:logger.warning(f"Self summarization failed with exception: {str(e)}. Falling back to self sliding window mode.")@@ -248,6 +251,7 @@ async def compact_messages(timezone=timezone,agent_tags=agent_tags,tools=tools,+ billing_context=billing_context,)summarization_mode_used = "self_compact_sliding_window"except Exception as e:@@ -267,6 +271,7 @@ async def compact_messages(agent_tags=agent_tags,run_id=run_id,step_id=step_id,+ billing_context=billing_context,)summarization_mode_used = "all"elif summarizer_config.mode == "self_compact_sliding_window":@@ -285,6 +290,7 @@ async def compact_messages(timezone=timezone,agent_tags=agent_tags,tools=tools,+ billing_context=billing_context,)except ContextWindowExceededError:raise@@ -306,6 +312,7 @@ async def compact_messages(agent_tags=agent_tags,run_id=run_id,step_id=step_id,+ billing_context=billing_context,)summarization_mode_used = "all"elif summarizer_config.mode == "all":@@ -318,6 +325,7 @@ async def compact_messages(agent_tags=agent_tags,run_id=run_id,step_id=step_id,+ billing_context=billing_context,)elif summarizer_config.mode == "sliding_window":try:@@ -331,6 +339,7 @@ async def compact_messages(agent_tags=agent_tags,run_id=run_id,step_id=step_id,+ billing_context=billing_context,)except ContextWindowExceededError:# If sliding window failed because the transcript was too large for@@ -353,6 +362,7 @@ async def compact_messages(agent_tags=agent_tags,run_id=run_id,step_id=step_id,+ billing_context=billing_context,)summarization_mode_used = "all"else:@@ -387,6 +397,7 @@ async def compact_messages(agent_tags=agent_tags,run_id=run_id,step_id=step_id,+ billing_context=billing_context,)summarization_mode_used = "all"diff --git a/letta/services/summarizer/self_summarizer.py b/letta/services/summarizer/self_summarizer.pyindex a74ab9000..b411af2d4 100644--- a/letta/services/summarizer/self_summarizer.py+++ b/letta/services/summarizer/self_summarizer.py@@ -10,6 +10,7 @@ from letta.schemas.enums import MessageRole, ProviderTypefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message+from letta.schemas.provider_trace import BillingContextfrom letta.schemas.user import Userfrom letta.services.summarizer.constants import SUMMARY_TRUNCATION_SUFFIXfrom letta.services.summarizer.summarizer_config import CompactionSettings, get_default_prompt_for_mode@@ -35,6 +36,7 @@ async def self_summarize_all(agent_tags: Optional[List[str]] = None,# For cache compatibility with regular agent requeststools: Optional[List[dict]] = None,+ billing_context: Optional[BillingContext] = None,) -> Tuple[str, List[Message], str]:"""Summary request is added as a user message, then the agent's LLM is called with the messages + request.The agent's summary response is parsed and returned.@@ -91,6 +93,7 @@ async def self_summarize_all(user_id=actor.id if actor.id else None,compaction_settings=compaction_settings.model_dump() if compaction_settings else None,actor=actor,+ billing_context=billing_context,)# Build request data using agent's llm_client@@ -163,6 +166,7 @@ async def self_summarize_sliding_window(agent_tags: Optional[List[str]] = None,# For cache compatibility with regular agent requeststools: Optional[List[dict]] = None,+ billing_context: Optional[BillingContext] = None,) -> Tuple[Message, List[Message], str]:"""Summary request is added as a user message, then the agent's LLM is called with the messages + request.The agent's summary response is parsed and returned.@@ -251,6 +255,7 @@ async def self_summarize_sliding_window(step_id=step_id,agent_tags=agent_tags,tools=tools,+ billing_context=billing_context,)# final_messages should just be the system promptdiff --git a/letta/services/summarizer/summarizer.py b/letta/services/summarizer/summarizer.pyindex 4a7a253c9..d11806e42 100644--- a/letta/services/summarizer/summarizer.py+++ b/letta/services/summarizer/summarizer.py@@ -21,6 +21,7 @@ from letta.schemas.enums import AgentType, LLMCallType, MessageRole, ProviderTypfrom letta.schemas.letta_message_content import ImageContent, TextContentfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message, MessageCreate+from letta.schemas.provider_trace import BillingContextfrom letta.schemas.user import Userfrom letta.services.agent_manager import AgentManagerfrom letta.services.message_manager import MessageManager@@ -496,6 +497,7 @@ async def simple_summary(run_id: str | None = None,step_id: str | None = None,compaction_settings: dict | None = None,+ billing_context: BillingContext | None = None,) -> str:"""Generate a simple summary from a list of messages.@@ -524,6 +526,7 @@ async def simple_summary(org_id=actor.organization_id if actor else None,user_id=actor.id if actor else None,compaction_settings=compaction_settings,+ billing_context=billing_context,)# Prepare the messages payload to send to the LLMdiff --git a/letta/services/summarizer/summarizer_all.py b/letta/services/summarizer/summarizer_all.pyindex d95d3ca9e..3e0ee713f 100644--- a/letta/services/summarizer/summarizer_all.py+++ b/letta/services/summarizer/summarizer_all.py@@ -4,6 +4,7 @@ from letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message, MessageRole+from letta.schemas.provider_trace import BillingContextfrom letta.schemas.user import Userfrom letta.services.summarizer.constants import SUMMARY_TRUNCATION_SUFFIXfrom letta.services.summarizer.summarizer import simple_summary@@ -26,6 +27,7 @@ async def summarize_all(agent_tags: Optional[List[str]] = None,run_id: Optional[str] = None,step_id: Optional[str] = None,+ billing_context: Optional[BillingContext] = None,) -> str:"""Summarize the entire conversation history into a single summary.@@ -73,6 +75,7 @@ async def summarize_all("mode": "summarize_all","clip_chars": summarizer_config.clip_chars,},+ billing_context=billing_context,)logger.info(f"Summarized {len(messages_to_summarize)} messages")diff --git a/letta/services/summarizer/summarizer_sliding_window.py b/letta/services/summarizer/summarizer_sliding_window.pyindex 9fd3cc7b6..a04f70cb4 100644--- a/letta/services/summarizer/summarizer_sliding_window.py+++ b/letta/services/summarizer/summarizer_sliding_window.py@@ -8,6 +8,7 @@ from letta.otel.tracing import trace_methodfrom letta.schemas.enums import MessageRolefrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message+from letta.schemas.provider_trace import BillingContextfrom letta.schemas.user import Userfrom letta.services.context_window_calculator.token_counter import create_token_counterfrom letta.services.summarizer.constants import SUMMARY_TRUNCATION_SUFFIX@@ -109,6 +110,7 @@ async def summarize_via_sliding_window(agent_tags: Optional[List[str]] = None,run_id: Optional[str] = None,step_id: Optional[str] = None,+ billing_context: Optional[BillingContext] = None,) -> Tuple[str, List[Message]]:"""If the total tokens is greater than the context window limit (or force=True),@@ -217,6 +219,7 @@ async def summarize_via_sliding_window("sliding_window_percentage": summarizer_config.sliding_window_percentage,"clip_chars": summarizer_config.clip_chars,},+ billing_context=billing_context,)logger.info(f"\n==================\nSummary message string: {summary_message_str[:100]}...\n==================\n")