Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: thread billing_context through summarization pipeline (#10089)

cthomasbe91c2f94c46
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.py
index 1e85ba076..9b9babffb 100644
--- a/letta/agents/letta_agent_v3.py
+++ b/letta/agents/letta_agent_v3.py
@@ -344,6 +344,7 @@ class LettaAgentV3(LettaAgentV2):
include_return_message_types=include_return_message_types,
request_start_timestamp_ns=request_start_timestamp_ns,
include_compaction_messages=include_compaction_messages,
+ billing_context=billing_context,
)
input_messages_to_persist = [] # clear after first step
@@ -582,6 +583,7 @@ class LettaAgentV3(LettaAgentV2):
include_return_message_types=include_return_message_types,
request_start_timestamp_ns=request_start_timestamp_ns,
include_compaction_messages=include_compaction_messages,
+ billing_context=billing_context,
)
input_messages_to_persist = [] # clear after first step
async for chunk in response:
@@ -897,6 +899,7 @@ class LettaAgentV3(LettaAgentV2):
dry_run: bool = False,
enforce_run_id_set: bool = True,
include_compaction_messages: bool = False,
+ billing_context: Optional["BillingContext"] = None,
) -> AsyncGenerator[LettaMessage | dict, None]:
"""
Execute a single agent step (one LLM call and tool execution).
@@ -1221,6 +1224,7 @@ class LettaAgentV3(LettaAgentV2):
trigger="context_window_exceeded",
context_tokens_before=context_tokens_before,
messages_count_before=messages_count_before,
+ billing_context=billing_context,
)
# Recompile the persisted system prompt after compaction so subsequent
@@ -1443,6 +1447,7 @@ class LettaAgentV3(LettaAgentV2):
trigger="post_step_context_check",
context_tokens_before=context_tokens_before,
messages_count_before=messages_count_before,
+ billing_context=billing_context,
)
# Recompile the persisted system prompt after compaction so subsequent
@@ -2057,6 +2062,7 @@ class LettaAgentV3(LettaAgentV2):
trigger: Optional[str] = None,
context_tokens_before: Optional[int] = None,
messages_count_before: Optional[int] = None,
+ billing_context: Optional["BillingContext"] = None,
) -> tuple[Message, list[Message], str]:
"""Compact the current in-context messages for this agent.
@@ -2096,6 +2102,7 @@ class LettaAgentV3(LettaAgentV2):
trigger=trigger,
context_tokens_before=context_tokens_before,
messages_count_before=messages_count_before,
+ billing_context=billing_context,
)
# Update the agent's context token estimate
diff --git a/letta/services/summarizer/compact.py b/letta/services/summarizer/compact.py
index 95d54d4f1..606d16f63 100644
--- a/letta/services/summarizer/compact.py
+++ b/letta/services/summarizer/compact.py
@@ -13,6 +13,7 @@ from letta.schemas.enums import MessageRole
from letta.schemas.letta_message_content import TextContent
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message, MessageCreate
+from letta.schemas.provider_trace import BillingContext
from letta.schemas.user import User
from letta.services.summarizer.self_summarizer import self_summarize_all, self_summarize_sliding_window
from letta.services.summarizer.summarizer_all import summarize_all
@@ -171,6 +172,7 @@ async def compact_messages(
trigger: Optional[str] = None,
context_tokens_before: Optional[int] = None,
messages_count_before: Optional[int] = None,
+ billing_context: Optional[BillingContext] = None,
) -> CompactResult:
"""Compact in-context messages using summarization.
@@ -224,6 +226,7 @@ async def compact_messages(
timezone=timezone,
agent_tags=agent_tags,
tools=tools,
+ billing_context=billing_context,
)
except Exception as e:
logger.warning(f"Self summarization failed with exception: {str(e)}. Falling back to self sliding window mode.")
@@ -248,6 +251,7 @@ async def compact_messages(
timezone=timezone,
agent_tags=agent_tags,
tools=tools,
+ billing_context=billing_context,
)
summarization_mode_used = "self_compact_sliding_window"
except Exception as e:
@@ -267,6 +271,7 @@ async def compact_messages(
agent_tags=agent_tags,
run_id=run_id,
step_id=step_id,
+ billing_context=billing_context,
)
summarization_mode_used = "all"
elif summarizer_config.mode == "self_compact_sliding_window":
@@ -285,6 +290,7 @@ async def compact_messages(
timezone=timezone,
agent_tags=agent_tags,
tools=tools,
+ billing_context=billing_context,
)
except ContextWindowExceededError:
raise
@@ -306,6 +312,7 @@ async def compact_messages(
agent_tags=agent_tags,
run_id=run_id,
step_id=step_id,
+ billing_context=billing_context,
)
summarization_mode_used = "all"
elif summarizer_config.mode == "all":
@@ -318,6 +325,7 @@ async def compact_messages(
agent_tags=agent_tags,
run_id=run_id,
step_id=step_id,
+ billing_context=billing_context,
)
elif summarizer_config.mode == "sliding_window":
try:
@@ -331,6 +339,7 @@ async def compact_messages(
agent_tags=agent_tags,
run_id=run_id,
step_id=step_id,
+ billing_context=billing_context,
)
except ContextWindowExceededError:
# If sliding window failed because the transcript was too large for
@@ -353,6 +362,7 @@ async def compact_messages(
agent_tags=agent_tags,
run_id=run_id,
step_id=step_id,
+ billing_context=billing_context,
)
summarization_mode_used = "all"
else:
@@ -387,6 +397,7 @@ async def compact_messages(
agent_tags=agent_tags,
run_id=run_id,
step_id=step_id,
+ billing_context=billing_context,
)
summarization_mode_used = "all"
diff --git a/letta/services/summarizer/self_summarizer.py b/letta/services/summarizer/self_summarizer.py
index a74ab9000..b411af2d4 100644
--- a/letta/services/summarizer/self_summarizer.py
+++ b/letta/services/summarizer/self_summarizer.py
@@ -10,6 +10,7 @@ from letta.schemas.enums import MessageRole, ProviderType
from letta.schemas.letta_message_content import TextContent
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message
+from letta.schemas.provider_trace import BillingContext
from letta.schemas.user import User
from letta.services.summarizer.constants import SUMMARY_TRUNCATION_SUFFIX
from letta.services.summarizer.summarizer_config import CompactionSettings, get_default_prompt_for_mode
@@ -35,6 +36,7 @@ async def self_summarize_all(
agent_tags: Optional[List[str]] = None,
# For cache compatibility with regular agent requests
tools: Optional[List[dict]] = None,
+ billing_context: Optional[BillingContext] = None,
) -> Tuple[str, List[Message], str]:
"""Summary request is added as a user message, then the agent's LLM is called with the messages + request.
The agent's summary response is parsed and returned.
@@ -91,6 +93,7 @@ async def self_summarize_all(
user_id=actor.id if actor.id else None,
compaction_settings=compaction_settings.model_dump() if compaction_settings else None,
actor=actor,
+ billing_context=billing_context,
)
# Build request data using agent's llm_client
@@ -163,6 +166,7 @@ async def self_summarize_sliding_window(
agent_tags: Optional[List[str]] = None,
# For cache compatibility with regular agent requests
tools: Optional[List[dict]] = None,
+ billing_context: Optional[BillingContext] = None,
) -> Tuple[Message, List[Message], str]:
"""Summary request is added as a user message, then the agent's LLM is called with the messages + request.
The agent's summary response is parsed and returned.
@@ -251,6 +255,7 @@ async def self_summarize_sliding_window(
step_id=step_id,
agent_tags=agent_tags,
tools=tools,
+ billing_context=billing_context,
)
# final_messages should just be the system prompt
diff --git a/letta/services/summarizer/summarizer.py b/letta/services/summarizer/summarizer.py
index 4a7a253c9..d11806e42 100644
--- a/letta/services/summarizer/summarizer.py
+++ b/letta/services/summarizer/summarizer.py
@@ -21,6 +21,7 @@ from letta.schemas.enums import AgentType, LLMCallType, MessageRole, ProviderTyp
from letta.schemas.letta_message_content import ImageContent, TextContent
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message, MessageCreate
+from letta.schemas.provider_trace import BillingContext
from letta.schemas.user import User
from letta.services.agent_manager import AgentManager
from letta.services.message_manager import MessageManager
@@ -496,6 +497,7 @@ async def simple_summary(
run_id: str | None = None,
step_id: str | None = None,
compaction_settings: dict | None = None,
+ billing_context: BillingContext | None = None,
) -> str:
"""Generate a simple summary from a list of messages.
@@ -524,6 +526,7 @@ async def simple_summary(
org_id=actor.organization_id if actor else None,
user_id=actor.id if actor else None,
compaction_settings=compaction_settings,
+ billing_context=billing_context,
)
# Prepare the messages payload to send to the LLM
diff --git a/letta/services/summarizer/summarizer_all.py b/letta/services/summarizer/summarizer_all.py
index d95d3ca9e..3e0ee713f 100644
--- a/letta/services/summarizer/summarizer_all.py
+++ b/letta/services/summarizer/summarizer_all.py
@@ -4,6 +4,7 @@ from letta.log import get_logger
from letta.otel.tracing import trace_method
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message, MessageRole
+from letta.schemas.provider_trace import BillingContext
from letta.schemas.user import User
from letta.services.summarizer.constants import SUMMARY_TRUNCATION_SUFFIX
from letta.services.summarizer.summarizer import simple_summary
@@ -26,6 +27,7 @@ async def summarize_all(
agent_tags: Optional[List[str]] = None,
run_id: Optional[str] = None,
step_id: Optional[str] = None,
+ billing_context: Optional[BillingContext] = None,
) -> str:
"""
Summarize the entire conversation history into a single summary.
@@ -73,6 +75,7 @@ async def summarize_all(
"mode": "summarize_all",
"clip_chars": summarizer_config.clip_chars,
},
+ billing_context=billing_context,
)
logger.info(f"Summarized {len(messages_to_summarize)} messages")
diff --git a/letta/services/summarizer/summarizer_sliding_window.py b/letta/services/summarizer/summarizer_sliding_window.py
index 9fd3cc7b6..a04f70cb4 100644
--- a/letta/services/summarizer/summarizer_sliding_window.py
+++ b/letta/services/summarizer/summarizer_sliding_window.py
@@ -8,6 +8,7 @@ from letta.otel.tracing import trace_method
from letta.schemas.enums import MessageRole
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message
+from letta.schemas.provider_trace import BillingContext
from letta.schemas.user import User
from letta.services.context_window_calculator.token_counter import create_token_counter
from letta.services.summarizer.constants import SUMMARY_TRUNCATION_SUFFIX
@@ -109,6 +110,7 @@ async def summarize_via_sliding_window(
agent_tags: Optional[List[str]] = None,
run_id: Optional[str] = None,
step_id: Optional[str] = None,
+ billing_context: Optional[BillingContext] = None,
) -> Tuple[str, List[Message]]:
"""
If the total tokens is greater than the context window limit (or force=True),
@@ -217,6 +219,7 @@ async def summarize_via_sliding_window(
"sliding_window_percentage": summarizer_config.sliding_window_percentage,
"clip_chars": summarizer_config.clip_chars,
},
+ billing_context=billing_context,
)
logger.info(f"\n==================\nSummary message string: {summary_message_str[:100]}...\n==================\n")