Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add provider trace persistence in llm adapters (#4442)
cthomasf2485daef791
diff --git a/letta/adapters/letta_llm_adapter.py b/letta/adapters/letta_llm_adapter.pyindex 8bd4c5b2e..022eb9fd7 100644--- a/letta/adapters/letta_llm_adapter.py+++ b/letta/adapters/letta_llm_adapter.py@@ -7,6 +7,8 @@ from letta.schemas.letta_message_content import ReasoningContent, RedactedReasonfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.openai.chat_completion_response import ChatCompletionResponse, ToolCallfrom letta.schemas.usage import LettaUsageStatistics+from letta.schemas.user import User+from letta.services.telemetry_manager import TelemetryManagerclass LettaLLMAdapter(ABC):@@ -18,7 +20,7 @@ class LettaLLMAdapter(ABC):through a consistent API."""- def __init__(self, llm_client: LLMClientBase, llm_config: LLMConfig):+ def __init__(self, llm_client: LLMClientBase, llm_config: LLMConfig) -> None:self.llm_client: LLMClientBase = llm_clientself.llm_config: LLMConfig = llm_configself.message_id: str | None = None@@ -28,6 +30,7 @@ class LettaLLMAdapter(ABC):self.reasoning_content: list[TextContent | ReasoningContent | RedactedReasoningContent] | None = Noneself.tool_call: ToolCall | None = Noneself.usage: LettaUsageStatistics = LettaUsageStatistics()+ self.telemetry_manager: TelemetryManager = TelemetryManager()@abstractmethodasync def invoke_llm(@@ -36,7 +39,9 @@ class LettaLLMAdapter(ABC):messages: list,tools: list,use_assistant_message: bool,- ) -> AsyncGenerator[LettaMessage, None]:+ step_id: str | None = None,+ actor: User | None = None,+ ) -> AsyncGenerator[LettaMessage | None, None]:"""Execute the LLM call and yield results as they become available.@@ -45,6 +50,8 @@ class LettaLLMAdapter(ABC):messages: The messages in context for the requesttools: The tools available for the LLM to useuse_assistant_message: If true, use assistant messages when streaming response+ step_id: The step ID associated with this request. If provided, logs request and response data.+ actor: The optional actor associated with this request for logging purposes.Yields:LettaMessage: Chunks of data for streaming adapters, or None for blocking adapters@@ -52,4 +59,20 @@ class LettaLLMAdapter(ABC):raise NotImplementedErrordef supports_token_streaming(self) -> bool:+ """+ Check if the adapter supports token-level streaming.++ Returns:+ bool: True if the adapter can stream back tokens as they are generated, False otherwise+ """return False++ def log_provider_trace(self, step_id: str | None, actor: User | None) -> None:+ """+ Log provider trace data for telemetry purposes.++ Args:+ step_id: The step ID associated with this request for logging purposes+ actor: The user associated with this request for logging purposes+ """+ raise NotImplementedErrordiff --git a/letta/adapters/letta_llm_request_adapter.py b/letta/adapters/letta_llm_request_adapter.pyindex b34dac393..3fd0803ae 100644--- a/letta/adapters/letta_llm_request_adapter.py+++ b/letta/adapters/letta_llm_request_adapter.py@@ -1,8 +1,12 @@+import asynciofrom typing import AsyncGeneratorfrom letta.adapters.letta_llm_adapter import LettaLLMAdapterfrom letta.schemas.letta_message import LettaMessagefrom letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, TextContent+from letta.schemas.provider_trace import ProviderTraceCreate+from letta.schemas.user import User+from letta.settings import settingsclass LettaLLMRequestAdapter(LettaLLMAdapter):@@ -20,7 +24,9 @@ class LettaLLMRequestAdapter(LettaLLMAdapter):messages: list,tools: list,use_assistant_message: bool,- ) -> AsyncGenerator[LettaMessage, None]:+ step_id: str | None = None,+ actor: str | None = None,+ ) -> AsyncGenerator[LettaMessage | None, None]:"""Execute a blocking LLM request and yield the response.@@ -70,5 +76,33 @@ class LettaLLMRequestAdapter(LettaLLMAdapter):self.usage.prompt_tokens = self.chat_completions_response.usage.prompt_tokensself.usage.total_tokens = self.chat_completions_response.usage.total_tokens+ self.log_provider_trace(step_id=step_id, actor=actor)+yield Nonereturn++ def log_provider_trace(self, step_id: str | None, actor: User | None) -> None:+ """+ Log provider trace data for telemetry purposes in a fire-and-forget manner.++ Creates an async task to log the request/response data without blocking+ the main execution flow. The task runs in the background.++ Args:+ step_id: The step ID associated with this request for logging purposes+ actor: The user associated with this request for logging purposes+ """+ if step_id is None or actor is None or not settings.track_provider_trace:+ return++ asyncio.create_task(+ self.telemetry_manager.create_provider_trace_async(+ actor=actor,+ provider_trace_create=ProviderTraceCreate(+ request_json=self.request_data,+ response_json=self.response_data,+ step_id=step_id, # Use original step_id for telemetry+ organization_id=actor.organization_id,+ ),+ )+ )diff --git a/letta/adapters/letta_llm_stream_adapter.py b/letta/adapters/letta_llm_stream_adapter.pyindex 69db429ce..d7323aa68 100644--- a/letta/adapters/letta_llm_stream_adapter.py+++ b/letta/adapters/letta_llm_stream_adapter.py@@ -1,3 +1,4 @@+import asynciofrom typing import AsyncGeneratorfrom letta.adapters.letta_llm_adapter import LettaLLMAdapter@@ -7,7 +8,10 @@ from letta.llm_api.llm_client_base import LLMClientBasefrom letta.schemas.enums import ProviderTypefrom letta.schemas.letta_message import LettaMessagefrom letta.schemas.llm_config import LLMConfig+from letta.schemas.provider_trace import ProviderTraceCreatefrom letta.schemas.usage import LettaUsageStatistics+from letta.schemas.user import User+from letta.settings import settingsclass LettaLLMStreamAdapter(LettaLLMAdapter):@@ -20,7 +24,7 @@ class LettaLLMStreamAdapter(LettaLLMAdapter):specific streaming formats."""- def __init__(self, llm_client: LLMClientBase, llm_config: LLMConfig):+ def __init__(self, llm_client: LLMClientBase, llm_config: LLMConfig) -> None:super().__init__(llm_client, llm_config)self.interface: OpenAIStreamingInterface | AnthropicStreamingInterface | None = None@@ -30,6 +34,8 @@ class LettaLLMStreamAdapter(LettaLLMAdapter):messages: list,tools: list,use_assistant_message: bool,+ step_id: str | None = None,+ actor: User | None = None,) -> AsyncGenerator[LettaMessage, None]:"""Execute a streaming LLM request and yield tokens/chunks as they arrive.@@ -109,5 +115,50 @@ class LettaLLMStreamAdapter(LettaLLMAdapter):# Store any additional data from the interfaceself.message_id = self.interface.letta_message_id+ # Log request and response data+ self.log_provider_trace(step_id=step_id, actor=actor)+def supports_token_streaming(self) -> bool:return True++ def log_provider_trace(self, step_id: str | None, actor: User | None) -> None:+ """+ Log provider trace data for telemetry purposes in a fire-and-forget manner.++ Creates an async task to log the request/response data without blocking+ the main execution flow. For streaming adapters, this includes the final+ tool call and reasoning content collected during streaming.++ Args:+ step_id: The step ID associated with this request for logging purposes+ actor: The user associated with this request for logging purposes+ """+ if step_id is None or actor is None or not settings.track_provider_trace:+ return++ asyncio.create_task(+ self.telemetry_manager.create_provider_trace_async(+ actor=actor,+ provider_trace_create=ProviderTraceCreate(+ request_json=self.request_data,+ response_json={+ "content": {+ "tool_call": self.tool_call.model_dump_json(),+ "reasoning": [content.model_dump_json() for content in self.reasoning_content],+ },+ "id": self.interface.message_id,+ "model": self.interface.model,+ "role": "assistant",+ # "stop_reason": "",+ # "stop_sequence": None,+ "type": "message",+ "usage": {+ "input_tokens": self.usage.prompt_tokens,+ "output_tokens": self.usage.completion_tokens,+ },+ },+ step_id=step_id, # Use original step_id for telemetry+ organization_id=actor.organization_id,+ ),+ )+ )diff --git a/letta/agents/letta_agent_v2.py b/letta/agents/letta_agent_v2.pyindex 180eb790d..3ffcfd222 100644--- a/letta/agents/letta_agent_v2.py+++ b/letta/agents/letta_agent_v2.py@@ -419,6 +419,8 @@ class LettaAgentV2(BaseAgentV2):messages=messages,tools=valid_tools,use_assistant_message=use_assistant_message,+ step_id=step_id,+ actor=self.actor,)async for chunk in invocation:if llm_adapter.supports_token_streaming():