Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: otel metrics and expanded collecting (#2647)
Andy Li80f6e97ca91b
diff --git a/letta/agent.py b/letta/agent.pyindex 6571f736a..f81e0f694 100644--- a/letta/agent.py+++ b/letta/agent.py@@ -41,6 +41,7 @@ from letta.log import get_loggerfrom letta.memory import summarize_messagesfrom letta.orm import Userfrom letta.orm.enums import ToolType+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.agent import AgentState, AgentStepResponse, UpdateAgent, get_prompt_template_for_agent_typefrom letta.schemas.block import BlockUpdatefrom letta.schemas.embedding_config import EmbeddingConfig@@ -72,7 +73,6 @@ from letta.services.tool_manager import ToolManagerfrom letta.settings import settings, summarizer_settingsfrom letta.streaming_interface import StreamingRefreshCLIInterfacefrom letta.system import get_heartbeat, get_token_limit_warning, package_function_response, package_summarize_message, package_user_message-from letta.tracing import log_event, trace_methodfrom letta.utils import count_tokens, get_friendly_error_msg, get_tool_call_id, log_telemetry, parse_json, validate_function_responselogger = get_logger(__name__)diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.pyindex 59acf6545..b259373f2 100644--- a/letta/agents/letta_agent.py+++ b/letta/agents/letta_agent.py@@ -16,7 +16,7 @@ from letta.agents.helpers import ()from letta.errors import ContextWindowExceededErrorfrom letta.helpers import ToolRulesSolver-from letta.helpers.datetime_helpers import get_utc_timestamp_ns+from letta.helpers.datetime_helpers import AsyncTimer, get_utc_timestamp_ns, ns_to_msfrom letta.helpers.tool_execution_helper import enable_strict_modefrom letta.interfaces.anthropic_streaming_interface import AnthropicStreamingInterfacefrom letta.interfaces.openai_streaming_interface import OpenAIStreamingInterface@@ -25,6 +25,9 @@ from letta.llm_api.llm_client_base import LLMClientBasefrom letta.local_llm.constants import INNER_THOUGHTS_KWARGfrom letta.log import get_loggerfrom letta.orm.enums import ToolType+from letta.otel.context import get_ctx_attributes+from letta.otel.metric_registry import MetricRegistry+from letta.otel.tracing import log_event, trace_method, tracerfrom letta.schemas.agent import AgentStatefrom letta.schemas.enums import MessageRole, MessageStreamStatusfrom letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, RedactedReasoningContent, TextContent@@ -48,7 +51,7 @@ from letta.services.telemetry_manager import NoopTelemetryManager, TelemetryManafrom letta.services.tool_executor.tool_execution_manager import ToolExecutionManagerfrom letta.settings import model_settingsfrom letta.system import package_function_response-from letta.tracing import log_event, trace_method, tracer+from letta.types import JsonDictfrom letta.utils import log_telemetry, validate_function_responselogger = get_logger(__name__)@@ -178,7 +181,7 @@ class LettaAgent(BaseAgent):# log llm request timenow = get_utc_timestamp_ns()llm_request_ns = now - step_start- agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": llm_request_ns // 1_000_000})+ agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": ns_to_ms(llm_request_ns)})response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)@@ -210,7 +213,7 @@ class LettaAgent(BaseAgent):# log LLM request timenow = get_utc_timestamp_ns()llm_request_ns = now - step_start- agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": llm_request_ns // 1_000_000})+ agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": ns_to_ms(llm_request_ns)})persisted_messages, should_continue = await self._handle_ai_response(tool_call,@@ -227,7 +230,7 @@ class LettaAgent(BaseAgent):# log step timenow = get_utc_timestamp_ns()step_ns = now - step_start- agent_step_span.add_event(name="step_ms", attributes={"duration_ms": step_ns // 1_000_000})+ agent_step_span.add_event(name="step_ms", attributes={"duration_ms": ns_to_ms(step_ns)})agent_step_span.end()# Log LLM Trace@@ -267,7 +270,7 @@ class LettaAgent(BaseAgent):if request_start_timestamp_ns:now = get_utc_timestamp_ns()request_ns = now - request_start_timestamp_ns- request_span.add_event(name="letta_request_ms", attributes={"duration_ms": request_ns // 1_000_000})+ request_span.add_event(name="letta_request_ms", attributes={"duration_ms": ns_to_ms(request_ns)})request_span.end()# Return back usage@@ -321,7 +324,7 @@ class LettaAgent(BaseAgent):# log LLM request timenow = get_utc_timestamp_ns()llm_request_ns = now - step_start- agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": llm_request_ns // 1_000_000})+ agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": ns_to_ms(llm_request_ns)})# TODO: add run_idusage.step_count += 1@@ -363,7 +366,7 @@ class LettaAgent(BaseAgent):# log step timenow = get_utc_timestamp_ns()step_ns = now - step_start- agent_step_span.add_event(name="step_ms", attributes={"duration_ms": step_ns // 1_000_000})+ agent_step_span.add_event(name="step_ms", attributes={"duration_ms": ns_to_ms(step_ns)})agent_step_span.end()# Log LLM Trace@@ -384,7 +387,7 @@ class LettaAgent(BaseAgent):if request_start_timestamp_ns:now = get_utc_timestamp_ns()request_ns = now - request_start_timestamp_ns- request_span.add_event(name="request_ms", attributes={"duration_ms": request_ns // 1_000_000})+ request_span.add_event(name="request_ms", attributes={"duration_ms": ns_to_ms(request_ns)})request_span.end()# Extend the in context message ids@@ -480,7 +483,7 @@ class LettaAgent(BaseAgent):if first_chunk and request_span is not None:now = get_utc_timestamp_ns()ttft_ns = now - request_start_timestamp_ns- request_span.add_event(name="time_to_first_token_ms", attributes={"ttft_ms": ttft_ns // 1_000_000})+ request_span.add_event(name="time_to_first_token_ms", attributes={"ttft_ms": ns_to_ms(ttft_ns)})first_chunk = Falseyield f"data: {chunk.model_dump_json()}\n\n"@@ -490,6 +493,9 @@ class LettaAgent(BaseAgent):usage.completion_tokens += interface.output_tokensusage.prompt_tokens += interface.input_tokensusage.total_tokens += interface.input_tokens + interface.output_tokens+ MetricRegistry().message_output_tokens.record(+ interface.output_tokens, dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model})+ )# Persist input messages if not already# Special strategy to lower TTFT@@ -500,7 +506,7 @@ class LettaAgent(BaseAgent):# log LLM request timenow = get_utc_timestamp_ns()llm_request_ns = now - step_start- agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": llm_request_ns // 1_000_000})+ agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": ns_to_ms(llm_request_ns)})# Process resulting stream contenttool_call = interface.get_tool_call_object()@@ -525,7 +531,7 @@ class LettaAgent(BaseAgent):# log total step timenow = get_utc_timestamp_ns()step_ns = now - step_start- agent_step_span.add_event(name="step_ms", attributes={"duration_ms": step_ns // 1_000_000})+ agent_step_span.add_event(name="step_ms", attributes={"duration_ms": ns_to_ms(step_ns)})agent_step_span.end()# TODO (cliandy): the stream POST request span has ended at this point, we should tie this to the stream@@ -576,7 +582,7 @@ class LettaAgent(BaseAgent):if request_start_timestamp_ns:now = get_utc_timestamp_ns()request_ns = now - request_start_timestamp_ns- request_span.add_event(name="letta_request_ms", attributes={"duration_ms": request_ns // 1_000_000})+ request_span.add_event(name="letta_request_ms", attributes={"duration_ms": ns_to_ms(request_ns)})request_span.end()# TODO: Also yield out a letta usage stats SSE@@ -603,10 +609,16 @@ class LettaAgent(BaseAgent):)log_event("agent.stream_no_tokens.llm_request.created")+ async with AsyncTimer() as timer:+ response = await llm_client.request_async(request_data, agent_state.llm_config)+ MetricRegistry().llm_execution_time_ms_histogram.record(+ timer.elapsed_ms,+ dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model}),+ )# Attempt LLM requestreturn (request_data,- await llm_client.request_async(request_data, agent_state.llm_config),+ response,current_in_context_messages,new_in_context_messages,)@@ -653,9 +665,7 @@ class LettaAgent(BaseAgent):if first_chunk and ttft_span is not None:provider_request_start_timestamp_ns = get_utc_timestamp_ns()provider_req_start_ns = provider_request_start_timestamp_ns - request_start_timestamp_ns- ttft_span.add_event(- name="provider_req_start_ns", attributes={"provider_req_start_ms": provider_req_start_ns // 1_000_000}- )+ ttft_span.add_event(name="provider_req_start_ns", attributes={"provider_req_start_ms": ns_to_ms(provider_req_start_ns)})# Attempt LLM requestreturn (@@ -861,6 +871,7 @@ class LettaAgent(BaseAgent):tool_args=tool_args,agent_state=agent_state,agent_step_span=agent_step_span,+ step_id=step_id,)log_telemetry(self.logger, "_handle_ai_response execute tool finish", tool_execution_result=tool_execution_result, tool_call_id=tool_call_id@@ -938,10 +949,15 @@ class LettaAgent(BaseAgent):@trace_methodasync def _execute_tool(- self, tool_name: str, tool_args: dict, agent_state: AgentState, agent_step_span: Optional["Span"] = None+ self,+ tool_name: str,+ tool_args: JsonDict,+ agent_state: AgentState,+ agent_step_span: Optional["Span"] = None,+ step_id: str | None = None,) -> "ToolExecutionResult":"""- Executes a tool and returns (result, success_flag).+ Executes a tool and returns the ToolExecutionResult."""from letta.schemas.tool_execution_result import ToolExecutionResult@@ -973,7 +989,10 @@ class LettaAgent(BaseAgent):# TODO: Integrate sandbox resultlog_event(name=f"start_{tool_name}_execution", attributes=tool_args)tool_execution_result = await tool_execution_manager.execute_tool_async(- function_name=tool_name, function_args=tool_args, tool=target_tool+ function_name=tool_name,+ function_args=tool_args,+ tool=target_tool,+ step_id=step_id,)if agent_step_span:end_time = get_utc_timestamp_ns()@@ -981,7 +1000,7 @@ class LettaAgent(BaseAgent):name="tool_execution_completed",attributes={"tool_name": target_tool.name,- "duration_ms": (end_time - start_time) // 1_000_000,+ "duration_ms": ns_to_ms((end_time - start_time)),"success": tool_execution_result.success_flag,"tool_type": target_tool.tool_type,"tool_id": target_tool.id,diff --git a/letta/agents/letta_agent_batch.py b/letta/agents/letta_agent_batch.pyindex dd1c71d07..f0810533d 100644--- a/letta/agents/letta_agent_batch.py+++ b/letta/agents/letta_agent_batch.py@@ -16,6 +16,7 @@ from letta.llm_api.llm_client import LLMClientfrom letta.local_llm.constants import INNER_THOUGHTS_KWARGfrom letta.log import get_loggerfrom letta.orm.enums import ToolType+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.agent import AgentState, AgentStepStatefrom letta.schemas.enums import AgentStepStatus, JobStatus, MessageStreamStatus, ProviderTypefrom letta.schemas.job import JobUpdate@@ -39,7 +40,6 @@ from letta.services.passage_manager import PassageManagerfrom letta.services.sandbox_config_manager import SandboxConfigManagerfrom letta.services.tool_executor.tool_execution_manager import ToolExecutionManagerfrom letta.settings import tool_settings-from letta.tracing import log_event, trace_methodlogger = get_logger(__name__)diff --git a/letta/agents/voice_sleeptime_agent.py b/letta/agents/voice_sleeptime_agent.pyindex f0e11d1e4..c8ebeb989 100644--- a/letta/agents/voice_sleeptime_agent.py+++ b/letta/agents/voice_sleeptime_agent.py@@ -3,6 +3,7 @@ from typing import AsyncGenerator, List, Optional, Tuple, Unionfrom letta.agents.helpers import _create_letta_response, serialize_message_historyfrom letta.agents.letta_agent import LettaAgentfrom letta.orm.enums import ToolType+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.block import BlockUpdatefrom letta.schemas.enums import MessageStreamStatus@@ -17,7 +18,7 @@ from letta.services.message_manager import MessageManagerfrom letta.services.passage_manager import PassageManagerfrom letta.services.summarizer.enums import SummarizationModefrom letta.services.summarizer.summarizer import Summarizer-from letta.tracing import trace_method+from letta.types import JsonDictclass VoiceSleeptimeAgent(LettaAgent):@@ -89,9 +90,16 @@ class VoiceSleeptimeAgent(LettaAgent):)@trace_method- async def _execute_tool(self, tool_name: str, tool_args: dict, agent_state: AgentState, agent_step_span: Optional["Span"] = None):+ async def _execute_tool(+ self,+ tool_name: str,+ tool_args: JsonDict,+ agent_state: AgentState,+ agent_step_span: Optional["Span"] = None,+ step_id: str | None = None,+ ) -> "ToolExecutionResult":"""- Executes a tool and returns (result, success_flag).+ Executes a tool and returns the ToolExecutionResult"""from letta.schemas.tool_execution_result import ToolExecutionResultdiff --git a/letta/groups/sleeptime_multi_agent_v2.py b/letta/groups/sleeptime_multi_agent_v2.pyindex 7d46ae43c..587a8ef89 100644--- a/letta/groups/sleeptime_multi_agent_v2.py+++ b/letta/groups/sleeptime_multi_agent_v2.py@@ -5,6 +5,7 @@ from typing import AsyncGenerator, List, Optionalfrom letta.agents.base_agent import BaseAgentfrom letta.agents.letta_agent import LettaAgentfrom letta.groups.helpers import stringify_message+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import JobStatusfrom letta.schemas.group import Group, ManagerTypefrom letta.schemas.job import JobUpdate@@ -21,7 +22,6 @@ from letta.services.message_manager import MessageManagerfrom letta.services.passage_manager import PassageManagerfrom letta.services.step_manager import NoopStepManager, StepManagerfrom letta.services.telemetry_manager import NoopTelemetryManager, TelemetryManager-from letta.tracing import trace_methodclass SleeptimeMultiAgentV2(BaseAgent):diff --git a/letta/helpers/datetime_helpers.py b/letta/helpers/datetime_helpers.pyindex 60d7d12a2..9c5cd825b 100644--- a/letta/helpers/datetime_helpers.py+++ b/letta/helpers/datetime_helpers.py@@ -1,7 +1,9 @@import reimport time-from datetime import datetime, timedelta, timezone+from datetime import datetime, timedelta+from datetime import timezone as dt_timezonefrom time import strftime+from typing import Callableimport pytz@@ -66,7 +68,7 @@ def get_local_time(timezone=None):def get_utc_time() -> datetime:"""Get the current UTC time"""# return datetime.now(pytz.utc)- return datetime.now(timezone.utc)+ return datetime.now(dt_timezone.utc)def get_utc_time_int() -> int:@@ -78,9 +80,13 @@ def get_utc_timestamp_ns() -> int:return int(time.time_ns())+def ns_to_ms(ns: int) -> int:+ return ns // 1_000_000++def timestamp_to_datetime(timestamp_seconds: int) -> datetime:"""Convert Unix timestamp in seconds to UTC datetime object"""- return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc)+ return datetime.fromtimestamp(timestamp_seconds, tz=dt_timezone.utc)def format_datetime(dt):@@ -105,3 +111,41 @@ def extract_date_from_timestamp(timestamp):def is_utc_datetime(dt: datetime) -> bool:return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) == timedelta(0)+++class AsyncTimer:+ """An async context manager for timing async code execution.++ Takes in an optional callback_func to call on exit with arguments+ taking in the elapsed_ms and exc if present.++ Do not use the start and end times outside of this function as they are relative.+ """++ def __init__(self, callback_func: Callable | None = None):+ self._start_time_ns = None+ self._end_time_ns = None+ self.elapsed_ns = None+ self.callback_func = callback_func++ async def __aenter__(self):+ self._start_time_ns = time.perf_counter_ns()+ return self++ async def __aexit__(self, exc_type, exc, tb):+ self._end_time_ns = time.perf_counter_ns()+ self.elapsed_ns = self._end_time_ns - self._start_time_ns+ if self.callback_func:+ from asyncio import iscoroutinefunction++ if iscoroutinefunction(self.callback_func):+ await self.callback_func(self.elapsed_ms, exc)+ else:+ self.callback_func(self.elapsed_ms, exc)+ return False++ @property+ def elapsed_ms(self):+ if self.elapsed_ns is not None:+ return ns_to_ms(self.elapsed_ns)+ return Nonediff --git a/letta/services/helpers/noop_helper.py b/letta/helpers/singleton.pysimilarity index 73%rename from letta/services/helpers/noop_helper.pyrename to letta/helpers/singleton.pyindex 7f32e628b..1d3824576 100644--- a/letta/services/helpers/noop_helper.py+++ b/letta/helpers/singleton.py@@ -1,7 +1,12 @@+# TODO (cliandy): consolidate with decorators later+from functools import wraps++def singleton(cls):"""Decorator to make a class a Singleton class."""instances = {}+ @wraps(cls)def get_instance(*args, **kwargs):if cls not in instances:instances[cls] = cls(*args, **kwargs)diff --git a/letta/interfaces/anthropic_streaming_interface.py b/letta/interfaces/anthropic_streaming_interface.pyindex 16622093a..65489166a 100644--- a/letta/interfaces/anthropic_streaming_interface.py+++ b/letta/interfaces/anthropic_streaming_interface.py@@ -23,7 +23,7 @@ from anthropic.types.beta import ()from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG-from letta.helpers.datetime_helpers import get_utc_timestamp_ns+from letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_msfrom letta.local_llm.constants import INNER_THOUGHTS_KWARGfrom letta.log import get_loggerfrom letta.schemas.letta_message import (@@ -131,7 +131,7 @@ class AnthropicStreamingInterface:now = get_utc_timestamp_ns()ttft_ns = now - provider_request_start_timestamp_nsttft_span.add_event(- name="anthropic_time_to_first_token_ms", attributes={"anthropic_time_to_first_token_ms": ttft_ns // 1_000_000}+ name="anthropic_time_to_first_token_ms", attributes={"anthropic_time_to_first_token_ms": ns_to_ms(ttft_ns)})first_chunk = Falsediff --git a/letta/interfaces/openai_streaming_interface.py b/letta/interfaces/openai_streaming_interface.pyindex 33ac60514..36c583c6e 100644--- a/letta/interfaces/openai_streaming_interface.py+++ b/letta/interfaces/openai_streaming_interface.py@@ -5,7 +5,7 @@ from openai import AsyncStreamfrom openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG-from letta.helpers.datetime_helpers import get_utc_timestamp_ns+from letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_msfrom letta.schemas.letta_message import AssistantMessage, LettaMessage, ReasoningMessage, ToolCallDelta, ToolCallMessagefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.message import Message@@ -85,7 +85,7 @@ class OpenAIStreamingInterface:now = get_utc_timestamp_ns()ttft_ns = now - provider_request_start_timestamp_nsttft_span.add_event(- name="openai_time_to_first_token_ms", attributes={"openai_time_to_first_token_ms": ttft_ns // 1_000_000}+ name="openai_time_to_first_token_ms", attributes={"openai_time_to_first_token_ms": ns_to_ms(ttft_ns)})first_chunk = Falsediff --git a/letta/llm_api/anthropic.py b/letta/llm_api/anthropic.pyindex fadc652d6..7d9cdf730 100644--- a/letta/llm_api/anthropic.py+++ b/letta/llm_api/anthropic.py@@ -26,6 +26,7 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functionsfrom letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTIONfrom letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messagesfrom letta.log import get_logger+from letta.otel.tracing import log_eventfrom letta.schemas.enums import ProviderCategoryfrom letta.schemas.message import Message as _Messagefrom letta.schemas.message import MessageRole as _MessageRole@@ -45,7 +46,6 @@ from letta.services.provider_manager import ProviderManagerfrom letta.services.user_manager import UserManagerfrom letta.settings import model_settingsfrom letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface-from letta.tracing import log_eventlogger = get_logger(__name__)diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.pyindex 3a60199a1..121955d73 100644--- a/letta/llm_api/anthropic_client.py+++ b/letta/llm_api/anthropic_client.py@@ -27,6 +27,7 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functions, unpack_all_infrom letta.llm_api.llm_client_base import LLMClientBasefrom letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTIONfrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import ProviderCategoryfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessage@@ -36,7 +37,6 @@ from letta.schemas.openai.chat_completion_response import Message as ChoiceMessafrom letta.schemas.openai.chat_completion_response import ToolCall, UsageStatisticsfrom letta.services.provider_manager import ProviderManagerfrom letta.settings import model_settings-from letta.tracing import trace_methodDUMMY_FIRST_USER_MESSAGE = "User initializing bootup sequence."diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.pyindex 60e800a6e..25b1e00fa 100644--- a/letta/llm_api/google_vertex_client.py+++ b/letta/llm_api/google_vertex_client.py@@ -12,12 +12,12 @@ from letta.llm_api.llm_client_base import LLMClientBasefrom letta.local_llm.json_parser import clean_json_string_extra_backslashfrom letta.local_llm.utils import count_tokensfrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.openai.chat_completion_request import Toolfrom letta.schemas.openai.chat_completion_response import ChatCompletionResponse, Choice, FunctionCall, Message, ToolCall, UsageStatisticsfrom letta.settings import model_settings, settings-from letta.tracing import trace_methodfrom letta.utils import get_tool_call_idlogger = get_logger(__name__)diff --git a/letta/llm_api/llm_api_tools.py b/letta/llm_api/llm_api_tools.pyindex a1af262f2..62d3bf3b9 100644--- a/letta/llm_api/llm_api_tools.py+++ b/letta/llm_api/llm_api_tools.py@@ -26,6 +26,7 @@ from letta.local_llm.chat_completion_proxy import get_chat_completionfrom letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTIONfrom letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messagesfrom letta.orm.user import User+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.enums import ProviderCategoryfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message@@ -35,7 +36,6 @@ from letta.schemas.provider_trace import ProviderTraceCreatefrom letta.services.telemetry_manager import TelemetryManagerfrom letta.settings import ModelSettingsfrom letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface-from letta.tracing import log_event, trace_methodLLM_API_PROVIDER_OPTIONS = ["openai", "azure", "anthropic", "google_ai", "cohere", "local", "groq", "deepseek"]diff --git a/letta/llm_api/llm_client_base.py b/letta/llm_api/llm_client_base.pyindex e88b60810..bd4054f46 100644--- a/letta/llm_api/llm_client_base.py+++ b/letta/llm_api/llm_client_base.py@@ -6,13 +6,13 @@ from openai import AsyncStream, Streamfrom openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta.errors import LLMError+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Messagefrom letta.schemas.openai.chat_completion_response import ChatCompletionResponsefrom letta.schemas.provider_trace import ProviderTraceCreatefrom letta.services.telemetry_manager import TelemetryManager-from letta.tracing import log_event, trace_methodif TYPE_CHECKING:from letta.orm import Userdiff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.pyindex fe3b77ccb..7372b1d0f 100644--- a/letta/llm_api/openai.py+++ b/letta/llm_api/openai.py@@ -19,6 +19,7 @@ from letta.llm_api.openai_client import (from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRSTfrom letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messagesfrom letta.log import get_logger+from letta.otel.tracing import log_eventfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as _Messagefrom letta.schemas.message import MessageRole as _MessageRole@@ -36,7 +37,6 @@ from letta.schemas.openai.chat_completion_response import ()from letta.schemas.openai.embedding_response import EmbeddingResponsefrom letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface-from letta.tracing import log_eventfrom letta.utils import get_tool_call_id, smart_urljoinlogger = get_logger(__name__)diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.pyindex 7be168352..737466e83 100644--- a/letta/llm_api/openai_client.py+++ b/letta/llm_api/openai_client.py@@ -23,6 +23,7 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_stfrom letta.llm_api.llm_client_base import LLMClientBasefrom letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRSTfrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import ProviderCategory, ProviderTypefrom letta.schemas.llm_config import LLMConfig@@ -34,7 +35,6 @@ from letta.schemas.openai.chat_completion_request import Tool as OpenAIToolfrom letta.schemas.openai.chat_completion_request import ToolFunctionChoice, cast_message_to_subtypefrom letta.schemas.openai.chat_completion_response import ChatCompletionResponsefrom letta.settings import model_settings-from letta.tracing import trace_methodlogger = get_logger(__name__)diff --git a/letta/local_llm/chat_completion_proxy.py b/letta/local_llm/chat_completion_proxy.pyindex 35db97ed8..214e04870 100644--- a/letta/local_llm/chat_completion_proxy.py+++ b/letta/local_llm/chat_completion_proxy.py@@ -20,9 +20,9 @@ from letta.local_llm.utils import count_tokens, get_available_wrappersfrom letta.local_llm.vllm.api import get_vllm_completionfrom letta.local_llm.webui.api import get_webui_completionfrom letta.local_llm.webui.legacy_api import get_webui_completion as get_webui_completion_legacy+from letta.otel.tracing import log_eventfrom letta.prompts.gpt_summarize import SYSTEM as SUMMARIZE_SYSTEM_MESSAGEfrom letta.schemas.openai.chat_completion_response import ChatCompletionResponse, Choice, Message, ToolCall, UsageStatistics-from letta.tracing import log_eventfrom letta.utils import get_tool_call_idhas_shown_warning = Falsediff --git a/letta/memory.py b/letta/memory.pyindex 818f45ca5..c6a03c143 100644--- a/letta/memory.py+++ b/letta/memory.py@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Callable, Dict, Listfrom letta.constants import MESSAGE_SUMMARY_REQUEST_ACKfrom letta.llm_api.llm_api_tools import createfrom letta.llm_api.llm_client import LLMClient+from letta.otel.tracing import trace_methodfrom letta.prompts.gpt_summarize import SYSTEM as SUMMARY_PROMPT_SYSTEMfrom letta.schemas.agent import AgentStatefrom letta.schemas.enums import MessageRole@@ -10,7 +11,6 @@ from letta.schemas.letta_message_content import TextContentfrom letta.schemas.memory import Memoryfrom letta.schemas.message import Messagefrom letta.settings import summarizer_settings-from letta.tracing import trace_methodfrom letta.utils import count_tokens, printdif TYPE_CHECKING:diff --git a/letta/otel/__init__.py b/letta/otel/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/letta/otel/context.py b/letta/otel/context.pynew file mode 100644index 000000000..4c83b98cf--- /dev/null+++ b/letta/otel/context.py@@ -0,0 +1,26 @@+from contextvars import ContextVar+from typing import Any, Dict++# Create context var at module level (outside middleware)+request_attributes: ContextVar[Dict[str, Any]] = ContextVar("request_attributes", default={})+++# Helper functions+def set_ctx_attributes(attrs: Dict[str, Any]):+ """Set attributes in current context"""+ current = request_attributes.get()+ new_attrs = {**current, **attrs}+ request_attributes.set(new_attrs)+++def add_ctx_attribute(key: str, value: Any):+ """Add single attribute to current context"""+ current = request_attributes.get()+ new_attrs = {**current, key: value}+ request_attributes.set(new_attrs)+++def get_ctx_attributes() -> Dict[str, Any]:+ """Get all attributes from current context"""+ print(request_attributes.get())+ return request_attributes.get()diff --git a/letta/otel/events.py b/letta/otel/events.pynew file mode 100644index 000000000..e69de29bbdiff --git a/letta/otel/metric_registry.py b/letta/otel/metric_registry.pynew file mode 100644index 000000000..60ece7134--- /dev/null+++ b/letta/otel/metric_registry.py@@ -0,0 +1,122 @@+from dataclasses import dataclass, field+from functools import partial++from opentelemetry import metrics+from opentelemetry.metrics import Counter, Histogram++from letta.helpers.singleton import singleton+from letta.otel.metrics import get_letta_meter+++@singleton+@dataclass(frozen=True)+class MetricRegistry:+ """Registry of all application metrics++ Metrics are composed of the following:+ - name+ - description+ - unit: UCUM unit of the metric (i.e. 'By' for bytes, 'ms' for milliseconds, '1' for count+ - bucket_bounds (list[float] | None): the explicit bucket bounds for histogram metrics++ and instruments are of types Counter, Histogram, and Gauge++ The relationship between the various models is as follows:+ project_id -N:1-> base_template_id -N:1-> template_id -N:1-> agent_id+ agent_id -1:1+-> model_name+ agent_id -1:N -> tool_name+ """++ Instrument = Counter | Histogram+ _metrics: dict[str, Instrument] = field(default_factory=dict, init=False)+ _meter: metrics.Meter = field(init=False)++ def __post_init__(self):+ object.__setattr__(self, "_meter", get_letta_meter())++ def _get_or_create_metric(self, name: str, factory):+ """Lazy initialization of metrics."""+ if name not in self._metrics:+ self._metrics[name] = factory()+ return self._metrics[name]++ # (includes base attributes: project, template_base, template, agent)+ @property+ def user_message_counter(self) -> Counter:+ return self._get_or_create_metric(+ "count_user_message",+ partial(+ self._meter.create_counter,+ name="count_user_message",+ description="Counts the number of messages sent by the user",+ unit="1",+ ),+ )++ # (includes tool_name, tool_execution_success, & step_id on failure)+ @property+ def tool_execution_counter(self) -> Counter:+ return self._get_or_create_metric(+ "count_tool_execution",+ partial(self._meter.create_counter, name="count_tool_execution", description="Counts the number of tools executed.", unit="1"),+ )++ # project_id + model+ @property+ def ttft_ms_histogram(self) -> Histogram:+ return self._get_or_create_metric(+ "hist_ttft_ms",+ partial(self._meter.create_histogram, name="hist_ttft_ms", description="Histogram for the Time to First Token (ms)", unit="ms"),+ )++ # (includes model name)+ @property+ def llm_execution_time_ms_histogram(self) -> Histogram:+ return self._get_or_create_metric(+ "hist_llm_execution_time_ms",+ partial(+ self._meter.create_histogram,+ name="hist_llm_execution_time_ms",+ description="Histogram for LLM execution time (ms)",+ unit="ms",+ ),+ )++ # (includes tool name)+ @property+ def tool_execution_time_ms_histogram(self) -> Histogram:+ return self._get_or_create_metric(+ "hist_tool_execution_time_ms",+ partial(+ self._meter.create_histogram,+ name="hist_tool_execution_time_ms",+ description="Histogram for tool execution time (ms)",+ unit="ms",+ ),+ )++ # TODO (cliandy): instrument this+ @property+ def message_cost(self) -> Histogram:+ return self._get_or_create_metric(+ "hist_message_cost_usd",+ partial(+ self._meter.create_histogram,+ name="hist_message_cost_usd",+ description="Histogram for cost of messages (usd) per step",+ unit="usd",+ ),+ )++ # (includes model name)+ @property+ def message_output_tokens(self) -> Histogram:+ return self._get_or_create_metric(+ "hist_message_output_tokens",+ partial(+ self._meter.create_histogram,+ name="hist_message_output_tokens",+ description="Histogram for output tokens generated by LLM per step",+ unit="1",+ ),+ )diff --git a/letta/otel/metrics.py b/letta/otel/metrics.pynew file mode 100644index 000000000..8e11f291b--- /dev/null+++ b/letta/otel/metrics.py@@ -0,0 +1,66 @@+from fastapi import FastAPI, Request+from opentelemetry import metrics+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter+from opentelemetry.metrics import NoOpMeter+from opentelemetry.sdk.metrics import MeterProvider+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader++from letta.log import get_logger+from letta.otel.context import add_ctx_attribute+from letta.otel.resource import get_resource, is_pytest_environment++logger = get_logger(__name__)++_meter: metrics.Meter = NoOpMeter("noop")+_is_metrics_initialized: bool = False+++async def _otel_metric_middleware(request: Request, call_next):+ if not _is_metrics_initialized:+ return await call_next(request)++ header_attributes = {+ "x-organization-id": "organization.id",+ "x-project-id": "project.id",+ "x-base-template-id": "base_template.id",+ "x-template-id": "template.id",+ "x-agent-id": "agent.id",+ }+ try:+ for header_key, otel_key in header_attributes.items():+ header_value = request.headers.get(header_key)+ if header_value:+ add_ctx_attribute(otel_key, header_value)+ return await call_next(request)+ except Exception:+ raise+++def setup_metrics(+ endpoint: str,+ app: FastAPI | None = None,+ service_name: str = "memgpt-server",+) -> None:+ if is_pytest_environment():+ return+ assert endpoint++ global _is_metrics_initialized, _meter++ otlp_metric_exporter = OTLPMetricExporter(endpoint=endpoint)+ metric_reader = PeriodicExportingMetricReader(exporter=otlp_metric_exporter)+ meter_provider = MeterProvider(resource=get_resource(service_name), metric_readers=[metric_reader])+ metrics.set_meter_provider(meter_provider)+ _meter = metrics.get_meter(__name__)++ if app:+ app.middleware("http")(_otel_metric_middleware)++ _is_metrics_initialized = True+++def get_letta_meter() -> metrics.Meter | None:+ """Returns the global letta meter if metrics are initialized."""+ if not _is_metrics_initialized or isinstance(_meter, NoOpMeter):+ logger.warning("Metrics are not initialized or meter is not available.")+ return _meterdiff --git a/letta/otel/resource.py b/letta/otel/resource.pynew file mode 100644index 000000000..d11e31044--- /dev/null+++ b/letta/otel/resource.py@@ -0,0 +1,26 @@+import os+import sys+import uuid++from opentelemetry.sdk.resources import Resource++from letta import __version__ as letta_version++_resources = {}+++def get_resource(service_name: str) -> Resource:+ _env = os.getenv("LETTA_ENVIRONMENT")+ if service_name not in _resources:+ resource_dict = {+ "service.name": service_name,+ "letta.version": letta_version,+ }+ if _env != "PRODUCTION":+ resource_dict["device.id"] = uuid.getnode() # MAC address as unique device identifier,+ _resources[(service_name, _env)] = Resource.create(resource_dict)+ return _resources[(service_name, _env)]+++def is_pytest_environment():+ return "pytest" in sys.modulesdiff --git a/letta/tracing.py b/letta/otel/tracing.pysimilarity index 66%rename from letta/tracing.pyrename to letta/otel/tracing.pyindex de3e4c5eb..05cd485a8 100644--- a/letta/tracing.py+++ b/letta/otel/tracing.py@@ -1,6 +1,5 @@import inspectimport re-import sysimport timefrom functools import wrapsfrom typing import Any, Dict, List, Optional@@ -11,15 +10,18 @@ from fastapi.responses import JSONResponsefrom opentelemetry import tracefrom opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporterfrom opentelemetry.instrumentation.requests import RequestsInstrumentor-from opentelemetry.sdk.resources import Resourcefrom opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import BatchSpanProcessorfrom opentelemetry.trace import Status, StatusCode-from letta import __version__ as letta_version+from letta.log import get_logger+from letta.otel.resource import get_resource, is_pytest_environment+from letta.settings import settings+logger = get_logger(__name__) # TODO: set up logger config for thistracer = trace.get_tracer(__name__)_is_tracing_initialized = False+_excluded_v1_endpoints_regex: List[str] = [# "^GET /v1/agents/(?P<agent_id>[^/]+)/messages$",# "^GET /v1/agents/(?P<agent_id>[^/]+)/context$",@@ -30,11 +32,7 @@ _excluded_v1_endpoints_regex: List[str] = []-def is_pytest_environment():- return "pytest" in sys.modules---async def trace_request_middleware(request: Request, call_next):+async def _trace_request_middleware(request: Request, call_next):if not _is_tracing_initialized:return await call_next(request)initial_span_name = f"{request.method} {request.url.path}"@@ -56,7 +54,7 @@ async def trace_request_middleware(request: Request, call_next):raise-async def update_trace_attributes(request: Request):+async def _update_trace_attributes(request: Request):"""Dependency to update trace attributes after FastAPI has processed the request"""if not _is_tracing_initialized:return@@ -78,35 +76,19 @@ async def update_trace_attributes(request: Request):for key, value in request.path_params.items():span.set_attribute(f"http.{key}", value)- # Add user ID if available- user_id = request.headers.get("user_id")- if user_id:- span.set_attribute("user.id", user_id)-- # Add organization_id if available- organization_id = request.headers.get("x-organization-id")- if organization_id:- span.set_attribute("organization.id", organization_id)-- # Add project_id if available- project_id = request.headers.get("x-project-id")- if project_id:- span.set_attribute("project.id", project_id)-- # Add agent_id if available- agent_id = request.headers.get("x-agent-id")- if agent_id:- span.set_attribute("agent.id", agent_id)-- # Add template_id if available- template_id = request.headers.get("x-template-id")- if template_id:- span.set_attribute("template.id", template_id)-- # Add base_template_id if available- base_template_id = request.headers.get("x-base-template-id")- if base_template_id:- span.set_attribute("base_template.id", base_template_id)+ # Add the following headers to span if available+ header_attributes = {+ "user_id": "user.id",+ "x-organization-id": "organization.id",+ "x-project-id": "project.id",+ "x-agent-id": "agent.id",+ "x-template-id": "template.id",+ "x-base-template-id": "base_template.id",+ }+ for header_key, span_key in header_attributes.items():+ header_value = request.headers.get(header_key)+ if header_value:+ span.set_attribute(span_key, header_value)# Add request body if availabletry:@@ -117,7 +99,7 @@ async def update_trace_attributes(request: Request):pass-async def trace_error_handler(_request: Request, exc: Exception) -> JSONResponse:+async def _trace_error_handler(_request: Request, exc: Exception) -> JSONResponse:status_code = getattr(exc, "status_code", 500)error_msg = str(exc)@@ -142,49 +124,44 @@ def setup_tracing() -> None:if is_pytest_environment():return+ assert endpointglobal _is_tracing_initialized- provider = TracerProvider(resource=Resource.create({"service.name": service_name}))- import uuid+ tracer_provider = TracerProvider(resource=get_resource(service_name))+ tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))+ _is_tracing_initialized = True+ trace.set_tracer_provider(tracer_provider)- provider = TracerProvider(- resource=Resource.create(- {- "service.name": service_name,- "device.id": uuid.getnode(), # MAC address as unique device identifier,- "letta.version": letta_version,- }- )- )- if endpoint:- provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))- _is_tracing_initialized = True- trace.set_tracer_provider(provider)-- def requests_callback(span: trace.Span, _: Any, response: Any) -> None:- if hasattr(response, "status_code"):- span.set_status(Status(StatusCode.OK if response.status_code < 400 else StatusCode.ERROR))-- RequestsInstrumentor().instrument(response_hook=requests_callback)-- if app:- # Add middleware first- app.middleware("http")(trace_request_middleware)-- # Add dependency to v1 routes- from letta.server.rest_api.routers.v1 import ROUTERS as v1_routes-- for router in v1_routes:- for route in router.routes:- full_path = ((next(iter(route.methods)) + " ") if route.methods else "") + "/v1" + route.path- if not any(re.match(regex, full_path) for regex in _excluded_v1_endpoints_regex):- route.dependencies.append(Depends(update_trace_attributes))-- # Register exception handlers- app.exception_handler(HTTPException)(trace_error_handler)- app.exception_handler(RequestValidationError)(trace_error_handler)- app.exception_handler(Exception)(trace_error_handler)+ # Instrumentors (e.g., RequestsInstrumentor)+ def requests_callback(span: trace.Span, _: Any, response: Any) -> None:+ if hasattr(response, "status_code"):+ span.set_status(Status(StatusCode.OK if response.status_code < 400 else StatusCode.ERROR))++ RequestsInstrumentor().instrument(response_hook=requests_callback)++ if settings.sqlalchemy_tracing:+ from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor++ SQLAlchemyInstrumentor().instrument()++ if app:+ # Add middleware first+ app.middleware("http")(_trace_request_middleware)++ # Add dependency to v1 routes+ from letta.server.rest_api.routers.v1 import ROUTERS as V1_ROUTES++ for router in V1_ROUTES:+ for route in router.routes:+ full_path = ((next(iter(route.methods)) + " ") if route.methods else "") + "/v1" + route.path+ if not any(re.match(regex, full_path) for regex in _excluded_v1_endpoints_regex):+ route.dependencies.append(Depends(_update_trace_attributes))++ # Register exception handlers for tracing+ app.exception_handler(HTTPException)(_trace_error_handler)+ app.exception_handler(RequestValidationError)(_trace_error_handler)+ app.exception_handler(Exception)(_trace_error_handler)def trace_method(func):diff --git a/letta/server/db.py b/letta/server/db.pyindex f50b09880..cf423c59e 100644--- a/letta/server/db.py+++ b/letta/server/db.py@@ -13,8 +13,8 @@ from sqlalchemy.orm import sessionmakerfrom letta.config import LettaConfigfrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.settings import settings-from letta.tracing import trace_methodlogger = get_logger(__name__)diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex 944fc2bd1..d7fecc74f 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -256,13 +256,15 @@ def create_application() -> "FastAPI":print(f"▶ Using OTLP tracing with endpoint: {otlp_endpoint}")env_name_suffix = os.getenv("ENV_NAME")service_name = f"letta-server-{env_name_suffix.lower()}" if env_name_suffix else "letta-server"- from letta.tracing import setup_tracing+ from letta.otel.metrics import setup_metrics+ from letta.otel.tracing import setup_tracingsetup_tracing(endpoint=otlp_endpoint,app=app,service_name=service_name,)+ setup_metrics(endpoint=otlp_endpoint, app=app, service_name=service_name)for route in v1_routes:app.include_router(route, prefix=API_PREFIX)@@ -339,14 +341,14 @@ def start_server(target="letta.server.rest_api.app:app",# factory=True,interface="asgi",- address=host or "localhost",+ address=host or "127.0.0.1", # Note granian address must be an ip addressport=port or REST_DEFAULT_PORT,workers=settings.uvicorn_workers,# threads=reload=reload or settings.uvicorn_reload,reload_ignore_patterns=["openapi_letta.json"],reload_ignore_worker_failure=True,- reload_tick=100,+ reload_tick=4000, # set to 4s to prevent crashing on weird state# log_level="info"ssl_keyfile="certs/localhost-key.pem",ssl_cert="certs/localhost.pem",@@ -380,14 +382,14 @@ def start_server(target="letta.server.rest_api.app:app",# factory=True,interface="asgi",- address=host or "localhost",+ address=host or "127.0.0.1", # Note granian address must be an ip addressport=port or REST_DEFAULT_PORT,workers=settings.uvicorn_workers,# threads=reload=reload or settings.uvicorn_reload,reload_ignore_patterns=["openapi_letta.json"],reload_ignore_worker_failure=True,- reload_tick=100,+ reload_tick=4000, # set to 4s to prevent crashing on weird state# log_level="info").serve()else:diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex fc1dbfcac..5d1873c0a 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -17,6 +17,8 @@ from letta.groups.sleeptime_multi_agent_v2 import SleeptimeMultiAgentV2from letta.helpers.datetime_helpers import get_utc_timestamp_nsfrom letta.log import get_loggerfrom letta.orm.errors import NoResultFound+from letta.otel.context import get_ctx_attributes+from letta.otel.metric_registry import MetricRegistryfrom letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgentfrom letta.schemas.block import Block, BlockUpdatefrom letta.schemas.group import Group@@ -663,6 +665,8 @@ async def send_message(Process a user message and return the agent's response.This endpoint accepts a message from a user and processes it through the agent."""+ MetricRegistry().user_message_counter.add(1, get_ctx_attributes())+actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)request_start_timestamp_ns = get_utc_timestamp_ns()# TODO: This is redundant, remove soon@@ -741,7 +745,8 @@ async def send_message_streaming(This endpoint accepts a message from a user and processes it through the agent.It will stream the steps of the response always, and stream the tokens if 'stream_tokens' is set to True."""- request_start_timestamp_ns = get_utc_timestamp_ns()+ MetricRegistry().user_message_counter.add(1, get_ctx_attributes())+actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)# TODO: This is redundant, remove soonagent = await server.agent_manager.get_agent_by_id_async(agent_id, actor, include_relationships=["multi_agent_group"])@@ -877,6 +882,7 @@ async def send_message_async(Asynchronously process a user message and return a run object.The actual processing happens in the background, and the status can be checked using the run ID."""+ MetricRegistry().user_message_counter.add(1, get_ctx_attributes())actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)# Create a new jobdiff --git a/letta/server/rest_api/utils.py b/letta/server/rest_api/utils.pyindex cd00af35a..f5ba0cbd0 100644--- a/letta/server/rest_api/utils.py+++ b/letta/server/rest_api/utils.py@@ -15,9 +15,12 @@ from pydantic import BaseModelfrom letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, FUNC_FAILED_HEARTBEAT_MESSAGE, REQ_HEARTBEAT_MESSAGEfrom letta.errors import ContextWindowExceededError, RateLimitExceededError-from letta.helpers.datetime_helpers import get_utc_time, get_utc_timestamp_ns+from letta.helpers.datetime_helpers import get_utc_time, get_utc_timestamp_ns, ns_to_msfrom letta.helpers.message_helper import convert_message_creates_to_messagesfrom letta.log import get_logger+from letta.otel.context import get_ctx_attributes+from letta.otel.metric_registry import MetricRegistry+from letta.otel.tracing import tracerfrom letta.schemas.enums import MessageRolefrom letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, RedactedReasoningContent, TextContentfrom letta.schemas.llm_config import LLMConfig@@ -27,7 +30,6 @@ from letta.schemas.usage import LettaUsageStatisticsfrom letta.schemas.user import Userfrom letta.server.rest_api.interface import StreamingServerInterfacefrom letta.system import get_heartbeat, package_function_response-from letta.tracing import tracerif TYPE_CHECKING:from letta.server.server import SyncServer@@ -81,8 +83,12 @@ async def sse_async_generator(if first_chunk and ttft_span is not None:now = get_utc_timestamp_ns()ttft_ns = now - request_start_timestamp_ns- ttft_span.add_event(name="time_to_first_token_ms", attributes={"ttft_ms": ttft_ns // 1_000_000})+ ttft_span.add_event(name="time_to_first_token_ms", attributes={"ttft_ms": ns_to_ms(ttft_ns)})ttft_span.end()+ metric_attributes = get_ctx_attributes()+ if llm_config:+ metric_attributes["model.name"] = llm_config.model+ MetricRegistry().ttft_ms_histogram.record(ns_to_ms(ttft_ns), metric_attributes)first_chunk = False# yield f"data: {json.dumps(chunk)}\n\n"diff --git a/letta/server/server.py b/letta/server/server.pyindex 347a97356..8ee45339c 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -34,6 +34,7 @@ from letta.interface import AgentInterface # abstractfrom letta.interface import CLIInterface # for printing to terminalfrom letta.log import get_loggerfrom letta.orm.errors import NoResultFound+from letta.otel.tracing import log_event, trace_methodfrom letta.prompts.gpt_system import get_system_textfrom letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgentfrom letta.schemas.block import Block, BlockUpdate, CreateBlock@@ -101,7 +102,6 @@ from letta.services.tool_executor.tool_execution_manager import ToolExecutionManfrom letta.services.tool_manager import ToolManagerfrom letta.services.user_manager import UserManagerfrom letta.settings import model_settings, settings, tool_settings-from letta.tracing import log_event, trace_methodfrom letta.utils import get_friendly_error_msg, get_persona_text, make_keyconfig = LettaConfig.load()diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 2f6c56123..0ceb9f582 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -37,6 +37,7 @@ from letta.orm.errors import NoResultFoundfrom letta.orm.sandbox_config import AgentEnvironmentVariablefrom letta.orm.sandbox_config import AgentEnvironmentVariable as AgentEnvironmentVariableModelfrom letta.orm.sqlalchemy_base import AccessType+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState as PydanticAgentStatefrom letta.schemas.agent import AgentType, CreateAgent, UpdateAgent, get_prompt_template_for_agent_typefrom letta.schemas.block import DEFAULT_BLOCKS@@ -86,7 +87,6 @@ from letta.services.message_manager import MessageManagerfrom letta.services.passage_manager import PassageManagerfrom letta.services.source_manager import SourceManagerfrom letta.services.tool_manager import ToolManager-from letta.tracing import trace_methodfrom letta.utils import enforce_types, united_difflogger = get_logger(__name__)diff --git a/letta/services/block_manager.py b/letta/services/block_manager.pyindex ce7311254..12156cd45 100644--- a/letta/services/block_manager.py+++ b/letta/services/block_manager.py@@ -9,12 +9,12 @@ from letta.orm.block import Block as BlockModelfrom letta.orm.block_history import BlockHistoryfrom letta.orm.enums import ActorTypefrom letta.orm.errors import NoResultFound+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState as PydanticAgentStatefrom letta.schemas.block import Block as PydanticBlockfrom letta.schemas.block import BlockUpdatefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typeslogger = get_logger(__name__)diff --git a/letta/services/files_agents_manager.py b/letta/services/files_agents_manager.pyindex cb1748648..a9bb7a6da 100644--- a/letta/services/files_agents_manager.py+++ b/letta/services/files_agents_manager.py@@ -5,11 +5,11 @@ from sqlalchemy import and_, func, select, updatefrom letta.orm.errors import NoResultFoundfrom letta.orm.files_agents import FileAgent as FileAgentModel+from letta.otel.tracing import trace_methodfrom letta.schemas.block import Block as PydanticBlockfrom letta.schemas.file import FileAgent as PydanticFileAgentfrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/group_manager.py b/letta/services/group_manager.pyindex 1884e5d06..57f532fb1 100644--- a/letta/services/group_manager.py+++ b/letta/services/group_manager.py@@ -7,13 +7,13 @@ from letta.orm.agent import Agent as AgentModelfrom letta.orm.errors import NoResultFoundfrom letta.orm.group import Group as GroupModelfrom letta.orm.message import Message as MessageModel+from letta.otel.tracing import trace_methodfrom letta.schemas.group import Group as PydanticGroupfrom letta.schemas.group import GroupCreate, GroupUpdate, ManagerTypefrom letta.schemas.letta_message import LettaMessagefrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/helpers/agent_manager_helper.py b/letta/services/helpers/agent_manager_helper.pyindex 3513a89ba..7d83b1ab4 100644--- a/letta/services/helpers/agent_manager_helper.py+++ b/letta/services/helpers/agent_manager_helper.py@@ -16,6 +16,7 @@ from letta.orm.agents_tags import AgentsTagsfrom letta.orm.errors import NoResultFoundfrom letta.orm.identity import Identityfrom letta.orm.sqlite_functions import adapt_array+from letta.otel.tracing import trace_methodfrom letta.prompts import gpt_systemfrom letta.schemas.agent import AgentState, AgentTypefrom letta.schemas.embedding_config import EmbeddingConfig@@ -27,7 +28,6 @@ from letta.schemas.tool_rule import ToolRulefrom letta.schemas.user import Userfrom letta.settings import settingsfrom letta.system import get_initial_boot_messages, get_login_event, package_function_response-from letta.tracing import trace_method# Static methodsdiff --git a/letta/services/identity_manager.py b/letta/services/identity_manager.pyindex f91854fd3..abf637bac 100644--- a/letta/services/identity_manager.py+++ b/letta/services/identity_manager.py@@ -7,11 +7,11 @@ from sqlalchemy.exc import NoResultFoundfrom letta.orm.agent import Agent as AgentModelfrom letta.orm.block import Block as BlockModelfrom letta.orm.identity import Identity as IdentityModel+from letta.otel.tracing import trace_methodfrom letta.schemas.identity import Identity as PydanticIdentityfrom letta.schemas.identity import IdentityCreate, IdentityProperty, IdentityType, IdentityUpdate, IdentityUpsertfrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/job_manager.py b/letta/services/job_manager.pyindex 11361b350..541d75141 100644--- a/letta/services/job_manager.py+++ b/letta/services/job_manager.py@@ -14,6 +14,7 @@ from letta.orm.message import Message as MessageModelfrom letta.orm.sqlalchemy_base import AccessTypefrom letta.orm.step import Stepfrom letta.orm.step import Step as StepModel+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import JobStatus, MessageRolefrom letta.schemas.job import BatchJob as PydanticBatchJobfrom letta.schemas.job import Job as PydanticJob@@ -25,7 +26,6 @@ from letta.schemas.step import Step as PydanticStepfrom letta.schemas.usage import LettaUsageStatisticsfrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/llm_batch_manager.py b/letta/services/llm_batch_manager.pyindex eeff0673f..4700d1467 100644--- a/letta/services/llm_batch_manager.py+++ b/letta/services/llm_batch_manager.py@@ -9,6 +9,7 @@ from letta.log import get_loggerfrom letta.orm import Message as MessageModelfrom letta.orm.llm_batch_items import LLMBatchItemfrom letta.orm.llm_batch_job import LLMBatchJob+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStepStatefrom letta.schemas.enums import AgentStepStatus, JobStatus, ProviderTypefrom letta.schemas.llm_batch_job import LLMBatchItem as PydanticLLMBatchItem@@ -17,7 +18,6 @@ from letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typeslogger = get_logger(__name__)diff --git a/letta/services/message_manager.py b/letta/services/message_manager.pyindex 017636896..45a6fc6f0 100644--- a/letta/services/message_manager.py+++ b/letta/services/message_manager.py@@ -7,13 +7,13 @@ from letta.log import get_loggerfrom letta.orm.agent import Agent as AgentModelfrom letta.orm.errors import NoResultFoundfrom letta.orm.message import Message as MessageModel+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import MessageRolefrom letta.schemas.letta_message import LettaMessageUpdateUnionfrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.message import MessageUpdatefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typeslogger = get_logger(__name__)diff --git a/letta/services/organization_manager.py b/letta/services/organization_manager.pyindex 08f8f70a1..23fe92c86 100644--- a/letta/services/organization_manager.py+++ b/letta/services/organization_manager.py@@ -2,10 +2,10 @@ from typing import List, Optionalfrom letta.orm.errors import NoResultFoundfrom letta.orm.organization import Organization as OrganizationModel+from letta.otel.tracing import trace_methodfrom letta.schemas.organization import Organization as PydanticOrganizationfrom letta.schemas.organization import OrganizationUpdatefrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/passage_manager.py b/letta/services/passage_manager.pyindex 99f7d6c41..b60e2e1b9 100644--- a/letta/services/passage_manager.py+++ b/letta/services/passage_manager.py@@ -11,11 +11,11 @@ from letta.constants import MAX_EMBEDDING_DIMfrom letta.embeddings import embedding_model, parse_and_chunk_textfrom letta.orm.errors import NoResultFoundfrom letta.orm.passage import AgentPassage, SourcePassage+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.passage import Passage as PydanticPassagefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/per_agent_lock_manager.py b/letta/services/per_agent_lock_manager.pyindex e8e2a0a43..aff76a1f5 100644--- a/letta/services/per_agent_lock_manager.py+++ b/letta/services/per_agent_lock_manager.py@@ -1,7 +1,7 @@import threadingfrom collections import defaultdict-from letta.tracing import trace_method+from letta.otel.tracing import trace_methodclass PerAgentLockManager:diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.pyindex 0e9cf9483..bdb7bd246 100644--- a/letta/services/provider_manager.py+++ b/letta/services/provider_manager.py@@ -1,12 +1,12 @@from typing import List, Optional, Unionfrom letta.orm.provider import Provider as ProviderModel+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import ProviderCategory, ProviderTypefrom letta.schemas.providers import Provider as PydanticProviderfrom letta.schemas.providers import ProviderCheck, ProviderCreate, ProviderUpdatefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/sandbox_config_manager.py b/letta/services/sandbox_config_manager.pyindex 211f69751..d9a8bffb8 100644--- a/letta/services/sandbox_config_manager.py+++ b/letta/services/sandbox_config_manager.py@@ -5,6 +5,7 @@ from letta.log import get_loggerfrom letta.orm.errors import NoResultFoundfrom letta.orm.sandbox_config import SandboxConfig as SandboxConfigModelfrom letta.orm.sandbox_config import SandboxEnvironmentVariable as SandboxEnvVarModel+from letta.otel.tracing import trace_methodfrom letta.schemas.environment_variables import SandboxEnvironmentVariable as PydanticEnvVarfrom letta.schemas.environment_variables import SandboxEnvironmentVariableCreate, SandboxEnvironmentVariableUpdatefrom letta.schemas.sandbox_config import LocalSandboxConfig@@ -12,7 +13,6 @@ from letta.schemas.sandbox_config import SandboxConfig as PydanticSandboxConfigfrom letta.schemas.sandbox_config import SandboxConfigCreate, SandboxConfigUpdate, SandboxTypefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_types, printdlogger = get_logger(__name__)diff --git a/letta/services/source_manager.py b/letta/services/source_manager.pyindex 099441b5c..831313837 100644--- a/letta/services/source_manager.py+++ b/letta/services/source_manager.py@@ -10,13 +10,13 @@ from letta.orm.file import FileContent as FileContentModelfrom letta.orm.file import FileMetadata as FileMetadataModelfrom letta.orm.source import Source as SourceModelfrom letta.orm.sqlalchemy_base import AccessType+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState as PydanticAgentStatefrom letta.schemas.file import FileMetadata as PydanticFileMetadatafrom letta.schemas.source import Source as PydanticSourcefrom letta.schemas.source import SourceUpdatefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.tracing import trace_methodfrom letta.utils import enforce_types, printddiff --git a/letta/services/step_manager.py b/letta/services/step_manager.pyindex ea30e2d5f..9401af487 100644--- a/letta/services/step_manager.py+++ b/letta/services/step_manager.py@@ -5,16 +5,16 @@ from sqlalchemy import selectfrom sqlalchemy.ext.asyncio import AsyncSessionfrom sqlalchemy.orm import Session+from letta.helpers.singleton import singletonfrom letta.orm.errors import NoResultFoundfrom letta.orm.job import Job as JobModelfrom letta.orm.sqlalchemy_base import AccessTypefrom letta.orm.step import Step as StepModel+from letta.otel.tracing import get_trace_id, trace_methodfrom letta.schemas.openai.chat_completion_response import UsageStatisticsfrom letta.schemas.step import Step as PydanticStepfrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.services.helpers.noop_helper import singleton-from letta.tracing import get_trace_id, trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/services/summarizer/summarizer.py b/letta/services/summarizer/summarizer.pyindex 3fc23dbf3..fa6d18d5c 100644--- a/letta/services/summarizer/summarizer.py+++ b/letta/services/summarizer/summarizer.py@@ -6,11 +6,11 @@ from typing import List, Optional, Tuple, Unionfrom letta.agents.ephemeral_summary_agent import EphemeralSummaryAgentfrom letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARGfrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.schemas.enums import MessageRolefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.message import Message, MessageCreatefrom letta.services.summarizer.enums import SummarizationMode-from letta.tracing import trace_methodlogger = get_logger(__name__)diff --git a/letta/services/telemetry_manager.py b/letta/services/telemetry_manager.pyindex 2dc14ca95..213bdb09b 100644--- a/letta/services/telemetry_manager.py+++ b/letta/services/telemetry_manager.py@@ -1,11 +1,11 @@from letta.helpers.json_helpers import json_dumps, json_loads+from letta.helpers.singleton import singletonfrom letta.orm.provider_trace import ProviderTrace as ProviderTraceModelfrom letta.schemas.provider_trace import ProviderTrace as PydanticProviderTracefrom letta.schemas.provider_trace import ProviderTraceCreatefrom letta.schemas.step import Step as PydanticStepfrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registry-from letta.services.helpers.noop_helper import singletonfrom letta.utils import enforce_typesdiff --git a/letta/services/tool_executor/builtin_tool_executor.py b/letta/services/tool_executor/builtin_tool_executor.pyindex e1993e197..cc320a475 100644--- a/letta/services/tool_executor/builtin_tool_executor.py+++ b/letta/services/tool_executor/builtin_tool_executor.py@@ -3,6 +3,7 @@ from textwrap import shortenfrom typing import Any, Dict, Literal, Optionalfrom letta.constants import WEB_SEARCH_CLIP_CONTENT, WEB_SEARCH_INCLUDE_SCORE, WEB_SEARCH_SEPARATOR+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfigfrom letta.schemas.tool import Tool@@ -10,7 +11,6 @@ from letta.schemas.tool_execution_result import ToolExecutionResultfrom letta.schemas.user import Userfrom letta.services.tool_executor.tool_executor_base import ToolExecutorfrom letta.settings import tool_settings-from letta.tracing import trace_methodclass LettaBuiltinToolExecutor(ToolExecutor):diff --git a/letta/services/tool_executor/composio_tool_executor.py b/letta/services/tool_executor/composio_tool_executor.pyindex 4f4dddb87..8053c521e 100644--- a/letta/services/tool_executor/composio_tool_executor.py+++ b/letta/services/tool_executor/composio_tool_executor.py@@ -3,13 +3,13 @@ from typing import Any, Dict, Optionalfrom letta.constants import COMPOSIO_ENTITY_ENV_VAR_KEYfrom letta.functions.composio_helpers import execute_composio_action_async, generate_composio_action_from_func_namefrom letta.helpers.composio_helpers import get_composio_api_key_async+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfigfrom letta.schemas.tool import Toolfrom letta.schemas.tool_execution_result import ToolExecutionResultfrom letta.schemas.user import Userfrom letta.services.tool_executor.tool_executor_base import ToolExecutor-from letta.tracing import trace_methodclass ExternalComposioToolExecutor(ToolExecutor):diff --git a/letta/services/tool_executor/mcp_tool_executor.py b/letta/services/tool_executor/mcp_tool_executor.pyindex eaef6ea3e..1640c1455 100644--- a/letta/services/tool_executor/mcp_tool_executor.py+++ b/letta/services/tool_executor/mcp_tool_executor.py@@ -1,6 +1,7 @@from typing import Any, Dict, Optionalfrom letta.constants import MCP_TOOL_TAG_NAME_PREFIX+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfigfrom letta.schemas.tool import Tool@@ -8,7 +9,6 @@ from letta.schemas.tool_execution_result import ToolExecutionResultfrom letta.schemas.user import Userfrom letta.services.mcp_manager import MCPManagerfrom letta.services.tool_executor.tool_executor_base import ToolExecutor-from letta.tracing import trace_methodclass ExternalMCPToolExecutor(ToolExecutor):diff --git a/letta/services/tool_executor/tool_execution_manager.py b/letta/services/tool_executor/tool_execution_manager.pyindex 085019ee0..2b4dad6a2 100644--- a/letta/services/tool_executor/tool_execution_manager.py+++ b/letta/services/tool_executor/tool_execution_manager.py@@ -2,8 +2,12 @@ import tracebackfrom typing import Any, Dict, Optional, Typefrom letta.constants import FUNCTION_RETURN_VALUE_TRUNCATED+from letta.helpers.datetime_helpers import AsyncTimerfrom letta.log import get_loggerfrom letta.orm.enums import ToolType+from letta.otel.context import get_ctx_attributes+from letta.otel.metric_registry import MetricRegistry+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfigfrom letta.schemas.tool import Tool@@ -21,7 +25,6 @@ from letta.services.tool_executor.mcp_tool_executor import ExternalMCPToolExecutfrom letta.services.tool_executor.multi_agent_tool_executor import LettaMultiAgentToolExecutorfrom letta.services.tool_executor.tool_executor import SandboxToolExecutorfrom letta.services.tool_executor.tool_executor_base import ToolExecutor-from letta.tracing import trace_methodfrom letta.utils import get_friendly_error_msg@@ -85,10 +88,13 @@ class ToolExecutionManager:self.sandbox_env_vars = sandbox_env_vars@trace_method- async def execute_tool_async(self, function_name: str, function_args: dict, tool: Tool) -> ToolExecutionResult:+ async def execute_tool_async(+ self, function_name: str, function_args: dict, tool: Tool, step_id: str | None = None+ ) -> ToolExecutionResult:"""Execute a tool asynchronously and persist any state changes."""+ status = "error" # set as default for tracking purposestry:executor = ToolExecutorFactory.get_executor(tool.tool_type,@@ -98,9 +104,17 @@ class ToolExecutionManager:passage_manager=self.passage_manager,actor=self.actor,)- result = await executor.execute(- function_name, function_args, tool, self.actor, self.agent_state, self.sandbox_config, self.sandbox_env_vars- )++ def _metrics_callback(exec_time_ms: int, exc):+ return MetricRegistry().tool_execution_time_ms_histogram.record(+ exec_time_ms, dict(get_ctx_attributes(), **{"tool.name": tool.name})+ )++ async with AsyncTimer(callback_func=_metrics_callback):+ result = await executor.execute(+ function_name, function_args, tool, self.actor, self.agent_state, self.sandbox_config, self.sandbox_env_vars+ )+ status = result.status# trim resultreturn_str = str(result.func_return)@@ -110,6 +124,7 @@ class ToolExecutionManager:return resultexcept Exception as e:+ status = "error"self.logger.error(f"Error executing tool {function_name}: {str(e)}")error_message = get_friendly_error_msg(function_name=function_name,@@ -121,3 +136,8 @@ class ToolExecutionManager:func_return=error_message,stderr=[traceback.format_exc()],)+ finally:+ metric_attrs = {"tool.name": tool.name, "tool.execution_success": status == "success"}+ if status == "error" and step_id:+ metric_attrs["step.id"] = step_id+ MetricRegistry().tool_execution_counter.add(1, dict(get_ctx_attributes(), **metric_attrs))diff --git a/letta/services/tool_executor/tool_execution_sandbox.py b/letta/services/tool_executor/tool_execution_sandbox.pyindex 4d60de8f7..e41089ea1 100644--- a/letta/services/tool_executor/tool_execution_sandbox.py+++ b/letta/services/tool_executor/tool_execution_sandbox.py@@ -11,6 +11,7 @@ from typing import Any, Dict, Optionalfrom letta.functions.helpers import generate_model_from_args_json_schemafrom letta.log import get_logger+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfig, SandboxTypefrom letta.schemas.tool import Tool@@ -27,7 +28,6 @@ from letta.services.organization_manager import OrganizationManagerfrom letta.services.sandbox_config_manager import SandboxConfigManagerfrom letta.services.tool_manager import ToolManagerfrom letta.settings import tool_settings-from letta.tracing import log_event, trace_methodfrom letta.utils import get_friendly_error_msglogger = get_logger(__name__)diff --git a/letta/services/tool_executor/tool_executor.py b/letta/services/tool_executor/tool_executor.pyindex f34e7d01e..b57d87cbf 100644--- a/letta/services/tool_executor/tool_executor.py+++ b/letta/services/tool_executor/tool_executor.py@@ -3,6 +3,7 @@ from typing import Any, Dict, Optionalfrom letta.functions.ast_parsers import coerce_dict_args_by_annotations, get_function_annotations_from_sourcefrom letta.log import get_logger+from letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfigfrom letta.schemas.tool import Tool@@ -13,7 +14,6 @@ from letta.services.tool_executor.tool_executor_base import ToolExecutorfrom letta.services.tool_sandbox.e2b_sandbox import AsyncToolSandboxE2Bfrom letta.services.tool_sandbox.local_sandbox import AsyncToolSandboxLocalfrom letta.settings import tool_settings-from letta.tracing import trace_methodfrom letta.types import JsonDictfrom letta.utils import get_friendly_error_msgdiff --git a/letta/services/tool_manager.py b/letta/services/tool_manager.pyindex 491e4bca6..78daed8a4 100644--- a/letta/services/tool_manager.py+++ b/letta/services/tool_manager.py@@ -24,12 +24,12 @@ from letta.orm.enums import ToolType# TODO: Remove this once we translate all of these to the ORMfrom letta.orm.errors import NoResultFoundfrom letta.orm.tool import Tool as ToolModel+from letta.otel.tracing import trace_methodfrom letta.schemas.tool import Tool as PydanticToolfrom letta.schemas.tool import ToolCreate, ToolUpdatefrom letta.schemas.user import User as PydanticUserfrom letta.server.db import db_registryfrom letta.services.mcp.types import SSEServerConfig, StdioServerConfig-from letta.tracing import trace_methodfrom letta.utils import enforce_types, printdlogger = get_logger(__name__)diff --git a/letta/services/tool_sandbox/e2b_sandbox.py b/letta/services/tool_sandbox/e2b_sandbox.pyindex ea07f1f11..fb44fc944 100644--- a/letta/services/tool_sandbox/e2b_sandbox.py+++ b/letta/services/tool_sandbox/e2b_sandbox.py@@ -3,13 +3,13 @@ from typing import TYPE_CHECKING, Any, Dict, Optionalfrom e2b_code_interpreter import AsyncSandboxfrom letta.log import get_logger+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfig, SandboxTypefrom letta.schemas.tool import Toolfrom letta.schemas.tool_execution_result import ToolExecutionResultfrom letta.services.helpers.tool_parser_helper import parse_stdout_best_effortfrom letta.services.tool_sandbox.base import AsyncToolSandboxBase-from letta.tracing import log_event, trace_methodfrom letta.types import JsonDictfrom letta.utils import get_friendly_error_msgdiff --git a/letta/services/tool_sandbox/local_sandbox.py b/letta/services/tool_sandbox/local_sandbox.pyindex 8fe34870a..08529b6f3 100644--- a/letta/services/tool_sandbox/local_sandbox.py+++ b/letta/services/tool_sandbox/local_sandbox.py@@ -8,6 +8,7 @@ from typing import Any, Dict, Optionalfrom pydantic.config import JsonDict+from letta.otel.tracing import log_event, trace_methodfrom letta.schemas.agent import AgentStatefrom letta.schemas.sandbox_config import SandboxConfig, SandboxTypefrom letta.schemas.tool import Tool@@ -20,7 +21,6 @@ from letta.services.helpers.tool_execution_helper import (from letta.services.helpers.tool_parser_helper import parse_stdout_best_effortfrom letta.services.tool_sandbox.base import AsyncToolSandboxBasefrom letta.settings import tool_settings-from letta.tracing import log_event, trace_methodfrom letta.utils import get_friendly_error_msg, parse_stderr_error_msgdiff --git a/letta/services/user_manager.py b/letta/services/user_manager.pyindex c48006ec5..eab42743b 100644--- a/letta/services/user_manager.py+++ b/letta/services/user_manager.py@@ -5,11 +5,11 @@ from sqlalchemy import select, textfrom letta.orm.errors import NoResultFoundfrom letta.orm.organization import Organization as OrganizationModelfrom letta.orm.user import User as UserModel+from letta.otel.tracing import trace_methodfrom letta.schemas.user import User as PydanticUserfrom letta.schemas.user import UserUpdatefrom letta.server.db import db_registryfrom letta.services.organization_manager import OrganizationManager-from letta.tracing import trace_methodfrom letta.utils import enforce_typesdiff --git a/letta/settings.py b/letta/settings.pyindex 0982c827a..eaa033a43 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -210,6 +210,7 @@ class Settings(BaseSettings):use_uvloop: bool = Falseuse_granian: bool = False+ sqlalchemy_tracing: bool = False# event loop parallelismevent_loop_threadpool_max_workers: int = 43diff --git a/otel/otel-collector-config-clickhouse-dev.yaml b/otel/otel-collector-config-clickhouse-dev.yamlindex 1c9c0cca0..f4a7c3744 100644--- a/otel/otel-collector-config-clickhouse-dev.yaml+++ b/otel/otel-collector-config-clickhouse-dev.yaml@@ -12,12 +12,18 @@ processors:send_batch_size: 1024exporters:- file:+ file/traces:path: ${HOME}/.letta/logs/traces.jsonrotation:max_megabytes: 100max_days: 7max_backups: 5+ file/metrics:+ path: ${HOME}/.letta/logs/metrics.json+ rotation:+ max_megabytes: 100+ max_days: 7+ max_backups: 5clickhouse:endpoint: ${CLICKHOUSE_ENDPOINT}database: ${CLICKHOUSE_DATABASE}@@ -40,4 +46,8 @@ service:traces:receivers: [otlp]processors: [batch]- exporters: [file, clickhouse]+ exporters: [file/traces, clickhouse]+ metrics:+ receivers: [otlp]+ processors: [batch]+ exporters: [file/metrics, clickhouse]