Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add otel collector to prod deployment (#1320)
cthomas6962a6bcb3f4
diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex 6212e584e..c2b1c137b 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -257,7 +257,7 @@ def create_application() -> "FastAPI":# Set up OpenTelemetry tracingotlp_endpoint = settings.otel_exporter_otlp_endpoint- if otlp_endpoint:+ if otlp_endpoint and not settings.disable_tracing:print(f"▶ Using OTLP tracing with endpoint: {otlp_endpoint}")env_name_suffix = os.getenv("ENV_NAME")service_name = f"letta-server-{env_name_suffix.lower()}" if env_name_suffix else "letta-server"diff --git a/letta/settings.py b/letta/settings.pyindex 5acfd532e..f6ade17ec 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -180,6 +180,7 @@ class Settings(BaseSettings):# telemetry loggingverbose_telemetry_logging: bool = Falseotel_exporter_otlp_endpoint: str = "http://localhost:4317"+ disable_tracing: bool = False# uvicorn settingsuvicorn_workers: int = 1diff --git a/otel-collector-config-clickhouse-prod.yaml b/otel-collector-config-clickhouse-prod.yamlnew file mode 100644index 000000000..84cdcfc96--- /dev/null+++ b/otel-collector-config-clickhouse-prod.yaml@@ -0,0 +1,74 @@+receivers:+ otlp:+ protocols:+ grpc:+ endpoint: 0.0.0.0:4317+ http:+ endpoint: 0.0.0.0:4318+ filelog:+ include:+ - /root/.letta/logs/Letta.log+ multiline:+ line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}+ operators:+ # Extract timestamp and other fields+ - type: regex_parser+ regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+-\s+(?P<component>[\w\.-]+)\s+-\s+(?P<severity>\w+)\s+-\s+(?P<body>.*)$'+ # Parse the timestamp+ - type: time_parser+ parse_from: attributes.timestamp+ layout: '%Y-%m-%d %H:%M:%S,%L'+ # Set severity+ - type: severity_parser+ parse_from: attributes.severity+ mapping:+ debug: DEBUG+ info: INFO+ warning: WARN+ error: ERROR+ critical: FATAL+ # Add resource attributes+ - type: add+ field: resource.service_name+ value: letta-server+ - type: add+ field: resource.environment+ value: ${ENV_NAME}++processors:+ batch:+ timeout: 1s+ send_batch_size: 1024++exporters:+ clickhouse:+ endpoint: ${CLICKHOUSE_ENDPOINT}+ database: ${CLICKHOUSE_DATABASE}+ username: ${CLICKHOUSE_USERNAME}+ password: ${CLICKHOUSE_PASSWORD}+ timeout: 5s+ sending_queue:+ queue_size: 100+ retry_on_failure:+ enabled: true+ initial_interval: 5s+ max_interval: 30s+ max_elapsed_time: 300s++service:+ telemetry:+ logs:+ level: error+ metrics:+ level: debug+ traces:+ level: debug+ pipelines:+ traces:+ receivers: [otlp]+ processors: [batch]+ exporters: [clickhouse]+ logs:+ receivers: [filelog]+ processors: [batch]+ exporters: [clickhouse]diff --git a/otel-collector-config-clickhouse.yaml b/otel-collector-config-clickhouse.yamlindex 6840610f1..aa00ce0db 100644--- a/otel-collector-config-clickhouse.yaml+++ b/otel-collector-config-clickhouse.yaml@@ -5,35 +5,6 @@ receivers:endpoint: 0.0.0.0:4317http:endpoint: 0.0.0.0:4318- filelog:- include:- - /root/.letta/logs/Letta.log- multiline:- line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}- operators:- # Extract timestamp and other fields- - type: regex_parser- regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+-\s+(?P<component>[\w\.-]+)\s+-\s+(?P<severity>\w+)\s+-\s+(?P<body>.*)$'- # Parse the timestamp- - type: time_parser- parse_from: attributes.timestamp- layout: '%Y-%m-%d %H:%M:%S,%L'- # Set severity- - type: severity_parser- parse_from: attributes.severity- mapping:- debug: DEBUG- info: INFO- warning: WARN- error: ERROR- critical: FATAL- # Add resource attributes- - type: add- field: resource.service_name- value: letta-server- - type: add- field: resource.environment- value: ${ENV_NAME}processors:batch:@@ -70,7 +41,3 @@ service:receivers: [otlp]processors: [batch]exporters: [file, clickhouse]- logs:- receivers: [filelog]- processors: [batch]- exporters: [clickhouse]