Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: Support Otel Logs as an option in evals export (#1025)
Ishan Jain48343eb1418d
diff --git a/docs/latest/sdk/configuration.mdx b/docs/latest/sdk/configuration.mdxindex af47266c..a5257226 100644--- a/docs/latest/sdk/configuration.mdx+++ b/docs/latest/sdk/configuration.mdx@@ -59,6 +59,14 @@ These options apply to database instrumentations like PostgreSQL (psycopg3):| `capture_parameters` | `--capture_parameters` | `OPENLIT_CAPTURE_PARAMETERS` | Capture database query parameters in spans (security risk - may expose sensitive data) | `False` | No || `enable_sqlcommenter` | `--enable_sqlcommenter` | `OPENLIT_ENABLE_SQLCOMMENTER` | Inject OpenTelemetry trace context as SQL comments for database log correlation | `False` | No |+### Evaluation export options++Configure how evaluation results are exported:++| Parameter | CLI Argument | Environment Variable | Description | Default | Required |+|-----------|--------------|---------------------|-------------|---------|----------|+| `evals_logs_export` | `--evals_logs_export` | `OPENLIT_EVALS_LOGS_EXPORT` | Emit evaluation results as OTel Log Records instead of OTel Events. Useful for backends like Grafana Loki that handle logs better than events. | `True` | No |+<Warning>**Security Notice**: Enabling `capture_parameters` records query parameters (the values passed to `$1`, `$2`, etc.) in your traces. This may expose sensitive data like passwords, API keys, or personal information. Only enable in development environments or when you're certain parameters don't contain sensitive data.</Warning>diff --git a/docs/latest/sdk/features/evaluations.mdx b/docs/latest/sdk/features/evaluations.mdxindex 4914b779..57e27df8 100644--- a/docs/latest/sdk/features/evaluations.mdx+++ b/docs/latest/sdk/features/evaluations.mdx@@ -22,6 +22,23 @@ The OpenLIT SDK provides tools to evaluate AI-generated text, ensuring it aligns</Card></CardGroup>+## Export configuration++By default, evaluation results are emitted as **OTel Log Records**. If your backend supports OTel Events natively, you can switch to event-based export:++```python+import openlit++# Use OTel Events instead of Log Records+openlit.init(evals_logs_export=False)+```++| Parameter | Environment Variable | Description | Default |+|-----------|---------------------|-------------|---------|+| `evals_logs_export` | `OPENLIT_EVALS_LOGS_EXPORT` | Emit evaluation results as OTel Log Records instead of OTel Events | `True` |++<Info>Both modes carry the same keys and value types — the only difference is the transport (event attributes vs JSON in the log body).</Info>+## Evaluations### Hallucination Detectiondiff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex daa928e4..06a869e7 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.37.2"+version = "1.37.4"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex 459186ad..ccafa4c3 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -82,6 +82,7 @@ class OpenlitConfig:# Database instrumentation optionscls.capture_parameters = Falsecls.enable_sqlcommenter = False+ cls.evals_logs_export = True@classmethoddef update_config(@@ -100,6 +101,7 @@ class OpenlitConfig:detailed_tracing,capture_parameters=False,enable_sqlcommenter=False,+ evals_logs_export=True,):"""Updates the configuration based on provided parameters.@@ -120,6 +122,7 @@ class OpenlitConfig:detailed_tracing (bool): Flag to enable detailed component-level tracing.capture_parameters (bool): Capture database query parameters (security risk).enable_sqlcommenter (bool): Inject trace context as SQL comments.+ evals_logs_export (bool): Emit evaluation results as OTEL Log Records instead of OTEL Events."""cls.environment = environmentcls.application_name = application_name@@ -135,6 +138,7 @@ class OpenlitConfig:cls.detailed_tracing = detailed_tracingcls.capture_parameters = capture_parameterscls.enable_sqlcommenter = enable_sqlcommenter+ cls.evals_logs_export = evals_logs_exportdef module_exists(module_name):@@ -229,6 +233,7 @@ def init(collect_system_metrics=False,capture_parameters=False,enable_sqlcommenter=False,+ evals_logs_export=True,):"""Initializes the openLIT configuration and setups tracing.@@ -308,6 +313,8 @@ def init(capture_parameters = env_config["capture_parameters"]if enable_sqlcommenter is False and "enable_sqlcommenter" in env_config:enable_sqlcommenter = env_config["enable_sqlcommenter"]+ if evals_logs_export is True and "evals_logs_export" in env_config:+ evals_logs_export = env_config["evals_logs_export"]except ImportError:# Fallback if config module is not available - continue without env var support@@ -392,6 +399,7 @@ def init(detailed_tracing,capture_parameters,enable_sqlcommenter,+ evals_logs_export,)# Create instrumentor instances dynamicallydiff --git a/sdk/python/src/openlit/cli/config.py b/sdk/python/src/openlit/cli/config.pyindex bb3e796d..33314363 100644--- a/sdk/python/src/openlit/cli/config.py+++ b/sdk/python/src/openlit/cli/config.py@@ -110,6 +110,12 @@ PARAMETER_CONFIG = {"cli_help": "Inject OpenTelemetry trace context as SQL comments for database log correlation","cli_type": bool,},+ "evals_logs_export": {+ "default": True,+ "env_var": "OPENLIT_EVALS_LOGS_EXPORT",+ "cli_help": "Emit evaluation results as OTEL Log Records instead of OTEL Events",+ "cli_type": bool,+ },# Parameters that are not exposed via CLI (internal use only)"tracer": {"default": None,diff --git a/sdk/python/src/openlit/evals/utils.py b/sdk/python/src/openlit/evals/utils.pyindex 55a39f26..6d3050aa 100644--- a/sdk/python/src/openlit/evals/utils.py+++ b/sdk/python/src/openlit/evals/utils.py@@ -145,13 +145,13 @@ def llm_response_openai(prompt: str, model: str, base_url: str) -> str:if base_url is None:base_url = "https://api.openai.com/v1"- response = client.beta.chat.completions.parse(+ response = client.chat.completions.create(model=model,messages=[{"role": "user", "content": prompt},],temperature=0.0,- response_format=JsonOutput,+ response_format={"type": "json_object"},)return response.choices[0].message.content@@ -271,6 +271,22 @@ def get_event_provider():return None+def get_evals_logs_export_config():+ """+ Safely retrieve the evals_logs_export flag from OpenLIT's global configuration.++ Returns:+ True if OTEL Log Records should be used instead of Events, else False.+ """+ try:+ # pylint: disable=cyclic-import+ from openlit import OpenlitConfig++ return getattr(OpenlitConfig, "evals_logs_export", True)+ except (ImportError, AttributeError):+ return True++def emit_evaluation_event(event_provider,evaluation_name,@@ -281,7 +297,14 @@ def emit_evaluation_event(error_type=None,):"""- Emit gen_ai.evaluation.result event per OTel semantic conventions.+ Emit gen_ai.evaluation.result as an OTel Event or OTel Log Record.++ By default, emits as an OTel Log Record. When ``evals_logs_export=False``+ is set in ``openlit.init()``, emits as an OTel Event instead (useful for+ backends that support events natively).++ Both paths carry the same keys and value types; the only difference is+ the transport (event attributes vs JSON string in the log body).Args:event_provider: The OTel event provider@@ -296,18 +319,13 @@ def emit_evaluation_event(if not event_provider:return- from openlit.__helpers import otel_event-- # Build event attributes per OTel semantic convention specattributes = {SemanticConvention.GEN_AI_EVALUATION_NAME: evaluation_name,}- # If error occurred, record error.type instead of score attributesif error_type:attributes[SemanticConvention.ERROR_TYPE] = error_typeelse:- # Record evaluation score and label (conditionally required if no error)if score_value is not None:attributes[SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE] = float(score_value@@ -317,20 +335,40 @@ def emit_evaluation_event(score_label)- # Add recommended attributes when availableif explanation:attributes[SemanticConvention.GEN_AI_EVALUATION_EXPLANATION] = explanationif response_id:attributes[SemanticConvention.GEN_AI_RESPONSE_ID] = response_id- # Create and emit event per OTel spec- event = otel_event(- name=SemanticConvention.GEN_AI_EVALUATION_RESULT,- attributes=attributes,- body="", # Per spec, all data must be in attributes, body is empty- )-- event_provider.emit(event)+ if get_evals_logs_export_config():+ _emit_as_log_record(json.dumps(attributes))+ else:+ _emit_as_event(event_provider, attributes)except Exception as e:logger.warning("Failed to emit evaluation event: %s", e, exc_info=True)+++def _emit_as_event(event_provider, attributes):+ """Emit evaluation result as an OTel Event (default path)."""+ from openlit.__helpers import otel_event++ event = otel_event(+ name=SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes=attributes,+ body="",+ )+ event_provider.emit(event)+++def _emit_as_log_record(body):+ """Emit evaluation result as an OTel Log Record via the Logger API directly."""+ from opentelemetry._logs import get_logger_provider, LogRecord, SeverityNumber++ otel_logger = get_logger_provider().get_logger("openlit.evals")+ otel_logger.emit(+ LogRecord(+ body=body,+ severity_number=SeverityNumber.INFO,+ )+ )diff --git a/sdk/python/src/openlit/guard/utils.py b/sdk/python/src/openlit/guard/utils.pyindex c8fc43e2..9a8c7469 100644--- a/sdk/python/src/openlit/guard/utils.py+++ b/sdk/python/src/openlit/guard/utils.py@@ -98,13 +98,13 @@ def llm_response_openai(prompt: str, model: str, base_url: str) -> str:if base_url is None:base_url = "https://api.openai.com/v1"- response = client.beta.chat.completions.parse(+ response = client.chat.completions.create(model=model,messages=[{"role": "user", "content": prompt},],temperature=0.0,- response_format=JsonOutput,+ response_format={"type": "json_object"},)return response.choices[0].message.content