Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

feat: Update Evals to align with OTel semconv (#1022)

Ishan Jain2a837651b02e
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index 4b105200..daa928e4 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.37.1"
+version = "1.37.2"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/evals/all.py b/sdk/python/src/openlit/evals/all.py
index f3763c91..d682914e 100644
--- a/sdk/python/src/openlit/evals/all.py
+++ b/sdk/python/src/openlit/evals/all.py
@@ -1,8 +1,9 @@
-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes
+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import
"""
Module for finding Hallucination, Bias and Toxicity in text.
"""
+import logging
from typing import Optional, List, Dict
from openlit.evals.utils import (
setup_provider,
@@ -10,10 +11,10 @@ from openlit.evals.utils import (
format_prompt,
llm_response,
parse_llm_response,
- eval_metrics,
- eval_metric_attributes,
)
+logger = logging.getLogger(__name__)
+
def get_system_prompt(
custom_categories: Optional[Dict[str, str]] = None,
@@ -123,7 +124,6 @@ class All:
model: Optional[str] = None,
base_url: Optional[str] = None,
custom_categories: Optional[Dict[str, str]] = None,
- collect_metrics: Optional[bool] = False,
threshold_score: Optional[float] = 0.5,
event_provider=None,
):
@@ -147,10 +147,11 @@ class All:
self.api_key, self.model, self.base_url = setup_provider(
provider, api_key, model, base_url
)
- self.collect_metrics = collect_metrics
self.custom_categories = custom_categories
self.threshold_score = threshold_score
- self.event_provider = event_provider
+ # Note: event_provider parameter retained for explicit passing, but defaults to
+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()
+ self._explicit_event_provider = event_provider
self.system_prompt = get_system_prompt(
self.custom_categories, self.threshold_score
)
@@ -160,54 +161,82 @@ class All:
prompt: Optional[str] = "",
contexts: Optional[List[str]] = None,
text: Optional[str] = None,
+ response_id: Optional[str] = None,
) -> JsonOutput:
"""
- Detects toxicity in AI output using LLM or custom rules.
+ Detects hallucination, bias and toxicity in AI output using LLM or custom rules.
Args:
prompt (Optional[str]): The prompt provided by the user.
contexts (Optional[List[str]]): A list of context sentences relevant to the task.
text (Optional[str]): The text to analyze.
+ response_id (Optional[str]): The unique identifier for the completion being evaluated.
Returns:
JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of evaluation.
"""
+ from openlit.evals.utils import get_event_provider
- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)
- llm_result = parse_llm_response(response)
- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
-
- result = JsonOutput(
- score=llm_result.score,
- evaluation=llm_result.evaluation,
- classification=llm_result.classification,
- explanation=llm_result.explanation,
- verdict=result_verdict,
- )
+ # Use explicitly passed provider if available, else auto-retrieve from config
+ event_provider = self._explicit_event_provider or get_event_provider()
- if self.collect_metrics:
- eval_counter = eval_metrics()
- attributes = eval_metric_attributes(
- result_verdict,
- result.score,
- result.evaluation,
- result.classification,
- result.explanation,
+ try:
+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
+ response = llm_response(
+ self.provider, llm_prompt, self.model, self.base_url
)
- eval_counter.add(1, attributes)
-
- # Emit evaluation event
- if self.event_provider:
- from openlit.evals.utils import emit_evaluation_event
-
- emit_evaluation_event(
- event_provider=self.event_provider,
- evaluation_name=result.evaluation,
- score_value=result.score,
- score_label=result_verdict,
- explanation=result.explanation,
- response_id=None,
+ llm_result = parse_llm_response(response)
+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
+
+ result = JsonOutput(
+ score=llm_result.score,
+ evaluation=llm_result.evaluation,
+ classification=llm_result.classification,
+ explanation=llm_result.explanation,
+ verdict=result_verdict,
)
- return result
+ # Emit evaluation event with OTel-compliant semantic label
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ # Map verdict to pass/fail per OTel spec
+ score_label = (
+ "pass" if llm_result.score <= self.threshold_score else "fail"
+ )
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name=result.evaluation,
+ score_value=result.score,
+ score_label=score_label,
+ explanation=result.explanation,
+ response_id=response_id,
+ )
+
+ return result
+
+ except Exception as e:
+ logger.error("Evaluation failed: %s", e, exc_info=True)
+
+ # Emit error event if provider available
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name="evaluation",
+ error_type="provider_error"
+ if "provider" in str(e).lower()
+ else "unknown",
+ response_id=response_id,
+ )
+
+ # Return neutral result to allow continued processing
+ return JsonOutput(
+ score=0.0,
+ evaluation="evaluation",
+ classification="error",
+ explanation=f"Evaluation failed: {str(e)}",
+ verdict="no",
+ )
diff --git a/sdk/python/src/openlit/evals/bias_detection.py b/sdk/python/src/openlit/evals/bias_detection.py
index b68eec95..9e104d06 100644
--- a/sdk/python/src/openlit/evals/bias_detection.py
+++ b/sdk/python/src/openlit/evals/bias_detection.py
@@ -1,8 +1,9 @@
-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes
+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import
"""
Module for finding Bias in text.
"""
+import logging
from typing import Optional, List, Dict
from openlit.evals.utils import (
setup_provider,
@@ -10,10 +11,10 @@ from openlit.evals.utils import (
format_prompt,
llm_response,
parse_llm_response,
- eval_metrics,
- eval_metric_attributes,
)
+logger = logging.getLogger(__name__)
+
def get_system_prompt(
custom_categories: Optional[Dict[str, str]] = None,
@@ -125,7 +126,6 @@ class BiasDetector:
model: Optional[str] = None,
base_url: Optional[str] = None,
custom_categories: Optional[Dict[str, str]] = None,
- collect_metrics: Optional[bool] = False,
threshold_score: Optional[float] = 0.5,
event_provider=None,
):
@@ -149,10 +149,11 @@ class BiasDetector:
self.api_key, self.model, self.base_url = setup_provider(
provider, api_key, model, base_url
)
- self.collect_metrics = collect_metrics
self.custom_categories = custom_categories
self.threshold_score = threshold_score
- self.event_provider = event_provider
+ # Note: event_provider parameter retained for explicit passing, but defaults to
+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()
+ self._explicit_event_provider = event_provider
self.system_prompt = get_system_prompt(
self.custom_categories, self.threshold_score
)
@@ -162,54 +163,82 @@ class BiasDetector:
prompt: Optional[str] = "",
contexts: Optional[List[str]] = None,
text: Optional[str] = None,
+ response_id: Optional[str] = None,
) -> JsonOutput:
"""
- Detects toxicity in AI output using LLM or custom rules.
+ Detects bias in AI output using LLM or custom rules.
Args:
prompt (Optional[str]): The prompt provided by the user.
contexts (Optional[List[str]]): A list of context sentences relevant to the task.
text (Optional[str]): The text to analyze.
+ response_id (Optional[str]): The unique identifier for the completion being evaluated.
Returns:
JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of bias detection.
"""
+ from openlit.evals.utils import get_event_provider
- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)
- llm_result = parse_llm_response(response)
- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
-
- result = JsonOutput(
- score=llm_result.score,
- evaluation=llm_result.evaluation,
- classification=llm_result.classification,
- explanation=llm_result.explanation,
- verdict=result_verdict,
- )
+ # Use explicitly passed provider if available, else auto-retrieve from config
+ event_provider = self._explicit_event_provider or get_event_provider()
- if self.collect_metrics:
- eval_counter = eval_metrics()
- attributes = eval_metric_attributes(
- result_verdict,
- result.score,
- result.evaluation,
- result.classification,
- result.explanation,
+ try:
+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
+ response = llm_response(
+ self.provider, llm_prompt, self.model, self.base_url
)
- eval_counter.add(1, attributes)
-
- # Emit evaluation event
- if self.event_provider:
- from openlit.evals.utils import emit_evaluation_event
-
- emit_evaluation_event(
- event_provider=self.event_provider,
- evaluation_name=result.evaluation,
- score_value=result.score,
- score_label=result_verdict,
- explanation=result.explanation,
- response_id=None,
+ llm_result = parse_llm_response(response)
+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
+
+ result = JsonOutput(
+ score=llm_result.score,
+ evaluation=llm_result.evaluation,
+ classification=llm_result.classification,
+ explanation=llm_result.explanation,
+ verdict=result_verdict,
)
- return result
+ # Emit evaluation event with OTel-compliant semantic label
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ # Map verdict to pass/fail per OTel spec
+ score_label = (
+ "pass" if llm_result.score <= self.threshold_score else "fail"
+ )
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name=result.evaluation,
+ score_value=result.score,
+ score_label=score_label,
+ explanation=result.explanation,
+ response_id=response_id,
+ )
+
+ return result
+
+ except Exception as e:
+ logger.error("Bias detection failed: %s", e, exc_info=True)
+
+ # Emit error event if provider available
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name="bias_detection",
+ error_type="provider_error"
+ if "provider" in str(e).lower()
+ else "unknown",
+ response_id=response_id,
+ )
+
+ # Return neutral result to allow continued processing
+ return JsonOutput(
+ score=0.0,
+ evaluation="bias_detection",
+ classification="error",
+ explanation=f"Evaluation failed: {str(e)}",
+ verdict="no",
+ )
diff --git a/sdk/python/src/openlit/evals/hallucination.py b/sdk/python/src/openlit/evals/hallucination.py
index 7b11388f..b9efa054 100644
--- a/sdk/python/src/openlit/evals/hallucination.py
+++ b/sdk/python/src/openlit/evals/hallucination.py
@@ -1,8 +1,9 @@
-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes
+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import
"""
Module for finding Hallucination in text.
"""
+import logging
from typing import Optional, List, Dict
from openlit.evals.utils import (
setup_provider,
@@ -10,10 +11,10 @@ from openlit.evals.utils import (
format_prompt,
llm_response,
parse_llm_response,
- eval_metrics,
- eval_metric_attributes,
)
+logger = logging.getLogger(__name__)
+
# pylint: disable=unused-argument
def get_system_prompt(
@@ -125,7 +126,6 @@ class Hallucination:
model: Optional[str] = None,
base_url: Optional[str] = None,
custom_categories: Optional[Dict[str, str]] = None,
- collect_metrics: Optional[bool] = False,
threshold_score: Optional[float] = 0.5,
event_provider=None,
):
@@ -151,10 +151,11 @@ class Hallucination:
self.api_key, self.model, self.base_url = setup_provider(
provider, api_key, model, base_url
)
- self.collect_metrics = collect_metrics
self.custom_categories = custom_categories
self.threshold_score = threshold_score
- self.event_provider = event_provider
+ # Note: event_provider parameter retained for explicit passing, but defaults to
+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()
+ self._explicit_event_provider = event_provider
self.system_prompt = get_system_prompt(
self.custom_categories, self.threshold_score
)
@@ -164,6 +165,7 @@ class Hallucination:
prompt: Optional[str] = "",
contexts: Optional[List[str]] = None,
text: Optional[str] = None,
+ response_id: Optional[str] = None,
) -> JsonOutput:
"""
Detects hallucinations in AI output using LLM or custom rules.
@@ -172,45 +174,72 @@ class Hallucination:
prompt (Optional[str]): The prompt provided by the user.
contexts (Optional[List[str]]): A list of context sentences relevant to the task.
text (Optional[str]): The text to analyze.
+ response_id (Optional[str]): The unique identifier for the completion being evaluated.
Returns:
JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of hallucination detection.
"""
+ from openlit.evals.utils import get_event_provider
- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)
- llm_result = parse_llm_response(response)
- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
- result = JsonOutput(
- score=llm_result.score,
- evaluation=llm_result.evaluation,
- classification=llm_result.classification,
- explanation=llm_result.explanation,
- verdict=result_verdict,
- )
+ # Use explicitly passed provider if available, else auto-retrieve from config
+ event_provider = self._explicit_event_provider or get_event_provider()
- if self.collect_metrics:
- eval_counter = eval_metrics()
- attributes = eval_metric_attributes(
- result_verdict,
- result.score,
- result.evaluation,
- result.classification,
- result.explanation,
+ try:
+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
+ response = llm_response(
+ self.provider, llm_prompt, self.model, self.base_url
)
- eval_counter.add(1, attributes)
-
- # Emit evaluation event
- if self.event_provider:
- from openlit.evals.utils import emit_evaluation_event
-
- emit_evaluation_event(
- event_provider=self.event_provider,
- evaluation_name=result.evaluation,
- score_value=result.score,
- score_label=result_verdict,
- explanation=result.explanation,
- response_id=None,
+ llm_result = parse_llm_response(response)
+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
+ result = JsonOutput(
+ score=llm_result.score,
+ evaluation=llm_result.evaluation,
+ classification=llm_result.classification,
+ explanation=llm_result.explanation,
+ verdict=result_verdict,
)
- return result
+ # Emit evaluation event with OTel-compliant semantic label
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ # Map verdict to pass/fail per OTel spec
+ score_label = (
+ "pass" if llm_result.score <= self.threshold_score else "fail"
+ )
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name=result.evaluation,
+ score_value=result.score,
+ score_label=score_label,
+ explanation=result.explanation,
+ response_id=response_id,
+ )
+
+ return result
+
+ except Exception as e:
+ logger.error("Hallucination detection failed: %s", e, exc_info=True)
+
+ # Emit error event if provider available
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name="hallucination",
+ error_type="provider_error"
+ if "provider" in str(e).lower()
+ else "unknown",
+ response_id=response_id,
+ )
+
+ # Return neutral result to allow continued processing
+ return JsonOutput(
+ score=0.0,
+ evaluation="hallucination",
+ classification="error",
+ explanation=f"Evaluation failed: {str(e)}",
+ verdict="no",
+ )
diff --git a/sdk/python/src/openlit/evals/toxicity.py b/sdk/python/src/openlit/evals/toxicity.py
index 98d1acd3..bb952f4a 100644
--- a/sdk/python/src/openlit/evals/toxicity.py
+++ b/sdk/python/src/openlit/evals/toxicity.py
@@ -1,8 +1,9 @@
-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes
+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import
"""
Module for finding Toxicity in text.
"""
+import logging
from typing import Optional, List, Dict
from openlit.evals.utils import (
setup_provider,
@@ -10,10 +11,10 @@ from openlit.evals.utils import (
format_prompt,
llm_response,
parse_llm_response,
- eval_metrics,
- eval_metric_attributes,
)
+logger = logging.getLogger(__name__)
+
def get_system_prompt(
custom_categories: Optional[Dict[str, str]] = None,
@@ -120,7 +121,6 @@ class ToxicityDetector:
model: Optional[str] = None,
base_url: Optional[str] = None,
custom_categories: Optional[Dict[str, str]] = None,
- collect_metrics: Optional[bool] = False,
threshold_score: Optional[float] = 0.5,
event_provider=None,
):
@@ -146,10 +146,11 @@ class ToxicityDetector:
self.api_key, self.model, self.base_url = setup_provider(
provider, api_key, model, base_url
)
- self.collect_metrics = collect_metrics
self.custom_categories = custom_categories
self.threshold_score = threshold_score
- self.event_provider = event_provider
+ # Note: event_provider parameter retained for explicit passing, but defaults to
+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()
+ self._explicit_event_provider = event_provider
self.system_prompt = get_system_prompt(
self.custom_categories, self.threshold_score
)
@@ -159,6 +160,7 @@ class ToxicityDetector:
prompt: Optional[str] = "",
contexts: Optional[List[str]] = "",
text: Optional[str] = None,
+ response_id: Optional[str] = None,
) -> JsonOutput:
"""
Detects toxicity in AI output using LLM or custom rules.
@@ -167,46 +169,73 @@ class ToxicityDetector:
prompt (Optional[str]): The prompt provided by the user.
contexts (Optional[List[str]]): A list of context sentences relevant to the task.
text (Optional[str]): The text to analyze.
+ response_id (Optional[str]): The unique identifier for the completion being evaluated.
Returns:
JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of toxicity detection.
"""
+ from openlit.evals.utils import get_event_provider
- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)
- llm_result = parse_llm_response(response)
- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
-
- result = JsonOutput(
- score=llm_result.score,
- evaluation=llm_result.evaluation,
- classification=llm_result.classification,
- explanation=llm_result.explanation,
- verdict=result_verdict,
- )
+ # Use explicitly passed provider if available, else auto-retrieve from config
+ event_provider = self._explicit_event_provider or get_event_provider()
- if self.collect_metrics:
- eval_counter = eval_metrics()
- attributes = eval_metric_attributes(
- result_verdict,
- result.score,
- result.evaluation,
- result.classification,
- result.explanation,
+ try:
+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)
+ response = llm_response(
+ self.provider, llm_prompt, self.model, self.base_url
)
- eval_counter.add(1, attributes)
-
- # Emit evaluation event
- if self.event_provider:
- from openlit.evals.utils import emit_evaluation_event
-
- emit_evaluation_event(
- event_provider=self.event_provider,
- evaluation_name=result.evaluation,
- score_value=result.score,
- score_label=result_verdict,
- explanation=result.explanation,
- response_id=None,
+ llm_result = parse_llm_response(response)
+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"
+
+ result = JsonOutput(
+ score=llm_result.score,
+ evaluation=llm_result.evaluation,
+ classification=llm_result.classification,
+ explanation=llm_result.explanation,
+ verdict=result_verdict,
)
- return result
+ # Emit evaluation event with OTel-compliant semantic label
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ # Map verdict to pass/fail per OTel spec
+ score_label = (
+ "pass" if llm_result.score <= self.threshold_score else "fail"
+ )
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name=result.evaluation,
+ score_value=result.score,
+ score_label=score_label,
+ explanation=result.explanation,
+ response_id=response_id,
+ )
+
+ return result
+
+ except Exception as e:
+ logger.error("Toxicity detection failed: %s", e, exc_info=True)
+
+ # Emit error event if provider available
+ if event_provider:
+ from openlit.evals.utils import emit_evaluation_event
+
+ emit_evaluation_event(
+ event_provider=event_provider,
+ evaluation_name="toxicity_detection",
+ error_type="provider_error"
+ if "provider" in str(e).lower()
+ else "unknown",
+ response_id=response_id,
+ )
+
+ # Return neutral result to allow continued processing
+ return JsonOutput(
+ score=0.0,
+ evaluation="toxicity_detection",
+ classification="error",
+ explanation=f"Evaluation failed: {str(e)}",
+ verdict="no",
+ )
diff --git a/sdk/python/src/openlit/evals/utils.py b/sdk/python/src/openlit/evals/utils.py
index 5e6954aa..55a39f26 100644
--- a/sdk/python/src/openlit/evals/utils.py
+++ b/sdk/python/src/openlit/evals/utils.py
@@ -6,8 +6,6 @@ import os
import logging
from typing import Optional, Tuple, List
from pydantic import BaseModel
-from opentelemetry.metrics import get_meter
-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
from anthropic import Anthropic
from openai import OpenAI
from openlit.semcov import SemanticConvention
@@ -252,71 +250,47 @@ def parse_llm_response(response) -> JsonOutput:
)
-def eval_metrics():
+def get_event_provider():
"""
- Initializes OpenTelemetry meter and counter.
-
- Returns:
- counter: The initialized telemetry counter.
- """
-
- meter = get_meter(
- __name__,
- "0.1.0",
- schema_url="https://opentelemetry.io/schemas/1.11.0",
- )
-
- guard_requests = meter.create_counter(
- name=SemanticConvention.EVAL_REQUESTS,
- description="Counter for evaluation requests",
- unit="1",
- )
-
- return guard_requests
+ Safely retrieve the event provider from OpenLIT's global configuration.
-
-def eval_metric_attributes(verdict, score, validator, classification, explanation):
- """
- Initializes OpenTelemetry attributes for metrics.
-
- Args:
- score (float): The name of the attribute for eval Score.
- validator (str): The name of the attribute for eval.
- classification (str): The name of the attribute for eval classification.
- explaination (str): The name of the attribute for eval explanation.
+ This function enables evaluators to auto-wire event emission without storing
+ references that cause cyclic imports. The provider is retrieved at call time,
+ allowing OpenLIT initialization to complete before evaluation measure() calls.
Returns:
- counter: The initialized telemetry counter.
+ The event provider if OpenLIT has been initialized with telemetry, else None.
"""
+ try:
+ # pylint: disable=cyclic-import
+ # (Import is inside function body, executed only after openlit.init())
+ from openlit import OpenlitConfig
- return {
- TELEMETRY_SDK_NAME: "openlit",
- SemanticConvention.EVAL_VERDICT: verdict,
- SemanticConvention.EVAL_SCORE: score,
- SemanticConvention.EVAL_VALIDATOR: validator,
- SemanticConvention.EVAL_CLASSIFICATION: classification,
- SemanticConvention.EVAL_EXPLANATION: explanation,
- }
+ return OpenlitConfig.event_provider
+ except (ImportError, AttributeError):
+ return None
def emit_evaluation_event(
event_provider,
evaluation_name,
- score_value,
- score_label,
- explanation,
+ score_value=None,
+ score_label=None,
+ explanation=None,
response_id=None,
+ error_type=None,
):
"""
- Emit gen_ai.evaluation.result event.
+ Emit gen_ai.evaluation.result event per OTel semantic conventions.
Args:
event_provider: The OTel event provider
evaluation_name: Name of evaluation (hallucination, bias_detection, toxicity_detection)
- score_value: Numerical score 0.0-1.0
- score_label: Human-readable label (yes/no or pass/fail)
- explanation: Brief explanation of evaluation result
- response_id: Optional response ID for correlation
+ score_value: Numerical score 0.0-1.0 (conditionally required)
+ score_label: Human-readable label (yes/no or pass/fail) (conditionally required)
+ explanation: Brief explanation of evaluation result (recommended)
+ response_id: Optional response ID for correlation (recommended when available)
+ error_type: Error type if evaluation failed (conditionally required if error occurs)
"""
try:
if not event_provider:
@@ -324,24 +298,36 @@ def emit_evaluation_event(
from openlit.__helpers import otel_event
- # Build event attributes per OTel spec
+ # Build event attributes per OTel semantic convention spec
attributes = {
SemanticConvention.GEN_AI_EVALUATION_NAME: evaluation_name,
- SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: float(score_value),
- SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: score_label,
}
- # Add recommended attributes
+ # If error occurred, record error.type instead of score attributes
+ if error_type:
+ attributes[SemanticConvention.ERROR_TYPE] = error_type
+ else:
+ # Record evaluation score and label (conditionally required if no error)
+ if score_value is not None:
+ attributes[SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE] = float(
+ score_value
+ )
+ if score_label:
+ attributes[SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL] = (
+ score_label
+ )
+
+ # Add recommended attributes when available
if explanation:
attributes[SemanticConvention.GEN_AI_EVALUATION_EXPLANATION] = explanation
if response_id:
attributes[SemanticConvention.GEN_AI_RESPONSE_ID] = response_id
- # Create and emit event
+ # Create and emit event per OTel spec
event = otel_event(
name=SemanticConvention.GEN_AI_EVALUATION_RESULT,
attributes=attributes,
- body="", # Per spec, all data in attributes
+ body="", # Per spec, all data must be in attributes, body is empty
)
event_provider.emit(event)
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index 1ece3e94..a089c59a 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -918,15 +918,9 @@ class SemanticConvention:
GUARD_VALIDATOR = "guard.validator"
GUARD_EXPLANATION = "guard.explanation"
- # Evals
- EVAL_REQUESTS = "evals.requests"
- EVAL_VERDICT = "evals.verdict"
- EVAL_SCORE = "evals.score"
- EVAL_CLASSIFICATION = "evals.classification"
- EVAL_VALIDATOR = "evals.validator"
- EVAL_EXPLANATION = "evals.explanation"
-
- # GenAI Evaluation Event (OTel Semconv)
+ # GenAI Evaluation Event (OTel Semantic Convention)
+ # Per OpenTelemetry semantic conventions for generative AI
+ # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-events.md#event-gen_aievaluationresult
GEN_AI_EVALUATION_RESULT = "gen_ai.evaluation.result"
GEN_AI_EVALUATION_NAME = "gen_ai.evaluation.name"
GEN_AI_EVALUATION_SCORE_VALUE = "gen_ai.evaluation.score.value"