Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: Update Evals to align with OTel semconv (#1022)
Ishan Jain2a837651b02e
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 4b105200..daa928e4 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.37.1"+version = "1.37.2"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/evals/all.py b/sdk/python/src/openlit/evals/all.pyindex f3763c91..d682914e 100644--- a/sdk/python/src/openlit/evals/all.py+++ b/sdk/python/src/openlit/evals/all.py@@ -1,8 +1,9 @@-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import"""Module for finding Hallucination, Bias and Toxicity in text."""+import loggingfrom typing import Optional, List, Dictfrom openlit.evals.utils import (setup_provider,@@ -10,10 +11,10 @@ from openlit.evals.utils import (format_prompt,llm_response,parse_llm_response,- eval_metrics,- eval_metric_attributes,)+logger = logging.getLogger(__name__)+def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,@@ -123,7 +124,6 @@ class All:model: Optional[str] = None,base_url: Optional[str] = None,custom_categories: Optional[Dict[str, str]] = None,- collect_metrics: Optional[bool] = False,threshold_score: Optional[float] = 0.5,event_provider=None,):@@ -147,10 +147,11 @@ class All:self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)- self.collect_metrics = collect_metricsself.custom_categories = custom_categoriesself.threshold_score = threshold_score- self.event_provider = event_provider+ # Note: event_provider parameter retained for explicit passing, but defaults to+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()+ self._explicit_event_provider = event_providerself.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)@@ -160,54 +161,82 @@ class All:prompt: Optional[str] = "",contexts: Optional[List[str]] = None,text: Optional[str] = None,+ response_id: Optional[str] = None,) -> JsonOutput:"""- Detects toxicity in AI output using LLM or custom rules.+ Detects hallucination, bias and toxicity in AI output using LLM or custom rules.Args:prompt (Optional[str]): The prompt provided by the user.contexts (Optional[List[str]]): A list of context sentences relevant to the task.text (Optional[str]): The text to analyze.+ response_id (Optional[str]): The unique identifier for the completion being evaluated.Returns:JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of evaluation."""+ from openlit.evals.utils import get_event_provider- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)- llm_result = parse_llm_response(response)- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"-- result = JsonOutput(- score=llm_result.score,- evaluation=llm_result.evaluation,- classification=llm_result.classification,- explanation=llm_result.explanation,- verdict=result_verdict,- )+ # Use explicitly passed provider if available, else auto-retrieve from config+ event_provider = self._explicit_event_provider or get_event_provider()- if self.collect_metrics:- eval_counter = eval_metrics()- attributes = eval_metric_attributes(- result_verdict,- result.score,- result.evaluation,- result.classification,- result.explanation,+ try:+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)+ response = llm_response(+ self.provider, llm_prompt, self.model, self.base_url)- eval_counter.add(1, attributes)-- # Emit evaluation event- if self.event_provider:- from openlit.evals.utils import emit_evaluation_event-- emit_evaluation_event(- event_provider=self.event_provider,- evaluation_name=result.evaluation,- score_value=result.score,- score_label=result_verdict,- explanation=result.explanation,- response_id=None,+ llm_result = parse_llm_response(response)+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"++ result = JsonOutput(+ score=llm_result.score,+ evaluation=llm_result.evaluation,+ classification=llm_result.classification,+ explanation=llm_result.explanation,+ verdict=result_verdict,)- return result+ # Emit evaluation event with OTel-compliant semantic label+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ # Map verdict to pass/fail per OTel spec+ score_label = (+ "pass" if llm_result.score <= self.threshold_score else "fail"+ )++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name=result.evaluation,+ score_value=result.score,+ score_label=score_label,+ explanation=result.explanation,+ response_id=response_id,+ )++ return result++ except Exception as e:+ logger.error("Evaluation failed: %s", e, exc_info=True)++ # Emit error event if provider available+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name="evaluation",+ error_type="provider_error"+ if "provider" in str(e).lower()+ else "unknown",+ response_id=response_id,+ )++ # Return neutral result to allow continued processing+ return JsonOutput(+ score=0.0,+ evaluation="evaluation",+ classification="error",+ explanation=f"Evaluation failed: {str(e)}",+ verdict="no",+ )diff --git a/sdk/python/src/openlit/evals/bias_detection.py b/sdk/python/src/openlit/evals/bias_detection.pyindex b68eec95..9e104d06 100644--- a/sdk/python/src/openlit/evals/bias_detection.py+++ b/sdk/python/src/openlit/evals/bias_detection.py@@ -1,8 +1,9 @@-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import"""Module for finding Bias in text."""+import loggingfrom typing import Optional, List, Dictfrom openlit.evals.utils import (setup_provider,@@ -10,10 +11,10 @@ from openlit.evals.utils import (format_prompt,llm_response,parse_llm_response,- eval_metrics,- eval_metric_attributes,)+logger = logging.getLogger(__name__)+def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,@@ -125,7 +126,6 @@ class BiasDetector:model: Optional[str] = None,base_url: Optional[str] = None,custom_categories: Optional[Dict[str, str]] = None,- collect_metrics: Optional[bool] = False,threshold_score: Optional[float] = 0.5,event_provider=None,):@@ -149,10 +149,11 @@ class BiasDetector:self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)- self.collect_metrics = collect_metricsself.custom_categories = custom_categoriesself.threshold_score = threshold_score- self.event_provider = event_provider+ # Note: event_provider parameter retained for explicit passing, but defaults to+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()+ self._explicit_event_provider = event_providerself.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)@@ -162,54 +163,82 @@ class BiasDetector:prompt: Optional[str] = "",contexts: Optional[List[str]] = None,text: Optional[str] = None,+ response_id: Optional[str] = None,) -> JsonOutput:"""- Detects toxicity in AI output using LLM or custom rules.+ Detects bias in AI output using LLM or custom rules.Args:prompt (Optional[str]): The prompt provided by the user.contexts (Optional[List[str]]): A list of context sentences relevant to the task.text (Optional[str]): The text to analyze.+ response_id (Optional[str]): The unique identifier for the completion being evaluated.Returns:JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of bias detection."""+ from openlit.evals.utils import get_event_provider- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)- llm_result = parse_llm_response(response)- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"-- result = JsonOutput(- score=llm_result.score,- evaluation=llm_result.evaluation,- classification=llm_result.classification,- explanation=llm_result.explanation,- verdict=result_verdict,- )+ # Use explicitly passed provider if available, else auto-retrieve from config+ event_provider = self._explicit_event_provider or get_event_provider()- if self.collect_metrics:- eval_counter = eval_metrics()- attributes = eval_metric_attributes(- result_verdict,- result.score,- result.evaluation,- result.classification,- result.explanation,+ try:+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)+ response = llm_response(+ self.provider, llm_prompt, self.model, self.base_url)- eval_counter.add(1, attributes)-- # Emit evaluation event- if self.event_provider:- from openlit.evals.utils import emit_evaluation_event-- emit_evaluation_event(- event_provider=self.event_provider,- evaluation_name=result.evaluation,- score_value=result.score,- score_label=result_verdict,- explanation=result.explanation,- response_id=None,+ llm_result = parse_llm_response(response)+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"++ result = JsonOutput(+ score=llm_result.score,+ evaluation=llm_result.evaluation,+ classification=llm_result.classification,+ explanation=llm_result.explanation,+ verdict=result_verdict,)- return result+ # Emit evaluation event with OTel-compliant semantic label+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ # Map verdict to pass/fail per OTel spec+ score_label = (+ "pass" if llm_result.score <= self.threshold_score else "fail"+ )++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name=result.evaluation,+ score_value=result.score,+ score_label=score_label,+ explanation=result.explanation,+ response_id=response_id,+ )++ return result++ except Exception as e:+ logger.error("Bias detection failed: %s", e, exc_info=True)++ # Emit error event if provider available+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name="bias_detection",+ error_type="provider_error"+ if "provider" in str(e).lower()+ else "unknown",+ response_id=response_id,+ )++ # Return neutral result to allow continued processing+ return JsonOutput(+ score=0.0,+ evaluation="bias_detection",+ classification="error",+ explanation=f"Evaluation failed: {str(e)}",+ verdict="no",+ )diff --git a/sdk/python/src/openlit/evals/hallucination.py b/sdk/python/src/openlit/evals/hallucination.pyindex 7b11388f..b9efa054 100644--- a/sdk/python/src/openlit/evals/hallucination.py+++ b/sdk/python/src/openlit/evals/hallucination.py@@ -1,8 +1,9 @@-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import"""Module for finding Hallucination in text."""+import loggingfrom typing import Optional, List, Dictfrom openlit.evals.utils import (setup_provider,@@ -10,10 +11,10 @@ from openlit.evals.utils import (format_prompt,llm_response,parse_llm_response,- eval_metrics,- eval_metric_attributes,)+logger = logging.getLogger(__name__)+# pylint: disable=unused-argumentdef get_system_prompt(@@ -125,7 +126,6 @@ class Hallucination:model: Optional[str] = None,base_url: Optional[str] = None,custom_categories: Optional[Dict[str, str]] = None,- collect_metrics: Optional[bool] = False,threshold_score: Optional[float] = 0.5,event_provider=None,):@@ -151,10 +151,11 @@ class Hallucination:self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)- self.collect_metrics = collect_metricsself.custom_categories = custom_categoriesself.threshold_score = threshold_score- self.event_provider = event_provider+ # Note: event_provider parameter retained for explicit passing, but defaults to+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()+ self._explicit_event_provider = event_providerself.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)@@ -164,6 +165,7 @@ class Hallucination:prompt: Optional[str] = "",contexts: Optional[List[str]] = None,text: Optional[str] = None,+ response_id: Optional[str] = None,) -> JsonOutput:"""Detects hallucinations in AI output using LLM or custom rules.@@ -172,45 +174,72 @@ class Hallucination:prompt (Optional[str]): The prompt provided by the user.contexts (Optional[List[str]]): A list of context sentences relevant to the task.text (Optional[str]): The text to analyze.+ response_id (Optional[str]): The unique identifier for the completion being evaluated.Returns:JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of hallucination detection."""+ from openlit.evals.utils import get_event_provider- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)- llm_result = parse_llm_response(response)- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"- result = JsonOutput(- score=llm_result.score,- evaluation=llm_result.evaluation,- classification=llm_result.classification,- explanation=llm_result.explanation,- verdict=result_verdict,- )+ # Use explicitly passed provider if available, else auto-retrieve from config+ event_provider = self._explicit_event_provider or get_event_provider()- if self.collect_metrics:- eval_counter = eval_metrics()- attributes = eval_metric_attributes(- result_verdict,- result.score,- result.evaluation,- result.classification,- result.explanation,+ try:+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)+ response = llm_response(+ self.provider, llm_prompt, self.model, self.base_url)- eval_counter.add(1, attributes)-- # Emit evaluation event- if self.event_provider:- from openlit.evals.utils import emit_evaluation_event-- emit_evaluation_event(- event_provider=self.event_provider,- evaluation_name=result.evaluation,- score_value=result.score,- score_label=result_verdict,- explanation=result.explanation,- response_id=None,+ llm_result = parse_llm_response(response)+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"+ result = JsonOutput(+ score=llm_result.score,+ evaluation=llm_result.evaluation,+ classification=llm_result.classification,+ explanation=llm_result.explanation,+ verdict=result_verdict,)- return result+ # Emit evaluation event with OTel-compliant semantic label+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ # Map verdict to pass/fail per OTel spec+ score_label = (+ "pass" if llm_result.score <= self.threshold_score else "fail"+ )++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name=result.evaluation,+ score_value=result.score,+ score_label=score_label,+ explanation=result.explanation,+ response_id=response_id,+ )++ return result++ except Exception as e:+ logger.error("Hallucination detection failed: %s", e, exc_info=True)++ # Emit error event if provider available+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name="hallucination",+ error_type="provider_error"+ if "provider" in str(e).lower()+ else "unknown",+ response_id=response_id,+ )++ # Return neutral result to allow continued processing+ return JsonOutput(+ score=0.0,+ evaluation="hallucination",+ classification="error",+ explanation=f"Evaluation failed: {str(e)}",+ verdict="no",+ )diff --git a/sdk/python/src/openlit/evals/toxicity.py b/sdk/python/src/openlit/evals/toxicity.pyindex 98d1acd3..bb952f4a 100644--- a/sdk/python/src/openlit/evals/toxicity.py+++ b/sdk/python/src/openlit/evals/toxicity.py@@ -1,8 +1,9 @@-# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes+# pylint: disable=duplicate-code, line-too-long, too-few-public-methods, too-many-instance-attributes, cyclic-import"""Module for finding Toxicity in text."""+import loggingfrom typing import Optional, List, Dictfrom openlit.evals.utils import (setup_provider,@@ -10,10 +11,10 @@ from openlit.evals.utils import (format_prompt,llm_response,parse_llm_response,- eval_metrics,- eval_metric_attributes,)+logger = logging.getLogger(__name__)+def get_system_prompt(custom_categories: Optional[Dict[str, str]] = None,@@ -120,7 +121,6 @@ class ToxicityDetector:model: Optional[str] = None,base_url: Optional[str] = None,custom_categories: Optional[Dict[str, str]] = None,- collect_metrics: Optional[bool] = False,threshold_score: Optional[float] = 0.5,event_provider=None,):@@ -146,10 +146,11 @@ class ToxicityDetector:self.api_key, self.model, self.base_url = setup_provider(provider, api_key, model, base_url)- self.collect_metrics = collect_metricsself.custom_categories = custom_categoriesself.threshold_score = threshold_score- self.event_provider = event_provider+ # Note: event_provider parameter retained for explicit passing, but defaults to+ # auto-retrieval from OpenlitConfig via get_event_provider() in measure()+ self._explicit_event_provider = event_providerself.system_prompt = get_system_prompt(self.custom_categories, self.threshold_score)@@ -159,6 +160,7 @@ class ToxicityDetector:prompt: Optional[str] = "",contexts: Optional[List[str]] = "",text: Optional[str] = None,+ response_id: Optional[str] = None,) -> JsonOutput:"""Detects toxicity in AI output using LLM or custom rules.@@ -167,46 +169,73 @@ class ToxicityDetector:prompt (Optional[str]): The prompt provided by the user.contexts (Optional[List[str]]): A list of context sentences relevant to the task.text (Optional[str]): The text to analyze.+ response_id (Optional[str]): The unique identifier for the completion being evaluated.Returns:JsonOutput: The result containing score, evaluation, classification, explanation, and verdict of toxicity detection."""+ from openlit.evals.utils import get_event_provider- llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)- response = llm_response(self.provider, llm_prompt, self.model, self.base_url)- llm_result = parse_llm_response(response)- result_verdict = "yes" if llm_result.score > self.threshold_score else "no"-- result = JsonOutput(- score=llm_result.score,- evaluation=llm_result.evaluation,- classification=llm_result.classification,- explanation=llm_result.explanation,- verdict=result_verdict,- )+ # Use explicitly passed provider if available, else auto-retrieve from config+ event_provider = self._explicit_event_provider or get_event_provider()- if self.collect_metrics:- eval_counter = eval_metrics()- attributes = eval_metric_attributes(- result_verdict,- result.score,- result.evaluation,- result.classification,- result.explanation,+ try:+ llm_prompt = format_prompt(self.system_prompt, prompt, contexts, text)+ response = llm_response(+ self.provider, llm_prompt, self.model, self.base_url)- eval_counter.add(1, attributes)-- # Emit evaluation event- if self.event_provider:- from openlit.evals.utils import emit_evaluation_event-- emit_evaluation_event(- event_provider=self.event_provider,- evaluation_name=result.evaluation,- score_value=result.score,- score_label=result_verdict,- explanation=result.explanation,- response_id=None,+ llm_result = parse_llm_response(response)+ result_verdict = "yes" if llm_result.score > self.threshold_score else "no"++ result = JsonOutput(+ score=llm_result.score,+ evaluation=llm_result.evaluation,+ classification=llm_result.classification,+ explanation=llm_result.explanation,+ verdict=result_verdict,)- return result+ # Emit evaluation event with OTel-compliant semantic label+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ # Map verdict to pass/fail per OTel spec+ score_label = (+ "pass" if llm_result.score <= self.threshold_score else "fail"+ )++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name=result.evaluation,+ score_value=result.score,+ score_label=score_label,+ explanation=result.explanation,+ response_id=response_id,+ )++ return result++ except Exception as e:+ logger.error("Toxicity detection failed: %s", e, exc_info=True)++ # Emit error event if provider available+ if event_provider:+ from openlit.evals.utils import emit_evaluation_event++ emit_evaluation_event(+ event_provider=event_provider,+ evaluation_name="toxicity_detection",+ error_type="provider_error"+ if "provider" in str(e).lower()+ else "unknown",+ response_id=response_id,+ )++ # Return neutral result to allow continued processing+ return JsonOutput(+ score=0.0,+ evaluation="toxicity_detection",+ classification="error",+ explanation=f"Evaluation failed: {str(e)}",+ verdict="no",+ )diff --git a/sdk/python/src/openlit/evals/utils.py b/sdk/python/src/openlit/evals/utils.pyindex 5e6954aa..55a39f26 100644--- a/sdk/python/src/openlit/evals/utils.py+++ b/sdk/python/src/openlit/evals/utils.py@@ -6,8 +6,6 @@ import osimport loggingfrom typing import Optional, Tuple, Listfrom pydantic import BaseModel-from opentelemetry.metrics import get_meter-from opentelemetry.sdk.resources import TELEMETRY_SDK_NAMEfrom anthropic import Anthropicfrom openai import OpenAIfrom openlit.semcov import SemanticConvention@@ -252,71 +250,47 @@ def parse_llm_response(response) -> JsonOutput:)-def eval_metrics():+def get_event_provider():"""- Initializes OpenTelemetry meter and counter.-- Returns:- counter: The initialized telemetry counter.- """-- meter = get_meter(- __name__,- "0.1.0",- schema_url="https://opentelemetry.io/schemas/1.11.0",- )-- guard_requests = meter.create_counter(- name=SemanticConvention.EVAL_REQUESTS,- description="Counter for evaluation requests",- unit="1",- )-- return guard_requests+ Safely retrieve the event provider from OpenLIT's global configuration.--def eval_metric_attributes(verdict, score, validator, classification, explanation):- """- Initializes OpenTelemetry attributes for metrics.-- Args:- score (float): The name of the attribute for eval Score.- validator (str): The name of the attribute for eval.- classification (str): The name of the attribute for eval classification.- explaination (str): The name of the attribute for eval explanation.+ This function enables evaluators to auto-wire event emission without storing+ references that cause cyclic imports. The provider is retrieved at call time,+ allowing OpenLIT initialization to complete before evaluation measure() calls.Returns:- counter: The initialized telemetry counter.+ The event provider if OpenLIT has been initialized with telemetry, else None."""+ try:+ # pylint: disable=cyclic-import+ # (Import is inside function body, executed only after openlit.init())+ from openlit import OpenlitConfig- return {- TELEMETRY_SDK_NAME: "openlit",- SemanticConvention.EVAL_VERDICT: verdict,- SemanticConvention.EVAL_SCORE: score,- SemanticConvention.EVAL_VALIDATOR: validator,- SemanticConvention.EVAL_CLASSIFICATION: classification,- SemanticConvention.EVAL_EXPLANATION: explanation,- }+ return OpenlitConfig.event_provider+ except (ImportError, AttributeError):+ return Nonedef emit_evaluation_event(event_provider,evaluation_name,- score_value,- score_label,- explanation,+ score_value=None,+ score_label=None,+ explanation=None,response_id=None,+ error_type=None,):"""- Emit gen_ai.evaluation.result event.+ Emit gen_ai.evaluation.result event per OTel semantic conventions.Args:event_provider: The OTel event providerevaluation_name: Name of evaluation (hallucination, bias_detection, toxicity_detection)- score_value: Numerical score 0.0-1.0- score_label: Human-readable label (yes/no or pass/fail)- explanation: Brief explanation of evaluation result- response_id: Optional response ID for correlation+ score_value: Numerical score 0.0-1.0 (conditionally required)+ score_label: Human-readable label (yes/no or pass/fail) (conditionally required)+ explanation: Brief explanation of evaluation result (recommended)+ response_id: Optional response ID for correlation (recommended when available)+ error_type: Error type if evaluation failed (conditionally required if error occurs)"""try:if not event_provider:@@ -324,24 +298,36 @@ def emit_evaluation_event(from openlit.__helpers import otel_event- # Build event attributes per OTel spec+ # Build event attributes per OTel semantic convention specattributes = {SemanticConvention.GEN_AI_EVALUATION_NAME: evaluation_name,- SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: float(score_value),- SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: score_label,}- # Add recommended attributes+ # If error occurred, record error.type instead of score attributes+ if error_type:+ attributes[SemanticConvention.ERROR_TYPE] = error_type+ else:+ # Record evaluation score and label (conditionally required if no error)+ if score_value is not None:+ attributes[SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE] = float(+ score_value+ )+ if score_label:+ attributes[SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL] = (+ score_label+ )++ # Add recommended attributes when availableif explanation:attributes[SemanticConvention.GEN_AI_EVALUATION_EXPLANATION] = explanationif response_id:attributes[SemanticConvention.GEN_AI_RESPONSE_ID] = response_id- # Create and emit event+ # Create and emit event per OTel specevent = otel_event(name=SemanticConvention.GEN_AI_EVALUATION_RESULT,attributes=attributes,- body="", # Per spec, all data in attributes+ body="", # Per spec, all data must be in attributes, body is empty)event_provider.emit(event)diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex 1ece3e94..a089c59a 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -918,15 +918,9 @@ class SemanticConvention:GUARD_VALIDATOR = "guard.validator"GUARD_EXPLANATION = "guard.explanation"- # Evals- EVAL_REQUESTS = "evals.requests"- EVAL_VERDICT = "evals.verdict"- EVAL_SCORE = "evals.score"- EVAL_CLASSIFICATION = "evals.classification"- EVAL_VALIDATOR = "evals.validator"- EVAL_EXPLANATION = "evals.explanation"-- # GenAI Evaluation Event (OTel Semconv)+ # GenAI Evaluation Event (OTel Semantic Convention)+ # Per OpenTelemetry semantic conventions for generative AI+ # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-events.md#event-gen_aievaluationresultGEN_AI_EVALUATION_RESULT = "gen_ai.evaluation.result"GEN_AI_EVALUATION_NAME = "gen_ai.evaluation.name"GEN_AI_EVALUATION_SCORE_VALUE = "gen_ai.evaluation.score.value"