Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat(sdk): add log_score for user feedback and external scores (#1268)
Vanshika Mehta15ea57cc359a
diff --git a/sdk/python/README.md b/sdk/python/README.mdindex 98f703af..82bdcb7d 100644--- a/sdk/python/README.md+++ b/sdk/python/README.md@@ -260,6 +260,38 @@ if result:# contexts = result.get("entity_data", {})```+### Score ingestion - `openlit.log_score()`++Attach numeric, boolean, or categorical scores and user feedback to a GenAI span. When called inside an instrumented LLM request, the score auto-attaches to the active span.++| Parameter | Description |+|-----------|-------------|+| `name` | Score name, for example `user_feedback` or `quality`. |+| `value` | Numeric, boolean, or categorical score value. |+| `span` | Optional explicit OpenTelemetry span. |+| `trace_id` | Optional trace ID for async feedback against a past trace. |+| `span_id` | Optional span ID for async feedback against a past trace. |+| `comment` | Optional explanation or reviewer comment. |+| `idempotency_key` | Optional key to deduplicate score updates. |+| `metadata` | Optional dictionary of extra event attributes. |++#### Example++```python+import openlit++openlit.init()++# Thumbs up/down style feedback on the current span+openlit.log_score("user_feedback", True, comment="Helpful response")++# Numeric reviewer score+openlit.log_score("quality", 0.85, metadata={"reviewer": "human"})++# Categorical label+openlit.log_score("category", "accurate")+```+## 🛣️ RoadmapWe are dedicated to continuously improving OpenLIT SDKs. Here's a look at what's been accomplished and what's on the horizon:diff --git a/sdk/python/src/openlit/__init__.py b/sdk/python/src/openlit/__init__.pyindex 47fef907..f74ccb7b 100644--- a/sdk/python/src/openlit/__init__.py+++ b/sdk/python/src/openlit/__init__.py@@ -782,6 +782,51 @@ def log_agent_tool_error(agent_name, tool_name, system=None, model=None):logger.debug("Failed to record agent tool error: %s", e)+def log_score(+ name,+ value,+ *,+ span=None,+ trace_id=None,+ span_id=None,+ comment=None,+ idempotency_key=None,+ metadata=None,+):+ """+ Record an external evaluation score or user feedback on a GenAI span.++ When called inside an instrumented LLM request, the score auto-attaches to+ the active span. You can also pass ``span``, or ``trace_id`` and ``span_id``+ for async feedback against a past trace.++ Returns ``True`` when a score event was emitted and ``False`` otherwise.++ Usage:+ openlit.log_score("user_feedback", True, comment="Helpful response")+ openlit.log_score("quality", 0.85, metadata={"reviewer": "human"})+ openlit.log_score("category", "accurate")+ """+ from openlit.score import record_evaluation_score++ try:+ return record_evaluation_score(+ name,+ value,+ span=span,+ trace_id=trace_id,+ span_id=span_id,+ comment=comment,+ idempotency_key=idempotency_key,+ metadata=metadata,+ )+ except (ValueError, TypeError):+ raise+ except Exception as e:+ logger.debug("Failed to record evaluation score: %s", e)+ return False++@contextmanagerdef agent_context(name):"""diff --git a/sdk/python/src/openlit/score.py b/sdk/python/src/openlit/score.pynew file mode 100644index 00000000..6a4d2848--- /dev/null+++ b/sdk/python/src/openlit/score.py@@ -0,0 +1,173 @@+"""+Evaluation score telemetry helpers for attaching external scores and user feedback+to GenAI spans.+"""++from __future__ import annotations++import logging+import re+from typing import Any, Dict, Optional, Union++from opentelemetry import _logs, trace+from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags++from openlit._config import OpenlitConfig+from openlit.__helpers import get_custom_attributes+from openlit.semcov import SemanticConvention++logger = logging.getLogger(__name__)++ScoreValue = Union[float, int, bool, str]++_OTEL_SAFE_METADATA_TYPES = (str, int, float, bool)++_HEX_RE = re.compile(r"^[0-9a-fA-F]+$")+++def _normalize_score_value(value: ScoreValue) -> Dict[str, Any]:+ if isinstance(value, bool):+ return {+ SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: 1.0 if value else 0.0,+ SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: "true" if value else "false",+ }+ if isinstance(value, (int, float)):+ return {SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: float(value)}+ if isinstance(value, str):+ return {SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: value}+ raise TypeError("value must be numeric, boolean, or string")+++def _merge_metadata(+ event_attributes: Dict[str, Any],+ metadata: Optional[Dict[str, Any]],+) -> None:+ if not metadata:+ return+ for key, value in metadata.items():+ if not isinstance(key, str) or value is None:+ continue+ if isinstance(value, _OTEL_SAFE_METADATA_TYPES):+ event_attributes[key] = value+ elif isinstance(value, (list, tuple)) and all(+ isinstance(item, _OTEL_SAFE_METADATA_TYPES) for item in value+ ):+ event_attributes[key] = list(value)+++def _merge_custom_event_attributes(event_attributes: Dict[str, Any]) -> None:+ global_attrs = getattr(OpenlitConfig, "custom_span_attributes", None)+ if global_attrs:+ for key, value in global_attrs.items():+ event_attributes.setdefault(key, value)++ context_attrs = get_custom_attributes()+ if context_attrs:+ event_attributes.update(context_attrs)+++def _valid_hex_id(value: str, expected_len: int) -> bool:+ return len(value) == expected_len and _HEX_RE.match(value) is not None+++def _span_from_ids(trace_id: str, span_id: str) -> Optional[NonRecordingSpan]:+ if not _valid_hex_id(trace_id, 32) or not _valid_hex_id(span_id, 16):+ logger.debug("Invalid trace_id/span_id: trace_id=%s span_id=%s", trace_id, span_id)+ return None+ return NonRecordingSpan(+ SpanContext(+ trace_id=int(trace_id, 16),+ span_id=int(span_id, 16),+ is_remote=True,+ trace_flags=TraceFlags(TraceFlags.SAMPLED),+ )+ )+++def _resolve_target_span(span=None, trace_id: Optional[str] = None, span_id: Optional[str] = None):+ if span is not None:+ return span+ current_span = trace.get_current_span()+ if current_span and current_span.is_recording():+ return current_span+ if trace_id and span_id:+ return _span_from_ids(trace_id, span_id)+ if current_span:+ return current_span+ return None+++def _events_disabled() -> bool:+ return bool(getattr(OpenlitConfig, "disable_events", False))+++def _emit_score_log_event(event_attributes: Dict[str, Any], target_span) -> bool:+ if _events_disabled():+ return False+ try:+ from opentelemetry._logs import LogRecord++ event_logger = _logs.get_logger_provider().get_logger(__name__)+ span_context = target_span.get_span_context()+ event = LogRecord( # pylint: disable=unexpected-keyword-arg+ attributes=event_attributes,+ body="",+ event_name=SemanticConvention.GEN_AI_EVALUATION_RESULT,+ trace_id=span_context.trace_id,+ span_id=span_context.span_id,+ trace_flags=span_context.trace_flags,+ )+ event_logger.emit(event)+ return True+ except Exception as exc:+ logger.debug("Failed to emit evaluation score log event: %s", exc)+ return False+++def record_evaluation_score(+ name: str,+ value: ScoreValue,+ *,+ span=None,+ trace_id: Optional[str] = None,+ span_id: Optional[str] = None,+ comment: Optional[str] = None,+ idempotency_key: Optional[str] = None,+ metadata: Optional[Dict[str, Any]] = None,+) -> bool:+ """+ Attach a ``gen_ai.evaluation.result`` event to a GenAI span.++ Returns ``True`` when an event is emitted and ``False`` when no target span+ is available or telemetry could not be exported.+ """+ if not name:+ raise ValueError("name is required")++ target_span = _resolve_target_span(span=span, trace_id=trace_id, span_id=span_id)+ if not target_span:+ return False++ event_attributes = {+ SemanticConvention.GEN_AI_EVALUATION_NAME: name,+ **_normalize_score_value(value),+ }+ if comment:+ event_attributes[SemanticConvention.GEN_AI_EVALUATION_EXPLANATION] = comment+ if idempotency_key:+ event_attributes[SemanticConvention.OPENLIT_SCORE_IDEMPOTENCY_KEY] = idempotency_key+ _merge_metadata(event_attributes, metadata)+ _merge_custom_event_attributes(event_attributes)++ emitted = False+ if target_span.is_recording():+ target_span.add_event(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes=event_attributes,+ )+ emitted = True++ if _emit_score_log_event(event_attributes, target_span):+ emitted = True++ return emitteddiff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex eef0a177..5cd3757e 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -1068,6 +1068,7 @@ class SemanticConvention:GEN_AI_EVALUATION_SCORE_VALUE = "gen_ai.evaluation.score.value"GEN_AI_EVALUATION_SCORE_LABEL = "gen_ai.evaluation.score.label"GEN_AI_EVALUATION_EXPLANATION = "gen_ai.evaluation.explanation"+ OPENLIT_SCORE_IDEMPOTENCY_KEY = "openlit.score.idempotency_key"# === FRAMEWORK OPERATIONS (Generic attributes for all RAG/AI frameworks) ===diff --git a/sdk/python/tests/test_log_score.py b/sdk/python/tests/test_log_score.pynew file mode 100644index 00000000..082cf78f--- /dev/null+++ b/sdk/python/tests/test_log_score.py@@ -0,0 +1,202 @@+# pylint: disable=missing-function-docstring+"""+Tests for the OpenTelemetry-native evaluation score helper.+"""++from unittest.mock import MagicMock, Mock, patch++import pytest++from openlit.score import record_evaluation_score+from openlit.semcov import SemanticConvention+++def test_record_evaluation_score_adds_numeric_score_to_explicit_span():+ span = Mock()+ span.is_recording.return_value = True++ emitted = record_evaluation_score("quality", 0.85, span=span)++ assert emitted is True+ span.add_event.assert_called_once_with(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes={+ SemanticConvention.GEN_AI_EVALUATION_NAME: "quality",+ SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: 0.85,+ },+ )+++def test_record_evaluation_score_maps_true_boolean_score():+ span = Mock()+ span.is_recording.return_value = True++ emitted = record_evaluation_score("user_feedback", True, span=span)++ assert emitted is True+ span.add_event.assert_called_once_with(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes={+ SemanticConvention.GEN_AI_EVALUATION_NAME: "user_feedback",+ SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: 1.0,+ SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: "true",+ },+ )+++def test_record_evaluation_score_maps_false_boolean_score():+ span = Mock()+ span.is_recording.return_value = True++ emitted = record_evaluation_score("user_feedback", False, span=span)++ assert emitted is True+ span.add_event.assert_called_once_with(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes={+ SemanticConvention.GEN_AI_EVALUATION_NAME: "user_feedback",+ SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: 0.0,+ SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: "false",+ },+ )+++def test_record_evaluation_score_maps_categorical_score():+ span = Mock()+ span.is_recording.return_value = True++ emitted = record_evaluation_score("category", "accurate", span=span)++ assert emitted is True+ span.add_event.assert_called_once_with(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes={+ SemanticConvention.GEN_AI_EVALUATION_NAME: "category",+ SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL: "accurate",+ },+ )+++def test_record_evaluation_score_uses_current_span_when_span_not_provided():+ span = Mock()+ span.is_recording.return_value = True++ with patch("opentelemetry.trace.get_current_span", return_value=span):+ emitted = record_evaluation_score("quality", 0.5)++ assert emitted is True+ span.add_event.assert_called_once()+++def test_record_evaluation_score_targets_span_from_trace_and_span_ids():+ mock_logger = MagicMock()+ mock_provider = MagicMock()+ mock_provider.get_logger.return_value = mock_logger++ with patch("openlit.score._events_disabled", return_value=False), patch(+ "opentelemetry._logs.get_logger_provider", return_value=mock_provider+ ), patch("opentelemetry.trace.get_current_span", return_value=Mock(is_recording=Mock(return_value=False))):+ emitted = record_evaluation_score(+ "user_feedback",+ False,+ trace_id="0123456789abcdef0123456789abcdef",+ span_id="0123456789abcdef",+ )++ assert emitted is True+ mock_logger.emit.assert_called_once()+++def test_record_evaluation_score_returns_false_for_invalid_trace_and_span_ids():+ with patch("openlit.score._events_disabled", return_value=True), patch(+ "opentelemetry.trace.get_current_span", return_value=Mock(is_recording=Mock(return_value=False))+ ):+ emitted = record_evaluation_score(+ "quality",+ 0.5,+ trace_id="not-a-valid-trace-id",+ span_id="bad",+ )++ assert emitted is False+++def test_record_evaluation_score_returns_false_without_target_span():+ inactive_span = Mock()+ inactive_span.is_recording.return_value = False++ with patch("openlit.score._events_disabled", return_value=True), patch(+ "opentelemetry.trace.get_current_span", return_value=inactive_span+ ):+ emitted = record_evaluation_score("quality", 0.5)++ assert emitted is False+ inactive_span.add_event.assert_not_called()+++def test_record_evaluation_score_requires_name():+ with pytest.raises(ValueError, match="name is required"):+ record_evaluation_score("", 0.5)+++def test_record_evaluation_score_includes_comment_metadata_and_idempotency_key():+ span = Mock()+ span.is_recording.return_value = True++ record_evaluation_score(+ "quality",+ 0.9,+ span=span,+ comment="Looks good",+ idempotency_key="score-123",+ metadata={"reviewer": "human", "ignored": None},+ )++ span.add_event.assert_called_once_with(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes={+ SemanticConvention.GEN_AI_EVALUATION_NAME: "quality",+ SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE: 0.9,+ SemanticConvention.GEN_AI_EVALUATION_EXPLANATION: "Looks good",+ SemanticConvention.OPENLIT_SCORE_IDEMPOTENCY_KEY: "score-123",+ "reviewer": "human",+ },+ )+++def test_record_evaluation_score_includes_custom_span_attributes():+ span = Mock()+ span.is_recording.return_value = True++ with patch(+ "openlit.score.OpenlitConfig.custom_span_attributes",+ {"session.id": "sess-1"},+ create=True,+ ), patch("openlit.score.get_custom_attributes", return_value={"user.id": "user-1"}), patch(+ "openlit.score._events_disabled", return_value=True+ ):+ record_evaluation_score("quality", 0.5, span=span)++ attributes = span.add_event.call_args.kwargs["attributes"]+ assert attributes["session.id"] == "sess-1"+ assert attributes["user.id"] == "user-1"+++def test_log_score_returns_emitted_result():+ span = Mock()+ span.is_recording.return_value = True++ import openlit++ with patch("openlit.score.record_evaluation_score", return_value=True) as mock_record:+ emitted = openlit.log_score("quality", 0.5, span=span)++ assert emitted is True+ mock_record.assert_called_once()+++def test_log_score_reraises_value_error():+ import openlit++ with pytest.raises(ValueError, match="name is required"):+ openlit.log_score("", 0.5)diff --git a/sdk/typescript/README.md b/sdk/typescript/README.mdindex d074106d..d7d8b746 100644--- a/sdk/typescript/README.md+++ b/sdk/typescript/README.md@@ -268,6 +268,33 @@ if (!('err' in result)) {}```+### Score ingestion - `Openlit.logScore()`++Attach numeric, boolean, or categorical scores and user feedback to a GenAI span. When called inside an instrumented LLM request, the score auto-attaches to the active span.++| Parameter | Description |+|-----------|-------------|+| `name` | Score name, for example `user_feedback` or `quality`. |+| `value` | Numeric, boolean, or categorical score value. |+| `span` | Optional explicit OpenTelemetry span. |+| `traceId` | Optional trace ID for async feedback against a past trace. |+| `spanId` | Optional span ID for async feedback against a past trace. |+| `comment` | Optional explanation or reviewer comment. |+| `idempotencyKey` | Optional key to deduplicate score updates. |+| `metadata` | Optional object of extra event attributes. |++#### Example++```typescript+import Openlit from 'openlit';++Openlit.init();++Openlit.logScore({ name: 'user_feedback', value: true, comment: 'Helpful response' });+Openlit.logScore({ name: 'quality', value: 0.85, metadata: { reviewer: 'human' } });+Openlit.logScore({ name: 'category', value: 'accurate' });+```+## 🌱 ContributingWhether it's big or small, we love contributions 💚. Check out our [Contribution guide](../../CONTRIBUTING.md) to get starteddiff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.tsindex d406b0f3..0e30e22f 100644--- a/sdk/typescript/src/index.ts+++ b/sdk/typescript/src/index.ts@@ -17,6 +17,7 @@ import {getCurrentAgentVersion,} from './helpers';import { runEval, runEvalBatch, fetchEvalTypes } from './evals';+import { logScore } from './score';import Metrics from './otel/metrics';import SemanticConvention from './semantic-convention';import { parseBoolEnv } from './otel/utils';@@ -144,6 +145,7 @@ class Openlit extends BaseOpenlit {static eval = runEval;static evalBatch = runEvalBatch;static getEvalTypes = fetchEvalTypes;+ static logScore = logScore;/*** Public API: stamp every subsequent chat span / inference event in the@@ -225,8 +227,9 @@ const openlit = Openlit as typeof Openlit & {(openlit as any).injectAdditionalAttributes = injectAdditionalAttributes;export default openlit;-export { Openlit, usingAttributes, injectAdditionalAttributes };+export { Openlit, usingAttributes, injectAdditionalAttributes, logScore };export type { OpenlitOptions } from './types';+export type { LogScoreOptions } from './score';// Guard re-exports for named imports: import { PII, Pipeline } from 'openlit'export {diff --git a/sdk/typescript/src/score/__tests__/score.test.ts b/sdk/typescript/src/score/__tests__/score.test.tsnew file mode 100644index 00000000..0ee4beca--- /dev/null+++ b/sdk/typescript/src/score/__tests__/score.test.ts@@ -0,0 +1,194 @@+import { trace } from '@opentelemetry/api';+import OpenlitConfig from '../../config';+import Events from '../../otel/events';+import SemanticConvention from '../../semantic-convention';+import { logScore } from '../score';++function createMockSpan(isRecording = true) {+ return {+ isRecording: jest.fn().mockReturnValue(isRecording),+ addEvent: jest.fn(),+ spanContext: jest.fn().mockReturnValue({+ traceId: '0123456789abcdef0123456789abcdef',+ spanId: '0123456789abcdef',+ traceFlags: 1,+ }),+ };+}++beforeEach(() => {+ jest.restoreAllMocks();+ OpenlitConfig.disableEvents = undefined;+ (Events as any).logger = undefined;+});++describe('logScore', () => {+ test('adds numeric score to explicit span', () => {+ const span = createMockSpan();++ const emitted = logScore({ name: 'quality', value: 0.85, span: span as any });++ expect(emitted).toBe(true);+ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ {+ [SemanticConvention.GEN_AI_EVALUATION_NAME]: 'quality',+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE]: 0.85,+ }+ );+ });++ test('maps true boolean score', () => {+ const span = createMockSpan();++ logScore({ name: 'user_feedback', value: true, span: span as any });++ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ {+ [SemanticConvention.GEN_AI_EVALUATION_NAME]: 'user_feedback',+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE]: 1.0,+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL]: 'true',+ }+ );+ });++ test('maps false boolean score', () => {+ const span = createMockSpan();++ logScore({ name: 'user_feedback', value: false, span: span as any });++ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ {+ [SemanticConvention.GEN_AI_EVALUATION_NAME]: 'user_feedback',+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE]: 0.0,+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL]: 'false',+ }+ );+ });++ test('maps categorical score', () => {+ const span = createMockSpan();++ logScore({ name: 'category', value: 'accurate', span: span as any });++ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ {+ [SemanticConvention.GEN_AI_EVALUATION_NAME]: 'category',+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL]: 'accurate',+ }+ );+ });++ test('auto-attaches to active span', () => {+ const span = createMockSpan();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span as any);++ const emitted = logScore({ name: 'quality', value: 0.5 });++ expect(emitted).toBe(true);+ expect(span.addEvent).toHaveBeenCalledTimes(1);+ });++ test('targets span from trace and span ids via log event', () => {+ OpenlitConfig.disableEvents = false;+ (Events as any).logger = { emit: jest.fn() };+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(createMockSpan(false) as any);++ const emitted = logScore({+ name: 'user_feedback',+ value: false,+ traceId: '0123456789abcdef0123456789abcdef',+ spanId: '0123456789abcdef',+ });++ expect(emitted).toBe(true);+ expect((Events as any).logger.emit).toHaveBeenCalledTimes(1);+ });++ test('returns false for invalid trace and span ids', () => {+ OpenlitConfig.disableEvents = true;+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(createMockSpan(false) as any);++ const emitted = logScore({+ name: 'quality',+ value: 0.5,+ traceId: 'not-a-valid-trace-id',+ spanId: 'bad',+ });++ expect(emitted).toBe(false);+ });++ test('returns false without target span when events disabled', () => {+ OpenlitConfig.disableEvents = true;+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(createMockSpan(false) as any);++ const emitted = logScore({ name: 'quality', value: 0.5 });++ expect(emitted).toBe(false);+ });++ test('requires name', () => {+ expect(() => logScore({ name: '', value: 0.5 })).toThrow('name is required');+ });++ test('includes comment metadata and idempotency key', () => {+ const span = createMockSpan();++ logScore({+ name: 'quality',+ value: 0.9,+ span: span as any,+ comment: 'Looks good',+ idempotencyKey: 'score-123',+ metadata: { reviewer: 'human' },+ });++ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ {+ [SemanticConvention.GEN_AI_EVALUATION_NAME]: 'quality',+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE]: 0.9,+ [SemanticConvention.GEN_AI_EVALUATION_EXPLANATION]: 'Looks good',+ [SemanticConvention.OPENLIT_SCORE_IDEMPOTENCY_KEY]: 'score-123',+ reviewer: 'human',+ }+ );+ });++ test('includes array metadata values', () => {+ const span = createMockSpan();++ logScore({+ name: 'quality',+ value: 0.9,+ span: span as any,+ metadata: { tags: ['human', 'reviewed'] },+ });++ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ expect.objectContaining({+ tags: ['human', 'reviewed'],+ })+ );+ });++ test('includes custom span attributes', () => {+ const span = createMockSpan();+ const helpers = require('../../helpers');+ jest.spyOn(helpers, 'getMergedCustomAttributes').mockReturnValue({ 'session.id': 'sess-1' });++ logScore({ name: 'quality', value: 0.5, span: span as any });++ expect(span.addEvent).toHaveBeenCalledWith(+ SemanticConvention.GEN_AI_EVALUATION_RESULT,+ expect.objectContaining({+ 'session.id': 'sess-1',+ })+ );+ });+});diff --git a/sdk/typescript/src/score/index.ts b/sdk/typescript/src/score/index.tsnew file mode 100644index 00000000..5f56c129--- /dev/null+++ b/sdk/typescript/src/score/index.ts@@ -0,0 +1,2 @@+export { logScore } from './score';+export type { LogScoreOptions } from './score';diff --git a/sdk/typescript/src/score/score.ts b/sdk/typescript/src/score/score.tsnew file mode 100644index 00000000..0dccf58d--- /dev/null+++ b/sdk/typescript/src/score/score.ts@@ -0,0 +1,163 @@+import { Attributes, AttributeValue, Span, SpanContext, TraceFlags, trace, context as otelContext } from '@opentelemetry/api';+import { SeverityNumber } from '@opentelemetry/api-logs';+import OpenlitConfig from '../config';+import { getMergedCustomAttributes } from '../helpers';+import Events from '../otel/events';+import SemanticConvention from '../semantic-convention';++type OtelSafeMetadataValue = string | number | boolean;+type OtelSafeMetadataArray = OtelSafeMetadataValue[];++export interface LogScoreOptions {+ name: string;+ value: number | boolean | string;+ span?: Span;+ traceId?: string;+ spanId?: string;+ comment?: string;+ idempotencyKey?: string;+ metadata?: Record<string, OtelSafeMetadataValue | OtelSafeMetadataArray>;+}++type ScoreValue = number | boolean | string;++function normalizeScoreValue(value: ScoreValue): Attributes {+ if (typeof value === 'boolean') {+ return {+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE]: value ? 1.0 : 0.0,+ [SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL]: value ? 'true' : 'false',+ };+ }+ if (typeof value === 'number') {+ return { [SemanticConvention.GEN_AI_EVALUATION_SCORE_VALUE]: value };+ }+ return { [SemanticConvention.GEN_AI_EVALUATION_SCORE_LABEL]: value };+}++function eventsDisabled(): boolean {+ return Boolean(OpenlitConfig.disableEvents);+}++function isOtelSafeMetadataValue(value: unknown): value is OtelSafeMetadataValue {+ return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';+}++function mergeMetadata(+ eventAttributes: Attributes,+ metadata?: Record<string, OtelSafeMetadataValue | OtelSafeMetadataArray>+): void {+ if (!metadata) {+ return;+ }+ for (const [key, value] of Object.entries(metadata)) {+ if (value === undefined || value === null) {+ continue;+ }+ if (isOtelSafeMetadataValue(value)) {+ eventAttributes[key] = value;+ continue;+ }+ if (Array.isArray(value) && value.every(isOtelSafeMetadataValue)) {+ eventAttributes[key] = value as AttributeValue;+ }+ }+}++function mergeCustomEventAttributes(eventAttributes: Attributes): void {+ const customAttrs = getMergedCustomAttributes();+ for (const [key, value] of Object.entries(customAttrs)) {+ if (value !== undefined && value !== null) {+ if (!(key in eventAttributes)) {+ eventAttributes[key] = value;+ }+ }+ }+}++const HEX_RE = /^[0-9a-fA-F]+$/;++function validHexId(value: string, expectedLen: number): boolean {+ return value.length === expectedLen && HEX_RE.test(value);+}++function spanFromIds(traceId: string, spanId: string): Span | undefined {+ if (!validHexId(traceId, 32) || !validHexId(spanId, 16)) {+ return undefined;+ }+ const spanContext: SpanContext = {+ traceId,+ spanId,+ traceFlags: TraceFlags.SAMPLED,+ isRemote: true,+ };+ return trace.wrapSpanContext(spanContext);+}++function resolveTargetSpan(options: LogScoreOptions): Span | undefined {+ if (options.span) {+ return options.span;+ }+ const activeSpan = trace.getActiveSpan();+ if (activeSpan?.isRecording()) {+ return activeSpan;+ }+ if (options.traceId && options.spanId) {+ return spanFromIds(options.traceId, options.spanId);+ }+ return activeSpan ?? undefined;+}++function emitScoreLogEvent(eventAttributes: Attributes, targetSpan: Span): boolean {+ if (eventsDisabled() || !Events.logger) {+ return false;+ }+ Events.logger.emit({+ eventName: SemanticConvention.GEN_AI_EVALUATION_RESULT,+ context: trace.setSpan(otelContext.active(), targetSpan),+ severityNumber: SeverityNumber.INFO,+ severityText: 'INFO',+ body: SemanticConvention.GEN_AI_EVALUATION_RESULT,+ attributes: {+ ...eventAttributes,+ 'event.name': SemanticConvention.GEN_AI_EVALUATION_RESULT,+ },+ });+ return true;+}++export function logScore(options: LogScoreOptions): boolean {+ const { name, value, comment, idempotencyKey, metadata } = options;+ if (!name) {+ throw new Error('name is required');+ }++ const targetSpan = resolveTargetSpan(options);+ if (!targetSpan) {+ return false;+ }++ const eventAttributes: Attributes = {+ [SemanticConvention.GEN_AI_EVALUATION_NAME]: name,+ ...normalizeScoreValue(value),+ };+ if (comment) {+ eventAttributes[SemanticConvention.GEN_AI_EVALUATION_EXPLANATION] = comment;+ }+ if (idempotencyKey) {+ eventAttributes[SemanticConvention.OPENLIT_SCORE_IDEMPOTENCY_KEY] = idempotencyKey;+ }+ mergeMetadata(eventAttributes, metadata);+ mergeCustomEventAttributes(eventAttributes);++ let emitted = false;+ if (targetSpan.isRecording()) {+ targetSpan.addEvent(SemanticConvention.GEN_AI_EVALUATION_RESULT, eventAttributes);+ emitted = true;+ }++ if (emitScoreLogEvent(eventAttributes, targetSpan)) {+ emitted = true;+ }++ return emitted;+}diff --git a/sdk/typescript/src/semantic-convention.ts b/sdk/typescript/src/semantic-convention.tsindex 1426eff3..c76e3476 100644--- a/sdk/typescript/src/semantic-convention.ts+++ b/sdk/typescript/src/semantic-convention.ts@@ -513,4 +513,12 @@ export default class SemanticConvention {static GUARD_LATENCY_MS = 'guard.latency_ms';static GUARD_DENIED = 'guard.denied';static GUARD_REQUESTS_COUNTER = 'guard.requests';++ // GenAI Evaluation Event (OTel Semantic Convention)+ static GEN_AI_EVALUATION_RESULT = 'gen_ai.evaluation.result';+ static GEN_AI_EVALUATION_NAME = 'gen_ai.evaluation.name';+ static GEN_AI_EVALUATION_SCORE_VALUE = 'gen_ai.evaluation.score.value';+ static GEN_AI_EVALUATION_SCORE_LABEL = 'gen_ai.evaluation.score.label';+ static GEN_AI_EVALUATION_EXPLANATION = 'gen_ai.evaluation.explanation';+ static OPENLIT_SCORE_IDEMPOTENCY_KEY = 'openlit.score.idempotency_key';}