Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(trace): send Langfuse traces over OpenTelemetry (#45606)

Xiyuan Chen0be453d27b22
diff --git a/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py
index 620ecd12388..e4eed3faa68 100644
--- a/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py
+++ b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py
@@ -1,39 +1,33 @@
-"""Langfuse destination over the public ingestion API.
+"""Langfuse destination over its OpenTelemetry endpoint.
-Each trace unit becomes ingestion events: the root span is the Langfuse trace
-and every other span an observation (a generation for LLM calls). Langfuse
-upserts by ID, so re-sending a unit after a partial failure is safe.
+Langfuse does not deduplicate re-sent spans, so a unit is retried only when
+none of it was accepted.
"""
import base64
-from collections.abc import Mapping, Sequence
-from datetime import datetime, timedelta
+from datetime import timedelta
from typing import override
-from urllib.parse import quote
-from uuid import uuid5
from dify_trace_sdk.errors import (
EgressError,
ExportOutcomeUnknownError,
- ExportRejectedError,
ExportRetryableError,
VerificationError,
)
-from dify_trace_sdk.exporting import export_endpoint, json_value, send_export_request
-from dify_trace_sdk.ids import span_uuid, trace_uuid
-from dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace
+from dify_trace_sdk.exporting import export_endpoint, json_value
+from dify_trace_sdk.model import Span, SpanKind, Trace
+from dify_trace_sdk.otlp import encode_trace, post_otlp
from dify_trace_sdk.provider import ProviderCapabilities, SendResult, TraceProvider
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
from pydantic import JsonValue
from dify_trace_langfuse.config import LangfuseConfig
+DESTINATION = "Langfuse"
+SERVICE_NAME = "dify"
VERIFY_TIMEOUT_SECONDS = 10.0
-MAX_BATCH_BYTES = 3_000_000
-"""Stay below Langfuse Cloud's ingestion request limit."""
-
-
-def _iso(value: datetime) -> str:
- return value.isoformat()
+MAX_REQUEST_BYTES = 3_000_000
+INGESTION_VERSION_HEADER = {"x-langfuse-ingestion-version": "4"}
def _messages(value: JsonValue, *, role: str) -> JsonValue:
@@ -50,15 +44,104 @@ def _messages(value: JsonValue, *, role: str) -> JsonValue:
return value
+def _json(value: JsonValue) -> str | None:
+ return None if value is None else json_value(value)
+
+
+def _trace_attributes(trace: Trace) -> dict[str, object]:
+ """Repeated on every span so Langfuse can filter observations by them."""
+ return {
+ "langfuse.trace.name": trace.root.name if trace.parent is None else None,
+ "langfuse.user.id": trace.user_id,
+ "langfuse.session.id": trace.session_id,
+ }
+
+
+def _trace_metadata(trace: Trace) -> dict[str, object]:
+ if trace.parent is not None:
+ return {}
+ metadata = {
+ **trace.root.attributes,
+ "dify_trace_id": trace.trace_id,
+ "external_trace_id": trace.external_trace_id,
+ }
+ return {f"langfuse.trace.metadata.{key}": value for key, value in metadata.items()}
+
+
+def _generation_attributes(span: Span) -> dict[str, object]:
+ attributes: dict[str, object] = {"langfuse.observation.model.name": span.model_name}
+ if span.usage is not None:
+ attributes["langfuse.observation.usage_details"] = json_value(
+ {
+ "input": span.usage.prompt_tokens,
+ "output": span.usage.completion_tokens,
+ "total": span.usage.total_tokens,
+ }
+ )
+ if span.cost is not None and span.cost.currency.upper() == "USD":
+ attributes["langfuse.observation.cost_details"] = json_value({"total": span.cost.amount})
+ if span.time_to_first_token is not None:
+ attributes["langfuse.observation.completion_start_time"] = (
+ span.start_time + timedelta(seconds=span.time_to_first_token)
+ ).isoformat()
+ return attributes
+
+
+def _span_attributes(span: Span) -> dict[str, object]:
+ # Other span kinds match Langfuse's type names.
+ attributes: dict[str, object] = {
+ "langfuse.observation.type": "generation" if span.kind is SpanKind.LLM else span.kind.value,
+ "langfuse.observation.input": _json(_messages(span.inputs, role="user")),
+ "langfuse.observation.output": _json(_messages(span.outputs, role="assistant")),
+ "langfuse.observation.metadata.dify_span_kind": span.kind.value,
+ **{f"langfuse.observation.metadata.{key}": value for key, value in span.attributes.items()},
+ }
+ if span.kind is SpanKind.LLM:
+ attributes.update(_generation_attributes(span))
+ return attributes
+
+
+def _export_request(trace: Trace) -> ExportTraceServiceRequest:
+ trace_attributes = _trace_attributes(trace)
+ trace_metadata = _trace_metadata(trace)
+ return encode_trace(
+ trace,
+ resource={"service.name": SERVICE_NAME},
+ attributes=lambda span: {
+ **trace_attributes,
+ **_span_attributes(span),
+ **(trace_metadata if span.span_id == trace.root.span_id else {}),
+ },
+ )
+
+
+def _split(request: ExportTraceServiceRequest) -> list[ExportTraceServiceRequest]:
+ """An oversized span travels alone."""
+ (resource_spans,) = request.resource_spans
+ (scope_spans,) = resource_spans.scope_spans
+ chunks: list[ExportTraceServiceRequest] = []
+ size = 0
+ for span in scope_spans.spans:
+ span_size = span.ByteSize()
+ if not chunks or size + span_size > MAX_REQUEST_BYTES:
+ chunk = ExportTraceServiceRequest()
+ chunk.resource_spans.add(resource=resource_spans.resource).scope_spans.add(scope=scope_spans.scope)
+ chunks.append(chunk)
+ size = 0
+ chunks[-1].resource_spans[0].scope_spans[0].spans.append(span)
+ size += span_size
+ return chunks
+
+
class LangfuseProvider(TraceProvider[LangfuseConfig]):
id = "langfuse"
config_model = LangfuseConfig
capabilities = ProviderCapabilities(native_cost=True, trace_url=True, span_url=True)
@staticmethod
- def _headers(config: LangfuseConfig) -> dict[str, str]:
+ def _authorization(config: LangfuseConfig) -> dict[str, str]:
token = base64.b64encode(f"{config.public_key}:{config.secret_key}".encode()).decode()
- return {"Authorization": f"Basic {token}", "Accept": "application/json"}
+ return {"Authorization": f"Basic {token}"}
@override
def verify(self, config: LangfuseConfig) -> LangfuseConfig:
@@ -66,7 +149,7 @@ class LangfuseProvider(TraceProvider[LangfuseConfig]):
response = self.egress.request(
"GET",
f"{config.host.rstrip('/')}/api/public/projects",
- headers=self._headers(config),
+ headers={**self._authorization(config), "Accept": "application/json"},
content=None,
timeout=VERIFY_TIMEOUT_SECONDS,
sensitive_url=False,
@@ -94,147 +177,21 @@ class LangfuseProvider(TraceProvider[LangfuseConfig]):
@override
def send(self, config: LangfuseConfig, trace: Trace) -> SendResult:
- host = export_endpoint(config.host)
- trace_key = trace.external_trace_id or str(trace_uuid(trace.trace_id))
- events = self._events(trace, trace_key)
- for batch in self._batches(events):
+ url = f"{export_endpoint(config.host)}/api/public/otel/v1/traces"
+ headers = {**self._authorization(config), **INGESTION_VERSION_HEADER}
+ for index, request in enumerate(_split(_export_request(trace))):
try:
- response = send_export_request(
- self.egress,
- "POST",
- f"{host}/api/public/ingestion",
- destination="Langfuse",
- headers={**self._headers(config), "Content-Type": "application/json"},
- content=batch,
- sensitive_url=False,
- )
- except ExportOutcomeUnknownError:
- raise ExportRetryableError("Langfuse ingestion outcome is unknown; re-sending is idempotent") from None
- try:
- body = response.json() if response.content else {}
- except ValueError:
- # Something between Dify and Langfuse answered; re-sending is idempotent.
- raise ExportRetryableError("Langfuse ingestion answered with an unreadable body") from None
- self._check_results(body)
+ post_otlp(self.egress, url, request, destination=DESTINATION, headers=headers, sensitive_url=False)
+ except ExportRetryableError:
+ if index == 0:
+ raise
+ raise ExportOutcomeUnknownError(f"{DESTINATION} accepted part of the trace") from None
trace_url = (
- f"{self.project_url(config).rstrip('/')}/traces/{quote(trace_key, safe='')}"
+ f"{self.project_url(config).rstrip('/')}/traces/{trace.trace_id}"
if config.project_key or config.project_url
else None
)
span_urls = (
- {
- span.span_id: f"{trace_url}?observation={self._observation_id(trace, span.span_id)}"
- for span in trace.spans
- }
- if trace_url
- else {}
+ {span.span_id: f"{trace_url}?observation={span.span_id}" for span in trace.spans} if trace_url else {}
)
- return SendResult(trace_id=trace_key, trace_url=trace_url, span_urls=span_urls)
-
- @staticmethod
- def _observation_id(trace: Trace, span_id: str) -> str:
- return str(span_uuid(trace.trace_id, span_id))
-
- def _events(self, trace: Trace, trace_key: str) -> list[dict[str, object]]:
- events: list[dict[str, object]] = []
- if trace.parent is None:
- root = trace.root
- events.append(
- self._event(
- trace,
- "trace-create",
- {
- "id": trace_key,
- "name": root.name,
- "timestamp": _iso(root.start_time),
- "userId": trace.user_id,
- "sessionId": trace.session_id,
- "input": _messages(root.inputs, role="user"),
- "output": _messages(root.outputs, role="assistant"),
- "metadata": {**root.attributes, "dify_trace_id": trace.trace_id},
- },
- )
- )
- # The trace itself represents the root; its children are top-level observations.
- trace_level_span_id = root.span_id
- observations: Sequence[Span] = trace.spans[1:]
- else:
- trace_level_span_id = trace.parent.span_id
- observations = trace.spans
- for span in observations:
- parent = (
- None
- if span.parent_span_id == trace_level_span_id or span.parent_span_id is None
- else self._observation_id(trace, span.parent_span_id)
- )
- body: dict[str, object] = {
- "id": self._observation_id(trace, span.span_id),
- "traceId": trace_key,
- "parentObservationId": parent,
- "name": span.name,
- "startTime": _iso(span.start_time),
- "endTime": _iso(span.end_time),
- "input": _messages(span.inputs, role="user"),
- "output": _messages(span.outputs, role="assistant"),
- "metadata": {**span.attributes, "dify_span_kind": span.kind.value},
- "level": "ERROR" if span.status is SpanStatus.ERROR else "DEFAULT",
- "statusMessage": span.error,
- }
- if span.kind is SpanKind.LLM:
- body.update(self._generation_fields(span))
- events.append(self._event(trace, "generation-create", body))
- else:
- events.append(self._event(trace, "span-create", body))
- return events
-
- @staticmethod
- def _generation_fields(span: Span) -> dict[str, object]:
- fields: dict[str, object] = {"model": span.model_name}
- if span.usage is not None:
- usage: dict[str, object] = {
- "input": span.usage.prompt_tokens,
- "output": span.usage.completion_tokens,
- "total": span.usage.total_tokens,
- "unit": "TOKENS",
- }
- if span.cost is not None and span.cost.currency.upper() == "USD":
- usage["totalCost"] = span.cost.amount
- fields["usage"] = usage
- if span.time_to_first_token is not None:
- fields["completionStartTime"] = _iso(span.start_time + timedelta(seconds=span.time_to_first_token))
- return fields
-
- @staticmethod
- def _event(trace: Trace, event_type: str, body: Mapping[str, object]) -> dict[str, object]:
- # Deterministic event IDs let Langfuse deduplicate a redelivered unit.
- return {
- "id": str(uuid5(trace_uuid(trace.trace_id), f"{event_type}:{body['id']}")),
- "type": event_type,
- "timestamp": body.get("startTime") or body.get("timestamp"),
- "body": {key: value for key, value in body.items() if value is not None},
- }
-
- @staticmethod
- def _batches(events: list[dict[str, object]]) -> list[bytes]:
- batches: list[bytes] = []
- current: list[bytes] = []
- size = 0
- for event in events:
- encoded = json_value(event).encode()
- if current and size + len(encoded) > MAX_BATCH_BYTES:
- batches.append(b'{"batch":[' + b",".join(current) + b"]}")
- current, size = [], 0
- current.append(encoded)
- size += len(encoded) + 1
- batches.append(b'{"batch":[' + b",".join(current) + b"]}")
- return batches
-
- @staticmethod
- def _check_results(body: object) -> None:
- errors = body.get("errors") if isinstance(body, dict) else None
- if not isinstance(errors, list) or not errors:
- return
- statuses = [error.get("status") for error in errors if isinstance(error, dict)]
- if any(isinstance(status, int) and (status == 429 or status >= 500) for status in statuses):
- raise ExportRetryableError("Langfuse could not ingest every event; re-sending is idempotent")
- raise ExportRejectedError(f"Langfuse rejected {len(errors)} ingestion events")
+ return SendResult(trace_id=trace.trace_id, trace_url=trace_url, span_urls=span_urls)
diff --git a/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py b/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py
index 8227f931cca..32f1616f838 100644
--- a/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py
+++ b/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py
@@ -1,20 +1,36 @@
import base64
+import json
import pytest
from dify_trace_langfuse.config import LangfuseConfig
-from dify_trace_langfuse.provider import MAX_BATCH_BYTES, LangfuseProvider
+from dify_trace_langfuse.provider import MAX_REQUEST_BYTES, LangfuseProvider
from dify_trace_sdk.egress import EgressResponse
from dify_trace_sdk.errors import (
EgressError,
+ ExportOutcomeUnknownError,
ExportRejectedError,
ExportRetryableError,
VerificationError,
)
-from dify_trace_sdk.ids import span_uuid
-from dify_trace_sdk.testing import MESSAGE_ID, RecordingEgress, message_trace, response, tool_fragment, workflow_trace
+from dify_trace_sdk.ids import span_id_for
+from dify_trace_sdk.model import Cost, Trace
+from dify_trace_sdk.testing import (
+ MESSAGE_ID,
+ RecordingEgress,
+ message_trace,
+ otlp_attributes,
+ otlp_spans,
+ response,
+ tool_fragment,
+ workflow_trace,
+)
+from opentelemetry.proto.trace.v1.trace_pb2 import Status
from pydantic import JsonValue
CONFIG = LangfuseConfig(public_key="pk-lf-1", secret_key="sk-lf-1", host="https://cloud.langfuse.com")
+OTEL_URL = "https://cloud.langfuse.com/api/public/otel/v1/traces"
+# Langfuse replies with its queued job, not an OTLP acknowledgment.
+ACK = response(200, {"name": "otel-ingestion-job", "data": {"id": "job-1"}, "opts": {"attempts": 6}, "progress": 0})
def _provider(*responses: EgressResponse | Exception) -> tuple[LangfuseProvider, RecordingEgress]:
@@ -22,11 +38,8 @@ def _provider(*responses: EgressResponse | Exception) -> tuple[LangfuseProvider,
return LangfuseProvider(egress), egress
-def _sent_events(egress: RecordingEgress) -> list[dict]:
- (sent,) = egress.requests
- body = sent.json()
- assert isinstance(body, dict)
- return body["batch"]
+def _sent_attributes(egress: RecordingEgress) -> list[dict[str, object]]:
+ return [otlp_attributes(span) for span in otlp_spans(egress)]
def test_verify_resolves_the_project() -> None:
@@ -41,7 +54,7 @@ def test_verify_resolves_the_project() -> None:
def test_legacy_config_keeps_project_and_trace_links_without_verification() -> None:
- provider, egress = _provider(response(200, {}))
+ provider, egress = _provider(ACK)
config = LangfuseConfig.model_validate(
{
"public_key": "pk-lf-1",
@@ -50,16 +63,17 @@ def test_legacy_config_keeps_project_and_trace_links_without_verification() -> N
"project_url": "https://dashboard.example/langfuse/project/legacy-project/",
}
)
+ trace = message_trace()
- result = provider.send(config, message_trace())
+ result = provider.send(config, trace)
assert provider.project_url(config) == config.project_url
- trace_url = f"https://dashboard.example/langfuse/project/legacy-project/traces/{MESSAGE_ID}"
+ trace_url = f"https://dashboard.example/langfuse/project/legacy-project/traces/{trace.trace_id}"
assert result.trace_url == trace_url
- assert result.span_urls
- assert all(url.startswith(trace_url + "?observation=") for url in result.span_urls.values())
- assert len(egress.requests) == 1
- assert egress.requests[0].url == "https://collector.example/langfuse/api/public/ingestion"
+ assert result.span_urls == {span.span_id: f"{trace_url}?observation={span.span_id}" for span in trace.spans}
+ assert [request.url for request in egress.requests] == [
+ "https://collector.example/langfuse/api/public/otel/v1/traces"
+ ]
verified = config.model_copy(update={"project_key": "new-project"})
assert provider.project_url(verified) == "https://collector.example/langfuse/project/new-project"
@@ -81,27 +95,61 @@ def test_verify_failures_have_stable_codes(outcome: EgressResponse | Exception,
assert caught.value.code == code
-def test_message_trace_becomes_trace_and_generation() -> None:
- provider, egress = _provider(response(207, {"successes": [], "errors": []}))
+def test_send_posts_otlp_protobuf_with_v4_ingestion_header() -> None:
+ provider, egress = _provider(ACK)
+
+ provider.send(CONFIG, message_trace())
+
+ (sent,) = egress.requests
+ assert (sent.method, sent.url) == ("POST", OTEL_URL)
+ assert sent.headers["Authorization"] == "Basic " + base64.b64encode(b"pk-lf-1:sk-lf-1").decode()
+ assert sent.headers["Content-Type"] == "application/x-protobuf"
+ assert sent.headers["x-langfuse-ingestion-version"] == "4"
+
+
+def test_message_trace_becomes_a_root_observation_and_a_generation() -> None:
+ provider, egress = _provider(ACK)
trace = message_trace()
+
result = provider.send(CONFIG.model_copy(update={"project_key": "project-1"}), trace)
- trace_event, generation = _sent_events(egress)
- assert trace_event["type"] == "trace-create"
- assert trace_event["body"]["id"] == MESSAGE_ID
- assert trace_event["body"]["sessionId"] == "conversation-1"
- assert trace_event["body"]["userId"] == "end-user-1"
- assert generation["type"] == "generation-create"
- body = generation["body"]
- assert body["traceId"] == MESSAGE_ID
- assert "parentObservationId" not in body
- assert body["model"] == "gpt-4o"
- assert body["usage"] == {"input": 12, "output": 8, "total": 20, "unit": "TOKENS", "totalCost": 0.0004}
- assert body["completionStartTime"] == "2026-10-01T08:00:00.250000+00:00"
- assert body["input"] == [{"role": "user", "content": "What is Dify?"}]
- assert body["output"] == [{"role": "assistant", "content": "An LLM application platform."}]
- assert result.trace_id == MESSAGE_ID
- assert result.trace_url == f"https://cloud.langfuse.com/project/project-1/traces/{MESSAGE_ID}"
+ root, llm = otlp_spans(egress)
+ assert root.trace_id.hex() == llm.trace_id.hex() == trace.trace_id
+ assert root.span_id.hex() == trace.spans[0].span_id
+ assert root.parent_span_id == b""
+ assert llm.parent_span_id.hex() == trace.spans[0].span_id
+ root_attributes, llm_attributes = otlp_attributes(root), otlp_attributes(llm)
+ assert root_attributes["langfuse.observation.type"] == "chain"
+ assert root_attributes["langfuse.observation.metadata.app_id"] == "app-1"
+ assert json.loads(str(root_attributes["langfuse.observation.output"])) == [
+ {"role": "assistant", "content": "An LLM application platform."}
+ ]
+ assert llm_attributes["langfuse.observation.type"] == "generation"
+ assert llm_attributes["langfuse.observation.model.name"] == "gpt-4o"
+ assert json.loads(str(llm_attributes["langfuse.observation.usage_details"])) == {
+ "input": 12,
+ "output": 8,
+ "total": 20,
+ }
+ assert json.loads(str(llm_attributes["langfuse.observation.cost_details"])) == {"total": 0.0004}
+ assert llm_attributes["langfuse.observation.completion_start_time"] == "2026-10-01T08:00:00.250000+00:00"
+ assert json.loads(str(llm_attributes["langfuse.observation.input"])) == [
+ {"role": "user", "content": "What is Dify?"}
+ ]
+ assert result.trace_id == trace.trace_id
+ assert result.trace_url == f"https://cloud.langfuse.com/project/project-1/traces/{trace.trace_id}"
+ assert result.span_urls[trace.spans[1].span_id] == f"{result.trace_url}?observation={trace.spans[1].span_id}"
+
+
+def test_trace_fields_are_repeated_on_every_span() -> None:
+ provider, egress = _provider(ACK)
+
+ provider.send(CONFIG, workflow_trace())
+
+ for attributes in _sent_attributes(egress):
+ assert attributes["langfuse.trace.name"] == "workflow"
+ assert attributes["langfuse.user.id"] == "account-1"
+ assert attributes["langfuse.session.id"] == workflow_trace().session_id
@pytest.mark.parametrize(
@@ -137,73 +185,134 @@ def test_message_trace_becomes_trace_and_generation() -> None:
],
)
def test_chat_inputs_keep_the_langfuse_preview_format(inputs: JsonValue, expected: JsonValue) -> None:
- provider, egress = _provider(response(200, {}))
+ provider, egress = _provider(ACK)
trace = message_trace()
trace = trace.model_copy(update={"spans": (trace.root, trace.spans[1].model_copy(update={"inputs": inputs}))})
source = trace.model_dump()
provider.send(CONFIG, trace)
- _, generation = _sent_events(egress)
- assert generation["body"]["input"] == expected
+ _, generation = _sent_attributes(egress)
+ assert json.loads(str(generation["langfuse.observation.input"])) == expected
assert trace.model_dump() == source
+def test_absent_values_are_omitted() -> None:
+ provider, egress = _provider(ACK)
+ trace = message_trace()
+ llm = trace.spans[1].model_copy(
+ update={"inputs": None, "outputs": None, "usage": None, "cost": None, "time_to_first_token": None}
+ )
+ trace = trace.model_copy(update={"spans": (trace.root, llm), "user_id": None, "session_id": None})
+
+ provider.send(CONFIG, trace)
+
+ _, generation = _sent_attributes(egress)
+ for key in (
+ "langfuse.observation.input",
+ "langfuse.observation.output",
+ "langfuse.observation.usage_details",
+ "langfuse.observation.cost_details",
+ "langfuse.observation.completion_start_time",
+ "langfuse.user.id",
+ "langfuse.session.id",
+ ):
+ assert key not in generation
+
+
+def test_cost_in_another_currency_is_left_to_langfuse_pricing() -> None:
+ provider, egress = _provider(ACK)
+ trace = message_trace()
+ llm = trace.spans[1].model_copy(update={"cost": Cost(amount=0.003, currency="CNY")})
+
+ provider.send(CONFIG, trace.model_copy(update={"spans": (trace.root, llm)}))
+
+ _, generation = _sent_attributes(egress)
+ assert "langfuse.observation.cost_details" not in generation
+ assert "langfuse.observation.usage_details" in generation
+
+
def test_workflow_spans_keep_hierarchy_and_errors() -> None:
- provider, egress = _provider(response(200, {}))
+ provider, egress = _provider(ACK)
trace = workflow_trace()
+
provider.send(CONFIG, trace)
- _, llm, tool = _sent_events(egress)
- assert "parentObservationId" not in llm["body"]
- assert tool["body"]["parentObservationId"] == str(span_uuid(trace.trace_id, trace.spans[1].span_id))
- assert tool["type"] == "span-create"
- assert tool["body"]["level"] == "ERROR"
- assert tool["body"]["statusMessage"] == "Tool failed"
+ _, llm, tool = otlp_spans(egress)
+ assert llm.parent_span_id.hex() == trace.spans[0].span_id
+ assert tool.parent_span_id.hex() == trace.spans[1].span_id
+ assert otlp_attributes(tool)["langfuse.observation.type"] == "tool"
+ assert tool.status.code == Status.STATUS_CODE_ERROR
+ assert tool.status.message == "Tool failed"
+ assert otlp_attributes(llm)["langfuse.observation.type"] == "generation"
+ assert llm.status.code == Status.STATUS_CODE_OK
+
+def test_fragment_joins_the_message_trace_without_renaming_it() -> None:
+ provider, egress = _provider(ACK)
-def test_fragment_joins_the_message_trace_without_recreating_it() -> None:
- provider, egress = _provider(response(207, {"successes": [{"id": "x", "status": 201}], "errors": []}))
provider.send(CONFIG, tool_fragment())
- (tool,) = _sent_events(egress)
- assert tool["type"] == "span-create"
- assert tool["body"]["traceId"] == MESSAGE_ID
- assert "parentObservationId" not in tool["body"]
+ (tool,) = otlp_spans(egress)
+ assert tool.trace_id.hex() == message_trace().trace_id
+ assert tool.parent_span_id.hex() == span_id_for(MESSAGE_ID)
+ attributes = otlp_attributes(tool)
+ assert "langfuse.trace.name" not in attributes
+ assert attributes["langfuse.user.id"] == "end-user-1"
+
+
+def test_root_carries_the_trace_metadata() -> None:
+ provider, egress = _provider(ACK)
+ trace = message_trace()
+
+ provider.send(CONFIG, trace)
+
+ root, llm = _sent_attributes(egress)
+ assert {key: value for key, value in root.items() if key.startswith("langfuse.trace.metadata.")} == {
+ "langfuse.trace.metadata.message_id": MESSAGE_ID,
+ "langfuse.trace.metadata.app_id": "app-1",
+ "langfuse.trace.metadata.tenant_id": "tenant-1",
+ "langfuse.trace.metadata.dify_trace_id": trace.trace_id,
+ }
+ assert not any(key.startswith("langfuse.trace.metadata.") for key in llm)
-def test_external_trace_id_is_kept_verbatim() -> None:
- provider, egress = _provider(response(200, {}))
+def test_external_trace_id_is_kept_as_trace_metadata() -> None:
+ provider, egress = _provider(ACK)
trace = message_trace().model_copy(update={"external_trace_id": "caller-trace-1"})
- assert provider.send(CONFIG, trace).trace_id == "caller-trace-1"
- trace_event, generation = _sent_events(egress)
- assert trace_event["body"]["id"] == "caller-trace-1"
- assert generation["body"]["traceId"] == "caller-trace-1"
+ assert provider.send(CONFIG, trace).trace_id == trace.trace_id
+
+ root, _ = _sent_attributes(egress)
+ assert root["langfuse.trace.metadata.external_trace_id"] == "caller-trace-1"
+
+
+def test_fragment_does_not_overwrite_the_trace_metadata() -> None:
+ provider, egress = _provider(ACK)
+
+ provider.send(CONFIG, tool_fragment())
+
+ (tool,) = _sent_attributes(egress)
+ assert not any(key.startswith("langfuse.trace.metadata.") for key in tool)
-def test_event_ids_are_stable_across_redelivery() -> None:
- first, first_egress = _provider(response(200, {}))
- second, second_egress = _provider(response(200, {}))
+
+def test_span_ids_are_stable_across_redelivery() -> None:
+ first, first_egress = _provider(ACK)
+ second, second_egress = _provider(ACK)
first.send(CONFIG, workflow_trace())
second.send(CONFIG, workflow_trace())
- assert [event["id"] for event in _sent_events(first_egress)] == [
- event["id"] for event in _sent_events(second_egress)
- ]
+ assert [span.span_id for span in otlp_spans(first_egress)] == [span.span_id for span in otlp_spans(second_egress)]
@pytest.mark.parametrize(
("outcome", "expected"),
[
- (
- response(207, {"successes": [], "errors": [{"id": "e", "status": 400, "message": "bad"}]}),
- ExportRejectedError,
- ),
- (
- response(207, {"successes": [], "errors": [{"id": "e", "status": 500, "message": "busy"}]}),
- ExportRetryableError,
- ),
(response(401, {}), ExportRejectedError),
- (EgressError("read timeout", request_sent=True), ExportRetryableError),
+ (response(413, {}), ExportRejectedError),
+ (response(503, {}), ExportRetryableError),
+ (EgressError("refused", request_sent=False), ExportRetryableError),
+ # Not retried: Langfuse would duplicate spans it already accepted.
+ (EgressError("read timeout", request_sent=True), ExportOutcomeUnknownError),
],
)
def test_send_failures_are_classified(outcome: EgressResponse | Exception, expected: type[Exception]) -> None:
@@ -212,25 +321,46 @@ def test_send_failures_are_classified(outcome: EgressResponse | Exception, expec
provider.send(CONFIG, message_trace())
-def test_unreadable_ingestion_bodies_are_resent() -> None:
- provider, _ = _provider(EgressResponse(status_code=200, headers={}, content=b"<html>proxy</html>"))
- with pytest.raises(ExportRetryableError):
- provider.send(CONFIG, message_trace())
+ "accepted",
+ [
+ response(200, {}),
+ EgressResponse(status_code=200, headers={}, content=b""),
+ EgressResponse(status_code=204, headers={"content-type": "text/plain"}, content=b"accepted"),
+ ],
+)
+def test_any_success_status_means_the_request_was_accepted(accepted: EgressResponse) -> None:
+ provider, _ = _provider(accepted)
+ assert provider.send(CONFIG, message_trace()).trace_id == message_trace().trace_id
-def test_batches_are_sized_in_bytes() -> None:
- wide = message_trace().model_copy(
+def _wide_trace() -> Trace:
+ trace = message_trace()
+ return trace.model_copy(
update={
- "spans": tuple(
- span.model_copy(update={"inputs": "中" * (MAX_BATCH_BYTES // 4)}) for span in message_trace().spans
- )
+ "spans": tuple(span.model_copy(update={"inputs": "中" * (MAX_REQUEST_BYTES // 4)}) for span in trace.spans)
}
)
- provider, egress = _provider(
- response(207, {"successes": [], "errors": []}), response(207, {"successes": [], "errors": []})
- )
- provider.send(CONFIG, wide)
- assert len(egress.requests) >= 2
- assert all(len(request.content or b"") <= MAX_BATCH_BYTES * 1.1 for request in egress.requests)
+def test_large_units_are_split_by_size() -> None:
+ provider, egress = _provider(ACK, ACK)
+
+ provider.send(CONFIG, _wide_trace())
+
+ assert len(egress.requests) == 2
+ assert all(len(request.content or b"") <= MAX_REQUEST_BYTES for request in egress.requests)
+ assert [span.span_id.hex() for span in otlp_spans(egress)] == [span.span_id for span in _wide_trace().spans]
+
+
+def test_a_failure_after_an_accepted_part_is_not_resent() -> None:
+ provider, _ = _provider(ACK, response(503, {}))
+ with pytest.raises(ExportOutcomeUnknownError):
+ provider.send(CONFIG, _wide_trace())
+
+
+def test_a_failure_of_the_first_part_is_resent() -> None:
+ provider, egress = _provider(response(503, {}))
+ with pytest.raises(ExportRetryableError):
+ provider.send(CONFIG, _wide_trace())
+ assert len(egress.requests) == 1
diff --git a/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py b/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py
index 846d3871606..193388aae2a 100644
--- a/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py
+++ b/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py
@@ -7,6 +7,9 @@ from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import grpc # type: ignore[import-untyped]
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue
+from opentelemetry.proto.trace.v1.trace_pb2 import Span as OtlpSpan
from dify_trace_sdk.egress import EgressResponse
from dify_trace_sdk.errors import EgressBlockedError
@@ -74,6 +77,35 @@ class RecordingEgress:
return grpc.insecure_channel(self._grpc_target)
+def otlp_spans(egress: RecordingEgress) -> list[OtlpSpan]:
+ """Every span ``egress`` recorded in OTLP requests, in send order."""
+ return [
+ span
+ for request in egress.requests
+ for resource_spans in ExportTraceServiceRequest.FromString(request.content or b"").resource_spans
+ for scope_spans in resource_spans.scope_spans
+ for span in scope_spans.spans
+ ]
+
+
+def _scalar(value: AnyValue) -> object:
+ match value.WhichOneof("value"):
+ case "string_value":
+ return value.string_value
+ case "int_value":
+ return value.int_value
+ case "double_value":
+ return value.double_value
+ case "bool_value":
+ return value.bool_value
+ case other:
+ raise AssertionError(f"unexpected attribute value type {other}")
+
+
+def otlp_attributes(span: OtlpSpan) -> dict[str, object]:
+ return {item.key: _scalar(item.value) for item in span.attributes}
+
+
def response(status_code: int, body: object) -> EgressResponse:
content = body if isinstance(body, bytes) else json.dumps(body).encode()
content_type = "application/octet-stream" if isinstance(body, bytes) else "application/json"
diff --git a/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py b/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py
index 7a97ac9c91c..606026d30ba 100644
--- a/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py
+++ b/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py
@@ -1,4 +1,13 @@
-from dify_trace_sdk.testing import message_trace, tool_fragment, workflow_trace
+from dify_trace_sdk.otlp import encode_trace
+from dify_trace_sdk.testing import (
+ RecordedRequest,
+ RecordingEgress,
+ message_trace,
+ otlp_attributes,
+ otlp_spans,
+ tool_fragment,
+ workflow_trace,
+)
def test_sample_traces_are_valid_parent_first_trees() -> None:
@@ -7,3 +16,24 @@ def test_sample_traces_are_valid_parent_first_trees() -> None:
fragment = tool_fragment()
assert fragment.parent is not None
assert fragment.root.parent_span_id == fragment.parent.span_id
+
+
+def test_otlp_helpers_read_back_what_was_encoded() -> None:
+ trace = message_trace()
+ request = encode_trace(trace, resource={}, attributes=lambda span: {"name": span.name, "tokens": 3, "ok": True})
+ egress = RecordingEgress([], grpc_target=None)
+ egress.requests.append(
+ RecordedRequest(
+ method="POST",
+ url="https://collector.test/v1/traces",
+ headers={},
+ content=request.SerializeToString(),
+ timeout=1.0,
+ sensitive_url=False,
+ )
+ )
+
+ spans = otlp_spans(egress)
+
+ assert [span.span_id.hex() for span in spans] == [span.span_id for span in trace.spans]
+ assert otlp_attributes(spans[0]) == {"name": "message", "tokens": 3, "ok": True}
diff --git a/api/tests/unit_tests/services/tracing/test_runtime_bridge.py b/api/tests/unit_tests/services/tracing/test_runtime_bridge.py
index 9cf24c29429..3ed99f53f0f 100644
--- a/api/tests/unit_tests/services/tracing/test_runtime_bridge.py
+++ b/api/tests/unit_tests/services/tracing/test_runtime_bridge.py
@@ -23,7 +23,7 @@ from dify_trace_sdk.events import (
)
from dify_trace_sdk.ids import span_id_for
from dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace
-from dify_trace_sdk.testing import RecordingEgress, response
+from dify_trace_sdk.testing import RecordingEgress, otlp_attributes, otlp_spans, response
from flask import Flask
from sqlalchemy import URL, Engine, create_engine, delete, event, func, select
from sqlalchemy.orm import Session, sessionmaker
@@ -201,27 +201,11 @@ def exported() -> RecordingEgress:
return RecordingEgress([response(200, {})], grpc_target=None)
-def ingested(destination: RecordingEgress) -> list[dict[str, object]]:
- """The Langfuse ingestion events the provider sent."""
- events: list[dict[str, object]] = []
- for request in destination.requests:
- body = request.json()
- assert isinstance(body, dict)
- batch = body.get("batch")
- assert isinstance(batch, list)
- events.extend(item for item in batch if isinstance(item, dict))
- return events
-
-
-def observations(destination: RecordingEgress) -> list[dict[str, object]]:
- """The Langfuse observations the provider sent: every span except the trace's root."""
- bodies: list[dict[str, object]] = []
- for item in ingested(destination):
- body = item["body"]
- assert isinstance(body, dict)
- if item["type"] != "trace-create":
- bodies.append(body)
- return bodies
+def root_metadata(destination: RecordingEgress) -> dict[str, object]:
+ """The Langfuse metadata of the root observation the provider sent."""
+ root = next(span for span in otlp_spans(destination) if not span.parent_span_id)
+ prefix = "langfuse.observation.metadata."
+ return {key.removeprefix(prefix): value for key, value in otlp_attributes(root).items() if key.startswith(prefix)}
def run_count(sessions: sessionmaker[Session]) -> int | None:
@@ -654,12 +638,10 @@ class TestApplicationTraceHandoffCases:
assert child_case.service.link_managed_parent(tenant_id=child.tenant_id, info=record) is record
destination = exported()
deliver(child_case, event, destination)
- trace = next(item for item in ingested(destination) if item["type"] == "trace-create")
- body = trace["body"]
- assert isinstance(body, dict)
- assert body["metadata"]["parent_workflow_run_id"] == parent.id
- assert body["metadata"]["parent_node_execution_id"] == identifier("parent-tool-node")
- assert "linked_parent_workflow_run_id" not in body["metadata"]
+ metadata = root_metadata(destination)
+ assert metadata["parent_workflow_run_id"] == parent.id
+ assert metadata["parent_node_execution_id"] == identifier("parent-tool-node")
+ assert "linked_parent_workflow_run_id" not in metadata
assert run_count(sqlite_session_factory) == 0
@staticmethod
@@ -706,11 +688,9 @@ class TestApplicationTraceHandoffCases:
add_app_owner(child_case.collection, "provider-child")
destination = exported()
deliver(child_case, workflow_event(child, parent), destination)
- trace = next(item for item in ingested(destination) if item["type"] == "trace-create")
- body = trace["body"]
- assert isinstance(body, dict)
- assert body["metadata"]["linked_parent_trace_id"] == parent_trace.trace_id
- assert "parent_workflow_run_id" not in body["metadata"]
+ metadata = root_metadata(destination)
+ assert metadata["linked_parent_trace_id"] == parent_trace.trace_id
+ assert "parent_workflow_run_id" not in metadata
@staticmethod
def test_collector_database_failure_redelivers_without_provider_fallback(
@@ -834,12 +814,17 @@ class TestApplicationTraceHandoffCases:
delivered = case.collection.repository.get(managed.id)
assert delivered is not None
assert delivered.state is TraceRunState.ACCEPTED
- assert [request.url for request in destination.requests] == ["https://langfuse.test/api/public/ingestion"]
- generations = [item["body"] for item in ingested(destination) if item["type"] == "generation-create"]
- usages = [body["usage"] for body in generations if isinstance(body, dict)]
+ assert [request.url for request in destination.requests] == ["https://langfuse.test/api/public/otel/v1/traces"]
+ generations = [
+ attributes
+ for attributes in map(otlp_attributes, otlp_spans(destination))
+ if attributes["langfuse.observation.type"] == "generation"
+ ]
+ usages = [json.loads(str(attributes["langfuse.observation.usage_details"])) for attributes in generations]
+ costs = [json.loads(str(attributes["langfuse.observation.cost_details"])) for attributes in generations]
assert len(usages) == 3
assert sum(usage["total"] for usage in usages) == 25
- assert sum(usage["totalCost"] for usage in usages) == pytest.approx(0.04)
+ assert sum(cost["total"] for cost in costs) == pytest.approx(0.04)
"""Verify execution hook ownership, policy gating, and fail-open business behavior."""
@@ -1462,10 +1447,12 @@ class TestWorkflowToolTracingBoundaryCases:
assert current is not None
# Assembly rejects any span whose parent is missing, so acceptance proves a complete tree.
assert current.state is TraceRunState.ACCEPTED
- assert [request.url for request in destination.requests] == ["https://tracing.test/root/api/public/ingestion"]
- sent = observations(destination)
- emitted = {body["id"] for body in sent}
- assert all(body["parentObservationId"] in emitted for body in sent if "parentObservationId" in body)
+ assert [request.url for request in destination.requests] == [
+ "https://tracing.test/root/api/public/otel/v1/traces"
+ ]
+ sent = otlp_spans(destination)
+ emitted = {span.span_id for span in sent}
+ assert all(span.parent_span_id in emitted for span in sent if span.parent_span_id)
assert current.provider_config == frozen
assert current.provider == "langfuse"
assert current.capture_content is False
diff --git a/api/tests/unit_tests/tasks/test_application_trace_task.py b/api/tests/unit_tests/tasks/test_application_trace_task.py
index da70d0865f7..8eaec0c1e92 100644
--- a/api/tests/unit_tests/tasks/test_application_trace_task.py
+++ b/api/tests/unit_tests/tasks/test_application_trace_task.py
@@ -2,7 +2,7 @@
from dataclasses import replace
from datetime import UTC, datetime, timedelta
-from uuid import UUID, uuid4
+from uuid import uuid4
import httpx
import pytest
@@ -100,14 +100,13 @@ def test_worker_task_exports_once_through_the_ssrf_proxy_and_replay_observes_che
monkeypatch.setattr(ssrf_proxy, "make_request", send)
export_application_trace.run(case.run.id)
export_application_trace.run(case.run.id)
- assert proxied == [("POST", "https://langfuse.test/api/public/ingestion")]
+ assert proxied == [("POST", "https://langfuse.test/api/public/otel/v1/traces")]
result = case.repository.get(case.run.id)
assert result is not None
assert result.state is TraceRunState.ACCEPTED
assert result.attempts == 1
- provider_trace_id = str(UUID(hex=case.run.trace_id))
- assert result.provider_trace_id == provider_trace_id
- assert result.trace_url == f"https://langfuse.test/project/original/traces/{provider_trace_id}"
+ assert result.provider_trace_id == case.run.trace_id
+ assert result.trace_url == f"https://langfuse.test/project/original/traces/{case.run.trace_id}"
def test_worker_records_carry_no_enterprise_names(sqlite_session_factory: sessionmaker[Session]) -> None: