Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(trace): send Langfuse traces over OpenTelemetry (#45606)
Xiyuan Chen0be453d27b22
diff --git a/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.pyindex 620ecd12388..e4eed3faa68 100644--- a/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py+++ b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/provider.py@@ -1,39 +1,33 @@-"""Langfuse destination over the public ingestion API.+"""Langfuse destination over its OpenTelemetry endpoint.-Each trace unit becomes ingestion events: the root span is the Langfuse trace-and every other span an observation (a generation for LLM calls). Langfuse-upserts by ID, so re-sending a unit after a partial failure is safe.+Langfuse does not deduplicate re-sent spans, so a unit is retried only when+none of it was accepted."""import base64-from collections.abc import Mapping, Sequence-from datetime import datetime, timedelta+from datetime import timedeltafrom typing import override-from urllib.parse import quote-from uuid import uuid5from dify_trace_sdk.errors import (EgressError,ExportOutcomeUnknownError,- ExportRejectedError,ExportRetryableError,VerificationError,)-from dify_trace_sdk.exporting import export_endpoint, json_value, send_export_request-from dify_trace_sdk.ids import span_uuid, trace_uuid-from dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace+from dify_trace_sdk.exporting import export_endpoint, json_value+from dify_trace_sdk.model import Span, SpanKind, Trace+from dify_trace_sdk.otlp import encode_trace, post_otlpfrom dify_trace_sdk.provider import ProviderCapabilities, SendResult, TraceProvider+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequestfrom pydantic import JsonValuefrom dify_trace_langfuse.config import LangfuseConfig+DESTINATION = "Langfuse"+SERVICE_NAME = "dify"VERIFY_TIMEOUT_SECONDS = 10.0-MAX_BATCH_BYTES = 3_000_000-"""Stay below Langfuse Cloud's ingestion request limit."""---def _iso(value: datetime) -> str:- return value.isoformat()+MAX_REQUEST_BYTES = 3_000_000+INGESTION_VERSION_HEADER = {"x-langfuse-ingestion-version": "4"}def _messages(value: JsonValue, *, role: str) -> JsonValue:@@ -50,15 +44,104 @@ def _messages(value: JsonValue, *, role: str) -> JsonValue:return value+def _json(value: JsonValue) -> str | None:+ return None if value is None else json_value(value)+++def _trace_attributes(trace: Trace) -> dict[str, object]:+ """Repeated on every span so Langfuse can filter observations by them."""+ return {+ "langfuse.trace.name": trace.root.name if trace.parent is None else None,+ "langfuse.user.id": trace.user_id,+ "langfuse.session.id": trace.session_id,+ }+++def _trace_metadata(trace: Trace) -> dict[str, object]:+ if trace.parent is not None:+ return {}+ metadata = {+ **trace.root.attributes,+ "dify_trace_id": trace.trace_id,+ "external_trace_id": trace.external_trace_id,+ }+ return {f"langfuse.trace.metadata.{key}": value for key, value in metadata.items()}+++def _generation_attributes(span: Span) -> dict[str, object]:+ attributes: dict[str, object] = {"langfuse.observation.model.name": span.model_name}+ if span.usage is not None:+ attributes["langfuse.observation.usage_details"] = json_value(+ {+ "input": span.usage.prompt_tokens,+ "output": span.usage.completion_tokens,+ "total": span.usage.total_tokens,+ }+ )+ if span.cost is not None and span.cost.currency.upper() == "USD":+ attributes["langfuse.observation.cost_details"] = json_value({"total": span.cost.amount})+ if span.time_to_first_token is not None:+ attributes["langfuse.observation.completion_start_time"] = (+ span.start_time + timedelta(seconds=span.time_to_first_token)+ ).isoformat()+ return attributes+++def _span_attributes(span: Span) -> dict[str, object]:+ # Other span kinds match Langfuse's type names.+ attributes: dict[str, object] = {+ "langfuse.observation.type": "generation" if span.kind is SpanKind.LLM else span.kind.value,+ "langfuse.observation.input": _json(_messages(span.inputs, role="user")),+ "langfuse.observation.output": _json(_messages(span.outputs, role="assistant")),+ "langfuse.observation.metadata.dify_span_kind": span.kind.value,+ **{f"langfuse.observation.metadata.{key}": value for key, value in span.attributes.items()},+ }+ if span.kind is SpanKind.LLM:+ attributes.update(_generation_attributes(span))+ return attributes+++def _export_request(trace: Trace) -> ExportTraceServiceRequest:+ trace_attributes = _trace_attributes(trace)+ trace_metadata = _trace_metadata(trace)+ return encode_trace(+ trace,+ resource={"service.name": SERVICE_NAME},+ attributes=lambda span: {+ **trace_attributes,+ **_span_attributes(span),+ **(trace_metadata if span.span_id == trace.root.span_id else {}),+ },+ )+++def _split(request: ExportTraceServiceRequest) -> list[ExportTraceServiceRequest]:+ """An oversized span travels alone."""+ (resource_spans,) = request.resource_spans+ (scope_spans,) = resource_spans.scope_spans+ chunks: list[ExportTraceServiceRequest] = []+ size = 0+ for span in scope_spans.spans:+ span_size = span.ByteSize()+ if not chunks or size + span_size > MAX_REQUEST_BYTES:+ chunk = ExportTraceServiceRequest()+ chunk.resource_spans.add(resource=resource_spans.resource).scope_spans.add(scope=scope_spans.scope)+ chunks.append(chunk)+ size = 0+ chunks[-1].resource_spans[0].scope_spans[0].spans.append(span)+ size += span_size+ return chunks++class LangfuseProvider(TraceProvider[LangfuseConfig]):id = "langfuse"config_model = LangfuseConfigcapabilities = ProviderCapabilities(native_cost=True, trace_url=True, span_url=True)@staticmethod- def _headers(config: LangfuseConfig) -> dict[str, str]:+ def _authorization(config: LangfuseConfig) -> dict[str, str]:token = base64.b64encode(f"{config.public_key}:{config.secret_key}".encode()).decode()- return {"Authorization": f"Basic {token}", "Accept": "application/json"}+ return {"Authorization": f"Basic {token}"}@overridedef verify(self, config: LangfuseConfig) -> LangfuseConfig:@@ -66,7 +149,7 @@ class LangfuseProvider(TraceProvider[LangfuseConfig]):response = self.egress.request("GET",f"{config.host.rstrip('/')}/api/public/projects",- headers=self._headers(config),+ headers={**self._authorization(config), "Accept": "application/json"},content=None,timeout=VERIFY_TIMEOUT_SECONDS,sensitive_url=False,@@ -94,147 +177,21 @@ class LangfuseProvider(TraceProvider[LangfuseConfig]):@overridedef send(self, config: LangfuseConfig, trace: Trace) -> SendResult:- host = export_endpoint(config.host)- trace_key = trace.external_trace_id or str(trace_uuid(trace.trace_id))- events = self._events(trace, trace_key)- for batch in self._batches(events):+ url = f"{export_endpoint(config.host)}/api/public/otel/v1/traces"+ headers = {**self._authorization(config), **INGESTION_VERSION_HEADER}+ for index, request in enumerate(_split(_export_request(trace))):try:- response = send_export_request(- self.egress,- "POST",- f"{host}/api/public/ingestion",- destination="Langfuse",- headers={**self._headers(config), "Content-Type": "application/json"},- content=batch,- sensitive_url=False,- )- except ExportOutcomeUnknownError:- raise ExportRetryableError("Langfuse ingestion outcome is unknown; re-sending is idempotent") from None- try:- body = response.json() if response.content else {}- except ValueError:- # Something between Dify and Langfuse answered; re-sending is idempotent.- raise ExportRetryableError("Langfuse ingestion answered with an unreadable body") from None- self._check_results(body)+ post_otlp(self.egress, url, request, destination=DESTINATION, headers=headers, sensitive_url=False)+ except ExportRetryableError:+ if index == 0:+ raise+ raise ExportOutcomeUnknownError(f"{DESTINATION} accepted part of the trace") from Nonetrace_url = (- f"{self.project_url(config).rstrip('/')}/traces/{quote(trace_key, safe='')}"+ f"{self.project_url(config).rstrip('/')}/traces/{trace.trace_id}"if config.project_key or config.project_urlelse None)span_urls = (- {- span.span_id: f"{trace_url}?observation={self._observation_id(trace, span.span_id)}"- for span in trace.spans- }- if trace_url- else {}+ {span.span_id: f"{trace_url}?observation={span.span_id}" for span in trace.spans} if trace_url else {})- return SendResult(trace_id=trace_key, trace_url=trace_url, span_urls=span_urls)-- @staticmethod- def _observation_id(trace: Trace, span_id: str) -> str:- return str(span_uuid(trace.trace_id, span_id))-- def _events(self, trace: Trace, trace_key: str) -> list[dict[str, object]]:- events: list[dict[str, object]] = []- if trace.parent is None:- root = trace.root- events.append(- self._event(- trace,- "trace-create",- {- "id": trace_key,- "name": root.name,- "timestamp": _iso(root.start_time),- "userId": trace.user_id,- "sessionId": trace.session_id,- "input": _messages(root.inputs, role="user"),- "output": _messages(root.outputs, role="assistant"),- "metadata": {**root.attributes, "dify_trace_id": trace.trace_id},- },- )- )- # The trace itself represents the root; its children are top-level observations.- trace_level_span_id = root.span_id- observations: Sequence[Span] = trace.spans[1:]- else:- trace_level_span_id = trace.parent.span_id- observations = trace.spans- for span in observations:- parent = (- None- if span.parent_span_id == trace_level_span_id or span.parent_span_id is None- else self._observation_id(trace, span.parent_span_id)- )- body: dict[str, object] = {- "id": self._observation_id(trace, span.span_id),- "traceId": trace_key,- "parentObservationId": parent,- "name": span.name,- "startTime": _iso(span.start_time),- "endTime": _iso(span.end_time),- "input": _messages(span.inputs, role="user"),- "output": _messages(span.outputs, role="assistant"),- "metadata": {**span.attributes, "dify_span_kind": span.kind.value},- "level": "ERROR" if span.status is SpanStatus.ERROR else "DEFAULT",- "statusMessage": span.error,- }- if span.kind is SpanKind.LLM:- body.update(self._generation_fields(span))- events.append(self._event(trace, "generation-create", body))- else:- events.append(self._event(trace, "span-create", body))- return events-- @staticmethod- def _generation_fields(span: Span) -> dict[str, object]:- fields: dict[str, object] = {"model": span.model_name}- if span.usage is not None:- usage: dict[str, object] = {- "input": span.usage.prompt_tokens,- "output": span.usage.completion_tokens,- "total": span.usage.total_tokens,- "unit": "TOKENS",- }- if span.cost is not None and span.cost.currency.upper() == "USD":- usage["totalCost"] = span.cost.amount- fields["usage"] = usage- if span.time_to_first_token is not None:- fields["completionStartTime"] = _iso(span.start_time + timedelta(seconds=span.time_to_first_token))- return fields-- @staticmethod- def _event(trace: Trace, event_type: str, body: Mapping[str, object]) -> dict[str, object]:- # Deterministic event IDs let Langfuse deduplicate a redelivered unit.- return {- "id": str(uuid5(trace_uuid(trace.trace_id), f"{event_type}:{body['id']}")),- "type": event_type,- "timestamp": body.get("startTime") or body.get("timestamp"),- "body": {key: value for key, value in body.items() if value is not None},- }-- @staticmethod- def _batches(events: list[dict[str, object]]) -> list[bytes]:- batches: list[bytes] = []- current: list[bytes] = []- size = 0- for event in events:- encoded = json_value(event).encode()- if current and size + len(encoded) > MAX_BATCH_BYTES:- batches.append(b'{"batch":[' + b",".join(current) + b"]}")- current, size = [], 0- current.append(encoded)- size += len(encoded) + 1- batches.append(b'{"batch":[' + b",".join(current) + b"]}")- return batches-- @staticmethod- def _check_results(body: object) -> None:- errors = body.get("errors") if isinstance(body, dict) else None- if not isinstance(errors, list) or not errors:- return- statuses = [error.get("status") for error in errors if isinstance(error, dict)]- if any(isinstance(status, int) and (status == 429 or status >= 500) for status in statuses):- raise ExportRetryableError("Langfuse could not ingest every event; re-sending is idempotent")- raise ExportRejectedError(f"Langfuse rejected {len(errors)} ingestion events")+ return SendResult(trace_id=trace.trace_id, trace_url=trace_url, span_urls=span_urls)diff --git a/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py b/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.pyindex 8227f931cca..32f1616f838 100644--- a/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py+++ b/api/providers/trace/trace-langfuse/tests/unit_tests/src/dify_trace_langfuse/test_provider.py@@ -1,20 +1,36 @@import base64+import jsonimport pytestfrom dify_trace_langfuse.config import LangfuseConfig-from dify_trace_langfuse.provider import MAX_BATCH_BYTES, LangfuseProvider+from dify_trace_langfuse.provider import MAX_REQUEST_BYTES, LangfuseProviderfrom dify_trace_sdk.egress import EgressResponsefrom dify_trace_sdk.errors import (EgressError,+ ExportOutcomeUnknownError,ExportRejectedError,ExportRetryableError,VerificationError,)-from dify_trace_sdk.ids import span_uuid-from dify_trace_sdk.testing import MESSAGE_ID, RecordingEgress, message_trace, response, tool_fragment, workflow_trace+from dify_trace_sdk.ids import span_id_for+from dify_trace_sdk.model import Cost, Trace+from dify_trace_sdk.testing import (+ MESSAGE_ID,+ RecordingEgress,+ message_trace,+ otlp_attributes,+ otlp_spans,+ response,+ tool_fragment,+ workflow_trace,+)+from opentelemetry.proto.trace.v1.trace_pb2 import Statusfrom pydantic import JsonValueCONFIG = LangfuseConfig(public_key="pk-lf-1", secret_key="sk-lf-1", host="https://cloud.langfuse.com")+OTEL_URL = "https://cloud.langfuse.com/api/public/otel/v1/traces"+# Langfuse replies with its queued job, not an OTLP acknowledgment.+ACK = response(200, {"name": "otel-ingestion-job", "data": {"id": "job-1"}, "opts": {"attempts": 6}, "progress": 0})def _provider(*responses: EgressResponse | Exception) -> tuple[LangfuseProvider, RecordingEgress]:@@ -22,11 +38,8 @@ def _provider(*responses: EgressResponse | Exception) -> tuple[LangfuseProvider,return LangfuseProvider(egress), egress-def _sent_events(egress: RecordingEgress) -> list[dict]:- (sent,) = egress.requests- body = sent.json()- assert isinstance(body, dict)- return body["batch"]+def _sent_attributes(egress: RecordingEgress) -> list[dict[str, object]]:+ return [otlp_attributes(span) for span in otlp_spans(egress)]def test_verify_resolves_the_project() -> None:@@ -41,7 +54,7 @@ def test_verify_resolves_the_project() -> None:def test_legacy_config_keeps_project_and_trace_links_without_verification() -> None:- provider, egress = _provider(response(200, {}))+ provider, egress = _provider(ACK)config = LangfuseConfig.model_validate({"public_key": "pk-lf-1",@@ -50,16 +63,17 @@ def test_legacy_config_keeps_project_and_trace_links_without_verification() -> N"project_url": "https://dashboard.example/langfuse/project/legacy-project/",})+ trace = message_trace()- result = provider.send(config, message_trace())+ result = provider.send(config, trace)assert provider.project_url(config) == config.project_url- trace_url = f"https://dashboard.example/langfuse/project/legacy-project/traces/{MESSAGE_ID}"+ trace_url = f"https://dashboard.example/langfuse/project/legacy-project/traces/{trace.trace_id}"assert result.trace_url == trace_url- assert result.span_urls- assert all(url.startswith(trace_url + "?observation=") for url in result.span_urls.values())- assert len(egress.requests) == 1- assert egress.requests[0].url == "https://collector.example/langfuse/api/public/ingestion"+ assert result.span_urls == {span.span_id: f"{trace_url}?observation={span.span_id}" for span in trace.spans}+ assert [request.url for request in egress.requests] == [+ "https://collector.example/langfuse/api/public/otel/v1/traces"+ ]verified = config.model_copy(update={"project_key": "new-project"})assert provider.project_url(verified) == "https://collector.example/langfuse/project/new-project"@@ -81,27 +95,61 @@ def test_verify_failures_have_stable_codes(outcome: EgressResponse | Exception,assert caught.value.code == code-def test_message_trace_becomes_trace_and_generation() -> None:- provider, egress = _provider(response(207, {"successes": [], "errors": []}))+def test_send_posts_otlp_protobuf_with_v4_ingestion_header() -> None:+ provider, egress = _provider(ACK)++ provider.send(CONFIG, message_trace())++ (sent,) = egress.requests+ assert (sent.method, sent.url) == ("POST", OTEL_URL)+ assert sent.headers["Authorization"] == "Basic " + base64.b64encode(b"pk-lf-1:sk-lf-1").decode()+ assert sent.headers["Content-Type"] == "application/x-protobuf"+ assert sent.headers["x-langfuse-ingestion-version"] == "4"+++def test_message_trace_becomes_a_root_observation_and_a_generation() -> None:+ provider, egress = _provider(ACK)trace = message_trace()+result = provider.send(CONFIG.model_copy(update={"project_key": "project-1"}), trace)- trace_event, generation = _sent_events(egress)- assert trace_event["type"] == "trace-create"- assert trace_event["body"]["id"] == MESSAGE_ID- assert trace_event["body"]["sessionId"] == "conversation-1"- assert trace_event["body"]["userId"] == "end-user-1"- assert generation["type"] == "generation-create"- body = generation["body"]- assert body["traceId"] == MESSAGE_ID- assert "parentObservationId" not in body- assert body["model"] == "gpt-4o"- assert body["usage"] == {"input": 12, "output": 8, "total": 20, "unit": "TOKENS", "totalCost": 0.0004}- assert body["completionStartTime"] == "2026-10-01T08:00:00.250000+00:00"- assert body["input"] == [{"role": "user", "content": "What is Dify?"}]- assert body["output"] == [{"role": "assistant", "content": "An LLM application platform."}]- assert result.trace_id == MESSAGE_ID- assert result.trace_url == f"https://cloud.langfuse.com/project/project-1/traces/{MESSAGE_ID}"+ root, llm = otlp_spans(egress)+ assert root.trace_id.hex() == llm.trace_id.hex() == trace.trace_id+ assert root.span_id.hex() == trace.spans[0].span_id+ assert root.parent_span_id == b""+ assert llm.parent_span_id.hex() == trace.spans[0].span_id+ root_attributes, llm_attributes = otlp_attributes(root), otlp_attributes(llm)+ assert root_attributes["langfuse.observation.type"] == "chain"+ assert root_attributes["langfuse.observation.metadata.app_id"] == "app-1"+ assert json.loads(str(root_attributes["langfuse.observation.output"])) == [+ {"role": "assistant", "content": "An LLM application platform."}+ ]+ assert llm_attributes["langfuse.observation.type"] == "generation"+ assert llm_attributes["langfuse.observation.model.name"] == "gpt-4o"+ assert json.loads(str(llm_attributes["langfuse.observation.usage_details"])) == {+ "input": 12,+ "output": 8,+ "total": 20,+ }+ assert json.loads(str(llm_attributes["langfuse.observation.cost_details"])) == {"total": 0.0004}+ assert llm_attributes["langfuse.observation.completion_start_time"] == "2026-10-01T08:00:00.250000+00:00"+ assert json.loads(str(llm_attributes["langfuse.observation.input"])) == [+ {"role": "user", "content": "What is Dify?"}+ ]+ assert result.trace_id == trace.trace_id+ assert result.trace_url == f"https://cloud.langfuse.com/project/project-1/traces/{trace.trace_id}"+ assert result.span_urls[trace.spans[1].span_id] == f"{result.trace_url}?observation={trace.spans[1].span_id}"+++def test_trace_fields_are_repeated_on_every_span() -> None:+ provider, egress = _provider(ACK)++ provider.send(CONFIG, workflow_trace())++ for attributes in _sent_attributes(egress):+ assert attributes["langfuse.trace.name"] == "workflow"+ assert attributes["langfuse.user.id"] == "account-1"+ assert attributes["langfuse.session.id"] == workflow_trace().session_id@pytest.mark.parametrize(@@ -137,73 +185,134 @@ def test_message_trace_becomes_trace_and_generation() -> None:],)def test_chat_inputs_keep_the_langfuse_preview_format(inputs: JsonValue, expected: JsonValue) -> None:- provider, egress = _provider(response(200, {}))+ provider, egress = _provider(ACK)trace = message_trace()trace = trace.model_copy(update={"spans": (trace.root, trace.spans[1].model_copy(update={"inputs": inputs}))})source = trace.model_dump()provider.send(CONFIG, trace)- _, generation = _sent_events(egress)- assert generation["body"]["input"] == expected+ _, generation = _sent_attributes(egress)+ assert json.loads(str(generation["langfuse.observation.input"])) == expectedassert trace.model_dump() == source+def test_absent_values_are_omitted() -> None:+ provider, egress = _provider(ACK)+ trace = message_trace()+ llm = trace.spans[1].model_copy(+ update={"inputs": None, "outputs": None, "usage": None, "cost": None, "time_to_first_token": None}+ )+ trace = trace.model_copy(update={"spans": (trace.root, llm), "user_id": None, "session_id": None})++ provider.send(CONFIG, trace)++ _, generation = _sent_attributes(egress)+ for key in (+ "langfuse.observation.input",+ "langfuse.observation.output",+ "langfuse.observation.usage_details",+ "langfuse.observation.cost_details",+ "langfuse.observation.completion_start_time",+ "langfuse.user.id",+ "langfuse.session.id",+ ):+ assert key not in generation+++def test_cost_in_another_currency_is_left_to_langfuse_pricing() -> None:+ provider, egress = _provider(ACK)+ trace = message_trace()+ llm = trace.spans[1].model_copy(update={"cost": Cost(amount=0.003, currency="CNY")})++ provider.send(CONFIG, trace.model_copy(update={"spans": (trace.root, llm)}))++ _, generation = _sent_attributes(egress)+ assert "langfuse.observation.cost_details" not in generation+ assert "langfuse.observation.usage_details" in generation++def test_workflow_spans_keep_hierarchy_and_errors() -> None:- provider, egress = _provider(response(200, {}))+ provider, egress = _provider(ACK)trace = workflow_trace()+provider.send(CONFIG, trace)- _, llm, tool = _sent_events(egress)- assert "parentObservationId" not in llm["body"]- assert tool["body"]["parentObservationId"] == str(span_uuid(trace.trace_id, trace.spans[1].span_id))- assert tool["type"] == "span-create"- assert tool["body"]["level"] == "ERROR"- assert tool["body"]["statusMessage"] == "Tool failed"+ _, llm, tool = otlp_spans(egress)+ assert llm.parent_span_id.hex() == trace.spans[0].span_id+ assert tool.parent_span_id.hex() == trace.spans[1].span_id+ assert otlp_attributes(tool)["langfuse.observation.type"] == "tool"+ assert tool.status.code == Status.STATUS_CODE_ERROR+ assert tool.status.message == "Tool failed"+ assert otlp_attributes(llm)["langfuse.observation.type"] == "generation"+ assert llm.status.code == Status.STATUS_CODE_OK++def test_fragment_joins_the_message_trace_without_renaming_it() -> None:+ provider, egress = _provider(ACK)-def test_fragment_joins_the_message_trace_without_recreating_it() -> None:- provider, egress = _provider(response(207, {"successes": [{"id": "x", "status": 201}], "errors": []}))provider.send(CONFIG, tool_fragment())- (tool,) = _sent_events(egress)- assert tool["type"] == "span-create"- assert tool["body"]["traceId"] == MESSAGE_ID- assert "parentObservationId" not in tool["body"]+ (tool,) = otlp_spans(egress)+ assert tool.trace_id.hex() == message_trace().trace_id+ assert tool.parent_span_id.hex() == span_id_for(MESSAGE_ID)+ attributes = otlp_attributes(tool)+ assert "langfuse.trace.name" not in attributes+ assert attributes["langfuse.user.id"] == "end-user-1"+++def test_root_carries_the_trace_metadata() -> None:+ provider, egress = _provider(ACK)+ trace = message_trace()++ provider.send(CONFIG, trace)++ root, llm = _sent_attributes(egress)+ assert {key: value for key, value in root.items() if key.startswith("langfuse.trace.metadata.")} == {+ "langfuse.trace.metadata.message_id": MESSAGE_ID,+ "langfuse.trace.metadata.app_id": "app-1",+ "langfuse.trace.metadata.tenant_id": "tenant-1",+ "langfuse.trace.metadata.dify_trace_id": trace.trace_id,+ }+ assert not any(key.startswith("langfuse.trace.metadata.") for key in llm)-def test_external_trace_id_is_kept_verbatim() -> None:- provider, egress = _provider(response(200, {}))+def test_external_trace_id_is_kept_as_trace_metadata() -> None:+ provider, egress = _provider(ACK)trace = message_trace().model_copy(update={"external_trace_id": "caller-trace-1"})- assert provider.send(CONFIG, trace).trace_id == "caller-trace-1"- trace_event, generation = _sent_events(egress)- assert trace_event["body"]["id"] == "caller-trace-1"- assert generation["body"]["traceId"] == "caller-trace-1"+ assert provider.send(CONFIG, trace).trace_id == trace.trace_id++ root, _ = _sent_attributes(egress)+ assert root["langfuse.trace.metadata.external_trace_id"] == "caller-trace-1"+++def test_fragment_does_not_overwrite_the_trace_metadata() -> None:+ provider, egress = _provider(ACK)++ provider.send(CONFIG, tool_fragment())++ (tool,) = _sent_attributes(egress)+ assert not any(key.startswith("langfuse.trace.metadata.") for key in tool)-def test_event_ids_are_stable_across_redelivery() -> None:- first, first_egress = _provider(response(200, {}))- second, second_egress = _provider(response(200, {}))++def test_span_ids_are_stable_across_redelivery() -> None:+ first, first_egress = _provider(ACK)+ second, second_egress = _provider(ACK)first.send(CONFIG, workflow_trace())second.send(CONFIG, workflow_trace())- assert [event["id"] for event in _sent_events(first_egress)] == [- event["id"] for event in _sent_events(second_egress)- ]+ assert [span.span_id for span in otlp_spans(first_egress)] == [span.span_id for span in otlp_spans(second_egress)]@pytest.mark.parametrize(("outcome", "expected"),[- (- response(207, {"successes": [], "errors": [{"id": "e", "status": 400, "message": "bad"}]}),- ExportRejectedError,- ),- (- response(207, {"successes": [], "errors": [{"id": "e", "status": 500, "message": "busy"}]}),- ExportRetryableError,- ),(response(401, {}), ExportRejectedError),- (EgressError("read timeout", request_sent=True), ExportRetryableError),+ (response(413, {}), ExportRejectedError),+ (response(503, {}), ExportRetryableError),+ (EgressError("refused", request_sent=False), ExportRetryableError),+ # Not retried: Langfuse would duplicate spans it already accepted.+ (EgressError("read timeout", request_sent=True), ExportOutcomeUnknownError),],)def test_send_failures_are_classified(outcome: EgressResponse | Exception, expected: type[Exception]) -> None:@@ -212,25 +321,46 @@ def test_send_failures_are_classified(outcome: EgressResponse | Exception, expecprovider.send(CONFIG, message_trace())-def test_unreadable_ingestion_bodies_are_resent() -> None:- provider, _ = _provider(EgressResponse(status_code=200, headers={}, content=b"<html>proxy</html>"))- with pytest.raises(ExportRetryableError):- provider.send(CONFIG, message_trace())+ "accepted",+ [+ response(200, {}),+ EgressResponse(status_code=200, headers={}, content=b""),+ EgressResponse(status_code=204, headers={"content-type": "text/plain"}, content=b"accepted"),+ ],+)+def test_any_success_status_means_the_request_was_accepted(accepted: EgressResponse) -> None:+ provider, _ = _provider(accepted)+ assert provider.send(CONFIG, message_trace()).trace_id == message_trace().trace_id-def test_batches_are_sized_in_bytes() -> None:- wide = message_trace().model_copy(+def _wide_trace() -> Trace:+ trace = message_trace()+ return trace.model_copy(update={- "spans": tuple(- span.model_copy(update={"inputs": "中" * (MAX_BATCH_BYTES // 4)}) for span in message_trace().spans- )+ "spans": tuple(span.model_copy(update={"inputs": "中" * (MAX_REQUEST_BYTES // 4)}) for span in trace.spans)})- provider, egress = _provider(- response(207, {"successes": [], "errors": []}), response(207, {"successes": [], "errors": []})- )- provider.send(CONFIG, wide)- assert len(egress.requests) >= 2- assert all(len(request.content or b"") <= MAX_BATCH_BYTES * 1.1 for request in egress.requests)+def test_large_units_are_split_by_size() -> None:+ provider, egress = _provider(ACK, ACK)++ provider.send(CONFIG, _wide_trace())++ assert len(egress.requests) == 2+ assert all(len(request.content or b"") <= MAX_REQUEST_BYTES for request in egress.requests)+ assert [span.span_id.hex() for span in otlp_spans(egress)] == [span.span_id for span in _wide_trace().spans]+++def test_a_failure_after_an_accepted_part_is_not_resent() -> None:+ provider, _ = _provider(ACK, response(503, {}))+ with pytest.raises(ExportOutcomeUnknownError):+ provider.send(CONFIG, _wide_trace())+++def test_a_failure_of_the_first_part_is_resent() -> None:+ provider, egress = _provider(response(503, {}))+ with pytest.raises(ExportRetryableError):+ provider.send(CONFIG, _wide_trace())+ assert len(egress.requests) == 1diff --git a/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py b/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.pyindex 846d3871606..193388aae2a 100644--- a/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py+++ b/api/providers/trace/trace-sdk/src/dify_trace_sdk/testing.py@@ -7,6 +7,9 @@ from dataclasses import dataclassfrom datetime import UTC, datetime, timedeltaimport grpc # type: ignore[import-untyped]+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+from opentelemetry.proto.common.v1.common_pb2 import AnyValue+from opentelemetry.proto.trace.v1.trace_pb2 import Span as OtlpSpanfrom dify_trace_sdk.egress import EgressResponsefrom dify_trace_sdk.errors import EgressBlockedError@@ -74,6 +77,35 @@ class RecordingEgress:return grpc.insecure_channel(self._grpc_target)+def otlp_spans(egress: RecordingEgress) -> list[OtlpSpan]:+ """Every span ``egress`` recorded in OTLP requests, in send order."""+ return [+ span+ for request in egress.requests+ for resource_spans in ExportTraceServiceRequest.FromString(request.content or b"").resource_spans+ for scope_spans in resource_spans.scope_spans+ for span in scope_spans.spans+ ]+++def _scalar(value: AnyValue) -> object:+ match value.WhichOneof("value"):+ case "string_value":+ return value.string_value+ case "int_value":+ return value.int_value+ case "double_value":+ return value.double_value+ case "bool_value":+ return value.bool_value+ case other:+ raise AssertionError(f"unexpected attribute value type {other}")+++def otlp_attributes(span: OtlpSpan) -> dict[str, object]:+ return {item.key: _scalar(item.value) for item in span.attributes}++def response(status_code: int, body: object) -> EgressResponse:content = body if isinstance(body, bytes) else json.dumps(body).encode()content_type = "application/octet-stream" if isinstance(body, bytes) else "application/json"diff --git a/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py b/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.pyindex 7a97ac9c91c..606026d30ba 100644--- a/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py+++ b/api/providers/trace/trace-sdk/tests/unit_tests/src/dify_trace_sdk/test_testing.py@@ -1,4 +1,13 @@-from dify_trace_sdk.testing import message_trace, tool_fragment, workflow_trace+from dify_trace_sdk.otlp import encode_trace+from dify_trace_sdk.testing import (+ RecordedRequest,+ RecordingEgress,+ message_trace,+ otlp_attributes,+ otlp_spans,+ tool_fragment,+ workflow_trace,+)def test_sample_traces_are_valid_parent_first_trees() -> None:@@ -7,3 +16,24 @@ def test_sample_traces_are_valid_parent_first_trees() -> None:fragment = tool_fragment()assert fragment.parent is not Noneassert fragment.root.parent_span_id == fragment.parent.span_id+++def test_otlp_helpers_read_back_what_was_encoded() -> None:+ trace = message_trace()+ request = encode_trace(trace, resource={}, attributes=lambda span: {"name": span.name, "tokens": 3, "ok": True})+ egress = RecordingEgress([], grpc_target=None)+ egress.requests.append(+ RecordedRequest(+ method="POST",+ url="https://collector.test/v1/traces",+ headers={},+ content=request.SerializeToString(),+ timeout=1.0,+ sensitive_url=False,+ )+ )++ spans = otlp_spans(egress)++ assert [span.span_id.hex() for span in spans] == [span.span_id for span in trace.spans]+ assert otlp_attributes(spans[0]) == {"name": "message", "tokens": 3, "ok": True}diff --git a/api/tests/unit_tests/services/tracing/test_runtime_bridge.py b/api/tests/unit_tests/services/tracing/test_runtime_bridge.pyindex 9cf24c29429..3ed99f53f0f 100644--- a/api/tests/unit_tests/services/tracing/test_runtime_bridge.py+++ b/api/tests/unit_tests/services/tracing/test_runtime_bridge.py@@ -23,7 +23,7 @@ from dify_trace_sdk.events import ()from dify_trace_sdk.ids import span_id_forfrom dify_trace_sdk.model import Span, SpanKind, SpanStatus, Trace-from dify_trace_sdk.testing import RecordingEgress, response+from dify_trace_sdk.testing import RecordingEgress, otlp_attributes, otlp_spans, responsefrom flask import Flaskfrom sqlalchemy import URL, Engine, create_engine, delete, event, func, selectfrom sqlalchemy.orm import Session, sessionmaker@@ -201,27 +201,11 @@ def exported() -> RecordingEgress:return RecordingEgress([response(200, {})], grpc_target=None)-def ingested(destination: RecordingEgress) -> list[dict[str, object]]:- """The Langfuse ingestion events the provider sent."""- events: list[dict[str, object]] = []- for request in destination.requests:- body = request.json()- assert isinstance(body, dict)- batch = body.get("batch")- assert isinstance(batch, list)- events.extend(item for item in batch if isinstance(item, dict))- return events---def observations(destination: RecordingEgress) -> list[dict[str, object]]:- """The Langfuse observations the provider sent: every span except the trace's root."""- bodies: list[dict[str, object]] = []- for item in ingested(destination):- body = item["body"]- assert isinstance(body, dict)- if item["type"] != "trace-create":- bodies.append(body)- return bodies+def root_metadata(destination: RecordingEgress) -> dict[str, object]:+ """The Langfuse metadata of the root observation the provider sent."""+ root = next(span for span in otlp_spans(destination) if not span.parent_span_id)+ prefix = "langfuse.observation.metadata."+ return {key.removeprefix(prefix): value for key, value in otlp_attributes(root).items() if key.startswith(prefix)}def run_count(sessions: sessionmaker[Session]) -> int | None:@@ -654,12 +638,10 @@ class TestApplicationTraceHandoffCases:assert child_case.service.link_managed_parent(tenant_id=child.tenant_id, info=record) is recorddestination = exported()deliver(child_case, event, destination)- trace = next(item for item in ingested(destination) if item["type"] == "trace-create")- body = trace["body"]- assert isinstance(body, dict)- assert body["metadata"]["parent_workflow_run_id"] == parent.id- assert body["metadata"]["parent_node_execution_id"] == identifier("parent-tool-node")- assert "linked_parent_workflow_run_id" not in body["metadata"]+ metadata = root_metadata(destination)+ assert metadata["parent_workflow_run_id"] == parent.id+ assert metadata["parent_node_execution_id"] == identifier("parent-tool-node")+ assert "linked_parent_workflow_run_id" not in metadataassert run_count(sqlite_session_factory) == 0@staticmethod@@ -706,11 +688,9 @@ class TestApplicationTraceHandoffCases:add_app_owner(child_case.collection, "provider-child")destination = exported()deliver(child_case, workflow_event(child, parent), destination)- trace = next(item for item in ingested(destination) if item["type"] == "trace-create")- body = trace["body"]- assert isinstance(body, dict)- assert body["metadata"]["linked_parent_trace_id"] == parent_trace.trace_id- assert "parent_workflow_run_id" not in body["metadata"]+ metadata = root_metadata(destination)+ assert metadata["linked_parent_trace_id"] == parent_trace.trace_id+ assert "parent_workflow_run_id" not in metadata@staticmethoddef test_collector_database_failure_redelivers_without_provider_fallback(@@ -834,12 +814,17 @@ class TestApplicationTraceHandoffCases:delivered = case.collection.repository.get(managed.id)assert delivered is not Noneassert delivered.state is TraceRunState.ACCEPTED- assert [request.url for request in destination.requests] == ["https://langfuse.test/api/public/ingestion"]- generations = [item["body"] for item in ingested(destination) if item["type"] == "generation-create"]- usages = [body["usage"] for body in generations if isinstance(body, dict)]+ assert [request.url for request in destination.requests] == ["https://langfuse.test/api/public/otel/v1/traces"]+ generations = [+ attributes+ for attributes in map(otlp_attributes, otlp_spans(destination))+ if attributes["langfuse.observation.type"] == "generation"+ ]+ usages = [json.loads(str(attributes["langfuse.observation.usage_details"])) for attributes in generations]+ costs = [json.loads(str(attributes["langfuse.observation.cost_details"])) for attributes in generations]assert len(usages) == 3assert sum(usage["total"] for usage in usages) == 25- assert sum(usage["totalCost"] for usage in usages) == pytest.approx(0.04)+ assert sum(cost["total"] for cost in costs) == pytest.approx(0.04)"""Verify execution hook ownership, policy gating, and fail-open business behavior."""@@ -1462,10 +1447,12 @@ class TestWorkflowToolTracingBoundaryCases:assert current is not None# Assembly rejects any span whose parent is missing, so acceptance proves a complete tree.assert current.state is TraceRunState.ACCEPTED- assert [request.url for request in destination.requests] == ["https://tracing.test/root/api/public/ingestion"]- sent = observations(destination)- emitted = {body["id"] for body in sent}- assert all(body["parentObservationId"] in emitted for body in sent if "parentObservationId" in body)+ assert [request.url for request in destination.requests] == [+ "https://tracing.test/root/api/public/otel/v1/traces"+ ]+ sent = otlp_spans(destination)+ emitted = {span.span_id for span in sent}+ assert all(span.parent_span_id in emitted for span in sent if span.parent_span_id)assert current.provider_config == frozenassert current.provider == "langfuse"assert current.capture_content is Falsediff --git a/api/tests/unit_tests/tasks/test_application_trace_task.py b/api/tests/unit_tests/tasks/test_application_trace_task.pyindex da70d0865f7..8eaec0c1e92 100644--- a/api/tests/unit_tests/tasks/test_application_trace_task.py+++ b/api/tests/unit_tests/tasks/test_application_trace_task.py@@ -2,7 +2,7 @@from dataclasses import replacefrom datetime import UTC, datetime, timedelta-from uuid import UUID, uuid4+from uuid import uuid4import httpximport pytest@@ -100,14 +100,13 @@ def test_worker_task_exports_once_through_the_ssrf_proxy_and_replay_observes_chemonkeypatch.setattr(ssrf_proxy, "make_request", send)export_application_trace.run(case.run.id)export_application_trace.run(case.run.id)- assert proxied == [("POST", "https://langfuse.test/api/public/ingestion")]+ assert proxied == [("POST", "https://langfuse.test/api/public/otel/v1/traces")]result = case.repository.get(case.run.id)assert result is not Noneassert result.state is TraceRunState.ACCEPTEDassert result.attempts == 1- provider_trace_id = str(UUID(hex=case.run.trace_id))- assert result.provider_trace_id == provider_trace_id- assert result.trace_url == f"https://langfuse.test/project/original/traces/{provider_trace_id}"+ assert result.provider_trace_id == case.run.trace_id+ assert result.trace_url == f"https://langfuse.test/project/original/traces/{case.run.trace_id}"def test_worker_records_carry_no_enterprise_names(sqlite_session_factory: sessionmaker[Session]) -> None: