Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat(tracing): export Agent app and Workflow Agent execution trees (#44144)

zyssyz1230a620ddd95c2
diff --git a/api/clients/agent_backend/request_builder.py b/api/clients/agent_backend/request_builder.py
index 75e8f060002..84584faafa8 100644
--- a/api/clients/agent_backend/request_builder.py
+++ b/api/clients/agent_backend/request_builder.py
@@ -138,7 +138,8 @@ _AGENT_MODEL_SETTINGS_PASSTHROUGH_KEYS = (
"frequency_penalty",
"max_tokens",
)
-_AGENT_MODEL_SETTINGS_KNOWN_KEYS = frozenset({*_AGENT_MODEL_SETTINGS_PASSTHROUGH_KEYS, "stop", "response_format"})
+# response_format uses extra_body so both plugin strings and saved native objects reach the plugin unchanged.
+_AGENT_MODEL_SETTINGS_KNOWN_KEYS = frozenset({*_AGENT_MODEL_SETTINGS_PASSTHROUGH_KEYS, "stop"})
def _agent_model_settings(settings: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None:
@@ -281,7 +282,9 @@ class AgentBackendRunRequestBuilder:
),
RunLayerSpec(
name=DIFY_EXECUTION_CONTEXT_LAYER_ID,
- config=(run_input.execution_context).model_dump(mode="json"),
+ config=run_input.execution_context.model_dump(
+ mode="json", exclude={"tracing"} if run_input.execution_context.tracing is None else set()
+ ),
),
]
)
@@ -415,7 +418,9 @@ class AgentBackendRunRequestBuilder:
),
RunLayerSpec(
name=DIFY_EXECUTION_CONTEXT_LAYER_ID,
- config=(run_input.execution_context).model_dump(mode="json"),
+ config=run_input.execution_context.model_dump(
+ mode="json", exclude={"tracing"} if run_input.execution_context.tracing is None else set()
+ ),
),
]
)
@@ -531,7 +536,11 @@ def redact_for_agent_backend_log(value: object) -> object:
is_multimodal_file = value.get("type") == "image" and "filename" in value and "mime_type" in value
for key, item in value.items():
key_text = str(key).lower()
- if any(part in key_text for part in _SENSITIVE_KEY_PARTS) or (is_multimodal_file and key_text == "url"):
+ if (
+ any(part in key_text for part in _SENSITIVE_KEY_PARTS)
+ or (is_multimodal_file and key_text == "url")
+ or (key_text == "tracing" and item is not None)
+ ):
redacted[key] = "[REDACTED]"
else:
redacted[key] = redact_for_agent_backend_log(item)
diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py
index 54473bf9a1f..2b3f972c508 100644
--- a/api/configs/feature/__init__.py
+++ b/api/configs/feature/__init__.py
@@ -1416,6 +1416,31 @@ class MultiModalTransferConfig(BaseSettings):
class OpsTraceConfig(BaseSettings):
+ OPS_AGENT_TRACE_ENABLED: bool = Field(
+ default=False, description="Enable application-scoped Agent tracing collection and delivery."
+ )
+ OPS_AGENT_TRACE_MAX_SPANS: PositiveInt = Field(
+ default=4096, description="Maximum runtime spans per application trace."
+ )
+ OPS_AGENT_TRACE_MAX_BYTES: PositiveInt = Field(
+ default=16777216, description="Maximum runtime payload bytes per application trace."
+ )
+ OPS_AGENT_TRACE_BATCH_MAX_BYTES: PositiveInt = Field(
+ default=2097152, description="Maximum inner tracing request body bytes."
+ )
+ OPS_AGENT_TRACE_COLLECTION_TIMEOUT: PositiveInt = Field(
+ default=180, description="Seconds after terminal execution to wait for final Agent batches."
+ )
+ OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS: PositiveInt = Field(
+ default=120, description="Export lease before unconfirmed delivery becomes uncertain."
+ )
+ OPS_AGENT_TRACE_MAX_ATTEMPTS: PositiveInt = Field(
+ default=8, description="Attempts for known retryable trace export failures."
+ )
+ OPS_AGENT_TRACE_RETENTION_DAYS: PositiveInt = Field(
+ default=7, description="Retention of trace payloads and encrypted destination snapshots."
+ )
+
OPS_TRACE_UNIFIED_ENABLED: bool = Field(
description="Enable unified ops tracing for providers registered in the unified registry.",
default=False,
@@ -1456,6 +1481,10 @@ class CeleryBeatConfig(BaseSettings):
class CeleryScheduleTasksConfig(BaseSettings):
+ ENABLE_APPLICATION_TRACE_RECONCILE_TASK: bool = Field(
+ description="Enable periodic application trace recovery and retention cleanup when Agent tracing is enabled",
+ default=True,
+ )
ENABLE_CONVERSATION_CLEANUP_TASK: bool = Field(
description="Enable periodic recovery of soft-deleted conversation cleanup",
default=True,
diff --git a/api/controllers/API_SCHEMA_GUIDE.md b/api/controllers/API_SCHEMA_GUIDE.md
index 1ef002cfbfe..81f556cf63a 100644
--- a/api/controllers/API_SCHEMA_GUIDE.md
+++ b/api/controllers/API_SCHEMA_GUIDE.md
@@ -19,7 +19,7 @@ requested during root SSR, so treat this response as a minimal public bootstrap
configuration endpoint, a feature registry, or a mirror of environment and Enterprise settings. Existing fields are
legacy inventory and do not establish precedent for new fields.
-A new field is eligible only when all of the following are true:
+Apart from the explicit Agent tracing exception below, a new field is eligible only when all of the following are true:
1. Both Console and Web have named production consumers for the field.
2. Both consumers need the value before authentication and tenant/workspace bootstrap to render initial state or
@@ -44,9 +44,27 @@ Route excluded values to their actual owner:
and failure cost explicitly.
- Load large, slow, or page-specific data lazily through a consumer-owned query.
+### Agent tracing deployment discovery exception
+
+`agent_tracing_api_collection_enabled` is an explicit deployment-discovery exception to the pre-authentication
+consumer requirements above. Both public Console and Web `/system-features` responses expose the same required
+boolean from `OPS_AGENT_TRACE_ENABLED`; the authenticated Console tracing configuration and Workflow inheritance
+surfaces consume it independently of workspace, application, or provider configuration. It has no schema default:
+a missing field must not be mistaken for a confirmed disabled deployment.
+
+This field reports only the API collection gate. It grants no configuration, credential, or log access and says
+nothing about Agent runtime, worker, or provider health, export acknowledgement, or platform queryability. The
+exception does not admit other environment settings or integration toggles, or broaden this response into a
+configuration registry. Keep workspace policy and entitlements in their authenticated owners.
+
+The existing Console and Web bootstrap requests gain one boolean and no additional configuration or provider I/O.
+Their authentication, SSR request timing, and cache ownership are unchanged; consumers must retain an unknown
+state when the response is missing or fails rather than infer a disabled or healthy deployment.
+
Every pull request that adds a System Features field must:
-- Name both production consumer paths and explain why they require the value before authentication.
+- Name both production consumer paths and explain why they require the value before authentication. For the named
+ Agent tracing exception, document the authenticated Console consumers and intentional shared Web exposure instead.
- Document the root SSR request, payload, caching, and failure-mode impact.
- Update the Pydantic owner, regenerate OpenAPI Markdown and TypeScript/Zod contracts, and update shared fixtures.
- Add Console and Web schema regression coverage. Do not hand-edit generated contracts or add compatibility defaults.
diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py
index 43ff40c5098..47956be121e 100644
--- a/api/controllers/console/__init__.py
+++ b/api/controllers/console/__init__.py
@@ -153,6 +153,8 @@ def _register_controllers() -> tuple[ModuleType, ...]:
)
from .agent import composer as agent_composer
from .agent import roster as agent_roster
+ from .agent import tracing as agent_tracing
+ from .agent import tracing_runs as agent_tracing_runs
# Import app controllers
from .app import (
@@ -270,6 +272,8 @@ def _register_controllers() -> tuple[ModuleType, ...]:
workflow_run_archive,
agent_composer,
agent_roster,
+ agent_tracing,
+ agent_tracing_runs,
advanced_prompt_template,
agent,
agent_app_access,
@@ -365,6 +369,8 @@ def _register_controllers() -> tuple[ModuleType, ...]:
workflow_run_archive,
agent_composer,
agent_roster,
+ agent_tracing,
+ agent_tracing_runs,
advanced_prompt_template,
agent,
agent_app_access,
@@ -458,6 +464,8 @@ __all__ = [
"agent_config_inspector",
"agent_providers",
"agent_roster",
+ "agent_tracing",
+ "agent_tracing_runs",
"annotation",
"api",
"apikey",
diff --git a/api/controllers/console/agent/tracing.py b/api/controllers/console/agent/tracing.py
new file mode 100644
index 00000000000..32c775991c5
--- /dev/null
+++ b/api/controllers/console/agent/tracing.py
@@ -0,0 +1,248 @@
+"""Agent tracing settings address Agent resources, never hidden inline Apps."""
+
+from collections.abc import Callable
+from typing import Literal
+from uuid import UUID
+
+from flask import request
+from flask_restx import Resource
+from pydantic import Field
+from werkzeug.exceptions import NotFound
+
+from configs import dify_config
+from controllers.common.rbac import AgentId, PlainApp, RBACCheck
+from controllers.common.schema import query_params_from_model, register_response_schema_models
+from controllers.console import console_ns
+from controllers.console.app.app import AppTracePayload
+from controllers.console.app.error import (
+ AppNotFoundError,
+ InvalidTracingConfigError,
+ TracingConfigAlreadyExistsError,
+ TracingConfigConflictError,
+ TracingConfigNotFoundError,
+ TracingConfigProcessingError,
+ TracingConfigVerificationFailedError,
+ TracingProviderUnavailableError,
+ UnsupportedTracingProviderError,
+)
+from controllers.console.app.ops_trace import TraceAppConfigResponse, TraceConfigPayload, TraceProviderQuery
+from controllers.console.flask_admission import console_account_admission
+from controllers.console.wraps import RBACPermission
+from extensions.ext_application_services import application_services
+from fields.app_fields import AppTraceResponse
+from fields.base import ResponseModel
+from libs.external_api import error_responses
+from libs.helper import dump_response
+from machinery.context import RequestContext
+from models.account import TenantAccountRole
+from services.app.agent_app_contracts import AgentAppNotFoundError
+from services.app.console_service import ConsoleAppNotFoundError
+from services.app_tracing_config_service import (
+ AppTraceSettings,
+ AppTracingConfigAlreadyExistsError,
+ AppTracingConfigAppNotFoundError,
+ AppTracingConfigConflictError,
+ AppTracingConfigInvalidConfigurationError,
+ AppTracingConfigInvalidProviderError,
+ AppTracingConfigNotFoundError,
+ AppTracingConfigProcessingError,
+ AppTracingConfigProviderUnavailableError,
+ AppTracingConfigVerificationFailedError,
+)
+
+_EDIT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR})
+
+
+class TraceProviderSupportResponse(ResponseModel):
+ """Adapter support declarations, not successful-delivery or platform-verification results."""
+
+ provider: str
+ agent_spans: bool
+ session: bool
+ native_usage: bool
+ native_cost: bool
+ trace_url: bool
+ span_url: bool
+ export_ack: Literal["accepted"] = Field(description="Exporter acknowledgement does not confirm query visibility")
+
+
+class TraceProviderCapabilitiesResponse(TraceProviderSupportResponse):
+ api_collection_enabled: bool = Field(
+ description="Current API collection gate only; does not probe Agent runtime or confirm delivery"
+ )
+
+
+register_response_schema_models(console_ns, TraceProviderSupportResponse, TraceProviderCapabilitiesResponse)
+
+
+def _tracing_call[T](call: Callable[[], T]) -> T:
+ """Translate configuration-domain failures without returning credentials or provider errors."""
+ try:
+ return call()
+ except (AgentAppNotFoundError, ConsoleAppNotFoundError) as error:
+ raise NotFound("Agent not found") from error
+ except AppTracingConfigAppNotFoundError as error:
+ raise AppNotFoundError() from error
+ except AppTracingConfigAlreadyExistsError as error:
+ raise TracingConfigAlreadyExistsError() from error
+ except AppTracingConfigConflictError as error:
+ raise TracingConfigConflictError() from error
+ except AppTracingConfigNotFoundError as error:
+ raise TracingConfigNotFoundError() from error
+ except AppTracingConfigInvalidProviderError as error:
+ raise UnsupportedTracingProviderError() from error
+ except AppTracingConfigProviderUnavailableError as error:
+ raise TracingProviderUnavailableError() from error
+ except AppTracingConfigInvalidConfigurationError as error:
+ raise InvalidTracingConfigError() from error
+ except AppTracingConfigVerificationFailedError as error:
+ raise TracingConfigVerificationFailedError() from error
+ except AppTracingConfigProcessingError as error:
+ raise TracingConfigProcessingError() from error
+
+
+@console_ns.route("/agent/<uuid:agent_id>/trace")
+class AgentTraceApi(Resource):
+ @console_ns.doc("get_agent_trace")
+ @console_ns.response(200, "Agent tracing settings", console_ns.models[AppTraceResponse.__name__])
+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.AGENT_MONITOR, AgentId())])
+ def get(self, context: RequestContext, agent_id: UUID):
+ settings = _tracing_call(lambda: application_services().agent_tracing_configs.get_trace(context, str(agent_id)))
+ return dump_response(AppTraceResponse, settings)
+
+ @error_responses(
+ TracingProviderUnavailableError,
+ TracingConfigNotFoundError,
+ TracingConfigConflictError,
+ InvalidTracingConfigError,
+ TracingConfigProcessingError,
+ UnsupportedTracingProviderError,
+ )
+ @console_ns.doc("update_agent_trace")
+ @console_ns.expect(console_ns.models[AppTracePayload.__name__])
+ @console_ns.response(200, "Updated Agent tracing settings", console_ns.models[AppTraceResponse.__name__])
+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])
+ def post(self, context: RequestContext, agent_id: UUID):
+ payload = AppTracePayload.model_validate(console_ns.payload or {})
+ settings = AppTraceSettings(**payload.model_dump())
+ _tracing_call(lambda: application_services().agent_tracing_configs.set_trace(context, str(agent_id), settings))
+ return dump_response(AppTraceResponse, settings)
+
+
+@console_ns.route("/agent/<uuid:agent_id>/trace-config")
+class AgentTraceConfigApi(Resource):
+ @error_responses(
+ AppNotFoundError, TracingConfigProcessingError, TracingProviderUnavailableError, UnsupportedTracingProviderError
+ )
+ @console_ns.doc("get_agent_trace_config", params=query_params_from_model(TraceProviderQuery))
+ @console_ns.response(200, "Masked Agent provider configuration", console_ns.models[TraceAppConfigResponse.__name__])
+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])
+ def get(self, context: RequestContext, agent_id: UUID):
+ query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))
+ record = _tracing_call(
+ lambda: application_services().agent_tracing_configs.get(context, str(agent_id), query.tracing_provider)
+ )
+ if record is None:
+ return dump_response(TraceAppConfigResponse, {"has_not_configured": True}, exclude_none=True)
+ return dump_response(TraceAppConfigResponse, TraceAppConfigResponse.from_record(record), exclude_none=True)
+
+ @error_responses(
+ AppNotFoundError,
+ InvalidTracingConfigError,
+ TracingConfigAlreadyExistsError,
+ TracingConfigProcessingError,
+ TracingConfigVerificationFailedError,
+ TracingProviderUnavailableError,
+ UnsupportedTracingProviderError,
+ )
+ @console_ns.doc("create_agent_trace_config")
+ @console_ns.expect(console_ns.models[TraceConfigPayload.__name__])
+ @console_ns.response(201, "Agent provider configured", console_ns.models[TraceAppConfigResponse.__name__])
+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])
+ def post(self, context: RequestContext, agent_id: UUID):
+ payload = TraceConfigPayload.model_validate(console_ns.payload or {})
+ _tracing_call(
+ lambda: application_services().agent_tracing_configs.create(
+ context, str(agent_id), payload.tracing_provider, payload.tracing_config, activate=payload.activate
+ )
+ )
+ return dump_response(TraceAppConfigResponse, {"result": "success"}, exclude_none=True), 201
+
+ @error_responses(
+ AppNotFoundError,
+ InvalidTracingConfigError,
+ TracingConfigNotFoundError,
+ TracingConfigConflictError,
+ TracingConfigProcessingError,
+ TracingConfigVerificationFailedError,
+ TracingProviderUnavailableError,
+ UnsupportedTracingProviderError,
+ )
+ @console_ns.doc("update_agent_trace_config")
+ @console_ns.expect(console_ns.models[TraceConfigPayload.__name__])
+ @console_ns.response(200, "Agent provider updated", console_ns.models[TraceAppConfigResponse.__name__])
+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])
+ def patch(self, context: RequestContext, agent_id: UUID):
+ payload = TraceConfigPayload.model_validate(console_ns.payload or {})
+ _tracing_call(
+ lambda: application_services().agent_tracing_configs.update(
+ context, str(agent_id), payload.tracing_provider, payload.tracing_config, activate=payload.activate
+ )
+ )
+ return dump_response(TraceAppConfigResponse, {"result": "success"}, exclude_none=True)
+
+ @error_responses(
+ AppNotFoundError, TracingConfigNotFoundError, TracingConfigProcessingError, UnsupportedTracingProviderError
+ )
+ @console_ns.doc("delete_agent_trace_config", params=query_params_from_model(TraceProviderQuery))
+ @console_ns.response(204, "Agent provider configuration deleted")
+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])
+ def delete(self, context: RequestContext, agent_id: UUID):
+ query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))
+ _tracing_call(
+ lambda: application_services().agent_tracing_configs.delete(context, str(agent_id), query.tracing_provider)
+ )
+ return "", 204
+
+
+def _provider_capabilities():
+ from services.tracing.provider_dispatch import provider_capabilities
+
+ query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))
+ try:
+ capabilities = provider_capabilities(query.tracing_provider)
+ except ValueError as error:
+ raise UnsupportedTracingProviderError() from error
+ return dump_response(
+ TraceProviderCapabilitiesResponse,
+ {**capabilities, "api_collection_enabled": dify_config.OPS_AGENT_TRACE_ENABLED},
+ )
+
+
+@console_ns.route("/agent/<uuid:agent_id>/trace-capabilities")
+class AgentTraceCapabilitiesApi(Resource):
+ @error_responses(UnsupportedTracingProviderError)
+ @console_ns.doc("get_agent_trace_capabilities", params=query_params_from_model(TraceProviderQuery))
+ @console_ns.response(
+ 200, "Provider support declarations", console_ns.models[TraceProviderCapabilitiesResponse.__name__]
+ )
+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.AGENT_MONITOR, AgentId())])
+ def get(self, context: RequestContext, agent_id: UUID):
+ _tracing_call(lambda: application_services().agent_tracing_configs.get_trace(context, str(agent_id)))
+ return _provider_capabilities()
+
+
+@console_ns.route("/apps/<uuid:app_id>/trace-capabilities")
+class AppTraceCapabilitiesApi(Resource):
+ @error_responses(UnsupportedTracingProviderError)
+ @console_ns.doc("get_app_trace_capabilities", params=query_params_from_model(TraceProviderQuery))
+ @console_ns.response(
+ 200, "Provider support declarations", console_ns.models[TraceProviderCapabilitiesResponse.__name__]
+ )
+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.APP_MONITOR, PlainApp())])
+ def get(self, context: RequestContext, app_id: UUID):
+ try:
+ application_services().apps.console.get_trace(context, str(app_id))
+ except ConsoleAppNotFoundError as error:
+ raise NotFound("App not found") from error
+ return _provider_capabilities()
diff --git a/api/controllers/console/agent/tracing_runs.py b/api/controllers/console/agent/tracing_runs.py
new file mode 100644
index 00000000000..60149273af5
--- /dev/null
+++ b/api/controllers/console/agent/tracing_runs.py
@@ -0,0 +1,246 @@
+"""Authorized trace delivery status and explicit retries of frozen destinations."""
+
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Literal
+from uuid import UUID
+
+from flask import request
+from flask_restx import Resource
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator
+from werkzeug.exceptions import NotFound
+
+from controllers.common.rbac import AgentId, PlainApp, RBACCheck
+from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
+from controllers.console import console_ns
+from controllers.console.agent.tracing import TraceProviderSupportResponse
+from controllers.console.flask_admission import console_account_admission
+from controllers.console.wraps import RBACPermission
+from extensions.ext_application_services import application_services
+from fields.base import ResponseModel
+from libs.exception import BaseHTTPException
+from libs.external_api import error_responses
+from libs.helper import dump_response
+from machinery.context import RequestContext
+from models.account import TenantAccountRole
+from repositories.application_trace_repository import TraceCollectionError
+from services.app.agent_app_contracts import AgentAppNotFoundError
+from services.tracing.service import application_tracing_service
+
+type TraceSourceKind = Literal["message", "workflow", "node"]
+_source_kind_adapter = TypeAdapter(TraceSourceKind)
+_EDIT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR})
+
+
+class TraceRunQuery(BaseModel):
+ node_execution_id: UUID | None = Field(default=None, description="Select an Agent node invocation within this run")
+
+
+class TraceRunRetryPayload(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ acknowledge_possible_duplicates: bool = Field(
+ default=False,
+ strict=True,
+ description="Acknowledge possible duplicate records when retrying an uncertain export",
+ )
+
+
+class TraceRunStatusResponse(ResponseModel):
+ status: Literal["not_collected", "collecting", "ready", "exporting", "retrying", "accepted", "failed", "uncertain"]
+ source_kind: TraceSourceKind
+ source_id: str
+ provider: str | None = None
+ trace_id: str | None = None
+ trace_url: str | None = None
+ span_url: str | None = None
+ project_url: str | None = Field(
+ default=None,
+ description="Frozen destination project URL; null when unavailable, never resolved from current config",
+ )
+ config_revision: str | None = Field(default=None, description="Frozen destination configuration revision")
+ capture_content: bool | None = None
+ capture_agent: bool | None = None
+ span_count: int | None = None
+ attempts: int | None = None
+ error_code: str | None = None
+ accepted_at: datetime | None = Field(
+ default=None, description="Ingestion acknowledgement time, not query visibility"
+ )
+ queryable: bool | None = Field(default=None, description="Null when third-party query visibility is not verified")
+ capabilities: TraceProviderSupportResponse | None = None
+
+ @field_validator("accepted_at")
+ @classmethod
+ def serialize_database_time_as_utc(cls, value: datetime | None) -> datetime | None:
+ # Database timestamps are naive UTC. Emit a timezone so the generated
+ # ISO datetime contract and browser parsing agree on the same instant.
+ if value is None:
+ return None
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+
+
+class TraceSourceNotFoundError(BaseHTTPException):
+ error_code = "tracing_source_not_found"
+ code = 404
+ description = "The tracing source does not exist in this application."
+
+
+class TraceNodeNotFoundError(BaseHTTPException):
+ error_code = "tracing_node_not_found"
+ code = 404
+ description = "The Agent node invocation does not belong to this trace."
+
+
+class TraceRunNotFoundError(BaseHTTPException):
+ error_code = "tracing_run_not_found"
+ code = 404
+ description = "No collected trace is available to retry."
+
+
+class TraceAlreadyAcceptedError(BaseHTTPException):
+ error_code = "tracing_already_accepted"
+ code = 409
+ description = "An accepted trace cannot be exported again."
+
+
+class TraceRetryNotAllowedError(BaseHTTPException):
+ error_code = "tracing_retry_not_allowed"
+ code = 409
+ description = "This trace is not in a retryable state."
+
+
+class TraceDuplicateAcknowledgementRequiredError(BaseHTTPException):
+ error_code = "tracing_duplicate_acknowledgement_required"
+ code = 409
+ description = "Acknowledge possible duplicates before retrying an uncertain export."
+
+
+_TRACE_ERRORS: dict[str, type[BaseHTTPException]] = {
+ cls.error_code: cls
+ for cls in (
+ TraceSourceNotFoundError,
+ TraceNodeNotFoundError,
+ TraceRunNotFoundError,
+ TraceAlreadyAcceptedError,
+ TraceRetryNotAllowedError,
+ TraceDuplicateAcknowledgementRequiredError,
+ )
+}
+
+register_schema_models(console_ns, TraceRunQuery, TraceRunRetryPayload)
+register_response_schema_models(console_ns, TraceRunStatusResponse)
+
+
+def _run_call[T](call: Callable[[], T]) -> T:
+ try:
+ return call()
+ except AgentAppNotFoundError as error:
+ raise NotFound("Agent not found") from error
+ except TraceCollectionError as error:
+ if error_type := _TRACE_ERRORS.get(error.code):
+ raise error_type() from error
+ raise
+
+
+def _query_node_id() -> str | None:
+ query = TraceRunQuery.model_validate(request.args.to_dict(flat=True))
+ return str(query.node_execution_id) if query.node_execution_id else None
+
+
+def _agent_app_id(context: RequestContext, agent_id: UUID) -> str:
+ return _run_call(lambda: application_services().agent_tracing_configs.resolve_app_id(context, str(agent_id)))
+
+
+def _status(context: RequestContext, app_id: str, kind: TraceSourceKind, source_id: str):
+ node_id = _query_node_id()
+ result = _run_call(
+ lambda: application_tracing_service().status(
+ tenant_id=context.active_workspace_id,
+ app_id=app_id,
+ source_kind=kind,
+ source_id=source_id,
+ node_execution_id=node_id,
+ )
+ )
+ return dump_response(TraceRunStatusResponse, result)
+
+
+def _retry(context: RequestContext, app_id: str, kind: TraceSourceKind, source_id: str):
+ node_id = _query_node_id()
+ payload = TraceRunRetryPayload.model_validate(console_ns.payload or {})
+ result = _run_call(
+ lambda: application_tracing_service().retry(
+ tenant_id=context.active_workspace_id,
+ app_id=app_id,
+ source_kind=kind,
+ source_id=source_id,
+ node_execution_id=node_id,
+ acknowledge_possible_duplicates=payload.acknowledge_possible_duplicates,
+ )
+ )
+ return dump_response(TraceRunStatusResponse, result)
+
+
+@console_ns.route("/agent/<uuid:agent_id>/trace-runs/<uuid:message_id>")
+class AgentTraceRunApi(Resource):
+ @error_responses(TraceSourceNotFoundError, TraceNodeNotFoundError)
+ @console_ns.doc("get_agent_trace_run", params=query_params_from_model(TraceRunQuery))
+ @console_ns.response(
+ 200, "Frozen trace destination and export state", console_ns.models[TraceRunStatusResponse.__name__]
+ )
+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.AGENT_LOG_MANAGE, AgentId())])
+ def get(self, context: RequestContext, agent_id: UUID, message_id: UUID):
+ return _status(context, _agent_app_id(context, agent_id), "message", str(message_id))
+
+
+@console_ns.route("/agent/<uuid:agent_id>/trace-runs/<uuid:message_id>/retry")
+class AgentTraceRunRetryApi(Resource):
+ @error_responses(*_TRACE_ERRORS.values())
+ @console_ns.doc("retry_agent_trace_run", params=query_params_from_model(TraceRunQuery))
+ @console_ns.expect(console_ns.models[TraceRunRetryPayload.__name__])
+ @console_ns.response(
+ 200, "Retry state for the same frozen destination", console_ns.models[TraceRunStatusResponse.__name__]
+ )
+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])
+ def post(self, context: RequestContext, agent_id: UUID, message_id: UUID):
+ return _retry(context, _agent_app_id(context, agent_id), "message", str(message_id))
+
+
+@console_ns.route("/apps/<uuid:app_id>/trace-runs/<string:source_kind>/<uuid:source_id>")
+class AppTraceRunApi(Resource):
+ @error_responses(TraceSourceNotFoundError, TraceNodeNotFoundError)
+ @console_ns.doc(
+ "get_app_trace_run",
+ params={
+ **query_params_from_model(TraceRunQuery),
+ "source_kind": {"enum": ["message", "workflow", "node"]},
+ },
+ )
+ @console_ns.response(
+ 200, "Frozen trace destination and export state", console_ns.models[TraceRunStatusResponse.__name__]
+ )
+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.APP_LOG_AND_ANNOTATION, PlainApp())])
+ def get(self, context: RequestContext, app_id: UUID, source_kind: str, source_id: UUID):
+ return _status(context, str(app_id), _source_kind_adapter.validate_python(source_kind), str(source_id))
+
+
+@console_ns.route("/apps/<uuid:app_id>/trace-runs/<string:source_kind>/<uuid:source_id>/retry")
+class AppTraceRunRetryApi(Resource):
+ @error_responses(*_TRACE_ERRORS.values())
+ @console_ns.doc(
+ "retry_app_trace_run",
+ params={
+ **query_params_from_model(TraceRunQuery),
+ "source_kind": {"enum": ["message", "workflow", "node"]},
+ },
+ )
+ @console_ns.expect(console_ns.models[TraceRunRetryPayload.__name__])
+ @console_ns.response(
+ 200, "Retry state for the same frozen destination", console_ns.models[TraceRunStatusResponse.__name__]
+ )
+ @console_account_admission(
+ allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.APP_TRACING_CONFIG, PlainApp())]
+ )
+ def post(self, context: RequestContext, app_id: UUID, source_kind: str, source_id: UUID):
+ return _retry(context, str(app_id), _source_kind_adapter.validate_python(source_kind), str(source_id))
diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py
index 8e3faec0f3f..cb2f3e2e0c9 100644
--- a/api/controllers/console/app/app.py
+++ b/api/controllers/console/app/app.py
@@ -19,7 +19,15 @@ from controllers.common.schema import (
register_schema_models,
)
from controllers.console import console_ns
-from controllers.console.app.error import AppNotFoundError, TracingProviderUnavailableError
+from controllers.console.app.error import (
+ AppNotFoundError,
+ InvalidTracingConfigError,
+ TracingConfigConflictError,
+ TracingConfigNotFoundError,
+ TracingConfigProcessingError,
+ TracingProviderUnavailableError,
+ UnsupportedTracingProviderError,
+)
from controllers.console.flask_admission import console_account_admission
from controllers.console.workspace.models import LoadBalancingPayload
from controllers.console.wraps import (
@@ -64,7 +72,13 @@ from services.app.console_service import (
InvalidAppExportError,
)
from services.app_tracing_config_service import (
+ AppTraceSettings,
+ AppTracingConfigAppNotFoundError,
+ AppTracingConfigConflictError,
+ AppTracingConfigInvalidConfigurationError,
AppTracingConfigInvalidProviderError,
+ AppTracingConfigNotFoundError,
+ AppTracingConfigProcessingError,
AppTracingConfigProviderUnavailableError,
)
from services.entities.app_entities import (
@@ -72,7 +86,6 @@ from services.entities.app_entities import (
AppListParams,
AppListSortBy,
AppRecord,
- AppTraceSettings,
CopyAppParams,
CreateAppParams,
StarredAppListParams,
@@ -232,13 +245,14 @@ class AppApiStatusPayload(BaseModel):
class AppTracePayload(BaseModel):
enabled: bool = Field(..., description="Enable or disable tracing")
tracing_provider: str | None = Field(default=None, description="Tracing provider")
+ capture_content: bool = Field(default=False, description="Include prompt, output and tool content in new traces")
+ capture_agent: bool = Field(default=True, description="Capture Agent internals when deployment support is enabled")
- @field_validator("tracing_provider")
- @classmethod
- def validate_tracing_provider(cls, value: str | None, info) -> str | None:
- if info.data.get("enabled") and not value:
+ @model_validator(mode="after")
+ def validate_tracing_provider(self) -> "AppTracePayload":
+ if self.enabled and not self.tracing_provider:
raise ValueError("tracing_provider is required when enabled is True")
- return value
+ return self
register_enum_models(console_ns, RetrievalMethod, WorkflowExecutionStatus, DatasetPermissionEnum)
@@ -657,7 +671,15 @@ class AppTraceApi(AppResource):
def get(self, context: RequestContext, app_id: uuid.UUID):
return dump_response(AppTraceResponse, application_services().apps.console.get_trace(context, str(app_id)))
- @error_responses(TracingProviderUnavailableError)
+ @error_responses(
+ AppNotFoundError,
+ TracingProviderUnavailableError,
+ TracingConfigNotFoundError,
+ TracingConfigConflictError,
+ TracingConfigProcessingError,
+ InvalidTracingConfigError,
+ UnsupportedTracingProviderError,
+ )
@console_ns.doc("update_app_trace")
@console_ns.doc(description="Update app tracing configuration")
@console_ns.doc(params={"app_id": "Application ID"})
@@ -679,6 +701,18 @@ class AppTraceApi(AppResource):
str(app_id),
AppTraceSettings(**payload.model_dump()),
)
+ except AppTracingConfigAppNotFoundError as error:
+ raise AppNotFoundError() from error
+ except AppTracingConfigConflictError as error:
+ raise TracingConfigConflictError() from error
+ except AppTracingConfigNotFoundError as error:
+ raise TracingConfigNotFoundError() from error
+ except AppTracingConfigProcessingError as error:
+ raise TracingConfigProcessingError() from error
+ except AppTracingConfigInvalidConfigurationError as error:
+ raise InvalidTracingConfigError() from error
+ except AppTracingConfigInvalidProviderError as error:
+ raise UnsupportedTracingProviderError() from error
except AppTracingConfigProviderUnavailableError as error:
raise TracingProviderUnavailableError() from error
return SimpleResultResponse(result="success").model_dump(mode="json")
diff --git a/api/controllers/console/app/error.py b/api/controllers/console/app/error.py
index adb0fbcf366..22e8bc7ed2a 100644
--- a/api/controllers/console/app/error.py
+++ b/api/controllers/console/app/error.py
@@ -187,3 +187,9 @@ class AgentNotBoundError(BaseHTTPException):
error_code = "agent_not_bound"
description = "no agent is bound for this app/node"
code = 400
+
+
+class TracingConfigConflictError(BaseHTTPException):
+ error_code = "tracing_config_conflict"
+ description = "Tracing configuration changed. Refresh and try again."
+ code = 409
diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py
index 6d41deb4d1f..ce1463d8d28 100644
--- a/api/controllers/console/app/ops_trace.py
+++ b/api/controllers/console/app/ops_trace.py
@@ -10,6 +10,7 @@ from controllers.console.app.error import (
AppNotFoundError,
InvalidTracingConfigError,
TracingConfigAlreadyExistsError,
+ TracingConfigConflictError,
TracingConfigNotFoundError,
TracingConfigProcessingError,
TracingConfigVerificationFailedError,
@@ -30,6 +31,7 @@ from models.account import TenantAccountRole
from services.app_tracing_config_service import (
AppTracingConfigAlreadyExistsError,
AppTracingConfigAppNotFoundError,
+ AppTracingConfigConflictError,
AppTracingConfigInvalidConfigurationError,
AppTracingConfigInvalidProviderError,
AppTracingConfigNotFoundError,
@@ -53,6 +55,10 @@ class TraceProviderQuery(BaseModel):
class TraceConfigPayload(BaseModel):
+ activate: bool = Field(
+ default=False,
+ description="Save, select this provider, and enable tracing atomically; preserve capture settings",
+ )
tracing_provider: str = Field(..., description="Tracing provider name")
tracing_config: dict[str, str | None] = Field(
...,
@@ -186,6 +192,7 @@ class TraceAppConfigApi(Resource):
app_id=str(app_id),
tracing_provider=req_data.tracing_provider,
tracing_config=req_data.tracing_config,
+ activate=req_data.activate,
)
except AppTracingConfigAppNotFoundError as error:
raise AppNotFoundError() from error
@@ -210,6 +217,7 @@ class TraceAppConfigApi(Resource):
AppNotFoundError,
InvalidTracingConfigError,
TracingConfigNotFoundError,
+ TracingConfigConflictError,
TracingConfigProcessingError,
TracingConfigVerificationFailedError,
TracingProviderUnavailableError,
@@ -246,9 +254,12 @@ class TraceAppConfigApi(Resource):
app_id=str(app_id),
tracing_provider=req_data.tracing_provider,
tracing_config=req_data.tracing_config,
+ activate=req_data.activate,
)
except AppTracingConfigAppNotFoundError as error:
raise AppNotFoundError() from error
+ except AppTracingConfigConflictError as error:
+ raise TracingConfigConflictError() from error
except AppTracingConfigNotFoundError as error:
raise TracingConfigNotFoundError() from error
except AppTracingConfigInvalidProviderError as error:
diff --git a/api/controllers/inner_api/__init__.py b/api/controllers/inner_api/__init__.py
index cc1752198fd..bc0610ee212 100644
--- a/api/controllers/inner_api/__init__.py
+++ b/api/controllers/inner_api/__init__.py
@@ -31,7 +31,7 @@ def _register_controllers() -> tuple[ModuleType, ...]:
"""Load route modules after their namespace exists."""
from . import mail, runtime_credentials
- from .agent import files, llm, runtime_usage, tools
+ from .agent import files, llm, runtime_usage, tools, tracing
from .app import app, dsl, file_grants
from .knowledge import retrieval
from .plugin import agent_config, plugin, skills
@@ -44,6 +44,7 @@ def _register_controllers() -> tuple[ModuleType, ...]:
llm,
runtime_usage,
tools,
+ tracing,
app,
dsl,
file_grants,
@@ -62,6 +63,7 @@ def _register_controllers() -> tuple[ModuleType, ...]:
_agent_llm,
_agent_runtime_usage,
_agent_tools,
+ _agent_tracing,
_app_app,
_app_dsl,
_app_file_grants,
diff --git a/api/controllers/inner_api/agent/tracing.py b/api/controllers/inner_api/agent/tracing.py
new file mode 100644
index 00000000000..945f42eb09f
--- /dev/null
+++ b/api/controllers/inner_api/agent/tracing.py
@@ -0,0 +1,49 @@
+"""Authenticated, idempotent ingestion of Agent observations."""
+
+from typing import Literal
+from uuid import UUID
+
+from dify_agent.protocol.tracing import AgentTraceBatch
+from flask import request
+from flask_restx import Resource
+from pydantic import ValidationError
+from werkzeug.exceptions import BadRequest, Conflict, NotFound, RequestEntityTooLarge
+
+from configs import dify_config
+from controllers.common.schema import register_response_schema_models, register_schema_models
+from controllers.inner_api import inner_api_ns
+from controllers.inner_api.wraps import agent_inner_api_only
+from fields.base import ResponseModel
+from repositories.application_trace_repository import TraceCollectionError
+from services.tracing.service import application_tracing_service
+
+
+class AgentTraceBatchResponse(ResponseModel):
+ result: Literal["success"]
+ batch_id: UUID
+ accepted_spans: int
+
+
+register_schema_models(inner_api_ns, AgentTraceBatch)
+register_response_schema_models(inner_api_ns, AgentTraceBatchResponse)
+
+
+@inner_api_ns.route("/agent/tracing/batches")
+class AgentTraceBatchesApi(Resource):
+ @agent_inner_api_only
+ @inner_api_ns.doc("inner_agent_trace_batches")
+ @inner_api_ns.expect(inner_api_ns.models[AgentTraceBatch.__name__])
+ @inner_api_ns.response(200, "Durably accepted", inner_api_ns.models[AgentTraceBatchResponse.__name__])
+ def post(self):
+ request.max_content_length = dify_config.OPS_AGENT_TRACE_BATCH_MAX_BYTES
+ try:
+ payload = AgentTraceBatch.model_validate(inner_api_ns.payload or {})
+ accepted = application_tracing_service().ingest(payload)
+ except ValidationError as error:
+ raise BadRequest("Invalid Agent trace batch") from error
+ except TraceCollectionError as error:
+ error_type = {404: NotFound, 409: Conflict, 413: RequestEntityTooLarge}.get(error.status, BadRequest)
+ raise error_type(error.code) from error
+ return AgentTraceBatchResponse(result="success", batch_id=payload.batch_id, accepted_spans=accepted).model_dump(
+ mode="json"
+ )
diff --git a/api/core/app/apps/agent_app/app_generator.py b/api/core/app/apps/agent_app/app_generator.py
index 6cb48fa0497..e33fde46f77 100644
--- a/api/core/app/apps/agent_app/app_generator.py
+++ b/api/core/app/apps/agent_app/app_generator.py
@@ -311,8 +311,21 @@ class AgentAppGenerator(MessageBasedAppGenerator):
),
)
except GenerateTaskStoppedError:
+ # The runner hands off only a runtime terminal it observed.
+ # A local stop or failed cancellation wait is still just intent.
pass
except Exception as e:
+ from core.ops.base_trace_instance import application_tracing_hooks
+
+ tracing_hooks = application_tracing_hooks()
+ if tracing_hooks is not None:
+ tracing_hooks.capture_agent_app_terminal(
+ tenant_id=application_generate_entity.app_config.tenant_id,
+ app_id=application_generate_entity.app_config.app_id,
+ message_id=message_id,
+ outcome="failed",
+ output=None,
+ )
logger.exception("Unknown Error in Agent App generate worker")
queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
finally:
diff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.py
index c8315afee5b..cfe4a1b6250 100644
--- a/api/core/app/apps/agent_app/app_runner.py
+++ b/api/core/app/apps/agent_app/app_runner.py
@@ -728,18 +728,41 @@ class AgentAppRunner:
message_id,
exc_info=True,
)
- self._publish_terminal_answer(
- queue_manager=queue_manager,
- model_name=model_name,
- answer=answer,
- query=query,
- usage=terminal_usage,
- )
+ try:
+ self._publish_terminal_answer(
+ queue_manager=queue_manager,
+ model_name=model_name,
+ answer=answer,
+ query=query,
+ usage=terminal_usage,
+ )
+ except GenerateTaskStoppedError:
+ # Stop can become visible after stream consumption returned its
+ # success but before the final chunk/end reaches the consumer.
+ self._finish_stopped_terminal(
+ terminal,
+ dify_context=dify_context,
+ session_scope=scope,
+ binding_id=runtime.binding_id,
+ message_id=message_id,
+ )
+ raise
self._save_session(
scope=scope,
binding_id=runtime.binding_id,
snapshot=terminal.session_snapshot,
)
+ from core.ops.base_trace_instance import application_tracing_hooks
+
+ tracing_hooks = application_tracing_hooks()
+ if tracing_hooks is not None:
+ tracing_hooks.capture_agent_app_terminal(
+ tenant_id=dify_context.tenant_id,
+ app_id=dify_context.app_id,
+ message_id=message_id,
+ outcome="succeeded",
+ output=answer,
+ )
def _build_session_scope(
self,
@@ -857,6 +880,20 @@ class AgentAppRunner:
for public_event in public_events:
if public_event.id is not None:
last_event_id = public_event.id
+ internal_events = self._event_adapter.adapt(public_event)
+ for internal_event in internal_events:
+ if isinstance(
+ internal_event,
+ AgentBackendRunSucceededInternalEvent
+ | AgentBackendRunFailedInternalEvent
+ | AgentBackendRunCancelledInternalEvent,
+ ):
+ # A terminal frame may already have arrived when stop
+ # becomes visible. Keep the observed runtime result.
+ terminal = internal_event
+ break
+ if terminal is not None:
+ break
if queue_manager.is_stopped():
flush_pending_agent_message_text()
self._cancel_run(
@@ -865,9 +902,11 @@ class AgentAppRunner:
session_scope=session_scope,
binding_id=binding_id,
message_id=message_id,
+ dify_context=dify_context,
+ queue_manager=queue_manager,
)
raise GenerateTaskStoppedError()
- for internal_event in self._event_adapter.adapt(public_event):
+ for internal_event in internal_events:
if queue_manager.is_stopped():
flush_pending_agent_message_text()
self._cancel_run(
@@ -876,6 +915,8 @@ class AgentAppRunner:
session_scope=session_scope,
binding_id=binding_id,
message_id=message_id,
+ dify_context=dify_context,
+ queue_manager=queue_manager,
)
raise GenerateTaskStoppedError()
if internal_event.type in (
@@ -904,11 +945,6 @@ class AgentAppRunner:
)
continue
continue
- flush_pending_agent_message_text()
- terminal = internal_event
- break
- if terminal is not None:
- break
except GenerateTaskStoppedError:
raise
except Exception as error:
@@ -919,19 +955,43 @@ class AgentAppRunner:
session_scope=session_scope,
binding_id=binding_id,
message_id=message_id,
+ dify_context=dify_context,
+ queue_manager=queue_manager,
)
if queue_manager.is_stopped():
raise GenerateTaskStoppedError() from error
raise
- flush_pending_agent_message_text()
+ try:
+ flush_pending_agent_message_text()
+ except GenerateTaskStoppedError:
+ if terminal is not None:
+ self._finish_stopped_terminal(
+ terminal,
+ dify_context=dify_context,
+ session_scope=session_scope,
+ binding_id=binding_id,
+ message_id=message_id,
+ )
+ raise
if queue_manager.is_stopped():
- self._cancel_run(
- run_id,
- after=last_event_id,
- session_scope=session_scope,
- binding_id=binding_id,
- message_id=message_id,
- )
+ if terminal is not None:
+ self._finish_stopped_terminal(
+ terminal,
+ dify_context=dify_context,
+ session_scope=session_scope,
+ binding_id=binding_id,
+ message_id=message_id,
+ )
+ else:
+ self._cancel_run(
+ run_id,
+ after=last_event_id,
+ session_scope=session_scope,
+ binding_id=binding_id,
+ message_id=message_id,
+ dify_context=dify_context,
+ queue_manager=queue_manager,
+ )
raise GenerateTaskStoppedError()
return terminal, process_recorder
@@ -943,23 +1003,20 @@ class AgentAppRunner:
session_scope: AgentAppSessionScope,
binding_id: str,
message_id: str,
+ dify_context: DifyRunContext,
+ queue_manager: AppQueueManager,
) -> None:
try:
public_event = self._agent_backend_client.cancel_run_and_wait(run_id, after=after)
for internal_event in self._event_adapter.adapt(public_event):
- if (
- isinstance(internal_event, AgentBackendRunCancelledInternalEvent)
- and internal_event.session_snapshot is not None
- ):
- self._save_session(
- scope=session_scope,
- binding_id=binding_id,
- snapshot=internal_event.session_snapshot,
- )
if isinstance(internal_event, AgentBackendRunCancelledInternalEvent):
- self._persist_message_usage(
+ self._finish_stopped_terminal(
+ internal_event,
+ dify_context=dify_context,
+ session_scope=session_scope,
+ binding_id=binding_id,
message_id=message_id,
- usage=_llm_usage_from_agent_backend(internal_event.usage),
+ queue_manager=queue_manager,
)
except Exception:
logger.warning(
@@ -968,6 +1025,50 @@ class AgentAppRunner:
exc_info=True,
)
+ def _finish_stopped_terminal(
+ self,
+ terminal: AgentBackendRunSucceededInternalEvent
+ | AgentBackendRunFailedInternalEvent
+ | AgentBackendRunCancelledInternalEvent,
+ *,
+ dify_context: DifyRunContext,
+ session_scope: AgentAppSessionScope,
+ binding_id: str,
+ message_id: str,
+ queue_manager: AppQueueManager | None = None,
+ ) -> None:
+ """Keep the observed runtime result when the client has requested stop."""
+ if terminal.session_snapshot is not None:
+ self._save_session(scope=session_scope, binding_id=binding_id, snapshot=terminal.session_snapshot)
+ self._persist_message_usage(message_id=message_id, usage=_llm_usage_from_agent_backend(terminal.usage))
+ if queue_manager is not None and not queue_manager.is_stopped():
+ # A stream error can cancel its runtime only for cleanup. Keep its
+ # snapshot/usage, but let the original business failure own the root.
+ # Check after cleanup so a Stop observed during its wait still wins.
+ return
+ from core.ops.base_trace_instance import application_tracing_hooks
+
+ tracing_hooks = application_tracing_hooks()
+ if tracing_hooks is not None:
+ outcome: Literal["succeeded", "failed", "cancelled"]
+ if isinstance(terminal, AgentBackendRunSucceededInternalEvent):
+ outcome = "succeeded"
+ elif isinstance(terminal, AgentBackendRunFailedInternalEvent):
+ outcome = "failed"
+ else:
+ outcome = "cancelled"
+ tracing_hooks.capture_agent_app_terminal(
+ tenant_id=dify_context.tenant_id,
+ app_id=dify_context.app_id,
+ message_id=message_id,
+ outcome=outcome,
+ output=(
+ self._terminal_output_to_answer(terminal.output)
+ if isinstance(terminal, AgentBackendRunSucceededInternalEvent)
+ else None
+ ),
+ )
+
def _publish_terminal_answer(
self,
*,
@@ -992,7 +1093,11 @@ class AgentAppRunner:
if usage is None or (usage.total_tokens <= 0 and usage.total_price <= 0):
return
try:
- message = db.session.get(Message, message_id)
+ # A stopped consumer may have committed its stop intent/partial
+ # answer through another session. Merge usage with that latest row
+ # under the same short lock used by its save, not an identity-cache
+ # snapshot that can erase the intent or newer reported usage.
+ message = db.session.get(Message, message_id, populate_existing=True, with_for_update=True)
if message is None:
logger.warning("Cannot persist Agent App usage: message not found: %s", message_id)
return
diff --git a/api/core/app/apps/agent_app/runtime_request_builder.py b/api/core/app/apps/agent_app/runtime_request_builder.py
index cc4a63d45d6..c373d96ffd3 100644
--- a/api/core/app/apps/agent_app/runtime_request_builder.py
+++ b/api/core/app/apps/agent_app/runtime_request_builder.py
@@ -36,6 +36,7 @@ from clients.agent_backend import (
from configs import dify_config
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
from core.app.llm.model_access import resolve_model_context_window, resolve_model_supports_vision
+from core.ops.base_trace_instance import application_tracing_hooks
from core.plugin.provider_identity import normalize_plugin_daemon_provider_identity
from core.workflow.file_reference import build_file_reference, is_canonical_file_reference
from core.workflow.nodes.agent_v2.dify_tools_builder import (
@@ -167,6 +168,7 @@ class AgentAppRuntimeRequestBuilder:
image_detail_config=context.image_detail_config,
)
+ tracing_hooks = application_tracing_hooks()
request = self._request_builder.build_for_agent_app(
AgentBackendAgentAppRunInput(
model=AgentBackendModelConfig(
@@ -188,6 +190,7 @@ class AgentAppRuntimeRequestBuilder:
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
agent_mode="agent_app",
+ tracing=tracing_hooks.prepare_agent_app(context) if tracing_hooks is not None else None,
),
backend_binding_ref=context.backend_binding_ref,
# ENG-616: expand slash-menu mention tokens to canonical names so
diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py
index 8995e106a01..aa490fbf546 100644
--- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py
+++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py
@@ -318,18 +318,28 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
answer=output_moderation_answer
)
+ stopped_agent_message: Message | None = None
with session_factory.create_session() as session:
# A stopped Agent run may persist provider-reported usage after
# cancellation completes. Do not replace it with local token estimates.
if isinstance(event, QueueStopEvent):
- self._save_message(
+ saved_message = self._save_message(
session=session,
trace_manager=trace_manager,
preserve_existing_usage=True,
)
+ if self._app_config.app_mode == AppMode.AGENT:
+ stopped_agent_message = saved_message
else:
self._save_message(session=session, trace_manager=trace_manager)
session.commit()
+ if stopped_agent_message is not None:
+ # Provider/billing subscribers may perform external I/O.
+ # Release the stop-save row lock before notifying them.
+ message_was_created.send(
+ stopped_agent_message,
+ application_generate_entity=self._application_generate_entity,
+ )
message_end_resp = self._message_end_to_stream_response()
yield message_end_resp
case QueueRetrieverResourcesEvent():
@@ -428,7 +438,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
session: Session,
trace_manager: TraceQueueManager | None = None,
preserve_existing_usage: bool = False,
- ):
+ ) -> Message:
"""
Save message.
:return:
@@ -436,7 +446,10 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
llm_result = self._task_state.llm_result
usage = llm_result.usage
+ agent_stop_requested = preserve_existing_usage and self._app_config.app_mode == AppMode.AGENT
message_stmt = select(Message).where(Message.id == self._message_id)
+ if agent_stop_requested:
+ message_stmt = message_stmt.with_for_update().execution_options(populate_existing=True)
message = session.scalar(message_stmt)
if not message:
raise ValueError(f"message {self._message_id} not found")
@@ -481,9 +494,13 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
metadata = self._task_state.metadata.model_dump(mode="json")
if has_persisted_usage and "usage" in existing_metadata:
metadata["usage"] = existing_metadata["usage"]
+ if agent_stop_requested:
+ # Closing the consumer records a request, not a runtime outcome.
+ # The Agent worker owns the terminal after cancellation/cleanup.
+ metadata["agent_stop_requested"] = True
message.message_metadata = json.dumps(metadata, ensure_ascii=False)
- if trace_manager:
+ if trace_manager and not agent_stop_requested:
trace_manager.add_trace_task(
TraceTask(
TraceTaskName.MESSAGE_TRACE,
@@ -493,10 +510,12 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat
)
)
- message_was_created.send(
- message,
- application_generate_entity=self._application_generate_entity,
- )
+ if not agent_stop_requested:
+ message_was_created.send(
+ message,
+ application_generate_entity=self._application_generate_entity,
+ )
+ return message
def _handle_stop(self, event: QueueStopEvent):
"""
diff --git a/api/core/ops/base_trace_instance.py b/api/core/ops/base_trace_instance.py
index d0f27ad0c5f..bcd2a663584 100644
--- a/api/core/ops/base_trace_instance.py
+++ b/api/core/ops/base_trace_instance.py
@@ -1,13 +1,91 @@
+from __future__ import annotations
+
from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Literal, Protocol, cast
+from flask import Flask, current_app, has_app_context
from sqlalchemy import select
from sqlalchemy.orm import Session
+from configs import dify_config
from core.ops.entities.config_entity import BaseTracingConfig
from core.ops.entities.trace_entity import BaseTraceInfo
+from core.ops.exceptions import RetryableTraceDispatchError
from extensions.ext_database import db
from models import Account, App
+if TYPE_CHECKING:
+ from dify_agent.layers.execution_context import ApplicationTracingContext
+
+ from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
+ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
+ from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
+
+
+class ApplicationTracingHooks(Protocol):
+ """Execution-owned port for the application tracing lifecycle.
+
+ The composition root supplies an app-scoped implementation. Core callers
+ carry trusted execution context and do not construct services or vendor SDKs.
+ """
+
+ def prepare_agent_app(self, context: AgentAppRuntimeBuildContext) -> ApplicationTracingContext | None: ...
+
+ def prepare_workflow_agent(
+ self, context: WorkflowAgentRuntimeBuildContext, conversation_id: str | None
+ ) -> ApplicationTracingContext | None: ...
+
+ def freeze_workflow_run(
+ self,
+ entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,
+ *,
+ workflow_id: str,
+ workflow_run_id: str,
+ conversation_id: object,
+ graph_data: object,
+ ) -> None: ...
+
+ def queue_enabled_at_start(self, app_id: object) -> bool: ...
+
+ def capture_terminal(self, info: BaseTraceInfo) -> bool: ...
+
+ def capture_agent_app_terminal(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ message_id: str,
+ outcome: Literal["succeeded", "failed", "cancelled"],
+ output: str | None,
+ ) -> None: ...
+
+
+class ApplicationTracingHooksNotRegisteredError(RetryableTraceDispatchError):
+ """Enabled tracing must not silently fall back to a different export path."""
+
+ def __init__(self) -> None:
+ super().__init__("application_tracing_hooks_not_registered")
+
+
+_APPLICATION_TRACING_HOOKS_KEY = "application_tracing_hooks"
+
+
+def bind_application_tracing_hooks(app: Flask, hooks: ApplicationTracingHooks) -> None:
+ """Bind the execution port to one Flask application, never a process singleton."""
+ app.extensions[_APPLICATION_TRACING_HOOKS_KEY] = hooks
+
+
+def application_tracing_hooks() -> ApplicationTracingHooks | None:
+ """Resolve the current app's port; a disabled deployment needs no Flask context."""
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return None
+ if not has_app_context():
+ raise ApplicationTracingHooksNotRegisteredError()
+ hooks = current_app.extensions.get(_APPLICATION_TRACING_HOOKS_KEY)
+ if hooks is None:
+ raise ApplicationTracingHooksNotRegisteredError()
+ return cast(ApplicationTracingHooks, hooks)
+
class BaseTraceInstance(ABC):
"""
diff --git a/api/core/ops/exceptions.py b/api/core/ops/exceptions.py
index 35ceff6124d..e80eb3b9649 100644
--- a/api/core/ops/exceptions.py
+++ b/api/core/ops/exceptions.py
@@ -13,6 +13,8 @@ class TraceProviderNotInstalledError(ImportError):
class RetryableTraceDispatchError(RuntimeError):
"""Base class for transient trace dispatch failures that Celery may retry."""
+ acceptance_unknown: bool = False
+
class TraceParentContextAccessError(RetryableTraceDispatchError):
"""Raised when unified parent context storage is temporarily unavailable."""
diff --git a/api/core/ops/ops_trace_manager.py b/api/core/ops/ops_trace_manager.py
index 94829ffede2..742a8dabf47 100644
--- a/api/core/ops/ops_trace_manager.py
+++ b/api/core/ops/ops_trace_manager.py
@@ -20,6 +20,7 @@ from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from core.helper.encrypter import batch_decrypt_token, encrypt_token, obfuscated_token
from core.helper.trace_id_helper import ParentTraceContext
+from core.ops.base_trace_instance import application_tracing_hooks
from core.ops.entities.config_entity import (
BaseTracingConfig,
TracingProviderEnum,
@@ -390,7 +391,11 @@ class OpsTraceManager:
# Encrypt necessary keys
for key in secret_keys:
if key in tracing_config:
- if "*" in tracing_config[key]:
+ if tracing_config[key] is None:
+ # Preserve explicit clearing for optional credentials; the
+ # provider model still rejects null on a required secret.
+ new_config[key] = None
+ elif "*" in tracing_config[key]:
# If the key contains '*', retain the original value from the current config
if current_trace_config:
new_config[key] = current_trace_config.get(key, tracing_config[key])
@@ -439,8 +444,10 @@ class OpsTraceManager:
provider_config_map[tracing_provider]["secret_keys"],
provider_config_map[tracing_provider]["other_keys"],
)
- new_config: dict[str, Any] = {}
- keys_to_decrypt = [key for key in secret_keys if key in tracing_config]
+ new_config: dict[str, Any] = {
+ key: None for key in secret_keys if key in tracing_config and tracing_config[key] is None
+ }
+ keys_to_decrypt = [key for key in secret_keys if tracing_config.get(key) is not None]
if keys_to_decrypt:
decrypted_values = batch_decrypt_token(tenant_id, [tracing_config[key] for key in keys_to_decrypt])
new_config.update(zip(keys_to_decrypt, decrypted_values, strict=False))
@@ -1477,7 +1484,15 @@ class TraceQueueManager:
self.app_id = app_id
self.user_id = user_id
- self.trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)
+ self.trace_instance = None
+ self._application_tracing_enabled = False
+ if dify_config.OPS_AGENT_TRACE_ENABLED:
+ tracing_hooks = application_tracing_hooks()
+ self._application_tracing_enabled = (
+ tracing_hooks.queue_enabled_at_start(app_id) if tracing_hooks is not None else False
+ )
+ else:
+ self.trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)
self.flask_app = current_app._get_current_object() # type: ignore
from core.telemetry.gateway import is_enterprise_telemetry_enabled
@@ -1489,7 +1504,7 @@ class TraceQueueManager:
def add_trace_task(self, trace_task: TraceTask):
global trace_manager_timer, trace_manager_queue
try:
- if self._enterprise_telemetry_enabled or self.trace_instance:
+ if self._enterprise_telemetry_enabled or self._application_tracing_enabled or self.trace_instance:
trace_task.app_id = self.app_id
trace_manager_queue.put(trace_task)
except Exception:
diff --git a/api/core/workflow/nodes/agent_v2/runtime_request_builder.py b/api/core/workflow/nodes/agent_v2/runtime_request_builder.py
index 0fb791fe874..424f46148de 100644
--- a/api/core/workflow/nodes/agent_v2/runtime_request_builder.py
+++ b/api/core/workflow/nodes/agent_v2/runtime_request_builder.py
@@ -48,6 +48,7 @@ from clients.agent_backend import (
from configs import dify_config
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
from core.app.llm.model_access import resolve_model_context_window
+from core.ops.base_trace_instance import application_tracing_hooks
from core.plugin.provider_identity import normalize_plugin_daemon_provider_identity
from core.workflow.system_variables import SystemVariableKey, get_system_text, get_system_value
from graphon.file import File, FileTransferMethod
@@ -227,6 +228,7 @@ class WorkflowAgentRuntimeRequestBuilder:
agent_soul.model.plugin_id,
)
+ tracing_hooks = application_tracing_hooks()
request = self._request_builder.build_for_workflow_node(
AgentBackendWorkflowNodeRunInput(
model=AgentBackendModelConfig(
@@ -257,6 +259,13 @@ class WorkflowAgentRuntimeRequestBuilder:
agent_config_version_kind="snapshot",
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
+ tracing=(
+ tracing_hooks.prepare_workflow_agent(
+ context, get_system_text(context.variable_pool, SystemVariableKey.CONVERSATION_ID)
+ )
+ if tracing_hooks is not None
+ else None
+ ),
),
backend_binding_ref=context.backend_binding_ref,
agent_soul_prompt=soul_prompt or None,
diff --git a/api/extensions/application_services/app.py b/api/extensions/application_services/app.py
index 8459402af50..6eb6f2b5d9f 100644
--- a/api/extensions/application_services/app.py
+++ b/api/extensions/application_services/app.py
@@ -28,7 +28,7 @@ from services.app.import_service import AppImportService
from services.app.query_service import AppDiscoveryService
from services.app_dsl_service import AppDslService
from services.app_package_service import AppPackageService
-from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigService
from services.oauth_server_service import OAuthServerService
from services.rbac.members import MemberService
from services.recommended_app_package_service import RecommendedAppPackageService
@@ -52,6 +52,7 @@ def build_app_services(
recommended_packages: RecommendedAppPackageService,
repository: ConsoleAppRepository,
rbac_members: MemberService,
+ tracing_configs: AppTracingConfigService,
) -> AppServices:
transfers = AppTransferGateway(
session_factory=database_client,
@@ -77,7 +78,7 @@ def build_app_services(
access=EnterpriseConsoleAppAccess(members=rbac_members),
transfers=transfers,
creators=CreatorsPlatformGateway(oauth=oauth),
- tracing=OpsTraceManagerGateway(),
+ tracing=tracing_configs,
lifecycle=AppLifecycleGateway(session_factory=database_client),
),
enterprise=EnterpriseAppService(
diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py
index d4105d8db27..381099f5c1b 100644
--- a/api/extensions/ext_application_services.py
+++ b/api/extensions/ext_application_services.py
@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session, sessionmaker
from configs import dify_config
from constants.dsl_version import CURRENT_APP_DSL_VERSION
from core.db.session_factory import get_session_maker
+from core.ops.base_trace_instance import bind_application_tracing_hooks
from core.schemas.schema_manager import SchemaManager
from core.tools.tool_file_manager import ToolFileManager
from enums import DeploymentEdition, WebAppAccessMode
@@ -52,6 +53,7 @@ from repositories.account.repository import SQLAlchemyAccountRepository
from repositories.account_activation_repository import SQLAlchemyAccountActivationRepository
from repositories.account_integration_repository import SQLAlchemyAccountIntegrationRepository
from repositories.api_based_extension_repository import APIBasedExtensionRepository
+from repositories.app.agent_app_repository import AgentAppRepository
from repositories.app.console_repository import ConsoleAppRepository
from repositories.app.mcp_server_repository import AppMCPServerRepository
from repositories.app.site_command_repository import AppSiteCommandRepository
@@ -99,6 +101,7 @@ from services.account.login_adapters import RedisConsoleAuthSecurityGateway
from services.account.service import AccountSetupProvisioner
from services.account_password_hasher import DefaultAccountPasswordHasher
from services.agent.roster_package_exporter import RosterAgentPackageExporter
+from services.agent_tracing_config_service import AgentTracingConfigService
from services.api_based_extension_adapters import APIBasedExtensionPingProbe, WorkspaceTokenCipher
from services.api_based_extension_application_service import APIBasedExtensionApplicationService
from services.app.advanced_prompt_template_service import AdvancedPromptTemplateService
@@ -187,6 +190,7 @@ from services.step_by_step_tour_service import StepByStepTourService
from services.system_feature_service import SystemFeatureService
from services.tag_application_service import TagApplicationService
from services.tool_file_download_service import ToolFileDownloadService
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from services.upload_file_delivery_service import UploadFileDeliveryService
from services.web_app_runtime_query_service import WebAppRuntimeQueryService
from services.web_authentication_adapters import (
@@ -274,6 +278,7 @@ class ApplicationServices:
conversation_variables: ConversationVariableQuery
app_statistics: AppStatisticQuery
app_tracing_configs: AppTracingConfigService
+ agent_tracing_configs: AgentTracingConfigService
billing_portal: BillingPortalService
compliance_downloads: ComplianceDownloadService
data_sources: DataSourceServices
@@ -460,12 +465,17 @@ def build_application_services(
apps=console_apps,
datasets=dataset_dependencies.datasets,
)
+ tracing_configs = AppTracingConfigService(
+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),
+ provider=OpsTraceManagerGateway(),
+ )
apps = build_app_services(
database_client=database_client,
oauth=oauth_server,
recommended_packages=recommended_app_packages,
repository=console_apps,
rbac_members=rbac.members,
+ tracing_configs=tracing_configs,
)
tags = TagApplicationService(tags=TagRepository(session_factory=database_client))
knowledge = build_knowledge_services(
@@ -549,9 +559,11 @@ def build_application_services(
),
resource_access_tokens=build_resource_access_token_service(database_client=database_client),
app_statistics=AppStatisticQueryRepository(session_factory=database_client),
- app_tracing_configs=AppTracingConfigService(
- configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),
- provider=OpsTraceManagerGateway(),
+ app_tracing_configs=tracing_configs,
+ agent_tracing_configs=AgentTracingConfigService(
+ owners=AgentAppRepository(session_factory=database_client),
+ settings=apps.console,
+ configs=tracing_configs,
),
billing_portal=BillingPortalService(
accounts=accounts,
@@ -749,6 +761,7 @@ def init_app(app: Flask) -> None:
redis=redis_client,
)
app.extensions[_EXTENSION_KEY] = services
+ bind_application_tracing_hooks(app, RuntimeApplicationTracingHooks())
bind_account_loader(app, services.accounts.identity.load_user)
diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.py
index 6368a7fd1dd..4e58a0b0af0 100644
--- a/api/extensions/ext_celery.py
+++ b/api/extensions/ext_celery.py
@@ -188,6 +188,14 @@ def init_app(app: DifyApp) -> Celery:
# if you add a new task, please add the switch to CeleryScheduleTasksConfig
beat_schedule: dict[str, CeleryBeatScheduleEntry] = {}
+ if dify_config.OPS_AGENT_TRACE_ENABLED:
+ imports.append("tasks.application_trace_task")
+ if dify_config.OPS_AGENT_TRACE_ENABLED and dify_config.ENABLE_APPLICATION_TRACE_RECONCILE_TASK:
+ beat_schedule["reconcile_application_traces"] = {
+ "task": "tasks.application_trace_task.reconcile_application_traces",
+ "schedule": timedelta(seconds=30),
+ "options": {"expires": 25},
+ }
if dify_config.AGENT_SANDBOX_METERING_ENABLED:
try:
interval = int(dify_config.AGENT_SANDBOX_METERING_INTERVAL_SECONDS)
diff --git a/api/fields/app_fields.py b/api/fields/app_fields.py
index 92408bac755..d1c3ffc45e3 100644
--- a/api/fields/app_fields.py
+++ b/api/fields/app_fields.py
@@ -17,6 +17,8 @@ from services.entities.dsl_entities import DslImportWarning, ImportStatus
class AppTraceResponse(ResponseModel):
enabled: bool = False
tracing_provider: str | None = None
+ capture_content: bool = False
+ capture_agent: bool = True
class Tag(ResponseModel):
diff --git a/api/migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py b/api/migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py
new file mode 100644
index 00000000000..a7cab08de98
--- /dev/null
+++ b/api/migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py
@@ -0,0 +1,86 @@
+"""Add durable application tracing collection and delivery state.
+
+Revision ID: a17c2980f3e6
+Revises: f1d3c5b7a902
+Create Date: 2026-10-15 03:30:00
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from models.types import StringUUID
+
+revision = "a17c2980f3e6"
+down_revision = "f1d3c5b7a902"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "application_trace_runs",
+ sa.Column("id", StringUUID(), primary_key=True),
+ sa.Column("tenant_id", StringUUID(), nullable=False),
+ sa.Column("app_id", StringUUID(), nullable=False),
+ sa.Column("source_kind", sa.String(16), nullable=False),
+ sa.Column("source_id", StringUUID(), nullable=False),
+ sa.Column("provider", sa.String(32), nullable=False),
+ sa.Column("config_revision", sa.String(64), nullable=False),
+ sa.Column("provider_config", sa.JSON(), nullable=False),
+ sa.Column("trace_id", sa.String(32), nullable=False),
+ sa.Column("root_span_id", sa.String(16), nullable=False),
+ sa.Column("session_id", sa.String(255), nullable=False),
+ sa.Column("capture_content", sa.Boolean(), nullable=False),
+ sa.Column("capture_agent", sa.Boolean(), nullable=False),
+ sa.Column("state", sa.String(24), nullable=False),
+ sa.Column("outer_trace", sa.JSON(), nullable=True),
+ sa.Column("span_count", sa.Integer(), nullable=False),
+ sa.Column("payload_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("attempts", sa.Integer(), nullable=False),
+ sa.Column("error_code", sa.String(96), nullable=True),
+ sa.Column("trace_url", sa.Text(), nullable=True),
+ sa.Column("provider_trace_id", sa.String(255), nullable=True),
+ sa.Column("span_urls", sa.JSON(), nullable=False),
+ sa.Column("export_token", StringUUID(), nullable=True),
+ sa.Column("export_started_at", sa.DateTime(), nullable=True),
+ sa.Column("export_heartbeat_at", sa.DateTime(), nullable=True),
+ sa.Column("next_attempt_at", sa.DateTime(), nullable=True),
+ sa.Column("accepted_at", sa.DateTime(), nullable=True),
+ sa.Column("terminal_at", sa.DateTime(), nullable=True),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(), nullable=False),
+ sa.UniqueConstraint("tenant_id", "app_id", "source_kind", "source_id", name="application_trace_source_unique"),
+ )
+ op.create_index("application_trace_delivery_idx", "application_trace_runs", ["state", "next_attempt_at"])
+ op.create_index("application_trace_retention_idx", "application_trace_runs", ["created_at"])
+ op.create_table(
+ "application_trace_invocations",
+ sa.Column("id", StringUUID(), primary_key=True),
+ sa.Column("run_id", StringUUID(), nullable=False),
+ sa.Column("agent_id", StringUUID(), nullable=False),
+ sa.Column("parent_span_id", sa.String(16), nullable=False),
+ sa.Column("span_id", sa.String(16), nullable=False),
+ sa.Column("node_execution_id", StringUUID(), nullable=True),
+ sa.Column("attempt", sa.Integer(), nullable=False),
+ sa.Column("finalized", sa.Boolean(), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("finalized_at", sa.DateTime(), nullable=True),
+ )
+ op.create_index("application_trace_invocation_run_idx", "application_trace_invocations", ["run_id"])
+ op.create_table(
+ "application_trace_batches",
+ sa.Column("id", StringUUID(), primary_key=True),
+ sa.Column("run_id", StringUUID(), nullable=False),
+ sa.Column("invocation_id", StringUUID(), nullable=False),
+ sa.Column("payload_hash", sa.String(64), nullable=False),
+ sa.Column("spans", sa.JSON(), nullable=False),
+ sa.Column("final", sa.Boolean(), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ )
+ op.create_index("application_trace_batch_run_idx", "application_trace_batches", ["run_id", "created_at"])
+
+
+def downgrade() -> None:
+ op.drop_table("application_trace_batches")
+ op.drop_table("application_trace_invocations")
+ op.drop_table("application_trace_runs")
diff --git a/api/migrations/versions/2026_10_19_1340-f7e2c9a6d481_merge_tracing_and_workflow_pause_heads.py b/api/migrations/versions/2026_10_19_1340-f7e2c9a6d481_merge_tracing_and_workflow_pause_heads.py
new file mode 100644
index 00000000000..04630caaf6a
--- /dev/null
+++ b/api/migrations/versions/2026_10_19_1340-f7e2c9a6d481_merge_tracing_and_workflow_pause_heads.py
@@ -0,0 +1,19 @@
+"""Merge application tracing and workflow pause migration heads.
+
+Revision ID: f7e2c9a6d481
+Revises: a17c2980f3e6, c6f4a90d8b31
+Create Date: 2026-10-19 13:40:00
+"""
+
+revision = "f7e2c9a6d481"
+down_revision = ("a17c2980f3e6", "c6f4a90d8b31")
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ pass
+
+
+def downgrade() -> None:
+ pass
diff --git a/api/migrations/versions/2026_10_21_1030-d6b9e2f1a407_merge_tracing_and_workflow_tool_heads.py b/api/migrations/versions/2026_10_21_1030-d6b9e2f1a407_merge_tracing_and_workflow_tool_heads.py
new file mode 100644
index 00000000000..5d32480c057
--- /dev/null
+++ b/api/migrations/versions/2026_10_21_1030-d6b9e2f1a407_merge_tracing_and_workflow_tool_heads.py
@@ -0,0 +1,19 @@
+"""Merge application tracing and workflow tool migration heads.
+
+Revision ID: d6b9e2f1a407
+Revises: a8c3e7f4b912, f7e2c9a6d481
+Create Date: 2026-10-21 10:30:00
+"""
+
+revision = "d6b9e2f1a407"
+down_revision = ("a8c3e7f4b912", "f7e2c9a6d481")
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ pass
+
+
+def downgrade() -> None:
+ pass
diff --git a/api/models/__init__.py b/api/models/__init__.py
index 21ff0d9fe65..a35befda04f 100644
--- a/api/models/__init__.py
+++ b/api/models/__init__.py
@@ -32,6 +32,7 @@ from .agent import (
)
from .agent_sandbox_usage import AgentSandboxExecution, AgentSandboxUsageEvent
from .api_based_extension import APIBasedExtension, APIBasedExtensionPoint
+from .application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun
from .comment import (
WorkflowComment,
WorkflowCommentMention,
@@ -201,6 +202,9 @@ __all__ = [
"AppTrigger",
"AppTriggerStatus",
"AppTriggerType",
+ "ApplicationTraceBatch",
+ "ApplicationTraceInvocation",
+ "ApplicationTraceRun",
"BuiltinToolProvider",
"CeleryTask",
"CeleryTaskSet",
diff --git a/api/models/agent_config_entities.py b/api/models/agent_config_entities.py
index d3900fa3e77..8911a8888b4 100644
--- a/api/models/agent_config_entities.py
+++ b/api/models/agent_config_entities.py
@@ -536,7 +536,8 @@ class AgentSoulModelSettings(AgentFlexibleConfig):
frequency_penalty: float | None = None
max_tokens: int | None = None
stop: list[str] | None = None
- response_format: AgentModelResponseFormatConfig | None = None
+ # Plugin parameter rules use string enums; existing saved configs may use native objects.
+ response_format: str | AgentModelResponseFormatConfig | None = None
class AgentFeatureToggleConfig(AgentFlexibleConfig):
diff --git a/api/models/application_trace.py b/api/models/application_trace.py
new file mode 100644
index 00000000000..c95ced54ad7
--- /dev/null
+++ b/api/models/application_trace.py
@@ -0,0 +1,84 @@
+"""Durable application tracing envelopes, separate from business execution state.
+
+Provider configuration snapshots contain the existing workspace-encrypted secret
+values. They are never serialized by Console endpoints or sent to Agent runs.
+"""
+
+from datetime import datetime
+from typing import Any
+
+import sqlalchemy as sa
+from sqlalchemy.orm import Mapped, mapped_column
+
+from libs.datetime_utils import naive_utc_now
+from models.base import TypeBase
+from models.types import StringUUID
+
+
+class ApplicationTraceRun(TypeBase, kw_only=True):
+ __tablename__ = "application_trace_runs"
+ __table_args__ = (
+ sa.UniqueConstraint("tenant_id", "app_id", "source_kind", "source_id", name="application_trace_source_unique"),
+ sa.Index("application_trace_delivery_idx", "state", "next_attempt_at"),
+ sa.Index("application_trace_retention_idx", "created_at"),
+ )
+
+ id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
+ tenant_id: Mapped[str] = mapped_column(StringUUID)
+ app_id: Mapped[str] = mapped_column(StringUUID)
+ source_kind: Mapped[str] = mapped_column(sa.String(16))
+ source_id: Mapped[str] = mapped_column(StringUUID)
+ provider: Mapped[str] = mapped_column(sa.String(32))
+ config_revision: Mapped[str] = mapped_column(sa.String(64))
+ provider_config: Mapped[dict[str, Any]] = mapped_column(sa.JSON)
+ trace_id: Mapped[str] = mapped_column(sa.String(32))
+ root_span_id: Mapped[str] = mapped_column(sa.String(16))
+ session_id: Mapped[str] = mapped_column(sa.String(255))
+ capture_content: Mapped[bool] = mapped_column(sa.Boolean)
+ capture_agent: Mapped[bool] = mapped_column(sa.Boolean, default=True)
+ state: Mapped[str] = mapped_column(sa.String(24), default="collecting")
+ outer_trace: Mapped[dict[str, Any] | None] = mapped_column(sa.JSON, default=None)
+ span_count: Mapped[int] = mapped_column(sa.Integer, default=0)
+ payload_bytes: Mapped[int] = mapped_column(sa.BigInteger, default=0)
+ attempts: Mapped[int] = mapped_column(sa.Integer, default=0)
+ error_code: Mapped[str | None] = mapped_column(sa.String(96), default=None)
+ trace_url: Mapped[str | None] = mapped_column(sa.Text, default=None)
+ provider_trace_id: Mapped[str | None] = mapped_column(sa.String(255), default=None)
+ span_urls: Mapped[dict[str, str]] = mapped_column(sa.JSON, default_factory=dict)
+ export_token: Mapped[str | None] = mapped_column(StringUUID, default=None)
+ export_started_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)
+ export_heartbeat_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)
+ next_attempt_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)
+ accepted_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)
+ terminal_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)
+ created_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)
+ updated_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)
+
+
+class ApplicationTraceInvocation(TypeBase, kw_only=True):
+ __tablename__ = "application_trace_invocations"
+ __table_args__ = (sa.Index("application_trace_invocation_run_idx", "run_id"),)
+
+ id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
+ run_id: Mapped[str] = mapped_column(StringUUID)
+ agent_id: Mapped[str] = mapped_column(StringUUID)
+ parent_span_id: Mapped[str] = mapped_column(sa.String(16))
+ span_id: Mapped[str] = mapped_column(sa.String(16))
+ node_execution_id: Mapped[str | None] = mapped_column(StringUUID)
+ attempt: Mapped[int] = mapped_column(sa.Integer)
+ finalized: Mapped[bool] = mapped_column(sa.Boolean, default=False)
+ created_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)
+ finalized_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)
+
+
+class ApplicationTraceBatch(TypeBase, kw_only=True):
+ __tablename__ = "application_trace_batches"
+ __table_args__ = (sa.Index("application_trace_batch_run_idx", "run_id", "created_at"),)
+
+ id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
+ run_id: Mapped[str] = mapped_column(StringUUID)
+ invocation_id: Mapped[str] = mapped_column(StringUUID)
+ payload_hash: Mapped[str] = mapped_column(sa.String(64))
+ spans: Mapped[list[dict[str, Any]]] = mapped_column(sa.JSON)
+ final: Mapped[bool] = mapped_column(sa.Boolean)
+ created_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)
diff --git a/api/models/model.py b/api/models/model.py
index 11c7ec4128c..c4dd6642230 100644
--- a/api/models/model.py
+++ b/api/models/model.py
@@ -308,6 +308,7 @@ class MessageDict(TypedDict):
id: str
app_id: str
conversation_id: str
+ model_provider: NotRequired[str | None]
model_id: str | None
inputs: dict[str, Any]
query: str
@@ -1786,6 +1787,7 @@ class Message(Base):
"id": self.id,
"app_id": self.app_id,
"conversation_id": self.conversation_id,
+ "model_provider": self.model_provider,
"model_id": self.model_id,
"inputs": self.inputs_with_session(session=session),
"query": self.query,
@@ -1810,6 +1812,7 @@ class Message(Base):
id=data["id"],
app_id=data["app_id"],
conversation_id=data["conversation_id"],
+ model_provider=data.get("model_provider"),
model_id=data["model_id"],
inputs=data["inputs"],
total_price=data["total_price"],
diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md
index 69ac746956d..09c87762c2c 100644
--- a/api/openapi/markdown/console-openapi.md
+++ b/api/openapi/markdown/console-openapi.md
@@ -1471,6 +1471,185 @@ Get available TTS voices for an Agent and language
| 404 | Agent not found | **application/json**: [AgentNotFoundError](#agentnotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+### [GET] /agent/{agent_id}/trace
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Agent tracing settings | **application/json**: [AppTraceResponse](#apptraceresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [POST] /agent/{agent_id}/trace
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Request Body
+
+| Required | Schema |
+| -------- | ------ |
+| Yes | **application/json**: [AppTracePayload](#apptracepayload)<br> |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Updated Agent tracing settings | **application/json**: [AppTraceResponse](#apptraceresponse)<br> |
+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 404 | Not Found | **application/json**: [TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |
+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [GET] /agent/{agent_id}/trace-capabilities
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| tracing_provider | query | Tracing provider name | Yes | string |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Provider support declarations | **application/json**: [TraceProviderCapabilitiesResponse](#traceprovidercapabilitiesresponse)<br> |
+| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [DELETE] /agent/{agent_id}/trace-config
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| tracing_provider | query | Tracing provider name | Yes | string |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 204 | Agent provider configuration deleted | |
+| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [GET] /agent/{agent_id}/trace-config
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| tracing_provider | query | Tracing provider name | Yes | string |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Masked Agent provider configuration | **application/json**: [TraceAppConfigResponse](#traceappconfigresponse)<br> |
+| 400 | Bad Request | **application/json**: [TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [PATCH] /agent/{agent_id}/trace-config
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Request Body
+
+| Required | Schema |
+| -------- | ------ |
+| Yes | **application/json**: [TraceConfigPayload](#traceconfigpayload)<br> |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Agent provider updated | **application/json**: [TraceAppConfigResponse](#traceappconfigresponse)<br> |
+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingConfigVerificationFailedError](#tracingconfigverificationfailederror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |
+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [POST] /agent/{agent_id}/trace-config
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| agent_id | path | | Yes | string (uuid) |
+
+#### Request Body
+
+| Required | Schema |
+| -------- | ------ |
+| Yes | **application/json**: [TraceConfigPayload](#traceconfigpayload)<br> |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 201 | Agent provider configured | **application/json**: [TraceAppConfigResponse](#traceappconfigresponse)<br> |
+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingConfigVerificationFailedError](#tracingconfigverificationfailederror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TraceConfigAlreadyExistsError](#traceconfigalreadyexistserror)<br>[ErrorResponse](#errorresponse)<br> |
+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [GET] /agent/{agent_id}/trace-runs/{message_id}
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |
+| agent_id | path | | Yes | string (uuid) |
+| message_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Frozen trace destination and export state | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |
+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [POST] /agent/{agent_id}/trace-runs/{message_id}/retry
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |
+| agent_id | path | | Yes | string (uuid) |
+| message_id | path | | Yes | string (uuid) |
+
+#### Request Body
+
+| Required | Schema |
+| -------- | ------ |
+| Yes | **application/json**: [TraceRunRetryPayload](#tracerunretrypayload)<br> |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Retry state for the same frozen destination | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |
+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingRunNotFoundError](#tracingrunnotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TracingAlreadyAcceptedError](#tracingalreadyacceptederror)<br>[TracingDuplicateAcknowledgementRequiredError](#tracingduplicateacknowledgementrequirederror)<br>[TracingRetryNotAllowedError](#tracingretrynotallowederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
### [GET] /agent/{agent_id}/versions
#### Parameters
@@ -3611,8 +3790,27 @@ Update app tracing configuration
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Trace configuration updated successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)<br> |
-| 400 | Bad Request | **application/json**: [TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[ErrorResponse](#errorresponse)<br> |
+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
| 403 | Insufficient permissions | **application/json**: [ErrorResponse](#errorresponse)<br> |
+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |
+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [GET] /apps/{app_id}/trace-capabilities
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| tracing_provider | query | Tracing provider name | Yes | string |
+| app_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Provider support declarations | **application/json**: [TraceProviderCapabilitiesResponse](#traceprovidercapabilitiesresponse)<br> |
+| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
### [DELETE] /apps/{app_id}/trace-config
@@ -3683,6 +3881,7 @@ Update an existing tracing configuration for an application
| 400 | Invalid request parameters or tracing configuration | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingConfigVerificationFailedError](#tracingconfigverificationfailederror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |
| 403 | Insufficient permissions | **application/json**: [ErrorResponse](#errorresponse)<br> |
| 404 | Application or tracing configuration not found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |
| 500 | Tracing configuration processing failed | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
@@ -3715,6 +3914,49 @@ Create a new tracing configuration for an application
| 500 | Tracing configuration processing failed | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |
| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+### [GET] /apps/{app_id}/trace-runs/{source_kind}/{source_id}
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| source_kind | path | | Yes | string, <br>**Available values:** "message", "node", "workflow" |
+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |
+| app_id | path | | Yes | string (uuid) |
+| source_id | path | | Yes | string (uuid) |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Frozen trace destination and export state | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |
+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
+### [POST] /apps/{app_id}/trace-runs/{source_kind}/{source_id}/retry
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| source_kind | path | | Yes | string, <br>**Available values:** "message", "node", "workflow" |
+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |
+| app_id | path | | Yes | string (uuid) |
+| source_id | path | | Yes | string (uuid) |
+
+#### Request Body
+
+| Required | Schema |
+| -------- | ------ |
+| Yes | **application/json**: [TraceRunRetryPayload](#tracerunretrypayload)<br> |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Retry state for the same frozen destination | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |
+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingRunNotFoundError](#tracingrunnotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |
+| 409 | Conflict | **application/json**: [TracingAlreadyAcceptedError](#tracingalreadyacceptederror)<br>[TracingDuplicateAcknowledgementRequiredError](#tracingduplicateacknowledgementrequirederror)<br>[TracingRetryNotAllowedError](#tracingretrynotallowederror)<br>[ErrorResponse](#errorresponse)<br> |
+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |
+
### [POST] /apps/{app_id}/trigger-enable
**Update app trigger (enable/disable)**
@@ -17095,7 +17337,7 @@ rather than being dropped.
| frequency_penalty | number | | No |
| max_tokens | integer | | No |
| presence_penalty | number | | No |
-| response_format | [AgentModelResponseFormatConfig](#agentmodelresponseformatconfig) | | No |
+| response_format | string<br>[AgentModelResponseFormatConfig](#agentmodelresponseformatconfig) | | No |
| stop | [ string ] | | No |
| temperature | number | | No |
| top_p | number | | No |
@@ -18903,6 +19145,8 @@ Function Suggested questions after answer disabled.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
+| capture_agent | boolean, <br>**Default:** true | Capture Agent internals when deployment support is enabled | No |
+| capture_content | boolean | Include prompt, output and tool content in new traces | No |
| enabled | boolean | Enable or disable tracing | Yes |
| tracing_provider | string | Tracing provider | No |
@@ -18910,6 +19154,8 @@ Function Suggested questions after answer disabled.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
+| capture_agent | boolean, <br>**Default:** true | | No |
+| capture_content | boolean | | No |
| enabled | boolean | | No |
| tracing_provider | string | | No |
@@ -27371,6 +27617,7 @@ Non-sensitive bootstrap snapshot exposed before Console or Web authentication.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
+| agent_tracing_api_collection_enabled | boolean | Deployment-level API gate for Agent trace collection, independent of workspace and provider configuration. Does not indicate Agent runtime, worker, or provider health or confirm trace delivery. | Yes |
| branding | [BrandingModel](#brandingmodel) | | Yes |
| deployment_edition | [DeploymentEdition](#deploymentedition) | | Yes |
| enable_app_deploy | boolean | | Yes |
@@ -27869,15 +28116,101 @@ Tracing configuration not found.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
+| activate | boolean | Save, select this provider, and enable tracing atomically; preserve capture settings | No |
| tracing_config | object | Tracing configuration data | Yes |
| tracing_provider | string | Tracing provider name | Yes |
+#### TraceProviderCapabilitiesResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| agent_spans | boolean | | Yes |
+| api_collection_enabled | boolean | Current API collection gate only; does not probe Agent runtime or confirm delivery | Yes |
+| export_ack | string | Exporter acknowledgement does not confirm query visibility | Yes |
+| native_cost | boolean | | Yes |
+| native_usage | boolean | | Yes |
+| provider | string | | Yes |
+| session | boolean | | Yes |
+| span_url | boolean | | Yes |
+| trace_url | boolean | | Yes |
+
#### TraceProviderQuery
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| tracing_provider | string | Tracing provider name | Yes |
+#### TraceProviderSupportResponse
+
+Adapter support declarations, not successful-delivery or platform-verification results.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| agent_spans | boolean | | Yes |
+| export_ack | string | Exporter acknowledgement does not confirm query visibility | Yes |
+| native_cost | boolean | | Yes |
+| native_usage | boolean | | Yes |
+| provider | string | | Yes |
+| session | boolean | | Yes |
+| span_url | boolean | | Yes |
+| trace_url | boolean | | Yes |
+
+#### TraceRunQuery
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| node_execution_id | string | Select an Agent node invocation within this run | No |
+
+#### TraceRunRetryPayload
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| acknowledge_possible_duplicates | boolean | Acknowledge possible duplicate records when retrying an uncertain export | No |
+
+#### TraceRunStatusResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| accepted_at | string | Ingestion acknowledgement time, not query visibility | No |
+| attempts | integer | | No |
+| capabilities | [TraceProviderSupportResponse](#traceprovidersupportresponse) | | No |
+| capture_agent | boolean | | No |
+| capture_content | boolean | | No |
+| config_revision | string | Frozen destination configuration revision | No |
+| error_code | string | | No |
+| project_url | string | Frozen destination project URL; null when unavailable, never resolved from current config | No |
+| provider | string | | No |
+| queryable | boolean | Null when third-party query visibility is not verified | No |
+| source_id | string | | Yes |
+| source_kind | [TraceSourceKind](#tracesourcekind) | | Yes |
+| span_count | integer | | No |
+| span_url | string | | No |
+| status | string, <br>**Available values:** "accepted", "collecting", "exporting", "failed", "not_collected", "ready", "retrying", "uncertain" | *Enum:* `"accepted"`, `"collecting"`, `"exporting"`, `"failed"`, `"not_collected"`, `"ready"`, `"retrying"`, `"uncertain"` | Yes |
+| trace_id | string | | No |
+| trace_url | string | | No |
+
+#### TraceSourceKind
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TraceSourceKind | string | | |
+
+#### TracingAlreadyAcceptedError
+
+An accepted trace cannot be exported again.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingAlreadyAcceptedError | | An accepted trace cannot be exported again. | |
+
+#### TracingConfigConflictError
+
+Tracing configuration changed. Refresh and try again.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingConfigConflictError | | Tracing configuration changed. Refresh and try again. | |
+
#### TracingConfigProcessingFailedError
The tracing configuration could not be processed.
@@ -27894,6 +28227,22 @@ The tracing configuration could not be verified.
| ---- | ---- | ----------- | -------- |
| TracingConfigVerificationFailedError | | The tracing configuration could not be verified. | |
+#### TracingDuplicateAcknowledgementRequiredError
+
+Acknowledge possible duplicates before retrying an uncertain export.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingDuplicateAcknowledgementRequiredError | | Acknowledge possible duplicates before retrying an uncertain export. | |
+
+#### TracingNodeNotFoundError
+
+The Agent node invocation does not belong to this trace.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingNodeNotFoundError | | The Agent node invocation does not belong to this trace. | |
+
#### TracingProviderUnavailableError
The tracing provider is not available in this deployment.
@@ -27902,6 +28251,30 @@ The tracing provider is not available in this deployment.
| ---- | ---- | ----------- | -------- |
| TracingProviderUnavailableError | | The tracing provider is not available in this deployment. | |
+#### TracingRetryNotAllowedError
+
+This trace is not in a retryable state.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingRetryNotAllowedError | | This trace is not in a retryable state. | |
+
+#### TracingRunNotFoundError
+
+No collected trace is available to retry.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingRunNotFoundError | | No collected trace is available to retry. | |
+
+#### TracingSourceNotFoundError
+
+The tracing source does not exist in this application.
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| TracingSourceNotFoundError | | The tracing source does not exist in this application. | |
+
#### TrialAgentConfigQuery
| Name | Type | Description | Required |
diff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.md
index b6ed652be66..ed64e186f96 100644
--- a/api/openapi/markdown/web-openapi.md
+++ b/api/openapi/markdown/web-openapi.md
@@ -1950,6 +1950,7 @@ Non-sensitive bootstrap snapshot exposed before Console or Web authentication.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
+| agent_tracing_api_collection_enabled | boolean | Deployment-level API gate for Agent trace collection, independent of workspace and provider configuration. Does not indicate Agent runtime, worker, or provider health or confirm trace delivery. | Yes |
| branding | [BrandingModel](#brandingmodel) | | Yes |
| deployment_edition | [DeploymentEdition](#deploymentedition) | | Yes |
| enable_app_deploy | boolean | | Yes |
diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.py
new file mode 100644
index 00000000000..ec1b1a39fb3
--- /dev/null
+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.py
@@ -0,0 +1,27 @@
+"""Aliyun LLM tracing export with synchronous OTLP/HTTP acknowledgment."""
+
+from collections.abc import Mapping
+from typing import Any
+
+from core.ops.unified_trace.entities import CanonicalTrace
+from dify_trace_aliyun.configuration import AliyunConfigurationError, ingestion_url
+from services.tracing.genai_attributes import attributes
+from services.tracing.otlp_export import build_request, export_http
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ try:
+ url = ingestion_url(config)
+ except AliyunConfigurationError:
+ raise TraceExportRejectedError("Aliyun tracing configuration is invalid") from None
+ payload = build_request(
+ trace,
+ {
+ "service.name": str(config.get("app_name") or "dify_app"),
+ "acs.arms.service.feature": "genai_app",
+ },
+ lambda span: attributes(span, trace, provider),
+ )
+ export_http(provider, url, {}, payload)
+ return TraceExportResult(trace_id=trace.trace_id)
diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py
index 717c1ac8765..5b7efaca491 100644
--- a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py
+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py
@@ -18,9 +18,9 @@ from core.ops.entities.trace_entity import (
)
from core.repositories import DifyCoreRepositoryFactory
from dify_trace_aliyun.config import AliyunConfig
+from dify_trace_aliyun.configuration import console_url, ingestion_url, verify_configuration
from dify_trace_aliyun.data_exporter.traceclient import (
TraceClient,
- build_endpoint,
convert_datetime_to_nanoseconds,
convert_to_span_id,
convert_to_trace_id,
@@ -90,38 +90,38 @@ class AliyunDataTrace(BaseTraceInstance):
aliyun_config: AliyunConfig,
):
super().__init__(aliyun_config)
- endpoint = build_endpoint(aliyun_config.endpoint, aliyun_config.license_key)
- self.trace_client = TraceClient(service_name=aliyun_config.app_name, endpoint=endpoint)
+ self._config = aliyun_config.model_copy(deep=True)
+ self.trace_client = TraceClient(
+ service_name=self._config.app_name, endpoint=ingestion_url(self._config.model_dump())
+ )
@override
def trace(self, trace_info: BaseTraceInfo):
- match trace_info:
- case WorkflowTraceInfo():
- self.workflow_trace(trace_info)
- case MessageTraceInfo():
- self.message_trace(trace_info)
- case ModerationTraceInfo():
- pass
- case SuggestedQuestionTraceInfo():
- self.suggested_question_trace(trace_info)
- case DatasetRetrievalTraceInfo():
- self.dataset_retrieval_trace(trace_info)
- case ToolTraceInfo():
- self.tool_trace(trace_info)
- case GenerateNameTraceInfo():
- pass
- case _:
- pass
-
- def api_check(self):
- return self.trace_client.api_check()
+ with self.trace_client.batch():
+ match trace_info:
+ case WorkflowTraceInfo():
+ self.workflow_trace(trace_info)
+ case MessageTraceInfo():
+ self.message_trace(trace_info)
+ case ModerationTraceInfo():
+ pass
+ case SuggestedQuestionTraceInfo():
+ self.suggested_question_trace(trace_info)
+ case DatasetRetrievalTraceInfo():
+ self.dataset_retrieval_trace(trace_info)
+ case ToolTraceInfo():
+ self.tool_trace(trace_info)
+ case GenerateNameTraceInfo():
+ pass
+ case _:
+ pass
+
+ def api_check(self) -> bool:
+ verify_configuration(self._config)
+ return True
def get_project_url(self):
- try:
- return self.trace_client.get_project_url()
- except Exception as e:
- logger.info("Aliyun get project url failed: %s", str(e), exc_info=True)
- raise ValueError(f"Aliyun get project url failed: {str(e)}") from e
+ return console_url()
def _extract_app_id(self, trace_info: BaseTraceInfo) -> str:
"""Extract app_id from trace_info, trying metadata first then message_data."""
diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py
index e0133e6cc92..bae1c06d16d 100644
--- a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py
+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py
@@ -1,4 +1,4 @@
-from pydantic import ValidationInfo, field_validator
+from pydantic import ConfigDict, ValidationInfo, field_validator
from core.ops.entities.config_entity import BaseTracingConfig
from core.ops.utils import validate_url_with_path
@@ -9,6 +9,10 @@ class AliyunConfig(BaseTracingConfig):
Model class for Aliyun tracing config.
"""
+ # Endpoints may be pasted with an embedded LicenseKey. Even rejected input
+ # must not appear in formatted validation exceptions or chained API errors.
+ model_config = ConfigDict(hide_input_in_errors=True)
+
app_name: str = "dify_app"
license_key: str
endpoint: str
diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.py
new file mode 100644
index 00000000000..ca7c9bf689c
--- /dev/null
+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.py
@@ -0,0 +1,158 @@
+"""Aliyun destinations and a resource-only, zero-span collector access check.
+
+GET and URL helpers are pure. Verification makes one explicit collector request;
+it never constructs a tracer, creates span IDs, or exports business telemetry.
+"""
+
+import re
+from collections.abc import Mapping
+from typing import Literal
+from urllib.parse import quote, unquote, urlsplit, urlunsplit
+
+from dify_trace_aliyun.config import AliyunConfig
+
+type ConfigurationCode = Literal["invalid_endpoint", "invalid_app_name", "invalid_license_key", "verification_failed"]
+_MESSAGES: dict[ConfigurationCode, str] = {
+ "invalid_endpoint": "Aliyun requires an HTTP(S) base URL without credentials or a collector suffix.",
+ "invalid_app_name": "Aliyun requires an application name without control characters.",
+ "invalid_license_key": "Aliyun requires a nonempty license key without whitespace or control characters.",
+ "verification_failed": "Aliyun tracing configuration could not be verified.",
+}
+_DEFAULT_ENDPOINT = "https://tracing-analysis-dc-hz.aliyuncs.com"
+
+
+class AliyunConfigurationError(ValueError):
+ """Diagnostics never include submitted URLs, credentials, or response bodies."""
+
+ def __init__(self, code: ConfigurationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+def _endpoint(value: object) -> str:
+ if not isinstance(value, str):
+ raise AliyunConfigurationError("invalid_endpoint")
+ value = value.strip(" ") or _DEFAULT_ENDPOINT
+ try:
+ parsed = urlsplit(value)
+ host, port = parsed.hostname, parsed.port
+ path = parsed.path.rstrip("/")
+ # Decode only the public path to detect hidden collector routes. License
+ # keys are opaque values and must never be unquoted or otherwise altered.
+ decoded_path = unquote(path, errors="strict")
+ valid = (
+ parsed.scheme in {"http", "https"}
+ and host
+ and parsed.username is None
+ and parsed.password is None
+ and "?" not in value
+ and "#" not in value
+ and "\\" not in value
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ and not parsed.netloc.endswith(":")
+ and (port is None or 1 <= port <= 65535)
+ and "%" not in host
+ and not re.search(r"%(?![0-9a-fA-F]{2})", path)
+ and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)
+ and not any(char in decoded_path for char in "\\?#%")
+ and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)
+ and "//" not in parsed.path
+ and not any(part in {".", ".."} or part.lower().startswith("adapt_") for part in decoded_path.split("/"))
+ and not decoded_path.lower().endswith(("/v1/traces", "/api/otlp/traces"))
+ )
+ if not valid or host is None:
+ raise AliyunConfigurationError("invalid_endpoint")
+ host = host.encode("idna").decode("ascii").lower()
+ authority = f"[{host}]" if ":" in host else host
+ if port is not None:
+ authority = f"{authority}:{port}"
+ return urlunsplit((parsed.scheme, authority, path, "", ""))
+ except (UnicodeError, ValueError):
+ raise AliyunConfigurationError("invalid_endpoint") from None
+
+
+def _license_key(value: object) -> str:
+ if (
+ not isinstance(value, str)
+ or not value
+ or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
+ ):
+ raise AliyunConfigurationError("invalid_license_key")
+ try:
+ value.encode("utf-8")
+ except UnicodeError:
+ raise AliyunConfigurationError("invalid_license_key") from None
+ return value
+
+
+def normalize_configuration(values: Mapping[str, object]) -> AliyunConfig:
+ """Validate new writes without tightening the model used for historical GET.
+
+ Masked keys survive this phase so the gateway can restore the original
+ ciphertext on PATCH. Exporters reject masks after that restoration.
+ """
+ name = values.get("app_name", "dify_app")
+ if not isinstance(name, str) or any(ord(char) < 32 or ord(char) == 127 for char in name):
+ raise AliyunConfigurationError("invalid_app_name")
+ return AliyunConfig(
+ app_name=name.strip() or "dify_app",
+ license_key=_license_key(values.get("license_key")),
+ endpoint=_endpoint(values.get("endpoint")),
+ )
+
+
+def ingestion_url(config: Mapping[str, object]) -> str:
+ """Build one destination, encoding the raw license exactly once as a path segment."""
+ normalized = normalize_configuration(config)
+ if "*" in normalized.license_key:
+ raise AliyunConfigurationError("invalid_license_key")
+ hostname = (urlsplit(normalized.endpoint).hostname or "").removesuffix(".")
+ is_sls = hostname == "log.aliyuncs.com" or hostname.endswith(".log.aliyuncs.com")
+ suffix = "api/v1/traces" if is_sls else "api/otlp/traces"
+ # Aliyun's documented path-form token contains literal '@'. RFC 3986 path
+ # segments permit it, and this collector does not treat '%40' equivalently.
+ # Preserve only that required reserved character; never unquote raw keys.
+ return f"{normalized.endpoint}/adapt_{quote(normalized.license_key, safe='@')}/{suffix}"
+
+
+def console_url() -> str:
+ """Return the platform console, not a destination-specific project or trace link."""
+ return "https://arms.console.aliyun.com/#/llm"
+
+
+def verify_configuration(config: AliyunConfig) -> None:
+ """Check access with one resource envelope and strictly zero scopes/spans.
+
+ This collector distinguishes valid from invalid licenses for this request;
+ HEAD 405 and a completely empty request are not verification. A successful
+ acknowledgment does not establish business-span retention or queryability.
+ """
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+
+ from services.tracing.otlp_export import is_aliyun_http_success_ack
+ from services.tracing.provider_common import request
+
+ normalized = normalize_configuration(config.model_dump())
+ url = ingestion_url(normalized.model_dump())
+ payload = ExportTraceServiceRequest()
+ resource = payload.resource_spans.add().resource
+ resource.attributes.add(key="service.name").value.string_value = normalized.app_name
+ try:
+ response = request(
+ "POST",
+ url,
+ "aliyun",
+ headers={"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"},
+ content=payload.SerializeToString(),
+ )
+ try:
+ # Only the account-tested resource-only ACK establishes this check.
+ # A generic empty OTLP ACK remains valid for ordinary export, but
+ # cannot be substituted as evidence that this license was checked.
+ if not is_aliyun_http_success_ack(response.status_code, response.content):
+ raise AliyunConfigurationError("verification_failed")
+ finally:
+ response.close()
+ except Exception:
+ # No URL, LicenseKey, vendor body, or nested transport exception escapes.
+ raise AliyunConfigurationError("verification_failed") from None
diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py
index 451fe662315..adbb9aefee6 100644
--- a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py
+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py
@@ -1,18 +1,15 @@
import hashlib
-import logging
import random
import socket
-import threading
import uuid
-from collections import deque
-from collections.abc import Sequence
+from collections.abc import Generator, Sequence
+from contextlib import contextmanager
+from contextvars import ContextVar
from datetime import datetime
from typing import Final
-from urllib.parse import urljoin
-import httpx
from opentelemetry import trace as trace_api
-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
@@ -26,28 +23,25 @@ from opentelemetry.semconv.attributes import service_attributes
from opentelemetry.trace import Link, SpanContext, TraceFlags
from configs import dify_config
+from dify_trace_aliyun.configuration import ingestion_url
from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData
from dify_trace_aliyun.entities.semconv import ACS_ARMS_SERVICE_FEATURE
+from services.tracing.otlp_export import export_http
INVALID_SPAN_ID: Final[int] = 0x0000000000000000
INVALID_TRACE_ID: Final[int] = 0x00000000000000000000000000000000
-DEFAULT_TIMEOUT: Final[int] = 5
-DEFAULT_MAX_QUEUE_SIZE: Final[int] = 1000
-DEFAULT_SCHEDULE_DELAY_SEC: Final[int] = 5
-DEFAULT_MAX_EXPORT_BATCH_SIZE: Final[int] = 50
-
-logger = logging.getLogger(__name__)
class TraceClient:
- def __init__(
- self,
- service_name: str,
- endpoint: str,
- max_queue_size: int = DEFAULT_MAX_QUEUE_SIZE,
- schedule_delay_sec: int = DEFAULT_SCHEDULE_DELAY_SEC,
- max_export_batch_size: int = DEFAULT_MAX_EXPORT_BATCH_SIZE,
- ):
+ """Collect one dispatch locally and synchronously acknowledge its exact batch.
+
+ Instances are cached, so pending spans belong to the current context rather
+ than the client. A mapper failure discards only that dispatch; an uncertain
+ response propagates to the durable task instead of silently retrying a batch.
+ No SDK exporter, process-global configuration or background queue is created.
+ """
+
+ def __init__(self, service_name: str, endpoint: str) -> None:
self.endpoint = endpoint
self.resource = Resource(
attributes={
@@ -59,80 +53,31 @@ class TraceClient:
}
)
self.span_builder = SpanBuilder(self.resource)
- self.exporter = OTLPSpanExporter(endpoint=endpoint)
-
- self.max_queue_size = max_queue_size
- self.schedule_delay_sec = schedule_delay_sec
- self.max_export_batch_size = max_export_batch_size
-
- self.queue: deque = deque(maxlen=max_queue_size)
- self.condition = threading.Condition(threading.Lock())
- self.done = False
+ self._dispatch_spans: ContextVar[list[ReadableSpan] | None] = ContextVar("aliyun_dispatch_spans", default=None)
- self.worker_thread = threading.Thread(target=self._worker, daemon=True)
- self.worker_thread.start()
-
- self._spans_dropped = False
-
- def export(self, spans: Sequence[ReadableSpan]):
- self.exporter.export(spans)
-
- def api_check(self) -> bool:
+ @contextmanager
+ def batch(self) -> Generator[None]:
+ """Export only after the whole mapper succeeds, and always restore context."""
+ spans: list[ReadableSpan] = []
+ token = self._dispatch_spans.set(spans)
try:
- response = httpx.head(self.endpoint, timeout=DEFAULT_TIMEOUT)
- if response.status_code == 405:
- return True
- else:
- logger.warning("AliyunTrace API check failed: Unexpected status code: %s", response.status_code)
- return False
- except httpx.RequestError as e:
- logger.warning("AliyunTrace API check failed", exc_info=True)
- raise ValueError(f"AliyunTrace API check failed: {str(e)}") from e
-
- def get_project_url(self) -> str:
- return "https://arms.console.aliyun.com/#/llm"
+ yield
+ if spans:
+ self.export(spans)
+ finally:
+ self._dispatch_spans.reset(token)
+
+ def export(self, spans: Sequence[ReadableSpan]) -> None:
+ if spans:
+ export_http("aliyun", self.endpoint, {}, encode_spans(spans))
def add_span(self, span_data: SpanData | None) -> None:
if span_data is None:
return
-
- span: ReadableSpan = self.span_builder.build_span(span_data)
- with self.condition:
- if len(self.queue) == self.max_queue_size:
- if not self._spans_dropped:
- logger.warning("Queue is full, likely spans will be dropped.")
- self._spans_dropped = True
-
- self.queue.appendleft(span)
- if len(self.queue) >= self.max_export_batch_size:
- self.condition.notify()
-
- def _worker(self) -> None:
- while not self.done:
- with self.condition:
- if len(self.queue) < self.max_export_batch_size and not self.done:
- self.condition.wait(timeout=self.schedule_delay_sec)
- self._export_batch()
-
- def _export_batch(self) -> None:
- spans_to_export: list[ReadableSpan] = []
- with self.condition:
- while len(spans_to_export) < self.max_export_batch_size and self.queue:
- spans_to_export.append(self.queue.pop())
-
- if spans_to_export:
- try:
- self.exporter.export(spans_to_export)
- except Exception as e:
- logger.warning("Error exporting spans: %s", e)
-
- def shutdown(self) -> None:
- with self.condition:
- self.done = True
- self.condition.notify_all()
- self.worker_thread.join()
- self._export_batch()
- self.exporter.shutdown()
+ spans = self._dispatch_spans.get()
+ if spans is None:
+ raise RuntimeError("Aliyun spans require an active trace dispatch")
+ spans.append(self.span_builder.build_span(span_data))
class SpanBuilder:
@@ -238,7 +183,5 @@ def convert_datetime_to_nanoseconds(start_time_a: datetime | None) -> int | None
def build_endpoint(base_url: str, license_key: str) -> str:
- if "log.aliyuncs.com" in base_url: # cms2.0 endpoint
- return urljoin(base_url, f"adapt_{license_key}/api/v1/traces")
- else: # xtrace endpoint
- return urljoin(base_url, f"adapt_{license_key}/api/otlp/traces")
+ """Compatibility wrapper around the shared, credential-safe URL builder."""
+ return ingestion_url({"endpoint": base_url, "license_key": license_key})
diff --git a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py
index 8c3a4f3e4b2..2d520a644bd 100644
--- a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py
+++ b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py
@@ -1,10 +1,7 @@
-import logging
-import time
import uuid
from datetime import datetime
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
-import httpx
import pytest
from dify_trace_aliyun.data_exporter.traceclient import (
INVALID_SPAN_ID,
@@ -19,207 +16,29 @@ from dify_trace_aliyun.data_exporter.traceclient import (
generate_span_id,
)
from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData
+from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import SpanKind, Status, StatusCode
-def trace_client_factory():
- """Factory fixture for creating TraceClient instances with automatic cleanup."""
- clients_to_shutdown = []
-
- def _factory(**kwargs):
- client = TraceClient(**kwargs)
- clients_to_shutdown.append(client)
- return client
-
- yield _factory
-
- # Cleanup: shutdown all created clients
- for client in clients_to_shutdown:
- client.shutdown()
-
-
class TestTraceClient:
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- @patch("dify_trace_aliyun.data_exporter.traceclient.socket.gethostname")
- def test_init(
- self, mock_gethostname: MagicMock, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]
- ):
- mock_gethostname.return_value = "test-host"
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
-
- assert client.endpoint == "http://test-endpoint"
- assert client.max_queue_size == 1000
- assert client.schedule_delay_sec == 5
- assert client.done is False
- assert client.worker_thread.is_alive()
-
- client.shutdown()
- # pyrefly: ignore [unnecessary-comparison]
- assert client.done is True
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_export(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):
- mock_exporter = mock_exporter_class.return_value
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
- spans = [ReadableSpan(name="test-span")]
- client.export(spans)
- mock_exporter.export.assert_called_once_with(spans)
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.httpx.head")
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_api_check_success(
- self, mock_exporter_class: MagicMock, mock_head: MagicMock, trace_client_factory: type[TraceClient]
- ):
- mock_response = MagicMock()
- mock_response.status_code = 405
- mock_head.return_value = mock_response
-
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
- assert client.api_check() is True
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.httpx.head")
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_api_check_failure_status(
- self, mock_exporter_class: MagicMock, mock_head: MagicMock, trace_client_factory: type[TraceClient]
- ):
- mock_response = MagicMock()
- mock_response.status_code = 500
- mock_head.return_value = mock_response
-
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
- assert client.api_check() is False
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.httpx.head")
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_api_check_exception(
- self, mock_exporter_class: MagicMock, mock_head: MagicMock, trace_client_factory: type[TraceClient]
- ):
- mock_head.side_effect = httpx.RequestError("Connection error")
-
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
- with pytest.raises(ValueError, match="AliyunTrace API check failed: Connection error"):
- client.api_check()
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_get_project_url(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
- assert client.get_project_url() == "https://arms.console.aliyun.com/#/llm"
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_add_span(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):
- client = trace_client_factory(
- service_name="test-service",
- endpoint="http://test-endpoint",
- max_export_batch_size=2,
- )
-
- # Test add None
+ def test_constructor_has_no_background_exporter(self) -> None:
+ client = TraceClient(service_name="test-service", endpoint="https://collector.test")
+ assert client.endpoint == "https://collector.test"
+ assert not hasattr(client, "exporter")
+ assert not hasattr(client, "worker_thread")
+ assert not hasattr(client, "queue")
+
+ def test_add_requires_dispatch(self) -> None:
+ client = TraceClient(service_name="test-service", endpoint="https://collector.test")
client.add_span(None)
- assert len(client.queue) == 0
-
- # Test add valid SpanData
- span_data = SpanData(
- name="test-span",
- trace_id=123,
- span_id=456,
- parent_span_id=None,
- start_time=1000,
- end_time=2000,
- status=Status(StatusCode.OK),
- span_kind=SpanKind.INTERNAL,
- )
-
- span = ReadableSpan(name="test-span")
- client.span_builder.build_span = MagicMock(return_value=span)
-
- with patch.object(client.condition, "notify") as mock_notify:
- client.add_span(span_data)
- assert len(client.queue) == 1
- mock_notify.assert_not_called()
-
- client.add_span(span_data)
- assert len(client.queue) == 2
- mock_notify.assert_called_once()
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_add_span_queue_full(
- self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient], caplog: pytest.LogCaptureFixture
- ):
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint", max_queue_size=1)
-
- span_data = SpanData(
- name="test-span",
- trace_id=123,
- span_id=456,
- parent_span_id=None,
- start_time=1000,
- end_time=2000,
- status=Status(StatusCode.OK),
- span_kind=SpanKind.INTERNAL,
- )
- span = ReadableSpan(name="test-span")
- client.span_builder.build_span = MagicMock(return_value=span)
-
- client.add_span(span_data)
- assert len(client.queue) == 1
-
- with caplog.at_level(logging.WARNING):
- client.add_span(span_data)
- assert len(client.queue) == 1
- assert "Queue is full, likely spans will be dropped." in caplog.text
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_export_batch_error(
- self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient], caplog: pytest.LogCaptureFixture
- ):
- mock_exporter = mock_exporter_class.return_value
- mock_exporter.export.side_effect = Exception("Export failed")
-
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
- span = ReadableSpan(name="test-span")
- client.queue.append(span)
-
- with caplog.at_level(logging.WARNING):
- client._export_batch()
- assert "Error exporting spans" in caplog.text
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_worker_loop(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):
- # We need to test the wait timeout in _worker
- # But _worker runs in a thread. Let's mock condition.wait.
- client = trace_client_factory(
- service_name="test-service",
- endpoint="http://test-endpoint",
- schedule_delay_sec=1,
- )
-
- with patch.object(client.condition, "wait") as mock_wait:
- # Let it run for a bit then shut down
- time.sleep(0.2)
- client.shutdown()
- # mock_wait might have been called
- assert mock_wait.called or client.done
-
- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")
- def test_shutdown_flushes(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):
- mock_exporter = mock_exporter_class.return_value
- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")
-
- span = ReadableSpan(name="test-span")
- client.queue.append(span)
-
- client.shutdown()
- # Should have called export twice (once in worker/export_batch, once in shutdown)
- # or at least once if worker was waiting
- assert mock_exporter.export.called
- assert mock_exporter.shutdown.called
+ with pytest.raises(RuntimeError, match="active trace dispatch"):
+ client.add_span(SpanData(name="span", trace_id=123, span_id=456, start_time=1000, end_time=2000))
class TestSpanBuilder:
def test_build_span(self):
- resource = MagicMock()
+ resource = Resource({"service.name": "test"})
builder = SpanBuilder(resource)
span_data = SpanData(
@@ -248,7 +67,7 @@ class TestSpanBuilder:
assert span.attributes == {"attr1": "val1"}
def test_build_span_no_parent(self):
- resource = MagicMock()
+ resource = Resource({"service.name": "test"})
builder = SpanBuilder(resource)
span_data = SpanData(
diff --git a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py
index 4dce71a1cae..25ffc570c9c 100644
--- a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py
+++ b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py
@@ -1,16 +1,18 @@
from __future__ import annotations
import json
-import logging
+from contextlib import nullcontext
from datetime import UTC, datetime
from types import SimpleNamespace
-from typing import cast
+from typing import Any, cast
from unittest.mock import MagicMock
import dify_trace_aliyun.aliyun_trace as aliyun_trace_module
+import httpx
import pytest
from dify_trace_aliyun.aliyun_trace import AliyunDataTrace
from dify_trace_aliyun.config import AliyunConfig
+from dify_trace_aliyun.configuration import AliyunConfigurationError
from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData, TraceMetadata
from dify_trace_aliyun.entities.semconv import (
DIFY_NODE_TYPE,
@@ -44,8 +46,10 @@ from dify_trace_aliyun.entities.semconv import (
TOOL_TYPE_FUNCTION,
GenAISpanKind,
)
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
from opentelemetry.trace import Link, SpanContext, SpanKind, Status, StatusCode, TraceFlags
+from core.helper import ssrf_proxy
from core.ops.entities.trace_entity import (
DatasetRetrievalTraceInfo,
GenerateNameTraceInfo,
@@ -65,6 +69,9 @@ class RecordingTraceClient:
self.endpoint = endpoint
self.added_spans: list[object] = []
+ def batch(self) -> nullcontext[None]:
+ return nullcontext()
+
def add_span(self, span: object) -> None:
self.added_spans.append(span)
@@ -213,7 +220,6 @@ def _make_suggested_question_trace_info(**overrides) -> SuggestedQuestionTraceIn
@pytest.fixture
def trace_instance(monkeypatch: pytest.MonkeyPatch) -> AliyunDataTrace:
- monkeypatch.setattr(aliyun_trace_module, "build_endpoint", lambda base_url, license_key: "built-endpoint")
monkeypatch.setattr(aliyun_trace_module, "TraceClient", RecordingTraceClient)
# Mock get_service_account_with_tenant to avoid DB errors
monkeypatch.setattr(AliyunDataTrace, "get_service_account_with_tenant", lambda self, app_id: MagicMock())
@@ -223,17 +229,12 @@ def trace_instance(monkeypatch: pytest.MonkeyPatch) -> AliyunDataTrace:
return trace
-def test_init_builds_endpoint_and_client(monkeypatch: pytest.MonkeyPatch):
- build_endpoint = MagicMock(return_value="built")
- trace_client_cls = MagicMock()
- monkeypatch.setattr(aliyun_trace_module, "build_endpoint", build_endpoint)
- monkeypatch.setattr(aliyun_trace_module, "TraceClient", trace_client_cls)
-
- config = AliyunConfig(app_name="my-app", license_key="license", endpoint="https://example.com")
+def test_init_builds_endpoint_and_client() -> None:
+ config = AliyunConfig(app_name="my-app", license_key="license", endpoint="https://example.com/prefix")
trace = AliyunDataTrace(config)
-
- build_endpoint.assert_called_once_with("https://example.com", "license")
- trace_client_cls.assert_called_once_with(service_name="my-app", endpoint="built")
+ config.license_key = "mutated"
+ assert trace.trace_client.endpoint == "https://example.com/prefix/adapt_license/api/otlp/traces"
+ assert trace.trace_client.resource.attributes["service.name"] == "my-app"
assert trace.trace_config == config
@@ -269,25 +270,48 @@ def test_trace_dispatches_to_correct_methods(trace_instance: AliyunDataTrace, mo
trace_instance.trace(GenerateNameTraceInfo(tenant_id="t", metadata={}))
-def test_api_check_delegates(trace_instance: AliyunDataTrace):
- trace_instance.trace_client.api_check = MagicMock(return_value=False)
- assert trace_instance.api_check() is False
+ ("status", "body", "verified"),
+ [
+ (200, b'"success"', True),
+ (200, b"", False),
+ (200, b"{}", False),
+ (403, b"DO_NOT_EXPOSE_BODY", False),
+ (405, b"", False),
+ ],
+)
+def test_api_check_verifies_credentials_with_zero_spans(
+ trace_instance: AliyunDataTrace, monkeypatch: pytest.MonkeyPatch, status: int, body: bytes, verified: bool
+) -> None:
+ sent: list[ExportTraceServiceRequest] = []
+
+ def send(method: str, url: str, **options: Any) -> httpx.Response:
+ assert method == "POST"
+ assert url == "https://example.com/adapt_k/api/otlp/traces"
+ assert options["max_retries"] == 0
+ assert options["follow_redirects"] is False
+ payload = ExportTraceServiceRequest()
+ payload.ParseFromString(options["content"])
+ sent.append(payload)
+ return httpx.Response(status, content=body)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ if verified:
+ assert trace_instance.api_check() is True
+ else:
+ with pytest.raises(AliyunConfigurationError) as error:
+ trace_instance.api_check()
+ assert "DO_NOT_EXPOSE_BODY" not in str(error.value)
+ assert "adapt_" not in str(error.value)
+ assert len(sent) == 1
+ assert len(sent[0].resource_spans) == 1
+ resource = sent[0].resource_spans[0]
+ assert len(resource.scope_spans) == 0
+ assert {item.key: item.value.string_value for item in resource.resource.attributes} == {"service.name": "app"}
def test_get_project_url_success(trace_instance: AliyunDataTrace):
- assert trace_instance.get_project_url() == "project-url"
-
-
-def test_get_project_url_error(
- trace_instance: AliyunDataTrace, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
-):
- monkeypatch.setattr(trace_instance.trace_client, "get_project_url", MagicMock(side_effect=Exception("boom")))
-
- caplog.set_level(logging.INFO, logger=aliyun_trace_module.logger.name)
- with pytest.raises(ValueError, match=r"Aliyun get project url failed: boom"):
- trace_instance.get_project_url()
-
- assert "Aliyun get project url failed: boom" in caplog.text
+ assert trace_instance.get_project_url() == "https://arms.console.aliyun.com/#/llm"
def test_workflow_trace_adds_workflow_and_node_spans(trace_instance: AliyunDataTrace, monkeypatch: pytest.MonkeyPatch):
diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.py
new file mode 100644
index 00000000000..7100d76615e
--- /dev/null
+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.py
@@ -0,0 +1,54 @@
+"""Stable-ID OpenInference mapping for Phoenix HTTP and Arize gRPC ingestion."""
+
+from collections.abc import Mapping
+from typing import Any
+
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalTrace
+from services.tracing.otlp_export import build_request, export_grpc, export_http
+from services.tracing.provider_common import endpoint, json_value, token_usage
+from services.tracing.provider_dispatch import TraceExportResult
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ url = endpoint(
+ config.get("endpoint", "https://otlp.arize.com" if provider == "arize" else "https://app.phoenix.arize.com")
+ )
+ api_key = str(config.get("api_key") or "")
+ project = str(config.get("project") or "default")
+
+ def attributes(span: CanonicalSpan) -> dict[str, object]:
+ result: dict[str, object] = {
+ "openinference.span.kind": span.kind.value.upper(),
+ "input.value": json_value(span.inputs),
+ "input.mime_type": "application/json",
+ "output.value": json_value(span.outputs),
+ "output.mime_type": "application/json",
+ "metadata": json_value(span.metadata),
+ "session.id": trace.session_id,
+ "dify.span.id": span.id,
+ }
+ if span.kind is CanonicalSpanKind.LLM:
+ model = span.metadata.get("model") or span.metadata.get("model_name")
+ if model:
+ result["llm.model_name"] = str(model)
+ if span.metadata.get("model_provider"):
+ result["llm.provider"] = str(span.metadata["model_provider"])
+ names = {"input": "prompt", "output": "completion", "total": "total"}
+ result.update({f"llm.token_count.{names[key]}": value for key, value in token_usage(span).items()})
+ elif span.kind is CanonicalSpanKind.TOOL:
+ result["tool.name"] = span.name
+ result["tool.parameters"] = json_value(span.inputs)
+ return result
+
+ payload = build_request(
+ trace,
+ {"service.name": "dify-application", "openinference.project.name": project, "model_id": project},
+ attributes,
+ )
+ headers = {"api_key": api_key, "authorization": f"Bearer {api_key}"} if api_key or provider == "arize" else {}
+ if provider == "arize":
+ headers["space_id"] = str(config.get("space_id") or "")
+ export_grpc(provider, url, headers, payload)
+ else:
+ export_http(provider, f"{url}/v1/traces", headers, payload)
+ return TraceExportResult(trace_id=trace.trace_id)
diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.py
new file mode 100644
index 00000000000..32053dc40f4
--- /dev/null
+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.py
@@ -0,0 +1,304 @@
+"""Optional Arize Cloud project metadata, resolved only on explicit config writes.
+
+Ingestion verification is separate from metadata reads. A key without read access
+may still ingest successfully, so an unavailable project link is never a reason
+to reject that configuration. Readers use only the frozen destination cache.
+"""
+
+import hashlib
+import json
+import re
+import time
+from collections.abc import Mapping
+from typing import Literal
+from urllib.parse import quote, urlsplit
+
+from pydantic import JsonValue, TypeAdapter
+
+from core.helper import ssrf_proxy
+from dify_trace_arize_phoenix.config import ArizeConfig
+
+DESTINATION_KEY = "arize_project_destination"
+_API_BASE = "https://api.arize.com/v2"
+_UI_BASE = "https://app.arize.com"
+_BUDGET_SECONDS = 5.0
+_MAX_RESPONSE_BYTES = 1024 * 1024
+_PAGE_LIMIT = 100
+_MAX_PAGES = 3
+_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])
+_VERIFY_TIMEOUT_SECONDS = 5.0
+
+type VerificationCode = Literal[
+ "invalid_configuration", "authentication_failed", "permission_denied", "connection_failed", "unexpected_response"
+]
+
+
+class ArizeConfigurationError(ValueError):
+ """Fixed diagnostics never contain credentials, vendor details or endpoints."""
+
+ def __init__(self, code: VerificationCode) -> None:
+ self.code = code
+ super().__init__(f"Arize ingestion verification failed: {code}")
+
+
+def verify_configuration(config: ArizeConfig) -> None:
+ """Require one synchronous OTLP ACK, without creating any span or project.
+
+ The default Arize collector was verified to authenticate this resource-only
+ request for both valid and invalid credentials. It establishes this ingest
+ protocol's access, not REST read permission or a project's eventual creation.
+ No SDK exporter, span processor or global/environment credentials are used.
+ """
+ import grpc # type: ignore[import-untyped]
+ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub
+
+ try:
+ parsed = urlsplit(config.endpoint)
+ host, port = parsed.hostname, parsed.port
+ if (
+ parsed.scheme not in {"http", "https"}
+ or not host
+ or parsed.username is not None
+ or parsed.password is not None
+ or (port is not None and not 1 <= port <= 65535)
+ ):
+ raise ArizeConfigurationError("invalid_configuration")
+ for value in (config.api_key, config.space_id):
+ if (
+ not isinstance(value, str)
+ or not value
+ or "*" in value
+ or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)
+ ):
+ raise ArizeConfigurationError("invalid_configuration")
+ except ValueError:
+ raise ArizeConfigurationError("invalid_configuration") from None
+
+ payload = ExportTraceServiceRequest()
+ resource = payload.resource_spans.add().resource
+ for name in ("model_id", "openinference.project.name"):
+ resource.attributes.add(key=name).value.string_value = config.project or "default"
+ headers = (
+ ("api_key", config.api_key),
+ ("space_id", config.space_id),
+ ("authorization", f"Bearer {config.api_key}"),
+ )
+ authority = f"[{host}]" if ":" in host else host
+ target = f"{authority}:{port or 443}"
+ options = (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", _MAX_RESPONSE_BYTES))
+ channel: grpc.Channel | None = None
+ context = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
+ try:
+ try:
+ channel = (
+ grpc.insecure_channel(target, options=options)
+ if parsed.scheme == "http"
+ else grpc.secure_channel(target, grpc.ssl_channel_credentials(), options=options)
+ )
+ result = TraceServiceStub(channel).Export(
+ payload, metadata=headers, timeout=_VERIFY_TIMEOUT_SECONDS, wait_for_ready=False
+ )
+ if result.partial_success.rejected_spans or result.partial_success.error_message:
+ raise ArizeConfigurationError("unexpected_response")
+ finally:
+ if channel is not None:
+ channel.close()
+ except ArizeConfigurationError:
+ raise
+ except grpc.RpcError as error:
+ code = error.code()
+ if code is grpc.StatusCode.UNAUTHENTICATED:
+ raise ArizeConfigurationError("authentication_failed") from None
+ if code is grpc.StatusCode.PERMISSION_DENIED:
+ raise ArizeConfigurationError("permission_denied") from None
+ raise ArizeConfigurationError("connection_failed") from None
+ except Exception:
+ raise ArizeConfigurationError("connection_failed") from None
+ finally:
+ detach(context)
+
+
+def _opaque_id(value: object) -> bool:
+ # IDs remain opaque. In particular, never derive organization IDs by
+ # decoding the project or space IDs, or accept path/query metacharacters.
+ return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9+/_=-]{1,1024}", value) is not None
+
+
+def _identity(values: Mapping[str, object]) -> dict[str, str] | None:
+ try:
+ config = ArizeConfig.model_validate(
+ {key: values[key] for key in ("endpoint", "space_id", "project") if key in values}
+ )
+ endpoint = urlsplit(config.endpoint)
+ # These fixed API/UI hosts belong to the documented default collector.
+ # Custom or other regional collectors keep ingestion support, but must
+ # not send their credentials to an assumed Cloud metadata destination.
+ if (
+ endpoint.scheme != "https"
+ or endpoint.hostname != "otlp.arize.com"
+ or endpoint.port not in (None, 443)
+ or endpoint.username is not None
+ or endpoint.password is not None
+ or not _opaque_id(config.space_id)
+ ):
+ return None
+ except ValueError:
+ return None
+ assert config.space_id is not None
+ return {"endpoint": config.endpoint, "space_id": config.space_id, "project": config.project or "default"}
+
+
+def project_url(values: Mapping[str, object]) -> str | None:
+ """Compute only a verified cached destination; never perform vendor I/O."""
+ destination = values.get(DESTINATION_KEY)
+ if not isinstance(destination, Mapping):
+ return None
+ identity = _identity(destination)
+ # Legacy Arize encrypts space_id as well as api_key. Bind the public cache
+ # to this exact persisted record without decrypting it on status reads.
+ digest = _source_digest(values)
+ if identity is None or digest is None or destination.get("source_digest") != digest:
+ return None
+ organization_id, project_id = destination.get("organization_id"), destination.get("project_id")
+ if not _opaque_id(organization_id) or not _opaque_id(project_id):
+ return None
+ assert isinstance(organization_id, str)
+ assert isinstance(project_id, str)
+ return (
+ f"{_UI_BASE}/organizations/{quote(organization_id, safe='')}"
+ f"/spaces/{quote(identity['space_id'], safe='')}/projects/{quote(project_id, safe='')}"
+ )
+
+
+def _source_digest(values: Mapping[str, object]) -> str | None:
+ parts = [values.get(key) for key in ("endpoint", "space_id", "project")]
+ if not all(isinstance(part, str) for part in parts):
+ return None
+ return hashlib.sha256(json.dumps(parts, ensure_ascii=True, separators=(",", ":")).encode()).hexdigest()
+
+
+def bind_destination(values: Mapping[str, object], destination: dict[str, str]) -> dict[str, str]:
+ """Bind server-resolved metadata after legacy secret encryption has finished."""
+ digest = _source_digest(values)
+ if digest is None:
+ raise ValueError("Arize persisted destination is incomplete")
+ return {**destination, "source_digest": digest}
+
+
+def _get_json(
+ path: str, headers: dict[str, str], deadline: float, *, params: dict[str, str | int] | None = None
+) -> dict[str, JsonValue] | None:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return None
+ try:
+ response = ssrf_proxy.get(
+ _API_BASE + path,
+ headers=headers,
+ params=params,
+ auth=None,
+ timeout=remaining,
+ max_retries=0,
+ follow_redirects=False,
+ stream_response=True,
+ )
+ try:
+ if response.status_code != 200 or time.monotonic() >= deadline:
+ return None
+ encoding = response.headers.get("content-encoding", "identity").strip().lower()
+ if encoding not in {"", "identity"}:
+ return None
+ content = bytearray()
+ for chunk in response.iter_bytes():
+ if time.monotonic() >= deadline or len(content) + len(chunk) > _MAX_RESPONSE_BYTES:
+ return None
+ content.extend(chunk)
+ if time.monotonic() >= deadline:
+ return None
+ return _JSON_OBJECT.validate_json(content, strict=True)
+ finally:
+ response.close()
+ except Exception:
+ # Metadata is optional. Do not expose request headers, response bodies,
+ # credential-bearing exceptions, or turn read permission into an ingest
+ # permission requirement.
+ return None
+
+
+def resolve_project_destination(values: Mapping[str, object]) -> dict[str, str] | None:
+ """Resolve exact project/space ownership within one shared five-second budget.
+
+ The documented name filter is a substring match. Read at most three scoped
+ pages and require one exact name/space match. Missing organization IDs on
+ older servers, denied reads, malformed responses and budget exhaustion all
+ mean no optional link, without modifying ingestion configuration or history.
+ """
+ identity = _identity(values)
+ key = values.get("api_key")
+ if (
+ identity is None
+ or not isinstance(key, str)
+ or not key
+ or "*" in key
+ or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in key)
+ ):
+ return None
+ deadline = time.monotonic() + _BUDGET_SECONDS
+ headers = {
+ "Authorization": f"Bearer {key}",
+ "Accept": "application/json",
+ "Accept-Encoding": "identity",
+ "Cookie": "",
+ }
+ space = _get_json("/spaces/" + quote(identity["space_id"], safe=""), headers, deadline)
+ if space is None or space.get("id") != identity["space_id"] or not _opaque_id(space.get("organization_id")):
+ return None
+ organization_id = space["organization_id"]
+ assert isinstance(organization_id, str)
+ project_ids: set[str] = set()
+ seen_cursors: set[str] = set()
+ params: dict[str, str | int] = {
+ "space_id": identity["space_id"],
+ "name": identity["project"],
+ "limit": _PAGE_LIMIT,
+ }
+ for _ in range(_MAX_PAGES):
+ page = _get_json("/projects", headers, deadline, params=params)
+ if page is None:
+ return None
+ projects, pagination = page.get("projects"), page.get("pagination")
+ if not isinstance(projects, list) or len(projects) > _PAGE_LIMIT or not isinstance(pagination, dict):
+ return None
+ for project in projects:
+ if (
+ not isinstance(project, dict)
+ or not _opaque_id(project.get("id"))
+ or not isinstance(project.get("name"), str)
+ or project.get("space_id") != identity["space_id"]
+ ):
+ return None
+ if project["name"] == identity["project"]:
+ project_id = project["id"]
+ assert isinstance(project_id, str)
+ project_ids.add(project_id)
+ if len(project_ids) > 1:
+ return None
+ has_more = pagination.get("has_more")
+ if has_more is False:
+ if len(project_ids) != 1:
+ return None
+ return {**identity, "organization_id": organization_id, "project_id": next(iter(project_ids))}
+ cursor = pagination.get("next_cursor")
+ if (
+ has_more is not True
+ or not isinstance(cursor, str)
+ or not cursor
+ or len(cursor) > 4096
+ or cursor in seen_cursors
+ ):
+ return None
+ seen_cursors.add(cursor)
+ params = {**params, "cursor": cursor}
+ return None
diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py
index c9105574c0e..d5ab4a570a3 100644
--- a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py
+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py
@@ -209,10 +209,16 @@ def setup_tracer(arize_phoenix_config: ArizeConfig | PhoenixConfig) -> tuple[tra
)
else:
phoenix_endpoint = f"{base_endpoint}{path}/v1/traces"
- phoenix_headers = {
- "api_key": arize_phoenix_config.api_key or "",
- "authorization": f"Bearer {arize_phoenix_config.api_key or ''}",
- }
+ # Keep headers nonempty so the SDK does not fall back to ambient
+ # OTEL exporter credentials for an anonymous Phoenix destination.
+ phoenix_headers = {"Content-Type": "application/x-protobuf"}
+ if arize_phoenix_config.api_key:
+ phoenix_headers.update(
+ {
+ "api_key": arize_phoenix_config.api_key,
+ "authorization": f"Bearer {arize_phoenix_config.api_key}",
+ }
+ )
exporter = HttpOTLPSpanExporter(
endpoint=phoenix_endpoint,
headers=phoenix_headers,
@@ -1484,6 +1490,11 @@ class ArizePhoenixDataTrace(BaseTraceInstance):
return self.carrier
def api_check(self):
+ if isinstance(self.arize_phoenix_config, ArizeConfig):
+ from dify_trace_arize_phoenix.arize_configuration import verify_configuration
+
+ verify_configuration(self.arize_phoenix_config)
+ return True
try:
with self.tracer.start_span("api_check") as span:
span.set_attribute("test", "true")
@@ -1493,14 +1504,16 @@ class ArizePhoenixDataTrace(BaseTraceInstance):
raise ValueError(f"[Arize/Phoenix] API check failed: {str(e)}") from e
def get_project_url(self):
- """Build a redirect URL that forwards the user to the correct project for Arize/Phoenix."""
+ """Get a Phoenix redirect; Arize Cloud requires verified destination metadata."""
try:
project_name = self.arize_phoenix_config.project
endpoint = self.arize_phoenix_config.endpoint.rstrip("/")
- # Arize
+ # Arize Cloud links require verified organization/space/project IDs.
+ # Only explicit config writes resolve and cache them; this legacy
+ # instance has none of that metadata and cannot promise a project.
if isinstance(self.arize_phoenix_config, ArizeConfig):
- return f"https://app.arize.com/?redirect_project_name={project_name}"
+ return None
# Phoenix
return f"{endpoint}/projects/?redirect_project_name={project_name}"
diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/configuration.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/configuration.py
new file mode 100644
index 00000000000..8085c7462ea
--- /dev/null
+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/configuration.py
@@ -0,0 +1,182 @@
+"""Phoenix configuration checks never create a tracer, project, or test span.
+
+A bounded project-list GET establishes read access only, including on a keyless
+self-hosted server. It does not establish ingestion or project write permission.
+Legacy Arize/Phoenix runtime adapters remain separate owners.
+"""
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import quote, unquote, urlsplit, urlunsplit
+
+from pydantic import TypeAdapter
+
+from core.helper import ssrf_proxy
+from dify_trace_arize_phoenix.config import PhoenixConfig
+
+type VerificationCode = Literal[
+ "invalid_endpoint",
+ "invalid_project",
+ "invalid_api_key",
+ "authentication_failed",
+ "permission_denied",
+ "endpoint_unavailable",
+ "malformed_response",
+ "connection_failed",
+]
+_MESSAGES: dict[VerificationCode, str] = {
+ "invalid_endpoint": "Phoenix requires an HTTP(S) base URL, optionally with a path prefix, without /v1/traces.",
+ "invalid_project": "Phoenix requires a valid project name.",
+ "invalid_api_key": "Phoenix requires an API key without whitespace or mask placeholders, or no key.",
+ "authentication_failed": "Phoenix authentication failed.",
+ "permission_denied": "Phoenix project-list access was denied.",
+ "endpoint_unavailable": "Phoenix endpoint could not be verified.",
+ "malformed_response": "Phoenix verification returned an invalid response.",
+ "connection_failed": "Phoenix verification request failed.",
+}
+_JSON_OBJECT = TypeAdapter(dict[str, object])
+_MAX_RESPONSE_BYTES = 1024 * 1024
+_DEFAULT_ENDPOINT = "https://app.phoenix.arize.com"
+
+
+class PhoenixConfigurationError(ValueError):
+ """Safe diagnostics exclude credentials, request URLs, and vendor response bodies."""
+
+ def __init__(self, code: VerificationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+@dataclass(frozen=True, slots=True)
+class ReadablePhoenixEndpoint:
+ endpoint: str
+ project: str
+
+
+def _endpoint(value: object) -> str:
+ if not isinstance(value, str):
+ raise PhoenixConfigurationError("invalid_endpoint")
+ value = value.strip(" ") or _DEFAULT_ENDPOINT
+ try:
+ parsed = urlsplit(value)
+ host, port = parsed.hostname, parsed.port
+ path = parsed.path.rstrip("/")
+ decoded_path = unquote(path, errors="strict")
+ valid = (
+ parsed.scheme in {"http", "https"}
+ and host
+ and parsed.username is None
+ and parsed.password is None
+ and "?" not in value
+ and "#" not in value
+ and "\\" not in value
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ and not parsed.netloc.endswith(":")
+ and (port is None or 1 <= port <= 65535)
+ and "%" not in host
+ and not re.search(r"%(?![0-9a-fA-F]{2})", path)
+ and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)
+ and not any(char in decoded_path for char in "\\?#%")
+ and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)
+ and "//" not in parsed.path
+ and not any(part in {".", ".."} for part in decoded_path.split("/"))
+ and not decoded_path.endswith("/v1/traces")
+ )
+ if not valid or host is None:
+ raise PhoenixConfigurationError("invalid_endpoint")
+ host = host.encode("idna").decode("ascii").lower()
+ authority = f"[{host}]" if ":" in host else host
+ if port is not None:
+ authority = f"{authority}:{port}"
+ return urlunsplit((parsed.scheme, authority, path, "", ""))
+ except (UnicodeError, ValueError):
+ raise PhoenixConfigurationError("invalid_endpoint") from None
+
+
+def _project(value: object) -> str:
+ if value is None:
+ return "default"
+ if not isinstance(value, str) or any(ord(char) < 32 or ord(char) == 127 for char in value):
+ raise PhoenixConfigurationError("invalid_project")
+ result = value.strip() or "default"
+ if result in {".", ".."}:
+ raise PhoenixConfigurationError("invalid_project")
+ return result
+
+
+def normalize_configuration(values: Mapping[str, object]) -> PhoenixConfig:
+ """Normalize new writes without tightening the model used to read old records.
+
+ Secret contents are checked only after PATCH restores masked credentials.
+ The project redirect loader uses a GraphQL name variable, not REST's more
+ restrictive project-identifier path. Do not impose that REST limit here.
+ """
+ key = values.get("api_key")
+ if key is not None and not isinstance(key, str):
+ raise PhoenixConfigurationError("invalid_api_key")
+ return PhoenixConfig(
+ endpoint=_endpoint(values.get("endpoint", _DEFAULT_ENDPOINT)),
+ project=_project(values.get("project")),
+ api_key=key,
+ )
+
+
+def project_url(config: PhoenixConfig) -> str:
+ """Compute the configured UI redirect without SDK initialization or network I/O."""
+ return f"{_endpoint(config.endpoint)}/redirects/projects/{quote(_project(config.project), safe='')}"
+
+
+def verify_configuration(config: PhoenixConfig) -> ReadablePhoenixEndpoint:
+ """Read at most one page; an empty project list is valid and creates nothing."""
+ config = normalize_configuration(config.model_dump())
+ key = config.api_key
+ if key and ("*" in key or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in key)):
+ raise PhoenixConfigurationError("invalid_api_key")
+ headers = {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}
+ if key:
+ headers["Authorization"] = f"Bearer {key}"
+ try:
+ response = ssrf_proxy.get(
+ config.endpoint + "/v1/projects",
+ params={"limit": 1},
+ headers=headers,
+ auth=None,
+ timeout=30.0,
+ max_retries=0,
+ follow_redirects=False,
+ stream_response=True,
+ )
+ except Exception:
+ raise PhoenixConfigurationError("connection_failed") from None
+ try:
+ if response.status_code == 401:
+ raise PhoenixConfigurationError("authentication_failed")
+ if response.status_code == 403:
+ raise PhoenixConfigurationError("permission_denied")
+ if response.status_code != 200:
+ raise PhoenixConfigurationError("endpoint_unavailable")
+ try:
+ buffered = ssrf_proxy.buffer_response(response, max_response_bytes=_MAX_RESPONSE_BYTES)
+ data = _JSON_OBJECT.validate_python(buffered.json(), strict=True)
+ except (ValueError, TypeError):
+ raise PhoenixConfigurationError("malformed_response") from None
+ except Exception:
+ raise PhoenixConfigurationError("connection_failed") from None
+ finally:
+ response.close()
+ rows, cursor = data.get("data"), data.get("next_cursor")
+ if not isinstance(rows, list) or "next_cursor" not in data or (cursor is not None and not isinstance(cursor, str)):
+ raise PhoenixConfigurationError("malformed_response")
+ for row in rows:
+ if (
+ not isinstance(row, dict)
+ or not isinstance(row.get("id"), str)
+ or not row["id"]
+ or not isinstance(row.get("name"), str)
+ or not row["name"]
+ or (row.get("description") is not None and not isinstance(row["description"], str))
+ ):
+ raise PhoenixConfigurationError("malformed_response")
+ return ReadablePhoenixEndpoint(config.endpoint, _project(config.project))
diff --git a/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py b/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py
index 870a3df5b1e..6394aefafe5 100644
--- a/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py
+++ b/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py
@@ -2049,7 +2049,8 @@ def test_construct_llm_attributes_dict_branch(trace_instance):
assert attrs["llm.input_messages.0.message.role"] == "user"
-def test_api_check_success(trace_instance):
+def test_phoenix_api_check_success(trace_instance):
+ trace_instance.arize_phoenix_config = PhoenixConfig(project="test", endpoint="https://phoenix.test")
assert trace_instance.api_check() is True
diff --git a/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/agent_trace.py b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/agent_trace.py
new file mode 100644
index 00000000000..841bd1227c9
--- /dev/null
+++ b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/agent_trace.py
@@ -0,0 +1,75 @@
+"""Export complete Agent/Workflow trees through Langfuse v4 OTLP ingestion."""
+
+import base64
+from collections.abc import Mapping
+from datetime import timedelta
+from typing import Any
+from urllib.parse import quote
+
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.otlp_export import build_request, export_http
+from services.tracing.provider_common import endpoint, json_value, required, token_usage, usd_cost, utc
+from services.tracing.provider_dispatch import TraceExportResult
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ host = endpoint(config.get("host", "https://cloud.langfuse.com"))
+ public_key, secret_key = required(config, "public_key"), required(config, "secret_key")
+ auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode("ascii")
+ root = trace.spans[0]
+ project_url = config.get("project_url")
+ trace_url = f"{endpoint(project_url)}/traces/{trace.trace_id}" if project_url else None
+
+ def attributes(span: CanonicalSpan) -> dict[str, object]:
+ values: dict[str, object] = {
+ "langfuse.trace.name": root.name,
+ "langfuse.session.id": trace.session_id,
+ "langfuse.observation.type": "generation" if span.kind is CanonicalSpanKind.LLM else span.kind.value,
+ "langfuse.observation.input": json_value(span.inputs),
+ "langfuse.observation.output": json_value(span.outputs),
+ "langfuse.observation.level": "ERROR" if span.status is CanonicalSpanStatus.ERROR else "DEFAULT",
+ "dify.span.id": span.id,
+ }
+ for key, value in root.metadata.items():
+ if value is not None:
+ values[f"langfuse.trace.metadata.{key}"] = value if isinstance(value, str) else json_value(value)
+ for key, value in span.metadata.items():
+ if value is not None:
+ values[f"langfuse.observation.metadata.{key}"] = value if isinstance(value, str) else json_value(value)
+ user_id = span.metadata.get("user_id") or root.metadata.get("user_id")
+ if user_id:
+ values["langfuse.user.id"] = str(user_id)
+ if span.error:
+ values["langfuse.observation.status_message"] = span.error
+ if span.kind is CanonicalSpanKind.LLM:
+ model = span.metadata.get("model") or span.metadata.get("model_name")
+ if model:
+ values["langfuse.observation.model.name"] = str(model)
+ usage = token_usage(span)
+ if usage:
+ values["langfuse.observation.usage_details"] = json_value(usage)
+ cost = usd_cost(span)
+ if cost is not None:
+ values["langfuse.observation.cost_details"] = json_value({"total": cost})
+ ttft = span.metadata.get("time_to_first_token", span.metadata.get("ttft"))
+ if isinstance(ttft, (int, float)) and ttft >= 0:
+ values["langfuse.observation.completion_start_time"] = (
+ utc(span.start_time) + timedelta(seconds=ttft)
+ ).isoformat()
+ return values
+
+ payload = build_request(trace, {"service.name": "dify-application"}, attributes)
+ export_http(
+ provider,
+ f"{host}/api/public/otel/v1/traces",
+ {
+ "Authorization": f"Basic {auth}",
+ "x-langfuse-ingestion-version": "4",
+ },
+ payload,
+ )
+ return TraceExportResult(
+ trace_id=trace.trace_id,
+ trace_url=trace_url,
+ span_urls={span.id: f"{trace_url}?observation={quote(span.id)}" for span in trace.spans} if trace_url else {},
+ )
diff --git a/api/providers/trace/trace-langsmith/src/dify_trace_langsmith/agent_trace.py b/api/providers/trace/trace-langsmith/src/dify_trace_langsmith/agent_trace.py
new file mode 100644
index 00000000000..0f0be6c5e64
--- /dev/null
+++ b/api/providers/trace/trace-langsmith/src/dify_trace_langsmith/agent_trace.py
@@ -0,0 +1,105 @@
+"""Canonical Agent trees mapped to synchronous, isolated LangSmith runs."""
+
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import quote
+from uuid import NAMESPACE_URL, uuid5
+
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from core.ops.utils import generate_dotted_order
+from services.tracing.provider_common import endpoint, finished_at, required, token_usage, usd_cost, utc
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError
+from services.tracing.sdk_transport import SSRFSession
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ from langsmith import Client
+ from urllib3.util.retry import Retry
+
+ api_url = endpoint(config.get("endpoint", "https://api.smith.langchain.com"))
+ project = required(config, "project")
+ ids = {span.id: str(uuid5(NAMESPACE_URL, f"dify:{trace.trace_id}:{span.id}")) for span in trace.spans}
+ root_id = ids[trace.root_span_id]
+ orders: dict[str, str] = {}
+ accepted: set[str] = set()
+ client = Client(
+ api_key=required(config, "api_key"),
+ api_url=api_url,
+ session=SSRFSession(provider),
+ retry_config=Retry(total=0),
+ auto_batch_tracing=False,
+ tracing_sampling_rate=1.0,
+ tracing_mode="langsmith",
+ omit_traced_runtime_info=True,
+ hide_inputs=False,
+ hide_outputs=False,
+ info={},
+ )
+ try:
+ for span in trace.spans:
+ orders[span.id] = generate_dotted_order(
+ ids[span.id], utc(span.start_time), orders.get(span.parent_id or "")
+ )
+ metadata = {
+ **span.metadata,
+ "session_id": trace.session_id,
+ "external_trace_id": trace.trace_id,
+ "dify.span.id": span.id,
+ "dify.span.kind": span.kind.value,
+ }
+ model = span.metadata.get("model") or span.metadata.get("model_name")
+ if span.kind is CanonicalSpanKind.LLM and model:
+ metadata["ls_model_name"] = model
+ metadata["ls_provider"] = span.metadata.get("model_provider")
+ values: dict[str, Any] = {}
+ usage = token_usage(span)
+ values.update(
+ {f"{key}_tokens" if key != "total" else "total_tokens": value for key, value in usage.items()}
+ )
+ cost = usd_cost(span)
+ if cost is not None:
+ values["total_cost"] = cost
+ outputs = dict(span.outputs) if isinstance(span.outputs, Mapping) else {"output": span.outputs}
+ if usage:
+ outputs["usage_metadata"] = {f"{key}_tokens": value for key, value in usage.items()}
+ client.create_run(
+ id=ids[span.id],
+ name=span.name,
+ inputs=dict(span.inputs) if isinstance(span.inputs, Mapping) else {"input": span.inputs},
+ outputs=outputs,
+ run_type="chain" if span.kind is CanonicalSpanKind.AGENT else span.kind.value,
+ project_name=project,
+ start_time=utc(span.start_time),
+ end_time=finished_at(span),
+ parent_run_id=ids.get(span.parent_id or ""),
+ trace_id=root_id,
+ dotted_order=orders[span.id],
+ extra={"metadata": metadata},
+ error=(span.error or "Trace operation failed") if span.status is CanonicalSpanStatus.ERROR else None,
+ **values,
+ )
+ accepted.add(span.id)
+ except Exception as error:
+ if accepted:
+ raise TraceExportUncertainError(
+ "LangSmith accepted part of the trace; reconciliation is required"
+ ) from None
+ # LangSmith wraps injected transport exceptions; recover our sanitized outcome.
+ cause: BaseException | None = error
+ while cause is not None:
+ if isinstance(cause, (RetryableTraceDispatchError, TraceExportRejectedError)):
+ raise cause from None
+ cause = cause.__cause__
+ raise TraceExportRejectedError("LangSmith rejected the trace") from None
+ finally:
+ client.close()
+ project_url = config.get("project_url")
+ trace_url = f"{str(project_url).rstrip('/')}/r/{quote(root_id)}" if project_url else None
+ return TraceExportResult(
+ trace_id=root_id,
+ trace_url=trace_url,
+ span_urls={span.id: f"{str(project_url).rstrip('/')}/r/{ids[span.id]}" for span in trace.spans}
+ if project_url
+ else {},
+ )
diff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/agent_trace.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/agent_trace.py
new file mode 100644
index 00000000000..3bb26942e9b
--- /dev/null
+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/agent_trace.py
@@ -0,0 +1,259 @@
+"""MLflow SDK span serialization with per-export credentials and bounded HTTP.
+
+OSS uses its OTLP experiment endpoint (MLflow >= 3.4). Databricks retains the
+existing experiment destination through the SDK's V3 trace and artifact APIs;
+its UC-only OTLP endpoint is not interchangeable with an experiment ID.
+"""
+
+import base64
+import json
+from collections.abc import Mapping
+from typing import Any, override
+from urllib.parse import quote, urlencode, urlsplit
+
+from google.protobuf.json_format import ParseDict, ParseError
+from mlflow.entities import Span, TraceData, TraceInfo, TraceLocation, TraceState
+from mlflow.protos.databricks_artifacts_pb2 import (
+ ArtifactCredentialType,
+ DatabricksMlflowArtifactsService,
+ GetCredentialsForTraceDataUpload,
+)
+from mlflow.store.tracking.rest_store import RestStore
+from mlflow.tracing.constant import SpanAttributeKey, TraceMetadataKey, TraceTagKey
+from mlflow.tracing.utils.otlp import MLFLOW_EXPERIMENT_ID_HEADER, OTLP_TRACES_PATH
+from mlflow.utils.rest_utils import MlflowHostCreds, extract_api_info_for_service
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.trace import SpanContext, Status, StatusCode
+
+from core.ops.unified_trace.entities import CanonicalSpanStatus, CanonicalTrace
+from services.tracing.otlp_export import export_http
+from services.tracing.provider_common import endpoint, json_value, nanos, request, required, token_usage, usd_cost
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError
+
+
+def _chat_tools(value: object) -> list[dict[str, Any]] | None:
+ """Map OTel's flat function definitions to MLflow's chat tool UI schema.
+
+ Preserve the original gen_ai attribute separately. An explicit native field
+ prevents MLflow's OTLP translator from copying the incompatible flat shape.
+ Invalid/unsupported entries stay in the source metadata, not in chat tools.
+ """
+ if isinstance(value, str):
+ try:
+ value = json.loads(value)
+ except ValueError:
+ return None
+ if not isinstance(value, list):
+ return None
+ tools = []
+ for item in value:
+ if not isinstance(item, dict) or item.get("type") != "function":
+ continue
+ function = item.get("function")
+ if not isinstance(function, dict):
+ function = {key: item[key] for key in ("name", "description", "parameters", "strict") if key in item}
+ name = function.get("name")
+ if not isinstance(name, str) or not name.strip():
+ continue
+ tool = {"type": "function", "function": function}
+ parameters = function.get("parameters")
+ if parameters is not None and not isinstance(parameters, dict):
+ continue
+ # MLflow's Python ChatTool model accepts only a subset of JSON Schema
+ # (e.g. it requires properties and restricts enum values). Preserve the
+ # full schema for the UI instead of silently dropping valid definitions.
+ tools.append(tool)
+ return tools
+
+
+def _spans(trace: CanonicalTrace, experiment_id: str, request_id: str) -> list[Span]:
+ result = []
+ for span in trace.spans:
+ attributes: dict[str, Any] = {
+ **span.metadata,
+ SpanAttributeKey.REQUEST_ID: request_id,
+ SpanAttributeKey.EXPERIMENT_ID: experiment_id,
+ SpanAttributeKey.SPAN_TYPE: span.kind.value.upper(),
+ SpanAttributeKey.SESSION_ID: trace.session_id,
+ }
+ if span.inputs is not None:
+ attributes[SpanAttributeKey.INPUTS] = span.inputs
+ if span.outputs is not None:
+ attributes[SpanAttributeKey.OUTPUTS] = span.outputs
+ if SpanAttributeKey.CHAT_TOOLS not in attributes:
+ tools = _chat_tools(span.metadata.get("gen_ai.tool.definitions"))
+ if tools is not None:
+ attributes[SpanAttributeKey.CHAT_TOOLS] = tools
+ if usage := token_usage(span):
+ attributes[SpanAttributeKey.CHAT_USAGE] = {f"{key}_tokens": value for key, value in usage.items()}
+ if (cost := usd_cost(span)) is not None:
+ attributes[SpanAttributeKey.LLM_COST] = {"total_cost": cost}
+ if model := span.metadata.get("model_name") or span.metadata.get("model"):
+ attributes[SpanAttributeKey.MODEL] = model
+ context = SpanContext(trace_id=int(trace.trace_id, 16), span_id=int(span.id, 16), is_remote=False)
+ parent = (
+ SpanContext(trace_id=context.trace_id, span_id=int(span.parent_id, 16), is_remote=False)
+ if span.parent_id
+ else None
+ )
+ result.append(
+ Span(
+ ReadableSpan(
+ name=span.name,
+ context=context,
+ parent=parent,
+ start_time=nanos(span.start_time),
+ end_time=nanos(span.end_time) if span.end_time else None,
+ attributes={key: json_value(value) for key, value in attributes.items()},
+ status=Status(StatusCode.ERROR, span.error)
+ if span.status == CanonicalSpanStatus.ERROR
+ else Status(StatusCode.OK),
+ resource=Resource.get_empty(),
+ )
+ )
+ )
+ return result
+
+
+def _auth(provider: str, config: Mapping[str, Any], host: str) -> dict[str, str]:
+ if provider == "mlflow":
+ if config.get("username") and config.get("password"):
+ token = base64.b64encode(f"{config['username']}:{config['password']}".encode()).decode()
+ return {"Authorization": f"Basic {token}"}
+ return {}
+ if config.get("client_id") and config.get("client_secret"):
+ response = request(
+ "POST",
+ host + "/oidc/v1/token",
+ provider,
+ auth=(config["client_id"], config["client_secret"]),
+ data={"grant_type": "client_credentials", "scope": "all-apis"},
+ )
+ token = response.json().get("access_token")
+ if not isinstance(token, str) or not token:
+ raise TraceExportRejectedError("Databricks OAuth token was not returned")
+ else:
+ token = required(config, "personal_access_token")
+ return {"Authorization": f"Bearer {token}"}
+
+
+class _Store(RestStore):
+ """Reuse native V3 request/response schemas without process-wide SDK credentials."""
+
+ def __init__(self, host: str, headers: dict[str, str]) -> None:
+ super().__init__(lambda: MlflowHostCreds(host))
+ self._host = host
+ self._headers = headers
+
+ @override
+ def _call_endpoint(self, api, json_body=None, endpoint=None, retry_timeout_seconds=None, response_proto=None):
+ methods = self._V3_METHOD_TO_INFO if api in self._V3_APIS else self._METHOD_TO_INFO
+ default_endpoint, method = methods[api]
+ response = request(
+ method,
+ self._host + (endpoint or default_endpoint),
+ "databricks",
+ headers={"Content-Type": "application/json", **self._headers},
+ content=json_body,
+ )
+ try:
+ return ParseDict(response.json(), response_proto if response_proto is not None else api.Response())
+ except (ValueError, ParseError):
+ raise TraceExportUncertainError("Databricks returned an unreadable trace acknowledgment") from None
+
+
+def _upload_databricks(host: str, headers: dict[str, str], trace_id: str, payload: bytes) -> None:
+ path, method = extract_api_info_for_service(DatabricksMlflowArtifactsService, "/api/2.0")[
+ GetCredentialsForTraceDataUpload
+ ]
+ response = request(
+ method, host + path.replace("{request_id}", quote(trace_id, safe="")), "databricks", headers=headers
+ )
+ credential = ParseDict(response.json(), GetCredentialsForTraceDataUpload.Response()).credential_info
+ signed_url = credential.signed_uri
+ parsed = urlsplit(signed_url)
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
+ raise TraceExportRejectedError("Invalid Databricks artifact upload destination")
+ upload_headers = {header.name: header.value for header in credential.headers}
+ if credential.type == ArtifactCredentialType.AZURE_ADLS_GEN2_SAS_URI:
+ # The bounded trace payload fits a single append. Flush commits that append.
+ separator = "&" if parsed.query else "?"
+ create_headers = {key: value for key, value in upload_headers.items() if key.lower() != "x-ms-blob-type"}
+ request("PUT", signed_url + separator + "resource=file", "databricks", headers=create_headers)
+ request(
+ "PATCH",
+ signed_url + separator + "action=append&position=0&flush=true",
+ "databricks",
+ headers=create_headers,
+ content=payload,
+ )
+ elif credential.type in {
+ ArtifactCredentialType.AWS_PRESIGNED_URL,
+ ArtifactCredentialType.GCP_SIGNED_URL,
+ ArtifactCredentialType.AZURE_SAS_URI,
+ }:
+ if credential.type == ArtifactCredentialType.AZURE_SAS_URI:
+ upload_headers["x-ms-blob-type"] = "BlockBlob"
+ request("PUT", signed_url, "databricks", headers=upload_headers, content=payload)
+ else:
+ raise TraceExportRejectedError("Unsupported Databricks artifact credential type")
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ host = endpoint(required(config, "host" if provider == "databricks" else "tracking_uri"))
+ experiment = str(config.get("experiment_id") or "0")
+ if not experiment.isdigit():
+ raise TraceExportRejectedError("MLflow experiment ID must be numeric")
+ headers = _auth(provider, config, host)
+ trace_id = "tr-" + trace.trace_id
+ spans = _spans(trace, experiment, trace_id)
+ if provider == "mlflow":
+ payload = ExportTraceServiceRequest()
+ scope = payload.resource_spans.add().scope_spans.add()
+ scope.spans.extend(span.to_otel_proto() for span in spans)
+ export_http(provider, host + OTLP_TRACES_PATH, {MLFLOW_EXPERIMENT_ID_HEADER: experiment, **headers}, payload)
+ # MLflow 3.4 has classic (selectedTraceId) and GenAI-v3
+ # (selectedEvaluationId) views; 3.17's v4 reads traceId. On close/change,
+ # v4 removes its selectedEvaluationId fallback before updating traceId.
+ # Each older view reads only its own key and ignores the other aliases.
+ query = urlencode({"traceId": trace_id, "selectedTraceId": trace_id, "selectedEvaluationId": trace_id})
+ url = f"{host}/#/experiments/{experiment}/traces?{query}"
+ else:
+ root = next(span for span in trace.spans if span.id == trace.root_span_id)
+ metadata = {TraceMetadataKey.TRACE_SESSION: trace.session_id}
+ usages = [token_usage(span) for span in trace.spans]
+ if any(usages):
+ metadata[TraceMetadataKey.TOKEN_USAGE] = json_value(
+ {f"{key}_tokens": sum(usage.get(key, 0) for usage in usages) for key in ("input", "output", "total")}
+ )
+ costs = [cost for span in trace.spans if (cost := usd_cost(span)) is not None]
+ if costs:
+ metadata[TraceMetadataKey.COST] = json_value({"total_cost": sum(costs)})
+ info = TraceInfo(
+ trace_id=trace_id,
+ trace_location=TraceLocation.from_experiment_id(experiment),
+ request_time=nanos(root.start_time) // 1_000_000,
+ execution_duration=(nanos(root.end_time) - nanos(root.start_time)) // 1_000_000 if root.end_time else None,
+ state=TraceState.ERROR if root.status == CanonicalSpanStatus.ERROR else TraceState.OK,
+ request_preview=json_value(root.inputs)[:10000] if root.inputs is not None else None,
+ response_preview=json_value(root.outputs)[:10000] if root.outputs is not None else None,
+ trace_metadata=metadata,
+ tags={TraceTagKey.TRACE_NAME: root.name},
+ client_request_id=trace.trace_id,
+ )
+ stored = _Store(host, headers).start_trace(info)
+ try:
+ trace_id = stored.trace_id
+ if not trace_id:
+ raise TraceExportUncertainError("Databricks did not acknowledge a trace identifier")
+ data = TraceData(spans=_spans(trace, experiment, trace_id))
+ _upload_databricks(host, headers, trace_id, json_value(data.to_dict()).encode())
+ except Exception:
+ raise TraceExportUncertainError(
+ "Databricks accepted trace metadata but artifact delivery is incomplete"
+ ) from None
+ # Databricks opens its trace detail panel with a different key from OSS MLflow.
+ url = f"{host}/ml/experiments/{experiment}/traces?selectedEvaluationId={quote(trace_id, safe='')}"
+ return TraceExportResult(trace_id=trace_id, trace_url=url)
diff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/configuration.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/configuration.py
new file mode 100644
index 00000000000..326930318d2
--- /dev/null
+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/configuration.py
@@ -0,0 +1,163 @@
+"""Read Databricks experiment metadata without changing MLflow's global state.
+
+This verifies authentication and read access to an existing active experiment.
+It does not prove write permission, tracing API availability, or artifact delivery.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import urlsplit, urlunsplit
+
+from google.protobuf.json_format import ParseDict, ParseError
+from mlflow.protos.service_pb2 import GetExperiment
+from pydantic import TypeAdapter
+
+from core.helper import ssrf_proxy
+from dify_trace_mlflow.config import DatabricksConfig
+
+type VerificationCode = Literal[
+ "invalid_workspace_url",
+ "invalid_experiment_id",
+ "authentication_failed",
+ "permission_denied",
+ "experiment_not_found_or_inaccessible",
+ "experiment_not_active",
+ "experiment_id_mismatch",
+ "malformed_response",
+ "endpoint_unavailable",
+ "connection_failed",
+]
+
+_MESSAGES: dict[VerificationCode, str] = {
+ "invalid_workspace_url": "Databricks requires an HTTPS workspace origin.",
+ "invalid_experiment_id": "Databricks requires a numeric experiment ID.",
+ "authentication_failed": "Databricks authentication failed.",
+ "permission_denied": "Databricks experiment access was denied.",
+ "experiment_not_found_or_inaccessible": "The selected Databricks experiment was not found or is inaccessible.",
+ "experiment_not_active": "The selected Databricks experiment is not active.",
+ "experiment_id_mismatch": "Databricks returned a different experiment.",
+ "malformed_response": "Databricks verification returned an invalid response.",
+ "endpoint_unavailable": "Databricks endpoint could not be verified.",
+ "connection_failed": "Databricks verification request failed.",
+}
+_JSON_OBJECT = TypeAdapter(dict[str, object])
+
+
+class DatabricksConfigurationError(ValueError):
+ """Fixed safe diagnostics; never include credentials or a vendor response."""
+
+ def __init__(self, code: VerificationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+@dataclass(frozen=True, slots=True)
+class ReadableDatabricksExperiment:
+ host: str
+ experiment_id: str
+
+
+def _target(config: DatabricksConfig) -> ReadableDatabricksExperiment:
+ host = config.host.strip()
+ try:
+ parsed = urlsplit(host)
+ valid = (
+ parsed.scheme == "https"
+ and parsed.hostname
+ and parsed.username is None
+ and parsed.password is None
+ and parsed.path in {"", "/"}
+ and not parsed.query
+ and not parsed.fragment
+ and "\\" not in host
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in host)
+ )
+ _ = parsed.port
+ except ValueError:
+ raise DatabricksConfigurationError("invalid_workspace_url") from None
+ if not valid:
+ raise DatabricksConfigurationError("invalid_workspace_url")
+ experiment_id = config.experiment_id.strip()
+ if not experiment_id or not experiment_id.isascii() or not experiment_id.isdigit():
+ raise DatabricksConfigurationError("invalid_experiment_id")
+ return ReadableDatabricksExperiment(
+ host=urlunsplit((parsed.scheme, parsed.netloc, "", "", "")),
+ experiment_id=experiment_id.lstrip("0") or "0",
+ )
+
+
+def project_url(config: DatabricksConfig) -> str:
+ """Pure URL presentation, including for historical configurations."""
+ target = _target(config)
+ return f"{target.host}/ml/experiments/{target.experiment_id}/traces"
+
+
+def _request(method: Literal["GET", "POST"], url: str, **options: object) -> dict[str, object]:
+ request = ssrf_proxy.get if method == "GET" else ssrf_proxy.post
+ try:
+ response = request(url, max_retries=0, timeout=30.0, follow_redirects=False, **options)
+ except Exception:
+ raise DatabricksConfigurationError("connection_failed") from None
+ try:
+ if response.status_code == 401:
+ raise DatabricksConfigurationError("authentication_failed")
+ if response.status_code == 403:
+ raise DatabricksConfigurationError("permission_denied")
+ if response.status_code == 404 and method == "GET":
+ raise DatabricksConfigurationError("experiment_not_found_or_inaccessible")
+ if not 200 <= response.status_code < 300:
+ raise DatabricksConfigurationError("endpoint_unavailable")
+ try:
+ return _JSON_OBJECT.validate_python(response.json(), strict=True)
+ except ValueError:
+ raise DatabricksConfigurationError("malformed_response") from None
+ finally:
+ response.close()
+
+
+def verify_configuration(config: DatabricksConfig) -> ReadableDatabricksExperiment:
+ """Read the exact experiment with the exporter's existing PAT/M2M precedence.
+
+ GetExperiment uses the official MLflow request/response schema. No permissions,
+ SCIM or account API is needed, and no experiment, run or trace is created.
+ Read access alone is intentionally not presented as write authorization.
+ """
+ target = _target(config)
+ if config.client_id and config.client_secret:
+ data = _request(
+ "POST",
+ target.host + "/oidc/v1/token",
+ auth=(config.client_id, config.client_secret),
+ data={"grant_type": "client_credentials", "scope": "all-apis"},
+ )
+ access_token = data.get("access_token")
+ token_type = data.get("token_type")
+ if (
+ not isinstance(access_token, str)
+ or not access_token.strip()
+ or not isinstance(token_type, str)
+ or token_type.lower() != "bearer"
+ ):
+ raise DatabricksConfigurationError("authentication_failed")
+ else:
+ access_token = config.personal_access_token
+ if not access_token or not access_token.strip():
+ raise DatabricksConfigurationError("authentication_failed")
+ request = GetExperiment(experiment_id=target.experiment_id)
+ data = _request(
+ "GET",
+ target.host + "/api/2.0/mlflow/experiments/get",
+ params={"experiment_id": request.experiment_id},
+ headers={"Authorization": f"Bearer {access_token}", "Accept": "application/json"},
+ )
+ try:
+ result = ParseDict(data, GetExperiment.Response(), ignore_unknown_fields=True)
+ except (ValueError, TypeError, ParseError):
+ raise DatabricksConfigurationError("malformed_response") from None
+ if not result.HasField("experiment") or not result.experiment.experiment_id:
+ raise DatabricksConfigurationError("malformed_response")
+ if result.experiment.experiment_id != target.experiment_id:
+ raise DatabricksConfigurationError("experiment_id_mismatch")
+ if result.experiment.lifecycle_stage != "active":
+ raise DatabricksConfigurationError("experiment_not_active")
+ return target
diff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py
index c89a69b90f8..0f7ffb08de1 100644
--- a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py
+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py
@@ -1,13 +1,28 @@
+import json
import logging
import os
+from collections.abc import Callable
+from contextvars import ContextVar
from datetime import datetime, timedelta
from typing import Any, cast, override
import mlflow
-from mlflow.entities import Document, LiveSpan, Span, SpanEvent, SpanStatusCode, SpanType
+from mlflow.entities import Document, LiveSpan, Span, SpanEvent, SpanStatusCode, SpanType, TraceData, TraceInfo
+from mlflow.entities.span import NoOpSpan
+from mlflow.exceptions import RestException
+from mlflow.tracing.client import TracingClient
from mlflow.tracing.constant import SpanAttributeKey, TokenUsageKey, TraceMetadataKey
+from mlflow.tracing.export.mlflow_v3 import MlflowV3SpanExporter
from mlflow.tracing.fluent import start_span_no_context, update_current_trace
+from mlflow.tracing.processor.mlflow_v3 import MlflowV3SpanProcessor
from mlflow.tracing.provider import detach_span_from_context, set_span_in_context
+from mlflow.tracing.trace_manager import InMemoryTraceManager
+from mlflow.tracing.utils import encode_span_id, get_otel_attribute
+from opentelemetry import trace as otel_trace
+from opentelemetry.context import Context
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.sampling import ALWAYS_ON
from sqlalchemy import select
from core.ops.base_trace_instance import BaseTraceInstance
@@ -41,42 +56,77 @@ def datetime_to_nanoseconds(dt: datetime | None) -> int | None:
return int(dt.timestamp() * 1_000_000_000)
-def _start_span_no_context(
- *,
- name: str,
- span_type: str,
- parent_span: LiveSpan | None = None,
- inputs: object | None = None,
- attributes: SpanAttributes | None = None,
- start_time_ns: int | None = None,
-) -> LiveSpan:
- """Start an MLflow span while preserving structured Dify attributes.
-
- MLflow 3.11 annotates `start_span_no_context(..., attributes=...)` as `dict[str, str]`,
- but the implementation immediately calls `LiveSpan.set_attributes(dict[str, Any])`.
- `LiveSpan` JSON-serializes arbitrary values before storing them in OpenTelemetry, and
- reserved attributes like `mlflow.chat.tokenUsage` are expected to round-trip as dicts.
- """
- return start_span_no_context(
- name=name,
- span_type=span_type,
- parent_span=parent_span,
- inputs=inputs,
- attributes=cast(dict[str, str] | None, attributes),
- start_time_ns=start_time_ns,
- )
+class _CheckedTracingClient(TracingClient):
+ """Remember errors hidden by SDK exporters without changing SDK serialization or I/O."""
+
+ def __init__(self, tracking_uri: str, failed: ContextVar[bool]):
+ super().__init__(tracking_uri)
+ self._failed = failed
+
+ def _checked[T](self, operation: Callable[[], T]) -> T:
+ try:
+ return operation()
+ except Exception:
+ self._failed.set(True)
+ raise
+
+ @override
+ def start_trace(self, trace_info: TraceInfo) -> TraceInfo:
+ return self._checked(lambda: super(_CheckedTracingClient, self).start_trace(trace_info))
+
+ @override
+ def _upload_trace_data(self, trace_info: TraceInfo, trace_data: TraceData) -> None:
+ self._checked(lambda: super(_CheckedTracingClient, self)._upload_trace_data(trace_info, trace_data))
+
+ @override
+ def log_spans(self, location: str, spans: list[Span]) -> list[Span]:
+ try:
+ return super().log_spans(location, spans)
+ except NotImplementedError:
+ # SDK falls back to the full trace artifact on pre-3.4/file-store servers.
+ raise
+ except RestException as error:
+ if "REST OTLP span logging is not supported" not in error.message:
+ self._failed.set(True)
+ raise
+ except Exception:
+ self._failed.set(True)
+ raise
+
+
+class _SynchronousMlflowExporter(MlflowV3SpanExporter):
+ """Celery owns asynchronous dispatch; no SDK queue may survive adapter LRU eviction."""
+
+ def __init__(self, tracking_uri: str):
+ self.failed: ContextVar[bool] = ContextVar("mlflow_legacy_export_failed", default=False)
+ super().__init__(tracking_uri)
+ self._client = _CheckedTracingClient(tracking_uri, self.failed)
+
+ @override
+ def _should_enable_async_logging(self) -> bool:
+ return False
class MLflowDataTrace(BaseTraceInstance):
def __init__(self, config: MLflowConfig | DatabricksConfig):
super().__init__(config)
+ self._tracer: otel_trace.Tracer | None = None
+ self._exporter: _SynchronousMlflowExporter | None = None
+ self._experiment_id = config.experiment_id
if isinstance(config, DatabricksConfig):
+ # Preserve the SDK's existing Databricks/UC destination selection.
self._setup_databricks(config)
- else:
- self._setup_mlflow(config)
+ os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] = "true"
+ return
+ self._setup_mlflow(config)
- # Enable async logging to minimize performance overhead
- os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] = "true"
+ # The public MLflow destination API still replaces the SDK's shared provider
+ # and can mutate OTEL_RESOURCE_ATTRIBUTES. Keep this legacy SDK pipeline private.
+ # No SDK background queue or atexit callback may retain an evicted adapter.
+ self._provider = TracerProvider(resource=Resource({}), sampler=ALWAYS_ON, shutdown_on_exit=False)
+ self._exporter = _SynchronousMlflowExporter(tracking_uri=self._tracking_uri)
+ self._provider.add_span_processor(MlflowV3SpanProcessor(self._exporter, export_metrics=False))
+ self._tracer = self._provider.get_tracer(__name__)
def _setup_databricks(self, config: DatabricksConfig):
"""Setup connection to Databricks-managed MLflow instances"""
@@ -95,6 +145,7 @@ class MLflowDataTrace(BaseTraceInstance):
"See https://docs.databricks.com/aws/en/dev-tools/auth/#what-authorization-option-should-i-choose "
"for more information about the authorization options."
)
+ self._tracking_uri = "databricks"
mlflow.set_tracking_uri("databricks")
mlflow.set_experiment(experiment_id=config.experiment_id)
@@ -104,8 +155,7 @@ class MLflowDataTrace(BaseTraceInstance):
def _setup_mlflow(self, config: MLflowConfig):
"""Setup connection to MLflow instances"""
- mlflow.set_tracking_uri(config.tracking_uri)
- mlflow.set_experiment(experiment_id=config.experiment_id)
+ self._tracking_uri = config.tracking_uri
# Simple auth if provided
if config.username and config.password:
@@ -114,9 +164,73 @@ class MLflowDataTrace(BaseTraceInstance):
self._project_url = f"{config.tracking_uri}/#/experiments/{config.experiment_id}/traces"
+ def _start_span_no_context(
+ self,
+ *,
+ name: str,
+ span_type: str,
+ parent_span: LiveSpan | None = None,
+ inputs: object | None = None,
+ attributes: SpanAttributes | None = None,
+ start_time_ns: int | None = None,
+ ) -> LiveSpan:
+ """Use the real SDK processor without inheriting a process-wide destination.
+
+ Every span needs the experiment attribute: the SDK's incremental exporter
+ resolves each child independently, rather than inheriting the root's location.
+ A missing/nonrecording span is an export failure, never successful dispatch.
+ """
+ if self._tracer is None:
+ span = start_span_no_context(
+ name=name,
+ span_type=span_type,
+ parent_span=parent_span,
+ inputs=inputs,
+ attributes=cast(dict[str, str] | None, attributes),
+ start_time_ns=start_time_ns,
+ )
+ if isinstance(span, NoOpSpan):
+ raise RuntimeError("mlflow_legacy_span_not_recording")
+ return span
+ context = Context()
+ if parent_span is not None:
+ parent = parent_span._span
+ if not isinstance(parent, otel_trace.Span):
+ raise RuntimeError("mlflow_legacy_parent_span_invalid")
+ context = otel_trace.set_span_in_context(parent, context)
+ try:
+ otel_span = self._tracer.start_span(
+ name,
+ context=context,
+ attributes={SpanAttributeKey.EXPERIMENT_ID: json.dumps(self._experiment_id)},
+ start_time=start_time_ns,
+ )
+ except Exception:
+ raise RuntimeError("mlflow_legacy_span_start_failed") from None
+ if not otel_span.is_recording():
+ raise RuntimeError("mlflow_legacy_span_not_recording")
+ trace_id = get_otel_attribute(otel_span, SpanAttributeKey.REQUEST_ID)
+ manager = InMemoryTraceManager.get_instance()
+ span = (
+ manager.get_span_from_id(trace_id, encode_span_id(otel_span.get_span_context().span_id))
+ if trace_id
+ else None
+ )
+ if not isinstance(span, LiveSpan):
+ manager.pop_trace(otel_span.get_span_context().trace_id)
+ otel_span.end()
+ raise RuntimeError("mlflow_legacy_span_not_registered")
+ span.set_span_type(span_type)
+ if inputs is not None:
+ span.set_inputs(inputs)
+ span.set_attributes(attributes or {})
+ span.set_attribute(SpanAttributeKey.EXPERIMENT_ID, self._experiment_id)
+ return span
+
@override
def trace(self, trace_info: BaseTraceInfo):
- """Simple dispatch to trace methods"""
+ """Dispatch legacy data; synchronous OSS export failures must reach the task owner."""
+ failure_token = self._exporter.failed.set(False) if self._exporter is not None else None
try:
match trace_info:
case WorkflowTraceInfo():
@@ -133,9 +247,14 @@ class MLflowDataTrace(BaseTraceInstance):
self.suggested_question_trace(trace_info)
case GenerateNameTraceInfo():
self.generate_name_trace(trace_info)
+ if self._exporter is not None and self._exporter.failed.get():
+ raise RuntimeError("mlflow_legacy_export_failed")
except Exception:
logger.exception("[MLflow] Trace error")
raise
+ finally:
+ if self._exporter is not None and failure_token is not None:
+ self._exporter.failed.reset(failure_token)
def workflow_trace(self, trace_info: WorkflowTraceInfo):
"""Create workflow span as root, with node spans as children"""
@@ -147,7 +266,7 @@ class MLflowDataTrace(BaseTraceInstance):
if trace_info.query:
workflow_inputs["query"] = trace_info.query
- workflow_span = _start_span_no_context(
+ workflow_span = self._start_span_no_context(
name=TraceTaskName.WORKFLOW_TRACE.value,
span_type=SpanType.CHAIN,
inputs=workflow_inputs,
@@ -185,7 +304,7 @@ class MLflowDataTrace(BaseTraceInstance):
if not inputs:
inputs = JSON_DICT_ADAPTER.validate_json(node.inputs) if node.inputs else {}
- node_span = _start_span_no_context(
+ node_span = self._start_span_no_context(
name=node.title,
span_type=self._get_node_span_type(node.node_type),
parent_span=workflow_span,
@@ -294,7 +413,7 @@ class MLflowDataTrace(BaseTraceInstance):
base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001")
file_list.append(f"{base_url}/{message_file_data.url}")
- span = _start_span_no_context(
+ span = self._start_span_no_context(
name=TraceTaskName.MESSAGE_TRACE.value,
span_type=SpanType.LLM,
inputs=self._parse_prompts(trace_info.inputs),
@@ -358,7 +477,7 @@ class MLflowDataTrace(BaseTraceInstance):
return metadata.get("from_account_id") # type: ignore[return-value]
def tool_trace(self, trace_info: ToolTraceInfo):
- span = _start_span_no_context(
+ span = self._start_span_no_context(
name=trace_info.tool_name,
span_type=SpanType.TOOL,
inputs=trace_info.tool_inputs,
@@ -395,7 +514,7 @@ class MLflowDataTrace(BaseTraceInstance):
return
start_time = trace_info.start_time or trace_info.message_data.created_at
- span = _start_span_no_context(
+ span = self._start_span_no_context(
name=TraceTaskName.MODERATION_TRACE.value,
span_type=SpanType.TOOL,
inputs=trace_info.inputs or {},
@@ -419,7 +538,7 @@ class MLflowDataTrace(BaseTraceInstance):
if trace_info.message_data is None:
return
- span = _start_span_no_context(
+ span = self._start_span_no_context(
name=TraceTaskName.DATASET_RETRIEVAL_TRACE.value,
span_type=SpanType.RETRIEVER,
inputs=trace_info.inputs,
@@ -438,7 +557,7 @@ class MLflowDataTrace(BaseTraceInstance):
start_time = trace_info.start_time or trace_info.message_data.created_at
end_time = trace_info.end_time or trace_info.message_data.updated_at
- span = _start_span_no_context(
+ span = self._start_span_no_context(
name=TraceTaskName.SUGGESTED_QUESTION_TRACE.value,
span_type=SpanType.TOOL,
inputs=trace_info.inputs,
@@ -467,7 +586,7 @@ class MLflowDataTrace(BaseTraceInstance):
span.end(outputs=trace_info.suggested_question, end_time_ns=datetime_to_nanoseconds(end_time))
def generate_name_trace(self, trace_info: GenerateNameTraceInfo):
- span = _start_span_no_context(
+ span = self._start_span_no_context(
name=TraceTaskName.GENERATE_NAME_TRACE.value,
span_type=SpanType.CHAIN,
inputs=trace_info.inputs,
@@ -566,7 +685,7 @@ class MLflowDataTrace(BaseTraceInstance):
def api_check(self):
"""Simple connection test"""
try:
- mlflow.search_experiments(max_results=1)
+ mlflow.MlflowClient(tracking_uri=self._tracking_uri).search_experiments(max_results=1)
return True
except Exception as e:
raise ValueError(f"MLflow connection failed: {str(e)}") from e
diff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/oss_configuration.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/oss_configuration.py
new file mode 100644
index 00000000000..20c201e00f0
--- /dev/null
+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/oss_configuration.py
@@ -0,0 +1,201 @@
+"""Read one OSS MLflow experiment without changing SDK or authentication state.
+
+Configuration verification proves only read access to an existing active experiment.
+It does not establish trace ingestion, artifact delivery, or write permission. The
+legacy exporter remains a separate owner; configuration never constructs it.
+"""
+
+import base64
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import unquote, urlsplit, urlunsplit
+
+from google.protobuf.json_format import ParseDict, ParseError
+from mlflow.protos.service_pb2 import GetExperiment
+from pydantic import TypeAdapter
+
+from core.helper import ssrf_proxy
+from dify_trace_mlflow.config import MLflowConfig
+
+type VerificationCode = Literal[
+ "invalid_tracking_uri",
+ "invalid_experiment_id",
+ "incomplete_basic_auth",
+ "authentication_failed",
+ "permission_denied",
+ "experiment_not_found_or_inaccessible",
+ "experiment_not_active",
+ "experiment_id_mismatch",
+ "malformed_response",
+ "endpoint_unavailable",
+ "connection_failed",
+]
+
+_MESSAGES: dict[VerificationCode, str] = {
+ "invalid_tracking_uri": "MLflow requires an HTTP(S) tracking server URL with an optional path prefix.",
+ "invalid_experiment_id": "MLflow requires a numeric experiment ID.",
+ "incomplete_basic_auth": "MLflow username and password must both be supplied or both be empty.",
+ "authentication_failed": "MLflow authentication failed.",
+ "permission_denied": "MLflow experiment access was denied.",
+ "experiment_not_found_or_inaccessible": "The selected MLflow experiment was not found or is inaccessible.",
+ "experiment_not_active": "The selected MLflow experiment is not active.",
+ "experiment_id_mismatch": "MLflow returned a different experiment.",
+ "malformed_response": "MLflow verification returned an invalid response.",
+ "endpoint_unavailable": "MLflow endpoint could not be verified.",
+ "connection_failed": "MLflow verification request failed.",
+}
+_JSON_OBJECT = TypeAdapter(dict[str, object])
+_MAX_RESPONSE_BYTES = 1024 * 1024
+
+
+class MLflowConfigurationError(ValueError):
+ """Fixed diagnostics that never include request values or vendor response text."""
+
+ def __init__(self, code: VerificationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+@dataclass(frozen=True, slots=True)
+class ReadableMLflowExperiment:
+ tracking_uri: str
+ experiment_id: str
+
+
+def _tracking_uri(value: object) -> str:
+ if not isinstance(value, str):
+ raise MLflowConfigurationError("invalid_tracking_uri")
+ # Only surrounding ordinary spaces are normalization; URL parsers silently
+ # discard some control characters, so inspect the original input first.
+ value = value.strip(" ")
+ try:
+ parsed = urlsplit(value)
+ host, port = parsed.hostname, parsed.port
+ path = parsed.path.rstrip("/")
+ decoded_path = unquote(path, errors="strict")
+ valid = (
+ parsed.scheme in {"http", "https"}
+ and host
+ and parsed.username is None
+ and parsed.password is None
+ and "?" not in value
+ and "#" not in value
+ and "\\" not in value
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ and not parsed.netloc.endswith(":")
+ and (port is None or 1 <= port <= 65535)
+ and "%" not in host
+ and not re.search(r"%(?![0-9a-fA-F]{2})", path)
+ and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)
+ and not any(char in decoded_path for char in "\\?#%")
+ and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)
+ and "//" not in parsed.path
+ and not any(part in {".", ".."} for part in decoded_path.split("/"))
+ )
+ if not valid or host is None:
+ raise MLflowConfigurationError("invalid_tracking_uri")
+ host = host.encode("idna").decode("ascii").lower()
+ authority = f"[{host}]" if ":" in host else host
+ if port is not None:
+ authority = f"{authority}:{port}"
+ return urlunsplit((parsed.scheme, authority, path, "", ""))
+ except (UnicodeError, ValueError):
+ raise MLflowConfigurationError("invalid_tracking_uri") from None
+
+
+def _experiment_id(value: str) -> str:
+ value = value.strip()
+ if not value or not value.isascii() or not value.isdigit():
+ raise MLflowConfigurationError("invalid_experiment_id")
+ return value.lstrip("0") or "0"
+
+
+def normalize_configuration(values: Mapping[str, object]) -> MLflowConfig:
+ """Validate new destinations while retaining masked/nullable password values.
+
+ Pair validation belongs after masked credentials are restored and decrypted.
+ Keeping it out of MLflowConfig also leaves historical records readable and
+ avoids interpreting RSA ciphertext as a user-supplied password.
+ """
+ config = MLflowConfig.model_validate(values)
+ return config.model_copy(
+ update={
+ "tracking_uri": _tracking_uri(values.get("tracking_uri", config.tracking_uri)),
+ "experiment_id": _experiment_id(config.experiment_id),
+ }
+ )
+
+
+def project_url(config: MLflowConfig) -> str:
+ """Pure link calculation, independent of authentication and server access."""
+ host = _tracking_uri(config.tracking_uri)
+ experiment_id = _experiment_id(config.experiment_id)
+ return f"{host}/#/experiments/{experiment_id}/traces"
+
+
+def verify_configuration(config: MLflowConfig) -> ReadableMLflowExperiment:
+ """GET the exact experiment with only this request's optional Basic credentials."""
+ config = normalize_configuration(config.model_dump())
+ username, password = config.username, config.password
+ has_username, has_password = bool(username), bool(password)
+ if has_username != has_password or (
+ has_username
+ and (
+ username is None
+ or password is None
+ or not username.strip()
+ or not password.strip()
+ or ":" in username
+ or any(ord(char) < 32 or ord(char) == 127 for char in username + password)
+ )
+ ):
+ raise MLflowConfigurationError("incomplete_basic_auth")
+
+ # Explicit Cookie suppresses the pooled HTTP client's ambient cookie jar.
+ # No MLflow environment variables, SDK client, or process-wide auth is read.
+ headers = {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}
+ if username and password:
+ encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
+ headers["Authorization"] = f"Basic {encoded}"
+ request = GetExperiment(experiment_id=config.experiment_id)
+ try:
+ response = ssrf_proxy.get(
+ config.tracking_uri + "/api/2.0/mlflow/experiments/get",
+ params={"experiment_id": request.experiment_id},
+ headers=headers,
+ auth=None,
+ max_retries=0,
+ timeout=30.0,
+ follow_redirects=False,
+ stream_response=True,
+ )
+ except Exception:
+ raise MLflowConfigurationError("connection_failed") from None
+ try:
+ if response.status_code == 401:
+ raise MLflowConfigurationError("authentication_failed")
+ if response.status_code == 403:
+ raise MLflowConfigurationError("permission_denied")
+ if response.status_code == 404:
+ raise MLflowConfigurationError("experiment_not_found_or_inaccessible")
+ if not 200 <= response.status_code < 300:
+ raise MLflowConfigurationError("endpoint_unavailable")
+ try:
+ buffered = ssrf_proxy.buffer_response(response, max_response_bytes=_MAX_RESPONSE_BYTES)
+ data = _JSON_OBJECT.validate_python(buffered.json(), strict=True)
+ result = ParseDict(data, GetExperiment.Response(), ignore_unknown_fields=True)
+ except (ValueError, TypeError, ParseError):
+ raise MLflowConfigurationError("malformed_response") from None
+ except Exception:
+ raise MLflowConfigurationError("connection_failed") from None
+ finally:
+ response.close()
+ if not result.HasField("experiment") or not result.experiment.experiment_id:
+ raise MLflowConfigurationError("malformed_response")
+ if result.experiment.experiment_id != config.experiment_id:
+ raise MLflowConfigurationError("experiment_id_mismatch")
+ if result.experiment.lifecycle_stage != "active":
+ raise MLflowConfigurationError("experiment_not_active")
+ return ReadableMLflowExperiment(config.tracking_uri, config.experiment_id)
diff --git a/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py b/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py
index 7e1aa5f43f0..e4ecf4db806 100644
--- a/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py
+++ b/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py
@@ -188,7 +188,7 @@ def mock_mlflow():
def mock_tracing():
"""Patch all MLflow tracing functions used by the module."""
with (
- patch("dify_trace_mlflow.mlflow_trace.start_span_no_context") as mock_start,
+ patch("dify_trace_mlflow.mlflow_trace.MLflowDataTrace._start_span_no_context") as mock_start,
patch("dify_trace_mlflow.mlflow_trace.update_current_trace") as mock_update,
patch("dify_trace_mlflow.mlflow_trace.set_span_in_context") as mock_set,
patch("dify_trace_mlflow.mlflow_trace.detach_span_from_context") as mock_detach,
@@ -234,10 +234,10 @@ class TestInit:
def test_mlflow_config_no_auth(self, mock_mlflow):
config = MLflowConfig(tracking_uri="http://localhost:5000", experiment_id="0")
trace = MLflowDataTrace(config)
- mock_mlflow.set_tracking_uri.assert_called_with("http://localhost:5000")
- mock_mlflow.set_experiment.assert_called_with(experiment_id="0")
+ mock_mlflow.set_tracking_uri.assert_not_called()
+ mock_mlflow.set_experiment.assert_not_called()
assert trace.get_project_url() == "http://localhost:5000/#/experiments/0/traces"
- assert os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] == "true"
+ assert trace._tracking_uri == "http://localhost:5000"
def test_mlflow_config_with_auth(self, mock_mlflow):
config = MLflowConfig(
@@ -262,6 +262,7 @@ class TestInit:
assert os.environ["DATABRICKS_CLIENT_ID"] == "cid"
assert os.environ["DATABRICKS_CLIENT_SECRET"] == "csec"
mock_mlflow.set_tracking_uri.assert_called_with("databricks")
+ assert trace._tracking_uri == "databricks"
# Trailing slash stripped
assert trace.get_project_url() == "https://db.com/ml/experiments/42/traces"
@@ -1034,11 +1035,11 @@ class TestResolveToolCallIds:
class TestApiCheck:
def test_success(self, trace_instance, mock_mlflow):
- mock_mlflow.search_experiments.return_value = []
+ mock_mlflow.MlflowClient.return_value.search_experiments.return_value = []
assert trace_instance.api_check() is True
def test_failure(self, trace_instance, mock_mlflow):
- mock_mlflow.search_experiments.side_effect = ConnectionError("refused")
+ mock_mlflow.MlflowClient.return_value.search_experiments.side_effect = ConnectionError("refused")
with pytest.raises(ValueError, match="MLflow connection failed"):
trace_instance.api_check()
diff --git a/api/providers/trace/trace-opik/src/dify_trace_opik/agent_trace.py b/api/providers/trace/trace-opik/src/dify_trace_opik/agent_trace.py
new file mode 100644
index 00000000000..db03c1a8d36
--- /dev/null
+++ b/api/providers/trace/trace-opik/src/dify_trace_opik/agent_trace.py
@@ -0,0 +1,129 @@
+"""Synchronous Opik SDK ingestion with native UUIDv7, thread, and usage fields."""
+
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.provider_common import endpoint, finished_at, json_value, token_usage, usd_cost, utc
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError
+from services.tracing.sdk_transport import SSRFTransport
+
+
+def _provider_id(trace_id: str, span_id: str, started_at: datetime) -> str:
+ from opik.id_helpers import uuid4_to_uuid7
+
+ seeded = UUID(str(uuid5(NAMESPACE_URL, f"dify:{trace_id}:{span_id}")), version=4)
+ return str(uuid4_to_uuid7(utc(started_at), str(seeded)))
+
+
+def _content(value: Any) -> dict[str, Any] | list[dict[str, Any]] | str | None:
+ """Opik accepts objects, object lists and strings, but not every JSON value."""
+ if value is None or isinstance(value, (dict, str)):
+ return value
+ if isinstance(value, list) and all(isinstance(item, dict) for item in value):
+ return value
+ return json_value(value)
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ import httpx
+ from opik.rest_api.client import OpikApi
+ from opik.rest_api.types.error_info_write import ErrorInfoWrite
+ from opik.rest_api.types.span_write import SpanWrite
+
+ url = endpoint(config.get("url", "https://www.comet.com/opik/api"))
+ project = str(config.get("project") or "Default Project")
+ root = trace.spans[0]
+ trace_id = _provider_id(trace.trace_id, "trace", root.start_time)
+ ids = {span.id: _provider_id(trace.trace_id, span.id, span.start_time) for span in trace.spans}
+ accepted = False
+ with httpx.Client(transport=SSRFTransport(provider), timeout=30) as http_client:
+ client = OpikApi(
+ base_url=url,
+ api_key=config.get("api_key"),
+ workspace_name=str(config.get("workspace") or "default"),
+ httpx_client=http_client,
+ )
+ spans = []
+ for span in trace.spans:
+ usage = token_usage(span)
+ native_usage = {
+ "prompt_tokens" if key == "input" else "completion_tokens" if key == "output" else "total_tokens": value
+ for key, value in usage.items()
+ }
+ ttft = span.metadata.get("time_to_first_token", span.metadata.get("ttft"))
+ spans.append(
+ SpanWrite(
+ id=ids[span.id],
+ trace_id=trace_id,
+ parent_span_id=ids.get(span.parent_id or ""),
+ project_name=project,
+ name=span.name,
+ type="llm"
+ if span.kind is CanonicalSpanKind.LLM
+ else "tool"
+ if span.kind is CanonicalSpanKind.TOOL
+ else "general",
+ start_time=utc(span.start_time),
+ end_time=finished_at(span),
+ input=_content(span.inputs),
+ output=_content(span.outputs),
+ metadata={
+ **span.metadata,
+ "dify.span.kind": span.kind.value,
+ "dify.span.id": span.id,
+ "dify.trace_id": trace.trace_id,
+ "session_id": trace.session_id,
+ },
+ model=str(span.metadata.get("model") or span.metadata.get("model_name") or "")
+ if span.kind is CanonicalSpanKind.LLM
+ else None,
+ provider=str(span.metadata.get("model_provider") or "")
+ if span.kind is CanonicalSpanKind.LLM
+ else None,
+ usage=native_usage or None,
+ total_estimated_cost=usd_cost(span),
+ ttft=float(ttft) * 1000
+ if span.kind is CanonicalSpanKind.LLM and isinstance(ttft, (int, float)) and ttft >= 0
+ else None,
+ error_info=ErrorInfoWrite(
+ exception_type="DifyExecutionError",
+ message=span.error or "Trace operation failed",
+ traceback="",
+ )
+ if span.status is CanonicalSpanStatus.ERROR
+ else None,
+ )
+ )
+ try:
+ client.traces.create_trace(
+ id=trace_id,
+ project_name=project,
+ name=root.name,
+ start_time=utc(root.start_time),
+ end_time=finished_at(root),
+ input=_content(root.inputs),
+ output=_content(root.outputs),
+ metadata={**root.metadata, "dify.trace_id": trace.trace_id},
+ error_info=ErrorInfoWrite(
+ exception_type="DifyExecutionError",
+ message=root.error or "Trace operation failed",
+ traceback="",
+ )
+ if root.status is CanonicalSpanStatus.ERROR
+ else None,
+ thread_id=trace.session_id,
+ request_options={"max_retries": 0},
+ )
+ accepted = True
+ client.spans.create_spans(spans=spans, request_options={"max_retries": 0})
+ except Exception as error:
+ if accepted:
+ raise TraceExportUncertainError("Opik accepted part of the trace; reconciliation is required") from None
+ if isinstance(error, (RetryableTraceDispatchError, TraceExportRejectedError)):
+ raise
+ raise TraceExportRejectedError("Opik rejected the trace") from None
+ return TraceExportResult(trace_id=trace_id)
diff --git a/api/providers/trace/trace-opik/src/dify_trace_opik/configuration.py b/api/providers/trace/trace-opik/src/dify_trace_opik/configuration.py
new file mode 100644
index 00000000000..e2152b906fb
--- /dev/null
+++ b/api/providers/trace/trace-opik/src/dify_trace_opik/configuration.py
@@ -0,0 +1,156 @@
+"""Check Opik workspace access without SDK streamers or ambient credentials.
+
+The auth endpoint is read-only despite using POST. Verification does not create
+projects or traces and does not establish project write or ingestion permission.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import quote, urlencode, urlsplit
+
+from pydantic import TypeAdapter
+
+from core.helper import ssrf_proxy
+from dify_trace_opik.config import OpikConfig
+
+type VerificationCode = Literal[
+ "invalid_endpoint",
+ "invalid_workspace",
+ "invalid_project",
+ "authentication_failed",
+ "permission_denied",
+ "endpoint_unavailable",
+ "malformed_response",
+ "connection_failed",
+]
+
+_MESSAGES: dict[VerificationCode, str] = {
+ "invalid_endpoint": "Opik requires an HTTP(S) API URL ending in /api/.",
+ "invalid_workspace": "Opik requires a valid workspace name.",
+ "invalid_project": "Opik requires a valid project name.",
+ "authentication_failed": "Opik authentication failed.",
+ "permission_denied": "Opik workspace access was denied.",
+ "endpoint_unavailable": "Opik endpoint could not be verified.",
+ "malformed_response": "Opik verification returned an invalid response.",
+ "connection_failed": "Opik verification request failed.",
+}
+_JSON_OBJECT = TypeAdapter(dict[str, object])
+
+
+class OpikConfigurationError(ValueError):
+ """Fixed safe codes; no credentials, vendor bodies or request details."""
+
+ def __init__(self, code: VerificationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+@dataclass(frozen=True, slots=True)
+class VerifiedOpikWorkspace:
+ url: str
+ workspace: str
+ project: str
+
+
+def _api_url(config: OpikConfig) -> str:
+ value = config.url.strip()
+ try:
+ parsed = urlsplit(value)
+ valid = (
+ parsed.scheme in {"http", "https"}
+ and parsed.hostname
+ and parsed.username is None
+ and parsed.password is None
+ and not parsed.query
+ and not parsed.fragment
+ and parsed.path.endswith("/api/")
+ and "\\" not in value
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ )
+ _ = parsed.port
+ except ValueError:
+ raise OpikConfigurationError("invalid_endpoint") from None
+ if not valid:
+ raise OpikConfigurationError("invalid_endpoint")
+ return value
+
+
+def _name(value: str | None, default: str, code: Literal["invalid_workspace", "invalid_project"]) -> str:
+ result = value.strip() if value else default
+ result = result or default
+ if any(ord(char) < 32 or ord(char) == 127 for char in result) or (
+ code == "invalid_workspace" and result in {".", ".."}
+ ):
+ raise OpikConfigurationError(code)
+ return result
+
+
+def console_url(config: OpikConfig) -> str:
+ """Use the configured host and prefix, never the SDK's global OpikConfig."""
+ return _api_url(config).removesuffix("api/")
+
+
+def project_url(config: OpikConfig) -> str | None:
+ """Pure fallback; historical default workspaces cannot be resolved on GET."""
+ base = console_url(config)
+ workspace = _name(config.workspace, "default", "invalid_workspace")
+ if workspace == "default":
+ return None
+ project = _name(config.project, "Default Project", "invalid_project")
+ return f"{base}{quote(workspace, safe='')}/redirect/projects?{urlencode({'name': project})}"
+
+
+def _request(method: Literal["GET", "POST"], url: str, headers: dict[str, str]) -> dict[str, object] | None:
+ try:
+ response = ssrf_proxy.make_request(
+ method,
+ url,
+ headers=headers,
+ json={} if method == "POST" else None,
+ max_retries=0,
+ timeout=30.0,
+ follow_redirects=False,
+ )
+ except Exception:
+ raise OpikConfigurationError("connection_failed") from None
+ try:
+ if response.status_code == 401:
+ raise OpikConfigurationError("authentication_failed")
+ if response.status_code == 403:
+ raise OpikConfigurationError("permission_denied")
+ if not 200 <= response.status_code < 300:
+ raise OpikConfigurationError("endpoint_unavailable")
+ if method == "POST":
+ # The official auth contract is No Content, not an arbitrary 2xx page.
+ if response.content.strip():
+ raise OpikConfigurationError("malformed_response")
+ return None
+ try:
+ return _JSON_OBJECT.validate_python(response.json(), strict=True)
+ except ValueError:
+ raise OpikConfigurationError("malformed_response") from None
+ finally:
+ response.close()
+
+
+def verify_configuration(config: OpikConfig) -> VerifiedOpikWorkspace:
+ """Check auth/workspace via the same endpoints as Opik 1.11.2's auth_check.
+
+ /v1/private/auth checks workspace access and returns no content. Only the
+ sentinel default workspace needs /v1/private/auth/workspace resolution.
+ Project creation remains an ingestion concern, as in the legacy behavior.
+ """
+ url = _api_url(config)
+ workspace = _name(config.workspace, "default", "invalid_workspace")
+ project = _name(config.project, "Default Project", "invalid_project")
+ headers = {"Comet-Workspace": workspace, "Accept": "application/json"}
+ if config.api_key:
+ headers["Authorization"] = config.api_key
+ _request("POST", url + "v1/private/auth", headers)
+ if workspace == "default":
+ data = _request("GET", url + "v1/private/auth/workspace", headers)
+ resolved = data.get("workspace_name") if data else None
+ if not isinstance(resolved, str) or not resolved.strip():
+ raise OpikConfigurationError("malformed_response")
+ workspace = _name(resolved, "default", "invalid_workspace")
+ return VerifiedOpikWorkspace(url=url, workspace=workspace, project=project)
diff --git a/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py b/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py
index 710173ccdc8..7323542c0f6 100644
--- a/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py
+++ b/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py
@@ -63,12 +63,12 @@ def _seed_to_uuid4(seed: str) -> str:
return str(uuid.UUID(bytes=bytes(ba)))
-def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None):
- """Opik needs UUIDv7 while Dify uses UUIDv4 for identifier of most
- messages and objects. The type-hints of BaseTraceInfo indicates that
- objects start_time and message_id could be null which means we cannot map
- it to a UUIDv7. Given that we have no way to identify that object
- uniquely, generate a new random one UUIDv7 in that case.
+def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None) -> uuid.UUID:
+ """Preserve UUID4 mappings and deterministically adapt external trace IDs.
+
+ OpenTelemetry trace IDs and other external identifiers need not be UUID4,
+ which the Opik conversion helper requires. Reuse the existing stable seed
+ mapping for those identifiers. Missing identifiers retain random-ID behavior.
"""
if user_datetime is None:
@@ -77,6 +77,13 @@ def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None):
if user_uuid is None:
user_uuid = str(uuid.uuid4())
+ try:
+ is_uuid4 = uuid.UUID(user_uuid).version == 4
+ except ValueError:
+ is_uuid4 = False
+ if not is_uuid4:
+ user_uuid = _seed_to_uuid4(user_uuid)
+
return uuid4_to_uuid7(user_datetime, user_uuid)
diff --git a/api/providers/trace/trace-tencent/src/dify_trace_tencent/agent_trace.py b/api/providers/trace/trace-tencent/src/dify_trace_tencent/agent_trace.py
new file mode 100644
index 00000000000..78c99f94e61
--- /dev/null
+++ b/api/providers/trace/trace-tencent/src/dify_trace_tencent/agent_trace.py
@@ -0,0 +1,19 @@
+"""Tencent APM Agent spans over its existing authenticated OTLP/gRPC protocol."""
+
+from collections.abc import Mapping
+from typing import Any
+
+from core.ops.unified_trace.entities import CanonicalTrace
+from services.tracing.genai_attributes import attributes
+from services.tracing.otlp_export import build_request, export_grpc
+from services.tracing.provider_common import endpoint, required
+from services.tracing.provider_dispatch import TraceExportResult
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ url = endpoint(required(config, "endpoint"))
+ payload = build_request(
+ trace, {"service.name": required(config, "service_name")}, lambda span: attributes(span, trace, provider)
+ )
+ export_grpc(provider, url, {"authorization": f"Bearer {required(config, 'token')}"}, payload)
+ return TraceExportResult(trace_id=trace.trace_id)
diff --git a/api/providers/trace/trace-tencent/src/dify_trace_tencent/configuration.py b/api/providers/trace/trace-tencent/src/dify_trace_tencent/configuration.py
new file mode 100644
index 00000000000..04146a6e450
--- /dev/null
+++ b/api/providers/trace/trace-tencent/src/dify_trace_tencent/configuration.py
@@ -0,0 +1,142 @@
+"""Tencent configuration syntax and console navigation without exporter state.
+
+These checks do not establish token validity or ingestion permission.
+Collectors may acknowledge zero-span OTLP requests without authenticating
+them, so an empty Export acknowledgment must not be presented as authentication.
+"""
+
+import re
+from collections.abc import Mapping
+from typing import Literal
+from urllib.parse import urlsplit, urlunsplit
+
+from dify_trace_tencent.config import TencentConfig
+
+type ConfigurationCode = Literal[
+ "invalid_endpoint", "invalid_service_name", "invalid_token", "connection_failed", "unexpected_response"
+]
+
+_MESSAGES: dict[ConfigurationCode, str] = {
+ "invalid_endpoint": "Tencent APM requires an HTTP(S) collector origin with a valid host and port.",
+ "invalid_service_name": (
+ "Tencent APM service names must contain 1 to 63 lowercase letters, digits or hyphens, "
+ "start with a letter and end with a letter or digit."
+ ),
+ "invalid_token": "Tencent APM requires a nonempty token.",
+ "connection_failed": "Tencent APM collector protocol connection failed.",
+ "unexpected_response": "Tencent APM collector returned an unexpected protocol acknowledgment.",
+}
+_SERVICE_NAME = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?")
+
+
+class TencentConfigurationError(ValueError):
+ """Fixed safe errors, without request values or credentials."""
+
+ def __init__(self, code: ConfigurationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+def _endpoint(value: object) -> str:
+ if not isinstance(value, str):
+ raise TencentConfigurationError("invalid_endpoint")
+ value = value.strip()
+ try:
+ parsed = urlsplit(value)
+ host, port = parsed.hostname, parsed.port
+ valid = (
+ parsed.scheme in {"http", "https"}
+ and host
+ and parsed.username is None
+ and parsed.password is None
+ and parsed.path in {"", "/"}
+ and not parsed.query
+ and not parsed.fragment
+ and "?" not in value
+ and "#" not in value
+ and "\\" not in value
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ and not parsed.netloc.endswith(":")
+ and (port is None or 1 <= port <= 65535)
+ )
+ if not valid or host is None:
+ raise TencentConfigurationError("invalid_endpoint")
+ host = host.encode("idna").decode("ascii").lower()
+ authority = f"[{host}]" if ":" in host else host
+ if port is not None:
+ authority = f"{authority}:{port}"
+ return urlunsplit((parsed.scheme, authority, "", "", ""))
+ except (UnicodeError, ValueError):
+ raise TencentConfigurationError("invalid_endpoint") from None
+
+
+def normalize_configuration(values: Mapping[str, object]) -> TencentConfig:
+ """Validate new writes before legacy model normalization drops URL parts.
+
+ The permissive legacy model remains available to display existing stored
+ configurations. Masked tokens are retained here so PATCH can restore the
+ encrypted original through the shared configuration owner.
+ """
+ endpoint = _endpoint(values.get("endpoint"))
+ service_name = values.get("service_name")
+ if not isinstance(service_name, str) or not _SERVICE_NAME.fullmatch(service_name.strip()):
+ raise TencentConfigurationError("invalid_service_name")
+ token = values.get("token")
+ if not isinstance(token, str) or not token.strip():
+ raise TencentConfigurationError("invalid_token")
+ return TencentConfig(endpoint=endpoint, service_name=service_name.strip(), token=token)
+
+
+def console_url() -> str:
+ """Return the console entry, not a destination-specific project link."""
+ return "https://console.cloud.tencent.com/apm"
+
+
+def verify_configuration(config: TencentConfig) -> TencentConfig:
+ """Check syntax and one zero-span OTLP/gRPC exchange, not authentication.
+
+ A collector may acknowledge invalid-token or unauthenticated empty
+ requests. Actual credential validity and ingestion must therefore be
+ assessed from a business trace, never inferred from this connection check.
+ No tracer, span processor, metrics reader or global SDK state is created.
+ """
+ import grpc # type: ignore[import-untyped]
+ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub
+
+ config = normalize_configuration(config.model_dump())
+ parsed = urlsplit(config.endpoint)
+ host = parsed.hostname
+ assert host is not None # Validated above, before constructing a channel.
+ authority = f"[{host}]" if ":" in host else host
+ target = f"{authority}:{parsed.port or 4317}"
+ options = (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", 4 * 1024 * 1024))
+ channel: grpc.Channel | None = None
+ context = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
+ try:
+ try:
+ channel = (
+ grpc.insecure_channel(target, options=options)
+ if parsed.scheme == "http"
+ else grpc.secure_channel(target, grpc.ssl_channel_credentials(), options=options)
+ )
+ response = TraceServiceStub(channel).Export(
+ ExportTraceServiceRequest(),
+ metadata=(("authorization", f"Bearer {config.token}"),),
+ timeout=10.0,
+ wait_for_ready=False,
+ )
+ if response.partial_success.rejected_spans or response.partial_success.error_message:
+ raise TencentConfigurationError("unexpected_response")
+ finally:
+ if channel is not None:
+ channel.close()
+ except TencentConfigurationError:
+ raise
+ except Exception:
+ # gRPC errors may contain metadata or vendor details. Expose neither.
+ raise TencentConfigurationError("connection_failed") from None
+ finally:
+ detach(context)
+ return config
diff --git a/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py b/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py
index 763a85ffd71..a473e9141af 100644
--- a/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py
+++ b/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py
@@ -154,8 +154,9 @@ class TencentSpanBuilder:
outputs = node_execution.outputs or {}
usage_data = process_data.get("usage", {}) if "usage" in process_data else outputs.get("usage", {})
+ session_id = trace_info.metadata.get("conversation_id")
attributes = {
- GEN_AI_SESSION_ID: trace_info.metadata.get("conversation_id", ""),
+ GEN_AI_SESSION_ID: "" if session_id is None else session_id,
GEN_AI_SPAN_KIND: GenAISpanKind.GENERATION.value,
GEN_AI_FRAMEWORK: "dify",
GEN_AI_MODEL_NAME: process_data.get("model_name", ""),
@@ -293,6 +294,7 @@ class TencentSpanBuilder:
if node_execution.outputs:
output_value = json.dumps(node_execution.outputs.get("result", []), ensure_ascii=False)
+ session_id = trace_info.metadata.get("conversation_id")
return SpanData(
trace_id=trace_id,
parent_span_id=workflow_span_id,
@@ -301,7 +303,7 @@ class TencentSpanBuilder:
start_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.created_at),
end_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.finished_at),
attributes={
- GEN_AI_SESSION_ID: trace_info.metadata.get("conversation_id", ""),
+ GEN_AI_SESSION_ID: "" if session_id is None else session_id,
GEN_AI_SPAN_KIND: GenAISpanKind.RETRIEVER.value,
GEN_AI_FRAMEWORK: "dify",
RETRIEVAL_QUERY: input_value,
@@ -345,6 +347,7 @@ class TencentSpanBuilder:
trace_id: int, workflow_span_id: int, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution
) -> SpanData:
"""Build generic task span for workflow nodes."""
+ session_id = trace_info.metadata.get("conversation_id")
return SpanData(
trace_id=trace_id,
parent_span_id=workflow_span_id,
@@ -353,7 +356,7 @@ class TencentSpanBuilder:
start_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.created_at),
end_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.finished_at),
attributes={
- GEN_AI_SESSION_ID: trace_info.metadata.get("conversation_id", ""),
+ GEN_AI_SESSION_ID: "" if session_id is None else session_id,
GEN_AI_SPAN_KIND: GenAISpanKind.TASK.value,
GEN_AI_FRAMEWORK: "dify",
INPUT_VALUE: json.dumps(node_execution.inputs, ensure_ascii=False),
diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/agent_trace.py b/api/providers/trace/trace-weave/src/dify_trace_weave/agent_trace.py
new file mode 100644
index 00000000000..8799e78f0b8
--- /dev/null
+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/agent_trace.py
@@ -0,0 +1,89 @@
+"""Completed Agent trees through Weave's typed call API and an isolated transport."""
+
+from collections.abc import Mapping
+from typing import Any, cast
+from urllib.parse import quote
+from uuid import UUID, uuid5
+
+from weave.trace_server.trace_server_interface import (
+ CallEndReq,
+ CallStartReq,
+ EndedCallSchemaForInsert,
+ StartedCallSchemaForInsert,
+ SummaryInsertMap,
+)
+
+from core.ops.unified_trace.entities import CanonicalTrace
+from dify_trace_weave.transport import TraceServer, resolve_project
+from services.tracing.provider_common import endpoint, required, token_usage, utc
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ api_key = required(config, "api_key")
+ entity, project, ui_host = resolve_project(config, api_key)
+ # Weave's concrete HTTP client leaves unrelated optional API methods as protocol stubs.
+ server = TraceServer( # pyrefly: ignore[bad-instantiation]
+ endpoint(config.get("endpoint") or "https://trace.wandb.ai"), auth=("api", api_key)
+ )
+ project_id = f"{entity}/{project}"
+ trace_uuid = UUID(hex=trace.trace_id)
+ span_ids = {span.id: str(uuid5(trace_uuid, span.id)) for span in trace.spans}
+ # Weave counts calls whose id equals turn_id as conversation turns.
+ turn_id = span_ids[trace.root_span_id] if trace.session_id else None
+ wrote = False
+ try:
+ for span in trace.spans:
+ started = server.call_start(
+ CallStartReq(
+ start=StartedCallSchemaForInsert(
+ project_id=project_id,
+ id=span_ids[span.id],
+ op_name=span.name,
+ display_name=span.name,
+ trace_id=str(trace_uuid),
+ parent_id=span_ids.get(span.parent_id) if span.parent_id else None,
+ thread_id=trace.session_id or None,
+ turn_id=turn_id,
+ started_at=utc(span.start_time),
+ attributes={**span.metadata, "dify.span_kind": span.kind.value, "dify.span_id": span.id},
+ inputs=span.inputs if isinstance(span.inputs, dict) else {"input": span.inputs},
+ )
+ )
+ )
+ wrote = True
+ if started.id != span_ids[span.id] or started.trace_id != str(trace_uuid):
+ raise TraceExportUncertainError("Weave did not acknowledge the submitted identifiers")
+ for span in reversed(trace.spans):
+ usage = token_usage(span)
+ summary: dict[str, Any] = {}
+ if usage:
+ model = str(span.metadata.get("model_name") or span.metadata.get("model") or "unknown")
+ summary["usage"] = {
+ model: {
+ {"input": "prompt_tokens", "output": "completion_tokens", "total": "total_tokens"}[key]: value
+ for key, value in usage.items()
+ }
+ | {"requests": 1}
+ }
+ if span.end_time is None:
+ raise TraceExportRejectedError("Weave requires completed spans")
+ server.call_end(
+ CallEndReq(
+ end=EndedCallSchemaForInsert(
+ project_id=project_id,
+ id=span_ids[span.id],
+ ended_at=utc(span.end_time),
+ output=span.outputs,
+ exception=span.error or ("Agent span failed" if span.status.value == "error" else None),
+ summary=cast(SummaryInsertMap, summary),
+ )
+ )
+ )
+ except Exception:
+ if wrote:
+ raise TraceExportUncertainError("Weave trace was partially accepted") from None
+ raise
+ base = f"{ui_host}/{quote(entity, safe='')}/{quote(project, safe='')}/r/call/"
+ urls = {span_id: base + call_id for span_id, call_id in span_ids.items()}
+ return TraceExportResult(trace_id=str(trace_uuid), trace_url=urls[trace.root_span_id], span_urls=urls)
diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/configuration.py b/api/providers/trace/trace-weave/src/dify_trace_weave/configuration.py
new file mode 100644
index 00000000000..e5f92f7055f
--- /dev/null
+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/configuration.py
@@ -0,0 +1,250 @@
+"""Read-only Weave configuration checks without SDK login or global client state.
+
+Projects must already exist. Configuration verification never creates a project,
+trace, or local credential file; actual legacy exports retain their SDK lifecycle.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import quote, urlsplit, urlunsplit
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+from weave.trace_server.trace_server_interface import CallsQueryStatsReq, CallsQueryStatsRes
+
+from core.helper import ssrf_proxy
+from dify_trace_weave.config import WeaveConfig
+
+type VerificationCode = Literal[
+ "invalid_project",
+ "invalid_endpoint",
+ "authentication_failed",
+ "permission_denied",
+ "project_not_found_or_inaccessible",
+ "project_read_only",
+ "project_permission_unknown",
+ "graphql_error",
+ "malformed_response",
+ "endpoint_unavailable",
+ "connection_failed",
+]
+
+_MESSAGES: dict[VerificationCode, str] = {
+ "invalid_project": "Weave requires a valid entity and project name.",
+ "invalid_endpoint": "Weave endpoint is invalid.",
+ "authentication_failed": "Weave authentication failed.",
+ "permission_denied": "Weave permission was denied.",
+ "project_not_found_or_inaccessible": "Create the Weave project first or check project access.",
+ "project_read_only": "Weave project is read-only for these credentials.",
+ "project_permission_unknown": "Weave project write permission could not be verified.",
+ "graphql_error": "Weave project verification returned an error.",
+ "malformed_response": "Weave verification returned an invalid response.",
+ "endpoint_unavailable": "Weave endpoint could not be verified.",
+ "connection_failed": "Weave verification request failed.",
+}
+
+_VIEWER_QUERY = "query DifyWeaveViewer { viewer { id entity } }"
+_PROJECT_QUERY = """query DifyWeaveProject($entity: String!, $project: String!) {
+ viewer { id }
+ project(name: $project, entityName: $entity) { id name entityName readOnly }
+}"""
+_JSON_OBJECT = TypeAdapter(dict[str, object])
+
+
+class WeaveConfigurationError(ValueError):
+ """Fixed diagnostic codes: never include keys, request data, or server errors."""
+
+ def __init__(self, code: VerificationCode) -> None:
+ self.code = code
+ super().__init__(_MESSAGES[code])
+
+
+@dataclass(frozen=True, slots=True)
+class VerifiedWeaveProject:
+ entity: str
+ project: str
+ project_url: str
+
+
+class _Viewer(BaseModel):
+ id: str = Field(min_length=1)
+ entity: str | None = None
+ model_config = ConfigDict(strict=True)
+
+
+class _Project(BaseModel):
+ id: str = Field(min_length=1)
+ name: str = Field(min_length=1)
+ entity_name: str = Field(alias="entityName", min_length=1)
+ read_only: bool | None = Field(alias="readOnly")
+ model_config = ConfigDict(strict=True)
+
+
+def _base_url(value: str, *, allow_http: bool) -> str:
+ try:
+ parsed = urlsplit(value)
+ valid = (
+ parsed.scheme in ({"https", "http"} if allow_http else {"https"})
+ and parsed.hostname
+ and parsed.username is None
+ and parsed.password is None
+ and not parsed.query
+ and not parsed.fragment
+ and "\\" not in value
+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ )
+ _ = parsed.port
+ except ValueError:
+ raise WeaveConfigurationError("invalid_endpoint") from None
+ if not valid:
+ raise WeaveConfigurationError("invalid_endpoint")
+ return urlunsplit(parsed).rstrip("/")
+
+
+def _name(value: str) -> str:
+ value = value.strip()
+ if (
+ not value
+ or value in {".", ".."}
+ or "/" in value
+ or "\\" in value
+ or any(ord(char) < 32 or ord(char) == 127 for char in value)
+ ):
+ raise WeaveConfigurationError("invalid_project")
+ return value
+
+
+def _target(config: WeaveConfig) -> tuple[str | None, str]:
+ entity = config.entity.strip() if config.entity else None
+ project = config.project.strip()
+ if "/" in project:
+ parts = project.split("/")
+ if entity or len(parts) != 2:
+ raise WeaveConfigurationError("invalid_project")
+ entity, project = _name(parts[0]), parts[1]
+ return (_name(entity) if entity else None), _name(project)
+
+
+def console_url(config: WeaveConfig) -> str:
+ """Map the public API origin to its UI; preserve private origins and prefixes."""
+ host = _base_url(config.host or "https://api.wandb.ai", allow_http=True)
+ parsed = urlsplit(host)
+ if (
+ parsed.scheme == "https"
+ and parsed.hostname == "api.wandb.ai"
+ and parsed.port in {None, 443}
+ and not parsed.path
+ ):
+ return "https://wandb.ai"
+ return host
+
+
+def project_url(config: WeaveConfig) -> str | None:
+ """Pure presentation fallback; never resolve a default entity over the network."""
+ entity, project = _target(config)
+ if entity is None:
+ return None
+ return f"{console_url(config)}/{quote(entity, safe='')}/{quote(project, safe='')}"
+
+
+def _post(url: str, *, api_key: str, body: dict[str, object]) -> dict[str, object]:
+ try:
+ response = ssrf_proxy.post(
+ url,
+ auth=("api", api_key),
+ json=body,
+ max_retries=0,
+ timeout=30.0,
+ follow_redirects=False,
+ )
+ except Exception:
+ raise WeaveConfigurationError("connection_failed") from None
+ try:
+ if response.status_code == 401:
+ raise WeaveConfigurationError("authentication_failed")
+ if response.status_code == 403:
+ raise WeaveConfigurationError("permission_denied")
+ if not 200 <= response.status_code < 300:
+ raise WeaveConfigurationError("endpoint_unavailable")
+ try:
+ return _JSON_OBJECT.validate_python(response.json(), strict=True)
+ except ValueError:
+ raise WeaveConfigurationError("malformed_response") from None
+ finally:
+ response.close()
+
+
+def _graphql(host: str, *, api_key: str, body: dict[str, object]) -> dict[str, object]:
+ response = _post(host + "/graphql", api_key=api_key, body=body)
+ errors = response.get("errors")
+ if errors is not None:
+ if not isinstance(errors, list):
+ raise WeaveConfigurationError("malformed_response")
+ if errors:
+ raise WeaveConfigurationError("graphql_error")
+ try:
+ return _JSON_OBJECT.validate_python(response.get("data"), strict=True)
+ except ValidationError:
+ raise WeaveConfigurationError("malformed_response") from None
+
+
+def verify_configuration(config: WeaveConfig) -> VerifiedWeaveProject:
+ """Verify credentials, an existing writable project, and the trace read endpoint.
+
+ The GraphQL fields match W&B's Viewer, GetProject and IsProjectReadOnly
+ queries. CallsQueryStats uses the same typed read API as the Weave SDK.
+ These checks do not write a test trace or replace ingestion authorization.
+ """
+ if not config.api_key.strip():
+ raise WeaveConfigurationError("authentication_failed")
+ entity, project = _target(config)
+ host = _base_url(config.host or "https://api.wandb.ai", allow_http=True)
+ trace_endpoint = _base_url(config.endpoint, allow_http=False)
+ try:
+ if entity is None:
+ data = _graphql(host, api_key=config.api_key, body={"query": _VIEWER_QUERY})
+ if "viewer" not in data:
+ raise WeaveConfigurationError("malformed_response")
+ if data.get("viewer") is None:
+ raise WeaveConfigurationError("authentication_failed")
+ viewer = _Viewer.model_validate(data["viewer"])
+ if not viewer.entity:
+ raise WeaveConfigurationError("invalid_project")
+ entity = _name(viewer.entity)
+
+ data = _graphql(
+ host,
+ api_key=config.api_key,
+ body={"query": _PROJECT_QUERY, "variables": {"entity": entity, "project": project}},
+ )
+ if "viewer" not in data or "project" not in data:
+ raise WeaveConfigurationError("malformed_response")
+ if data.get("viewer") is None:
+ raise WeaveConfigurationError("authentication_failed")
+ _Viewer.model_validate(data["viewer"])
+ if data.get("project") is None:
+ raise WeaveConfigurationError("project_not_found_or_inaccessible")
+ resolved = _Project.model_validate(data["project"])
+ if resolved.name != project or resolved.entity_name != entity:
+ raise WeaveConfigurationError("invalid_project")
+ if resolved.read_only is True:
+ raise WeaveConfigurationError("project_read_only")
+ if resolved.read_only is None:
+ raise WeaveConfigurationError("project_permission_unknown")
+
+ stats_request = CallsQueryStatsReq(project_id=f"{entity}/{project}", limit=1)
+ stats_response = _post(
+ trace_endpoint + "/calls/query_stats",
+ api_key=config.api_key,
+ body=stats_request.model_dump(mode="json", exclude_none=True),
+ )
+ stats = CallsQueryStatsRes.model_validate(stats_response, strict=True)
+ if stats.count < 0:
+ raise WeaveConfigurationError("malformed_response")
+ except ValidationError:
+ raise WeaveConfigurationError("malformed_response") from None
+
+ return VerifiedWeaveProject(
+ entity=entity,
+ project=project,
+ project_url=f"{console_url(config)}/{quote(entity, safe='')}/{quote(project, safe='')}",
+ )
diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/transport.py b/api/providers/trace/trace-weave/src/dify_trace_weave/transport.py
new file mode 100644
index 00000000000..ed578b5d768
--- /dev/null
+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/transport.py
@@ -0,0 +1,56 @@
+"""Instance-authenticated Weave SDK transport shared by managed and legacy tracing."""
+
+from collections.abc import Mapping
+from typing import Any, override
+
+from pydantic import BaseModel
+from weave.trace_server_bindings.remote_http_trace_server import RemoteHTTPTraceServer
+
+from dify_trace_weave.config import WeaveConfig
+from dify_trace_weave.configuration import console_url
+from services.tracing.provider_common import endpoint, request, required
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError
+
+
+class TraceServer(RemoteHTTPTraceServer):
+ """Use SDK request/response validation without SDK retries or global W&B login."""
+
+ @override
+ def _post_request_executor(self, url: str, req: BaseModel, stream: bool = False):
+ return request(
+ "POST",
+ self.trace_server_url + url,
+ "weave",
+ auth=self._auth,
+ content=req.model_dump_json(by_alias=True).encode(),
+ headers={"Content-Type": "application/json"},
+ )
+
+ @override
+ def _generic_request(self, url, req, req_model, res_model, method="POST", params=None):
+ response = self._post_request_executor(url, req)
+ try:
+ return res_model.model_validate(response.json())
+ except ValueError:
+ raise TraceExportUncertainError("Weave returned an unreadable ingestion acknowledgment") from None
+
+
+def resolve_project(config: Mapping[str, Any], api_key: str) -> tuple[str, str, str]:
+ project = required(config, "project")
+ host = endpoint(config.get("host") or "https://api.wandb.ai")
+ entity = config.get("entity")
+ if not entity and "/" in project:
+ entity, project = project.split("/", 1)
+ if not entity:
+ response = request(
+ "POST",
+ host + "/graphql",
+ "weave",
+ auth=("api", api_key),
+ json={"query": "query DifyTraceEntity { viewer { entity } }"},
+ )
+ entity = (response.json().get("data") or {}).get("viewer", {}).get("entity")
+ if not isinstance(entity, str) or not entity or "/" in entity or not project or "/" in project:
+ raise TraceExportRejectedError("Weave project or entity could not be resolved")
+ ui_host = console_url(WeaveConfig(api_key=api_key, entity=entity, project=project, host=host))
+ return entity, project, ui_host
diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py b/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py
index 8a9fbefe4ab..6766536203e 100644
--- a/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py
+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py
@@ -1,11 +1,10 @@
import logging
import os
import uuid
+from contextvars import ContextVar
from datetime import UTC, datetime, timedelta
from typing import Any, cast, override
-import wandb
-import weave
from sqlalchemy.orm import sessionmaker
from weave.trace_server.trace_server_interface import (
CallEndReq,
@@ -28,14 +27,30 @@ from core.ops.entities.trace_entity import (
TraceTaskName,
WorkflowTraceInfo,
)
+from core.ops.exceptions import RetryableTraceDispatchError
from core.repositories import DifyCoreRepositoryFactory
from dify_trace_weave.config import WeaveConfig
+from dify_trace_weave.configuration import console_url, project_url, verify_configuration
from dify_trace_weave.entities.weave_trace_entity import WeaveTraceModel
+from dify_trace_weave.transport import TraceServer, resolve_project
from extensions.ext_database import db
from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey
from models import EndUser, MessageFile, WorkflowNodeExecutionTriggeredFrom
+from services.tracing.provider_common import endpoint
+from services.tracing.provider_dispatch import TraceExportUncertainError
logger = logging.getLogger(__name__)
+_dispatch_started_calls: ContextVar[dict[str, dict[str, Any]] | None] = ContextVar("weave_dispatch_calls", default=None)
+
+
+def _same_identifier(actual: str, expected: str) -> bool:
+ """Accept UUID formatting differences without rewriting legacy wire identifiers."""
+ if actual == expected:
+ return True
+ try:
+ return uuid.UUID(actual) == uuid.UUID(expected)
+ except ValueError:
+ return False
class WeaveDataTrace(BaseTraceInstance):
@@ -44,60 +59,65 @@ class WeaveDataTrace(BaseTraceInstance):
weave_config: WeaveConfig,
):
super().__init__(weave_config)
- self.weave_api_key = weave_config.api_key
- self.project_name = weave_config.project
- self.entity = weave_config.entity
- self.host = weave_config.host
-
- # Login with API key first, including host if provided
- if self.host:
- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True, host=self.host)
- else:
- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True)
-
- if not login_status:
- logger.error("Failed to login to Weights & Biases with the provided API key")
- raise ValueError("Weave login failed")
-
- # Then initialize weave client
- self.weave_client = weave.init(
- project_name=(f"{self.entity}/{self.project_name}" if self.entity else self.project_name)
+ # Construction is local: no global SDK client, login, netrc, or background queue.
+ self._config = weave_config.model_copy(deep=True)
+ self.weave_api_key = self._config.api_key
+ self.project_name = self._config.project
+ self.entity = self._config.entity
+ self.host = self._config.host
+ self.server = TraceServer( # pyrefly: ignore[bad-instantiation]
+ endpoint(self._config.endpoint), should_batch=False, auth=("api", self.weave_api_key)
)
+ self._project_id: str | None = None
self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001")
self.calls: dict[str, Any] = {}
- self.project_id = f"{self.weave_client.entity}/{self.weave_client.project}"
- def get_project_url(
- self,
- ):
- try:
- project_identifier = f"{self.entity}/{self.project_name}" if self.entity else self.project_name
- project_url = f"https://wandb.ai/{project_identifier}"
- return project_url
- except Exception as e:
- logger.debug("Weave get run url failed", exc_info=True)
- raise ValueError(f"Weave get run url failed: {str(e)}") from e
+ @property
+ def project_id(self) -> str:
+ # Existing saved configurations may omit entity. Resolve it lazily with
+ # this instance's credentials, never W&B's process-global defaults.
+ if self._project_id is None:
+ entity, project, _ = resolve_project(self._config.model_dump(), self.weave_api_key)
+ self._project_id = f"{entity}/{project}"
+ return self._project_id
+
+ def get_project_url(self):
+ return project_url(self._config) or console_url(self._config)
@override
def trace(self, trace_info: BaseTraceInfo):
logger.debug("Trace info: %s", trace_info)
- match trace_info:
- case WorkflowTraceInfo():
- self.workflow_trace(trace_info)
- case MessageTraceInfo():
- self.message_trace(trace_info)
- case ModerationTraceInfo():
- self.moderation_trace(trace_info)
- case SuggestedQuestionTraceInfo():
- self.suggested_question_trace(trace_info)
- case DatasetRetrievalTraceInfo():
- self.dataset_retrieval_trace(trace_info)
- case ToolTraceInfo():
- self.tool_trace(trace_info)
- case GenerateNameTraceInfo():
- self.generate_name_trace(trace_info)
- case _:
- pass
+ accepted_calls: dict[str, dict[str, Any]] = {}
+ token = _dispatch_started_calls.set(accepted_calls)
+ try:
+ match trace_info:
+ case WorkflowTraceInfo():
+ self.workflow_trace(trace_info)
+ case MessageTraceInfo():
+ self.message_trace(trace_info)
+ case ModerationTraceInfo():
+ self.moderation_trace(trace_info)
+ case SuggestedQuestionTraceInfo():
+ self.suggested_question_trace(trace_info)
+ case DatasetRetrievalTraceInfo():
+ self.dataset_retrieval_trace(trace_info)
+ case ToolTraceInfo():
+ self.tool_trace(trace_info)
+ case GenerateNameTraceInfo():
+ self.generate_name_trace(trace_info)
+ case _:
+ pass
+ except RetryableTraceDispatchError as error:
+ if accepted_calls and not error.acceptance_unknown:
+ raise TraceExportUncertainError(
+ "Weave trace was partially accepted; automatic retry is unsafe"
+ ) from None
+ raise
+ finally:
+ for call_id, call_meta in accepted_calls.items():
+ if self.calls.get(call_id) is call_meta:
+ self.calls.pop(call_id, None)
+ _dispatch_started_calls.reset(token)
def workflow_trace(self, trace_info: WorkflowTraceInfo):
trace_id = trace_info.trace_id or trace_info.message_id or trace_info.workflow_run_id
@@ -421,20 +441,9 @@ class WeaveDataTrace(BaseTraceInstance):
self.finish_call(name_run)
def api_check(self):
- try:
- if self.host:
- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True, host=self.host)
- else:
- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True)
-
- if not login_status:
- raise ValueError("Weave login failed")
- else:
- logger.info("Weave login successful")
- return True
- except Exception as e:
- logger.debug("Weave API check failed", exc_info=True)
- raise ValueError(f"Weave API check failed: {str(e)}") from e
+ # Read-only credential/project/endpoint verification; it creates no calls.
+ verify_configuration(self._config)
+ return True
def _normalize_time(self, dt: datetime | None) -> datetime:
if dt is None:
@@ -475,8 +484,14 @@ class WeaveDataTrace(BaseTraceInstance):
wb_user_id=None,
)
)
- self.weave_client.server.call_start(call_start_req)
- self.calls[run_data.id] = {"trace_id": trace_id, "parent_id": parent_run_id}
+ started = self.server.call_start(call_start_req)
+ if not _same_identifier(started.id, run_data.id) or not _same_identifier(started.trace_id, trace_id):
+ raise TraceExportUncertainError("Weave did not acknowledge the submitted identifiers")
+ call_meta = {"trace_id": trace_id, "parent_id": parent_run_id}
+ self.calls[run_data.id] = call_meta
+ dispatch_calls = _dispatch_started_calls.get()
+ if dispatch_calls is not None:
+ dispatch_calls[run_data.id] = call_meta
def finish_call(self, run_data: WeaveTraceModel):
call_meta = self.calls.get(run_data.id)
@@ -523,4 +538,7 @@ class WeaveDataTrace(BaseTraceInstance):
summary=cast(SummaryInsertMap, summary),
)
)
- self.weave_client.server.call_end(call_end_req)
+ try:
+ self.server.call_end(call_end_req)
+ finally:
+ self.calls.pop(run_data.id, None)
diff --git a/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py b/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py
index 4c0abbc8e4c..b836b06a861 100644
--- a/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py
+++ b/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py
@@ -8,9 +8,10 @@ from unittest.mock import MagicMock, patch
import pytest
from dify_trace_weave.config import WeaveConfig
+from dify_trace_weave.configuration import WeaveConfigurationError
from dify_trace_weave.entities.weave_trace_entity import WeaveTraceModel
from dify_trace_weave.weave_trace import WeaveDataTrace
-from weave.trace_server.trace_server_interface import TraceStatus
+from weave.trace_server.trace_server_interface import CallStartRes, TraceStatus
from core.ops.entities.trace_entity import (
DatasetRetrievalTraceInfo,
@@ -190,138 +191,65 @@ def _make_node(**overrides):
@pytest.fixture
-def mock_wandb():
- with patch("dify_trace_weave.weave_trace.wandb") as mock:
- mock.login.return_value = True
- yield mock
+def mock_server():
+ with patch("dify_trace_weave.weave_trace.TraceServer") as factory:
+ server = factory.return_value
+ server.call_start.side_effect = lambda req: CallStartRes(id=req.start.id, trace_id=req.start.trace_id)
+ yield factory
@pytest.fixture
-def mock_weave():
- with patch("dify_trace_weave.weave_trace.weave") as mock:
- client = MagicMock()
- client.entity = "my-entity"
- client.project = "my-project"
- mock.init.return_value = client
- yield mock, client
+def trace_instance(mock_server):
+ return WeaveDataTrace(_make_weave_config())
@pytest.fixture
-def trace_instance(mock_wandb, mock_weave):
- """Create a WeaveDataTrace instance with mocked wandb/weave."""
- _, weave_client = mock_weave
- config = _make_weave_config()
- instance = WeaveDataTrace(config)
- return instance
-
-
-def trace_instance_with_host(mock_wandb, mock_weave):
- """Create a WeaveDataTrace instance with host configured."""
- _, weave_client = mock_weave
- config = _make_weave_config(host="https://my.wandb.host")
- instance = WeaveDataTrace(config)
- return instance
-
-
-# ── TestInit ─────────────────────────────────────────────────────────────────
+def trace_instance_with_host(mock_server):
+ return WeaveDataTrace(_make_weave_config(host="https://my.wandb.host"))
class TestInit:
- def test_init_without_host(self, mock_wandb, mock_weave):
- """Test __init__ calls wandb.login without host."""
- mock_w, weave_client = mock_weave
- config = _make_weave_config(host=None)
- instance = WeaveDataTrace(config)
-
- mock_wandb.login.assert_called_once_with(key="wv-api-key", verify=True, relogin=True)
- mock_w.init.assert_called_once_with(project_name="my-entity/my-project")
+ def test_init_has_explicit_endpoint_and_auth_without_batching(self, mock_server):
+ instance = WeaveDataTrace(_make_weave_config(endpoint="https://trace.test/prefix"))
+ mock_server.assert_called_once_with("https://trace.test/prefix", should_batch=False, auth=("api", "wv-api-key"))
assert instance.weave_api_key == "wv-api-key"
- assert instance.project_name == "my-project"
- assert instance.entity == "my-entity"
+ assert instance.project_id == "my-entity/my-project"
assert instance.calls == {}
-
- def test_init_with_host(self, mock_wandb, mock_weave):
- """Test __init__ calls wandb.login with host."""
- config = _make_weave_config(host="https://my.wandb.host")
- instance = WeaveDataTrace(config)
-
- mock_wandb.login.assert_called_once_with(
- key="wv-api-key", verify=True, relogin=True, host="https://my.wandb.host"
- )
- assert instance.host == "https://my.wandb.host"
-
- def test_init_with_host_path(self, mock_wandb, mock_weave):
- """A self-hosted host keeps its path prefix all the way to wandb.login."""
- config = _make_weave_config(host="https://wandb.internal/api")
- instance = WeaveDataTrace(config)
-
- mock_wandb.login.assert_called_once_with(
- key="wv-api-key", verify=True, relogin=True, host="https://wandb.internal/api"
- )
- assert instance.host == "https://wandb.internal/api"
-
- def test_init_without_entity(self, mock_wandb, mock_weave):
- """Test __init__ initializes weave without entity prefix when entity is None."""
- mock_w, weave_client = mock_weave
- config = _make_weave_config(entity=None)
- WeaveDataTrace(config)
-
- mock_w.init.assert_called_once_with(project_name="my-project")
-
- def test_init_login_failure_raises(self, mock_wandb, mock_weave):
- """Test __init__ raises ValueError when wandb.login returns False."""
- mock_wandb.login.return_value = False
- config = _make_weave_config()
-
- with pytest.raises(ValueError, match="Weave login failed"):
- WeaveDataTrace(config)
-
- def test_init_files_url_from_env(self, mock_wandb, mock_weave, monkeypatch: pytest.MonkeyPatch):
- """Test FILES_URL is read from environment."""
+ mock_server.return_value.call_start.assert_not_called()
+
+ @pytest.mark.parametrize("host", [None, "https://my.wandb.host", "https://wandb.internal/api"])
+ def test_init_preserves_host(self, mock_server, host):
+ instance = WeaveDataTrace(_make_weave_config(host=host))
+ assert instance.host == host
+
+ def test_init_without_entity_does_not_resolve_over_network(self, mock_server):
+ with patch("dify_trace_weave.weave_trace.resolve_project") as resolve:
+ instance = WeaveDataTrace(_make_weave_config(entity=None))
+ resolve.assert_not_called()
+ assert instance.get_project_url() == "https://wandb.ai"
+ resolve.assert_not_called()
+
+ def test_qualified_project_does_not_require_default_entity(self, mock_server):
+ instance = WeaveDataTrace(_make_weave_config(entity=None, project="team/project"))
+ assert instance.project_id == "team/project"
+ assert instance.get_project_url() == "https://wandb.ai/team/project"
+
+ def test_init_files_url_from_env(self, mock_server, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("FILES_URL", "http://files.example.com")
- config = _make_weave_config()
- instance = WeaveDataTrace(config)
- assert instance.file_base_url == "http://files.example.com"
+ assert WeaveDataTrace(_make_weave_config()).file_base_url == "http://files.example.com"
- def test_init_files_url_default(self, mock_wandb, mock_weave, monkeypatch: pytest.MonkeyPatch):
- """Test FILES_URL defaults to http://127.0.0.1:5001."""
+ def test_init_files_url_default(self, mock_server, monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("FILES_URL", raising=False)
- config = _make_weave_config()
- instance = WeaveDataTrace(config)
- assert instance.file_base_url == "http://127.0.0.1:5001"
-
- def test_project_id_set_correctly(self, trace_instance):
- """Test that project_id is set from weave_client entity/project."""
- assert trace_instance.project_id == "my-entity/my-project"
-
-
-# ── TestGetProjectUrl ─────────────────────────────────────────────────────────
+ assert WeaveDataTrace(_make_weave_config()).file_base_url == "http://127.0.0.1:5001"
class TestGetProjectUrl:
def test_get_project_url_with_entity(self, trace_instance):
- """Returns wandb URL with entity/project."""
- url = trace_instance.get_project_url()
- assert url == "https://wandb.ai/my-entity/my-project"
-
- def test_get_project_url_without_entity(self, mock_wandb, mock_weave):
- """Returns wandb URL with project only when entity is None."""
- config = _make_weave_config(entity=None)
- instance = WeaveDataTrace(config)
- url = instance.get_project_url()
- assert url == "https://wandb.ai/my-project"
-
- def test_get_project_url_exception_raises(self, trace_instance, monkeypatch: pytest.MonkeyPatch):
- """Raises ValueError when exception occurs in get_project_url."""
- monkeypatch.setattr(trace_instance, "entity", None)
- monkeypatch.setattr(trace_instance, "project_name", None)
- # Force an error by making string formatting fail
- # Simulate exception via property
- trace_instance.entity = None
- trace_instance.project_name = None
- url = trace_instance.get_project_url()
- assert "https://wandb.ai/" in url
+ assert trace_instance.get_project_url() == "https://wandb.ai/my-entity/my-project"
+
+ def test_get_project_url_preserves_private_prefix(self, mock_server):
+ instance = WeaveDataTrace(_make_weave_config(host="https://wandb.internal/prefix"))
+ assert instance.get_project_url() == "https://wandb.internal/prefix/my-entity/my-project"
# ── TestTraceDispatcher ─────────────────────────────────────────────────────
@@ -408,7 +336,7 @@ class TestStartCall:
assert "run-1" in trace_instance.calls
assert trace_instance.calls["run-1"]["trace_id"] == "t-1"
assert trace_instance.calls["run-1"]["parent_id"] is None
- trace_instance.weave_client.server.call_start.assert_called_once()
+ trace_instance.server.call_start.assert_called_once()
def test_start_call_with_parent(self, trace_instance):
"""Test start_call records parent_run_id."""
@@ -431,7 +359,7 @@ class TestStartCall:
attributes={"trace_id": "t-2", "start_time": _dt()},
)
trace_instance.start_call(run)
- call_args = trace_instance.weave_client.server.call_start.call_args
+ call_args = trace_instance.server.call_start.call_args
req = call_args[0][0]
assert req.start.inputs == {}
@@ -444,7 +372,7 @@ class TestStartCall:
attributes={"trace_id": "t-3", "start_time": _dt()},
)
trace_instance.start_call(run)
- call_args = trace_instance.weave_client.server.call_start.call_args
+ call_args = trace_instance.server.call_start.call_args
req = call_args[0][0]
# String inputs gets converted by validator to a dict
assert isinstance(req.start.inputs, dict)
@@ -472,7 +400,7 @@ class TestStartCall:
# Manually override after construction
run.attributes = "some-attr-string"
trace_instance.start_call(run)
- call_args = trace_instance.weave_client.server.call_start.call_args
+ call_args = trace_instance.server.call_start.call_args
req = call_args[0][0]
assert isinstance(req.start.attributes, dict)
assert req.start.attributes == {"attributes": "some-attr-string"}
@@ -509,8 +437,8 @@ class TestFinishCall:
exception=None,
)
trace_instance.finish_call(run)
- trace_instance.weave_client.server.call_end.assert_called_once()
- call_args = trace_instance.weave_client.server.call_end.call_args
+ trace_instance.server.call_end.assert_called_once()
+ call_args = trace_instance.server.call_end.call_args
req = call_args[0][0]
assert req.end.summary["status_counts"][TraceStatus.SUCCESS] == 1
assert req.end.summary["status_counts"][TraceStatus.ERROR] == 0
@@ -528,7 +456,7 @@ class TestFinishCall:
exception="Something broke",
)
trace_instance.finish_call(run)
- call_args = trace_instance.weave_client.server.call_end.call_args
+ call_args = trace_instance.server.call_end.call_args
req = call_args[0][0]
assert req.end.summary["status_counts"][TraceStatus.ERROR] == 1
assert req.end.summary["status_counts"][TraceStatus.SUCCESS] == 0
@@ -557,7 +485,7 @@ class TestFinishCall:
},
)
trace_instance.finish_call(run)
- call_args = trace_instance.weave_client.server.call_end.call_args
+ call_args = trace_instance.server.call_end.call_args
req = call_args[0][0]
assert req.end.summary["weave"]["latency_ms"] == 0
@@ -571,7 +499,7 @@ class TestFinishCall:
attributes=None,
)
trace_instance.finish_call(run)
- trace_instance.weave_client.server.call_end.assert_called_once()
+ trace_instance.server.call_end.assert_called_once()
def test_finish_call_non_dict_attributes(self, trace_instance):
"""Test finish_call handles non-dict attributes."""
@@ -584,7 +512,7 @@ class TestFinishCall:
)
run.attributes = "some string attr"
trace_instance.finish_call(run)
- trace_instance.weave_client.server.call_end.assert_called_once()
+ trace_instance.server.call_end.assert_called_once()
# ── TestWorkflowTrace ─────────────────────────────────────────────────────────
@@ -1165,40 +1093,17 @@ class TestGenerateNameTrace:
class TestApiCheck:
- def test_api_check_success_without_host(self, trace_instance, mock_wandb):
- """api_check returns True on successful login without host."""
- trace_instance.host = None
- mock_wandb.login.return_value = True
-
- result = trace_instance.api_check()
-
- assert result is True
- mock_wandb.login.assert_called_with(key=trace_instance.weave_api_key, verify=True, relogin=True)
-
- def test_api_check_success_with_host(self, trace_instance, mock_wandb):
- """api_check returns True on successful login with host."""
- trace_instance.host = "https://my.wandb.host"
- mock_wandb.login.return_value = True
-
- result = trace_instance.api_check()
-
- assert result is True
- mock_wandb.login.assert_called_with(
- key=trace_instance.weave_api_key, verify=True, relogin=True, host="https://my.wandb.host"
- )
-
- def test_api_check_login_failure_raises(self, trace_instance, mock_wandb):
- """api_check raises ValueError when login returns False."""
- trace_instance.host = None
- mock_wandb.login.return_value = False
-
- with pytest.raises(ValueError, match="Weave API check failed"):
- trace_instance.api_check()
-
- def test_api_check_exception_raises_value_error(self, trace_instance, mock_wandb):
- """api_check raises ValueError when wandb.login raises exception."""
- trace_instance.host = None
- mock_wandb.login.side_effect = Exception("network error")
-
- with pytest.raises(ValueError, match="Weave API check failed: network error"):
- trace_instance.api_check()
+ def test_api_check_uses_read_only_verification(self, trace_instance):
+ with patch("dify_trace_weave.weave_trace.verify_configuration") as verify:
+ assert trace_instance.api_check() is True
+ verify.assert_called_once_with(trace_instance._config)
+ trace_instance.server.call_start.assert_not_called()
+ trace_instance.server.call_end.assert_not_called()
+
+ def test_api_check_propagates_safe_failure(self, trace_instance):
+ with patch(
+ "dify_trace_weave.weave_trace.verify_configuration",
+ side_effect=WeaveConfigurationError("authentication_failed"),
+ ):
+ with pytest.raises(WeaveConfigurationError, match="Weave authentication failed"):
+ trace_instance.api_check()
diff --git a/api/repositories/app/agent_app_repository.py b/api/repositories/app/agent_app_repository.py
index 328d0e32d7a..fef158cd315 100644
--- a/api/repositories/app/agent_app_repository.py
+++ b/api/repositories/app/agent_app_repository.py
@@ -18,6 +18,14 @@ class AgentAppRepository:
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
self._session_factory = session_factory
+ def resolve_roster_app_id(self, *, tenant_id: str, agent_id: str) -> str:
+ """Resolve a roster Agent's configuration owner without materializing inline Apps."""
+ with self._session_factory() as session:
+ try:
+ return AgentRosterService(session).get_agent_app_model(tenant_id=tenant_id, agent_id=agent_id).id
+ except AgentNotFoundError as exc:
+ raise AgentAppNotFoundError from exc
+
def resolve_runtime_app_id(self, *, tenant_id: str, agent_id: str) -> str:
with self._session_factory() as session:
try:
diff --git a/api/repositories/app/console_repository.py b/api/repositories/app/console_repository.py
index 64fa0844d09..08787fde139 100644
--- a/api/repositories/app/console_repository.py
+++ b/api/repositories/app/console_repository.py
@@ -7,7 +7,6 @@ existing use cases that still need attached models.
from __future__ import annotations
-import json
import logging
from collections.abc import Sequence
from dataclasses import replace
@@ -50,6 +49,7 @@ from services.agent.errors import (
from services.app.console_service import ConsoleAppNotFoundError, ConsoleApps
from services.app.query_service import AppDescription, DiscoveryApps, RelatedApps
from services.app_creation_records import create_installed_app_record, create_site_record
+from services.app_tracing_config_service import AppTraceSettings
from services.entities.app_entities import (
RECENT_APP_MODES,
AppAgentBinding,
@@ -65,7 +65,6 @@ from services.entities.app_entities import (
AppRecord,
AppReference,
AppSummary,
- AppTraceSettings,
AppUpdateArguments,
CreateAppParams,
RecentAppListItem,
@@ -326,12 +325,6 @@ class ConsoleAppRepository(ConsoleApps, DiscoveryApps, RelatedApps):
app = require_console_app(session, context, app_id)
return _app_trace_settings_adapter.validate_json(app.tracing) if app.tracing else AppTraceSettings()
- @override
- def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:
- with self._session_factory.begin() as session:
- app = require_console_app(session, context, app_id)
- app.tracing = json.dumps({"enabled": settings.enabled, "tracing_provider": settings.tracing_provider})
-
@staticmethod
def _draft_trigger_app_ids(session: Session, workspace_id: str, apps: list[App]) -> set[str]:
workflow_app_ids = [app.id for app in apps if app.mode in {"workflow", "advanced-chat"}]
diff --git a/api/repositories/app/tracing_config_repository.py b/api/repositories/app/tracing_config_repository.py
index 467380abdd9..afa4968b3b8 100644
--- a/api/repositories/app/tracing_config_repository.py
+++ b/api/repositories/app/tracing_config_repository.py
@@ -1,7 +1,10 @@
"""SQLAlchemy persistence adapter for app tracing provider configurations."""
+import json
+from dataclasses import asdict, replace
from typing import Any, override
+from pydantic import TypeAdapter
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
@@ -9,11 +12,16 @@ from models.enums import AppStatus
from models.model import App, TraceAppConfig
from repositories.app.console_visibility import console_visible_condition
from services.app_tracing_config_service import (
+ AppTraceSettings,
AppTracingConfigAppNotFoundError,
+ AppTracingConfigConflictError,
+ AppTracingConfigNotFoundError,
AppTracingConfigRecord,
AppTracingConfigStore,
)
+_trace_settings = TypeAdapter(AppTraceSettings)
+
class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
@@ -40,9 +48,10 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):
app_id: str,
tracing_provider: str,
tracing_config: dict[str, Any],
+ activate: bool = False,
) -> bool:
with self._session_factory.begin() as session:
- self._require_app(session, workspace_id, app_id)
+ app = self._require_app(session, workspace_id, app_id, lock=True)
if self._get_config(session, app_id, tracing_provider) is not None:
return False
@@ -53,6 +62,8 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):
tracing_config=dict(tracing_config),
)
)
+ if activate:
+ self._activate(app, tracing_provider)
return True
@override
@@ -63,14 +74,20 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):
app_id: str,
tracing_provider: str,
tracing_config: dict[str, Any],
+ expected: AppTracingConfigRecord,
+ activate: bool = False,
) -> bool:
with self._session_factory.begin() as session:
- self._require_app(session, workspace_id, app_id)
+ app = self._require_app(session, workspace_id, app_id, lock=True)
config = self._get_config(session, app_id, tracing_provider)
if config is None:
- return False
+ raise AppTracingConfigConflictError
+ self._check_expected(config, expected)
config.tracing_config = dict(tracing_config)
+ if activate:
+ config.is_active = True
+ self._activate(app, tracing_provider)
return True
@override
@@ -82,28 +99,65 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):
tracing_provider: str,
) -> bool:
with self._session_factory.begin() as session:
- self._require_app(session, workspace_id, app_id)
+ app = self._require_app(session, workspace_id, app_id, lock=True)
config = self._get_config(session, app_id, tracing_provider)
if config is None:
return False
+ settings = self._settings(app)
+ if settings.tracing_provider == tracing_provider:
+ app.tracing = json.dumps(asdict(replace(settings, enabled=False, tracing_provider=None)))
session.delete(config)
return True
+ @override
+ def set_trace(
+ self, *, workspace_id: str, app_id: str, settings: AppTraceSettings, expected: AppTracingConfigRecord | None
+ ) -> None:
+ with self._session_factory.begin() as session:
+ app = self._require_app(session, workspace_id, app_id, lock=True)
+ if settings.enabled:
+ if settings.tracing_provider is None or expected is None:
+ raise AppTracingConfigNotFoundError
+ config = self._get_config(session, app_id, settings.tracing_provider)
+ if config is None:
+ raise AppTracingConfigConflictError
+ self._check_expected(config, expected)
+ if not config.is_active or not config.tracing_config:
+ raise AppTracingConfigNotFoundError
+ app.tracing = json.dumps(asdict(settings))
+
@staticmethod
- def _require_app(session: Session, workspace_id: str, app_id: str) -> None:
- app_exists = session.scalar(
- select(App.id)
- .where(
- App.id == app_id,
- App.tenant_id == workspace_id,
- App.status == AppStatus.NORMAL,
- console_visible_condition(),
- )
- .limit(1)
+ def _settings(app: App) -> AppTraceSettings:
+ return _trace_settings.validate_json(app.tracing) if app.tracing else AppTraceSettings()
+
+ @classmethod
+ def _activate(cls, app: App, provider: str) -> None:
+ app.tracing = json.dumps(asdict(replace(cls._settings(app), enabled=True, tracing_provider=provider)))
+
+ @staticmethod
+ def _check_expected(config: TraceAppConfig, expected: AppTracingConfigRecord) -> None:
+ if (config.id, config.tracing_config, config.is_active) != (
+ expected.id,
+ expected.tracing_config,
+ expected.is_active,
+ ):
+ raise AppTracingConfigConflictError
+
+ @staticmethod
+ def _require_app(session: Session, workspace_id: str, app_id: str, *, lock: bool = False) -> App:
+ statement = select(App).where(
+ App.id == app_id,
+ App.tenant_id == workspace_id,
+ App.status == AppStatus.NORMAL,
+ console_visible_condition(),
)
- if app_exists is None:
+ if lock:
+ statement = statement.with_for_update()
+ app = session.scalar(statement)
+ if app is None:
raise AppTracingConfigAppNotFoundError
+ return app
@staticmethod
def _get_config(session: Session, app_id: str, tracing_provider: str) -> TraceAppConfig | None:
diff --git a/api/repositories/application_trace_repository.py b/api/repositories/application_trace_repository.py
new file mode 100644
index 00000000000..653e66ba6e5
--- /dev/null
+++ b/api/repositories/application_trace_repository.py
@@ -0,0 +1,505 @@
+"""Bounded transactions for application trace collection and delivery leases."""
+
+import hashlib
+import json
+from datetime import timedelta
+from typing import Any, Literal
+from uuid import UUID, uuid5
+
+from dify_agent.protocol.tracing import AgentTraceBatch
+from sqlalchemy import delete, or_, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from libs.datetime_utils import naive_utc_now
+from libs.uuid_utils import uuidv7
+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun
+from models.model import App, Message, TraceAppConfig
+from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowRun
+from repositories.factory import DifyAPIRepositoryFactory
+from services.tracing.normalization import safe_attributes, span_id
+
+
+class TraceCollectionError(Exception):
+ def __init__(self, code: str, status: int = 400) -> None:
+ self.code = code
+ self.status = status
+ super().__init__(code)
+
+
+class ApplicationTraceRepository:
+ def __init__(self, sessions: sessionmaker[Session]) -> None:
+ self.sessions = sessions
+
+ def _source(
+ self, *, tenant_id: str, app_id: str, kind: str, source_id: str
+ ) -> Message | WorkflowRun | WorkflowNodeExecutionModel | None:
+ # Close the SQL owner read before consulting a configured backend: LogStore
+ # can perform network I/O and need not have a corresponding SQL run/node row.
+ with self.sessions() as session:
+ if session.scalar(select(App.id).where(App.id == app_id, App.tenant_id == tenant_id)) is None:
+ return None
+ if kind == "message":
+ return session.scalar(select(Message).where(Message.id == source_id, Message.app_id == app_id))
+ source: WorkflowRun | WorkflowNodeExecutionModel | None
+ if kind == "workflow":
+ repository = DifyAPIRepositoryFactory.create_api_workflow_run_repository(self.sessions)
+ source = repository.get_workflow_run_by_id(tenant_id=tenant_id, app_id=app_id, run_id=source_id)
+ elif kind == "node":
+ node_repository = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(self.sessions)
+ source = node_repository.get_execution_by_id(execution_id=source_id, tenant_id=tenant_id)
+ else:
+ return None
+ # The node reader has no app filter. Check the entire chain on either
+ # backend's result before exposing a destination or authorizing a retry.
+ if source is None or (source.id, source.tenant_id, source.app_id) != (source_id, tenant_id, app_id):
+ return None
+ return source
+
+ def start(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ source_kind: Literal["message", "workflow", "node"],
+ source_id: str,
+ session_id: str,
+ workflow_id: str | None = None,
+ ) -> ApplicationTraceRun | None:
+ # Re-entry uses the already validated, frozen source. It must not depend
+ # on LogStore availability or indexing latency after collection began.
+ with self.sessions() as session:
+ if session.scalar(select(App.id).where(App.id == app_id, App.tenant_id == tenant_id)) is None:
+ return None
+ existing = session.scalar(
+ select(ApplicationTraceRun).where(
+ ApplicationTraceRun.tenant_id == tenant_id,
+ ApplicationTraceRun.app_id == app_id,
+ ApplicationTraceRun.source_kind == source_kind,
+ ApplicationTraceRun.source_id == source_id,
+ )
+ )
+ if existing is not None:
+ return existing
+ source = self._source(tenant_id=tenant_id, app_id=app_id, kind=source_kind, source_id=source_id)
+ # The app row serializes parallel node starts, so one workflow freezes one
+ # destination even when its first two Agent nodes start concurrently.
+ with self.sessions.begin() as session:
+ app = session.scalar(select(App).where(App.id == app_id, App.tenant_id == tenant_id).with_for_update())
+ if app is None:
+ return None
+ existing = session.scalar(
+ select(ApplicationTraceRun).where(
+ ApplicationTraceRun.tenant_id == tenant_id,
+ ApplicationTraceRun.app_id == app_id,
+ ApplicationTraceRun.source_kind == source_kind,
+ ApplicationTraceRun.source_id == source_id,
+ )
+ )
+ if existing is not None:
+ return existing
+ if source is None:
+ # A single-node preview allocates its execution ID before the
+ # node row is saved. Only the trusted workflow builder may reserve
+ # it, after checking its persisted owner workflow here.
+ if source_kind not in {"node", "workflow"} or workflow_id is None:
+ return None
+ workflow = session.scalar(
+ select(Workflow).where(
+ Workflow.id == workflow_id, Workflow.app_id == app_id, Workflow.tenant_id == tenant_id
+ )
+ )
+ if workflow is None:
+ return None
+ try:
+ policy = json.loads(app.tracing) if app.tracing else {}
+ except (TypeError, ValueError):
+ policy = {}
+ if not isinstance(policy, dict):
+ policy = {}
+ provider = policy.get("tracing_provider")
+ config = (
+ session.scalar(
+ select(TraceAppConfig).where(
+ TraceAppConfig.app_id == app_id,
+ TraceAppConfig.tracing_provider == provider,
+ )
+ )
+ if isinstance(provider, str) and policy.get("enabled") is True
+ else None
+ )
+ enabled = config is not None and isinstance(config.tracing_config, dict)
+ frozen = (
+ dict(config.tracing_config) if config is not None and isinstance(config.tracing_config, dict) else {}
+ )
+ revision = hashlib.sha256(
+ json.dumps({"config": frozen, "policy": policy}, sort_keys=True, default=str).encode()
+ ).hexdigest()
+ record = ApplicationTraceRun(
+ id=str(uuidv7()),
+ tenant_id=tenant_id,
+ app_id=app_id,
+ source_kind=source_kind,
+ source_id=source_id,
+ provider=provider if enabled and isinstance(provider, str) else "",
+ config_revision=revision,
+ provider_config=frozen,
+ trace_id=uuidv7().hex,
+ root_span_id=span_id(source_id),
+ session_id=session_id[:255] or source_id,
+ capture_content=enabled and policy.get("capture_content", False) is True,
+ capture_agent=enabled and policy.get("capture_agent", True) is True,
+ state="collecting" if enabled else "not_collected",
+ )
+ session.add(record)
+ session.flush()
+ return record
+
+ def source_run(
+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str
+ ) -> ApplicationTraceRun | None:
+ with self.sessions() as session:
+ return session.scalar(
+ select(ApplicationTraceRun).where(
+ ApplicationTraceRun.tenant_id == tenant_id,
+ ApplicationTraceRun.app_id == app_id,
+ ApplicationTraceRun.source_kind == source_kind,
+ ApplicationTraceRun.source_id == source_id,
+ )
+ )
+
+ def record_initialization_failure(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ source_kind: str,
+ source_id: str,
+ ) -> ApplicationTraceRun | None:
+ """A missing start snapshot must never fall back to an unmasked exporter."""
+ source = self._source(tenant_id=tenant_id, app_id=app_id, kind=source_kind, source_id=source_id)
+ if source is None:
+ return None
+ with self.sessions.begin() as session:
+ app = session.scalar(select(App).where(App.id == app_id, App.tenant_id == tenant_id).with_for_update())
+ if app is None:
+ return None
+ current = session.scalar(
+ select(ApplicationTraceRun).where(
+ ApplicationTraceRun.tenant_id == tenant_id,
+ ApplicationTraceRun.app_id == app_id,
+ ApplicationTraceRun.source_kind == source_kind,
+ ApplicationTraceRun.source_id == source_id,
+ )
+ )
+ if current is not None:
+ return current
+ current = ApplicationTraceRun(
+ id=str(uuidv7()),
+ tenant_id=tenant_id,
+ app_id=app_id,
+ source_kind=source_kind,
+ source_id=source_id,
+ provider="",
+ config_revision="",
+ provider_config={},
+ trace_id=uuidv7().hex,
+ root_span_id=span_id(source_id),
+ session_id=source_id,
+ capture_content=False,
+ capture_agent=False,
+ state="failed",
+ error_code="tracing_initialization_failed",
+ terminal_at=naive_utc_now(),
+ )
+ session.add(current)
+ return current
+
+ def invocation(
+ self, *, run_id: str, agent_id: str, node_execution_id: str | None, attempt: int, call_key: str
+ ) -> ApplicationTraceInvocation:
+ identifier = str(uuid5(UUID(run_id), call_key))
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.state not in {"collecting", "ready"}:
+ raise TraceCollectionError("tracing_run_not_collecting", 409)
+ invocation = session.get(ApplicationTraceInvocation, identifier)
+ if invocation is not None and (
+ invocation.agent_id != agent_id
+ or invocation.node_execution_id != node_execution_id
+ or invocation.attempt != attempt
+ ):
+ raise TraceCollectionError("tracing_invocation_conflict", 409)
+ if invocation is None:
+ invocation = ApplicationTraceInvocation(
+ id=identifier,
+ run_id=run_id,
+ agent_id=agent_id,
+ parent_span_id=span_id(node_execution_id) if node_execution_id else run.root_span_id,
+ span_id=span_id(f"agent-attempt:{identifier}"),
+ node_execution_id=node_execution_id,
+ attempt=attempt,
+ )
+ session.add(invocation)
+ return invocation
+
+ def ingest(self, batch: AgentTraceBatch, *, max_spans: int, max_bytes: int) -> tuple[int, str]:
+ # Hash the original validated payload. Sanitization cannot turn two
+ # conflicting producer payloads into a falsely successful replay.
+ data = batch.model_dump(mode="json")
+ canonical = json.dumps(data, sort_keys=True, separators=(",", ":")).encode()
+ digest = hashlib.sha256(canonical).hexdigest()
+ with self.sessions.begin() as session:
+ run = session.scalar(
+ select(ApplicationTraceRun).where(ApplicationTraceRun.id == str(batch.run_id)).with_for_update()
+ )
+ if run is None:
+ raise TraceCollectionError("tracing_run_not_found", 404)
+ app = session.scalar(select(App.id).where(App.id == run.app_id, App.tenant_id == run.tenant_id))
+ if app is None:
+ raise TraceCollectionError("tracing_owner_not_found", 404)
+ invocation = session.scalar(
+ select(ApplicationTraceInvocation).where(
+ ApplicationTraceInvocation.id == str(batch.invocation_id),
+ ApplicationTraceInvocation.run_id == run.id,
+ )
+ )
+ if invocation is None:
+ raise TraceCollectionError("tracing_invocation_not_found", 404)
+ previous = session.get(ApplicationTraceBatch, str(batch.batch_id))
+ if previous is not None:
+ if (
+ previous.run_id != run.id
+ or previous.invocation_id != invocation.id
+ or previous.payload_hash != digest
+ ):
+ raise TraceCollectionError("tracing_batch_conflict", 409)
+ return len(previous.spans), run.id
+ if run.state in {"accepted", "exporting", "uncertain", "failed"} or invocation.finalized:
+ raise TraceCollectionError("tracing_collection_closed", 409)
+ if any(span.trace_id != run.trace_id for span in batch.spans):
+ raise TraceCollectionError("tracing_trace_id_mismatch", 409)
+ if run.span_count + len(batch.spans) > max_spans or run.payload_bytes + len(canonical) > max_bytes:
+ raise TraceCollectionError("tracing_payload_limit", 413)
+ spans: list[dict[str, Any]] = []
+ for value in batch.spans:
+ span = value.model_dump(mode="json")
+ span["attributes"] = safe_attributes(span["attributes"], capture_content=run.capture_content)
+ if not run.capture_content:
+ span["events"] = []
+ span["status_message"] = "Agent operation failed" if span["status"] == "error" else None
+ spans.append(span)
+ session.add(
+ ApplicationTraceBatch(
+ id=str(batch.batch_id),
+ run_id=run.id,
+ invocation_id=invocation.id,
+ payload_hash=digest,
+ spans=spans,
+ final=batch.final,
+ )
+ )
+ run.span_count += len(spans)
+ run.payload_bytes += len(canonical)
+ run.updated_at = naive_utc_now()
+ if batch.final:
+ invocation.finalized = True
+ invocation.finalized_at = naive_utc_now()
+ return len(spans), run.id
+
+ def terminal(self, run_id: str, trace: dict[str, Any], root_span_id: str) -> None:
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.state in {"accepted", "exporting", "uncertain", "failed"}:
+ return
+ if run.outer_trace is not None and run.outer_trace != trace:
+ # A duplicate terminal task must not rewrite a completed immutable
+ # span. The first persisted terminal envelope owns the export.
+ return
+ run.outer_trace = trace
+ run.root_span_id = root_span_id
+ run.terminal_at = run.terminal_at or naive_utc_now()
+ run.updated_at = naive_utc_now()
+
+ def get(self, run_id: str) -> ApplicationTraceRun | None:
+ with self.sessions() as session:
+ return session.get(ApplicationTraceRun, run_id)
+
+ def collected(self, run_id: str) -> tuple[list[ApplicationTraceInvocation], list[ApplicationTraceBatch]]:
+ with self.sessions() as session:
+ invocations = list(
+ session.scalars(
+ select(ApplicationTraceInvocation)
+ .where(
+ ApplicationTraceInvocation.run_id == run_id,
+ )
+ .order_by(
+ ApplicationTraceInvocation.attempt,
+ ApplicationTraceInvocation.created_at,
+ ApplicationTraceInvocation.id,
+ )
+ )
+ )
+ batches = list(
+ session.scalars(
+ select(ApplicationTraceBatch)
+ .where(ApplicationTraceBatch.run_id == run_id)
+ .order_by(ApplicationTraceBatch.created_at)
+ )
+ )
+ return invocations, batches
+
+ def claim(
+ self, run_id: str, *, collection_timeout: int, lease_seconds: int
+ ) -> tuple[ApplicationTraceRun, str] | None:
+ now = naive_utc_now()
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.state in {"not_collected", "accepted", "failed", "uncertain"}:
+ return None
+ if run.state == "exporting":
+ last_heartbeat = run.export_heartbeat_at or run.export_started_at
+ if last_heartbeat and last_heartbeat + timedelta(seconds=lease_seconds) < now:
+ run.state, run.error_code = "uncertain", "export_lease_expired"
+ run.updated_at = now
+ return None
+ if run.outer_trace is None:
+ return None
+ incomplete = session.scalar(
+ select(ApplicationTraceInvocation.id)
+ .where(ApplicationTraceInvocation.run_id == run_id, ApplicationTraceInvocation.finalized.is_(False))
+ .limit(1)
+ )
+ if incomplete:
+ if run.terminal_at and run.terminal_at + timedelta(seconds=collection_timeout) < now:
+ run.state, run.error_code = "failed", "agent_trace_collection_incomplete"
+ run.updated_at = now
+ return None
+ if run.next_attempt_at and run.next_attempt_at > now:
+ return None
+ app = session.scalar(select(App.id).where(App.id == run.app_id, App.tenant_id == run.tenant_id))
+ if app is None:
+ run.state, run.error_code = "failed", "tracing_owner_not_found"
+ return None
+ token = str(uuidv7())
+ run.state = "exporting"
+ run.export_token = token
+ run.export_started_at = now
+ run.export_heartbeat_at = now
+ run.updated_at = now
+ run.attempts += 1
+ return run, token
+
+ def renew_export_lease(self, run_id: str, token: str) -> bool:
+ """Renew only the current export attempt, in a short independent transaction."""
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.state != "exporting" or run.export_token != token:
+ return False
+ heartbeat_at = naive_utc_now()
+ run.export_heartbeat_at = heartbeat_at
+ run.updated_at = heartbeat_at
+ return True
+
+ def complete(
+ self,
+ run_id: str,
+ token: str,
+ *,
+ state: str,
+ error_code: str | None = None,
+ result: Any = None,
+ retry_delay: int | None = None,
+ ) -> None:
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.export_token != token:
+ return
+ late_acceptance = (
+ state == "accepted"
+ and result is not None
+ and run.state == "uncertain"
+ and run.error_code == "export_lease_expired"
+ )
+ if run.state != "exporting" and not late_acceptance:
+ return
+ run.state, run.error_code = state, error_code
+ run.updated_at = naive_utc_now()
+ run.export_token = None
+ if result is not None:
+ run.provider_trace_id, run.trace_url, run.span_urls = (
+ result.trace_id,
+ result.trace_url,
+ result.span_urls,
+ )
+ run.accepted_at = naive_utc_now()
+ run.next_attempt_at = naive_utc_now() + timedelta(seconds=retry_delay) if retry_delay is not None else None
+
+ def recovery_checked(self, run_id: str, *, error_code: str | None = None) -> None:
+ """Rotate checked incomplete work without changing a concurrent export."""
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.outer_trace is not None or run.state not in {"collecting", "ready"}:
+ return
+ run.updated_at = naive_utc_now()
+ if error_code is not None:
+ run.state, run.error_code = "failed", error_code
+
+ def pending_ids(self, limit: int = 100) -> list[str]:
+ with self.sessions() as session:
+ return list(
+ session.scalars(
+ select(ApplicationTraceRun.id)
+ .where(
+ ApplicationTraceRun.state.in_(["collecting", "ready", "retrying", "exporting"]),
+ or_(
+ ApplicationTraceRun.next_attempt_at.is_(None),
+ ApplicationTraceRun.next_attempt_at <= naive_utc_now(),
+ ),
+ )
+ .order_by(ApplicationTraceRun.terminal_at.is_(None), ApplicationTraceRun.updated_at)
+ .limit(limit)
+ )
+ )
+
+ def retry(self, *, run_id: str, acknowledge_possible_duplicates: bool) -> None:
+ with self.sessions.begin() as session:
+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())
+ if run is None or run.state == "not_collected":
+ raise TraceCollectionError("tracing_run_not_found", 404)
+ if run.state == "accepted":
+ raise TraceCollectionError("tracing_already_accepted", 409)
+ if run.state not in {"failed", "uncertain"} or run.outer_trace is None:
+ raise TraceCollectionError("tracing_retry_not_allowed", 409)
+ incomplete = session.scalar(
+ select(ApplicationTraceInvocation.id)
+ .where(
+ ApplicationTraceInvocation.run_id == run.id,
+ ApplicationTraceInvocation.finalized.is_(False),
+ )
+ .limit(1)
+ )
+ if incomplete or (run.error_code or "").startswith("tracing_"):
+ raise TraceCollectionError("tracing_retry_not_allowed", 409)
+ if run.state == "uncertain" and not acknowledge_possible_duplicates:
+ raise TraceCollectionError("tracing_duplicate_acknowledgement_required", 409)
+ run.state, run.error_code, run.export_token = "ready", None, None
+ run.next_attempt_at = None
+ run.updated_at = naive_utc_now()
+
+ def purge(self, *, retention_days: int, limit: int = 100) -> int:
+ cutoff = naive_utc_now() - timedelta(days=retention_days)
+ with self.sessions.begin() as session:
+ ids = list(
+ session.scalars(
+ select(ApplicationTraceRun.id)
+ .where(ApplicationTraceRun.created_at < cutoff)
+ .limit(limit)
+ .with_for_update()
+ )
+ )
+ if not ids:
+ return 0
+ session.execute(delete(ApplicationTraceBatch).where(ApplicationTraceBatch.run_id.in_(ids)))
+ session.execute(delete(ApplicationTraceInvocation).where(ApplicationTraceInvocation.run_id.in_(ids)))
+ session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id.in_(ids)))
+ return len(ids)
diff --git a/api/services/agent_tracing_config_service.py b/api/services/agent_tracing_config_service.py
new file mode 100644
index 00000000000..c1dca1702c3
--- /dev/null
+++ b/api/services/agent_tracing_config_service.py
@@ -0,0 +1,71 @@
+"""Agent-scoped tracing configuration, backed by the existing App tracing owners.
+
+Only roster Agent Apps own third-party credentials. Workflow-only Agents inherit
+their caller's tracing policy and must not create a hidden configuration owner.
+"""
+
+from typing import Any, Protocol
+
+from machinery.context import RequestContext
+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigRecord, AppTracingConfigService
+
+
+class AgentTracingOwner(Protocol):
+ def resolve_roster_app_id(self, *, tenant_id: str, agent_id: str) -> str: ...
+
+
+class AppTracingSettings(Protocol):
+ def get_trace(self, context: RequestContext, app_id: str) -> AppTraceSettings: ...
+
+ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None: ...
+
+
+class AgentTracingConfigService:
+ def __init__(
+ self, *, owners: AgentTracingOwner, settings: AppTracingSettings, configs: AppTracingConfigService
+ ) -> None:
+ self._owners = owners
+ self._settings = settings
+ self._configs = configs
+
+ def resolve_app_id(self, context: RequestContext, agent_id: str) -> str:
+ """Resolve the roster Agent's tenant-scoped App without creating runtime state."""
+ return self._owners.resolve_roster_app_id(tenant_id=context.active_workspace_id, agent_id=agent_id)
+
+ def get_trace(self, context: RequestContext, agent_id: str) -> AppTraceSettings:
+ return self._settings.get_trace(context, self.resolve_app_id(context, agent_id))
+
+ def set_trace(self, context: RequestContext, agent_id: str, settings: AppTraceSettings) -> None:
+ self._settings.set_trace(context, self.resolve_app_id(context, agent_id), settings)
+
+ def get(self, context: RequestContext, agent_id: str, tracing_provider: str) -> AppTracingConfigRecord | None:
+ return self._configs.get(context, self.resolve_app_id(context, agent_id), tracing_provider)
+
+ def create(
+ self,
+ context: RequestContext,
+ agent_id: str,
+ tracing_provider: str,
+ tracing_config: dict[str, Any],
+ *,
+ activate: bool = False,
+ ) -> None:
+ self._configs.create(
+ context, self.resolve_app_id(context, agent_id), tracing_provider, tracing_config, activate=activate
+ )
+
+ def update(
+ self,
+ context: RequestContext,
+ agent_id: str,
+ tracing_provider: str,
+ tracing_config: dict[str, Any],
+ *,
+ activate: bool = False,
+ ) -> None:
+ self._configs.update(
+ context, self.resolve_app_id(context, agent_id), tracing_provider, tracing_config, activate=activate
+ )
+
+ def delete(self, context: RequestContext, agent_id: str, tracing_provider: str) -> None:
+ self._configs.delete(context, self.resolve_app_id(context, agent_id), tracing_provider)
diff --git a/api/services/app/console_service.py b/api/services/app/console_service.py
index 9ec2e62574e..c100f9aa6f9 100644
--- a/api/services/app/console_service.py
+++ b/api/services/app/console_service.py
@@ -11,6 +11,7 @@ from machinery.context import RequestContext
from models.model import AppMode
from services.agent.errors import InvalidRosterAgentPackageError
from services.agent.roster_package_entities import RosterAgentPackageExport
+from services.app_tracing_config_service import AppTraceSettings
from services.entities.app_entities import (
AppChange,
AppCreationSettings,
@@ -21,7 +22,6 @@ from services.entities.app_entities import (
AppPage,
AppRecord,
AppReference,
- AppTraceSettings,
CopyAppParams,
CreateAppParams,
ImportedAppPackage,
@@ -121,8 +121,6 @@ class ConsoleApps(Protocol):
def get_trace(self, context: RequestContext, app_id: str) -> AppTraceSettings: ...
- def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None: ...
-
class AppTransfers(Protocol):
def download_import(self, url: str) -> AbstractContextManager[BinaryIO]: ...
@@ -171,10 +169,8 @@ class CreatorsPlatform(Protocol):
def redirect_url(self, claim_code: str, oauth_code: str | None) -> str: ...
-class AppTraceProvider(Protocol):
- def validate_provider(self, tracing_provider: str) -> None: ...
-
- def require_provider_available(self, tracing_provider: str) -> None: ...
+class AppTracingSettings(Protocol):
+ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None: ...
class AppLifecycle(Protocol):
@@ -203,7 +199,7 @@ class ConsoleAppService:
access: ConsoleAppAccess,
transfers: AppTransfers,
creators: CreatorsPlatform,
- tracing: AppTraceProvider,
+ tracing: AppTracingSettings,
lifecycle: AppLifecycle,
) -> None:
self._apps = apps
@@ -457,9 +453,4 @@ class ConsoleAppService:
def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:
self._apps.get_reference(context, app_id)
- if settings.tracing_provider is not None:
- if settings.enabled:
- self._tracing.require_provider_available(settings.tracing_provider)
- else:
- self._tracing.validate_provider(settings.tracing_provider)
- self._apps.set_trace(context, app_id, settings)
+ self._tracing.set_trace(context, app_id, settings)
diff --git a/api/services/app_tracing_config_gateway.py b/api/services/app_tracing_config_gateway.py
index eb696f35afb..c513cb215f9 100644
--- a/api/services/app_tracing_config_gateway.py
+++ b/api/services/app_tracing_config_gateway.py
@@ -20,7 +20,6 @@ from services.app_tracing_config_service import (
logger = logging.getLogger(__name__)
_PROJECT_URL_FALLBACKS = {
- "arize": "https://app.arize.com/",
"phoenix": "https://app.phoenix.arize.com/projects/",
"langsmith": "https://smith.langchain.com/",
"opik": "https://www.comet.com/opik/",
@@ -41,6 +40,17 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
def require_provider_available(self, tracing_provider: str) -> None:
self._provider_config(tracing_provider)
+ @override
+ def validate_saved_config(
+ self, *, workspace_id: str, tracing_provider: str, tracing_config: dict[str, Any]
+ ) -> None:
+ provider_config = self._provider_config(tracing_provider)
+ try:
+ decrypted = OpsTraceManager.decrypt_tracing_config(workspace_id, tracing_provider, tracing_config)
+ except Exception as error:
+ raise AppTracingConfigProcessingError from error
+ self._validate_config(provider_config, decrypted)
+
@override
def prepare_new_config(
self,
@@ -50,10 +60,21 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
tracing_config: dict[str, Any],
) -> dict[str, Any]:
provider_config = self._provider_config(tracing_provider)
- normalized_config = self._normalize_config(provider_config, tracing_config)
+ if tracing_provider == "tencent":
+ normalized_config = self._normalize_tencent_config(tracing_config)
+ elif tracing_provider == "mlflow":
+ normalized_config = self._normalize_mlflow_config(tracing_config)
+ elif tracing_provider == "phoenix":
+ normalized_config = self._normalize_phoenix_config(tracing_config)
+ elif tracing_provider == "aliyun":
+ normalized_config = self._normalize_aliyun_config(tracing_config)
+ else:
+ normalized_config = self._normalize_config(provider_config, tracing_config)
self._verify_config(normalized_config, tracing_provider)
- project_url = self._get_project_url_for_create(normalized_config, tracing_provider)
+ project_url = (
+ None if tracing_provider == "arize" else self._get_project_url(normalized_config, tracing_provider)
+ )
try:
encrypted_config = OpsTraceManager.encrypt_tracing_config(
workspace_id,
@@ -62,7 +83,9 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
)
except Exception as error:
raise AppTracingConfigProcessingError from error
- if project_url:
+ if tracing_provider == "arize":
+ self._resolve_arize_destination(encrypted_config, normalized_config)
+ elif project_url:
encrypted_config["project_url"] = project_url
return encrypted_config
@@ -76,6 +99,16 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
current_tracing_config: dict[str, Any] | None,
) -> dict[str, Any]:
provider_config = self._provider_config(tracing_provider)
+ if tracing_provider == "tencent":
+ # Check the original URL before the legacy model discards its path
+ # and query. Masked tokens are restored by encryption below.
+ tracing_config = self._normalize_tencent_config(tracing_config)
+ elif tracing_provider == "mlflow":
+ tracing_config = self._normalize_mlflow_config(tracing_config)
+ elif tracing_provider == "phoenix":
+ tracing_config = self._normalize_phoenix_config(tracing_config)
+ elif tracing_provider == "aliyun":
+ tracing_config = self._normalize_aliyun_config(tracing_config)
self._validate_config(provider_config, tracing_config)
try:
encrypted_config = OpsTraceManager.encrypt_tracing_config(
@@ -93,6 +126,30 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
raise AppTracingConfigProcessingError from error
self._verify_config(decrypted_config, tracing_provider)
+ if tracing_provider == "weave":
+ # Verification resolves an optional entity and qualified project name.
+ # Encryption happened first to restore masked credentials; persist the
+ # verified non-secret destination alongside that unchanged ciphertext.
+ encrypted_config["entity"] = decrypted_config["entity"]
+ encrypted_config["project"] = decrypted_config["project"]
+ elif tracing_provider == "databricks":
+ encrypted_config["host"] = decrypted_config["host"]
+ encrypted_config["experiment_id"] = decrypted_config["experiment_id"]
+ elif tracing_provider == "mlflow":
+ encrypted_config["tracking_uri"] = decrypted_config["tracking_uri"]
+ encrypted_config["experiment_id"] = decrypted_config["experiment_id"]
+ elif tracing_provider == "phoenix":
+ encrypted_config["endpoint"] = decrypted_config["endpoint"]
+ encrypted_config["project"] = decrypted_config["project"]
+ elif tracing_provider == "opik":
+ for key in ("url", "workspace", "project"):
+ encrypted_config[key] = decrypted_config[key]
+ # Re-resolve after destination changes; never retain a previous project's link.
+ project_url = None if tracing_provider == "arize" else self._get_project_url(decrypted_config, tracing_provider)
+ if tracing_provider == "arize":
+ self._resolve_arize_destination(encrypted_config, decrypted_config)
+ elif project_url:
+ encrypted_config["project_url"] = project_url
return encrypted_config
@override
@@ -118,7 +175,70 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
except Exception as error:
raise AppTracingConfigProcessingError from error
- if tracing_provider == "langfuse" and not decrypted_config.get("project_key"):
+ if tracing_provider == "arize":
+ from dify_trace_arize_phoenix.arize_configuration import project_url
+
+ # The encrypted record carries server-owned non-secret metadata.
+ # Legacy decryption intentionally retains only provider input fields.
+ presented_config["project_url"] = project_url(tracing_config)
+ elif tracing_provider == "aliyun":
+ from dify_trace_aliyun.configuration import console_url
+
+ # GET never constructs the legacy queue/exporter or probes credentials.
+ presented_config["project_url"] = console_url()
+ elif tracing_provider == "tencent":
+ from dify_trace_tencent.configuration import console_url
+
+ # Historical names remain readable. GET performs no connection
+ # check and never constructs legacy tracing or metrics exporters.
+ presented_config["project_url"] = console_url()
+ elif tracing_provider == "opik":
+ from dify_trace_opik.config import OpikConfig
+ from dify_trace_opik.configuration import console_url, project_url
+
+ try:
+ config = OpikConfig.model_validate(decrypted_config)
+ presented_config["project_url"] = project_url(config) or console_url(config)
+ except ValueError:
+ presented_config["project_url"] = _PROJECT_URL_FALLBACKS["opik"]
+ elif tracing_provider == "phoenix":
+ from dify_trace_arize_phoenix.config import PhoenixConfig
+ from dify_trace_arize_phoenix.configuration import project_url
+
+ try:
+ presented_config["project_url"] = project_url(PhoenixConfig.model_validate(decrypted_config))
+ except ValueError:
+ # Historical destinations stay readable; never initialize a
+ # legacy exporter or invent a link for an unsafe base URL.
+ presented_config["project_url"] = None
+ elif tracing_provider == "mlflow":
+ from dify_trace_mlflow.config import MLflowConfig
+ from dify_trace_mlflow.oss_configuration import project_url
+
+ try:
+ presented_config["project_url"] = project_url(MLflowConfig.model_validate(decrypted_config))
+ except ValueError:
+ # OSS deployments have no universal project destination. Never
+ # initialize an SDK or substitute localhost for an invalid URL.
+ presented_config["project_url"] = None
+ elif tracing_provider == "databricks":
+ from dify_trace_mlflow.config import DatabricksConfig
+ from dify_trace_mlflow.configuration import project_url
+
+ try:
+ presented_config["project_url"] = project_url(DatabricksConfig.model_validate(decrypted_config))
+ except ValueError:
+ presented_config["project_url"] = _PROJECT_URL_FALLBACKS["databricks"]
+ elif tracing_provider == "weave":
+ from dify_trace_weave.config import WeaveConfig
+ from dify_trace_weave.configuration import console_url, project_url
+
+ try:
+ config = WeaveConfig.model_validate(decrypted_config)
+ presented_config["project_url"] = project_url(config) or console_url(config)
+ except ValueError:
+ presented_config["project_url"] = _PROJECT_URL_FALLBACKS["weave"]
+ elif tracing_provider == "langfuse" and not decrypted_config.get("project_key"):
try:
project_key = OpsTraceManager.get_trace_config_project_key(decrypted_config, tracing_provider)
presented_config["project_url"] = f"{decrypted_config.get('host')}/project/{project_key}"
@@ -135,6 +255,24 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
return presented_config
+ @staticmethod
+ def _resolve_arize_destination(encrypted_config: dict[str, Any], verified_config: dict[str, Any]) -> None:
+ from dify_trace_arize_phoenix.arize_configuration import (
+ DESTINATION_KEY,
+ bind_destination,
+ project_url,
+ resolve_project_destination,
+ )
+
+ # Only the explicit write path may replace this cache. Never trust IDs
+ # or URLs submitted by a client, or retain a previous target on failure.
+ encrypted_config.pop(DESTINATION_KEY, None)
+ encrypted_config.pop("project_url", None)
+ destination = resolve_project_destination(verified_config)
+ if destination is not None:
+ encrypted_config[DESTINATION_KEY] = bind_destination(encrypted_config, destination)
+ encrypted_config["project_url"] = project_url(encrypted_config)
+
@staticmethod
def _provider_config(tracing_provider: str) -> TracingProviderConfigEntry:
try:
@@ -158,6 +296,42 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
normalized_config[key] = default_values.get(key)
return normalized_config
+ @staticmethod
+ def _normalize_aliyun_config(tracing_config: dict[str, Any]) -> dict[str, Any]:
+ from dify_trace_aliyun.configuration import normalize_configuration
+
+ try:
+ return normalize_configuration(tracing_config).model_dump()
+ except ValueError:
+ raise AppTracingConfigInvalidConfigurationError from None
+
+ @staticmethod
+ def _normalize_tencent_config(tracing_config: dict[str, Any]) -> dict[str, Any]:
+ from dify_trace_tencent.configuration import TencentConfigurationError, normalize_configuration
+
+ try:
+ return normalize_configuration(tracing_config).model_dump()
+ except TencentConfigurationError as error:
+ raise AppTracingConfigInvalidConfigurationError from error
+
+ @staticmethod
+ def _normalize_mlflow_config(tracing_config: dict[str, Any]) -> dict[str, Any]:
+ from dify_trace_mlflow.oss_configuration import normalize_configuration
+
+ try:
+ return normalize_configuration(tracing_config).model_dump()
+ except ValueError as error:
+ raise AppTracingConfigInvalidConfigurationError from error
+
+ @staticmethod
+ def _normalize_phoenix_config(tracing_config: dict[str, Any]) -> dict[str, Any]:
+ from dify_trace_arize_phoenix.configuration import normalize_configuration
+
+ try:
+ return normalize_configuration(tracing_config).model_dump()
+ except ValueError as error:
+ raise AppTracingConfigInvalidConfigurationError from error
+
@staticmethod
def _validate_config(
provider_config: TracingProviderConfigEntry,
@@ -167,10 +341,105 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
try:
return config_class.model_validate(tracing_config)
except ValidationError as error:
+ # Aliyun credentials can be embedded in a pasted endpoint. Do not
+ # propagate a Pydantic error object whose errors() includes inputs.
+ if config_class.__module__ == "dify_trace_aliyun.config":
+ raise AppTracingConfigInvalidConfigurationError from None
raise AppTracingConfigInvalidConfigurationError from error
@staticmethod
def _verify_config(tracing_config: dict[str, Any], tracing_provider: str) -> None:
+ if tracing_provider == "arize":
+ from dify_trace_arize_phoenix.arize_configuration import ArizeConfigurationError, verify_configuration
+ from dify_trace_arize_phoenix.config import ArizeConfig
+
+ try:
+ verify_configuration(ArizeConfig.model_validate(tracing_config))
+ except ArizeConfigurationError as error:
+ logger.warning("Arize ingestion verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from None
+ return
+ if tracing_provider == "aliyun":
+ from dify_trace_aliyun.config import AliyunConfig
+ from dify_trace_aliyun.configuration import AliyunConfigurationError, verify_configuration
+
+ try:
+ verify_configuration(AliyunConfig.model_validate(tracing_config))
+ except AliyunConfigurationError as error:
+ logger.warning("Aliyun configuration verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from None
+ return
+ if tracing_provider == "phoenix":
+ from dify_trace_arize_phoenix.config import PhoenixConfig
+ from dify_trace_arize_phoenix.configuration import PhoenixConfigurationError, verify_configuration
+
+ try:
+ resolved = verify_configuration(PhoenixConfig.model_validate(tracing_config))
+ except PhoenixConfigurationError as error:
+ logger.warning("Phoenix configuration verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from error
+ tracing_config["endpoint"] = resolved.endpoint
+ tracing_config["project"] = resolved.project
+ return
+ if tracing_provider == "mlflow":
+ from dify_trace_mlflow.config import MLflowConfig
+ from dify_trace_mlflow.oss_configuration import MLflowConfigurationError, verify_configuration
+
+ try:
+ resolved = verify_configuration(MLflowConfig.model_validate(tracing_config))
+ except MLflowConfigurationError as error:
+ logger.warning("MLflow configuration verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from error
+ tracing_config["tracking_uri"] = resolved.tracking_uri
+ tracing_config["experiment_id"] = resolved.experiment_id
+ return
+ if tracing_provider == "tencent":
+ from dify_trace_tencent.config import TencentConfig
+ from dify_trace_tencent.configuration import TencentConfigurationError, verify_configuration
+
+ try:
+ verify_configuration(TencentConfig.model_validate(tracing_config))
+ except TencentConfigurationError as error:
+ logger.warning("Tencent collector connection check failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from error
+ return
+ if tracing_provider == "opik":
+ from dify_trace_opik.config import OpikConfig
+ from dify_trace_opik.configuration import OpikConfigurationError, verify_configuration
+
+ try:
+ resolved = verify_configuration(OpikConfig.model_validate(tracing_config))
+ except OpikConfigurationError as error:
+ logger.warning("Opik configuration verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from error
+ tracing_config["url"] = resolved.url
+ tracing_config["workspace"] = resolved.workspace
+ tracing_config["project"] = resolved.project
+ return
+ if tracing_provider == "databricks":
+ from dify_trace_mlflow.config import DatabricksConfig
+ from dify_trace_mlflow.configuration import DatabricksConfigurationError, verify_configuration
+
+ try:
+ resolved = verify_configuration(DatabricksConfig.model_validate(tracing_config))
+ except DatabricksConfigurationError as error:
+ logger.warning("Databricks configuration verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from error
+ tracing_config["host"] = resolved.host
+ tracing_config["experiment_id"] = resolved.experiment_id
+ return
+ if tracing_provider == "weave":
+ from dify_trace_weave.config import WeaveConfig
+ from dify_trace_weave.configuration import WeaveConfigurationError, verify_configuration
+
+ try:
+ resolved = verify_configuration(WeaveConfig.model_validate(tracing_config))
+ except WeaveConfigurationError as error:
+ logger.warning("Weave configuration verification failed: error_code=%s", error.code)
+ raise AppTracingConfigVerificationFailedError from error
+ tracing_config["entity"] = resolved.entity
+ tracing_config["project"] = resolved.project
+ return
try:
is_effective = OpsTraceManager.check_trace_config_is_effective(tracing_config, tracing_provider)
except ValueError as error:
@@ -180,14 +449,50 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):
raise AppTracingConfigVerificationFailedError
@staticmethod
- def _get_project_url_for_create(tracing_config: dict[str, Any], tracing_provider: str) -> str | None:
+ def _get_project_url(tracing_config: dict[str, Any], tracing_provider: str) -> str | None:
try:
- if tracing_provider in ("arize", "phoenix"):
- return OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider)
+ if tracing_provider == "aliyun":
+ from dify_trace_aliyun.configuration import console_url
+
+ return console_url()
+ if tracing_provider == "tencent":
+ from dify_trace_tencent.configuration import console_url
+
+ return console_url()
+ if tracing_provider == "opik":
+ from dify_trace_opik.config import OpikConfig
+ from dify_trace_opik.configuration import console_url, project_url
+
+ config = OpikConfig.model_validate(tracing_config)
+ return project_url(config) or console_url(config)
+ if tracing_provider == "databricks":
+ from dify_trace_mlflow.config import DatabricksConfig
+ from dify_trace_mlflow.configuration import project_url
+
+ return project_url(DatabricksConfig.model_validate(tracing_config))
+ if tracing_provider == "mlflow":
+ from dify_trace_mlflow.config import MLflowConfig
+ from dify_trace_mlflow.oss_configuration import project_url
+
+ return project_url(MLflowConfig.model_validate(tracing_config))
+ if tracing_provider == "weave":
+ from dify_trace_weave.config import WeaveConfig
+ from dify_trace_weave.configuration import project_url
+
+ return project_url(WeaveConfig.model_validate(tracing_config))
+ if tracing_provider == "phoenix":
+ from dify_trace_arize_phoenix.config import PhoenixConfig
+ from dify_trace_arize_phoenix.configuration import project_url
+
+ return project_url(PhoenixConfig.model_validate(tracing_config))
+ if tracing_provider == "arize":
+ from dify_trace_arize_phoenix.arize_configuration import project_url
+
+ return project_url(tracing_config)
if tracing_provider == "langfuse":
project_key = OpsTraceManager.get_trace_config_project_key(tracing_config, tracing_provider)
return f"{tracing_config.get('host')}/project/{project_key}"
- if tracing_provider in ("langsmith", "opik", "mlflow", "databricks", "tencent"):
+ if tracing_provider in ("langsmith", "opik"):
return OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider)
except Exception:
return None
diff --git a/api/services/app_tracing_config_service.py b/api/services/app_tracing_config_service.py
index 6d7867fa86b..11d3b1028ff 100644
--- a/api/services/app_tracing_config_service.py
+++ b/api/services/app_tracing_config_service.py
@@ -7,6 +7,14 @@ from typing import Any, Protocol
from machinery.context import RequestContext
+@dataclass(frozen=True, slots=True)
+class AppTraceSettings:
+ enabled: bool = False
+ tracing_provider: str | None = None
+ capture_content: bool = False
+ capture_agent: bool = True
+
+
@dataclass(frozen=True, slots=True)
class AppTracingConfigRecord:
id: str
@@ -34,6 +42,7 @@ class AppTracingConfigStore(Protocol):
app_id: str,
tracing_provider: str,
tracing_config: dict[str, Any],
+ activate: bool = False,
) -> bool: ...
def update(
@@ -43,6 +52,8 @@ class AppTracingConfigStore(Protocol):
app_id: str,
tracing_provider: str,
tracing_config: dict[str, Any],
+ expected: AppTracingConfigRecord,
+ activate: bool = False,
) -> bool: ...
def delete(
@@ -53,10 +64,18 @@ class AppTracingConfigStore(Protocol):
tracing_provider: str,
) -> bool: ...
+ def set_trace(
+ self, *, workspace_id: str, app_id: str, settings: AppTraceSettings, expected: AppTracingConfigRecord | None
+ ) -> None: ...
+
class TracingConfigProviderGateway(Protocol):
def validate_provider(self, tracing_provider: str) -> None: ...
+ def validate_saved_config(
+ self, *, workspace_id: str, tracing_provider: str, tracing_config: dict[str, Any]
+ ) -> None: ...
+
def prepare_new_config(
self,
*,
@@ -97,6 +116,11 @@ class AppTracingConfigAlreadyExistsError(AppTracingConfigError):
super().__init__("Trace config is exist.")
+class AppTracingConfigConflictError(AppTracingConfigError):
+ def __init__(self) -> None:
+ super().__init__("Tracing configuration changed. Refresh and try again.")
+
+
class AppTracingConfigNotFoundError(AppTracingConfigError):
def __init__(self) -> None:
super().__init__("Trace config not exist.")
@@ -167,6 +191,8 @@ class AppTracingConfigService:
app_id: str,
tracing_provider: str,
tracing_config: dict[str, Any],
+ *,
+ activate: bool = False,
) -> None:
workspace_id = context.active_workspace_id
current = self._configs.get(
@@ -188,6 +214,7 @@ class AppTracingConfigService:
app_id=app_id,
tracing_provider=tracing_provider,
tracing_config=encrypted_config,
+ activate=activate,
)
if not created:
raise AppTracingConfigAlreadyExistsError
@@ -198,6 +225,8 @@ class AppTracingConfigService:
app_id: str,
tracing_provider: str,
tracing_config: dict[str, Any],
+ *,
+ activate: bool = False,
) -> None:
workspace_id = context.active_workspace_id
current = self._configs.get(
@@ -220,6 +249,8 @@ class AppTracingConfigService:
app_id=app_id,
tracing_provider=tracing_provider,
tracing_config=encrypted_config,
+ expected=current,
+ activate=activate,
)
if not updated:
raise AppTracingConfigNotFoundError
@@ -233,3 +264,25 @@ class AppTracingConfigService:
)
if not deleted:
raise AppTracingConfigNotFoundError
+
+ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:
+ expected = None
+ provider = settings.tracing_provider
+ if provider is not None:
+ self._provider.validate_provider(provider)
+ if settings.enabled:
+ if provider is None:
+ raise AppTracingConfigNotFoundError
+ expected = self._configs.get(
+ workspace_id=context.active_workspace_id, app_id=app_id, tracing_provider=provider
+ )
+ if expected is None or not expected.is_active or not expected.tracing_config:
+ raise AppTracingConfigNotFoundError
+ self._provider.validate_saved_config(
+ workspace_id=context.active_workspace_id,
+ tracing_provider=provider,
+ tracing_config=expected.tracing_config,
+ )
+ self._configs.set_trace(
+ workspace_id=context.active_workspace_id, app_id=app_id, settings=settings, expected=expected
+ )
diff --git a/api/services/entities/app_entities.py b/api/services/entities/app_entities.py
index a14e3dc97d6..345f60505ef 100644
--- a/api/services/entities/app_entities.py
+++ b/api/services/entities/app_entities.py
@@ -225,12 +225,6 @@ class AppReference:
bound_agent_id: str | None
-@dataclass(frozen=True, slots=True)
-class AppTraceSettings:
- enabled: bool = False
- tracing_provider: str | None = None
-
-
@dataclass(frozen=True, slots=True)
class AppCreationSettings:
app: dict[str, Any]
diff --git a/api/services/entities/feature_entities.py b/api/services/entities/feature_entities.py
index 0bf39a385b8..20f9d149000 100644
--- a/api/services/entities/feature_entities.py
+++ b/api/services/entities/feature_entities.py
@@ -179,6 +179,10 @@ class SystemFeatureModel(FeatureResponseModel):
"""Non-sensitive bootstrap snapshot exposed before Console or Web authentication."""
deployment_edition: DeploymentEdition
+ agent_tracing_api_collection_enabled: bool = Field(
+ description="Deployment-level API gate for Agent trace collection, independent of workspace and provider "
+ "configuration. Does not indicate Agent runtime, worker, or provider health or confirm trace delivery.",
+ )
enable_app_deploy: bool = False
sso_enforced_for_signin: bool = False
sso_enforced_for_signin_protocol: SSOProtocol | None = None
diff --git a/api/services/system_feature_service.py b/api/services/system_feature_service.py
index d2056411907..58e3c7ab522 100644
--- a/api/services/system_feature_service.py
+++ b/api/services/system_feature_service.py
@@ -25,8 +25,11 @@ class SystemFeatureService:
@classmethod
def get_public_system_features(cls) -> feature_entities.SystemFeatureModel:
- """Build the non-sensitive bootstrap snapshot shared by Console and Web."""
- system_features = feature_entities.SystemFeatureModel(deployment_edition=dify_config.DEPLOYMENT_EDITION)
+ """Build the shared public snapshot, including the Agent tracing deployment gate."""
+ system_features = feature_entities.SystemFeatureModel(
+ deployment_edition=dify_config.DEPLOYMENT_EDITION,
+ agent_tracing_api_collection_enabled=dify_config.OPS_AGENT_TRACE_ENABLED,
+ )
system_features.rbac_enabled = dify_config.RBAC_ENABLED
cls._fulfill_system_params_from_env(system_features)
diff --git a/api/services/tracing/__init__.py b/api/services/tracing/__init__.py
new file mode 100644
index 00000000000..6aaaab24ebe
--- /dev/null
+++ b/api/services/tracing/__init__.py
@@ -0,0 +1 @@
+"""Application-owned tracing policy, collection, and delivery."""
diff --git a/api/services/tracing/genai_attributes.py b/api/services/tracing/genai_attributes.py
new file mode 100644
index 00000000000..1b1abcfe814
--- /dev/null
+++ b/api/services/tracing/genai_attributes.py
@@ -0,0 +1,58 @@
+"""GenAI semantic attributes shared by the Aliyun and Tencent adapters."""
+
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalTrace
+from services.tracing.provider_common import json_value, token_usage
+
+
+def attributes(span: CanonicalSpan, trace: CanonicalTrace, provider: str) -> dict[str, object]:
+ kind = span.kind.value.upper()
+ if provider == "tencent":
+ kind = {"LLM": "GENERATION", "CHAIN": "WORKFLOW"}.get(kind, kind)
+ result: dict[str, object] = {
+ "gen_ai.span.kind": kind,
+ "gen_ai.session.id": trace.session_id,
+ "gen_ai.framework": "dify",
+ "dify.span.id": span.id,
+ "dify.metadata": json_value(span.metadata),
+ "gen_ai.entity.input" if provider == "tencent" else "input.value": json_value(span.inputs),
+ "gen_ai.entity.output" if provider == "tencent" else "output.value": json_value(span.outputs),
+ }
+ if provider == "tencent" and span.id == trace.root_span_id:
+ result["gen_ai.is_entry"] = "true"
+ for key in ("tenant_id", "app_id", "agent_id", "workflow_run_id", "node_execution_id", "message_id"):
+ value = span.metadata.get(key) or trace.spans[0].metadata.get(key)
+ if value:
+ result[f"dify.{key}"] = str(value)
+ user_id = span.metadata.get("user_id") or trace.spans[0].metadata.get("user_id")
+ if user_id:
+ result["gen_ai.user.id"] = str(user_id)
+ if span.kind is CanonicalSpanKind.LLM:
+ result["gen_ai.operation.name"] = "chat"
+ result["gen_ai.prompt"] = json_value(span.inputs)
+ result["gen_ai.completion"] = json_value(span.outputs)
+ result["gen_ai.input.messages"] = json_value(span.inputs)
+ result["gen_ai.output.messages"] = json_value(span.outputs)
+ model = span.metadata.get("model") or span.metadata.get("model_name")
+ if model:
+ result["gen_ai.request.model"] = str(model)
+ result["gen_ai.response.model"] = str(model)
+ if span.metadata.get("model_provider"):
+ result["gen_ai.provider.name"] = str(span.metadata["model_provider"])
+ result.update({f"gen_ai.usage.{key}_tokens": value for key, value in token_usage(span).items()})
+ ttft = span.metadata.get("time_to_first_token", span.metadata.get("ttft"))
+ if isinstance(ttft, (int, float)) and ttft >= 0:
+ result["gen_ai.response.time_to_first_token"] = int(ttft * 1_000_000_000)
+ elif span.kind is CanonicalSpanKind.AGENT:
+ result["gen_ai.operation.name"] = "invoke_agent"
+ result["gen_ai.agent.name"] = span.name
+ elif span.kind is CanonicalSpanKind.TOOL:
+ result["gen_ai.operation.name"] = "execute_tool"
+ result["gen_ai.tool.name"] = span.name
+ result["gen_ai.tool.call.arguments"] = json_value(span.inputs)
+ result["gen_ai.tool.call.result"] = json_value(span.outputs)
+ result["tool.name"] = span.name
+ result["tool.parameters"] = json_value(span.inputs)
+ elif span.kind is CanonicalSpanKind.RETRIEVER:
+ result["retrieval.query"] = json_value(span.inputs)
+ result["retrieval.document"] = json_value(span.outputs)
+ return result
diff --git a/api/services/tracing/normalization.py b/api/services/tracing/normalization.py
new file mode 100644
index 00000000000..0216e12431e
--- /dev/null
+++ b/api/services/tracing/normalization.py
@@ -0,0 +1,346 @@
+"""Normalize Agent SDK observations without conflating business and provider IDs."""
+
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from typing import Any
+
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+
+MAX_VALUE_CHARS = 16000
+_SECRET_KEYS = frozenset(
+ {
+ "api_key",
+ "apikey",
+ "secret",
+ "secret_key",
+ "authorization",
+ "password",
+ "access_token",
+ "refresh_token",
+ "credentials",
+ }
+)
+_CONTENT_ATTRIBUTES = frozenset(
+ {
+ "gen_ai.input.messages",
+ "gen_ai.output.messages",
+ "gen_ai.system_instructions",
+ "gen_ai.prompt",
+ "gen_ai.completion",
+ "tool_arguments",
+ "tool_response",
+ "gen_ai.tool.call.arguments",
+ "gen_ai.tool.call.result",
+ "input",
+ "output",
+ "inputs",
+ "outputs",
+ }
+)
+
+
+def span_id(logical_id: str) -> str:
+ """Stable nonzero OTel span identity for Dify's persisted logical ID."""
+ value = hashlib.sha256(f"dify-application-span:{logical_id}".encode()).hexdigest()[:16]
+ return value if int(value, 16) else "0000000000000001"
+
+
+def sanitize(value: Any, *, depth: int = 0) -> Any:
+ if depth > 12:
+ return "[truncated]"
+ if isinstance(value, Mapping):
+ return {
+ str(k): "[redacted]" if str(k).lower() in _SECRET_KEYS else sanitize(v, depth=depth + 1)
+ for k, v in value.items()
+ }
+ if isinstance(value, (list, tuple)):
+ return [sanitize(item, depth=depth + 1) for item in value[:256]]
+ if isinstance(value, str):
+ if value.lstrip().startswith(("{", "[")):
+ try:
+ decoded = json.loads(value)
+ except ValueError:
+ pass
+ else:
+ if isinstance(decoded, (dict, list)):
+ return sanitize(decoded, depth=depth + 1)
+ return value[:MAX_VALUE_CHARS] + ("[truncated]" if len(value) > MAX_VALUE_CHARS else "")
+ if value is None or isinstance(value, (int, float, bool)):
+ return value
+ return str(value)[:MAX_VALUE_CHARS]
+
+
+def safe_attributes(attributes: Mapping[str, Any], *, capture_content: bool) -> dict[str, Any]:
+ """Content-off is an allowlist, including metadata and error attributes."""
+ if capture_content:
+ result = sanitize(attributes)
+ return dict(result) if isinstance(result, Mapping) else {}
+ safe = {
+ "gen_ai.system",
+ "gen_ai.provider.name",
+ "gen_ai.operation.name",
+ "gen_ai.agent.name",
+ "gen_ai.tool.name",
+ "gen_ai.tool.call.id",
+ "logfire.span_type",
+ "llm.model_name",
+ "openinference.span.kind",
+ "dify.llm.total_price",
+ "dify.llm.currency",
+ "dify.agent.execution_state",
+ "gen_ai.response.time_to_first_token",
+ "dify.llm.time_to_first_token_unit",
+ "dify.llm.time_to_first_token_source",
+ "gen_ai.request.model",
+ "gen_ai.response.model",
+ "gen_ai.response.id",
+ "gen_ai.request.max_tokens",
+ "gen_ai.request.temperature",
+ "gen_ai.request.top_p",
+ "dify.trace_id",
+ "dify.tenant_id",
+ "dify.app_id",
+ "dify.agent_id",
+ "dify.invoke_from",
+ "dify.conversation.id",
+ "dify.workflow.id",
+ "dify.workflow.run_id",
+ "dify.node.id",
+ "dify.node.execution_id",
+ "dify.tracing.run_id",
+ "dify.tracing.invocation_id",
+ "gen_ai.user.id",
+ }
+ return {
+ str(key): sanitize(value)
+ for key, value in attributes.items()
+ if key not in _CONTENT_ATTRIBUTES
+ and (
+ key in safe
+ and isinstance(value, (str, int, float, bool))
+ or str(key).startswith(("gen_ai.usage.", "llm.token_count."))
+ and isinstance(value, (int, float))
+ )
+ }
+
+
+def _decoded(value: Any) -> Any:
+ if isinstance(value, str):
+ try:
+ return json.loads(value)
+ except (ValueError, TypeError):
+ return value
+ return value
+
+
+def _first(attributes: Mapping[str, Any], keys: Sequence[str]) -> Any:
+ for key in keys:
+ value = attributes.get(key)
+ if value is not None:
+ return value
+ return None
+
+
+def normalize_agent_span(data: Mapping[str, Any], *, capture_content: bool) -> CanonicalSpan:
+ attributes = safe_attributes(data.get("attributes") or {}, capture_content=capture_content)
+ operation = str(attributes.get("gen_ai.operation.name", ""))
+ declared = str(attributes.get("openinference.span.kind", "")).lower()
+ if operation in {"invoke_agent", "agent"} or declared == "agent":
+ kind = CanonicalSpanKind.AGENT
+ elif operation in {"execute_tool", "tool"} or "gen_ai.tool.name" in attributes or declared == "tool":
+ kind = CanonicalSpanKind.TOOL
+ elif (
+ operation in {"chat", "text_completion", "generate_content", "embeddings"}
+ or "gen_ai.request.model" in attributes
+ or declared == "llm"
+ ):
+ kind = CanonicalSpanKind.LLM
+ elif declared == "retriever":
+ kind = CanonicalSpanKind.RETRIEVER
+ else:
+ kind = CanonicalSpanKind.CHAIN
+ metadata = {key: value for key, value in attributes.items() if key not in _CONTENT_ATTRIBUTES}
+ mappings = {
+ "model_name": ("gen_ai.response.model", "gen_ai.request.model", "llm.model_name"),
+ "model_provider": ("gen_ai.provider.name", "gen_ai.system"),
+ "prompt_tokens": ("gen_ai.usage.input_tokens", "llm.token_count.prompt"),
+ "completion_tokens": ("gen_ai.usage.output_tokens", "llm.token_count.completion"),
+ "cache_read_tokens": ("gen_ai.usage.cache_read.input_tokens", "gen_ai.usage.input_tokens.cached"),
+ "total_price": ("dify.llm.total_price",),
+ "currency": ("dify.llm.currency",),
+ "time_to_first_token": ("gen_ai.response.time_to_first_token",),
+ }
+ for key, candidates in mappings.items():
+ value = _first(attributes, candidates)
+ if value is not None:
+ metadata[key] = value
+ if kind == CanonicalSpanKind.LLM:
+ input_tokens, output_tokens = metadata.get("prompt_tokens"), metadata.get("completion_tokens")
+ if isinstance(input_tokens, (int, float)) and isinstance(output_tokens, (int, float)):
+ metadata["total_tokens"] = input_tokens + output_tokens
+ else:
+ # Agent/chain aggregate usage is useful context, never another billed generation.
+ for key in ("prompt_tokens", "completion_tokens", "total_tokens", "total_price"):
+ if key in metadata:
+ metadata[f"aggregate_{key}"] = metadata.pop(key)
+ error = str(data.get("status_message") or "Agent operation failed") if data.get("status") == "error" else None
+ if error and not capture_content:
+ error = "Agent operation failed"
+ inputs = (
+ _first(
+ attributes,
+ (
+ "gen_ai.input.messages",
+ "gen_ai.prompt",
+ "gen_ai.tool.call.arguments",
+ "tool_arguments",
+ "input",
+ "inputs",
+ ),
+ )
+ if capture_content
+ else None
+ )
+ outputs = (
+ _first(
+ attributes,
+ (
+ "gen_ai.output.messages",
+ "gen_ai.completion",
+ "gen_ai.tool.call.result",
+ "tool_response",
+ "output",
+ "outputs",
+ ),
+ )
+ if capture_content
+ else None
+ )
+ return CanonicalSpan(
+ id=str(data["span_id"]),
+ parent_id=data.get("parent_span_id"),
+ name=str(data["name"]),
+ kind=kind,
+ start_time=datetime.fromtimestamp(int(data["start_time_ns"]) / 1e9, UTC),
+ end_time=datetime.fromtimestamp(int(data["end_time_ns"]) / 1e9, UTC),
+ inputs=sanitize(_decoded(inputs)),
+ outputs=sanitize(_decoded(outputs)),
+ status=CanonicalSpanStatus.ERROR if error else CanonicalSpanStatus.OK,
+ error=error,
+ metadata=metadata,
+ )
+
+
+def remap_outer_trace(
+ trace: CanonicalTrace, *, trace_id: str, capture_content: bool, agent_message: bool
+) -> CanonicalTrace:
+ """Use the same IDs allocated before runtime and suppress synthetic Agent usage."""
+ safe_metadata = {
+ "workflow_id",
+ "workflow_run_id",
+ "workflow_app_log_id",
+ "node_id",
+ "node_execution_id",
+ "node_type",
+ "status",
+ "model_provider",
+ "model_name",
+ "prompt_tokens",
+ "completion_tokens",
+ "total_tokens",
+ "total_price",
+ "currency",
+ "trace_entity_type",
+ "wrapper_type",
+ "wrapper_index",
+ "container_execution_id",
+ "tenant_id",
+ "app_id",
+ "agent_id",
+ "user_id",
+ "conversation_id",
+ "external_trace_id",
+ "message_execution_state",
+ "linked_parent_workflow_run_id",
+ "linked_parent_node_execution_id",
+ "linked_parent_application_trace_id",
+ "linked_parent_trace_id",
+ }
+ spans = []
+ for original in trace.spans:
+ if agent_message and original.synthetic and original.kind == CanonicalSpanKind.LLM:
+ continue
+ metadata = (
+ sanitize(original.metadata)
+ if capture_content
+ else {k: sanitize(v) for k, v in original.metadata.items() if k in safe_metadata}
+ )
+ if trace.trace_id != trace_id:
+ metadata.setdefault("external_trace_id", trace.trace_id)
+ if trace.external_parent is not None:
+ # This pipeline exports a complete local tree. Cross-workflow
+ # correlation is explicit until a durable cross-run merge exists.
+ metadata["linked_parent_workflow_run_id"] = trace.external_parent.parent_workflow_run_id
+ if trace.external_parent.parent_node_execution_id:
+ metadata["linked_parent_node_execution_id"] = trace.external_parent.parent_node_execution_id
+ metadata.pop("parent_trace_context", None)
+ if original.kind != CanonicalSpanKind.LLM:
+ for key in ("prompt_tokens", "completion_tokens", "total_tokens", "total_price"):
+ if key in metadata:
+ metadata[f"aggregate_{key}"] = metadata.pop(key)
+ spans.append(
+ original.model_copy(
+ update={
+ "id": span_id(original.id),
+ "parent_id": span_id(original.parent_id) if original.parent_id else None,
+ "inputs": sanitize(original.inputs) if capture_content else None,
+ "outputs": sanitize(original.outputs) if capture_content else None,
+ "metadata": metadata,
+ "error": original.error
+ if capture_content or not original.error
+ else "Application execution failed",
+ "can_parent_workflow": False,
+ "publishes_parent_context": False,
+ }
+ )
+ )
+ return CanonicalTrace(
+ trace_id=trace_id, session_id=trace.session_id, root_span_id=span_id(trace.root_span_id), spans=tuple(spans)
+ )
+
+
+def order_spans(root_id: str, spans: Sequence[CanonicalSpan]) -> tuple[CanonicalSpan, ...]:
+ by_id: dict[str, CanonicalSpan] = {}
+ for span in spans:
+ prior = by_id.get(span.id)
+ if prior is not None and prior != span:
+ raise ValueError("Conflicting trace span identity")
+ by_id[span.id] = span
+ if root_id not in by_id:
+ raise ValueError("Trace root is missing")
+ ordered: list[CanonicalSpan] = []
+ complete: set[str] = set()
+ visiting: set[str] = set()
+
+ def visit(identifier: str) -> None:
+ if identifier in complete:
+ return
+ if identifier in visiting:
+ raise ValueError("Trace parent cycle")
+ span = by_id.get(identifier)
+ if span is None:
+ raise ValueError("Trace parent is missing")
+ visiting.add(identifier)
+ if span.parent_id is not None:
+ visit(span.parent_id)
+ elif span.id != root_id:
+ raise ValueError("Unexpected trace root")
+ visiting.remove(identifier)
+ complete.add(identifier)
+ ordered.append(span)
+
+ for span in spans:
+ visit(span.id)
+ return tuple(ordered)
diff --git a/api/services/tracing/otlp_export.py b/api/services/tracing/otlp_export.py
new file mode 100644
index 00000000000..9507e82f495
--- /dev/null
+++ b/api/services/tracing/otlp_export.py
@@ -0,0 +1,202 @@
+"""Encode completed spans with stable IDs and explicit provider ingestion ACKs.
+
+No tracer or global SDK configuration is created: a replay must preserve the
+original IDs and timestamps, and destination credentials belong to one export.
+"""
+
+import re
+from collections.abc import Callable, Mapping
+from datetime import datetime
+from typing import Any
+from urllib.parse import urlsplit
+from uuid import UUID
+
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.provider_common import json_value, nanos, request
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError
+
+
+def _id(value: str, length: int) -> bytes:
+ if not re.fullmatch(rf"[0-9a-f]{{{length}}}", value) or int(value, 16) == 0:
+ raise TraceExportRejectedError("Application trace contains an invalid OpenTelemetry identifier")
+ return bytes.fromhex(value)
+
+
+def build_request(
+ trace: CanonicalTrace,
+ resource: Mapping[str, object],
+ attributes: Callable[[CanonicalSpan], Mapping[str, object]],
+) -> Any:
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+ from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
+ from opentelemetry.proto.trace.v1.trace_pb2 import Status
+
+ def attribute(key: str, value: object) -> Any:
+ if isinstance(value, bool):
+ encoded = AnyValue(bool_value=value)
+ elif isinstance(value, int):
+ encoded = AnyValue(int_value=value)
+ elif isinstance(value, float):
+ encoded = AnyValue(double_value=value)
+ elif isinstance(value, str):
+ encoded = AnyValue(string_value=value)
+ else:
+ encoded = AnyValue(string_value=json_value(value))
+ return KeyValue(key=key, value=encoded)
+
+ trace_id = _id(trace.trace_id, 32)
+ payload = ExportTraceServiceRequest()
+ resource_spans = payload.resource_spans.add()
+ resource_spans.resource.attributes.extend(attribute(k, v) for k, v in resource.items() if v is not None)
+ scope = resource_spans.scope_spans.add()
+ scope.scope.name = "dify.application.tracing"
+ scope.scope.version = "1"
+ for span in trace.spans:
+ if span.end_time is None or nanos(span.end_time) < nanos(span.start_time):
+ raise TraceExportRejectedError("Application trace contains incomplete or invalid timestamps")
+ target = scope.spans.add(
+ trace_id=trace_id,
+ span_id=_id(span.id, 16),
+ parent_span_id=_id(span.parent_id, 16) if span.parent_id else b"",
+ name=span.name,
+ kind=1,
+ start_time_unix_nano=nanos(span.start_time),
+ end_time_unix_nano=nanos(span.end_time),
+ flags=1,
+ )
+ target.status.code = (
+ Status.STATUS_CODE_ERROR if span.status is CanonicalSpanStatus.ERROR else Status.STATUS_CODE_OK
+ )
+ if span.error:
+ target.status.message = span.error
+ target.attributes.extend(attribute(k, v) for k, v in attributes(span).items() if v is not None)
+ return payload
+
+
+def _check_response(response: Any, provider: str) -> None:
+ if response.partial_success.rejected_spans:
+ # Retrying the entire batch would duplicate the accepted subset.
+ raise TraceExportUncertainError(f"{provider} partially accepted the trace; reconciliation is required")
+
+
+def is_aliyun_http_success_ack(status_code: int, content: bytes) -> bool:
+ """Recognize the exact ACK observed from Aliyun's HTTP Trace collector.
+
+ This literal response is an observed compatibility behavior, not the
+ documented OTLP ExportTraceServiceResponse. Do not normalize other bodies
+ or accept it for another status or provider.
+ """
+ return status_code == 200 and content == b'"success"'
+
+
+def _langfuse_enqueue_ack(body: object) -> bool:
+ """Recognize Langfuse's queue acknowledgement without exposing its auth data.
+
+ The v4 HTTP route returns the job after blob upload and awaited queue.add,
+ rather than an OTLP ExportTraceServiceResponse. This confirms enqueue only.
+ Upstream: langfuse/langfuse@bc1ade0aacca75104bad934e92d70a454ec16e67,
+ web/src/server/otel/processOtelIngestion.ts:277 and
+ packages/shared/src/server/otel/OtelIngestionProcessor.ts:350.
+ """
+ if not isinstance(body, Mapping) or body.get("name") != "otel-ingestion-job":
+ return False
+ if any(key in body for key in ("partialSuccess", "partial_success", "error", "errors")) or body.get("failedReason"):
+ return False
+ data = body.get("data")
+ if not isinstance(data, Mapping) or data.get("name") != "otel-ingestion-job":
+ return False
+ identifier, timestamp = data.get("id"), data.get("timestamp")
+ if not isinstance(identifier, str) or not isinstance(timestamp, str):
+ return False
+ try:
+ if UUID(identifier).int == 0 or datetime.fromisoformat(timestamp).tzinfo is None:
+ return False
+ except ValueError:
+ return False
+ payload = data.get("payload")
+ if not isinstance(payload, Mapping):
+ return False
+ stored_data, auth = payload.get("data"), payload.get("authCheck")
+ if not isinstance(stored_data, Mapping) or not isinstance(auth, Mapping) or auth.get("validKey") is not True:
+ return False
+ scope = auth.get("scope")
+ if not isinstance(scope, Mapping) or scope.get("accessLevel") != "project":
+ return False
+ return all(
+ isinstance(value, str) and bool(value.strip())
+ for value in (stored_data.get("fileKey"), stored_data.get("publicKey"), scope.get("projectId"))
+ )
+
+
+def export_http(provider: str, url: str, headers: Mapping[str, str], payload: Any) -> None:
+ from google.protobuf.json_format import ParseDict, ParseError
+ from google.protobuf.message import DecodeError
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceResponse
+
+ response = request(
+ "POST",
+ url,
+ provider,
+ headers={"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf", **headers},
+ content=payload.SerializeToString(),
+ )
+ if provider == "aliyun" and is_aliyun_http_success_ack(response.status_code, response.content):
+ return
+ result = ExportTraceServiceResponse()
+ if response.content:
+ try:
+ if "json" in response.headers.get("content-type", "").lower():
+ body = response.json()
+ if not isinstance(body, dict):
+ raise ValueError("Invalid ingestion acknowledgment object")
+ if (
+ provider == "langfuse"
+ and response.status_code == 200
+ and any(
+ key.lower() == "x-langfuse-ingestion-version" and value == "4" for key, value in headers.items()
+ )
+ and _langfuse_enqueue_ack(body)
+ ):
+ return
+ ParseDict(body, result)
+ else:
+ result.ParseFromString(response.content)
+ except (DecodeError, ParseError, ValueError):
+ raise TraceExportUncertainError(f"{provider} returned an unreadable ingestion acknowledgment") from None
+ _check_response(result, provider)
+
+
+def export_grpc(provider: str, url: str, headers: Mapping[str, str], payload: Any) -> None:
+ """Use the provider's gRPC protocol; HTTP exports use the SSRF HTTP owner."""
+ import grpc # type: ignore[import-untyped]
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub
+
+ parsed = urlsplit(url)
+ if not parsed.hostname or parsed.username or parsed.password or parsed.scheme not in {"http", "https"}:
+ raise TraceExportRejectedError("Invalid OTLP gRPC endpoint")
+ port = parsed.port or (4317 if provider == "tencent" else 443)
+ host = f"[{parsed.hostname}]" if ":" in parsed.hostname else parsed.hostname
+ target = f"{host}:{port}"
+ channel = (
+ grpc.insecure_channel(target)
+ if parsed.scheme == "http"
+ else grpc.secure_channel(target, grpc.ssl_channel_credentials())
+ )
+ try:
+ result = TraceServiceStub(channel).Export(payload, metadata=tuple(headers.items()), timeout=30)
+ except grpc.RpcError as error:
+ code = error.code()
+ if code in {
+ grpc.StatusCode.DEADLINE_EXCEEDED,
+ grpc.StatusCode.CANCELLED,
+ grpc.StatusCode.UNKNOWN,
+ grpc.StatusCode.UNAVAILABLE,
+ }:
+ raise TraceExportUncertainError(f"{provider} ingestion acknowledgment is unknown") from None
+ if code in {grpc.StatusCode.RESOURCE_EXHAUSTED, grpc.StatusCode.ABORTED}:
+ raise RetryableTraceDispatchError(f"{provider} ingestion temporarily unavailable") from None
+ raise TraceExportRejectedError(f"{provider} ingestion rejected") from None
+ finally:
+ channel.close()
+ _check_response(result, provider)
diff --git a/api/services/tracing/provider_common.py b/api/services/tracing/provider_common.py
new file mode 100644
index 00000000000..88fdef0636a
--- /dev/null
+++ b/api/services/tracing/provider_common.py
@@ -0,0 +1,127 @@
+"""Shared serialization and HTTP boundaries for application trace adapters."""
+
+import json
+import math
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from typing import Any
+from urllib.parse import urlsplit, urlunsplit
+
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError
+from services.tracing.request_log_protection import protect_http_request_logs
+
+
+def json_value(value: object) -> str:
+ return json.dumps(value, ensure_ascii=False, default=str, allow_nan=False)
+
+
+def utc(value: datetime) -> datetime:
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+
+
+def nanos(value: datetime) -> int:
+ return int(utc(value).timestamp() * 1_000_000_000)
+
+
+def finished_at(span: CanonicalSpan) -> datetime:
+ if span.end_time is None:
+ raise TraceExportRejectedError("Application trace requires completed spans")
+ return utc(span.end_time)
+
+
+def endpoint(value: object) -> str:
+ """Validate a destination without allowing credentials in HTTP authority."""
+ if not isinstance(value, str):
+ raise TraceExportRejectedError("Tracing endpoint is required")
+ parsed = urlsplit(value)
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
+ raise TraceExportRejectedError("Invalid tracing endpoint")
+ if parsed.query or parsed.fragment:
+ raise TraceExportRejectedError("Tracing endpoint must not contain query parameters or a fragment")
+ return urlunsplit(parsed).rstrip("/")
+
+
+def required(config: Mapping[str, Any], name: str) -> str:
+ value = config.get(name)
+ if not isinstance(value, str) or not value.strip():
+ raise TraceExportRejectedError(f"Tracing configuration requires {name}")
+ return value
+
+
+def token_usage(span: CanonicalSpan) -> dict[str, int]:
+ """Only model-call spans carry billable usage; wrappers retain metadata."""
+ if span.kind is not CanonicalSpanKind.LLM:
+ return {}
+ result = {}
+ for source, target in (("prompt_tokens", "input"), ("completion_tokens", "output"), ("total_tokens", "total")):
+ value = span.metadata.get(source)
+ if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
+ result[target] = value
+ return result
+
+
+def usd_cost(span: CanonicalSpan) -> float | None:
+ if span.kind is not CanonicalSpanKind.LLM or str(span.metadata.get("currency", "USD")).upper() != "USD":
+ return None
+ try:
+ value = float(span.metadata["total_price"])
+ except (KeyError, TypeError, ValueError):
+ return None
+ return value if math.isfinite(value) and value >= 0 else None
+
+
+def check_http_status(status: int, provider: str) -> None:
+ if 200 <= status < 300:
+ return
+ if status in {408, 425, 429} or status >= 500:
+ raise RetryableTraceDispatchError(f"{provider} ingestion temporarily unavailable (HTTP {status})")
+ raise TraceExportRejectedError(f"{provider} ingestion rejected (HTTP {status})")
+
+
+def request(method: str, url: str, provider: str, **kwargs: Any):
+ """Use the configured SSRF boundary, with retries owned by the durable task."""
+ import httpx
+ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value
+
+ from core.helper import ssrf_proxy
+
+ token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
+ try:
+ with protect_http_request_logs(provider == "aliyun"):
+ if provider == "aliyun":
+ response = _request_sensitive_aliyun(method, url, **kwargs)
+ else:
+ response = ssrf_proxy.make_request(
+ method, url, max_retries=0, timeout=30.0, follow_redirects=False, **kwargs
+ )
+ except httpx.TransportError:
+ raise TraceExportUncertainError(f"{provider} ingestion outcome is unknown after transport failure") from None
+ finally:
+ detach(token)
+ check_http_status(response.status_code, provider)
+ return response
+
+
+def _request_sensitive_aliyun(method: str, url: str, **kwargs: Any):
+ """Discard private HTTP exception objects before raising a public failure."""
+ import httpx
+
+ from core.helper import ssrf_proxy
+
+ response: httpx.Response | None = None
+ failure: RuntimeError | None = None
+ try:
+ response = ssrf_proxy.make_request(method, url, max_retries=0, timeout=30.0, follow_redirects=False, **kwargs)
+ except httpx.RequestError:
+ failure = TraceExportUncertainError("aliyun ingestion outcome is unknown after transport failure")
+ except Exception:
+ failure = TraceExportRejectedError("aliyun HTTP request was rejected by the transport boundary")
+ # Raise after leaving the original exception handler so neither its cause
+ # nor implicit context retains a credential-bearing request or message.
+ if failure is not None:
+ raise failure from None
+ if response is None:
+ raise TraceExportUncertainError("aliyun HTTP transport returned no acknowledgment")
+ return response
diff --git a/api/services/tracing/provider_dispatch.py b/api/services/tracing/provider_dispatch.py
new file mode 100644
index 00000000000..06317c582e8
--- /dev/null
+++ b/api/services/tracing/provider_dispatch.py
@@ -0,0 +1,122 @@
+"""Dispatch a completed application trace without exposing destination credentials.
+
+An accepted export acknowledges ingestion, not query availability. Completed spans
+must not be exported again after acceptance; some providers do not deduplicate IDs.
+The durable task owner decides whether an uncertain transport outcome is retried.
+"""
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from importlib import import_module
+from typing import Any, Literal
+from urllib.parse import urlsplit
+
+from core.ops.exceptions import RetryableTraceDispatchError, TraceProviderNotInstalledError
+from core.ops.unified_trace.entities import CanonicalTrace
+
+
+@dataclass(frozen=True)
+class TraceExportResult:
+ trace_id: str
+ trace_url: str | None = None
+ span_urls: dict[str, str] = field(default_factory=dict)
+ status: Literal["accepted"] = "accepted"
+
+
+class TraceExportRejectedError(RuntimeError):
+ """A terminal export rejection, containing no credential or response body."""
+
+
+class TraceExportUncertainError(RetryableTraceDispatchError):
+ """The transport failed without proving whether ingestion accepted the data."""
+
+ acceptance_unknown = True
+
+
+def frozen_project_url(provider: str, config: Mapping[str, Any]) -> str | None:
+ """Read only persisted destination metadata, without credentials or vendor I/O."""
+ if provider == "arize":
+ try:
+ from dify_trace_arize_phoenix.arize_configuration import project_url
+ except ModuleNotFoundError:
+ return None
+
+ # Never substitute the current app destination for a historical run.
+ # Unverified legacy redirects remain stored, but are not presented as
+ # project links. A future config save does not backfill that snapshot.
+ return project_url(config)
+ # These adapters expose a platform console, not the run's destination project.
+ if provider in {"aliyun", "tencent"}:
+ return None
+ value = config.get("project_url")
+ if (
+ not isinstance(value, str)
+ or not value
+ or any(ord(char) <= 32 or ord(char) == 127 for char in value)
+ or "\\" in value
+ ):
+ return None
+ try:
+ parsed = urlsplit(value)
+ if (
+ parsed.scheme not in {"http", "https"}
+ or not parsed.hostname
+ or parsed.username is not None
+ or parsed.password is not None
+ ):
+ return None
+ _ = parsed.port
+ except ValueError:
+ return None
+ return value
+
+
+_MODULES = {
+ "langfuse": "dify_trace_langfuse.agent_trace",
+ "langsmith": "dify_trace_langsmith.agent_trace",
+ "phoenix": "dify_trace_arize_phoenix.agent_trace",
+ "arize": "dify_trace_arize_phoenix.agent_trace",
+ "opik": "dify_trace_opik.agent_trace",
+ "weave": "dify_trace_weave.agent_trace",
+ "mlflow": "dify_trace_mlflow.agent_trace",
+ "databricks": "dify_trace_mlflow.agent_trace",
+ "aliyun": "dify_trace_aliyun.agent_trace",
+ "tencent": "dify_trace_tencent.agent_trace",
+}
+
+
+def provider_capabilities(provider: str) -> dict[str, bool | str]:
+ """Describe implemented mappings, independently of installation/account tests."""
+ if provider not in _MODULES:
+ raise ValueError("Unsupported tracing provider")
+ return {
+ "provider": provider,
+ "agent_spans": True,
+ "session": True,
+ "native_usage": True,
+ "native_cost": provider in {"langfuse", "langsmith", "opik", "mlflow", "databricks"},
+ "trace_url": provider in {"langfuse", "langsmith", "weave", "mlflow", "databricks"},
+ "span_url": provider in {"langfuse", "langsmith", "weave"},
+ "export_ack": "accepted",
+ }
+
+
+def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:
+ """Export one complete tree to the caller's already-authorized destination."""
+ module_name = _MODULES.get(provider)
+ if module_name is None:
+ raise TraceExportRejectedError("Unsupported tracing provider")
+ if trace.external_parent is not None or trace.required_parent_context_id is not None:
+ raise TraceExportRejectedError("Application trace export requires a complete resolved tree")
+ if any(span.end_time is None for span in trace.spans):
+ raise TraceExportRejectedError("Application trace export requires completed spans")
+ try:
+ module = import_module(module_name)
+ return module.export_trace(provider, config, trace)
+ except (TraceExportRejectedError, RetryableTraceDispatchError):
+ raise
+ except ImportError as error:
+ raise TraceProviderNotInstalledError(provider, error.name or module_name) from None
+ except Exception:
+ # Vendor exceptions can contain full requests, headers, and endpoint secrets.
+ raise TraceExportRejectedError(f"{provider} trace export failed") from None
diff --git a/api/services/tracing/recovery.py b/api/services/tracing/recovery.py
new file mode 100644
index 00000000000..4e9c8729edb
--- /dev/null
+++ b/api/services/tracing/recovery.py
@@ -0,0 +1,200 @@
+"""Recover dropped in-memory terminal events from committed business records.
+
+Runtime finalization can precede the API's final answer transaction. In particular,
+AgentAppRunner persists usage independently, so usage alone is not proof that a
+Message is complete. Recovery never invents answers, usage, or success states.
+"""
+
+import json
+import logging
+import math
+from datetime import datetime, timedelta
+from typing import Protocol, cast
+
+from sqlalchemy import select
+from sqlalchemy.exc import SQLAlchemyError
+
+from configs import dify_config
+from core.ops.entities.trace_entity import BaseTraceInfo, MessageTraceInfo, TraceTaskName, WorkflowTraceInfo
+from core.ops.ops_trace_manager import TraceTask
+from graphon.enums import WorkflowExecutionStatus
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceInvocation, ApplicationTraceRun
+from models.enums import MessageStatus
+from models.model import App, AppMode, Conversation, Message, MessageDict
+from models.workflow import WorkflowRun
+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError
+
+logger = logging.getLogger(__name__)
+_SETTLE_SECONDS = 5
+_WORKFLOW_TERMINAL = {
+ WorkflowExecutionStatus.SUCCEEDED,
+ WorkflowExecutionStatus.PARTIAL_SUCCEEDED,
+ WorkflowExecutionStatus.FAILED,
+ WorkflowExecutionStatus.STOPPED,
+}
+
+
+class RecoveryTarget(Protocol):
+ repository: ApplicationTraceRepository
+
+ def capture_terminal(self, info: BaseTraceInfo) -> bool: ...
+
+
+def agent_stop_requested(message: Message) -> bool:
+ """Read the consumer's stop intent without inferring cancellation success."""
+ try:
+ metadata = json.loads(message.message_metadata) if message.message_metadata else {}
+ except (ValueError, TypeError):
+ return False
+ return isinstance(metadata, dict) and metadata.get("agent_stop_requested") is True
+
+
+def _complete_message_metadata(message: Message) -> bool:
+ """Recognize the EasyUI _save_message transaction, not an early usage flush."""
+ try:
+ metadata = json.loads(message.message_metadata) if message.message_metadata else {}
+ except (ValueError, TypeError):
+ return False
+ if (
+ not isinstance(metadata, dict)
+ or not {"annotation_reply", "retriever_resources", "reasoning"} <= metadata.keys()
+ ):
+ return False
+ usage = metadata.get("usage")
+ if not isinstance(usage, dict):
+ return False
+ for name in ("prompt_tokens", "completion_tokens"):
+ value = usage.get(name)
+ if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value) or value < 0:
+ return False
+ latency = message.provider_response_latency
+ return isinstance(latency, (int, float)) and math.isfinite(latency) and latency >= 0
+
+
+def _message_task(repository: ApplicationTraceRepository, run: ApplicationTraceRun, now: datetime) -> TraceTask | None:
+ with repository.sessions() as session:
+ source = session.execute(
+ select(Message, App.mode)
+ .join(App, App.id == Message.app_id)
+ .join(Conversation, Conversation.id == Message.conversation_id)
+ .where(
+ Message.id == run.source_id,
+ Message.app_id == run.app_id,
+ App.tenant_id == run.tenant_id,
+ Conversation.app_id == run.app_id,
+ )
+ ).one_or_none()
+ if source is None:
+ raise TraceCollectionError("tracing_source_not_found", 404)
+ message, app_mode = source
+ pending_agent_stop = app_mode == AppMode.AGENT and agent_stop_requested(message)
+ if message.status is MessageStatus.PAUSED and not pending_agent_stop:
+ return None
+ invocations = list(
+ session.scalars(select(ApplicationTraceInvocation).where(ApplicationTraceInvocation.run_id == run.id))
+ )
+ explicit_error = message.status is MessageStatus.ERROR and bool(message.error)
+ if any(not invocation.finalized or invocation.finalized_at is None for invocation in invocations):
+ if (explicit_error or pending_agent_stop) and message.updated_at + timedelta(
+ seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT
+ ) < now:
+ raise TraceCollectionError("agent_trace_collection_incomplete")
+ return None
+ finalized = [invocation.finalized_at for invocation in invocations if invocation.finalized_at is not None]
+ anchor = max([message.updated_at, *finalized])
+ if pending_agent_stop:
+ # Even a fully saved partial answer/usage cannot establish which
+ # runtime terminal won the stop race. Wait for the existing worker
+ # handoff, or converge to incomplete delivery after its deadline.
+ if anchor + timedelta(seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT) < now:
+ raise TraceCollectionError("business_terminal_not_persisted")
+ return None
+ if run.capture_agent and not invocations:
+ return None
+ if anchor + timedelta(seconds=_SETTLE_SECONDS) > now:
+ return None
+ complete = explicit_error or (
+ message.status is MessageStatus.NORMAL
+ and message.updated_at > message.created_at
+ and _complete_message_metadata(message)
+ )
+ if not complete:
+ if finalized and max(finalized) + timedelta(seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT) < now:
+ raise TraceCollectionError("business_terminal_not_persisted")
+ return None
+ return TraceTask(
+ TraceTaskName.MESSAGE_TRACE,
+ message_id=message.id,
+ conversation_id=message.conversation_id,
+ trace_session_id=run.session_id,
+ )
+
+
+def _workflow_task(repository: ApplicationTraceRepository, run: ApplicationTraceRun, now: datetime) -> TraceTask | None:
+ workflow = repository._source(tenant_id=run.tenant_id, app_id=run.app_id, kind="workflow", source_id=run.source_id)
+ # An asynchronous workflow repository may not have saved the reserved run yet.
+ if (
+ not isinstance(workflow, WorkflowRun)
+ or workflow.status not in _WORKFLOW_TERMINAL
+ or workflow.finished_at is None
+ ):
+ return None
+ if workflow.finished_at + timedelta(seconds=_SETTLE_SECONDS) > now:
+ return None
+ with repository.sessions() as session:
+ conversation_id = session.scalar(
+ select(Message.conversation_id)
+ .join(Conversation, Conversation.id == Message.conversation_id)
+ .where(
+ Message.app_id == run.app_id, Message.workflow_run_id == workflow.id, Conversation.app_id == run.app_id
+ )
+ .limit(1)
+ )
+ return TraceTask(
+ TraceTaskName.WORKFLOW_TRACE,
+ workflow_run_id=workflow.id,
+ conversation_id=conversation_id,
+ trace_session_id=run.session_id,
+ user_id=workflow.created_by,
+ )
+
+
+def recover_terminal(target: RecoveryTarget, run: ApplicationTraceRun) -> bool:
+ """Attempt one bounded recovery and rotate unresolved work behind other runs."""
+ if (
+ run.outer_trace is not None
+ or run.state not in {"collecting", "ready"}
+ or run.source_kind not in {"message", "workflow"}
+ ):
+ return False
+ error_code: str | None = None
+ try:
+ task = (_message_task if run.source_kind == "message" else _workflow_task)(
+ target.repository, run, naive_utc_now()
+ )
+ if task is None:
+ return False
+ info = task.execute()
+ if isinstance(info, MessageTraceInfo) and isinstance(info.message_data, dict):
+ info = info.model_copy(update={"message_data": Message.from_dict(cast(MessageDict, info.message_data))})
+ if isinstance(info, WorkflowTraceInfo) and isinstance(info.workflow_data, dict):
+ info = info.model_copy(update={"workflow_data": WorkflowRun.from_dict(info.workflow_data)})
+ if not isinstance(info, (MessageTraceInfo, WorkflowTraceInfo)):
+ raise TraceCollectionError("business_terminal_recovery_incomplete")
+ if not target.capture_terminal(info):
+ raise TraceCollectionError("business_terminal_recovery_unclaimed")
+ return True
+ except TraceCollectionError as error:
+ error_code = error.code
+ return False
+ except SQLAlchemyError:
+ # Leave the durable source intact. The next reconciler pass can retry.
+ logger.warning("Application trace recovery database unavailable: run_id=%s", run.id)
+ return False
+ except Exception as error:
+ error_code = "business_terminal_recovery_failed"
+ logger.warning("Application trace recovery failed: run_id=%s error_type=%s", run.id, type(error).__name__)
+ return False
+ finally:
+ target.repository.recovery_checked(run.id, error_code=error_code)
diff --git a/api/services/tracing/request_log_protection.py b/api/services/tracing/request_log_protection.py
new file mode 100644
index 00000000000..397290ab4f3
--- /dev/null
+++ b/api/services/tracing/request_log_protection.py
@@ -0,0 +1,98 @@
+"""Hide dynamic transport diagnostics only inside sensitive trace HTTP calls.
+
+The Aliyun collector carries credentials in its path. Do not try to recognize
+individual encodings of those credentials: replace dynamic content at the
+standard Logger.makeRecord entry before any record factory or handler sees it.
+The current record factory remains independently configurable. Other concurrent
+requests retain their normal logging, levels and handlers. This does not defend
+against arbitrary process code replacing the protected logging entry itself.
+"""
+
+import logging
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from threading import RLock
+
+type RecordMaker = Callable[..., logging.LogRecord]
+
+SAFE_HTTP_LOG_MESSAGE = "Sensitive tracing HTTP request; dynamic transport details redacted"
+_sensitive_request: ContextVar[bool] = ContextVar("sensitive_trace_http_request", default=False)
+
+
+def _protected_record_maker(previous: RecordMaker) -> RecordMaker:
+ def create_record(
+ self: logging.Logger,
+ name: str,
+ level: int,
+ fn: str,
+ lno: int,
+ msg: object,
+ args: object,
+ exc_info: object,
+ func: str | None = None,
+ extra: object = None,
+ sinfo: str | None = None,
+ ) -> logging.LogRecord:
+ if not _sensitive_request.get():
+ return previous(self, name, level, fn, lno, msg, args, exc_info, func, extra, sinfo)
+ # Sanitize before delegating, including extras, chained exceptions and
+ # stack information. Never stringify the original diagnostic values.
+ # The original makeRecord still invokes the dynamic current factory.
+ record = previous(self, name, level, fn, lno, SAFE_HTTP_LOG_MESSAGE, (), None, func, None, None)
+ record.msg = SAFE_HTTP_LOG_MESSAGE
+ record.args = ()
+ record.exc_info = None
+ record.exc_text = None
+ record.stack_info = None
+ return record
+
+ return create_record
+
+
+class _RecordMakerLeases:
+ def __init__(self) -> None:
+ self.lock = RLock()
+ self.users = 0
+ self.previous: RecordMaker | None = None
+ self.installed: RecordMaker | None = None
+
+ def acquire(self) -> None:
+ with self.lock:
+ if self.users == 0:
+ self.previous = logging.Logger.makeRecord
+ self.installed = _protected_record_maker(self.previous)
+ logging.Logger.makeRecord = self.installed
+ self.users += 1
+
+ def release(self) -> None:
+ with self.lock:
+ self.users -= 1
+ if self.users == 0:
+ # Preserve an independently installed method instead of
+ # overwriting another logging integration during cleanup.
+ if logging.Logger.makeRecord is self.installed and self.previous is not None:
+ logging.Logger.makeRecord = self.previous
+ self.previous = None
+ self.installed = None
+
+
+_record_maker_leases = _RecordMakerLeases()
+
+
+@contextmanager
+def protect_http_request_logs(sensitive: bool) -> Generator[None]:
+ """Scope protection to this call; nested public calls restore their parent."""
+ token = _sensitive_request.set(sensitive)
+ leased = False
+ try:
+ if sensitive:
+ _record_maker_leases.acquire()
+ leased = True
+ yield
+ finally:
+ try:
+ if leased:
+ _record_maker_leases.release()
+ finally:
+ _sensitive_request.reset(token)
diff --git a/api/services/tracing/runtime_bridge.py b/api/services/tracing/runtime_bridge.py
new file mode 100644
index 00000000000..f94aae3b192
--- /dev/null
+++ b/api/services/tracing/runtime_bridge.py
@@ -0,0 +1,205 @@
+"""Fail-open business hooks and durable terminal handoff for application tracing.
+
+The deployment gate is checked before constructing any database-backed service.
+Execution context carries opaque correlation and policy only, never provider keys.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Literal
+
+from dify_agent.layers.execution_context import ApplicationTracingContext
+
+from configs import dify_config
+
+if TYPE_CHECKING:
+ from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
+ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
+ from core.ops.entities.trace_entity import BaseTraceInfo
+ from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
+ from services.tracing.service import ApplicationTracingService
+
+logger = logging.getLogger(__name__)
+
+
+def workflow_has_agent_v2(graph: object) -> bool:
+ """Scope the new pipeline to graphs containing the Dify Agent v2 node."""
+ from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data
+
+ if isinstance(graph, str):
+ try:
+ graph = json.loads(graph)
+ except ValueError:
+ return False
+ if not isinstance(graph, Mapping) or not isinstance(graph.get("nodes"), list):
+ return False
+ return any(
+ isinstance(node, Mapping) and isinstance(node.get("data"), Mapping) and is_dify_agent_node_data(node["data"])
+ for node in graph["nodes"]
+ )
+
+
+def _service() -> ApplicationTracingService:
+ from services.tracing.service import application_tracing_service
+
+ return application_tracing_service()
+
+
+def _log_failure(stage: str, error: Exception) -> None:
+ from repositories.application_trace_repository import TraceCollectionError
+
+ code = error.code if isinstance(error, TraceCollectionError) else type(error).__name__
+ logger.warning("Application tracing hook skipped: stage=%s error_code=%s", stage, code)
+
+
+def prepare_agent_app(context: AgentAppRuntimeBuildContext) -> ApplicationTracingContext | None:
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return None
+ try:
+ return _service().prepare_invocation(
+ tenant_id=context.dify_context.tenant_id,
+ app_id=context.dify_context.app_id,
+ agent_id=context.agent_id,
+ source_kind="message",
+ source_id=context.idempotency_key,
+ session_id=context.dify_context.trace_session_id or context.conversation_id,
+ node_execution_id=None,
+ attempt=0,
+ call_key=context.idempotency_key,
+ )
+ except Exception as error:
+ _log_failure("agent_app", error)
+ return None
+
+
+def capture_agent_app_terminal(
+ *,
+ tenant_id: str,
+ app_id: str,
+ message_id: str,
+ outcome: Literal["succeeded", "failed", "cancelled"],
+ output: str | None,
+) -> None:
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return
+ try:
+ _service().capture_agent_app_terminal(
+ tenant_id=tenant_id, app_id=app_id, message_id=message_id, outcome=outcome, output=output
+ )
+ except Exception as error:
+ _log_failure("agent_app_terminal", error)
+
+
+def prepare_workflow_agent(
+ context: WorkflowAgentRuntimeBuildContext, conversation_id: str | None
+) -> ApplicationTracingContext | None:
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return None
+ try:
+ # Hidden Tool nodes own a different app but share the caller's run and are
+ # absent from its outer trace. Neither a source-app snapshot nor an orphan
+ # parent is safe. Keep this new nested scope uncollected until it has an
+ # explicit owner/parent contract; ordinary nodes and previews are unchanged.
+ if context.dify_context.workflow_tool_invocation_id is not None:
+ return None
+ source_id = context.workflow_run_id or context.node_execution_id
+ return _service().prepare_invocation(
+ tenant_id=context.dify_context.tenant_id,
+ app_id=context.dify_context.app_id,
+ agent_id=context.agent.id,
+ source_kind="workflow" if context.workflow_run_id else "node",
+ source_id=source_id,
+ session_id=context.dify_context.trace_session_id or conversation_id or source_id,
+ node_execution_id=context.node_execution_id,
+ attempt=context.attempt,
+ call_key=f"{context.node_execution_id}:{context.attempt}",
+ workflow_id=context.workflow_id,
+ )
+ except Exception as error:
+ _log_failure("workflow_agent", error)
+ return None
+
+
+def freeze_workflow_run(
+ entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,
+ *,
+ workflow_id: str,
+ workflow_run_id: str,
+ conversation_id: object,
+ graph_data: object,
+) -> None:
+ """Freeze policy immediately after the business workflow row is saved."""
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return
+ if not workflow_has_agent_v2(graph_data):
+ return
+ try:
+ explicit_session = entity.extras.get("trace_session_id")
+ session_id = next(
+ (
+ value
+ for value in (explicit_session, conversation_id, workflow_run_id)
+ if isinstance(value, str) and value
+ ),
+ workflow_run_id,
+ )
+ _service().start(
+ tenant_id=entity.app_config.tenant_id,
+ app_id=entity.app_config.app_id,
+ source_kind="workflow",
+ source_id=workflow_run_id,
+ session_id=session_id,
+ workflow_id=workflow_id,
+ )
+ except Exception as error:
+ _log_failure("workflow_start", error)
+
+
+def queue_enabled_at_start(app_id: object) -> bool:
+ """Read the existing application switch without constructing a vendor SDK."""
+ if not dify_config.OPS_AGENT_TRACE_ENABLED or app_id is None:
+ return False
+ from extensions.ext_database import db
+ from models.model import App
+
+ try:
+ app = db.session.get(App, str(app_id))
+ policy = json.loads(app.tracing) if app is not None and app.tracing else {}
+ return isinstance(policy, dict) and policy.get("enabled") is True
+ except Exception as error:
+ _log_failure("queue_policy", error)
+ return False
+
+
+def capture_terminal(info: BaseTraceInfo) -> bool:
+ """Transfer ownership before legacy dispatch; transient DB failures retry.
+
+ Unlike request construction, a worker must not fall back to a second export
+ path when the durable collector is temporarily inaccessible.
+ """
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return False
+ from sqlalchemy.exc import SQLAlchemyError
+
+ from core.ops.exceptions import RetryableTraceDispatchError
+
+ try:
+ return _service().capture_terminal(info)
+ except SQLAlchemyError:
+ raise RetryableTraceDispatchError("Application tracing persistence temporarily unavailable") from None
+
+
+class RuntimeApplicationTracingHooks:
+ """Stateless app-scoped adapter; database services are created only on invocation."""
+
+ __slots__ = ()
+
+ prepare_agent_app = staticmethod(prepare_agent_app)
+ prepare_workflow_agent = staticmethod(prepare_workflow_agent)
+ freeze_workflow_run = staticmethod(freeze_workflow_run)
+ queue_enabled_at_start = staticmethod(queue_enabled_at_start)
+ capture_terminal = staticmethod(capture_terminal)
+ capture_agent_app_terminal = staticmethod(capture_agent_app_terminal)
diff --git a/api/services/tracing/sdk_transport.py b/api/services/tracing/sdk_transport.py
new file mode 100644
index 00000000000..d78ee4bae9c
--- /dev/null
+++ b/api/services/tracing/sdk_transport.py
@@ -0,0 +1,42 @@
+"""Per-client SDK transports using Dify's HTTP egress policy, never global state."""
+
+from typing import Any, override
+
+import httpx
+import requests
+
+from services.tracing.provider_common import request as send_http
+from services.tracing.provider_dispatch import TraceExportRejectedError
+
+
+class SSRFTransport(httpx.BaseTransport):
+ def __init__(self, provider: str) -> None:
+ self.provider = provider
+
+ @override
+ def handle_request(self, request: httpx.Request) -> httpx.Response:
+ response = send_http(
+ request.method, str(request.url), self.provider, headers=dict(request.headers), content=request.read()
+ )
+ return httpx.Response(response.status_code, headers=response.headers, content=response.content, request=request)
+
+
+class SSRFSession(requests.Session):
+ def __init__(self, provider: str) -> None:
+ super().__init__()
+ self.provider = provider
+
+ @override
+ def send(self, request: requests.PreparedRequest, **kwargs: Any) -> requests.Response:
+ if not request.method or not request.url or not isinstance(request.body, (str, bytes, type(None))):
+ raise TraceExportRejectedError("Tracing SDK prepared an unsupported HTTP request")
+ received = send_http(
+ request.method, request.url, self.provider, headers=dict(request.headers), content=request.body
+ )
+ response = requests.Response()
+ response.status_code = received.status_code
+ response.headers.update(received.headers)
+ response._content = received.content
+ response.url = str(received.url)
+ response.request = request
+ return response
diff --git a/api/services/tracing/service.py b/api/services/tracing/service.py
new file mode 100644
index 00000000000..acce7315b1d
--- /dev/null
+++ b/api/services/tracing/service.py
@@ -0,0 +1,705 @@
+"""Application tracing orchestration; secrets and vendor I/O stay in the API worker."""
+
+import json
+import logging
+import threading
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
+from typing import Any, Literal, cast
+
+from dify_agent.layers.execution_context import ApplicationTracingContext
+from dify_agent.protocol.tracing import AgentTraceBatch
+from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
+
+from configs import dify_config
+from core.ops.entities.trace_entity import BaseTraceInfo, MessageTraceInfo, TraceTaskName, WorkflowTraceInfo
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.ops_trace_manager import OpsTraceManager, TraceTask
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from core.ops.unified_trace.trace_builder import CanonicalTraceBuilder, RepositoryWorkflowExecutionLoader
+from extensions.ext_database import db
+from models import Account, App, AppMode, Conversation, Message, Workflow, WorkflowNodeExecutionModel, WorkflowRun
+from models.application_trace import ApplicationTraceRun
+from models.model import MessageDict
+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError
+from services.tracing.normalization import normalize_agent_span, order_spans, remap_outer_trace, sanitize
+from services.tracing.provider_dispatch import (
+ TraceExportUncertainError,
+ export_trace,
+ frozen_project_url,
+ provider_capabilities,
+)
+from services.tracing.runtime_bridge import workflow_has_agent_v2
+
+logger = logging.getLogger(__name__)
+
+
+def _message_inputs_are_empty(inputs: object) -> bool:
+ if isinstance(inputs, str):
+ if not inputs.strip():
+ return True
+ try:
+ inputs = json.loads(inputs)
+ except ValueError:
+ return False
+ return inputs is None or isinstance(inputs, (str, dict, list)) and not inputs
+
+
+class ApplicationTracingService:
+ def __init__(self, repository: ApplicationTraceRepository) -> None:
+ self.repository = repository
+
+ def start(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ source_kind: Literal["message", "workflow", "node"],
+ source_id: str,
+ session_id: str,
+ workflow_id: str | None = None,
+ ) -> ApplicationTraceRun | None:
+ if not dify_config.OPS_AGENT_TRACE_ENABLED:
+ return None
+ run = self.repository.start(
+ tenant_id=tenant_id,
+ app_id=app_id,
+ source_kind=source_kind,
+ source_id=source_id,
+ session_id=session_id,
+ workflow_id=workflow_id,
+ )
+ return run if run is not None and run.state != "not_collected" else None
+
+ def prepare_invocation(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ agent_id: str,
+ source_kind: Literal["message", "workflow", "node"],
+ source_id: str,
+ session_id: str,
+ node_execution_id: str | None,
+ attempt: int,
+ call_key: str,
+ workflow_id: str | None = None,
+ ) -> ApplicationTracingContext | None:
+ run = self.start(
+ tenant_id=tenant_id,
+ app_id=app_id,
+ source_kind=source_kind,
+ source_id=source_id,
+ session_id=session_id,
+ workflow_id=workflow_id,
+ )
+ if run is None or not run.capture_agent:
+ return None
+ invocation = self.repository.invocation(
+ run_id=run.id, agent_id=agent_id, node_execution_id=node_execution_id, attempt=attempt, call_key=call_key
+ )
+ return ApplicationTracingContext(
+ run_id=run.id,
+ invocation_id=invocation.id,
+ trace_id=run.trace_id,
+ parent_span_id=invocation.span_id,
+ session_id=run.session_id,
+ capture_content=run.capture_content,
+ )
+
+ def ingest(self, batch: AgentTraceBatch) -> int:
+ accepted, run_id = self.repository.ingest(
+ batch,
+ max_spans=dify_config.OPS_AGENT_TRACE_MAX_SPANS,
+ max_bytes=dify_config.OPS_AGENT_TRACE_MAX_BYTES,
+ )
+ # Persistence commits before scheduling. Beat reconciles a broker outage.
+ self.schedule(run_id)
+ return accepted
+
+ def capture_agent_app_terminal(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ message_id: str,
+ outcome: Literal["succeeded", "failed", "cancelled"],
+ output: str | None,
+ ) -> None:
+ """Persist a worker-observed terminal even after its SSE consumer closes.
+
+ Usage-only Message writes are intentionally insufficient for recovery.
+ This handoff is made only after the runner observes a terminal result or
+ finishes handling an explicit stop, and it does not rewrite Message data.
+ """
+ source = self.repository._source(tenant_id=tenant_id, app_id=app_id, kind="message", source_id=message_id)
+ if not isinstance(source, Message):
+ raise TraceCollectionError("tracing_source_not_found", 404)
+ conversation_id = source.conversation_id
+ run = self.repository.source_run(
+ tenant_id=tenant_id, app_id=app_id, source_kind="message", source_id=message_id
+ )
+ if run is not None and run.state == "not_collected":
+ return
+ info = TraceTask(
+ TraceTaskName.MESSAGE_TRACE,
+ message_id=message_id,
+ conversation_id=conversation_id,
+ trace_session_id=run.session_id if run else conversation_id,
+ ).execute()
+ if not isinstance(info, MessageTraceInfo):
+ raise TraceCollectionError("tracing_terminal_not_supported")
+ if isinstance(info.message_data, dict):
+ info.message_data = Message.from_dict(cast(MessageDict, info.message_data))
+ if (
+ isinstance(info.message_data, Message)
+ and _message_inputs_are_empty(info.inputs)
+ and info.message_data.query
+ ):
+ # Early errors may precede formatted-message persistence. Preserve only
+ # the known user query on the root; this is not an inferred LLM prompt.
+ info.inputs = {"query": info.message_data.query}
+ if output is not None and isinstance(info.message_data, Message):
+ # This is a detached trace snapshot, not a write to the business row.
+ info.message_data.answer = output
+ info.outputs = output if output is not None else info.outputs
+ info.end_time = datetime.now(UTC)
+ info.metadata = {**info.metadata, "message_execution_state": outcome}
+ if outcome != "succeeded":
+ info.error = f"Agent message {outcome}"
+ self.capture_terminal(info)
+
+ @staticmethod
+ def schedule(run_id: str) -> None:
+ from tasks.application_trace_task import export_application_trace
+
+ try:
+ export_application_trace.delay(run_id)
+ except Exception:
+ logger.warning("Application trace is persisted; scheduling deferred: run_id=%s", run_id)
+
+ def _service_account(self, run: ApplicationTraceRun, app_id: str) -> Account:
+ with self.repository.sessions() as session:
+ app = session.scalar(
+ select(App).where(App.id == app_id, App.id == run.app_id, App.tenant_id == run.tenant_id)
+ )
+ if app is None or not app.created_by:
+ raise TraceCollectionError("tracing_owner_not_found", 404)
+ account = session.get(Account, app.created_by)
+ if account is None:
+ raise TraceCollectionError("tracing_owner_not_found", 404)
+ account.set_tenant_id_with_session(run.tenant_id, session=session)
+ return account
+
+ def _run_for_source(
+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str
+ ) -> ApplicationTraceRun | None:
+ run = self.repository.source_run(
+ tenant_id=tenant_id, app_id=app_id, source_kind=source_kind, source_id=source_id
+ )
+ if run is None and source_kind == "message":
+ with self.repository.sessions() as session:
+ workflow_run_id = session.scalar(
+ select(Message.workflow_run_id)
+ .join(App, App.id == Message.app_id)
+ .where(
+ Message.id == source_id,
+ Message.app_id == app_id,
+ App.tenant_id == tenant_id,
+ )
+ )
+ if workflow_run_id:
+ run = self.repository.source_run(
+ tenant_id=tenant_id, app_id=app_id, source_kind="workflow", source_id=workflow_run_id
+ )
+ return run
+
+ def _managed_owner(self, app: App, message: Message | None = None, workflow_run_id: str | None = None) -> bool:
+ """Honor frozen owners first; ordinary workflows retain legacy tracing."""
+ if message is not None:
+ if self._run_for_source(
+ tenant_id=app.tenant_id, app_id=app.id, source_kind="message", source_id=message.id
+ ):
+ return True
+ workflow_run_id = message.workflow_run_id or workflow_run_id
+ if app.mode == AppMode.AGENT:
+ return True
+ if workflow_run_id:
+ if self.repository.source_run(
+ tenant_id=app.tenant_id, app_id=app.id, source_kind="workflow", source_id=workflow_run_id
+ ):
+ return True
+ workflow = self.repository._source(
+ tenant_id=app.tenant_id, app_id=app.id, kind="workflow", source_id=workflow_run_id
+ )
+ if isinstance(workflow, WorkflowRun):
+ return workflow_has_agent_v2(workflow.graph)
+ if app.mode == AppMode.ADVANCED_CHAT or (app.mode == AppMode.WORKFLOW and workflow_run_id):
+ # Naming may run before Message.workflow_run_id is populated. Use
+ # the owned app workflow; an unknown snapshot fails closed. The same
+ # applies to a Workflow whose run is not yet indexed by LogStore.
+ with self.repository.sessions() as session:
+ definition = (
+ session.scalar(
+ select(Workflow).where(
+ Workflow.id == app.workflow_id,
+ Workflow.app_id == app.id,
+ Workflow.tenant_id == app.tenant_id,
+ )
+ )
+ if app.workflow_id
+ else None
+ )
+ return definition is None or workflow_has_agent_v2(definition.graph)
+ return False
+
+ def _capture_auxiliary(self, info: BaseTraceInfo) -> bool:
+ """Resolve real business owners even when message_id is a workflow-log ID."""
+ actual_message_id = info.message_data.id if isinstance(info.message_data, Message) else None
+ message_ids = [actual_message_id, info.metadata.get("message_id"), info.message_id]
+ message: Message | None = None
+ app: App | None = None
+ with self.repository.sessions() as session:
+ for message_id in message_ids:
+ if not isinstance(message_id, str):
+ continue
+ owner = session.execute(
+ select(Message, App).join(App, App.id == Message.app_id).where(Message.id == message_id)
+ ).first()
+ if owner is not None:
+ message, app = owner
+ break
+ conversation_id = info.metadata.get("conversation_id")
+ if app is None and isinstance(conversation_id, str):
+ app = session.scalar(
+ select(App)
+ .join(Conversation, Conversation.app_id == App.id)
+ .where(Conversation.id == conversation_id)
+ )
+ if app is not None:
+ message = session.scalar(
+ select(Message)
+ .where(Message.conversation_id == conversation_id, Message.app_id == app.id)
+ .order_by(Message.created_at.desc())
+ .limit(1)
+ )
+ if app is not None:
+ return self._managed_owner(app, message)
+ workflow_run_id = info.metadata.get("workflow_run_id")
+ if isinstance(workflow_run_id, str):
+ with self.repository.sessions() as session:
+ # A durable managed owner already passed the trusted start boundary.
+ # Suppress auxiliary content even before LogStore indexing catches up.
+ managed_app = session.scalar(
+ select(App)
+ .join(ApplicationTraceRun, ApplicationTraceRun.app_id == App.id)
+ .where(
+ ApplicationTraceRun.source_kind == "workflow",
+ ApplicationTraceRun.source_id == workflow_run_id,
+ ApplicationTraceRun.tenant_id == App.tenant_id,
+ )
+ )
+ if managed_app is not None:
+ return True
+ app_id = info.metadata.get("app_id")
+ app = (
+ session.scalar(select(App).where(App.id == app_id))
+ if isinstance(app_id, str)
+ else session.scalar(
+ select(App)
+ .join(WorkflowRun, WorkflowRun.app_id == App.id)
+ .where(WorkflowRun.id == workflow_run_id, WorkflowRun.tenant_id == App.tenant_id)
+ )
+ )
+ if app is not None:
+ return self._managed_owner(app, workflow_run_id=workflow_run_id)
+ return False
+
+ def _link_managed_parent(self, info: WorkflowTraceInfo, tenant_id: str) -> None:
+ """A legacy child cannot await a parent exported by a different pipeline."""
+ parent_run_id, parent_node_id = info.resolved_parent_context
+ if not parent_run_id:
+ return
+ with self.repository.sessions() as session:
+ parent = session.scalar(
+ select(ApplicationTraceRun)
+ .join(App, App.id == ApplicationTraceRun.app_id)
+ .where(
+ ApplicationTraceRun.source_kind == "workflow",
+ ApplicationTraceRun.source_id == parent_run_id,
+ ApplicationTraceRun.tenant_id == tenant_id,
+ App.tenant_id == tenant_id,
+ )
+ )
+ if parent is None:
+ return
+ if (
+ self.repository._source(tenant_id=tenant_id, app_id=parent.app_id, kind="workflow", source_id=parent_run_id)
+ is None
+ ):
+ return
+ metadata = dict(info.metadata)
+ metadata.pop("parent_trace_context", None)
+ metadata["linked_parent_workflow_run_id"] = parent_run_id
+ if parent_node_id:
+ metadata["linked_parent_node_execution_id"] = parent_node_id
+ metadata["linked_parent_application_trace_id"] = parent.id
+ metadata["linked_parent_trace_id"] = parent.trace_id
+ info.metadata = metadata
+
+ def capture_terminal(self, info: BaseTraceInfo) -> bool:
+ """Hand managed traces to durable collection while preserving legacy scope."""
+ if isinstance(info, WorkflowTraceInfo):
+ app_id = str(info.metadata.get("app_id", ""))
+ source_id, source_kind = info.workflow_run_id, "workflow"
+ elif isinstance(info, MessageTraceInfo) and isinstance(info.message_data, Message):
+ app_id = info.message_data.app_id
+ source_id, source_kind = info.message_data.id, "message"
+ else:
+ return self._capture_auxiliary(info)
+ with self.repository.sessions() as session:
+ app = session.scalar(select(App).where(App.id == app_id))
+ if app is None:
+ return False
+ message = (
+ session.scalar(select(Message).where(Message.id == source_id, Message.app_id == app_id))
+ if source_kind == "message"
+ else None
+ )
+ managed = self._managed_owner(app, message, source_id if source_kind == "workflow" else None)
+ run = self._run_for_source(tenant_id=app.tenant_id, app_id=app.id, source_kind=source_kind, source_id=source_id)
+ if run is not None and source_kind == "message" and run.source_kind == "workflow":
+ # Chatflow emits both events; the message is already in the workflow tree.
+ return True
+ if run is None:
+ if not managed:
+ if isinstance(info, WorkflowTraceInfo):
+ self._link_managed_parent(info, app.tenant_id)
+ return False
+ run = self.repository.record_initialization_failure(
+ tenant_id=app.tenant_id,
+ app_id=app.id,
+ source_kind=source_kind,
+ source_id=source_id,
+ )
+ logger.warning(
+ "Application trace snapshot missing; legacy export suppressed: app_id=%s source_id=%s",
+ app_id,
+ source_id,
+ )
+ if run is None:
+ return True
+ if run.state in {"not_collected", "failed", "uncertain", "accepted", "exporting"}:
+ return True
+ if (
+ app.mode == AppMode.AGENT
+ and message is not None
+ and info.metadata.get("message_execution_state") not in ("succeeded", "failed", "cancelled")
+ ):
+ from services.tracing.recovery import agent_stop_requested
+
+ if agent_stop_requested(message):
+ # An already queued consumer notification may arrive after a
+ # stop. Its ordinary Message snapshot cannot own the runtime
+ # outcome, even if span finalization has already completed.
+ return True
+ loader = RepositoryWorkflowExecutionLoader(lambda requested_app: self._service_account(run, requested_app))
+ trace = CanonicalTraceBuilder(loader).build(info)
+ if trace is None:
+ raise TraceCollectionError("tracing_terminal_not_supported")
+ trace = remap_outer_trace(
+ trace, trace_id=run.trace_id, capture_content=run.capture_content, agent_message=source_kind == "message"
+ )
+ trace = trace.model_copy(update={"session_id": run.session_id})
+ self.repository.terminal(run.id, trace.model_dump(mode="json"), trace.root_span_id)
+ self.schedule(run.id)
+ return True
+
+ def settle_preview(self, run: ApplicationTraceRun) -> None:
+ if run.source_kind != "node" or run.outer_trace is not None:
+ return
+ node = self.repository._source(tenant_id=run.tenant_id, app_id=run.app_id, kind="node", source_id=run.source_id)
+ if not isinstance(node, WorkflowNodeExecutionModel) or str(node.status) in {"running", "pending", "paused"}:
+ return
+ error = node.error
+ started_at = node.created_at.replace(tzinfo=UTC)
+ ended_at = started_at + timedelta(seconds=node.elapsed_time or 0)
+ span = CanonicalSpan(
+ id=run.root_span_id,
+ parent_id=None,
+ name=node.title or "Agent preview",
+ kind=CanonicalSpanKind.AGENT,
+ start_time=started_at,
+ end_time=ended_at,
+ inputs=sanitize(node.inputs) if run.capture_content else None,
+ outputs=sanitize(node.outputs) if run.capture_content else None,
+ status=CanonicalSpanStatus.ERROR if error else CanonicalSpanStatus.OK,
+ error=error if run.capture_content else "Node execution failed" if error else None,
+ metadata={"node_execution_id": node.id, "node_id": node.node_id, "source": "single_step"},
+ )
+ trace = CanonicalTrace(
+ trace_id=run.trace_id, session_id=run.session_id, root_span_id=run.root_span_id, spans=(span,)
+ )
+ self.repository.terminal(run.id, trace.model_dump(mode="json"), run.root_span_id)
+
+ def assemble(self, run: ApplicationTraceRun) -> CanonicalTrace:
+ if run.outer_trace is None:
+ raise TraceCollectionError("tracing_terminal_missing")
+ outer = CanonicalTrace.model_validate(run.outer_trace)
+ invocations, batches = self.repository.collected(run.id)
+ invocation_by_id = {inv.id: inv for inv in invocations}
+ spans = list(outer.spans)
+ internal_by_invocation: dict[str, dict[str, CanonicalSpan]] = {inv.id: {} for inv in invocations}
+ for batch in batches:
+ if batch.invocation_id not in invocation_by_id:
+ raise TraceCollectionError("tracing_invocation_not_found")
+ for data in batch.spans:
+ span = normalize_agent_span(data, capture_content=run.capture_content)
+ current = internal_by_invocation[batch.invocation_id].get(span.id)
+ if current is not None and current != span:
+ raise TraceCollectionError("tracing_span_conflict")
+ internal_by_invocation[batch.invocation_id][span.id] = span
+ for invocation in invocations:
+ children = list(internal_by_invocation[invocation.id].values())
+ if not children:
+ raise TraceCollectionError("tracing_invocation_empty")
+ valid_ids = {child.id for child in children}
+ if any(child.parent_id not in valid_ids and child.parent_id != invocation.span_id for child in children):
+ raise TraceCollectionError("tracing_invocation_parent_invalid")
+ root_children = [child for child in children if child.parent_id == invocation.span_id]
+ if not root_children:
+ raise TraceCollectionError("tracing_invocation_root_missing")
+ errors = [child.error for child in root_children if child.status == CanonicalSpanStatus.ERROR]
+ spans.append(
+ CanonicalSpan(
+ id=invocation.span_id,
+ parent_id=invocation.parent_span_id,
+ name=f"Agent attempt {invocation.attempt + 1}",
+ kind=CanonicalSpanKind.AGENT,
+ start_time=min(child.start_time for child in children),
+ end_time=max(child.end_time or child.start_time for child in children),
+ status=CanonicalSpanStatus.ERROR if errors else CanonicalSpanStatus.OK,
+ error=errors[0] if errors else None,
+ metadata={
+ "agent_id": invocation.agent_id,
+ "attempt": invocation.attempt,
+ "node_execution_id": invocation.node_execution_id,
+ "invocation_id": invocation.id,
+ },
+ )
+ )
+ spans.extend(children)
+ trusted = {
+ "tenant_id": run.tenant_id,
+ "app_id": run.app_id,
+ "session_id": run.session_id,
+ "dify.tenant_id": run.tenant_id,
+ "dify.app_id": run.app_id,
+ "dify.tracing.run_id": run.id,
+ "dify.source.id": run.source_id,
+ }
+ spans = [span.model_copy(update={"metadata": {**span.metadata, **trusted}}) for span in spans]
+ return CanonicalTrace(
+ trace_id=run.trace_id,
+ session_id=run.session_id,
+ root_span_id=outer.root_span_id,
+ spans=order_spans(outer.root_span_id, spans),
+ )
+
+ @contextmanager
+ def _export_lease_heartbeat(self, run_id: str, token: str) -> Generator[None]:
+ """Keep slow exporters leased without holding a transaction across vendor I/O.
+
+ A lost database connection is not proof that the exporter stopped. The
+ attempt continues; the token-checked completion can reconcile a late ACK
+ after lease expiry unless a manual retry has already replaced its token.
+ """
+ stopped = threading.Event()
+ interval = dify_config.OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS / 3
+
+ def heartbeat() -> None:
+ while not stopped.wait(interval):
+ try:
+ if not self.repository.renew_export_lease(run_id, token):
+ return
+ except Exception as error:
+ logger.warning(
+ "Application trace heartbeat failed: run_id=%s error_type=%s", run_id, type(error).__name__
+ )
+
+ worker = threading.Thread(target=heartbeat, name=f"application-trace-lease-{run_id}", daemon=True)
+ started = False
+ try:
+ worker.start()
+ started = True
+ except Exception as error:
+ logger.warning(
+ "Application trace heartbeat unavailable: run_id=%s error_type=%s", run_id, type(error).__name__
+ )
+ try:
+ yield
+ finally:
+ stopped.set()
+ # A wedged DB request must not hold the vendor completion path open.
+ # When it returns, the event stops the loop; renew is also token/state guarded.
+ if started:
+ worker.join(timeout=1.0)
+ if worker.is_alive():
+ logger.warning("Application trace heartbeat shutdown pending: run_id=%s", run_id)
+
+ def deliver(self, run_id: str) -> None:
+ record = self.repository.get(run_id)
+ if record is None:
+ return
+ from services.tracing.recovery import recover_terminal
+
+ recover_terminal(self, record)
+ record = self.repository.get(run_id)
+ if record is None:
+ return
+ self.settle_preview(record)
+ claim = self.repository.claim(
+ run_id,
+ collection_timeout=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT,
+ lease_seconds=dify_config.OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS,
+ )
+ if claim is None:
+ return
+ run, token = claim
+ with self._export_lease_heartbeat(run_id, token):
+ try:
+ try:
+ trace = self.assemble(run)
+ except ValueError as error:
+ # Invalid persisted trees cannot be repaired by retrying a
+ # provider request. ValidationError also inherits ValueError.
+ raise TraceCollectionError("tracing_payload_invalid") from error
+ config = dict(run.provider_config)
+ config.update(OpsTraceManager.decrypt_tracing_config(run.tenant_id, run.provider, run.provider_config))
+ result = export_trace(run.provider, config, trace)
+ except TraceExportUncertainError:
+ self.repository.complete(run_id, token, state="uncertain", error_code="provider_acceptance_unknown")
+ except RetryableTraceDispatchError:
+ if run.attempts >= dify_config.OPS_AGENT_TRACE_MAX_ATTEMPTS:
+ self.repository.complete(run_id, token, state="failed", error_code="provider_retry_exhausted")
+ else:
+ self.repository.complete(
+ run_id,
+ token,
+ state="retrying",
+ error_code="provider_temporarily_unavailable",
+ retry_delay=min(300, 5 * 2 ** min(run.attempts - 1, 6)),
+ )
+ except TraceCollectionError as error:
+ self.repository.complete(run_id, token, state="failed", error_code=error.code)
+ except Exception as error:
+ logger.warning(
+ "Application trace export failed: run_id=%s provider=%s error_type=%s",
+ run_id,
+ run.provider,
+ type(error).__name__,
+ )
+ self.repository.complete(run_id, token, state="failed", error_code="provider_export_rejected")
+ else:
+ self.repository.complete(run_id, token, state="accepted", result=result)
+
+ def status(
+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str, node_execution_id: str | None = None
+ ) -> dict[str, Any]:
+ run, node_execution_id = self._resolve_status_source(
+ tenant_id=tenant_id,
+ app_id=app_id,
+ source_kind=source_kind,
+ source_id=source_id,
+ node_execution_id=node_execution_id,
+ )
+ return self._status_for_run(run, source_kind, source_id, node_execution_id)
+
+ def _resolve_status_source(
+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str, node_execution_id: str | None
+ ) -> tuple[ApplicationTraceRun | None, str | None]:
+ """Resolve node log aliases through their owned workflow, outside SQL transactions."""
+ source = self.repository._source(tenant_id=tenant_id, app_id=app_id, kind=source_kind, source_id=source_id)
+ if source is None:
+ raise TraceCollectionError("tracing_source_not_found", 404)
+ if source_kind == "node" and node_execution_id is not None and node_execution_id != source_id:
+ raise TraceCollectionError("tracing_node_not_found", 404)
+ run = self._run_for_source(tenant_id=tenant_id, app_id=app_id, source_kind=source_kind, source_id=source_id)
+ # A standalone preview (including a frozen disabled decision) owns its
+ # status even if its source later carries a workflow run reference.
+ if run is None and isinstance(source, WorkflowNodeExecutionModel) and source.workflow_run_id:
+ workflow = self.repository._source(
+ tenant_id=tenant_id, app_id=app_id, kind="workflow", source_id=source.workflow_run_id
+ )
+ if (
+ not isinstance(workflow, WorkflowRun)
+ or not source.workflow_id
+ or workflow.workflow_id != source.workflow_id
+ ):
+ raise TraceCollectionError("tracing_source_not_found", 404)
+ run = self.repository.source_run(
+ tenant_id=tenant_id, app_id=app_id, source_kind="workflow", source_id=workflow.id
+ )
+ node_execution_id = source.id
+ return run, node_execution_id
+
+ def _status_for_run(
+ self, run: ApplicationTraceRun | None, source_kind: str, source_id: str, node_execution_id: str | None
+ ) -> dict[str, Any]:
+ if run is None or run.state == "not_collected":
+ return {"status": "not_collected", "source_kind": source_kind, "source_id": source_id}
+ invocations, _ = self.repository.collected(run.id)
+ selected = (
+ [inv for inv in invocations if inv.node_execution_id == node_execution_id] if node_execution_id else []
+ )
+ if node_execution_id and not selected and not (source_kind == "node" and run.source_kind == "workflow"):
+ raise TraceCollectionError("tracing_node_not_found", 404)
+ return {
+ "status": run.state,
+ "source_kind": source_kind,
+ "source_id": source_id,
+ "provider": run.provider or None,
+ "trace_id": run.provider_trace_id or run.trace_id,
+ "trace_url": run.trace_url,
+ "span_url": run.span_urls.get(selected[-1].span_id) if selected else None,
+ "project_url": frozen_project_url(run.provider, run.provider_config),
+ "config_revision": run.config_revision or None,
+ "capture_content": run.capture_content,
+ "capture_agent": run.capture_agent,
+ "span_count": run.span_count,
+ "attempts": run.attempts,
+ "error_code": run.error_code,
+ "accepted_at": run.accepted_at,
+ "queryable": None,
+ "capabilities": provider_capabilities(run.provider) if run.provider else None,
+ }
+
+ def retry(
+ self,
+ *,
+ tenant_id: str,
+ app_id: str,
+ source_kind: str,
+ source_id: str,
+ acknowledge_possible_duplicates: bool,
+ node_execution_id: str | None = None,
+ ) -> dict[str, Any]:
+ run, node_execution_id = self._resolve_status_source(
+ tenant_id=tenant_id,
+ app_id=app_id,
+ source_kind=source_kind,
+ source_id=source_id,
+ node_execution_id=node_execution_id,
+ )
+ if run is None:
+ raise TraceCollectionError("tracing_run_not_found", 404)
+ self._status_for_run(run, source_kind, source_id, node_execution_id)
+ self.repository.retry(run_id=run.id, acknowledge_possible_duplicates=acknowledge_possible_duplicates)
+ self.schedule(run.id)
+ return self._status_for_run(self.repository.get(run.id), source_kind, source_id, node_execution_id)
+
+
+def application_tracing_service() -> ApplicationTracingService:
+ return ApplicationTracingService(ApplicationTraceRepository(sessionmaker(bind=db.engine, expire_on_commit=False)))
diff --git a/api/services/workflow_persistence.py b/api/services/workflow_persistence.py
index 9244679933f..28bb6750b83 100644
--- a/api/services/workflow_persistence.py
+++ b/api/services/workflow_persistence.py
@@ -12,6 +12,7 @@ from typing import Any, override
from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
from core.app.workflow.retry_history import RETRY_HISTORY_PROCESS_DATA_KEY, WorkflowNodeRetryAttempt
from core.helper.trace_id_helper import ParentTraceContext
+from core.ops.base_trace_instance import application_tracing_hooks
from core.ops.entities.trace_entity import TraceTaskName
from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
@@ -200,6 +201,15 @@ class WorkflowPersistenceLayer(Layer):
self._workflow_execution_repository.save_synchronously(workflow_execution)
self._workflow_execution = workflow_execution
+ tracing_hooks = application_tracing_hooks()
+ if tracing_hooks is not None:
+ tracing_hooks.freeze_workflow_run(
+ self._application_generate_entity,
+ workflow_id=self._workflow_info.workflow_id,
+ workflow_run_id=execution_id,
+ conversation_id=self._system_variables().get(SystemVariableKey.CONVERSATION_ID.value),
+ graph_data=self._workflow_info.graph_data,
+ )
def _handle_graph_run_succeeded(self, event: GraphRunSucceededEvent) -> None:
execution = self._get_workflow_execution()
diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py
index 74fef8c57c4..700d58e7273 100644
--- a/api/services/workflow_service.py
+++ b/api/services/workflow_service.py
@@ -38,6 +38,7 @@ from core.workflow.node_factory import (
from core.workflow.node_runtime import (
apply_dify_debug_email_recipient,
)
+from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data
from core.workflow.nodes.human_input.callback import (
DifyHITLCallback,
render_form_content_before_submission,
@@ -46,7 +47,13 @@ from core.workflow.nodes.human_input.callback import (
from core.workflow.nodes.human_input.entities import FormInputConfig, HumanInputNodeData
from core.workflow.nodes.human_input.enums import HumanInputFormKind
from core.workflow.nodes.human_input.pause_reason import HumanInputRequired
-from core.workflow.system_variables import build_bootstrap_variables, build_system_variables, default_system_variables
+from core.workflow.system_variables import (
+ SystemVariableKey,
+ build_bootstrap_variables,
+ build_system_variables,
+ default_system_variables,
+ system_variable_selector,
+)
from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_pool
from core.workflow.workflow_entry import WorkflowEntry
from enterprise.telemetry.draft_trace import enqueue_draft_node_execution_trace
@@ -1158,14 +1165,19 @@ class WorkflowService:
Run draft workflow node
"""
files = files or []
+ node_config = draft_workflow.get_node_config_by_id(node_id)
+ node_type = Workflow.get_node_type_from_node_config(node_config)
+ node_data = node_config["data"]
+ is_agent_preview = is_dify_agent_node_data(node_data)
+ if is_agent_preview and (
+ draft_workflow.tenant_id != app_model.tenant_id or draft_workflow.app_id != app_model.id
+ ):
+ raise ValueError("Workflow does not belong to the application")
with Session(bind=db.engine, expire_on_commit=False) as session, session.begin():
draft_var_srv = WorkflowDraftVariableService(session)
draft_var_srv.prefill_conversation_variable_default_values(draft_workflow, user_id=account.id)
- node_config = draft_workflow.get_node_config_by_id(node_id)
- node_type = Workflow.get_node_type_from_node_config(node_config)
- node_data = node_config["data"]
if is_start_node_type(node_type):
with Session(bind=db.engine) as session, session.begin():
draft_var_srv = WorkflowDraftVariableService(session)
@@ -1234,37 +1246,59 @@ class WorkflowService:
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
)
node, node_events = run
+ if is_agent_preview:
+ # Standalone previews have no WorkflowRun. The general debug bootstrap
+ # inserts a synthetic run ID; do not use it as this caller's owner.
+ variable_pool.remove(system_variable_selector(SystemVariableKey.WORKFLOW_EXECUTION_ID))
+
started_event: NodeRunStartedEvent | None = None
+ started_execution: WorkflowNodeExecution | None = None
def persist_node_start() -> Generator[NodeEvent | ContainerAwaitRequest]:
- nonlocal started_event
+ nonlocal started_event, started_execution
for event in node_events:
if isinstance(event, NodeRunStartedEvent) and event.node_id == node_id:
started_event = event
# Agent workspace allocation requires this caller row before the node body runs.
- repository.save_synchronously(
- WorkflowNodeExecution(
- id=event.id,
- node_execution_id=event.id,
- workflow_id=draft_workflow.id,
- index=1,
- predecessor_node_id=event.predecessor_node_id,
- node_id=event.node_id,
- node_type=event.node_type,
- title=event.node_title,
- process_data=event.node_run_result.process_data or None,
- metadata=event.node_run_result.metadata,
- created_at=event.start_at,
- )
+ started_execution = WorkflowNodeExecution(
+ id=event.id,
+ node_execution_id=event.id,
+ workflow_id=draft_workflow.id,
+ index=1,
+ predecessor_node_id=event.predecessor_node_id,
+ node_id=event.node_id,
+ node_type=event.node_type,
+ title=event.node_title,
+ process_data=event.node_run_result.process_data or None,
+ metadata=event.node_run_result.metadata,
+ created_at=event.start_at,
)
+ repository.save_synchronously(started_execution)
yield event
start_at = time.perf_counter()
- node_execution = self._handle_single_step_result(
- invoke_node_fn=lambda: (node, persist_node_start()),
- start_at=start_at,
- node_id=node_id,
- )
+ try:
+ node_execution = self._handle_single_step_result(
+ invoke_node_fn=lambda: (node, persist_node_start()),
+ start_at=start_at,
+ node_id=node_id,
+ )
+ except BaseException:
+ if is_agent_preview and started_execution is not None:
+ started_execution.status = WorkflowNodeExecutionStatus.FAILED
+ started_execution.error = "Agent preview execution interrupted"
+ started_execution.elapsed_time = time.perf_counter() - start_at
+ started_execution.finished_at = naive_utc_now()
+ try:
+ repository.save_synchronously(started_execution)
+ # Keep the terminal append for LogStore and other async repositories.
+ repository.save(started_execution)
+ except Exception as persistence_error:
+ logging.getLogger(__name__).warning(
+ "Agent preview terminal persistence failed (%s)", type(persistence_error).__name__
+ )
+ raise
+
node_execution.workflow_id = draft_workflow.id
if started_event is not None:
node_execution.id = started_event.id
@@ -1706,7 +1740,6 @@ class WorkflowService:
created_at=naive_utc_now(),
finished_at=naive_utc_now(),
)
-
# Populate execution result data
self._populate_execution_result(node_execution, node_run_result, run_succeeded, error)
diff --git a/api/tasks/application_trace_task.py b/api/tasks/application_trace_task.py
new file mode 100644
index 00000000000..a7d68d69b8d
--- /dev/null
+++ b/api/tasks/application_trace_task.py
@@ -0,0 +1,22 @@
+"""Durable application trace delivery on the existing ops_trace worker queue."""
+
+from celery import shared_task
+
+from configs import dify_config
+
+
+@shared_task(queue="ops_trace")
+def export_application_trace(run_id: str) -> None:
+ from services.tracing.service import application_tracing_service
+
+ application_tracing_service().deliver(run_id)
+
+
+@shared_task(queue="ops_trace")
+def reconcile_application_traces() -> None:
+ from services.tracing.service import application_tracing_service
+
+ service = application_tracing_service()
+ for run_id in service.repository.pending_ids():
+ service.schedule(run_id)
+ service.repository.purge(retention_days=dify_config.OPS_AGENT_TRACE_RETENTION_DAYS)
diff --git a/api/tasks/ops_trace_task.py b/api/tasks/ops_trace_task.py
index acfa59df3f2..1295b3f97b4 100644
--- a/api/tasks/ops_trace_task.py
+++ b/api/tasks/ops_trace_task.py
@@ -1,8 +1,9 @@
"""
Celery task for asynchronous ops trace dispatch.
-Trace providers may report explicitly retryable dispatch failures through the
-core retryable exception contract. The task preserves the payload file only
+Trace providers may report retryable dispatch failures through the core
+exception contract. Unknown acceptance is terminal to prevent automatic replay
+of a possibly accepted prefix. The task preserves the payload file only
when Celery accepts the retry request; successful dispatches and terminal
failures clean up the stored payload.
@@ -23,6 +24,7 @@ from celery.exceptions import Retry
from flask import current_app
from configs import dify_config
+from core.ops.base_trace_instance import application_tracing_hooks
from core.ops.entities.config_entity import OPS_FILE_PATH, OPS_TRACE_FAILED_KEY
from core.ops.entities.trace_entity import trace_info_info_map
from core.ops.exceptions import RetryableTraceDispatchError
@@ -58,7 +60,6 @@ def process_trace_tasks(self, file_info):
trace_info = file_data.get("trace_info")
trace_info_type = file_data.get("trace_info_type")
enterprise_trace_dispatched = bool(file_data.get("_enterprise_trace_dispatched"))
- trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)
if trace_info.get("message_data"):
trace_info["message_data"] = Message.from_dict(data=trace_info["message_data"])
@@ -74,6 +75,13 @@ def process_trace_tasks(self, file_info):
if trace_type:
trace_info = trace_type(**trace_info)
+ application_trace_captured = False
+ if dify_config.OPS_AGENT_TRACE_ENABLED:
+ with current_app.app_context():
+ tracing_hooks = application_tracing_hooks()
+ if tracing_hooks is not None:
+ application_trace_captured = tracing_hooks.capture_terminal(trace_info)
+
from extensions.ext_enterprise_telemetry import is_enabled as is_ee_telemetry_enabled
if is_ee_telemetry_enabled() and not enterprise_trace_dispatched:
@@ -87,21 +95,29 @@ def process_trace_tasks(self, file_info):
file_data["_enterprise_trace_dispatched"] = True
enterprise_trace_dispatched = True
- if trace_instance:
- with current_app.app_context():
- trace_instance.trace(trace_info)
+ if not application_trace_captured:
+ trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)
+ if trace_instance:
+ with current_app.app_context():
+ trace_instance.trace(trace_info)
logger.info("Processing trace tasks success, app_id: %s", app_id)
except RetryableTraceDispatchError as e:
- # Retryable dispatch failures represent a transient provider-side
- # ordering gap, not corrupt payload data. Keep the payload only after
+ # Only confirmed retryable failures, including transient parent ordering
+ # gaps, may replay. Unknown acceptance must terminate without retry.
+ # Keep the payload only after
# Celery accepts the retry request; otherwise this attempt becomes a
# terminal failure and the stored file is cleaned up in `finally`.
#
# Enterprise telemetry runs before provider dispatch. If it already ran
# and provider dispatch asks for a retry, persist that private flag so
# the next attempt does not emit the same enterprise trace twice.
- if self.request.retries >= _RETRYABLE_TRACE_DISPATCH_LIMIT:
+ if e.acceptance_unknown:
+ # Provider exception details can contain private request content.
+ logger.exception("Trace acceptance unknown; automatic retry suppressed, app_id: %s", app_id, exc_info=False)
+ failed_key = f"{OPS_TRACE_FAILED_KEY}_{app_id}"
+ redis_client.incr(failed_key)
+ elif self.request.retries >= _RETRYABLE_TRACE_DISPATCH_LIMIT:
logger.exception("Retryable trace dispatch budget exhausted, app_id: %s", app_id)
failed_key = f"{OPS_TRACE_FAILED_KEY}_{app_id}"
redis_client.incr(failed_key)
diff --git a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py
index e11bdbe0eeb..169f6cf7a2c 100644
--- a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py
+++ b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py
@@ -1,9 +1,11 @@
+import json
from typing import Any, cast
import pytest
+from dify_agent.adapters.llm.model import _get_stop_sequences, _map_model_settings_to_parameters
from dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolsLayerConfig
from dify_agent.layers.dify_plugin import DifyPluginLLMLayerConfig, DifyPluginToolConfig, DifyPluginToolsLayerConfig
-from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
+from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfig
from dify_agent.layers.knowledge import DifyKnowledgeBaseLayerConfig
from dify_agent.layers.prompt import Config as PromptConfig
from dify_agent.layers.shell import DifyShellEnvVarConfig, DifyShellLayerConfig
@@ -14,7 +16,7 @@ from dify_agent.protocol import (
DIFY_AGENT_OUTPUT_LAYER_ID,
CreateRunRequest,
)
-from pydantic import ValidationError
+from pydantic import JsonValue, ValidationError
from clients.agent_backend import (
AGENT_SOUL_PROMPT_LAYER_ID,
@@ -32,6 +34,7 @@ from clients.agent_backend import (
redact_for_agent_backend_log,
)
from clients.agent_backend.request_builder import DIFY_SHELL_LAYER_ID
+from models.agent_config_entities import AgentSoulModelSettings
def _run_input() -> AgentBackendWorkflowNodeRunInput:
@@ -121,6 +124,70 @@ def test_request_builder_forwards_plugin_specific_model_settings_via_extra_body(
}
[email protected]("builder_kind", ["agent_app", "workflow_node"])
+ "settings",
+ [
+ pytest.param({}, id="default"),
+ pytest.param({"response_format": None}, id="none"),
+ pytest.param({"response_format": "text"}, id="plugin-text"),
+ pytest.param(
+ {
+ "response_format": "json_schema",
+ "json_schema": json.dumps(
+ {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}
+ ),
+ },
+ id="plugin-json-schema",
+ ),
+ pytest.param({"response_format": {"type": "text"}}, id="legacy-text"),
+ pytest.param(
+ {
+ "response_format": {
+ "type": "json_schema",
+ "json_schema": {"name": "response", "schema": {"type": "object"}},
+ }
+ },
+ id="legacy-json-schema",
+ ),
+ ],
+)
+def test_response_format_reaches_plugin_parameters_through_real_request_serialization(
+ builder_kind: str, settings: dict[str, JsonValue]
+) -> None:
+ soul_settings = AgentSoulModelSettings.model_validate(
+ {**settings, "temperature": 0.2, "stop": ["END"], "enable_thinking": True}
+ )
+ model = AgentBackendModelConfig(
+ plugin_id="langgenius/openai",
+ model_provider="openai",
+ model="gpt-test",
+ model_settings=soul_settings.model_dump(mode="json", exclude_none=True),
+ )
+ builder = AgentBackendRunRequestBuilder()
+ if builder_kind == "agent_app":
+ request = builder.build_for_agent_app(_agent_app_input().model_copy(update={"model": model}))
+ else:
+ request = builder.build_for_workflow_node(_run_input().model_copy(update={"model": model}))
+
+ request = CreateRunRequest.model_validate_json(request.model_dump_json())
+ model_layer = next(layer for layer in request.composition.layers if layer.name == DIFY_AGENT_MODEL_LAYER_ID)
+ model_config = DifyPluginLLMLayerConfig.model_validate(model_layer.config)
+ plugin_settings = {key: value for key, value in settings.items() if value is not None}
+
+ assert model_config.model_settings == {
+ "temperature": 0.2,
+ "stop_sequences": ["END"],
+ "extra_body": {"enable_thinking": True, **plugin_settings},
+ }
+ assert _map_model_settings_to_parameters(model_config.model_settings) == {
+ "temperature": 0.2,
+ "enable_thinking": True,
+ **plugin_settings,
+ }
+ assert _get_stop_sequences(model_config.model_settings) == ["END"]
+
+
@pytest.mark.parametrize("agent_config_version_kind", ["snapshot", "draft"])
def test_agent_app_request_builder_keeps_agent_soul_prompt_for_snapshot_and_draft(
agent_config_version_kind: str,
@@ -192,6 +259,38 @@ def test_request_builder_sets_model_and_output_layer_contract_ids() -> None:
assert layers[DIFY_AGENT_MODEL_LAYER_ID].config["execution_context"] == DIFY_EXECUTION_CONTEXT_LAYER_ID
[email protected]("mode", ["agent_app", "workflow_node"])
[email protected]("traced", [False, True])
+def test_context_wire_contract_omits_tracing_for_legacy_runtime(mode: str, traced: bool) -> None:
+ run_input = _agent_app_input() if mode == "agent_app" else _run_input()
+ tracing = ApplicationTracingContext(
+ run_id="00000000-0000-0000-0000-000000000001",
+ invocation_id="00000000-0000-0000-0000-000000000002",
+ trace_id="a" * 32,
+ parent_span_id="b" * 16,
+ session_id="session",
+ capture_content=False,
+ )
+ if traced:
+ run_input = run_input.model_copy(
+ update={"execution_context": run_input.execution_context.model_copy(update={"tracing": tracing})}
+ )
+ builder = AgentBackendRunRequestBuilder()
+ request = (
+ builder.build_for_agent_app(run_input)
+ if isinstance(run_input, AgentBackendAgentAppRunInput)
+ else builder.build_for_workflow_node(run_input)
+ )
+ payload = next(
+ layer.config for layer in request.composition.layers if layer.name == DIFY_EXECUTION_CONTEXT_LAYER_ID
+ )
+ if traced:
+ assert payload["tracing"] == tracing.model_dump(mode="json")
+ else:
+ # Pre-tracing runtimes reject even a null unknown field (extra=forbid).
+ assert "tracing" not in payload
+
+
def test_request_builder_adds_dify_plugin_tools_layer_when_configured() -> None:
run_input = _run_input()
run_input.tools = DifyPluginToolsLayerConfig(
diff --git a/api/tests/unit_tests/conftest.py b/api/tests/unit_tests/conftest.py
index 5bf81c3f0d3..d9a765d5746 100644
--- a/api/tests/unit_tests/conftest.py
+++ b/api/tests/unit_tests/conftest.py
@@ -354,12 +354,19 @@ def app_services(sqlite_session_factory: sessionmaker[Session], rbac_domain: RBA
from extensions.ext_application_services import _build_oauth_server_service
from extensions.ext_redis import redis_client
from repositories.app.console_repository import ConsoleAppRepository
+ from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+ from services.app_tracing_config_gateway import OpsTraceManagerGateway
+ from services.app_tracing_config_service import AppTracingConfigService
from services.recommended_app_package_service import RecommendedAppPackageService
return build_app_services(
database_client=sqlite_session_factory,
repository=ConsoleAppRepository(session_factory=sqlite_session_factory),
rbac_members=rbac_domain.rbac.members,
+ tracing_configs=AppTracingConfigService(
+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory),
+ provider=OpsTraceManagerGateway(),
+ ),
oauth=_build_oauth_server_service(database_client=sqlite_session_factory, redis=redis_client),
recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()),
)
diff --git a/api/tests/unit_tests/controllers/console/agent/test_tracing.py b/api/tests/unit_tests/controllers/console/agent/test_tracing.py
new file mode 100644
index 00000000000..ca1c59e3440
--- /dev/null
+++ b/api/tests/unit_tests/controllers/console/agent/test_tracing.py
@@ -0,0 +1,272 @@
+"""Agent tracing Console contracts and authorization stay separate from PlainApp."""
+
+import inspect
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from uuid import UUID
+
+import pytest
+from flask import Flask
+from pydantic import ValidationError
+from werkzeug.exceptions import Forbidden, NotFound
+
+from controllers.common.rbac import AgentId, RBACCheck
+from controllers.console import flask_admission
+from controllers.console.agent import tracing
+from controllers.console.app.app import AppTracePayload
+from extensions.ext_application_services import ApplicationServices
+from libs.login import AccountWithTenant
+from machinery.context import RequestContext
+from models.account import Account, AccountStatus, TenantAccountRole
+from services.app.agent_app_contracts import AgentAppNotFoundError
+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigRecord
+from tests.unit_tests.config_override import apply_config_overrides
+
+AGENT_ID = UUID("1fbc2f02-f7e0-4910-9032-cbe5bf6c65d5")
+CONTEXT = RequestContext("request", None, "account", "tenant")
+
+
+@dataclass
+class ConfigCase:
+ services: ApplicationServices
+ calls: list[tuple[object, ...]] = field(default_factory=list)
+ record: AppTracingConfigRecord | None = None
+ failure: Exception | None = None
+
+
+def service(monkeypatch: pytest.MonkeyPatch, account_application_services: ApplicationServices) -> ConfigCase:
+ case = ConfigCase(account_application_services)
+ target = account_application_services.agent_tracing_configs
+
+ def get_trace(context: RequestContext, agent_id: str) -> AppTraceSettings:
+ case.calls.append(("get_trace", context, agent_id))
+ if case.failure is not None:
+ raise case.failure
+ return AppTraceSettings()
+
+ def set_trace(context: RequestContext, agent_id: str, settings: AppTraceSettings) -> None:
+ case.calls.append(("set_trace", context, agent_id, settings))
+
+ def get(context: RequestContext, agent_id: str, provider: str) -> AppTracingConfigRecord | None:
+ case.calls.append(("get", context, agent_id, provider))
+ return case.record
+
+ def delete(context: RequestContext, agent_id: str, provider: str) -> None:
+ case.calls.append(("delete", context, agent_id, provider))
+
+ monkeypatch.setattr(target, "get_trace", get_trace)
+ monkeypatch.setattr(target, "set_trace", set_trace)
+ monkeypatch.setattr(target, "get", get)
+ monkeypatch.setattr(target, "delete", delete)
+ monkeypatch.setattr(tracing, "application_services", lambda: account_application_services)
+ return case
+
+
+def test_status_round_trips_content_policy(app: Flask, service: ConfigCase) -> None:
+ with app.test_request_context(
+ json={"enabled": True, "tracing_provider": "langfuse", "capture_content": True, "capture_agent": False}
+ ):
+ result = inspect.unwrap(tracing.AgentTraceApi.post)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)
+ assert result == {"enabled": True, "tracing_provider": "langfuse", "capture_content": True, "capture_agent": False}
+ assert service.calls == [("set_trace", CONTEXT, str(AGENT_ID), AppTraceSettings(True, "langfuse", True, False))]
+
+
+def test_enable_requires_provider_even_when_field_omitted() -> None:
+ with pytest.raises(ValidationError, match="tracing_provider is required"):
+ AppTracePayload.model_validate({"enabled": True})
+ payload = AppTracePayload.model_validate({"enabled": False})
+ assert payload.capture_content is False
+ assert payload.capture_agent is True
+
+
[email protected]("enabled", [False, True])
+def test_capabilities_reports_current_api_gate_without_claiming_runtime_delivery(
+ app: Flask, service: ConfigCase, monkeypatch: pytest.MonkeyPatch, enabled: bool
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=enabled)
+ with app.test_request_context("/?tracing_provider=langfuse"):
+ result = inspect.unwrap(tracing.AgentTraceCapabilitiesApi.get)(
+ tracing.AgentTraceCapabilitiesApi(), CONTEXT, AGENT_ID
+ )
+ assert service.calls == [("get_trace", CONTEXT, str(AGENT_ID))]
+ assert result["api_collection_enabled"] is enabled
+ assert result["agent_spans"] is True
+ assert result["export_ack"] == "accepted"
+ assert "runtime_enabled" not in result
+ assert "queryable" not in result
+
+
[email protected]("enabled", [False, True])
+def test_app_capabilities_reports_api_gate(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, enabled: bool, service: ConfigCase
+) -> None:
+ calls: list[tuple[RequestContext, str]] = []
+
+ def get_trace(context: RequestContext, app_id: str) -> AppTraceSettings:
+ calls.append((context, app_id))
+ return AppTraceSettings()
+
+ monkeypatch.setattr(service.services.apps.console, "get_trace", get_trace)
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=enabled)
+ with app.test_request_context("/?tracing_provider=mlflow"):
+ result = inspect.unwrap(tracing.AppTraceCapabilitiesApi.get)(
+ tracing.AppTraceCapabilitiesApi(), CONTEXT, AGENT_ID
+ )
+ assert result["api_collection_enabled"] is enabled
+ assert result["provider"] == "mlflow"
+ assert calls == [(CONTEXT, str(AGENT_ID))]
+
+
+def test_get_config_absent_and_delete_has_no_body(app: Flask, service: ConfigCase) -> None:
+ with app.test_request_context("/?tracing_provider=langfuse"):
+ assert inspect.unwrap(tracing.AgentTraceConfigApi.get)(tracing.AgentTraceConfigApi(), CONTEXT, AGENT_ID) == {
+ "has_not_configured": True
+ }
+ assert inspect.unwrap(tracing.AgentTraceConfigApi.delete)(tracing.AgentTraceConfigApi(), CONTEXT, AGENT_ID) == (
+ "",
+ 204,
+ )
+ assert service.calls == [
+ ("get", CONTEXT, str(AGENT_ID), "langfuse"),
+ ("delete", CONTEXT, str(AGENT_ID), "langfuse"),
+ ]
+
+
+def test_invalid_agent_returns_not_found(app: Flask, service: ConfigCase) -> None:
+ service.failure = AgentAppNotFoundError()
+ with app.test_request_context(), pytest.raises(NotFound):
+ inspect.unwrap(tracing.AgentTraceApi.get)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)
+
+
+ "method",
+ [
+ tracing.AgentTraceApi.post,
+ tracing.AgentTraceConfigApi.get,
+ tracing.AgentTraceConfigApi.post,
+ tracing.AgentTraceConfigApi.patch,
+ tracing.AgentTraceConfigApi.delete,
+ ],
+)
[email protected]("rbac_enabled", [False, True])
+def test_config_requires_editor_role_or_agent_edit_permission(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, method: Callable[..., object], rbac_enabled: bool
+) -> None:
+ account = Account(name="Reader", email="[email protected]", status=AccountStatus.ACTIVE)
+ account.id, account.role = "account", TenantAccountRole.NORMAL
+ apply_config_overrides(monkeypatch, RBAC_ENABLED=rbac_enabled)
+ monkeypatch.setattr(
+ flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account=account, tenant_id="tenant")
+ )
+ denied_calls: list[dict[str, object]] = []
+
+ def denied(**kwargs: object) -> None:
+ denied_calls.append(kwargs)
+ raise Forbidden()
+
+ monkeypatch.setattr(flask_admission, "enforce_rbac_checks", denied)
+ admission = inspect.unwrap(
+ method, stop=lambda candidate: "allowed_roles" in inspect.getclosurevars(candidate).nonlocals
+ )
+ with app.test_request_context(), pytest.raises(Forbidden):
+ admission(None, agent_id=AGENT_ID)
+ if rbac_enabled:
+ checks = denied_calls[0]["checks"]
+ assert isinstance(checks, (list, tuple))
+ (check,) = checks
+ assert isinstance(check, RBACCheck)
+ assert check.scene == tracing.RBACPermission.AGENT_EDIT
+ assert isinstance(check.locator, AgentId)
+ assert denied_calls[0]["path_args"] == {"agent_id": AGENT_ID}
+ else:
+ assert denied_calls == []
+
+
+def test_read_settings_and_masked_config_keep_agent_identity(app: Flask, service: ConfigCase) -> None:
+ from datetime import UTC, datetime
+
+ now = datetime(2026, 1, 1, tzinfo=UTC)
+ service.record = AppTracingConfigRecord(
+ id="config",
+ app_id="backing-app",
+ tracing_provider="langfuse",
+ tracing_config={"secret_key": "[__HIDDEN__]", "project_url": "https://example.test/project"},
+ is_active=True,
+ created_at=now,
+ updated_at=now,
+ )
+ with app.test_request_context("/?tracing_provider=langfuse"):
+ settings = inspect.unwrap(tracing.AgentTraceApi.get)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)
+ config = inspect.unwrap(tracing.AgentTraceConfigApi.get)(tracing.AgentTraceConfigApi(), CONTEXT, AGENT_ID)
+ assert settings == {"enabled": False, "tracing_provider": None, "capture_content": False, "capture_agent": True}
+ assert config["tracing_config"] == service.record.tracing_config
+ assert config["app_id"] == "backing-app"
+ assert service.calls == [("get_trace", CONTEXT, str(AGENT_ID)), ("get", CONTEXT, str(AGENT_ID), "langfuse")]
+
+
[email protected]("create", [True, False])
[email protected]("activate", [False, True])
+def test_config_write_forwards_credentials_but_response_does_not_echo_them(
+ app: Flask, service: ConfigCase, monkeypatch: pytest.MonkeyPatch, create: bool, activate: bool
+) -> None:
+ calls: list[tuple[RequestContext, str, str, dict[str, object], bool]] = []
+
+ def write(
+ context: RequestContext, agent_id: str, provider: str, config: dict[str, object], *, activate: bool
+ ) -> None:
+ calls.append((context, agent_id, provider, config, activate))
+
+ target = service.services.agent_tracing_configs
+ monkeypatch.setattr(target, "create" if create else "update", write)
+ credentials = {"secret_key": "never-echo-this-secret"}
+ with app.test_request_context(
+ method="POST" if create else "PATCH",
+ json={"tracing_provider": "langfuse", "tracing_config": credentials, "activate": activate},
+ ):
+ api = tracing.AgentTraceConfigApi()
+ result = inspect.unwrap(api.post if create else api.patch)(api, CONTEXT, AGENT_ID)
+ assert calls == [(CONTEXT, str(AGENT_ID), "langfuse", credentials, activate)]
+ assert result == (({"result": "success"}, 201) if create else {"result": "success"})
+ assert "never-echo-this-secret" not in str(result)
+
+
+ ("error", "expected"),
+ [
+ (tracing.ConsoleAppNotFoundError(), NotFound),
+ (tracing.AppTracingConfigAppNotFoundError(), tracing.AppNotFoundError),
+ (tracing.AppTracingConfigAlreadyExistsError(), tracing.TracingConfigAlreadyExistsError),
+ (tracing.AppTracingConfigNotFoundError(), tracing.TracingConfigNotFoundError),
+ (tracing.AppTracingConfigConflictError(), tracing.TracingConfigConflictError),
+ (
+ tracing.AppTracingConfigInvalidProviderError("private-provider-secret"),
+ tracing.UnsupportedTracingProviderError,
+ ),
+ (tracing.AppTracingConfigProviderUnavailableError(), tracing.TracingProviderUnavailableError),
+ (tracing.AppTracingConfigInvalidConfigurationError(), tracing.InvalidTracingConfigError),
+ (tracing.AppTracingConfigVerificationFailedError(), tracing.TracingConfigVerificationFailedError),
+ (tracing.AppTracingConfigProcessingError(), tracing.TracingConfigProcessingError),
+ ],
+)
+def test_config_domain_errors_use_public_contract_without_provider_details(
+ app: Flask, service: ConfigCase, error: Exception, expected: type[Exception]
+) -> None:
+ service.failure = error
+ with app.test_request_context(), pytest.raises(expected) as caught:
+ inspect.unwrap(tracing.AgentTraceApi.get)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)
+ assert "private-provider-secret" not in str(caught.value)
+
+
+def test_unknown_capability_and_missing_app_return_public_errors(
+ app: Flask, service: ConfigCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ with app.test_request_context("/?tracing_provider=unknown"), pytest.raises(tracing.UnsupportedTracingProviderError):
+ inspect.unwrap(tracing.AgentTraceCapabilitiesApi.get)(tracing.AgentTraceCapabilitiesApi(), CONTEXT, AGENT_ID)
+
+ def missing(_context: RequestContext, _app_id: str) -> AppTraceSettings:
+ raise tracing.ConsoleAppNotFoundError()
+
+ monkeypatch.setattr(service.services.apps.console, "get_trace", missing)
+ with app.test_request_context("/?tracing_provider=langfuse"), pytest.raises(NotFound):
+ inspect.unwrap(tracing.AppTraceCapabilitiesApi.get)(tracing.AppTraceCapabilitiesApi(), CONTEXT, AGENT_ID)
diff --git a/api/tests/unit_tests/controllers/console/agent/test_tracing_runs.py b/api/tests/unit_tests/controllers/console/agent/test_tracing_runs.py
new file mode 100644
index 00000000000..7fb163b5bf0
--- /dev/null
+++ b/api/tests/unit_tests/controllers/console/agent/test_tracing_runs.py
@@ -0,0 +1,294 @@
+"""Run tracing endpoints preserve resource ownership, acknowledgement and error contracts."""
+
+import inspect
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta, timezone
+from functools import wraps
+from uuid import UUID
+
+import pytest
+from flask import Flask
+from flask_restx import Resource
+from pydantic import ValidationError
+from sqlalchemy.orm import Session, sessionmaker
+from werkzeug.exceptions import Forbidden, NotFound
+
+from controllers.common.rbac import AgentId, PlainApp, RBACCheck
+from controllers.console import flask_admission
+from controllers.console.agent import tracing_runs
+from extensions.ext_application_services import ApplicationServices
+from libs.external_api import ExternalApi
+from libs.login import AccountWithTenant
+from machinery.context import RequestContext
+from models.account import Account, AccountStatus, TenantAccountRole
+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError
+from services.app.agent_app_contracts import AgentAppNotFoundError
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+
+CONTEXT = RequestContext("request", None, "account", "tenant")
+AGENT_ID = UUID("01a11111-1111-7111-8111-111111111111")
+APP_ID = UUID("01a22222-2222-7222-8222-222222222222")
+SOURCE_ID = UUID("01a33333-3333-7333-8333-333333333333")
+NODE_ID = UUID("01a44444-4444-7444-8444-444444444444")
+
+
+@dataclass
+class RunCase:
+ owners: ApplicationServices
+ runs: ApplicationTracingService
+ owner_calls: list[tuple[RequestContext, str]] = field(default_factory=list)
+ status_calls: list[dict[str, object]] = field(default_factory=list)
+ retry_calls: list[dict[str, object]] = field(default_factory=list)
+ response: dict[str, object] = field(
+ default_factory=lambda: {"status": "not_collected", "source_kind": "message", "source_id": str(SOURCE_ID)}
+ )
+ owner_error: Exception | None = None
+ retry_error: Exception | None = None
+
+
+def services(
+ monkeypatch: pytest.MonkeyPatch,
+ account_application_services: ApplicationServices,
+ sqlite_session_factory: sessionmaker[Session],
+) -> RunCase:
+ runs = ApplicationTracingService(ApplicationTraceRepository(sqlite_session_factory))
+ case = RunCase(account_application_services, runs)
+
+ def resolve(context: RequestContext, agent_id: str) -> str:
+ case.owner_calls.append((context, agent_id))
+ if case.owner_error is not None:
+ raise case.owner_error
+ return str(APP_ID)
+
+ def status(**kwargs: object) -> dict[str, object]:
+ case.status_calls.append(kwargs)
+ return case.response
+
+ def retry(**kwargs: object) -> dict[str, object]:
+ case.retry_calls.append(kwargs)
+ if case.retry_error is not None:
+ raise case.retry_error
+ return {**case.response, "status": "retrying"}
+
+ monkeypatch.setattr(account_application_services.agent_tracing_configs, "resolve_app_id", resolve)
+ monkeypatch.setattr(runs, "status", status)
+ monkeypatch.setattr(runs, "retry", retry)
+ monkeypatch.setattr(tracing_runs, "application_services", lambda: account_application_services)
+ monkeypatch.setattr(tracing_runs, "application_tracing_service", lambda: runs)
+ return case
+
+
+ "accepted_at",
+ [
+ datetime(2026, 1, 1),
+ datetime(2026, 1, 1, tzinfo=UTC),
+ datetime(2026, 1, 1, 8, tzinfo=timezone(timedelta(hours=8))),
+ ],
+)
+def test_agent_status_resolves_owner_and_never_exposes_destination_secrets(
+ app: Flask, services: RunCase, accepted_at: datetime
+) -> None:
+ services.response.update(
+ {
+ "status": "accepted",
+ "provider": "langfuse",
+ "trace_id": "trace-id",
+ "trace_url": "https://example.com/trace",
+ "project_url": "https://example.com/project/frozen",
+ "accepted_at": accepted_at,
+ "queryable": None,
+ "provider_config": {"secret_key": "do-not-return"},
+ "config_revision": "frozen-revision",
+ }
+ )
+ with app.test_request_context(f"/?node_execution_id={NODE_ID}"):
+ result = inspect.unwrap(tracing_runs.AgentTraceRunApi.get)(
+ tracing_runs.AgentTraceRunApi(), CONTEXT, AGENT_ID, SOURCE_ID
+ )
+ assert services.owner_calls == [(CONTEXT, str(AGENT_ID))]
+ assert services.status_calls == [
+ {
+ "tenant_id": "tenant",
+ "app_id": str(APP_ID),
+ "source_kind": "message",
+ "source_id": str(SOURCE_ID),
+ "node_execution_id": str(NODE_ID),
+ }
+ ]
+ assert result["accepted_at"] == "2026-01-01T00:00:00Z"
+ assert result["queryable"] is None
+ assert result["config_revision"] == "frozen-revision"
+ assert result["project_url"] == "https://example.com/project/frozen"
+ assert "provider_config" not in result
+ assert "do-not-return" not in str(result)
+
+
[email protected]("kind", ["message", "workflow", "node"])
+def test_app_status_source_scope_and_not_collected(app: Flask, services: RunCase, kind: str) -> None:
+ services.response["source_kind"] = kind
+ with app.test_request_context():
+ result = inspect.unwrap(tracing_runs.AppTraceRunApi.get)(
+ tracing_runs.AppTraceRunApi(), CONTEXT, APP_ID, kind, SOURCE_ID
+ )
+ assert services.owner_calls == []
+ assert result["status"] == "not_collected"
+ assert result["project_url"] is None
+ assert services.status_calls == [
+ {
+ "tenant_id": "tenant",
+ "app_id": str(APP_ID),
+ "source_kind": kind,
+ "source_id": str(SOURCE_ID),
+ "node_execution_id": None,
+ }
+ ]
+
+
[email protected]("ack", [False, True])
[email protected]("agent", [False, True])
+def test_retry_forwards_only_frozen_run_identity_and_explicit_ack(
+ app: Flask, services: RunCase, ack: bool, agent: bool
+) -> None:
+ with app.test_request_context(
+ f"/?node_execution_id={NODE_ID}", method="POST", json={"acknowledge_possible_duplicates": ack} if ack else {}
+ ):
+ if agent:
+ result = inspect.unwrap(tracing_runs.AgentTraceRunRetryApi.post)(
+ tracing_runs.AgentTraceRunRetryApi(), CONTEXT, AGENT_ID, SOURCE_ID
+ )
+ else:
+ result = inspect.unwrap(tracing_runs.AppTraceRunRetryApi.post)(
+ tracing_runs.AppTraceRunRetryApi(), CONTEXT, APP_ID, "message", SOURCE_ID
+ )
+ assert result["status"] == "retrying"
+ assert services.retry_calls == [
+ {
+ "tenant_id": "tenant",
+ "app_id": str(APP_ID),
+ "source_kind": "message",
+ "source_id": str(SOURCE_ID),
+ "node_execution_id": str(NODE_ID),
+ "acknowledge_possible_duplicates": ack,
+ }
+ ]
+
+
[email protected]("payload", [{"provider": "different-provider"}, {"acknowledge_possible_duplicates": "true"}])
+def test_retry_rejects_destination_changes_and_non_boolean_ack(
+ app: Flask, services: RunCase, payload: dict[str, str]
+) -> None:
+ with app.test_request_context(method="POST", json=payload), pytest.raises(ValidationError):
+ inspect.unwrap(tracing_runs.AppTraceRunRetryApi.post)(
+ tracing_runs.AppTraceRunRetryApi(), CONTEXT, APP_ID, "message", SOURCE_ID
+ )
+ assert services.retry_calls == []
+
+
+def test_source_kind_is_validated_before_service_lookup(app: Flask, services: RunCase) -> None:
+ with app.test_request_context(), pytest.raises(ValidationError):
+ inspect.unwrap(tracing_runs.AppTraceRunApi.get)(
+ tracing_runs.AppTraceRunApi(), CONTEXT, APP_ID, "unknown", SOURCE_ID
+ )
+ assert services.status_calls == []
+
+
+def test_agent_owner_rejection_prevents_run_lookup(app: Flask, services: RunCase) -> None:
+ services.owner_error = AgentAppNotFoundError()
+ with app.test_request_context(), pytest.raises(NotFound):
+ inspect.unwrap(tracing_runs.AgentTraceRunApi.get)(tracing_runs.AgentTraceRunApi(), CONTEXT, AGENT_ID, SOURCE_ID)
+ assert services.status_calls == []
+
+
+ ("code", "status"),
+ [
+ ("tracing_source_not_found", 404),
+ ("tracing_node_not_found", 404),
+ ("tracing_run_not_found", 404),
+ ("tracing_already_accepted", 409),
+ ("tracing_retry_not_allowed", 409),
+ ("tracing_duplicate_acknowledgement_required", 409),
+ ],
+)
+def test_retry_business_errors_match_served_error_contract(services: RunCase, code: str, status: int) -> None:
+ services.retry_error = TraceCollectionError(code, status)
+ target = tracing_runs.AppTraceRunRetryApi.post
+
+ @wraps(target)
+ def post(self: Resource) -> dict[str, object]:
+ return inspect.unwrap(target)(self, CONTEXT, APP_ID, "message", SOURCE_ID)
+
+ resource = type("RetryContractResource", (Resource,), {"post": post})
+ app = Flask("trace-error-contract")
+ api = ExternalApi(app)
+ api.add_resource(resource, "/retry")
+ with app.test_client() as client:
+ response = client.post("/retry", json={})
+ assert response.status_code == status
+ body = response.json
+ assert body is not None
+ assert body["code"] == code
+ assert body["status"] == status
+
+
+ ("method", "permission", "locator"),
+ [
+ (tracing_runs.AgentTraceRunApi.get, tracing_runs.RBACPermission.AGENT_LOG_MANAGE, AgentId),
+ (tracing_runs.AppTraceRunApi.get, tracing_runs.RBACPermission.APP_LOG_AND_ANNOTATION, PlainApp),
+ (tracing_runs.AgentTraceRunRetryApi.post, tracing_runs.RBACPermission.AGENT_EDIT, AgentId),
+ (tracing_runs.AppTraceRunRetryApi.post, tracing_runs.RBACPermission.APP_TRACING_CONFIG, PlainApp),
+ ],
+)
+def test_run_endpoints_enforce_resource_permissions(
+ app: Flask,
+ monkeypatch: pytest.MonkeyPatch,
+ method: Callable[..., object],
+ permission: tracing_runs.RBACPermission,
+ locator: type[AgentId] | type[PlainApp],
+) -> None:
+ account = Account(name="Reader", email="[email protected]", status=AccountStatus.ACTIVE)
+ account.id, account.role = "account", TenantAccountRole.NORMAL
+ apply_config_overrides(monkeypatch, RBAC_ENABLED=True)
+ monkeypatch.setattr(
+ flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account=account, tenant_id="tenant")
+ )
+ denied_calls: list[dict[str, object]] = []
+
+ def denied(**kwargs: object) -> None:
+ denied_calls.append(kwargs)
+ raise Forbidden()
+
+ monkeypatch.setattr(flask_admission, "enforce_rbac_checks", denied)
+ admission = inspect.unwrap(
+ method, stop=lambda candidate: "allowed_roles" in inspect.getclosurevars(candidate).nonlocals
+ )
+ with app.test_request_context(), pytest.raises(Forbidden):
+ admission(None, agent_id=AGENT_ID) if locator is AgentId else admission(None, app_id=APP_ID)
+ checks = denied_calls[0]["checks"]
+ assert isinstance(checks, (list, tuple))
+ (check,) = checks
+ assert isinstance(check, RBACCheck)
+ assert check.scene == permission
+ assert isinstance(check.locator, locator)
+
+
[email protected]("method", [tracing_runs.AgentTraceRunRetryApi.post, tracing_runs.AppTraceRunRetryApi.post])
+def test_retry_rejects_readonly_members_without_rbac(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, method: Callable[..., object]
+) -> None:
+ account = Account(name="Reader", email="[email protected]", status=AccountStatus.ACTIVE)
+ account.id, account.role = "account", TenantAccountRole.NORMAL
+ apply_config_overrides(monkeypatch, RBAC_ENABLED=False)
+ monkeypatch.setattr(
+ flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account=account, tenant_id="tenant")
+ )
+ admission = inspect.unwrap(
+ method, stop=lambda candidate: "allowed_roles" in inspect.getclosurevars(candidate).nonlocals
+ )
+ with app.test_request_context(), pytest.raises(Forbidden):
+ admission(None)
diff --git a/api/tests/unit_tests/controllers/console/app/test_console_app_api.py b/api/tests/unit_tests/controllers/console/app/test_console_app_api.py
index 06f44f45955..8dd0edd30b4 100644
--- a/api/tests/unit_tests/controllers/console/app/test_console_app_api.py
+++ b/api/tests/unit_tests/controllers/console/app/test_console_app_api.py
@@ -27,12 +27,12 @@ from services.app.console_service import (
CreatorsPlatformDisabledError,
InvalidAppExportError,
)
+from services.app_tracing_config_service import AppTraceSettings
from services.entities.app_entities import (
AppExportOptions,
AppListParams,
AppPage,
AppRecord,
- AppTraceSettings,
CopyAppParams,
CreateAppParams,
StarredAppListParams,
diff --git a/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py b/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py
index c73e9a3b98b..dc28f6eee73 100644
--- a/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py
+++ b/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py
@@ -19,6 +19,7 @@ from controllers.console.app.error import (
AppNotFoundError,
InvalidTracingConfigError,
TracingConfigAlreadyExistsError,
+ TracingConfigConflictError,
TracingConfigNotFoundError,
TracingConfigProcessingError,
TracingConfigVerificationFailedError,
@@ -33,6 +34,7 @@ from services.app_tracing_config_gateway import OpsTraceManagerGateway
from services.app_tracing_config_service import (
AppTracingConfigAlreadyExistsError,
AppTracingConfigAppNotFoundError,
+ AppTracingConfigConflictError,
AppTracingConfigInvalidConfigurationError,
AppTracingConfigInvalidProviderError,
AppTracingConfigNotFoundError,
@@ -140,10 +142,21 @@ def test_trace_config_without_sdk_can_be_read_when_absent_or_deleted(
configs.delete.assert_called_once_with(workspace_id=WORKSPACE_ID, app_id=APP_ID, tracing_provider="weave")
-def test_enable_tracing_maps_missing_dependency_to_unavailable(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- unavailable = AppTracingConfigProviderUnavailableError()
+ ("failure", "expected"),
+ [
+ (AppTracingConfigProviderUnavailableError(), TracingProviderUnavailableError),
+ (AppTracingConfigNotFoundError(), TracingConfigNotFoundError),
+ (AppTracingConfigConflictError(), TracingConfigConflictError),
+ (AppTracingConfigInvalidConfigurationError(), InvalidTracingConfigError),
+ (AppTracingConfigProcessingError(), TracingConfigProcessingError),
+ ],
+)
+def test_enable_tracing_maps_configuration_failures(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, failure: Exception, expected: type[BaseHTTPException]
+) -> None:
console_apps = MagicMock()
- console_apps.set_trace.side_effect = unavailable
+ console_apps.set_trace.side_effect = failure
monkeypatch.setattr(
app_module,
"application_services",
@@ -155,7 +168,7 @@ def test_enable_tracing_maps_missing_dependency_to_unavailable(app: Flask, monke
method="POST",
json={"enabled": True, "tracing_provider": "weave"},
),
- pytest.raises(TracingProviderUnavailableError) as caught,
+ pytest.raises(expected) as caught,
):
_original(app_module.AppTraceApi.post)(
app_module.AppTraceApi(),
@@ -163,9 +176,9 @@ def test_enable_tracing_maps_missing_dependency_to_unavailable(app: Flask, monke
UUID(APP_ID),
)
- assert caught.value.code == 400
- assert caught.value.error_code == "tracing_provider_unavailable"
- assert caught.value.__cause__ is unavailable
+ assert caught.value.code == expected.code
+ assert caught.value.error_code == expected.error_code
+ assert caught.value.__cause__ is failure
@pytest.mark.parametrize("method", _MUTATION_METHODS)
@@ -321,17 +334,20 @@ def test_trace_app_config_get_configured_returns_exact_legacy_body(
("mlflow", {"tracking_uri": "", "experiment_id": "0", "username": None, "password": "******"}),
],
)
[email protected]("activate", [False, True])
def test_trace_app_config_write_returns_expected_response(
app: Flask,
tracing_configs: MagicMock,
method_name: str,
expected_result: object,
+ activate: bool,
provider: str,
config: dict[str, str | None],
) -> None:
payload = ops_trace_module.TraceConfigPayload(
tracing_provider=provider,
tracing_config=config,
+ activate=activate,
)
with app.test_request_context("/", method=method_name.upper()):
@@ -348,6 +364,7 @@ def test_trace_app_config_write_returns_expected_response(
app_id=APP_ID,
tracing_provider=provider,
tracing_config=config,
+ activate=activate,
)
@@ -401,6 +418,14 @@ def test_trace_app_config_maps_missing_app_to_404(
@pytest.mark.parametrize(
("method_name", "service_error", "expected_http_error", "expected_status", "expected_code"),
[
+ pytest.param(
+ "patch",
+ AppTracingConfigConflictError(),
+ TracingConfigConflictError,
+ 409,
+ "tracing_config_conflict",
+ id="patch-concurrent-change",
+ ),
pytest.param(
"get",
AppTracingConfigProviderUnavailableError(),
diff --git a/api/tests/unit_tests/controllers/console/test_arize_tracing_ack_routes.py b/api/tests/unit_tests/controllers/console/test_arize_tracing_ack_routes.py
new file mode 100644
index 00000000000..cf158494655
--- /dev/null
+++ b/api/tests/unit_tests/controllers/console/test_arize_tracing_ack_routes.py
@@ -0,0 +1,193 @@
+"""Both real config Resources reach the same ACK gate before atomic SQL updates.
+
+HTTP dispatch, payload validation, owner resolution, DI services, encryption and
+SQLite persistence are real. Login admission supplies a local owner account and
+the existing RSA fixture supplies private-key storage; no external account is used.
+"""
+
+import json
+from copy import deepcopy
+from uuid import uuid4
+
+import pytest
+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace
+from flask import Flask
+from flask_restx import Api
+from sqlalchemy import select
+from sqlalchemy.orm import Session, sessionmaker
+
+from controllers.console import flask_admission, wraps
+from controllers.console.agent.tracing import AgentTraceConfigApi
+from controllers.console.app.ops_trace import TraceAppConfigApi
+from core.helper import ssrf_proxy
+from core.ops.ops_trace_manager import OpsTraceManager
+from enums import DeploymentEdition
+from extensions.ext_application_services import ApplicationServices
+from libs.login import AccountWithTenant
+from models.account import Account, AccountStatus, TenantAccountRole
+from models.agent import Agent, AgentScope, AgentSource, AgentStatus
+from models.model import App, AppMode, TraceAppConfig
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_arize_configuration_ack import (
+ collector as collector, # noqa: PLC0414 - Re-export the real local gRPC collector fixture.
+)
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
[email protected]("surface", ["app", "agent"])
[email protected]("method", ["POST", "PATCH"])
+def test_config_write_rejects_bad_ack_without_changing_old_rows_policy_or_cache(
+ workspace: str,
+ sqlite_session_factory: sessionmaker[Session],
+ account_application_services: ApplicationServices,
+ monkeypatch: pytest.MonkeyPatch,
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ surface: str,
+ method: str,
+) -> None:
+ endpoint, calls, _ = collector
+ app_id, agent_id, account_id = (str(uuid4()) for _ in range(3))
+ old_policy = json.dumps(
+ {"enabled": True, "tracing_provider": "langsmith", "capture_content": False, "capture_agent": False}
+ )
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ mode=AppMode.AGENT if surface == "agent" else AppMode.WORKFLOW,
+ tracing=old_policy,
+ )
+ )
+ if surface == "agent":
+ session.add(
+ Agent(
+ id=agent_id,
+ tenant_id=workspace,
+ name="ACK test",
+ scope=AgentScope.ROSTER,
+ source=AgentSource.AGENT_APP,
+ app_id=app_id,
+ status=AgentStatus.ACTIVE,
+ created_by=account_id,
+ updated_by=account_id,
+ )
+ )
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ original = OpsTraceManager.encrypt_tracing_config(
+ workspace, "langsmith", {"api_key": "other-provider-key", "project": "other"}
+ )
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=original)
+ if method == "PATCH":
+ old = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "arize",
+ {
+ "endpoint": endpoint,
+ "project": "old-project",
+ "api_key": "synthetic-good",
+ "space_id": "synthetic-space",
+ },
+ )
+ old.update(
+ {
+ "project_url": "https://app.arize.com/?redirect_project_name=old-project",
+ "arize_project_destination": {"legacy": "unchanged"},
+ }
+ )
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="arize", tracing_config=old)
+ with sqlite_session_factory() as session:
+ before = deepcopy(
+ list(
+ session.execute(
+ select(TraceAppConfig.__table__).where(TraceAppConfig.app_id == app_id).order_by(TraceAppConfig.id)
+ ).mappings()
+ )
+ )
+
+ account = Account(name="Owner", email="[email protected]", status=AccountStatus.ACTIVE)
+ account.id, account.role = account_id, TenantAccountRole.OWNER
+ identity = AccountWithTenant(account, workspace)
+ monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: identity)
+ monkeypatch.setattr(wraps, "current_account_with_tenant", lambda: identity)
+ apply_config_overrides(
+ monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, LOGIN_DISABLED=True, RBAC_ENABLED=False
+ )
+
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("Local custom collector verification must not perform optional Cloud HTTP reads")
+
+ monkeypatch.setattr(ssrf_proxy, "get", forbidden)
+ monkeypatch.setattr(ArizePhoenixDataTrace, "__init__", forbidden)
+ app = Flask(__name__)
+ app.extensions["application_services"] = account_application_services
+ api = Api(app)
+ api.add_resource(TraceAppConfigApi, "/console/api/apps/<uuid:app_id>/trace-config")
+ api.add_resource(AgentTraceConfigApi, "/console/api/agent/<uuid:agent_id>/trace-config")
+ url = (
+ f"/console/api/apps/{app_id}/trace-config"
+ if surface == "app"
+ else f"/console/api/agent/{agent_id}/trace-config"
+ )
+ values = {
+ "endpoint": endpoint,
+ "project": "not-created-by-verification",
+ "api_key": "synthetic-invalid",
+ "space_id": "synthetic-space",
+ }
+ with app.test_client() as client:
+ failed = client.open(
+ url, method=method, json={"tracing_provider": "arize", "tracing_config": values, "activate": True}
+ )
+ assert failed.status_code == 400
+ body = failed.json
+ assert isinstance(body, dict)
+ assert body["code"] == "tracing_config_verification_failed"
+ assert "private-vendor-details" not in failed.get_data(as_text=True)
+ assert "synthetic-invalid" not in failed.get_data(as_text=True)
+ with sqlite_session_factory() as session:
+ after = list(
+ session.execute(
+ select(TraceAppConfig.__table__).where(TraceAppConfig.app_id == app_id).order_by(TraceAppConfig.id)
+ ).mappings()
+ )
+ assert after == before
+ assert session.scalar(select(App.tracing).where(App.id == app_id)) == old_policy
+ values["api_key"] = "synthetic-good"
+ accepted = client.open(
+ url, method=method, json={"tracing_provider": "arize", "tracing_config": values, "activate": True}
+ )
+ assert accepted.status_code == (201 if method == "POST" else 200)
+ saved = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="arize")
+ assert saved is not None
+ assert saved.tracing_config is not None
+ assert (
+ OpsTraceManager.decrypt_tracing_config(workspace, "arize", saved.tracing_config)["api_key"] == "synthetic-good"
+ )
+ assert "project_url" not in saved.tracing_config
+ assert "arize_project_destination" not in saved.tracing_config
+ assert len(calls) == 2
+ with sqlite_session_factory() as session:
+ policy = json.loads(session.scalar(select(App.tracing).where(App.id == app_id)) or "{}")
+ assert policy == {"enabled": True, "tracing_provider": "arize", "capture_content": False, "capture_agent": False}
+ if method == "PATCH":
+ with app.test_client() as client:
+ masked = client.patch(
+ url,
+ json={
+ "tracing_provider": "arize",
+ "tracing_config": {**values, "project": "next-project", "api_key": "******", "space_id": "******"},
+ },
+ )
+ assert masked.status_code == 200
+ updated = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="arize")
+ assert updated is not None
+ assert updated.tracing_config is not None
+ assert updated.tracing_config["api_key"] == saved.tracing_config["api_key"]
+ assert updated.tracing_config["space_id"] == saved.tracing_config["space_id"]
+ assert calls[-1][1]["api_key"] == "synthetic-good"
+ assert calls[-1][1]["space_id"] == "synthetic-space"
+ assert len(calls) == 3
diff --git a/api/tests/unit_tests/controllers/console/test_feature.py b/api/tests/unit_tests/controllers/console/test_feature.py
index bf5ec1fe354..df58c5efec7 100644
--- a/api/tests/unit_tests/controllers/console/test_feature.py
+++ b/api/tests/unit_tests/controllers/console/test_feature.py
@@ -147,6 +147,7 @@ class TestSystemFeatureApi:
from controllers.console.feature import SystemFeatureApi
system_features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False,
deployment_edition=DeploymentEdition.COMMUNITY,
is_allow_register=True,
enable_learn_app=True,
diff --git a/api/tests/unit_tests/controllers/console/test_feature_agent_tracing.py b/api/tests/unit_tests/controllers/console/test_feature_agent_tracing.py
new file mode 100644
index 00000000000..3a65d7b8511
--- /dev/null
+++ b/api/tests/unit_tests/controllers/console/test_feature_agent_tracing.py
@@ -0,0 +1,120 @@
+"""Public deployment discovery is independent of workspace tracing configuration."""
+
+from typing import Never
+
+import pytest
+from flask import Flask
+from flask_restx import Api
+from pydantic import ValidationError
+
+from controllers.console import feature, wraps
+from controllers.web.feature import SystemFeatureApi as WebSystemFeatureApi
+from enums import DeploymentEdition
+from extensions.ext_application_services import ApplicationServices
+from extensions.ext_login import DifyLoginManager
+from libs import login
+from models.account import AccountStatus
+from services.entities.feature_entities import FeatureModel, SystemFeatureModel
+from services.feature_service import FeatureService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_account, make_tenant
+
+
[email protected]("enabled", [False, True])
[email protected]("surface", ["console", "web"])
+def test_public_system_features_exposes_gate_without_account_app_or_provider(
+ monkeypatch: pytest.MonkeyPatch,
+ account_application_services: ApplicationServices,
+ enabled: bool,
+ surface: str,
+) -> None:
+ apply_config_overrides(
+ monkeypatch,
+ DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY,
+ OPS_AGENT_TRACE_ENABLED=enabled,
+ LOGIN_DISABLED=False,
+ )
+
+ def forbidden_account() -> Never:
+ pytest.fail("public deployment discovery must not resolve a Console account")
+
+ def forbidden_workspace(_workspace_id: str, **_kwargs: object) -> Never:
+ pytest.fail("public deployment discovery must not query workspace features")
+
+ monkeypatch.setattr(login, "_resolve_current_user", forbidden_account)
+ monkeypatch.setattr(FeatureService, "get_features", forbidden_workspace)
+ app = Flask(__name__)
+ app.extensions["application_services"] = account_application_services
+ resource = feature.SystemFeatureApi if surface == "console" else WebSystemFeatureApi
+ path = "/console/api/system-features" if surface == "console" else "/api/system-features"
+ Api(app).add_resource(resource, path)
+
+ response = app.test_client().get(path)
+
+ assert response.status_code == 200
+ payload = response.get_json()
+ assert payload["agent_tracing_api_collection_enabled"] is enabled
+ assert "tracing_provider" not in payload
+ assert "runtime_enabled" not in payload
+ assert "queryable" not in payload
+ schema = SystemFeatureModel.model_json_schema(mode="serialization")
+ gate = schema["properties"]["agent_tracing_api_collection_enabled"]
+ assert gate["type"] == "boolean"
+ assert "default" not in gate
+ assert "agent_tracing_api_collection_enabled" in schema["required"]
+
+
[email protected]("enabled", [False, True])
+def test_authenticated_workspace_features_omits_deployment_gate(
+ monkeypatch: pytest.MonkeyPatch,
+ account_application_services: ApplicationServices,
+ enabled: bool,
+) -> None:
+ apply_config_overrides(
+ monkeypatch,
+ DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY,
+ OPS_AGENT_TRACE_ENABLED=enabled,
+ LOGIN_DISABLED=False,
+ )
+ account = make_account(status=AccountStatus.ACTIVE)
+ account._current_tenant = make_tenant()
+ monkeypatch.setattr(login, "_resolve_current_user", lambda: account)
+ monkeypatch.setattr(login, "check_csrf_token", lambda _request, _account_id: None)
+ monkeypatch.setattr(wraps, "_is_setup_completed", lambda: True)
+ monkeypatch.setattr(feature, "application_services", lambda: account_application_services)
+ app = Flask(__name__)
+ Api(app).add_resource(feature.FeatureApi, "/console/api/features")
+
+ response = app.test_client().get("/console/api/features")
+
+ assert response.status_code == 200
+ payload = response.get_json()
+ assert "agent_tracing_api_collection_enabled" not in payload
+ schema = FeatureModel.model_json_schema(mode="serialization")
+ assert "agent_tracing_api_collection_enabled" not in schema["properties"]
+
+
+def test_workspace_features_still_require_authentication(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, LOGIN_DISABLED=False, OPS_AGENT_TRACE_ENABLED=True)
+ monkeypatch.setattr(wraps, "_is_setup_completed", lambda: True)
+ monkeypatch.setattr(login, "_resolve_current_user", lambda: None)
+
+ def forbidden() -> ApplicationServices:
+ pytest.fail("anonymous requests must not reach feature queries")
+
+ monkeypatch.setattr(feature, "application_services", forbidden)
+ app = Flask(__name__)
+ DifyLoginManager(app)
+ Api(app).add_resource(feature.FeatureApi, "/console/api/features")
+
+ response = app.test_client().get("/console/api/features")
+
+ assert response.status_code == 401
+ assert b"agent_tracing_api_collection_enabled" not in response.data
+
+
+def test_missing_public_gate_cannot_default_to_disabled() -> None:
+ with pytest.raises(ValidationError) as error:
+ SystemFeatureModel.model_validate({"deployment_edition": DeploymentEdition.COMMUNITY})
+
+ assert error.value.errors(include_input=False)[0]["loc"] == ("agent_tracing_api_collection_enabled",)
diff --git a/api/tests/unit_tests/controllers/inner_api/test_agent_tracing.py b/api/tests/unit_tests/controllers/inner_api/test_agent_tracing.py
new file mode 100644
index 00000000000..df1b4576ec0
--- /dev/null
+++ b/api/tests/unit_tests/controllers/inner_api/test_agent_tracing.py
@@ -0,0 +1,96 @@
+"""HTTP authentication and acknowledgement backed by the real SQLite collector."""
+
+from collections.abc import Iterator
+from uuid import uuid4
+
+import pytest
+from flask import Flask
+from flask.testing import FlaskClient
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from controllers.inner_api import bp
+from models.application_trace import ApplicationTraceBatch
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import config_overrides_context
+from tests.unit_tests.services.tracing.test_application_trace_collection import batch_for, seed
+
+
+def client() -> Iterator[FlaskClient]:
+ app = Flask(__name__)
+ app.config["TESTING"] = True
+ app.register_blueprint(bp)
+ with config_overrides_context(
+ PLUGIN_DAEMON_KEY="daemon", INNER_API_KEY_FOR_PLUGIN="inner", OPS_AGENT_TRACE_ENABLED=True
+ ):
+ yield app.test_client()
+
+
+def test_trace_http_ack_follows_commit_and_replay_is_idempotent(
+ client: FlaskClient,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ collection = seed(sqlite_session_factory, "http", capture_content=False)
+ invocation = collection.invocation()
+ payload = batch_for(collection.run, invocation).model_dump(mode="json")
+ service = ApplicationTracingService(collection.repository)
+ monkeypatch.setattr("controllers.inner_api.agent.tracing.application_tracing_service", lambda: service)
+ scheduled = []
+
+ def schedule(run_id: str) -> None:
+ # A fresh transaction can see the batch before the caller gets an ACK.
+ with sqlite_session_factory() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 1
+ scheduled.append(run_id)
+
+ monkeypatch.setattr(service, "schedule", schedule)
+ for _ in range(2):
+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})
+ assert response.status_code == 200
+ assert response.get_json() == {"result": "success", "batch_id": payload["batch_id"], "accepted_spans": 1}
+ assert scheduled == [collection.run.id, collection.run.id]
+
+ payload["spans"][0]["name"] = "conflicting replay"
+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})
+ assert response.status_code == 409
+ assert len(scheduled) == 2
+
+
[email protected]("header", [{}, {"X-Inner-Api-Key": "wrong"}])
+def test_trace_http_requires_internal_auth(client: FlaskClient, header: dict[str, str]) -> None:
+ response = client.post("/inner/api/agent/tracing/batches", json={}, headers=header)
+ assert response.status_code == 404
+
+
+def test_trace_http_rejects_invalid_span_and_unknown_owner(
+ client: FlaskClient,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ collection = seed(sqlite_session_factory, "http-validation")
+ invocation = collection.invocation()
+ payload = batch_for(collection.run, invocation).model_dump(mode="json")
+ service = ApplicationTracingService(collection.repository)
+ monkeypatch.setattr("controllers.inner_api.agent.tracing.application_tracing_service", lambda: service)
+ payload["spans"][0]["end_time_ns"] = 0
+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})
+ assert response.status_code == 400
+ payload["spans"][0]["end_time_ns"] = 2_000_000_000
+ payload["invocation_id"] = str(uuid4())
+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})
+ assert response.status_code == 404
+ with sqlite_session_factory() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0
+
+
+def test_trace_http_enforces_body_limit_before_ingestion(client: FlaskClient, monkeypatch: pytest.MonkeyPatch) -> None:
+ calls = []
+ monkeypatch.setattr("controllers.inner_api.agent.tracing.application_tracing_service", lambda: calls.append(True))
+ with config_overrides_context(OPS_AGENT_TRACE_BATCH_MAX_BYTES=64):
+ response = client.post(
+ "/inner/api/agent/tracing/batches", json={"padding": "x" * 256}, headers={"X-Inner-Api-Key": "inner"}
+ )
+ assert response.status_code == 413
+ assert calls == []
diff --git a/api/tests/unit_tests/controllers/openapi/auth/_world.py b/api/tests/unit_tests/controllers/openapi/auth/_world.py
index 98985e191c8..9a50759f56d 100644
--- a/api/tests/unit_tests/controllers/openapi/auth/_world.py
+++ b/api/tests/unit_tests/controllers/openapi/auth/_world.py
@@ -115,6 +115,7 @@ def system_features(
webapp_auth: bool = False,
) -> SystemFeatureModel:
return SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False,
deployment_edition=DeploymentEdition.ENTERPRISE,
license=LicenseStatusModel(status=license_status),
webapp_auth=WebAppAuthModel(enabled=webapp_auth),
diff --git a/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py b/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py
index 6eafdd198f5..4e1a047a51b 100644
--- a/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py
+++ b/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py
@@ -1664,7 +1664,7 @@ def world(sqlite_session_factory: sessionmaker[Session], token_rows: dict[str, R
def _system_features(
*, edition: DeploymentEdition, webapp_auth: bool, license_status: LicenseStatus
) -> SystemFeatureModel:
- features = SystemFeatureModel(deployment_edition=edition)
+ features = SystemFeatureModel(agent_tracing_api_collection_enabled=False, deployment_edition=edition)
features.webapp_auth.enabled = webapp_auth
features.license.status = license_status
return features
diff --git a/api/tests/unit_tests/controllers/web/test_feature.py b/api/tests/unit_tests/controllers/web/test_feature.py
index 1a08908bc25..42b211aacb5 100644
--- a/api/tests/unit_tests/controllers/web/test_feature.py
+++ b/api/tests/unit_tests/controllers/web/test_feature.py
@@ -29,7 +29,9 @@ class TestSystemFeatureApi:
app: Flask,
mocker: MockerFixture,
) -> None:
- system_features = SystemFeatureModel(deployment_edition=deployment_edition)
+ system_features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=deployment_edition
+ )
feature_queries = _install_feature_queries(mocker)
feature_queries.get_public_system_features.return_value = system_features
diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py
index a4e2cc02ac4..7cec02c4fe3 100644
--- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py
+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py
@@ -479,6 +479,12 @@ class TestGenerateWorker:
def test_generate_task_stopped_is_swallowed(self, generator, mocker: MockerFixture):
self._wire(generator, mocker, run_side_effect=GenerateTaskStoppedError())
+ # A locally stopped consumer does not prove runtime cancellation. The
+ # runner must hand off the typed terminal it actually observed.
+ mocker.patch(
+ "core.ops.base_trace_instance.application_tracing_hooks",
+ side_effect=AssertionError("stop intent must not invent a tracing terminal"),
+ )
queue_manager = mocker.MagicMock()
self._call(generator, mocker, queue_manager)
queue_manager.publish_error.assert_not_called()
diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
index f7a122427f4..4ef45d54e3c 100644
--- a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
@@ -14,6 +14,7 @@ from dify_agent.layers.knowledge import DifyKnowledgeBaseLayerConfig
from dify_agent.layers.prompt import Config as PromptConfig
from dify_agent.layers.user_prompt import DifyUserPromptLayerConfig
from dify_agent.protocol.snapshot import SessionSnapshot
+from flask import Flask
from clients.agent_backend import (
DIFY_CONFIG_LAYER_ID,
@@ -30,13 +31,21 @@ from core.app.apps.agent_app.runtime_request_builder import (
AgentAppRuntimeRequestBuildError,
)
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
+from core.ops.base_trace_instance import bind_application_tracing_hooks
from core.workflow.file_reference import build_file_reference
+from core.workflow.nodes.agent_v2.dify_tools_builder import WorkflowAgentDifyToolsBuilder
from graphon.file import File, FileTransferMethod, FileType
from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
from models.agent_config_entities import AgentSoulConfig
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from tests.unit_tests.config_override import apply_config_overrides
[email protected](autouse=True)
+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+
+
@pytest.fixture(autouse=True)
def _no_runtime_agent_skills(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
@@ -112,6 +121,43 @@ class TestBuildForAgentApp:
assert [layer.name for layer in request.composition.layers][0] == "agent_app_user_prompt"
+def test_builder_carries_application_trace_context_but_redacts_it_from_request_logs(monkeypatch: pytest.MonkeyPatch):
+ from uuid import uuid4
+
+ from dify_agent.layers.execution_context import ApplicationTracingContext
+
+ tracing = ApplicationTracingContext(
+ run_id=uuid4(),
+ invocation_id=uuid4(),
+ trace_id="1" * 32,
+ parent_span_id="2" * 16,
+ session_id="private-session",
+ capture_content=False,
+ )
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ context = _ctx(_soul_with_model())
+ calls: list[AgentAppRuntimeBuildContext] = []
+
+ def prepare(value: AgentAppRuntimeBuildContext) -> ApplicationTracingContext:
+ calls.append(value)
+ return tracing
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "prepare_agent_app", staticmethod(prepare))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+
+ with app.app_context():
+ result = AgentAppRuntimeRequestBuilder(dify_tools_builder=WorkflowAgentDifyToolsBuilder()).build(context)
+ layer = next(item for item in result.request.composition.layers if item.name == "execution_context")
+ assert DifyExecutionContextLayerConfig.model_validate(layer.config).tracing == tracing
+ redacted = next(
+ item for item in result.redacted_request["composition"]["layers"] if item["name"] == "execution_context"
+ )
+ assert redacted["config"]["tracing"] == "[REDACTED]"
+ assert calls == [context]
+
+
class _NoToolsBuilder:
def build_layers(self, **kwargs):
del kwargs
diff --git a/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py b/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py
index 6046a70fabd..cdf3f984378 100644
--- a/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py
+++ b/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py
@@ -5,12 +5,14 @@ from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
import pytest
+from flask import Flask
from sqlalchemy import event as sqlalchemy_event
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
-from core.app.entities.app_invoke_entities import WorkflowAppGenerateEntity
+from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
from core.app.workflow.retry_history import RETRY_HISTORY_PROCESS_DATA_KEY
+from core.ops.base_trace_instance import bind_application_tracing_hooks
from core.ops.ops_trace_manager import TraceTask, TraceTaskName
from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository
from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
@@ -62,13 +64,21 @@ from graphon.runtime.execution import ROOT_FRAME_ID
from models import Account, WorkflowRun
from models.enums import WorkflowRunTriggeredFrom
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
+from repositories.entities.workflow_execution import WorkflowExecution
from repositories.sqlalchemy_api_workflow_node_execution_repository import (
DifyAPISQLAlchemyWorkflowNodeExecutionRepository,
)
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer
+from tests.unit_tests.config_override import apply_config_overrides
from tests.workflow_test_utils import build_test_graph_init_params
[email protected](autouse=True)
+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+
+
class _RepoRecorder:
def __init__(self) -> None:
self.saved: list[object] = []
@@ -679,6 +689,37 @@ def test_terminal_failure_batches_only_unfinished_nodes_per_source(
class TestWorkflowPersistenceLayer:
+ def test_start_freezes_tracing_only_after_workflow_save(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ layer, workflow_repo, _, _ = _make_layer(extras={"trace_session_id": "external-session"})
+ calls: list[tuple[str, str, str, str, object]] = []
+
+ def freeze(
+ entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,
+ *,
+ workflow_id: str,
+ workflow_run_id: str,
+ conversation_id: object,
+ graph_data: object,
+ ) -> None:
+ assert workflow_repo.synchronously_saved
+ assert graph_data == {"nodes": [], "edges": []}
+ saved_execution = workflow_repo.synchronously_saved[-1]
+ assert isinstance(saved_execution, WorkflowExecution)
+ assert saved_execution.id_ == workflow_run_id
+ calls.append(
+ (entity.app_config.app_id, entity.app_config.tenant_id, workflow_id, workflow_run_id, conversation_id)
+ )
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "freeze_workflow_run", staticmethod(freeze))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+
+ with app.app_context():
+ layer._handle_graph_run_started()
+ assert calls == [("app", "tenant", "workflow-id", "run-id", "conv-id")]
+
def test_get_execution_id_requires_system_variable(self):
layer, _, _, _ = _make_layer(build_system_variables())
diff --git a/api/tests/unit_tests/core/ops/test_application_tracing_hooks.py b/api/tests/unit_tests/core/ops/test_application_tracing_hooks.py
new file mode 100644
index 00000000000..994d0cad57b
--- /dev/null
+++ b/api/tests/unit_tests/core/ops/test_application_tracing_hooks.py
@@ -0,0 +1,52 @@
+"""The execution port is gated before Flask lookup and bound per application."""
+
+from contextvars import Context
+
+import pytest
+from flask import Flask, has_app_context
+
+from core.ops.base_trace_instance import (
+ ApplicationTracingHooksNotRegisteredError,
+ application_tracing_hooks,
+ bind_application_tracing_hooks,
+)
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
+from tests.unit_tests.config_override import apply_config_overrides
+
+
+def test_disabled_port_needs_no_application_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+ isolated_context = Context()
+ assert not isolated_context.run(has_app_context)
+ assert isolated_context.run(application_tracing_hooks) is None
+
+
+def test_enabled_port_reports_missing_registration_without_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ isolated_context = Context()
+ assert not isolated_context.run(has_app_context)
+ with pytest.raises(ApplicationTracingHooksNotRegisteredError, match="application_tracing_hooks_not_registered"):
+ isolated_context.run(application_tracing_hooks)
+
+
+def test_enabled_port_reports_missing_registration_in_app(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ with Flask(__name__).app_context():
+ with pytest.raises(ApplicationTracingHooksNotRegisteredError, match="application_tracing_hooks_not_registered"):
+ application_tracing_hooks()
+
+
+def test_port_does_not_leak_between_flask_applications(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ first, second = Flask("first"), Flask("second")
+ first_hooks = RuntimeApplicationTracingHooks()
+ second_hooks = RuntimeApplicationTracingHooks()
+ bind_application_tracing_hooks(first, first_hooks)
+ bind_application_tracing_hooks(second, second_hooks)
+ with first.app_context():
+ assert application_tracing_hooks() is first_hooks
+ with second.app_context():
+ assert application_tracing_hooks() is second_hooks
+ assert application_tracing_hooks() is first_hooks
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+ assert application_tracing_hooks() is None
diff --git a/api/tests/unit_tests/core/ops/test_ops_trace_manager.py b/api/tests/unit_tests/core/ops/test_ops_trace_manager.py
index ba456261407..a7cc2ba847a 100644
--- a/api/tests/unit_tests/core/ops/test_ops_trace_manager.py
+++ b/api/tests/unit_tests/core/ops/test_ops_trace_manager.py
@@ -467,14 +467,20 @@ def test_message_config_lookup_uses_real_conversation_and_model_config(database:
assert OpsTraceManager.get_app_config_through_message_id("missing") is None
[email protected]("module_name", ["dify_trace_weave", "wandb"])
-def test_provider_loader_identifies_missing_optional_dependency(
- monkeypatch: pytest.MonkeyPatch, module_name: str
-) -> None:
- monkeypatch.delitem(sys.modules, "dify_trace_weave.weave_trace", raising=False)
- monkeypatch.setitem(sys.modules, module_name, None)
- with pytest.raises(TraceProviderNotInstalledError, match=f"weave.*{module_name}") as caught:
- module.OpsTraceProviderConfigMap()["weave"]
[email protected]("imports_cached", [False, True], ids=["cold", "warm"])
[email protected]("module_name", ["dify_trace_weave", "weave"])
+def test_provider_loader_identifies_missing_optional_dependency(module_name: str, imports_cached: bool) -> None:
+ with patch.dict(sys.modules):
+ if imports_cached:
+ module.OpsTraceProviderConfigMap()["weave"]
+ # Cached typed SDK submodules can bypass a missing top-level package.
+ # Simulate its absence consistently and restore the complete cache after.
+ for cached_name in list(sys.modules):
+ if cached_name.partition(".")[0] in {"dify_trace_weave", "weave"}:
+ del sys.modules[cached_name]
+ with patch.dict(sys.modules, {module_name: None}):
+ with pytest.raises(TraceProviderNotInstalledError, match=f"weave.*{module_name}") as caught:
+ module.OpsTraceProviderConfigMap()["weave"]
cause = caught.value.__cause__
assert isinstance(cause, ModuleNotFoundError)
diff --git a/api/tests/unit_tests/core/ops/test_trace_queue_manager.py b/api/tests/unit_tests/core/ops/test_trace_queue_manager.py
index eb11d6197c4..51a924127ee 100644
--- a/api/tests/unit_tests/core/ops/test_trace_queue_manager.py
+++ b/api/tests/unit_tests/core/ops/test_trace_queue_manager.py
@@ -14,6 +14,16 @@ import queue
from unittest.mock import MagicMock, patch
import pytest
+from flask import Flask
+
+from core.ops.base_trace_instance import bind_application_tracing_hooks
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
+from tests.unit_tests.config_override import apply_config_overrides
+
+
[email protected](autouse=True)
+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
@pytest.fixture
@@ -158,3 +168,37 @@ class TestTraceQueueManagerTelemetryGuard:
assert called_task is trace_task
assert trace_queue.empty()
assert called_task.app_id == "expected-app-id"
+
+
[email protected]("enabled_at_start", [False, True])
+def test_application_pipeline_snapshots_switch_without_constructing_sdk(
+ trace_queue_manager_and_task, monkeypatch: pytest.MonkeyPatch, enabled_at_start: bool
+):
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ trace_queue_manager, trace_task_class, trace_task_name = trace_queue_manager_and_task
+ trace_queue = queue.Queue()
+ calls: list[object] = []
+ current_enabled = enabled_at_start
+
+ def queue_enabled(app_id: object) -> bool:
+ calls.append(app_id)
+ return current_enabled
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "queue_enabled_at_start", staticmethod(queue_enabled))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+ with (
+ app.app_context(),
+ patch("core.telemetry.gateway.is_enterprise_telemetry_enabled", return_value=False),
+ patch("core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance") as legacy,
+ patch("core.ops.ops_trace_manager.trace_manager_queue", trace_queue),
+ ):
+ manager = trace_queue_manager(app_id="app-id")
+ current_enabled = not enabled_at_start
+ task = trace_task_class(trace_type=trace_task_name.WORKFLOW_TRACE)
+ manager.add_trace_task(task)
+ legacy.assert_not_called()
+ assert calls == ["app-id"]
+ assert manager.trace_instance is None
+ assert trace_queue.qsize() == int(enabled_at_start)
diff --git a/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py b/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py
index 1402b82c34b..81166c68e6a 100644
--- a/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py
+++ b/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py
@@ -4,6 +4,7 @@ import json
from datetime import datetime
import pytest
+import requests
from sqlalchemy.orm import Session
from core.ops.entities.trace_entity import WorkflowTraceInfo
@@ -12,11 +13,20 @@ from extensions.ext_database import db
from models import Account
from models.enums import CreatorUserRole
from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
+from tests.unit_tests.services.tracing.test_mlflow_legacy_destination import (
+ exported_spans,
+)
+from tests.unit_tests.services.tracing.test_mlflow_legacy_destination import (
+ transport as transport, # noqa: PLC0414 - Reuse the real SDK's isolated requests boundary.
+)
@pytest.mark.parametrize("exporter", ["canonical", "mlflow"])
def test_external_workflow_trace_omits_source_internals(
- exporter: str, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch
+ exporter: str,
+ sqlite_session: Session,
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
) -> None:
started_at = datetime(2025, 1, 1)
public_inputs = {"query": "public tool input"}
@@ -94,31 +104,28 @@ def test_external_workflow_trace_omits_source_internals(
from dify_trace_mlflow.config import MLflowConfig
from dify_trace_mlflow.mlflow_trace import MLflowDataTrace
from mlflow.entities import Span
- from mlflow.tracing import provider
- from mlflow.tracing.processor.mlflow_v3 import MlflowV3SpanProcessor
from mlflow.tracing.trace_manager import InMemoryTraceManager
- from opentelemetry.sdk.trace import TracerProvider
- from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
- monkeypatch.setattr("dify_trace_mlflow.mlflow_trace.mlflow.set_tracking_uri", lambda _uri: None)
- monkeypatch.setattr("dify_trace_mlflow.mlflow_trace.mlflow.set_experiment", lambda **_kwargs: None)
- monkeypatch.setattr("mlflow.tracking.fluent._active_experiment_id", "0")
monkeypatch.setattr(InMemoryTraceManager, "_instance", InMemoryTraceManager())
- monkeypatch.setenv("MLFLOW_ENABLE_ASYNC_TRACE_LOGGING", "true")
- span_exporter = InMemorySpanExporter()
- tracer_provider = TracerProvider()
- tracer_provider.add_span_processor(MlflowV3SpanProcessor(span_exporter, export_metrics=False))
- monkeypatch.setattr(provider, "_get_tracer", tracer_provider.get_tracer)
+ # OSS owns a private synchronous SDK pipeline. Inspect its real wire spans
+ # instead of replacing the unrelated process-wide MLflow tracer.
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://workflow-boundary.example", experiment_id="0"))
try:
- MLflowDataTrace(MLflowConfig()).workflow_trace(info)
- spans = [Span(span) for span in span_exporter.get_finished_spans()]
+ instance.trace(info)
+ spans = [Span.from_otel_proto(span) for span in exported_spans(transport)]
assert len(spans) == 2
payload = json.dumps([span.to_dict() for span in spans])
tool = next(span for span in spans if span.name == "outer-tool")
assert tool.inputs == public_inputs
assert tool.outputs == public_outputs
+ assert transport
+ assert {request.url for request in transport} <= {
+ "https://workflow-boundary.example/version",
+ "https://workflow-boundary.example/api/3.0/mlflow/traces",
+ "https://workflow-boundary.example/v1/traces",
+ }
finally:
- tracer_provider.shutdown()
+ instance._provider.shutdown()
assert "private source" not in payload
assert "source-llm" not in payload
assert "recursive-llm" not in payload
diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
index 5651da46785..3ad29c87207 100644
--- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
+++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
@@ -10,6 +10,7 @@ from dify_agent.layers.dify_plugin import DifyPluginToolConfig, DifyPluginToolsL
from dify_agent.layers.prompt import Config as PromptConfig
from dify_agent.protocol import DIFY_AGENT_HISTORY_LAYER_ID, DIFY_AGENT_MODEL_LAYER_ID, DIFY_AGENT_OUTPUT_LAYER_ID
from dify_agent.protocol.snapshot import SessionSnapshot
+from flask import Flask
from clients.agent_backend import (
DIFY_CONFIG_LAYER_ID,
@@ -19,6 +20,7 @@ from clients.agent_backend import (
)
from clients.agent_backend.request_builder import DIFY_SHELL_LAYER_ID
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
+from core.ops.base_trace_instance import bind_application_tracing_hooks
from core.workflow.file_reference import build_file_reference
from core.workflow.nodes.agent_v2.dify_tools_builder import WorkflowAgentDifyToolsBuilder
from core.workflow.nodes.agent_v2.runtime_request_builder import (
@@ -47,9 +49,15 @@ from models.agent_config_entities import (
DeclaredOutputType,
WorkflowNodeJobConfig,
)
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from tests.unit_tests.config_override import apply_config_overrides
[email protected](autouse=True)
+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+
+
@pytest.fixture(autouse=True)
def _no_runtime_agent_skills(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
@@ -232,6 +240,42 @@ def _context() -> WorkflowAgentRuntimeBuildContext:
)
[email protected]("preview", [False, True])
+def test_builder_passes_inline_or_preview_context_to_application_tracing(
+ monkeypatch: pytest.MonkeyPatch, preview: bool
+):
+ from uuid import uuid4
+
+ from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfig
+
+ tracing = ApplicationTracingContext(
+ run_id=uuid4(),
+ invocation_id=uuid4(),
+ trace_id="1" * 32,
+ parent_span_id="2" * 16,
+ session_id="session",
+ capture_content=False,
+ )
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ context = replace(_context(), workflow_run_id=None if preview else "run-1", attempt=2)
+ calls: list[tuple[WorkflowAgentRuntimeBuildContext, str | None]] = []
+
+ def prepare(value: WorkflowAgentRuntimeBuildContext, conversation_id: str | None) -> ApplicationTracingContext:
+ calls.append((value, conversation_id))
+ return tracing
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "prepare_workflow_agent", staticmethod(prepare))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+
+ with app.app_context():
+ result = WorkflowAgentRuntimeRequestBuilder(dify_tools_builder=WorkflowAgentDifyToolsBuilder()).build(context)
+ layer = next(item for item in result.request.composition.layers if item.name == DIFY_EXECUTION_CONTEXT_LAYER_ID)
+ assert DifyExecutionContextLayerConfig.model_validate(layer.config).tracing == tracing
+ assert calls == [(context, None)]
+
+
def _request_layers(result) -> dict[str, dict[str, object]]:
dumped = result.request.model_dump(mode="json")
return {layer["name"]: layer for layer in dumped["composition"]["layers"]}
diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.py
index 32de236ce22..3bafee7497b 100644
--- a/api/tests/unit_tests/extensions/test_ext_application_services.py
+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py
@@ -6,7 +6,7 @@ from collections.abc import Mapping
from decimal import Decimal
from io import BytesIO
from types import SimpleNamespace
-from typing import cast
+from typing import Never, cast
from unittest.mock import MagicMock, call, create_autospec, patch
from uuid import uuid4
@@ -20,6 +20,7 @@ from sqlalchemy.orm import Session, sessionmaker
from core.credit_usage import CreditUsageAppType
from core.llm_generator.llm_generator import LLMGenerator
from core.model_context import get_credit_usage_metadata, use_credit_usage_metadata
+from core.ops.base_trace_instance import application_tracing_hooks
from core.tools.tool_file_manager import ToolFileManager
from enums import DeploymentEdition, WebAppAccessMode
from extensions import ext_application_services
@@ -151,6 +152,7 @@ from services.retention.workflow_run.archive_download_task_cache import Workflow
from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveService
from services.tag_application_service import TagApplicationService
from services.tool_file_download_service import ToolFileDownloadService
+from services.tracing import runtime_bridge
from services.upload_file_delivery_service import UploadFileDeliveryService
from services.web_authentication_adapters import (
AccountWebAuthenticationSecurityGateway,
@@ -269,6 +271,40 @@ def test_init_app_registers_services_for_the_current_app(
assert creators._oauth is services.oauth_server
+def test_init_app_registers_distinct_tracing_ports_without_constructing_trace_services(
+ monkeypatch: pytest.MonkeyPatch,
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ first_app, second_app = Flask("first"), Flask("second")
+ monkeypatch.setattr(ext_application_services, "get_session_maker", lambda: sqlite_session_factory)
+ apply_config_overrides(
+ monkeypatch,
+ DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY,
+ INIT_PASSWORD="",
+ OPS_AGENT_TRACE_ENABLED=True,
+ )
+ construction_calls: list[bool] = []
+
+ def construct_trace_service() -> Never:
+ construction_calls.append(True)
+ raise AssertionError("Composition must not construct a trace service")
+
+ monkeypatch.setattr(runtime_bridge, "_service", construct_trace_service)
+
+ ext_application_services.init_app(first_app)
+ ext_application_services.init_app(second_app)
+
+ with first_app.app_context():
+ first_hooks = application_tracing_hooks()
+ assert isinstance(first_hooks, runtime_bridge.RuntimeApplicationTracingHooks)
+ with second_app.app_context():
+ second_hooks = application_tracing_hooks()
+ assert isinstance(second_hooks, runtime_bridge.RuntimeApplicationTracingHooks)
+ assert second_hooks is not first_hooks
+ assert application_tracing_hooks() is first_hooks
+ assert construction_calls == []
+
+
def test_build_application_services_preserves_composed_boundaries(
sqlite_session_factory: sessionmaker[Session],
) -> None:
diff --git a/api/tests/unit_tests/migrations/test_application_agent_tracing.py b/api/tests/unit_tests/migrations/test_application_agent_tracing.py
new file mode 100644
index 00000000000..571ed2afb73
--- /dev/null
+++ b/api/tests/unit_tests/migrations/test_application_agent_tracing.py
@@ -0,0 +1,69 @@
+"""The migration can create and remove the three tables without business tables."""
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+import sqlalchemy as sa
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from alembic.script import ScriptDirectory
+
+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun
+
+
+ ("starting_revision", "required_revisions"),
+ [
+ ("f1d3c5b7a902", ("a17c2980f3e6", "c6f4a90d8b31", "f7e2c9a6d481")),
+ ("a17c2980f3e6", ("c6f4a90d8b31", "f7e2c9a6d481")),
+ ("c6f4a90d8b31", ("a17c2980f3e6", "f7e2c9a6d481")),
+ ],
+)
+def test_tracing_and_workflow_pause_upgrade_paths_join(
+ starting_revision: str, required_revisions: tuple[str, ...]
+) -> None:
+ scripts = ScriptDirectory(str(Path(__file__).resolve().parents[3] / "migrations"))
+ merge = scripts.get_revision("f7e2c9a6d481")
+ assert merge is not None
+ assert isinstance(merge.down_revision, tuple)
+ assert set(merge.down_revision) == {"a17c2980f3e6", "c6f4a90d8b31"}
+
+ # Use Alembic's upgrade planner so either already-applied branch still runs
+ # its missing sibling before the merge, without rewriting either migration.
+ steps = scripts._upgrade_revs(merge.revision, starting_revision)
+ assert {step.revision.revision for step in steps} == set(required_revisions)
+ assert steps[-1].revision.revision == merge.revision
+
+
+def test_application_trace_migration_matches_models_and_roundtrips() -> None:
+ path = (
+ Path(__file__).resolve().parents[3]
+ / "migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py"
+ )
+ spec = importlib.util.spec_from_file_location("application_trace_migration", path)
+ assert spec is not None
+ assert spec.loader is not None
+ migration = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(migration)
+ assert migration.down_revision == "f1d3c5b7a902"
+ engine = sa.create_engine("sqlite://")
+ try:
+ with engine.begin() as connection:
+ with Operations.context(MigrationContext.configure(connection)):
+ migration.upgrade()
+ inspector = sa.inspect(connection)
+ for model in (ApplicationTraceRun, ApplicationTraceInvocation, ApplicationTraceBatch):
+ actual = {column["name"]: column for column in inspector.get_columns(model.__tablename__)}
+ assert set(actual) == set(model.__table__.columns.keys())
+ for column in model.__table__.columns:
+ assert actual[column.name]["nullable"] == column.nullable
+ assert {index["name"] for index in inspector.get_indexes("application_trace_runs")} == {
+ "application_trace_delivery_idx",
+ "application_trace_retention_idx",
+ }
+ with Operations.context(MigrationContext.configure(connection)):
+ migration.downgrade()
+ assert sa.inspect(connection).get_table_names() == []
+ finally:
+ engine.dispose()
diff --git a/api/tests/unit_tests/models/test_agent_config_entities.py b/api/tests/unit_tests/models/test_agent_config_entities.py
index dbd6f84c30e..6a3b816887f 100644
--- a/api/tests/unit_tests/models/test_agent_config_entities.py
+++ b/api/tests/unit_tests/models/test_agent_config_entities.py
@@ -1,4 +1,5 @@
import pytest
+from pydantic import ValidationError
from core.workflow.file_reference import build_file_reference
from models.agent_config_entities import (
@@ -28,6 +29,30 @@ def test_agent_soul_model_settings_preserves_plugin_declared_parameters() -> Non
assert dumped["thinking_budget"] == 4096
+ "response_format",
+ [
+ "text",
+ "json_schema",
+ {"type": "text"},
+ {"type": "json_schema", "json_schema": {"name": "response", "schema": {"type": "object"}}},
+ None,
+ ],
+)
+def test_agent_model_response_format_preserves_plugin_and_legacy_shapes(response_format: object) -> None:
+ settings = AgentSoulModelSettings.model_validate({"response_format": response_format})
+
+ assert settings.model_dump(mode="json", exclude_none=True) == (
+ {"response_format": response_format} if response_format is not None else {}
+ )
+
+
[email protected]("response_format", [[], ["text"], 1, True])
+def test_agent_model_response_format_rejects_non_string_or_object_values(response_format: object) -> None:
+ with pytest.raises(ValidationError, match="response_format"):
+ AgentSoulModelSettings.model_validate({"response_format": response_format})
+
+
def test_file_default_value_accepts_canonical_reference_mapping() -> None:
reference = build_file_reference(record_id="tool-file-1")
diff --git a/api/tests/unit_tests/models/test_message_trace_serialization.py b/api/tests/unit_tests/models/test_message_trace_serialization.py
new file mode 100644
index 00000000000..755dbf158eb
--- /dev/null
+++ b/api/tests/unit_tests/models/test_message_trace_serialization.py
@@ -0,0 +1,161 @@
+"""Keep the persisted provider through private trace JSON and the real consumer."""
+
+import json
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from uuid import uuid4
+
+import pytest
+from dify_trace_arize_phoenix import arize_phoenix_trace
+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace
+from dify_trace_arize_phoenix.config import ArizeConfig
+from openinference.semconv.trace import SpanAttributes
+from opentelemetry.sdk.trace import Tracer, TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from pydantic import TypeAdapter
+from sqlalchemy.orm import Session
+
+from core.ops.entities.trace_entity import MessageTraceInfo
+from core.ops.ops_trace_manager import OpsTraceManager, TaskData
+from models.enums import ConversationFromSource
+from models.model import Message, MessageDict
+from tasks import ops_trace_task
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_message
+
+_MESSAGE = TypeAdapter(MessageDict)
+
+
+def persisted_message(sqlite_session: Session) -> Message:
+ now = datetime(2026, 10, 20, tzinfo=UTC)
+ message = make_message(
+ message_id=str(uuid4()),
+ app_id=str(uuid4()),
+ conversation_id=str(uuid4()),
+ inputs={},
+ query="Synthetic input",
+ message={},
+ answer="Synthetic output",
+ message_unit_price=Decimal(0),
+ answer_unit_price=Decimal(0),
+ total_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.CONSOLE,
+ created_at=now,
+ updated_at=now + timedelta(seconds=1),
+ )
+ message.model_provider = "langgenius/openai/openai"
+ message.model_id = "gpt-4o-mini"
+ sqlite_session.add(message)
+ sqlite_session.commit()
+ sqlite_session.expire_all()
+ restored = sqlite_session.get(Message, message.id)
+ assert restored is not None
+ return restored
+
+
[email protected]("provider", ["langgenius/openai/openai", "Custom/Provider:Exact", None])
+def test_message_provider_survives_real_json_round_trip(
+ sqlite_session: Session,
+ persisted_message: Message,
+ provider: str | None,
+) -> None:
+ persisted_message.model_provider = provider
+ sqlite_session.commit()
+ sqlite_session.refresh(persisted_message)
+ wire = json.dumps(persisted_message.to_dict(session=sqlite_session), default=str)
+ payload = _MESSAGE.validate_json(wire)
+ restored = Message.from_dict(payload)
+ assert "model_provider" in payload
+ assert restored.model_provider == provider
+ assert restored.model_id == persisted_message.model_id
+ assert restored.id == persisted_message.id
+
+
[email protected]("shape", ["current", "legacy_missing", "explicit_null"])
+def test_private_trace_task_preserves_provider_into_real_arize_span(
+ sqlite_session: Session,
+ persisted_message: Message,
+ monkeypatch: pytest.MonkeyPatch,
+ shape: str,
+) -> None:
+ message_data = persisted_message.to_dict(session=sqlite_session)
+ if shape == "legacy_missing":
+ message_data.pop("model_provider", None)
+ elif shape == "explicit_null":
+ message_data["model_provider"] = None
+ start = datetime(2026, 10, 20, tzinfo=UTC)
+ info = MessageTraceInfo(
+ message_id=persisted_message.id,
+ message_data=message_data,
+ conversation_model="chat",
+ conversation_mode="chat",
+ inputs={"query": "Synthetic input"},
+ outputs="Synthetic output",
+ start_time=start,
+ end_time=start + timedelta(seconds=1),
+ metadata={},
+ message_tokens=1,
+ answer_tokens=1,
+ total_tokens=2,
+ )
+ encoded = (
+ TaskData(app_id=persisted_message.app_id, trace_info_type="MessageTraceInfo", trace_info=info.model_dump())
+ .model_dump_json()
+ .encode()
+ )
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ processor = SimpleSpanProcessor(exporter)
+ provider.add_span_processor(processor)
+
+ def setup(_config: ArizeConfig) -> tuple[Tracer, SimpleSpanProcessor]:
+ tracer = provider.get_tracer("message-provider-serialization")
+ assert isinstance(tracer, Tracer)
+ return tracer, processor
+
+ monkeypatch.setattr(arize_phoenix_trace, "setup_tracer", setup)
+ instance = ArizePhoenixDataTrace(ArizeConfig(api_key="unused-synthetic-key", space_id="synthetic-space"))
+ paths: list[str] = []
+ deleted: list[str] = []
+
+ def load(path: str) -> bytes:
+ paths.append(path)
+ return encoded
+
+ def delete(path: str) -> None:
+ deleted.append(path)
+
+ def lookup(app_id: str) -> ArizePhoenixDataTrace:
+ assert app_id == persisted_message.app_id
+ return instance
+
+ def failed(_key: str) -> None:
+ pytest.fail("The real trace consumer must not hide a failed dispatch")
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+ monkeypatch.setattr(ops_trace_task.storage, "load", load)
+ monkeypatch.setattr(ops_trace_task.storage, "delete", delete)
+ monkeypatch.setattr(ops_trace_task.redis_client, "incr", failed)
+ monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lookup)
+ monkeypatch.setattr("extensions.ext_enterprise_telemetry.is_enabled", lambda: False)
+ try:
+ ops_trace_task.process_trace_tasks.apply(
+ args=({"app_id": persisted_message.app_id, "file_id": "owned-synthetic"},), throw=True
+ ).get()
+ spans = exporter.get_finished_spans()
+ llm = [span for span in spans if span.name == "llm"]
+ assert len(llm) == 1
+ attributes = llm[0].attributes
+ assert attributes is not None
+ assert attributes[SpanAttributes.LLM_MODEL_NAME] == persisted_message.model_id
+ if shape == "current":
+ assert attributes[SpanAttributes.LLM_PROVIDER] == persisted_message.model_provider
+ else:
+ assert SpanAttributes.LLM_PROVIDER not in attributes
+ assert paths == [f"ops_trace/{persisted_message.app_id}/owned-synthetic.json"]
+ assert deleted == paths
+ finally:
+ provider.shutdown()
diff --git a/api/tests/unit_tests/repositories/app/test_agent_app_repository.py b/api/tests/unit_tests/repositories/app/test_agent_app_repository.py
index f123e8a54c6..aa9300f2001 100644
--- a/api/tests/unit_tests/repositories/app/test_agent_app_repository.py
+++ b/api/tests/unit_tests/repositories/app/test_agent_app_repository.py
@@ -11,10 +11,23 @@ from models.agent import Agent, AgentKind, AgentScope, AgentSource, AgentStatus
from models.model import App, AppModelConfig
from repositories.app.agent_app_repository import AgentAppRepository
from services.agent.roster_service import AgentRosterService
+from services.app.agent_app_contracts import AgentAppNotFoundError
type Resolver = Literal["repository", "roster"]
[email protected]("_inline_agent")
+def test_tracing_owner_rejects_inline_without_materializing_app(sqlite_session_factory: sessionmaker[Session]) -> None:
+ repository = AgentAppRepository(session_factory=sqlite_session_factory)
+ with pytest.raises(AgentAppNotFoundError):
+ repository.resolve_roster_app_id(tenant_id="tenant-1", agent_id="inline-agent")
+ with sqlite_session_factory() as session:
+ assert session.scalar(select(App)) is None
+ agent = session.get(Agent, "inline-agent")
+ assert agent is not None
+ assert agent.backing_app_id is None
+
+
@pytest.fixture
def _inline_agent(sqlite_session_factory: sessionmaker[Session]) -> None:
with sqlite_session_factory.begin() as session:
diff --git a/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py b/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py
index abb31de5b28..4c94bde58b1 100644
--- a/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py
+++ b/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py
@@ -1,4 +1,5 @@
from collections.abc import Callable
+from datetime import datetime
import pytest
from sqlalchemy import select
@@ -6,7 +7,12 @@ from sqlalchemy.orm import Session, sessionmaker
from models.model import App, AppMode, TraceAppConfig
from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
-from services.app_tracing_config_service import AppTracingConfigAppNotFoundError, AppTracingConfigRecord
+from services.app_tracing_config_service import (
+ AppTraceSettings,
+ AppTracingConfigAppNotFoundError,
+ AppTracingConfigConflictError,
+ AppTracingConfigRecord,
+)
from tests.unit_tests.repositories.app.console_visibility import UNADDRESSABLE_IN_WORKSPACE, MakeUnaddressable
_APP_ID = "11111111-1111-1111-1111-111111111111"
@@ -69,6 +75,7 @@ def test_config_lifecycle_is_persisted_by_owned_transactions(
app_id=_APP_ID,
tracing_provider=_PROVIDER,
tracing_config={"public_key": "updated"},
+ expected=record,
)
with sqlite_session_factory() as session:
config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == _APP_ID))
@@ -76,12 +83,14 @@ def test_config_lifecycle_is_persisted_by_owned_transactions(
assert config.tracing_config == {"public_key": "updated"}
assert repository.delete(workspace_id=_WORKSPACE_ID, app_id=_APP_ID, tracing_provider=_PROVIDER)
- assert not repository.update(
- workspace_id=_WORKSPACE_ID,
- app_id=_APP_ID,
- tracing_provider=_PROVIDER,
- tracing_config={"public_key": "missing"},
- )
+ with pytest.raises(AppTracingConfigConflictError):
+ repository.update(
+ workspace_id=_WORKSPACE_ID,
+ app_id=_APP_ID,
+ tracing_provider=_PROVIDER,
+ tracing_config={"public_key": "missing"},
+ expected=record,
+ )
assert not repository.delete(workspace_id=_WORKSPACE_ID, app_id=_APP_ID, tracing_provider=_PROVIDER)
with sqlite_session_factory() as session:
assert session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == _APP_ID)) is None
@@ -106,6 +115,9 @@ def test_all_operations_reject_apps_console_cannot_address(
repository = _repository(sqlite_session_factory)
operations: tuple[Callable[[], object], ...] = (
lambda: repository.get(workspace_id=workspace_id, app_id=_APP_ID, tracing_provider=_PROVIDER),
+ lambda: repository.set_trace(
+ workspace_id=workspace_id, app_id=_APP_ID, settings=AppTraceSettings(), expected=None
+ ),
lambda: repository.create(
workspace_id=workspace_id,
app_id=_APP_ID,
@@ -117,6 +129,7 @@ def test_all_operations_reject_apps_console_cannot_address(
app_id=_APP_ID,
tracing_provider=_PROVIDER,
tracing_config={},
+ expected=AppTracingConfigRecord("config", _APP_ID, _PROVIDER, {}, True, datetime.now(), datetime.now()),
),
lambda: repository.delete(workspace_id=workspace_id, app_id=_APP_ID, tracing_provider=_PROVIDER),
)
diff --git a/api/tests/unit_tests/repositories/app/test_tracing_settings_repository.py b/api/tests/unit_tests/repositories/app/test_tracing_settings_repository.py
new file mode 100644
index 00000000000..e76f69e4233
--- /dev/null
+++ b/api/tests/unit_tests/repositories/app/test_tracing_settings_repository.py
@@ -0,0 +1,51 @@
+"""Tracing policy round-trips without dropping content or Agent capture choices."""
+
+import json
+
+import pytest
+from sqlalchemy.orm import Session, sessionmaker
+
+from machinery.context import RequestContext
+from models.model import App
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigAppNotFoundError
+
+
+def test_capture_policy_round_trip_and_legacy_defaults(sqlite_session_factory: sessionmaker[Session]) -> None:
+ context = RequestContext("request", None, "account", "tenant")
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ App(
+ id="app",
+ tenant_id="tenant",
+ name="Trace app",
+ mode="workflow",
+ enable_site=False,
+ enable_api=False,
+ tracing=json.dumps({"enabled": True, "tracing_provider": "langfuse"}),
+ )
+ )
+ repository = ConsoleAppRepository(session_factory=sqlite_session_factory)
+ assert repository.get_trace(context, "app") == AppTraceSettings(True, "langfuse", False, True)
+ writer = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ writer.create(
+ workspace_id="tenant", app_id="app", tracing_provider="langfuse", tracing_config={"secret": "encrypted"}
+ )
+ expected = writer.get(workspace_id="tenant", app_id="app", tracing_provider="langfuse")
+ policy = AppTraceSettings(True, "langfuse", True, False)
+ writer.set_trace(workspace_id="tenant", app_id="app", settings=policy, expected=expected)
+ assert repository.get_trace(context, "app") == policy
+ with sqlite_session_factory() as session:
+ app = session.get(App, "app")
+ assert app is not None
+ assert app.tracing is not None
+ assert json.loads(app.tracing) == {
+ "enabled": True,
+ "tracing_provider": "langfuse",
+ "capture_content": True,
+ "capture_agent": False,
+ }
+ with pytest.raises(AppTracingConfigAppNotFoundError):
+ writer.set_trace(workspace_id="other-tenant", app_id="app", settings=AppTraceSettings(), expected=None)
+ assert repository.get_trace(context, "app") == policy
diff --git a/api/tests/unit_tests/repositories/test_console_app_repository.py b/api/tests/unit_tests/repositories/test_console_app_repository.py
index ce6b3d6580b..44f12f5ccc6 100644
--- a/api/tests/unit_tests/repositories/test_console_app_repository.py
+++ b/api/tests/unit_tests/repositories/test_console_app_repository.py
@@ -18,10 +18,10 @@ from models.model import App, AppMode, AppModelConfig, AppStar, IconType, Site
from models.workflow import Workflow, WorkflowType
from repositories.app.console_repository import ConsoleAppRepository
from services.app.console_service import ConsoleAppNotFoundError
+from services.app_tracing_config_service import AppTraceSettings
from services.entities.app_entities import (
AppListParams,
AppRecord,
- AppTraceSettings,
StarredAppListParams,
UpdateAppParams,
)
@@ -160,7 +160,6 @@ def test_authorization_maintainer_requires_normal_app(
lambda repo, ctx, app_id: repo.rename(ctx, app_id, "Renamed"),
lambda repo, ctx, app_id: repo.set_starred(ctx, app_id, True),
lambda repo, ctx, app_id: repo.get_trace(ctx, app_id),
- lambda repo, ctx, app_id: repo.set_trace(ctx, app_id, AppTraceSettings(True, "langfuse")),
lambda repo, ctx, app_id: repo.get_reference(ctx, app_id),
],
)
@@ -332,14 +331,9 @@ def test_openapi_visible_app_list_only_includes_apps_with_api_enabled(
assert page.total == 1
-def test_trace_settings_commit_and_default(repository: ConsoleAppRepository, sqlite_session: Session) -> None:
+def test_trace_settings_default(repository: ConsoleAppRepository, sqlite_session: Session) -> None:
app = persist_app(sqlite_session)
assert repository.get_trace(CONTEXT, app.id) == AppTraceSettings()
- settings = AppTraceSettings(True, "langfuse")
- repository.set_trace(CONTEXT, app.id, settings)
- assert repository.get_trace(CONTEXT, app.id) == settings
- repository.set_trace(CONTEXT, app.id, AppTraceSettings())
- assert repository.get_trace(CONTEXT, app.id) == AppTraceSettings()
def test_trace_settings_preserve_stored_json_validation(
diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.py
index afd0c71be99..7e0f00317f4 100644
--- a/api/tests/unit_tests/services/app/test_console_service.py
+++ b/api/tests/unit_tests/services/app/test_console_service.py
@@ -39,6 +39,7 @@ from services.app.console_service import (
CreatorsPlatformDisabledError,
InvalidAppExportError,
)
+from services.app_tracing_config_service import AppTraceSettings
from services.entities.app_entities import (
AppCreationSettings,
AppDeletion,
@@ -47,7 +48,6 @@ from services.entities.app_entities import (
AppPage,
AppRecord,
AppReference,
- AppTraceSettings,
CopyAppParams,
CreateAppParams,
ImportedAppPackage,
@@ -254,13 +254,11 @@ class Creators:
class Tracing:
- def validate_provider(self, tracing_provider: str) -> None:
- if tracing_provider != "langfuse":
+ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:
+ del context, app_id
+ if settings.tracing_provider not in (None, "langfuse"):
raise ValueError("Invalid tracing provider")
- def require_provider_available(self, tracing_provider: str) -> None:
- self.validate_provider(tracing_provider)
-
Ports = tuple[ConsoleAppService, Apps, Access, Transfers, Creators]
@@ -464,7 +462,7 @@ def test_trace_update_checks_ownership_and_provider_without_parsing_old_settings
assert apps.calls == [("get_reference", CONTEXT, "app")]
else:
service.set_trace(CONTEXT, "app", settings)
- assert apps.calls == [("get_reference", CONTEXT, "app"), ("set_trace", CONTEXT, "app", settings)]
+ assert apps.calls == [("get_reference", CONTEXT, "app")]
class Lifecycle:
diff --git a/api/tests/unit_tests/services/test_agent_tracing_config_service.py b/api/tests/unit_tests/services/test_agent_tracing_config_service.py
new file mode 100644
index 00000000000..f1c4953bc04
--- /dev/null
+++ b/api/tests/unit_tests/services/test_agent_tracing_config_service.py
@@ -0,0 +1,88 @@
+"""Agent tracing resolves the complete owner chain before existing App operations."""
+
+from collections.abc import Callable
+
+import pytest
+from sqlalchemy.orm import Session, sessionmaker
+
+from extensions.application_services.app import AppServices
+from machinery.context import RequestContext
+from repositories.app.agent_app_repository import AgentAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.agent_tracing_config_service import AgentTracingConfigService
+from services.app.agent_app_contracts import AgentAppNotFoundError
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigService
+
+type TraceConfigArguments = tuple[()] | tuple[AppTraceSettings] | tuple[str] | tuple[str, dict[str, str]]
+type RecordedCall = tuple[str, tuple[object, ...]]
+
+CONTEXT = RequestContext("request", None, "account", "tenant")
+
+
+def invoke(service: AgentTracingConfigService, method: str) -> TraceConfigArguments:
+ settings = AppTraceSettings(True, "langfuse", capture_content=True, capture_agent=False)
+ config = {"secret_key": "masked"}
+ if method == "get_trace":
+ service.get_trace(CONTEXT, "agent")
+ return ()
+ if method == "set_trace":
+ service.set_trace(CONTEXT, "agent", settings)
+ return (settings,)
+ if method == "get":
+ service.get(CONTEXT, "agent", "langfuse")
+ return ("langfuse",)
+ if method == "delete":
+ service.delete(CONTEXT, "agent", "langfuse")
+ return ("langfuse",)
+ if method == "create":
+ service.create(CONTEXT, "agent", "langfuse", config)
+ else:
+ assert method == "update"
+ service.update(CONTEXT, "agent", "langfuse", config)
+ return ("langfuse", config)
+
+
[email protected]("method", ["get_trace", "set_trace", "get", "create", "update", "delete"])
[email protected]("valid_owner", [True, False])
+def test_agent_operations_resolve_tenant_owner_before_app_services(
+ method: str,
+ valid_owner: bool,
+ sqlite_session_factory: sessionmaker[Session],
+ app_services: AppServices,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ owners = AgentAppRepository(session_factory=sqlite_session_factory)
+ settings = app_services.console
+ configs = AppTracingConfigService(
+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory),
+ provider=OpsTraceManagerGateway(),
+ )
+ calls: list[RecordedCall] = []
+
+ def resolve(*, tenant_id: str, agent_id: str) -> str:
+ calls.append(("resolve", (tenant_id, agent_id)))
+ if not valid_owner:
+ raise AgentAppNotFoundError
+ return "backing-app"
+
+ def recorder(name: str) -> Callable[..., None]:
+ def record(*args: object, **kwargs: object) -> None:
+ assert kwargs == ({"activate": False} if name in ("create", "update") else {})
+ calls.append((name, args))
+
+ return record
+
+ monkeypatch.setattr(owners, "resolve_roster_app_id", resolve)
+ for name in ("get_trace", "set_trace"):
+ monkeypatch.setattr(settings, name, recorder(name))
+ for name in ("get", "create", "update", "delete"):
+ monkeypatch.setattr(configs, name, recorder(name))
+ service = AgentTracingConfigService(owners=owners, settings=settings, configs=configs)
+ if not valid_owner:
+ with pytest.raises(AgentAppNotFoundError):
+ invoke(service, method)
+ assert calls == [("resolve", ("tenant", "agent"))]
+ return
+ args = invoke(service, method)
+ assert calls == [("resolve", ("tenant", "agent")), (method, (CONTEXT, "backing-app", *args))]
diff --git a/api/tests/unit_tests/services/test_aliyun_tracing_config_gateway.py b/api/tests/unit_tests/services/test_aliyun_tracing_config_gateway.py
new file mode 100644
index 00000000000..b8dc0fca5e1
--- /dev/null
+++ b/api/tests/unit_tests/services/test_aliyun_tracing_config_gateway.py
@@ -0,0 +1,265 @@
+"""Real RSA/SQL/controller Aliyun configuration through a recorded HTTP boundary."""
+
+import inspect
+import json
+import traceback
+from dataclasses import dataclass, field, replace
+from typing import Never
+from urllib.parse import quote
+from uuid import UUID, uuid4
+
+import httpx
+import pytest
+from dify_trace_aliyun.aliyun_trace import AliyunDataTrace
+from flask import Flask
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+from sqlalchemy.orm import Session, sessionmaker
+
+from controllers.console.app import ops_trace as controller
+from controllers.console.app.error import InvalidTracingConfigError
+from core.helper import ssrf_proxy
+from core.ops.ops_trace_manager import OpsTraceManager
+from extensions.ext_application_services import ApplicationServices
+from machinery.context import RequestContext
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import (
+ AppTracingConfigInvalidConfigurationError,
+ AppTracingConfigRecord,
+ AppTracingConfigService,
+ AppTracingConfigVerificationFailedError,
+)
+from tests.unit_tests.model_factories import make_app
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+@dataclass
+class AliyunCase:
+ gateway: OpsTraceManagerGateway
+ service: AppTracingConfigService
+ store: SQLAlchemyAppTracingConfigRepository
+ reader: ConsoleAppRepository
+ context: RequestContext
+ app_id: str
+ verification_calls: list[tuple[str, dict[str, object]]] = field(default_factory=list)
+ verification_succeeds: bool = True
+
+ def read(self, provider: str = "aliyun") -> AppTracingConfigRecord | None:
+ return self.store.get(
+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider
+ )
+
+
+def case(workspace: str, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch) -> AliyunCase:
+ app_id = str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ tracing=json.dumps(
+ {
+ "enabled": True,
+ "tracing_provider": "langsmith",
+ "capture_content": True,
+ "capture_agent": False,
+ }
+ ),
+ )
+ )
+ gateway = OpsTraceManagerGateway()
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ old = OpsTraceManager.encrypt_tracing_config(
+ workspace, "langsmith", {"api_key": "synthetic-other", "project": "other", "endpoint": "https://other.test"}
+ )
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=old)
+ result = AliyunCase(
+ gateway,
+ AppTracingConfigService(configs=store, provider=gateway),
+ store,
+ ConsoleAppRepository(session_factory=sqlite_session_factory),
+ RequestContext("request", None, "account", workspace),
+ app_id,
+ )
+
+ def verification_boundary(method: str, url: str, **options: object) -> httpx.Response:
+ assert method == "POST"
+ content = options["content"]
+ assert isinstance(content, bytes)
+ payload = ExportTraceServiceRequest.FromString(content)
+ assert len(payload.resource_spans) == 1
+ assert len(payload.resource_spans[0].scope_spans) == 0
+ assert [attribute.key for attribute in payload.resource_spans[0].resource.attributes] == ["service.name"]
+ result.verification_calls.append((url, dict(options)))
+ status = 200 if result.verification_succeeds else 403
+ return httpx.Response(status, content=b'"success"', request=httpx.Request(method, url))
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("Configuration must not initialize a legacy exporter or fetch its console link")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", verification_boundary)
+ monkeypatch.setattr(OpsTraceManager, "check_trace_config_is_effective", forbidden)
+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)
+ monkeypatch.setattr(AliyunDataTrace, "__init__", forbidden)
+ return result
+
+
+def test_post_masked_patch_and_get_preserve_real_ciphertext_and_policy(case: AliyunCase) -> None:
+ other = case.read("langsmith")
+ submitted = {
+ "endpoint": " HTTP://TRACING-ANALYSIS-DC-QD.ALIYUNCS.COM/reverse/ ",
+ "app_name": " 中文 app_Name ",
+ "license_key": "literal%2F@synthetic",
+ }
+ original = dict(submitted)
+ case.service.create(case.context, case.app_id, "aliyun", submitted, activate=True)
+ before = case.read()
+ assert before is not None
+ assert before.tracing_config is not None
+ stored = before.tracing_config
+ assert submitted == original
+ assert stored["endpoint"] == "http://tracing-analysis-dc-qd.aliyuncs.com/reverse"
+ assert stored["app_name"] == "中文 app_Name"
+ assert stored["license_key"] != submitted["license_key"]
+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "aliyun", stored)[
+ "license_key"
+ ] == ("literal%2F@synthetic")
+ shown = case.service.get(case.context, case.app_id, "aliyun")
+ assert shown is not None
+ assert shown.tracing_config is not None
+ masked = shown.tracing_config["license_key"]
+ assert isinstance(masked, str)
+ assert "*" in masked
+ assert masked != submitted["license_key"]
+ assert shown.tracing_config["project_url"] == "https://arms.console.aliyun.com/#/llm"
+ assert len(case.verification_calls) == 1
+ case.service.update(
+ case.context,
+ case.app_id,
+ "aliyun",
+ {
+ "endpoint": "https://project.region.log.aliyuncs.com/prefix/",
+ "app_name": " second_app ",
+ "license_key": masked,
+ },
+ )
+ after = case.read()
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["license_key"] == stored["license_key"]
+ assert after.tracing_config["app_name"] == "second_app"
+ assert after.tracing_config["endpoint"] == "https://project.region.log.aliyuncs.com/prefix"
+ assert case.verification_calls[-1][0] == (
+ "https://project.region.log.aliyuncs.com/prefix/adapt_literal%252F@synthetic/api/v1/traces"
+ )
+ assert case.service.get(case.context, case.app_id, "aliyun") is not None
+ assert len(case.verification_calls) == 2
+ assert case.read("langsmith") == other
+ policy = case.reader.get_trace(case.context, case.app_id)
+ assert policy.tracing_provider == "aliyun"
+ assert policy.enabled
+ assert policy.capture_content is True
+ assert policy.capture_agent is False
+
+
[email protected]("update", [False, True])
[email protected]("failure", ["invalid-schema", "verification-failed"])
+def test_failed_save_preserves_rows_and_active_policy(case: AliyunCase, update: bool, failure: str) -> None:
+ original = {"endpoint": "https://host.test", "app_name": "original", "license_key": "synthetic-secret"}
+ if update:
+ case.service.create(case.context, case.app_id, "aliyun", original)
+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)
+ submitted = {**original, "app_name": "changed"}
+ if failure == "invalid-schema":
+ submitted["endpoint"] = "https://host.test/adapt_DO-NOT-PERSIST/api/otlp/traces"
+ expected = AppTracingConfigInvalidConfigurationError
+ else:
+ case.verification_succeeds = False
+ expected = AppTracingConfigVerificationFailedError
+ save = case.service.update if update else case.service.create
+ if update:
+ submitted["license_key"] = "******"
+ with pytest.raises(expected):
+ save(case.context, case.app_id, "aliyun", submitted, activate=True)
+ assert case.read() == before
+ assert case.read("langsmith") == other
+ assert case.reader.get_trace(case.context, case.app_id) == policy
+
+
[email protected]("cached_url", [None, "https://wrong.test/stale"])
+def test_historical_get_is_pure_even_for_a_now_invalid_ingestion_url(case: AliyunCase, cached_url: str | None) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "aliyun",
+ {
+ "endpoint": "https://host.test/adapt_old/api/otlp/traces",
+ "app_name": "historical_name",
+ "license_key": "synthetic-key",
+ },
+ )
+ if cached_url:
+ stored["project_url"] = cached_url
+ before = dict(stored)
+ result = case.gateway.present_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="aliyun", tracing_config=stored
+ )
+ assert result["project_url"] == "https://arms.console.aliyun.com/#/llm"
+ assert isinstance(result["license_key"], str)
+ assert "*" in result["license_key"]
+ assert stored == before
+ assert case.verification_calls == []
+
+
[email protected]("update", [False, True])
[email protected]("encoded", [False, True])
+def test_controller_rejects_embedded_license_without_echo_or_state_change(
+ case: AliyunCase,
+ app: Flask,
+ account_application_services: ApplicationServices,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ update: bool,
+ encoded: bool,
+) -> None:
+ old_key = "SYNTHETIC_OLD_LICENSE"
+ original = {"endpoint": "https://host.test", "app_name": "old-app", "license_key": old_key}
+ if update:
+ case.service.create(case.context, case.app_id, "aliyun", original, activate=True)
+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)
+ before_checks = len(case.verification_calls)
+ services = replace(account_application_services, app_tracing_configs=case.service)
+ monkeypatch.setattr(controller, "application_services", lambda: services)
+ key = "SYNTHETIC_PRIVATE_LICENSE@+/%2F"
+ url_key = quote(key, safe="") if encoded else key
+ payload = controller.TraceConfigPayload(
+ tracing_provider="aliyun",
+ tracing_config={
+ "endpoint": f"https://host.test/adapt_{url_key}/api/otlp/traces",
+ "app_name": "should-not-save",
+ "license_key": "******" if update else key,
+ },
+ activate=True,
+ )
+ save = inspect.unwrap(controller.TraceAppConfigApi.patch if update else controller.TraceAppConfigApi.post)
+ with app.test_request_context():
+ with pytest.raises(InvalidTracingConfigError) as caught:
+ save(controller.TraceAppConfigApi(), payload, case.context, UUID(case.app_id))
+ # The real controller exposes this fixed BaseHTTPException body. Neither
+ # Pydantic inputs nor a credential-bearing exception chain reaches it.
+ assert caught.value.data == {
+ "code": "invalid_tracing_config",
+ "message": "The tracing configuration is invalid.",
+ "status": 400,
+ }
+ rendered = json.dumps(caught.value.data) + str(caught.value) + repr(caught.value)
+ rendered += "".join(traceback.format_exception(caught.value)) + caplog.text
+ assert key not in rendered
+ assert quote(key, safe="") not in rendered
+ assert old_key not in rendered
+ assert case.read() == before
+ assert case.read("langsmith") == other
+ assert case.reader.get_trace(case.context, case.app_id) == policy
+ assert len(case.verification_calls) == before_checks
diff --git a/api/tests/unit_tests/services/test_app_tracing_atomic_operations.py b/api/tests/unit_tests/services/test_app_tracing_atomic_operations.py
new file mode 100644
index 00000000000..092bec779b9
--- /dev/null
+++ b/api/tests/unit_tests/services/test_app_tracing_atomic_operations.py
@@ -0,0 +1,225 @@
+"""Public tracing operations keep credentials and policy in one transaction."""
+
+import json
+from dataclasses import dataclass
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from machinery.context import RequestContext
+from models.model import App, TraceAppConfig
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import (
+ AppTraceSettings,
+ AppTracingConfigConflictError,
+ AppTracingConfigNotFoundError,
+ AppTracingConfigRecord,
+ AppTracingConfigService,
+ AppTracingConfigVerificationFailedError,
+)
+
+CONTEXT = RequestContext("request", None, "account", "tenant")
+
+
+@dataclass
+class Case:
+ sessions: sessionmaker[Session]
+ store: SQLAlchemyAppTracingConfigRepository
+ service: AppTracingConfigService
+ gateway: OpsTraceManagerGateway
+ reader: ConsoleAppRepository
+
+ def policy(self) -> AppTraceSettings:
+ return self.reader.get_trace(CONTEXT, "app")
+
+ def config(self, provider: str = "langfuse") -> AppTracingConfigRecord | None:
+ return self.store.get(workspace_id="tenant", app_id="app", tracing_provider=provider)
+
+
+def case(sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch) -> Case:
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ App(
+ id="app",
+ tenant_id="tenant",
+ name="Trace",
+ mode="workflow",
+ enable_site=False,
+ enable_api=False,
+ tracing=json.dumps(
+ {"enabled": False, "tracing_provider": None, "capture_content": True, "capture_agent": False}
+ ),
+ )
+ )
+ gateway = OpsTraceManagerGateway()
+
+ def prepare(*, tracing_config: dict[str, object], **_: object) -> dict[str, object]:
+ return dict(tracing_config)
+
+ monkeypatch.setattr(gateway, "prepare_new_config", prepare)
+ monkeypatch.setattr(gateway, "prepare_updated_config", prepare)
+ monkeypatch.setattr(gateway, "validate_saved_config", lambda **_: None)
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ return Case(
+ sqlite_session_factory,
+ store,
+ AppTracingConfigService(configs=store, provider=gateway),
+ gateway,
+ ConsoleAppRepository(session_factory=sqlite_session_factory),
+ )
+
+
[email protected]("activate", [False, True])
+def test_create_and_update_preserve_capture_and_explicit_activation(case: Case, activate: bool) -> None:
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"}, activate=activate)
+ assert case.policy() == AppTraceSettings(activate, "langfuse" if activate else None, True, False)
+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "updated"}, activate=True)
+ assert case.policy() == AppTraceSettings(True, "langfuse", True, False)
+ record = case.config()
+ assert record is not None
+ assert record.tracing_config == {"secret": "updated"}
+
+
+def test_save_activation_uses_capture_policy_changed_during_vendor_validation(
+ case: Case, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def prepare(**_: object) -> dict[str, object]:
+ case.service.set_trace(CONTEXT, "app", AppTraceSettings(False, None, False, True))
+ return {"secret": "validated"}
+
+ monkeypatch.setattr(case.gateway, "prepare_new_config", prepare)
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "input"}, activate=True)
+ assert case.policy() == AppTraceSettings(True, "langfuse", False, True)
+
+
[email protected]("operation", ["create", "update", "delete"])
+def test_transaction_failure_rolls_back_credentials_and_policy(case: Case, operation: str) -> None:
+ if operation != "create":
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"}, activate=True)
+ before_policy, before_record = case.policy(), case.config()
+
+ def fail_commit(session: Session) -> None:
+ session.flush()
+ raise RuntimeError("commit failed")
+
+ event.listen(case.sessions, "before_commit", fail_commit)
+ try:
+
+ def mutate() -> None:
+ if operation == "delete":
+ case.service.delete(CONTEXT, "app", "langfuse")
+ elif operation == "create":
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "new"}, activate=True)
+ else:
+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "new"}, activate=True)
+
+ with pytest.raises(RuntimeError, match="commit failed"):
+ mutate()
+ finally:
+ event.remove(case.sessions, "before_commit", fail_commit)
+ assert case.policy() == before_policy
+ assert case.config() == before_record
+
+
[email protected]("enabled", [False, True])
+def test_delete_current_clears_selection_but_other_provider_preserves_policy(case: Case, enabled: bool) -> None:
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "selected"}, activate=True)
+ case.service.create(CONTEXT, "app", "langsmith", {"secret": "other"})
+ policy = AppTraceSettings(enabled, "langfuse", True, False)
+ case.service.set_trace(CONTEXT, "app", policy)
+ case.service.delete(CONTEXT, "app", "langsmith")
+ assert case.policy() == policy
+ case.service.delete(CONTEXT, "app", "langfuse")
+ assert case.policy() == AppTraceSettings(False, None, True, False)
+ assert case.config() is None
+
+
+def test_validation_failure_writes_neither_config_nor_policy(case: Case, monkeypatch: pytest.MonkeyPatch) -> None:
+ def reject(**_: object) -> dict[str, object]:
+ raise AppTracingConfigVerificationFailedError
+
+ monkeypatch.setattr(case.gateway, "prepare_new_config", reject)
+ with pytest.raises(AppTracingConfigVerificationFailedError):
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "bad"}, activate=True)
+ assert case.config() is None
+ assert case.policy() == AppTraceSettings(False, None, True, False)
+
+
[email protected]("state", ["missing", "empty", "inactive"])
+def test_enable_requires_usable_owned_config(case: Case, state: str) -> None:
+ if state != "missing":
+ case.service.create(CONTEXT, "app", "langfuse", {} if state == "empty" else {"secret": "valid"})
+ if state == "inactive":
+ with case.sessions.begin() as session:
+ record = session.scalar(select(TraceAppConfig))
+ assert record is not None
+ record.is_active = False
+ with pytest.raises(AppTracingConfigNotFoundError):
+ case.service.set_trace(CONTEXT, "app", AppTraceSettings(True, "langfuse"))
+ assert case.policy() == AppTraceSettings(False, None, True, False)
+
+
+def test_update_rejects_credentials_replaced_while_validation_was_in_flight(
+ case: Case, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})
+
+ def prepare(**_: object) -> dict[str, object]:
+ current = case.config()
+ assert current is not None
+ case.store.update(
+ workspace_id="tenant",
+ app_id="app",
+ tracing_provider="langfuse",
+ tracing_config={"secret": "concurrent"},
+ expected=current,
+ )
+ return {"secret": "stale"}
+
+ monkeypatch.setattr(case.gateway, "prepare_updated_config", prepare)
+ with pytest.raises(AppTracingConfigConflictError):
+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "masked"}, activate=True)
+ saved = case.config()
+ assert saved is not None
+ assert saved.tracing_config == {"secret": "concurrent"}
+ assert case.policy() == AppTraceSettings(False, None, True, False)
+
+
+def test_enable_rejects_configuration_deleted_during_validation(case: Case, monkeypatch: pytest.MonkeyPatch) -> None:
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})
+
+ def validate(**_: object) -> None:
+ case.service.delete(CONTEXT, "app", "langfuse")
+
+ monkeypatch.setattr(case.gateway, "validate_saved_config", validate)
+ with pytest.raises(AppTracingConfigConflictError):
+ case.service.set_trace(CONTEXT, "app", AppTraceSettings(True, "langfuse"))
+ assert case.policy() == AppTraceSettings(False, None, True, False)
+
+
[email protected]("recreate", [False, True])
+def test_patch_rejects_configuration_deleted_during_validation(
+ case: Case, monkeypatch: pytest.MonkeyPatch, recreate: bool
+) -> None:
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})
+
+ def prepare(**_: object) -> dict[str, object]:
+ case.service.delete(CONTEXT, "app", "langfuse")
+ if recreate:
+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})
+ return {"secret": "stale"}
+
+ monkeypatch.setattr(case.gateway, "prepare_updated_config", prepare)
+ with pytest.raises(AppTracingConfigConflictError):
+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "masked"}, activate=True)
+ assert case.policy() == AppTraceSettings(False, None, True, False)
+ saved = case.config()
+ if recreate:
+ assert saved is not None
+ assert saved.tracing_config == {"secret": "original"}
+ else:
+ assert saved is None
diff --git a/api/tests/unit_tests/services/test_app_tracing_config_gateway.py b/api/tests/unit_tests/services/test_app_tracing_config_gateway.py
index df619b8e4e0..ba47e3a7dff 100644
--- a/api/tests/unit_tests/services/test_app_tracing_config_gateway.py
+++ b/api/tests/unit_tests/services/test_app_tracing_config_gateway.py
@@ -109,7 +109,7 @@ def test_prepare_new_config_applies_defaults_validates_and_encrypts(monkeypatch:
monkeypatch.setattr(
gateway_module,
"provider_config_map",
- {"arize": _provider_entry(other_keys=["endpoint", "project"])},
+ {"langsmith": _provider_entry(other_keys=["endpoint", "project"])},
)
submitted = {"api_key": "plain", "endpoint": "", "project": ""}
@@ -120,7 +120,7 @@ def test_prepare_new_config_applies_defaults_validates_and_encrypts(monkeypatch:
result = OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config=submitted,
)
@@ -131,14 +131,14 @@ def test_prepare_new_config_applies_defaults_validates_and_encrypts(monkeypatch:
}
assert submitted == {"api_key": "plain", "endpoint": "", "project": ""}
assert result == {"api_key": "encrypted", "project_url": "https://project.example.com"}
- manager.check_trace_config_is_effective.assert_called_once_with(normalized, "arize")
- manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "arize", normalized)
+ manager.check_trace_config_is_effective.assert_called_once_with(normalized, "langsmith")
+ manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "langsmith", normalized)
def test_prepare_new_config_reports_failed_verification_before_encryption(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
with patch.object(gateway_module, "OpsTraceManager") as manager:
manager.check_trace_config_is_effective.return_value = False
@@ -146,7 +146,7 @@ def test_prepare_new_config_reports_failed_verification_before_encryption(
with pytest.raises(AppTracingConfigVerificationFailedError):
OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={"api_key": "plain"},
)
@@ -156,7 +156,7 @@ def test_prepare_new_config_reports_failed_verification_before_encryption(
def test_prepare_new_config_keeps_success_when_project_url_lookup_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
with patch.object(gateway_module, "OpsTraceManager") as manager:
manager.check_trace_config_is_effective.return_value = True
@@ -165,7 +165,7 @@ def test_prepare_new_config_keeps_success_when_project_url_lookup_fails(
result = OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={"api_key": "plain"},
)
@@ -192,7 +192,7 @@ def test_prepare_new_langfuse_config_builds_project_url(monkeypatch: pytest.Monk
def test_prepare_new_config_reports_provider_check_exception_as_failed_verification(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
with patch.object(gateway_module, "OpsTraceManager") as manager:
manager.check_trace_config_is_effective.side_effect = ValueError("verification failed")
@@ -200,7 +200,7 @@ def test_prepare_new_config_reports_provider_check_exception_as_failed_verificat
with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={},
)
@@ -210,7 +210,7 @@ def test_prepare_new_config_reports_provider_check_exception_as_failed_verificat
def test_prepare_new_config_propagates_unexpected_verification_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
failure = RuntimeError("unexpected provider bug")
with patch.object(gateway_module, "OpsTraceManager") as manager:
@@ -219,7 +219,7 @@ def test_prepare_new_config_propagates_unexpected_verification_failure(
with pytest.raises(RuntimeError) as caught:
OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={},
)
@@ -230,13 +230,13 @@ def test_prepare_new_config_propagates_unexpected_verification_failure(
def test_prepare_new_config_rejects_invalid_schema_before_verification(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
with patch.object(gateway_module, "OpsTraceManager") as manager:
with pytest.raises(AppTracingConfigInvalidConfigurationError):
OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={"endpoint": {"invalid": "value"}},
)
@@ -245,7 +245,7 @@ def test_prepare_new_config_rejects_invalid_schema_before_verification(
def test_prepare_new_config_reports_encryption_failure_as_processing_error(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
failure = RuntimeError("key provider unavailable")
with patch.object(gateway_module, "OpsTraceManager") as manager:
@@ -256,7 +256,7 @@ def test_prepare_new_config_reports_encryption_failure_as_processing_error(monke
with pytest.raises(AppTracingConfigProcessingError) as caught:
OpsTraceManagerGateway().prepare_new_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={},
)
@@ -266,7 +266,7 @@ def test_prepare_new_config_reports_encryption_failure_as_processing_error(monke
def test_prepare_updated_config_preserves_masked_secret_from_current_config(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
submitted = {"api_key": "******", "project": "new-project"}
current = {"api_key": "old-encrypted", "project": "old-project"}
encrypted = {"api_key": "old-encrypted", "project": "new-project"}
@@ -276,29 +276,48 @@ def test_prepare_updated_config_preserves_masked_secret_from_current_config(
manager.encrypt_tracing_config.return_value = encrypted
manager.decrypt_tracing_config.return_value = decrypted
manager.check_trace_config_is_effective.return_value = True
+ manager.get_trace_config_project_url.return_value = "https://example.com/project/new"
result = OpsTraceManagerGateway().prepare_updated_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config=submitted,
current_tracing_config=current,
)
assert result == encrypted
+ assert result["project_url"] == "https://example.com/project/new"
assert submitted == {"api_key": "******", "project": "new-project"}
- manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "arize", submitted, current)
- manager.decrypt_tracing_config.assert_called_once_with("workspace-1", "arize", encrypted)
- manager.check_trace_config_is_effective.assert_called_once_with(decrypted, "arize")
+ manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "langsmith", submitted, current)
+ manager.decrypt_tracing_config.assert_called_once_with("workspace-1", "langsmith", encrypted)
+ manager.check_trace_config_is_effective.assert_called_once_with(decrypted, "langsmith")
+ manager.get_trace_config_project_url.assert_called_once_with(decrypted, "langsmith")
+
+
+def test_updated_destination_drops_old_project_when_lookup_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
+ with patch.object(gateway_module, "OpsTraceManager") as manager:
+ manager.encrypt_tracing_config.return_value = {"api_key": "encrypted", "project": "new-project"}
+ manager.decrypt_tracing_config.return_value = {"api_key": "plain", "project": "new-project"}
+ manager.check_trace_config_is_effective.return_value = True
+ manager.get_trace_config_project_url.side_effect = RuntimeError("metadata unavailable")
+ result = OpsTraceManagerGateway().prepare_updated_config(
+ workspace_id="workspace-1",
+ tracing_provider="langsmith",
+ tracing_config={"project": "new-project"},
+ current_tracing_config={"project_url": "https://example.com/project/old", "api_key": "encrypted"},
+ )
+ assert "project_url" not in result
def test_prepare_updated_config_validates_schema_before_encryption(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
with patch.object(gateway_module, "OpsTraceManager") as manager:
with pytest.raises(AppTracingConfigInvalidConfigurationError):
OpsTraceManagerGateway().prepare_updated_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={"endpoint": {"invalid": "value"}},
current_tracing_config={"api_key": "old-encrypted"},
)
@@ -311,7 +330,7 @@ def test_prepare_updated_config_validates_schema_before_encryption(monkeypatch:
def test_prepare_updated_config_reports_decryption_failure_as_processing_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
failure = RuntimeError("stored credential cannot be decrypted")
with patch.object(gateway_module, "OpsTraceManager") as manager:
@@ -321,7 +340,7 @@ def test_prepare_updated_config_reports_decryption_failure_as_processing_error(
with pytest.raises(AppTracingConfigProcessingError) as caught:
OpsTraceManagerGateway().prepare_updated_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={"project": "new-project"},
current_tracing_config={"api_key": "old-encrypted"},
)
@@ -333,20 +352,20 @@ def test_prepare_updated_config_reports_decryption_failure_as_processing_error(
@pytest.mark.parametrize(
("provider", "fallback_url"),
[
- ("arize", "https://app.arize.com/"),
- ("phoenix", "https://app.phoenix.arize.com/projects/"),
+ ("arize", None),
+ ("phoenix", "https://app.phoenix.arize.com/redirects/projects/default"),
("langsmith", "https://smith.langchain.com/"),
("opik", "https://www.comet.com/opik/"),
("weave", "https://wandb.ai/"),
- ("aliyun", "https://arms.console.aliyun.com/"),
+ ("aliyun", "https://arms.console.aliyun.com/#/llm"),
("tencent", "https://console.cloud.tencent.com/apm"),
- ("mlflow", "http://localhost:5000/"),
+ ("mlflow", "http://localhost:5000/#/experiments/0/traces"),
("databricks", "https://www.databricks.com/"),
],
)
def test_present_config_uses_provider_fallback_when_project_lookup_fails(
provider: str,
- fallback_url: str,
+ fallback_url: str | None,
) -> None:
with patch.object(gateway_module, "OpsTraceManager") as manager:
decrypted_config: dict[str, object] = {}
@@ -361,7 +380,11 @@ def test_present_config_uses_provider_fallback_when_project_lookup_fails(
tracing_config={"encrypted": "config"},
)
- assert result == {"project_url": fallback_url}
+ assert result == {"project_url": fallback_url}
+ if provider in {"arize", "aliyun", "mlflow", "phoenix"}:
+ # Provider defaults produce a pure link; the failing legacy lookup is
+ # deliberately not reached, even when no cached URL is present.
+ manager.get_trace_config_project_url.assert_not_called()
def test_present_langfuse_config_builds_project_url() -> None:
@@ -398,7 +421,7 @@ def test_present_config_rejects_missing_stored_config() -> None:
with pytest.raises(AppTracingConfigProcessingError, match="processing failed"):
OpsTraceManagerGateway().present_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config=None,
)
@@ -412,9 +435,40 @@ def test_present_config_reports_decryption_failure_as_processing_error() -> None
with pytest.raises(AppTracingConfigProcessingError) as caught:
OpsTraceManagerGateway().present_config(
workspace_id="workspace-1",
- tracing_provider="arize",
+ tracing_provider="langsmith",
tracing_config={"encrypted": "config"},
)
assert caught.value.__cause__ is failure
manager.obfuscated_decrypt_token.assert_not_called()
+
+
[email protected]("malformed", [False, True])
+def test_validate_saved_config_decrypts_and_validates_without_vendor_io(
+ monkeypatch: pytest.MonkeyPatch, malformed: bool
+) -> None:
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
+ with patch.object(gateway_module, "OpsTraceManager") as manager:
+ manager.decrypt_tracing_config.return_value = {"project": ["invalid-project"] if malformed else "saved-project"}
+ gateway = OpsTraceManagerGateway()
+ if malformed:
+ with pytest.raises(AppTracingConfigInvalidConfigurationError):
+ gateway.validate_saved_config(
+ workspace_id="workspace", tracing_provider="langsmith", tracing_config={"api_key": "encrypted"}
+ )
+ else:
+ gateway.validate_saved_config(
+ workspace_id="workspace", tracing_provider="langsmith", tracing_config={"api_key": "encrypted"}
+ )
+ manager.decrypt_tracing_config.assert_called_once_with("workspace", "langsmith", {"api_key": "encrypted"})
+ manager.check_trace_config_is_effective.assert_not_called()
+
+
+def test_validate_saved_config_rejects_undecryptable_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})
+ with patch.object(gateway_module, "OpsTraceManager") as manager:
+ manager.decrypt_tracing_config.side_effect = ValueError("invalid ciphertext")
+ with pytest.raises(AppTracingConfigProcessingError):
+ OpsTraceManagerGateway().validate_saved_config(
+ workspace_id="workspace", tracing_provider="langsmith", tracing_config={"api_key": "broken"}
+ )
diff --git a/api/tests/unit_tests/services/test_app_tracing_config_project_url.py b/api/tests/unit_tests/services/test_app_tracing_config_project_url.py
new file mode 100644
index 00000000000..32915c4e0cd
--- /dev/null
+++ b/api/tests/unit_tests/services/test_app_tracing_config_project_url.py
@@ -0,0 +1,238 @@
+"""Real provider config serialization must retain verified project links on update."""
+
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from uuid import uuid4
+
+import httpx
+import pytest
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.helper import ssrf_proxy
+from core.ops import ops_trace_manager
+from core.ops.ops_trace_manager import OpsTraceManager
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from machinery.context import RequestContext
+from models.enums import ConversationFromSource
+from models.model import AppMode
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from repositories.application_trace_repository import ApplicationTraceRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError
+from services.tracing.provider_dispatch import export_trace
+from tests.unit_tests.model_factories import make_app, make_message
+from tests.unit_tests.services.tracing.test_opik_configuration import AccessAPI
+
+
+@dataclass
+class ProviderBoundary:
+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)
+ effective: bool = True
+ lookup_fails: bool = False
+
+ def verify(self, config: Mapping[str, object], provider: str) -> bool:
+ self.calls.append(("verify", provider, dict(config)))
+ return self.effective
+
+ def project_url(self, config: Mapping[str, object], provider: str) -> str:
+ self.calls.append(("url", provider, dict(config)))
+ if self.lookup_fails:
+ raise RuntimeError("Synthetic project lookup failure")
+ return f"https://tracing.test/{provider}/{config['project']}"
+
+ def project_key(self, config: Mapping[str, object], provider: str) -> str:
+ self.calls.append(("key", provider, dict(config)))
+ return "verified-project"
+
+
+def boundary(monkeypatch: pytest.MonkeyPatch) -> ProviderBoundary:
+ boundary = ProviderBoundary()
+
+ def encrypt(_tenant: str, value: str) -> str:
+ return "test-cipher:" + value
+
+ def decrypt(_tenant: str, values: Sequence[str]) -> list[str]:
+ assert all(value.startswith("test-cipher:") for value in values)
+ return [value.removeprefix("test-cipher:") for value in values]
+
+ monkeypatch.setattr(ops_trace_manager, "encrypt_token", encrypt)
+ monkeypatch.setattr(ops_trace_manager, "batch_decrypt_token", decrypt)
+ monkeypatch.setattr(OpsTraceManager, "check_trace_config_is_effective", boundary.verify)
+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", boundary.project_url)
+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_key", boundary.project_key)
+ return boundary
+
+
[email protected]("provider", ["langsmith", "opik"])
[email protected]("project", ["original", "changed"])
+def test_update_resolves_project_link_using_verified_unmasked_configuration(
+ boundary: ProviderBoundary, provider: str, project: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ gateway = OpsTraceManagerGateway()
+ workspace = str(uuid4())
+ api = AccessAPI()
+ if provider == "opik":
+ monkeypatch.setattr(ssrf_proxy, "make_request", api.request)
+ target = {"workspace": "team", "url": "https://tracing.test/opik/api/"} if provider == "opik" else {}
+ current = gateway.prepare_new_config(
+ workspace_id=workspace,
+ tracing_provider=provider,
+ tracing_config={"api_key": "synthetic-key", "project": "original", **target},
+ )
+ submitted = {"api_key": "******", "project": project, "project_url": "https://untrusted.test/stale", **target}
+ updated = gateway.prepare_updated_config(
+ workspace_id=workspace, tracing_provider=provider, tracing_config=submitted, current_tracing_config=current
+ )
+
+ prefix = f"https://tracing.test/{provider}/" + ("team/redirect/projects?name=" if provider == "opik" else "")
+ assert updated["project_url"] == prefix + project
+ assert updated["api_key"] == current["api_key"] == "test-cipher:synthetic-key"
+ assert current["project_url"] == prefix + "original"
+ assert submitted["api_key"] == "******"
+ if provider == "opik":
+ assert boundary.calls == []
+ assert len(api.calls) == 2
+ assert api.calls[-1][2]["headers"] == {
+ "Authorization": "synthetic-key",
+ "Comet-Workspace": "team",
+ "Accept": "application/json",
+ }
+ else:
+ assert [call[0] for call in boundary.calls] == ["verify", "url", "verify", "url"]
+ assert boundary.calls[-1][2]["api_key"] == "synthetic-key"
+ assert boundary.calls[-1][2]["project"] == project
+
+
+def test_langfuse_update_uses_current_host_and_resolved_project_key(boundary: ProviderBoundary) -> None:
+ gateway = OpsTraceManagerGateway()
+ workspace = str(uuid4())
+ current = gateway.prepare_new_config(
+ workspace_id=workspace,
+ tracing_provider="langfuse",
+ tracing_config={"public_key": "synthetic-public", "secret_key": "synthetic-secret", "host": "https://old.test"},
+ )
+ updated = gateway.prepare_updated_config(
+ workspace_id=workspace,
+ tracing_provider="langfuse",
+ tracing_config={"public_key": "******", "secret_key": "******", "host": "https://new.test"},
+ current_tracing_config=current,
+ )
+ assert updated["project_url"] == "https://new.test/project/verified-project"
+ assert current["project_url"] == "https://old.test/project/verified-project"
+ assert boundary.calls[-1][0] == "key"
+ assert boundary.calls[-1][2]["secret_key"] == "synthetic-secret"
+
+
+def test_failed_optional_link_lookup_does_not_reuse_stale_project_url(boundary: ProviderBoundary) -> None:
+ gateway = OpsTraceManagerGateway()
+ workspace = str(uuid4())
+ current = gateway.prepare_new_config(
+ workspace_id=workspace,
+ tracing_provider="langsmith",
+ tracing_config={"api_key": "synthetic-key", "project": "original"},
+ )
+ boundary.lookup_fails = True
+ updated = gateway.prepare_updated_config(
+ workspace_id=workspace,
+ tracing_provider="langsmith",
+ tracing_config={"api_key": "******", "project": "changed"},
+ current_tracing_config=current,
+ )
+ assert updated["project"] == "changed"
+ assert "project_url" not in updated
+ assert boundary.calls[-1][0] == "url"
+
+
+def test_invalid_updated_configuration_never_resolves_a_project_url(boundary: ProviderBoundary) -> None:
+ boundary.effective = False
+ with pytest.raises(AppTracingConfigVerificationFailedError):
+ OpsTraceManagerGateway().prepare_updated_config(
+ workspace_id=str(uuid4()),
+ tracing_provider="langsmith",
+ tracing_config={"api_key": "synthetic-key", "project": "changed"},
+ current_tracing_config=None,
+ )
+ assert [call[0] for call in boundary.calls] == ["verify"]
+
+
+def test_updated_project_url_is_frozen_and_used_by_real_langsmith_serializer(
+ boundary: ProviderBoundary,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ workspace, app_id, message_id = (str(uuid4()) for _ in range(3))
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ mode=AppMode.AGENT,
+ tracing=json.dumps({"enabled": True, "tracing_provider": "langsmith"}),
+ )
+ )
+ session.add(
+ make_message(
+ message_id=message_id,
+ app_id=app_id,
+ inputs={},
+ query="test input",
+ message={},
+ answer="test output",
+ message_unit_price=Decimal(0),
+ answer_unit_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.CONSOLE,
+ )
+ )
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ configs = AppTracingConfigService(configs=store, provider=OpsTraceManagerGateway())
+ context = RequestContext("request", None, "account", workspace)
+ configs.create(context, app_id, "langsmith", {"api_key": "synthetic-key", "project": "original"})
+ configs.update(context, app_id, "langsmith", {"api_key": "******", "project": "changed"})
+ repository = ApplicationTraceRepository(sqlite_session_factory)
+ run = repository.start(
+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id, session_id="test-session"
+ )
+ assert run is not None
+ assert run.provider_config["project_url"] == "https://tracing.test/langsmith/changed"
+ configs.update(context, app_id, "langsmith", {"api_key": "******", "project": "later"})
+ frozen = repository.source_run(tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id)
+ assert frozen is not None
+ assert frozen.provider_config["project_url"] == "https://tracing.test/langsmith/changed"
+
+ requests: list[str] = []
+
+ def send(method: str, url: str, **_kwargs: object) -> httpx.Response:
+ requests.append(url)
+ return httpx.Response(200, request=httpx.Request(method, url))
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ config = dict(frozen.provider_config)
+ config.update(OpsTraceManager.decrypt_tracing_config(workspace, "langsmith", frozen.provider_config))
+ start = datetime(2026, 10, 16, tzinfo=UTC)
+ trace = CanonicalTrace(
+ trace_id=frozen.trace_id,
+ session_id=frozen.session_id,
+ root_span_id=frozen.root_span_id,
+ spans=(
+ CanonicalSpan(
+ id=frozen.root_span_id,
+ name="Agent",
+ kind=CanonicalSpanKind.AGENT,
+ parent_id=None,
+ status=CanonicalSpanStatus.OK,
+ start_time=start,
+ end_time=start + timedelta(seconds=1),
+ ),
+ ),
+ )
+ result = export_trace("langsmith", config, trace)
+ assert result.status == "accepted"
+ assert result.trace_url == f"https://tracing.test/langsmith/changed/r/{result.trace_id}"
+ assert result.span_urls[frozen.root_span_id] == result.trace_url
+ assert len(requests) == 1
+ assert boundary.calls[-1][2]["project"] == "later"
diff --git a/api/tests/unit_tests/services/test_app_tracing_config_service.py b/api/tests/unit_tests/services/test_app_tracing_config_service.py
index 208b5d4a972..8325cf96bf1 100644
--- a/api/tests/unit_tests/services/test_app_tracing_config_service.py
+++ b/api/tests/unit_tests/services/test_app_tracing_config_service.py
@@ -114,6 +114,7 @@ def test_create_calls_provider_between_separate_store_operations() -> None:
app_id="app-1",
tracing_provider="arize",
tracing_config={"api_key": "encrypted"},
+ activate=False,
)
@@ -170,6 +171,8 @@ def test_update_validates_reads_prepares_and_writes_in_order() -> None:
app_id="app-1",
tracing_provider="arize",
tracing_config={"api_key": "new-encrypted"},
+ expected=current,
+ activate=False,
)
diff --git a/api/tests/unit_tests/services/test_arize_tracing_project_url.py b/api/tests/unit_tests/services/test_arize_tracing_project_url.py
new file mode 100644
index 00000000000..f11cf7421a2
--- /dev/null
+++ b/api/tests/unit_tests/services/test_arize_tracing_project_url.py
@@ -0,0 +1,292 @@
+"""Arize project links use verified metadata without changing ingestion permissions."""
+
+import json
+from collections import deque
+from copy import deepcopy
+from decimal import Decimal
+from typing import Never
+from uuid import uuid4
+
+import httpx
+import pytest
+from dify_trace_arize_phoenix import arize_configuration
+from dify_trace_arize_phoenix.arize_configuration import DESTINATION_KEY
+from dify_trace_arize_phoenix.config import ArizeConfig
+from sqlalchemy import select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.helper import ssrf_proxy
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from models.application_trace import ApplicationTraceRun
+from models.enums import ConversationFromSource
+from models.model import AppMode
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from repositories.application_trace_repository import ApplicationTraceRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError
+from services.tracing.provider_dispatch import frozen_project_url
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.model_factories import make_app, make_message
+from tests.unit_tests.services.tracing.test_arize_configuration import (
+ DESTINATION,
+ SPACE,
+ URL,
+ VALUES,
+ page,
+ project,
+ response,
+ space,
+)
+from tests.unit_tests.services.tracing.test_arize_configuration import (
+ calls as calls, # noqa: PLC0414 - Re-export the HTTP boundary fixture for pytest.
+)
+from tests.unit_tests.services.tracing.test_arize_configuration import (
+ responses as responses, # noqa: PLC0414 - Re-export the HTTP boundary fixture for pytest.
+)
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+def verified(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]:
+ calls: list[dict[str, object]] = []
+
+ def verify(config: ArizeConfig) -> None:
+ calls.append(config.model_dump())
+
+ monkeypatch.setattr(arize_configuration, "verify_configuration", verify)
+ return calls
+
+
+def test_legacy_arize_get_hides_unverified_redirect_without_network_or_rewriting(
+ workspace: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "arize",
+ {"api_key": "synthetic-ingestion-key", "space_id": "U3BhY2U6dGVzdA==", "project": "original"},
+ )
+ stored["project_url"] = "https://app.arize.com/?redirect_project_name=original"
+ untouched = deepcopy(stored)
+ calls: list[str] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ calls.append("legacy or network")
+ raise AssertionError("Configuration GET must not initialize a tracer or resolve vendor metadata")
+
+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)
+ monkeypatch.setattr(ssrf_proxy, "get", forbidden)
+ shown = OpsTraceManagerGateway().present_config(
+ workspace_id=workspace, tracing_provider="arize", tracing_config=stored
+ )
+
+ assert shown["project_url"] is None
+ assert calls == []
+ assert stored == untouched
+
+
+def test_real_rsa_create_masked_update_get_and_frozen_history(
+ workspace: str,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+ verified: list[dict[str, object]],
+ calls: list[tuple[str, dict[str, object]]],
+ responses: deque[httpx.Response | Exception],
+) -> None:
+ app_id, message_id = str(uuid4()), str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ mode=AppMode.AGENT,
+ tracing=json.dumps({"enabled": True, "tracing_provider": "arize"}),
+ )
+ )
+ session.add(
+ make_message(
+ message_id=message_id,
+ app_id=app_id,
+ inputs={},
+ query="test",
+ message={},
+ answer="ok",
+ message_unit_price=Decimal(0),
+ answer_unit_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.CONSOLE,
+ )
+ )
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ gateway = OpsTraceManagerGateway()
+ service = AppTracingConfigService(configs=store, provider=gateway)
+ context = RequestContext("request", None, "account", workspace)
+ legacy = OpsTraceManager.encrypt_tracing_config(workspace, "arize", VALUES)
+ legacy["project_url"] = "https://app.arize.com/?redirect_project_name=exact"
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="arize", tracing_config=legacy)
+ repository = ApplicationTraceRepository(sqlite_session_factory)
+ run = repository.start(
+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id, session_id="session"
+ )
+ assert run is not None
+ with sqlite_session_factory.begin() as session:
+ accepted = session.get(ApplicationTraceRun, run.id)
+ assert accepted is not None
+ accepted.state = "accepted"
+ accepted.attempts = 1
+ with sqlite_session_factory() as session:
+ accepted_row = deepcopy(
+ dict(
+ session.execute(select(ApplicationTraceRun.__table__).where(ApplicationTraceRun.id == run.id))
+ .mappings()
+ .one()
+ )
+ )
+ frozen_before = deepcopy(run.provider_config)
+
+ # Both legacy secret fields are really encrypted, rather than identity stubs.
+ assert legacy["api_key"] != VALUES["api_key"]
+ assert legacy["space_id"] != SPACE
+ responses.extend([space(), page(project())])
+ submitted = {
+ **VALUES,
+ "api_key": "******",
+ "space_id": "******",
+ DESTINATION_KEY: {"organization_id": "forged", "project_id": "forged"},
+ "project_url": "https://untrusted.test/forged",
+ }
+ untouched = deepcopy(submitted)
+ service.update(context, app_id, "arize", submitted)
+ current = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="arize")
+ assert current is not None
+ assert current.tracing_config is not None
+ saved = current.tracing_config
+ assert saved["api_key"] == legacy["api_key"]
+ assert saved["space_id"] == legacy["space_id"]
+ assert saved[DESTINATION_KEY] == {**DESTINATION, "source_digest": saved[DESTINATION_KEY]["source_digest"]}
+ assert saved["project_url"] == URL
+ assert submitted == untouched
+ assert len(calls) == 2
+ assert verified[-1]["api_key"] == VALUES["api_key"]
+ assert verified[-1]["space_id"] == SPACE
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("Reading current configuration or frozen history must not use vendor I/O")
+
+ monkeypatch.setattr(ssrf_proxy, "get", forbidden)
+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)
+ shown = service.get(context, app_id, "arize")
+ assert shown is not None
+ assert shown.tracing_config is not None
+ assert shown.tracing_config["project_url"] == URL
+ assert DESTINATION_KEY not in shown.tracing_config
+ frozen = repository.source_run(tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id)
+ assert frozen is not None
+ assert (frozen.state, frozen.attempts, frozen.provider_config) == ("accepted", 1, frozen_before)
+ assert frozen_project_url("arize", frozen.provider_config) is None
+ assert frozen.provider_config == frozen_before
+ monkeypatch.setattr(OpsTraceManager, "decrypt_tracing_config", forbidden)
+ status = ApplicationTracingService(repository).status(
+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id
+ )
+ assert status["project_url"] is None
+ assert status["status"] == "accepted"
+ with sqlite_session_factory() as session:
+ after_row = dict(
+ session.execute(select(ApplicationTraceRun.__table__).where(ApplicationTraceRun.id == run.id))
+ .mappings()
+ .one()
+ )
+ assert after_row == accepted_row
+
+ # A future source freezes the verified destination; it never repairs the old run.
+ later_id = str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_message(
+ message_id=later_id,
+ app_id=app_id,
+ inputs={},
+ query="later",
+ message={},
+ answer="ok",
+ message_unit_price=Decimal(0),
+ answer_unit_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.CONSOLE,
+ )
+ )
+ later = repository.start(
+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=later_id, session_id="session"
+ )
+ assert later is not None
+ assert frozen_project_url("arize", later.provider_config) == URL
+
+
[email protected]("status", [401, 403, 404, 500])
+def test_ingest_verified_key_without_metadata_read_permission_still_saves(
+ workspace: str,
+ verified: list[dict[str, object]],
+ calls: list[tuple[str, dict[str, object]]],
+ responses: deque[httpx.Response | Exception],
+ status: int,
+) -> None:
+ responses.append(response({"message": "private-response"}, status))
+ saved = OpsTraceManagerGateway().prepare_new_config(
+ workspace_id=workspace, tracing_provider="arize", tracing_config=VALUES
+ )
+ assert "project_url" not in saved
+ assert DESTINATION_KEY not in saved
+ assert OpsTraceManager.decrypt_tracing_config(workspace, "arize", saved)["api_key"] == VALUES["api_key"]
+ assert len(verified) == len(calls) == 1
+
+
+def test_changed_destination_cannot_keep_old_or_client_supplied_cache(
+ workspace: str,
+ verified: list[dict[str, object]],
+ calls: list[tuple[str, dict[str, object]]],
+ responses: deque[httpx.Response | Exception],
+) -> None:
+ gateway = OpsTraceManagerGateway()
+ responses.extend([space(), page(project())])
+ before = gateway.prepare_new_config(workspace_id=workspace, tracing_provider="arize", tracing_config=VALUES)
+ assert before["project_url"] == URL
+ responses.append(response({}, 403))
+ after = gateway.prepare_updated_config(
+ workspace_id=workspace,
+ tracing_provider="arize",
+ current_tracing_config=before,
+ tracing_config={
+ **VALUES,
+ "api_key": "******",
+ "space_id": "******",
+ "project": "later",
+ DESTINATION_KEY: before[DESTINATION_KEY],
+ "project_url": URL,
+ },
+ )
+ assert after["project"] == "later"
+ assert after["api_key"] == before["api_key"]
+ assert after["space_id"] == before["space_id"]
+ assert "project_url" not in after
+ assert DESTINATION_KEY not in after
+ assert before["project_url"] == URL
+ assert len(calls) == 3
+ assert len(verified) == 2
+
+
+def test_failed_original_ingestion_verification_never_resolves_metadata(
+ workspace: str,
+ monkeypatch: pytest.MonkeyPatch,
+ calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ def rejected(_config: ArizeConfig) -> Never:
+ raise arize_configuration.ArizeConfigurationError("permission_denied")
+
+ monkeypatch.setattr(arize_configuration, "verify_configuration", rejected)
+ with pytest.raises(AppTracingConfigVerificationFailedError):
+ OpsTraceManagerGateway().prepare_new_config(
+ workspace_id=workspace, tracing_provider="arize", tracing_config=VALUES
+ )
+ assert calls == []
diff --git a/api/tests/unit_tests/services/test_databricks_tracing_config_gateway.py b/api/tests/unit_tests/services/test_databricks_tracing_config_gateway.py
new file mode 100644
index 00000000000..4704659408a
--- /dev/null
+++ b/api/tests/unit_tests/services/test_databricks_tracing_config_gateway.py
@@ -0,0 +1,218 @@
+"""Databricks config persistence must not initialize a process-global MLflow client."""
+
+from collections.abc import Callable, Iterator, Sequence
+from dataclasses import dataclass, field
+from typing import Never
+from uuid import uuid4
+
+import mlflow
+import pytest
+from dify_trace_mlflow.configuration import DatabricksConfigurationError
+from dify_trace_mlflow.mlflow_trace import MLflowDataTrace
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.helper import ssrf_proxy
+from core.ops import ops_trace_manager
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_databricks_configuration import ExperimentAPI, response
+
+
+@dataclass
+class GatewayEnvironment:
+ gateway: OpsTraceManagerGateway = field(default_factory=OpsTraceManagerGateway)
+ api: ExperimentAPI = field(default_factory=ExperimentAPI)
+ forbidden_calls: list[str] = field(default_factory=list)
+
+
+def environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[GatewayEnvironment]:
+ environment = GatewayEnvironment()
+
+ def encrypt(_tenant: str, value: str) -> str:
+ return "test-cipher:" + value
+
+ def decrypt(_tenant: str, values: Sequence[str]) -> list[str]:
+ assert all(value.startswith("test-cipher:") for value in values)
+ return [value.removeprefix("test-cipher:") for value in values]
+
+ def forbidden(name: str) -> Callable[..., Never]:
+ def block(*_args: object, **_kwargs: object) -> Never:
+ environment.forbidden_calls.append(name)
+ raise AssertionError("Configuration must not initialize a stateful exporter")
+
+ return block
+
+ monkeypatch.setattr(ops_trace_manager, "encrypt_token", encrypt)
+ monkeypatch.setattr(ops_trace_manager, "batch_decrypt_token", decrypt)
+ monkeypatch.setattr(ops_trace_manager, "obfuscated_token", lambda value: "******" if value else value)
+ monkeypatch.setattr(ssrf_proxy, "get", environment.api.get)
+ monkeypatch.setattr(ssrf_proxy, "post", environment.api.post)
+ monkeypatch.setattr(MLflowDataTrace, "__init__", forbidden("legacy_constructor"))
+ monkeypatch.setattr(mlflow, "set_tracking_uri", forbidden("tracking_uri"))
+ monkeypatch.setattr(mlflow, "set_experiment", forbidden("experiment"))
+ monkeypatch.setattr(mlflow, "search_experiments", forbidden("search"))
+ monkeypatch.setattr(mlflow.config, "enable_async_logging", forbidden("async_logging"))
+ yield environment
+ assert environment.forbidden_calls == []
+
+
[email protected]("oauth", [False, True])
+def test_create_and_patch_persist_verified_target_without_replacing_masked_credentials(
+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], oauth: bool
+) -> None:
+ workspace, app_id = str(uuid4()), str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id=app_id, tenant_id=workspace))
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ service = AppTracingConfigService(configs=store, provider=environment.gateway)
+ context = RequestContext("request", None, "account", workspace)
+ credentials = (
+ {"client_id": "synthetic-client", "client_secret": "synthetic-secret"}
+ if oauth
+ else {"personal_access_token": "synthetic-pat"}
+ )
+ secret_field = "client_secret" if oauth else "personal_access_token"
+ service.create(
+ context, app_id, "databricks", {"host": " https://first.test/ ", "experiment_id": "00012", **credentials}
+ )
+ before = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")
+ assert before is not None
+ assert before.tracing_config is not None
+ assert before.tracing_config["host"] == "https://first.test"
+ assert before.tracing_config["experiment_id"] == "12"
+ service.update(
+ context,
+ app_id,
+ "databricks",
+ {
+ "host": " https://second.test/ ",
+ "experiment_id": "00034",
+ **credentials,
+ secret_field: "******",
+ },
+ )
+ after = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["host"] == "https://second.test"
+ assert after.tracing_config["experiment_id"] == "34"
+ assert after.tracing_config[secret_field] == before.tracing_config[secret_field]
+ assert after.tracing_config["project_url"] == "https://second.test/ml/experiments/34/traces"
+ assert before.tracing_config["project_url"] == "https://first.test/ml/experiments/12/traces"
+ before_get = len(environment.api.calls)
+ presented = service.get(context, app_id, "databricks")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config[secret_field] == "******"
+ assert presented.tracing_config["project_url"] == after.tracing_config["project_url"]
+ environment.gateway.validate_saved_config(
+ workspace_id=workspace, tracing_provider="databricks", tracing_config=after.tracing_config
+ )
+ assert len(environment.api.calls) == before_get
+
+
[email protected]("saved_url", [None, "https://stale.test/old-project"])
+def test_get_is_pure_for_historical_and_current_configs(environment: GatewayEnvironment, saved_url: str | None) -> None:
+ workspace = str(uuid4())
+ encrypted = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "databricks",
+ {"host": "https://workspace.test/", "experiment_id": "12", "personal_access_token": "synthetic-pat"},
+ )
+ if saved_url is not None:
+ encrypted["project_url"] = saved_url
+ result = environment.gateway.present_config(
+ workspace_id=workspace, tracing_provider="databricks", tracing_config=encrypted
+ )
+ assert result["project_url"] == "https://workspace.test/ml/experiments/12/traces"
+ assert result["personal_access_token"] == "******"
+ assert environment.api.calls == []
+
+
+def test_invalid_historical_host_uses_safe_pure_fallback(environment: GatewayEnvironment) -> None:
+ workspace = str(uuid4())
+ encrypted = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "databricks",
+ {
+ "host": "https://user:[email protected]",
+ "experiment_id": "12",
+ "personal_access_token": "synthetic-pat",
+ },
+ )
+ result = environment.gateway.present_config(
+ workspace_id=workspace, tracing_provider="databricks", tracing_config=encrypted
+ )
+ assert result["project_url"] == "https://www.databricks.com/"
+ assert environment.api.calls == []
+
+
+def test_failed_patch_keeps_original_destination_and_logs_only_safe_code(
+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], caplog: pytest.LogCaptureFixture
+) -> None:
+ workspace, app_id = str(uuid4()), str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id=app_id, tenant_id=workspace))
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ service = AppTracingConfigService(configs=store, provider=environment.gateway)
+ context = RequestContext("request", None, "account", workspace)
+ original = {
+ "host": "https://workspace.test",
+ "experiment_id": "12",
+ "personal_access_token": "synthetic-private-pat",
+ }
+ service.create(context, app_id, "databricks", original)
+ before = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")
+ assert before is not None
+ environment.api.override = lambda _method, url, _options: response(url, {"message": "private-vendor-detail"}, 403)
+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
+ service.update(
+ context, app_id, "databricks", {**original, "experiment_id": "34", "personal_access_token": "******"}
+ )
+ assert isinstance(caught.value.__cause__, DatabricksConfigurationError)
+ assert caught.value.__cause__.code == "permission_denied"
+ after = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")
+ assert after is not None
+ assert after.tracing_config == before.tracing_config
+ assert "permission_denied" in caplog.text
+ assert "private-vendor-detail" not in caplog.text
+ assert "synthetic-private-pat" not in caplog.text
+
+
[email protected]("start_with_oauth", [False, True])
+def test_switching_auth_clears_unselected_credentials_in_persistence_and_requests(
+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], start_with_oauth: bool
+) -> None:
+ workspace, app_id = str(uuid4()), str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id=app_id, tenant_id=workspace))
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ service = AppTracingConfigService(configs=store, provider=environment.gateway)
+ context = RequestContext("request", None, "account", workspace)
+ pat = {"personal_access_token": "synthetic-pat", "client_id": None, "client_secret": None}
+ oauth = {"personal_access_token": None, "client_id": "synthetic-client", "client_secret": "synthetic-secret"}
+ target = {"host": "https://workspace.test", "experiment_id": "12"}
+ service.create(context, app_id, "databricks", {**target, **(oauth if start_with_oauth else pat)})
+ environment.api.calls.clear()
+ service.update(context, app_id, "databricks", {**target, **(pat if start_with_oauth else oauth)})
+ stored = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")
+ assert stored is not None
+ assert stored.tracing_config is not None
+ assert stored.tracing_config["client_secret" if start_with_oauth else "personal_access_token"] is None
+ assert [method for method, _url, _options in environment.api.calls] == (
+ ["GET"] if start_with_oauth else ["POST", "GET"]
+ )
+ expected_token = "synthetic-pat" if start_with_oauth else "synthetic-oauth-token"
+ assert environment.api.calls[-1][2]["headers"] == {
+ "Authorization": f"Bearer {expected_token}",
+ "Accept": "application/json",
+ }
+ presented = service.get(context, app_id, "databricks")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config["client_secret" if start_with_oauth else "personal_access_token"] is None
diff --git a/api/tests/unit_tests/services/test_feature_query_service.py b/api/tests/unit_tests/services/test_feature_query_service.py
index d66340e9a03..08ab5f740bb 100644
--- a/api/tests/unit_tests/services/test_feature_query_service.py
+++ b/api/tests/unit_tests/services/test_feature_query_service.py
@@ -43,7 +43,9 @@ def test_workspace_queries_use_workspace_from_request_context() -> None:
def test_deployment_queries_delegate_without_request_context() -> None:
gateway = create_autospec(FeatureQueryGateway, instance=True, spec_set=True)
- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)
+ system_features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY
+ )
license_model = LicenseModel()
gateway.get_public_system_features.return_value = system_features
gateway.get_license.return_value = license_model
@@ -83,6 +85,7 @@ def test_workspace_id_queries_delegate_without_request_context() -> None:
def test_enterprise_license_admission(status: LicenseStatus, expected: bool) -> None:
gateway = create_autospec(FeatureQueryGateway, instance=True, spec_set=True)
gateway.get_public_system_features.return_value = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False,
deployment_edition=DeploymentEdition.ENTERPRISE,
license={"status": status},
)
diff --git a/api/tests/unit_tests/services/test_feature_service_gateway.py b/api/tests/unit_tests/services/test_feature_service_gateway.py
index c4f8a2efcfd..fbfb02a1e56 100644
--- a/api/tests/unit_tests/services/test_feature_service_gateway.py
+++ b/api/tests/unit_tests/services/test_feature_service_gateway.py
@@ -8,7 +8,9 @@ from services.system_feature_service import SystemFeatureService
def test_public_system_features_delegate_to_existing_service(mocker: MockerFixture) -> None:
- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)
+ system_features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY
+ )
get_system_features = mocker.patch.object(
SystemFeatureService,
"get_public_system_features",
diff --git a/api/tests/unit_tests/services/test_mlflow_tracing_config_gateway.py b/api/tests/unit_tests/services/test_mlflow_tracing_config_gateway.py
new file mode 100644
index 00000000000..bd024bc1677
--- /dev/null
+++ b/api/tests/unit_tests/services/test_mlflow_tracing_config_gateway.py
@@ -0,0 +1,273 @@
+"""MLflow config persistence uses real RSA/SQLite without constructing an exporter."""
+
+import json
+from dataclasses import dataclass
+from uuid import uuid4
+
+import pytest
+from dify_trace_mlflow.oss_configuration import MLflowConfigurationError
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import (
+ AppTracingConfigInvalidConfigurationError,
+ AppTracingConfigRecord,
+ AppTracingConfigService,
+ AppTracingConfigVerificationFailedError,
+)
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_databricks_configuration import ExperimentAPI, response
+from tests.unit_tests.services.tracing.test_mlflow_configuration import (
+ experiment_api as experiment_api, # noqa: PLC0414 - Re-export pytest fixtures only in this module.
+)
+from tests.unit_tests.services.tracing.test_mlflow_configuration import (
+ isolated_sdk_state as isolated_sdk_state, # noqa: PLC0414 - Re-export pytest fixtures only in this module.
+)
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+@dataclass
+class ConfigurationCase:
+ api: ExperimentAPI
+ gateway: OpsTraceManagerGateway
+ service: AppTracingConfigService
+ store: SQLAlchemyAppTracingConfigRepository
+ reader: ConsoleAppRepository
+ context: RequestContext
+ app_id: str
+
+ def read(self, provider: str = "mlflow") -> AppTracingConfigRecord | None:
+ return self.store.get(
+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider
+ )
+
+
+def case(
+ workspace: str, sqlite_session_factory: sessionmaker[Session], experiment_api: ExperimentAPI
+) -> ConfigurationCase:
+ app_id = str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ tracing=json.dumps(
+ {"enabled": True, "tracing_provider": "langsmith", "capture_content": True, "capture_agent": False}
+ ),
+ )
+ )
+ gateway = OpsTraceManagerGateway()
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ previous = OpsTraceManager.encrypt_tracing_config(
+ workspace, "langsmith", {"api_key": "synthetic-other-key", "project": "other", "endpoint": "https://other.test"}
+ )
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=previous)
+ return ConfigurationCase(
+ experiment_api,
+ gateway,
+ AppTracingConfigService(configs=store, provider=gateway),
+ store,
+ ConsoleAppRepository(session_factory=sqlite_session_factory),
+ RequestContext("request", None, "account", workspace),
+ app_id,
+ )
+
+
+def test_post_masked_patch_and_get_preserve_ciphertext_and_exact_destination(case: ConfigurationCase) -> None:
+ original = case.read("langsmith")
+ submitted = {
+ "tracking_uri": " HTTP://FIRST.Example:5000/team/ ",
+ "experiment_id": "00012",
+ "username": "test-user",
+ "password": "synthetic-password",
+ }
+ untouched = dict(submitted)
+ case.service.create(case.context, case.app_id, "mlflow", submitted, activate=True)
+ before = case.read()
+ assert before is not None
+ assert before.tracing_config is not None
+ config = before.tracing_config
+ assert config["tracking_uri"] == "http://first.example:5000/team"
+ assert config["experiment_id"] == "12"
+ assert config["password"] != "synthetic-password"
+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "mlflow", config)["password"] == (
+ "synthetic-password"
+ )
+ assert submitted == untouched
+ shown = case.service.get(case.context, case.app_id, "mlflow")
+ assert shown is not None
+ assert shown.tracing_config is not None
+ masked = shown.tracing_config["password"]
+ assert isinstance(masked, str)
+ assert "*" in masked
+ assert masked != "synthetic-password"
+ assert len(case.api.calls) == 1
+ case.service.update(
+ case.context,
+ case.app_id,
+ "mlflow",
+ {
+ "tracking_uri": "https://SECOND.test/new-prefix/",
+ "experiment_id": "00034",
+ "username": "test-user",
+ "password": masked,
+ },
+ )
+ after = case.read()
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["password"] == config["password"]
+ assert after.tracing_config["tracking_uri"] == "https://second.test/new-prefix"
+ assert after.tracing_config["experiment_id"] == "34"
+ assert after.tracing_config["project_url"] == "https://second.test/new-prefix/#/experiments/34/traces"
+ assert len(case.api.calls) == 2
+ assert case.api.calls[-1][1] == "https://second.test/new-prefix/api/2.0/mlflow/experiments/get"
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=after.tracing_config
+ )
+ assert case.service.get(case.context, case.app_id, "mlflow") is not None
+ assert len(case.api.calls) == 2
+ assert case.read("langsmith") == original
+ policy = case.reader.get_trace(case.context, case.app_id)
+ assert policy.enabled
+ assert policy.tracing_provider == "mlflow"
+ assert policy.capture_content is True
+ assert policy.capture_agent is False
+
+
[email protected]("clear_with_null", [False, True])
+def test_basic_to_anonymous_and_back_does_not_reuse_credentials(case: ConfigurationCase, clear_with_null: bool) -> None:
+ destination = {"tracking_uri": "https://same.test/prefix", "experiment_id": "12"}
+ case.service.create(
+ case.context, case.app_id, "mlflow", {**destination, "username": "first", "password": "first-secret"}
+ )
+ empty = None if clear_with_null else ""
+ case.service.update(case.context, case.app_id, "mlflow", {**destination, "username": empty, "password": empty})
+ assert case.api.calls[-1][2]["headers"] == {
+ "Accept": "application/json",
+ "Accept-Encoding": "identity",
+ "Cookie": "",
+ }
+ stored = case.read()
+ assert stored is not None
+ assert stored.tracing_config is not None
+ assert (
+ OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "mlflow", stored.tracing_config)[
+ "password"
+ ]
+ == empty
+ )
+ case.service.update(
+ case.context, case.app_id, "mlflow", {**destination, "username": "second", "password": "second-secret"}
+ )
+ assert case.api.calls[-1][2]["headers"] != case.api.calls[0][2]["headers"]
+
+
+def test_failed_patch_keeps_config_policy_and_other_provider(
+ case: ConfigurationCase, caplog: pytest.LogCaptureFixture
+) -> None:
+ original = {
+ "tracking_uri": "https://first.test",
+ "experiment_id": "12",
+ "username": "user",
+ "password": "private-password",
+ }
+ case.service.create(case.context, case.app_id, "mlflow", original)
+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)
+ case.api.override = lambda _method, url, _options: response(url, {"message": "private-vendor-response"}, 403)
+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
+ case.service.update(
+ case.context,
+ case.app_id,
+ "mlflow",
+ {**original, "tracking_uri": "https://second.test", "password": "******"},
+ activate=True,
+ )
+ assert isinstance(caught.value.__cause__, MLflowConfigurationError)
+ assert caught.value.__cause__.code == "permission_denied"
+ assert case.read() == before
+ assert case.read("langsmith") == other
+ assert case.reader.get_trace(case.context, case.app_id) == policy
+ assert "permission_denied" in caplog.text
+ assert "private-vendor-response" not in caplog.text
+ assert "private-password" not in caplog.text
+
+
+def test_failed_auth_pair_patch_does_not_erase_a_password(case: ConfigurationCase) -> None:
+ original = {"tracking_uri": "https://first.test", "experiment_id": "12", "username": "user", "password": "password"}
+ case.service.create(case.context, case.app_id, "mlflow", original)
+ before = case.read()
+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
+ case.service.update(case.context, case.app_id, "mlflow", {**original, "password": None})
+ assert isinstance(caught.value.__cause__, MLflowConfigurationError)
+ assert caught.value.__cause__.code == "incomplete_basic_auth"
+ assert len(case.api.calls) == 1
+ assert case.read() == before
+
+
[email protected]("saved_url", [None, "https://stale.test/wrong-experiment"])
+def test_historical_get_is_pure_even_with_partial_auth(case: ConfigurationCase, saved_url: str | None) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "mlflow",
+ {
+ "tracking_uri": "https://server.test/prefix/",
+ "experiment_id": "00012",
+ "username": "historical-user",
+ "password": None,
+ },
+ )
+ if saved_url is not None:
+ stored["project_url"] = saved_url
+ shown = case.gateway.present_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=stored
+ )
+ assert shown["project_url"] == "https://server.test/prefix/#/experiments/12/traces"
+ assert shown["password"] is None
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=stored
+ )
+ assert case.api.calls == []
+
+
+def test_invalid_historical_url_has_no_fabricated_link_or_network(case: ConfigurationCase) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "mlflow",
+ {
+ "tracking_uri": "https://user:[email protected]?token=private",
+ "experiment_id": "12",
+ "username": None,
+ "password": None,
+ },
+ )
+ shown = case.gateway.present_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=stored
+ )
+ assert shown["project_url"] is None
+ assert case.api.calls == []
+
+
+def test_invalid_new_destination_never_reaches_storage_or_network(case: ConfigurationCase) -> None:
+ with pytest.raises(AppTracingConfigInvalidConfigurationError):
+ case.service.create(
+ case.context,
+ case.app_id,
+ "mlflow",
+ {
+ "tracking_uri": "https://server.test/prefix/../other",
+ "experiment_id": "12",
+ "username": "user",
+ "password": "secret",
+ },
+ activate=True,
+ )
+ assert case.read() is None
+ assert case.api.calls == []
+ assert case.reader.get_trace(case.context, case.app_id).tracing_provider == "langsmith"
diff --git a/api/tests/unit_tests/services/test_opik_tracing_config_gateway.py b/api/tests/unit_tests/services/test_opik_tracing_config_gateway.py
new file mode 100644
index 00000000000..e72e22706b1
--- /dev/null
+++ b/api/tests/unit_tests/services/test_opik_tracing_config_gateway.py
@@ -0,0 +1,211 @@
+"""Opik configuration transactions use real RSA and never initialize an SDK streamer."""
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Never
+from uuid import uuid4
+
+import httpx
+import opik
+import pytest
+from dify_trace_opik.configuration import OpikConfigurationError
+from dify_trace_opik.opik_trace import OpikDataTrace
+from opik.config import OpikConfig as SDKOpikConfig
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.helper import ssrf_proxy
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from models.model import App
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_opik_configuration import AccessAPI
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+@dataclass
+class ConfigurationCase:
+ api: AccessAPI
+ gateway: OpsTraceManagerGateway
+ service: AppTracingConfigService
+ store: SQLAlchemyAppTracingConfigRepository
+ reader: ConsoleAppRepository
+ context: RequestContext
+ app_id: str
+ sessions: sessionmaker[Session]
+
+
+def case(
+ workspace: str, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> Iterator[ConfigurationCase]:
+ app_id = str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id=app_id, tenant_id=workspace))
+ api = AccessAPI()
+ gateway = OpsTraceManagerGateway()
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ forbidden_calls: list[str] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ forbidden_calls.append("sdk_constructor")
+ raise AssertionError("Configuration must not initialize an SDK client or ambient configuration")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", api.request)
+ monkeypatch.setattr(OpikDataTrace, "__init__", forbidden)
+ monkeypatch.setattr(opik.Opik, "__init__", forbidden)
+ monkeypatch.setattr(SDKOpikConfig, "__init__", forbidden)
+ yield ConfigurationCase(
+ api,
+ gateway,
+ AppTracingConfigService(configs=store, provider=gateway),
+ store,
+ ConsoleAppRepository(session_factory=sqlite_session_factory),
+ RequestContext("request", None, "account", workspace),
+ app_id,
+ sqlite_session_factory,
+ )
+ assert forbidden_calls == []
+
+
+def test_post_get_patch_preserve_real_ciphertext_and_verified_destination(case: ConfigurationCase) -> None:
+ secret = "synthetic-private-opik-key"
+ case.service.create(
+ case.context,
+ case.app_id,
+ "opik",
+ {"api_key": secret, "workspace": "", "project": " first ", "url": "https://first.test/prefix/api/"},
+ activate=True,
+ )
+ before = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")
+ assert before is not None
+ assert before.tracing_config is not None
+ config = before.tracing_config
+ assert config["api_key"] != secret
+ assert config["workspace"] == "resolved-team"
+ assert config["project"] == "first"
+ assert config["project_url"] == "https://first.test/prefix/resolved-team/redirect/projects?name=first"
+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "opik", config)["api_key"] == secret
+ assert case.reader.get_trace(case.context, case.app_id).tracing_provider == "opik"
+
+ before_get = len(case.api.calls)
+ presented = case.service.get(case.context, case.app_id, "opik")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ masked = presented.tracing_config["api_key"]
+ assert masked != secret
+ assert "*" in masked
+ assert len(case.api.calls) == before_get
+ case.api.default_workspace = "new-team"
+ case.service.update(
+ case.context,
+ case.app_id,
+ "opik",
+ {"api_key": masked, "workspace": "default", "project": "A & B", "url": "https://second.test/api/"},
+ )
+ after = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["api_key"] == config["api_key"]
+ assert after.tracing_config["workspace"] == "new-team"
+ assert after.tracing_config["project"] == "A & B"
+ assert after.tracing_config["url"] == "https://second.test/api/"
+ assert after.tracing_config["project_url"] == "https://second.test/new-team/redirect/projects?name=A+%26+B"
+ assert case.api.calls[-1][2]["headers"] == {
+ "Authorization": secret,
+ "Comet-Workspace": "default",
+ "Accept": "application/json",
+ }
+ before_get = len(case.api.calls)
+ presented = case.service.get(case.context, case.app_id, "opik")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config["project_url"] == after.tracing_config["project_url"]
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="opik", tracing_config=after.tracing_config
+ )
+ assert len(case.api.calls) == before_get
+
+
[email protected]("saved_url", [None, "https://wrong-global.test/old-project"])
[email protected]("workspace_name", ["team", "default", None])
+def test_historical_get_uses_only_configuration_host_without_network(
+ case: ConfigurationCase, saved_url: str | None, workspace_name: str | None
+) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "opik",
+ {
+ "api_key": "synthetic-key",
+ "workspace": workspace_name,
+ "project": "A & B",
+ "url": "https://private.test/opik/api/",
+ },
+ )
+ if saved_url:
+ stored["project_url"] = saved_url
+ result = case.gateway.present_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="opik", tracing_config=stored
+ )
+ expected = (
+ "https://private.test/opik/team/redirect/projects?name=A+%26+B"
+ if workspace_name == "team"
+ else "https://private.test/opik/"
+ )
+ assert result["project_url"] == expected
+ assert result["api_key"] != "synthetic-key"
+ assert case.api.calls == []
+
+
[email protected]("status", [401, 403])
[email protected]("existing", [False, True])
+def test_bad_key_or_workspace_does_not_save_or_change_active_provider(
+ case: ConfigurationCase, status: int, existing: bool, caplog: pytest.LogCaptureFixture
+) -> None:
+ original = {"api_key": "synthetic-private-key", "workspace": "team", "project": "original"}
+ if existing:
+ case.service.create(case.context, case.app_id, "opik", original)
+ with case.sessions.begin() as session:
+ app = session.get(App, case.app_id)
+ assert app is not None
+ app.tracing = json.dumps({"enabled": True, "tracing_provider": "langsmith", "capture_content": False})
+ before_policy = case.reader.get_trace(case.context, case.app_id)
+ before = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")
+ case.api.override = lambda method, url, _options: httpx.Response(
+ status, text="private-vendor-details", request=httpx.Request(method, url)
+ )
+ operation = case.service.update if existing else case.service.create
+ with pytest.raises(AppTracingConfigVerificationFailedError) as error:
+ operation(case.context, case.app_id, "opik", {**original, "workspace": "inaccessible"}, activate=True)
+ assert isinstance(error.value.__cause__, OpikConfigurationError)
+ assert error.value.__cause__.code == ("authentication_failed" if status == 401 else "permission_denied")
+ after = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")
+ assert after == before
+ assert case.reader.get_trace(case.context, case.app_id) == before_policy
+ assert "private-vendor-details" not in caplog.text
+ assert "synthetic-private-key" not in caplog.text
+
+
+def test_clear_optional_api_key_keeps_keyless_self_host_valid(
+ case: ConfigurationCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ target = {"url": "http://local.test:5173/api/", "workspace": "team", "project": "sample"}
+ case.service.create(case.context, case.app_id, "opik", {**target, "api_key": "synthetic-key"})
+ monkeypatch.setenv("OPIK_API_KEY", "unrelated-key")
+ case.api.calls.clear()
+ case.service.update(case.context, case.app_id, "opik", {**target, "api_key": None})
+ stored = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")
+ assert stored is not None
+ assert stored.tracing_config is not None
+ assert stored.tracing_config["api_key"] is None
+ presented = case.service.get(case.context, case.app_id, "opik")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config["api_key"] is None
+ assert case.api.calls[0][2]["headers"] == {"Comet-Workspace": "team", "Accept": "application/json"}
+ assert len(case.api.calls) == 1
diff --git a/api/tests/unit_tests/services/test_phoenix_tracing_config_gateway.py b/api/tests/unit_tests/services/test_phoenix_tracing_config_gateway.py
new file mode 100644
index 00000000000..3f07363c34f
--- /dev/null
+++ b/api/tests/unit_tests/services/test_phoenix_tracing_config_gateway.py
@@ -0,0 +1,285 @@
+"""Persist verified Phoenix configuration while preserving owner policy and masked keys."""
+
+import json
+from collections import deque
+from dataclasses import dataclass
+from uuid import uuid4
+
+import httpx
+import pytest
+from dify_trace_arize_phoenix.configuration import PhoenixConfigurationError
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import (
+ AppTracingConfigInvalidConfigurationError,
+ AppTracingConfigRecord,
+ AppTracingConfigService,
+ AppTracingConfigVerificationFailedError,
+)
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_phoenix_configuration import (
+ api_calls as api_calls, # noqa: PLC0414 - Re-export pytest fixtures only in this module.
+)
+from tests.unit_tests.services.tracing.test_phoenix_configuration import (
+ api_responses as api_responses, # noqa: PLC0414 - Re-export pytest fixtures only in this module.
+)
+from tests.unit_tests.services.tracing.test_phoenix_configuration import (
+ no_legacy_or_environment_auth as no_legacy_or_environment_auth, # noqa: PLC0414 - Re-export pytest fixtures.
+)
+from tests.unit_tests.services.tracing.test_phoenix_configuration import response
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+@dataclass
+class PhoenixCase:
+ gateway: OpsTraceManagerGateway
+ service: AppTracingConfigService
+ store: SQLAlchemyAppTracingConfigRepository
+ reader: ConsoleAppRepository
+ context: RequestContext
+ app_id: str
+
+ def read(self, provider: str = "phoenix") -> AppTracingConfigRecord | None:
+ return self.store.get(
+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider
+ )
+
+
+def case(workspace: str, sqlite_session_factory: sessionmaker[Session]) -> PhoenixCase:
+ app_id = str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ tracing=json.dumps(
+ {
+ "enabled": True,
+ "tracing_provider": "langsmith",
+ "capture_content": True,
+ "capture_agent": False,
+ }
+ ),
+ )
+ )
+ gateway = OpsTraceManagerGateway()
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ old = OpsTraceManager.encrypt_tracing_config(
+ workspace, "langsmith", {"api_key": "synthetic-other-key", "project": "other", "endpoint": "https://other.test"}
+ )
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=old)
+ return PhoenixCase(
+ gateway,
+ AppTracingConfigService(configs=store, provider=gateway),
+ store,
+ ConsoleAppRepository(session_factory=sqlite_session_factory),
+ RequestContext("request", None, "account", workspace),
+ app_id,
+ )
+
+
+def test_post_masked_patch_get_keep_normalized_destination_and_ciphertext(
+ case: PhoenixCase,
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ other = case.read("langsmith")
+ submitted = {
+ "endpoint": " HTTP://FIRST.Example:6006/reverse/ ",
+ "project": " first project ",
+ "api_key": "synthetic-secret",
+ }
+ untouched = dict(submitted)
+ case.service.create(case.context, case.app_id, "phoenix", submitted, activate=True)
+ before = case.read()
+ assert before is not None
+ assert before.tracing_config is not None
+ stored = before.tracing_config
+ assert stored["endpoint"] == "http://first.example:6006/reverse"
+ assert stored["project"] == "first project"
+ assert stored["api_key"] != "synthetic-secret"
+ assert submitted == untouched
+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "phoenix", stored)["api_key"] == (
+ "synthetic-secret"
+ )
+ shown = case.service.get(case.context, case.app_id, "phoenix")
+ assert shown is not None
+ assert shown.tracing_config is not None
+ masked = shown.tracing_config["api_key"]
+ assert isinstance(masked, str)
+ assert "*" in masked
+ assert masked != "synthetic-secret"
+ assert shown.tracing_config["project_url"] == "http://first.example:6006/reverse/redirects/projects/first%20project"
+ assert len(api_calls) == 1
+ case.service.update(
+ case.context,
+ case.app_id,
+ "phoenix",
+ {
+ "endpoint": " https://SECOND.test/s/space/ ",
+ "project": " second & 项目 ",
+ "api_key": masked,
+ },
+ )
+ after = case.read()
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["api_key"] == stored["api_key"]
+ assert after.tracing_config["endpoint"] == "https://second.test/s/space"
+ assert after.tracing_config["project"] == "second & 项目"
+ assert after.tracing_config["project_url"] == (
+ "https://second.test/s/space/redirects/projects/second%20%26%20%E9%A1%B9%E7%9B%AE"
+ )
+ assert api_calls[-1][0] == "https://second.test/s/space/v1/projects"
+ headers = api_calls[-1][1]["headers"]
+ assert isinstance(headers, dict)
+ assert headers["Authorization"] == "Bearer synthetic-secret"
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=after.tracing_config
+ )
+ assert case.service.get(case.context, case.app_id, "phoenix") is not None
+ assert len(api_calls) == 2
+ assert case.read("langsmith") == other
+ policy = case.reader.get_trace(case.context, case.app_id)
+ assert policy.enabled
+ assert policy.tracing_provider == "phoenix"
+ assert policy.capture_content is True
+ assert policy.capture_agent is False
+
+
[email protected]("empty", [None, ""])
+def test_bearer_to_keyless_and_back_is_explicit(
+ case: PhoenixCase,
+ api_calls: list[tuple[str, dict[str, object]]],
+ empty: str | None,
+) -> None:
+ destination = {"endpoint": "https://host.test", "project": "project"}
+ case.service.create(case.context, case.app_id, "phoenix", {**destination, "api_key": "first-key"})
+ case.service.update(case.context, case.app_id, "phoenix", {**destination, "api_key": empty})
+ assert api_calls[-1][1]["headers"] == {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}
+ stored = case.read()
+ assert stored is not None
+ assert stored.tracing_config is not None
+ assert (
+ OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "phoenix", stored.tracing_config)[
+ "api_key"
+ ]
+ == empty
+ )
+ case.service.update(case.context, case.app_id, "phoenix", {**destination, "api_key": "second-key"})
+ assert api_calls[-1][1]["headers"] != api_calls[0][1]["headers"]
+ assert len(api_calls) == 3
+
+
[email protected]("failure", ["denied", "wrong-path-html"])
+def test_failed_patch_preserves_policy_config_and_other_provider(
+ case: PhoenixCase,
+ api_calls: list[tuple[str, dict[str, object]]],
+ api_responses: deque[httpx.Response | Exception],
+ caplog: pytest.LogCaptureFixture,
+ failure: str,
+) -> None:
+ original = {"endpoint": "https://host.test", "project": "project", "api_key": "private-secret"}
+ case.service.create(case.context, case.app_id, "phoenix", original)
+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)
+ if failure == "denied":
+ api_responses.append(response({"secret": "private-vendor-response"}, 403))
+ else:
+ api_responses.append(
+ httpx.Response(
+ 200,
+ text="<html>private-vendor-response</html>",
+ request=httpx.Request("GET", "https://host.test/not-a-collector/v1/projects"),
+ )
+ )
+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
+ case.service.update(
+ case.context,
+ case.app_id,
+ "phoenix",
+ {
+ **original,
+ "endpoint": "https://host.test/not-a-collector",
+ "api_key": "******",
+ },
+ activate=True,
+ )
+ assert isinstance(caught.value.__cause__, PhoenixConfigurationError)
+ assert caught.value.__cause__.code == ("permission_denied" if failure == "denied" else "malformed_response")
+ assert case.read() == before
+ assert case.read("langsmith") == other
+ assert case.reader.get_trace(case.context, case.app_id) == policy
+ assert "private-vendor-response" not in caplog.text
+ assert "private-secret" not in caplog.text
+ assert len(api_calls) == 2
+
+
[email protected]("saved_url", [None, "https://stale.test/wrong-project"])
+def test_historical_get_calculates_url_without_verification_or_sdk(
+ case: PhoenixCase,
+ api_calls: list[tuple[str, dict[str, object]]],
+ saved_url: str | None,
+) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "phoenix",
+ {
+ "endpoint": "https://host.test/s/space/",
+ "project": "historical project",
+ "api_key": None,
+ },
+ )
+ if saved_url:
+ stored["project_url"] = saved_url
+ shown = case.gateway.present_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=stored
+ )
+ assert shown["project_url"] == "https://host.test/s/space/redirects/projects/historical%20project"
+ assert shown["api_key"] is None
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=stored
+ )
+ assert api_calls == []
+
+
[email protected]("endpoint", ["https://user:[email protected]?token=private", "https://host.test/v1/traces"])
+def test_unsafe_historical_destination_is_readable_without_link_or_io(
+ case: PhoenixCase,
+ api_calls: list[tuple[str, dict[str, object]]],
+ endpoint: str,
+) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "phoenix",
+ {
+ "endpoint": endpoint,
+ "project": "old-project",
+ "api_key": None,
+ },
+ )
+ shown = case.gateway.present_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=stored
+ )
+ assert shown["endpoint"] == endpoint
+ assert shown["project_url"] is None
+ assert api_calls == []
+
+
[email protected]("values", [{"endpoint": "https://host.test/v1/traces"}, {"project": "bad\nproject"}])
+def test_invalid_new_destination_has_no_io_or_storage_effect(
+ case: PhoenixCase,
+ api_calls: list[tuple[str, dict[str, object]]],
+ values: dict[str, str],
+) -> None:
+ policy = case.reader.get_trace(case.context, case.app_id)
+ with pytest.raises(AppTracingConfigInvalidConfigurationError):
+ case.service.create(case.context, case.app_id, "phoenix", {"api_key": "synthetic-key", **values}, activate=True)
+ assert case.read() is None
+ assert case.reader.get_trace(case.context, case.app_id) == policy
+ assert api_calls == []
diff --git a/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py b/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py
index ddf4ff03538..c8ee83d8668 100644
--- a/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py
+++ b/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py
@@ -13,7 +13,27 @@ from services.system_feature_service import SystemFeatureService
def test_system_feature_model_requires_deployment_edition() -> None:
with pytest.raises(ValidationError):
- SystemFeatureModel.model_validate({})
+ SystemFeatureModel.model_validate({"agent_tracing_api_collection_enabled": False})
+
+
[email protected]("edition", list(DeploymentEdition))
[email protected]("enabled", [False, True])
+def test_public_agent_tracing_gate_uses_configured_value_in_every_edition(
+ monkeypatch: pytest.MonkeyPatch,
+ config_overrides: Callable[..., None],
+ edition: DeploymentEdition,
+ enabled: bool,
+) -> None:
+ config_overrides(DEPLOYMENT_EDITION=edition, OPS_AGENT_TRACE_ENABLED=enabled)
+ monkeypatch.setattr(
+ "services.system_feature_service.EnterpriseService.get_info",
+ staticmethod(lambda: {}),
+ )
+
+ result = SystemFeatureService.get_public_system_features()
+
+ assert result.agent_tracing_api_collection_enabled is enabled
+ assert result.model_dump(mode="json")["agent_tracing_api_collection_enabled"] is enabled
@pytest.mark.parametrize(
diff --git a/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py b/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py
index 0741f6a6f32..8fbf8cc674e 100644
--- a/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py
+++ b/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py
@@ -33,7 +33,9 @@ def test_fulfill_params_from_enterprise_enable_app_deploy(
staticmethod(lambda: enterprise_info),
)
- features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)
+ features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY
+ )
features.enable_app_deploy = initial
SystemFeatureService._fulfill_params_from_enterprise(features)
diff --git a/api/tests/unit_tests/services/test_system_feature_service_learn_app.py b/api/tests/unit_tests/services/test_system_feature_service_learn_app.py
index 95e53bb51aa..370db54a8bf 100644
--- a/api/tests/unit_tests/services/test_system_feature_service_learn_app.py
+++ b/api/tests/unit_tests/services/test_system_feature_service_learn_app.py
@@ -10,7 +10,9 @@ from services.system_feature_service import SystemFeatureService
def test_system_feature_model_defaults_enable_learn_app() -> None:
- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)
+ system_features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY
+ )
assert system_features.enable_learn_app is True
assert system_features.enable_step_by_step_tour is False
diff --git a/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py b/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py
index cc2490b6983..9ed70688d3e 100644
--- a/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py
+++ b/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py
@@ -89,7 +89,9 @@ def test_system_features_exposes_only_validated_plugin_installation_policy(
}
),
)
- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)
+ features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE
+ )
SystemFeatureService._fulfill_params_from_enterprise(features)
diff --git a/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py b/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py
index b889282e24b..48e42cfe8b3 100644
--- a/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py
+++ b/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py
@@ -24,7 +24,9 @@ def test_system_features_exposes_valid_enterprise_sso_protocols(
}
),
)
- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)
+ features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE
+ )
SystemFeatureService._fulfill_params_from_enterprise(features)
@@ -48,7 +50,9 @@ def test_system_features_normalizes_empty_enterprise_sso_protocols_to_none(
}
),
)
- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)
+ features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE
+ )
SystemFeatureService._fulfill_params_from_enterprise(features)
@@ -73,7 +77,9 @@ def test_system_features_rejects_invalid_enterprise_sso_protocols(
}
),
)
- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)
+ features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE
+ )
with caplog.at_level(logging.ERROR, logger="services.system_feature_service"):
SystemFeatureService._fulfill_params_from_enterprise(features)
@@ -84,7 +90,9 @@ def test_system_features_rejects_invalid_enterprise_sso_protocols(
def test_system_features_defaults_sso_protocols_to_none() -> None:
- features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)
+ features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY
+ )
assert features.sso_enforced_for_signin_protocol is None
assert features.webapp_auth.sso_config.protocol is None
diff --git a/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py b/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py
index 69757ebd7cd..6764dbf39d8 100644
--- a/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py
+++ b/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py
@@ -24,7 +24,9 @@ def test_fulfill_system_params_from_env_sets_allow_public_access(
) -> None:
config_overrides(WEBAPP_PUBLIC_ACCESS_ENABLED=env_value)
- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)
+ system_features = SystemFeatureModel(
+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY
+ )
SystemFeatureService._fulfill_system_params_from_env(system_features)
assert system_features.webapp_auth.allow_public_access is expected
diff --git a/api/tests/unit_tests/services/test_tencent_tracing_config_gateway.py b/api/tests/unit_tests/services/test_tencent_tracing_config_gateway.py
new file mode 100644
index 00000000000..ff5ab51464e
--- /dev/null
+++ b/api/tests/unit_tests/services/test_tencent_tracing_config_gateway.py
@@ -0,0 +1,324 @@
+"""Tencent configuration persistence uses real RSA and bounded protocol checks."""
+
+import json
+import os
+from dataclasses import dataclass
+from typing import Never
+from uuid import uuid4
+
+import grpc # type: ignore[import-untyped]
+import pytest
+from dify_trace_tencent.configuration import TencentConfigurationError
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import (
+ AppTracingConfigInvalidConfigurationError,
+ AppTracingConfigRecord,
+ AppTracingConfigService,
+ AppTracingConfigVerificationFailedError,
+)
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_tencent_configuration import CollectorBoundary
+from tests.unit_tests.services.tracing.test_tencent_configuration import (
+ _forbid_provider_io as _forbid_provider_io, # noqa: PLC0414 - Re-export pytest fixtures only in this module.
+)
+from tests.unit_tests.services.tracing.test_tencent_configuration import (
+ collector as collector, # noqa: PLC0414 - Re-export pytest fixtures only in this module.
+)
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+@dataclass
+class ConfigurationCase:
+ collector: CollectorBoundary
+ gateway: OpsTraceManagerGateway
+ service: AppTracingConfigService
+ store: SQLAlchemyAppTracingConfigRepository
+ reader: ConsoleAppRepository
+ context: RequestContext
+ app_id: str
+
+ def read(self, provider: str = "tencent") -> AppTracingConfigRecord | None:
+ return self.store.get(
+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider
+ )
+
+
+def case(
+ workspace: str, sqlite_session_factory: sessionmaker[Session], collector: CollectorBoundary
+) -> ConfigurationCase:
+ app_id = str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=workspace,
+ tracing=json.dumps(
+ {
+ "enabled": True,
+ "tracing_provider": "langsmith",
+ "capture_content": True,
+ "capture_agent": False,
+ }
+ ),
+ )
+ )
+ gateway = OpsTraceManagerGateway()
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ old_config = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "langsmith",
+ {"api_key": "synthetic-existing-key", "project": "existing-project", "endpoint": "https://existing.example"},
+ )
+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=old_config)
+ return ConfigurationCase(
+ collector,
+ gateway,
+ AppTracingConfigService(configs=store, provider=gateway),
+ store,
+ ConsoleAppRepository(session_factory=sqlite_session_factory),
+ RequestContext("request", None, "account", workspace),
+ app_id,
+ )
+
+
+def test_post_get_masked_patch_persist_normalization_and_preserve_real_ciphertext(case: ConfigurationCase) -> None:
+ secret = "synthetic-private-tencent-token"
+ old_provider = case.read("langsmith")
+ submitted = {"token": secret, "endpoint": " HTTPS://FIRST.Example:4320/ ", "service_name": " first-app "}
+ original_submission = dict(submitted)
+
+ case.service.create(case.context, case.app_id, "tencent", submitted, activate=True)
+
+ before = case.read()
+ assert before is not None
+ assert before.tracing_config is not None
+ config = before.tracing_config
+ assert config["token"] != secret
+ assert config["endpoint"] == "https://first.example:4320"
+ assert config["service_name"] == "first-app"
+ assert config["project_url"] == "https://console.cloud.tencent.com/apm"
+ assert (
+ OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "tencent", config)["token"] == secret
+ )
+ assert submitted == original_submission
+ assert len(case.collector.channels) == 1
+ assert case.collector.channels[0].calls[0].metadata == (("authorization", f"Bearer {secret}"),)
+ policy = case.reader.get_trace(case.context, case.app_id)
+ assert policy.enabled is True
+ assert policy.tracing_provider == "tencent"
+ assert policy.capture_content is True
+ assert policy.capture_agent is False
+ assert case.read("langsmith") == old_provider
+
+ presented = case.service.get(case.context, case.app_id, "tencent")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ masked = presented.tracing_config["token"]
+ assert isinstance(masked, str)
+ assert masked != secret
+ assert "*" in masked
+ assert presented.tracing_config["project_url"] == "https://console.cloud.tencent.com/apm"
+ assert len(case.collector.channels) == 1
+
+ replacement = {"token": masked, "endpoint": " http://SECOND.Example:4319/ ", "service_name": " second-app "}
+ original_replacement = dict(replacement)
+ case.service.update(case.context, case.app_id, "tencent", replacement, activate=True)
+
+ after = case.read()
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["token"] == config["token"]
+ assert after.tracing_config["endpoint"] == "http://second.example:4319"
+ assert after.tracing_config["service_name"] == "second-app"
+ assert after.tracing_config["project_url"] == "https://console.cloud.tencent.com/apm"
+ assert replacement == original_replacement
+ assert len(case.collector.channels) == 2
+ channel = case.collector.channels[-1]
+ assert channel.target == "second.example:4319"
+ assert channel.secure is False
+ assert channel.calls[0].metadata == (("authorization", f"Bearer {secret}"),)
+ assert all(channel.close_count == 1 for channel in case.collector.channels)
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="tencent", tracing_config=after.tracing_config
+ )
+ assert case.service.get(case.context, case.app_id, "tencent") is not None
+ assert len(case.collector.channels) == 2
+ assert case.read("langsmith") == old_provider
+
+
[email protected]("saved_url", [None, "https://wrong.example/previous-project"])
+def test_historical_name_remains_readable_and_saved_validation_does_not_connect(
+ case: ConfigurationCase, saved_url: str | None
+) -> None:
+ stored = OpsTraceManager.encrypt_tracing_config(
+ case.context.active_workspace_id,
+ "tencent",
+ {"token": "synthetic-historical-token", "endpoint": "https://legacy.example:4320", "service_name": "dify_app"},
+ )
+ if saved_url is not None:
+ stored["project_url"] = saved_url
+ assert case.store.create(
+ workspace_id=case.context.active_workspace_id,
+ app_id=case.app_id,
+ tracing_provider="tencent",
+ tracing_config=stored,
+ )
+ before = case.read()
+ before_policy = case.reader.get_trace(case.context, case.app_id)
+
+ presented = case.service.get(case.context, case.app_id, "tencent")
+ case.gateway.validate_saved_config(
+ workspace_id=case.context.active_workspace_id, tracing_provider="tencent", tracing_config=stored
+ )
+
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config["service_name"] == "dify_app"
+ assert presented.tracing_config["project_url"] == "https://console.cloud.tencent.com/apm"
+ assert presented.tracing_config["token"] != "synthetic-historical-token"
+ assert "*" in presented.tracing_config["token"]
+ assert case.collector.channels == []
+ assert case.read() == before
+ assert case.reader.get_trace(case.context, case.app_id) == before_policy
+
+
[email protected]("existing", [False, True])
+ ("field_name", "invalid_value", "error_code"),
+ [
+ ("service_name", "dify_app", "invalid_service_name"),
+ ("service_name", "", "invalid_service_name"),
+ ("endpoint", "https://collector.example:4320/v1/traces", "invalid_endpoint"),
+ ("endpoint", "https://collector.example:4320?token=private-endpoint-detail", "invalid_endpoint"),
+ ("endpoint", "https://", "invalid_endpoint"),
+ ],
+)
+def test_invalid_create_or_patch_rejects_before_rpc_encryption_or_persistence(
+ case: ConfigurationCase,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ existing: bool,
+ field_name: str,
+ invalid_value: str,
+ error_code: str,
+) -> None:
+ original = {
+ "token": "synthetic-private-token",
+ "endpoint": "https://original.example:4320",
+ "service_name": "original-app",
+ }
+ if existing:
+ case.service.create(case.context, case.app_id, "tencent", original)
+ case.collector.channels.clear()
+ before = case.read()
+ old_provider = case.read("langsmith")
+ before_policy = case.reader.get_trace(case.context, case.app_id)
+ forbidden_attempts: list[str] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ forbidden_attempts.append("encryption_or_write")
+ raise AssertionError("Invalid new input must fail before encryption or persistence")
+
+ monkeypatch.setattr(OpsTraceManager, "encrypt_tracing_config", forbidden)
+ monkeypatch.setattr(case.store, "create", forbidden)
+ monkeypatch.setattr(case.store, "update", forbidden)
+ operation = case.service.update if existing else case.service.create
+ with pytest.raises(AppTracingConfigInvalidConfigurationError) as caught:
+ operation(case.context, case.app_id, "tencent", {**original, field_name: invalid_value}, activate=True)
+
+ assert isinstance(caught.value.__cause__, TencentConfigurationError)
+ assert caught.value.__cause__.code == error_code
+ assert forbidden_attempts == []
+ assert case.collector.channels == []
+ assert case.read() == before
+ assert case.read("langsmith") == old_provider
+ assert case.reader.get_trace(case.context, case.app_id) == before_policy
+ assert "synthetic-private-token" not in caplog.text
+ assert "private-endpoint-detail" not in caplog.text
+
+
[email protected]("existing", [False, True])
[email protected]("endpoint", ["https://collector.example:4320", "http://localhost:4317"])
+def test_rpc_failure_never_saves_or_activates_even_for_localhost(
+ case: ConfigurationCase, caplog: pytest.LogCaptureFixture, existing: bool, endpoint: str
+) -> None:
+ secret = "synthetic-private-token"
+ original = {"token": secret, "endpoint": "https://original.example:4320", "service_name": "original-app"}
+ if existing:
+ case.service.create(case.context, case.app_id, "tencent", original)
+ case.collector.channels.clear()
+ before = case.read()
+ old_provider = case.read("langsmith")
+ before_policy = case.reader.get_trace(case.context, case.app_id)
+ case.collector.failure = grpc.RpcError("private-provider-details synthetic-private-token")
+ operation = case.service.update if existing else case.service.create
+
+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
+ operation(
+ case.context,
+ case.app_id,
+ "tencent",
+ {"token": "******" if existing else secret, "endpoint": endpoint, "service_name": "changed-app"},
+ activate=True,
+ )
+
+ assert isinstance(caught.value.__cause__, TencentConfigurationError)
+ assert caught.value.__cause__.code == "connection_failed"
+ assert len(case.collector.channels) == 1
+ channel = case.collector.channels[0]
+ assert channel.close_count == 1
+ assert len(channel.calls) == 1
+ assert channel.calls[0].metadata == (("authorization", f"Bearer {secret}"),)
+ assert case.read() == before
+ assert case.read("langsmith") == old_provider
+ assert case.reader.get_trace(case.context, case.app_id) == before_policy
+ assert "connection_failed" in caplog.text
+ assert "private-provider-details" not in caplog.text
+ assert secret not in caplog.text
+ assert secret not in str(caught.value)
+ assert secret not in str(caught.value.__cause__)
+
+
+def test_explicit_configuration_ignores_and_preserves_otel_environment(
+ case: ConfigurationCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ environment = {
+ "OTEL_EXPORTER_OTLP_ENDPOINT": "http://ambient.example:4317",
+ "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://ambient-traces.example:4317",
+ "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer unrelated-token",
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer unrelated-traces-token",
+ "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
+ "OTEL_SERVICE_NAME": "unrelated-service",
+ "OTEL_RESOURCE_ATTRIBUTES": "token=unrelated-resource-token,service.name=ambient-app",
+ }
+ for key, value in environment.items():
+ monkeypatch.setenv(key, value)
+
+ case.service.create(
+ case.context,
+ case.app_id,
+ "tencent",
+ {"token": "explicit-token", "endpoint": "https://explicit.example:4320", "service_name": "explicit-app"},
+ )
+
+ assert len(case.collector.channels) == 1
+ channel = case.collector.channels[0]
+ assert channel.target == "explicit.example:4320"
+ assert channel.secure is True
+ assert channel.calls[0].metadata == (("authorization", "Bearer explicit-token"),)
+ assert channel.calls[0].request.SerializeToString() == b""
+ assert channel.close_count == 1
+ presented = case.service.get(case.context, case.app_id, "tencent")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config["service_name"] == "explicit-app"
+ assert len(case.collector.channels) == 1
+ assert {key: os.environ.get(key) for key in environment} == environment
diff --git a/api/tests/unit_tests/services/test_weave_tracing_config_gateway.py b/api/tests/unit_tests/services/test_weave_tracing_config_gateway.py
new file mode 100644
index 00000000000..918d30bc8cc
--- /dev/null
+++ b/api/tests/unit_tests/services/test_weave_tracing_config_gateway.py
@@ -0,0 +1,167 @@
+"""Config reads and writes must not construct Weave's stateful legacy exporter."""
+
+from collections.abc import Callable, Iterator, Sequence
+from dataclasses import dataclass, field
+from typing import Never
+from uuid import uuid4
+
+import pytest
+import wandb
+import weave
+from dify_trace_weave.configuration import WeaveConfigurationError
+from dify_trace_weave.weave_trace import WeaveDataTrace
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.helper import ssrf_proxy
+from core.ops import ops_trace_manager
+from core.ops.ops_trace_manager import OpsTraceManager
+from machinery.context import RequestContext
+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError
+from tests.unit_tests.model_factories import make_app
+from tests.unit_tests.services.tracing.test_weave_configuration import ReadAPI
+
+
+@dataclass
+class GatewayEnvironment:
+ gateway: OpsTraceManagerGateway = field(default_factory=OpsTraceManagerGateway)
+ api: ReadAPI = field(default_factory=ReadAPI)
+ forbidden_calls: list[str] = field(default_factory=list)
+
+
+def environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[GatewayEnvironment]:
+ environment = GatewayEnvironment()
+
+ def encrypt(_tenant: str, value: str) -> str:
+ return "test-cipher:" + value
+
+ def decrypt(_tenant: str, values: Sequence[str]) -> list[str]:
+ assert all(value.startswith("test-cipher:") for value in values)
+ return [value.removeprefix("test-cipher:") for value in values]
+
+ def mask(_value: str) -> str:
+ return "******"
+
+ def forbidden(name: str) -> Callable[..., Never]:
+ def block(*_args: object, **_kwargs: object) -> Never:
+ environment.forbidden_calls.append(name)
+ raise AssertionError("Configuration must not initialize a stateful exporter")
+
+ return block
+
+ monkeypatch.setattr(ops_trace_manager, "encrypt_token", encrypt)
+ monkeypatch.setattr(ops_trace_manager, "batch_decrypt_token", decrypt)
+ monkeypatch.setattr(ops_trace_manager, "obfuscated_token", mask)
+ monkeypatch.setattr(ssrf_proxy, "post", environment.api.post)
+ monkeypatch.setattr(WeaveDataTrace, "__init__", forbidden("legacy_constructor"))
+ monkeypatch.setattr(wandb, "login", forbidden("wandb_login"))
+ monkeypatch.setattr(weave, "init", forbidden("weave_init"))
+ yield environment
+ assert environment.forbidden_calls == []
+
+
+def test_create_persists_resolved_default_entity_and_canonical_project(environment: GatewayEnvironment) -> None:
+ submitted = {"api_key": "synthetic-key", "project": " demo "}
+ stored = environment.gateway.prepare_new_config(
+ workspace_id=str(uuid4()), tracing_provider="weave", tracing_config=submitted
+ )
+ assert stored["entity"] == "default-team"
+ assert stored["project"] == "demo"
+ assert stored["project_url"] == "https://wandb.ai/default-team/demo"
+ assert stored["api_key"] == "test-cipher:synthetic-key"
+ assert submitted == {"api_key": "synthetic-key", "project": " demo "}
+ assert len(environment.api.calls) == 3
+
+
+def test_patch_persists_verified_destination_with_the_existing_ciphertext(
+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session]
+) -> None:
+ workspace, app_id = str(uuid4()), str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id=app_id, tenant_id=workspace))
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ service = AppTracingConfigService(configs=store, provider=environment.gateway)
+ context = RequestContext("request", None, "account", workspace)
+ service.create(context, app_id, "weave", {"api_key": "synthetic-key", "entity": "old", "project": "original"})
+ before = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="weave")
+ assert before is not None
+ assert before.tracing_config is not None
+
+ submitted = {"api_key": "******", "project": "new-team/changed"}
+ service.update(context, app_id, "weave", submitted)
+ after = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="weave")
+ assert after is not None
+ assert after.tracing_config is not None
+ assert after.tracing_config["entity"] == "new-team"
+ assert after.tracing_config["project"] == "changed"
+ assert after.tracing_config["project_url"] == "https://wandb.ai/new-team/changed"
+ assert after.tracing_config["api_key"] == before.tracing_config["api_key"] == "test-cipher:synthetic-key"
+ assert before.tracing_config["project_url"] == "https://wandb.ai/old/original"
+ assert submitted == {"api_key": "******", "project": "new-team/changed"}
+ assert len(environment.api.calls) == 4
+
+ presented = service.get(context, app_id, "weave")
+ assert presented is not None
+ assert presented.tracing_config is not None
+ assert presented.tracing_config["api_key"] == "******"
+ assert presented.tracing_config["project_url"] == after.tracing_config["project_url"]
+ environment.gateway.validate_saved_config(
+ workspace_id=workspace, tracing_provider="weave", tracing_config=after.tracing_config
+ )
+ assert len(environment.api.calls) == 4
+
+
+ ("entity", "project", "host", "expected"),
+ [
+ ("team", "demo", None, "https://wandb.ai/team/demo"),
+ (None, "team/demo", None, "https://wandb.ai/team/demo"),
+ (None, "demo", None, "https://wandb.ai"),
+ (None, "demo", "http://private.test/wandb", "http://private.test/wandb"),
+ ("team", "a & b", "https://private.test/wandb", "https://private.test/wandb/team/a%20%26%20b"),
+ ],
+)
+def test_get_computes_project_or_console_link_without_vendor_requests(
+ environment: GatewayEnvironment, entity: str | None, project: str, host: str | None, expected: str
+) -> None:
+ workspace = str(uuid4())
+ encrypted = OpsTraceManager.encrypt_tracing_config(
+ workspace, "weave", {"api_key": "synthetic-key", "entity": entity, "project": project, "host": host}
+ )
+ encrypted["project_url"] = "https://stale.test/previous-project"
+ result = environment.gateway.present_config(
+ workspace_id=workspace, tracing_provider="weave", tracing_config=encrypted
+ )
+ assert result["project_url"] == expected
+ assert result["api_key"] == "******"
+ assert environment.api.calls == []
+
+
+def test_inaccessible_project_is_a_sanitized_domain_error_and_is_not_saved(
+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], caplog: pytest.LogCaptureFixture
+) -> None:
+ workspace, app_id = str(uuid4()), str(uuid4())
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id=app_id, tenant_id=workspace))
+ environment.api.project_exists = False
+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)
+ service = AppTracingConfigService(configs=store, provider=environment.gateway)
+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:
+ service.create(
+ RequestContext("request", None, "account", workspace),
+ app_id,
+ "weave",
+ {
+ "api_key": "synthetic-private-key",
+ "entity": "team",
+ "project": "missing",
+ },
+ )
+ assert isinstance(caught.value.__cause__, WeaveConfigurationError)
+ assert caught.value.__cause__.code == "project_not_found_or_inaccessible"
+ assert store.get(workspace_id=workspace, app_id=app_id, tracing_provider="weave") is None
+ assert "synthetic-private-key" not in caplog.text
+ assert "project_not_found_or_inaccessible" in caplog.text
+ assert len(environment.api.calls) == 1
diff --git a/api/tests/unit_tests/services/test_workflow_agent_preview_identity.py b/api/tests/unit_tests/services/test_workflow_agent_preview_identity.py
new file mode 100644
index 00000000000..dd4c5c56375
--- /dev/null
+++ b/api/tests/unit_tests/services/test_workflow_agent_preview_identity.py
@@ -0,0 +1,195 @@
+"""Agent preview callers are committed before execution and retain their identity."""
+
+import json
+from collections.abc import Generator, Iterator
+from unittest.mock import patch
+from uuid import uuid4
+
+import pytest
+from flask import Flask
+from sqlalchemy import Engine, func, select
+from sqlalchemy.orm import sessionmaker
+
+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
+from core.workflow.nodes.agent_v2 import DifyAgentNode
+from core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStore
+from core.workflow.system_variables import SystemVariableKey, get_system_text
+from extensions.ext_database import db
+from graphon.engine_events import NodeRunFailedEvent, NodeRunStartedEvent, NodeRunSucceededEvent
+from graphon.enums import WorkflowNodeExecutionStatus
+from graphon.node_events import NodeRunResult
+from libs.datetime_utils import naive_utc_now
+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
+from services.workflow_service import WorkflowService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.services.test_workflow_service import TestWorkflowAssociatedDataFactory as Factory
+
+
+class PreviewCancelled(BaseException):
+ """Cancellation can bypass Exception handlers, like a stopped greenlet."""
+
+
+def preview_flask_app(sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch) -> Iterator[Flask]:
+ apply_config_overrides(monkeypatch, AGENT_BACKEND_BASE_URL="https://agent.test")
+ app = Flask(__name__)
+ app.config["SQLALCHEMY_DATABASE_URI"] = sqlite_engine.url.render_as_string(hide_password=False)
+ db.init_app(app)
+ with app.app_context():
+ yield app
+
+
[email protected]("outcome", ["success", "failure", "exception", "cancel"])
[email protected]("preview_flask_app")
+def test_agent_preview_commits_runtime_identity_and_updates_same_row(sqlite_engine: Engine, outcome: str) -> None:
+ tenant_id, app_id, workflow_id, account_id = (str(uuid4()) for _ in range(4))
+ node_id = "agent-preview"
+ app = Factory.create_app(app_id=app_id, tenant_id=tenant_id)
+ account = Factory.create_account(account_id=account_id)
+ workflow = Factory.create_workflow(
+ workflow_id=workflow_id,
+ tenant_id=tenant_id,
+ app_id=app_id,
+ graph={
+ "nodes": [
+ {
+ "id": node_id,
+ "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Agent"},
+ }
+ ],
+ "edges": [],
+ },
+ )
+ sessions = sessionmaker(sqlite_engine, expire_on_commit=False)
+ with sessions.begin() as session:
+ session.add_all([app, account, workflow])
+ repository = SQLAlchemyWorkflowNodeExecutionRepository(
+ sessions, tenant_id, account, app_id, WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP
+ )
+ observed_created_at = []
+ execution_ids: list[str] = []
+
+ def events(node: DifyAgentNode) -> Generator[NodeRunStartedEvent | NodeRunSucceededEvent | NodeRunFailedEvent]:
+ execution_id = node.execution_id
+ execution_ids.append(execution_id)
+ started_at = naive_utc_now()
+ # Node.run now emits its typed start before entering the node body. The
+ # service observer must commit that event before resuming this generator.
+ yield NodeRunStartedEvent(
+ id=execution_id,
+ node_id=node_id,
+ node_type="agent",
+ node_version="2",
+ node_title="Agent",
+ start_at=started_at,
+ )
+ # A separate transaction must see the committed caller before any
+ # Agent workspace or tracing request can be dispatched.
+ variable_pool = node.runtime_state.variable_pool
+ workflow_run_id = get_system_text(variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID)
+ assert workflow_run_id is None
+ with sessions.begin() as session:
+ caller = WorkflowAgentWorkspaceStore._load_execution_by_identity(
+ session=session,
+ tenant_id=tenant_id,
+ app_id=app_id,
+ workflow_id=workflow_id,
+ workflow_run_id=workflow_run_id,
+ node_id=node_id,
+ node_execution_id=execution_id,
+ )
+ assert caller.id == caller.node_execution_id == execution_id
+ assert caller.status == WorkflowNodeExecutionStatus.RUNNING
+ assert caller.finished_at is None
+ observed_created_at.append(caller.created_at)
+ # Session materialization writes these independently; final persistence
+ # must not discard either piece of caller ownership.
+ caller.agent_workspace_binding_id = "workspace-binding"
+ caller.process_data = json.dumps({"workflow_agent_binding_id": "workflow-binding"})
+ if outcome == "exception":
+ raise RuntimeError("private runtime error details")
+ if outcome == "cancel":
+ raise PreviewCancelled()
+ status = WorkflowNodeExecutionStatus.SUCCEEDED if outcome == "success" else WorkflowNodeExecutionStatus.FAILED
+ result = NodeRunResult(
+ status=status, outputs={"answer": "done"}, error="" if outcome == "success" else "failed"
+ )
+ if outcome == "success":
+ yield NodeRunSucceededEvent(
+ id=execution_id, node_id=node_id, node_type="agent", node_run_result=result, start_at=started_at
+ )
+ else:
+ yield NodeRunFailedEvent(
+ id=execution_id,
+ node_id=node_id,
+ node_type="agent",
+ node_run_result=result,
+ start_at=started_at,
+ error="failed",
+ )
+
+ service = WorkflowService(sessions)
+ with (
+ patch("services.workflow_service.WorkflowDraftVariableService.prefill_conversation_variable_default_values"),
+ patch.object(DifyAgentNode, "run", events),
+ patch(
+ "services.workflow_service.DifyCoreRepositoryFactory.create_workflow_node_execution_repository",
+ return_value=repository,
+ ),
+ patch("services.workflow_service.DraftVariableSaver.save"),
+ patch("services.workflow_service.enqueue_draft_node_execution_trace"),
+ ):
+ if outcome in {"exception", "cancel"}:
+ with pytest.raises(RuntimeError if outcome == "exception" else PreviewCancelled):
+ service.run_draft_workflow_node(app, workflow, node_id, {}, account)
+ else:
+ response = service.run_draft_workflow_node(app, workflow, node_id, {}, account)
+ assert response.id == response.node_execution_id == execution_ids[0]
+
+ assert len(execution_ids) == 1
+ execution_id = execution_ids[0]
+ with sessions() as session:
+ assert session.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 1
+ row = session.get(WorkflowNodeExecutionModel, execution_id)
+ assert row is not None
+ assert row.id == row.node_execution_id == execution_id
+ assert row.workflow_run_id is None
+ assert row.workflow_id == workflow_id
+ assert row.tenant_id == tenant_id
+ assert row.app_id == app_id
+ assert row.created_at == observed_created_at[0]
+ assert row.finished_at is not None
+ assert row.elapsed_time >= 0
+ assert row.agent_workspace_binding_id == "workspace-binding"
+ assert row.process_data_dict is not None
+ assert row.process_data_dict["workflow_agent_binding_id"] == "workflow-binding"
+ assert row.status == (
+ WorkflowNodeExecutionStatus.SUCCEEDED if outcome == "success" else WorkflowNodeExecutionStatus.FAILED
+ )
+ if outcome in {"exception", "cancel"}:
+ assert row.error == "Agent preview execution interrupted"
+
+
[email protected]("preview_flask_app")
+def test_agent_preview_rejects_mismatched_workflow_before_creating_caller(sqlite_engine: Engine) -> None:
+ app = Factory.create_app(app_id="app", tenant_id="tenant")
+ account = Factory.create_account()
+ workflow = Factory.create_workflow(
+ app_id="another-app",
+ tenant_id="another-tenant",
+ graph={
+ "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}],
+ "edges": [],
+ },
+ )
+ sessions = sessionmaker(sqlite_engine, expire_on_commit=False)
+ with (
+ patch(
+ "services.workflow_service.WorkflowDraftVariableService.prefill_conversation_variable_default_values"
+ ) as prefill,
+ pytest.raises(ValueError, match="does not belong"),
+ ):
+ WorkflowService(sessions).run_draft_workflow_node(app, workflow, "agent", {}, account)
+ prefill.assert_not_called()
+ with sessions() as session:
+ assert session.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 0
diff --git a/api/tests/unit_tests/services/tracing/test_aliyun_configuration.py b/api/tests/unit_tests/services/tracing/test_aliyun_configuration.py
new file mode 100644
index 00000000000..e679dc996b8
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_aliyun_configuration.py
@@ -0,0 +1,276 @@
+"""Pure Aliyun destination rules; these tests do not claim account verification."""
+
+import traceback
+from dataclasses import dataclass, field
+from urllib.parse import unquote, urlsplit
+
+import httpx
+import pytest
+from dify_trace_aliyun.aliyun_trace import AliyunDataTrace
+from dify_trace_aliyun.config import AliyunConfig
+from dify_trace_aliyun.configuration import (
+ AliyunConfigurationError,
+ console_url,
+ ingestion_url,
+ normalize_configuration,
+ verify_configuration,
+)
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+from pydantic import ValidationError
+
+from core.helper import ssrf_proxy
+
+
+def values(**overrides: object) -> dict[str, object]:
+ return {"endpoint": "http://tracing-analysis-dc-qd.aliyuncs.com", "license_key": "synthetic-key", **overrides}
+
+
+def test_normalization_preserves_opaque_license_prefix_and_application_name() -> None:
+ submitted = values(
+ endpoint=" HTTPS://Project.CN-QINGDAO.LOG.ALIYUNCS.COM:443/apm/trace/opentelemetry/ ",
+ app_name=" 中文 agent_Name 01 ",
+ license_key="literal%2F@a+b/c",
+ )
+ original = dict(submitted)
+ result = normalize_configuration(submitted)
+ assert result.endpoint == "https://project.cn-qingdao.log.aliyuncs.com:443/apm/trace/opentelemetry"
+ assert result.app_name == "中文 agent_Name 01"
+ assert result.license_key == "literal%2F@a+b/c"
+ assert submitted == original
+ assert ingestion_url(result.model_dump()) == (
+ "https://project.cn-qingdao.log.aliyuncs.com:443/apm/trace/opentelemetry/"
+ "adapt_literal%252F@a%2Bb%2Fc/api/v1/traces"
+ )
+
+
[email protected]("key", ["raw%2F", "raw%252F", "a/b?c#d", "left@right_one+two", "密钥"])
+def test_ingestion_path_round_trips_the_raw_license_exactly_once(key: str) -> None:
+ url = ingestion_url(values(license_key=key))
+ path = urlsplit(url).path
+ assert path.startswith("/adapt_")
+ assert path.endswith("/api/otlp/traces")
+ encoded = path.removeprefix("/adapt_").removesuffix("/api/otlp/traces")
+ assert "/" not in encoded
+ assert unquote(encoded) == key
+
+
[email protected]("name", ["", " ", "dify_app", "应用 & Space"])
+def test_application_names_keep_aliyun_semantics(name: str) -> None:
+ assert normalize_configuration(values(app_name=name)).app_name == (name.strip() or "dify_app")
+
+
+def test_legacy_default_endpoint_and_app_name_remain_available() -> None:
+ result = normalize_configuration(values(endpoint=""))
+ assert result.endpoint == "https://tracing-analysis-dc-hz.aliyuncs.com"
+ assert result.app_name == "dify_app"
+
+
+ ("base", "expected"),
+ [
+ ("https://log.aliyuncs.com", "https://log.aliyuncs.com/adapt_synthetic-key/api/v1/traces"),
+ (
+ "https://CN-QINGDAO.LOG.ALIYUNCS.COM.:443/prefix/",
+ "https://cn-qingdao.log.aliyuncs.com.:443/prefix/adapt_synthetic-key/api/v1/traces",
+ ),
+ (
+ "https://log.aliyuncs.com.evil.test",
+ "https://log.aliyuncs.com.evil.test/adapt_synthetic-key/api/otlp/traces",
+ ),
+ ("https://evil-log.aliyuncs.com", "https://evil-log.aliyuncs.com/adapt_synthetic-key/api/otlp/traces"),
+ (
+ "https://collector.test/log.aliyuncs.com",
+ "https://collector.test/log.aliyuncs.com/adapt_synthetic-key/api/otlp/traces",
+ ),
+ ("http://[::1]:4318/reverse/", "http://[::1]:4318/reverse/adapt_synthetic-key/api/otlp/traces"),
+ ],
+)
+def test_ingestion_family_uses_only_the_hostname(base: str, expected: str) -> None:
+ assert ingestion_url(values(endpoint=base)) == expected
+
+
+ "endpoint",
+ [
+ None,
+ 1,
+ "ftp://host.test",
+ "https:///missing",
+ "https://user:[email protected]",
+ "https://host.test?",
+ "https://host.test#",
+ "https://host.test/path?key=secret",
+ "https://host.test:bad",
+ "https://host.test:0",
+ "https://host.test:65536",
+ "https://host.test:",
+ "https://[invalid",
+ "\nhttps://host.test",
+ "https://ho\nst.test",
+ "https://host.test\\@elsewhere",
+ "https://host.test/a/../b",
+ "https://host.test/a/%2e%2e/b",
+ "https://host.test/a%2fb",
+ "https://host.test/%252e%252e",
+ "https://host.test/%zz",
+ "https://host.test/double//path",
+ "https://host.test/adapt_DO-NOT-PRINT/api/otlp/traces",
+ "https://host.test/reverse/adapt_DO-NOT-PRINT/api/v1/traces/",
+ "https://host.test/%61dapt_DO-NOT-PRINT",
+ "https://host.test/api/otlp/traces",
+ "https://host.test/v1/traces",
+ ],
+)
+def test_new_endpoints_reject_unsafe_or_already_completed_routes(endpoint: object) -> None:
+ with pytest.raises(AliyunConfigurationError) as error:
+ normalize_configuration(values(endpoint=endpoint))
+ assert error.value.code == "invalid_endpoint"
+ assert "DO-NOT-PRINT" not in str(error.value)
+ assert "host.test" not in str(error.value)
+
+
[email protected]("field", ["endpoint", "license_key"])
+def test_required_fields_are_not_invented(field: str) -> None:
+ submitted = values()
+ del submitted[field]
+ with pytest.raises(AliyunConfigurationError):
+ normalize_configuration(submitted)
+
+
[email protected]("key", [None, 1, "", " ", " secret", "secret ", "secret\n", "secret\x00", "secret\ud800"])
+def test_invalid_license_is_rejected_without_echoing_it(key: object) -> None:
+ with pytest.raises(AliyunConfigurationError) as error:
+ normalize_configuration(values(license_key=key))
+ assert error.value.code == "invalid_license_key"
+ assert "secret" not in str(error.value)
+
+
[email protected]("name", [None, 1, "app\nname", "app\x00name"])
+def test_application_control_characters_are_rejected(name: object) -> None:
+ with pytest.raises(AliyunConfigurationError) as error:
+ normalize_configuration(values(app_name=name))
+ assert error.value.code == "invalid_app_name"
+
+
+def test_masks_survive_patch_normalization_but_cannot_be_sent() -> None:
+ result = normalize_configuration(values(license_key="***masked"))
+ assert result.license_key == "***masked"
+ with pytest.raises(AliyunConfigurationError):
+ ingestion_url(result.model_dump())
+
+
+def test_console_link_and_historical_model_do_not_require_new_write_validation() -> None:
+ old = AliyunConfig(license_key="synthetic-key", endpoint="https://host.test/adapt_old/api/otlp/traces")
+ assert old.endpoint == "https://host.test/adapt_old/api/otlp/traces"
+ assert console_url() == "https://arms.console.aliyun.com/#/llm"
+
+
+def test_historical_model_validation_does_not_format_sensitive_endpoint_inputs() -> None:
+ secret = "SYNTHETIC_LICENSE_NOT_IN_ERRORS"
+ with pytest.raises(ValidationError) as caught:
+ AliyunConfig(license_key=secret, endpoint=f"ftp://host.test/adapt_{secret}/api/otlp/traces")
+ rendered = str(caught.value) + repr(caught.value) + "".join(traceback.format_exception(caught.value))
+ assert secret not in rendered
+
+
+@dataclass
+class CollectorBoundary:
+ status: int = 200
+ content: bytes = b'"success"'
+ failure: Exception | None = None
+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)
+ responses: list[httpx.Response] = field(default_factory=list)
+
+
+def collector(monkeypatch: pytest.MonkeyPatch) -> CollectorBoundary:
+ boundary = CollectorBoundary()
+
+ def send(method: str, url: str, **options: object) -> httpx.Response:
+ boundary.calls.append((method, url, dict(options)))
+ if boundary.failure is not None:
+ raise boundary.failure
+ response = httpx.Response(boundary.status, content=boundary.content, request=httpx.Request(method, url))
+ boundary.responses.append(response)
+ return response
+
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ raise AssertionError("Resource-only verification must not initialize a legacy exporter")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ monkeypatch.setattr(AliyunDataTrace, "__init__", forbidden)
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://ambient.test")
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=ambient-credential")
+ return boundary
+
+
+def test_verification_emits_one_resource_and_zero_business_spans(collector: CollectorBoundary) -> None:
+ config = normalize_configuration(values(app_name="test_app", license_key="raw%2F@a+b/c"))
+ verify_configuration(config)
+ assert len(collector.calls) == 1
+ method, url, options = collector.calls[0]
+ assert method == "POST"
+ assert url == "http://tracing-analysis-dc-qd.aliyuncs.com/adapt_raw%252F@a%2Bb%2Fc/api/otlp/traces"
+ assert options["headers"] == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}
+ assert options["max_retries"] == 0
+ assert options["follow_redirects"] is False
+ body = options["content"]
+ assert isinstance(body, bytes)
+ assert body
+ payload = ExportTraceServiceRequest.FromString(body)
+ assert len(payload.resource_spans) == 1
+ resource = payload.resource_spans[0]
+ assert len(resource.scope_spans) == 0
+ assert len(resource.resource.attributes) == 1
+ assert resource.resource.attributes[0].key == "service.name"
+ assert resource.resource.attributes[0].value.string_value == "test_app"
+ assert collector.responses[0].is_closed
+
+
+ ("status", "body"),
+ [
+ (200, b""),
+ (200, b"{}"),
+ (200, b"success"),
+ (200, b'"success"\n'),
+ (200, b'{"ret":0}'),
+ (200, b'"failure"'),
+ (200, b"<html>private-response</html>"),
+ (201, b'"success"'),
+ (204, b""),
+ (400, b"private-response"),
+ (401, b"private-response"),
+ (403, b"private-response"),
+ (404, b"private-response"),
+ (405, b"private-response"),
+ (429, b"private-response"),
+ (503, b"private-response"),
+ ],
+)
+def test_only_the_observed_resource_ack_verifies_access(
+ collector: CollectorBoundary, status: int, body: bytes, caplog: pytest.LogCaptureFixture
+) -> None:
+ collector.status = status
+ collector.content = body
+ config = normalize_configuration(values(license_key="SYNTHETIC_PRIVATE_LICENSE@key"))
+ with pytest.raises(AliyunConfigurationError) as error:
+ verify_configuration(config)
+ assert error.value.code == "verification_failed"
+ rendered = str(error.value) + repr(error.value) + "".join(traceback.format_exception(error.value)) + caplog.text
+ assert "SYNTHETIC_PRIVATE_LICENSE" not in rendered
+ assert "private-response" not in rendered
+ assert len(collector.calls) == 1
+ assert all(response.is_closed for response in collector.responses)
+
+
+def test_transport_failure_does_not_escape_with_private_url(collector: CollectorBoundary) -> None:
+ collector.failure = httpx.ConnectError("request failed at /adapt_PRIVATE%40KEY/api/otlp/traces")
+ config = normalize_configuration(values(license_key="PRIVATE@KEY"))
+ with pytest.raises(AliyunConfigurationError) as error:
+ verify_configuration(config)
+ assert error.value.code == "verification_failed"
+ rendered = "".join(traceback.format_exception(error.value))
+ assert "PRIVATE@KEY" not in rendered
+ assert "PRIVATE%40KEY" not in rendered
+ assert len(collector.calls) == 1
diff --git a/api/tests/unit_tests/services/tracing/test_aliyun_otlp_ack.py b/api/tests/unit_tests/services/tracing/test_aliyun_otlp_ack.py
new file mode 100644
index 00000000000..d1c028d324f
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_aliyun_otlp_ack.py
@@ -0,0 +1,241 @@
+"""Exercise Aliyun's observed HTTP ACK through both real protobuf owners."""
+
+import logging
+from collections.abc import Mapping
+from datetime import UTC, datetime, timedelta
+from typing import Literal, TypedDict, Unpack
+
+import httpx
+import pytest
+from dify_trace_aliyun.data_exporter.traceclient import TraceClient
+from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData
+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+ ExportTraceServiceRequest,
+ ExportTraceServiceResponse,
+)
+
+from core.helper import ssrf_proxy
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.otlp_export import export_http, is_aliyun_http_success_ack
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError, export_trace
+
+ENDPOINT = "https://collector.test/adapt_synthetic-license/api/otlp/traces"
+START = datetime(2026, 10, 19, tzinfo=UTC)
+TRACE_ID = "1" * 32
+SPAN_ID = "2" * 16
+
+type Owner = Literal["managed", "legacy"]
+
+
+class RequestOptions(TypedDict, total=False):
+ content: bytes
+ headers: Mapping[str, str]
+ max_retries: int
+ timeout: float
+ follow_redirects: bool
+
+
+def boundary(
+ monkeypatch: pytest.MonkeyPatch, status: int, content: bytes, content_type: str
+) -> list[ExportTraceServiceRequest]:
+ received: list[ExportTraceServiceRequest] = []
+
+ def send(method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ assert method == "POST"
+ assert url == ENDPOINT
+ assert options["max_retries"] == 0
+ assert options["follow_redirects"] is False
+ assert options["headers"] == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True
+ received.append(ExportTraceServiceRequest.FromString(options["content"]))
+ headers = {"Content-Type": content_type} if content_type else {}
+ return httpx.Response(status, content=content, headers=headers)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ return received
+
+
+def dispatch(owner: Owner) -> None:
+ if owner == "managed":
+ trace = CanonicalTrace(
+ trace_id=TRACE_ID,
+ root_span_id=SPAN_ID,
+ session_id="synthetic-conversation",
+ spans=(
+ CanonicalSpan(
+ id=SPAN_ID,
+ parent_id=None,
+ name="synthetic root",
+ kind=CanonicalSpanKind.AGENT,
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ status=CanonicalSpanStatus.OK,
+ ),
+ ),
+ )
+ result = export_trace(
+ "aliyun",
+ {"endpoint": "https://collector.test", "license_key": "synthetic-license", "app_name": "synthetic"},
+ trace,
+ )
+ assert result.status == "accepted"
+ else:
+ client = TraceClient("synthetic", ENDPOINT)
+ with client.batch():
+ client.add_span(
+ SpanData(
+ trace_id=int(TRACE_ID, 16),
+ span_id=int(SPAN_ID, 16),
+ name="synthetic root",
+ start_time=int(START.timestamp()) * 1_000_000_000,
+ end_time=int((START + timedelta(seconds=1)).timestamp()) * 1_000_000_000,
+ )
+ )
+ assert client._dispatch_spans.get() is None
+
+
+def assert_one_batch(received: list[ExportTraceServiceRequest]) -> None:
+ assert len(received) == 1
+ spans = received[0].resource_spans[0].scope_spans[0].spans
+ assert len(spans) == 1
+ assert spans[0].trace_id.hex() == TRACE_ID
+ assert spans[0].span_id.hex() == SPAN_ID
+
+
[email protected]("owner", ["managed", "legacy"])
[email protected]("content_type", ["", "application/json", "application/x-protobuf"])
+def test_exact_observed_ack_accepts_one_batch_without_changing_its_ids(
+ monkeypatch: pytest.MonkeyPatch, owner: Owner, content_type: str
+) -> None:
+ received = boundary(monkeypatch, 200, b'"success"', content_type)
+ dispatch(owner)
+ assert_one_batch(received)
+
+
[email protected]("owner", ["managed", "legacy"])
+ ("content", "content_type"),
+ [
+ (b"success", ""),
+ (b'"SUCCESS"', ""),
+ (b' "success"', "application/json"),
+ (b'"success"\n', "application/json"),
+ (b'{"success":true}', "application/json"),
+ (b'{"message":"success"}', "application/json"),
+ (b"true", "application/json"),
+ (b"not protobuf", ""),
+ ],
+)
+def test_other_ack_bodies_are_uncertain_and_not_replayed(
+ monkeypatch: pytest.MonkeyPatch, owner: Owner, content: bytes, content_type: str
+) -> None:
+ received = boundary(monkeypatch, 200, content, content_type)
+ with pytest.raises(TraceExportUncertainError, match="unreadable ingestion acknowledgment") as error:
+ dispatch(owner)
+ assert error.value.acceptance_unknown is True
+ assert_one_batch(received)
+
+
[email protected]("owner", ["managed", "legacy"])
[email protected]("status", [201, 202, 204])
+def test_literal_ack_does_not_accept_another_success_status(
+ monkeypatch: pytest.MonkeyPatch, owner: Owner, status: int
+) -> None:
+ received = boundary(monkeypatch, status, b'"success"', "")
+ assert is_aliyun_http_success_ack(status, b'"success"') is False
+ with pytest.raises(TraceExportUncertainError) as error:
+ dispatch(owner)
+ assert error.value.acceptance_unknown is True
+ assert_one_batch(received)
+
+
[email protected]("owner", ["managed", "legacy"])
[email protected]("status", [403, 429, 503])
+def test_literal_ack_does_not_override_http_rejection_or_known_retry(
+ monkeypatch: pytest.MonkeyPatch, owner: Owner, status: int
+) -> None:
+ received = boundary(monkeypatch, status, b'"success"', "")
+ expected = TraceExportRejectedError if status == 403 else RetryableTraceDispatchError
+ with pytest.raises(expected) as error:
+ dispatch(owner)
+ if isinstance(error.value, RetryableTraceDispatchError):
+ assert error.value.acceptance_unknown is False
+ assert_one_batch(received)
+
+
[email protected]("owner", ["managed", "legacy"])
[email protected]("encoding", ["protobuf", "json"])
[email protected]("rejected_spans", [0, 1])
+def test_standard_otlp_ack_and_partial_success_remain_validated(
+ monkeypatch: pytest.MonkeyPatch, owner: Owner, encoding: str, rejected_spans: int
+) -> None:
+ if encoding == "protobuf":
+ response = ExportTraceServiceResponse()
+ response.partial_success.rejected_spans = rejected_spans
+ response.partial_success.error_message = "synthetic warning"
+ content = response.SerializeToString()
+ content_type = "application/x-protobuf"
+ else:
+ content = b'{"partialSuccess":{"rejectedSpans":"%d","errorMessage":"synthetic warning"}}' % rejected_spans
+ content_type = "application/json"
+ received = boundary(monkeypatch, 200, content, content_type)
+ if rejected_spans:
+ with pytest.raises(TraceExportUncertainError, match="partially accepted") as error:
+ dispatch(owner)
+ assert error.value.acceptance_unknown is True
+ else:
+ dispatch(owner)
+ assert_one_batch(received)
+
+
[email protected]("owner", ["managed", "legacy"])
+def test_damaged_compressed_ack_is_uncertain_after_real_http_submission(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, owner: Owner
+) -> None:
+ received: list[ExportTraceServiceRequest] = []
+ previous_suppression = get_value(_SUPPRESS_INSTRUMENTATION_KEY)
+
+ def send(wire: httpx.Request) -> httpx.Response:
+ assert wire.method == "POST"
+ assert str(wire.url) == ENDPOINT
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True
+ received.append(ExportTraceServiceRequest.FromString(wire.content))
+ # Reading a malformed compressed response raises DecodingError, which
+ # is a RequestError but not a TransportError. Submission already ran.
+ return httpx.Response(
+ 200,
+ headers={"Content-Encoding": "gzip"},
+ stream=httpx.ByteStream(b"invalid gzip acknowledgment"),
+ request=wire,
+ )
+
+ with httpx.Client(transport=httpx.MockTransport(send)) as client:
+
+ def get_client(ssl_verify_enabled: bool) -> httpx.Client:
+ assert isinstance(ssl_verify_enabled, bool)
+ return client
+
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", get_client)
+ with caplog.at_level(logging.INFO, logger="httpx"):
+ with pytest.raises(TraceExportUncertainError) as error:
+ dispatch(owner)
+ assert error.value.acceptance_unknown is True
+ assert error.value.__cause__ is None
+ assert error.value.__context__ is None
+ assert "synthetic-license" not in str(error.value) + caplog.text
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == previous_suppression
+ assert_one_batch(received)
+
+
[email protected]("provider", ["langfuse", "phoenix", "tencent"])
[email protected]("content_type", ["", "application/json"])
+def test_literal_success_ack_does_not_expand_other_providers(
+ monkeypatch: pytest.MonkeyPatch, provider: str, content_type: str
+) -> None:
+ received = boundary(monkeypatch, 200, b'"success"', content_type)
+ with pytest.raises(TraceExportUncertainError, match="unreadable ingestion acknowledgment") as error:
+ export_http(provider, ENDPOINT, {}, ExportTraceServiceRequest())
+ assert error.value.acceptance_unknown is True
+ assert len(received) == 1
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_collection.py b/api/tests/unit_tests/services/tracing/test_application_trace_collection.py
new file mode 100644
index 00000000000..84f421cc9f2
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_collection.py
@@ -0,0 +1,519 @@
+"""Exercise collection transactions and delivery state against real SQLite rows."""
+
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
+
+import pytest
+from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceEvent, AgentTraceSpan
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.entities.trace_entity import MessageTraceInfo
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun
+from models.enums import ConversationFromSource
+from models.model import App, AppMode, Message, TraceAppConfig
+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError
+from services.tracing import service as service_module
+from services.tracing.provider_dispatch import TraceExportResult, TraceExportUncertainError
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_app, make_message
+
+
+def identifier(value: str) -> str:
+ return str(uuid5(NAMESPACE_URL, value))
+
+
+@dataclass
+class Collection:
+ repository: ApplicationTraceRepository
+ sessions: sessionmaker[Session]
+ run: ApplicationTraceRun
+
+ def invocation(self, name: str = "first") -> ApplicationTraceInvocation:
+ return self.repository.invocation(
+ run_id=self.run.id, agent_id=identifier("agent"), node_execution_id=None, attempt=0, call_key=name
+ )
+
+
+def seed(
+ sessions: sessionmaker[Session],
+ name: str,
+ *,
+ capture_content: bool = False,
+ enabled: bool = True,
+ capture_agent: bool = True,
+) -> Collection:
+ app_id, tenant_id, message_id = identifier(f"app-{name}"), identifier("tenant"), identifier(f"message-{name}")
+ with sessions.begin() as session:
+ session.add(
+ make_app(
+ app_id=app_id,
+ tenant_id=tenant_id,
+ mode=AppMode.AGENT,
+ tracing=json.dumps(
+ {
+ "enabled": enabled,
+ "tracing_provider": "langfuse",
+ "capture_content": capture_content,
+ "capture_agent": capture_agent,
+ }
+ ),
+ )
+ )
+ session.add(
+ make_message(
+ message_id=message_id,
+ app_id=app_id,
+ conversation_id=identifier(f"conversation-{name}"),
+ inputs={},
+ query="hello",
+ message={},
+ answer="done",
+ message_unit_price=Decimal(0),
+ answer_unit_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.CONSOLE,
+ )
+ )
+ session.add(
+ TraceAppConfig(
+ app_id=app_id,
+ tracing_provider="langfuse",
+ tracing_config={
+ "host": "https://langfuse.test",
+ "secret_key": "encrypted-original",
+ "project_url": "https://langfuse.test/project/original",
+ },
+ )
+ )
+ repository = ApplicationTraceRepository(sessions)
+ run = repository.start(
+ tenant_id=tenant_id, app_id=app_id, source_kind="message", source_id=message_id, session_id="session-original"
+ )
+ assert run is not None
+ return Collection(repository, sessions, run)
+
+
+def collection(sqlite_session_factory: sessionmaker[Session]) -> Collection:
+ return seed(sqlite_session_factory, "first")
+
+
+def batch_for(
+ run: ApplicationTraceRun,
+ invocation: ApplicationTraceInvocation,
+ *,
+ final: bool = True,
+ batch_id: UUID | None = None,
+ span_id: str = "a" * 16,
+) -> AgentTraceBatch:
+ return AgentTraceBatch(
+ run_id=UUID(run.id),
+ invocation_id=UUID(invocation.id),
+ batch_id=batch_id or uuid4(),
+ final=final,
+ spans=[
+ AgentTraceSpan(
+ trace_id=run.trace_id,
+ span_id=span_id,
+ parent_span_id=invocation.span_id,
+ name="Agent execution",
+ start_time_ns=1_000_000_000,
+ end_time_ns=2_000_000_000,
+ status="ok",
+ attributes={"gen_ai.operation.name": "invoke_agent"},
+ )
+ ],
+ )
+
+
+def ingest(collection: Collection, batch: AgentTraceBatch) -> tuple[int, str]:
+ return collection.repository.ingest(batch, max_spans=100, max_bytes=1_000_000)
+
+
+def terminal(collection: Collection) -> None:
+ run = collection.run
+ root = CanonicalSpan(
+ id=run.root_span_id,
+ parent_id=None,
+ name="Agent turn",
+ kind=CanonicalSpanKind.CHAIN,
+ start_time=datetime.fromtimestamp(0, UTC),
+ end_time=datetime.fromtimestamp(3, UTC),
+ status=CanonicalSpanStatus.OK,
+ )
+ trace = CanonicalTrace(trace_id=run.trace_id, session_id=run.session_id, root_span_id=root.id, spans=(root,))
+ collection.repository.terminal(run.id, trace.model_dump(mode="json"), root.id)
+
+
+def claim(collection: Collection) -> tuple[ApplicationTraceRun, str] | None:
+ return collection.repository.claim(collection.run.id, collection_timeout=60, lease_seconds=30)
+
+
+def test_batch_duplicate_is_idempotent_after_final_and_acceptance(collection: Collection) -> None:
+ invocation = collection.invocation()
+ batch = batch_for(collection.run, invocation)
+ assert ingest(collection, batch) == (1, collection.run.id)
+ assert ingest(collection, batch) == (1, collection.run.id)
+ terminal(collection)
+ leased = claim(collection)
+ assert leased is not None
+ collection.repository.complete(
+ collection.run.id, leased[1], state="accepted", result=TraceExportResult("provider-id")
+ )
+ assert ingest(collection, batch) == (1, collection.run.id)
+ stored = collection.repository.get(collection.run.id)
+ assert stored is not None
+ assert stored.span_count == 1
+ with collection.sessions() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 1
+
+
+def test_conflicting_batch_id_rolls_back_without_replacing_payload(collection: Collection) -> None:
+ batch = batch_for(collection.run, collection.invocation(), final=False)
+ ingest(collection, batch)
+ changed = batch.model_copy(update={"final": True})
+ with pytest.raises(TraceCollectionError, match="tracing_batch_conflict"):
+ ingest(collection, changed)
+ invocations, batches = collection.repository.collected(collection.run.id)
+ assert invocations[0].finalized is False
+ assert batches[0].final is False
+
+
+def test_run_cannot_ingest_another_runs_invocation(collection: Collection) -> None:
+ other = seed(collection.sessions, "other")
+ foreign = other.invocation()
+ forged = batch_for(collection.run, foreign)
+ with pytest.raises(TraceCollectionError, match="tracing_invocation_not_found"):
+ ingest(collection, forged)
+ with collection.sessions() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0
+
+
+def test_payload_limits_reject_whole_transaction(collection: Collection) -> None:
+ batch = batch_for(collection.run, collection.invocation())
+ with pytest.raises(TraceCollectionError, match="tracing_payload_limit"):
+ collection.repository.ingest(batch, max_spans=100, max_bytes=1)
+ invocations, batches = collection.repository.collected(collection.run.id)
+ assert not batches
+ assert invocations[0].finalized is False
+ stored = collection.repository.get(collection.run.id)
+ assert stored is not None
+ assert stored.span_count == 0
+
+
+def test_wrong_trace_id_is_rejected_before_persistence(collection: Collection) -> None:
+ batch = batch_for(collection.run, collection.invocation())
+ altered = batch.model_copy(update={"spans": [batch.spans[0].model_copy(update={"trace_id": "f" * 32})]})
+ with pytest.raises(TraceCollectionError, match="tracing_trace_id_mismatch"):
+ ingest(collection, altered)
+ _, batches = collection.repository.collected(collection.run.id)
+ assert not batches
+
+
+def test_content_off_cleans_the_durable_batch_not_only_the_export(collection: Collection) -> None:
+ original = batch_for(collection.run, collection.invocation(), final=False)
+ observation = original.spans[0].model_copy(
+ update={
+ "status": "error",
+ "status_message": "private error",
+ "attributes": {
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": "model-one",
+ "gen_ai.input.messages": "private prompt",
+ "gen_ai.tool.call.arguments": "private arguments",
+ "dify.payload": "private metadata",
+ "authorization": "private credential",
+ },
+ "events": [
+ AgentTraceEvent(
+ name="exception", timestamp_ns=1_500_000_000, attributes={"exception.message": "private event"}
+ )
+ ],
+ }
+ )
+ batch = original.model_copy(update={"spans": [observation]})
+ ingest(collection, batch)
+ _, batches = collection.repository.collected(collection.run.id)
+ assert "private" not in json.dumps(batches[0].spans)
+ assert batches[0].spans[0]["events"] == []
+ assert batches[0].spans[0]["status"] == "error"
+ assert batches[0].spans[0]["attributes"]["gen_ai.request.model"] == "model-one"
+ # A changed original payload is a conflict even if content-off sanitization
+ # would produce exactly the same persisted span.
+ changed = batch.model_copy(
+ update={"spans": [observation.model_copy(update={"status_message": "different private error"})]}
+ )
+ with pytest.raises(TraceCollectionError, match="tracing_batch_conflict"):
+ ingest(collection, changed)
+
+
+def test_invocation_key_replay_must_not_change_agent_binding(collection: Collection) -> None:
+ original = collection.invocation()
+ repeated = collection.invocation()
+ assert original.id == repeated.id
+ with pytest.raises(TraceCollectionError, match="tracing_invocation_conflict"):
+ collection.repository.invocation(
+ run_id=collection.run.id,
+ agent_id=identifier("different-agent"),
+ node_execution_id=identifier("different-node"),
+ attempt=2,
+ call_key="first",
+ )
+
+
+def test_claim_waits_for_all_invocations_final_and_then_only_one_lease(collection: Collection) -> None:
+ first, second = collection.invocation("first"), collection.invocation("second")
+ terminal(collection)
+ ingest(collection, batch_for(collection.run, first))
+ assert claim(collection) is None
+ ingest(collection, batch_for(collection.run, second, span_id="b" * 16))
+ leased = claim(collection)
+ assert leased is not None
+ assert claim(collection) is None
+ assert leased[0].attempts == 1
+
+
+def test_expired_delivery_lease_accepts_confirmed_late_ack_for_same_attempt(collection: Collection) -> None:
+ ingest(collection, batch_for(collection.run, collection.invocation()))
+ terminal(collection)
+ leased = claim(collection)
+ assert leased is not None
+ with collection.sessions.begin() as session:
+ stored = session.get(ApplicationTraceRun, collection.run.id)
+ assert stored is not None
+ stored.export_started_at = naive_utc_now() - timedelta(seconds=60)
+ stored.export_heartbeat_at = stored.export_started_at
+ assert claim(collection) is None
+ current = collection.repository.get(collection.run.id)
+ assert current is not None
+ assert current.state == "uncertain"
+ collection.repository.complete(collection.run.id, leased[1], state="accepted", result=TraceExportResult("late"))
+ stored = collection.repository.get(collection.run.id)
+ assert stored is not None
+ assert stored.state == "accepted"
+ assert stored.error_code is None
+ assert stored.accepted_at is not None
+ assert stored.provider_trace_id == "late"
+ assert claim(collection) is None
+
+
+def test_incomplete_final_times_out_without_export(collection: Collection) -> None:
+ collection.invocation()
+ terminal(collection)
+ with collection.sessions.begin() as session:
+ stored = session.get(ApplicationTraceRun, collection.run.id)
+ assert stored is not None
+ stored.terminal_at = naive_utc_now() - timedelta(seconds=120)
+ assert claim(collection) is None
+ stored = collection.repository.get(collection.run.id)
+ assert stored is not None
+ assert stored.state == "failed"
+ assert stored.error_code == "agent_trace_collection_incomplete"
+
+
+def test_running_trace_freezes_credentials_content_policy_and_session(collection: Collection) -> None:
+ original = collection.run
+ with collection.sessions.begin() as session:
+ app = session.get(App, original.app_id)
+ assert app is not None
+ app.tracing = json.dumps({"enabled": False, "tracing_provider": "opik", "capture_content": True})
+ config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == original.app_id))
+ assert config is not None
+ config.tracing_config = {"secret_key": "encrypted-new"}
+ repeated = collection.repository.start(
+ tenant_id=original.tenant_id,
+ app_id=original.app_id,
+ source_kind="message",
+ source_id=original.source_id,
+ session_id="changed",
+ )
+ assert repeated is not None
+ assert repeated.provider == "langfuse"
+ assert repeated.provider_config["secret_key"] == "encrypted-original"
+ assert repeated.capture_content is False
+ assert repeated.config_revision == original.config_revision
+ assert repeated.session_id == "session-original"
+
+
[email protected]("outcome", ["accepted", "uncertain", "retrying"])
+def test_delivery_checkpoint_controls_replays(
+ collection: Collection, monkeypatch: pytest.MonkeyPatch, outcome: str
+) -> None:
+ ingest(collection, batch_for(collection.run, collection.invocation()))
+ terminal(collection)
+ exports: list[tuple[str, dict[str, object], CanonicalTrace]] = []
+ decryptions: list[dict[str, object]] = []
+
+ def decrypt(_tenant: str, _provider: str, config: dict[str, object]) -> dict[str, object]:
+ decryptions.append(config)
+ return {"secret_key": "plaintext-only-in-worker"}
+
+ def export(provider: str, config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:
+ exports.append((provider, config, trace))
+ if outcome == "uncertain":
+ raise TraceExportUncertainError("ack lost")
+ if outcome == "retrying":
+ raise RetryableTraceDispatchError("busy")
+ return TraceExportResult("vendor-trace", "https://trace.test/example")
+
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", decrypt)
+ monkeypatch.setattr(service_module, "export_trace", export)
+ service = ApplicationTracingService(collection.repository)
+ service.deliver(collection.run.id)
+ service.deliver(collection.run.id)
+ assert len(exports) == 1
+ assert decryptions[0]["secret_key"] == "encrypted-original"
+ stored = collection.repository.get(collection.run.id)
+ assert stored is not None
+ assert stored.state == outcome
+ assert stored.provider_config["secret_key"] == "encrypted-original"
+ assert "plaintext-only-in-worker" not in json.dumps(stored.outer_trace)
+ assert exports[0][1]["host"] == "https://langfuse.test"
+
+
+def test_source_owner_checks_app_and_tenant(collection: Collection) -> None:
+ run = collection.run
+ assert (
+ collection.repository.start(
+ tenant_id=identifier("foreign-tenant"),
+ app_id=run.app_id,
+ source_kind="message",
+ source_id=run.source_id,
+ session_id="x",
+ )
+ is None
+ )
+ other = seed(collection.sessions, "second-app")
+ assert (
+ collection.repository.start(
+ tenant_id=run.tenant_id,
+ app_id=other.run.app_id,
+ source_kind="message",
+ source_id=run.source_id,
+ session_id="x",
+ )
+ is None
+ )
+
+
+def test_disabled_source_decision_stays_disabled_after_config_enable(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ case = seed(sqlite_session_factory, "initially-disabled", enabled=False)
+ service = ApplicationTracingService(case.repository)
+ assert case.run.state == "not_collected"
+ with case.sessions.begin() as session:
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.tracing = json.dumps({"enabled": True, "tracing_provider": "langfuse", "capture_content": True})
+ message = session.get(Message, case.run.source_id)
+ assert message is not None
+ assert (
+ service.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="message",
+ source_id=case.run.source_id,
+ session_id="later-session",
+ )
+ is None
+ )
+ info = MessageTraceInfo(
+ metadata={},
+ message_data=message,
+ message_id=message.id,
+ conversation_model="agent",
+ conversation_mode="agent",
+ message_tokens=1,
+ answer_tokens=1,
+ total_tokens=2,
+ )
+ assert service.capture_terminal(info) is True
+ stored = case.repository.get(case.run.id)
+ assert stored is not None
+ assert stored.state == "not_collected"
+ assert stored.provider_config == {}
+ assert stored.outer_trace is None
+
+
+def test_capture_agent_disabled_preserves_outer_only_content_policy(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ case = seed(sqlite_session_factory, "outer-only", capture_agent=False)
+ service = ApplicationTracingService(case.repository)
+ scheduled: list[str] = []
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ assert (
+ service.prepare_invocation(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ agent_id=identifier("agent"),
+ source_kind="message",
+ source_id=case.run.source_id,
+ session_id="session",
+ node_execution_id=None,
+ attempt=0,
+ call_key="call",
+ )
+ is None
+ )
+ with case.sessions() as session:
+ message = session.get(Message, case.run.source_id)
+ assert message is not None
+ info = MessageTraceInfo(
+ metadata={},
+ message_data=message,
+ message_id=message.id,
+ conversation_model="agent",
+ conversation_mode="agent",
+ message_tokens=1,
+ answer_tokens=1,
+ total_tokens=2,
+ inputs="private input",
+ outputs="private output",
+ )
+ assert service.capture_terminal(info) is True
+ assert scheduled == [case.run.id]
+ stored = case.repository.get(case.run.id)
+ assert stored is not None
+ assert stored.outer_trace is not None
+ assert "private" not in json.dumps(stored.outer_trace)
+ assert len(stored.outer_trace["spans"]) == 1
+ invocations, _ = case.repository.collected(case.run.id)
+ assert invocations == []
+
+
+def test_deleting_selected_configuration_keeps_historical_destination(collection: Collection) -> None:
+ from machinery.context import RequestContext
+ from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository
+ from services.app_tracing_config_gateway import OpsTraceManagerGateway
+ from services.app_tracing_config_service import AppTracingConfigService
+
+ original = collection.run
+ service = AppTracingConfigService(
+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=collection.sessions),
+ provider=OpsTraceManagerGateway(),
+ )
+ service.delete(RequestContext("request", None, "account", original.tenant_id), original.app_id, "langfuse")
+ stored = collection.repository.get(original.id)
+ assert stored is not None
+ assert stored.provider_config == original.provider_config
+ assert stored.provider_config["project_url"] == "https://langfuse.test/project/original"
+ assert stored.config_revision == original.config_revision
+ with collection.sessions() as session:
+ assert session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == original.app_id)) is None
+ app = session.get(App, original.app_id)
+ assert app is not None
+ assert app.tracing is not None
+ assert json.loads(app.tracing)["enabled"] is False
+ assert json.loads(app.tracing)["tracing_provider"] is None
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_handoff.py b/api/tests/unit_tests/services/tracing/test_application_trace_handoff.py
new file mode 100644
index 00000000000..55ed619edfa
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_handoff.py
@@ -0,0 +1,468 @@
+"""SQLite ownership regressions at the actual legacy-task handoff boundary."""
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from uuid import UUID, uuid4
+
+import httpx
+import pytest
+from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceSpan
+from flask import Flask
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+from sqlalchemy import delete, func, select
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
+from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
+from core.ops.base_trace_instance import bind_application_tracing_hooks
+from core.ops.entities.trace_entity import (
+ BaseTraceInfo,
+ GenerateNameTraceInfo,
+ MessageTraceInfo,
+ ModerationTraceInfo,
+ WorkflowTraceInfo,
+)
+from core.ops.ops_trace_manager import OpsTraceManager
+from extensions.ext_database import db
+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus
+from models.agent_config_entities import AgentSoulConfig
+from models.application_trace import ApplicationTraceRun
+from models.enums import ConversationFromSource, CreatorUserRole, WorkflowRunTriggeredFrom
+from models.model import App, AppMode, Message
+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun, WorkflowType
+from services.tracing import runtime_bridge
+from services.tracing.provider_dispatch import export_trace
+from services.tracing.service import ApplicationTracingService
+from tasks import ops_trace_task
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_account, make_conversation, make_workflow
+from tests.unit_tests.services.tracing.test_application_trace_collection import Collection, identifier, seed, terminal
+
+AGENT_GRAPH = {
+ "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}],
+ "edges": list[object](),
+}
+
+
+@dataclass
+class Case:
+ collection: Collection
+ service: ApplicationTracingService
+ scheduled: list[str]
+
+
+def setup(
+ sessions: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, name: str, mode: AppMode, *, snapshot: bool
+) -> Case:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ collection = seed(sessions, name)
+ with sessions.begin() as session:
+ app = session.get(App, collection.run.app_id)
+ message = session.get(Message, collection.run.source_id)
+ assert app is not None
+ assert message is not None
+ app.mode = mode
+ session.add(
+ make_conversation(
+ conversation_id=message.conversation_id,
+ app_id=app.id,
+ mode=mode,
+ inputs={},
+ from_source=ConversationFromSource.CONSOLE,
+ )
+ )
+ if not snapshot:
+ session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id == collection.run.id))
+ service = ApplicationTracingService(collection.repository)
+ scheduled: list[str] = []
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ monkeypatch.setattr(runtime_bridge, "_service", lambda: service)
+ return Case(collection, service, scheduled)
+
+
+def message_info(case: Case) -> MessageTraceInfo:
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ return MessageTraceInfo(
+ metadata={"app_id": message.app_id},
+ message_id=message.id,
+ message_data=message.to_dict(session=session),
+ conversation_model="agent",
+ conversation_mode="agent",
+ message_tokens=0,
+ answer_tokens=0,
+ total_tokens=0,
+ inputs="private input",
+ outputs="private output",
+ )
+
+
+def dispatch(case: Case, info: BaseTraceInfo, monkeypatch: pytest.MonkeyPatch) -> list[BaseTraceInfo]:
+ calls: list[BaseTraceInfo] = []
+ failures: list[str] = []
+
+ class Legacy:
+ def trace(self, restored: BaseTraceInfo) -> None:
+ calls.append(restored)
+
+ payload = json.dumps({"trace_info_type": type(info).__name__, "trace_info": info.model_dump(mode="json")})
+ monkeypatch.setattr(ops_trace_task.storage, "load", lambda _path: payload)
+ monkeypatch.setattr(ops_trace_task.storage, "delete", lambda _path: None)
+ monkeypatch.setattr(ops_trace_task.redis_client, "incr", failures.append)
+ monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lambda _app: Legacy())
+ monkeypatch.setattr("extensions.ext_enterprise_telemetry.is_enabled", lambda: False)
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, runtime_bridge.RuntimeApplicationTracingHooks())
+ with app.app_context():
+ ops_trace_task.process_trace_tasks.run({"app_id": case.collection.run.app_id, "file_id": "fixture"})
+ assert failures == []
+ return calls
+
+
+def workflow(
+ case: Case, name: str, *, has_agent: bool, snapshot: bool
+) -> tuple[WorkflowRun, ApplicationTraceRun | None]:
+ graph = AGENT_GRAPH if has_agent else {"nodes": [{"data": {"type": "llm"}}], "edges": list[object]()}
+ now = datetime(2026, 10, 15)
+ with case.collection.sessions.begin() as session:
+ definition = make_workflow(
+ workflow_id=identifier(f"workflow-{name}"),
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ graph=graph,
+ )
+ session.add(definition)
+ app = session.get(App, case.collection.run.app_id)
+ assert app is not None
+ app.workflow_id = definition.id
+ row = WorkflowRun(
+ id=identifier(f"workflow-run-{name}"),
+ app_id=app.id,
+ tenant_id=app.tenant_id,
+ workflow_id=definition.id,
+ type=WorkflowType.WORKFLOW,
+ version="1",
+ graph=json.dumps(graph),
+ inputs="{}",
+ outputs="{}",
+ status=WorkflowExecutionStatus.SUCCEEDED,
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by=identifier("owner"),
+ created_at=now,
+ finished_at=now + timedelta(seconds=2),
+ elapsed_time=2,
+ total_tokens=0,
+ total_steps=0,
+ )
+ session.add(row)
+ trace_run = (
+ case.collection.repository.start(
+ tenant_id=row.tenant_id,
+ app_id=row.app_id,
+ source_kind="workflow",
+ source_id=row.id,
+ session_id="workflow-session",
+ )
+ if snapshot
+ else None
+ )
+ return row, trace_run
+
+
+def workflow_info(row: WorkflowRun, parent: WorkflowRun | None = None) -> WorkflowTraceInfo:
+ metadata: dict[str, object] = {"app_id": row.app_id}
+ if parent is not None:
+ metadata["parent_trace_context"] = {
+ "parent_workflow_run_id": parent.id,
+ "parent_node_execution_id": identifier("parent-tool-node"),
+ }
+ return WorkflowTraceInfo(
+ workflow_data=row,
+ workflow_id=row.workflow_id,
+ tenant_id=row.tenant_id,
+ workflow_run_id=row.id,
+ workflow_run_elapsed_time=2,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={},
+ workflow_run_outputs={},
+ workflow_run_version="1",
+ total_tokens=0,
+ file_list=[],
+ query="",
+ metadata=metadata,
+ start_time=row.created_at,
+ end_time=row.finished_at,
+ )
+
+
+def test_chatflow_message_event_cannot_export_again_via_legacy(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ case = setup(sqlite_session_factory, monkeypatch, "chatflow", AppMode.ADVANCED_CHAT, snapshot=False)
+ row, trace_run = workflow(case, "chatflow", has_agent=True, snapshot=True)
+ assert trace_run is not None
+ terminal(Collection(case.collection.repository, case.collection.sessions, trace_run))
+ before = case.collection.repository.get(trace_run.id)
+ assert before is not None
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.workflow_run_id = row.id
+ assert dispatch(case, message_info(case), monkeypatch) == []
+ after = case.collection.repository.get(trace_run.id)
+ assert after is not None
+ assert after.outer_trace == before.outer_trace
+ assert case.scheduled == []
+ with case.collection.sessions() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceRun)) == 1
+
+
[email protected]("mode", [AppMode.AGENT, AppMode.ADVANCED_CHAT])
+def test_snapshot_failure_is_persisted_and_never_falls_back(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, mode: AppMode
+) -> None:
+ case = setup(sqlite_session_factory, monkeypatch, "failure", mode, snapshot=False)
+
+ def fail_start(**_kwargs: object) -> None:
+ raise OperationalError("snapshot", {}, Exception("temporary database unavailable"))
+
+ monkeypatch.setattr(case.collection.repository, "start", fail_start)
+ context = AgentAppRuntimeBuildContext(
+ dify_context=DifyRunContext(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ user_id=identifier("user"),
+ user_from=UserFrom.ACCOUNT,
+ invoke_from=InvokeFrom.DEBUGGER,
+ ),
+ agent_id=identifier("agent"),
+ idempotency_key=case.collection.run.source_id,
+ conversation_id="conversation",
+ agent_config_snapshot_id=identifier("snapshot"),
+ agent_soul=AgentSoulConfig(),
+ user_query="query",
+ binding_id=identifier("binding"),
+ backend_binding_ref="backend",
+ )
+ assert runtime_bridge.prepare_agent_app(context) is None
+ assert dispatch(case, message_info(case), monkeypatch) == []
+ stored = case.collection.repository.source_run(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ source_kind="message",
+ source_id=case.collection.run.source_id,
+ )
+ assert stored is not None
+ assert stored.state == "failed"
+ assert stored.error_code == "tracing_initialization_failed"
+ assert stored.provider_config == {}
+
+
[email protected]("mode", [AppMode.AGENT, AppMode.ADVANCED_CHAT, AppMode.CHAT])
[email protected]("kind", ["title", "moderation", "aliased_moderation"])
+def test_auxiliary_events_resolve_owner_without_bypassing_content_policy(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, mode: AppMode, kind: str
+) -> None:
+ case = setup(sqlite_session_factory, monkeypatch, "aux", mode, snapshot=False)
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ if kind == "title":
+ info: BaseTraceInfo = GenerateNameTraceInfo(
+ tenant_id=case.collection.run.tenant_id,
+ conversation_id=message.conversation_id,
+ metadata={"conversation_id": message.conversation_id},
+ inputs="private conversation",
+ outputs="private generated title",
+ )
+ else:
+ info = ModerationTraceInfo(
+ message_id=identifier("workflow-app-log") if kind == "aliased_moderation" else message.id,
+ message_data=message.to_dict(session=session),
+ metadata={"message_id": message.id},
+ flagged=False,
+ action="direct_output",
+ preset_response="",
+ query="private prompt",
+ inputs="private prompt",
+ outputs="private output",
+ )
+ legacy = dispatch(case, info, monkeypatch)
+ assert len(legacy) == (1 if mode is AppMode.CHAT else 0)
+ assert case.scheduled == []
+
+
+def test_plain_workflow_preserves_legacy_parent_context(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ parent_case = setup(sqlite_session_factory, monkeypatch, "plain-parent", AppMode.WORKFLOW, snapshot=False)
+ parent, _ = workflow(parent_case, "plain-parent", has_agent=False, snapshot=False)
+ child_case = setup(sqlite_session_factory, monkeypatch, "plain-child", AppMode.WORKFLOW, snapshot=False)
+ child, _ = workflow(child_case, "plain-child", has_agent=False, snapshot=False)
+ info = workflow_info(child, parent)
+ original = dict(info.metadata)
+ assert child_case.service.capture_terminal(info) is False
+ assert info.metadata == original
+ with child_case.collection.sessions() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceRun)) == 0
+
+
+def test_plain_chatflow_auxiliary_events_keep_the_legacy_route(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ case = setup(sqlite_session_factory, monkeypatch, "plain-chatflow", AppMode.ADVANCED_CHAT, snapshot=False)
+ row, _ = workflow(case, "plain-chatflow", has_agent=False, snapshot=False)
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.workflow_run_id = row.id
+ info = GenerateNameTraceInfo(
+ tenant_id=row.tenant_id,
+ conversation_id=message.conversation_id,
+ metadata={"conversation_id": message.conversation_id},
+ inputs="legacy input",
+ outputs="legacy title",
+ )
+ assert len(dispatch(case, info, monkeypatch)) == 1
+
+
+def test_legacy_child_of_managed_parent_uses_explicit_link_not_unavailable_context(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ parent_case = setup(sqlite_session_factory, monkeypatch, "managed-parent", AppMode.WORKFLOW, snapshot=False)
+ parent, parent_trace = workflow(parent_case, "managed-parent", has_agent=True, snapshot=True)
+ assert parent_trace is not None
+ child_case = setup(sqlite_session_factory, monkeypatch, "legacy-child", AppMode.WORKFLOW, snapshot=False)
+ child, _ = workflow(child_case, "legacy-child", has_agent=False, snapshot=False)
+ info = workflow_info(child, parent)
+ assert child_case.service.capture_terminal(info) is False
+ assert info.resolved_parent_context == (None, None)
+ assert info.metadata["linked_parent_workflow_run_id"] == parent.id
+ assert info.metadata["linked_parent_node_execution_id"] == identifier("parent-tool-node")
+ assert info.metadata["linked_parent_application_trace_id"] == parent_trace.id
+ assert info.metadata["linked_parent_trace_id"] == parent_trace.trace_id
+
+
+def test_mixed_workflow_exports_normal_llm_and_agent_calls_without_double_usage(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ case = setup(sqlite_session_factory, monkeypatch, "mixed", AppMode.WORKFLOW, snapshot=False)
+ row, managed = workflow(case, "mixed", has_agent=True, snapshot=True)
+ assert managed is not None
+ agent_node_id = identifier("agent-execution")
+ owner = identifier("mixed-owner")
+ with case.collection.sessions.begin() as session:
+ app = session.get(App, row.app_id)
+ assert app is not None
+ app.created_by = owner
+ session.add(make_account(account_id=owner))
+ for index, kind in enumerate(("llm", "agent")):
+ node_id = identifier("normal-execution") if kind == "llm" else agent_node_id
+ usage: dict[str, int] = (
+ {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} if kind == "llm" else {}
+ )
+ session.add(
+ WorkflowNodeExecutionModel(
+ id=node_id,
+ node_execution_id=node_id,
+ tenant_id=row.tenant_id,
+ app_id=row.app_id,
+ workflow_id=row.workflow_id,
+ workflow_run_id=row.id,
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
+ index=index,
+ node_id=kind,
+ node_type=kind,
+ title=kind,
+ inputs="{}",
+ outputs="{}",
+ process_data=json.dumps(
+ {
+ "model_mode": "chat",
+ "model_name": "ordinary-model",
+ "model_provider": "openai",
+ "usage": usage,
+ }
+ )
+ if kind == "llm"
+ else "{}",
+ status=WorkflowNodeExecutionStatus.SUCCEEDED,
+ elapsed_time=1,
+ execution_metadata=json.dumps(
+ {"total_tokens": 15 if kind == "llm" else 10, "total_price": "0.02", "currency": "USD"}
+ ),
+ created_at=row.created_at,
+ finished_at=row.finished_at,
+ created_by=owner,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ )
+ )
+ invocation = case.collection.repository.invocation(
+ run_id=managed.id, agent_id=identifier("agent"), node_execution_id=agent_node_id, attempt=0, call_key="agent:0"
+ )
+ spans = [
+ AgentTraceSpan(
+ trace_id=managed.trace_id,
+ span_id="a" * 16,
+ parent_span_id=invocation.span_id,
+ name="agent run",
+ start_time_ns=1_000_000_000,
+ end_time_ns=3_000_000_000,
+ status="ok",
+ attributes={"gen_ai.operation.name": "invoke_agent"},
+ )
+ ]
+ spans.extend(
+ AgentTraceSpan(
+ trace_id=managed.trace_id,
+ span_id=key * 16,
+ parent_span_id="a" * 16,
+ name="model call",
+ start_time_ns=1_000_000_000,
+ end_time_ns=2_000_000_000,
+ status="ok",
+ attributes={
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": "agent-model",
+ "gen_ai.usage.input_tokens": 3,
+ "gen_ai.usage.output_tokens": 2,
+ "dify.llm.total_price": "0.01",
+ "dify.llm.currency": "USD",
+ },
+ )
+ for key in ("b", "c")
+ )
+ case.collection.repository.ingest(
+ AgentTraceBatch(
+ run_id=UUID(managed.id), invocation_id=UUID(invocation.id), batch_id=uuid4(), final=True, spans=spans
+ ),
+ max_spans=100,
+ max_bytes=100000,
+ )
+ monkeypatch.setattr(type(db), "engine", property(lambda _db: sqlite_session_factory.kw["bind"]))
+ assert case.service.capture_terminal(workflow_info(row)) is True
+ stored = case.collection.repository.get(managed.id)
+ assert stored is not None
+ trace = case.service.assemble(stored)
+ payloads: list[ExportTraceServiceRequest] = []
+
+ def send(method: str, url: str, **kwargs: object) -> httpx.Response:
+ body = kwargs["content"]
+ assert isinstance(body, bytes)
+ payloads.append(ExportTraceServiceRequest.FromString(body))
+ return httpx.Response(200, content=b"", request=httpx.Request(method, url))
+
+ monkeypatch.setattr("core.helper.ssrf_proxy.make_request", send)
+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)
+ observations = payloads[0].resource_spans[0].scope_spans[0].spans
+ attributes = [{item.key: item.value.string_value for item in span.attributes} for span in observations]
+ generations = [item for item in attributes if item["langfuse.observation.type"] == "generation"]
+ assert len(generations) == 3
+ assert sum(json.loads(item["langfuse.observation.usage_details"])["total"] for item in generations) == 25
+ assert sum(json.loads(item["langfuse.observation.cost_details"])["total"] for item in generations) == pytest.approx(
+ 0.04
+ )
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_logstore.py b/api/tests/unit_tests/services/tracing/test_application_trace_logstore.py
new file mode 100644
index 00000000000..22bc2132e8f
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_logstore.py
@@ -0,0 +1,422 @@
+"""Configured LogStore readers work without SQL mirrors and retain source ownership."""
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import Literal
+
+import pytest
+from aliyun.log.logclient import LogClient
+from sqlalchemy import Connection, Engine, event, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.entities.trace_entity import ModerationTraceInfo
+from extensions.logstore.aliyun_logstore import AliyunLogStore
+from extensions.logstore.repositories.logstore_api_workflow_run_repository import _dict_to_workflow_run
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceRun
+from models.model import App, AppMode
+from models.workflow import WorkflowNodeExecutionModel, WorkflowRun
+from repositories.application_trace_repository import TraceCollectionError
+from services.tracing.provider_dispatch import TraceExportResult
+from services.tracing.recovery import _workflow_task
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_workflow
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ Collection,
+ batch_for,
+ claim,
+ identifier,
+ ingest,
+ seed,
+ terminal,
+)
+from tests.unit_tests.services.tracing.test_application_trace_handoff import AGENT_GRAPH, workflow_info
+
+
+@dataclass
+class StoredSource:
+ case: Collection
+ kind: Literal["workflow", "node"]
+ row: dict[str, object]
+ requests: list[str]
+
+ def start(self) -> ApplicationTraceRun:
+ run = self.case.repository.start(
+ tenant_id=self.case.run.tenant_id,
+ app_id=self.case.run.app_id,
+ source_kind=self.kind,
+ source_id=str(self.row["id"]),
+ session_id="logstore-session",
+ )
+ assert run is not None
+ return run
+
+ def service(self) -> ApplicationTracingService:
+ return ApplicationTracingService(self.case.repository)
+
+ def status(self, node_execution_id: str | None = None) -> dict[str, object]:
+ return self.service().status(
+ tenant_id=self.case.run.tenant_id,
+ app_id=self.case.run.app_id,
+ source_kind=self.kind,
+ source_id=str(self.row["id"]),
+ node_execution_id=node_execution_id,
+ )
+
+
[email protected](params=["workflow", "node"])
+def stored(
+ request: pytest.FixtureRequest,
+ sqlite_engine: Engine,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> Iterator[StoredSource]:
+ kind = request.param
+ assert kind in {"workflow", "node"}
+ case = seed(sqlite_session_factory, "logstore")
+ with case.sessions.begin() as session:
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.mode = AppMode.WORKFLOW
+ now = naive_utc_now()
+ row: dict[str, object] = {
+ "id": identifier("logstore-source"),
+ "tenant_id": case.run.tenant_id,
+ "app_id": case.run.app_id,
+ "workflow_id": identifier("definition"),
+ "created_by": identifier("owner"),
+ "created_at": (now - timedelta(seconds=30)).isoformat(),
+ "finished_at": (now - timedelta(seconds=10)).isoformat(),
+ "status": "succeeded",
+ "elapsed_time": 20,
+ "inputs": json.dumps({"query": "private input"}),
+ "outputs": json.dumps({"result": "private output"}),
+ "node_id": "agent-node",
+ "node_type": "agent",
+ "title": "Agent preview",
+ "graph": json.dumps(AGENT_GRAPH),
+ }
+ apply_config_overrides(
+ monkeypatch,
+ LOGSTORE_DUAL_WRITE_ENABLED=False,
+ API_WORKFLOW_RUN_REPOSITORY=(
+ "extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository"
+ ),
+ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=(
+ "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository."
+ "LogstoreAPIWorkflowNodeExecutionRepository"
+ ),
+ )
+ monkeypatch.setenv("LOGSTORE_DUAL_READ_ENABLED", "false")
+ monkeypatch.setenv("LOGSTORE_PG_MODE_ENABLED", "false")
+ monkeypatch.setenv("ALIYUN_SLS_ACCESS_KEY_ID", "test-access-key")
+ monkeypatch.setenv("ALIYUN_SLS_ACCESS_KEY_SECRET", "test-secret")
+ monkeypatch.setenv("ALIYUN_SLS_ENDPOINT", "https://sls.example.test")
+ monkeypatch.setenv("ALIYUN_SLS_REGION", "test-region")
+ monkeypatch.setattr(AliyunLogStore, "_instance", None)
+ monkeypatch.setattr(AliyunLogStore, "_initialized", False)
+ monkeypatch.setattr(AliyunLogStore, "_check_endpoint_connectivity", staticmethod(lambda _endpoint, _timeout: None))
+
+ def reject_http_transport(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("Configured LogStore test escaped its isolated transport boundary")
+
+ monkeypatch.setattr(LogClient, "_getHttpResponse", reject_http_transport)
+ transactions: set[Connection] = set()
+
+ def begun(connection: Connection) -> None:
+ transactions.add(connection)
+
+ def ended(connection: Connection) -> None:
+ transactions.discard(connection)
+
+ requests: list[str] = []
+
+ def get_logs(_client: AliyunLogStore, *, query: str, logstore: str, **_kwargs: object) -> list[dict[str, object]]:
+ # Exercise real configured repository construction, scoped query generation,
+ # and model decoding; replace only the remote transport boundary.
+ assert not transactions, "LogStore I/O must not hold a SQL transaction"
+ assert f'tenant_id:"{case.run.tenant_id}"' in query.replace(": ", ":")
+ assert logstore == (
+ AliyunLogStore.workflow_execution_logstore
+ if kind == "workflow"
+ else AliyunLogStore.workflow_node_execution_logstore
+ )
+ if kind == "workflow":
+ assert f'app_id:"{case.run.app_id}"' in query
+ requests.append(query)
+ return [dict(row)] if row else []
+
+ monkeypatch.setattr(AliyunLogStore, "get_logs", get_logs)
+
+ def execute_sql(
+ client: AliyunLogStore, *, sql: str, query: str, logstore: str, **_kwargs: object
+ ) -> list[dict[str, object]]:
+ # The node reader now uses SQL in both SDK and PG modes. Preserve the
+ # real repository's query/decoding path while isolating this transport.
+ assert kind == "node"
+ assert f'id:"{identifier("logstore-source")}"' in query.replace(": ", ":")
+ assert f"tenant_id = '{case.run.tenant_id}'" in sql
+ assert f"id = '{identifier('logstore-source')}'" in sql
+ assert f'FROM "{logstore}"' in sql
+ assert "ORDER BY log_version DESC LIMIT 1" in sql
+ return get_logs(client, query=query, logstore=logstore)
+
+ monkeypatch.setattr(AliyunLogStore, "execute_sql", execute_sql)
+ event.listen(sqlite_engine, "begin", begun)
+ event.listen(sqlite_engine, "commit", ended)
+ event.listen(sqlite_engine, "rollback", ended)
+ try:
+ yield StoredSource(case, kind, row, requests)
+ finally:
+ event.remove(sqlite_engine, "begin", begun)
+ event.remove(sqlite_engine, "commit", ended)
+ event.remove(sqlite_engine, "rollback", ended)
+
+
+def test_status_and_retry_use_logstore_without_sql_source_rows(
+ stored: StoredSource, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ run = stored.start()
+ with stored.case.sessions() as session:
+ assert session.scalar(select(WorkflowRun)) is None
+ assert session.scalar(select(WorkflowNodeExecutionModel)) is None
+ with stored.case.sessions.begin() as session:
+ current = session.get(ApplicationTraceRun, run.id)
+ assert current is not None
+ current.state = "failed"
+ current.error_code = "provider_export_rejected"
+ current.outer_trace = {"persisted": True}
+ assert stored.status()["status"] == "failed"
+ scheduled: list[str] = []
+ service = stored.service()
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ stored.requests.clear()
+ result = service.retry(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ source_kind=stored.kind,
+ source_id=run.source_id,
+ acknowledge_possible_duplicates=False,
+ )
+ assert result["status"] == "ready"
+ assert scheduled == [run.id]
+ assert len(stored.requests) == 1
+
+
[email protected]("field", ["id", "tenant_id", "app_id"])
+def test_backend_result_must_match_complete_source_owner(stored: StoredSource, field: str) -> None:
+ run = stored.start()
+ stored.row[field] = identifier("foreign")
+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found") as caught:
+ stored.service().status(
+ tenant_id=run.tenant_id, app_id=run.app_id, source_kind=stored.kind, source_id=run.source_id
+ )
+ assert caught.value.status == 404
+ assert stored.case.repository.get(run.id) is not None
+
+
+def test_foreign_app_owner_is_rejected_before_logstore_request(stored: StoredSource) -> None:
+ stored.start()
+ stored.requests.clear()
+ with stored.case.sessions.begin() as session:
+ app = session.get(App, stored.case.run.app_id)
+ assert app is not None
+ app.tenant_id = identifier("other-tenant")
+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found"):
+ stored.status()
+ assert stored.requests == []
+
+
+def test_initialization_failure_uses_configured_source_reader(stored: StoredSource) -> None:
+ run = stored.case.repository.record_initialization_failure(
+ tenant_id=stored.case.run.tenant_id,
+ app_id=stored.case.run.app_id,
+ source_kind=stored.kind,
+ source_id=str(stored.row["id"]),
+ )
+ assert run is not None
+ assert run.state == "failed"
+ assert run.error_code == "tracing_initialization_failed"
+
+
+def test_existing_snapshot_reentry_does_not_consult_logstore(stored: StoredSource) -> None:
+ original = stored.start()
+ stored.requests.clear()
+ stored.row.clear()
+ current = stored.case.repository.start(
+ tenant_id=original.tenant_id,
+ app_id=original.app_id,
+ source_kind=stored.kind,
+ source_id=original.source_id,
+ session_id="must-not-replace",
+ )
+ assert current is not None
+ assert current.id == original.id
+ assert current.session_id == original.session_id
+ assert stored.requests == []
+
+
[email protected]("stored", ["workflow"], indirect=True)
+def test_workflow_node_deep_link_is_scoped_to_durable_invocation(stored: StoredSource) -> None:
+ run = stored.start()
+ case = Collection(stored.case.repository, stored.case.sessions, run)
+ node_id = identifier("logstore-agent-node")
+ invocation = case.repository.invocation(
+ run_id=run.id, agent_id=identifier("agent"), node_execution_id=node_id, attempt=0, call_key="node"
+ )
+ ingest(case, batch_for(run, invocation))
+ terminal(case)
+ lease = claim(case)
+ assert lease is not None
+ case.repository.complete(
+ run.id,
+ lease[1],
+ state="accepted",
+ result=TraceExportResult("provider-id", span_urls={invocation.span_id: "https://trace.test/span"}),
+ )
+ assert stored.status(node_id)["span_url"] == "https://trace.test/span"
+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found"):
+ stored.status(identifier("another-run-node"))
+
+
+def test_preview_settlement_and_workflow_recovery_use_configured_source(stored: StoredSource) -> None:
+ run = stored.start()
+ if stored.kind == "workflow":
+ task = _workflow_task(stored.case.repository, run, naive_utc_now())
+ assert task is not None
+ assert task.workflow_run_id == run.source_id
+ assert task.kwargs["trace_session_id"] == run.session_id
+ stored.row["status"] = "running"
+ assert _workflow_task(stored.case.repository, run, naive_utc_now()) is None
+ else:
+ stored.service().settle_preview(run)
+ current = stored.case.repository.get(run.id)
+ assert current is not None
+ assert current.outer_trace is not None
+ root = current.outer_trace["spans"][0]
+ assert root["status"] == "ok"
+ assert root["inputs"] is None
+ assert root["outputs"] is None
+ assert root["metadata"]["node_execution_id"] == run.source_id
+
+
[email protected]("stored", ["workflow"], indirect=True)
+def test_missing_snapshot_classifies_logstore_agent_workflow_and_persists_failure(stored: StoredSource) -> None:
+ info = workflow_info(_dict_to_workflow_run(stored.row))
+ assert stored.service().capture_terminal(info) is True
+ result = stored.status()
+ assert result["status"] == "failed"
+ assert result["error_code"] == "tracing_initialization_failed"
+
+
[email protected]("stored", ["workflow"], indirect=True)
[email protected]("has_agent", [True, False])
+def test_unindexed_workflow_uses_owned_definition_for_safe_legacy_decision(
+ stored: StoredSource, has_agent: bool
+) -> None:
+ info = workflow_info(_dict_to_workflow_run(stored.row))
+ with stored.case.sessions.begin() as session:
+ app = session.get(App, stored.case.run.app_id)
+ assert app is not None
+ definition = make_workflow(
+ workflow_id=str(stored.row["workflow_id"]),
+ tenant_id=app.tenant_id,
+ app_id=app.id,
+ graph=AGENT_GRAPH if has_agent else {"nodes": [{"data": {"type": "llm"}}]},
+ )
+ app.workflow_id = definition.id
+ session.add(definition)
+ stored.row.clear()
+ assert stored.service().capture_terminal(info) is has_agent
+
+
[email protected]("stored", ["workflow"], indirect=True)
[email protected]("snapshot", [True, False])
+def test_auxiliary_content_is_suppressed_for_logstore_workflow(stored: StoredSource, snapshot: bool) -> None:
+ if snapshot:
+ stored.start()
+ info = ModerationTraceInfo(
+ message_id=identifier("workflow-log"),
+ metadata={
+ "workflow_run_id": stored.row["id"],
+ **({} if snapshot else {"app_id": stored.case.run.app_id}),
+ },
+ flagged=False,
+ action="direct_output",
+ preset_response="",
+ query="private query",
+ inputs="private input",
+ outputs="private output",
+ )
+ assert stored.service().capture_terminal(info) is True
+ if snapshot:
+ # Durable ownership is sufficient to suppress auxiliary payload even when
+ # the remote run has not been indexed yet.
+ stored.row.clear()
+ assert stored.service().capture_terminal(info) is True
+ else:
+ stored.row["graph"] = json.dumps({"nodes": [{"data": {"type": "llm"}}]})
+ assert stored.service().capture_terminal(info) is False
+
+
[email protected]("stored", ["workflow"], indirect=True)
[email protected]("mismatch", [None, "tenant_id", "app_id"])
+def test_linked_parent_requires_logstore_run_and_durable_owner_to_agree(
+ stored: StoredSource, mismatch: str | None
+) -> None:
+ parent = stored.start()
+ info = workflow_info(_dict_to_workflow_run(stored.row), _dict_to_workflow_run(stored.row))
+ original = dict(info.metadata)
+ if mismatch:
+ stored.row[mismatch] = identifier("foreign")
+ stored.service()._link_managed_parent(info, parent.tenant_id)
+ if mismatch:
+ assert info.metadata == original
+ else:
+ assert info.resolved_parent_context == (None, None)
+ assert info.metadata["linked_parent_trace_id"] == parent.trace_id
+ assert info.metadata["linked_parent_workflow_run_id"] == parent.source_id
+
+
[email protected]("stored", ["workflow"], indirect=True)
+def test_auxiliary_durable_owner_rejects_app_reassigned_to_other_tenant(stored: StoredSource) -> None:
+ run = stored.start()
+ with stored.case.sessions.begin() as session:
+ app = session.get(App, run.app_id)
+ assert app is not None
+ app.tenant_id = identifier("foreign")
+ info = ModerationTraceInfo(
+ metadata={"workflow_run_id": run.source_id}, query="", flagged=False, action="direct_output", preset_response=""
+ )
+ assert stored.service().capture_terminal(info) is False
+
+
+def test_sql_workflow_source_still_uses_scoped_configured_reader(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ case = seed(sqlite_session_factory, "sql-reader")
+ row = _dict_to_workflow_run(
+ {
+ "id": identifier("sql-run"),
+ "tenant_id": case.run.tenant_id,
+ "app_id": case.run.app_id,
+ "workflow_id": identifier("definition"),
+ "created_by": identifier("owner"),
+ }
+ )
+ with case.sessions.begin() as session:
+ session.add(row)
+ source = case.repository._source(
+ tenant_id=case.run.tenant_id, app_id=case.run.app_id, kind="workflow", source_id=row.id
+ )
+ assert isinstance(source, WorkflowRun)
+ assert source.id == row.id
+ assert (
+ case.repository._source(
+ tenant_id=identifier("wrong-tenant"), app_id=case.run.app_id, kind="workflow", source_id=row.id
+ )
+ is None
+ )
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_node_alias.py b/api/tests/unit_tests/services/tracing/test_application_trace_node_alias.py
new file mode 100644
index 00000000000..59f9116f177
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_node_alias.py
@@ -0,0 +1,386 @@
+"""Node logs resolve the caller's trace using SQL or configured LogStore ownership."""
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+
+import pytest
+from aliyun.log.logclient import LogClient
+from sqlalchemy import Connection, Engine, event, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from extensions.logstore.aliyun_logstore import AliyunLogStore
+from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (
+ _dict_to_workflow_node_execution_model,
+)
+from extensions.logstore.repositories.logstore_api_workflow_run_repository import _dict_to_workflow_run
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceRun
+from models.model import App, AppMode
+from models.workflow import WorkflowNodeExecutionModel, WorkflowRun
+from repositories.application_trace_repository import TraceCollectionError
+from services.tracing.provider_dispatch import TraceExportResult
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ Collection,
+ batch_for,
+ claim,
+ identifier,
+ ingest,
+ seed,
+ terminal,
+)
+
+
+@dataclass
+class NodeAlias:
+ case: Collection
+ node: dict[str, object]
+ workflow: dict[str, object]
+ requests: list[str]
+ logstore: bool
+
+ def service(self) -> ApplicationTracingService:
+ return ApplicationTracingService(self.case.repository)
+
+ def status(self, node_execution_id: str | None = None) -> dict[str, object]:
+ return self.service().status(
+ tenant_id=self.case.run.tenant_id,
+ app_id=self.case.run.app_id,
+ source_kind="node",
+ source_id=str(self.node["id"]),
+ node_execution_id=node_execution_id,
+ )
+
+ def accept(self, *, spans: bool = True) -> tuple[str, str]:
+ span_urls: dict[str, str] = {}
+ for attempt in (0, 1):
+ invocation = self.case.repository.invocation(
+ run_id=self.case.run.id,
+ agent_id=identifier("referenced-agent"),
+ node_execution_id=str(self.node["id"]),
+ attempt=attempt,
+ call_key=f"attempt-{attempt}",
+ )
+ ingest(self.case, batch_for(self.case.run, invocation, span_id=str(attempt + 1) * 16))
+ span_urls[invocation.span_id] = f"https://provider.test/span/{attempt}"
+ terminal(self.case)
+ lease = claim(self.case)
+ assert lease is not None
+ self.case.repository.complete(
+ self.case.run.id,
+ lease[1],
+ state="accepted",
+ result=TraceExportResult(
+ "provider-trace", "https://provider.test/trace", span_urls=span_urls if spans else {}
+ ),
+ )
+ return "https://provider.test/trace", "https://provider.test/span/1"
+
+
[email protected](params=["sql", "logstore"])
+def alias(
+ request: pytest.FixtureRequest,
+ sqlite_engine: Engine,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> Iterator[NodeAlias]:
+ case = seed(sqlite_session_factory, "node-alias")
+ with case.sessions.begin() as session:
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.mode = AppMode.WORKFLOW
+ common: dict[str, object] = {
+ "tenant_id": case.run.tenant_id,
+ "app_id": case.run.app_id,
+ "workflow_id": identifier("definition"),
+ "created_by": identifier("actor"),
+ "created_at": naive_utc_now().isoformat(),
+ "status": "succeeded",
+ }
+ workflow = {**common, "id": identifier("workflow-source"), "graph": json.dumps({"nodes": []})}
+ node = {
+ **common,
+ "id": identifier("node-source"),
+ "workflow_run_id": workflow["id"],
+ "node_id": "reference-agent",
+ "node_type": "agent",
+ "title": "Reference Agent",
+ }
+ requests: list[str] = []
+ transactions: set[Connection] = set()
+
+ def begun(connection: Connection) -> None:
+ transactions.add(connection)
+
+ def ended(connection: Connection) -> None:
+ transactions.discard(connection)
+
+ if request.param == "sql":
+ with case.sessions.begin() as session:
+ session.add(_dict_to_workflow_run(workflow))
+ session.add(_dict_to_workflow_node_execution_model(node))
+ else:
+ apply_config_overrides(
+ monkeypatch,
+ LOGSTORE_DUAL_WRITE_ENABLED=False,
+ API_WORKFLOW_RUN_REPOSITORY=(
+ "extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository"
+ ),
+ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=(
+ "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository."
+ "LogstoreAPIWorkflowNodeExecutionRepository"
+ ),
+ )
+ for key, value in {
+ "LOGSTORE_DUAL_READ_ENABLED": "false",
+ "LOGSTORE_PG_MODE_ENABLED": "false",
+ "ALIYUN_SLS_ACCESS_KEY_ID": "test-access-key",
+ "ALIYUN_SLS_ACCESS_KEY_SECRET": "test-secret",
+ "ALIYUN_SLS_ENDPOINT": "https://sls.example.test",
+ "ALIYUN_SLS_REGION": "test-region",
+ }.items():
+ monkeypatch.setenv(key, value)
+ monkeypatch.setattr(AliyunLogStore, "_instance", None)
+ monkeypatch.setattr(AliyunLogStore, "_initialized", False)
+ monkeypatch.setattr(
+ AliyunLogStore, "_check_endpoint_connectivity", staticmethod(lambda _endpoint, _timeout: None)
+ )
+
+ def reject_http_transport(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("Configured LogStore test escaped its isolated transport boundary")
+
+ monkeypatch.setattr(LogClient, "_getHttpResponse", reject_http_transport)
+
+ def get_logs(
+ _client: AliyunLogStore, *, query: str, logstore: str, **_kwargs: object
+ ) -> list[dict[str, object]]:
+ assert not transactions, "Configured source reads must not hold a SQL transaction"
+ assert f'tenant_id:"{case.run.tenant_id}"' in query.replace(": ", ":")
+ assert logstore in {
+ AliyunLogStore.workflow_execution_logstore,
+ AliyunLogStore.workflow_node_execution_logstore,
+ }
+ is_workflow = logstore == AliyunLogStore.workflow_execution_logstore
+ if is_workflow:
+ assert f'app_id:"{case.run.app_id}"' in query
+ requests.append("workflow" if is_workflow else "node")
+ row = workflow if is_workflow else node
+ return [dict(row)] if row else []
+
+ monkeypatch.setattr(AliyunLogStore, "get_logs", get_logs)
+
+ def execute_sql(
+ client: AliyunLogStore, *, sql: str, query: str, logstore: str, **_kwargs: object
+ ) -> list[dict[str, object]]:
+ # Node lookups use the SQL transport in SDK mode too; the real
+ # service still validates the returned node-to-workflow owner chain.
+ assert logstore == AliyunLogStore.workflow_node_execution_logstore
+ assert f'id:"{identifier("node-source")}"' in query.replace(": ", ":")
+ assert f"tenant_id = '{case.run.tenant_id}'" in sql
+ assert f"id = '{identifier('node-source')}'" in sql
+ assert f'FROM "{logstore}"' in sql
+ assert "ORDER BY log_version DESC LIMIT 1" in sql
+ return get_logs(client, query=query, logstore=logstore)
+
+ monkeypatch.setattr(AliyunLogStore, "execute_sql", execute_sql)
+ with case.sessions() as session:
+ assert session.scalar(select(WorkflowRun)) is None
+ assert session.scalar(select(WorkflowNodeExecutionModel)) is None
+ event.listen(sqlite_engine, "begin", begun)
+ event.listen(sqlite_engine, "commit", ended)
+ event.listen(sqlite_engine, "rollback", ended)
+ try:
+ run = case.repository.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="workflow",
+ source_id=str(workflow["id"]),
+ session_id="caller-session",
+ )
+ assert run is not None
+ requests.clear()
+ yield NodeAlias(
+ Collection(case.repository, case.sessions, run), node, workflow, requests, request.param == "logstore"
+ )
+ finally:
+ event.remove(sqlite_engine, "begin", begun)
+ event.remove(sqlite_engine, "commit", ended)
+ event.remove(sqlite_engine, "rollback", ended)
+
+
[email protected]("spans", [True, False])
+def test_alias_returns_parent_trace_and_latest_owned_invocation(alias: NodeAlias, spans: bool) -> None:
+ trace_url, span_url = alias.accept(spans=spans)
+ status = alias.status()
+ assert status["status"] == "accepted"
+ assert status["source_kind"] == "node"
+ assert status["source_id"] == alias.node["id"]
+ assert status["trace_id"] == "provider-trace"
+ assert status["trace_url"] == trace_url
+ assert status["span_url"] == (span_url if spans else None)
+ assert status["config_revision"] == alias.case.run.config_revision
+ if alias.logstore:
+ assert alias.requests == ["node", "workflow"]
+
+
[email protected]("capture_agent", [True, False])
[email protected]("accepted", [True, False])
+def test_alias_without_internal_invocation_returns_parent_status(
+ alias: NodeAlias, capture_agent: bool, accepted: bool
+) -> None:
+ with alias.case.sessions.begin() as session:
+ run = session.get(ApplicationTraceRun, alias.case.run.id)
+ assert run is not None
+ run.capture_agent = capture_agent
+ if accepted:
+ terminal(alias.case)
+ lease = claim(alias.case)
+ assert lease is not None
+ alias.case.repository.complete(
+ alias.case.run.id,
+ lease[1],
+ state="accepted",
+ result=TraceExportResult("outer-only", "https://provider.test/outer"),
+ )
+ status = alias.status()
+ assert status["status"] == ("accepted" if accepted else "collecting")
+ assert status["span_url"] is None
+ assert status["capture_agent"] is capture_agent
+ assert status["trace_url"] == ("https://provider.test/outer" if accepted else None)
+
+
[email protected]("state", ["collecting", "not_collected"])
+def test_standalone_node_snapshot_wins_over_parent(alias: NodeAlias, state: str) -> None:
+ alias.accept()
+ node_run = alias.case.repository.start(
+ tenant_id=alias.case.run.tenant_id,
+ app_id=alias.case.run.app_id,
+ source_kind="node",
+ source_id=str(alias.node["id"]),
+ session_id="preview-session",
+ )
+ assert node_run is not None
+ with alias.case.sessions.begin() as session:
+ run = session.get(ApplicationTraceRun, node_run.id)
+ assert run is not None
+ run.state = state
+ alias.requests.clear()
+ status = alias.status()
+ assert status["status"] == state
+ assert status.get("trace_url") is None
+ if alias.logstore:
+ assert alias.requests == ["node"]
+
+
+def test_explicit_other_node_filter_cannot_select_sibling_span(
+ alias: NodeAlias, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ sibling = alias.case.repository.invocation(
+ run_id=alias.case.run.id,
+ agent_id=identifier("referenced-agent"),
+ node_execution_id=identifier("sibling-node"),
+ attempt=0,
+ call_key="sibling",
+ )
+ ingest(alias.case, batch_for(alias.case.run, sibling, span_id="f" * 16))
+ alias.accept()
+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found") as error:
+ alias.status(identifier("sibling-node"))
+ assert error.value.status == 404
+ service = alias.service()
+ scheduled: list[str] = []
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found"):
+ service.retry(
+ tenant_id=alias.case.run.tenant_id,
+ app_id=alias.case.run.app_id,
+ source_kind="node",
+ source_id=str(alias.node["id"]),
+ node_execution_id=identifier("sibling-node"),
+ acknowledge_possible_duplicates=True,
+ )
+ assert not scheduled
+
+
[email protected]("state", ["failed", "uncertain", "accepted"])
+def test_alias_retry_uses_parent_guards_and_resolves_sources_once(
+ alias: NodeAlias, state: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ alias.accept()
+ with alias.case.sessions.begin() as session:
+ run = session.get(ApplicationTraceRun, alias.case.run.id)
+ assert run is not None
+ run.state = state
+ service = alias.service()
+ scheduled: list[str] = []
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+
+ def retry(acknowledge: bool) -> dict[str, object]:
+ alias.requests.clear()
+ return service.retry(
+ tenant_id=alias.case.run.tenant_id,
+ app_id=alias.case.run.app_id,
+ source_kind="node",
+ source_id=str(alias.node["id"]),
+ acknowledge_possible_duplicates=acknowledge,
+ )
+
+ if state == "accepted":
+ with pytest.raises(TraceCollectionError, match="tracing_already_accepted") as error:
+ retry(True)
+ assert error.value.status == 409
+ assert not scheduled
+ else:
+ if state == "uncertain":
+ with pytest.raises(TraceCollectionError, match="tracing_duplicate_acknowledgement_required"):
+ retry(False)
+ assert not scheduled
+ assert retry(state == "uncertain")["status"] == "ready"
+ assert scheduled == [alias.case.run.id]
+ if alias.logstore:
+ assert alias.requests == ["node", "workflow"]
+
+
[email protected]("source", ["node", "workflow"])
[email protected]("field", ["tenant_id", "app_id", "workflow_id"])
+def test_alias_rejects_broken_owner_chain(alias: NodeAlias, source: str, field: str) -> None:
+ alias.accept()
+ row = alias.node if source == "node" else alias.workflow
+ if alias.logstore:
+ row[field] = identifier("foreign")
+ else:
+ with alias.case.sessions.begin() as session:
+ model = session.get(WorkflowNodeExecutionModel if source == "node" else WorkflowRun, str(row["id"]))
+ assert model is not None
+ # Keep field assignments explicit so ownership failures are visible.
+ if field == "tenant_id":
+ model.tenant_id = identifier("foreign")
+ elif field == "app_id":
+ model.app_id = identifier("foreign")
+ else:
+ model.workflow_id = identifier("foreign")
+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found") as error:
+ alias.status()
+ assert error.value.status == 404
+
+
+def test_alias_missing_parent_business_source_is_not_a_durable_owner_bypass(alias: NodeAlias) -> None:
+ if alias.logstore:
+ alias.workflow.clear()
+ else:
+ with alias.case.sessions.begin() as session:
+ parent = session.get(WorkflowRun, str(alias.workflow["id"]))
+ assert parent is not None
+ session.delete(parent)
+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found"):
+ alias.status()
+
+
+def test_owned_alias_without_parent_trace_is_not_collected(alias: NodeAlias) -> None:
+ with alias.case.sessions.begin() as session:
+ run = session.get(ApplicationTraceRun, alias.case.run.id)
+ assert run is not None
+ session.delete(run)
+ assert alias.status() == {"status": "not_collected", "source_kind": "node", "source_id": alias.node["id"]}
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_normalization.py b/api/tests/unit_tests/services/tracing/test_application_trace_normalization.py
new file mode 100644
index 00000000000..870b812dc2c
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_normalization.py
@@ -0,0 +1,218 @@
+"""Content policy and model-call accounting across all canonical tree layers."""
+
+import json
+from datetime import UTC, datetime
+
+import pytest
+
+from core.helper.trace_id_helper import ParentTraceContext
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.normalization import normalize_agent_span, order_spans, remap_outer_trace, safe_attributes
+from services.tracing.provider_common import token_usage, usd_cost
+
+
+def agent_data(
+ attributes: dict[str, object], *, span_id: str = "2" * 16, parent_id: str = "1" * 16
+) -> dict[str, object]:
+ return {
+ "span_id": span_id,
+ "parent_span_id": parent_id,
+ "name": "operation",
+ "start_time_ns": 1_000_000_000,
+ "end_time_ns": 2_000_000_000,
+ "status": "error",
+ "status_message": "sensitive error output",
+ "attributes": attributes,
+ }
+
+
+def test_content_off_is_safe_for_all_metadata_namespaces() -> None:
+ normalized = normalize_agent_span(
+ agent_data(
+ {
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": "model-one",
+ "gen_ai.usage.input_tokens": 3,
+ "gen_ai.usage.output_tokens": 2,
+ "gen_ai.input.messages": [{"content": "private input"}],
+ "gen_ai.output.messages": [{"content": "private output"}],
+ "exception.message": "private exception",
+ "dify.app_id": "app-one",
+ "dify.custom_payload": {"prompt": "private metadata"},
+ "gen_ai.request.messages": "private request metadata",
+ "gen_ai.response.output": "private response metadata",
+ "gen_ai.usage.custom_prompt": "private usage metadata",
+ }
+ ),
+ capture_content=False,
+ )
+ assert normalized.inputs is None
+ assert normalized.outputs is None
+ assert "private" not in normalized.model_dump_json()
+ assert "sensitive error" not in normalized.model_dump_json()
+ assert normalized.metadata["dify.app_id"] == "app-one"
+ assert normalized.metadata["model_name"] == "model-one"
+ assert normalized.metadata["total_tokens"] == 5
+
+
+def test_json_encoded_tool_credentials_are_redacted_from_inputs_and_metadata() -> None:
+ normalized = normalize_agent_span(
+ agent_data(
+ {
+ "gen_ai.operation.name": "execute_tool",
+ "gen_ai.tool.name": "search",
+ "gen_ai.tool.call.arguments": json.dumps({"api_key": "tool-secret-123", "query": "public query"}),
+ "gen_ai.tool.call.result": json.dumps(
+ {"nested": {"authorization": "bearer-secret-123"}, "result": "okay"}
+ ),
+ }
+ ),
+ capture_content=True,
+ )
+ serialized = normalized.model_dump_json()
+ assert "tool-secret-123" not in serialized
+ assert "bearer-secret-123" not in serialized
+ assert "public query" in serialized
+ assert normalized.inputs["api_key"] == "[redacted]"
+
+
+def test_content_off_drops_error_event_payload_and_keeps_numeric_usage() -> None:
+ attrs = safe_attributes(
+ {
+ "input": "user text",
+ "authorization": "secret",
+ "exception.stacktrace": "sensitive",
+ "gen_ai.usage.input_tokens": 10,
+ "gen_ai.tool.name": "lookup",
+ },
+ capture_content=False,
+ )
+ assert attrs == {"gen_ai.usage.input_tokens": 10, "gen_ai.tool.name": "lookup"}
+
+
+def test_real_runtime_accounting_attributes_survive_content_off() -> None:
+ normalized = normalize_agent_span(
+ agent_data(
+ {
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": "gpt-4o-mini",
+ "gen_ai.usage.input_tokens": 8,
+ "gen_ai.usage.output_tokens": 4,
+ "dify.llm.total_price": "0.03",
+ "dify.llm.currency": "USD",
+ "gen_ai.response.time_to_first_token": 0.2,
+ "dify.llm.time_to_first_token_unit": "s",
+ "dify.llm.time_to_first_token_source": "gateway_usage",
+ "gen_ai.input.messages": "private input",
+ }
+ ),
+ capture_content=False,
+ )
+ assert usd_cost(normalized) == 0.03
+ assert normalized.metadata["currency"] == "USD"
+ assert normalized.metadata["time_to_first_token"] == 0.2
+ assert token_usage(normalized) == {"input": 8, "output": 4, "total": 12}
+ assert normalized.inputs is None
+
+
+def span(identifier: str, parent: str | None, kind: CanonicalSpanKind, *, synthetic: bool = False) -> CanonicalSpan:
+ return CanonicalSpan(
+ id=identifier,
+ parent_id=parent,
+ name=identifier,
+ kind=kind,
+ start_time=datetime.fromtimestamp(0, UTC),
+ end_time=datetime.fromtimestamp(3, UTC),
+ status=CanonicalSpanStatus.OK,
+ inputs={"content": "private"},
+ outputs="private",
+ metadata={
+ "prompt_tokens": 6,
+ "completion_tokens": 4,
+ "total_tokens": 10,
+ "total_price": "0.03",
+ "prompt": "private",
+ },
+ synthetic=synthetic,
+ )
+
+
+def test_two_native_model_calls_replace_synthetic_aggregate_generation() -> None:
+ outer = CanonicalTrace(
+ trace_id="old",
+ session_id="session",
+ root_span_id="message",
+ spans=(
+ span("message", None, CanonicalSpanKind.CHAIN),
+ span("message:llm", "message", CanonicalSpanKind.LLM, synthetic=True),
+ ),
+ )
+ remapped = remap_outer_trace(outer, trace_id="1" * 32, capture_content=False, agent_message=True)
+ calls = [
+ normalize_agent_span(
+ agent_data(
+ {
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": "model",
+ "gen_ai.usage.input_tokens": 3,
+ "gen_ai.usage.output_tokens": 2,
+ },
+ span_id=identifier,
+ parent_id=remapped.root_span_id,
+ ),
+ capture_content=False,
+ )
+ for identifier in ("2" * 16, "3" * 16)
+ ]
+ combined = order_spans(remapped.root_span_id, [calls[1], *remapped.spans, calls[0]])
+ assert combined[0].id == remapped.root_span_id
+ generations = [item for item in combined if item.kind is CanonicalSpanKind.LLM]
+ assert len(generations) == 2
+ assert sum(token_usage(item)["total"] for item in generations) == 10
+ assert token_usage(combined[0]) == {}
+ assert combined[0].metadata["aggregate_total_tokens"] == 10
+ assert "private" not in json.dumps([item.model_dump(mode="json") for item in combined])
+
+
+def test_non_agent_workflow_retains_its_real_llm_node() -> None:
+ trace = CanonicalTrace(
+ trace_id="old",
+ session_id="session",
+ root_span_id="workflow",
+ spans=(
+ span("workflow", None, CanonicalSpanKind.CHAIN),
+ span("llm-node", "workflow", CanonicalSpanKind.LLM),
+ ),
+ )
+ remapped = remap_outer_trace(trace, trace_id="1" * 32, capture_content=True, agent_message=False)
+ assert len(remapped.spans) == 2
+ assert remapped.spans[1].kind is CanonicalSpanKind.LLM
+ assert token_usage(remapped.spans[1])["total"] == 10
+
+
+def test_order_spans_rejects_orphans_cycles_and_conflicting_ids() -> None:
+ root = span("root", None, CanonicalSpanKind.CHAIN)
+ with pytest.raises(ValueError, match="parent is missing"):
+ order_spans("root", [root, span("orphan", "missing", CanonicalSpanKind.TOOL)])
+ with pytest.raises(ValueError, match="parent cycle"):
+ order_spans(
+ "root", [root, span("one", "two", CanonicalSpanKind.TOOL), span("two", "one", CanonicalSpanKind.TOOL)]
+ )
+ with pytest.raises(ValueError, match="Conflicting"):
+ order_spans("root", [root, root.model_copy(update={"name": "other"})])
+
+
+def test_managed_external_parent_is_preserved_as_explicit_link_when_content_off() -> None:
+ trace = CanonicalTrace(
+ trace_id="old",
+ session_id="session",
+ root_span_id="workflow",
+ spans=(span("workflow", None, CanonicalSpanKind.CHAIN),),
+ external_parent=ParentTraceContext(
+ parent_workflow_run_id="outer-workflow", parent_node_execution_id="outer-tool"
+ ),
+ )
+ remapped = remap_outer_trace(trace, trace_id="1" * 32, capture_content=False, agent_message=False)
+ assert remapped.external_parent is None
+ assert remapped.spans[0].metadata["linked_parent_workflow_run_id"] == "outer-workflow"
+ assert remapped.spans[0].metadata["linked_parent_node_execution_id"] == "outer-tool"
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_recovery.py b/api/tests/unit_tests/services/tracing/test_application_trace_recovery.py
new file mode 100644
index 00000000000..471366713c5
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_recovery.py
@@ -0,0 +1,1215 @@
+"""Recover lost terminal notifications using real business rows and TraceTask."""
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from decimal import Decimal
+from typing import Literal
+
+import pytest
+from sqlalchemy import Engine
+from sqlalchemy.orm import Session, scoped_session, sessionmaker
+
+from core.app.entities.app_invoke_entities import ChatAppGenerateEntity
+from core.ops.ops_trace_manager import TraceTask
+from core.ops.unified_trace.entities import CanonicalTrace
+from extensions.ext_database import db
+from graphon.enums import WorkflowExecutionStatus
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceInvocation
+from models.enums import ConversationFromSource, CreatorUserRole, MessageStatus, WorkflowRunTriggeredFrom
+from models.model import App, AppMode, Message
+from models.workflow import WorkflowRun, WorkflowType
+from services.tracing import recovery
+from services.tracing import service as service_module
+from services.tracing.normalization import span_id
+from services.tracing.provider_dispatch import TraceExportResult, TraceExportUncertainError
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.model_factories import make_account, make_conversation
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ Collection,
+ batch_for,
+ identifier,
+ ingest,
+ seed,
+)
+
+
+@dataclass
+class RecoveryCase:
+ collection: Collection
+ service: ApplicationTracingService
+ now: datetime
+ scheduled: list[str]
+
+
+def attach_conversation(case: Collection) -> None:
+ with case.sessions.begin() as session:
+ message = session.get(Message, case.run.source_id)
+ assert message is not None
+ session.add(
+ make_conversation(
+ conversation_id=message.conversation_id,
+ app_id=case.run.app_id,
+ mode=AppMode.AGENT,
+ inputs={},
+ from_source=ConversationFromSource.CONSOLE,
+ )
+ )
+
+
+def case(
+ sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> RecoveryCase:
+ collection = seed(sqlite_session_factory, "recover", capture_content=True)
+ attach_conversation(collection)
+ invocation = collection.invocation()
+ ingest(collection, batch_for(collection.run, invocation))
+ now = naive_utc_now()
+ with sqlite_session_factory.begin() as session:
+ message = session.get(Message, collection.run.source_id)
+ assert message is not None
+ stored_invocation = session.get(ApplicationTraceInvocation, invocation.id)
+ assert message is not None
+ assert stored_invocation is not None
+ message.created_at = now - timedelta(seconds=30)
+ message.updated_at = now - timedelta(seconds=10)
+ message.answer = "persisted final answer"
+ message.provider_response_latency = 1.0
+ message.message_tokens = 2
+ message.answer_tokens = 1
+ stored_invocation.finalized_at = now - timedelta(seconds=10)
+ monkeypatch.setattr(type(db), "engine", property(lambda _db: sqlite_engine))
+ monkeypatch.setattr(db, "session", scoped_session(sqlite_session_factory))
+ monkeypatch.setattr(TraceTask, "_workflow_run_repo", None)
+ monkeypatch.setattr("core.telemetry.gateway.is_enterprise_telemetry_enabled", lambda: False)
+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: now)
+ service = ApplicationTracingService(collection.repository)
+ scheduled: list[str] = []
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ return RecoveryCase(collection, service, now, scheduled)
+
+
+def complete_metadata() -> str:
+ return json.dumps(
+ {
+ "annotation_reply": None,
+ "retriever_resources": [],
+ "reasoning": {},
+ "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
+ }
+ )
+
+
+def save_agent_stop(
+ case: RecoveryCase,
+ monkeypatch: pytest.MonkeyPatch,
+ answer: str,
+ *,
+ mode: AppMode = AppMode.AGENT,
+ consumer_session: Session | None = None,
+ process_stream: bool = False,
+ notify: Callable[[Message, ChatAppGenerateEntity], None] | None = None,
+) -> list[TraceTask]:
+ """Run the real consumer save and tracing queue against the owned SQL rows."""
+ import queue
+
+ from flask import Flask
+
+ from core.app.app_config.entities import (
+ EasyUIBasedAppConfig,
+ EasyUIBasedAppModelConfigFrom,
+ ModelConfigEntity,
+ PromptTemplateEntity,
+ )
+ from core.app.entities.app_invoke_entities import InvokeFrom
+ from core.app.entities.queue_entities import QueueStopEvent
+ from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline
+ from core.ops import ops_trace_manager
+ from core.ops.base_trace_instance import bind_application_tracing_hooks
+ from services.tracing import runtime_bridge
+ from tests.unit_tests.config_override import apply_config_overrides
+ from tests.unit_tests.core.app.task_pipeline.test_easy_ui_based_generate_task_pipeline import (
+ _QueueManager,
+ queue_message,
+ )
+ from tests.unit_tests.core.model_fixtures import make_model_config
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ monkeypatch.setattr(runtime_bridge, "_service", lambda: case.service)
+ monkeypatch.setattr(ops_trace_manager, "trace_manager_queue", queue.Queue())
+ monkeypatch.setattr(ops_trace_manager.TraceQueueManager, "start_timer", lambda _manager: None)
+
+ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity) -> None:
+ if notify is not None:
+ notify(message, application_generate_entity)
+
+ monkeypatch.setattr("core.app.task_pipeline.easy_ui_based_generate_task_pipeline.message_was_created.send", send)
+ monkeypatch.setattr(
+ "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.naive_utc_now",
+ lambda: case.now - timedelta(seconds=10),
+ )
+ with case.collection.sessions.begin() as session:
+ persisted = session.get(Message, case.collection.run.source_id)
+ assert persisted is not None
+ if persisted.message_metadata is None:
+ persisted.message_metadata = complete_metadata()
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ conversation = session.get(service_module.Conversation, message.conversation_id)
+ assert conversation is not None
+ entity = ChatAppGenerateEntity(
+ task_id=identifier("stop-race-task"),
+ app_config=EasyUIBasedAppConfig(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ app_mode=mode,
+ app_model_config_from=EasyUIBasedAppModelConfigFrom.APP_LATEST_CONFIG,
+ app_model_config_dict={},
+ model=ModelConfigEntity(provider="test-provider", model="test-model"),
+ prompt_template=PromptTemplateEntity(prompt_type=PromptTemplateEntity.PromptType.SIMPLE),
+ ),
+ model_conf=make_model_config(provider="test-provider", model="test-model", mode="chat"),
+ inputs={},
+ files=[],
+ user_id=identifier("stop-race-user"),
+ stream=True,
+ invoke_from=InvokeFrom.DEBUGGER,
+ )
+ queue_manager = _QueueManager()
+ pipeline = EasyUIBasedGenerateTaskPipeline(entity, queue_manager, conversation, message, stream=True)
+ pipeline._task_state.llm_result.message.content = answer
+ application = Flask(__name__)
+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())
+ with application.app_context():
+ trace_manager = ops_trace_manager.TraceQueueManager(app_id=case.collection.run.app_id)
+ if process_stream:
+ queue_manager.messages = [queue_message(QueueStopEvent(stopped_by=QueueStopEvent.StopBy.USER_MANUAL))]
+ # Token estimates and moderation are separate external boundaries;
+ # exercise real stream dispatch, SQL save, commit and notification.
+ monkeypatch.setattr(pipeline, "_handle_stop", lambda _event: None)
+ monkeypatch.setattr(pipeline, "handle_output_moderation_when_task_finished", lambda _answer: None)
+ list(pipeline._process_stream_response(publisher=None, trace_manager=trace_manager))
+ elif consumer_session is None:
+ with case.collection.sessions.begin() as session:
+ pipeline._save_message(session=session, trace_manager=trace_manager, preserve_existing_usage=True)
+ else:
+ pipeline._save_message(session=consumer_session, trace_manager=trace_manager, preserve_existing_usage=True)
+ consumer_session.commit()
+ return trace_manager.collect_tasks()
+
+
[email protected]("receiver_fails", [False, True])
+def test_stopped_agent_notifies_the_real_provider_receiver_once_after_commit(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, receiver_fails: bool
+) -> None:
+ from sqlalchemy import inspect
+
+ from core.entities.provider_entities import ProviderQuotaType, QuotaConfiguration, QuotaUnit, SystemConfiguration
+ from events.event_handlers import update_provider_when_message_created as receiver
+
+ calls: list[str] = []
+ provider_updates: list[str] = []
+ monkeypatch.setattr(
+ receiver,
+ "_execute_provider_updates",
+ lambda updates: provider_updates.extend(operation.description for operation in updates),
+ )
+
+ def notify(message: Message, entity: ChatAppGenerateEntity) -> None:
+ # The production session factory does not expire on commit. Check real
+ # detached fields used by the actual receiver, not a mocked send order.
+ assert inspect(message).detached is True
+ assert not inspect(message).expired_attributes
+ assert message.answer == "partial answer"
+ with case.collection.sessions() as session:
+ saved = session.get(Message, message.id)
+ assert saved is not None
+ assert recovery.agent_stop_requested(saved) is True
+ assert saved.answer == message.answer
+ system = SystemConfiguration(
+ enabled=True,
+ current_quota_type=ProviderQuotaType.TRIAL,
+ quota_configurations=[
+ QuotaConfiguration(
+ quota_type=ProviderQuotaType.TRIAL,
+ quota_unit=QuotaUnit.TOKENS,
+ quota_limit=100,
+ quota_used=0,
+ is_valid=True,
+ )
+ ],
+ )
+ assert receiver._calculate_quota_usage(message=message, system_configuration=system, model_name="test") == 3
+ receiver.handle(message, application_generate_entity=entity)
+ calls.append(message.id)
+ if receiver_fails:
+ raise RuntimeError("receiver failed after commit")
+
+ if receiver_fails:
+ with pytest.raises(RuntimeError, match="receiver failed after commit"):
+ save_agent_stop(case, monkeypatch, "partial answer", process_stream=True, notify=notify)
+ else:
+ assert save_agent_stop(case, monkeypatch, "partial answer", process_stream=True, notify=notify) == []
+ assert calls == [case.collection.run.source_id]
+ assert provider_updates == ["basic_last_used_update"]
+ with case.collection.sessions() as session:
+ saved = session.get(Message, case.collection.run.source_id)
+ assert saved is not None
+ assert recovery.agent_stop_requested(saved) is True
+
+
[email protected]("answer", ["", "partial answer before stop"])
[email protected]("outcome", ["succeeded", "failed", "cancelled"])
[email protected]("delivery", ["accepted", "uncertain"])
+def test_stopped_consumer_cannot_freeze_a_terminal_before_the_runtime_worker(
+ case: RecoveryCase,
+ monkeypatch: pytest.MonkeyPatch,
+ answer: str,
+ outcome: Literal["succeeded", "failed", "cancelled"],
+ delivery: str,
+) -> None:
+ from typing import cast
+
+ from core.ops.entities.trace_entity import MessageTraceInfo, TraceTaskName
+ from models.model import MessageDict
+
+ # Simulate an old queued notification as well as the current consumer save:
+ # persisted state, not a detached event payload, must decide who owns the end.
+ notification = TraceTask(
+ TraceTaskName.MESSAGE_TRACE,
+ message_id=case.collection.run.source_id,
+ conversation_id=identifier("conversation-recover"),
+ )
+ info = notification.execute()
+ assert isinstance(info, MessageTraceInfo)
+ assert isinstance(info.message_data, dict)
+ info.message_data = Message.from_dict(cast(MessageDict, info.message_data))
+ exported: list[CanonicalTrace] = []
+
+ def export(_provider: str, _config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:
+ exported.append(trace)
+ if delivery == "uncertain":
+ raise TraceExportUncertainError("acknowledgment lost")
+ return TraceExportResult(trace_id=trace.trace_id)
+
+ monkeypatch.setattr(service_module, "export_trace", export)
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ queued = save_agent_stop(case, monkeypatch, answer)
+ assert case.service.capture_terminal(info) is True
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.outer_trace is None, "a stopped SSE save is not the runtime terminal"
+ assert queued == []
+ assert recovery.recover_terminal(case.service, stored) is False
+ assert case.collection.repository.claim(stored.id, collection_timeout=60, lease_seconds=30) is None
+ case.service.deliver(stored.id)
+ assert exported == []
+
+ output = "real runtime answer" if outcome == "succeeded" else None
+ case.service.capture_agent_app_terminal(
+ tenant_id=stored.tenant_id,
+ app_id=stored.app_id,
+ message_id=stored.source_id,
+ outcome=outcome,
+ output=output,
+ )
+ terminal = case.collection.repository.get(stored.id)
+ assert terminal is not None
+ trace = case.service.assemble(terminal)
+ root = next(span for span in trace.spans if span.id == trace.root_span_id)
+ assert root.metadata["message_execution_state"] == outcome
+ assert root.status.value == ("ok" if outcome == "succeeded" else "error")
+ assert len(trace.spans) == 3
+
+ # A duplicate worker result and late consumer notification cannot rewrite
+ # the frozen end or create a second ingestion attempt after acceptance.
+ case.service.deliver(stored.id)
+ case.service.deliver(stored.id)
+ assert len(exported) == 1
+ assert exported[0] == trace
+ frozen = case.collection.repository.get(stored.id)
+ assert frozen is not None
+ assert frozen.state == delivery
+ case.service.capture_agent_app_terminal(
+ tenant_id=stored.tenant_id,
+ app_id=stored.app_id,
+ message_id=stored.source_id,
+ outcome="succeeded" if outcome != "succeeded" else "cancelled",
+ output="late result must not replace the accepted envelope",
+ )
+ assert case.service.capture_terminal(info) is True
+ unchanged = case.collection.repository.get(stored.id)
+ assert unchanged is not None
+ assert unchanged.outer_trace == frozen.outer_trace
+ assert unchanged.attempts == frozen.attempts == 1
+ case.service.deliver(stored.id)
+ assert len(exported) == 1
+ with case.collection.sessions() as session:
+ message = session.get(Message, stored.source_id)
+ assert message is not None
+ assert message.status is MessageStatus.NORMAL
+ assert message.answer == answer
+ assert message.message_tokens == 2
+ assert message.answer_tokens == 1
+
+
[email protected]("mode", [AppMode.CHAT, AppMode.AGENT_CHAT])
+def test_classic_stopped_consumer_keeps_its_existing_trace_notification(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, mode: AppMode
+) -> None:
+ queued = save_agent_stop(case, monkeypatch, "classic partial answer", mode=mode)
+ assert len(queued) == 1
+ assert queued[0].trace_type is service_module.TraceTaskName.MESSAGE_TRACE
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ assert "agent_stop_requested" not in json.loads(message.message_metadata or "{}")
+
+
+def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from core.app.apps.agent_app.app_runner import AgentAppRunner
+ from graphon.model_runtime.entities.llm_entities import LLMUsage
+
+ # The worker's scoped session may already have loaded this Message before
+ # the consumer commits its independent stop transaction.
+ cached_message = db.session.get(Message, case.collection.run.source_id)
+ assert cached_message is not None
+ assert cached_message.message_metadata is None
+ save_agent_stop(case, monkeypatch, "partial answer")
+ AgentAppRunner._persist_message_usage(
+ message_id=cached_message.id,
+ usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}),
+ )
+ with case.collection.sessions() as session:
+ message = session.get(Message, cached_message.id)
+ assert message is not None
+ assert recovery.agent_stop_requested(message) is True
+ assert message.answer == "partial answer"
+ assert message.message_tokens == 13
+ assert message.answer_tokens == 8
+ assert json.loads(message.message_metadata or "{}")["usage"]["total_tokens"] == 21
+
+
+def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_read(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from core.app.apps.agent_app.app_runner import AgentAppRunner
+ from graphon.model_runtime.entities.llm_entities import LLMUsage
+
+ with case.collection.sessions.begin() as session:
+ initial = session.get(Message, case.collection.run.source_id)
+ assert initial is not None
+ initial.message_tokens = 0
+ initial.answer_tokens = 0
+ with case.collection.sessions() as consumer:
+ cached = consumer.get(Message, case.collection.run.source_id)
+ assert cached is not None
+ assert cached.message_tokens == 0
+ AgentAppRunner._persist_message_usage(
+ message_id=cached.id,
+ usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}),
+ )
+ save_agent_stop(case, monkeypatch, "partial answer", consumer_session=consumer)
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ assert recovery.agent_stop_requested(message) is True
+ assert message.message_tokens == 13
+ assert message.answer_tokens == 8
+ assert message.total_price == Decimal("0.21")
+ assert json.loads(message.message_metadata or "{}")["usage"]["total_tokens"] == 21
+
+
[email protected]("runtime_outcome", ["succeeded", "failed", "cancelled", "cancel_wait_failed"])
[email protected]("final_batch_arrived", [False, True])
[email protected]("snapshot_save_fails", [False, True])
+def test_stopped_runner_hands_off_only_an_observed_runtime_terminal(
+ case: RecoveryCase,
+ monkeypatch: pytest.MonkeyPatch,
+ runtime_outcome: str,
+ final_batch_arrived: bool,
+ snapshot_save_fails: bool,
+) -> None:
+ from typing import cast
+
+ from flask import Flask
+
+ from clients.agent_backend import FakeAgentBackendRunClient, FakeAgentBackendScenario
+ from core.app.apps.agent_app.session_store import AgentAppSessionScope
+ from core.app.apps.base_app_queue_manager import AppQueueManager
+ from core.app.apps.exc import GenerateTaskStoppedError
+ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
+ from core.ops.base_trace_instance import bind_application_tracing_hooks
+ from services.tracing import runtime_bridge
+ from tests.unit_tests.config_override import apply_config_overrides
+ from tests.unit_tests.core.app.apps.agent_app.test_app_runner import (
+ _CancelAndWaitFailingClient,
+ _ExplodingSessionStore,
+ _FakeSessionStore,
+ _runner,
+ _StoppedQueueManager,
+ )
+
+ save_agent_stop(case, monkeypatch, "partial answer")
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_COLLECTION_TIMEOUT=60)
+ pending_invocation = None
+ if not final_batch_arrived:
+ pending_invocation = case.collection.invocation("awaiting-stop-final-batch")
+ ingest(case.collection, batch_for(case.collection.run, pending_invocation, final=False, span_id="b" * 16))
+ client = (
+ _CancelAndWaitFailingClient()
+ if runtime_outcome == "cancel_wait_failed"
+ else FakeAgentBackendRunClient(
+ scenario=FakeAgentBackendScenario.FAILED
+ if runtime_outcome == "failed"
+ else FakeAgentBackendScenario.SUCCESS
+ )
+ )
+ if runtime_outcome in {"succeeded", "failed"}:
+ # The terminal frame has arrived, but stop is already visible at the
+ # next check. Do not discard this known result and ask SDK cancellation
+ # to reclassify it (the SDK rejects non-cancelled terminal events).
+ arrived_terminal = client._events(client.run_id)[-1]
+ monkeypatch.setattr(client, "stream_events", lambda _run_id, **_kwargs: iter([arrived_terminal]))
+
+ def unexpected_cancel(*_args: object, **_kwargs: object) -> None:
+ raise AssertionError("a known runtime terminal must not be cancelled again")
+
+ monkeypatch.setattr(client, "cancel_run_and_wait", unexpected_cancel)
+ store = _ExplodingSessionStore() if snapshot_save_fails else _FakeSessionStore()
+ runner = _runner(client, store)
+ run = case.collection.run
+ context = DifyRunContext(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ user_id=identifier("stop-race-user"),
+ user_from=UserFrom.ACCOUNT,
+ invoke_from=InvokeFrom.DEBUGGER,
+ )
+ scope = AgentAppSessionScope(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ conversation_id=identifier("conversation-recover"),
+ agent_id=identifier("stop-agent"),
+ agent_config_snapshot_id=identifier("stop-snapshot"),
+ home_snapshot_id=None,
+ )
+ application = Flask(__name__)
+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())
+ with application.app_context(), pytest.raises(GenerateTaskStoppedError):
+ runner._consume_stream(
+ client.run_id,
+ dify_context=context,
+ message_id=run.source_id,
+ queue_manager=cast(AppQueueManager, _StoppedQueueManager()),
+ model_name="test-model",
+ query="test query",
+ session_scope=scope,
+ binding_id=identifier("stop-binding"),
+ )
+ frozen = case.collection.repository.get(run.id)
+ assert frozen is not None
+ if runtime_outcome == "cancel_wait_failed":
+ assert frozen.outer_trace is None
+ assert store.saved == []
+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: case.now + timedelta(seconds=120))
+ assert recovery.recover_terminal(case.service, frozen) is False
+ expired = case.collection.repository.get(run.id)
+ assert expired is not None
+ assert expired.state == "failed"
+ assert expired.error_code == (
+ "business_terminal_not_persisted" if final_batch_arrived else "agent_trace_collection_incomplete"
+ )
+ return
+ assert frozen.outer_trace is not None
+ trace = case.service.assemble(frozen)
+ root = next(span for span in trace.spans if span.id == trace.root_span_id)
+ assert root.metadata["message_execution_state"] == runtime_outcome
+ assert root.status.value == ("ok" if runtime_outcome == "succeeded" else "error")
+ assert len(store.saved) == (0 if snapshot_save_fails else 1)
+ exported: list[CanonicalTrace] = []
+
+ def export(_provider: str, _config: dict[str, object], payload: CanonicalTrace) -> TraceExportResult:
+ exported.append(payload)
+ return TraceExportResult(trace_id=payload.trace_id)
+
+ monkeypatch.setattr(service_module, "export_trace", export)
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ if pending_invocation is not None:
+ assert case.collection.repository.claim(run.id, collection_timeout=60, lease_seconds=30) is None
+ case.service.deliver(run.id)
+ assert exported == []
+ ingest(case.collection, batch_for(run, pending_invocation, final=True, span_id="b" * 16))
+ case.service.deliver(run.id)
+ case.service.deliver(run.id)
+ assert len(exported) == 1
+ assert len(exported[0].spans) == (3 if final_batch_arrived else 5)
+ accepted = case.collection.repository.get(run.id)
+ assert accepted is not None
+ assert accepted.state == "accepted"
+ assert accepted.attempts == 1
+ with case.collection.sessions() as session:
+ message = session.get(Message, run.source_id)
+ assert message is not None
+ assert message.answer == "partial answer"
+ assert recovery.agent_stop_requested(message) is True
+
+
[email protected]("incomplete", [False, True])
+def test_missing_runtime_stop_terminal_converges_without_inventing_success(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, incomplete: bool
+) -> None:
+ from tests.unit_tests.config_override import apply_config_overrides
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_COLLECTION_TIMEOUT=60)
+ save_agent_stop(case, monkeypatch, "partial answer")
+ if incomplete:
+ invocation = case.collection.invocation("still-collecting")
+ ingest(case.collection, batch_for(case.collection.run, invocation, final=False, span_id="b" * 16))
+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: case.now + timedelta(seconds=120))
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.state == "failed"
+ assert stored.error_code == (
+ "agent_trace_collection_incomplete" if incomplete else "business_terminal_not_persisted"
+ )
+ assert stored.outer_trace is None
+ assert case.collection.repository.claim(stored.id, collection_timeout=60, lease_seconds=30) is None
+
+
[email protected]("stop_on_end", [False, True])
+def test_stop_during_final_queue_publication_preserves_the_already_observed_success(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, stop_on_end: bool
+) -> None:
+ from cachetools import TTLCache
+ from flask import Flask
+
+ from clients.agent_backend import FakeAgentBackendRunClient
+ from core.app.apps import base_app_queue_manager
+ from core.app.apps.base_app_queue_manager import PublishFrom
+ from core.app.apps.exc import GenerateTaskStoppedError
+ from core.app.apps.execution_coordinator import set_app_task_stop_flag
+ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager
+ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
+ from core.app.entities.queue_entities import (
+ AppQueueEvent,
+ MessageQueueMessage,
+ QueueLLMChunkEvent,
+ QueueMessageEndEvent,
+ )
+ from core.ops.base_trace_instance import bind_application_tracing_hooks
+ from services.tracing import runtime_bridge
+ from tests.unit_tests.core.app.apps.agent_app.test_app_runner import _FakeSessionStore, _runner, _soul
+
+ save_agent_stop(case, monkeypatch, "partial answer")
+ # Only Redis storage is in memory. Use the real queue's publish stop check
+ # rather than an already-stopped fake consumer or an exception-only stub.
+ flags: dict[str, object] = {}
+ monkeypatch.setattr(base_app_queue_manager.redis_client, "get", flags.get)
+ monkeypatch.setattr(
+ base_app_queue_manager.redis_client, "setex", lambda key, _ttl, value: flags.__setitem__(key, value)
+ )
+ monkeypatch.setattr(
+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None
+ )
+ task_id = identifier("stop-at-final-publish")
+ queue_manager = MessageBasedAppQueueManager(
+ task_id=task_id,
+ user_id=identifier("stop-race-user"),
+ invoke_from=InvokeFrom.DEBUGGER,
+ conversation_id=identifier("conversation-recover"),
+ app_mode=AppMode.AGENT,
+ message_id=case.collection.run.source_id,
+ )
+ clock = [0.0]
+ queue_manager._stopped_cache = TTLCache(maxsize=1, ttl=1, timer=lambda: clock[0])
+ store = _FakeSessionStore()
+ runner = _runner(FakeAgentBackendRunClient(), store)
+ original_publish = queue_manager._publish
+
+ def publish_after_stop(event: AppQueueEvent, pub_from: PublishFrom) -> None:
+ if isinstance(event, QueueMessageEndEvent if stop_on_end else QueueLLMChunkEvent):
+ set_app_task_stop_flag(task_id)
+ clock[0] = 2.0 # Expire the real queue's earlier cached false result.
+ original_publish(event, pub_from)
+
+ monkeypatch.setattr(queue_manager, "_publish", publish_after_stop)
+ run = case.collection.run
+ context = DifyRunContext(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ user_id=identifier("stop-race-user"),
+ user_from=UserFrom.ACCOUNT,
+ invoke_from=InvokeFrom.DEBUGGER,
+ )
+ application = Flask(__name__)
+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())
+ with application.app_context(), pytest.raises(GenerateTaskStoppedError):
+ runner.run(
+ dify_context=context,
+ agent_id=identifier("stop-agent"),
+ agent_config_snapshot_id=identifier("stop-snapshot"),
+ agent_soul=_soul(),
+ home_snapshot_id=None,
+ conversation_id=identifier("conversation-recover"),
+ query="test query",
+ message_id=run.source_id,
+ model_name="test-model",
+ queue_manager=queue_manager,
+ )
+ published = queue_manager._q.get_nowait()
+ assert isinstance(published, MessageQueueMessage)
+ assert isinstance(published.event, QueueLLMChunkEvent)
+ if stop_on_end:
+ published_end = queue_manager._q.get_nowait()
+ assert isinstance(published_end, MessageQueueMessage)
+ assert isinstance(published_end.event, QueueMessageEndEvent)
+ frozen = case.collection.repository.get(run.id)
+ assert frozen is not None
+ assert frozen.outer_trace is not None, "success arrived before the final queue publication saw Stop"
+ trace = CanonicalTrace.model_validate(frozen.outer_trace)
+ root = next(span for span in trace.spans if span.id == trace.root_span_id)
+ assert root.metadata["message_execution_state"] == "succeeded"
+ assert root.status.value == "ok"
+ assert len(store.saved) == 1
+
+
[email protected]("stop_during_cleanup", [False, True])
+def test_runtime_cleanup_cancellation_does_not_reclassify_a_business_stream_failure(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, stop_during_cleanup: bool
+) -> None:
+ from collections.abc import Iterator
+ from typing import cast
+
+ from dify_agent.protocol import RunEvent
+ from flask import Flask
+
+ from core.app.apps.agent_app.session_store import AgentAppSessionScope
+ from core.app.apps.base_app_queue_manager import AppQueueManager
+ from core.app.apps.exc import GenerateTaskStoppedError
+ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
+ from core.ops.base_trace_instance import bind_application_tracing_hooks
+ from services.tracing import runtime_bridge
+ from tests.unit_tests.config_override import apply_config_overrides
+ from tests.unit_tests.core.app.apps.agent_app.test_app_runner import (
+ _FakeQueueManager,
+ _FakeSessionStore,
+ _runner,
+ _UsageCancellationClient,
+ )
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ monkeypatch.setattr(runtime_bridge, "_service", lambda: case.service)
+ client = _UsageCancellationClient()
+ queue_manager = _FakeQueueManager()
+
+ def disconnected_stream(*_args: object, **_kwargs: object) -> Iterator[RunEvent]:
+ raise RuntimeError("stream transport failed")
+
+ monkeypatch.setattr(client, "stream_events", disconnected_stream)
+ if stop_during_cleanup:
+ original_cancel = client.cancel_run_and_wait
+
+ def cancel_then_stop(*_args: object, **_kwargs: object) -> RunEvent:
+ terminal = original_cancel(client.run_id)
+ queue_manager.request_stop()
+ return terminal
+
+ monkeypatch.setattr(client, "cancel_run_and_wait", cancel_then_stop)
+ store = _FakeSessionStore()
+ runner = _runner(client, store)
+ run = case.collection.run
+ context = DifyRunContext(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ user_id=identifier("cleanup-user"),
+ user_from=UserFrom.ACCOUNT,
+ invoke_from=InvokeFrom.DEBUGGER,
+ )
+ scope = AgentAppSessionScope(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ conversation_id=identifier("conversation-recover"),
+ agent_id=identifier("cleanup-agent"),
+ agent_config_snapshot_id=identifier("cleanup-snapshot"),
+ home_snapshot_id=None,
+ )
+ application = Flask(__name__)
+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())
+ with application.app_context():
+ with pytest.raises(GenerateTaskStoppedError if stop_during_cleanup else RuntimeError):
+ runner._consume_stream(
+ client.run_id,
+ dify_context=context,
+ message_id=run.source_id,
+ queue_manager=cast(AppQueueManager, queue_manager),
+ model_name="test-model",
+ query="test query",
+ session_scope=scope,
+ binding_id=identifier("cleanup-binding"),
+ )
+ if not stop_during_cleanup:
+ # This is the generator's existing generic exception handoff. A
+ # cleanup child-run cancellation must not win the application root.
+ runtime_bridge.capture_agent_app_terminal(
+ tenant_id=run.tenant_id,
+ app_id=run.app_id,
+ message_id=run.source_id,
+ outcome="failed",
+ output=None,
+ )
+ frozen = case.collection.repository.get(run.id)
+ assert frozen is not None
+ assert frozen.outer_trace is not None
+ trace = CanonicalTrace.model_validate(frozen.outer_trace)
+ root = next(span for span in trace.spans if span.id == trace.root_span_id)
+ assert root.metadata["message_execution_state"] == ("cancelled" if stop_during_cleanup else "failed")
+ assert root.status.value == "error"
+ assert len(store.saved) == 1
+ with case.collection.sessions() as session:
+ message = session.get(Message, run.source_id)
+ assert message is not None
+ assert message.message_tokens == 13
+ assert message.answer_tokens == 8
+ assert message.total_price == Decimal("0.21")
+ exported: list[CanonicalTrace] = []
+
+ def export(_provider: str, _config: dict[str, object], payload: CanonicalTrace) -> TraceExportResult:
+ exported.append(payload)
+ return TraceExportResult(trace_id=payload.trace_id)
+
+ monkeypatch.setattr(service_module, "export_trace", export)
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ case.service.deliver(run.id)
+ case.service.deliver(run.id)
+ accepted = case.collection.repository.get(run.id)
+ assert accepted is not None
+ assert accepted.state == "accepted"
+ assert accepted.attempts == 1
+ assert len(exported) == 1
+
+
[email protected]("capture_agent", [False, True])
+def test_missing_stop_worker_without_runtime_spans_also_reaches_a_delivery_failure(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, capture_agent: bool
+) -> None:
+ from tests.unit_tests.config_override import apply_config_overrides
+
+ collection = seed(case.collection.sessions, f"stop-without-spans-{capture_agent}", capture_agent=capture_agent)
+ attach_conversation(collection)
+ stopped = RecoveryCase(collection, ApplicationTracingService(collection.repository), case.now, [])
+ monkeypatch.setattr(stopped.service, "schedule", stopped.scheduled.append)
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_COLLECTION_TIMEOUT=60)
+ save_agent_stop(stopped, monkeypatch, "")
+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: case.now + timedelta(seconds=120))
+ assert recovery.recover_terminal(stopped.service, collection.run) is False
+ stored = collection.repository.get(collection.run.id)
+ assert stored is not None
+ assert stored.state == "failed"
+ assert stored.error_code == "business_terminal_not_persisted"
+ assert stored.outer_trace is None
+
+
[email protected]("outcome", ["succeeded", "cancelled", "failed"])
+def test_worker_terminal_survives_closed_sse_without_inventing_a_message_write(
+ case: RecoveryCase, outcome: Literal["succeeded", "failed", "cancelled"]
+) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.message_metadata = json.dumps({"usage": {"prompt_tokens": 2, "completion_tokens": 1}})
+ message.answer = ""
+ # A runtime final/usage flush still does not imply that an API message ended.
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ output = "actual worker result" if outcome == "succeeded" else None
+ case.service.capture_agent_app_terminal(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ message_id=case.collection.run.source_id,
+ outcome=outcome,
+ output=output,
+ )
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ trace = CanonicalTrace.model_validate(stored.outer_trace)
+ root = trace.spans[0]
+ assert root.metadata["message_execution_state"] == outcome
+ assert root.status.value == ("ok" if outcome == "succeeded" else "error")
+ assert root.outputs == (output or "")
+ assert case.scheduled == [stored.id]
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ assert message.answer == ""
+ assert message.message_metadata is not None
+ assert set(json.loads(message.message_metadata)) == {"usage"}
+
+
+def test_worker_terminal_revalidates_message_tenant(case: RecoveryCase) -> None:
+ from repositories.application_trace_repository import TraceCollectionError
+
+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found"):
+ case.service.capture_agent_app_terminal(
+ tenant_id=identifier("foreign-tenant"),
+ app_id=case.collection.run.app_id,
+ message_id=case.collection.run.source_id,
+ outcome="succeeded",
+ output="must not export",
+ )
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.outer_trace is None
+
+
+def test_recovers_real_message_terminal_once_through_existing_trace_task(case: RecoveryCase) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.message_metadata = complete_metadata()
+ message.updated_at = case.now - timedelta(seconds=11)
+ assert recovery.recover_terminal(case.service, case.collection.run) is True
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.outer_trace is not None
+ assert stored.outer_trace["session_id"] == "session-original"
+ assert stored.outer_trace["spans"][0]["outputs"] == "persisted final answer"
+ assert recovery.recover_terminal(case.service, stored) is False
+ assert case.scheduled == [stored.id]
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ assert message.message_tokens == 2
+ assert message.answer_tokens == 1
+ assert message.answer == "persisted final answer"
+
+
+def test_deliver_recovers_and_exports_once_after_terminal_queue_loss(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.message_metadata = complete_metadata()
+ message.updated_at = case.now - timedelta(seconds=11)
+ exported: list[CanonicalTrace] = []
+
+ def export(_provider: str, _config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:
+ exported.append(trace)
+ return TraceExportResult(trace_id="provider-id")
+
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ monkeypatch.setattr(service_module, "export_trace", export)
+ case.service.deliver(case.collection.run.id)
+ case.service.deliver(case.collection.run.id)
+ assert len(exported) == 1
+ assert exported[0].spans[0].outputs == "persisted final answer"
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.state == "accepted"
+ assert stored.attempts == 1
+
+
+def test_early_usage_commit_is_not_treated_as_a_final_answer(case: RecoveryCase) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.message_metadata = json.dumps({"usage": {"prompt_tokens": 2, "completion_tokens": 1}})
+ message.answer = ""
+ message.updated_at = case.now - timedelta(seconds=11)
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.outer_trace is None
+ assert stored.state == "collecting"
+ assert not case.scheduled
+
+
+def test_final_without_business_commit_times_out_as_failed_not_success(case: RecoveryCase) -> None:
+ invocations, _ = case.collection.repository.collected(case.collection.run.id)
+ with case.collection.sessions.begin() as session:
+ invocation = session.get(ApplicationTraceInvocation, invocations[0].id)
+ assert invocation is not None
+ invocation.finalized_at = case.now - timedelta(hours=1)
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.state == "failed"
+ assert stored.error_code == "business_terminal_not_persisted"
+ assert stored.outer_trace is None
+
+
+def test_final_settle_window_and_all_invocations_must_finish(case: RecoveryCase) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.message_metadata = complete_metadata()
+ message.updated_at = case.now - timedelta(seconds=1)
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ second = case.collection.invocation("pending-second")
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.updated_at = case.now - timedelta(seconds=10)
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ with case.collection.sessions() as session:
+ invocation = session.get(ApplicationTraceInvocation, second.id)
+ assert invocation is not None
+ assert invocation.finalized is False
+
+
+def test_explicit_message_failure_can_recover_without_fabricating_usage(case: RecoveryCase) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.status = MessageStatus.ERROR
+ message.error = "model initialization failed"
+ message.answer = ""
+ message.message_metadata = None
+ message.message_tokens = 0
+ message.answer_tokens = 0
+ message.updated_at = case.now - timedelta(seconds=11)
+ assert recovery.recover_terminal(case.service, case.collection.run) is True
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.outer_trace is not None
+ assert stored.outer_trace["spans"][0]["status"] == "error"
+ assert stored.outer_trace["spans"][0]["outputs"] == ""
+ assert stored.outer_trace["spans"][0]["metadata"]["aggregate_total_tokens"] == 0
+
+
+def test_cross_tenant_source_is_rejected_before_trace_task_executes(
+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ with case.collection.sessions.begin() as session:
+ app = session.get(App, case.collection.run.app_id)
+ assert app is not None
+ app.tenant_id = identifier("other-tenant")
+ called: list[bool] = []
+
+ def forbidden(_task: TraceTask) -> None:
+ called.append(True)
+
+ monkeypatch.setattr(TraceTask, "execute", forbidden)
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ assert called == []
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.error_code == "tracing_source_not_found"
+
+
+def test_unresolved_recovery_rotates_behind_unchecked_sources(case: RecoveryCase) -> None:
+ other = seed(case.collection.sessions, "other-recovery")
+ assert case.collection.repository.pending_ids(limit=1) == [case.collection.run.id]
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ assert case.collection.repository.pending_ids(limit=1) == [other.run.id]
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert stored.state == "collecting"
+
+
[email protected]("status", [MessageStatus.PAUSED, MessageStatus.NORMAL])
+def test_no_final_marker_or_paused_message_is_not_recovered(case: RecoveryCase, status: MessageStatus) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.status = status
+ message.updated_at = case.now - timedelta(seconds=10)
+ assert recovery.recover_terminal(case.service, case.collection.run) is False
+ assert not case.scheduled
+
+
+ ("status", "finished"),
+ [
+ (WorkflowExecutionStatus.RUNNING, False),
+ (WorkflowExecutionStatus.PAUSED, True),
+ (WorkflowExecutionStatus.SUCCEEDED, False),
+ (WorkflowExecutionStatus.SUCCEEDED, True),
+ (WorkflowExecutionStatus.PARTIAL_SUCCEEDED, True),
+ (WorkflowExecutionStatus.FAILED, True),
+ (WorkflowExecutionStatus.STOPPED, True),
+ ],
+)
+def test_workflow_recovery_requires_committed_terminal_state(
+ case: RecoveryCase, status: WorkflowExecutionStatus, finished: bool
+) -> None:
+ owner_id, workflow_run_id = identifier("workflow-owner"), identifier("recover-workflow-run")
+ with case.collection.sessions.begin() as session:
+ app = session.get(App, case.collection.run.app_id)
+ assert app is not None
+ app.created_by = owner_id
+ session.add(make_account(account_id=owner_id))
+ session.add(
+ WorkflowRun(
+ id=workflow_run_id,
+ tenant_id=case.collection.run.tenant_id,
+ app_id=app.id,
+ workflow_id=identifier("workflow"),
+ type=WorkflowType.WORKFLOW,
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ version="1",
+ graph="{}",
+ inputs="{}",
+ outputs=json.dumps({"result": "persisted workflow output"}),
+ status=status,
+ error="actual workflow failure"
+ if status in {WorkflowExecutionStatus.FAILED, WorkflowExecutionStatus.STOPPED}
+ else None,
+ created_by=owner_id,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_at=case.now - timedelta(seconds=30),
+ finished_at=case.now - timedelta(seconds=10) if finished else None,
+ elapsed_time=20,
+ total_tokens=0,
+ total_steps=0,
+ )
+ )
+ run = case.collection.repository.start(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ source_kind="workflow",
+ source_id=workflow_run_id,
+ session_id="workflow-session",
+ )
+ assert run is not None
+ expected = finished and status in {
+ WorkflowExecutionStatus.SUCCEEDED,
+ WorkflowExecutionStatus.PARTIAL_SUCCEEDED,
+ WorkflowExecutionStatus.FAILED,
+ WorkflowExecutionStatus.STOPPED,
+ }
+ assert recovery.recover_terminal(case.service, run) is expected
+ stored = case.collection.repository.get(run.id)
+ assert stored is not None
+ if expected:
+ assert stored.outer_trace is not None
+ assert stored.outer_trace["root_span_id"] == span_id(workflow_run_id)
+ assert stored.outer_trace["spans"][0]["outputs"] == {"result": "persisted workflow output"}
+ else:
+ assert stored.outer_trace is None
+
+
[email protected]("formatted", [None, "", " \n\t", "{}", "[]", "null", " { } ", "[\n]", {}, []])
[email protected]("capture_content", [False, True])
+def test_early_error_root_preserves_known_query_only_when_content_is_enabled(
+ case: RecoveryCase, formatted: object, capture_content: bool
+) -> None:
+ from sqlalchemy import update
+
+ from models.application_trace import ApplicationTraceRun
+
+ query = "KNOWN_USER_QUERY_BEFORE_FORMATTED_MESSAGE"
+ with case.collection.sessions.begin() as session:
+ # Historical rows can contain JSON strings/lists despite the ORM's dict annotation.
+ session.execute(
+ update(Message.metadata.tables["messages"])
+ .where(Message.id == case.collection.run.source_id)
+ .values(message=formatted, query=query, answer="")
+ )
+ run = session.get(ApplicationTraceRun, case.collection.run.id)
+ assert run is not None
+ run.capture_content = capture_content
+ case.service.capture_agent_app_terminal(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ message_id=case.collection.run.source_id,
+ outcome="failed",
+ output=None,
+ )
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ trace = CanonicalTrace.model_validate(stored.outer_trace)
+ root = trace.spans[0]
+ assert root.status.value == "error"
+ assert root.metadata["message_execution_state"] == "failed"
+ assert root.inputs == ({"query": query} if capture_content else None)
+ assert root.outputs == ("" if capture_content else None)
+ assembled = case.service.assemble(stored)
+ assert (query in assembled.model_dump_json()) is capture_content
+ assert all(span.inputs is None for span in assembled.spans if span.id != assembled.root_span_id)
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ assert message.message == formatted
+ assert message.query == query
+ assert message.answer == ""
+
+
+ ("formatted", "expected"),
+ [
+ (
+ {"messages": [{"role": "user", "content": "actual formatted input"}]},
+ {"messages": [{"role": "user", "content": "actual formatted input"}]},
+ ),
+ (
+ '{"messages":[{"role":"user","content":"actual formatted input"}]}',
+ {"messages": [{"role": "user", "content": "actual formatted input"}]},
+ ),
+ (
+ [{"role": "user", "content": "actual formatted input"}],
+ [{"role": "user", "content": "actual formatted input"}],
+ ),
+ (
+ '[{"role":"user","content":"actual formatted input"}]',
+ [{"role": "user", "content": "actual formatted input"}],
+ ),
+ ("opaque non-JSON formatted input", "opaque non-JSON formatted input"),
+ ("0", "0"),
+ ("false", "false"),
+ ],
+)
+def test_worker_handoff_never_replaces_existing_formatted_inputs(
+ case: RecoveryCase, formatted: object, expected: object
+) -> None:
+ from sqlalchemy import update
+
+ query = "QUERY_MUST_NOT_REPLACE_FORMATTED_INPUTS"
+ with case.collection.sessions.begin() as session:
+ session.execute(
+ update(Message.metadata.tables["messages"])
+ .where(Message.id == case.collection.run.source_id)
+ .values(message=formatted, query=query, answer="")
+ )
+ case.service.capture_agent_app_terminal(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ message_id=case.collection.run.source_id,
+ outcome="failed",
+ output=None,
+ )
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ trace = CanonicalTrace.model_validate(stored.outer_trace)
+ assert trace.spans[0].inputs == expected
+ assert query not in trace.model_dump_json()
+ with case.collection.sessions() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ assert message.message == formatted
+
+
+def test_worker_handoff_does_not_invent_input_when_query_is_empty(case: RecoveryCase) -> None:
+ with case.collection.sessions.begin() as session:
+ message = session.get(Message, case.collection.run.source_id)
+ assert message is not None
+ message.message = {}
+ message.query = ""
+ message.answer = ""
+ case.service.capture_agent_app_terminal(
+ tenant_id=case.collection.run.tenant_id,
+ app_id=case.collection.run.app_id,
+ message_id=case.collection.run.source_id,
+ outcome="failed",
+ output=None,
+ )
+ stored = case.collection.repository.get(case.collection.run.id)
+ assert stored is not None
+ assert CanonicalTrace.model_validate(stored.outer_trace).spans[0].inputs == {}
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_repository_boundaries.py b/api/tests/unit_tests/services/tracing/test_application_trace_repository_boundaries.py
new file mode 100644
index 00000000000..7122829140b
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_repository_boundaries.py
@@ -0,0 +1,209 @@
+"""Persistence boundaries that must fail closed or remain immutable across retries."""
+
+from datetime import timedelta
+from uuid import UUID
+
+import pytest
+from sqlalchemy import delete, func, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun
+from models.model import App
+from repositories.application_trace_repository import TraceCollectionError
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ batch_for,
+ claim,
+ identifier,
+ ingest,
+ seed,
+ terminal,
+)
+
+
[email protected]("policy", ["invalid-json", "null", "[]"])
+def test_malformed_policy_freezes_disabled_decision(sqlite_session_factory: sessionmaker[Session], policy: str) -> None:
+ case = seed(sqlite_session_factory, "malformed")
+ with case.sessions.begin() as session:
+ session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id == case.run.id))
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.tracing = policy
+ run = case.repository.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="message",
+ source_id=case.run.source_id,
+ session_id="",
+ )
+ assert run is not None
+ assert run.state == "not_collected"
+ assert run.provider_config == {}
+ assert run.session_id == case.run.source_id
+ assert not run.capture_content
+ assert not run.capture_agent
+
+
+def test_reserved_preview_requires_persisted_owner_workflow(sqlite_session_factory: sessionmaker[Session]) -> None:
+ case = seed(sqlite_session_factory, "missing-preview-owner")
+ assert (
+ case.repository.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="node",
+ source_id=identifier("preview"),
+ session_id="preview",
+ workflow_id=identifier("missing-workflow"),
+ )
+ is None
+ )
+
+
+def test_initialization_failure_cannot_replace_snapshot_or_create_foreign_row(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ case = seed(sqlite_session_factory, "failure-snapshot")
+ same = case.repository.record_initialization_failure(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="message",
+ source_id=case.run.source_id,
+ )
+ assert same is not None
+ assert same.id == case.run.id
+ assert same.state == "collecting"
+ assert same.provider_config == case.run.provider_config
+ assert (
+ case.repository.record_initialization_failure(
+ tenant_id=identifier("foreign-tenant"),
+ app_id=case.run.app_id,
+ source_kind="message",
+ source_id=case.run.source_id,
+ )
+ is None
+ )
+
+
[email protected]("target", ["missing_run", "revoked_owner", "closed_invocation"])
+def test_ingestion_rejects_missing_or_closed_owner_without_writing(
+ sqlite_session_factory: sessionmaker[Session], target: str
+) -> None:
+ case = seed(sqlite_session_factory, "ingestion-owner")
+ invocation = case.invocation()
+ batch = batch_for(case.run, invocation)
+ if target == "missing_run":
+ batch = batch.model_copy(update={"run_id": UUID(identifier("missing-run"))})
+ error = "tracing_run_not_found"
+ elif target == "revoked_owner":
+ with case.sessions.begin() as session:
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.tenant_id = identifier("another-tenant")
+ error = "tracing_owner_not_found"
+ else:
+ with case.sessions.begin() as session:
+ stored = session.get(ApplicationTraceInvocation, invocation.id)
+ assert stored is not None
+ stored.finalized = True
+ error = "tracing_collection_closed"
+ with pytest.raises(TraceCollectionError, match=error):
+ ingest(case, batch)
+ with case.sessions() as session:
+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0
+ stored_run = case.repository.get(case.run.id)
+ assert stored_run is not None
+ assert stored_run.span_count == 0
+
+
+def test_first_terminal_is_immutable_and_revoked_owner_is_never_exported(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ case = seed(sqlite_session_factory, "terminal-owner")
+ terminal(case)
+ before = case.repository.get(case.run.id)
+ assert before is not None
+ case.repository.terminal(case.run.id, {"replacement": True}, "f" * 16)
+ after = case.repository.get(case.run.id)
+ assert after is not None
+ assert after.outer_trace == before.outer_trace
+ assert after.root_span_id == before.root_span_id
+ with case.sessions.begin() as session:
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.tenant_id = identifier("moved-owner")
+ assert claim(case) is None
+ failed = case.repository.get(case.run.id)
+ assert failed is not None
+ assert failed.error_code == "tracing_owner_not_found"
+ assert failed.state == "failed"
+ case.repository.terminal(case.run.id, {"replacement": True}, "f" * 16)
+ frozen = case.repository.get(case.run.id)
+ assert frozen is not None
+ assert frozen.outer_trace == before.outer_trace
+
+
+ ("state", "error"),
+ [
+ ("not_collected", "tracing_run_not_found"),
+ ("accepted", "tracing_already_accepted"),
+ ("collecting", "tracing_retry_not_allowed"),
+ ],
+)
+def test_retry_rejects_ineligible_states(sqlite_session_factory: sessionmaker[Session], state: str, error: str) -> None:
+ case = seed(sqlite_session_factory, "retry-state")
+ terminal(case)
+ with case.sessions.begin() as session:
+ row = session.get(ApplicationTraceRun, case.run.id)
+ assert row is not None
+ row.state = state
+ with pytest.raises(TraceCollectionError, match=error):
+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)
+ row = case.repository.get(case.run.id)
+ assert row is not None
+ assert row.state == state
+
+
+def test_retry_unknown_run_and_invocation_reject_before_creation(sqlite_session_factory: sessionmaker[Session]) -> None:
+ case = seed(sqlite_session_factory, "retry-unknown")
+ with pytest.raises(TraceCollectionError, match="tracing_run_not_found"):
+ case.repository.retry(run_id=identifier("missing"), acknowledge_possible_duplicates=True)
+ with pytest.raises(TraceCollectionError, match="tracing_run_not_collecting"):
+ case.repository.invocation(
+ run_id=identifier("missing"),
+ agent_id=identifier("agent"),
+ node_execution_id=None,
+ attempt=0,
+ call_key="x",
+ )
+ assert case.repository.claim(identifier("missing"), collection_timeout=1, lease_seconds=1) is None
+
+
+def test_collection_without_terminal_is_not_exportable(sqlite_session_factory: sessionmaker[Session]) -> None:
+ case = seed(sqlite_session_factory, "no-terminal")
+ ingest(case, batch_for(case.run, case.invocation()))
+ assert claim(case) is None
+ row = case.repository.get(case.run.id)
+ assert row is not None
+ assert row.attempts == 0
+ assert row.state == "collecting"
+
+
+def test_retention_removes_all_payload_rows_and_preserves_newer_runs(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ expired = seed(sqlite_session_factory, "expired")
+ fresh = seed(sqlite_session_factory, "fresh")
+ for case in (expired, fresh):
+ ingest(case, batch_for(case.run, case.invocation()))
+ with expired.sessions.begin() as session:
+ row = session.get(ApplicationTraceRun, expired.run.id)
+ assert row is not None
+ row.created_at = naive_utc_now() - timedelta(days=8)
+ assert expired.repository.purge(retention_days=7) == 1
+ assert expired.repository.get(expired.run.id) is None
+ assert expired.repository.collected(expired.run.id) == ([], [])
+ assert fresh.repository.get(fresh.run.id) is not None
+ invocations, batches = fresh.repository.collected(fresh.run.id)
+ assert len(invocations) == len(batches) == 1
+ assert expired.repository.purge(retention_days=7) == 0
diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_service_boundaries.py b/api/tests/unit_tests/services/tracing/test_application_trace_service_boundaries.py
new file mode 100644
index 00000000000..52baaeaf778
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_application_trace_service_boundaries.py
@@ -0,0 +1,425 @@
+"""Exercise public orchestration, privacy, and delivery decisions against durable rows."""
+
+from datetime import datetime
+from uuid import UUID
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.exceptions import RetryableTraceDispatchError
+from graphon.enums import WorkflowNodeExecutionStatus
+from models.application_trace import ApplicationTraceRun
+from models.enums import CreatorUserRole
+from models.model import Message, TraceAppConfig
+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
+from repositories.application_trace_repository import TraceCollectionError
+from services.tracing import service as service_module
+from services.tracing.provider_dispatch import TraceExportResult
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_message, make_workflow
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ Collection,
+ batch_for,
+ claim,
+ identifier,
+ ingest,
+ seed,
+ terminal,
+)
+
+
+def source(case: Collection) -> dict[str, str]:
+ return {
+ "tenant_id": case.run.tenant_id,
+ "app_id": case.run.app_id,
+ "source_kind": "message",
+ "source_id": case.run.source_id,
+ }
+
+
+def test_prepare_invocation_returns_only_opaque_context_and_respects_capture_gate(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ case = seed(sqlite_session_factory, "prepare", capture_content=True)
+ service = ApplicationTracingService(case.repository)
+ context = service.prepare_invocation(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ agent_id=identifier("agent"),
+ source_kind="message",
+ source_id=case.run.source_id,
+ session_id="ignored-new-session",
+ node_execution_id=None,
+ attempt=0,
+ call_key="call",
+ )
+ assert context is not None
+ assert context.run_id == UUID(case.run.id)
+ assert context.capture_content is True
+ assert context.session_id == "session-original"
+ assert "encrypted-original" not in context.model_dump_json()
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+ assert (
+ service.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="message",
+ source_id=case.run.source_id,
+ session_id="new",
+ )
+ is None
+ )
+
+
[email protected]("missing", ["tenant", "app", "message"])
+def test_status_denies_foreign_or_absent_source_without_destination_details(
+ sqlite_session_factory: sessionmaker[Session], missing: str
+) -> None:
+ case = seed(sqlite_session_factory, "scope")
+ identity = source(case)
+ identity[{"tenant": "tenant_id", "app": "app_id", "message": "source_id"}[missing]] = identifier("foreign")
+ with pytest.raises(TraceCollectionError) as caught:
+ ApplicationTracingService(case.repository).status(**identity)
+ assert caught.value.code == "tracing_source_not_found"
+ assert caught.value.status == 404
+ assert "encrypted-original" not in str(caught.value)
+
+
+def test_status_uses_frozen_destination_and_selects_latest_node_attempt(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ case = seed(sqlite_session_factory, "status")
+ with case.sessions.begin() as session:
+ config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == case.run.app_id))
+ assert config is not None
+ config.tracing_config = {"project_url": "https://new.example/project/new", "secret_key": "encrypted-new"}
+
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("status must not decrypt credentials or resolve current provider metadata")
+
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", forbidden)
+ monkeypatch.setattr(service_module.OpsTraceManager, "get_trace_config_project_url", forbidden)
+ node_id = identifier("selected-node")
+ invocation = case.repository.invocation(
+ run_id=case.run.id, agent_id=identifier("agent"), node_execution_id=node_id, attempt=1, call_key="node:1"
+ )
+ ingest(case, batch_for(case.run, invocation))
+ terminal(case)
+ lease = claim(case)
+ assert lease is not None
+ case.repository.complete(
+ case.run.id,
+ lease[1],
+ state="accepted",
+ result=TraceExportResult(
+ "provider-trace",
+ trace_url="https://frozen.example/trace",
+ span_urls={invocation.span_id: "https://frozen.example/span"},
+ ),
+ )
+ service = ApplicationTracingService(case.repository)
+ result = service.status(**source(case), node_execution_id=node_id)
+ assert result["status"] == "accepted"
+ assert result["trace_id"] == "provider-trace"
+ assert result["span_url"] == "https://frozen.example/span"
+ assert result["project_url"] == "https://langfuse.test/project/original"
+ assert result["config_revision"] == case.run.config_revision
+ assert result["queryable"] is None
+ assert result["capture_content"] is False
+ assert "provider_config" not in result
+ assert "encrypted-original" not in str(result)
+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found"):
+ service.status(**source(case), node_execution_id=identifier("another-node"))
+
+
+def test_new_run_freezes_updated_project_without_changing_previous_run(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ case = seed(sqlite_session_factory, "updated-project")
+ new_id = identifier("new-message")
+ with case.sessions.begin() as session:
+ config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == case.run.app_id))
+ assert config is not None
+ config.tracing_config = {"project_url": "https://new.example/project/new", "secret_key": "encrypted-new"}
+ message = session.get(Message, case.run.source_id)
+ assert message is not None
+ session.add(
+ make_message(
+ message_id=new_id,
+ app_id=case.run.app_id,
+ conversation_id=message.conversation_id,
+ inputs={},
+ query="new run",
+ message={},
+ answer="done",
+ message_unit_price=message.message_unit_price,
+ answer_unit_price=message.answer_unit_price,
+ currency=message.currency,
+ from_source=message.from_source,
+ )
+ )
+ new_run = case.repository.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="message",
+ source_id=new_id,
+ session_id="new-session",
+ )
+ assert new_run is not None
+ service = ApplicationTracingService(case.repository)
+ assert service.status(**source(case))["project_url"] == "https://langfuse.test/project/original"
+ assert service.status(**{**source(case), "source_id": new_id})["project_url"] == "https://new.example/project/new"
+
+
+ "project_url",
+ [
+ None,
+ "",
+ 1,
+ "javascript:alert(1)",
+ "//example.com/project",
+ "https://user:[email protected]/project",
+ "https://@example.com",
+ "https://example.com\\@other.test",
+ "https://example.com:invalid/project",
+ "https://[invalid/project",
+ "https://example.com/\nproject",
+ ],
+)
+def test_status_hides_unavailable_or_unsafe_frozen_project_urls(
+ sqlite_session_factory: sessionmaker[Session], project_url: object
+) -> None:
+ case = seed(sqlite_session_factory, "invalid-project")
+ with case.sessions.begin() as session:
+ run = session.get(ApplicationTraceRun, case.run.id)
+ assert run is not None
+ run.provider_config = {
+ "secret_key": "encrypted-original",
+ **({"project_url": project_url} if project_url is not None else {}),
+ }
+ result = ApplicationTracingService(case.repository).status(**source(case))
+ assert result["project_url"] is None
+ assert "encrypted-original" not in str(result)
+
+
[email protected]("provider", ["aliyun", "tencent"])
+def test_status_does_not_present_platform_console_as_project(
+ sqlite_session_factory: sessionmaker[Session], provider: str
+) -> None:
+ case = seed(sqlite_session_factory, "platform-console")
+ with case.sessions.begin() as session:
+ run = session.get(ApplicationTraceRun, case.run.id)
+ assert run is not None
+ run.provider = provider
+ run.provider_config = {"project_url": "https://console.example.com"}
+ assert ApplicationTracingService(case.repository).status(**source(case))["project_url"] is None
+
+
[email protected]("disabled", [True, False])
+def test_uncollected_status_and_missing_run_retry_do_not_construct_provider(
+ sqlite_session_factory: sessionmaker[Session], disabled: bool
+) -> None:
+ case = seed(sqlite_session_factory, "missing", enabled=not disabled)
+ if not disabled:
+ with case.sessions.begin() as session:
+ stored = session.get(ApplicationTraceRun, case.run.id)
+ assert stored is not None
+ session.delete(stored)
+ service = ApplicationTracingService(case.repository)
+ assert service.status(**source(case)) == {
+ "status": "not_collected",
+ "source_kind": "message",
+ "source_id": case.run.source_id,
+ }
+ if not disabled:
+ with pytest.raises(TraceCollectionError, match="tracing_run_not_found"):
+ service.retry(**source(case), acknowledge_possible_duplicates=True)
+
+
+def test_retry_acknowledgement_guards_schedule_and_keeps_frozen_snapshot(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ case = seed(sqlite_session_factory, "retry")
+ ingest(case, batch_for(case.run, case.invocation()))
+ terminal(case)
+ lease = claim(case)
+ assert lease is not None
+ case.repository.complete(case.run.id, lease[1], state="uncertain", error_code="provider_acceptance_unknown")
+ scheduled: list[str] = []
+ service = ApplicationTracingService(case.repository)
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ with pytest.raises(TraceCollectionError, match="tracing_duplicate_acknowledgement_required"):
+ service.retry(**source(case), acknowledge_possible_duplicates=False)
+ assert scheduled == []
+ result = service.retry(**source(case), acknowledge_possible_duplicates=True)
+ assert scheduled == [case.run.id]
+ assert result["status"] == "ready"
+ assert result["config_revision"] == case.run.config_revision
+ current = case.repository.get(case.run.id)
+ assert current is not None
+ assert current.provider_config == case.run.provider_config
+
+
+ ("failure", "code"),
+ [
+ (RetryableTraceDispatchError("private-transport-error"), "provider_retry_exhausted"),
+ (ValueError("private-credential-error"), "provider_export_rejected"),
+ ],
+)
+def test_delivery_exhaustion_and_rejection_never_log_vendor_details(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ failure: Exception,
+ code: str,
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_MAX_ATTEMPTS=1)
+ case = seed(sqlite_session_factory, "delivery")
+ ingest(case, batch_for(case.run, case.invocation()))
+ terminal(case)
+
+ def export(*_args: object) -> TraceExportResult:
+ raise failure
+
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ monkeypatch.setattr(service_module, "export_trace", export)
+ service = ApplicationTracingService(case.repository)
+ service.deliver(case.run.id)
+ current = case.repository.get(case.run.id)
+ assert current is not None
+ assert current.state == "failed"
+ assert current.error_code == code
+ assert "private-" not in caplog.text
+ service.deliver(identifier("missing-run"))
+
+
[email protected]("capture_content", [False, True])
+ "status",
+ [WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.SUCCEEDED, WorkflowNodeExecutionStatus.FAILED],
+)
+def test_preview_terminal_uses_persisted_node_state_and_frozen_privacy(
+ sqlite_session_factory: sessionmaker[Session], status: WorkflowNodeExecutionStatus, capture_content: bool
+) -> None:
+ case = seed(sqlite_session_factory, "preview", capture_content=capture_content)
+ node_id = identifier("preview-node")
+ workflow_id = identifier("preview-definition")
+ with case.sessions.begin() as session:
+ session.add(make_workflow(workflow_id=workflow_id, tenant_id=case.run.tenant_id, app_id=case.run.app_id))
+ session.add(
+ WorkflowNodeExecutionModel(
+ id=node_id,
+ node_execution_id=node_id,
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ workflow_id=workflow_id,
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
+ index=0,
+ node_id="agent",
+ node_type="agent",
+ title="Agent preview",
+ inputs='{"secret":"private-input"}',
+ outputs='{"answer":"private-output"}',
+ process_data="{}",
+ status=status,
+ elapsed_time=2,
+ error="private-model-error" if status == WorkflowNodeExecutionStatus.FAILED else None,
+ created_at=datetime(2026, 1, 1),
+ created_by=identifier("account"),
+ created_by_role=CreatorUserRole.ACCOUNT,
+ )
+ )
+ run = case.repository.start(
+ tenant_id=case.run.tenant_id,
+ app_id=case.run.app_id,
+ source_kind="node",
+ source_id=node_id,
+ workflow_id=workflow_id,
+ session_id="preview-session",
+ )
+ assert run is not None
+ ApplicationTracingService(case.repository).settle_preview(run)
+ current = case.repository.get(run.id)
+ assert current is not None
+ if status == WorkflowNodeExecutionStatus.RUNNING:
+ assert current.outer_trace is None
+ else:
+ assert current.outer_trace is not None
+ trace = service_module.CanonicalTrace.model_validate(current.outer_trace)
+ root = trace.spans[0]
+ assert root.status.value == ("error" if status == WorkflowNodeExecutionStatus.FAILED else "ok")
+ assert ("private-" in trace.model_dump_json()) is capture_content
+ assert root.metadata["node_execution_id"] == node_id
+ if not capture_content and status == WorkflowNodeExecutionStatus.FAILED:
+ assert root.error == "Node execution failed"
+
+
[email protected]("owner_state", ["no_creator", "missing_account", "wrong_app"])
+def test_missing_owner_identity_cannot_authorize_trace_expansion(
+ sqlite_session_factory: sessionmaker[Session], owner_state: str
+) -> None:
+ from models.model import App
+
+ case = seed(sqlite_session_factory, "owner")
+ with case.sessions.begin() as session:
+ app = session.get(App, case.run.app_id)
+ assert app is not None
+ app.created_by = None if owner_state == "no_creator" else identifier("missing-owner")
+ app_id = identifier("another-app") if owner_state == "wrong_app" else case.run.app_id
+ with pytest.raises(TraceCollectionError, match="tracing_owner_not_found"):
+ ApplicationTracingService(case.repository)._service_account(case.run, app_id)
+
+
+ ("invalid", "code"),
+ [
+ ("empty", "tracing_invocation_empty"),
+ ("foreign_parent", "tracing_invocation_parent_invalid"),
+ ("no_invocation_root", "tracing_invocation_root_missing"),
+ ("conflicting_span", "tracing_span_conflict"),
+ ],
+)
+def test_invalid_runtime_tree_is_rejected_before_provider_credentials_or_io(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, invalid: str, code: str
+) -> None:
+ case = seed(sqlite_session_factory, "invalid-runtime")
+ batch = batch_for(case.run, case.invocation())
+ if invalid == "empty":
+ batch.spans = []
+ elif invalid == "foreign_parent":
+ batch.spans[0] = batch.spans[0].model_copy(update={"parent_span_id": "f" * 16})
+ elif invalid == "no_invocation_root":
+ batch.spans[0] = batch.spans[0].model_copy(update={"parent_span_id": batch.spans[0].span_id})
+ else:
+ ingest(case, batch.model_copy(update={"final": False}))
+ from uuid import uuid4
+
+ batch = batch.model_copy(
+ update={"batch_id": uuid4(), "spans": [batch.spans[0].model_copy(update={"name": "conflicting-version"})]}
+ )
+ ingest(case, batch)
+ terminal(case)
+
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("invalid trees must fail before credentials or provider requests")
+
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", forbidden)
+ monkeypatch.setattr(service_module, "export_trace", forbidden)
+ service = ApplicationTracingService(case.repository)
+ service.deliver(case.run.id)
+ current = case.repository.get(case.run.id)
+ assert current is not None
+ assert current.state == "failed"
+ assert current.error_code == code
+
+
+def test_assemble_requires_a_persisted_business_terminal(sqlite_session_factory: sessionmaker[Session]) -> None:
+ case = seed(sqlite_session_factory, "no-terminal")
+ with pytest.raises(TraceCollectionError, match="tracing_terminal_missing"):
+ ApplicationTracingService(case.repository).assemble(case.run)
diff --git a/api/tests/unit_tests/services/tracing/test_arize_configuration.py b/api/tests/unit_tests/services/tracing/test_arize_configuration.py
new file mode 100644
index 00000000000..7e6a1e587d1
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_arize_configuration.py
@@ -0,0 +1,260 @@
+"""Arize metadata reads are optional, bounded and pinned to the exact destination."""
+
+from collections import deque
+from collections.abc import Iterator
+from typing import Never
+
+import httpx
+import pytest
+from dify_trace_arize_phoenix import arize_configuration as configuration
+from dify_trace_arize_phoenix.arize_configuration import (
+ DESTINATION_KEY,
+ bind_destination,
+ project_url,
+ resolve_project_destination,
+)
+
+from core.helper import ssrf_proxy
+
+SPACE = "U3BhY2U6dGVzdA=="
+ORG = "T3JnOnRlc3Q="
+PROJECT = "UHJvamVjdDovKz0="
+VALUES = {"endpoint": "https://otlp.arize.com", "space_id": SPACE, "project": "exact", "api_key": "synthetic-key"}
+DESTINATION = {
+ "endpoint": VALUES["endpoint"],
+ "space_id": SPACE,
+ "project": "exact",
+ "organization_id": ORG,
+ "project_id": PROJECT,
+}
+URL = "https://app.arize.com/organizations/T3JnOnRlc3Q%3D/spaces/U3BhY2U6dGVzdA%3D%3D/projects/UHJvamVjdDovKz0%3D"
+
+
+def response(body: object, status: int = 200) -> httpx.Response:
+ return httpx.Response(status, json=body, request=httpx.Request("GET", "https://api.arize.com/v2/projects"))
+
+
+def space() -> httpx.Response:
+ return response({"id": SPACE, "organization_id": ORG})
+
+
+def page(*projects: dict[str, str], more: bool = False, cursor: str | None = None) -> httpx.Response:
+ return response({"projects": list(projects), "pagination": {"has_more": more, "next_cursor": cursor}})
+
+
+def project(name: str = "exact", project_id: str = PROJECT) -> dict[str, str]:
+ return {"id": project_id, "name": name, "space_id": SPACE}
+
+
+def responses() -> deque[httpx.Response | Exception]:
+ return deque()
+
+
+def calls(
+ monkeypatch: pytest.MonkeyPatch, responses: deque[httpx.Response | Exception]
+) -> list[tuple[str, dict[str, object]]]:
+ calls: list[tuple[str, dict[str, object]]] = []
+
+ def get(url: str, **options: object) -> httpx.Response:
+ calls.append((url, options))
+ assert responses, "Unplanned metadata request"
+ item = responses.popleft()
+ if isinstance(item, Exception):
+ raise item
+ return item
+
+ monkeypatch.setattr(ssrf_proxy, "get", get)
+ return calls
+
+
+def test_exact_scoped_project_and_opaque_ids_use_official_route(
+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception]
+) -> None:
+ received = [space(), page(project("exact-substring"), more=True, cursor="next"), page(project())]
+ responses.extend(received)
+ assert resolve_project_destination(VALUES) == DESTINATION
+ assert (
+ project_url(
+ {**VALUES, DESTINATION_KEY: bind_destination(VALUES, DESTINATION), "project_url": "https://untrusted.test"}
+ )
+ == URL
+ )
+ assert len(calls) == 3
+ assert calls[0][0] == "https://api.arize.com/v2/spaces/U3BhY2U6dGVzdA%3D%3D"
+ assert calls[1][1]["params"] == {"space_id": SPACE, "name": "exact", "limit": 100}
+ assert calls[2][1]["params"] == {"space_id": SPACE, "name": "exact", "limit": 100, "cursor": "next"}
+ for url, options in calls:
+ assert url.startswith("https://api.arize.com/v2/")
+ assert options["headers"] == {
+ "Authorization": "Bearer synthetic-key",
+ "Accept": "application/json",
+ "Accept-Encoding": "identity",
+ "Cookie": "",
+ }
+ assert options["auth"] is None
+ assert options["follow_redirects"] is False
+ assert options["max_retries"] == 0
+ assert options["stream_response"] is True
+ timeout = options["timeout"]
+ assert isinstance(timeout, float)
+ assert 0 < timeout <= 5
+ assert all(item.is_closed for item in received)
+
+
[email protected]("status", [301, 401, 403, 404, 429, 500])
+def test_unreadable_metadata_has_no_link_and_closes_response(
+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], status: int
+) -> None:
+ denied = response({"error": "private-response"}, status)
+ responses.append(denied)
+ assert resolve_project_destination(VALUES) is None
+ assert len(calls) == 1
+ assert denied.is_closed
+
+
+ "body", [{"id": SPACE}, {"id": "other", "organization_id": ORG}, {"id": SPACE, "organization_id": "../bad"}]
+)
+def test_missing_or_mismatched_space_metadata_is_not_a_project(
+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], body: dict[str, str]
+) -> None:
+ responses.append(response(body))
+ assert resolve_project_destination(VALUES) is None
+ assert len(calls) == 1
+
+
+ "body",
+ [
+ {"projects": [project("EXACT")], "pagination": {"has_more": False}},
+ {"projects": [project(), project(project_id="other")], "pagination": {"has_more": False}},
+ {"projects": [{**project(), "space_id": "other"}], "pagination": {"has_more": False}},
+ {"projects": [{**project(), "id": "../bad"}], "pagination": {"has_more": False}},
+ {"projects": [project()] * 101, "pagination": {"has_more": False}},
+ {"projects": [project()], "pagination": {"has_more": True}},
+ {"projects": [project()], "pagination": {"has_more": "false"}},
+ {"projects": [project()]},
+ {"data": [project()], "pagination": {"has_more": False}},
+ ],
+)
+def test_incomplete_ambiguous_or_non_exact_projects_do_not_fabricate_links(
+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], body: object
+) -> None:
+ responses.extend([space(), response(body)])
+ assert resolve_project_destination(VALUES) is None
+ assert len(calls) == 2
+
+
[email protected]("repeat", [False, True])
+def test_pagination_is_bounded_and_never_accepts_a_partial_match(
+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], repeat: bool
+) -> None:
+ responses.append(space())
+ responses.extend(page(project(), more=True, cursor="same" if repeat else str(index)) for index in range(3))
+ assert resolve_project_destination(VALUES) is None
+ assert len(calls) == (3 if repeat else 4)
+
+
+ "item",
+ [
+ httpx.ReadTimeout("synthetic-timeout"),
+ httpx.Response(200, content=b"not-json"),
+ httpx.Response(200, json=[]),
+ httpx.Response(200, content=b" " * (1024 * 1024 + 1)),
+ httpx.Response(200, json={}, headers={"content-encoding": "unsupported"}),
+ ],
+)
+def test_transport_and_bounded_decode_failures_remain_optional(
+ calls: list[tuple[str, dict[str, object]]],
+ responses: deque[httpx.Response | Exception],
+ item: httpx.Response | Exception,
+) -> None:
+ responses.append(item)
+ assert resolve_project_destination(VALUES) is None
+ assert len(calls) == 1
+ if isinstance(item, httpx.Response):
+ assert item.is_closed
+
+
+def test_all_pages_share_one_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ clock = [100.0]
+ calls: list[float] = []
+ received = [space(), page(project(), more=True, cursor="next")]
+
+ def get(_url: str, **options: object) -> httpx.Response:
+ timeout = options["timeout"]
+ assert isinstance(timeout, float)
+ calls.append(timeout)
+ clock[0] += 2.5
+ return received[len(calls) - 1]
+
+ monkeypatch.setattr(configuration.time, "monotonic", lambda: clock[0])
+ monkeypatch.setattr(ssrf_proxy, "get", get)
+ assert resolve_project_destination(VALUES) is None
+ assert calls == [5.0, 2.5]
+ assert all(item.is_closed for item in received)
+
+
+def test_slow_response_chunks_stop_at_the_shared_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ clock = [100.0]
+ chunks: list[int] = []
+ received = httpx.Response(200)
+
+ def stream() -> Iterator[bytes]:
+ for index in range(10):
+ clock[0] += 2.0
+ chunks.append(index)
+ yield b" "
+
+ monkeypatch.setattr(configuration.time, "monotonic", lambda: clock[0])
+ monkeypatch.setattr(received, "iter_bytes", stream)
+ monkeypatch.setattr(ssrf_proxy, "get", lambda _url, **_options: received)
+ assert resolve_project_destination(VALUES) is None
+ assert chunks == [0, 1, 2]
+ assert received.is_closed
+
+
+ "change",
+ [
+ {"endpoint": "https://custom.test"},
+ {"endpoint": "https://otlp.eu-west-1a.arize.com"},
+ {"endpoint": "http://otlp.arize.com"},
+ {"endpoint": "https://user:[email protected]"},
+ {"endpoint": "https://otlp.arize.com:444"},
+ {"space_id": "../bad"},
+ {"api_key": None},
+ {"api_key": ""},
+ {"api_key": "******"},
+ {"api_key": "key\nheader"},
+ ],
+)
+def test_unsupported_destination_or_credentials_do_not_send_to_an_assumed_host(
+ calls: list[tuple[str, dict[str, object]]], change: dict[str, str | None]
+) -> None:
+ assert resolve_project_destination({**VALUES, **change}) is None
+ assert calls == []
+
+
+ "change",
+ [
+ {"endpoint": "https://otlp.arize.com:443"},
+ {"space_id": "other"},
+ {"project": "later"},
+ {DESTINATION_KEY: {**DESTINATION, "organization_id": "bad?query"}},
+ {DESTINATION_KEY: {**DESTINATION, "project_id": "../bad"}},
+ {DESTINATION_KEY: None},
+ ],
+)
+def test_get_formatter_rejects_stale_or_forged_metadata_without_io(
+ monkeypatch: pytest.MonkeyPatch, change: dict[str, object]
+) -> None:
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("GET must not call the vendor")
+
+ monkeypatch.setattr(ssrf_proxy, "get", forbidden)
+ assert project_url({**VALUES, DESTINATION_KEY: bind_destination(VALUES, DESTINATION), **change}) is None
diff --git a/api/tests/unit_tests/services/tracing/test_arize_configuration_ack.py b/api/tests/unit_tests/services/tracing/test_arize_configuration_ack.py
new file mode 100644
index 00000000000..1bf6b77dcfc
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_arize_configuration_ack.py
@@ -0,0 +1,315 @@
+"""A real gRPC collector must acknowledge Arize credentials before config persistence."""
+
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from threading import Event
+from typing import Never
+
+import grpc # type: ignore[import-untyped]
+import httpx
+import pytest
+from dify_trace_arize_phoenix import arize_configuration as configuration
+from dify_trace_arize_phoenix.arize_configuration import ArizeConfigurationError, verify_configuration
+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace
+from dify_trace_arize_phoenix.config import ArizeConfig
+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+ ExportTraceServiceRequest,
+ ExportTraceServiceResponse,
+)
+
+from core.helper import ssrf_proxy
+from core.ops.ops_trace_manager import OpsTraceManager
+from services.app_tracing_config_gateway import OpsTraceManagerGateway
+from services.app_tracing_config_service import AppTracingConfigVerificationFailedError
+
+pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)
+
+
+def collector() -> Iterator[tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]]]:
+ calls: list[tuple[bytes, dict[str, str]]] = []
+ state: dict[str, object] = {"status": grpc.StatusCode.OK}
+ release = Event()
+ state["release"] = release
+
+ def export(request: ExportTraceServiceRequest, context: grpc.ServicerContext) -> ExportTraceServiceResponse:
+ headers = {
+ key: value.decode() if isinstance(value, bytes) else value for key, value in context.invocation_metadata()
+ }
+ calls.append((request.SerializeToString(), headers))
+ if headers.get("api_key") != "synthetic-good" or headers.get("space_id") != "synthetic-space":
+ context.abort(grpc.StatusCode.PERMISSION_DENIED, "private-vendor-details")
+ code = state["status"]
+ if isinstance(code, grpc.StatusCode) and code is not grpc.StatusCode.OK:
+ context.abort(code, "private-vendor-details")
+ if state.get("delay"):
+ release.wait(2)
+ result = ExportTraceServiceResponse()
+ if state.get("partial"):
+ result.partial_success.rejected_spans = 1
+ if state.get("message"):
+ result.partial_success.error_message = "private-vendor-details"
+ return result
+
+ def dispatch(*args: object, **kwargs: object) -> ExportTraceServiceResponse:
+ assert kwargs == {}
+ request, context = args
+ assert isinstance(request, ExportTraceServiceRequest)
+ assert isinstance(context, grpc.ServicerContext)
+ return export(request, context)
+
+ handler = grpc.unary_unary_rpc_method_handler(
+ dispatch,
+ request_deserializer=ExportTraceServiceRequest.FromString,
+ response_serializer=ExportTraceServiceResponse.SerializeToString,
+ )
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ server = grpc.server(pool)
+ server.add_generic_rpc_handlers(
+ (
+ grpc.method_handlers_generic_handler(
+ "opentelemetry.proto.collector.trace.v1.TraceService", {"Export": handler}
+ ),
+ )
+ )
+ port = server.add_insecure_port("127.0.0.1:0")
+ assert port > 0
+ server.start()
+ try:
+ yield f"http://127.0.0.1:{port}", calls, state
+ finally:
+ release.set()
+ server.stop(0).wait(2)
+
+
+def test_rejected_real_collector_ack_cannot_be_saved(
+ workspace: str,
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+) -> None:
+ endpoint, calls, _ = collector
+ with pytest.raises(AppTracingConfigVerificationFailedError):
+ OpsTraceManagerGateway().prepare_new_config(
+ workspace_id=workspace,
+ tracing_provider="arize",
+ tracing_config={
+ "endpoint": endpoint,
+ "project": "synthetic-project",
+ "space_id": "synthetic-space",
+ "api_key": "synthetic-invalid",
+ },
+ )
+ assert len(calls) == 1
+
+
+def test_synchronous_ack_is_resource_only_and_has_no_sdk_or_environment_side_effects(
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ endpoint, calls, _ = collector
+ attempted: list[bool] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ attempted.append(True)
+ raise AssertionError("Arize verification must not construct the SDK tracer")
+
+ monkeypatch.setattr(ArizePhoenixDataTrace, "__init__", forbidden)
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "api_key=ambient-invalid,space_id=ambient-other")
+ verify_configuration(
+ ArizeConfig(
+ endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space", project="synthetic-project"
+ )
+ )
+ assert attempted == []
+ assert len(calls) == 1
+ payload = ExportTraceServiceRequest.FromString(calls[0][0])
+ assert len(payload.resource_spans) == 1
+ assert {a.key: a.value.string_value for a in payload.resource_spans[0].resource.attributes} == {
+ "model_id": "synthetic-project",
+ "openinference.project.name": "synthetic-project",
+ }
+ assert len(payload.resource_spans[0].scope_spans) == 0
+ assert calls[0][1]["api_key"] == "synthetic-good"
+ assert calls[0][1]["space_id"] == "synthetic-space"
+ assert calls[0][1]["authorization"] == "Bearer synthetic-good"
+ assert "ambient" not in str(calls)
+
+
+ ("code", "expected"),
+ [
+ (grpc.StatusCode.UNAUTHENTICATED, "authentication_failed"),
+ (grpc.StatusCode.PERMISSION_DENIED, "permission_denied"),
+ (grpc.StatusCode.UNAVAILABLE, "connection_failed"),
+ (grpc.StatusCode.RESOURCE_EXHAUSTED, "connection_failed"),
+ ],
+)
+def test_real_rpc_rejection_uses_only_safe_fixed_diagnostics(
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ code: grpc.StatusCode,
+ expected: str,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ endpoint, calls, state = collector
+ state["status"] = code
+ with pytest.raises(ArizeConfigurationError) as caught:
+ verify_configuration(ArizeConfig(endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space"))
+ assert caught.value.code == expected
+ assert caught.value.__suppress_context__
+ assert "synthetic-good" not in str(caught.value)
+ assert "private-vendor-details" not in str(caught.value)
+ assert "private-vendor-details" not in caplog.text
+ assert len(calls) == 1
+
+
[email protected]("partial", ["partial", "message"])
+def test_partial_or_error_ack_is_not_success(
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ partial: str,
+) -> None:
+ endpoint, calls, state = collector
+ state[partial] = True
+ with pytest.raises(ArizeConfigurationError, match="unexpected_response"):
+ verify_configuration(ArizeConfig(endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space"))
+ assert len(calls) == 1
+
+
+def test_deadline_is_finite_and_does_not_retry(
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ endpoint, calls, state = collector
+ state["delay"] = True
+ monkeypatch.setattr(configuration, "_VERIFY_TIMEOUT_SECONDS", 0.05)
+ with pytest.raises(ArizeConfigurationError, match="connection_failed"):
+ verify_configuration(ArizeConfig(endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space"))
+ assert len(calls) == 1
+
+
[email protected]("reject", [False, True])
+def test_tls_channel_is_explicit_suppressed_and_closed_on_all_outcomes(
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ monkeypatch: pytest.MonkeyPatch,
+ reject: bool,
+) -> None:
+ endpoint, _, state = collector
+ if reject:
+ state["status"] = grpc.StatusCode.PERMISSION_DENIED
+ original = grpc.insecure_channel
+ options_seen: list[tuple[str, object, object]] = []
+ closed: list[bool] = []
+
+ def secure(target: str, credentials: grpc.ChannelCredentials, options: object) -> grpc.Channel:
+ assert isinstance(credentials, grpc.ChannelCredentials)
+ options_seen.append((target, options, get_value(_SUPPRESS_INSTRUMENTATION_KEY)))
+ channel = original(endpoint.removeprefix("http://"))
+ close = channel.close
+
+ def close_recording() -> None:
+ closed.append(True)
+ close()
+
+ monkeypatch.setattr(channel, "close", close_recording)
+ return channel
+
+ monkeypatch.setattr(grpc, "secure_channel", secure)
+ before = get_value(_SUPPRESS_INSTRUMENTATION_KEY)
+ config = ArizeConfig(api_key="synthetic-good", space_id="synthetic-space")
+ if reject:
+ with pytest.raises(ArizeConfigurationError):
+ verify_configuration(config)
+ else:
+ verify_configuration(config)
+ assert options_seen == [
+ ("otlp.arize.com:443", (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", 1024 * 1024)), True)
+ ]
+ assert closed == [True]
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == before
+
+
+ ("field", "value"),
+ [
+ ("api_key", None),
+ ("api_key", ""),
+ ("api_key", "******"),
+ ("api_key", "key\nheader"),
+ ("space_id", None),
+ ("space_id", " "),
+ ("space_id", "******"),
+ ("endpoint", "https://user:[email protected]"),
+ ("endpoint", "https://otlp.arize.com:0"),
+ ],
+)
+def test_invalid_auth_values_never_construct_a_channel(
+ monkeypatch: pytest.MonkeyPatch,
+ field: str,
+ value: str | None,
+) -> None:
+ attempted: list[bool] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ attempted.append(True)
+ raise AssertionError("Invalid configuration must be rejected before transport construction")
+
+ monkeypatch.setattr(grpc, "secure_channel", forbidden)
+ monkeypatch.setattr(grpc, "insecure_channel", forbidden)
+ config = ArizeConfig.model_validate({"api_key": "synthetic-good", "space_id": "synthetic-space", field: value})
+ with pytest.raises(ArizeConfigurationError, match="invalid_configuration"):
+ verify_configuration(config)
+ assert attempted == []
+
+
[email protected]("accepted", [False, True])
+def test_legacy_api_check_also_requires_the_real_ack(
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ accepted: bool,
+) -> None:
+ endpoint, calls, _ = collector
+ instance = ArizePhoenixDataTrace(
+ ArizeConfig(
+ endpoint=endpoint, api_key="synthetic-good" if accepted else "synthetic-invalid", space_id="synthetic-space"
+ )
+ )
+ try:
+ if accepted:
+ assert instance.api_check()
+ else:
+ with pytest.raises(ArizeConfigurationError, match="permission_denied"):
+ instance.api_check()
+ finally:
+ instance.processor.shutdown()
+ assert len(calls) == 1
+ assert len(ExportTraceServiceRequest.FromString(calls[0][0]).resource_spans[0].scope_spans) == 0
+
+
+def test_real_ingest_ack_without_rest_read_permission_can_still_save(
+ workspace: str,
+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ endpoint, rpc_calls, _ = collector
+ original = grpc.insecure_channel
+ reads: list[str] = []
+
+ def secure(_target: str, _credentials: grpc.ChannelCredentials, **_options: object) -> grpc.Channel:
+ return original(endpoint.removeprefix("http://"))
+
+ def get(url: str, **_options: object) -> httpx.Response:
+ reads.append(url)
+ return httpx.Response(403, json={"message": "private-vendor-details"})
+
+ monkeypatch.setattr(grpc, "secure_channel", secure)
+ monkeypatch.setattr(ssrf_proxy, "get", get)
+ saved = OpsTraceManagerGateway().prepare_new_config(
+ workspace_id=workspace,
+ tracing_provider="arize",
+ tracing_config={"api_key": "synthetic-good", "space_id": "synthetic-space", "project": "not-created"},
+ )
+ assert len(rpc_calls) == len(reads) == 1
+ assert reads[0] == "https://api.arize.com/v2/spaces/synthetic-space"
+ assert "project_url" not in saved
+ assert "arize_project_destination" not in saved
+ assert OpsTraceManager.decrypt_tracing_config(workspace, "arize", saved)["api_key"] == "synthetic-good"
+ payload = ExportTraceServiceRequest.FromString(rpc_calls[0][0])
+ assert len(payload.resource_spans[0].scope_spans) == 0
diff --git a/api/tests/unit_tests/services/tracing/test_databricks_configuration.py b/api/tests/unit_tests/services/tracing/test_databricks_configuration.py
new file mode 100644
index 00000000000..e16ff2891e9
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_databricks_configuration.py
@@ -0,0 +1,230 @@
+"""Official experiment schemas at a request-scoped, read-only HTTP boundary."""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+import httpx
+import pytest
+from dify_trace_mlflow.config import DatabricksConfig
+from dify_trace_mlflow.configuration import DatabricksConfigurationError, project_url, verify_configuration
+from google.protobuf.json_format import MessageToDict
+from mlflow.protos.service_pb2 import Experiment, GetExperiment
+
+from core.helper import ssrf_proxy
+
+
+def response(url: str, body: object, status: int = 200) -> httpx.Response:
+ return httpx.Response(status, json=body, request=httpx.Request("GET", url))
+
+
+@dataclass
+class ExperimentAPI:
+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)
+ override: Callable[[str, str, dict[str, object]], httpx.Response] | None = None
+
+ def get(self, url: str, **options: object) -> httpx.Response:
+ self.calls.append(("GET", url, options))
+ if self.override:
+ return self.override("GET", url, options)
+ assert url.endswith("/api/2.0/mlflow/experiments/get")
+ params = options["params"]
+ assert isinstance(params, dict)
+ requested = GetExperiment(experiment_id=params["experiment_id"])
+ return response(
+ url,
+ MessageToDict(
+ GetExperiment.Response(
+ experiment=Experiment(experiment_id=requested.experiment_id, lifecycle_stage="active")
+ ),
+ preserving_proto_field_name=True,
+ ),
+ )
+
+ def post(self, url: str, **options: object) -> httpx.Response:
+ self.calls.append(("POST", url, options))
+ assert url.endswith("/oidc/v1/token"), "configuration must never create experiments, runs or traces"
+ if self.override:
+ return self.override("POST", url, options)
+ return response(url, {"access_token": "synthetic-oauth-token", "token_type": "Bearer", "expires_in": 3600})
+
+
+def experiment_api(monkeypatch: pytest.MonkeyPatch) -> ExperimentAPI:
+ api = ExperimentAPI()
+ monkeypatch.setattr(ssrf_proxy, "get", api.get)
+ monkeypatch.setattr(ssrf_proxy, "post", api.post)
+ return api
+
+
+def test_pat_requests_are_isolated_and_read_only(experiment_api: ExperimentAPI) -> None:
+ for host, token, experiment_id in [
+ ("https://first.test/", "first-pat", "000123"),
+ ("https://second.test", "second-pat", "456"),
+ ]:
+ config = DatabricksConfig(host=host, experiment_id=experiment_id, personal_access_token=token)
+ result = verify_configuration(config)
+ assert result.host == host.rstrip("/")
+ assert result.experiment_id == experiment_id.lstrip("0")
+ method, url, options = experiment_api.calls[-1]
+ assert method == "GET"
+ assert url == host.rstrip("/") + "/api/2.0/mlflow/experiments/get"
+ assert options["headers"] == {"Authorization": f"Bearer {token}", "Accept": "application/json"}
+ assert options["max_retries"] == 0
+ assert options["timeout"] == 30.0
+ assert options["follow_redirects"] is False
+ assert len(experiment_api.calls) == 2
+
+
+def test_oauth_precedence_matches_exporter_without_reusing_pat(experiment_api: ExperimentAPI) -> None:
+ config = DatabricksConfig(
+ host="https://workspace.test",
+ experiment_id="12",
+ personal_access_token="unused-pat",
+ client_id="synthetic-client",
+ client_secret="synthetic-secret",
+ )
+ assert verify_configuration(config).experiment_id == "12"
+ method, url, options = experiment_api.calls[0]
+ assert method == "POST"
+ assert url == "https://workspace.test/oidc/v1/token"
+ assert options["auth"] == ("synthetic-client", "synthetic-secret")
+ assert options["data"] == {"grant_type": "client_credentials", "scope": "all-apis"}
+ assert "headers" not in options
+ assert options["follow_redirects"] is False
+ assert experiment_api.calls[1][2]["headers"] == {
+ "Authorization": "Bearer synthetic-oauth-token",
+ "Accept": "application/json",
+ }
+
+
+def test_incomplete_oauth_uses_explicit_pat_as_existing_exporter_does(experiment_api: ExperimentAPI) -> None:
+ verify_configuration(
+ DatabricksConfig(
+ host="https://workspace.test", experiment_id="12", client_id="unused-client", personal_access_token="pat"
+ )
+ )
+ assert len(experiment_api.calls) == 1
+ assert experiment_api.calls[0][0] == "GET"
+ assert experiment_api.calls[0][2]["headers"] == {"Authorization": "Bearer pat", "Accept": "application/json"}
+
+
+def test_non_ascii_experiment_ids_are_not_sent(experiment_api: ExperimentAPI) -> None:
+ with pytest.raises(DatabricksConfigurationError) as caught:
+ verify_configuration(
+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="pat")
+ )
+ assert caught.value.code == "invalid_experiment_id"
+ assert experiment_api.calls == []
+
+
+ "credentials",
+ [
+ {},
+ {"client_id": "client"},
+ {"client_secret": "secret"},
+ {"personal_access_token": " "},
+ ],
+)
+def test_missing_credentials_fail_before_vendor_io(experiment_api: ExperimentAPI, credentials: dict[str, str]) -> None:
+ with pytest.raises(DatabricksConfigurationError) as caught:
+ verify_configuration(DatabricksConfig(host="https://workspace.test", experiment_id="12", **credentials))
+ assert caught.value.code == "authentication_failed"
+ assert experiment_api.calls == []
+
+
+ "host",
+ [
+ "http://workspace.test",
+ "https://user:[email protected]",
+ "https://workspace.test/path",
+ "https://workspace.test?o=1",
+ "https://workspace.test/#/experiments/12",
+ "https://workspace.test\\@evil.test",
+ "https://workspace.test:\n443",
+ "https://workspace.test:bad",
+ "https://[invalid",
+ ],
+)
+def test_invalid_workspace_origins_do_not_send_credentials(experiment_api: ExperimentAPI, host: str) -> None:
+ with pytest.raises(DatabricksConfigurationError) as caught:
+ verify_configuration(DatabricksConfig(host=host, experiment_id="12", personal_access_token="synthetic-key"))
+ assert caught.value.code == "invalid_workspace_url"
+ assert experiment_api.calls == []
+
+
+def test_project_url_is_pure_and_supports_explicit_https_port(experiment_api: ExperimentAPI) -> None:
+ config = DatabricksConfig(host=" https://Workspace.Example:443/ ", experiment_id="00012")
+ assert project_url(config) == "https://Workspace.Example:443/ml/experiments/12/traces"
+ assert experiment_api.calls == []
+
+
+ ("status", "code"),
+ [
+ (401, "authentication_failed"),
+ (403, "permission_denied"),
+ (404, "experiment_not_found_or_inaccessible"),
+ (302, "endpoint_unavailable"),
+ (500, "endpoint_unavailable"),
+ ],
+)
+def test_http_failures_are_safe_and_responses_are_closed(experiment_api: ExperimentAPI, status: int, code: str) -> None:
+ result = response("https://workspace.test", {"message": "private-key-vendor-response"}, status)
+ experiment_api.override = lambda _method, _url, _options: result
+ with pytest.raises(DatabricksConfigurationError) as caught:
+ verify_configuration(
+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="synthetic-key")
+ )
+ assert caught.value.code == code
+ assert "private-key" not in str(caught.value)
+ assert "synthetic-key" not in str(caught.value)
+ assert result.is_closed
+
+
+ ("body", "code"),
+ [
+ ({}, "malformed_response"),
+ ([], "malformed_response"),
+ ({"experiment": "invalid"}, "malformed_response"),
+ ({"experiment": {"experiment_id": "other", "lifecycle_stage": "active"}}, "experiment_id_mismatch"),
+ ({"experiment": {"experiment_id": "12", "lifecycle_stage": "deleted"}}, "experiment_not_active"),
+ ({"experiment": {"experiment_id": "12"}}, "experiment_not_active"),
+ ],
+)
+def test_response_must_identify_selected_active_experiment(
+ experiment_api: ExperimentAPI, body: object, code: str
+) -> None:
+ experiment_api.override = lambda _method, url, _options: response(url, body)
+ with pytest.raises(DatabricksConfigurationError) as caught:
+ verify_configuration(
+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="synthetic-key")
+ )
+ assert caught.value.code == code
+
+
[email protected]("body", [{}, {"access_token": ""}, {"access_token": "token", "token_type": "other"}])
+def test_invalid_oauth_response_never_reaches_experiment_api(experiment_api: ExperimentAPI, body: object) -> None:
+ experiment_api.override = lambda _method, url, _options: response(url, body)
+ with pytest.raises(DatabricksConfigurationError, match="authentication failed"):
+ verify_configuration(
+ DatabricksConfig(
+ host="https://workspace.test", experiment_id="12", client_id="client", client_secret="secret"
+ )
+ )
+ assert len(experiment_api.calls) == 1
+
+
+def test_transport_errors_do_not_disclose_request_details(experiment_api: ExperimentAPI) -> None:
+ def fail(_method: str, _url: str, _options: dict[str, object]) -> httpx.Response:
+ raise httpx.ConnectError("private-key-vendor-response")
+
+ experiment_api.override = fail
+ with pytest.raises(DatabricksConfigurationError) as caught:
+ verify_configuration(
+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="synthetic-key")
+ )
+ assert caught.value.code == "connection_failed"
+ assert "private-key" not in str(caught.value)
diff --git a/api/tests/unit_tests/services/tracing/test_export_lease.py b/api/tests/unit_tests/services/tracing/test_export_lease.py
new file mode 100644
index 00000000000..7d742207d6b
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_export_lease.py
@@ -0,0 +1,223 @@
+"""Real SQLite state transitions and background renewal for delivery leases.
+
+PostgreSQL row-lock scheduling and the deployed Celery pool still need dev
+validation; these tests establish token and acknowledgement semantics.
+"""
+
+import threading
+from datetime import timedelta
+
+import pytest
+from sqlalchemy.orm import Session, sessionmaker
+
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceRun
+from repositories import application_trace_repository as repository_module
+from repositories.application_trace_repository import TraceCollectionError
+from services.tracing import service as service_module
+from services.tracing.provider_dispatch import TraceExportResult, TraceExportUncertainError
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ Collection,
+ batch_for,
+ claim,
+ ingest,
+ seed,
+ terminal,
+)
+
+
+def case(sqlite_session_factory: sessionmaker[Session]) -> Collection:
+ result = seed(sqlite_session_factory, "lease-test")
+ ingest(result, batch_for(result.run, result.invocation()))
+ terminal(result)
+ return result
+
+
+def _stored(case: Collection) -> ApplicationTraceRun:
+ row = case.repository.get(case.run.id)
+ assert row is not None
+ return row
+
+
+def _expire(case: Collection) -> str:
+ leased = claim(case)
+ assert leased is not None
+ with case.sessions.begin() as session:
+ row = session.get(ApplicationTraceRun, case.run.id)
+ assert row is not None
+ row.export_started_at = naive_utc_now() - timedelta(seconds=60)
+ row.export_heartbeat_at = row.export_started_at
+ assert claim(case) is None
+ assert _stored(case).state == "uncertain"
+ return leased[1]
+
+
+def test_renewal_keeps_long_export_alive_then_missing_heartbeat_expires(
+ case: Collection, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ clock = [naive_utc_now()]
+ monkeypatch.setattr(repository_module, "naive_utc_now", lambda: clock[0])
+ leased = claim(case)
+ assert leased is not None
+ token = leased[1]
+ clock[0] += timedelta(seconds=20)
+ assert case.repository.renew_export_lease(case.run.id, token)
+ clock[0] += timedelta(seconds=20)
+ assert claim(case) is None
+ current = _stored(case)
+ assert current.state == "exporting"
+ assert current.attempts == 1
+ assert current.export_started_at is not None
+ assert current.export_started_at + timedelta(seconds=30) < clock[0]
+ clock[0] += timedelta(seconds=11)
+ assert claim(case) is None
+ expired = _stored(case)
+ assert expired.state == "uncertain"
+ assert expired.error_code == "export_lease_expired"
+ assert expired.export_token == token
+ assert not case.repository.renew_export_lease(case.run.id, token)
+
+
+def test_old_rows_without_heartbeat_use_started_at(case: Collection) -> None:
+ leased = claim(case)
+ assert leased is not None
+ with case.sessions.begin() as session:
+ row = session.get(ApplicationTraceRun, case.run.id)
+ assert row is not None
+ row.export_started_at = naive_utc_now() - timedelta(seconds=60)
+ row.export_heartbeat_at = None
+ assert claim(case) is None
+ current = _stored(case)
+ assert current.state == "uncertain"
+ assert current.export_token == leased[1]
+
+
+def test_wrong_token_cannot_renew_or_complete(case: Collection) -> None:
+ leased = claim(case)
+ assert leased is not None
+ before = _stored(case)
+ assert not case.repository.renew_export_lease(case.run.id, "other-token")
+ case.repository.complete(case.run.id, "other-token", state="accepted", result=TraceExportResult("wrong"))
+ current = _stored(case)
+ assert current.export_token == leased[1]
+ assert current.export_heartbeat_at == before.export_heartbeat_at
+ assert current.state == "exporting"
+ assert current.provider_trace_id is None
+
+
+def test_late_success_wins_before_manual_retry_and_is_idempotent(case: Collection) -> None:
+ token = _expire(case)
+ result = TraceExportResult("confirmed", "https://example.com/trace")
+ case.repository.complete(case.run.id, token, state="accepted", result=result)
+ current = _stored(case)
+ assert current.state == "accepted"
+ assert current.error_code is None
+ assert current.export_token is None
+ assert current.accepted_at is not None
+ assert current.trace_url == result.trace_url
+ case.repository.complete(case.run.id, token, state="accepted", result=TraceExportResult("duplicate-ack"))
+ assert _stored(case).provider_trace_id == "confirmed"
+ with pytest.raises(TraceCollectionError, match="tracing_already_accepted"):
+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)
+
+
+def test_manual_retry_revokes_old_heartbeat_and_ack_before_and_after_new_claim(case: Collection) -> None:
+ old_token = _expire(case)
+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)
+ assert not case.repository.renew_export_lease(case.run.id, old_token)
+ case.repository.complete(case.run.id, old_token, state="accepted", result=TraceExportResult("old"))
+ assert _stored(case).state == "ready"
+ second = claim(case)
+ assert second is not None
+ assert second[1] != old_token
+ case.repository.complete(case.run.id, old_token, state="accepted", result=TraceExportResult("old"))
+ current = _stored(case)
+ assert current.state == "exporting"
+ assert current.export_token == second[1]
+ assert current.provider_trace_id is None
+ case.repository.complete(case.run.id, second[1], state="accepted", result=TraceExportResult("new"))
+ assert _stored(case).provider_trace_id == "new"
+
+
[email protected]("late_state", ["failed", "retrying", "uncertain"])
+def test_late_non_success_cannot_overwrite_lease_expiry(case: Collection, late_state: str) -> None:
+ token = _expire(case)
+ case.repository.complete(case.run.id, token, state=late_state, error_code="late-failure")
+ current = _stored(case)
+ assert current.state == "uncertain"
+ assert current.error_code == "export_lease_expired"
+
+
+def test_other_uncertain_reason_cannot_be_promoted_by_late_ack(case: Collection) -> None:
+ token = _expire(case)
+ with case.sessions.begin() as session:
+ row = session.get(ApplicationTraceRun, case.run.id)
+ assert row is not None
+ row.error_code = "provider_acceptance_unknown"
+ case.repository.complete(case.run.id, token, state="accepted", result=TraceExportResult("late"))
+ assert _stored(case).state == "uncertain"
+
+
[email protected]("transient_failure", [False, True])
[email protected]("outcome", ["accepted", "uncertain"])
+def test_service_heartbeats_during_export_and_stops_on_every_outcome(
+ case: Collection,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ transient_failure: bool,
+ outcome: str,
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS=1)
+ clock = [naive_utc_now()]
+ monkeypatch.setattr(repository_module, "naive_utc_now", lambda: clock[0])
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ renewed = threading.Event()
+ calls = []
+ original = case.repository.renew_export_lease
+
+ def renew(run_id: str, token: str) -> bool:
+ calls.append(threading.get_ident())
+ if transient_failure and len(calls) == 1:
+ raise RuntimeError("private-provider-value")
+ result = original(run_id, token)
+ renewed.set()
+ return result
+
+ monkeypatch.setattr(case.repository, "renew_export_lease", renew)
+
+ def export(*_args: object) -> TraceExportResult:
+ # The original start time is now beyond the lease; only a real background
+ # transaction can prevent a second claimant from marking it uncertain.
+ clock[0] += timedelta(seconds=3)
+ assert renewed.wait(3), "background heartbeat did not commit"
+ assert case.repository.claim(case.run.id, collection_timeout=60, lease_seconds=1) is None
+ assert _stored(case).state == "exporting"
+ if outcome == "uncertain":
+ raise TraceExportUncertainError("lost acknowledgement")
+ return TraceExportResult("ack")
+
+ monkeypatch.setattr(service_module, "export_trace", export)
+ ApplicationTracingService(case.repository).deliver(case.run.id)
+ assert _stored(case).state == outcome
+ assert calls
+ assert all(thread_id != threading.get_ident() for thread_id in calls)
+ assert not any(thread.name == f"application-trace-lease-{case.run.id}" for thread in threading.enumerate())
+ assert "private-provider-value" not in caplog.text
+
+
+def test_heartbeat_stops_even_when_completion_database_write_raises(
+ case: Collection, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ monkeypatch.setattr(service_module, "export_trace", lambda *_args: TraceExportResult("ack"))
+
+ def fail_complete(*_args: object, **_kwargs: object) -> None:
+ raise RuntimeError("database unavailable")
+
+ monkeypatch.setattr(case.repository, "complete", fail_complete)
+ with pytest.raises(RuntimeError, match="database unavailable"):
+ ApplicationTracingService(case.repository).deliver(case.run.id)
+ assert not any(thread.name == f"application-trace-lease-{case.run.id}" for thread in threading.enumerate())
diff --git a/api/tests/unit_tests/services/tracing/test_invalid_trace_delivery.py b/api/tests/unit_tests/services/tracing/test_invalid_trace_delivery.py
new file mode 100644
index 00000000000..fbfd62dd790
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_invalid_trace_delivery.py
@@ -0,0 +1,55 @@
+"""Malformed durable traces must never be sent or offered for provider retry."""
+
+from uuid import uuid4
+
+import pytest
+from dify_agent.protocol.tracing import AgentTraceSpan
+from sqlalchemy.orm import Session, sessionmaker
+
+from models.application_trace import ApplicationTraceRun
+from repositories.application_trace_repository import TraceCollectionError
+from services.tracing import service as service_module
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.services.tracing.test_application_trace_collection import batch_for, ingest, seed, terminal
+
+
[email protected]("invalid", ["cycle", "invalid_envelope"])
+def test_invalid_tree_is_permanent_before_decryption_or_provider_io(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, invalid: str
+) -> None:
+ case = seed(sqlite_session_factory, str(uuid4()))
+ batch = batch_for(case.run, case.invocation())
+ if invalid == "cycle":
+ for span_id, parent_id in [("b" * 16, "c" * 16), ("c" * 16, "b" * 16)]:
+ batch.spans.append(
+ AgentTraceSpan(
+ trace_id=case.run.trace_id,
+ span_id=span_id,
+ parent_span_id=parent_id,
+ name="cyclic tool",
+ start_time_ns=1_000_000_000,
+ end_time_ns=2_000_000_000,
+ status="ok",
+ )
+ )
+ ingest(case, batch)
+ terminal(case)
+ if invalid == "invalid_envelope":
+ with case.sessions.begin() as session:
+ row = session.get(ApplicationTraceRun, case.run.id)
+ assert row is not None
+ row.outer_trace = {"corrupt": True}
+
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("invalid traces must fail before credentials or provider I/O")
+
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", forbidden)
+ monkeypatch.setattr(service_module, "export_trace", forbidden)
+ ApplicationTracingService(case.repository).deliver(case.run.id)
+ current = case.repository.get(case.run.id)
+ assert current is not None
+ assert current.state == "failed"
+ assert current.error_code == "tracing_payload_invalid"
+ assert current.attempts == 1
+ with pytest.raises(TraceCollectionError, match="tracing_retry_not_allowed"):
+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)
diff --git a/api/tests/unit_tests/services/tracing/test_legacy_aliyun_transport.py b/api/tests/unit_tests/services/tracing/test_legacy_aliyun_transport.py
new file mode 100644
index 00000000000..811cf01feb4
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_legacy_aliyun_transport.py
@@ -0,0 +1,344 @@
+"""Persisted legacy sources through real OTel protobuf, replacing only HTTP I/O."""
+
+import json
+import os
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+from threading import Barrier, Thread
+from typing import TypedDict, Unpack
+from uuid import UUID
+
+import httpx
+import pytest
+from dify_trace_aliyun.aliyun_trace import AliyunDataTrace
+from dify_trace_aliyun.config import AliyunConfig
+from dify_trace_aliyun.data_exporter.traceclient import TraceClient, convert_to_span_id
+from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData
+from dify_trace_aliyun.entities.semconv import GEN_AI_USAGE_TOTAL_TOKENS
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+ ExportTraceServiceRequest,
+ ExportTraceServiceResponse,
+)
+from sqlalchemy.orm import Session
+
+from core.helper import ssrf_proxy
+from core.ops.entities.trace_entity import MessageTraceInfo, ModerationTraceInfo, WorkflowTraceInfo
+from core.ops.exceptions import RetryableTraceDispatchError
+from graphon.enums import BuiltinNodeTypes, WorkflowExecutionStatus, WorkflowNodeExecutionStatus
+from models.account import TenantAccountJoin, TenantAccountRole
+from models.enums import ConversationFromSource, CreatorUserRole, MessageStatus
+from models.model import AppMode
+from models.workflow import (
+ WorkflowNodeExecutionModel,
+ WorkflowNodeExecutionTriggeredFrom,
+ WorkflowRun,
+ WorkflowRunTriggeredFrom,
+ WorkflowType,
+)
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError
+from tests.unit_tests.model_factories import make_account, make_app, make_message, make_tenant
+
+START = datetime(2026, 10, 19, tzinfo=UTC)
+RUN_ID = "a0000000-0000-4000-8000-000000000001"
+MESSAGE_ID = "a0000000-0000-4000-8000-000000000002"
+NODE_IDS = [f"b0000000-0000-4000-8000-00000000000{i}" for i in range(3)]
+
+
+class RequestOptions(TypedDict, total=False):
+ content: bytes
+ headers: Mapping[str, str]
+ max_retries: int
+ follow_redirects: bool
+ timeout: float
+
+
+def config(**overrides: object) -> AliyunConfig:
+ return AliyunConfig.model_validate(
+ {
+ "app_name": "owned-service",
+ "license_key": "owned-key",
+ "endpoint": "https://collector.test/prefix",
+ **overrides,
+ }
+ )
+
+
+def decoded(options: RequestOptions) -> ExportTraceServiceRequest:
+ payload = ExportTraceServiceRequest()
+ payload.ParseFromString(options["content"])
+ return payload
+
+
+def test_constructor_and_presentation_have_no_network_sdk_queue_or_environment_mutation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("Constructor and presentation must not create I/O, exporters or worker threads")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", forbidden)
+ monkeypatch.setattr(OTLPSpanExporter, "__init__", forbidden)
+ monkeypatch.setattr(Thread, "start", forbidden)
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://ambient.test")
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=ambient")
+ before = dict(os.environ)
+ original = config()
+ adapter = AliyunDataTrace(original)
+ original.license_key = "mutated-after-construction"
+ assert adapter.trace_client.endpoint == "https://collector.test/prefix/adapt_owned-key/api/otlp/traces"
+ assert adapter.get_project_url() == "https://arms.console.aliyun.com/#/llm"
+ assert not hasattr(adapter.trace_client, "exporter")
+ assert not hasattr(adapter.trace_client, "queue")
+ assert not hasattr(adapter.trace_client, "worker_thread")
+ assert dict(os.environ) == before
+
+
[email protected]("agent_runtime_app")
[email protected]("failure", [None, 400, 401, 403, 429, 503, "timeout", "partial", "malformed", "redirect"])
+def test_persisted_workflow_uses_one_acknowledged_protobuf_batch_and_preserves_business_state(
+ sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, failure: int | str | None
+) -> None:
+ tenant = make_tenant(tenant_id="tenant")
+ account = make_account(account_id="owner", tenant=tenant)
+ app = make_app(app_id="app", tenant_id=tenant.id, mode=AppMode.WORKFLOW, created_by=account.id)
+ membership = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole.OWNER)
+ run = WorkflowRun(
+ id=RUN_ID,
+ tenant_id=tenant.id,
+ app_id=app.id,
+ workflow_id="workflow",
+ type=WorkflowType.WORKFLOW,
+ triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
+ version="v1",
+ inputs=json.dumps({"query": "hello"}),
+ outputs=json.dumps({"answer": "ok"}),
+ status=WorkflowExecutionStatus.SUCCEEDED,
+ total_tokens=32,
+ created_by=account.id,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_at=START,
+ finished_at=START + timedelta(seconds=1),
+ elapsed_time=1,
+ )
+ sqlite_session.add_all([tenant, account, app, membership, run])
+ nodes = [
+ WorkflowNodeExecutionModel(
+ id=NODE_IDS[i],
+ tenant_id=tenant.id,
+ app_id=app.id,
+ workflow_id="workflow",
+ workflow_run_id=RUN_ID,
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
+ index=i,
+ node_id=f"node-{i}",
+ title=title,
+ node_type=kind,
+ status=WorkflowNodeExecutionStatus.SUCCEEDED,
+ created_by=account.id,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ inputs=json.dumps({"synthetic": True}),
+ outputs=json.dumps(
+ {"text": "ok", "usage": {"prompt_tokens": 21, "completion_tokens": 11, "total_tokens": 32}}
+ ),
+ created_at=START + timedelta(milliseconds=i),
+ finished_at=START + timedelta(milliseconds=i + 100),
+ elapsed_time=0.1,
+ process_data=json.dumps(
+ {"prompts": [{"role": "user", "text": "hello"}], "model_provider": "test", "model_name": "test-model"}
+ )
+ if i == 1
+ else None,
+ )
+ for i, (title, kind) in enumerate(
+ [("Start", BuiltinNodeTypes.START), ("LLM", BuiltinNodeTypes.LLM), ("End", BuiltinNodeTypes.END)]
+ )
+ ]
+ sqlite_session.add_all(nodes)
+ sqlite_session.commit()
+ info = WorkflowTraceInfo(
+ query="hello",
+ workflow_id="workflow",
+ tenant_id=tenant.id,
+ workflow_run_id=RUN_ID,
+ workflow_run_elapsed_time=1.0,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={"query": "hello"},
+ workflow_run_outputs={"answer": "ok"},
+ workflow_run_version="v1",
+ total_tokens=32,
+ file_list=[],
+ metadata={"app_id": app.id, "conversation_id": "owned-conversation"},
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ )
+ sent: list[tuple[str, RequestOptions, ExportTraceServiceRequest]] = []
+
+ def send(method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ assert method == "POST"
+ assert url == "https://collector.test/prefix/adapt_owned-key/api/otlp/traces"
+ sent.append((url, options, decoded(options)))
+ if len(sent) == 1:
+ if failure == "timeout":
+ raise httpx.ReadTimeout("synthetic lost ACK", request=httpx.Request("POST", url))
+ if isinstance(failure, int):
+ return httpx.Response(failure, content=b"DO_NOT_EXPOSE_BODY")
+ if failure == "malformed":
+ return httpx.Response(200, content=b"not protobuf")
+ if failure == "partial":
+ response = ExportTraceServiceResponse()
+ response.partial_success.rejected_spans = 1
+ response.partial_success.error_message = "DO_NOT_EXPOSE_BODY"
+ return httpx.Response(200, content=response.SerializeToString())
+ if failure == "redirect":
+ return httpx.Response(307, headers={"Location": "https://unrelated.test"})
+ return httpx.Response(200, content=ExportTraceServiceResponse().SerializeToString())
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ adapter = AliyunDataTrace(config())
+ if failure is None:
+ adapter.trace(info)
+ else:
+ expected = (
+ RetryableTraceDispatchError
+ if failure in {429, 503}
+ else TraceExportUncertainError
+ if failure in {"timeout", "partial", "malformed"}
+ else TraceExportRejectedError
+ )
+ with pytest.raises(expected) as error:
+ adapter.trace(info)
+ assert "DO_NOT_EXPOSE_BODY" not in str(error.value)
+ if isinstance(error.value, RetryableTraceDispatchError):
+ assert error.value.acceptance_unknown is (expected is TraceExportUncertainError)
+ assert len(sent) == 1 # No SDK retry, per-span request, batching thread or silent queue success.
+ _, options, payload = sent[0]
+ assert options["max_retries"] == 0
+ assert options["follow_redirects"] is False
+ assert options["headers"] == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}
+ resource = payload.resource_spans[0]
+ attributes = {item.key: item.value for item in resource.resource.attributes}
+ assert attributes["service.name"].string_value == "owned-service"
+ spans = list(resource.scope_spans[0].spans)
+ assert len(spans) == 4
+ assert [s.name for s in spans] == ["workflow", "Start", "LLM", "End"]
+ root_id = convert_to_span_id(RUN_ID, "workflow").to_bytes(8)
+ assert [s.span_id for s in spans] == [root_id, *[convert_to_span_id(i, "node").to_bytes(8) for i in NODE_IDS]]
+ assert all(s.trace_id == UUID(RUN_ID).bytes for s in spans)
+ assert [s.parent_span_id for s in spans] == [b"", root_id, root_id, root_id]
+ assert spans[0].start_time_unix_nano == int(START.timestamp() * 1_000_000_000)
+ assert spans[0].end_time_unix_nano == int((START + timedelta(seconds=1)).timestamp() * 1_000_000_000)
+ model_attributes = {item.key: item.value for item in spans[2].attributes}
+ assert model_attributes[GEN_AI_USAGE_TOTAL_TOKENS].string_value == "32"
+ assert all(s.status.code == 1 for s in spans)
+ assert adapter.trace_client._dispatch_spans.get() is None
+ assert info.workflow_run_status == "succeeded"
+ sqlite_session.refresh(run)
+ assert run.status == WorkflowExecutionStatus.SUCCEEDED
+ assert run.total_tokens == 32
+ assert all(node.status == WorkflowNodeExecutionStatus.SUCCEEDED for node in nodes)
+ # A fresh business operation does not inherit any old batch after success or error.
+ adapter.trace(ModerationTraceInfo(flagged=False, action="allow", preset_response="", query="", metadata={}))
+ assert len(sent) == 1
+ adapter.trace(info.model_copy(update={"workflow_run_id": "a0000000-0000-4000-8000-000000000003"}))
+ assert len(sent) == 2
+ recovered_spans = list(sent[1][2].resource_spans[0].scope_spans[0].spans)
+ assert [s.name for s in recovered_spans] == ["workflow"]
+
+
[email protected]("failed", [False, True])
+def test_message_mapping_keeps_original_ids_raw_usage_error_and_business_state(
+ sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, failed: bool
+) -> None:
+ message = make_message(
+ message_id=MESSAGE_ID,
+ app_id="app",
+ conversation_id="conversation",
+ query="hello",
+ answer="ok",
+ status=MessageStatus.ERROR if failed else MessageStatus.NORMAL,
+ inputs={},
+ message={},
+ message_unit_price=Decimal(0),
+ answer_unit_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.CONSOLE,
+ from_account_id="owner",
+ message_tokens=21,
+ answer_tokens=11,
+ )
+ sqlite_session.add(message)
+ sqlite_session.commit()
+ info = MessageTraceInfo(
+ conversation_model="chat",
+ conversation_mode="chat",
+ message_id=MESSAGE_ID,
+ message_data=message,
+ inputs={"prompt": "hello"},
+ outputs="ok",
+ message_tokens=21,
+ answer_tokens=11,
+ total_tokens=32,
+ error="synthetic model failure" if failed else None,
+ metadata={
+ "app_id": "app",
+ "conversation_id": "conversation",
+ "ls_model_name": "test-model",
+ "ls_provider": "test",
+ },
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ )
+ sent: list[ExportTraceServiceRequest] = []
+
+ def send(_method: str, _url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ sent.append(decoded(options))
+ return httpx.Response(200, content=b"")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ adapter = AliyunDataTrace(config())
+ adapter.trace(info)
+ assert len(sent) == 1
+ spans = list(sent[0].resource_spans[0].scope_spans[0].spans)
+ assert [s.name for s in spans] == ["message", "llm"]
+ root_id = convert_to_span_id(MESSAGE_ID, "message").to_bytes(8)
+ assert [s.parent_span_id for s in spans] == [b"", root_id]
+ assert [s.span_id for s in spans] == [root_id, convert_to_span_id(MESSAGE_ID, "llm").to_bytes(8)]
+ assert all(s.trace_id == UUID(MESSAGE_ID).bytes and s.status.code == (2 if failed else 1) for s in spans)
+ assert {a.key: a.value for a in spans[1].attributes}[GEN_AI_USAGE_TOTAL_TOKENS].string_value == "32"
+ assert message.status == (MessageStatus.ERROR if failed else MessageStatus.NORMAL)
+
+
+def test_one_cached_client_keeps_interleaved_batches_and_failure_cleanup_isolated(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ client = TraceClient(service_name="owned-service", endpoint="https://collector.test/adapt_key/api/otlp/traces")
+ barrier = Barrier(2)
+ observed: list[list[str]] = []
+
+ def send(_method: str, _url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ payload = decoded(options)
+ observed.append([s.name for rs in payload.resource_spans for scope in rs.scope_spans for s in scope.spans])
+ return httpx.Response(200, content=b"")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+
+ def run(label: str) -> None:
+ try:
+ with client.batch():
+ client.add_span(SpanData(name=label, trace_id=1, span_id=2, start_time=1, end_time=2))
+ barrier.wait(timeout=5)
+ if label == "failed":
+ raise ValueError("mapper failed before submission")
+ client.add_span(SpanData(name=label + "-child", trace_id=1, span_id=3, start_time=1, end_time=2))
+ except ValueError:
+ assert label == "failed"
+ assert client._dispatch_spans.get() is None
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ list(pool.map(run, ["accepted", "failed"]))
+ assert observed == [["accepted", "accepted-child"]]
+ with client.batch():
+ client.add_span(SpanData(name="next", trace_id=1, span_id=4, start_time=1, end_time=2))
+ assert observed == [["accepted", "accepted-child"], ["next"]]
diff --git a/api/tests/unit_tests/services/tracing/test_legacy_weave_transport.py b/api/tests/unit_tests/services/tracing/test_legacy_weave_transport.py
new file mode 100644
index 00000000000..6ab4749fd89
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_legacy_weave_transport.py
@@ -0,0 +1,521 @@
+"""Real Weave SDK serialization with only the outbound HTTP boundary replaced."""
+
+import atexit
+import json
+import os
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from threading import Barrier
+from typing import TypedDict, Unpack
+
+import httpx
+import pytest
+import wandb
+import weave
+from dify_trace_weave.config import WeaveConfig
+from dify_trace_weave.entities.weave_trace_entity import WeaveTraceModel
+from dify_trace_weave.weave_trace import WeaveDataTrace
+from sqlalchemy.orm import Session
+from weave.trace_server.trace_server_interface import CallEndReq, CallStartReq, TraceStatus
+
+from core.helper import ssrf_proxy
+from core.ops.entities.trace_entity import GenerateNameTraceInfo, WorkflowTraceInfo
+from core.ops.exceptions import RetryableTraceDispatchError
+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus
+from models.account import TenantAccountJoin, TenantAccountRole
+from models.enums import CreatorUserRole
+from models.model import AppMode
+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError
+from tests.unit_tests.model_factories import make_account, make_app, make_tenant
+
+START = datetime(2026, 10, 19, tzinfo=UTC)
+
+
+class RequestOptions(TypedDict, total=False):
+ content: bytes
+ json: dict[str, object]
+ auth: tuple[str, str]
+ headers: Mapping[str, str]
+ max_retries: int
+ follow_redirects: bool
+ timeout: float
+
+
+def config(**overrides: object) -> WeaveConfig:
+ return WeaveConfig.model_validate({"api_key": "fixture-key", "entity": "team", "project": "project", **overrides})
+
+
+def call(call_id: str = "call-1", **overrides: object) -> WeaveTraceModel:
+ return WeaveTraceModel.model_validate(
+ {
+ "id": call_id,
+ "op": "tool",
+ "inputs": {"synthetic": True},
+ "outputs": {"result": "ok"},
+ "attributes": {"trace_id": call_id, "start_time": START, "end_time": START + timedelta(seconds=1)},
+ **overrides,
+ }
+ )
+
+
+def response(url: str, body: object, *, status: int = 200) -> httpx.Response:
+ return httpx.Response(status, request=httpx.Request("POST", url), json=body)
+
+
+def wire(
+ monkeypatch: pytest.MonkeyPatch, failure: tuple[str, str, int | str] | None
+) -> list[tuple[str, RequestOptions, dict[str, object]]]:
+ sent: list[tuple[str, RequestOptions, dict[str, object]]] = []
+ fault_pending = True
+
+ def send(method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ nonlocal fault_pending
+ assert method == "POST"
+ assert url in {"https://trace.wandb.ai/call/start", "https://trace.wandb.ai/call/end"}
+ body = json.loads(options["content"])
+ sent.append((url, options, body))
+ stage = "start" if "start" in body else "end"
+ record = body[stage]
+ if fault_pending and failure is not None and (stage, record["id"]) == failure[:2]:
+ fault_pending = False
+ outcome = failure[2]
+ if outcome == "timeout":
+ raise httpx.ReadTimeout("synthetic lost acknowledgment", request=httpx.Request("POST", url))
+ if outcome == "malformed":
+ return httpx.Response(200, request=httpx.Request("POST", url), content=b"invalid-json")
+ assert isinstance(outcome, int)
+ return response(url, {"error": "DO_NOT_EXPOSE_BODY"}, status=outcome)
+ if url.endswith("/call/start"):
+ req = CallStartReq.model_validate(body)
+ return response(url, {"id": req.start.id, "trace_id": req.start.trace_id})
+ CallEndReq.model_validate(body)
+ return response(url, {})
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ return sent
+
+
+def test_constructor_and_presentation_have_no_login_network_files_or_background_state(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def forbidden(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("Constructor must not initialize global credentials, files, I/O or background work")
+
+ monkeypatch.setattr(Path, "touch", forbidden)
+ monkeypatch.setattr(wandb, "login", forbidden)
+ monkeypatch.setattr(weave, "init", forbidden)
+ monkeypatch.setattr("wandb.sdk.lib.wbauth.wbnetrc.write_netrc_auth", forbidden)
+ monkeypatch.setattr("weave.trace_server_bindings.remote_http_trace_server.AsyncBatchProcessor", forbidden)
+ monkeypatch.setattr("weave.trace_server_bindings.remote_http_trace_server.CallBatchProcessor", forbidden)
+ monkeypatch.setattr(ssrf_proxy, "make_request", forbidden)
+ monkeypatch.setattr(atexit, "register", forbidden)
+ monkeypatch.setenv("WANDB_API_KEY", "unrelated-key")
+ monkeypatch.setenv("WANDB_BASE_URL", "https://unrelated.test")
+ monkeypatch.setenv("WEAVE_TRACE_SERVER", "https://unrelated-traces.test")
+ before = dict(os.environ)
+ original = config(entity=None)
+ adapter = WeaveDataTrace(original)
+ original.api_key = "changed-after-construction"
+ assert adapter.weave_api_key == "fixture-key"
+ assert adapter.get_project_url() == "https://wandb.ai"
+ assert adapter.server.trace_server_url == "https://trace.wandb.ai"
+ assert adapter.server._auth == ("api", "fixture-key")
+ assert not adapter.server.should_batch
+ assert adapter.server.call_processor is None
+ assert adapter.server.feedback_processor is None
+ assert dict(os.environ) == before
+
+
[email protected]("agent_runtime_app")
+ "failure",
+ [None]
+ + [
+ (stage, call_id, status)
+ for stage, call_id in [("start", "legacy-run"), ("end", "legacy-run"), ("start", "node-0"), ("end", "node-0")]
+ for status in [429, 503]
+ ]
+ + [
+ ("start", "legacy-run", "timeout"),
+ ("start", "legacy-run", "malformed"),
+ ("end", "node-0", "malformed"),
+ ("end", "legacy-run", "timeout"),
+ ]
+ + [("start", "legacy-run", 401), ("end", "legacy-run", 401)],
+)
+def test_workflow_emits_four_real_sdk_calls_with_original_ids_payload_and_status(
+ sqlite_session: Session,
+ wire: list[tuple[str, RequestOptions, dict[str, object]]],
+ failure: tuple[str, str, int | str] | None,
+) -> None:
+ # The normal Flask/SQLite fixtures bind the actual service-account lookup and
+ # production repository to persisted rows. Only outbound HTTP is substituted.
+ tenant = make_tenant(tenant_id="tenant")
+ account = make_account(account_id="owner", tenant=tenant)
+ app = make_app(app_id="app", tenant_id=tenant.id, mode=AppMode.WORKFLOW, created_by=account.id)
+ membership = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole.OWNER)
+ sqlite_session.add_all([tenant, account, app, membership])
+ nodes = [
+ WorkflowNodeExecutionModel(
+ id=f"node-{i}",
+ tenant_id=tenant.id,
+ app_id=app.id,
+ workflow_id="workflow",
+ workflow_run_id="legacy-run",
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
+ index=i,
+ node_id=f"node-{i}",
+ title=title,
+ node_type=kind,
+ status=WorkflowNodeExecutionStatus.SUCCEEDED,
+ created_by=account.id,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ inputs=json.dumps({"synthetic": True}),
+ outputs=json.dumps({"answer": "ok"}),
+ created_at=START + timedelta(milliseconds=i),
+ finished_at=START + timedelta(milliseconds=i + 100),
+ elapsed_time=0.1,
+ process_data=json.dumps(
+ {
+ "prompts": [{"role": "user", "text": "hello"}],
+ "model_mode": "chat",
+ "model_provider": "test",
+ "model_name": "test-model",
+ }
+ )
+ if i == 1
+ else None,
+ execution_metadata=json.dumps(
+ {"total_tokens": 32, "total_price": "0.0000098", "currency": "USD"} if i == 1 else {}
+ ),
+ )
+ for i, (title, kind) in enumerate(
+ [("Start", BuiltinNodeTypes.START), ("LLM", BuiltinNodeTypes.LLM), ("End", BuiltinNodeTypes.END)]
+ )
+ ]
+ sqlite_session.add_all(nodes)
+ sqlite_session.commit()
+ adapter = WeaveDataTrace(config())
+ info = WorkflowTraceInfo(
+ query="hello",
+ workflow_id="workflow",
+ tenant_id="tenant",
+ workflow_run_id="legacy-run",
+ workflow_run_elapsed_time=1.0,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={"query": "hello"},
+ workflow_run_outputs={"answer": "ok"},
+ workflow_run_version="v1",
+ total_tokens=32,
+ file_list=[],
+ metadata={"app_id": "app", "conversation_id": None},
+ trace_id="external-correlation",
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ )
+
+ if failure is not None:
+ stage, call_id, outcome = failure
+ first_start = (stage, call_id) == ("start", "legacy-run")
+ expected = (
+ TraceExportRejectedError
+ if outcome == 401
+ else RetryableTraceDispatchError
+ if first_start and outcome in {429, 503}
+ else TraceExportUncertainError
+ )
+ with pytest.raises(expected) as error:
+ adapter.trace(info)
+ if isinstance(error.value, RetryableTraceDispatchError):
+ assert error.value.acceptance_unknown is (expected is TraceExportUncertainError)
+ assert len(wire) == (1 if first_start else 8 if call_id == "legacy-run" else 2 if stage == "start" else 3)
+ assert "DO_NOT_EXPOSE" not in str(error.value)
+ assert adapter.calls == {}
+ assert info.workflow_run_status == "succeeded"
+ assert all(node.status == WorkflowNodeExecutionStatus.SUCCEEDED for node in nodes)
+ before_recovery = len(wire)
+ adapter.trace(
+ GenerateNameTraceInfo(
+ tenant_id=tenant.id,
+ inputs="new business operation",
+ outputs="new title",
+ metadata={},
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ )
+ )
+ assert len(wire) == before_recovery + 2
+ assert adapter.calls == {}
+ return
+
+ adapter.trace(info)
+
+ starts = [CallStartReq.model_validate(body).start for _, _, body in wire if "start" in body]
+ ends = [CallEndReq.model_validate(body).end for _, _, body in wire if "end" in body]
+ assert len(starts) == len(ends) == 4
+ assert [s.id for s in starts] == ["legacy-run", "node-0", "node-1", "node-2"]
+ assert [s.parent_id for s in starts] == [None, "legacy-run", "legacy-run", "legacy-run"]
+ assert all(s.trace_id == "external-correlation" and s.project_id == "team/project" for s in starts)
+ # Legacy has no native thread/turn contract: keep the existing raw metadata.
+ assert all(s.thread_id is None and s.turn_id is None for s in starts)
+ assert starts[0].attributes["conversation_id"] is None
+ model = starts[2]
+ assert model.attributes["total_tokens"] == 32
+ assert model.attributes["total_price"] == "0.0000098"
+ assert model.inputs["messages"][0]["content"] == "hello"
+ model_end = next(e for e in ends if e.id == "node-1")
+ assert isinstance(model_end.output, dict)
+ assert model_end.output["usage_metadata"]["total_tokens"] == 32
+ assert all(e.exception is None and e.summary["status_counts"][TraceStatus.SUCCESS] == 1 for e in ends)
+ assert all("usage" not in e.summary for e in ends) # Do not invent native aggregation for the old mapper.
+ assert all(options["auth"] == ("api", "fixture-key") and options["max_retries"] == 0 for _, options, _ in wire)
+ assert adapter.calls == {}
+
+
+def test_interleaved_instances_resolve_only_their_own_credentials_and_targets(monkeypatch: pytest.MonkeyPatch) -> None:
+ barrier = Barrier(2)
+ observed: list[tuple[str, tuple[str, str], str]] = []
+ monkeypatch.setenv("WANDB_API_KEY", "ambient-key")
+ monkeypatch.setenv("WANDB_ENTITY", "ambient-entity")
+ monkeypatch.setenv("WANDB_PROJECT", "ambient-project")
+ before = dict(os.environ)
+
+ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ label = (
+ "a"
+ if url
+ in {
+ "https://control-a.test/prefix/graphql",
+ "https://trace-a.test/prefix/call/start",
+ "https://trace-a.test/prefix/call/end",
+ }
+ else "b"
+ )
+ assert url in {
+ f"https://control-{label}.test/prefix/graphql",
+ f"https://trace-{label}.test/prefix/call/start",
+ f"https://trace-{label}.test/prefix/call/end",
+ }
+ assert options["auth"] == ("api", f"key-{label}")
+ if url.endswith("/graphql"):
+ return response(url, {"data": {"viewer": {"entity": f"team-{label}"}}})
+ body = json.loads(options["content"])
+ record = body.get("start", body.get("end"))
+ assert record["project_id"] == f"team-{label}/project-{label}"
+ observed.append((url, options["auth"], record["project_id"]))
+ if "start" in body:
+ barrier.wait(timeout=5)
+ return response(url, {"id": record["id"], "trace_id": record["trace_id"]})
+ return response(url, {})
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+
+ def run(label: str) -> None:
+ adapter = WeaveDataTrace(
+ config(
+ api_key=f"key-{label}",
+ entity=None,
+ project=f"project-{label}",
+ host=f"https://control-{label}.test/prefix",
+ endpoint=f"https://trace-{label}.test/prefix/",
+ )
+ )
+ data = call(f"call-{label}")
+ adapter.start_call(data)
+ adapter.finish_call(data)
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ list(pool.map(run, ["a", "b"]))
+ assert len(observed) == 4
+ assert dict(os.environ) == before
+
+
+def test_concurrent_dispatches_on_one_cached_instance_keep_prefix_state_isolated(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ barrier = Barrier(2)
+
+ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ body = json.loads(options["content"])
+ if "start" in body:
+ start = CallStartReq.model_validate(body).start
+ if start.inputs["label"] == "unaccepted":
+ barrier.wait(timeout=5)
+ return response(url, {}, status=503)
+ return response(url, {"id": start.id, "trace_id": start.trace_id})
+ CallEndReq.model_validate(body)
+ barrier.wait(timeout=5)
+ return response(url, {}, status=503)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ adapter = WeaveDataTrace(config())
+
+ def run(label: str) -> RetryableTraceDispatchError:
+ info = GenerateNameTraceInfo(
+ tenant_id="tenant",
+ inputs={"label": label},
+ outputs={"label": label},
+ metadata={},
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ )
+ with pytest.raises(RetryableTraceDispatchError) as error:
+ adapter.trace(info)
+ return error.value
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ accepted, unaccepted = list(pool.map(run, ["accepted", "unaccepted"]))
+ assert isinstance(accepted, TraceExportUncertainError)
+ assert accepted.acceptance_unknown
+ assert not unaccepted.acceptance_unknown
+ assert adapter.calls == {}
+
+
+ ("stage", "failure"),
+ [
+ (stage, failure)
+ for stage in ("start", "end")
+ for failure in ("rejected", "retryable", "malformed", "mismatched_id")
+ if not (stage == "end" and failure == "mismatched_id")
+ ],
+)
+def test_synchronous_failures_propagate_and_next_call_recovers(
+ monkeypatch: pytest.MonkeyPatch, stage: str, failure: str
+) -> None:
+ broken = True
+ requests = []
+
+ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ body = json.loads(options["content"])
+ requests.append(url)
+ if broken and url.endswith("/call/" + stage):
+ if failure == "rejected":
+ return response(url, {"error": "DO_NOT_EXPOSE_KEY_OR_BODY"}, status=401)
+ if failure == "retryable":
+ return response(url, {}, status=503)
+ if failure == "malformed":
+ return httpx.Response(200, request=httpx.Request("POST", url), content=b"invalid-json")
+ return response(url, {"id": "wrong-id", "trace_id": "wrong-trace"})
+ if "start" in body:
+ return response(url, {"id": body["start"]["id"], "trace_id": body["start"]["trace_id"]})
+ return response(url, {})
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ adapter = WeaveDataTrace(config())
+ data = call()
+ expected = {
+ "rejected": TraceExportRejectedError,
+ "retryable": RetryableTraceDispatchError,
+ "malformed": TraceExportUncertainError,
+ "mismatched_id": TraceExportUncertainError,
+ }[failure]
+ if stage == "end":
+ adapter.start_call(data)
+ with pytest.raises(expected) as error:
+ adapter.start_call(data) if stage == "start" else adapter.finish_call(data)
+ assert "DO_NOT_EXPOSE" not in str(error.value)
+ assert len(requests) == (1 if stage == "start" else 2)
+ assert adapter.calls == {}
+ broken = False
+ recovered = call("next-call", exception="synthetic tool error")
+ adapter.start_call(recovered)
+ adapter.finish_call(recovered)
+ assert adapter.calls == {}
+
+
+def test_legacy_api_check_uses_read_only_project_and_stats_requests(monkeypatch: pytest.MonkeyPatch) -> None:
+ sent = []
+
+ def post(url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ assert options["auth"] == ("api", "fixture-key")
+ assert options["follow_redirects"] is False
+ sent.append(url)
+ if url == "https://api.wandb.ai/graphql":
+ return response(
+ url,
+ {
+ "data": {
+ "viewer": {"id": "viewer"},
+ "project": {"id": "project-id", "name": "project", "entityName": "team", "readOnly": False},
+ }
+ },
+ )
+ assert url == "https://trace.wandb.ai/calls/query_stats"
+ assert options["json"]["project_id"] == "team/project"
+ return response(url, {"count": 0})
+
+ monkeypatch.setattr(ssrf_proxy, "post", post)
+ adapter = WeaveDataTrace(config())
+ assert sent == []
+ assert adapter.api_check()
+ assert sent == ["https://api.wandb.ai/graphql", "https://trace.wandb.ai/calls/query_stats"]
+ assert adapter.calls == {}
+
+
+ ("sent_id", "ack_id"),
+ [
+ ("1234567812345678123456781234abcd", "12345678-1234-5678-1234-56781234abcd"),
+ ("12345678-1234-5678-1234-56781234abcd", "1234567812345678123456781234ABCD"),
+ ("12345678-1234-5678-1234-56781234ABCD", "1234567812345678123456781234abcd"),
+ ],
+)
+def test_uuid_equivalent_start_ack_preserves_sent_ids(
+ monkeypatch: pytest.MonkeyPatch, sent_id: str, ack_id: str
+) -> None:
+ sent: list[dict[str, object]] = []
+
+ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ body = json.loads(options["content"])
+ sent.append(body)
+ if "start" in body:
+ start = CallStartReq.model_validate(body).start
+ assert start.id == sent_id
+ assert start.trace_id == sent_id
+ return response(url, {"id": ack_id, "trace_id": ack_id})
+ end = CallEndReq.model_validate(body).end
+ assert end.id == sent_id
+ return response(url, {})
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ adapter = WeaveDataTrace(config())
+ data = call(sent_id)
+ adapter.start_call(data)
+ assert adapter.calls[sent_id]["trace_id"] == sent_id
+ adapter.finish_call(data)
+ assert len(sent) == 2
+ assert adapter.calls == {}
+
+
[email protected]("field", ["id", "trace_id"])
+ ("sent_id", "wrong_ack"),
+ [
+ ("1234567812345678123456781234abcd", "12345678-1234-5678-1234-56781234abce"),
+ ("legacy-correlation", "LEGACY-CORRELATION"),
+ ("1234567812345678123456781234abcd", "not-a-uuid"),
+ ],
+)
+def test_start_ack_rejects_different_uuid_or_opaque_identifiers(
+ monkeypatch: pytest.MonkeyPatch, field: str, sent_id: str, wrong_ack: str
+) -> None:
+ sent = []
+
+ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:
+ start = CallStartReq.model_validate_json(options["content"]).start
+ sent.append(start)
+ ack = {"id": start.id, "trace_id": start.trace_id}
+ ack[field] = wrong_ack
+ return response(url, ack)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ adapter = WeaveDataTrace(config())
+ with pytest.raises(TraceExportUncertainError, match="did not acknowledge the submitted identifiers"):
+ adapter.start_call(call(sent_id))
+ assert len(sent) == 1
+ assert adapter.calls == {}
diff --git a/api/tests/unit_tests/services/tracing/test_mlflow_configuration.py b/api/tests/unit_tests/services/tracing/test_mlflow_configuration.py
new file mode 100644
index 00000000000..d8f1bd5e5ab
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_mlflow_configuration.py
@@ -0,0 +1,317 @@
+"""OSS MLflow verification reads one experiment with isolated Basic credentials."""
+
+import base64
+import os
+from collections.abc import Callable, Iterator
+from concurrent.futures import ThreadPoolExecutor
+from threading import Barrier
+from typing import Never
+
+import httpx
+import mlflow
+import pytest
+from dify_trace_mlflow.config import MLflowConfig
+from dify_trace_mlflow.mlflow_trace import MLflowDataTrace
+from dify_trace_mlflow.oss_configuration import (
+ MLflowConfigurationError,
+ normalize_configuration,
+ project_url,
+ verify_configuration,
+)
+from pydantic import ValidationError
+
+from core.helper import ssrf_proxy
+from tests.unit_tests.services.tracing.test_databricks_configuration import ExperimentAPI, response
+
+
[email protected](autouse=True)
+def isolated_sdk_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ names = (
+ "MLFLOW_TRACKING_URI",
+ "MLFLOW_TRACKING_USERNAME",
+ "MLFLOW_TRACKING_PASSWORD",
+ "MLFLOW_TRACKING_TOKEN",
+ "MLFLOW_EXPERIMENT_ID",
+ "MLFLOW_ENABLE_ASYNC_TRACE_LOGGING",
+ "DATABRICKS_HOST",
+ "DATABRICKS_TOKEN",
+ )
+ for name in names:
+ monkeypatch.setenv(name, "synthetic-ambient-value")
+ forbidden_calls: list[str] = []
+
+ def forbidden(name: str) -> Callable[..., Never]:
+ def block(*_args: object, **_kwargs: object) -> Never:
+ forbidden_calls.append(name)
+ raise AssertionError("Configuration must not initialize an exporter or global SDK")
+
+ return block
+
+ monkeypatch.setattr(MLflowDataTrace, "__init__", forbidden("legacy_constructor"))
+ for name in ("set_tracking_uri", "set_experiment", "search_experiments"):
+ monkeypatch.setattr(mlflow, name, forbidden(name))
+ monkeypatch.setattr(mlflow.config, "enable_async_logging", forbidden("async_logging"))
+ yield
+ assert forbidden_calls == []
+ assert all(os.environ[name] == "synthetic-ambient-value" for name in names)
+
+
+def experiment_api(monkeypatch: pytest.MonkeyPatch) -> ExperimentAPI:
+ api = ExperimentAPI()
+ monkeypatch.setattr(ssrf_proxy, "get", api.get)
+
+ def forbid_post(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("OSS verification must not create an experiment, run, trace, or session")
+
+ monkeypatch.setattr(ssrf_proxy, "post", forbid_post)
+ return api
+
+
+def test_exact_experiment_and_path_prefix_use_only_explicit_basic_auth(experiment_api: ExperimentAPI) -> None:
+ values = {
+ "tracking_uri": " HTTP://FIRST.Example:5000/team/mlflow/ ",
+ "experiment_id": "00012",
+ "username": "first-user",
+ "password": "first-password",
+ }
+ config = normalize_configuration(values)
+ target = verify_configuration(config)
+ assert target.tracking_uri == "http://first.example:5000/team/mlflow"
+ assert target.experiment_id == "12"
+ assert project_url(config) == "http://first.example:5000/team/mlflow/#/experiments/12/traces"
+ method, url, options = experiment_api.calls[0]
+ assert method == "GET"
+ assert url == "http://first.example:5000/team/mlflow/api/2.0/mlflow/experiments/get"
+ assert options["params"] == {"experiment_id": "12"}
+ assert options["headers"] == {
+ "Accept": "application/json",
+ "Accept-Encoding": "identity",
+ "Cookie": "",
+ "Authorization": "Basic " + base64.b64encode(b"first-user:first-password").decode(),
+ }
+ assert options["follow_redirects"] is False
+ assert options["stream_response"] is True
+ assert options["timeout"] == 30.0
+ assert options["max_retries"] == 0
+ assert values["tracking_uri"].startswith(" HTTP")
+
+
+def test_anonymous_request_does_not_inherit_previous_or_environment_credentials(experiment_api: ExperimentAPI) -> None:
+ verify_configuration(
+ MLflowConfig(tracking_uri="https://same.test", experiment_id="12", username="user", password="password")
+ )
+ verify_configuration(MLflowConfig(tracking_uri="https://same.test", experiment_id="34"))
+ assert len(experiment_api.calls) == 2
+ assert experiment_api.calls[1][2]["headers"] == {
+ "Accept": "application/json",
+ "Accept-Encoding": "identity",
+ "Cookie": "",
+ }
+
+
+def test_concurrent_destinations_keep_experiment_and_auth_together(experiment_api: ExperimentAPI) -> None:
+ barrier = Barrier(2)
+ expected_destinations = {
+ "https://first.test/prefix/api/2.0/mlflow/experiments/get": ("first", "12"),
+ "https://second.test/prefix/api/2.0/mlflow/experiments/get": ("second", "34"),
+ }
+
+ def read(_method: str, url: str, options: dict[str, object]) -> httpx.Response:
+ params = options["params"]
+ headers = options["headers"]
+ assert isinstance(params, dict)
+ assert isinstance(headers, dict)
+ label, experiment_id = expected_destinations[url]
+ assert params["experiment_id"] == experiment_id
+ assert headers["Authorization"] == "Basic " + base64.b64encode(f"{label}:{label}-secret".encode()).decode()
+ barrier.wait(timeout=5)
+ return response(url, {"experiment": {"experiment_id": params["experiment_id"], "lifecycle_stage": "active"}})
+
+ experiment_api.override = read
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ pending = [
+ pool.submit(
+ verify_configuration,
+ MLflowConfig(
+ tracking_uri=f"https://{label}.test/prefix",
+ experiment_id=experiment,
+ username=label,
+ password=f"{label}-secret",
+ ),
+ )
+ for label, experiment in (("first", "12"), ("second", "34"))
+ ]
+ assert [item.result().experiment_id for item in pending] == ["12", "34"]
+ assert sorted(url for _method, url, _options in experiment_api.calls) == sorted(expected_destinations)
+
+
+ "uri",
+ [
+ "",
+ "ftp://server.test",
+ "https:///missing",
+ "https://user:[email protected]",
+ "https://@server.test",
+ "https://server.test?",
+ "https://server.test#",
+ "https://server.test?key=secret",
+ "https://server.test/#/experiments/1",
+ "https://server.test:bad",
+ "https://server.test:0",
+ "https://server.test:",
+ "https://[invalid",
+ "https://server.test\\@other.test",
+ "\nhttps://server.test",
+ "https://ser\nver.test",
+ "https://server.test/./prefix",
+ "https://server.test/prefix/../other",
+ "https://server.test/%2e%2e/other",
+ "https://server.test/prefix%2fother",
+ "https://server.test/%252fother",
+ "https://server.test/%00",
+ "https://server.test/%zz",
+ "https://server.test//prefix",
+ ],
+)
+def test_ambiguous_destinations_fail_before_request(experiment_api: ExperimentAPI, uri: str) -> None:
+ with pytest.raises((MLflowConfigurationError, ValidationError)):
+ verify_configuration(normalize_configuration({"tracking_uri": uri, "experiment_id": "12"}))
+ assert experiment_api.calls == []
+
+
[email protected]("experiment", ["", "-1", "1.5", "12", "abc"])
+def test_experiment_ids_are_ascii_integer_identifiers(experiment_api: ExperimentAPI, experiment: str) -> None:
+ with pytest.raises((MLflowConfigurationError, ValidationError)):
+ normalize_configuration({"tracking_uri": "https://server.test", "experiment_id": experiment})
+ assert experiment_api.calls == []
+
+
+ ("username", "password"),
+ [("user", None), (None, "password"), ("user", ""), (" ", "password"), ("user", " "), ("user:name", "password")],
+)
+def test_partial_or_ambiguous_basic_auth_never_sends_credentials(
+ experiment_api: ExperimentAPI, username: str | None, password: str | None
+) -> None:
+ with pytest.raises(MLflowConfigurationError) as caught:
+ verify_configuration(MLflowConfig(tracking_uri="https://server.test", username=username, password=password))
+ assert caught.value.code == "incomplete_basic_auth"
+ assert experiment_api.calls == []
+
+
+ ("status", "code"),
+ [
+ (401, "authentication_failed"),
+ (403, "permission_denied"),
+ (404, "experiment_not_found_or_inaccessible"),
+ (302, "endpoint_unavailable"),
+ (307, "endpoint_unavailable"),
+ (500, "endpoint_unavailable"),
+ ],
+)
+def test_http_failures_do_not_follow_redirect_or_disclose_body(
+ experiment_api: ExperimentAPI, status: int, code: str
+) -> None:
+ result = response("https://server.test", {"message": "private-vendor-body"}, status)
+ result.headers["Location"] = "https://other.test/secret"
+ experiment_api.override = lambda _method, _url, _options: result
+ with pytest.raises(MLflowConfigurationError) as caught:
+ verify_configuration(
+ MLflowConfig(tracking_uri="https://server.test", username="user", password="private-password")
+ )
+ assert caught.value.code == code
+ assert "private" not in str(caught.value)
+ assert result.is_closed
+ assert len(experiment_api.calls) == 1
+
+
+ ("body", "code"),
+ [
+ ({}, "malformed_response"),
+ ([], "malformed_response"),
+ ({"experiment": "invalid"}, "malformed_response"),
+ ({"experiment": {"experiment_id": "34", "lifecycle_stage": "active"}}, "experiment_id_mismatch"),
+ ({"experiment": {"experiment_id": "12", "lifecycle_stage": "deleted"}}, "experiment_not_active"),
+ ({"experiment": {"experiment_id": "12"}}, "experiment_not_active"),
+ ],
+)
+def test_response_must_match_exact_active_experiment(experiment_api: ExperimentAPI, body: object, code: str) -> None:
+ experiment_api.override = lambda _method, url, _options: response(url, body)
+ with pytest.raises(MLflowConfigurationError) as caught:
+ verify_configuration(MLflowConfig(tracking_uri="https://server.test", experiment_id="12"))
+ assert caught.value.code == code
+
+
+def test_transport_failure_and_oversized_response_are_bounded_and_safe(experiment_api: ExperimentAPI) -> None:
+ def fail(_method: str, _url: str, _options: dict[str, object]) -> httpx.Response:
+ raise httpx.ConnectError("private-url-and-password")
+
+ experiment_api.override = fail
+ with pytest.raises(MLflowConfigurationError, match="MLflow verification request failed"):
+ verify_configuration(MLflowConfig(tracking_uri="https://server.test"))
+ oversized = response("https://server.test", {"message": "x" * (1024 * 1024)})
+ experiment_api.override = lambda _method, _url, _options: oversized
+ with pytest.raises(MLflowConfigurationError) as caught:
+ verify_configuration(MLflowConfig(tracking_uri="https://server.test"))
+ assert caught.value.code == "malformed_response"
+ assert oversized.is_closed
+
+
+def test_project_link_requires_no_auth_or_io_and_preserves_valid_prefix(experiment_api: ExperimentAPI) -> None:
+ assert project_url(MLflowConfig(tracking_uri="https://Server.test/team/mlflow/", experiment_id="000")) == (
+ "https://server.test/team/mlflow/#/experiments/0/traces"
+ )
+ assert experiment_api.calls == []
+
+
+def test_real_http_transport_overrides_ambient_auth_and_cookies(monkeypatch: pytest.MonkeyPatch) -> None:
+ requests: list[httpx.Request] = []
+
+ def handle(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ assert request.method == "GET"
+ assert request.url.path == "/prefix/api/2.0/mlflow/experiments/get"
+ assert request.headers.get("cookie") == ""
+ return httpx.Response(
+ 200,
+ headers={"Set-Cookie": "session=synthetic-vendor-session; Path=/"},
+ json={"experiment": {"experiment_id": request.url.params["experiment_id"], "lifecycle_stage": "active"}},
+ )
+
+ with httpx.Client(
+ transport=httpx.MockTransport(handle),
+ auth=httpx.BasicAuth("ambient-user", "ambient-password"),
+ cookies={"session": "synthetic-ambient-session"},
+ trust_env=False,
+ ) as client:
+
+ def get_client(_ssl_verify_enabled: bool) -> httpx.Client:
+ return client
+
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", get_client)
+ verify_configuration(
+ MLflowConfig(
+ tracking_uri="https://server.test/prefix", experiment_id="12", username="owner", password="secret"
+ )
+ )
+ verify_configuration(MLflowConfig(tracking_uri="https://server.test/prefix", experiment_id="34"))
+ assert len(requests) == 2
+ assert requests[0].headers["authorization"] == "Basic " + base64.b64encode(b"owner:secret").decode()
+ assert "authorization" not in requests[1].headers
+
+
[email protected]("compressed", [False, True])
+def test_invalid_encoding_or_json_response_is_closed(experiment_api: ExperimentAPI, compressed: bool) -> None:
+ result = httpx.Response(200, content=b"private-vendor-text", request=httpx.Request("GET", "https://server.test"))
+ if compressed:
+ result.headers["Content-Encoding"] = "gzip"
+ experiment_api.override = lambda _method, _url, _options: result
+ with pytest.raises(MLflowConfigurationError) as caught:
+ verify_configuration(MLflowConfig(tracking_uri="https://server.test"))
+ assert caught.value.code == "malformed_response"
+ assert "private-vendor-text" not in str(caught.value)
+ assert result.is_closed
diff --git a/api/tests/unit_tests/services/tracing/test_mlflow_legacy_destination.py b/api/tests/unit_tests/services/tracing/test_mlflow_legacy_destination.py
new file mode 100644
index 00000000000..9a2ab3b6bea
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_mlflow_legacy_destination.py
@@ -0,0 +1,637 @@
+"""Real MLflow SDK export must not select the application's ambient OTLP destination."""
+
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from threading import Barrier
+from typing import Never
+
+import mlflow
+import pytest
+import requests
+from dify_trace_mlflow.config import DatabricksConfig, MLflowConfig
+from dify_trace_mlflow.mlflow_trace import MLflowDataTrace
+from google.protobuf.json_format import Parse
+from mlflow.entities import SpanType, Trace
+from mlflow.entities.trace_location import MlflowExperimentLocation
+from mlflow.protos.service_pb2 import StartTraceV3
+from mlflow.tracing import provider as sdk_provider
+from mlflow.tracing.constant import SpanAttributeKey, TraceMetadataKey, TraceTagKey
+from mlflow.tracing.trace_manager import InMemoryTraceManager
+from mlflow.tracing.utils.otlp import _decode_otel_proto_anyvalue
+from opentelemetry import trace
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+from opentelemetry.proto.trace.v1.trace_pb2 import Span as ProtoSpan
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.trace import INVALID_SPAN_CONTEXT, NonRecordingSpan
+
+from core.ops.entities.trace_entity import (
+ DatasetRetrievalTraceInfo,
+ GenerateNameTraceInfo,
+ MessageTraceInfo,
+ ModerationTraceInfo,
+ SuggestedQuestionTraceInfo,
+ ToolTraceInfo,
+ WorkflowTraceInfo,
+)
+from graphon.enums import BuiltinNodeTypes
+from models.model import Message
+from models.workflow import WorkflowNodeExecutionModel
+
+
+def transport(monkeypatch: pytest.MonkeyPatch) -> list[requests.PreparedRequest]:
+ calls: list[requests.PreparedRequest] = []
+
+ def send(_session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:
+ calls.append(request)
+ response = requests.Response()
+ response.status_code = 200
+ response.request = request
+ response.url = request.url or ""
+ if response.url.endswith("/version"):
+ response._content = b"3.17.1"
+ elif response.url.endswith("/v1/traces"):
+ response._content = b"{}"
+ elif response.url.endswith("/api/3.0/mlflow/traces"):
+ assert isinstance(request.body, (str, bytes))
+ body = json.loads(request.body)
+ info = body["trace"]["trace_info"]
+ info.setdefault("tags", {})[TraceTagKey.SPANS_LOCATION] = "TRACKING_STORE"
+ response._content = json.dumps(body).encode()
+ else:
+ raise AssertionError(f"Unexpected SDK endpoint: {request.method} {request.path_url}")
+ return response
+
+ monkeypatch.setattr(requests.Session, "send", send)
+ monkeypatch.setenv("MLFLOW_DISABLE_TELEMETRY", "true")
+ monkeypatch.delenv("_MLFLOW_TELEMETRY_SESSION_ID", raising=False)
+ monkeypatch.setenv("MLFLOW_ENABLE_ASYNC_TRACE_LOGGING", "false")
+ monkeypatch.setenv("MLFLOW_ENABLE_OTLP_EXPORTER", "true")
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://dify-internal.example:4318")
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http")
+ monkeypatch.setenv("OTEL_SERVICE_NAME", "dify-api")
+ monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=unit-test")
+ return calls
+
+
+def tool_info() -> ToolTraceInfo:
+ started = datetime(2026, 1, 1, tzinfo=UTC)
+ return ToolTraceInfo(
+ tool_name="code",
+ tool_inputs={"a": 17},
+ tool_outputs="323",
+ tool_config={},
+ tool_parameters={},
+ time_cost=1,
+ metadata={},
+ inputs={"a": 17},
+ outputs="323",
+ start_time=started,
+ end_time=started + timedelta(seconds=1),
+ )
+
+
[email protected]("default_provider", ["true", "false"])
+def test_real_sdk_uses_private_mlflow_exporter_without_touching_shared_state(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+ default_provider: str,
+) -> None:
+ monkeypatch.setenv("MLFLOW_USE_DEFAULT_TRACER_PROVIDER", default_provider)
+ main_provider = TracerProvider()
+ monkeypatch.setattr(trace, "_TRACER_PROVIDER", main_provider)
+ sdk_before = sdk_provider.provider.get()
+ destination_before = sdk_provider._MLFLOW_TRACE_USER_DESTINATION.get()
+ unrelated = MlflowExperimentLocation("999")
+ monkeypatch.setattr(sdk_provider._MLFLOW_TRACE_USER_DESTINATION, "_global_value", unrelated)
+ destination_before = sdk_provider._MLFLOW_TRACE_USER_DESTINATION.get()
+ environment_before = dict(os.environ)
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("Legacy exporter must not initialize or reset the process SDK")
+
+ for name in ("set_tracking_uri", "set_experiment"):
+ monkeypatch.setattr(mlflow, name, forbidden)
+ monkeypatch.setattr(sdk_provider, "set_destination", forbidden)
+ monkeypatch.setattr(sdk_provider, "reset", forbidden)
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://legacy.example", experiment_id="12"))
+ assert transport == []
+ instance.trace(tool_info())
+ assert sdk_provider.provider.get() is sdk_before
+ assert sdk_provider._MLFLOW_TRACE_USER_DESTINATION.get() is destination_before
+ assert trace.get_tracer_provider() is main_provider
+ assert dict(os.environ) == environment_before
+ otlp_requests = [request for request in transport if (request.url or "").endswith("/v1/traces")]
+ assert len(otlp_requests) == 1
+ assert otlp_requests[0].url == "https://legacy.example/v1/traces"
+ assert otlp_requests[0].headers["x-mlflow-experiment-id"] == "12"
+ body = otlp_requests[0].body
+ assert isinstance(body, bytes)
+ spans = ExportTraceServiceRequest.FromString(body).resource_spans[0].scope_spans[0].spans
+ assert len(spans) == 1
+ assert spans[0].name == "code"
+ assert len(spans[0].trace_id) == 16
+ assert any(spans[0].trace_id)
+ assert any((request.url or "").endswith("/api/3.0/mlflow/traces") for request in transport)
+ main_provider.shutdown()
+
+
+def exported_spans(calls: list[requests.PreparedRequest]) -> list[ProtoSpan]:
+ spans: list[ProtoSpan] = []
+ for request in calls:
+ if (request.url or "").endswith("/v1/traces"):
+ assert isinstance(request.body, bytes)
+ spans.extend(ExportTraceServiceRequest.FromString(request.body).resource_spans[0].scope_spans[0].spans)
+ return spans
+
+
+def exported_traces(calls: list[requests.PreparedRequest]) -> list[Trace]:
+ results: list[Trace] = []
+ for request in calls:
+ if (request.url or "").endswith("/api/3.0/mlflow/traces"):
+ assert isinstance(request.body, (bytes, str))
+ results.append(Trace.from_proto(Parse(request.body, StartTraceV3()).trace))
+ return results
+
+
+def test_workflow_parent_children_usage_metadata_and_explicit_experiment(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+) -> None:
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://workflow.example", experiment_id="42"))
+ started = datetime(2026, 1, 1, tzinfo=UTC)
+ nodes = [
+ WorkflowNodeExecutionModel(
+ id=f"node-{index}",
+ tenant_id="tenant",
+ app_id="app",
+ title=name,
+ node_type=node_type,
+ status="succeeded",
+ inputs='{"question":"synthetic"}',
+ outputs='{"text":"answer"}',
+ created_at=started + timedelta(milliseconds=index),
+ elapsed_time=0.1,
+ process_data=json.dumps(
+ {
+ "prompts": [{"role": "user", "text": "synthetic"}],
+ "model_name": "test-model",
+ "model_provider": "test",
+ "usage": {"prompt_tokens": 25, "completion_tokens": 7, "total_tokens": 32},
+ }
+ )
+ if node_type == BuiltinNodeTypes.LLM
+ else None,
+ )
+ for index, (name, node_type) in enumerate(
+ [
+ ("Start", BuiltinNodeTypes.START),
+ ("LLM", BuiltinNodeTypes.LLM),
+ ("End", BuiltinNodeTypes.END),
+ ]
+ )
+ ]
+ monkeypatch.setattr(instance, "_get_workflow_nodes", lambda _run_id: nodes)
+ info = WorkflowTraceInfo(
+ workflow_id="workflow",
+ tenant_id="tenant",
+ workflow_run_id="run",
+ workflow_run_elapsed_time=1,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={"q": "synthetic", "sys.hidden": "drop"},
+ workflow_run_outputs={"answer": "answer"},
+ workflow_run_version="1",
+ total_tokens=32,
+ file_list=[],
+ query="",
+ conversation_id="session",
+ metadata={"user_id": "user"},
+ start_time=started,
+ end_time=started + timedelta(seconds=1),
+ )
+ ambient_provider = TracerProvider()
+ with ambient_provider.get_tracer("ambient").start_as_current_span("unrelated"):
+ instance.trace(info)
+ spans = exported_spans(transport)
+ assert len(spans) == 4
+ root = next(span for span in spans if span.name == "workflow")
+ assert not root.parent_span_id
+ assert all(span.trace_id == root.trace_id for span in spans)
+ assert all(span.parent_span_id == root.span_id for span in spans if span is not root)
+ assert all(
+ request.headers["x-mlflow-experiment-id"] == "42"
+ for request in transport
+ if (request.url or "").endswith("/v1/traces")
+ )
+ native = exported_traces(transport)
+ assert len(native) == 1
+ location = native[0].info.trace_location.mlflow_experiment
+ assert location is not None
+ assert location.experiment_id == "42"
+ assert native[0].info.trace_metadata[TraceMetadataKey.TRACE_SESSION] == "session"
+ assert native[0].info.trace_metadata[TraceMetadataKey.TRACE_USER] == "user"
+ assert json.loads(native[0].info.trace_metadata[TraceMetadataKey.TOKEN_USAGE]) == {
+ "input_tokens": 25,
+ "output_tokens": 7,
+ "total_tokens": 32,
+ }
+ attributes = {
+ attribute.key: _decode_otel_proto_anyvalue(attribute.value)
+ for attribute in root.attributes
+ if attribute.key in (SpanAttributeKey.INPUTS, SpanAttributeKey.OUTPUTS)
+ }
+ assert attributes[SpanAttributeKey.INPUTS] == {"q": "synthetic"}
+ assert attributes[SpanAttributeKey.OUTPUTS] == {"answer": "answer"}
+ ambient_provider.shutdown()
+
+
+def test_interleaved_instances_keep_children_on_their_own_uri_and_experiment(
+ transport: list[requests.PreparedRequest],
+) -> None:
+ barrier = Barrier(2)
+ instances = [
+ MLflowDataTrace(MLflowConfig(tracking_uri=f"https://{name}.example", experiment_id=experiment))
+ for name, experiment in [("first", "12"), ("second", "34")]
+ ]
+
+ def emit(instance: MLflowDataTrace) -> tuple[str, str]:
+ root = instance._start_span_no_context(name="workflow", span_type=SpanType.CHAIN)
+ barrier.wait(timeout=5)
+ child = instance._start_span_no_context(name="node", span_type=SpanType.TOOL, parent_span=root)
+ child.end(outputs="ok")
+ barrier.wait(timeout=5)
+ root.end(outputs="ok")
+ return root.trace_id, root.span_id
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ ids = list(pool.map(emit, instances))
+ assert ids[0][0] != ids[1][0]
+ for host, experiment in [("first", "12"), ("second", "34")]:
+ requests_for_host = [item for item in transport if (item.url or "").startswith(f"https://{host}.example/")]
+ spans = exported_spans(requests_for_host)
+ assert len(spans) == 2
+ assert spans[0].trace_id == spans[1].trace_id
+ root = next(span for span in spans if not span.parent_span_id)
+ child = next(span for span in spans if span.parent_span_id)
+ assert child.parent_span_id == root.span_id
+ assert all(
+ item.headers["x-mlflow-experiment-id"] == experiment
+ for item in requests_for_host
+ if (item.url or "").endswith("/v1/traces")
+ )
+ location = exported_traces(requests_for_host)[0].info.trace_location.mlflow_experiment
+ assert location is not None
+ assert location.experiment_id == experiment
+
+
+ "method", ["message", "moderation", "dataset_retrieval", "suggested_question", "generate_name"]
+)
+def test_other_legacy_entrypoints_use_the_same_private_sdk(
+ method: str,
+ transport: list[requests.PreparedRequest],
+) -> None:
+ started = datetime(2026, 1, 1, tzinfo=UTC)
+ message = Message(
+ model_provider="test",
+ model_id="model",
+ answer="answer",
+ total_price=0,
+ created_at=started,
+ updated_at=started + timedelta(seconds=1),
+ )
+ common = {
+ "metadata": {},
+ "message_data": message,
+ "inputs": "synthetic",
+ "outputs": "answer",
+ "start_time": started,
+ "end_time": started + timedelta(seconds=1),
+ }
+ match method:
+ case "message":
+ info = MessageTraceInfo(
+ **common,
+ conversation_model="chat",
+ conversation_mode="chat",
+ message_tokens=3,
+ answer_tokens=2,
+ total_tokens=5,
+ )
+ case "moderation":
+ info = ModerationTraceInfo(**common, flagged=False, action="allow", preset_response="", query="synthetic")
+ case "dataset_retrieval":
+ info = DatasetRetrievalTraceInfo(**common, documents=[{"content": "synthetic"}])
+ case "suggested_question":
+ info = SuggestedQuestionTraceInfo(**common, total_tokens=5, suggested_question=["synthetic"], level="info")
+ case _:
+ info = GenerateNameTraceInfo(**common, tenant_id="tenant")
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://other.example", experiment_id="16"))
+ instance.trace(info)
+ assert len(exported_spans(transport)) == 1
+ location = exported_traces(transport)[0].info.trace_location.mlflow_experiment
+ assert location is not None
+ assert location.experiment_id == "16"
+
+
[email protected]("failure", ["exception", "noop", "unregistered"])
+def test_span_creation_failure_is_explicit_and_does_not_export_success(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+ failure: str,
+) -> None:
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://failure.example", experiment_id="10"))
+ if failure == "exception":
+
+ def fail(*_args: object, **_kwargs: object) -> Never:
+ raise ValueError("must-not-leak-vendor-details")
+
+ monkeypatch.setattr(instance._tracer, "start_span", fail)
+ expected = "mlflow_legacy_span_start_failed"
+ elif failure == "noop":
+
+ def noop(*_args: object, **_kwargs: object) -> NonRecordingSpan:
+ return NonRecordingSpan(INVALID_SPAN_CONTEXT)
+
+ monkeypatch.setattr(instance._tracer, "start_span", noop)
+ expected = "mlflow_legacy_span_not_recording"
+ else:
+ manager = InMemoryTraceManager.get_instance()
+ monkeypatch.setattr(manager, "get_span_from_id", lambda *_args: None)
+ expected = "mlflow_legacy_span_not_registered"
+ with pytest.raises(RuntimeError, match=f"^{expected}$"):
+ instance.trace(tool_info())
+ assert not exported_spans(transport)
+ assert not exported_traces(transport)
+
+
+def test_legacy_basic_credentials_remain_compatible_at_transport_boundary(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+) -> None:
+ # This preserves legacy process-level credentials; it is not an isolation claim.
+ for key in ("MLFLOW_TRACKING_USERNAME", "MLFLOW_TRACKING_PASSWORD"):
+ monkeypatch.delenv(key, raising=False)
+ instance = MLflowDataTrace(
+ MLflowConfig(
+ tracking_uri="https://basic.example",
+ experiment_id="12",
+ username="synthetic-user",
+ password="synthetic-secret",
+ )
+ )
+ instance.trace(tool_info())
+ import base64
+
+ expected = "Basic " + base64.b64encode(b"synthetic-user:synthetic-secret").decode()
+ assert all(request.headers["Authorization"] == expected for request in transport)
+
+
[email protected]("noop", [False, True])
+def test_databricks_preserves_legacy_sdk_setup_and_span_selection(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+ noop: bool,
+) -> None:
+ import dify_trace_mlflow.mlflow_trace as legacy
+ from mlflow.entities import LiveSpan
+ from mlflow.entities.span import NoOpSpan
+
+ setup: list[tuple[str, str]] = []
+ provider = TracerProvider()
+ raw = provider.get_tracer("databricks-legacy").start_span("code")
+ span = LiveSpan(raw, "tr-synthetic-databricks")
+ options: list[dict[str, object]] = []
+
+ def start(**kwargs: object) -> LiveSpan | NoOpSpan:
+ options.append(kwargs)
+ return NoOpSpan() if noop else span
+
+ monkeypatch.setattr(mlflow, "set_tracking_uri", lambda uri: setup.append(("uri", uri)))
+ monkeypatch.setattr(mlflow, "set_experiment", lambda *, experiment_id: setup.append(("experiment", experiment_id)))
+ monkeypatch.setattr(legacy, "start_span_no_context", start)
+ for name in ("DATABRICKS_HOST", "DATABRICKS_TOKEN", "MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"):
+ monkeypatch.delenv(name, raising=False)
+ instance = MLflowDataTrace(
+ DatabricksConfig(host="https://workspace.example", experiment_id="123", personal_access_token="synthetic-pat")
+ )
+ assert setup == [("uri", "databricks"), ("experiment", "123")]
+ assert instance._tracer is None
+ assert instance._exporter is None
+ if noop:
+ with pytest.raises(RuntimeError, match="^mlflow_legacy_span_not_recording$"):
+ instance.trace(tool_info())
+ raw.end()
+ else:
+ instance.trace(tool_info())
+ assert span.outputs == "323"
+ assert options[0]["name"] == "code"
+ assert options[0]["inputs"] == {"a": 17}
+ assert transport == []
+ provider.shutdown()
+
+
+def test_private_exporter_is_synchronous_and_does_not_retain_evicted_adapters(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+) -> None:
+ import atexit
+ import gc
+ import weakref
+
+ from mlflow.tracing.export import mlflow_v3
+
+ monkeypatch.setenv("MLFLOW_ENABLE_ASYNC_TRACE_LOGGING", "true")
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ raise AssertionError("A private legacy exporter must not retain an SDK queue or exit callback")
+
+ monkeypatch.setattr(mlflow_v3, "AsyncTraceExportQueue", forbidden)
+ monkeypatch.setattr(atexit, "register", forbidden)
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://synchronous.example", experiment_id="77"))
+ reference = weakref.ref(instance)
+ provider_reference = weakref.ref(instance._provider)
+ instance.trace(tool_info())
+ instance.trace(tool_info())
+ assert len(exported_traces(transport)) == 2 # Complete before return, no global flush call.
+ assert os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] == "true"
+ del instance
+ gc.collect()
+ assert reference() is None
+ assert provider_reference() is None
+
+
[email protected]("endpoint", ["/v1/traces", "/api/3.0/mlflow/traces"])
+def test_synchronous_http_failure_reaches_task_boundary_and_does_not_poison_next_trace(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+ endpoint: str,
+) -> None:
+ original_send = requests.Session.send
+ fail = True
+
+ def send(session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:
+ if fail and (request.url or "").endswith(endpoint):
+ response = requests.Response()
+ response.status_code = 403
+ response._content = b'{"error_code":"PERMISSION_DENIED","message":"synthetic denial"}'
+ response.request = request
+ response.url = request.url or ""
+ return response
+ return original_send(session, request)
+
+ monkeypatch.setattr(requests.Session, "send", send)
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://failure-transport.example", experiment_id="70"))
+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):
+ instance.trace(tool_info())
+ assert instance._exporter is not None
+ assert instance._exporter.failed.get() is False
+ fail = False
+ transport.clear()
+ instance.trace(tool_info())
+ assert len(exported_spans(transport)) == len(exported_traces(transport)) == 1
+
+
[email protected]("artifact_fails", [False, True])
+def test_expected_501_falls_back_to_sdk_artifact_and_only_final_failure_is_reported(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+ artifact_fails: bool,
+) -> None:
+ from mlflow.entities import TraceData, TraceInfo
+ from mlflow.tracing.client import TracingClient
+
+ assert transport == []
+ original_send = requests.Session.send
+ uploaded: list[TraceData] = []
+
+ def send(session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:
+ if (request.url or "").endswith("/v1/traces"):
+ response = requests.Response()
+ response.status_code = 501
+ response._content = b'{"error_code":"NOT_IMPLEMENTED","message":"REST OTLP span logging is not supported"}'
+ response.request = request
+ response.url = request.url or ""
+ return response
+ response = original_send(session, request)
+ if (request.url or "").endswith("/api/3.0/mlflow/traces"):
+ body = response.json()
+ body["trace"]["trace_info"]["tags"].pop(TraceTagKey.SPANS_LOCATION)
+ response._content = json.dumps(body).encode()
+ return response
+
+ def upload(_client: TracingClient, _info: TraceInfo, data: TraceData) -> None:
+ if artifact_fails:
+ raise requests.HTTPError("synthetic artifact failure")
+ uploaded.append(data)
+
+ monkeypatch.setattr(requests.Session, "send", send)
+ monkeypatch.setattr(TracingClient, "_upload_trace_data", upload)
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://file-store.example", experiment_id="80"))
+ if artifact_fails:
+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):
+ instance.trace(tool_info())
+ else:
+ instance.trace(tool_info())
+ assert len(uploaded) == 1
+ assert len(uploaded[0].spans) == 1
+ assert uploaded[0].spans[0].outputs == "323"
+
+
[email protected]("fallback_fails", [False, True])
+def test_v3_endpoint_not_found_keeps_real_sdk_v2_fallback(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+ fallback_fails: bool,
+) -> None:
+ from mlflow.entities.trace_info_v2 import TraceInfoV2
+ from mlflow.exceptions import MlflowException
+ from mlflow.protos.databricks_pb2 import ENDPOINT_NOT_FOUND
+ from mlflow.store.tracking.rest_store import RestStore
+
+ assert transport == []
+ original_endpoint = RestStore._call_endpoint
+ actions: list[str] = []
+
+ def endpoint(store: RestStore, api: object, json_body: str, **kwargs: object) -> object:
+ if api is StartTraceV3:
+ actions.append("v3-not-found")
+ raise MlflowException("old server", error_code=ENDPOINT_NOT_FOUND)
+ return original_endpoint(store, api, json_body, **kwargs)
+
+ def start_v2(_store: RestStore, **kwargs: object) -> TraceInfoV2:
+ actions.append("v2-start")
+ if fallback_fails:
+ raise requests.HTTPError("synthetic v2 failure")
+ return TraceInfoV2.from_dict(
+ {
+ "request_id": "tr-v2",
+ "experiment_id": str(kwargs["experiment_id"]),
+ "timestamp_ms": 1,
+ "execution_time_ms": 1,
+ "status": "OK",
+ "request_metadata": {},
+ "tags": {TraceTagKey.SPANS_LOCATION: "TRACKING_STORE"},
+ }
+ )
+
+ def end_v2(_store: RestStore, **_kwargs: object) -> None:
+ actions.append("v2-end")
+
+ monkeypatch.setattr(RestStore, "_call_endpoint", endpoint)
+ monkeypatch.setattr(RestStore, "deprecated_start_trace_v2", start_v2)
+ monkeypatch.setattr(RestStore, "deprecated_end_trace_v2", end_v2)
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://old-v2.example", experiment_id="90"))
+ if fallback_fails:
+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):
+ instance.trace(tool_info())
+ assert actions == ["v3-not-found", "v2-start"]
+ else:
+ instance.trace(tool_info())
+ assert actions == ["v3-not-found", "v2-start", "v2-end"]
+
+
+def test_concurrent_dispatches_on_cached_adapter_do_not_share_failure_state(
+ monkeypatch: pytest.MonkeyPatch,
+ transport: list[requests.PreparedRequest],
+) -> None:
+ from contextvars import ContextVar
+
+ original_send = requests.Session.send
+ fail_this_dispatch: ContextVar[bool] = ContextVar("fail_this_dispatch", default=False)
+ barrier = Barrier(2)
+
+ def send(session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:
+ if (request.url or "").endswith("/v1/traces"):
+ barrier.wait(timeout=5)
+ if fail_this_dispatch.get():
+ response = requests.Response()
+ response.status_code = 403
+ response._content = b'{"error_code":"PERMISSION_DENIED","message":"synthetic denial"}'
+ response.request = request
+ response.url = request.url or ""
+ return response
+ return original_send(session, request)
+
+ monkeypatch.setattr(requests.Session, "send", send)
+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://concurrent.example", experiment_id="71"))
+
+ def emit(fail: bool) -> bool:
+ token = fail_this_dispatch.set(fail)
+ try:
+ if fail:
+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):
+ instance.trace(tool_info())
+ return False
+ instance.trace(tool_info())
+ return True
+ finally:
+ fail_this_dispatch.reset(token)
+ assert instance._exporter is not None
+ assert not instance._exporter.failed.get()
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert list(pool.map(emit, [True, False])) == [False, True]
+ assert len(exported_spans(transport)) == 1
diff --git a/api/tests/unit_tests/services/tracing/test_native_weave_mlflow.py b/api/tests/unit_tests/services/tracing/test_native_weave_mlflow.py
new file mode 100644
index 00000000000..14a7b9964ad
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_native_weave_mlflow.py
@@ -0,0 +1,516 @@
+"""Real SDK schema/serialization boundaries with HTTP transport substituted.
+
+These tests do not claim acceptance by a live third-party account.
+"""
+
+import base64
+import json
+import os
+from collections.abc import Mapping
+from copy import deepcopy
+from datetime import UTC, datetime, timedelta
+from typing import TypedDict, Unpack
+from urllib.parse import parse_qs, urlsplit
+
+import httpx
+import pytest
+from mlflow.entities import Span, TraceData
+from mlflow.tracing.constant import SpanAttributeKey
+from mlflow.tracing.otel.translation import translate_span_when_storing
+from mlflow.types.chat import ChatTool
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+ ExportTraceServiceRequest,
+ ExportTraceServiceResponse,
+)
+from weave.trace_server.trace_server_interface import StartedCallSchemaForInsert
+
+from core.helper import ssrf_proxy
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.normalization import remap_outer_trace
+from services.tracing.provider_dispatch import TraceExportUncertainError, export_trace
+
+
+class TransportOptions(TypedDict, total=False):
+ content: bytes | str
+ headers: Mapping[str, str]
+ auth: tuple[str, str]
+ max_retries: int
+
+
+def trace() -> CanonicalTrace:
+ start = datetime(2026, 1, 1, tzinfo=UTC)
+ root = CanonicalSpan(
+ id="0000000000000001",
+ parent_id=None,
+ name="Agent",
+ kind=CanonicalSpanKind.AGENT,
+ start_time=start,
+ end_time=start + timedelta(seconds=2),
+ inputs={"query": "hello"},
+ outputs="answer",
+ status=CanonicalSpanStatus.OK,
+ metadata={"total_price": 999, "total_tokens": 999},
+ )
+ child = CanonicalSpan(
+ id="0000000000000002",
+ parent_id=root.id,
+ name="Model",
+ kind=CanonicalSpanKind.LLM,
+ start_time=start,
+ end_time=start + timedelta(seconds=1),
+ inputs={"prompt": "hello"},
+ outputs="answer",
+ status=CanonicalSpanStatus.OK,
+ metadata={
+ "model_name": "test-model",
+ "prompt_tokens": 2,
+ "completion_tokens": 3,
+ "total_tokens": 5,
+ "total_price": "0.01",
+ "currency": "USD",
+ },
+ )
+ return CanonicalTrace(
+ trace_id="1234567890abcdef1234567890abcdef", session_id="session-1", root_span_id=root.id, spans=(root, child)
+ )
+
+
+def _response(url: str, *, body: object = None, content: bytes | None = None, status: int = 200) -> httpx.Response:
+ request = httpx.Request("POST", url)
+ if content is not None:
+ return httpx.Response(status, request=request, content=content)
+ return httpx.Response(status, request=request, json=body or {})
+
+
+def weave_started_calls(monkeypatch: pytest.MonkeyPatch) -> list[StartedCallSchemaForInsert]:
+ starts: list[StartedCallSchemaForInsert] = []
+
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ body = json.loads(kwargs["content"])
+ if start := body.get("start"):
+ started = StartedCallSchemaForInsert.model_validate(start)
+ starts.append(started)
+ return _response(url, body={"id": started.id, "trace_id": started.trace_id})
+ return _response(url)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ return starts
+
+
[email protected]("host", ["https://api.wandb.ai", "https://API.WANDB.AI:443"])
+def test_weave_sdk_exports_tree_session_usage_without_global_credentials(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, host: str
+) -> None:
+ calls = []
+ before = dict(os.environ)
+
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ body = json.loads(kwargs["content"])
+ calls.append((_method, url, kwargs, body))
+ start = body.get("start")
+ return _response(url, body={"id": start["id"], "trace_id": start["trace_id"]} if start else {})
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ result = export_trace("weave", {"api_key": "tenant-key", "entity": "team", "project": "demo", "host": host}, trace)
+ assert len(calls) == 4
+ root, child = calls[0][3]["start"], calls[1][3]["start"]
+ assert child["parent_id"] == root["id"]
+ assert child["thread_id"] == root["thread_id"] == "session-1"
+ assert child["turn_id"] == root["turn_id"] == root["id"]
+ assert calls[2][3]["end"]["summary"]["usage"]["test-model"]["total_tokens"] == 5
+ assert calls[3][3]["end"]["summary"] == {}
+ assert result.trace_url == f"https://wandb.ai/team/demo/r/call/{root['id']}"
+ assert result.span_urls[trace.spans[1].id].endswith(child["id"])
+ assert all(call[2]["auth"] == ("api", "tenant-key") and call[2]["max_retries"] == 0 for call in calls)
+ assert dict(os.environ) == before
+
+
[email protected]("root_kind", [CanonicalSpanKind.CHAIN, CanonicalSpanKind.AGENT])
+def test_weave_nested_agent_calls_form_one_outer_turn(
+ trace: CanonicalTrace, weave_started_calls: list[StartedCallSchemaForInsert], root_kind: CanonicalSpanKind
+) -> None:
+ root = trace.spans[0].model_copy(update={"kind": root_kind})
+ agent = trace.spans[1].model_copy(update={"kind": CanonicalSpanKind.AGENT})
+ model = trace.spans[1].model_copy(update={"id": "0000000000000003", "parent_id": agent.id})
+ nested = trace.model_copy(update={"spans": (root, agent, model)})
+
+ export_trace("weave", {"api_key": "tenant-key", "entity": "team", "project": "demo"}, nested)
+
+ outer, inner_agent, inner_model = weave_started_calls
+ assert inner_agent.parent_id == outer.id
+ assert inner_model.parent_id == inner_agent.id
+ assert all(call.thread_id == trace.session_id and call.turn_id == outer.id for call in weave_started_calls)
+ # Native Threads queries count only call.id == call.turn_id, including for
+ # Workflow and single-step preview outer roots with nested Agent details.
+ assert [call.id for call in weave_started_calls if call.id == call.turn_id] == [outer.id]
+
+
+def test_weave_same_session_has_distinct_stable_turns(
+ trace: CanonicalTrace, weave_started_calls: list[StartedCallSchemaForInsert]
+) -> None:
+ config = {"api_key": "tenant-key", "entity": "team", "project": "demo"}
+ first = export_trace("weave", config, trace)
+ repeated = export_trace("weave", config, trace)
+ next_trace = trace.model_copy(update={"trace_id": "abcdef1234567890abcdef1234567890"})
+ second = export_trace("weave", config, next_trace)
+
+ first_calls, repeated_calls, second_calls = (
+ weave_started_calls[:2],
+ weave_started_calls[2:4],
+ weave_started_calls[4:],
+ )
+ assert repeated_calls == first_calls
+ assert repeated == first
+ assert second.trace_url != first.trace_url
+ assert all(call.thread_id == trace.session_id for call in weave_started_calls)
+ assert first_calls[0].turn_id == first_calls[0].id
+ assert second_calls[0].turn_id == second_calls[0].id
+ assert second_calls[0].turn_id != first_calls[0].turn_id
+ assert all(call.turn_id == first_calls[0].id for call in first_calls)
+ assert all(call.turn_id == second_calls[0].id for call in second_calls)
+
+
+def test_weave_without_session_does_not_fabricate_a_thread_or_turn(
+ trace: CanonicalTrace, weave_started_calls: list[StartedCallSchemaForInsert]
+) -> None:
+ trace = trace.model_copy(update={"session_id": ""})
+
+ export_trace("weave", {"api_key": "tenant-key", "entity": "team", "project": "demo"}, trace)
+
+ assert len(weave_started_calls) == len(trace.spans)
+ assert all(call.thread_id is None and call.turn_id is None for call in weave_started_calls)
+
+
+def test_weave_resolves_optional_entity_using_only_supplied_credentials(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace
+) -> None:
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ assert kwargs["auth"] == ("api", "tenant-key")
+ if url.endswith("/graphql"):
+ return _response(url, body={"data": {"viewer": {"entity": "default-team"}}})
+ body = json.loads(kwargs["content"])
+ if start := body.get("start"):
+ assert start["project_id"] == "default-team/demo"
+ return _response(url, body={"id": start["id"], "trace_id": start["trace_id"]})
+ return _response(url)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ result = export_trace("weave", {"api_key": "tenant-key", "project": "demo"}, trace)
+ assert result.trace_url is not None
+ assert "/default-team/demo/" in result.trace_url
+
+
+def test_weave_partial_tree_is_uncertain_and_not_retried(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace
+) -> None:
+ calls = []
+
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ calls.append(url)
+ if len(calls) > 1:
+ return _response(url, status=503)
+ start = json.loads(kwargs["content"])["start"]
+ return _response(url, body={"id": start["id"], "trace_id": start["trace_id"]})
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ with pytest.raises(TraceExportUncertainError, match="partially accepted"):
+ export_trace("weave", {"api_key": "secret", "entity": "team", "project": "demo"}, trace)
+ assert len(calls) == 2
+
+
[email protected]("tracking_uri", ["https://mlflow.example", "https://mlflow.example/proxy/mlflow"])
+def test_mlflow_sdk_span_roundtrip_and_credential_isolation(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, tracking_uri: str
+) -> None:
+ calls = []
+ before = dict(os.environ)
+
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ calls.append((url, kwargs))
+ return _response(url, content=b"")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ for username in ("tenant-a", "tenant-b"):
+ result = export_trace(
+ "mlflow",
+ {
+ "tracking_uri": tracking_uri,
+ "experiment_id": "42",
+ "username": username,
+ "password": "password",
+ },
+ trace,
+ )
+ assert result.trace_id == "tr-" + trace.trace_id
+ assert result.trace_url is not None
+ fragment_path, _, query = urlsplit(result.trace_url).fragment.partition("?")
+ assert result.trace_url.startswith(tracking_uri + "/#/")
+ assert fragment_path == "/experiments/42/traces"
+ assert parse_qs(query) == {
+ "traceId": [result.trace_id],
+ "selectedTraceId": [result.trace_id],
+ "selectedEvaluationId": [result.trace_id],
+ }
+ for index, (url, options) in enumerate(calls):
+ assert url == tracking_uri + "/v1/traces"
+ assert options["headers"]["x-mlflow-experiment-id"] == "42"
+ assert (
+ options["headers"]["Authorization"]
+ == "Basic " + base64.b64encode(f"tenant-{'a' if index == 0 else 'b'}:password".encode()).decode()
+ )
+ content = options["content"]
+ assert isinstance(content, bytes)
+ payload = ExportTraceServiceRequest.FromString(content)
+ spans = [Span.from_otel_proto(span) for span in payload.resource_spans[0].scope_spans[0].spans]
+ assert spans[1].parent_id == spans[0].span_id
+ assert spans[1].get_attribute(SpanAttributeKey.CHAT_USAGE) == {
+ "input_tokens": 2,
+ "output_tokens": 3,
+ "total_tokens": 5,
+ }
+ assert spans[1].get_attribute(SpanAttributeKey.LLM_COST) == {"total_cost": 0.01}
+ assert spans[0].get_attribute(SpanAttributeKey.CHAT_USAGE) is None
+ assert spans[0].get_attribute(SpanAttributeKey.SESSION_ID) == "session-1"
+ assert dict(os.environ) == before
+
+
+def _tool_definitions() -> list[dict[str, object]]:
+ return [
+ {
+ "type": "function",
+ "name": "calculate",
+ "description": "TOOL_CONTENT_CANARY",
+ "parameters": {
+ "type": "object",
+ "properties": {"value": {"type": "integer"}},
+ "required": ["value"],
+ "additionalProperties": False,
+ "$comment": "Preserve schema extension fields",
+ },
+ "strict": True,
+ },
+ {"type": "function", "function": {"name": "already_nested", "description": "Nested definition"}},
+ ]
+
+
[email protected]("serialized", [False, True])
+def test_mlflow_tool_definitions_have_native_chat_schema_without_losing_source(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, serialized: bool
+) -> None:
+ definitions = _tool_definitions()
+ source = json.dumps(definitions) if serialized else definitions
+ trace.spans[1].metadata["gen_ai.tool.definitions"] = source
+ before = deepcopy(trace.model_dump())
+ native: list[Span] = []
+
+ def send(_method: str, url: str, **options: Unpack[TransportOptions]) -> httpx.Response:
+ content = options["content"]
+ assert isinstance(content, bytes)
+ payload = ExportTraceServiceRequest.FromString(content)
+ native.extend(Span.from_otel_proto(item) for item in payload.resource_spans[0].scope_spans[0].spans)
+ return _response(url, content=b"")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ export_trace("mlflow", {"tracking_uri": "https://mlflow.example", "experiment_id": "42"}, trace)
+ tools = native[1].get_attribute(SpanAttributeKey.CHAT_TOOLS)
+ assert tools == [
+ {"type": "function", "function": {key: value for key, value in definitions[0].items() if key != "type"}},
+ definitions[1],
+ ]
+ assert isinstance(tools, list)
+ for tool in tools:
+ ChatTool.model_validate(tool, strict=True)
+ assert native[1].get_attribute("gen_ai.tool.definitions") == source
+ stored = translate_span_when_storing(native[1])
+ assert json.loads(stored["attributes"][SpanAttributeKey.CHAT_TOOLS]) == tools
+ assert trace.model_dump() == before
+
+
+def test_mlflow_redacted_trace_does_not_recreate_tool_definition_content(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace
+) -> None:
+ trace.spans[1].metadata["gen_ai.tool.definitions"] = _tool_definitions()
+ trace.spans[1].metadata[SpanAttributeKey.CHAT_TOOLS] = [
+ {"type": "function", "function": {"name": "TOOL_CONTENT_CANARY"}}
+ ]
+ redacted = remap_outer_trace(trace, trace_id=trace.trace_id, capture_content=False, agent_message=False)
+ native: list[Span] = []
+
+ def send(_method: str, url: str, **options: Unpack[TransportOptions]) -> httpx.Response:
+ content = options["content"]
+ assert isinstance(content, bytes)
+ assert b"TOOL_CONTENT_CANARY" not in content
+ payload = ExportTraceServiceRequest.FromString(content)
+ native.extend(Span.from_otel_proto(item) for item in payload.resource_spans[0].scope_spans[0].spans)
+ return _response(url, content=b"")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ export_trace("mlflow", {"tracking_uri": "https://mlflow.example", "experiment_id": "42"}, redacted)
+ for span in native:
+ assert span.get_attribute(SpanAttributeKey.CHAT_TOOLS) is None
+ assert span.get_attribute("gen_ai.tool.definitions") is None
+ assert span.inputs is None
+ assert span.outputs is None
+
+
[email protected]("definitions", [[None, 12, {"type": "other"}, {"type": "function"}], "invalid-json"])
+def test_mlflow_unsupported_tool_shapes_remain_in_metadata_without_fake_functions(
+ trace: CanonicalTrace, definitions: object
+) -> None:
+ from dify_trace_mlflow.agent_trace import _spans
+
+ trace.spans[1].metadata["gen_ai.tool.definitions"] = definitions
+ before = deepcopy(trace.model_dump())
+ result = _spans(trace, "42", "tr-" + trace.trace_id)
+ assert result[1].get_attribute("gen_ai.tool.definitions") == definitions
+ assert result[1].get_attribute(SpanAttributeKey.CHAT_TOOLS) == ([] if isinstance(definitions, list) else None)
+ assert trace.model_dump() == before
+
+
+def test_mlflow_existing_native_chat_tools_take_precedence(trace: CanonicalTrace) -> None:
+ from dify_trace_mlflow.agent_trace import _spans
+
+ native_tools = [{"type": "function", "function": {"name": "already_native"}}]
+ trace.spans[1].metadata[SpanAttributeKey.CHAT_TOOLS] = native_tools
+ trace.spans[1].metadata["gen_ai.tool.definitions"] = _tool_definitions()
+ result = _spans(trace, "42", "tr-" + trace.trace_id)
+ assert result[1].get_attribute(SpanAttributeKey.CHAT_TOOLS) == native_tools
+
+
+def test_mlflow_preserves_json_schema_beyond_the_sdk_chat_tool_model(trace: CanonicalTrace) -> None:
+ from dify_trace_mlflow.agent_trace import _spans
+
+ parameters = {
+ "$ref": "#/$defs/Arguments",
+ "$defs": {
+ "Arguments": {
+ "type": "object",
+ "properties": {"choice": {"enum": [None, {"mode": "auto"}]}},
+ "additionalProperties": {"type": "string"},
+ }
+ },
+ }
+ source = [{"type": "function", "name": "schema_extension", "parameters": parameters}]
+ trace.spans[1].metadata["gen_ai.tool.definitions"] = source
+ result = _spans(trace, "42", "tr-" + trace.trace_id)
+ expected = [{"type": "function", "function": {"name": "schema_extension", "parameters": parameters}}]
+ assert result[1].get_attribute(SpanAttributeKey.CHAT_TOOLS) == expected
+ assert result[1].get_attribute("gen_ai.tool.definitions") == source
+ stored = translate_span_when_storing(result[1])
+ assert json.loads(stored["attributes"][SpanAttributeKey.CHAT_TOOLS]) == expected
+
+
+def test_mlflow_partial_otlp_ack_is_uncertain(monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace) -> None:
+ ack = ExportTraceServiceResponse()
+ ack.partial_success.rejected_spans = 1
+ monkeypatch.setattr(
+ ssrf_proxy, "make_request", lambda _method, url, **_kwargs: _response(url, content=ack.SerializeToString())
+ )
+ with pytest.raises(TraceExportUncertainError, match="partially accepted"):
+ export_trace("mlflow", {"tracking_uri": "https://mlflow.example", "experiment_id": "42"}, trace)
+
+
+ "credential_type", ["AWS_PRESIGNED_URL", "GCP_SIGNED_URL", "AZURE_SAS_URI", "AZURE_ADLS_GEN2_SAS_URI"]
+)
[email protected]("oauth", [False, True])
+def test_databricks_sdk_v3_metadata_and_native_artifact(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, credential_type: str, oauth: bool
+) -> None:
+ calls: list[tuple[str, str, TransportOptions]] = []
+ uploaded: list[TraceData] = []
+ before = dict(os.environ)
+ server_trace_id = "tr-abcdef0123456789abcdef0123456789"
+ definitions = _tool_definitions()
+ trace.spans[1].metadata["gen_ai.tool.definitions"] = definitions
+
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ calls.append((_method, url, kwargs))
+ if url.endswith("/oidc/v1/token"):
+ assert kwargs["auth"] == ("client", "client-secret")
+ return _response(url, body={"access_token": "oauth-token"})
+ if url.endswith("/api/3.0/mlflow/traces"):
+ assert kwargs["headers"]["Authorization"] == ("Bearer oauth-token" if oauth else "Bearer pat-token")
+ body = json.loads(kwargs["content"])
+ assert body["trace"]["trace_info"]["trace_metadata"]["mlflow.trace.session"] == "session-1"
+ body["trace"]["trace_info"]["trace_id"] = server_trace_id
+ return _response(url, body=body)
+ if url.endswith("/credentials-for-data-upload"):
+ assert f"/traces/{server_trace_id}/" in url
+ return _response(
+ url,
+ body={
+ "credential_info": {
+ "signed_uri": "https://storage.example/trace?sig=fake",
+ "type": credential_type,
+ "headers": [],
+ }
+ },
+ )
+ assert "Authorization" not in kwargs["headers"]
+ if kwargs.get("content"):
+ uploaded.append(TraceData.from_dict(json.loads(kwargs["content"])))
+ return _response(url)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ config = {"host": "https://databricks.example", "experiment_id": "42"}
+ config.update(
+ {"client_id": "client", "client_secret": "client-secret"} if oauth else {"personal_access_token": "pat-token"}
+ )
+ result = export_trace("databricks", config, trace)
+ assert result.trace_id == server_trace_id
+ assert result.trace_url == (
+ f"https://databricks.example/ml/experiments/42/traces?selectedEvaluationId={server_trace_id}"
+ )
+ data = uploaded[0]
+ assert len(data.spans) == 2
+ assert all(span.trace_id == server_trace_id for span in data.spans)
+ assert data.spans[1].parent_id == data.spans[0].span_id
+ usage = data.spans[1].get_attribute(SpanAttributeKey.CHAT_USAGE)
+ assert usage is not None
+ assert usage["total_tokens"] == 5
+ assert data.spans[1].get_attribute(SpanAttributeKey.LLM_COST) == {"total_cost": 0.01}
+ tools = data.spans[1].get_attribute(SpanAttributeKey.CHAT_TOOLS)
+ assert isinstance(tools, list)
+ assert len(tools) == 2
+ for tool in tools:
+ ChatTool.model_validate(tool, strict=True)
+ assert data.spans[1].get_attribute("gen_ai.tool.definitions") == definitions
+ assert dict(os.environ) == before
+ assert len(calls) == 3 + int(oauth) + int(credential_type == "AZURE_ADLS_GEN2_SAS_URI")
+
+
+def test_databricks_partial_metadata_delivery_is_uncertain(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace
+) -> None:
+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:
+ if url.endswith("/api/3.0/mlflow/traces"):
+ return _response(url, body=json.loads(kwargs["content"]))
+ return _response(url, status=503)
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ with pytest.raises(TraceExportUncertainError, match="metadata"):
+ export_trace(
+ "databricks",
+ {"host": "https://databricks.example", "experiment_id": "42", "personal_access_token": "secret"},
+ trace,
+ )
+
+
[email protected]("provider", ["weave", "databricks"])
+def test_success_status_with_unreadable_ack_is_uncertain(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, provider: str
+) -> None:
+ monkeypatch.setattr(ssrf_proxy, "make_request", lambda _method, url, **_kwargs: _response(url, content=b"not-json"))
+ config = {
+ "api_key": "secret",
+ "entity": "team",
+ "project": "demo",
+ "host": "https://example.com",
+ "experiment_id": "42",
+ "personal_access_token": "secret",
+ }
+ with pytest.raises(TraceExportUncertainError, match="unreadable"):
+ export_trace(provider, config, trace)
diff --git a/api/tests/unit_tests/services/tracing/test_nullable_tracing_credentials.py b/api/tests/unit_tests/services/tracing/test_nullable_tracing_credentials.py
new file mode 100644
index 00000000000..fa8d4215b31
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_nullable_tracing_credentials.py
@@ -0,0 +1,92 @@
+"""Nullable provider credentials never reach RSA/base64 as non-string tokens."""
+
+from uuid import uuid4
+
+import pytest
+from Crypto.PublicKey import RSA
+from pydantic import ValidationError
+from sqlalchemy.orm import Session
+
+import models.engine as model_engine
+from core.ops.ops_trace_manager import OpsTraceManager
+from extensions.ext_key_provider import key_provider_manager
+from libs import gmpy2_pkcs10aep_cipher, rsa
+from libs.key_providers.rsa_key_provider import RSAKeyProvider
+from tests.unit_tests.model_factories import make_tenant
+
+
+def workspace(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> str:
+ key = RSA.generate(2048)
+ tenant_id = str(uuid4())
+ sqlite_session.add(make_tenant(tenant_id=tenant_id, encrypt_public_key=key.publickey().export_key().decode()))
+ sqlite_session.commit()
+ monkeypatch.setattr(model_engine.db, "session", sqlite_session)
+ monkeypatch.setattr(key_provider_manager, "_provider", RSAKeyProvider())
+ monkeypatch.setattr(rsa, "get_decrypt_decoding", lambda _tenant: (key, gmpy2_pkcs10aep_cipher.new(key)))
+ return tenant_id
+
+
+def test_databricks_switches_auth_with_real_encryption_and_explicit_null(workspace: str) -> None:
+ pat = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "databricks",
+ {
+ "host": "https://workspace.test",
+ "experiment_id": "12",
+ "personal_access_token": "synthetic-pat",
+ "client_id": None,
+ "client_secret": None,
+ },
+ )
+ assert pat["personal_access_token"] != "synthetic-pat"
+ decrypted_pat = OpsTraceManager.decrypt_tracing_config(workspace, "databricks", pat)
+ assert decrypted_pat["personal_access_token"] == "synthetic-pat"
+ assert decrypted_pat["client_secret"] is None
+
+ oauth = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "databricks",
+ {
+ "host": "https://workspace.test",
+ "experiment_id": "12",
+ "personal_access_token": None,
+ "client_id": "synthetic-client",
+ "client_secret": "synthetic-secret",
+ },
+ pat,
+ )
+ assert oauth["personal_access_token"] is None
+ assert oauth["client_secret"] != "synthetic-secret"
+ decrypted_oauth = OpsTraceManager.decrypt_tracing_config(workspace, "databricks", oauth)
+ assert decrypted_oauth["personal_access_token"] is None
+ assert decrypted_oauth["client_secret"] == "synthetic-secret"
+
+
+def test_other_optional_secret_model_can_be_cleared_without_losing_validation(workspace: str) -> None:
+ password = OpsTraceManager.encrypt_tracing_config(
+ workspace,
+ "mlflow",
+ {
+ "tracking_uri": "https://mlflow.test",
+ "experiment_id": "0",
+ "username": "user",
+ "password": "synthetic-password",
+ },
+ )
+ assert OpsTraceManager.decrypt_tracing_config(workspace, "mlflow", password)["password"] == "synthetic-password"
+ cleared = OpsTraceManager.encrypt_tracing_config(workspace, "mlflow", {**password, "password": None}, password)
+ assert cleared["password"] is None
+ assert OpsTraceManager.decrypt_tracing_config(workspace, "mlflow", cleared)["password"] is None
+
+
+def test_non_nullable_secret_is_still_rejected_by_provider_model(workspace: str) -> None:
+ with pytest.raises(ValidationError, match="secret_key"):
+ OpsTraceManager.encrypt_tracing_config(
+ workspace, "langfuse", {"public_key": "synthetic-public", "secret_key": None}
+ )
+ encrypted = OpsTraceManager.encrypt_tracing_config(
+ workspace, "langfuse", {"public_key": "synthetic-public", "secret_key": "synthetic-secret"}
+ )
+ with pytest.raises(ValidationError, match="secret_key"):
+ OpsTraceManager.decrypt_tracing_config(workspace, "langfuse", {**encrypted, "secret_key": None})
diff --git a/api/tests/unit_tests/services/tracing/test_opik_configuration.py b/api/tests/unit_tests/services/tracing/test_opik_configuration.py
new file mode 100644
index 00000000000..073b5bb8233
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_opik_configuration.py
@@ -0,0 +1,188 @@
+"""Opik auth contracts at the read-only HTTP boundary, without SDK streamers."""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from urllib.parse import parse_qs, urlsplit
+
+import httpx
+import pytest
+from dify_trace_opik.config import OpikConfig
+from dify_trace_opik.configuration import OpikConfigurationError, console_url, project_url, verify_configuration
+
+from core.helper import ssrf_proxy
+
+
+@dataclass
+class AccessAPI:
+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)
+ default_workspace: object = "resolved-team"
+ override: Callable[[str, str, dict[str, object]], httpx.Response] | None = None
+
+ def request(self, method: str, url: str, **options: object) -> httpx.Response:
+ self.calls.append((method, url, options))
+ if self.override:
+ return self.override(method, url, options)
+ request = httpx.Request(method, url)
+ if method == "POST":
+ assert url.endswith("/v1/private/auth"), "configuration must never create projects, spans or traces"
+ assert options["json"] == {}
+ return httpx.Response(204, request=request)
+ assert method == "GET"
+ assert url.endswith("/v1/private/auth/workspace")
+ return httpx.Response(200, json={"workspace_name": self.default_workspace}, request=request)
+
+
+def access_api(monkeypatch: pytest.MonkeyPatch) -> AccessAPI:
+ api = AccessAPI()
+ monkeypatch.setattr(ssrf_proxy, "make_request", api.request)
+ return api
+
+
+def test_named_workspaces_use_isolated_auth_without_default_lookup(access_api: AccessAPI) -> None:
+ for host, key, workspace in [
+ ("https://first.test/opik/api/", "first-key", "first-team"),
+ ("https://second.test/api/", "second-key", "second-team"),
+ ]:
+ result = verify_configuration(OpikConfig(api_key=key, workspace=workspace, project="not-created", url=host))
+ assert (result.url, result.workspace, result.project) == (host, workspace, "not-created")
+ method, url, options = access_api.calls[-1]
+ assert method == "POST"
+ assert url == host + "v1/private/auth"
+ assert options["headers"] == {
+ "Authorization": key,
+ "Comet-Workspace": workspace,
+ "Accept": "application/json",
+ }
+ assert options["json"] == {}
+ assert options["max_retries"] == 0
+ assert options["follow_redirects"] is False
+ assert options["timeout"] == 30.0
+ assert len(access_api.calls) == 2
+
+
[email protected]("workspace", [None, "", "default", " default "])
+def test_default_workspace_is_resolved_only_during_verification(access_api: AccessAPI, workspace: str | None) -> None:
+ result = verify_configuration(OpikConfig(api_key="synthetic-key", workspace=workspace))
+ assert result.workspace == "resolved-team"
+ assert result.project == "Default Project"
+ assert [method for method, _, _ in access_api.calls] == ["POST", "GET"]
+ assert access_api.calls[1][2]["headers"] == access_api.calls[0][2]["headers"]
+
+
[email protected]("key", [None, ""])
+def test_keyless_self_host_does_not_fall_back_to_environment(
+ access_api: AccessAPI, monkeypatch: pytest.MonkeyPatch, key: str | None
+) -> None:
+ monkeypatch.setenv("OPIK_API_KEY", "unrelated-global-key")
+ monkeypatch.setenv("OPIK_WORKSPACE", "unrelated-global-team")
+ monkeypatch.setenv("OPIK_URL_OVERRIDE", "https://unrelated.test/api/")
+ result = verify_configuration(OpikConfig(api_key=key, workspace="local", url="http://selfhost.test:5173/api/"))
+ assert result.url == "http://selfhost.test:5173/api/"
+ assert access_api.calls[0][2]["headers"] == {"Comet-Workspace": "local", "Accept": "application/json"}
+ assert len(access_api.calls) == 1
+
+
+ "url",
+ [
+ "https://user:[email protected]/api/",
+ "https://private.test:invalid/api/",
+ "http:///api/",
+ "https://private.test/api/?next=/api/",
+ "https://private.test/api/#/api/",
+ "https://private.test/with space/api/",
+ "https://private.test/back\\slash/api/",
+ ],
+)
+def test_invalid_destination_is_rejected_before_sending_credentials(access_api: AccessAPI, url: str) -> None:
+ with pytest.raises(OpikConfigurationError) as error:
+ verify_configuration(OpikConfig(api_key="private-key", url=url))
+ assert error.value.code == "invalid_endpoint"
+ assert access_api.calls == []
+ assert "private-key" not in str(error.value)
+
+
+ ("status", "code"),
+ [
+ (401, "authentication_failed"),
+ (403, "permission_denied"),
+ (302, "endpoint_unavailable"),
+ (500, "endpoint_unavailable"),
+ ],
+)
+def test_auth_failures_are_sanitized(access_api: AccessAPI, status: int, code: str) -> None:
+ access_api.override = lambda method, url, _options: httpx.Response(
+ status, text="private-vendor-error", request=httpx.Request(method, url)
+ )
+ with pytest.raises(OpikConfigurationError) as error:
+ verify_configuration(OpikConfig(api_key="private-key", workspace="team"))
+ assert error.value.code == code
+ assert "private" not in str(error.value)
+ assert len(access_api.calls) == 1
+
+
+def test_auth_html_success_is_not_a_verified_workspace(access_api: AccessAPI) -> None:
+ access_api.override = lambda method, url, _options: httpx.Response(
+ 200, text="<html>Login</html>", request=httpx.Request(method, url)
+ )
+ with pytest.raises(OpikConfigurationError, match="invalid response"):
+ verify_configuration(OpikConfig(workspace="team"))
+
+
[email protected]("workspace", [None, "", " ", 123, []])
+def test_default_workspace_response_requires_a_name(access_api: AccessAPI, workspace: object) -> None:
+ access_api.default_workspace = workspace
+ with pytest.raises(OpikConfigurationError) as error:
+ verify_configuration(OpikConfig())
+ assert error.value.code == "malformed_response"
+
+
+def test_connection_error_never_exposes_request_or_key(access_api: AccessAPI) -> None:
+ def fail(method: str, url: str, _options: dict[str, object]) -> httpx.Response:
+ raise httpx.ReadTimeout("private-key in vendor detail", request=httpx.Request(method, url))
+
+ access_api.override = fail
+ with pytest.raises(OpikConfigurationError) as error:
+ verify_configuration(OpikConfig(api_key="private-key"))
+ assert error.value.code == "connection_failed"
+ assert "private-key" not in str(error.value)
+
+
+def test_project_url_uses_configured_host_and_encodes_names_without_network(access_api: AccessAPI) -> None:
+ config = OpikConfig(url="https://private.test:7443/prefix/api/", workspace="team/name", project="A & B/? 中文")
+ url = project_url(config)
+ assert url is not None
+ parsed = urlsplit(url)
+ assert parsed.netloc == "private.test:7443"
+ assert parsed.path == "/prefix/team%2Fname/redirect/projects"
+ assert parse_qs(parsed.query) == {"name": ["A & B/? 中文"]}
+ assert access_api.calls == []
+
+
[email protected]("workspace", [None, "", "default"])
+def test_historical_default_workspace_get_has_a_pure_console_fallback(
+ access_api: AccessAPI, workspace: str | None
+) -> None:
+ config = OpikConfig(url="http://selfhost.test:5173/prefix/api/", workspace=workspace)
+ assert project_url(config) is None
+ assert console_url(config) == "http://selfhost.test:5173/prefix/"
+ assert access_api.calls == []
+
+
+ ("workspace", "project", "code"),
+ [
+ ("..", "ok", "invalid_workspace"),
+ ("bad\nworkspace", "ok", "invalid_workspace"),
+ ("team", "bad\nproject", "invalid_project"),
+ ],
+)
+def test_invalid_names_cannot_escape_the_console_path_or_headers(
+ access_api: AccessAPI, workspace: str, project: str, code: str
+) -> None:
+ with pytest.raises(OpikConfigurationError) as error:
+ verify_configuration(OpikConfig(workspace=workspace, project=project))
+ assert error.value.code == code
+ assert access_api.calls == []
diff --git a/api/tests/unit_tests/services/tracing/test_opik_legacy_identifiers.py b/api/tests/unit_tests/services/tracing/test_opik_legacy_identifiers.py
new file mode 100644
index 00000000000..af48f2234e9
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_opik_legacy_identifiers.py
@@ -0,0 +1,214 @@
+"""Legacy external IDs retain stable SDK batches and workflow parent relations."""
+
+import json
+from datetime import datetime, timedelta
+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
+
+import pytest
+from dify_trace_opik.opik_trace import OpikDataTrace, prepare_opik_uuid
+from opik import Opik
+from opik.config import OpikConfig as SDKConfig
+from opik.id_helpers import uuid4_to_uuid7
+from opik.message_processing.batching.batchers import CreateSpanMessageBatcher, CreateTraceMessageBatcher
+from opik.message_processing.messages import (
+ BaseMessage,
+ CreateSpanMessage,
+ CreateSpansBatchMessage,
+ CreateTraceBatchMessage,
+ CreateTraceMessage,
+)
+from opik.message_processing.streamer import Streamer
+from sqlalchemy import Engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.entities.trace_entity import WorkflowTraceInfo
+from extensions.ext_database import db
+from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (
+ _dict_to_workflow_node_execution_model,
+)
+from models.account import TenantAccountJoin, TenantAccountRole
+from tests.unit_tests.model_factories import make_account, make_app, make_tenant
+
+START = datetime(2026, 10, 16, 15, 18, 10, 792586)
+WORKFLOW_ID = "11111111-2222-4333-8444-555555555555"
+EXTERNAL_IDS = [
+ "0123456789abcdef0123456789abcdef",
+ "019140fb-6972-7d74-a219-6cba9eb543b1",
+ str(uuid5(NAMESPACE_URL, "external-trace")),
+ "external-trace_correlation",
+]
+
+
[email protected]("identifier", [WORKFLOW_ID, WORKFLOW_ID.upper(), UUID(WORKFLOW_ID).hex])
+def test_valid_uuid4_keeps_the_original_sdk_mapping(identifier: str) -> None:
+ assert prepare_opik_uuid(START, identifier) == uuid4_to_uuid7(START, identifier)
+
+
[email protected]("identifier", EXTERNAL_IDS)
+def test_external_identifiers_become_stable_uuid7_with_unchanged_time(identifier: str) -> None:
+ mapped = prepare_opik_uuid(START, identifier)
+ assert mapped.version == 7
+ assert mapped == prepare_opik_uuid(START, identifier)
+ assert int.from_bytes(mapped.bytes[:6]) == int(START.timestamp() * 1000)
+ assert mapped != prepare_opik_uuid(START, identifier + "-different")
+
+
+def test_missing_identifier_keeps_random_fallback() -> None:
+ first, second = prepare_opik_uuid(START, None), prepare_opik_uuid(START, None)
+ assert first.version == second.version == 7
+ assert first != second
+ assert int.from_bytes(first.bytes[:6]) == int(START.timestamp() * 1000)
+ assert prepare_opik_uuid(None, WORKFLOW_ID).version == 7
+
+
[email protected]("external_id", [*EXTERNAL_IDS, WORKFLOW_ID, None])
+def test_workflow_external_ids_reach_real_sdk_batches_with_complete_owned_tree(
+ external_id: str | None,
+ sqlite_engine: Engine,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ tenant_id, app_id, account_id, definition_id = (str(uuid4()) for _ in range(4))
+ node_ids = [str(uuid4()) for _ in range(3)]
+ with sqlite_session_factory.begin() as session:
+ session.add(make_tenant(tenant_id=tenant_id))
+ session.add(make_account(account_id=account_id))
+ session.add(
+ TenantAccountJoin(tenant_id=tenant_id, account_id=account_id, role=TenantAccountRole.OWNER, current=True)
+ )
+ session.add(make_app(app_id=app_id, tenant_id=tenant_id, created_by=account_id))
+ for index, kind in enumerate(("start", "llm", "end")):
+ usage = {"prompt_tokens": 27, "completion_tokens": 5, "total_tokens": 32}
+ process: dict[str, object] = (
+ {
+ "model_mode": "chat",
+ "model_name": "test-model",
+ "model_provider": "provider/test",
+ "usage": usage,
+ "prompts": [{"role": "user", "content": "synthetic"}],
+ }
+ if kind == "llm"
+ else {}
+ )
+ session.add(
+ _dict_to_workflow_node_execution_model(
+ {
+ "id": node_ids[index],
+ "tenant_id": tenant_id,
+ "app_id": app_id,
+ "workflow_id": definition_id,
+ "workflow_run_id": WORKFLOW_ID,
+ "node_id": kind,
+ "node_type": kind,
+ "title": kind,
+ "created_by": account_id,
+ "created_at": (START + timedelta(milliseconds=index * 100)).isoformat(),
+ "status": "succeeded",
+ "index": index,
+ "elapsed_time": 0.1,
+ "inputs": json.dumps({}),
+ "outputs": json.dumps({"answer": "synthetic"}),
+ "process_data": json.dumps(process),
+ "execution_metadata": json.dumps({"total_tokens": 32} if kind == "llm" else {}),
+ }
+ )
+ )
+ # A node from another run must not enter this trace's SDK batch.
+ session.add(
+ _dict_to_workflow_node_execution_model(
+ {
+ "id": str(uuid4()),
+ "tenant_id": tenant_id,
+ "app_id": app_id,
+ "workflow_id": definition_id,
+ "workflow_run_id": str(uuid4()),
+ "node_id": "foreign",
+ "node_type": "llm",
+ "title": "foreign",
+ "created_by": account_id,
+ "created_at": START.isoformat(),
+ "status": "succeeded",
+ "elapsed_time": 0.1,
+ }
+ )
+ )
+
+ # Bind the real SQLAlchemy owner to the test engine; no Session/repository double.
+ monkeypatch.setattr(type(db), "engine", property(lambda _owner: sqlite_engine))
+ queued: list[BaseMessage] = []
+ streamer = object.__new__(Streamer)
+ monkeypatch.setattr(streamer, "put", queued.append)
+ client = object.__new__(Opik)
+ client._streamer = streamer
+ client._config = SDKConfig.model_construct(url_override="https://offline.invalid/api", project_name="test-project")
+ client._project_name = "test-project"
+ client._project_name_most_recent_trace = None
+ instance = object.__new__(OpikDataTrace)
+ instance.opik_client = client
+ instance.project = "test-project"
+ info = WorkflowTraceInfo(
+ workflow_id=definition_id,
+ tenant_id=tenant_id,
+ workflow_run_id=WORKFLOW_ID,
+ workflow_run_elapsed_time=1,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={"query": "synthetic"},
+ workflow_run_outputs={"answer": "synthetic"},
+ workflow_run_version="draft",
+ total_tokens=32,
+ file_list=[],
+ query="synthetic",
+ metadata={"app_id": app_id},
+ start_time=START,
+ end_time=START + timedelta(seconds=1),
+ trace_id=external_id,
+ )
+ instance.workflow_trace(info)
+ trace_batches: list[BaseMessage] = []
+ span_batches: list[BaseMessage] = []
+ trace_batcher = CreateTraceMessageBatcher(trace_batches.append, 1000, 2)
+ span_batcher = CreateSpanMessageBatcher(span_batches.append, 1000, 2)
+ for message in queued:
+ if isinstance(message, CreateTraceMessage):
+ trace_batcher.add(message)
+ elif isinstance(message, CreateSpanMessage):
+ span_batcher.add(message)
+ else:
+ pytest.fail("Unexpected SDK message type")
+ trace_batcher.flush()
+ span_batcher.flush()
+ assert len(trace_batches) == len(span_batches) == 1
+ trace_batch, span_batch = trace_batches[0], span_batches[0]
+ assert isinstance(trace_batch, CreateTraceBatchMessage)
+ assert isinstance(span_batch, CreateSpansBatchMessage)
+ assert len(trace_batch.batch) == 1
+ assert len(span_batch.batch) == 4
+ trace = trace_batch.batch[0]
+ root = next(span for span in span_batch.batch if span.parent_span_id is None)
+ assert trace.id == str(prepare_opik_uuid(START, external_id or WORKFLOW_ID))
+ assert trace.start_time.timestamp() == START.timestamp()
+ assert root.id != trace.id
+ assert {span.trace_id for span in span_batch.batch} == {trace.id}
+ assert len({span.id for span in span_batch.batch}) == 4
+ assert all(UUID(str(span.id)).version == 7 for span in span_batch.batch)
+ children = [span for span in span_batch.batch if span.parent_span_id is not None]
+ assert all(span.parent_span_id == root.id for span in children)
+ assert {span.metadata["node_execution_id"] for span in children if isinstance(span.metadata, dict)} == set(node_ids)
+ for index, node_id in enumerate(node_ids):
+ child = next(
+ span
+ for span in children
+ if isinstance(span.metadata, dict) and span.metadata["node_execution_id"] == node_id
+ )
+ assert child.start_time.timestamp() == (START + timedelta(milliseconds=index * 100)).timestamp()
+ assert child.end_time is not None
+ assert (child.end_time - child.start_time).total_seconds() == pytest.approx(0.1)
+ llm = next(span for span in children if span.type == "llm")
+ assert llm.usage is not None
+ assert {key: llm.usage[key] for key in ("prompt_tokens", "completion_tokens", "total_tokens")} == {
+ "prompt_tokens": 27,
+ "completion_tokens": 5,
+ "total_tokens": 32,
+ }
+ assert sum(span.usage.get("total_tokens", 0) for span in span_batch.batch if span.usage) == 32
+ assert trace.output == {"answer": "synthetic"}
diff --git a/api/tests/unit_tests/services/tracing/test_phoenix_configuration.py b/api/tests/unit_tests/services/tracing/test_phoenix_configuration.py
new file mode 100644
index 00000000000..032b511aa25
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_phoenix_configuration.py
@@ -0,0 +1,336 @@
+"""Phoenix configuration verifies only a bounded project-list read, with isolated auth."""
+
+import os
+from collections import deque
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from threading import Barrier
+from typing import Never
+
+import httpx
+import pytest
+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace
+from dify_trace_arize_phoenix.config import PhoenixConfig
+from dify_trace_arize_phoenix.configuration import (
+ PhoenixConfigurationError,
+ normalize_configuration,
+ project_url,
+ verify_configuration,
+)
+
+from core.helper import ssrf_proxy
+from core.ops.ops_trace_manager import OpsTraceManager
+
+
+def response(body: object, status: int = 200) -> httpx.Response:
+ return httpx.Response(status, json=body, request=httpx.Request("GET", "https://phoenix.test/v1/projects"))
+
+
[email protected](autouse=True)
+def no_legacy_or_environment_auth(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ names = ("PHOENIX_API_KEY", "PHOENIX_PROJECT_NAME", "PHOENIX_COLLECTOR_ENDPOINT", "OTEL_EXPORTER_OTLP_HEADERS")
+ for name in names:
+ monkeypatch.setenv(name, "synthetic-ambient-value")
+ called: list[bool] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ called.append(True)
+ raise AssertionError("Phoenix configuration must not initialize a tracer or write test spans")
+
+ monkeypatch.setattr(ArizePhoenixDataTrace, "__init__", forbidden)
+ monkeypatch.setattr(OpsTraceManager, "check_trace_config_is_effective", forbidden)
+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)
+ for method in ("post", "put", "patch", "delete"):
+ monkeypatch.setattr(ssrf_proxy, method, forbidden)
+ yield
+ assert called == []
+ assert all(os.environ[name] == "synthetic-ambient-value" for name in names)
+
+
+def api_responses() -> deque[httpx.Response | Exception]:
+ return deque()
+
+
+def api_calls(
+ monkeypatch: pytest.MonkeyPatch, api_responses: deque[httpx.Response | Exception]
+) -> list[tuple[str, dict[str, object]]]:
+ calls: list[tuple[str, dict[str, object]]] = []
+
+ def get(url: str, **options: object) -> httpx.Response:
+ calls.append((url, options))
+ result = api_responses.popleft() if api_responses else response({"data": [], "next_cursor": None})
+ if isinstance(result, Exception):
+ raise result
+ return result
+
+ monkeypatch.setattr(ssrf_proxy, "get", get)
+ return calls
+
+
+def test_normalization_preserves_prefix_and_does_not_mutate_input(
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ values = {"endpoint": " HTTP://Phoenix.TEST:6006/proxy/s/space/ ", "project": " project name ", "api_key": None}
+ untouched = dict(values)
+ config = normalize_configuration(values)
+ result = verify_configuration(config)
+ assert (result.endpoint, result.project) == ("http://phoenix.test:6006/proxy/s/space", "project name")
+ assert values == untouched
+ assert api_calls == [
+ (
+ "http://phoenix.test:6006/proxy/s/space/v1/projects",
+ {
+ "params": {"limit": 1},
+ "headers": {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""},
+ "auth": None,
+ "timeout": 30.0,
+ "max_retries": 0,
+ "follow_redirects": False,
+ "stream_response": True,
+ },
+ )
+ ]
+
+
[email protected]("key", [None, "", "synthetic-key"])
+def test_optional_key_does_not_inherit_environment(
+ api_calls: list[tuple[str, dict[str, object]]],
+ key: str | None,
+) -> None:
+ result = verify_configuration(PhoenixConfig(endpoint="https://phoenix.test", project="not-created", api_key=key))
+ assert result.project == "not-created"
+ expected = {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}
+ if key:
+ expected["Authorization"] = "Bearer synthetic-key"
+ assert api_calls[0][1]["headers"] == expected
+ assert len(api_calls) == 1
+
+
[email protected]("project", [None, "", " "])
+def test_empty_project_uses_default(project: str | None, api_calls: list[tuple[str, dict[str, object]]]) -> None:
+ config = normalize_configuration({"project": project})
+ assert config.project == "default"
+ assert project_url(config) == "https://app.phoenix.arize.com/redirects/projects/default"
+ assert api_calls == []
+
+
+def test_project_redirect_encodes_name_for_graphql_loader_without_io(
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ config = PhoenixConfig(endpoint="https://host.test:8443/s/space/", project="A /? & #中文")
+ assert (
+ project_url(config)
+ == "https://host.test:8443/s/space/redirects/projects/A%20%2F%3F%20%26%20%23%E4%B8%AD%E6%96%87"
+ )
+ assert api_calls == []
+
+
+ "endpoint",
+ [
+ "ftp://host.test",
+ "https:///missing",
+ "https://user:[email protected]",
+ "https://host.test?",
+ "https://host.test#",
+ "https://host.test/path?token=secret",
+ "https://host.test/path#fragment",
+ "https://host.test:bad",
+ "https://host.test:0",
+ "https://host.test:65536",
+ "https://host.test:",
+ "https://[invalid",
+ "https://host.test\\@elsewhere",
+ "\nhttps://host.test",
+ "https://ho\nst.test",
+ "https://host.test/a/../b",
+ "https://host.test/a/%2e%2e/b",
+ "https://host.test/a%2fb",
+ "https://host.test/a%5cb",
+ "https://host.test/a%252fb",
+ "https://host.test//proxy",
+ "https://host.test/%00",
+ "https://host.test/%zz",
+ "https://host.test/v1/traces",
+ "https://host.test/s/space/v1/traces/",
+ None,
+ ],
+)
+def test_invalid_base_is_rejected_before_io(endpoint: object, api_calls: list[tuple[str, dict[str, object]]]) -> None:
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ normalize_configuration({"endpoint": endpoint, "api_key": "private-secret"})
+ assert caught.value.code == "invalid_endpoint"
+ assert "private-secret" not in str(caught.value)
+ assert api_calls == []
+
+
[email protected]("project", ["bad\nname", "\tname", ".", "..", 123])
+def test_invalid_project_is_rejected_before_io(project: object, api_calls: list[tuple[str, dict[str, object]]]) -> None:
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ normalize_configuration({"project": project})
+ assert caught.value.code == "invalid_project"
+ assert api_calls == []
+
+
[email protected]("key", [" ", "line\nbreak", "invalid token", "******", "ab***cd"])
+def test_header_or_mask_key_is_rejected_after_restore(key: str, api_calls: list[tuple[str, dict[str, object]]]) -> None:
+ config = normalize_configuration({"api_key": key})
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ verify_configuration(config)
+ assert caught.value.code == "invalid_api_key"
+ assert api_calls == []
+
+
+ ("status", "code"),
+ [
+ (401, "authentication_failed"),
+ (403, "permission_denied"),
+ (302, "endpoint_unavailable"),
+ (404, "endpoint_unavailable"),
+ (500, "endpoint_unavailable"),
+ (204, "endpoint_unavailable"),
+ ],
+)
+def test_http_failures_are_safe_and_close_response(
+ status: int,
+ code: str,
+ api_responses: deque[httpx.Response | Exception],
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ item = response({"private-vendor-detail": "private-secret"}, status)
+ api_responses.append(item)
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ verify_configuration(PhoenixConfig(api_key="private-secret"))
+ assert caught.value.code == code
+ assert "private-secret" not in str(caught.value)
+ assert item.is_closed
+ assert len(api_calls) == 1
+
+
+ "body",
+ [
+ None,
+ [],
+ {},
+ {"data": []},
+ {"data": {}, "next_cursor": None},
+ {"data": [], "next_cursor": 3},
+ {"data": [None], "next_cursor": None},
+ {"data": [{"id": "1"}], "next_cursor": None},
+ {"data": [{"id": 1, "name": "x"}], "next_cursor": None},
+ {"data": [{"id": "1", "name": ""}], "next_cursor": None},
+ {"data": [{"id": "1", "name": "x", "description": {}}], "next_cursor": None},
+ ],
+)
+def test_project_list_shape_is_validated(
+ body: object,
+ api_responses: deque[httpx.Response | Exception],
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ item = response(body)
+ api_responses.append(item)
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ verify_configuration(PhoenixConfig())
+ assert caught.value.code == "malformed_response"
+ assert item.is_closed
+ assert len(api_calls) == 1
+
+
+def test_existing_project_page_does_not_require_selected_project_or_follow_cursor(
+ api_responses: deque[httpx.Response | Exception],
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ api_responses.append(
+ response({"data": [{"id": "Project:1", "name": "other", "description": None}], "next_cursor": "opaque-cursor"})
+ )
+ assert verify_configuration(PhoenixConfig(project="new-project")).project == "new-project"
+ assert len(api_calls) == 1
+
+
[email protected]("kind", ["html", "gzip", "oversized"])
+def test_wrong_path_html_compressed_and_oversized_responses_are_not_success(
+ kind: str,
+ api_responses: deque[httpx.Response | Exception],
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ content = b"x" * (1024 * 1024 + 1) if kind == "oversized" else b"<html>private-login-page</html>"
+ item = httpx.Response(
+ 200, content=content, request=httpx.Request("GET", "https://host.test/not-a-collector/v1/projects")
+ )
+ if kind == "gzip":
+ item.headers["Content-Encoding"] = "gzip"
+ api_responses.append(item)
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ verify_configuration(PhoenixConfig(endpoint="https://host.test/not-a-collector"))
+ assert caught.value.code == "malformed_response"
+ assert "private-login-page" not in str(caught.value)
+ assert item.is_closed
+ assert len(api_calls) == 1
+
+
+def test_transport_error_does_not_leak_url_or_credentials(
+ api_responses: deque[httpx.Response | Exception],
+ api_calls: list[tuple[str, dict[str, object]]],
+) -> None:
+ api_responses.append(httpx.ReadTimeout("private-key in vendor response"))
+ with pytest.raises(PhoenixConfigurationError) as caught:
+ verify_configuration(PhoenixConfig(api_key="private-key"))
+ assert caught.value.code == "connection_failed"
+ assert "private-key" not in str(caught.value)
+ assert caught.value.__suppress_context__
+ assert len(api_calls) == 1
+
+
+def test_concurrent_targets_and_credentials_remain_isolated(monkeypatch: pytest.MonkeyPatch) -> None:
+ barrier = Barrier(2)
+ requests: list[httpx.Request] = []
+ targets = {
+ "https://first.test/s/one/v1/projects?limit=1": "first-key",
+ "https://second.test/reverse/v1/projects?limit=1": "second-key",
+ }
+
+ def handle(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ assert request.method == "GET"
+ assert request.headers["Authorization"] == "Bearer " + targets[str(request.url)]
+ barrier.wait(timeout=5)
+ return httpx.Response(200, json={"data": [], "next_cursor": None})
+
+ with httpx.Client(transport=httpx.MockTransport(handle), trust_env=False) as client:
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _ssl_verify_enabled: client)
+ configs = [
+ PhoenixConfig(endpoint="https://first.test/s/one", api_key="first-key", project="one"),
+ PhoenixConfig(endpoint="https://second.test/reverse", api_key="second-key", project="two"),
+ ]
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert [item.project for item in pool.map(verify_configuration, configs)] == ["one", "two"]
+ assert sorted(str(request.url) for request in requests) == sorted(targets)
+
+
+def test_real_proxy_client_has_no_ambient_auth_or_cookie_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
+ requests: list[httpx.Request] = []
+
+ def handle(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ assert str(request.url) == "https://host.test/s/space/v1/projects?limit=1"
+ assert request.headers["Cookie"] == ""
+ return httpx.Response(
+ 200, headers={"Set-Cookie": "session=vendor; Path=/"}, json={"data": [], "next_cursor": None}
+ )
+
+ with httpx.Client(
+ transport=httpx.MockTransport(handle),
+ auth=httpx.BasicAuth("ambient", "secret"),
+ cookies={"session": "ambient"},
+ trust_env=False,
+ ) as client:
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _ssl_verify_enabled: client)
+ verify_configuration(PhoenixConfig(endpoint="https://host.test/s/space", api_key="explicit-key"))
+ verify_configuration(PhoenixConfig(endpoint="https://host.test/s/space", api_key=None))
+ assert requests[0].headers["Authorization"] == "Bearer explicit-key"
+ assert "Authorization" not in requests[1].headers
+ assert len(requests) == 2
diff --git a/api/tests/unit_tests/services/tracing/test_phoenix_export_auth.py b/api/tests/unit_tests/services/tracing/test_phoenix_export_auth.py
new file mode 100644
index 00000000000..ed5f8c1d3ac
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_phoenix_export_auth.py
@@ -0,0 +1,142 @@
+"""Exercise anonymous Phoenix auth at the HTTP/1.1 and legacy SDK boundaries."""
+
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import httpcore
+import httpx
+import pytest
+import requests
+from dify_trace_arize_phoenix.arize_phoenix_trace import setup_tracer
+from dify_trace_arize_phoenix.config import PhoenixConfig
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+
+from core.helper import ssrf_proxy
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.provider_dispatch import export_trace
+
+
+def trace() -> CanonicalTrace:
+ start = datetime(2026, 10, 20, tzinfo=UTC)
+ return CanonicalTrace(
+ trace_id="1" * 32,
+ session_id="phoenix-auth-session",
+ root_span_id="2" * 16,
+ spans=(
+ CanonicalSpan(
+ id="2" * 16,
+ parent_id=None,
+ name="Phoenix auth regression",
+ kind=CanonicalSpanKind.AGENT,
+ start_time=start,
+ end_time=start + timedelta(seconds=1),
+ status=CanonicalSpanStatus.OK,
+ ),
+ ),
+ )
+
+
+def http11_wire(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[bytes]]:
+ """Keep real HTTPX/httpcore/h11 serialization, replacing only socket I/O."""
+ chunks: list[bytes] = []
+
+ def write(_stream: httpcore.MockStream, buffer: bytes, timeout: float | None = None) -> None:
+ del timeout
+ chunks.append(buffer)
+
+ monkeypatch.setattr(httpcore.MockStream, "write", write)
+ transport = httpx.HTTPTransport()
+ transport.close()
+ transport._pool = httpcore.ConnectionPool(
+ network_backend=httpcore.MockBackend([b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"])
+ )
+ with httpx.Client(transport=transport, trust_env=False) as client:
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _verify: client)
+ yield chunks
+
+
+def test_managed_exports_serialize_optional_auth_without_cross_destination_credentials(
+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, http11_wire: list[bytes]
+) -> None:
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer%20ambient-general,api_key=ambient-general")
+ monkeypatch.setenv(
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer%20ambient-traces,api_key=ambient-traces"
+ )
+
+ for index, key in enumerate(("first-synthetic-key", "", None, "second-synthetic-key", None, "")):
+ previous_chunks = len(http11_wire)
+ result = export_trace(
+ "phoenix", {"endpoint": "http://phoenix.test/proxy", "api_key": key, "project": f"project-{index}"}, trace
+ )
+ assert result.status == "accepted"
+ assert result.trace_id == trace.trace_id
+ head, separator, body = b"".join(http11_wire[previous_chunks:]).partition(b"\r\n\r\n")
+ assert separator
+ lines = head.split(b"\r\n")
+ assert lines[0] == b"POST /proxy/v1/traces HTTP/1.1"
+ header_values: list[tuple[bytes, bytes]] = []
+ for line in lines[1:]:
+ name, value = line.split(b": ", 1)
+ header_values.append((name, value))
+ headers = httpx.Headers(header_values)
+ assert headers["Content-Type"] == "application/x-protobuf"
+ if key:
+ assert headers["api_key"] == key
+ assert headers["authorization"] == f"Bearer {key}"
+ else:
+ assert "api_key" not in headers
+ assert "authorization" not in headers
+ payload = ExportTraceServiceRequest.FromString(body)
+ resource = payload.resource_spans[0]
+ assert {item.key: item.value.string_value for item in resource.resource.attributes}[
+ "openinference.project.name"
+ ] == f"project-{index}"
+ assert resource.scope_spans[0].spans[0].trace_id.hex() == trace.trace_id
+
+
+def test_legacy_sdk_prepares_anonymous_requests_without_ambient_or_previous_auth(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer%20ambient-general,api_key=ambient-general")
+ monkeypatch.setenv(
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer%20ambient-traces,api_key=ambient-traces"
+ )
+ monkeypatch.delenv("OTEL_EXPORTER_OTLP_COMPRESSION", raising=False)
+ monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", raising=False)
+ prepared: list[requests.PreparedRequest] = []
+
+ def send(_session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:
+ prepared.append(request)
+ response = requests.Response()
+ response.status_code = 200
+ response._content = b""
+ response.request = request
+ return response
+
+ monkeypatch.setattr(requests.Session, "send", send)
+ for index, key in enumerate(("first-synthetic-key", "", None, "second-synthetic-key", None, "")):
+ tracer, processor = setup_tracer(
+ PhoenixConfig(endpoint="http://phoenix.test/proxy", api_key=key, project=f"project-{index}")
+ )
+ try:
+ tracer.start_span("Legacy Phoenix auth regression").end()
+ finally:
+ processor.shutdown()
+ assert len(prepared) == index + 1
+ request = prepared[-1]
+ assert request.method == "POST"
+ assert request.url == "http://phoenix.test/proxy/v1/traces"
+ assert request.headers["Content-Type"] == "application/x-protobuf"
+ if key:
+ assert request.headers["api_key"] == key
+ assert request.headers["authorization"] == f"Bearer {key}"
+ else:
+ assert "api_key" not in request.headers
+ assert "authorization" not in request.headers
+ assert isinstance(request.body, bytes)
+ payload = ExportTraceServiceRequest.FromString(request.body)
+ assert {item.key: item.value.string_value for item in payload.resource_spans[0].resource.attributes}[
+ "openinference.project.name"
+ ] == f"project-{index}"
diff --git a/api/tests/unit_tests/services/tracing/test_provider_dispatch.py b/api/tests/unit_tests/services/tracing/test_provider_dispatch.py
new file mode 100644
index 00000000000..32ffd59b8e7
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_provider_dispatch.py
@@ -0,0 +1,715 @@
+"""Exercise real vendor serializers and OTLP stubs at their network boundary."""
+
+import json
+from collections.abc import Callable, Mapping, Sequence
+from datetime import UTC, datetime, timedelta
+from typing import Never, NotRequired, TypedDict, cast
+from uuid import UUID
+
+import httpx
+import pytest
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+ ExportTraceServiceRequest,
+ ExportTraceServiceResponse,
+)
+from opentelemetry.proto.trace.v1.trace_pb2 import Span
+
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace
+from services.tracing.normalization import remap_outer_trace
+from services.tracing.provider_dispatch import (
+ TraceExportRejectedError,
+ TraceExportUncertainError,
+ export_trace,
+ provider_capabilities,
+)
+
+
+class HTTPArguments(TypedDict):
+ content: bytes
+ headers: Mapping[str, str]
+ max_retries: NotRequired[int]
+ follow_redirects: NotRequired[bool]
+
+
+type HTTPCall = tuple[str, str, HTTPArguments]
+
+
+def trace() -> CanonicalTrace:
+ start = datetime(2026, 10, 15, 1, tzinfo=UTC)
+ return CanonicalTrace(
+ trace_id="1" * 32,
+ session_id="conversation-1",
+ root_span_id="1" * 16,
+ spans=(
+ CanonicalSpan(
+ id="1" * 16,
+ parent_id=None,
+ name="Agent turn",
+ kind=CanonicalSpanKind.AGENT,
+ start_time=start,
+ end_time=start + timedelta(seconds=3),
+ status=CanonicalSpanStatus.OK,
+ inputs={"question": "hi"},
+ outputs={"answer": "done"},
+ metadata={"app_id": "app-one", "tenant_id": "tenant-one", "total_tokens": 12, "total_price": "0.03"},
+ ),
+ CanonicalSpan(
+ id="2" * 16,
+ parent_id="1" * 16,
+ name="model call",
+ kind=CanonicalSpanKind.LLM,
+ start_time=start,
+ end_time=start + timedelta(seconds=1),
+ status=CanonicalSpanStatus.OK,
+ inputs=[{"role": "user", "content": "hi"}],
+ outputs="done",
+ metadata={
+ "model": "gpt-4o-mini",
+ "model_provider": "openai",
+ "prompt_tokens": 8,
+ "completion_tokens": 4,
+ "total_tokens": 12,
+ "total_price": "0.03",
+ "currency": "USD",
+ "ttft": 0.2,
+ },
+ ),
+ CanonicalSpan(
+ id="3" * 16,
+ parent_id="1" * 16,
+ name="search",
+ kind=CanonicalSpanKind.TOOL,
+ start_time=start + timedelta(seconds=1),
+ end_time=start + timedelta(seconds=2),
+ status=CanonicalSpanStatus.ERROR,
+ inputs={"q": "x"},
+ outputs=None,
+ error="tool failed",
+ ),
+ ),
+ )
+
+
+def http_calls(monkeypatch: pytest.MonkeyPatch) -> list[HTTPCall]:
+ from core.helper import ssrf_proxy
+
+ calls: list[HTTPCall] = []
+
+ def send(method: str, url: str, **kwargs: object) -> httpx.Response:
+ calls.append((method, url, cast(HTTPArguments, kwargs)))
+ return httpx.Response(200, content=b"", request=httpx.Request(method, url))
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ return calls
+
+
+def spans_from_call(call: HTTPCall) -> list[Span]:
+ request = ExportTraceServiceRequest.FromString(call[2]["content"])
+ return list(request.resource_spans[0].scope_spans[0].spans)
+
+
+def attributes(span: Span) -> dict[str, object]:
+ result: dict[str, object] = {}
+ for item in span.attributes:
+ kind = item.value.WhichOneof("value")
+ assert kind is not None
+ result[item.key] = {
+ "string_value": item.value.string_value,
+ "bool_value": item.value.bool_value,
+ "int_value": item.value.int_value,
+ "double_value": item.value.double_value,
+ "array_value": item.value.array_value,
+ "kvlist_value": item.value.kvlist_value,
+ "bytes_value": item.value.bytes_value,
+ }[kind]
+ return result
+
+
+def test_langfuse_v4_tree_and_native_usage(trace: CanonicalTrace, http_calls: list[HTTPCall]) -> None:
+ config = {
+ "host": "https://us.cloud.langfuse.com",
+ "public_key": "pk-test",
+ "secret_key": "sk-test",
+ "project_url": "https://us.cloud.langfuse.com/project/example",
+ }
+ result = export_trace("langfuse", config, trace)
+ assert result.status == "accepted"
+ assert result.trace_id == trace.trace_id
+ assert len(http_calls) == 1
+ _, url, args = http_calls[0]
+ assert url.endswith("/api/public/otel/v1/traces")
+ assert args["headers"]["x-langfuse-ingestion-version"] == "4"
+ assert args["max_retries"] == 0
+ assert args["follow_redirects"] is False
+ spans = spans_from_call(http_calls[0])
+ assert [span.span_id.hex() for span in spans] == [s.id for s in trace.spans]
+ assert spans[1].parent_span_id == spans[0].span_id
+ assert spans[2].status.code == 2
+ root, llm = attributes(spans[0]), attributes(spans[1])
+ assert root["langfuse.observation.type"] == "agent"
+ assert "langfuse.observation.usage_details" not in root
+ usage_details, cost_details = llm["langfuse.observation.usage_details"], llm["langfuse.observation.cost_details"]
+ assert isinstance(usage_details, str)
+ assert isinstance(cost_details, str)
+ assert json.loads(usage_details) == {"input": 8, "output": 4, "total": 12}
+ assert json.loads(cost_details) == {"total": 0.03}
+ assert all(attributes(span)["langfuse.session.id"] == trace.session_id for span in spans)
+ assert result.span_urls[trace.spans[1].id].endswith(trace.spans[1].id)
+
+
+ ("provider", "config", "suffix", "kind_key", "kind"),
+ [
+ (
+ "phoenix",
+ {"endpoint": "https://phoenix.test/proxy", "api_key": "key", "project": "one"},
+ "/proxy/v1/traces",
+ "openinference.span.kind",
+ "AGENT",
+ ),
+ (
+ "aliyun",
+ {"endpoint": "https://region.log.aliyuncs.com", "license_key": "license", "app_name": "one"},
+ "/adapt_license/api/v1/traces",
+ "gen_ai.span.kind",
+ "AGENT",
+ ),
+ ],
+)
+def test_other_http_providers_have_native_semantics(
+ trace: CanonicalTrace,
+ http_calls: list[HTTPCall],
+ provider: str,
+ config: dict[str, str],
+ suffix: str,
+ kind_key: str,
+ kind: str,
+) -> None:
+ result = export_trace(provider, config, trace)
+ assert result.status == "accepted"
+ assert http_calls[0][1].endswith(suffix)
+ spans = spans_from_call(http_calls[0])
+ assert attributes(spans[0])[kind_key] == kind
+ token_key = "llm.token_count.prompt" if provider == "phoenix" else "gen_ai.usage.input_tokens"
+ assert attributes(spans[1])[token_key] == 8
+ assert token_key not in attributes(spans[0])
+
+
+ ("endpoint_url", "suffix"),
+ [
+ ("https://log.aliyuncs.com", "api/v1/traces"),
+ ("https://cn-hangzhou.log.aliyuncs.com", "api/v1/traces"),
+ ("https://project.cn-heyuan.log.aliyuncs.com/apm/trace/opentelemetry", "api/v1/traces"),
+ ("https://project.cn-heyuan-intranet.log.aliyuncs.com/apm/trace/opentelemetry", "api/v1/traces"),
+ ("https://cn-shanghai-share.log.aliyuncs.com", "api/v1/traces"),
+ ("https://cn-beijing.dualstack.log.aliyuncs.com", "api/v1/traces"),
+ ("https://cn-hangzhou.log.aliyuncs.com.:443", "api/v1/traces"),
+ ("https://tracing-analysis-dc-hz.aliyuncs.com", "api/otlp/traces"),
+ ("https://collector.example.test:4318/custom-prefix", "api/otlp/traces"),
+ ("https://evil-log.aliyuncs.com.evil.test", "api/otlp/traces"),
+ ("https://log.aliyuncs.com.evil.test", "api/otlp/traces"),
+ ("https://evil-log.aliyuncs.com", "api/otlp/traces"),
+ ("https://collector.example.test/log.aliyuncs.com", "api/otlp/traces"),
+ ],
+)
+def test_aliyun_endpoint_family_uses_dns_labels_and_preserves_custom_destinations(
+ trace: CanonicalTrace, http_calls: list[HTTPCall], endpoint_url: str, suffix: str
+) -> None:
+ result = export_trace(
+ "aliyun", {"endpoint": endpoint_url, "license_key": "synthetic-license", "app_name": "test-app"}, trace
+ )
+ assert result.status == "accepted"
+ assert len(http_calls) == 1
+ assert http_calls[0][1] == f"{endpoint_url}/adapt_synthetic-license/{suffix}"
+ spans = spans_from_call(http_calls[0])
+ assert [span.span_id.hex() for span in spans] == [span.id for span in trace.spans]
+ assert spans[1].parent_span_id == spans[0].span_id
+
+
+def test_aliyun_raw_license_is_encoded_once_at_the_real_otlp_boundary(
+ trace: CanonicalTrace, http_calls: list[HTTPCall]
+) -> None:
+ result = export_trace(
+ "aliyun",
+ {"endpoint": "https://COLLECTOR.test:8443/prefix/", "license_key": "raw%2F@a+b/c", "app_name": "test_app"},
+ trace,
+ )
+ assert result.status == "accepted"
+ assert len(http_calls) == 1
+ assert http_calls[0][1] == "https://collector.test:8443/prefix/adapt_raw%252F@a%2Bb%2Fc/api/otlp/traces"
+ assert [span.span_id.hex() for span in spans_from_call(http_calls[0])] == [span.id for span in trace.spans]
+
+
+def test_aliyun_complete_token_url_is_rejected_without_any_ingestion(
+ trace: CanonicalTrace, http_calls: list[HTTPCall]
+) -> None:
+ with pytest.raises(TraceExportRejectedError, match="Aliyun tracing configuration is invalid"):
+ export_trace(
+ "aliyun",
+ {"endpoint": "https://collector.test/adapt_private/api/otlp/traces", "license_key": "private"},
+ trace,
+ )
+ assert http_calls == []
+
+
+ ("provider", "config", "header"),
+ [
+ (
+ "arize",
+ {"endpoint": "https://otlp.arize.com", "api_key": "key", "space_id": "space", "project": "one"},
+ ("space_id", "space"),
+ ),
+ (
+ "tencent",
+ {"endpoint": "https://apm.test:4317", "token": "token", "service_name": "one"},
+ ("authorization", "Bearer token"),
+ ),
+ ],
+)
+def test_grpc_real_stub_serialization(
+ trace: CanonicalTrace,
+ monkeypatch: pytest.MonkeyPatch,
+ provider: str,
+ config: dict[str, str],
+ header: tuple[str, str],
+) -> None:
+ calls: list[tuple[ExportTraceServiceRequest, Sequence[tuple[str, str]]]] = []
+
+ class Channel:
+ def unary_unary(
+ self,
+ method: str,
+ request_serializer: Callable[[ExportTraceServiceRequest], bytes],
+ response_deserializer: Callable[[bytes], ExportTraceServiceResponse],
+ **_kwargs: object,
+ ) -> Callable[..., ExportTraceServiceResponse]:
+ assert method == "/opentelemetry.proto.collector.trace.v1.TraceService/Export"
+
+ def invoke(
+ payload: ExportTraceServiceRequest, metadata: Sequence[tuple[str, str]], timeout: float
+ ) -> ExportTraceServiceResponse:
+ assert timeout == 30
+ calls.append((ExportTraceServiceRequest.FromString(request_serializer(payload)), metadata))
+ return response_deserializer(ExportTraceServiceResponse().SerializeToString())
+
+ return invoke
+
+ def close(self) -> None:
+ pass
+
+ monkeypatch.setattr("grpc.secure_channel", lambda *_args, **_kwargs: Channel())
+ result = export_trace(provider, config, trace)
+ assert result.status == "accepted"
+ assert header in calls[0][1]
+ spans = calls[0][0].resource_spans[0].scope_spans[0].spans
+ assert spans[1].parent_span_id == spans[0].span_id
+
+
+def test_langsmith_real_sdk_uses_isolated_sync_runs(trace: CanonicalTrace, http_calls: list[HTTPCall]) -> None:
+ result = export_trace(
+ "langsmith", {"api_key": "ls-test", "endpoint": "https://langsmith.test", "project": "one"}, trace
+ )
+ writes = [json.loads(call[2]["content"]) for call in http_calls if call[0] == "POST"]
+ assert len(writes) == 3
+ root, llm, tool = writes
+ assert root["run_type"] == "chain"
+ assert llm["run_type"] == "llm"
+ assert llm["parent_run_id"] == root["id"]
+ assert llm["dotted_order"].startswith(root["dotted_order"])
+ assert llm["input_tokens"] == 8
+ assert llm["output_tokens"] == 4
+ assert llm["total_cost"] == 0.03
+ assert "input_tokens" not in root
+ assert all(write["extra"]["metadata"]["session_id"] == trace.session_id for write in writes)
+ assert tool["error"] == "tool failed"
+ assert UUID(result.trace_id)
+
+
+def test_opik_real_sdk_preserves_tree_thread_and_native_usage(
+ trace: CanonicalTrace, http_calls: list[HTTPCall]
+) -> None:
+ result = export_trace(
+ "opik",
+ {"api_key": "opik-test", "url": "https://opik.test/api", "workspace": "org-one", "project": "one"},
+ trace,
+ )
+ assert len(http_calls) == 2
+ root = json.loads(http_calls[0][2]["content"])
+ batch = json.loads(http_calls[1][2]["content"])
+ spans = batch["spans"]
+ assert UUID(result.trace_id).version == 7
+ assert root["thread_id"] == trace.session_id
+ assert spans[1]["parent_span_id"] == spans[0]["id"]
+ assert spans[1]["usage"]["prompt_tokens"] == 8
+ assert spans[1]["total_estimated_cost"] == 0.03
+ assert not spans[0].get("usage")
+ assert spans[2]["error_info"]["message"] == "tool failed"
+ assert http_calls[0][2]["headers"]["comet-workspace"] == "org-one"
+
+
+ ("value", "wire_value"),
+ [
+ (None, None),
+ ("323", "323"),
+ ({"result": 323}, {"result": 323}),
+ ([{"result": 323}], [{"result": 323}]),
+ ([], []),
+ (323, "323"),
+ (0, "0"),
+ (-5, "-5"),
+ (3.5, "3.5"),
+ (True, "true"),
+ (False, "false"),
+ ([1, 2], "[1, 2]"),
+ (["one", "two"], '["one", "two"]'),
+ ([{"result": 1}, "text"], '[{"result": 1}, "text"]'),
+ ([[1], [2]], "[[1], [2]]"),
+ ],
+)
+def test_opik_real_sdk_preserves_json_content_across_trace_and_span_boundaries(
+ trace: CanonicalTrace, http_calls: list[HTTPCall], value: object, wire_value: object
+) -> None:
+ content_trace = trace.model_copy(
+ update={"spans": tuple(span.model_copy(update={"inputs": value, "outputs": value}) for span in trace.spans)}
+ )
+ result = export_trace(
+ "opik",
+ {"api_key": "synthetic-key", "url": "https://opik.test/api", "workspace": "team", "project": "one"},
+ content_trace,
+ )
+ assert result.status == "accepted"
+ assert len(http_calls) == 2
+ root = json.loads(http_calls[0][2]["content"])
+ spans = json.loads(http_calls[1][2]["content"])["spans"]
+ for payload in [root, *spans]:
+ assert payload.get("input") == wire_value
+ assert payload.get("output") == wire_value
+ assert spans[1]["parent_span_id"] == spans[0]["id"]
+ assert spans[2]["parent_span_id"] == spans[0]["id"]
+ assert root["thread_id"] == trace.session_id
+ assert spans[1]["usage"]["total_tokens"] == 12
+ assert not spans[0].get("usage")
+ assert not spans[2].get("usage")
+
+
[email protected]("capture_content", [True, False])
[email protected]("root_error", [True, False])
+def test_opik_trace_error_matches_sanitized_root_and_not_handled_tool_failure(
+ trace: CanonicalTrace, http_calls: list[HTTPCall], capture_content: bool, root_error: bool
+) -> None:
+ root = trace.spans[0].model_copy(
+ update={
+ "status": CanonicalSpanStatus.ERROR if root_error else CanonicalSpanStatus.OK,
+ "error": "private-model-error" if root_error else None,
+ "inputs": {"query": "private-error-input"},
+ }
+ )
+ source = trace.model_copy(update={"spans": (root, *trace.spans[1:])})
+ canonical = remap_outer_trace(source, trace_id=trace.trace_id, capture_content=capture_content, agent_message=True)
+ export_trace(
+ "opik",
+ {"api_key": "synthetic-key", "url": "https://opik.test/api", "workspace": "team", "project": "one"},
+ canonical,
+ )
+ root_payload = json.loads(http_calls[0][2]["content"])
+ spans = json.loads(http_calls[1][2]["content"])["spans"]
+ assert spans[2]["error_info"] is not None
+ if root_error:
+ expected = {
+ "exception_type": "DifyExecutionError",
+ "message": "private-model-error" if capture_content else "Application execution failed",
+ "traceback": "",
+ }
+ assert root_payload["error_info"] == spans[0]["error_info"] == expected
+ else:
+ assert root_payload.get("error_info") is None
+ assert spans[0].get("error_info") is None
+ if not capture_content:
+ assert all(b"private-model-error" not in call[2]["content"] for call in http_calls)
+ assert all(b"private-error-input" not in call[2]["content"] for call in http_calls)
+
+
+ ("status", "error"),
+ [(401, TraceExportRejectedError), (429, RetryableTraceDispatchError), (503, RetryableTraceDispatchError)],
+)
+def test_explicit_http_rejections_are_classified_without_response_body(
+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch, status: int, error: type[Exception]
+) -> None:
+ from core.helper import ssrf_proxy
+
+ monkeypatch.setattr(
+ ssrf_proxy,
+ "make_request",
+ lambda method, url, **_kwargs: httpx.Response(status, text="secret-token", request=httpx.Request(method, url)),
+ )
+ with pytest.raises(error) as caught:
+ export_trace(
+ "langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "secret-token"}, trace
+ )
+ assert "secret-token" not in str(caught.value)
+
+
+def test_partial_otlp_acceptance_requires_reconciliation(
+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from core.helper import ssrf_proxy
+
+ response = ExportTraceServiceResponse()
+ response.partial_success.rejected_spans = 1
+ monkeypatch.setattr(
+ ssrf_proxy,
+ "make_request",
+ lambda method, url, **_kwargs: httpx.Response(
+ 200, content=response.SerializeToString(), request=httpx.Request(method, url)
+ ),
+ )
+ with pytest.raises(TraceExportUncertainError):
+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)
+
+
+def _langfuse_queue_job_ack() -> dict[str, object]:
+ # Synthetic values only. Real acknowledgements contain auth context and must
+ # never be logged or persisted as acceptance-test evidence.
+ return {
+ "name": "otel-ingestion-job",
+ "data": {
+ "id": "11111111-2222-4333-8444-555555555555",
+ "timestamp": "2026-10-15T01:00:00.000Z",
+ "name": "otel-ingestion-job",
+ "payload": {
+ "data": {"fileKey": "otel/test/payload.json", "publicKey": "never-log-public-key"},
+ "authCheck": {
+ "validKey": True,
+ "scope": {"projectId": "never-log-project", "accessLevel": "project", "orgId": "test-org"},
+ },
+ "sdkName": "test-sdk",
+ "ingestionVersion": "4",
+ },
+ },
+ }
+
+
+def _replace_ack_value(ack: dict[str, object], path: tuple[str, ...], value: object) -> None:
+ for field in path[:-1]:
+ nested = ack[field]
+ assert isinstance(nested, dict)
+ ack = cast(dict[str, object], nested)
+ ack[path[-1]] = value
+
+
+def test_langfuse_v4_queue_ack_confirms_ingestion_acceptance_only(
+ trace: CanonicalTrace,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ from core.helper import ssrf_proxy
+
+ monkeypatch.setattr(
+ ssrf_proxy,
+ "make_request",
+ lambda method, url, **_kwargs: httpx.Response(
+ 200, json=_langfuse_queue_job_ack(), request=httpx.Request(method, url)
+ ),
+ )
+ result = export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)
+ assert result.status == "accepted"
+ assert result.trace_id == trace.trace_id
+ assert result.trace_url is None
+ captured = capsys.readouterr()
+ evidence = caplog.text + captured.out + captured.err + repr(result)
+ assert "never-log-public-key" not in evidence
+ assert "never-log-project" not in evidence
+ assert "authCheck" not in evidence
+
+
+ "scenario",
+ [
+ "wrong_name",
+ "wrong_nested_name",
+ "invalid_id",
+ "invalid_timestamp",
+ "invalid_auth",
+ "missing_scope",
+ "missing_file",
+ "partial_success",
+ "error",
+ "failed_job",
+ "unknown",
+ "null",
+ "array",
+ ],
+)
+def test_langfuse_ambiguous_json_ack_remains_uncertain(
+ trace: CanonicalTrace,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ capsys: pytest.CaptureFixture[str],
+ scenario: str,
+) -> None:
+ from core.helper import ssrf_proxy
+
+ ack = _langfuse_queue_job_ack()
+ body: object = ack
+ if scenario == "wrong_name":
+ ack["name"] = "different-job"
+ elif scenario == "wrong_nested_name":
+ _replace_ack_value(ack, ("data", "name"), "different-job")
+ elif scenario == "invalid_id":
+ _replace_ack_value(ack, ("data", "id"), "not-a-uuid")
+ elif scenario == "invalid_timestamp":
+ _replace_ack_value(ack, ("data", "timestamp"), "2026-10-15T01:00:00")
+ elif scenario == "invalid_auth":
+ _replace_ack_value(ack, ("data", "payload", "authCheck", "validKey"), False)
+ elif scenario == "missing_scope":
+ _replace_ack_value(ack, ("data", "payload", "authCheck", "scope"), dict[str, object]())
+ elif scenario == "missing_file":
+ _replace_ack_value(ack, ("data", "payload", "data", "fileKey"), "")
+ elif scenario == "partial_success":
+ ack["partialSuccess"] = {"rejectedSpans": "1"}
+ elif scenario == "error":
+ ack["error"] = "never-log-public-key"
+ elif scenario == "failed_job":
+ ack["failedReason"] = "never-log-public-key"
+ elif scenario == "unknown":
+ body = {"accepted": True, "publicKey": "never-log-public-key"}
+ elif scenario == "null":
+ body = None
+ elif scenario == "array":
+ body = list[object]()
+ monkeypatch.setattr(
+ ssrf_proxy,
+ "make_request",
+ lambda method, url, **_kwargs: httpx.Response(
+ 200,
+ content=json.dumps(body).encode(),
+ headers={"Content-Type": "application/json"},
+ request=httpx.Request(method, url),
+ ),
+ )
+ with pytest.raises(TraceExportUncertainError) as caught:
+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)
+ captured = capsys.readouterr()
+ evidence = caplog.text + captured.out + captured.err + str(caught.value)
+ assert "never-log-public-key" not in evidence
+ assert "never-log-project" not in evidence
+ assert "authCheck" not in evidence
+
+
+ ("provider", "version", "status"), [("phoenix", "4", 200), ("langfuse", "3", 200), ("langfuse", "4", 202)]
+)
+def test_langfuse_queue_ack_is_not_a_generic_otlp_success(
+ monkeypatch: pytest.MonkeyPatch, provider: str, version: str, status: int
+) -> None:
+ from core.helper import ssrf_proxy
+ from services.tracing.otlp_export import export_http
+
+ monkeypatch.setattr(
+ ssrf_proxy,
+ "make_request",
+ lambda method, url, **_kwargs: httpx.Response(
+ status, json=_langfuse_queue_job_ack(), request=httpx.Request(method, url)
+ ),
+ )
+ with pytest.raises(TraceExportUncertainError):
+ export_http(
+ provider,
+ "https://collector.test/v1/traces",
+ {"x-langfuse-ingestion-version": version},
+ ExportTraceServiceRequest(),
+ )
+
+
[email protected]("body", [{}, {"partialSuccess": {}}, {"partialSuccess": {"rejectedSpans": "1"}}])
+def test_langfuse_standard_otlp_json_ack_remains_supported(
+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch, body: dict[str, dict[str, str]]
+) -> None:
+ from core.helper import ssrf_proxy
+
+ monkeypatch.setattr(
+ ssrf_proxy,
+ "make_request",
+ lambda method, url, **_kwargs: httpx.Response(200, json=body, request=httpx.Request(method, url)),
+ )
+ config = {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}
+ if body.get("partialSuccess", {}).get("rejectedSpans"):
+ with pytest.raises(TraceExportUncertainError):
+ export_trace("langfuse", config, trace)
+ else:
+ assert export_trace("langfuse", config, trace).status == "accepted"
+
+
+def test_lost_http_ack_is_uncertain_not_rejected(trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch) -> None:
+ from core.helper import ssrf_proxy
+
+ def timeout(*_args: object, **_kwargs: object) -> Never:
+ raise httpx.ReadTimeout("response lost")
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", timeout)
+ with pytest.raises(TraceExportUncertainError):
+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)
+
+
[email protected]("provider", ["langsmith", "opik"])
[email protected]("accepted_prefix", [False, True])
+def test_native_sdk_preserves_first_failure_or_partial_acceptance(
+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch, provider: str, accepted_prefix: bool
+) -> None:
+ from core.helper import ssrf_proxy
+
+ calls: list[str] = []
+
+ def send(method: str, url: str, **_kwargs: object) -> httpx.Response:
+ calls.append(url)
+ status = 200 if accepted_prefix and len(calls) == 1 else 429
+ return httpx.Response(status, content=b"", request=httpx.Request(method, url))
+
+ monkeypatch.setattr(ssrf_proxy, "make_request", send)
+ config = {"api_key": "test-key", "project": "one", "endpoint": "https://smith.test", "url": "https://opik.test/api"}
+ expected = TraceExportUncertainError if accepted_prefix else RetryableTraceDispatchError
+ with pytest.raises(expected) as caught:
+ export_trace(provider, config, trace)
+ assert isinstance(caught.value, TraceExportUncertainError) is accepted_prefix
+ assert len(calls) == (2 if accepted_prefix else 1)
+
+
+def test_http_exports_keep_destination_credentials_local(trace: CanonicalTrace, http_calls: list[HTTPCall]) -> None:
+ for project in ("a", "b"):
+ export_trace(
+ "langfuse", {"host": f"https://{project}.test", "public_key": project, "secret_key": project}, trace
+ )
+ assert http_calls[0][1].startswith("https://a.test/")
+ assert http_calls[1][1].startswith("https://b.test/")
+ assert http_calls[0][2]["headers"]["Authorization"] != http_calls[1][2]["headers"]["Authorization"]
+ assert http_calls[0][2]["content"] == http_calls[1][2]["content"]
+
+
+ "provider",
+ ["langfuse", "langsmith", "phoenix", "arize", "opik", "weave", "mlflow", "databricks", "aliyun", "tencent"],
+)
+def test_capabilities_state_implemented_contract_not_platform_validation(provider: str) -> None:
+ result = provider_capabilities(provider)
+ assert result["agent_spans"] is True
+ assert result["export_ack"] == "accepted"
+ assert "platform_validated" not in result
+
+
+def test_rejects_unfinished_and_unknown_provider_before_sending(
+ trace: CanonicalTrace, http_calls: list[HTTPCall]
+) -> None:
+ unfinished = trace.model_copy(update={"spans": (trace.spans[0].model_copy(update={"end_time": None}),)})
+ with pytest.raises(TraceExportRejectedError):
+ export_trace("langfuse", {}, unfinished)
+ with pytest.raises(TraceExportRejectedError):
+ export_trace("unknown", {}, trace)
+ assert not http_calls
diff --git a/api/tests/unit_tests/services/tracing/test_request_log_protection.py b/api/tests/unit_tests/services/tracing/test_request_log_protection.py
new file mode 100644
index 00000000000..ebff6db3cac
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_request_log_protection.py
@@ -0,0 +1,292 @@
+"""Real HTTPX/SSRF diagnostics with synthetic credential-bearing destinations."""
+
+import logging
+import traceback
+from collections.abc import Callable, Iterator
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import ExitStack
+from threading import Barrier, Event
+from urllib.parse import quote
+
+import httpx
+import pytest
+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value
+
+from core.helper import ssrf_proxy
+from core.logging.structured_formatter import StructuredJSONFormatter
+from core.ops.exceptions import RetryableTraceDispatchError
+from services.tracing.provider_common import request
+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError
+from services.tracing.request_log_protection import SAFE_HTTP_LOG_MESSAGE, protect_http_request_logs
+
+SYNTHETIC_KEY = "SYNTHETIC-NOT-A-REAL-LICENSE/+?=测试"
+ENCODED_KEY = quote(SYNTHETIC_KEY, safe="")
+PRIVATE_URL = f"https://collector.test/prefix/adapt_{ENCODED_KEY}/api/otlp/traces"
+type HTTPHandler = Callable[[httpx.Request], httpx.Response]
+type InstallClient = Callable[[HTTPHandler], None]
+
+
[email protected](autouse=True)
+def enable_http_diagnostics(caplog: pytest.LogCaptureFixture) -> Iterator[None]:
+ """Exercise INFO records even if an imported SDK changed HTTPX's level."""
+ with caplog.at_level(logging.INFO, logger="httpx"):
+ yield
+
+
+def install_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[InstallClient]:
+ with ExitStack() as stack:
+
+ def install(handler: HTTPHandler) -> None:
+ client = stack.enter_context(httpx.Client(transport=httpx.MockTransport(handler)))
+
+ def get_client(ssl_verify_enabled: bool) -> httpx.Client:
+ assert isinstance(ssl_verify_enabled, bool)
+ return client
+
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", get_client)
+
+ yield install
+
+
+def evidence(caplog: pytest.LogCaptureFixture) -> str:
+ formatter = StructuredJSONFormatter()
+ return caplog.text + "".join(repr(record.__dict__) + formatter.format(record) for record in caplog.records)
+
+
[email protected]("outcome", [200, 503, 302, "squid", "timeout", "invalid"])
[email protected]("scheme", ["http", "https"])
+def test_aliyun_keeps_real_wire_url_but_protects_transport_logs_and_exception_chains(
+ install_client: InstallClient, caplog: pytest.LogCaptureFixture, outcome: int | str, scheme: str
+) -> None:
+ caplog.set_level(logging.INFO)
+ original_factory = logging.getLogRecordFactory()
+ original_make_record = logging.Logger.makeRecord
+ previous_suppression = get_value(_SUPPRESS_INSTRUMENTATION_KEY)
+ sent: list[httpx.Request] = []
+ private_url = PRIVATE_URL.replace("https://", scheme + "://", 1)
+
+ def handler(wire: httpx.Request) -> httpx.Response:
+ sent.append(wire)
+ assert str(wire.url) == private_url
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True
+ assert set(wire.extensions["timeout"].values()) == {30.0}
+ logger = logging.getLogger("core.helper.ssrf_proxy")
+ logger.warning(
+ "Synthetic URL %s raw %s encoded %s",
+ wire.url,
+ SYNTHETIC_KEY,
+ quote(ENCODED_KEY, safe=""),
+ extra={"private_url": str(wire.url), "private_payload": {"license": SYNTHETIC_KEY}},
+ )
+ assert logging.getLogRecordFactory() is original_factory
+ if outcome in {"timeout", "invalid"}:
+ try:
+ raise ValueError(SYNTHETIC_KEY + ENCODED_KEY)
+ except ValueError as cause:
+ logger.exception("Synthetic transport exception %s", wire.url)
+ if outcome == "invalid":
+ raise ValueError(SYNTHETIC_KEY) from cause
+ raise httpx.ReadError("lost acknowledgment " + str(wire.url), request=wire) from cause
+ if outcome == "squid":
+ return httpx.Response(403, headers={"Server": "squid"})
+ assert isinstance(outcome, int)
+ return httpx.Response(outcome, headers={"Location": "https://redirect.test/"})
+
+ install_client(handler)
+ if outcome == 200:
+ result = request("POST", private_url, "aliyun", content=b"synthetic")
+ assert result.status_code == 200
+ rendered_exception = ""
+ else:
+ expected = (
+ TraceExportUncertainError
+ if outcome == "timeout"
+ else RetryableTraceDispatchError
+ if outcome == 503
+ else TraceExportRejectedError
+ )
+ with pytest.raises(expected) as caught:
+ request("POST", private_url, "aliyun", content=b"synthetic")
+ assert caught.value.__cause__ is None
+ assert caught.value.__context__ is None
+ rendered_exception = "".join(traceback.format_exception(caught.value))
+ assert len(sent) == 1 # Real SSRF max_retries=0 and no redirects remain in force.
+ assert logging.getLogRecordFactory() is original_factory
+ assert logging.Logger.makeRecord is original_make_record
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == previous_suppression
+ assert any(
+ record.name == "core.helper.ssrf_proxy" and record.levelno == logging.WARNING for record in caplog.records
+ )
+ if outcome in {200, 503, 302, "squid"}:
+ assert any(record.name == "httpx" and record.levelno == logging.INFO for record in caplog.records)
+ assert all(record.getMessage() == SAFE_HTTP_LOG_MESSAGE for record in caplog.records)
+ rendered = evidence(caplog) + rendered_exception
+ for private in [SYNTHETIC_KEY, ENCODED_KEY, quote(ENCODED_KEY, safe="")]:
+ assert private.casefold() not in rendered.casefold()
+ logging.getLogger("httpx").info("AFTER-REQUEST-PUBLIC-LOG")
+ assert "AFTER-REQUEST-PUBLIC-LOG" in caplog.text
+
+
+def test_overlapping_private_requests_keep_public_request_logs_and_restore_factory(
+ install_client: InstallClient, caplog: pytest.LogCaptureFixture
+) -> None:
+ caplog.set_level(logging.INFO)
+ original_factory = logging.getLogRecordFactory()
+ original_make_record = logging.Logger.makeRecord
+ levels = (logging.root.level, logging.getLogger("httpx").level, logging.getLogger("core.helper.ssrf_proxy").level)
+ barrier = Barrier(3)
+ first_finished = Event()
+ keys = {label: f"SYNTHETIC-PRIVATE-{label}/+?" for label in ["a", "b"]}
+ urls = {label: f"https://{label}.test/adapt_{quote(key, safe='')}/api/otlp/traces" for label, key in keys.items()}
+ urls["public"] = "https://public.test/PUBLIC-DESTINATION-VISIBLE"
+
+ def handler(wire: httpx.Request) -> httpx.Response:
+ label = wire.url.host.split(".")[0]
+ barrier.wait(timeout=5)
+ if label != "a":
+ assert first_finished.wait(timeout=5)
+ logging.getLogger("core.helper.ssrf_proxy").warning("diagnostic URL %s", wire.url)
+ return httpx.Response(200)
+
+ install_client(handler)
+
+ def run(label: str) -> int:
+ result = request("POST", urls[label], "aliyun" if label in keys else "weave")
+ if label == "a":
+ first_finished.set()
+ return result.status_code
+
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ assert list(pool.map(run, ["a", "b", "public"])) == [200, 200, 200]
+ rendered = evidence(caplog)
+ for key in keys.values():
+ assert key not in rendered
+ assert quote(key, safe="") not in rendered
+ assert "PUBLIC-DESTINATION-VISIBLE" in rendered
+ assert logging.getLogRecordFactory() is original_factory
+ assert logging.Logger.makeRecord is original_make_record
+ assert levels == (
+ logging.root.level,
+ logging.getLogger("httpx").level,
+ logging.getLogger("core.helper.ssrf_proxy").level,
+ )
+
+
+def test_nested_scopes_sanitize_before_existing_factory_and_restore_public_logging(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ caplog.set_level(logging.INFO)
+ original = logging.getLogRecordFactory()
+ observed: list[str] = []
+
+ def existing_factory(*args: object, **kwargs: object) -> logging.LogRecord:
+ record = original(*args, **kwargs)
+ observed.append(record.getMessage())
+ return record
+
+ logging.setLogRecordFactory(existing_factory)
+ try:
+ with protect_http_request_logs(True):
+ assert logging.getLogRecordFactory() is existing_factory
+ logging.getLogger("httpcore.connection").info("outer %s", SYNTHETIC_KEY)
+ with protect_http_request_logs(False):
+ logging.getLogger("httpx").info("NESTED-PUBLIC-LOG")
+ with protect_http_request_logs(True):
+ logging.getLogger("httpx").info("inner %s", ENCODED_KEY)
+ logging.getLogger("core.helper.ssrf_proxy").info("outer restored %s", SYNTHETIC_KEY)
+ assert logging.getLogRecordFactory() is existing_factory
+ assert observed == [SAFE_HTTP_LOG_MESSAGE, "NESTED-PUBLIC-LOG", SAFE_HTTP_LOG_MESSAGE, SAFE_HTTP_LOG_MESSAGE]
+ finally:
+ logging.setLogRecordFactory(original)
+
+
+def test_non_chaining_factory_replacement_during_request_receives_no_private_fields(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ caplog.set_level(logging.INFO)
+ original = logging.getLogRecordFactory()
+ observed: list[tuple[object, ...]] = []
+ try:
+ with protect_http_request_logs(True):
+ assert logging.getLogRecordFactory() is original
+
+ def externally_installed(*args: object, **kwargs: object) -> logging.LogRecord:
+ observed.append((args[4], args[5], args[6], args[8]))
+ return original(*args, **kwargs)
+
+ logging.setLogRecordFactory(externally_installed)
+ logging.getLogger("httpx").info(
+ "private %s", SYNTHETIC_KEY, extra={"url": PRIVATE_URL, "license": ENCODED_KEY}
+ )
+ assert logging.getLogRecordFactory() is externally_installed
+ assert observed == [(SAFE_HTTP_LOG_MESSAGE, (), None, None)]
+ assert "url" not in caplog.records[0].__dict__
+ assert "license" not in caplog.records[0].__dict__
+ logging.getLogger("httpx").info("EXTERNAL-FACTORY-PUBLIC-LOG")
+ assert SYNTHETIC_KEY not in evidence(caplog)
+ assert "EXTERNAL-FACTORY-PUBLIC-LOG" in caplog.text
+ finally:
+ logging.setLogRecordFactory(original)
+
+
+def test_protection_never_sets_the_record_factory(monkeypatch: pytest.MonkeyPatch) -> None:
+ original = logging.getLogRecordFactory()
+
+ def forbidden(_factory: object) -> None:
+ pytest.fail("Request protection must not configure the LogRecordFactory")
+
+ monkeypatch.setattr(logging, "setLogRecordFactory", forbidden)
+ with protect_http_request_logs(True):
+ logging.getLogger("httpx").warning("private %s", SYNTHETIC_KEY)
+ assert logging.getLogRecordFactory() is original
+ assert logging.getLogRecordFactory() is original
+
+
+def test_public_make_record_signature_preserves_public_fields_and_redacts_private_fields() -> None:
+ logger = logging.getLogger("httpx")
+ error_info = (ValueError, ValueError(SYNTHETIC_KEY), None)
+ original_factory = logging.getLogRecordFactory()
+ for sensitive in [True, False]:
+ with protect_http_request_logs(sensitive):
+ record = logger.makeRecord(
+ name="httpx",
+ level=logging.WARNING,
+ fn="fixture.py",
+ lno=42,
+ msg="message %s",
+ args=(SYNTHETIC_KEY,),
+ exc_info=error_info,
+ func="request",
+ extra={"license": ENCODED_KEY},
+ sinfo=SYNTHETIC_KEY,
+ )
+ assert logging.getLogRecordFactory() is original_factory
+ assert record.name == "httpx"
+ assert record.levelno == logging.WARNING
+ if sensitive:
+ assert record.getMessage() == SAFE_HTTP_LOG_MESSAGE
+ assert record.exc_info is None
+ assert record.stack_info is None
+ assert "license" not in record.__dict__
+ else:
+ assert record.getMessage() == "message " + SYNTHETIC_KEY
+ assert record.exc_info is error_info
+ assert record.stack_info == SYNTHETIC_KEY
+ assert record.__dict__["license"] == ENCODED_KEY
+
+
+def test_exit_does_not_overwrite_a_foreign_make_record_method() -> None:
+ original_method: Callable[..., logging.LogRecord] = logging.Logger.makeRecord
+ original_factory = logging.getLogRecordFactory()
+ try:
+ with protect_http_request_logs(True):
+
+ def externally_installed(*args: object, **kwargs: object) -> logging.LogRecord:
+ return original_method(*args, **kwargs)
+
+ logging.Logger.makeRecord = externally_installed
+ assert logging.Logger.makeRecord is externally_installed
+ assert logging.getLogRecordFactory() is original_factory
+ finally:
+ logging.Logger.makeRecord = original_method
diff --git a/api/tests/unit_tests/services/tracing/test_runtime_bridge.py b/api/tests/unit_tests/services/tracing/test_runtime_bridge.py
new file mode 100644
index 00000000000..bb2ae7ca369
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_runtime_bridge.py
@@ -0,0 +1,367 @@
+"""Verify execution hook ownership, policy gating, and fail-open business behavior."""
+
+from dataclasses import dataclass, field
+from typing import cast
+from uuid import uuid4
+
+import pytest
+from dify_agent.layers.execution_context import ApplicationTracingContext
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.app.app_config.entities import WorkflowUIBasedAppConfig
+from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext
+from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom, WorkflowAppGenerateEntity
+from core.ops.entities.trace_entity import BaseTraceInfo
+from core.ops.exceptions import RetryableTraceDispatchError
+from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
+from graphon.runtime import VariablePool
+from models.agent import (
+ Agent,
+ AgentConfigSnapshot,
+ AgentScope,
+ AgentSource,
+ WorkflowAgentBindingType,
+ WorkflowAgentNodeBinding,
+)
+from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig
+from models.model import AppMode
+from repositories.application_trace_repository import ApplicationTraceRepository
+from services.tracing import runtime_bridge
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+
+
+@dataclass
+class BridgeCase:
+ target: ApplicationTracingService
+ calls: list[tuple[str, dict[str, object]]] = field(default_factory=list)
+ failure: Exception | None = None
+ context: ApplicationTracingContext = field(
+ default_factory=lambda: ApplicationTracingContext(
+ run_id=uuid4(),
+ invocation_id=uuid4(),
+ trace_id="1" * 32,
+ parent_span_id="2" * 16,
+ session_id="session",
+ capture_content=False,
+ )
+ )
+
+
+def service(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> BridgeCase:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ target = ApplicationTracingService(ApplicationTraceRepository(sqlite_session_factory))
+ case = BridgeCase(target)
+
+ def prepare(**values: object) -> ApplicationTracingContext:
+ if case.failure is not None:
+ raise case.failure
+ case.calls.append(("prepare", values))
+ return case.context
+
+ def start(**values: object) -> None:
+ if case.failure is not None:
+ raise case.failure
+ case.calls.append(("start", values))
+
+ def terminal(_info: BaseTraceInfo) -> bool:
+ if case.failure is not None:
+ raise case.failure
+ return True
+
+ def agent_terminal(**values: object) -> None:
+ if case.failure is not None:
+ raise case.failure
+ case.calls.append(("terminal", values))
+
+ monkeypatch.setattr(target, "prepare_invocation", prepare)
+ monkeypatch.setattr(target, "start", start)
+ monkeypatch.setattr(target, "capture_terminal", terminal)
+ monkeypatch.setattr(target, "capture_agent_app_terminal", agent_terminal)
+ monkeypatch.setattr(runtime_bridge, "_service", lambda: target)
+ return case
+
+
+def run_context(app_id: str) -> DifyRunContext:
+ return DifyRunContext(
+ tenant_id="tenant",
+ app_id=app_id,
+ user_id="account",
+ user_from=UserFrom.ACCOUNT,
+ invoke_from=InvokeFrom.DEBUGGER,
+ )
+
+
+def agent_context() -> AgentAppRuntimeBuildContext:
+ return AgentAppRuntimeBuildContext(
+ dify_context=run_context("agent-app"),
+ agent_id="agent",
+ agent_config_snapshot_id="snapshot",
+ agent_soul=AgentSoulConfig(),
+ conversation_id="conversation",
+ user_query="query",
+ idempotency_key="message-id",
+ binding_id="binding",
+ backend_binding_ref="reference",
+ )
+
+
+def workflow_context(preview: bool) -> WorkflowAgentRuntimeBuildContext:
+ return WorkflowAgentRuntimeBuildContext(
+ dify_context=run_context("workflow-app"),
+ agent=Agent(
+ id="referenced-agent",
+ tenant_id="tenant",
+ name="Agent",
+ scope=AgentScope.ROSTER,
+ source=AgentSource.ROSTER,
+ ),
+ snapshot=AgentConfigSnapshot(
+ id="snapshot", tenant_id="tenant", agent_id="referenced-agent", version=1, config_snapshot=AgentSoulConfig()
+ ),
+ binding=WorkflowAgentNodeBinding(
+ id="binding",
+ tenant_id="tenant",
+ app_id="workflow-app",
+ workflow_id="workflow",
+ workflow_version="draft",
+ node_id="node",
+ binding_type=WorkflowAgentBindingType.ROSTER_AGENT,
+ agent_id="referenced-agent",
+ node_job_config=WorkflowNodeJobConfig(),
+ ),
+ workflow_id="workflow",
+ workflow_run_id=None if preview else "run",
+ node_id="node",
+ node_execution_id="node-execution",
+ attempt=2,
+ variable_pool=VariablePool(),
+ binding_id="binding",
+ backend_binding_ref="reference",
+ )
+
+
+def workflow_entity(extras: dict[str, object] | None = None) -> WorkflowAppGenerateEntity:
+ return WorkflowAppGenerateEntity.model_construct(
+ app_config=WorkflowUIBasedAppConfig(
+ app_id="app", tenant_id="tenant", app_mode=AppMode.WORKFLOW, workflow_id="workflow"
+ ),
+ extras=extras or {},
+ )
+
+
+def test_disabled_gate_never_constructs_database_service_or_reads_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+ calls = []
+
+ def forbidden() -> ApplicationTracingService:
+ calls.append(True)
+ raise AssertionError("disabled tracing must not construct a service")
+
+ monkeypatch.setattr(runtime_bridge, "_service", forbidden)
+ assert runtime_bridge.prepare_agent_app(cast(AgentAppRuntimeBuildContext, object())) is None
+ assert runtime_bridge.prepare_workflow_agent(cast(WorkflowAgentRuntimeBuildContext, object()), None) is None
+ runtime_bridge.freeze_workflow_run(
+ cast(WorkflowAppGenerateEntity, object()),
+ workflow_id="workflow",
+ workflow_run_id="run",
+ conversation_id=None,
+ graph_data={},
+ )
+ assert runtime_bridge.capture_terminal(cast(BaseTraceInfo, object())) is False
+ assert runtime_bridge.queue_enabled_at_start("app") is False
+ assert calls == []
+
+
+def test_agent_app_uses_message_identity_and_stable_conversation(service: BridgeCase) -> None:
+ assert runtime_bridge.prepare_agent_app(agent_context()) is service.context
+ assert service.calls == [
+ (
+ "prepare",
+ {
+ "tenant_id": "tenant",
+ "app_id": "agent-app",
+ "agent_id": "agent",
+ "source_kind": "message",
+ "source_id": "message-id",
+ "session_id": "conversation",
+ "node_execution_id": None,
+ "attempt": 0,
+ "call_key": "message-id",
+ },
+ )
+ ]
+
+
[email protected]("preview", [False, True])
+def test_workflow_invocation_uses_owner_app_and_execution_attempt(service: BridgeCase, preview: bool) -> None:
+ assert runtime_bridge.prepare_workflow_agent(workflow_context(preview), None) is service.context
+ values = service.calls[0][1]
+ assert values["app_id"] == "workflow-app"
+ assert values["agent_id"] == "referenced-agent"
+ assert values["source_kind"] == ("node" if preview else "workflow")
+ assert values["source_id"] == ("node-execution" if preview else "run")
+ assert values["session_id"] == values["source_id"]
+ assert values["call_key"] == "node-execution:2"
+ assert values["workflow_id"] == "workflow"
+
+
+def test_workflow_start_prefers_explicit_trace_session(service: BridgeCase) -> None:
+ entity = workflow_entity({"trace_session_id": "external-session"})
+ runtime_bridge.freeze_workflow_run(
+ entity,
+ workflow_id="workflow",
+ workflow_run_id="run",
+ conversation_id="conversation",
+ graph_data={"nodes": [{"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}]},
+ )
+ assert service.calls == [
+ (
+ "start",
+ {
+ "tenant_id": "tenant",
+ "app_id": "app",
+ "source_kind": "workflow",
+ "source_id": "run",
+ "session_id": "external-session",
+ "workflow_id": "workflow",
+ },
+ )
+ ]
+
+
+def test_business_hook_failure_logs_type_without_sensitive_exception_message(
+ service: BridgeCase, caplog: pytest.LogCaptureFixture
+) -> None:
+ service.failure = ValueError("password=private-secret")
+ assert runtime_bridge.prepare_agent_app(agent_context()) is None
+ assert "ValueError" in caplog.text
+ assert "private-secret" not in caplog.text
+
+
+def test_worker_database_failure_retries_without_legacy_fallback(service: BridgeCase) -> None:
+ service.failure = OperationalError("statement", {}, Exception("sensitive database error"))
+ with pytest.raises(RetryableTraceDispatchError, match="persistence temporarily unavailable"):
+ runtime_bridge.capture_terminal(cast(BaseTraceInfo, object()))
+
+
+def test_ordinary_workflow_does_not_create_managed_snapshot(service: BridgeCase) -> None:
+ runtime_bridge.freeze_workflow_run(
+ cast(WorkflowAppGenerateEntity, object()),
+ workflow_id="workflow",
+ workflow_run_id="run",
+ conversation_id=None,
+ graph_data={"nodes": [{"data": {"type": "llm"}}, {"data": {"type": "agent", "version": "1"}}]},
+ )
+ assert service.calls == []
+
+
+ "graph", ["not-json", "null", "[]", {}, {"nodes": {}}, {"nodes": [None, {"data": {"type": "llm"}}]}]
+)
+def test_invalid_or_ordinary_graph_cannot_enable_agent_collection(graph: object) -> None:
+ assert runtime_bridge.workflow_has_agent_v2(graph) is False
+
+
+def test_graph_string_with_agent_is_recognized() -> None:
+ assert (
+ runtime_bridge.workflow_has_agent_v2(
+ '{"nodes":[{"data":{"type":"agent","version":"2","agent_node_kind":"dify_agent"}}]}'
+ )
+ is True
+ )
+
+
[email protected]("outcome", ["succeeded", "failed", "cancelled"])
+def test_worker_terminal_handoff_preserves_outcome_without_sse(service: BridgeCase, outcome: str) -> None:
+ from typing import Literal, cast
+
+ runtime_bridge.capture_agent_app_terminal(
+ tenant_id="tenant",
+ app_id="agent-app",
+ message_id="message",
+ output="answer",
+ outcome=cast(Literal["succeeded", "failed", "cancelled"], outcome),
+ )
+ assert service.calls == [
+ (
+ "terminal",
+ {
+ "tenant_id": "tenant",
+ "app_id": "agent-app",
+ "message_id": "message",
+ "outcome": outcome,
+ "output": "answer",
+ },
+ )
+ ]
+
+
+def test_terminal_handoff_gate_and_fail_open_log_redact_sensitive_details(
+ service: BridgeCase, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ from repositories.application_trace_repository import TraceCollectionError
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
+ runtime_bridge.capture_agent_app_terminal(
+ tenant_id="tenant", app_id="app", message_id="message", outcome="failed", output=None
+ )
+ assert service.calls == []
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ service.failure = TraceCollectionError("safe_error_code")
+ runtime_bridge.capture_agent_app_terminal(
+ tenant_id="tenant", app_id="app", message_id="message", outcome="failed", output="private-content"
+ )
+ assert "safe_error_code" in caplog.text
+ assert "private-content" not in caplog.text
+
+
+def test_workflow_fail_open_logs_error_kind_without_content(
+ service: BridgeCase, caplog: pytest.LogCaptureFixture
+) -> None:
+ service.failure = ValueError("private-prompt")
+ assert runtime_bridge.prepare_workflow_agent(workflow_context(False), "conversation") is None
+ runtime_bridge.freeze_workflow_run(
+ workflow_entity(),
+ workflow_id="workflow",
+ workflow_run_id="run",
+ conversation_id="conversation",
+ graph_data={"nodes": [{"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}]},
+ )
+ assert "workflow_agent" in caplog.text
+ assert "workflow_start" in caplog.text
+ assert "ValueError" in caplog.text
+ assert "private-prompt" not in caplog.text
+
+
+ ("policy", "expected"),
+ [
+ (None, False),
+ ('{"enabled":true}', True),
+ ('{"enabled":"true"}', False),
+ ("[]", False),
+ ("invalid-private", False),
+ ],
+)
+def test_queue_policy_reads_real_app_rows_without_constructing_vendor_sdk(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ policy: str | None,
+ expected: bool,
+) -> None:
+ from extensions.ext_database import db
+ from tests.unit_tests.model_factories import make_app
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ with sqlite_session_factory.begin() as session:
+ session.add(make_app(app_id="queue-app", tenant_id="tenant", tracing=policy))
+ with sqlite_session_factory() as session:
+ monkeypatch.setattr(db, "session", session)
+ assert runtime_bridge.queue_enabled_at_start("queue-app") is expected
+ assert runtime_bridge.queue_enabled_at_start("missing") is False
+ assert runtime_bridge.queue_enabled_at_start(None) is False
+ assert "invalid-private" not in caplog.text
diff --git a/api/tests/unit_tests/services/tracing/test_tencent_configuration.py b/api/tests/unit_tests/services/tracing/test_tencent_configuration.py
new file mode 100644
index 00000000000..c3cd8d08cb2
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_tencent_configuration.py
@@ -0,0 +1,390 @@
+"""Tencent configuration checks must reject malformed input without provider I/O."""
+
+from collections.abc import Callable, Iterator, Sequence
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import Never
+
+import grpc # type: ignore[import-untyped]
+import pytest
+from dify_trace_tencent.configuration import (
+ TencentConfigurationError,
+ console_url,
+ normalize_configuration,
+ verify_configuration,
+)
+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+ ExportTraceServiceRequest,
+ ExportTraceServiceResponse,
+)
+
+
+def configuration_values(**overrides: object) -> dict[str, object]:
+ return {
+ "endpoint": "https://collector.example:4320",
+ "service_name": "dify-agent",
+ "token": "synthetic-token",
+ **overrides,
+ }
+
+
[email protected](autouse=True)
+def _forbid_provider_io(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ attempts: list[str] = []
+
+ def forbidden(*_args: object, **_kwargs: object) -> Never:
+ attempts.append("provider_io")
+ raise AssertionError("Local configuration validation must not construct clients or perform network I/O")
+
+ for target in (
+ "dify_trace_tencent.tencent_trace.TencentDataTrace.__init__",
+ "dify_trace_tencent.client.TencentTraceClient.__init__",
+ "socket.socket",
+ "socket.create_connection",
+ "socket.getaddrinfo",
+ "grpc.secure_channel",
+ "grpc.insecure_channel",
+ ):
+ monkeypatch.setattr(target, forbidden)
+ yield
+ assert attempts == []
+
+
+def test_normalization_accepts_read_only_mapping_and_preserves_opaque_token() -> None:
+ values = configuration_values(
+ endpoint=" https://COLLECTOR.Example:4320/ ",
+ service_name=" dify-agent ",
+ token=" opaque token/+== ",
+ )
+ original = dict(values)
+
+ result = normalize_configuration(MappingProxyType(values))
+
+ assert result.model_dump() == {
+ "endpoint": "https://collector.example:4320",
+ "service_name": "dify-agent",
+ "token": " opaque token/+== ",
+ }
+ assert values == original
+
+
[email protected]("service_name", ["a", "a" * 63, "app-1", "a--b"])
+def test_accepts_documented_service_name_boundaries(service_name: str) -> None:
+ result = normalize_configuration(configuration_values(service_name=service_name))
+
+ assert result.service_name == service_name
+
+
+ "service_name",
+ [None, 42, "", " ", "dify_app", "Dify", "1app", "-app", "app-", "a" * 64, "应用", "a.b", "a/b", "a b", "a\nb"],
+)
+def test_rejects_invalid_service_names_before_provider_io(service_name: object) -> None:
+ with pytest.raises(TencentConfigurationError) as caught:
+ normalize_configuration(configuration_values(service_name=service_name))
+
+ assert caught.value.code == "invalid_service_name"
+
+
+ ("endpoint", "expected"),
+ [
+ ("https://COLLECTOR.Example:4320/", "https://collector.example:4320"),
+ ("http://collector.example:4319", "http://collector.example:4319"),
+ ("https://collector.example/", "https://collector.example"),
+ ("http://collector.example", "http://collector.example"),
+ ("https://collector.example:443", "https://collector.example:443"),
+ ("http://127.0.0.1:1/", "http://127.0.0.1:1"),
+ ("https://[2001:DB8::1]:65535/", "https://[2001:db8::1]:65535"),
+ ("https://BÜCHER.example:4320/", "https://xn--bcher-kva.example:4320"),
+ ],
+)
+def test_normalizes_origins_without_changing_explicit_port(endpoint: str, expected: str) -> None:
+ result = normalize_configuration(configuration_values(endpoint=endpoint))
+
+ assert result.endpoint == expected
+
+
+ "endpoint",
+ [
+ None,
+ 4320,
+ "",
+ " ",
+ "collector.example:4320",
+ "ftp://collector.example:4320",
+ "https://",
+ "https:///",
+ "https://user:[email protected]:4320",
+ "https://@collector.example:4320",
+ "https://collector.example:4320/v1/traces",
+ "https://collector.example:4320//",
+ "https://collector.example:4320?token=synthetic-secret",
+ "https://collector.example:4320?",
+ "https://collector.example:4320#project",
+ "https://collector.example:4320#",
+ "https://collector.example\\@other.example:4320",
+ "https://col lector.example:4320",
+ "https://collector.example:\n4320",
+ "https://collector.example:\t4320",
+ "https://collector\x00.example:4320",
+ "https://collector.example:0",
+ "https://collector.example:65536",
+ "https://collector.example:-1",
+ "https://collector.example:invalid",
+ "https://collector.example:",
+ "https://[2001:db8::1",
+ "https://[not-an-ipv6-address]:4320",
+ ],
+)
+def test_rejects_invalid_endpoints_before_provider_io(endpoint: object) -> None:
+ with pytest.raises(TencentConfigurationError) as caught:
+ normalize_configuration(configuration_values(endpoint=endpoint))
+
+ assert caught.value.code == "invalid_endpoint"
+ assert "synthetic-secret" not in str(caught.value)
+ assert "synthetic-token" not in str(caught.value)
+
+
[email protected]("token", [None, 42, "", " ", "\t\n"])
+def test_rejects_missing_or_blank_tokens_without_provider_io(token: object) -> None:
+ with pytest.raises(TencentConfigurationError) as caught:
+ normalize_configuration(configuration_values(token=token))
+
+ assert caught.value.code == "invalid_token"
+
+
+def test_masked_token_survives_local_validation_for_patch_restoration() -> None:
+ result = normalize_configuration(configuration_values(token="***"))
+
+ assert result.token == "***"
+
+
+ ("field", "error_code"),
+ [("endpoint", "invalid_endpoint"), ("service_name", "invalid_service_name"), ("token", "invalid_token")],
+)
+def test_missing_fields_do_not_receive_legacy_defaults(field: str, error_code: str) -> None:
+ values = configuration_values()
+ del values[field]
+
+ with pytest.raises(TencentConfigurationError) as caught:
+ normalize_configuration(values)
+
+ assert caught.value.code == error_code
+
+
+def test_console_url_is_constant_without_configuration_or_provider_io() -> None:
+ assert console_url() == "https://console.cloud.tencent.com/apm"
+
+
+@dataclass
+class ExportCall:
+ request: ExportTraceServiceRequest
+ metadata: tuple[tuple[str, str], ...]
+ timeout: float
+ wait_for_ready: bool
+
+
+@dataclass
+class CollectorChannel:
+ target: str
+ secure: bool
+ options: tuple[tuple[str, int], ...]
+ response_bytes: bytes
+ failure: Exception | None = None
+ fail_stub_construction: bool = False
+ calls: list[ExportCall] = field(default_factory=list)
+ close_count: int = 0
+
+ def unary_unary(
+ self,
+ method: str,
+ request_serializer: Callable[[ExportTraceServiceRequest], bytes],
+ response_deserializer: Callable[[bytes], ExportTraceServiceResponse],
+ **_kwargs: object,
+ ) -> Callable[..., ExportTraceServiceResponse]:
+ assert method == "/opentelemetry.proto.collector.trace.v1.TraceService/Export"
+ if self.fail_stub_construction:
+ raise RuntimeError("private-stub-detail synthetic-token")
+
+ def invoke(
+ payload: ExportTraceServiceRequest,
+ metadata: Sequence[tuple[str, str]],
+ timeout: float,
+ wait_for_ready: bool,
+ ) -> ExportTraceServiceResponse:
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True
+ request = ExportTraceServiceRequest.FromString(request_serializer(payload))
+ self.calls.append(
+ ExportCall(request=request, metadata=tuple(metadata), timeout=timeout, wait_for_ready=wait_for_ready)
+ )
+ if self.failure is not None:
+ raise self.failure
+ return response_deserializer(self.response_bytes)
+
+ return invoke
+
+ def close(self) -> None:
+ self.close_count += 1
+
+
+@dataclass
+class CollectorBoundary:
+ response_bytes: bytes = b""
+ failure: Exception | None = None
+ fail_stub_construction: bool = False
+ fail_channel_construction: bool = False
+ channels: list[CollectorChannel] = field(default_factory=list)
+
+ def open_channel(self, target: str, secure: bool, options: Sequence[tuple[str, int]]) -> CollectorChannel:
+ if self.fail_channel_construction:
+ raise RuntimeError("private-channel-detail synthetic-token")
+ channel = CollectorChannel(
+ target=target,
+ secure=secure,
+ options=tuple(options),
+ response_bytes=self.response_bytes,
+ failure=self.failure,
+ fail_stub_construction=self.fail_stub_construction,
+ )
+ self.channels.append(channel)
+ return channel
+
+ def secure_channel(self, target: str, credentials: object, options: Sequence[tuple[str, int]]) -> CollectorChannel:
+ assert credentials is not None
+ return self.open_channel(target, secure=True, options=options)
+
+ def insecure_channel(self, target: str, options: Sequence[tuple[str, int]]) -> CollectorChannel:
+ return self.open_channel(target, secure=False, options=options)
+
+
+def collector(monkeypatch: pytest.MonkeyPatch, _forbid_provider_io: None) -> Iterator[CollectorBoundary]:
+ prior_suppression = get_value(_SUPPRESS_INSTRUMENTATION_KEY)
+ boundary = CollectorBoundary()
+ monkeypatch.setattr("grpc.secure_channel", boundary.secure_channel)
+ monkeypatch.setattr("grpc.insecure_channel", boundary.insecure_channel)
+ yield boundary
+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == prior_suppression
+
+
+ ("endpoint", "target", "secure"),
+ [
+ ("https://collector.example", "collector.example:4317", True),
+ ("http://collector.example", "collector.example:4317", False),
+ ("https://collector.example:4320", "collector.example:4320", True),
+ ("http://collector.example:4319", "collector.example:4319", False),
+ ("https://[2001:db8::1]:4320", "[2001:db8::1]:4320", True),
+ ],
+)
+def test_probe_serializes_one_empty_export_and_closes_channel(
+ collector: CollectorBoundary, endpoint: str, target: str, secure: bool
+) -> None:
+ config = normalize_configuration(configuration_values(endpoint=endpoint))
+
+ assert verify_configuration(config) == config
+
+ assert len(collector.channels) == 1
+ channel = collector.channels[0]
+ assert (channel.target, channel.secure) == (target, secure)
+ assert dict(channel.options) == {"grpc.enable_retries": 0, "grpc.max_receive_message_length": 4 * 1024 * 1024}
+ assert len(channel.calls) == 1
+ call = channel.calls[0]
+ assert list(call.request.resource_spans) == []
+ assert call.request.SerializeToString() == b""
+ assert call.metadata == (("authorization", "Bearer synthetic-token"),)
+ assert call.timeout == 10
+ assert call.wait_for_ready is False
+ assert channel.close_count == 1
+
+
+def test_probe_credentials_belong_to_each_request(collector: CollectorBoundary) -> None:
+ for token in ("first-synthetic-token", "second-synthetic-token"):
+ verify_configuration(normalize_configuration(configuration_values(token=token)))
+
+ assert len(collector.channels) == 2
+ assert [channel.calls[0].metadata for channel in collector.channels] == [
+ (("authorization", "Bearer first-synthetic-token"),),
+ (("authorization", "Bearer second-synthetic-token"),),
+ ]
+ assert all(channel.close_count == 1 for channel in collector.channels)
+
+
+ ("rejected_spans", "error_message"),
+ [(1, ""), (0, "private-collector-detail synthetic-token")],
+)
+def test_probe_rejects_partial_response_and_closes_channel(
+ collector: CollectorBoundary, rejected_spans: int, error_message: str
+) -> None:
+ response = ExportTraceServiceResponse()
+ response.partial_success.rejected_spans = rejected_spans
+ response.partial_success.error_message = error_message
+ collector.response_bytes = response.SerializeToString()
+
+ with pytest.raises(TencentConfigurationError) as caught:
+ verify_configuration(normalize_configuration(configuration_values()))
+
+ assert caught.value.code == "unexpected_response"
+ assert "private-collector-detail" not in str(caught.value)
+ assert "synthetic-token" not in str(caught.value)
+ assert collector.channels[0].close_count == 1
+
+
+ "failure",
+ [
+ grpc.RpcError("private-rpc-detail synthetic-token"),
+ ValueError("private-serialization-detail synthetic-token"),
+ ],
+)
+def test_probe_transport_failures_are_safe_and_close_channel(collector: CollectorBoundary, failure: Exception) -> None:
+ collector.failure = failure
+
+ with pytest.raises(TencentConfigurationError) as caught:
+ verify_configuration(normalize_configuration(configuration_values()))
+
+ assert caught.value.code == "connection_failed"
+ assert "private-" not in str(caught.value)
+ assert "synthetic-token" not in str(caught.value)
+ assert len(collector.channels[0].calls) == 1
+ assert collector.channels[0].close_count == 1
+
+
+def test_probe_unreadable_response_is_safe_and_closes_channel(collector: CollectorBoundary) -> None:
+ collector.response_bytes = b"\xff"
+
+ with pytest.raises(TencentConfigurationError) as caught:
+ verify_configuration(normalize_configuration(configuration_values()))
+
+ assert caught.value.code == "connection_failed"
+ assert collector.channels[0].close_count == 1
+
+
+def test_probe_stub_construction_failure_is_safe_and_closes_channel(collector: CollectorBoundary) -> None:
+ collector.fail_stub_construction = True
+
+ with pytest.raises(TencentConfigurationError) as caught:
+ verify_configuration(normalize_configuration(configuration_values()))
+
+ assert caught.value.code == "connection_failed"
+ assert "private-stub-detail" not in str(caught.value)
+ assert "synthetic-token" not in str(caught.value)
+ assert collector.channels[0].calls == []
+ assert collector.channels[0].close_count == 1
+
+
+def test_probe_channel_construction_failure_has_fixed_safe_error(collector: CollectorBoundary) -> None:
+ collector.fail_channel_construction = True
+
+ with pytest.raises(TencentConfigurationError) as caught:
+ verify_configuration(normalize_configuration(configuration_values()))
+
+ assert caught.value.code == "connection_failed"
+ assert "private-channel-detail" not in str(caught.value)
+ assert "synthetic-token" not in str(caught.value)
+ assert collector.channels == []
diff --git a/api/tests/unit_tests/services/tracing/test_tencent_legacy_sessions.py b/api/tests/unit_tests/services/tracing/test_tencent_legacy_sessions.py
new file mode 100644
index 00000000000..7ab192b9903
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_tencent_legacy_sessions.py
@@ -0,0 +1,194 @@
+"""Pure legacy Tencent session regressions; IDs here are builder-local IDs."""
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from dify_trace_tencent.entities.tencent_trace_entity import SpanData
+from dify_trace_tencent.span_builder import TencentSpanBuilder
+from dify_trace_tencent.utils import TencentTraceUtils
+from opentelemetry.trace import StatusCode
+from pydantic import ValidationError
+
+from core.ops.entities.trace_entity import WorkflowTraceInfo
+from graphon.entities import WorkflowNodeExecution
+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus
+
+RUN_ID = "00000000-0000-4000-8000-000000000001"
+WORKFLOW_ID = "00000000-0000-4000-8000-000000000002"
+TENANT_ID = "00000000-0000-4000-8000-000000000003"
+STARTED = datetime(2026, 10, 17, tzinfo=UTC)
+
+
+def workflow_trace(metadata: dict[str, object]) -> WorkflowTraceInfo:
+ session_id = metadata.get("conversation_id")
+ return WorkflowTraceInfo(
+ workflow_id=WORKFLOW_ID,
+ tenant_id=TENANT_ID,
+ workflow_run_id=RUN_ID,
+ workflow_run_elapsed_time=3.0,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={"query": "synthetic input"},
+ workflow_run_outputs={"answer": "synthetic answer"},
+ workflow_run_version="1",
+ error="",
+ total_tokens=32,
+ prompt_tokens=13,
+ completion_tokens=19,
+ file_list=[],
+ query="synthetic input",
+ metadata=metadata,
+ conversation_id=session_id if isinstance(session_id, str) else None,
+ start_time=STARTED,
+ end_time=STARTED + timedelta(seconds=3),
+ )
+
+
+def workflow_node(node_type: str, index: int = 1) -> WorkflowNodeExecution:
+ return WorkflowNodeExecution(
+ id=f"00000000-0000-4000-8000-{index + 10:012d}",
+ workflow_id=WORKFLOW_ID,
+ workflow_execution_id=RUN_ID,
+ index=index,
+ node_id=f"node-{index}",
+ node_type=node_type,
+ title={BuiltinNodeTypes.START: "Start", BuiltinNodeTypes.END: "End"}.get(node_type, "Synthetic node"),
+ inputs={"query": "synthetic input"},
+ process_data={"model_name": "synthetic-model", "model_provider": "synthetic-provider"},
+ outputs={
+ "text": "synthetic answer",
+ "usage": {"prompt_tokens": 13, "completion_tokens": 19, "total_tokens": 32},
+ "finish_reason": "stop",
+ },
+ status=WorkflowNodeExecutionStatus.SUCCEEDED,
+ created_at=STARTED + timedelta(seconds=index - 1),
+ finished_at=STARTED + timedelta(seconds=index),
+ )
+
+
+def build_node_span(trace: WorkflowTraceInfo, node: WorkflowNodeExecution) -> SpanData:
+ trace_id = TencentTraceUtils.convert_to_trace_id(trace.workflow_run_id)
+ workflow_span_id = TencentTraceUtils.convert_to_span_id(trace.workflow_run_id, "workflow")
+ if node.node_type == BuiltinNodeTypes.LLM:
+ return TencentSpanBuilder.build_workflow_llm_span(trace_id, workflow_span_id, trace, node)
+ if node.node_type == BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL:
+ return TencentSpanBuilder.build_workflow_retrieval_span(trace_id, workflow_span_id, trace, node)
+ return TencentSpanBuilder.build_workflow_task_span(trace_id, workflow_span_id, trace, node)
+
+
+ "node_type", [BuiltinNodeTypes.START, BuiltinNodeTypes.LLM, BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL]
+)
+ ("metadata", "expected_session"),
+ [
+ pytest.param({}, "", id="missing"),
+ pytest.param({"conversation_id": None}, "", id="none"),
+ pytest.param({"conversation_id": ""}, "", id="empty"),
+ pytest.param({"conversation_id": "owned-conversation"}, "owned-conversation", id="conversation"),
+ ],
+)
+def test_workflow_builders_accept_optional_sessions_without_rewriting_metadata(
+ node_type: str, metadata: dict[str, object], expected_session: str
+) -> None:
+ trace = workflow_trace(metadata)
+ before = dict(trace.metadata)
+
+ span = build_node_span(trace, workflow_node(node_type))
+
+ assert span.attributes["gen_ai.session.id"] == expected_session
+ assert trace.metadata == before
+
+
+def complete_workflow_spans(trace: WorkflowTraceInfo) -> list[SpanData]:
+ spans = TencentSpanBuilder.build_workflow_spans(
+ trace, TencentTraceUtils.convert_to_trace_id(trace.workflow_run_id), "synthetic-user"
+ )
+ spans.extend(
+ build_node_span(trace, workflow_node(node_type, index))
+ for index, node_type in enumerate((BuiltinNodeTypes.START, BuiltinNodeTypes.LLM, BuiltinNodeTypes.END), start=1)
+ )
+ return spans
+
+
+def assert_usage_is_not_duplicated(spans: list[SpanData]) -> None:
+ generations = [span for span in spans if span.attributes.get("gen_ai.span.kind") == "GENERATION"]
+ assert len(generations) == 1
+ generation = generations[0]
+ assert generation.attributes["gen_ai.usage.input_tokens"] == "13"
+ assert generation.attributes["gen_ai.usage.output_tokens"] == "19"
+ assert generation.attributes["gen_ai.usage.total_tokens"] == "32"
+ assert sum(int(span.attributes.get("gen_ai.usage.total_tokens", "0")) for span in spans) == 32
+
+
+def test_workflow_without_conversation_keeps_start_llm_end_under_one_root() -> None:
+ trace = workflow_trace({"conversation_id": None})
+
+ spans = complete_workflow_spans(trace)
+
+ assert [span.name for span in spans] == ["workflow", "Start", "GENERATION", "End"]
+ root = spans[0]
+ assert root.parent_span_id is None
+ assert all(span.parent_span_id == root.span_id for span in spans[1:])
+ assert all(span.trace_id == root.trace_id for span in spans)
+ assert len({span.span_id for span in spans}) == 4
+ assert all(span.attributes["gen_ai.session.id"] == "" for span in spans[1:])
+ assert trace.metadata["conversation_id"] is None
+ assert_usage_is_not_duplicated(spans)
+
+
+def test_chatflow_keeps_conversation_and_message_workflow_parent_layer() -> None:
+ trace = workflow_trace({"conversation_id": "owned-conversation"})
+
+ spans = complete_workflow_spans(trace)
+
+ assert [span.name for span in spans] == ["message", "workflow", "Start", "GENERATION", "End"]
+ message, workflow = spans[:2]
+ assert message.parent_span_id is None
+ assert workflow.parent_span_id == message.span_id
+ assert all(span.parent_span_id == workflow.span_id for span in spans[2:])
+ assert all(span.trace_id == message.trace_id for span in spans)
+ assert len({span.span_id for span in spans}) == 5
+ assert message.attributes["gen_ai.session.id"] == "owned-conversation"
+ assert all(span.attributes["gen_ai.session.id"] == "owned-conversation" for span in spans[2:])
+ assert_usage_is_not_duplicated(spans)
+
+
[email protected]("status", [WorkflowNodeExecutionStatus.FAILED, WorkflowNodeExecutionStatus.EXCEPTION])
+def test_error_status_and_description_survive_missing_session(status: WorkflowNodeExecutionStatus) -> None:
+ trace = workflow_trace({"conversation_id": None})
+ trace.error = "synthetic workflow failure"
+ trace.workflow_run_status = "failed"
+ node = workflow_node(BuiltinNodeTypes.LLM)
+ node.status = status
+ node.error = "synthetic node failure"
+
+ root = TencentSpanBuilder.build_workflow_spans(
+ trace, TencentTraceUtils.convert_to_trace_id(trace.workflow_run_id), "synthetic-user"
+ )[0]
+ span = build_node_span(trace, node)
+
+ assert root.status.status_code == StatusCode.ERROR
+ assert root.status.description == "synthetic workflow failure"
+ assert span.status.status_code == StatusCode.ERROR
+ assert span.status.description == "synthetic node failure"
+ assert span.parent_span_id == root.span_id
+ assert span.attributes["gen_ai.session.id"] == ""
+ assert_usage_is_not_duplicated([root, span])
+
+
+ "node_type", [BuiltinNodeTypes.START, BuiltinNodeTypes.LLM, BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL]
+)
+ "invalid_session", [0, False, [], {}, 123], ids=["zero", "false", "empty-list", "empty-map", "int"]
+)
+def test_non_string_sessions_remain_validation_errors(node_type: str, invalid_session: object) -> None:
+ trace = workflow_trace({"conversation_id": invalid_session})
+
+ with pytest.raises(ValidationError) as caught:
+ build_node_span(trace, workflow_node(node_type))
+
+ errors = caught.value.errors(include_input=False, include_url=False)
+ assert len(errors) == 1
+ assert errors[0]["loc"] == ("attributes", "gen_ai.session.id")
+ assert errors[0]["type"] == "string_type"
diff --git a/api/tests/unit_tests/services/tracing/test_weave_configuration.py b/api/tests/unit_tests/services/tracing/test_weave_configuration.py
new file mode 100644
index 00000000000..63d4983119b
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_weave_configuration.py
@@ -0,0 +1,258 @@
+"""Read-only Weave validation at the real config/schema and HTTP boundaries."""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import cast
+
+import httpx
+import pytest
+from dify_trace_weave.config import WeaveConfig
+from dify_trace_weave.configuration import (
+ WeaveConfigurationError,
+ console_url,
+ project_url,
+ verify_configuration,
+)
+from weave.trace_server.trace_server_interface import CallsQueryStatsReq
+
+from core.helper import ssrf_proxy
+
+
+def object_dict(value: object) -> dict[str, object]:
+ assert isinstance(value, dict)
+ return cast(dict[str, object], value)
+
+
+def response(url: str, body: object, *, status: int = 200) -> httpx.Response:
+ return httpx.Response(status, json=body, request=httpx.Request("POST", url))
+
+
+@dataclass
+class ReadAPI:
+ calls: list[tuple[str, dict[str, object]]] = field(default_factory=list)
+ override: Callable[[str, dict[str, object]], httpx.Response] | None = None
+ project_fields: dict[str, object] = field(default_factory=dict)
+ project_exists: bool = True
+ viewer_exists: bool = True
+ count: object = 0
+
+ def post(self, url: str, **options: object) -> httpx.Response:
+ self.calls.append((url, options))
+ body = object_dict(options["json"])
+ if self.override is not None:
+ return self.override(url, body)
+ if url.endswith("/graphql"):
+ query = body["query"]
+ assert isinstance(query, str)
+ assert "mutation" not in query
+ if "DifyWeaveViewer" in query:
+ return response(url, {"data": {"viewer": {"id": "viewer-id", "entity": "default-team"}}})
+ variables = object_dict(body["variables"])
+ project: dict[str, object] = {
+ "id": "project-id",
+ "name": variables["project"],
+ "entityName": variables["entity"],
+ "readOnly": False,
+ }
+ project.update(self.project_fields)
+ return response(
+ url,
+ {
+ "data": {
+ "viewer": {"id": "viewer-id"} if self.viewer_exists else None,
+ "project": project if self.project_exists else None,
+ }
+ },
+ )
+ assert url.endswith("/calls/query_stats")
+ query_request = CallsQueryStatsReq.model_validate(body)
+ assert query_request.limit == 1
+ return response(url, {"count": self.count})
+
+
+def read_api(monkeypatch: pytest.MonkeyPatch) -> ReadAPI:
+ api = ReadAPI()
+ monkeypatch.setattr(ssrf_proxy, "post", api.post)
+ return api
+
+
+ ("entity", "project", "expected_entity", "requests"),
+ [("team", "demo", "team", 2), (None, "team/demo", "team", 2), (None, "demo", "default-team", 3)],
+)
+def test_verify_resolves_project_with_request_scoped_authentication(
+ read_api: ReadAPI, entity: str | None, project: str, expected_entity: str, requests: int
+) -> None:
+ result = verify_configuration(WeaveConfig(api_key="synthetic-key", entity=entity, project=project))
+ assert result.entity == expected_entity
+ assert result.project == "demo"
+ assert result.project_url == f"https://wandb.ai/{expected_entity}/demo"
+ assert len(read_api.calls) == requests
+ for _url, options in read_api.calls:
+ assert options["auth"] == ("api", "synthetic-key")
+ assert options["max_retries"] == 0
+ assert options["follow_redirects"] is False
+ assert options["timeout"] == 30.0
+ assert object_dict(read_api.calls[-1][1]["json"])["project_id"] == f"{expected_entity}/demo"
+
+
+ ("entity", "project"),
+ [
+ ("team", ""),
+ ("team", ".."),
+ ("/", "demo"),
+ (None, "/demo"),
+ (None, "team/"),
+ (None, "team/sub/demo"),
+ ("other", "team/demo"),
+ ("team", "de\\mo"),
+ ],
+)
+def test_invalid_target_is_rejected_before_network(read_api: ReadAPI, entity: str | None, project: str) -> None:
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity=entity, project=project))
+ assert caught.value.code == "invalid_project"
+ assert read_api.calls == []
+
+
+def test_private_endpoints_preserve_paths_and_encode_project_links(read_api: ReadAPI) -> None:
+ config = WeaveConfig(
+ api_key="synthetic-key",
+ entity="team one",
+ project="demo & notes#",
+ host="http://wandb.internal/wandb/",
+ endpoint="https://trace.internal/proxy/",
+ )
+ result = verify_configuration(config)
+ assert result.project_url == "http://wandb.internal/wandb/team%20one/demo%20%26%20notes%23"
+ assert [url for url, _options in read_api.calls] == [
+ "http://wandb.internal/wandb/graphql",
+ "https://trace.internal/proxy/calls/query_stats",
+ ]
+
+
+ "host", ["https://user:[email protected]", "https://host.test/?secret=x", "https://host.test/#x"]
+)
+def test_unsafe_host_is_rejected_without_authentication_request(read_api: ReadAPI, host: str) -> None:
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo", host=host))
+ assert caught.value.code == "invalid_endpoint"
+ assert read_api.calls == []
+
+
+ ("status", "code"),
+ [
+ (401, "authentication_failed"),
+ (403, "permission_denied"),
+ (302, "endpoint_unavailable"),
+ (404, "endpoint_unavailable"),
+ (429, "endpoint_unavailable"),
+ (503, "endpoint_unavailable"),
+ ],
+)
+def test_http_errors_are_sanitized_and_never_retried(read_api: ReadAPI, status: int, code: str) -> None:
+ read_api.override = lambda url, _body: response(url, {"error": "never-log-secret"}, status=status)
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="never-log-secret", entity="team", project="demo"))
+ assert caught.value.code == code
+ assert "never-log-secret" not in str(caught.value)
+ assert len(read_api.calls) == 1
+
+
+ ("fields", "code"),
+ [
+ ({"readOnly": True}, "project_read_only"),
+ ({"readOnly": None}, "project_permission_unknown"),
+ ({"readOnly": "false"}, "malformed_response"),
+ ({"id": ""}, "malformed_response"),
+ ({"name": "other"}, "invalid_project"),
+ ({"entityName": "other"}, "invalid_project"),
+ ],
+)
+def test_project_permission_and_identity_are_checked(read_api: ReadAPI, fields: dict[str, object], code: str) -> None:
+ read_api.project_fields = fields
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))
+ assert caught.value.code == code
+ assert len(read_api.calls) == 1
+
+
+def test_missing_project_is_not_created(read_api: ReadAPI) -> None:
+ read_api.project_exists = False
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))
+ assert caught.value.code == "project_not_found_or_inaccessible"
+ assert len(read_api.calls) == 1
+
+
+def test_anonymous_viewer_cannot_validate_a_public_project(read_api: ReadAPI) -> None:
+ read_api.viewer_exists = False
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))
+ assert caught.value.code == "authentication_failed"
+
+
[email protected]("count", [-1, True, "0", None])
+def test_invalid_trace_endpoint_response_is_rejected(read_api: ReadAPI, count: object) -> None:
+ read_api.count = count
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))
+ assert caught.value.code == "malformed_response"
+ assert len(read_api.calls) == 2
+
+
+ ("body", "code"),
+ [
+ ({"errors": [{"message": "never-log-secret"}]}, "graphql_error"),
+ ({"errors": "never-log-secret"}, "malformed_response"),
+ ({"data": []}, "malformed_response"),
+ ({"data": {}}, "malformed_response"),
+ ],
+)
+def test_graphql_errors_and_invalid_envelopes_are_rejected(
+ read_api: ReadAPI, body: dict[str, object], code: str
+) -> None:
+ read_api.override = lambda url, _body: response(url, body)
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="never-log-secret", entity="team", project="demo"))
+ assert caught.value.code == code
+ assert "never-log-secret" not in str(caught.value)
+
+
+def test_connection_failure_has_no_raw_exception_or_secret(read_api: ReadAPI) -> None:
+ def fail(_url: str, _body: dict[str, object]) -> httpx.Response:
+ raise httpx.ReadTimeout("never-log-secret")
+
+ read_api.override = fail
+ with pytest.raises(WeaveConfigurationError) as caught:
+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))
+ assert caught.value.code == "connection_failed"
+ assert "never-log-secret" not in str(caught.value)
+ assert caught.value.__cause__ is None
+
+
+def test_sequential_tenants_use_only_their_supplied_credentials(read_api: ReadAPI) -> None:
+ for tenant in ("a", "b"):
+ verify_configuration(WeaveConfig(api_key=f"synthetic-{tenant}", entity=f"team-{tenant}", project="demo"))
+ assert [options["auth"] for _url, options in read_api.calls] == [
+ ("api", "synthetic-a"),
+ ("api", "synthetic-a"),
+ ("api", "synthetic-b"),
+ ("api", "synthetic-b"),
+ ]
+
+
+def test_project_url_and_console_fallback_are_pure(read_api: ReadAPI) -> None:
+ assert project_url(WeaveConfig(api_key="synthetic", project="demo")) is None
+ assert project_url(WeaveConfig(api_key="synthetic", project="team/demo")) == "https://wandb.ai/team/demo"
+ assert console_url(WeaveConfig(api_key="synthetic", project="demo")) == "https://wandb.ai"
+ assert console_url(WeaveConfig(api_key="synthetic", project="demo", host="https://private.test/wandb")) == (
+ "https://private.test/wandb"
+ )
+ assert read_api.calls == []
diff --git a/api/tests/unit_tests/services/tracing/test_workflow_tool_tracing_boundary.py b/api/tests/unit_tests/services/tracing/test_workflow_tool_tracing_boundary.py
new file mode 100644
index 00000000000..4c0a881ba12
--- /dev/null
+++ b/api/tests/unit_tests/services/tracing/test_workflow_tool_tracing_boundary.py
@@ -0,0 +1,292 @@
+"""Hidden Workflow Tool Agents must not invent a trace owned by the source app."""
+
+import json
+from copy import deepcopy
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.entities.trace_entity import WorkflowTraceInfo
+from core.ops.unified_trace.trace_builder import CanonicalTraceBuilder
+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository
+from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext
+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType
+from models.application_trace import ApplicationTraceRun
+from models.enums import CreatorUserRole
+from models.model import App, AppMode, TraceAppConfig
+from models.workflow import (
+ WorkflowNodeExecutionModel,
+ WorkflowNodeExecutionTriggeredFrom,
+ WorkflowRun,
+ WorkflowRunTriggeredFrom,
+)
+from repositories.application_trace_repository import ApplicationTraceRepository
+from services.tracing import runtime_bridge
+from services.tracing.normalization import remap_outer_trace
+from services.tracing.service import ApplicationTracingService
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.model_factories import make_account, make_app, make_workflow
+from tests.unit_tests.services.tracing.test_application_trace_collection import (
+ Collection,
+ batch_for,
+ identifier,
+ ingest,
+)
+from tests.unit_tests.services.tracing.test_runtime_bridge import workflow_context, workflow_entity
+
+
+def seed(
+ sessions: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ root_has_agent: bool,
+ source_enabled: bool = True,
+) -> tuple[ApplicationTracingService, WorkflowAgentRuntimeBuildContext, dict[str, str]]:
+ ids = {
+ name: identifier("tool-boundary-" + name)
+ for name in [
+ "tenant",
+ "root",
+ "source",
+ "root_workflow",
+ "source_workflow",
+ "run",
+ "root_agent",
+ "root_tool",
+ "child",
+ "actor",
+ ]
+ }
+ root_graph: dict[str, list[dict[str, object]]] = {
+ "nodes": [{"id": "tool", "data": {"type": "tool", "title": "Root tool"}}],
+ "edges": [],
+ }
+ if root_has_agent:
+ root_graph["nodes"].append(
+ {
+ "id": "root-agent",
+ "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Root Agent"},
+ }
+ )
+ now = datetime(2026, 10, 21, tzinfo=UTC)
+ with sessions.begin() as session:
+ for app, provider, enabled, content in [
+ ("root", "langfuse", True, False),
+ ("source", "langsmith", source_enabled, True),
+ ]:
+ session.add(
+ make_app(
+ app_id=ids[app],
+ tenant_id=ids["tenant"],
+ mode=AppMode.WORKFLOW,
+ tracing=json.dumps({"enabled": enabled, "tracing_provider": provider, "capture_content": content}),
+ )
+ )
+ session.add(
+ make_workflow(
+ workflow_id=ids[app + "_workflow"],
+ app_id=ids[app],
+ tenant_id=ids["tenant"],
+ graph=root_graph if app == "root" else {"nodes": [], "edges": []},
+ )
+ )
+ session.add(
+ TraceAppConfig(
+ app_id=ids[app],
+ tracing_provider=provider,
+ tracing_config={"project_url": "https://tracing.test/" + app},
+ )
+ )
+ session.add(
+ WorkflowRun(
+ id=ids["run"],
+ tenant_id=ids["tenant"],
+ app_id=ids["root"],
+ workflow_id=ids["root_workflow"],
+ type=WorkflowType.WORKFLOW,
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ version="1",
+ graph=json.dumps(root_graph),
+ inputs="{}",
+ status=WorkflowExecutionStatus.SUCCEEDED,
+ created_by=ids["actor"],
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_at=now,
+ finished_at=now + timedelta(seconds=3),
+ )
+ )
+ for key, node_type in [("root_agent", "agent"), ("root_tool", "tool"), ("child", "agent")]:
+ child = key == "child"
+ app = "source" if child else "root"
+ session.add(
+ WorkflowNodeExecutionModel(
+ id=ids[key],
+ tenant_id=ids["tenant"],
+ app_id=ids[app],
+ workflow_id=ids[app + "_workflow"],
+ workflow_run_id=ids["run"],
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL
+ if child
+ else WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
+ triggered_from_workflow_id=ids["root_workflow"] if child else None,
+ triggered_from_node_execution_id=ids["root_tool"] if child else None,
+ node_execution_id=ids[key],
+ index=1,
+ node_id=key,
+ node_type=node_type,
+ title=key,
+ status=WorkflowNodeExecutionStatus.SUCCEEDED,
+ created_by=ids["actor"],
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_at=now,
+ finished_at=now + timedelta(seconds=2),
+ inputs="{}",
+ outputs="{}",
+ process_data=json.dumps(
+ {"workflow_tool_root_app_id": ids["root"], "workflow_tool_invocation_id": "tool-call"}
+ )
+ if child
+ else "{}",
+ )
+ )
+ service = ApplicationTracingService(ApplicationTraceRepository(sessions))
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ monkeypatch.setattr(runtime_bridge, "_service", lambda: service)
+ entity = workflow_entity()
+ entity.app_config = entity.app_config.model_copy(
+ update={"app_id": ids["root"], "tenant_id": ids["tenant"], "workflow_id": ids["root_workflow"]}
+ )
+ runtime_bridge.freeze_workflow_run(
+ entity,
+ workflow_id=ids["root_workflow"],
+ workflow_run_id=ids["run"],
+ conversation_id=None,
+ graph_data=root_graph,
+ )
+ original = workflow_context(False)
+ context = replace(
+ original,
+ dify_context=original.dify_context.model_copy(
+ update={"tenant_id": ids["tenant"], "app_id": ids["source"], "workflow_tool_invocation_id": "tool-call"}
+ ),
+ workflow_id=ids["source_workflow"],
+ workflow_run_id=ids["run"],
+ node_execution_id=ids["child"],
+ )
+ return service, context, ids
+
+
[email protected]("root_has_agent", [False, True])
[email protected]("source_enabled", [False, True])
+def test_hidden_child_never_reserves_source_policy_or_changes_root_freeze(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+ root_has_agent: bool,
+ source_enabled: bool,
+) -> None:
+ service, context, ids = seed(
+ sqlite_session_factory, monkeypatch, root_has_agent=root_has_agent, source_enabled=source_enabled
+ )
+ with sqlite_session_factory() as session:
+ before = deepcopy([dict(row) for row in session.execute(select(ApplicationTraceRun.__table__)).mappings()])
+ service_lookups: list[bool] = []
+
+ def lookup() -> ApplicationTracingService:
+ service_lookups.append(True)
+ return service
+
+ monkeypatch.setattr(runtime_bridge, "_service", lookup)
+ # Marked children never consult policy or ownership, even for malformed
+ # empty markers or a foreign tenant. They cannot create an orphan B run.
+ for tenant in (ids["tenant"], identifier("foreign-tool-tenant")):
+ for marker in ("tool-call", ""):
+ child = replace(
+ context,
+ dify_context=context.dify_context.model_copy(
+ update={
+ "tenant_id": tenant,
+ "workflow_tool_invocation_id": marker,
+ }
+ ),
+ )
+ assert runtime_bridge.prepare_workflow_agent(child, None) is None
+ assert service_lookups == []
+ with sqlite_session_factory() as session:
+ observed = [
+ (r.app_id, r.provider, r.capture_content, r.source_id) for r in session.scalars(select(ApplicationTraceRun))
+ ]
+ assert observed == ([(ids["root"], "langfuse", False, ids["run"])] if root_has_agent else [])
+ after = [dict(row) for row in session.execute(select(ApplicationTraceRun.__table__)).mappings()]
+ assert after == before
+ assert (
+ service.repository.source_run(
+ tenant_id=ids["tenant"], app_id=ids["source"], source_kind="workflow", source_id=ids["run"]
+ )
+ is None
+ )
+
+
+def test_hidden_child_skip_keeps_normal_root_agent_trace_assemblable(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ service, child, ids = seed(sqlite_session_factory, monkeypatch, root_has_agent=True)
+ assert runtime_bridge.prepare_workflow_agent(child, None) is None
+ root_context = replace(
+ child,
+ dify_context=child.dify_context.model_copy(update={"app_id": ids["root"], "workflow_tool_invocation_id": None}),
+ workflow_id=ids["root_workflow"],
+ node_execution_id=ids["root_agent"],
+ )
+ runtime = runtime_bridge.prepare_workflow_agent(root_context, None)
+ assert runtime is not None
+ run = service.repository.get(str(runtime.run_id))
+ assert run is not None
+ assert run.app_id == ids["root"]
+ frozen = deepcopy(run.provider_config)
+ with sqlite_session_factory.begin() as session:
+ owner = session.get(App, ids["root"])
+ assert owner is not None
+ owner.tracing = json.dumps({"enabled": True, "tracing_provider": "langsmith", "capture_content": True})
+ workflow = session.get(WorkflowRun, ids["run"])
+ assert workflow is not None
+ actor = make_account(account_id=ids["actor"])
+ nodes = SQLAlchemyWorkflowNodeExecutionRepository(
+ session_factory=sqlite_session_factory,
+ tenant_id=ids["tenant"],
+ user=actor,
+ app_id=ids["root"],
+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
+ )
+ info = WorkflowTraceInfo(
+ workflow_data=workflow,
+ workflow_id=ids["root_workflow"],
+ tenant_id=ids["tenant"],
+ workflow_run_id=ids["run"],
+ workflow_run_elapsed_time=3,
+ workflow_run_status="succeeded",
+ workflow_run_inputs={},
+ workflow_run_outputs={},
+ workflow_run_version="1",
+ total_tokens=0,
+ file_list=[],
+ query="",
+ metadata={"app_id": ids["root"]},
+ )
+ outer = CanonicalTraceBuilder(lambda _info: nodes.get_by_workflow_execution(ids["run"])).build(info)
+ assert outer is not None
+ outer = remap_outer_trace(outer, trace_id=run.trace_id, capture_content=False, agent_message=False)
+ service.repository.terminal(run.id, outer.model_dump(mode="json"), outer.root_span_id)
+ invocation = service.repository.collected(run.id)[0][0]
+ ingest(Collection(service.repository, sqlite_session_factory, run), batch_for(run, invocation))
+ current = service.repository.get(run.id)
+ assert current is not None
+ trace = service.assemble(current)
+ span_ids = {span.id for span in trace.spans}
+ assert all(span.parent_id is None or span.parent_id in span_ids for span in trace.spans)
+ assert current.provider_config == frozen
+ assert current.provider == "langfuse"
+ assert current.capture_content is False
diff --git a/api/tests/unit_tests/tasks/test_application_trace_task.py b/api/tests/unit_tests/tasks/test_application_trace_task.py
new file mode 100644
index 00000000000..f9713f5be4a
--- /dev/null
+++ b/api/tests/unit_tests/tasks/test_application_trace_task.py
@@ -0,0 +1,72 @@
+"""Exercise Celery entrypoints against durable application trace state."""
+
+from datetime import timedelta
+
+import pytest
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.ops.unified_trace.entities import CanonicalTrace
+from libs.datetime_utils import naive_utc_now
+from models.application_trace import ApplicationTraceRun
+from services.tracing import service as service_module
+from services.tracing.provider_dispatch import TraceExportResult
+from services.tracing.service import ApplicationTracingService
+from tasks.application_trace_task import export_application_trace, reconcile_application_traces
+from tests.unit_tests.config_override import apply_config_overrides
+from tests.unit_tests.services.tracing.test_application_trace_collection import batch_for, ingest, seed, terminal
+
+
+def test_worker_task_exports_once_and_replayed_task_observes_checkpoint(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ case = seed(sqlite_session_factory, "worker-task")
+ ingest(case, batch_for(case.run, case.invocation()))
+ terminal(case)
+ service = ApplicationTracingService(case.repository)
+ monkeypatch.setattr(service_module, "application_tracing_service", lambda: service)
+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})
+ exports: list[CanonicalTrace] = []
+
+ def export(_provider: str, _config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:
+ exports.append(trace)
+ return TraceExportResult("vendor-id", "https://example.test/trace/vendor-id")
+
+ monkeypatch.setattr(service_module, "export_trace", export)
+ export_application_trace.run(case.run.id)
+ export_application_trace.run(case.run.id)
+ assert len(exports) == 1
+ assert exports[0].trace_id == case.run.trace_id
+ result = case.repository.get(case.run.id)
+ assert result is not None
+ assert result.state == "accepted"
+ assert result.attempts == 1
+ assert result.trace_url == "https://example.test/trace/vendor-id"
+
+
+def test_reconcile_only_schedules_due_pending_rows_and_applies_retention(
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_RETENTION_DAYS=3)
+ due = seed(sqlite_session_factory, "due")
+ deferred = seed(sqlite_session_factory, "future")
+ accepted = seed(sqlite_session_factory, "accepted-expired")
+ with sqlite_session_factory.begin() as session:
+ future = session.get(ApplicationTraceRun, deferred.run.id)
+ expired = session.get(ApplicationTraceRun, accepted.run.id)
+ assert future is not None
+ assert expired is not None
+ future.state = "retrying"
+ future.next_attempt_at = naive_utc_now() + timedelta(hours=1)
+ expired.state = "accepted"
+ expired.created_at = naive_utc_now() - timedelta(days=4)
+ service = ApplicationTracingService(due.repository)
+ scheduled: list[str] = []
+ monkeypatch.setattr(service, "schedule", scheduled.append)
+ monkeypatch.setattr(service_module, "application_tracing_service", lambda: service)
+ reconcile_application_traces.run()
+ assert scheduled == [due.run.id]
+ assert due.repository.get(accepted.run.id) is None
+ assert due.repository.get(deferred.run.id) is not None
+ assert due.repository.get(due.run.id) is not None
diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
index 33e000371dc..79dc983f267 100644
--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
@@ -22,7 +22,6 @@ from unittest.mock import MagicMock, patch
import pytest
from celery.exceptions import Retry
-from flask import Flask
from sqlalchemy import Engine, event
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session, sessionmaker
@@ -134,10 +133,14 @@ def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:
@pytest.fixture
-def elasticsearch_delete_endpoint() -> Iterator[tuple[int, list[int], list[tuple[str, int]]]]:
+def elasticsearch_delete_endpoint(
+ monkeypatch: pytest.MonkeyPatch,
+) -> Iterator[tuple[int, list[int], list[tuple[str, int]]]]:
"""Serve real SDK requests with Elasticsearch's missing-index response shape."""
pytest.importorskip("elasticsearch")
- pytest.importorskip("dify_vdb_elasticsearch.elasticsearch_vector")
+ elasticsearch_vector = pytest.importorskip("dify_vdb_elasticsearch.elasticsearch_vector")
+ from elasticsearch import Elasticsearch
+
responses: list[int] = []
requests: list[tuple[str, int]] = []
@@ -174,9 +177,23 @@ def elasticsearch_delete_endpoint() -> Iterator[tuple[int, list[int], list[tuple
server = HTTPServer(("127.0.0.1", 0), Handler)
thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)
thread.start()
+ client = Elasticsearch(
+ [f"http://127.0.0.1:{server.server_port}"],
+ request_timeout=2,
+ max_retries=0,
+ retry_on_timeout=False,
+ )
+
+ def local_client(**_options: object) -> Elasticsearch:
+ return client
+
+ # Celery can enter a different Flask app from the test fixture's app.
+ # Bind only SDK construction; keep real ping/info/delete and error decoding.
+ monkeypatch.setattr(elasticsearch_vector, "Elasticsearch", local_client)
try:
yield server.server_port, responses, requests
finally:
+ client.close()
server.shutdown()
thread.join(timeout=1)
server.server_close()
@@ -662,8 +679,6 @@ class TestIndexProcessorParameters:
def test_missing_elasticsearch_index_allows_parent_child_cleanup_after_retry(
self,
commit_fails: bool,
- app: Flask,
- monkeypatch: pytest.MonkeyPatch,
elasticsearch_delete_endpoint: tuple[int, list[int], list[tuple[str, int]]],
dataset_id: str,
tenant_id: str,
@@ -672,20 +687,9 @@ class TestIndexProcessorParameters:
mock_storage: MagicMock,
caplog: pytest.LogCaptureFixture,
) -> None:
- port, responses, requests = elasticsearch_delete_endpoint
+ _port, responses, requests = elasticsearch_delete_endpoint
expected_statuses = [200, 404] if commit_fails else [404]
responses.extend(expected_statuses)
- for key, value in {
- "ELASTICSEARCH_USE_CLOUD": False,
- "ELASTICSEARCH_HOST": "http://127.0.0.1",
- "ELASTICSEARCH_PORT": port,
- "ELASTICSEARCH_USERNAME": "elastic",
- "ELASTICSEARCH_PASSWORD": "test",
- "ELASTICSEARCH_REQUEST_TIMEOUT": 2,
- "ELASTICSEARCH_MAX_RETRIES": 0,
- "ELASTICSEARCH_RETRY_ON_TIMEOUT": False,
- }.items():
- monkeypatch.setitem(app.config, key, value)
document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
segment = _persist_segment(
orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id
diff --git a/api/tests/unit_tests/tasks/test_ops_trace_task.py b/api/tests/unit_tests/tasks/test_ops_trace_task.py
index c23372250ae..95960c69239 100644
--- a/api/tests/unit_tests/tasks/test_ops_trace_task.py
+++ b/api/tests/unit_tests/tasks/test_ops_trace_task.py
@@ -1,16 +1,30 @@
import json
import sys
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
from types import ModuleType
from unittest.mock import MagicMock, patch
import pytest
from celery.exceptions import Retry
+from flask import Flask
from configs import dify_config
+from core.ops.base_trace_instance import (
+ ApplicationTracingHooksNotRegisteredError,
+ bind_application_tracing_hooks,
+)
from core.ops.entities.config_entity import OPS_TRACE_FAILED_KEY
-from core.ops.exceptions import RetryableTraceDispatchError
+from core.ops.entities.trace_entity import BaseTraceInfo
+from core.ops.exceptions import PendingTraceParentContextError, RetryableTraceDispatchError
+from services.tracing.provider_dispatch import TraceExportUncertainError
+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks
from tasks.ops_trace_task import process_trace_tasks
+from tests.unit_tests.config_override import apply_config_overrides
+
+
[email protected](autouse=True)
+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)
@contextmanager
@@ -81,10 +95,20 @@ def _run_task(file_info: dict[str, str], retries: int = 0) -> None:
process_trace_tasks.pop_request()
-def test_process_trace_tasks_retries_retryable_dispatch_failure_and_preserves_payload():
+ "pending_error",
+ [
+ _retryable_dispatch_error(),
+ PendingTraceParentContextError("parent-node"),
+ TraceExportUncertainError("DO_NOT_EXPOSE_PROVIDER_BODY"),
+ ],
+)
+def test_process_trace_tasks_retries_only_confirmed_retryable_dispatch_failures(
+ pending_error: RetryableTraceDispatchError,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
file_info = {"app_id": "app-id", "file_id": "file-id"}
trace_instance = MagicMock()
- pending_error = _retryable_dispatch_error()
trace_instance.trace.side_effect = pending_error
retry_error = Retry()
@@ -95,16 +119,23 @@ def test_process_trace_tasks_retries_retryable_dispatch_failure_and_preserves_pa
patch("tasks.ops_trace_task.storage.delete") as mock_delete,
patch("tasks.ops_trace_task.redis_client.incr") as mock_incr,
patch.object(process_trace_tasks, "retry", side_effect=retry_error) as mock_retry,
- pytest.raises(Retry),
+ pytest.raises(Retry) if not pending_error.acceptance_unknown else nullcontext(),
):
_run_task(file_info)
- mock_retry.assert_called_once_with(
- exc=pending_error,
- countdown=process_trace_tasks.default_retry_delay,
- )
- mock_delete.assert_not_called()
- mock_incr.assert_not_called()
+ trace_instance.trace.assert_called_once_with({})
+ if pending_error.acceptance_unknown:
+ mock_retry.assert_not_called()
+ mock_delete.assert_called_once_with("ops_trace/app-id/file-id.json")
+ mock_incr.assert_called_once_with(f"{OPS_TRACE_FAILED_KEY}_app-id")
+ assert "DO_NOT_EXPOSE_PROVIDER_BODY" not in caplog.text
+ else:
+ mock_retry.assert_called_once_with(
+ exc=pending_error,
+ countdown=process_trace_tasks.default_retry_delay,
+ )
+ mock_delete.assert_not_called()
+ mock_incr.assert_not_called()
def test_process_trace_tasks_marks_enterprise_trace_dispatched_before_retryable_dispatch_retry():
@@ -303,3 +334,150 @@ def test_process_trace_tasks_deletes_payload_and_counts_exhausted_retryable_disp
mock_retry.assert_not_called()
mock_delete.assert_called_once_with("ops_trace/app-id/file-id.json")
mock_incr.assert_called_once_with(f"{OPS_TRACE_FAILED_KEY}_app-id")
+
+
+def test_managed_application_capture_receives_restored_info_and_preserves_enterprise(monkeypatch: pytest.MonkeyPatch):
+ from core.ops.entities.trace_entity import MessageTraceInfo
+
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ calls: list[BaseTraceInfo] = []
+
+ def capture(info: BaseTraceInfo) -> bool:
+ calls.append(info)
+ return True
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "capture_terminal", staticmethod(capture))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+ enterprise_calls: list[BaseTraceInfo] = []
+
+ class RecordingEnterpriseTrace:
+ def trace(self, info: BaseTraceInfo) -> None:
+ enterprise_calls.append(info)
+
+ payload = json.dumps(
+ {
+ "trace_info_type": "MessageTraceInfo",
+ "trace_info": {
+ "metadata": {},
+ "message_id": "message-id",
+ "conversation_model": "agent",
+ "conversation_mode": "agent",
+ "message_tokens": 0,
+ "answer_tokens": 0,
+ "total_tokens": 0,
+ },
+ }
+ )
+ with (
+ app.app_context(),
+ patch("extensions.ext_enterprise_telemetry.is_enabled", return_value=True),
+ patch("enterprise.telemetry.enterprise_trace.EnterpriseOtelTrace", new=RecordingEnterpriseTrace),
+ patch(
+ "core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance",
+ side_effect=AssertionError("managed traces must not instantiate legacy providers"),
+ ) as lookup,
+ patch("tasks.ops_trace_task.storage.load", return_value=payload),
+ patch("tasks.ops_trace_task.storage.delete") as deleted,
+ ):
+ _run_task({"app_id": "app-id", "file_id": "file-id"})
+ assert len(calls) == 1
+ restored = calls[0]
+ assert isinstance(restored, MessageTraceInfo)
+ assert restored.message_id == "message-id"
+ assert enterprise_calls == [restored]
+ lookup.assert_not_called()
+ deleted.assert_called_once()
+
+
[email protected]("enabled", [False, True])
+def test_nonmanaged_task_uses_legacy_and_disabled_gate_never_calls_collector(
+ monkeypatch: pytest.MonkeyPatch, enabled: bool
+):
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=enabled)
+ legacy_calls: list[object] = []
+
+ class RecordingLegacyTrace:
+ def trace(self, info: object) -> None:
+ legacy_calls.append(info)
+
+ legacy = RecordingLegacyTrace()
+ calls: list[BaseTraceInfo] = []
+
+ def capture(info: BaseTraceInfo) -> bool:
+ calls.append(info)
+ return False
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "capture_terminal", staticmethod(capture))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+ with (
+ app.app_context(),
+ patch("extensions.ext_enterprise_telemetry.is_enabled", return_value=False),
+ patch("core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance", return_value=legacy) as lookup,
+ patch("tasks.ops_trace_task.storage.load", return_value=_make_payload()),
+ patch("tasks.ops_trace_task.storage.delete"),
+ ):
+ _run_task({"app_id": "app-id", "file_id": "file-id"})
+ assert calls == ([{}] if enabled else [])
+ assert legacy_calls == [{}]
+ lookup.assert_called_once_with("app-id")
+
+
+def test_capture_persistence_failure_preserves_payload_without_legacy_export(monkeypatch: pytest.MonkeyPatch):
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ pending_error = RetryableTraceDispatchError("database busy")
+ calls: list[BaseTraceInfo] = []
+
+ def capture(info: BaseTraceInfo) -> bool:
+ calls.append(info)
+ raise pending_error
+
+ monkeypatch.setattr(RuntimeApplicationTracingHooks, "capture_terminal", staticmethod(capture))
+ hooks = RuntimeApplicationTracingHooks()
+ app = Flask(__name__)
+ bind_application_tracing_hooks(app, hooks)
+ with (
+ app.app_context(),
+ patch(
+ "core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance",
+ side_effect=AssertionError("capture failures must not instantiate legacy providers"),
+ ) as lookup,
+ patch("tasks.ops_trace_task.storage.load", return_value=_make_payload()),
+ patch("tasks.ops_trace_task.storage.delete") as deleted,
+ patch.object(process_trace_tasks, "retry", side_effect=Retry()) as retry,
+ pytest.raises(Retry),
+ ):
+ _run_task({"app_id": "app-id", "file_id": "file-id"})
+ lookup.assert_not_called()
+ deleted.assert_not_called()
+
+ assert calls == [{}]
+ retry.assert_called_once_with(exc=pending_error, countdown=process_trace_tasks.default_retry_delay)
+
+
+def test_unregistered_application_hooks_preserve_payload_without_legacy_export(monkeypatch: pytest.MonkeyPatch) -> None:
+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)
+ app = Flask(__name__)
+ with (
+ app.app_context(),
+ patch(
+ "core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance",
+ side_effect=AssertionError("unregistered hooks must not instantiate legacy providers"),
+ ) as lookup,
+ patch("tasks.ops_trace_task.storage.load", return_value=_make_payload()),
+ patch("tasks.ops_trace_task.storage.delete") as deleted,
+ patch("tasks.ops_trace_task.redis_client.incr") as failed,
+ patch.object(process_trace_tasks, "retry", side_effect=Retry()) as retry,
+ pytest.raises(Retry),
+ ):
+ _run_task({"app_id": "app-id", "file_id": "file-id"})
+
+ deleted.assert_not_called()
+ failed.assert_not_called()
+ retry.assert_called_once()
+ lookup.assert_not_called()
+ assert isinstance(retry.call_args.kwargs["exc"], ApplicationTracingHooksNotRegisteredError)
+ assert retry.call_args.kwargs["countdown"] == process_trace_tasks.default_retry_delay
diff --git a/dify-agent/.example.env b/dify-agent/.example.env
index 00afe2698aa..cf45162451e 100644
--- a/dify-agent/.example.env
+++ b/dify-agent/.example.env
@@ -108,3 +108,5 @@ DIFY_AGENT_LOCAL_SANDBOX_MATERIALIZED_HOME_ROOT=/home/dify
DIFY_AGENT_LOCAL_SANDBOX_WORKSPACE_ROOT=/workspace
DIFY_AGENT_LOCAL_SANDBOX_HOME_SNAPSHOT_ROOT=/home/dify/.snapshots
LOGFIRE_CONSOLE=false
+# Application tracing uses only API-issued run policy and the inner API above.
+DIFY_AGENT_APPLICATION_TRACING_ENABLED=false
diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md
index 812f6971b63..bcfd6cf9eb3 100644
--- a/dify-agent/docs/dify-agent/guide/index.md
+++ b/dify-agent/docs/dify-agent/guide/index.md
@@ -82,6 +82,7 @@ in [E2B runtime metering](runtime-metering.md).
| `DIFY_AGENT_OUTBOUND_HTTP_MAX_KEEPALIVE_CONNECTIONS` | `20` | Maximum idle shared outbound HTTP connections. |
| `DIFY_AGENT_OUTBOUND_HTTP_KEEPALIVE_EXPIRY` | `30` | Idle keep-alive expiry in seconds. |
| `DIFY_AGENT_TRAJECTORY_ENABLED` | `false` | Opt-in switch for the separate Agent trajectory export pipeline. |
+| `DIFY_AGENT_APPLICATION_TRACING_ENABLED` | `false` | Collect API-authorized application spans into the Dify inner API tracing inbox. Requires the server inner API URL and key. |
| `DIFY_AGENT_TRAJECTORY_OTLP_TRACES_ENDPOINT` | empty | Full HTTP(S) OTLP traces endpoint required when trajectory export is enabled. |
| `DIFY_AGENT_TRAJECTORY_OTLP_HEADERS` | `{}` | JSON object of request headers sent with Agent OTLP trace exports. |
| `DIFY_AGENT_TRAJECTORY_SERVICE_NAME` | `dify-agent-trajectory` | Resource service name attached to Agent trajectory spans. |
@@ -119,6 +120,48 @@ separate. The mode is fixed at process initialization and requires a restart
to change. No extra shared root span is added; incomplete traces may still
appear temporarily while a run or export is in progress.
+Application tracing is a third, independent pipeline. When enabled, a run's
+API-issued `execution_context.tracing` supplies its durable run/invocation IDs,
+OpenTelemetry parent, session and content/sampling policy. Its spans are sent as
+JSON to `POST /inner/api/agent/tracing/batches`, authenticated with the server's
+`DIFY_AGENT_INNER_API_KEY` in `X-Inner-Api-Key`. Provider credentials and routing
+remain in the Dify API. Application spans do not inherit platform or deployment
+trajectory destinations, and one Agent is never instrumented by both application
+and deployment pipelines. Binary content is excluded.
+
+The actual model-call span also carries the Dify gateway's billed amount as
+`dify.llm.total_price` (a decimal string) and `dify.llm.currency` when a usage
+payload is returned. `gen_ai.response.time_to_first_token` is measured in seconds
+(`dify.llm.time_to_first_token_unit=s`): gateway-provided TTFT takes precedence;
+otherwise it measures request issue to the first content/tool-call chunk.
+`dify.llm.time_to_first_token_source` distinguishes those two sources. Missing
+accounting is left absent. The current Graphon LLM gateway does not expose cache
+token counters, so the Agent adapter does not manufacture them.
+
+Each batch has a stable `batch_id`. Retries reuse the same payload, and a 2xx
+response is accepted only with `{result: "success", batch_id, accepted_spans}`
+matching that batch. An invocation emits an empty `final: true` batch after its
+ended spans are flushed and acknowledged, before its normal terminal event when
+delivery completes within five seconds. Failure or cancellation still flushes.
+Collection failures do not fail the Agent run; outstanding delivery can finish
+in the background after the terminal-event deadline. A full in-memory queue or
+unacknowledged data suppresses completion instead of claiming a complete trace.
+The queue is bounded (2048 spans, batches of at most 128), with up to 128 unacknowledged
+batches retained until invocation finalization. Batches are split below 1 MiB
+of UTF-8 JSON. A single oversized span is dropped and marks its invocation
+incomplete rather than silently truncating content or sending a success marker.
+This is not a durable local
+spool: process loss can leave an incomplete trace in the API. Shutdown drains the
+collector after active runs finish.
+
+If setup fails or cancellation arrives before Pydantic AI starts its Agent span,
+the collector emits one failed `invoke_agent` span with the measured setup
+interval, then finalizes normally. It records `dify.agent.execution_state` as
+`initialization_failed` or `cancelled`. Runs that already started their native
+Agent span keep that span without an extra wrapper. Sampling-off invocations
+remain span-free; successful but unexplained empty collections are not converted
+to synthetic success.
+
Example `.env`:
```env
diff --git a/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md b/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md
index c17b4626a19..a81958c10a2 100644
--- a/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md
+++ b/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md
@@ -5,3 +5,33 @@ Capability. Config carries tenant/user/app/invoke identity and correlation IDs.
Other modules refer to its registered name in their Config. Server-only daemon
settings and clients are borrowed from Deps.services. Knowledge retrieval
requires its complete caller identity before any external request.
+
+The optional `tracing` field is an API-issued `ApplicationTracingContext`:
+
+```json
+{
+ "version": 1,
+ "run_id": "429cab49-8663-42b4-a701-ac1038ee4fa7",
+ "invocation_id": "95bdeed9-4652-4708-8322-b866f159ad62",
+ "trace_id": "81b617d638eb419ba2b209c14984d908",
+ "parent_span_id": "716db0b6c736b395",
+ "session_id": "conversation-id",
+ "capture_content": false,
+ "sampled": true
+}
+```
+
+`run_id` identifies the API's persisted application trace. `invocation_id`
+identifies this particular Agent execution; multiple nodes or attempts can
+belong to one application trace. Both are UUIDs. The trace and parent span IDs
+are nonzero, lowercase OpenTelemetry hex IDs (32 and 16 characters). The parent
+is the API-owned message or Agent-node invocation, and is restored as the actual
+OpenTelemetry parent rather than attached only as metadata. Tenant/app identity
+continues to come from the containing execution context.
+
+Application collection requires `DIFY_AGENT_APPLICATION_TRACING_ENABLED=true`
+on the Agent server. The API owns sampling and content policy per invocation;
+no third-party destination or credential is accepted in a run request. When an
+application context is selected it replaces deployment trajectory instrumentation
+for that invocation, including when `sampled` is false. Without an application
+context, the existing deployment trajectory behavior is unchanged.
diff --git a/dify-agent/src/dify_agent/adapters/llm/model.py b/dify-agent/src/dify_agent/adapters/llm/model.py
index 1f4c93a161e..98d1ede1bef 100644
--- a/dify-agent/src/dify_agent/adapters/llm/model.py
+++ b/dify-agent/src/dify_agent/adapters/llm/model.py
@@ -10,13 +10,18 @@ and reads that accumulated usage after all model/tool rounds finish.
from __future__ import annotations
import base64
+import logging
+import math
import re
+import time
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from contextlib import asynccontextmanager
from dataclasses import KW_ONLY, InitVar, dataclass, field
from datetime import datetime, timezone
from typing import cast
+from opentelemetry.trace import Span, StatusCode, get_current_span
+
from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMUsage
from graphon.model_runtime.entities.message_entities import (
AssistantPromptMessage,
@@ -78,6 +83,7 @@ _THINK_OPEN_TAG = "<think>"
_THINK_CLOSE_TAG = "</think>"
_THINK_TAG_PATTERN = re.compile(r"<think>(.*?)</think>", re.DOTALL)
_DETAIL_HIGH = "high"
+logger = logging.getLogger(__name__)
@dataclass(slots=True)
@@ -233,11 +239,34 @@ class DifyStreamedResponse(StreamedResponse):
_timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
_embedded_thinking_parser: "_EmbeddedThinkingParser" = field(default_factory=lambda: _EmbeddedThinkingParser())
_dify_usage: LLMUsage | None = field(default=None, init=False, repr=False)
+ # Pydantic AI opens its actual chat span before calling the wrapped model.
+ # Capture that span here; HTTP transport spans may be current while iterating.
+ _model_span: Span = field(default_factory=get_current_span, init=False, repr=False)
+ _request_started_at: float = field(default_factory=time.perf_counter, init=False, repr=False)
+ _first_content_seconds: float | None = field(default=None, init=False, repr=False)
+ _usage_ttft_seconds: float | None = field(default=None, init=False, repr=False)
@override
async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]:
+ try:
+ async for event in self._iterate_gateway_events():
+ yield event
+ except BaseException as error:
+ # Some instrumentation versions end the chat span without marking
+ # errors raised while consuming an asynchronous response. Keep the
+ # actual model call consistent with the failed Agent envelope.
+ try:
+ if self._model_span.is_recording():
+ self._model_span.set_status(StatusCode.ERROR, "Model request did not complete")
+ self._model_span.set_attribute("error.type", type(error).__name__)
+ except Exception:
+ logger.debug("Unable to record model trace failure")
+ raise
+
+ async def _iterate_gateway_events(self) -> AsyncIterator[ModelResponseStreamEvent]:
chunk_sequence = 0
async for chunk in self.chunks:
+ self._observe_trace_usage(chunk)
if chunk.delta.usage is not None:
self._dify_usage = chunk.delta.usage
self._usage: RequestUsage = _map_usage(chunk.delta.usage)
@@ -257,6 +286,52 @@ class DifyStreamedResponse(StreamedResponse):
for event in self._embedded_thinking_parser.flush(self._parts_manager, self.provider_name_value):
yield event
+ def _observe_trace_usage(self, chunk: LLMResultChunk) -> None:
+ """Attach gateway accounting to the real model span, without inventing usage.
+
+ Dify's usage TTFT is in seconds. If absent, measure from gateway request
+ issue to the first content/tool-call chunk, excluding empty usage chunks.
+ Price is the gateway's billed decimal amount in its accompanying currency.
+ Graphon's current LLMUsage has no cache counters, so none are synthesized.
+ """
+ try:
+ if not self._model_span.is_recording():
+ return
+ if self._first_content_seconds is None and (chunk.delta.message.content or chunk.delta.message.tool_calls):
+ self._first_content_seconds = time.perf_counter() - self._request_started_at
+ usage = chunk.delta.usage
+ attributes: dict[str, str | int | float] = {}
+ ttft = self._first_content_seconds
+ source = "gateway_first_content"
+ if usage is not None:
+ attributes.update(
+ {
+ "gen_ai.usage.input_tokens": usage.prompt_tokens,
+ "gen_ai.usage.output_tokens": usage.completion_tokens,
+ "gen_ai.usage.total_tokens": usage.total_tokens,
+ }
+ )
+ if usage.total_price.is_finite() and usage.total_price >= 0 and usage.currency:
+ attributes["dify.llm.total_price"] = str(usage.total_price)
+ attributes["dify.llm.currency"] = usage.currency
+ if (
+ usage.time_to_first_token is not None
+ and math.isfinite(usage.time_to_first_token)
+ and usage.time_to_first_token >= 0
+ ):
+ self._usage_ttft_seconds = usage.time_to_first_token
+ if self._usage_ttft_seconds is not None:
+ ttft = self._usage_ttft_seconds
+ source = "gateway_usage"
+ if ttft is not None and math.isfinite(ttft) and ttft >= 0:
+ attributes["gen_ai.response.time_to_first_token"] = ttft
+ attributes["dify.llm.time_to_first_token_unit"] = "s"
+ attributes["dify.llm.time_to_first_token_source"] = source
+ self._model_span.set_attributes(attributes)
+ except Exception:
+ # Observability must never change a successful/failed model outcome.
+ logger.warning("Failed to annotate Dify LLM accounting span", exc_info=True)
+
@property
def dify_usage(self) -> LLMUsage | None:
"""Return Dify's complete usage for this model request."""
diff --git a/dify-agent/src/dify_agent/adapters/llm/provider.py b/dify-agent/src/dify_agent/adapters/llm/provider.py
index 7c5176a232e..9143b15a78f 100644
--- a/dify-agent/src/dify_agent/adapters/llm/provider.py
+++ b/dify-agent/src/dify_agent/adapters/llm/provider.py
@@ -97,7 +97,9 @@ class DifyApiLLMClient:
if missing:
raise UserError(f"Agent LLM Gateway requires execution context fields: {', '.join(missing)}")
- caller = context.model_dump(mode="json")
+ # Application tracing policy belongs to the collector, not the strict
+ # metering-gateway caller DTO. Keep it out even when its value is None.
+ caller = context.model_dump(mode="json", exclude={"tracing"})
caller.update(
{
"invocation_id": invocation_id,
diff --git a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py
index 0b8c5725e48..b2a4b232a93 100644
--- a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py
+++ b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py
@@ -13,6 +13,8 @@ from copy import deepcopy
from dataclasses import replace
from typing import cast, override
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.trace import StatusCode, get_current_span
from pydantic import JsonValue
from pydantic_ai import RunContext, Tool
from pydantic_ai.tools import ToolDefinition
@@ -93,9 +95,11 @@ class Toolset(FunctionToolset[Deps]):
tool_parameters=cast(dict[str, JsonValue], tool_arguments),
)
return response.observation
- except DifyCoreToolsClientConfigurationError:
+ except DifyCoreToolsClientConfigurationError as exc:
+ _mark_tool_span_failed(_ctx, exc)
return "Tool is unavailable because required execution context is missing."
except DifyCoreToolsClientError as exc:
+ _mark_tool_span_failed(_ctx, exc)
return _tool_error_text(tool_name=tool_name, error=exc)
async def prepare_tool_definition(_ctx: RunContext[Deps], tool_def: ToolDefinition) -> ToolDefinition:
@@ -110,6 +114,30 @@ class Toolset(FunctionToolset[Deps]):
)
+def _mark_tool_span_failed(ctx: RunContext[Deps], error: DifyCoreToolsClientError) -> None:
+ """Keep handled API errors on this native tool span without failing the Agent.
+
+ Pydantic AI wraps tool execution in the current ``execute_tool`` span. When
+ instrumentation is disabled, an ambient Agent/platform span must stay intact.
+ Only record a safe exception type; the observation retains its existing text
+ and the instrumentation's content policy controls whether that text is traced.
+ """
+ span = get_current_span()
+ attributes = span.attributes if isinstance(span, ReadableSpan) else None
+ if (
+ not span.is_recording()
+ or not attributes
+ or attributes.get("gen_ai.operation.name") != "execute_tool"
+ or ctx.tool_call_id is None
+ or attributes.get("gen_ai.tool.call.id") != ctx.tool_call_id
+ or attributes.get("gen_ai.tool.name") != ctx.tool_name
+ ):
+ return
+ error_type = type(error).__name__
+ span.set_attribute("error.type", error_type)
+ span.set_status(StatusCode.ERROR, error_type)
+
+
def _tool_error_text(*, tool_name: str, error: DifyCoreToolsClientError) -> str:
if error.retryable:
return TEMPORARY_UNAVAILABLE_OBSERVATION
diff --git a/dify-agent/src/dify_agent/layers/execution_context/__init__.py b/dify-agent/src/dify_agent/layers/execution_context/__init__.py
index 356653ae65c..44060ce0f85 100644
--- a/dify-agent/src/dify_agent/layers/execution_context/__init__.py
+++ b/dify-agent/src/dify_agent/layers/execution_context/__init__.py
@@ -7,6 +7,7 @@ DTO, but that runtime implementation still lives in sibling modules.
"""
from dify_agent.layers.execution_context.configs import (
+ ApplicationTracingContext,
DifyExecutionContextAgentConfigVersionKind,
DifyExecutionContextAgentMode,
DifyExecutionContextInvokeFrom,
@@ -15,6 +16,7 @@ from dify_agent.layers.execution_context.configs import (
)
__all__ = [
+ "ApplicationTracingContext",
"DifyExecutionContextAgentConfigVersionKind",
"DifyExecutionContextAgentMode",
"DifyExecutionContextInvokeFrom",
diff --git a/dify-agent/src/dify_agent/layers/execution_context/configs.py b/dify-agent/src/dify_agent/layers/execution_context/configs.py
index 1f8d1d51453..1e5242bf2f7 100644
--- a/dify-agent/src/dify_agent/layers/execution_context/configs.py
+++ b/dify-agent/src/dify_agent/layers/execution_context/configs.py
@@ -11,9 +11,10 @@ Server-only plugin-daemon settings are injected by the runtime runtime services
and therefore do not appear in this public schema.
"""
-from typing import ClassVar, Literal, TypeAlias
+from typing import Annotated, ClassVar, Literal, TypeAlias
+from uuid import UUID
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field, field_validator
DifyExecutionContextAgentMode: TypeAlias = Literal[
@@ -37,6 +38,32 @@ DifyExecutionContextInvokeFrom: TypeAlias = Literal[
]
+class ApplicationTracingContext(BaseModel):
+ """API-issued application tracing policy; never contains exporter credentials.
+
+ ``run_id`` refers to the API's durable tracing run, not the Agent backend's
+ execution id. The parent identifies the owning message or workflow node.
+ """
+
+ version: Literal[1] = 1
+ run_id: UUID
+ invocation_id: UUID
+ trace_id: Annotated[str, Field(pattern=r"^[0-9a-f]{32}$")]
+ parent_span_id: Annotated[str, Field(pattern=r"^[0-9a-f]{16}$")]
+ session_id: str
+ capture_content: bool
+ sampled: bool = True
+
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ @field_validator("trace_id", "parent_span_id")
+ @classmethod
+ def validate_nonzero_id(cls, value: str) -> str:
+ if int(value, 16) == 0:
+ raise ValueError("OpenTelemetry ids must be nonzero")
+ return value
+
+
class DifyExecutionContextLayerConfig(BaseModel):
"""Public config for Dify execution identity and daemon transport context."""
@@ -55,11 +82,13 @@ class DifyExecutionContextLayerConfig(BaseModel):
agent_mode: DifyExecutionContextAgentMode
invoke_from: DifyExecutionContextInvokeFrom
trace_id: str | None = None
+ tracing: ApplicationTracingContext | None = None
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
__all__ = [
+ "ApplicationTracingContext",
"DifyExecutionContextAgentConfigVersionKind",
"DifyExecutionContextAgentMode",
"DifyExecutionContextInvokeFrom",
diff --git a/dify-agent/src/dify_agent/protocol/tracing.py b/dify-agent/src/dify_agent/protocol/tracing.py
new file mode 100644
index 00000000000..7cdd6ca28cb
--- /dev/null
+++ b/dify-agent/src/dify_agent/protocol/tracing.py
@@ -0,0 +1,59 @@
+"""JSON delivery contract between the Agent collector and the Dify inner API."""
+
+from typing import Annotated, Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
+
+
+class AgentTraceEvent(BaseModel):
+ name: str
+ timestamp_ns: int = Field(ge=0)
+ attributes: dict[str, JsonValue] = Field(default_factory=dict)
+
+ model_config = ConfigDict(extra="forbid")
+
+
+class AgentTraceSpan(BaseModel):
+ trace_id: Annotated[str, Field(pattern=r"^[0-9a-f]{32}$")]
+ span_id: Annotated[str, Field(pattern=r"^[0-9a-f]{16}$")]
+ parent_span_id: Annotated[str, Field(pattern=r"^[0-9a-f]{16}$")] | None = None
+ name: str
+ start_time_ns: int = Field(ge=0)
+ end_time_ns: int = Field(ge=0)
+ attributes: dict[str, JsonValue] = Field(default_factory=dict)
+ status: Literal["ok", "error"]
+ status_message: str | None = None
+ events: list[AgentTraceEvent] = Field(default_factory=list)
+
+ model_config = ConfigDict(extra="forbid")
+
+ @model_validator(mode="after")
+ def validate_span(self) -> "AgentTraceSpan":
+ for value in (self.trace_id, self.span_id, self.parent_span_id):
+ if value is not None and int(value, 16) == 0:
+ raise ValueError("OpenTelemetry ids must be nonzero")
+ if self.end_time_ns < self.start_time_ns:
+ raise ValueError("span end time must not precede start time")
+ return self
+
+
+class AgentTraceBatch(BaseModel):
+ run_id: UUID
+ invocation_id: UUID
+ batch_id: UUID
+ spans: list[AgentTraceSpan]
+ final: bool = False
+
+ model_config = ConfigDict(extra="forbid")
+
+
+class AgentTraceBatchAcknowledgement(BaseModel):
+ result: Literal["success"]
+ batch_id: UUID
+ accepted_spans: int = Field(ge=0)
+
+ model_config = ConfigDict(extra="forbid")
+
+
+__all__ = ["AgentTraceBatch", "AgentTraceBatchAcknowledgement", "AgentTraceEvent", "AgentTraceSpan"]
diff --git a/dify-agent/src/dify_agent/runtime/application_trace_exporter.py b/dify-agent/src/dify_agent/runtime/application_trace_exporter.py
new file mode 100644
index 00000000000..585b227431d
--- /dev/null
+++ b/dify-agent/src/dify_agent/runtime/application_trace_exporter.py
@@ -0,0 +1,352 @@
+"""Bounded application-span delivery to the API-owned, durable tracing inbox.
+
+The Agent backend never holds third-party credentials. A successful export means
+the inner API acknowledged this exact batch. Exhausted retries are retained until
+run finalization; a run with dropped or unacknowledged spans never sends ``final``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import threading
+import time
+from collections import defaultdict, deque
+from collections.abc import Sequence
+from typing import Final, override
+from uuid import UUID, uuid5
+
+import httpx
+from opentelemetry.context import Context
+from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+from opentelemetry.trace import StatusCode
+from pydantic import JsonValue, TypeAdapter
+
+from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceBatchAcknowledgement, AgentTraceEvent, AgentTraceSpan
+
+DIFY_TRACING_RUN_ID_ATTRIBUTE: Final[str] = "dify.tracing.run_id"
+DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: Final[str] = "dify.tracing.invocation_id"
+_JSON_ATTRIBUTES = TypeAdapter(dict[str, JsonValue])
+logger = logging.getLogger(__name__)
+
+
+def _scope(span: ReadableSpan) -> tuple[UUID, UUID] | None:
+ attributes = span.attributes or {}
+ try:
+ return (
+ UUID(str(attributes[DIFY_TRACING_RUN_ID_ATTRIBUTE])),
+ UUID(str(attributes[DIFY_TRACING_INVOCATION_ID_ATTRIBUTE])),
+ )
+ except (KeyError, ValueError):
+ return None
+
+
+def _serialize_span(span: ReadableSpan) -> AgentTraceSpan:
+ context = span.get_span_context()
+ if context is None or span.start_time is None or span.end_time is None:
+ raise ValueError("Application tracing requires a completed span")
+ return AgentTraceSpan(
+ trace_id=f"{context.trace_id:032x}",
+ span_id=f"{context.span_id:016x}",
+ parent_span_id=f"{span.parent.span_id:016x}" if span.parent else None,
+ name=span.name,
+ start_time_ns=span.start_time,
+ end_time_ns=span.end_time,
+ attributes=_JSON_ATTRIBUTES.validate_json(json.dumps(dict(span.attributes or {}), allow_nan=False)),
+ status="error" if span.status.status_code == StatusCode.ERROR else "ok",
+ status_message=span.status.description,
+ events=[
+ AgentTraceEvent(
+ name=event.name,
+ timestamp_ns=event.timestamp,
+ attributes=_JSON_ATTRIBUTES.validate_json(json.dumps(dict(event.attributes or {}), allow_nan=False)),
+ )
+ for event in span.events
+ ],
+ )
+
+
+class JSONSpanExporter(SpanExporter):
+ """Send idempotent JSON batches using the server-owned inner API credential."""
+
+ def __init__(
+ self,
+ *,
+ inner_api_url: str,
+ inner_api_key: str,
+ client: httpx.Client | None = None,
+ max_attempts: int = 3,
+ retry_delay_seconds: float = 0.1,
+ max_pending_batches: int = 128,
+ max_batch_bytes: int = 1024 * 1024,
+ ) -> None:
+ self._url = f"{inner_api_url.rstrip('/')}/inner/api/agent/tracing/batches"
+ self._headers = {"X-Inner-Api-Key": inner_api_key}
+ self._client = client or httpx.Client(timeout=2.0, trust_env=False)
+ self._owns_client = client is None
+ self._max_attempts = max(1, max_attempts)
+ self._retry_delay_seconds = retry_delay_seconds
+ self._max_pending_batches = max_pending_batches
+ self._max_batch_bytes = max_batch_bytes
+ self._pending: dict[UUID, AgentTraceBatch] = {}
+ self._incomplete_runs: set[tuple[UUID, UUID]] = set()
+ self._state_lock = threading.Lock()
+ self._delivery_lock = threading.RLock()
+
+ def mark_incomplete(self, scope: tuple[UUID, UUID]) -> None:
+ with self._state_lock:
+ self._incomplete_runs.add(scope)
+
+ @override
+ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+ grouped: dict[tuple[UUID, UUID], list[AgentTraceSpan]] = defaultdict(list)
+ successful = True
+ for span in spans:
+ scope = _scope(span)
+ if scope is None:
+ successful = False
+ continue
+ try:
+ grouped[scope].append(_serialize_span(span))
+ except Exception:
+ self.mark_incomplete(scope)
+ successful = False
+ logger.warning("Invalid application trace span; run_id=%s invocation_id=%s", *scope)
+ with self._delivery_lock:
+ for (run_id, invocation_id), run_spans in grouped.items():
+ batches, oversized = self._batches(run_id, invocation_id, run_spans)
+ successful = successful and not oversized
+ for batch in batches:
+ if not self._deliver(batch):
+ successful = False
+ if len(self._pending) < self._max_pending_batches:
+ self._pending[batch.batch_id] = batch
+ else:
+ self.mark_incomplete((run_id, invocation_id))
+ return SpanExportResult.SUCCESS if successful else SpanExportResult.FAILURE
+
+ def _batches(
+ self, run_id: UUID, invocation_id: UUID, spans: list[AgentTraceSpan]
+ ) -> tuple[list[AgentTraceBatch], bool]:
+ batches: list[AgentTraceBatch] = []
+ current: list[AgentTraceSpan] = []
+ # Reserve more than the fixed envelope, commas and list punctuation.
+ size = 1024
+ oversized = False
+
+ def append_batch() -> None:
+ encoded = json.dumps(
+ [span.model_dump(mode="json") for span in current],
+ sort_keys=True,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+ batches.append(
+ AgentTraceBatch(
+ run_id=run_id,
+ invocation_id=invocation_id,
+ batch_id=uuid5(invocation_id, hashlib.sha256(encoded.encode()).hexdigest()),
+ spans=list(current),
+ )
+ )
+
+ for span in spans:
+ span_size = len(span.model_dump_json().encode()) + 1
+ if span_size + 1024 > self._max_batch_bytes:
+ oversized = True
+ self.mark_incomplete((run_id, invocation_id))
+ logger.warning("Application trace span exceeds delivery limit; invocation_id=%s", invocation_id)
+ continue
+ if size + span_size > self._max_batch_bytes:
+ append_batch()
+ current = []
+ size = 1024
+ current.append(span)
+ size += span_size
+ if current:
+ append_batch()
+ return batches, oversized
+
+ def _deliver(self, batch: AgentTraceBatch) -> bool:
+ payload = batch.model_dump(mode="json")
+ if len(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()) > self._max_batch_bytes:
+ self.mark_incomplete((batch.run_id, batch.invocation_id))
+ return False
+ for attempt in range(self._max_attempts):
+ retryable = True
+ try:
+ response = self._client.post(self._url, headers=self._headers, json=payload)
+ if response.is_success:
+ acknowledgement = AgentTraceBatchAcknowledgement.model_validate(response.json())
+ if acknowledgement.batch_id == batch.batch_id and acknowledgement.accepted_spans == len(
+ batch.spans
+ ):
+ return True
+ retryable = response.status_code >= 500 or response.status_code in (408, 429)
+ except (httpx.TransportError, ValueError):
+ pass
+ except Exception:
+ retryable = False
+ if not retryable or attempt + 1 == self._max_attempts:
+ break
+ time.sleep(self._retry_delay_seconds * (2**attempt))
+ logger.warning(
+ "Application trace batch was not acknowledged; run_id=%s batch_id=%s", batch.run_id, batch.batch_id
+ )
+ return False
+
+ def finish_invocation(self, run_id: UUID, invocation_id: UUID) -> bool:
+ """Retry pending data, then acknowledge completion only after all data."""
+ with self._delivery_lock:
+ successful = True
+ for batch_id, batch in tuple(self._pending.items()):
+ if (batch.run_id, batch.invocation_id) != (run_id, invocation_id):
+ continue
+ if not self._deliver(batch):
+ successful = False
+ del self._pending[batch_id]
+ with self._state_lock:
+ incomplete = (run_id, invocation_id) in self._incomplete_runs
+ if not successful or incomplete:
+ self.mark_incomplete((run_id, invocation_id))
+ return False
+ return self._deliver(
+ AgentTraceBatch(
+ run_id=run_id,
+ invocation_id=invocation_id,
+ batch_id=uuid5(invocation_id, "final"),
+ spans=[],
+ final=True,
+ )
+ )
+
+ @override
+ def shutdown(self) -> None:
+ with self._delivery_lock:
+ for batch in self._pending.values():
+ self._deliver(batch)
+ self._pending.clear()
+ with self._state_lock:
+ self._incomplete_runs.clear()
+ if self._owns_client:
+ self._client.close()
+
+ @override
+ def force_flush(self, timeout_millis: int = 30000) -> bool:
+ # Delivery is synchronous; the processor owns buffering and flush order.
+ return True
+
+
+class ApplicationBatchSpanProcessor(SpanProcessor):
+ """Batch spans without silently declaring a queue-overflowed run complete.
+
+ OTel's default batch processor silently drops oldest spans on overflow. This
+ collector must tell the API whether a run is complete, so a drop suppresses
+ its final marker. Network I/O is confined to the worker/explicit flush thread.
+ """
+
+ def __init__(
+ self,
+ exporter: JSONSpanExporter,
+ *,
+ max_queue_size: int = 2048,
+ max_export_batch_size: int = 128,
+ schedule_delay_seconds: float = 1.0,
+ ) -> None:
+ self.exporter = exporter
+ self._max_queue_size = max_queue_size
+ self._max_export_batch_size = max_export_batch_size
+ self._schedule_delay_seconds = schedule_delay_seconds
+ self._queue: deque[ReadableSpan] = deque()
+ self._started_invocations: set[tuple[UUID, UUID]] = set()
+ self._condition = threading.Condition()
+ self._export_lock = threading.Lock()
+ self._stopping = False
+ self._worker_thread = threading.Thread(target=self._worker, name="dify-application-traces", daemon=True)
+ self._worker_thread.start()
+
+ @override
+ def on_start(self, span: Span, parent_context: Context | None = None) -> None:
+ if (span.attributes or {}).get("gen_ai.operation.name") == "invoke_agent":
+ if scope := _scope(span):
+ with self._condition:
+ self._started_invocations.add(scope)
+
+ def invocation_started(self, scope: tuple[UUID, UUID]) -> bool:
+ with self._condition:
+ return scope in self._started_invocations
+
+ def release_invocation(self, scope: tuple[UUID, UUID]) -> None:
+ with self._condition:
+ self._started_invocations.discard(scope)
+
+ @override
+ def on_end(self, span: ReadableSpan) -> None:
+ with self._condition:
+ if self._stopping or len(self._queue) >= self._max_queue_size:
+ if scope := _scope(span):
+ self.exporter.mark_incomplete(scope)
+ return
+ self._queue.append(span)
+ if len(self._queue) >= self._max_export_batch_size:
+ self._condition.notify()
+
+ def _drain_locked(self) -> None:
+ # Flush a finite snapshot: unrelated busy runs must not indefinitely
+ # extend a completed invocation's finalization barrier.
+ with self._condition:
+ remaining = len(self._queue)
+ while remaining:
+ with self._condition:
+ batch = [self._queue.popleft() for _ in range(min(remaining, self._max_export_batch_size))]
+ remaining -= len(batch)
+ if not batch:
+ return
+ try:
+ self.exporter.export(batch)
+ except Exception:
+ for span in batch:
+ if scope := _scope(span):
+ self.exporter.mark_incomplete(scope)
+ logger.warning("Application trace batch export failed", exc_info=True)
+
+ def _worker(self) -> None:
+ try:
+ while True:
+ with self._condition:
+ if not self._stopping and len(self._queue) < self._max_export_batch_size:
+ self._condition.wait(self._schedule_delay_seconds)
+ stopping = self._stopping
+ with self._export_lock:
+ self._drain_locked()
+ if stopping:
+ return
+ finally:
+ self.exporter.shutdown()
+
+ @override
+ def force_flush(self, timeout_millis: int = 30000) -> bool:
+ if not self._export_lock.acquire(timeout=timeout_millis / 1000):
+ return False
+ try:
+ self._drain_locked()
+ return True
+ finally:
+ self._export_lock.release()
+
+ @override
+ def shutdown(self) -> None:
+ with self._condition:
+ self._stopping = True
+ self._condition.notify_all()
+ self._worker_thread.join(timeout=10)
+
+
+__all__ = [
+ "ApplicationBatchSpanProcessor",
+ "DIFY_TRACING_RUN_ID_ATTRIBUTE",
+ "DIFY_TRACING_INVOCATION_ID_ATTRIBUTE",
+ "JSONSpanExporter",
+]
diff --git a/dify-agent/src/dify_agent/runtime/observability.py b/dify-agent/src/dify_agent/runtime/observability.py
index b7b4d0de3e6..58de42b4ee8 100644
--- a/dify-agent/src/dify_agent/runtime/observability.py
+++ b/dify-agent/src/dify_agent/runtime/observability.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import logging
from collections.abc import Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Literal, override
@@ -10,8 +11,13 @@ from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import (
INVALID_SPAN,
Link,
+ NonRecordingSpan,
Span,
+ SpanContext,
SpanKind,
+ Status,
+ StatusCode,
+ TraceFlags,
Tracer,
TracerProvider,
get_current_span,
@@ -19,7 +25,13 @@ from opentelemetry.trace import (
)
from opentelemetry.util.types import Attributes
-from dify_agent.layers.execution_context.configs import DifyExecutionContextLayerConfig
+from dify_agent.layers.execution_context.configs import ApplicationTracingContext, DifyExecutionContextLayerConfig
+from dify_agent.runtime.application_trace_exporter import (
+ ApplicationBatchSpanProcessor,
+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE,
+ DIFY_TRACING_RUN_ID_ATTRIBUTE,
+ JSONSpanExporter,
+)
if TYPE_CHECKING:
from logfire import Logfire
@@ -36,6 +48,7 @@ DIFY_WORKFLOW_RUN_ID_ATTRIBUTE: Final[str] = "dify.workflow.run_id"
DIFY_NODE_ID_ATTRIBUTE: Final[str] = "dify.node.id"
DIFY_NODE_EXECUTION_ID_ATTRIBUTE: Final[str] = "dify.node.execution_id"
GEN_AI_USER_ID_ATTRIBUTE: Final[str] = "gen_ai.user.id"
+logger = logging.getLogger(__name__)
def dify_run_attributes(execution_context: DifyExecutionContextLayerConfig) -> tuple[tuple[str, str], ...]:
@@ -179,6 +192,9 @@ class _RunScopedTracer(Tracer):
def _run_attributes(self, attributes: Attributes) -> Attributes:
return {**(attributes or {}), **dict(self.attributes)}
+ def _parent_context(self, context: Context | None) -> Context | None:
+ return context
+
@override
def start_span(
self,
@@ -193,7 +209,7 @@ class _RunScopedTracer(Tracer):
) -> Span:
return self.delegate.start_span(
name,
- context=context,
+ context=self._parent_context(context),
kind=kind,
attributes=self._run_attributes(attributes),
links=links,
@@ -217,7 +233,7 @@ class _RunScopedTracer(Tracer):
):
return self.delegate.start_as_current_span(
name,
- context=context,
+ context=self._parent_context(context),
kind=kind,
attributes=self._run_attributes(attributes),
links=links,
@@ -239,6 +255,7 @@ class AgentObservability:
agent: Agent[Any, Any],
*,
execution_context: DifyExecutionContextLayerConfig | None = None,
+ include_content: bool | None = None,
) -> None:
"""Instrument one run's agent and attribute its spans to its Dify context.
@@ -249,7 +266,7 @@ class AgentObservability:
"""
self.client.instrument_pydantic_ai(
agent,
- include_content=self.include_content,
+ include_content=self.include_content if include_content is None else include_content,
include_binary_content=False,
tracer_provider=self._tracer_provider(execution_context),
)
@@ -277,6 +294,163 @@ class AgentObservability:
await asyncio.to_thread(self.client.shutdown, timeout_millis=5000)
+@dataclass(frozen=True)
+class _ApplicationRunTracer(_RunScopedTracer):
+ tracing: ApplicationTracingContext
+
+ @override
+ def _parent_context(self, context: Context | None) -> Context:
+ current = get_current_span(context)
+ # Preserve only this invocation's descendants. Platform spans, a previous
+ # invocation, or caller-supplied ambient context must not choose the root.
+ if isinstance(current, ReadableSpan):
+ attributes = current.attributes or {}
+ current_context = current.get_span_context()
+ if (
+ attributes.get(DIFY_TRACING_RUN_ID_ATTRIBUTE) == str(self.tracing.run_id)
+ and attributes.get(DIFY_TRACING_INVOCATION_ID_ATTRIBUTE) == str(self.tracing.invocation_id)
+ and current_context is not None
+ and current_context.trace_id == int(self.tracing.trace_id, 16)
+ ):
+ return context if context is not None else set_span_in_context(current)
+ parent = SpanContext(
+ trace_id=int(self.tracing.trace_id, 16),
+ span_id=int(self.tracing.parent_span_id, 16),
+ is_remote=True,
+ trace_flags=TraceFlags(TraceFlags.SAMPLED if self.tracing.sampled else TraceFlags.DEFAULT),
+ )
+ return set_span_in_context(NonRecordingSpan(parent), Context())
+
+
+@dataclass(frozen=True)
+class _ApplicationRunTracerProvider(RunScopedTracerProvider):
+ tracing: ApplicationTracingContext
+
+ @override
+ def get_tracer(
+ self,
+ instrumenting_module_name: str,
+ instrumenting_library_version: str | None = None,
+ schema_url: str | None = None,
+ attributes: Attributes = None,
+ ) -> Tracer:
+ delegate = self.delegate.get_tracer(
+ instrumenting_module_name,
+ instrumenting_library_version,
+ schema_url=schema_url,
+ attributes=attributes,
+ )
+ return _ApplicationRunTracer(delegate, self.attributes, self.tracing)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ApplicationAgentObservability(AgentObservability):
+ """Per-invocation policy over one process-owned internal export pipeline."""
+
+ exporter: JSONSpanExporter
+ processor: ApplicationBatchSpanProcessor
+
+ @override
+ def instrument(
+ self,
+ agent: Agent[Any, Any],
+ *,
+ execution_context: DifyExecutionContextLayerConfig | None = None,
+ include_content: bool | None = None,
+ ) -> None:
+ tracing = execution_context.tracing if execution_context else None
+ if tracing is None or not tracing.sampled:
+ agent.instrument = False
+ return
+ # The immutable API policy always wins over process defaults/callers.
+ AgentObservability.instrument(
+ self, agent, execution_context=execution_context, include_content=tracing.capture_content
+ )
+
+ @override
+ def _tracer_provider(self, execution_context: DifyExecutionContextLayerConfig | None) -> TracerProvider:
+ if execution_context is None or execution_context.tracing is None:
+ raise ValueError("Application tracing requires API-issued execution context")
+ tracing = execution_context.tracing
+ attributes = dict(dify_run_attributes(execution_context))
+ attributes.update(
+ {
+ DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),
+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),
+ DIFY_TRACE_ID_ATTRIBUTE: tracing.trace_id,
+ "gen_ai.session.id": tracing.session_id,
+ }
+ )
+ return _ApplicationRunTracerProvider(
+ self.client.config.get_tracer_provider(), tuple(attributes.items()), tracing
+ )
+
+ async def finish_invocation(self, tracing: ApplicationTracingContext) -> bool:
+ """Flush ended spans before the terminal event exposes the completed run."""
+
+ def flush_and_finish() -> bool:
+ if not self.processor.force_flush(timeout_millis=30000):
+ self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))
+ return False
+ try:
+ return self.exporter.finish_invocation(tracing.run_id, tracing.invocation_id)
+ finally:
+ self.processor.release_invocation((tracing.run_id, tracing.invocation_id))
+
+ try:
+ # The worker may continue its bounded retries after this deadline;
+ # collection must not hold the Agent's terminal event indefinitely.
+ return await asyncio.wait_for(asyncio.to_thread(flush_and_finish), timeout=5)
+ except TimeoutError:
+ logger.warning("Application trace finalization timed out; invocation_id=%s", tracing.invocation_id)
+ return False
+ except Exception:
+ logger.warning(
+ "Application trace finalization failed; invocation_id=%s", tracing.invocation_id, exc_info=True
+ )
+ return False
+
+ def record_unstarted_invocation(
+ self,
+ execution_context: DifyExecutionContextLayerConfig,
+ *,
+ start_time_ns: int,
+ end_time_ns: int,
+ error: BaseException | None,
+ cancelled: bool,
+ ) -> None:
+ """Represent setup failure/cancellation only when no native run span began.
+
+ Successful invocations keep the native tree. An unexplained empty
+ collector stays incomplete rather than being invented as success.
+ """
+ tracing = execution_context.tracing
+ if tracing is None or not tracing.sampled or (error is None and not cancelled):
+ return
+ if self.processor.invocation_started((tracing.run_id, tracing.invocation_id)):
+ return
+ error_type = type(error).__name__ if error is not None else "CancelledError"
+ message = "Agent cancelled before execution" if cancelled else error_type
+ if not cancelled and error is not None and tracing.capture_content:
+ message = str(error) or error_type
+ try:
+ tracer = self._tracer_provider(execution_context).get_tracer("dify-agent.initialization")
+ span = tracer.start_span(
+ "invoke_agent dify-agent",
+ start_time=start_time_ns,
+ attributes={
+ "gen_ai.operation.name": "invoke_agent",
+ "dify.agent.execution_state": "cancelled" if cancelled else "initialization_failed",
+ "error.type": error_type,
+ },
+ )
+ span.set_status(Status(StatusCode.ERROR, message))
+ span.end(end_time=max(start_time_ns, end_time_ns))
+ except Exception:
+ self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))
+ logger.warning("Unable to record Agent initialization failure; invocation_id=%s", tracing.invocation_id)
+
+
__all__ = [
"DIFY_AGENT_ID_ATTRIBUTE",
"DIFY_APP_ID_ATTRIBUTE",
diff --git a/dify-agent/src/dify_agent/runtime/run_scheduler.py b/dify-agent/src/dify_agent/runtime/run_scheduler.py
index dfebfdadade..dfdae91c432 100644
--- a/dify-agent/src/dify_agent/runtime/run_scheduler.py
+++ b/dify-agent/src/dify_agent/runtime/run_scheduler.py
@@ -28,7 +28,7 @@ from dify_agent.runtime.event_coalescer import (
DEFAULT_TEXT_DELTA_MAX_CHARS,
)
from dify_agent.runtime.event_sink import RunEventSink, RunFinalizationResult, emit_run_failed
-from dify_agent.runtime.observability import AgentObservability
+from dify_agent.runtime.observability import AgentObservability, ApplicationAgentObservability
from dify_agent.runtime.runner import DEFAULT_AGENT_RUN_TIMEOUT_SECONDS, AgentRunRunner
from dify_agent.server.schemas import RunRecord
@@ -118,6 +118,7 @@ class RunScheduler:
runner_factory: RunRunnerFactory | None
services: Services
agent_observability: AgentObservability | None
+ application_observability: ApplicationAgentObservability | None
_lifecycle_lock: asyncio.Lock
def __init__(
@@ -132,6 +133,7 @@ class RunScheduler:
stream_text_delta_max_chars: int = DEFAULT_TEXT_DELTA_MAX_CHARS,
runner_factory: RunRunnerFactory | None = None,
agent_observability: AgentObservability | None = None,
+ application_observability: ApplicationAgentObservability | None = None,
) -> None:
self.store = store
self.shutdown_grace_seconds = shutdown_grace_seconds
@@ -144,6 +146,7 @@ class RunScheduler:
self.services = services
self.runner_factory = runner_factory
self.agent_observability = agent_observability
+ self.application_observability = application_observability
self._lifecycle_lock = asyncio.Lock()
async def create_run(self, request: CreateRunRequest) -> RunRecord:
@@ -316,6 +319,7 @@ class RunScheduler:
stream_text_delta_flush_interval_seconds=self.stream_text_delta_flush_interval_seconds,
stream_text_delta_max_chars=self.stream_text_delta_max_chars,
agent_observability=self.agent_observability,
+ application_observability=self.application_observability,
)
def _discard_active_run(self, run_id: str) -> None:
diff --git a/dify-agent/src/dify_agent/runtime/runner.py b/dify-agent/src/dify_agent/runtime/runner.py
index 0ad20ca75f0..3e13fd73a7e 100644
--- a/dify-agent/src/dify_agent/runtime/runner.py
+++ b/dify-agent/src/dify_agent/runtime/runner.py
@@ -8,6 +8,8 @@ scopes. Workspace and Binding retirement remain API-owned.
"""
import asyncio
+import logging
+import time
from collections.abc import AsyncIterable, Callable, Mapping
from dataclasses import dataclass
from typing import Protocol, cast, runtime_checkable
@@ -53,13 +55,15 @@ from dify_agent.runtime.event_sink import (
emit_run_started,
emit_run_succeeded,
)
-from dify_agent.runtime.observability import AgentObservability
+from dify_agent.layers.execution_context.configs import ApplicationTracingContext, DifyExecutionContextLayerConfig
+from dify_agent.runtime.observability import AgentObservability, ApplicationAgentObservability
from dify_agent.runtime.user_prompt_validation import EMPTY_USER_PROMPTS_ERROR, has_non_blank_user_prompt
_AGENT_OUTPUT_ADAPTER = TypeAdapter(object)
_MAX_AGENT_STEPS_PER_RUN = 500
DEFAULT_AGENT_RUN_TIMEOUT_SECONDS = 60 * 60
+logger = logging.getLogger(__name__)
@runtime_checkable
@@ -136,6 +140,7 @@ class AgentRunRunner:
stream_text_delta_flush_interval_seconds: float
stream_text_delta_max_chars: int
agent_observability: AgentObservability | None
+ application_observability: ApplicationAgentObservability | None
_terminal_session_snapshot: SessionSnapshot | None
_terminal_usage: AgentRunUsage | None
@@ -153,6 +158,7 @@ class AgentRunRunner:
stream_text_delta_flush_interval_seconds: float = DEFAULT_TEXT_DELTA_FLUSH_INTERVAL_SECONDS,
stream_text_delta_max_chars: int = DEFAULT_TEXT_DELTA_MAX_CHARS,
agent_observability: AgentObservability | None = None,
+ application_observability: ApplicationAgentObservability | None = None,
) -> None:
if stream_text_delta_flush_interval_seconds <= 0:
raise ValueError("stream_text_delta_flush_interval_seconds must be positive")
@@ -169,6 +175,7 @@ class AgentRunRunner:
self.stream_text_delta_flush_interval_seconds = stream_text_delta_flush_interval_seconds
self.stream_text_delta_max_chars = stream_text_delta_max_chars
self.agent_observability = agent_observability
+ self.application_observability = application_observability
self._terminal_session_snapshot = None
self._terminal_usage = None
@@ -186,12 +193,27 @@ class AgentRunRunner:
"""Execute the run and emit the documented event sequence."""
self._terminal_session_snapshot = None
self._terminal_usage = None
+ execution_context = self._application_execution_context()
+ started_at_ns = time.time_ns()
if self.is_cancelled():
+ await self._finalize_application_tracing(execution_context, started_at_ns, error=None, cancelled=True)
return
+ failure: BaseException | None = None
try:
- _ = await emit_run_started(self.sink, run_id=self.run_id)
- outcome = await self._run_agent()
+ try:
+ _ = await emit_run_started(self.sink, run_id=self.run_id)
+ outcome = await self._run_agent()
+ except BaseException as exc:
+ failure = exc
+ raise
+ finally:
+ await self._finalize_application_tracing(
+ execution_context,
+ started_at_ns,
+ error=failure,
+ cancelled=isinstance(failure, asyncio.CancelledError) or self.is_cancelled(),
+ )
except Exception as exc:
if self.is_cancelled():
return
@@ -219,7 +241,38 @@ class AgentRunRunner:
usage=outcome.usage,
)
+ def _application_execution_context(self) -> DifyExecutionContextLayerConfig | None:
+ """Resolve valid correlation even if later model/resource setup fails."""
+ if self.application_observability is None:
+ return None
+ try:
+ layers = {layer.name: layer for layer in self.request.composition.layers}
+ llm_config = llm_module.Config.model_validate(layers[DIFY_AGENT_MODEL_LAYER_ID].config)
+ return DifyExecutionContextLayerConfig.model_validate(layers[llm_config.execution_context].config)
+ except (KeyError, TypeError, ValueError):
+ return None
+
+ async def _finalize_application_tracing(
+ self,
+ execution_context: DifyExecutionContextLayerConfig | None,
+ start_time_ns: int,
+ *,
+ error: BaseException | None,
+ cancelled: bool,
+ ) -> None:
+ if execution_context is None or execution_context.tracing is None or self.application_observability is None:
+ return
+ self.application_observability.record_unstarted_invocation(
+ execution_context,
+ start_time_ns=start_time_ns,
+ end_time_ns=time.time_ns(),
+ error=error,
+ cancelled=cancelled,
+ )
+ await self.application_observability.finish_invocation(execution_context.tracing)
+
async def _run_agent(self) -> RunSuccessOutcome:
+ application_tracing: ApplicationTracingContext | None = None
try:
deps = load_modules(self.request, self.services, self.run_id, registry=self.registry)
modules = create_modules(deps, registry=self.registry)
@@ -260,15 +313,23 @@ class AgentRunRunner:
capabilities=capabilities,
output_type=output_type,
)
- if self.agent_observability is not None:
- from dify_agent.layers.execution_context.configs import DifyExecutionContextLayerConfig
-
- self.agent_observability.instrument(
- agent,
- execution_context=DifyExecutionContextLayerConfig.model_validate(
- deps.layers[llm_config.execution_context]["config"]
- ),
- )
+ execution_context = DifyExecutionContextLayerConfig.model_validate(
+ deps.layers[llm_config.execution_context]["config"]
+ )
+ observability: AgentObservability | None = self.agent_observability
+ if execution_context.tracing is not None and self.application_observability is not None:
+ application_tracing = execution_context.tracing
+ observability = self.application_observability
+ if observability is not None:
+ try:
+ observability.instrument(agent, execution_context=execution_context)
+ except Exception:
+ agent.instrument = False
+ if application_tracing is not None and self.application_observability is not None:
+ self.application_observability.exporter.mark_incomplete(
+ (application_tracing.run_id, application_tracing.invocation_id)
+ )
+ logger.warning("Agent tracing instrumentation failed; run_id=%s", self.run_id, exc_info=True)
except (KeyError, TypeError, ValueError) as exc:
raise AgentRunValidationError(str(exc)) from exc
diff --git a/dify-agent/src/dify_agent/server/app.py b/dify-agent/src/dify_agent/server/app.py
index abae36a5e0a..ef1be498bef 100644
--- a/dify-agent/src/dify_agent/server/app.py
+++ b/dify-agent/src/dify_agent/server/app.py
@@ -30,7 +30,11 @@ from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
from dify_agent.runtime.context import Services
from dify_agent.runtime.run_scheduler import RunScheduler
from dify_agent.server.auth import create_bearer_token_dependency
-from dify_agent.server.observability import configure_agent_observability, configure_server_observability
+from dify_agent.server.observability import (
+ configure_agent_observability,
+ configure_application_observability,
+ configure_server_observability,
+)
from dify_agent.server.routes.e2b_usage import create_e2b_usage_router
from dify_agent.server.routes.runs import create_runs_router
from dify_agent.server.routes.execution_bindings import create_execution_bindings_router
@@ -93,7 +97,9 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
@asynccontextmanager
async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
agent_observability = configure_agent_observability(resolved_settings)
+ application_observability = configure_application_observability(resolved_settings)
_app.state.agent_observability = agent_observability
+ _app.state.application_observability = application_observability
redis = Redis.from_url(resolved_settings.redis_url)
plugin_daemon_http_client = create_plugin_daemon_http_client(resolved_settings)
dify_api_inner_http_client = create_dify_api_inner_http_client(resolved_settings)
@@ -123,6 +129,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
stream_text_delta_flush_interval_seconds=(resolved_settings.stream_text_delta_flush_interval_ms / 1000),
stream_text_delta_max_chars=resolved_settings.stream_text_delta_max_chars,
agent_observability=agent_observability,
+ application_observability=application_observability,
)
state["store"] = store
state["scheduler"] = scheduler
@@ -136,6 +143,8 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
# Closed last so runs draining above still reach the exporter.
if agent_observability is not None:
await agent_observability.aclose()
+ if application_observability is not None:
+ await application_observability.aclose()
app = FastAPI(title="Dify Agent Run Server", version="0.1.0", lifespan=lifespan)
app.state.platform_observability = configure_server_observability(app, settings=resolved_settings)
diff --git a/dify-agent/src/dify_agent/server/observability.py b/dify-agent/src/dify_agent/server/observability.py
index 01fecfd0cee..97ffcf94a86 100644
--- a/dify-agent/src/dify_agent/server/observability.py
+++ b/dify-agent/src/dify_agent/server/observability.py
@@ -20,6 +20,7 @@ code that mutates ``os.environ`` concurrently.
from __future__ import annotations
import os
+import logging
from collections.abc import Iterator
from contextlib import contextmanager
from threading import RLock
@@ -31,12 +32,14 @@ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExport
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ALWAYS_ON, ParentBased
-from dify_agent.runtime.observability import AgentObservability, IsolatedTracerProvider
+from dify_agent.runtime.application_trace_exporter import ApplicationBatchSpanProcessor, JSONSpanExporter
+from dify_agent.runtime.observability import AgentObservability, ApplicationAgentObservability, IsolatedTracerProvider
from dify_agent.server.settings import ServerSettings
_global_instrumentation_ready = False
_global_trace_context_mode: Literal["isolated", "shared"] | None = None
_observability_configuration_lock = RLock()
+logger = logging.getLogger(__name__)
@contextmanager
@@ -174,8 +177,42 @@ def configure_agent_observability(settings: ServerSettings) -> AgentObservabilit
)
+def configure_application_observability(settings: ServerSettings) -> ApplicationAgentObservability | None:
+ """Configure application collection once, separately from platform/trajectory.
+
+ Application policy is API-issued per invocation; destinations and provider
+ credentials remain inside the Dify API. No SDK environment is inherited.
+ """
+ if not settings.application_tracing_enabled:
+ return None
+ if not settings.inner_api_key:
+ logger.warning("Application tracing is enabled without an inner API credential; collector disabled")
+ return None
+ exporter = JSONSpanExporter(inner_api_url=settings.inner_api_url, inner_api_key=settings.inner_api_key)
+ processor = ApplicationBatchSpanProcessor(exporter)
+ try:
+ with _isolated_agent_environment():
+ client = logfire.configure(
+ local=True,
+ send_to_logfire=False,
+ service_name="dify-agent-application",
+ console=False,
+ metrics=False,
+ inspect_arguments=False,
+ add_baggage_to_attributes=False,
+ sampling=logfire.SamplingOptions(head=ParentBased(ALWAYS_ON)),
+ additional_span_processors=[processor],
+ )
+ except Exception:
+ processor.shutdown()
+ logger.warning("Application tracing setup failed; collector disabled", exc_info=True)
+ return None
+ return ApplicationAgentObservability(client=client, exporter=exporter, processor=processor)
+
+
__all__ = [
"AgentObservability",
"configure_agent_observability",
+ "configure_application_observability",
"configure_server_observability",
]
diff --git a/dify-agent/src/dify_agent/server/settings.py b/dify-agent/src/dify_agent/server/settings.py
index a5b5623a9ba..9bab69c5ea7 100644
--- a/dify-agent/src/dify_agent/server/settings.py
+++ b/dify-agent/src/dify_agent/server/settings.py
@@ -131,6 +131,7 @@ class ServerSettings(BaseSettings):
server_secret_key: str | None = None
api_token: str | None = None
trajectory_enabled: bool = False
+ application_tracing_enabled: bool = False
trajectory_otlp_traces_endpoint: AnyHttpUrl | None = None
trajectory_otlp_headers: dict[str, SecretStr] = Field(default_factory=dict, repr=False)
trajectory_service_name: str = Field(default="dify-agent-trajectory", min_length=1)
diff --git a/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py b/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py
index eb6df382054..83bc1319467 100644
--- a/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py
+++ b/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py
@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
from decimal import Decimal
from typing import cast
from unittest.mock import patch
+from uuid import uuid4
import httpx
import pytest
@@ -25,10 +26,15 @@ from pydantic_ai.messages import (
UserPromptPart,
)
from pydantic_ai.models import ModelRequestParameters
+from pydantic_ai.models.instrumented import InstrumentationSettings, InstrumentedModel
from pydantic_ai.tools import ToolDefinition
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import StatusCode
from dify_agent.adapters.llm import DifyApiLLMProvider, DifyLLMAdapterModel
-from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
+from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfig
from ._test_support import (
AssistantPromptMessage,
@@ -158,6 +164,7 @@ class DifyLLMAdapterModelTests(unittest.IsolatedAsyncioTestCase):
payload = json.loads(request.content.decode("utf-8"))
self.assertEqual(payload["caller"]["user_id"], "user-123")
+ self.assertNotIn("tracing", payload["caller"])
data = payload["target"]
self.assertEqual(data["provider"], "langgenius/openai/openai")
self.assertEqual(data["model"], "demo-model")
@@ -209,6 +216,152 @@ class DifyLLMAdapterModelTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(response.parts[0].part_kind, "text")
self.assertEqual(cast(TextPart, response.parts[0]).content, "adapter response")
+ async def test_application_tracing_policy_is_excluded_from_gateway_caller(self) -> None:
+ provider = self.make_provider()
+ provider.execution_context.tracing = ApplicationTracingContext(
+ run_id=uuid4(),
+ invocation_id=uuid4(),
+ trace_id=uuid4().hex,
+ parent_span_id=uuid4().hex[:16],
+ session_id="test-session",
+ capture_content=True,
+ )
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ payload = json.loads(request.content)
+ self.assertNotIn("tracing", payload["caller"])
+ self.assertEqual(payload["caller"]["tenant_id"], "tenant-1")
+ return build_stream_response(*single_text_chunk("hello"))
+
+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):
+ adapter = DifyLLMAdapterModel("demo-model", provider, model_provider="openai")
+ await adapter.request([ModelRequest(parts=[UserPromptPart("hello")])], None, ModelRequestParameters())
+
+ async def test_real_model_span_records_gateway_accounting_on_streaming_and_nonstreaming_requests(self) -> None:
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+ usage = make_usage(
+ prompt_tokens=17,
+ completion_tokens=9,
+ total_price=Decimal("0.001234"),
+ currency="CNY",
+ time_to_first_token=0.125,
+ )
+
+ def handler(_request: httpx.Request) -> httpx.Response:
+ return build_stream_response(
+ LLMResultChunk(
+ model="demo-model",
+ delta=LLMResultChunkDelta(
+ index=0,
+ message=AssistantPromptMessage(content="hello"),
+ usage=usage,
+ ),
+ ),
+ LLMResultChunk(
+ model="demo-model",
+ delta=LLMResultChunkDelta(
+ index=1,
+ message=AssistantPromptMessage(content=" world"),
+ ),
+ ),
+ )
+
+ try:
+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):
+ adapter = InstrumentedModel(
+ DifyLLMAdapterModel("demo-model", self.make_provider(), model_provider="openai"),
+ InstrumentationSettings(tracer_provider=tracer_provider),
+ )
+ messages = [ModelRequest(parts=[UserPromptPart("test")])]
+ await adapter.request(messages, None, ModelRequestParameters())
+ async with adapter.request_stream(messages, None, ModelRequestParameters()) as stream:
+ async for _ in stream:
+ pass
+ spans = exporter.get_finished_spans()
+ self.assertEqual(len(spans), 2)
+ for span in spans:
+ attributes = span.attributes or {}
+ self.assertEqual(attributes["gen_ai.operation.name"], "chat")
+ self.assertEqual(attributes["dify.llm.total_price"], "0.001234")
+ self.assertEqual(attributes["dify.llm.currency"], "CNY")
+ self.assertEqual(attributes["gen_ai.response.time_to_first_token"], 0.125)
+ self.assertEqual(attributes["dify.llm.time_to_first_token_unit"], "s")
+ self.assertEqual(attributes["dify.llm.time_to_first_token_source"], "gateway_usage")
+ self.assertEqual(attributes["gen_ai.usage.input_tokens"], 17)
+ self.assertEqual(attributes["gen_ai.usage.output_tokens"], 9)
+ self.assertEqual(attributes["gen_ai.usage.total_tokens"], 26)
+ self.assertNotIn("gen_ai.usage.cache_read.input_tokens", attributes)
+ finally:
+ tracer_provider.shutdown()
+
+ async def test_streaming_gateway_failure_marks_the_real_chat_span_as_error(self) -> None:
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+
+ def handler(_request: httpx.Request) -> httpx.Response:
+ return build_stream_error("EndpointSetupFailedError", "sensitive-provider-detail")
+
+ try:
+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):
+ adapter = InstrumentedModel(
+ DifyLLMAdapterModel("demo-model", self.make_provider(), model_provider="openai"),
+ InstrumentationSettings(tracer_provider=tracer_provider),
+ )
+ with self.assertRaises(UserError):
+ async with adapter.request_stream(
+ [ModelRequest(parts=[UserPromptPart("test")])], None, ModelRequestParameters()
+ ) as stream:
+ async for _ in stream:
+ pass
+ spans = exporter.get_finished_spans()
+ self.assertEqual(len(spans), 1)
+ self.assertEqual(spans[0].status.status_code, StatusCode.ERROR)
+ self.assertEqual((spans[0].attributes or {}).get("error.type"), "UserError")
+ self.assertNotIn("dify.llm.total_price", spans[0].attributes or {})
+ finally:
+ tracer_provider.shutdown()
+
+ async def test_model_span_measures_first_content_without_inventing_missing_cost(self) -> None:
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+
+ def handler(_request: httpx.Request) -> httpx.Response:
+ return build_stream_response(
+ LLMResultChunk(
+ model="demo-model",
+ delta=LLMResultChunkDelta(
+ index=0,
+ message=AssistantPromptMessage(content=""),
+ ),
+ ),
+ LLMResultChunk(
+ model="demo-model",
+ delta=LLMResultChunkDelta(
+ index=1,
+ message=AssistantPromptMessage(content="hello"),
+ ),
+ ),
+ )
+
+ try:
+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):
+ adapter = InstrumentedModel(
+ DifyLLMAdapterModel("demo-model", self.make_provider(), model_provider="openai"),
+ InstrumentationSettings(tracer_provider=tracer_provider),
+ )
+ await adapter.request([ModelRequest(parts=[UserPromptPart("test")])], None, ModelRequestParameters())
+ attributes = exporter.get_finished_spans()[0].attributes or {}
+ self.assertGreaterEqual(attributes["gen_ai.response.time_to_first_token"], 0)
+ self.assertEqual(attributes["dify.llm.time_to_first_token_source"], "gateway_first_content")
+ self.assertNotIn("dify.llm.total_price", attributes)
+ self.assertNotIn("dify.llm.currency", attributes)
+ finally:
+ tracer_provider.shutdown()
+
async def test_request_accumulates_complete_dify_usage_across_model_rounds(self) -> None:
usages = [
make_usage(
diff --git a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py
new file mode 100644
index 00000000000..dc0f7716253
--- /dev/null
+++ b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py
@@ -0,0 +1,188 @@
+"""Real Pydantic AI tool spans retain handled API failure status without failing the run."""
+
+import json
+from collections.abc import Iterator
+
+import httpx
+import logfire
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import StatusCode
+from pydantic_ai import Agent
+from pydantic_ai.messages import ModelMessage, ModelRequest, ModelResponse, TextPart, ToolCallPart, ToolReturnPart
+from pydantic_ai.models.function import AgentInfo, FunctionModel
+
+from dify_agent.layers.dify_core_tools.layer import Config, Toolset
+from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
+from dify_agent.runtime.application_trace_exporter import _serialize_span
+from dify_agent.runtime.context import Deps, Services
+from dify_agent.runtime.observability import AgentObservability
+
+
+def observability(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[AgentObservability, InMemorySpanExporter]]:
+ monkeypatch.setattr(Agent, "_instrument_default", False)
+ exporter = InMemorySpanExporter()
+ client = logfire.configure(
+ local=True,
+ send_to_logfire=False,
+ console=False,
+ metrics=False,
+ inspect_arguments=False,
+ additional_span_processors=[SimpleSpanProcessor(exporter)],
+ )
+ yield AgentObservability(client), exporter
+ client.shutdown()
+
+
+def tool_model(messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse:
+ returned = [
+ part
+ for message in messages
+ if isinstance(message, ModelRequest)
+ for part in message.parts
+ if isinstance(part, ToolReturnPart)
+ ]
+ if not returned:
+ return ModelResponse(parts=[ToolCallPart("calculate", {}, tool_call_id="owned-call")])
+ return ModelResponse(parts=[TextPart(f"Handled: {returned[-1].content}")])
+
+
+def dependencies(client: httpx.AsyncClient, *, missing_context: bool = False) -> Deps:
+ config = Config(
+ tools=[
+ {
+ "provider_type": "builtin",
+ "provider_id": "code",
+ "tool_name": "calculate",
+ "parameters_json_schema": {"type": "object", "properties": {}},
+ }
+ ]
+ )
+ context = DifyExecutionContextLayerConfig(
+ tenant_id="tenant",
+ user_id=None if missing_context else "user",
+ user_from="account",
+ app_id="app",
+ agent_mode="agent_app",
+ invoke_from="web-app",
+ )
+ return Deps(
+ layers={
+ "core_tools": {"config": config.model_dump(mode="json"), "state": {}},
+ "execution_context": {"config": context.model_dump(mode="json"), "state": {}},
+ },
+ services=Services(client, client, inner_api_key="synthetic-key"),
+ run_id="test",
+ )
+
+
[email protected]("include_content", [False, True])
[email protected]("failure", ["invocation", "timeout", "configuration"])
+async def test_handled_failure_marks_only_its_native_tool_span(
+ observability: tuple[AgentObservability, InMemorySpanExporter],
+ include_content: bool,
+ failure: str,
+) -> None:
+ telemetry, exporter = observability
+ calls: list[str] = []
+
+ def handle(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ if failure == "timeout":
+ raise httpx.ReadTimeout("private-transport-content", request=request)
+ return httpx.Response(422, json={"code": "agent_tool_invoke_failed", "message": "private-error-content"})
+
+ async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client:
+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])
+ telemetry.instrument(agent, include_content=include_content)
+ result = await agent.run("Call the tool", deps=dependencies(client, missing_context=failure == "configuration"))
+ assert result.output.startswith("Handled:")
+ if failure == "invocation":
+ assert "tool invoke error: private-error-content" in result.output
+ elif failure == "timeout":
+ assert "temporarily unavailable" in result.output
+ else:
+ assert "required execution context is missing" in result.output
+ assert len(calls) == (0 if failure == "configuration" else 1)
+ telemetry.client.force_flush()
+ spans = exporter.get_finished_spans()
+ tools = [span for span in spans if (span.attributes or {}).get("gen_ai.operation.name") == "execute_tool"]
+ roots = [span for span in spans if (span.attributes or {}).get("gen_ai.operation.name") == "invoke_agent"]
+ assert len(tools) == 1
+ assert len(roots) == 1
+ assert tools[0].status.status_code is StatusCode.ERROR
+ assert roots[0].status.status_code is not StatusCode.ERROR
+ serialized = _serialize_span(tools[0])
+ assert serialized.status == "error"
+ assert serialized.status_message in {"DifyCoreToolsClientError", "DifyCoreToolsClientConfigurationError"}
+ if not include_content:
+ assert "private-error-content" not in json.dumps(
+ [_serialize_span(span).model_dump(mode="json") for span in spans]
+ )
+ assert "private-transport-content" not in serialized.model_dump_json()
+
+
+async def test_successful_observation_is_not_classified_by_its_text(
+ observability: tuple[AgentObservability, InMemorySpanExporter],
+) -> None:
+ telemetry, exporter = observability
+ observation = "tool invoke error timeout is only sample text in a successful result"
+ async with httpx.AsyncClient(
+ transport=httpx.MockTransport(lambda _request: httpx.Response(200, json={"observation": observation}))
+ ) as client:
+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])
+ telemetry.instrument(agent)
+ result = await agent.run("Call the tool", deps=dependencies(client))
+ assert observation in result.output
+ telemetry.client.force_flush()
+ tool = next(
+ span
+ for span in exporter.get_finished_spans()
+ if (span.attributes or {}).get("gen_ai.operation.name") == "execute_tool"
+ )
+ assert tool.status.status_code is not StatusCode.ERROR
+ assert _serialize_span(tool).status == "ok"
+
+
+ "attributes",
+ [
+ {"gen_ai.operation.name": "invoke_agent"},
+ {"gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "calculate", "gen_ai.tool.call.id": "other-call"},
+ {
+ "gen_ai.operation.name": "execute_tool",
+ "gen_ai.tool.name": "other-tool",
+ "gen_ai.tool.call.id": "owned-call",
+ },
+ ],
+)
+async def test_uninstrumented_tool_failure_does_not_mark_an_unrelated_ambient_span(
+ attributes: dict[str, str],
+) -> None:
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ try:
+ async with httpx.AsyncClient(
+ transport=httpx.MockTransport(
+ lambda _request: httpx.Response(
+ 422, json={"code": "agent_tool_invoke_failed", "message": "handled failure"}
+ )
+ )
+ ) as client:
+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])
+ agent.instrument = False
+ with provider.get_tracer("ambient").start_as_current_span("Ambient", attributes=attributes):
+ result = await agent.run("Call the tool", deps=dependencies(client))
+ assert "tool invoke error: handled failure" in result.output
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code is not StatusCode.ERROR
+ finally:
+ provider.shutdown()
diff --git a/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py b/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py
index 34aa332c5d8..3048560b0cf 100644
--- a/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py
+++ b/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py
@@ -6,6 +6,7 @@ from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
def test_execution_context_package_exports_client_safe_config_symbols_only() -> None:
assert execution_context_exports.__all__ == [
+ "ApplicationTracingContext",
"DifyExecutionContextAgentConfigVersionKind",
"DifyExecutionContextAgentMode",
"DifyExecutionContextInvokeFrom",
diff --git a/dify-agent/tests/local/dify_agent/runtime/test_application_tracing.py b/dify-agent/tests/local/dify_agent/runtime/test_application_tracing.py
new file mode 100644
index 00000000000..e9d37cbaa00
--- /dev/null
+++ b/dify-agent/tests/local/dify_agent/runtime/test_application_tracing.py
@@ -0,0 +1,511 @@
+"""Exercise the actual Pydantic AI/Logfire tree and JSON delivery over local HTTP."""
+
+import asyncio
+import json
+import os
+import threading
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from uuid import uuid4
+
+import httpx
+import logfire
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export import SpanExportResult
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags, use_span
+from pydantic import ValidationError
+from pydantic_ai import Tool
+from pydantic_ai.models.test import TestModel
+from pydantic_ai.models.function import FunctionModel
+from pydantic_ai.toolsets import FunctionToolset
+
+from dify_agent.layers.dify_plugin import llm_layer
+from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfig
+from dify_agent.protocol.schemas import CreateRunRequest, RunComposition, RunLayerSpec, RunSucceededEvent
+from dify_agent.protocol.tracing import AgentTraceBatch
+from dify_agent.runtime.agent_factory import create_agent
+from dify_agent.runtime.application_trace_exporter import (
+ ApplicationBatchSpanProcessor,
+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE,
+ DIFY_TRACING_RUN_ID_ATTRIBUTE,
+ JSONSpanExporter,
+)
+from dify_agent.runtime.context import Services
+from dify_agent.runtime.event_sink import InMemoryRunEventSink
+from dify_agent.runtime.observability import AgentObservability
+from dify_agent.runtime.runner import AgentRunRunner
+from dify_agent.server.observability import configure_application_observability
+from dify_agent.server.settings import ServerSettings
+
+
+@dataclass
+class TraceInbox:
+ batches: list[dict[str, object]] = field(default_factory=list)
+ headers: list[str | None] = field(default_factory=list)
+ paths: list[str] = field(default_factory=list)
+ failures: int = 0
+ status: int | None = None
+ url: str = ""
+
+
[email protected](autouse=True)
+def isolated_environment(monkeypatch, tmp_path) -> None:
+ monkeypatch.chdir(tmp_path)
+ for key in tuple(os.environ):
+ if key.startswith(("OTEL_", "LOGFIRE_", "DIFY_AGENT_")):
+ monkeypatch.delenv(key)
+
+
+def inbox() -> Iterator[TraceInbox]:
+ state = TraceInbox()
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ payload = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ batch = AgentTraceBatch.model_validate(payload)
+ state.batches.append(payload)
+ state.headers.append(self.headers.get("X-Inner-Api-Key"))
+ state.paths.append(self.path)
+ status = state.status or (503 if state.failures else 200)
+ state.failures = max(0, state.failures - 1)
+ response = json.dumps(
+ {"result": "success", "batch_id": str(batch.batch_id), "accepted_spans": len(batch.spans)}
+ ).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(response)))
+ self.end_headers()
+ self.wfile.write(response)
+
+ def log_message(self, *_args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)
+ thread.start()
+ state.url = f"http://127.0.0.1:{server.server_port}"
+ try:
+ yield state
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=1)
+
+
+def tracing_context(*, capture_content=False, sampled=True, run_id=None):
+ return ApplicationTracingContext(
+ run_id=run_id or uuid4(),
+ invocation_id=uuid4(),
+ trace_id=uuid4().hex,
+ parent_span_id=uuid4().hex[:16],
+ session_id="test-session",
+ capture_content=capture_content,
+ sampled=sampled,
+ )
+
+
+def execution_context(tracing):
+ return DifyExecutionContextLayerConfig(
+ tenant_id="test-tenant",
+ app_id="test-app",
+ agent_id="test-agent",
+ agent_mode="agent_app",
+ invoke_from="web-app",
+ tracing=tracing,
+ )
+
+
+def configured_collector(inbox):
+ collector = configure_application_observability(
+ ServerSettings(
+ application_tracing_enabled=True,
+ inner_api_url=inbox.url,
+ inner_api_key="test-inner-key",
+ )
+ )
+ assert collector is not None
+ return collector
+
+
+def _tool() -> str:
+ return "tool-private-content"
+
+
[email protected]("capture_content", [False, True])
+def test_actual_agent_tree_uses_api_parent_and_per_run_content_policy(inbox, capture_content):
+ collector = configured_collector(inbox)
+ tracing = tracing_context(capture_content=capture_content)
+ agent = create_agent(
+ TestModel(custom_output_text="answer-private-content"),
+ toolsets=[FunctionToolset(tools=[Tool(_tool)])],
+ capabilities=[],
+ )
+ collector.instrument(agent, execution_context=execution_context(tracing))
+ unrelated = NonRecordingSpan(SpanContext(trace_id=1, span_id=2, is_remote=True, trace_flags=TraceFlags(1)))
+ try:
+ with use_span(unrelated):
+ assert agent.run_sync("prompt-private-content").output == "answer-private-content"
+ assert asyncio.run(collector.finish_invocation(tracing))
+ spans = [span for batch in inbox.batches for span in batch["spans"]]
+ assert spans
+ assert any("execute_tool" in span["name"] or "_tool" in span["name"] for span in spans)
+ roots = [span for span in spans if span["parent_span_id"] == tracing.parent_span_id]
+ assert len(roots) == 1
+ assert roots[0]["name"] == "invoke_agent dify-agent"
+ ids = {span["span_id"] for span in spans}
+ for span in spans:
+ assert span["trace_id"] == tracing.trace_id
+ assert span["parent_span_id"] == tracing.parent_span_id or span["parent_span_id"] in ids
+ assert span["attributes"]["dify.tenant_id"] == "test-tenant"
+ assert span["attributes"][DIFY_TRACING_INVOCATION_ID_ATTRIBUTE] == str(tracing.invocation_id)
+ serialized = json.dumps(spans)
+ assert ("prompt-private-content" in serialized) is capture_content
+ assert ("tool-private-content" in serialized) is capture_content
+ assert inbox.batches[-1]["final"] is True
+ assert inbox.batches[-1]["spans"] == []
+ assert set(inbox.paths) == {"/inner/api/agent/tracing/batches"}
+ assert set(inbox.headers) == {"test-inner-key"}
+ finally:
+ asyncio.run(collector.aclose())
+
+
+def test_retry_reuses_exact_batch_and_completion_waits_for_ack(inbox):
+ collector = configured_collector(inbox)
+ tracing = tracing_context()
+ inbox.failures = 1
+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])
+ collector.instrument(agent, execution_context=execution_context(tracing))
+ try:
+ agent.run_sync("hello")
+ assert asyncio.run(collector.finish_invocation(tracing))
+ assert len(inbox.batches) >= 3
+ assert inbox.batches[0] == inbox.batches[1]
+ assert inbox.batches[-1]["final"] is True
+ finally:
+ asyncio.run(collector.aclose())
+
+
+def test_unacknowledged_invocation_never_emits_final_or_blocks_another_invocation(inbox):
+ collector = configured_collector(inbox)
+ first = tracing_context()
+ second = tracing_context(run_id=first.run_id)
+ try:
+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])
+ collector.instrument(agent, execution_context=execution_context(first))
+ agent.run_sync("hello")
+ inbox.status = 503
+ assert not asyncio.run(collector.finish_invocation(first))
+ assert not any(batch["final"] for batch in inbox.batches)
+ inbox.status = None
+ collector.instrument(agent, execution_context=execution_context(second))
+ agent.run_sync("second")
+ assert asyncio.run(collector.finish_invocation(second))
+ finals = [batch for batch in inbox.batches if batch["final"]]
+ assert [batch["invocation_id"] for batch in finals] == [str(second.invocation_id)]
+ finally:
+ asyncio.run(collector.aclose())
+
+
+def test_sampling_off_sends_only_completion(inbox):
+ collector = configured_collector(inbox)
+ tracing = tracing_context(sampled=False)
+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])
+ collector.instrument(agent, execution_context=execution_context(tracing))
+ try:
+ agent.run_sync("hello")
+ assert asyncio.run(collector.finish_invocation(tracing))
+ assert len(inbox.batches) == 1
+ assert inbox.batches[0]["final"] is True
+ assert inbox.batches[0]["spans"] == []
+ finally:
+ asyncio.run(collector.aclose())
+
+
+def test_shutdown_drains_ended_spans_without_inventing_completion(inbox):
+ collector = configured_collector(inbox)
+ tracing = tracing_context()
+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])
+ collector.instrument(agent, execution_context=execution_context(tracing))
+ agent.run_sync("hello")
+ asyncio.run(collector.aclose())
+ assert any(batch["spans"] for batch in inbox.batches)
+ assert not any(batch["final"] for batch in inbox.batches)
+
+
+def test_concurrent_invocations_keep_parent_and_content_policy_separate(inbox):
+ collector = configured_collector(inbox)
+ first = tracing_context(capture_content=False)
+ second = tracing_context(capture_content=True, run_id=first.run_id)
+
+ async def invoke(tracing, prompt):
+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])
+ collector.instrument(agent, execution_context=execution_context(tracing))
+ await agent.run(prompt)
+ assert await collector.finish_invocation(tracing)
+
+ try:
+
+ async def both():
+ await asyncio.gather(invoke(first, "private-first"), invoke(second, "visible-second"))
+
+ asyncio.run(both())
+ for tracing in (first, second):
+ batches = [batch for batch in inbox.batches if batch["invocation_id"] == str(tracing.invocation_id)]
+ assert batches[-1]["final"] is True
+ spans = [span for batch in batches for span in batch["spans"]]
+ roots = [span for span in spans if span["parent_span_id"] == tracing.parent_span_id]
+ assert len(roots) == 1
+ assert {span["trace_id"] for span in spans} == {tracing.trace_id}
+ assert "private-first" not in json.dumps(spans)
+ assert ("visible-second" in json.dumps(spans)) is tracing.capture_content
+ finally:
+ asyncio.run(collector.aclose())
+
+
+def test_queue_overflow_does_not_acknowledge_complete_run(inbox):
+ tracing = tracing_context()
+ exporter = JSONSpanExporter(inner_api_url=inbox.url, inner_api_key="test-inner-key")
+ processor = ApplicationBatchSpanProcessor(
+ exporter, max_queue_size=1, max_export_batch_size=10, schedule_delay_seconds=3600
+ )
+ provider = TracerProvider()
+ provider.add_span_processor(processor)
+ tracer = provider.get_tracer("test")
+ attributes = {
+ DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),
+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),
+ "choices": ("a", "b"),
+ }
+ try:
+ for name in ("one", "two"):
+ with tracer.start_as_current_span(name, attributes=attributes):
+ pass
+ assert processor.force_flush()
+ assert not exporter.finish_invocation(tracing.run_id, tracing.invocation_id)
+ assert not exporter.finish_invocation(tracing.run_id, tracing.invocation_id)
+ assert not any(batch["final"] for batch in inbox.batches)
+ assert inbox.batches[0]["spans"][0]["attributes"]["choices"] == ["a", "b"]
+ finally:
+ provider.shutdown()
+
+
[email protected]("delivery_status", [None, 503])
+def test_runner_prefers_application_pipeline_and_finishes_before_success(inbox, monkeypatch, delivery_status):
+ collector = configured_collector(inbox)
+ inbox.status = delivery_status
+ tracing = tracing_context()
+ deployment_exporter = InMemorySpanExporter()
+ deployment_client = logfire.configure(
+ local=True,
+ send_to_logfire=False,
+ console=False,
+ metrics=False,
+ additional_span_processors=[SimpleSpanProcessor(deployment_exporter)],
+ )
+ monkeypatch.setattr(llm_layer.Capability, "build_model", lambda self, deps: TestModel())
+ incoming = CreateRunRequest(
+ composition=RunComposition(
+ layers=[
+ RunLayerSpec(name="execution_context", config=execution_context(tracing).model_dump(mode="json")),
+ RunLayerSpec(name="llm", config={"plugin_id": "test", "model_provider": "test", "model": "test"}),
+ RunLayerSpec(name="prompt", config={"user": "hello"}),
+ ]
+ )
+ )
+
+ async def run():
+ async with httpx.AsyncClient(trust_env=False) as client:
+ sink = InMemoryRunEventSink()
+ runner = AgentRunRunner(
+ sink=sink,
+ request=incoming,
+ run_id="backend-run",
+ services=Services(client, client),
+ agent_observability=AgentObservability(client=deployment_client),
+ application_observability=collector,
+ )
+ await runner.run()
+ assert isinstance(sink.events["backend-run"][-1], RunSucceededEvent)
+ assert inbox.batches[-1]["final"] is (delivery_status is None)
+
+ try:
+ asyncio.run(run())
+ deployment_client.force_flush()
+ assert not deployment_exporter.get_finished_spans()
+ finally:
+ asyncio.run(collector.aclose())
+ deployment_client.shutdown()
+
+
[email protected]("oversized", [False, True])
+def test_batches_respect_utf8_byte_limit_and_oversized_spans_prevent_final(inbox, oversized):
+ tracing = tracing_context()
+ memory = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(memory))
+ tracer = provider.get_tracer("test")
+ attributes = {
+ DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),
+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),
+ "unicode-content": "汉" * (5000 if oversized else 600),
+ }
+ for index in range(3):
+ with tracer.start_as_current_span(str(index), attributes=attributes):
+ pass
+ exporter = JSONSpanExporter(inner_api_url=inbox.url, inner_api_key="test-inner-key", max_batch_bytes=4096)
+ try:
+ result = exporter.export(memory.get_finished_spans())
+ assert result == (SpanExportResult.FAILURE if oversized else SpanExportResult.SUCCESS)
+ assert exporter.finish_invocation(tracing.run_id, tracing.invocation_id) is not oversized
+ if oversized:
+ assert not inbox.batches
+ else:
+ assert len([batch for batch in inbox.batches if batch["spans"]]) == 3
+ assert all(
+ len(json.dumps(batch, ensure_ascii=False, separators=(",", ":")).encode()) <= 4096
+ for batch in inbox.batches
+ )
+ finally:
+ provider.shutdown()
+ exporter.shutdown()
+
+
[email protected]("cancel", [False, True])
+def test_runner_flushes_on_model_error_and_cancellation(inbox, monkeypatch, cancel):
+ collector = configured_collector(inbox)
+ tracing = tracing_context()
+
+ async def run():
+ entered = asyncio.Event()
+
+ async def stream(_messages, _info):
+ entered.set()
+ if cancel:
+ await asyncio.Event().wait()
+ raise RuntimeError("test-only model failure")
+ yield "unreachable"
+
+ monkeypatch.setattr(
+ llm_layer.Capability, "build_model", lambda self, deps: FunctionModel(stream_function=stream)
+ )
+ incoming = CreateRunRequest(
+ composition=RunComposition(
+ layers=[
+ RunLayerSpec(name="execution_context", config=execution_context(tracing).model_dump(mode="json")),
+ RunLayerSpec(name="llm", config={"plugin_id": "test", "model_provider": "test", "model": "test"}),
+ RunLayerSpec(name="prompt", config={"user": "hello"}),
+ ]
+ )
+ )
+ async with httpx.AsyncClient(trust_env=False) as client:
+ runner = AgentRunRunner(
+ sink=InMemoryRunEventSink(),
+ request=incoming,
+ run_id="backend-run",
+ services=Services(client, client),
+ application_observability=collector,
+ )
+ task = asyncio.create_task(runner.run())
+ await entered.wait()
+ if cancel:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError if cancel else RuntimeError):
+ await task
+ assert inbox.batches[-1]["final"] is True
+ assert any(batch["spans"] for batch in inbox.batches)
+ roots = [
+ span
+ for batch in inbox.batches
+ for span in batch["spans"]
+ if span["attributes"].get("gen_ai.operation.name") == "invoke_agent"
+ ]
+ assert len(roots) == 1
+ assert "dify.agent.execution_state" not in roots[0]["attributes"]
+
+ try:
+ asyncio.run(run())
+ finally:
+ asyncio.run(collector.aclose())
+
+
[email protected]("early_cancel", [False, True])
[email protected]("sampled", [False, True])
+def test_setup_failure_or_early_cancel_has_one_timed_fallback_only_when_sampled(
+ inbox, monkeypatch, early_cancel, sampled
+):
+ collector = configured_collector(inbox)
+ tracing = tracing_context(sampled=sampled, capture_content=True)
+
+ def fail_model_setup(self, deps):
+ raise ValueError("test-only invalid model configuration")
+
+ monkeypatch.setattr(llm_layer.Capability, "build_model", fail_model_setup)
+ incoming = CreateRunRequest(
+ composition=RunComposition(
+ layers=[
+ RunLayerSpec(name="execution_context", config=execution_context(tracing).model_dump(mode="json")),
+ RunLayerSpec(name="llm", config={"plugin_id": "test", "model_provider": "test", "model": "test"}),
+ RunLayerSpec(name="prompt", config={"user": "hello"}),
+ ]
+ )
+ )
+ before = time.time_ns()
+
+ async def run():
+ async with httpx.AsyncClient(trust_env=False) as client:
+ runner = AgentRunRunner(
+ sink=InMemoryRunEventSink(),
+ request=incoming,
+ run_id="backend-run",
+ services=Services(client, client),
+ application_observability=collector,
+ is_cancelled=lambda: early_cancel,
+ )
+ if early_cancel:
+ await runner.run()
+ else:
+ with pytest.raises(ValueError, match="invalid model configuration"):
+ await runner.run()
+
+ try:
+ asyncio.run(run())
+ after = time.time_ns()
+ assert inbox.batches[-1]["final"] is True
+ spans = [span for batch in inbox.batches for span in batch["spans"]]
+ if not sampled:
+ assert spans == []
+ return
+ assert len(spans) == 1
+ span = spans[0]
+ assert span["trace_id"] == tracing.trace_id
+ assert span["parent_span_id"] == tracing.parent_span_id
+ assert span["attributes"]["gen_ai.operation.name"] == "invoke_agent"
+ assert span["attributes"]["dify.agent.execution_state"] == (
+ "cancelled" if early_cancel else "initialization_failed"
+ )
+ assert span["status"] == "error"
+ assert before <= span["start_time_ns"] <= span["end_time_ns"] <= after
+ assert span["status_message"] == (
+ "Agent cancelled before execution" if early_cancel else "test-only invalid model configuration"
+ )
+ finally:
+ asyncio.run(collector.aclose())
+
+
[email protected]("field,value", [("version", 2), ("trace_id", "0" * 32), ("parent_span_id", "x" * 16)])
+def test_invalid_api_trace_context_is_rejected(field, value):
+ payload = tracing_context().model_dump(mode="json")
+ payload[field] = value
+ with pytest.raises(ValidationError):
+ ApplicationTracingContext.model_validate(payload)
+
+
+def test_application_collection_is_disabled_by_default():
+ assert ServerSettings().application_tracing_enabled is False
+ assert configure_application_observability(ServerSettings()) is None
diff --git a/dify-agent/tests/local/dify_agent/server/test_app.py b/dify-agent/tests/local/dify_agent/server/test_app.py
index 3841cd7c662..d97b24a348c 100644
--- a/dify-agent/tests/local/dify_agent/server/test_app.py
+++ b/dify-agent/tests/local/dify_agent/server/test_app.py
@@ -99,6 +99,7 @@ class FakeRunScheduler:
stream_text_delta_flush_interval_seconds: float,
stream_text_delta_max_chars: int,
agent_observability: object | None = None,
+ application_observability: object | None = None,
) -> None:
self.store = store
self.shutdown_grace_seconds = shutdown_grace_seconds
@@ -110,6 +111,7 @@ class FakeRunScheduler:
self.plugin_daemon_http_client = services.plugin_daemon_http_client
self.dify_api_http_client = services.dify_api_http_client
self.agent_observability = agent_observability
+ self.application_observability = application_observability
self.shutdown_called = False
self.created.append(self)
diff --git a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py
index 4cff38d7f13..65f6753457c 100644
--- a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py
+++ b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py
@@ -79,6 +79,15 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Pat
assert agent_stub_shell_env_module.build_shell_agent_stub_env is not None
assert shell_module.DifyShellLayerConfig is not None
assert execution_context_module.DifyExecutionContextLayerConfig is not None
+ tracing = execution_context_module.ApplicationTracingContext(
+ run_id="11111111-1111-4111-8111-111111111111",
+ invocation_id="22222222-2222-4222-8222-222222222222",
+ trace_id="1" * 32,
+ parent_span_id="2" * 16,
+ session_id="client-session",
+ capture_content=False,
+ )
+ assert execution_context_module.ApplicationTracingContext.model_validate_json(tracing.model_dump_json()) == tracing
assert plugin_module.DifyPluginLLMLayerConfig is not None
assert output_module.DifyOutputLayerConfig is not None
assert user_prompt_module.DifyUserPromptLayerConfig is not None
diff --git a/docker/envs/core-services/dify-agent.env.example b/docker/envs/core-services/dify-agent.env.example
index 42094c3a1c5..1c202a4f266 100644
--- a/docker/envs/core-services/dify-agent.env.example
+++ b/docker/envs/core-services/dify-agent.env.example
@@ -30,6 +30,8 @@ DIFY_AGENT_OUTBOUND_HTTP_MAX_KEEPALIVE_CONNECTIONS=20
DIFY_AGENT_OUTBOUND_HTTP_KEEPALIVE_EXPIRY=30
DIFY_AGENT_TRAJECTORY_ENABLED=false
+# Collect API-issued application traces into the Dify inner API inbox.
+DIFY_AGENT_APPLICATION_TRACING_ENABLED=false
DIFY_AGENT_TRAJECTORY_OTLP_TRACES_ENDPOINT=
DIFY_AGENT_TRAJECTORY_OTLP_HEADERS={}
DIFY_AGENT_TRAJECTORY_SERVICE_NAME=dify-agent-trajectory
diff --git a/docker/envs/core-services/shared.env.example b/docker/envs/core-services/shared.env.example
index cedce027c4d..2b3250e0cc3 100644
--- a/docker/envs/core-services/shared.env.example
+++ b/docker/envs/core-services/shared.env.example
@@ -104,6 +104,9 @@ DEBUG=false
FLASK_DEBUG=false
ENABLE_REQUEST_LOGGING=False
OPS_TRACE_UNIFIED_ENABLED=false
+# Application-level Agent tracing; also enable the collector in dify-agent.env.
+OPS_AGENT_TRACE_ENABLED=false
+ENABLE_APPLICATION_TRACE_RECONCILE_TASK=true
OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES=780
OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS=5
# OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS should cover the retry window
diff --git a/packages/contracts/generated/api/console/agent/orpc.gen.ts b/packages/contracts/generated/api/console/agent/orpc.gen.ts
index 2c3d85a2ce4..5ae49bd98bd 100644
--- a/packages/contracts/generated/api/console/agent/orpc.gen.ts
+++ b/packages/contracts/generated/api/console/agent/orpc.gen.ts
@@ -13,6 +13,7 @@ import {
zAgentNotPublishedError,
zAgentSoulLockedError,
zAgentVersionNotFoundError,
+ zAppNotFoundError,
zAppSuggestedQuestionsAfterAnswerDisabledError,
zAppUnavailableError,
zAudioTooLargeError,
@@ -20,6 +21,7 @@ import {
zCompletionRequestError,
zInspectorUnavailableError,
zInvalidComposerConfigError,
+ zInvalidTracingConfigError,
zMaxKeysExceededError,
zModelCurrentlyNotSupportError,
zNoActiveBindingError,
@@ -31,7 +33,20 @@ import {
zProviderQuotaExceededError,
zRateLimitError,
zSpeechToTextDisabledError,
+ zTraceConfigAlreadyExistsError,
+ zTraceConfigNotFoundError,
+ zTracingAlreadyAcceptedError,
+ zTracingConfigConflictError,
+ zTracingConfigProcessingFailedError,
+ zTracingConfigVerificationFailedError,
+ zTracingDuplicateAcknowledgementRequiredError,
+ zTracingNodeNotFoundError,
+ zTracingProviderUnavailableError,
+ zTracingRetryNotAllowedError,
+ zTracingRunNotFoundError,
+ zTracingSourceNotFoundError,
zUnsupportedAudioTypeError,
+ zUnsupportedTracingProviderError,
} from './zod.gen.ts'
import {
zDeleteAgentByAgentIdApiKeysByApiKeyIdPath,
@@ -46,6 +61,9 @@ import {
zDeleteAgentByAgentIdConfigSkillsByNameResponse,
zDeleteAgentByAgentIdPath,
zDeleteAgentByAgentIdResponse,
+ zDeleteAgentByAgentIdTraceConfigPath,
+ zDeleteAgentByAgentIdTraceConfigQuery,
+ zDeleteAgentByAgentIdTraceConfigResponse,
zGetAgentByAgentIdApiAccessPath,
zGetAgentByAgentIdApiAccessResponse,
zGetAgentByAgentIdApiKeysPath,
@@ -119,6 +137,17 @@ import {
zGetAgentByAgentIdTextToAudioVoicesPath,
zGetAgentByAgentIdTextToAudioVoicesQuery,
zGetAgentByAgentIdTextToAudioVoicesResponse,
+ zGetAgentByAgentIdTraceCapabilitiesPath,
+ zGetAgentByAgentIdTraceCapabilitiesQuery,
+ zGetAgentByAgentIdTraceCapabilitiesResponse,
+ zGetAgentByAgentIdTraceConfigPath,
+ zGetAgentByAgentIdTraceConfigQuery,
+ zGetAgentByAgentIdTraceConfigResponse,
+ zGetAgentByAgentIdTracePath,
+ zGetAgentByAgentIdTraceResponse,
+ zGetAgentByAgentIdTraceRunsByMessageIdPath,
+ zGetAgentByAgentIdTraceRunsByMessageIdQuery,
+ zGetAgentByAgentIdTraceRunsByMessageIdResponse,
zGetAgentByAgentIdVersionsByVersionIdPath,
zGetAgentByAgentIdVersionsByVersionIdResponse,
zGetAgentByAgentIdVersionsPath,
@@ -127,6 +156,9 @@ import {
zGetAgentInviteOptionsResponse,
zGetAgentQuery,
zGetAgentResponse,
+ zPatchAgentByAgentIdTraceConfigBody,
+ zPatchAgentByAgentIdTraceConfigPath,
+ zPatchAgentByAgentIdTraceConfigResponse,
zPostAgentBody,
zPostAgentByAgentIdApiEnableBody,
zPostAgentByAgentIdApiEnablePath,
@@ -176,6 +208,16 @@ import {
zPostAgentByAgentIdTextToAudioBody,
zPostAgentByAgentIdTextToAudioPath,
zPostAgentByAgentIdTextToAudioResponse,
+ zPostAgentByAgentIdTraceBody,
+ zPostAgentByAgentIdTraceConfigBody,
+ zPostAgentByAgentIdTraceConfigPath,
+ zPostAgentByAgentIdTraceConfigResponse,
+ zPostAgentByAgentIdTracePath,
+ zPostAgentByAgentIdTraceResponse,
+ zPostAgentByAgentIdTraceRunsByMessageIdRetryBody,
+ zPostAgentByAgentIdTraceRunsByMessageIdRetryPath,
+ zPostAgentByAgentIdTraceRunsByMessageIdRetryQuery,
+ zPostAgentByAgentIdTraceRunsByMessageIdRetryResponse,
zPostAgentByAgentIdVersionsByVersionIdRestorePath,
zPostAgentByAgentIdVersionsByVersionIdRestoreResponse,
zPostAgentResponse,
@@ -1370,7 +1412,234 @@ export const textToAudio = {
voices,
}
+export const get29 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAgentByAgentIdTrace',
+ path: '/agent/{agent_id}/trace',
+ tags: ['console'],
+ })
+ .input(z.object({ params: zGetAgentByAgentIdTracePath }))
+ .output(zGetAgentByAgentIdTraceResponse)
+
export const post18 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'POST',
+ operationId: 'postAgentByAgentIdTrace',
+ path: '/agent/{agent_id}/trace',
+ tags: ['console'],
+ })
+ .input(z.object({ body: zPostAgentByAgentIdTraceBody, params: zPostAgentByAgentIdTracePath }))
+ .output(zPostAgentByAgentIdTraceResponse)
+ .errors({
+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },
+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },
+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },
+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },
+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },
+ })
+
+export const trace = {
+ get: get29,
+ post: post18,
+}
+
+export const get30 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAgentByAgentIdTraceCapabilities',
+ path: '/agent/{agent_id}/trace-capabilities',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zGetAgentByAgentIdTraceCapabilitiesPath,
+ query: zGetAgentByAgentIdTraceCapabilitiesQuery,
+ }),
+ )
+ .output(zGetAgentByAgentIdTraceCapabilitiesResponse)
+ .errors({ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError } })
+
+export const traceCapabilities = {
+ get: get30,
+}
+
+export const delete5 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'DELETE',
+ operationId: 'deleteAgentByAgentIdTraceConfig',
+ path: '/agent/{agent_id}/trace-config',
+ successStatus: 204,
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zDeleteAgentByAgentIdTraceConfigPath,
+ query: zDeleteAgentByAgentIdTraceConfigQuery,
+ }),
+ )
+ .output(zDeleteAgentByAgentIdTraceConfigResponse)
+ .errors({
+ app_not_found: { status: 404, data: zAppNotFoundError },
+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },
+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },
+ })
+
+export const get31 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAgentByAgentIdTraceConfig',
+ path: '/agent/{agent_id}/trace-config',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zGetAgentByAgentIdTraceConfigPath,
+ query: zGetAgentByAgentIdTraceConfigQuery,
+ }),
+ )
+ .output(zGetAgentByAgentIdTraceConfigResponse)
+ .errors({
+ app_not_found: { status: 404, data: zAppNotFoundError },
+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },
+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },
+ })
+
+export const patch = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'PATCH',
+ operationId: 'patchAgentByAgentIdTraceConfig',
+ path: '/agent/{agent_id}/trace-config',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ body: zPatchAgentByAgentIdTraceConfigBody,
+ params: zPatchAgentByAgentIdTraceConfigPath,
+ }),
+ )
+ .output(zPatchAgentByAgentIdTraceConfigResponse)
+ .errors({
+ app_not_found: { status: 404, data: zAppNotFoundError },
+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },
+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },
+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },
+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
+ tracing_config_verification_failed: {
+ status: 400,
+ data: zTracingConfigVerificationFailedError,
+ },
+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },
+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },
+ })
+
+export const post19 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'POST',
+ operationId: 'postAgentByAgentIdTraceConfig',
+ path: '/agent/{agent_id}/trace-config',
+ successStatus: 201,
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ body: zPostAgentByAgentIdTraceConfigBody,
+ params: zPostAgentByAgentIdTraceConfigPath,
+ }),
+ )
+ .output(zPostAgentByAgentIdTraceConfigResponse)
+ .errors({
+ app_not_found: { status: 404, data: zAppNotFoundError },
+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },
+ trace_config_already_exists: { status: 409, data: zTraceConfigAlreadyExistsError },
+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
+ tracing_config_verification_failed: {
+ status: 400,
+ data: zTracingConfigVerificationFailedError,
+ },
+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },
+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },
+ })
+
+export const traceConfig = {
+ delete: delete5,
+ get: get31,
+ patch,
+ post: post19,
+}
+
+export const post20 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'POST',
+ operationId: 'postAgentByAgentIdTraceRunsByMessageIdRetry',
+ path: '/agent/{agent_id}/trace-runs/{message_id}/retry',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ body: zPostAgentByAgentIdTraceRunsByMessageIdRetryBody,
+ params: zPostAgentByAgentIdTraceRunsByMessageIdRetryPath,
+ query: zPostAgentByAgentIdTraceRunsByMessageIdRetryQuery.optional(),
+ }),
+ )
+ .output(zPostAgentByAgentIdTraceRunsByMessageIdRetryResponse)
+ .errors({
+ tracing_already_accepted: { status: 409, data: zTracingAlreadyAcceptedError },
+ tracing_duplicate_acknowledgement_required: {
+ status: 409,
+ data: zTracingDuplicateAcknowledgementRequiredError,
+ },
+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },
+ tracing_retry_not_allowed: { status: 409, data: zTracingRetryNotAllowedError },
+ tracing_run_not_found: { status: 404, data: zTracingRunNotFoundError },
+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },
+ })
+
+export const retry = {
+ post: post20,
+}
+
+export const get32 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAgentByAgentIdTraceRunsByMessageId',
+ path: '/agent/{agent_id}/trace-runs/{message_id}',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zGetAgentByAgentIdTraceRunsByMessageIdPath,
+ query: zGetAgentByAgentIdTraceRunsByMessageIdQuery.optional(),
+ }),
+ )
+ .output(zGetAgentByAgentIdTraceRunsByMessageIdResponse)
+ .errors({
+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },
+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },
+ })
+
+export const byMessageId3 = {
+ get: get32,
+ retry,
+}
+
+export const traceRuns = {
+ byMessageId: byMessageId3,
+}
+
+export const post21 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -1386,10 +1655,10 @@ export const post18 = oc
})
export const restore = {
- post: post18,
+ post: post21,
}
-export const get29 = oc
+export const get33 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1405,11 +1674,11 @@ export const get29 = oc
})
export const byVersionId = {
- get: get29,
+ get: get33,
restore,
}
-export const get30 = oc
+export const get34 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1422,11 +1691,11 @@ export const get30 = oc
.errors({ agent_not_found_error: { status: 404, data: zAgentNotFoundError } })
export const versions = {
- get: get30,
+ get: get34,
byVersionId,
}
-export const delete5 = oc
+export const delete6 = oc
.route({
inputStructure: 'detailed',
method: 'DELETE',
@@ -1439,7 +1708,7 @@ export const delete5 = oc
.output(zDeleteAgentByAgentIdResponse)
.errors({ agent_not_found_error: { status: 404, data: zAgentNotFoundError } })
-export const get31 = oc
+export const get35 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1467,8 +1736,8 @@ export const put3 = oc
})
export const byAgentId = {
- delete: delete5,
- get: get31,
+ delete: delete6,
+ get: get35,
put: put3,
apiAccess,
apiEnable,
@@ -1491,10 +1760,14 @@ export const byAgentId = {
sandbox,
statistics,
textToAudio,
+ trace,
+ traceCapabilities,
+ traceConfig,
+ traceRuns,
versions,
}
-export const get32 = oc
+export const get36 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1506,7 +1779,7 @@ export const get32 = oc
.output(zGetAgentResponse)
.errors({ agent_not_found_error: { status: 404, data: zAgentNotFoundError } })
-export const post19 = oc
+export const post22 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -1523,8 +1796,8 @@ export const post19 = oc
})
export const agent = {
- get: get32,
- post: post19,
+ get: get36,
+ post: post22,
inviteOptions,
byAgentId,
}
diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.ts
index d01e6f92cb8..23eb12f6779 100644
--- a/packages/contracts/generated/api/console/agent/types.gen.ts
+++ b/packages/contracts/generated/api/console/agent/types.gen.ts
@@ -572,6 +572,162 @@ export type ProviderNotSupportTextToSpeechError = ErrorResponse & {
export type TextToSpeechVoiceListResponse = Array<TextToSpeechVoiceResponse>
+export type AppTraceResponse = {
+ capture_agent?: boolean
+ capture_content?: boolean
+ enabled?: boolean
+ tracing_provider?: string | null
+}
+
+export type AppTracePayload = {
+ capture_agent?: boolean
+ capture_content?: boolean
+ enabled: boolean
+ tracing_provider?: string | null
+}
+
+export type InvalidTracingConfigError = ErrorResponse & {
+ code: 'invalid_tracing_config'
+ status: 400
+}
+
+export type TracingProviderUnavailableError = ErrorResponse & {
+ code: 'tracing_provider_unavailable'
+ status: 400
+}
+
+export type UnsupportedTracingProviderError = ErrorResponse & {
+ code: 'unsupported_tracing_provider'
+ status: 400
+}
+
+export type TraceConfigNotFoundError = ErrorResponse & {
+ code: 'trace_config_not_found'
+ status: 404
+}
+
+export type TracingConfigConflictError = ErrorResponse & {
+ code: 'tracing_config_conflict'
+ status: 409
+}
+
+export type TracingConfigProcessingFailedError = ErrorResponse & {
+ code: 'tracing_config_processing_failed'
+ status: 500
+}
+
+export type TraceProviderCapabilitiesResponse = {
+ agent_spans: boolean
+ api_collection_enabled: boolean
+ export_ack: 'accepted'
+ native_cost: boolean
+ native_usage: boolean
+ provider: string
+ session: boolean
+ span_url: boolean
+ trace_url: boolean
+}
+
+export type AppNotFoundError = ErrorResponse & {
+ code: 'app_not_found'
+ status: 404
+}
+
+export type TraceAppConfigResponse = {
+ app_id?: string | null
+ created_at?: string | null
+ error?: string | null
+ has_not_configured?: boolean | null
+ id?: string | null
+ is_active?: boolean | null
+ result?: string | null
+ tracing_config?: {
+ [key: string]: string | null
+ } | null
+ tracing_provider?: string | null
+ updated_at?: string | null
+}
+
+export type TraceConfigPayload = {
+ activate?: boolean
+ tracing_config: {
+ [key: string]: string | null
+ }
+ tracing_provider: string
+}
+
+export type TracingConfigVerificationFailedError = ErrorResponse & {
+ code: 'tracing_config_verification_failed'
+ status: 400
+}
+
+export type TraceConfigAlreadyExistsError = ErrorResponse & {
+ code: 'trace_config_already_exists'
+ status: 409
+}
+
+export type TraceRunStatusResponse = {
+ accepted_at?: string | null
+ attempts?: number | null
+ capabilities?: TraceProviderSupportResponse | null
+ capture_agent?: boolean | null
+ capture_content?: boolean | null
+ config_revision?: string | null
+ error_code?: string | null
+ project_url?: string | null
+ provider?: string | null
+ queryable?: boolean | null
+ source_id: string
+ source_kind: TraceSourceKind
+ span_count?: number | null
+ span_url?: string | null
+ status:
+ | 'accepted'
+ | 'collecting'
+ | 'exporting'
+ | 'failed'
+ | 'not_collected'
+ | 'ready'
+ | 'retrying'
+ | 'uncertain'
+ trace_id?: string | null
+ trace_url?: string | null
+}
+
+export type TracingNodeNotFoundError = ErrorResponse & {
+ code: 'tracing_node_not_found'
+ status: 404
+}
+
+export type TracingSourceNotFoundError = ErrorResponse & {
+ code: 'tracing_source_not_found'
+ status: 404
+}
+
+export type TraceRunRetryPayload = {
+ acknowledge_possible_duplicates?: boolean
+}
+
+export type TracingRunNotFoundError = ErrorResponse & {
+ code: 'tracing_run_not_found'
+ status: 404
+}
+
+export type TracingAlreadyAcceptedError = ErrorResponse & {
+ code: 'tracing_already_accepted'
+ status: 409
+}
+
+export type TracingDuplicateAcknowledgementRequiredError = ErrorResponse & {
+ code: 'tracing_duplicate_acknowledgement_required'
+ status: 409
+}
+
+export type TracingRetryNotAllowedError = ErrorResponse & {
+ code: 'tracing_retry_not_allowed'
+ status: 409
+}
+
export type AgentConfigSnapshotListResponse = {
data: Array<AgentConfigSnapshotSummaryResponse>
}
@@ -1144,6 +1300,19 @@ export type TextToSpeechVoiceResponse = {
value: string
}
+export type TraceProviderSupportResponse = {
+ agent_spans: boolean
+ export_ack: 'accepted'
+ native_cost: boolean
+ native_usage: boolean
+ provider: string
+ session: boolean
+ span_url: boolean
+ trace_url: boolean
+}
+
+export type TraceSourceKind = 'message' | 'node' | 'workflow'
+
export type AgentConfigRevisionResponse = {
created_at?: number | null
created_by?: string | null
@@ -1921,7 +2090,7 @@ export type AgentSoulModelSettings = {
frequency_penalty?: number | null
max_tokens?: number | null
presence_penalty?: number | null
- response_format?: AgentModelResponseFormatConfig | null
+ response_format?: string | AgentModelResponseFormatConfig | null
stop?: Array<string> | null
temperature?: number | null
top_p?: number | null
@@ -3961,6 +4130,269 @@ export type GetAgentByAgentIdTextToAudioVoicesResponses = {
export type GetAgentByAgentIdTextToAudioVoicesResponse =
GetAgentByAgentIdTextToAudioVoicesResponses[keyof GetAgentByAgentIdTextToAudioVoicesResponses]
+export type GetAgentByAgentIdTraceData = {
+ body?: never
+ path: {
+ agent_id: string
+ }
+ query?: never
+ url: '/agent/{agent_id}/trace'
+}
+
+export type GetAgentByAgentIdTraceErrors = {
+ default: ErrorResponse
+}
+
+export type GetAgentByAgentIdTraceError =
+ GetAgentByAgentIdTraceErrors[keyof GetAgentByAgentIdTraceErrors]
+
+export type GetAgentByAgentIdTraceResponses = {
+ 200: AppTraceResponse
+}
+
+export type GetAgentByAgentIdTraceResponse =
+ GetAgentByAgentIdTraceResponses[keyof GetAgentByAgentIdTraceResponses]
+
+export type PostAgentByAgentIdTraceData = {
+ body: AppTracePayload
+ path: {
+ agent_id: string
+ }
+ query?: never
+ url: '/agent/{agent_id}/trace'
+}
+
+export type PostAgentByAgentIdTraceErrors = {
+ 400:
+ | InvalidTracingConfigError
+ | TracingProviderUnavailableError
+ | UnsupportedTracingProviderError
+ | ErrorResponse
+ 404: TraceConfigNotFoundError | ErrorResponse
+ 409: TracingConfigConflictError | ErrorResponse
+ 500: TracingConfigProcessingFailedError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type PostAgentByAgentIdTraceError =
+ PostAgentByAgentIdTraceErrors[keyof PostAgentByAgentIdTraceErrors]
+
+export type PostAgentByAgentIdTraceResponses = {
+ 200: AppTraceResponse
+}
+
+export type PostAgentByAgentIdTraceResponse =
+ PostAgentByAgentIdTraceResponses[keyof PostAgentByAgentIdTraceResponses]
+
+export type GetAgentByAgentIdTraceCapabilitiesData = {
+ body?: never
+ path: {
+ agent_id: string
+ }
+ query: {
+ tracing_provider: string
+ }
+ url: '/agent/{agent_id}/trace-capabilities'
+}
+
+export type GetAgentByAgentIdTraceCapabilitiesErrors = {
+ 400: UnsupportedTracingProviderError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type GetAgentByAgentIdTraceCapabilitiesError =
+ GetAgentByAgentIdTraceCapabilitiesErrors[keyof GetAgentByAgentIdTraceCapabilitiesErrors]
+
+export type GetAgentByAgentIdTraceCapabilitiesResponses = {
+ 200: TraceProviderCapabilitiesResponse
+}
+
+export type GetAgentByAgentIdTraceCapabilitiesResponse =
+ GetAgentByAgentIdTraceCapabilitiesResponses[keyof GetAgentByAgentIdTraceCapabilitiesResponses]
+
+export type DeleteAgentByAgentIdTraceConfigData = {
+ body?: never
+ path: {
+ agent_id: string
+ }
+ query: {
+ tracing_provider: string
+ }
+ url: '/agent/{agent_id}/trace-config'
+}
+
+export type DeleteAgentByAgentIdTraceConfigErrors = {
+ 400: UnsupportedTracingProviderError | ErrorResponse
+ 404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse
+ 500: TracingConfigProcessingFailedError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type DeleteAgentByAgentIdTraceConfigError =
+ DeleteAgentByAgentIdTraceConfigErrors[keyof DeleteAgentByAgentIdTraceConfigErrors]
+
+export type DeleteAgentByAgentIdTraceConfigResponses = {
+ 204: void
+}
+
+export type DeleteAgentByAgentIdTraceConfigResponse =
+ DeleteAgentByAgentIdTraceConfigResponses[keyof DeleteAgentByAgentIdTraceConfigResponses]
+
+export type GetAgentByAgentIdTraceConfigData = {
+ body?: never
+ path: {
+ agent_id: string
+ }
+ query: {
+ tracing_provider: string
+ }
+ url: '/agent/{agent_id}/trace-config'
+}
+
+export type GetAgentByAgentIdTraceConfigErrors = {
+ 400: TracingProviderUnavailableError | UnsupportedTracingProviderError | ErrorResponse
+ 404: AppNotFoundError | ErrorResponse
+ 500: TracingConfigProcessingFailedError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type GetAgentByAgentIdTraceConfigError =
+ GetAgentByAgentIdTraceConfigErrors[keyof GetAgentByAgentIdTraceConfigErrors]
+
+export type GetAgentByAgentIdTraceConfigResponses = {
+ 200: TraceAppConfigResponse
+}
+
+export type GetAgentByAgentIdTraceConfigResponse =
+ GetAgentByAgentIdTraceConfigResponses[keyof GetAgentByAgentIdTraceConfigResponses]
+
+export type PatchAgentByAgentIdTraceConfigData = {
+ body: TraceConfigPayload
+ path: {
+ agent_id: string
+ }
+ query?: never
+ url: '/agent/{agent_id}/trace-config'
+}
+
+export type PatchAgentByAgentIdTraceConfigErrors = {
+ 400:
+ | InvalidTracingConfigError
+ | TracingConfigVerificationFailedError
+ | TracingProviderUnavailableError
+ | UnsupportedTracingProviderError
+ | ErrorResponse
+ 404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse
+ 409: TracingConfigConflictError | ErrorResponse
+ 500: TracingConfigProcessingFailedError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type PatchAgentByAgentIdTraceConfigError =
+ PatchAgentByAgentIdTraceConfigErrors[keyof PatchAgentByAgentIdTraceConfigErrors]
+
+export type PatchAgentByAgentIdTraceConfigResponses = {
+ 200: TraceAppConfigResponse
+}
+
+export type PatchAgentByAgentIdTraceConfigResponse =
+ PatchAgentByAgentIdTraceConfigResponses[keyof PatchAgentByAgentIdTraceConfigResponses]
+
+export type PostAgentByAgentIdTraceConfigData = {
+ body: TraceConfigPayload
+ path: {
+ agent_id: string
+ }
+ query?: never
+ url: '/agent/{agent_id}/trace-config'
+}
+
+export type PostAgentByAgentIdTraceConfigErrors = {
+ 400:
+ | InvalidTracingConfigError
+ | TracingConfigVerificationFailedError
+ | TracingProviderUnavailableError
+ | UnsupportedTracingProviderError
+ | ErrorResponse
+ 404: AppNotFoundError | ErrorResponse
+ 409: TraceConfigAlreadyExistsError | ErrorResponse
+ 500: TracingConfigProcessingFailedError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type PostAgentByAgentIdTraceConfigError =
+ PostAgentByAgentIdTraceConfigErrors[keyof PostAgentByAgentIdTraceConfigErrors]
+
+export type PostAgentByAgentIdTraceConfigResponses = {
+ 201: TraceAppConfigResponse
+}
+
+export type PostAgentByAgentIdTraceConfigResponse =
+ PostAgentByAgentIdTraceConfigResponses[keyof PostAgentByAgentIdTraceConfigResponses]
+
+export type GetAgentByAgentIdTraceRunsByMessageIdData = {
+ body?: never
+ path: {
+ agent_id: string
+ message_id: string
+ }
+ query?: {
+ node_execution_id?: string
+ }
+ url: '/agent/{agent_id}/trace-runs/{message_id}'
+}
+
+export type GetAgentByAgentIdTraceRunsByMessageIdErrors = {
+ 404: TracingNodeNotFoundError | TracingSourceNotFoundError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type GetAgentByAgentIdTraceRunsByMessageIdError =
+ GetAgentByAgentIdTraceRunsByMessageIdErrors[keyof GetAgentByAgentIdTraceRunsByMessageIdErrors]
+
+export type GetAgentByAgentIdTraceRunsByMessageIdResponses = {
+ 200: TraceRunStatusResponse
+}
+
+export type GetAgentByAgentIdTraceRunsByMessageIdResponse =
+ GetAgentByAgentIdTraceRunsByMessageIdResponses[keyof GetAgentByAgentIdTraceRunsByMessageIdResponses]
+
+export type PostAgentByAgentIdTraceRunsByMessageIdRetryData = {
+ body: TraceRunRetryPayload
+ path: {
+ agent_id: string
+ message_id: string
+ }
+ query?: {
+ node_execution_id?: string
+ }
+ url: '/agent/{agent_id}/trace-runs/{message_id}/retry'
+}
+
+export type PostAgentByAgentIdTraceRunsByMessageIdRetryErrors = {
+ 404:
+ | TracingNodeNotFoundError
+ | TracingRunNotFoundError
+ | TracingSourceNotFoundError
+ | ErrorResponse
+ 409:
+ | TracingAlreadyAcceptedError
+ | TracingDuplicateAcknowledgementRequiredError
+ | TracingRetryNotAllowedError
+ | ErrorResponse
+ default: ErrorResponse
+}
+
+export type PostAgentByAgentIdTraceRunsByMessageIdRetryError =
+ PostAgentByAgentIdTraceRunsByMessageIdRetryErrors[keyof PostAgentByAgentIdTraceRunsByMessageIdRetryErrors]
+
+export type PostAgentByAgentIdTraceRunsByMessageIdRetryResponses = {
+ 200: TraceRunStatusResponse
+}
+
+export type PostAgentByAgentIdTraceRunsByMessageIdRetryResponse =
+ PostAgentByAgentIdTraceRunsByMessageIdRetryResponses[keyof PostAgentByAgentIdTraceRunsByMessageIdRetryResponses]
+
export type GetAgentByAgentIdVersionsData = {
body?: never
path: {
diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.ts
index cf27ddb2219..f6d8eb36780 100644
--- a/packages/contracts/generated/api/console/agent/zod.gen.ts
+++ b/packages/contracts/generated/api/console/agent/zod.gen.ts
@@ -569,6 +569,253 @@ export const zProviderNotSupportTextToSpeechError = zErrorResponse.and(
}),
)
+/**
+ * AppTraceResponse
+ */
+export const zAppTraceResponse = z.object({
+ capture_agent: z.boolean().optional().default(true),
+ capture_content: z.boolean().optional().default(false),
+ enabled: z.boolean().optional().default(false),
+ tracing_provider: z.string().nullish(),
+})
+
+/**
+ * AppTracePayload
+ */
+export const zAppTracePayload = z.object({
+ capture_agent: z.boolean().optional().default(true),
+ capture_content: z.boolean().optional().default(false),
+ enabled: z.boolean(),
+ tracing_provider: z.string().nullish(),
+})
+
+/**
+ * InvalidTracingConfigError
+ *
+ * The tracing configuration is invalid.
+ */
+export const zInvalidTracingConfigError = zErrorResponse.and(
+ z.object({
+ code: z.literal('invalid_tracing_config'),
+ status: z.literal(400),
+ }),
+)
+
+/**
+ * TracingProviderUnavailableError
+ *
+ * The tracing provider is not available in this deployment.
+ */
+export const zTracingProviderUnavailableError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_provider_unavailable'),
+ status: z.literal(400),
+ }),
+)
+
+/**
+ * UnsupportedTracingProviderError
+ *
+ * The tracing provider is not supported.
+ */
+export const zUnsupportedTracingProviderError = zErrorResponse.and(
+ z.object({
+ code: z.literal('unsupported_tracing_provider'),
+ status: z.literal(400),
+ }),
+)
+
+/**
+ * TraceConfigNotFoundError
+ *
+ * Tracing configuration not found.
+ */
+export const zTraceConfigNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('trace_config_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TracingConfigConflictError
+ *
+ * Tracing configuration changed. Refresh and try again.
+ */
+export const zTracingConfigConflictError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_config_conflict'),
+ status: z.literal(409),
+ }),
+)
+
+/**
+ * TracingConfigProcessingFailedError
+ *
+ * The tracing configuration could not be processed.
+ */
+export const zTracingConfigProcessingFailedError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_config_processing_failed'),
+ status: z.literal(500),
+ }),
+)
+
+/**
+ * TraceProviderCapabilitiesResponse
+ */
+export const zTraceProviderCapabilitiesResponse = z.object({
+ agent_spans: z.boolean(),
+ api_collection_enabled: z.boolean(),
+ export_ack: z.literal('accepted'),
+ native_cost: z.boolean(),
+ native_usage: z.boolean(),
+ provider: z.string(),
+ session: z.boolean(),
+ span_url: z.boolean(),
+ trace_url: z.boolean(),
+})
+
+/**
+ * AppNotFoundError
+ *
+ * App not found.
+ */
+export const zAppNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('app_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TraceAppConfigResponse
+ */
+export const zTraceAppConfigResponse = z.object({
+ app_id: z.string().nullish(),
+ created_at: z.string().nullish(),
+ error: z.string().nullish(),
+ has_not_configured: z.boolean().nullish(),
+ id: z.string().nullish(),
+ is_active: z.boolean().nullish(),
+ result: z.string().nullish(),
+ tracing_config: z.record(z.string(), z.string().nullable()).nullish(),
+ tracing_provider: z.string().nullish(),
+ updated_at: z.string().nullish(),
+})
+
+/**
+ * TraceConfigPayload
+ */
+export const zTraceConfigPayload = z.object({
+ activate: z.boolean().optional().default(false),
+ tracing_config: z.record(z.string(), z.string().nullable()),
+ tracing_provider: z.string(),
+})
+
+/**
+ * TracingConfigVerificationFailedError
+ *
+ * The tracing configuration could not be verified.
+ */
+export const zTracingConfigVerificationFailedError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_config_verification_failed'),
+ status: z.literal(400),
+ }),
+)
+
+/**
+ * TraceConfigAlreadyExistsError
+ *
+ * A tracing configuration already exists for this provider.
+ */
+export const zTraceConfigAlreadyExistsError = zErrorResponse.and(
+ z.object({
+ code: z.literal('trace_config_already_exists'),
+ status: z.literal(409),
+ }),
+)
+
+/**
+ * TracingNodeNotFoundError
+ *
+ * The Agent node invocation does not belong to this trace.
+ */
+export const zTracingNodeNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_node_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TracingSourceNotFoundError
+ *
+ * The tracing source does not exist in this application.
+ */
+export const zTracingSourceNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_source_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TraceRunRetryPayload
+ */
+export const zTraceRunRetryPayload = z.object({
+ acknowledge_possible_duplicates: z.boolean().optional().default(false),
+})
+
+/**
+ * TracingRunNotFoundError
+ *
+ * No collected trace is available to retry.
+ */
+export const zTracingRunNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_run_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TracingAlreadyAcceptedError
+ *
+ * An accepted trace cannot be exported again.
+ */
+export const zTracingAlreadyAcceptedError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_already_accepted'),
+ status: z.literal(409),
+ }),
+)
+
+/**
+ * TracingDuplicateAcknowledgementRequiredError
+ *
+ * Acknowledge possible duplicates before retrying an uncertain export.
+ */
+export const zTracingDuplicateAcknowledgementRequiredError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_duplicate_acknowledgement_required'),
+ status: z.literal(409),
+ }),
+)
+
+/**
+ * TracingRetryNotAllowedError
+ *
+ * This trace is not in a retryable state.
+ */
+export const zTracingRetryNotAllowedError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_retry_not_allowed'),
+ status: z.literal(409),
+ }),
+)
+
/**
* AgentConfigSnapshotRestoreResponse
*/
@@ -1106,6 +1353,56 @@ export const zTextToSpeechVoiceResponse = z.object({
*/
export const zTextToSpeechVoiceListResponse = z.array(zTextToSpeechVoiceResponse)
+/**
+ * TraceProviderSupportResponse
+ *
+ * Adapter support declarations, not successful-delivery or platform-verification results.
+ */
+export const zTraceProviderSupportResponse = z.object({
+ agent_spans: z.boolean(),
+ export_ack: z.literal('accepted'),
+ native_cost: z.boolean(),
+ native_usage: z.boolean(),
+ provider: z.string(),
+ session: z.boolean(),
+ span_url: z.boolean(),
+ trace_url: z.boolean(),
+})
+
+export const zTraceSourceKind = z.enum(['message', 'node', 'workflow'])
+
+/**
+ * TraceRunStatusResponse
+ */
+export const zTraceRunStatusResponse = z.object({
+ accepted_at: z.iso.datetime().nullish(),
+ attempts: z.int().nullish(),
+ capabilities: zTraceProviderSupportResponse.nullish(),
+ capture_agent: z.boolean().nullish(),
+ capture_content: z.boolean().nullish(),
+ config_revision: z.string().nullish(),
+ error_code: z.string().nullish(),
+ project_url: z.string().nullish(),
+ provider: z.string().nullish(),
+ queryable: z.boolean().nullish(),
+ source_id: z.string(),
+ source_kind: zTraceSourceKind,
+ span_count: z.int().nullish(),
+ span_url: z.string().nullish(),
+ status: z.enum([
+ 'accepted',
+ 'collecting',
+ 'exporting',
+ 'failed',
+ 'not_collected',
+ 'ready',
+ 'retrying',
+ 'uncertain',
+ ]),
+ trace_id: z.string().nullish(),
+ trace_url: z.string().nullish(),
+})
+
/**
* AgentAppPublishedReferenceResponse
*/
@@ -2774,7 +3071,7 @@ export const zAgentSoulModelSettings = z.object({
frequency_penalty: z.number().nullish(),
max_tokens: z.int().nullish(),
presence_penalty: z.number().nullish(),
- response_format: zAgentModelResponseFormatConfig.nullish(),
+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),
stop: z.array(z.string()).nullish(),
temperature: z.number().nullish(),
top_p: z.number().nullish(),
@@ -4217,6 +4514,117 @@ export const zGetAgentByAgentIdTextToAudioVoicesQuery = z.object({
*/
export const zGetAgentByAgentIdTextToAudioVoicesResponse = zTextToSpeechVoiceListResponse
+export const zGetAgentByAgentIdTracePath = z.object({
+ agent_id: z.uuid(),
+})
+
+/**
+ * Agent tracing settings
+ */
+export const zGetAgentByAgentIdTraceResponse = zAppTraceResponse
+
+export const zPostAgentByAgentIdTraceBody = zAppTracePayload
+
+export const zPostAgentByAgentIdTracePath = z.object({
+ agent_id: z.uuid(),
+})
+
+/**
+ * Updated Agent tracing settings
+ */
+export const zPostAgentByAgentIdTraceResponse = zAppTraceResponse
+
+export const zGetAgentByAgentIdTraceCapabilitiesPath = z.object({
+ agent_id: z.uuid(),
+})
+
+export const zGetAgentByAgentIdTraceCapabilitiesQuery = z.object({
+ tracing_provider: z.string(),
+})
+
+/**
+ * Provider support declarations
+ */
+export const zGetAgentByAgentIdTraceCapabilitiesResponse = zTraceProviderCapabilitiesResponse
+
+export const zDeleteAgentByAgentIdTraceConfigPath = z.object({
+ agent_id: z.uuid(),
+})
+
+export const zDeleteAgentByAgentIdTraceConfigQuery = z.object({
+ tracing_provider: z.string(),
+})
+
+/**
+ * Agent provider configuration deleted
+ */
+export const zDeleteAgentByAgentIdTraceConfigResponse = z.void()
+
+export const zGetAgentByAgentIdTraceConfigPath = z.object({
+ agent_id: z.uuid(),
+})
+
+export const zGetAgentByAgentIdTraceConfigQuery = z.object({
+ tracing_provider: z.string(),
+})
+
+/**
+ * Masked Agent provider configuration
+ */
+export const zGetAgentByAgentIdTraceConfigResponse = zTraceAppConfigResponse
+
+export const zPatchAgentByAgentIdTraceConfigBody = zTraceConfigPayload
+
+export const zPatchAgentByAgentIdTraceConfigPath = z.object({
+ agent_id: z.uuid(),
+})
+
+/**
+ * Agent provider updated
+ */
+export const zPatchAgentByAgentIdTraceConfigResponse = zTraceAppConfigResponse
+
+export const zPostAgentByAgentIdTraceConfigBody = zTraceConfigPayload
+
+export const zPostAgentByAgentIdTraceConfigPath = z.object({
+ agent_id: z.uuid(),
+})
+
+/**
+ * Agent provider configured
+ */
+export const zPostAgentByAgentIdTraceConfigResponse = zTraceAppConfigResponse
+
+export const zGetAgentByAgentIdTraceRunsByMessageIdPath = z.object({
+ agent_id: z.uuid(),
+ message_id: z.uuid(),
+})
+
+export const zGetAgentByAgentIdTraceRunsByMessageIdQuery = z.object({
+ node_execution_id: z.uuid().optional(),
+})
+
+/**
+ * Frozen trace destination and export state
+ */
+export const zGetAgentByAgentIdTraceRunsByMessageIdResponse = zTraceRunStatusResponse
+
+export const zPostAgentByAgentIdTraceRunsByMessageIdRetryBody = zTraceRunRetryPayload
+
+export const zPostAgentByAgentIdTraceRunsByMessageIdRetryPath = z.object({
+ agent_id: z.uuid(),
+ message_id: z.uuid(),
+})
+
+export const zPostAgentByAgentIdTraceRunsByMessageIdRetryQuery = z.object({
+ node_execution_id: z.uuid().optional(),
+})
+
+/**
+ * Retry state for the same frozen destination
+ */
+export const zPostAgentByAgentIdTraceRunsByMessageIdRetryResponse = zTraceRunStatusResponse
+
export const zGetAgentByAgentIdVersionsPath = z.object({
agent_id: z.uuid(),
})
diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts
index b19f277e3c2..43e89f94d39 100644
--- a/packages/contracts/generated/api/console/apps/orpc.gen.ts
+++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts
@@ -46,9 +46,16 @@ import {
zTooManyFilesError,
zTraceConfigAlreadyExistsError,
zTraceConfigNotFoundError,
+ zTracingAlreadyAcceptedError,
+ zTracingConfigConflictError,
zTracingConfigProcessingFailedError,
zTracingConfigVerificationFailedError,
+ zTracingDuplicateAcknowledgementRequiredError,
+ zTracingNodeNotFoundError,
zTracingProviderUnavailableError,
+ zTracingRetryNotAllowedError,
+ zTracingRunNotFoundError,
+ zTracingSourceNotFoundError,
zUnsupportedAudioTypeError,
zUnsupportedTracingProviderError,
zWorkflowRunNotFoundError,
@@ -202,11 +209,17 @@ import {
zGetAppsByAppIdTextToAudioVoicesPath,
zGetAppsByAppIdTextToAudioVoicesQuery,
zGetAppsByAppIdTextToAudioVoicesResponse,
+ zGetAppsByAppIdTraceCapabilitiesPath,
+ zGetAppsByAppIdTraceCapabilitiesQuery,
+ zGetAppsByAppIdTraceCapabilitiesResponse,
zGetAppsByAppIdTraceConfigPath,
zGetAppsByAppIdTraceConfigQuery,
zGetAppsByAppIdTraceConfigResponse,
zGetAppsByAppIdTracePath,
zGetAppsByAppIdTraceResponse,
+ zGetAppsByAppIdTraceRunsBySourceKindBySourceIdPath,
+ zGetAppsByAppIdTraceRunsBySourceKindBySourceIdQuery,
+ zGetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse,
zGetAppsByAppIdTriggersPath,
zGetAppsByAppIdTriggersResponse,
zGetAppsByAppIdWorkflowAppLogsPath,
@@ -413,6 +426,10 @@ import {
zPostAppsByAppIdTraceConfigResponse,
zPostAppsByAppIdTracePath,
zPostAppsByAppIdTraceResponse,
+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryBody,
+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryPath,
+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryQuery,
+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse,
zPostAppsByAppIdTriggerEnableBody,
zPostAppsByAppIdTriggerEnablePath,
zPostAppsByAppIdTriggerEnableResponse,
@@ -2683,13 +2700,42 @@ export const post35 = oc
})
.input(z.object({ body: zPostAppsByAppIdTraceBody, params: zPostAppsByAppIdTracePath }))
.output(zPostAppsByAppIdTraceResponse)
- .errors({ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError } })
+ .errors({
+ app_not_found: { status: 404, data: zAppNotFoundError },
+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },
+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },
+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },
+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },
+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },
+ })
export const trace = {
get: get45,
post: post35,
}
+export const get46 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAppsByAppIdTraceCapabilities',
+ path: '/apps/{app_id}/trace-capabilities',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zGetAppsByAppIdTraceCapabilitiesPath,
+ query: zGetAppsByAppIdTraceCapabilitiesQuery,
+ }),
+ )
+ .output(zGetAppsByAppIdTraceCapabilitiesResponse)
+ .errors({ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError } })
+
+export const traceCapabilities = {
+ get: get46,
+}
+
/**
* Delete an existing trace app configuration
*
@@ -2723,7 +2769,7 @@ export const delete8 = oc
/**
* Get tracing configuration for an application
*/
-export const get46 = oc
+export const get47 = oc
.route({
description: 'Get tracing configuration for an application',
inputStructure: 'detailed',
@@ -2766,6 +2812,7 @@ export const patch = oc
app_not_found: { status: 404, data: zAppNotFoundError },
invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },
trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },
+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },
tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },
tracing_config_verification_failed: {
status: 400,
@@ -2810,15 +2857,80 @@ export const post36 = oc
export const traceConfig = {
delete: delete8,
- get: get46,
+ get: get47,
patch,
post: post36,
}
+export const post37 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'POST',
+ operationId: 'postAppsByAppIdTraceRunsBySourceKindBySourceIdRetry',
+ path: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}/retry',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ body: zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryBody,
+ params: zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryPath,
+ query: zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryQuery.optional(),
+ }),
+ )
+ .output(zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse)
+ .errors({
+ tracing_already_accepted: { status: 409, data: zTracingAlreadyAcceptedError },
+ tracing_duplicate_acknowledgement_required: {
+ status: 409,
+ data: zTracingDuplicateAcknowledgementRequiredError,
+ },
+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },
+ tracing_retry_not_allowed: { status: 409, data: zTracingRetryNotAllowedError },
+ tracing_run_not_found: { status: 404, data: zTracingRunNotFoundError },
+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },
+ })
+
+export const retry = {
+ post: post37,
+}
+
+export const get48 = oc
+ .route({
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAppsByAppIdTraceRunsBySourceKindBySourceId',
+ path: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zGetAppsByAppIdTraceRunsBySourceKindBySourceIdPath,
+ query: zGetAppsByAppIdTraceRunsBySourceKindBySourceIdQuery.optional(),
+ }),
+ )
+ .output(zGetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse)
+ .errors({
+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },
+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },
+ })
+
+export const bySourceId = {
+ get: get48,
+ retry,
+}
+
+export const bySourceKind = {
+ bySourceId,
+}
+
+export const traceRuns = {
+ bySourceKind,
+}
+
/**
* Update app trigger (enable/disable)
*/
-export const post37 = oc
+export const post38 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -2837,13 +2949,13 @@ export const post37 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const triggerEnable = {
- post: post37,
+ post: post38,
}
/**
* Get app triggers list
*/
-export const get47 = oc
+export const get49 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -2857,7 +2969,7 @@ export const get47 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const triggers = {
- get: get47,
+ get: get49,
}
/**
@@ -2865,7 +2977,7 @@ export const triggers = {
*
* Get workflow application execution logs
*/
-export const get48 = oc
+export const get50 = oc
.route({
description: 'Get workflow application execution logs',
inputStructure: 'detailed',
@@ -2885,7 +2997,7 @@ export const get48 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const workflowAppLogs = {
- get: get48,
+ get: get50,
}
/**
@@ -2893,7 +3005,7 @@ export const workflowAppLogs = {
*
* Get workflow runs count statistics
*/
-export const get49 = oc
+export const get51 = oc
.route({
description: 'Get workflow runs count statistics',
inputStructure: 'detailed',
@@ -2913,7 +3025,7 @@ export const get49 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const count3 = {
- get: get49,
+ get: get51,
}
/**
@@ -2921,7 +3033,7 @@ export const count3 = {
*
* Stop running workflow task
*/
-export const post38 = oc
+export const post39 = oc
.route({
description: 'Stop running workflow task',
inputStructure: 'detailed',
@@ -2936,7 +3048,7 @@ export const post38 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const stop3 = {
- post: post38,
+ post: post39,
}
export const byTaskId3 = {
@@ -2950,7 +3062,7 @@ export const tasks = {
/**
* Get the internal node executions of one workflow tool invocation
*/
-export const get50 = oc
+export const get52 = oc
.route({
description: 'Get the internal node executions of one workflow tool invocation',
inputStructure: 'detailed',
@@ -2968,7 +3080,7 @@ export const get50 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const children = {
- get: get50,
+ get: get52,
}
export const byNodeExecutionId = {
@@ -2980,7 +3092,7 @@ export const byNodeExecutionId = {
*
* Get workflow run node execution list
*/
-export const get51 = oc
+export const get53 = oc
.route({
description: 'Get workflow run node execution list',
inputStructure: 'detailed',
@@ -2995,7 +3107,7 @@ export const get51 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const nodeExecutions = {
- get: get51,
+ get: get53,
byNodeExecutionId,
}
@@ -3004,7 +3116,7 @@ export const nodeExecutions = {
*
* Get workflow run detail
*/
-export const get52 = oc
+export const get54 = oc
.route({
description: 'Get workflow run detail',
inputStructure: 'detailed',
@@ -3019,14 +3131,14 @@ export const get52 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const byRunId = {
- get: get52,
+ get: get54,
nodeExecutions,
}
/**
* Create a ToolFile from one workflow Agent Binding file and return its download URL
*/
-export const post39 = oc
+export const post40 = oc
.route({
description:
'Create a ToolFile from one workflow Agent Binding file and return its download URL',
@@ -3052,13 +3164,13 @@ export const post39 = oc
})
export const download4 = {
- post: post39,
+ post: post40,
}
/**
* Read a text/binary preview file in a workflow Agent node sandbox
*/
-export const get53 = oc
+export const get55 = oc
.route({
description: 'Read a text/binary preview file in a workflow Agent node sandbox',
inputStructure: 'detailed',
@@ -3082,13 +3194,13 @@ export const get53 = oc
})
export const read = {
- get: get53,
+ get: get55,
}
/**
* List a directory in a workflow Agent node sandbox
*/
-export const get54 = oc
+export const get56 = oc
.route({
description: 'List a directory in a workflow Agent node sandbox',
inputStructure: 'detailed',
@@ -3112,7 +3224,7 @@ export const get54 = oc
})
export const files3 = {
- get: get54,
+ get: get56,
download: download4,
read,
}
@@ -3138,7 +3250,7 @@ export const byWorkflowRunId = {
*
* Get workflow run list
*/
-export const get55 = oc
+export const get57 = oc
.route({
description: 'Get workflow run list',
inputStructure: 'detailed',
@@ -3158,7 +3270,7 @@ export const get55 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const workflowRuns2 = {
- get: get55,
+ get: get57,
count: count3,
tasks,
byRunId,
@@ -3170,7 +3282,7 @@ export const workflowRuns2 = {
*
* Get all users in current tenant for mentions
*/
-export const get56 = oc
+export const get58 = oc
.route({
description: 'Get all users in current tenant for mentions',
inputStructure: 'detailed',
@@ -3188,7 +3300,7 @@ export const get56 = oc
})
export const mentionUsers = {
- get: get56,
+ get: get58,
}
/**
@@ -3251,7 +3363,7 @@ export const byReplyId = {
*
* Add a reply to a workflow comment
*/
-export const post40 = oc
+export const post41 = oc
.route({
description: 'Add a reply to a workflow comment',
inputStructure: 'detailed',
@@ -3275,7 +3387,7 @@ export const post40 = oc
})
export const replies = {
- post: post40,
+ post: post41,
byReplyId,
}
@@ -3284,7 +3396,7 @@ export const replies = {
*
* Resolve a workflow comment
*/
-export const post41 = oc
+export const post42 = oc
.route({
description: 'Resolve a workflow comment',
inputStructure: 'detailed',
@@ -3302,7 +3414,7 @@ export const post41 = oc
})
export const resolve = {
- post: post41,
+ post: post42,
}
/**
@@ -3333,7 +3445,7 @@ export const delete10 = oc
*
* Get a specific workflow comment
*/
-export const get57 = oc
+export const get59 = oc
.route({
description: 'Get a specific workflow comment',
inputStructure: 'detailed',
@@ -3379,7 +3491,7 @@ export const put4 = oc
export const byCommentId = {
delete: delete10,
- get: get57,
+ get: get59,
put: put4,
replies,
resolve,
@@ -3390,7 +3502,7 @@ export const byCommentId = {
*
* Get all comments for a workflow
*/
-export const get58 = oc
+export const get60 = oc
.route({
description: 'Get all comments for a workflow',
inputStructure: 'detailed',
@@ -3412,7 +3524,7 @@ export const get58 = oc
*
* Create a new workflow comment
*/
-export const post42 = oc
+export const post43 = oc
.route({
description: 'Create a new workflow comment',
inputStructure: 'detailed',
@@ -3436,8 +3548,8 @@ export const post42 = oc
})
export const comments = {
- get: get58,
- post: post42,
+ get: get60,
+ post: post43,
mentionUsers,
byCommentId,
}
@@ -3445,7 +3557,7 @@ export const comments = {
/**
* Get workflow average app interaction statistics
*/
-export const get59 = oc
+export const get61 = oc
.route({
description: 'Get workflow average app interaction statistics',
inputStructure: 'detailed',
@@ -3464,13 +3576,13 @@ export const get59 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const averageAppInteractions = {
- get: get59,
+ get: get61,
}
/**
* Get workflow daily runs statistics
*/
-export const get60 = oc
+export const get62 = oc
.route({
description: 'Get workflow daily runs statistics',
inputStructure: 'detailed',
@@ -3489,13 +3601,13 @@ export const get60 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const dailyConversations2 = {
- get: get60,
+ get: get62,
}
/**
* Get workflow daily terminals statistics
*/
-export const get61 = oc
+export const get63 = oc
.route({
description: 'Get workflow daily terminals statistics',
inputStructure: 'detailed',
@@ -3514,13 +3626,13 @@ export const get61 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const dailyTerminals = {
- get: get61,
+ get: get63,
}
/**
* Get workflow daily token cost statistics
*/
-export const get62 = oc
+export const get64 = oc
.route({
description: 'Get workflow daily token cost statistics',
inputStructure: 'detailed',
@@ -3539,7 +3651,7 @@ export const get62 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const tokenCosts2 = {
- get: get62,
+ get: get64,
}
export const statistics2 = {
@@ -3559,7 +3671,7 @@ export const workflow = {
*
* Get default block configuration by type
*/
-export const get63 = oc
+export const get65 = oc
.route({
description: 'Get default block configuration by type',
inputStructure: 'detailed',
@@ -3579,7 +3691,7 @@ export const get63 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const byBlockType = {
- get: get63,
+ get: get65,
}
/**
@@ -3587,7 +3699,7 @@ export const byBlockType = {
*
* Get default block configurations for workflow
*/
-export const get64 = oc
+export const get66 = oc
.route({
description: 'Get default block configurations for workflow',
inputStructure: 'detailed',
@@ -3602,14 +3714,14 @@ export const get64 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const defaultWorkflowBlockConfigs = {
- get: get64,
+ get: get66,
byBlockType,
}
/**
* Get conversation variables for workflow
*/
-export const get65 = oc
+export const get67 = oc
.route({
description: 'Get conversation variables for workflow',
inputStructure: 'detailed',
@@ -3625,7 +3737,7 @@ export const get65 = oc
/**
* Update conversation variables for workflow draft
*/
-export const post43 = oc
+export const post44 = oc
.route({
description: 'Update conversation variables for workflow draft',
inputStructure: 'detailed',
@@ -3644,8 +3756,8 @@ export const post43 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const conversationVariables2 = {
- get: get65,
- post: post43,
+ get: get67,
+ post: post44,
}
/**
@@ -3653,7 +3765,7 @@ export const conversationVariables2 = {
*
* Get environment variables for workflow
*/
-export const get66 = oc
+export const get68 = oc
.route({
description: 'Get environment variables for workflow',
inputStructure: 'detailed',
@@ -3673,7 +3785,7 @@ export const get66 = oc
/**
* Update environment variables for workflow draft
*/
-export const post44 = oc
+export const post45 = oc
.route({
description: 'Update environment variables for workflow draft',
inputStructure: 'detailed',
@@ -3692,14 +3804,14 @@ export const post44 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const environmentVariables = {
- get: get66,
- post: post44,
+ get: get68,
+ post: post45,
}
/**
* Update draft workflow features
*/
-export const post45 = oc
+export const post46 = oc
.route({
description: 'Update draft workflow features',
inputStructure: 'detailed',
@@ -3718,7 +3830,7 @@ export const post45 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const features = {
- post: post45,
+ post: post46,
}
/**
@@ -3726,7 +3838,7 @@ export const features = {
*
* Test human input delivery for workflow
*/
-export const post46 = oc
+export const post47 = oc
.route({
description: 'Test human input delivery for workflow',
inputStructure: 'detailed',
@@ -3746,7 +3858,7 @@ export const post46 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const deliveryTest = {
- post: post46,
+ post: post47,
}
/**
@@ -3754,7 +3866,7 @@ export const deliveryTest = {
*
* Get human input form preview for workflow
*/
-export const post47 = oc
+export const post48 = oc
.route({
description: 'Get human input form preview for workflow',
inputStructure: 'detailed',
@@ -3774,7 +3886,7 @@ export const post47 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const preview4 = {
- post: post47,
+ post: post48,
}
/**
@@ -3782,7 +3894,7 @@ export const preview4 = {
*
* Submit human input form preview for workflow
*/
-export const post48 = oc
+export const post49 = oc
.route({
description: 'Submit human input form preview for workflow',
inputStructure: 'detailed',
@@ -3802,7 +3914,7 @@ export const post48 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const run5 = {
- post: post48,
+ post: post49,
}
export const form2 = {
@@ -3828,7 +3940,7 @@ export const humanInput2 = {
*
* Run draft workflow iteration node
*/
-export const post49 = oc
+export const post50 = oc
.route({
description: 'Run draft workflow iteration node',
inputStructure: 'detailed',
@@ -3848,7 +3960,7 @@ export const post49 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const run6 = {
- post: post49,
+ post: post50,
}
export const byNodeId6 = {
@@ -3868,7 +3980,7 @@ export const iteration2 = {
*
* Run draft workflow loop node
*/
-export const post50 = oc
+export const post51 = oc
.route({
description: 'Run draft workflow loop node',
inputStructure: 'detailed',
@@ -3888,7 +4000,7 @@ export const post50 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const run7 = {
- post: post50,
+ post: post51,
}
export const byNodeId7 = {
@@ -3903,7 +4015,7 @@ export const loop2 = {
nodes: nodes6,
}
-export const get67 = oc
+export const get69 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -3918,10 +4030,10 @@ export const get67 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const candidates = {
- get: get67,
+ get: get69,
}
-export const post51 = oc
+export const post52 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -3945,10 +4057,10 @@ export const post51 = oc
})
export const copyFromRoster = {
- post: post51,
+ post: post52,
}
-export const post52 = oc
+export const post53 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -3966,10 +4078,10 @@ export const post52 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const impact = {
- post: post52,
+ post: post53,
}
-export const post53 = oc
+export const post54 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -3995,10 +4107,10 @@ export const post53 = oc
})
export const saveToRoster = {
- post: post53,
+ post: post54,
}
-export const post54 = oc
+export const post55 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -4021,10 +4133,10 @@ export const post54 = oc
})
export const validate = {
- post: post54,
+ post: post55,
}
-export const get68 = oc
+export const get70 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -4071,7 +4183,7 @@ export const put5 = oc
})
export const agentComposer = {
- get: get68,
+ get: get70,
put: put5,
candidates,
copyFromRoster,
@@ -4083,7 +4195,7 @@ export const agentComposer = {
/**
* Get last run result for draft workflow node
*/
-export const get69 = oc
+export const get71 = oc
.route({
description: 'Get last run result for draft workflow node',
inputStructure: 'detailed',
@@ -4097,7 +4209,7 @@ export const get69 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const lastRun = {
- get: get69,
+ get: get71,
}
/**
@@ -4105,7 +4217,7 @@ export const lastRun = {
*
* Run draft workflow node
*/
-export const post55 = oc
+export const post56 = oc
.route({
description: 'Run draft workflow node',
inputStructure: 'detailed',
@@ -4125,7 +4237,7 @@ export const post55 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const run8 = {
- post: post55,
+ post: post56,
}
/**
@@ -4133,7 +4245,7 @@ export const run8 = {
*
* Poll for trigger events and execute single node when event arrives
*/
-export const post56 = oc
+export const post57 = oc
.route({
description: 'Poll for trigger events and execute single node when event arrives',
inputStructure: 'detailed',
@@ -4148,7 +4260,7 @@ export const post56 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const run9 = {
- post: post56,
+ post: post57,
}
export const trigger = {
@@ -4175,7 +4287,7 @@ export const delete11 = oc
/**
* Get variables for a specific node
*/
-export const get70 = oc
+export const get72 = oc
.route({
description: 'Get variables for a specific node',
inputStructure: 'detailed',
@@ -4190,7 +4302,7 @@ export const get70 = oc
export const variables = {
delete: delete11,
- get: get70,
+ get: get72,
}
export const byNodeId8 = {
@@ -4210,7 +4322,7 @@ export const nodes7 = {
*
* Run draft workflow
*/
-export const post57 = oc
+export const post58 = oc
.route({
description: 'Run draft workflow',
inputStructure: 'detailed',
@@ -4233,13 +4345,13 @@ export const post57 = oc
})
export const run10 = {
- post: post57,
+ post: post58,
}
/**
* Server-Sent Events stream of inspector deltas for a draft workflow run.
*/
-export const get71 = oc
+export const get73 = oc
.route({
description: 'Server-Sent Events stream of inspector deltas for a draft workflow run.',
inputStructure: 'detailed',
@@ -4256,13 +4368,13 @@ export const get71 = oc
})
export const events = {
- get: get71,
+ get: get73,
}
/**
* Full value for one declared output, including signed download URL for files.
*/
-export const get72 = oc
+export const get74 = oc
.route({
description: 'Full value for one declared output, including signed download URL for files.',
inputStructure: 'detailed',
@@ -4285,7 +4397,7 @@ export const get72 = oc
})
export const preview5 = {
- get: get72,
+ get: get74,
}
export const byOutputName = {
@@ -4295,7 +4407,7 @@ export const byOutputName = {
/**
* One node's declared outputs for a draft workflow run.
*/
-export const get73 = oc
+export const get75 = oc
.route({
description: "One node's declared outputs for a draft workflow run.",
inputStructure: 'detailed',
@@ -4313,14 +4425,14 @@ export const get73 = oc
})
export const byNodeId9 = {
- get: get73,
+ get: get75,
byOutputName,
}
/**
* Snapshot of every node's declared outputs for a draft workflow run.
*/
-export const get74 = oc
+export const get76 = oc
.route({
description: "Snapshot of every node's declared outputs for a draft workflow run.",
inputStructure: 'detailed',
@@ -4337,7 +4449,7 @@ export const get74 = oc
})
export const nodeOutputs = {
- get: get74,
+ get: get76,
events,
byNodeId: byNodeId9,
}
@@ -4353,7 +4465,7 @@ export const runs = {
/**
* Get system variables for workflow
*/
-export const get75 = oc
+export const get77 = oc
.route({
description: 'Get system variables for workflow',
inputStructure: 'detailed',
@@ -4367,7 +4479,7 @@ export const get75 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const systemVariables = {
- get: get75,
+ get: get77,
}
/**
@@ -4375,7 +4487,7 @@ export const systemVariables = {
*
* Poll for trigger events and execute full workflow when event arrives
*/
-export const post58 = oc
+export const post59 = oc
.route({
description: 'Poll for trigger events and execute full workflow when event arrives',
inputStructure: 'detailed',
@@ -4398,7 +4510,7 @@ export const post58 = oc
})
export const run11 = {
- post: post58,
+ post: post59,
}
/**
@@ -4406,7 +4518,7 @@ export const run11 = {
*
* Full workflow debug when the start node is a trigger
*/
-export const post59 = oc
+export const post60 = oc
.route({
description: 'Full workflow debug when the start node is a trigger',
inputStructure: 'detailed',
@@ -4429,7 +4541,7 @@ export const post59 = oc
})
export const runAll = {
- post: post59,
+ post: post60,
}
export const trigger2 = {
@@ -4477,7 +4589,7 @@ export const delete12 = oc
/**
* Get a specific workflow variable
*/
-export const get76 = oc
+export const get78 = oc
.route({
description: 'Get a specific workflow variable',
inputStructure: 'detailed',
@@ -4513,7 +4625,7 @@ export const patch2 = oc
export const byVariableId = {
delete: delete12,
- get: get76,
+ get: get78,
patch: patch2,
reset,
}
@@ -4540,7 +4652,7 @@ export const delete13 = oc
*
* Get draft workflow variables
*/
-export const get77 = oc
+export const get79 = oc
.route({
description: 'Get draft workflow variables',
inputStructure: 'detailed',
@@ -4564,7 +4676,7 @@ export const get77 = oc
export const variables2 = {
delete: delete13,
- get: get77,
+ get: get79,
byVariableId,
}
@@ -4573,7 +4685,7 @@ export const variables2 = {
*
* Get draft workflow for an application
*/
-export const get78 = oc
+export const get80 = oc
.route({
description: 'Get draft workflow for an application',
inputStructure: 'detailed',
@@ -4595,7 +4707,7 @@ export const get78 = oc
*
* Sync draft workflow configuration
*/
-export const post60 = oc
+export const post61 = oc
.route({
description: 'Sync draft workflow configuration',
inputStructure: 'detailed',
@@ -4618,8 +4730,8 @@ export const post60 = oc
})
export const draft2 = {
- get: get78,
- post: post60,
+ get: get80,
+ post: post61,
conversationVariables: conversationVariables2,
environmentVariables,
features,
@@ -4639,7 +4751,7 @@ export const draft2 = {
*
* Get published workflow for an application
*/
-export const get79 = oc
+export const get81 = oc
.route({
description: 'Get published workflow for an application',
inputStructure: 'detailed',
@@ -4656,7 +4768,7 @@ export const get79 = oc
/**
* Publish workflow
*/
-export const post61 = oc
+export const post62 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -4675,14 +4787,14 @@ export const post61 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const publish = {
- get: get79,
- post: post61,
+ get: get81,
+ post: post62,
}
/**
* Server-Sent Events stream of inspector deltas for a published workflow run.
*/
-export const get80 = oc
+export const get82 = oc
.route({
description: 'Server-Sent Events stream of inspector deltas for a published workflow run.',
inputStructure: 'detailed',
@@ -4699,13 +4811,13 @@ export const get80 = oc
})
export const events2 = {
- get: get80,
+ get: get82,
}
/**
* Full value for one declared output of a published run.
*/
-export const get81 = oc
+export const get83 = oc
.route({
description: 'Full value for one declared output of a published run.',
inputStructure: 'detailed',
@@ -4732,7 +4844,7 @@ export const get81 = oc
})
export const preview6 = {
- get: get81,
+ get: get83,
}
export const byOutputName2 = {
@@ -4742,7 +4854,7 @@ export const byOutputName2 = {
/**
* One node's declared outputs for a published workflow run.
*/
-export const get82 = oc
+export const get84 = oc
.route({
description: "One node's declared outputs for a published workflow run.",
inputStructure: 'detailed',
@@ -4760,14 +4872,14 @@ export const get82 = oc
})
export const byNodeId10 = {
- get: get82,
+ get: get84,
byOutputName: byOutputName2,
}
/**
* Snapshot of every node's declared outputs for a published workflow run.
*/
-export const get83 = oc
+export const get85 = oc
.route({
description: "Snapshot of every node's declared outputs for a published workflow run.",
inputStructure: 'detailed',
@@ -4784,7 +4896,7 @@ export const get83 = oc
})
export const nodeOutputs2 = {
- get: get83,
+ get: get85,
events: events2,
byNodeId: byNodeId10,
}
@@ -4804,7 +4916,7 @@ export const published = {
/**
* Get webhook trigger for a node
*/
-export const get84 = oc
+export const get86 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -4823,7 +4935,7 @@ export const get84 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const webhook = {
- get: get84,
+ get: get86,
}
export const triggers2 = {
@@ -4833,7 +4945,7 @@ export const triggers2 = {
/**
* Restore a published workflow version into the draft workflow
*/
-export const post62 = oc
+export const post63 = oc
.route({
description: 'Restore a published workflow version into the draft workflow',
inputStructure: 'detailed',
@@ -4847,7 +4959,7 @@ export const post62 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const restore = {
- post: post62,
+ post: post63,
}
/**
@@ -4902,7 +5014,7 @@ export const byWorkflowId = {
*
* Get all published workflows for an application
*/
-export const get85 = oc
+export const get87 = oc
.route({
description: 'Get all published workflows for an application',
inputStructure: 'detailed',
@@ -4922,7 +5034,7 @@ export const get85 = oc
.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })
export const workflows3 = {
- get: get85,
+ get: get87,
defaultWorkflowBlockConfigs,
draft: draft2,
publish,
@@ -4950,7 +5062,7 @@ export const delete15 = oc
/**
* Get application details
*/
-export const get86 = oc
+export const get88 = oc
.route({
description: 'Get application details',
inputStructure: 'detailed',
@@ -4979,7 +5091,7 @@ export const put7 = oc
export const byAppId2 = {
delete: delete15,
- get: get86,
+ get: get88,
put: put7,
advancedChat,
agent,
@@ -5011,7 +5123,9 @@ export const byAppId2 = {
statistics,
textToAudio,
trace,
+ traceCapabilities,
traceConfig,
+ traceRuns,
triggerEnable,
triggers,
workflowAppLogs,
@@ -5043,7 +5157,7 @@ export const byApiKeyId = {
/**
* Get all API keys for an app
*/
-export const get87 = oc
+export const get89 = oc
.route({
description: 'Get all API keys for an app',
inputStructure: 'detailed',
@@ -5058,7 +5172,7 @@ export const get87 = oc
/**
* Create a new API key for an app
*/
-export const post63 = oc
+export const post64 = oc
.route({
description: 'Create a new API key for an app',
inputStructure: 'detailed',
@@ -5076,8 +5190,8 @@ export const post63 = oc
})
export const apiKeys = {
- get: get87,
- post: post63,
+ get: get89,
+ post: post64,
byApiKeyId,
}
@@ -5088,7 +5202,7 @@ export const byResourceId = {
/**
* Get list of applications with pagination and filtering
*/
-export const get88 = oc
+export const get90 = oc
.route({
description: 'Get list of applications with pagination and filtering',
inputStructure: 'detailed',
@@ -5103,7 +5217,7 @@ export const get88 = oc
/**
* Create a new application
*/
-export const post64 = oc
+export const post65 = oc
.route({
description: 'Create a new application',
inputStructure: 'detailed',
@@ -5117,8 +5231,8 @@ export const post64 = oc
.output(zPostAppsResponse)
export const apps = {
- get: get88,
- post: post64,
+ get: get90,
+ post: post65,
imports,
recent,
starred,
diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts
index 96355856cfe..d2ba8be55c7 100644
--- a/packages/contracts/generated/api/console/apps/types.gen.ts
+++ b/packages/contracts/generated/api/console/apps/types.gen.ts
@@ -851,15 +851,24 @@ export type ProviderNotSupportTextToSpeechError = ErrorResponse & {
export type TextToSpeechVoiceListResponse = Array<TextToSpeechVoiceResponse>
export type AppTraceResponse = {
+ capture_agent?: boolean
+ capture_content?: boolean
enabled?: boolean
tracing_provider?: string | null
}
export type AppTracePayload = {
+ capture_agent?: boolean
+ capture_content?: boolean
enabled: boolean
tracing_provider?: string | null
}
+export type InvalidTracingConfigError = ErrorResponse & {
+ code: 'invalid_tracing_config'
+ status: 400
+}
+
export type TracingProviderUnavailableError = ErrorResponse & {
code: 'tracing_provider_unavailable'
status: 400
@@ -875,11 +884,28 @@ export type TraceConfigNotFoundError = ErrorResponse & {
status: 404
}
+export type TracingConfigConflictError = ErrorResponse & {
+ code: 'tracing_config_conflict'
+ status: 409
+}
+
export type TracingConfigProcessingFailedError = ErrorResponse & {
code: 'tracing_config_processing_failed'
status: 500
}
+export type TraceProviderCapabilitiesResponse = {
+ agent_spans: boolean
+ api_collection_enabled: boolean
+ export_ack: 'accepted'
+ native_cost: boolean
+ native_usage: boolean
+ provider: string
+ session: boolean
+ span_url: boolean
+ trace_url: boolean
+}
+
export type TraceAppConfigResponse = {
app_id?: string | null
created_at?: string | null
@@ -896,17 +922,13 @@ export type TraceAppConfigResponse = {
}
export type TraceConfigPayload = {
+ activate?: boolean
tracing_config: {
[key: string]: string | null
}
tracing_provider: string
}
-export type InvalidTracingConfigError = ErrorResponse & {
- code: 'invalid_tracing_config'
- status: 400
-}
-
export type TracingConfigVerificationFailedError = ErrorResponse & {
code: 'tracing_config_verification_failed'
status: 400
@@ -917,6 +939,68 @@ export type TraceConfigAlreadyExistsError = ErrorResponse & {
status: 409
}
+export type TraceRunStatusResponse = {
+ accepted_at?: string | null
+ attempts?: number | null
+ capabilities?: TraceProviderSupportResponse | null
+ capture_agent?: boolean | null
+ capture_content?: boolean | null
+ config_revision?: string | null
+ error_code?: string | null
+ project_url?: string | null
+ provider?: string | null
+ queryable?: boolean | null
+ source_id: string
+ source_kind: TraceSourceKind
+ span_count?: number | null
+ span_url?: string | null
+ status:
+ | 'accepted'
+ | 'collecting'
+ | 'exporting'
+ | 'failed'
+ | 'not_collected'
+ | 'ready'
+ | 'retrying'
+ | 'uncertain'
+ trace_id?: string | null
+ trace_url?: string | null
+}
+
+export type TracingNodeNotFoundError = ErrorResponse & {
+ code: 'tracing_node_not_found'
+ status: 404
+}
+
+export type TracingSourceNotFoundError = ErrorResponse & {
+ code: 'tracing_source_not_found'
+ status: 404
+}
+
+export type TraceRunRetryPayload = {
+ acknowledge_possible_duplicates?: boolean
+}
+
+export type TracingRunNotFoundError = ErrorResponse & {
+ code: 'tracing_run_not_found'
+ status: 404
+}
+
+export type TracingAlreadyAcceptedError = ErrorResponse & {
+ code: 'tracing_already_accepted'
+ status: 409
+}
+
+export type TracingDuplicateAcknowledgementRequiredError = ErrorResponse & {
+ code: 'tracing_duplicate_acknowledgement_required'
+ status: 409
+}
+
+export type TracingRetryNotAllowedError = ErrorResponse & {
+ code: 'tracing_retry_not_allowed'
+ status: 409
+}
+
export type ParserEnable = {
enable_trigger: boolean
trigger_id: string
@@ -2186,6 +2270,19 @@ export type TextToSpeechVoiceResponse = {
value: string
}
+export type TraceProviderSupportResponse = {
+ agent_spans: boolean
+ export_ack: 'accepted'
+ native_cost: boolean
+ native_usage: boolean
+ provider: string
+ session: boolean
+ span_url: boolean
+ trace_url: boolean
+}
+
+export type TraceSourceKind = 'message' | 'node' | 'workflow'
+
export type WorkflowAppLogPartialResponse = {
created_at?: number | null
created_by_account?: SimpleAccountResponse | null
@@ -4007,7 +4104,7 @@ export type AgentSoulModelSettings = {
frequency_penalty?: number | null
max_tokens?: number | null
presence_penalty?: number | null
- response_format?: AgentModelResponseFormatConfig | null
+ response_format?: string | AgentModelResponseFormatConfig | null
stop?: Array<string> | null
temperature?: number | null
top_p?: number | null
@@ -7036,8 +7133,15 @@ export type PostAppsByAppIdTraceData = {
}
export type PostAppsByAppIdTraceErrors = {
- 400: TracingProviderUnavailableError | ErrorResponse
+ 400:
+ | InvalidTracingConfigError
+ | TracingProviderUnavailableError
+ | UnsupportedTracingProviderError
+ | ErrorResponse
403: ErrorResponse
+ 404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse
+ 409: TracingConfigConflictError | ErrorResponse
+ 500: TracingConfigProcessingFailedError | ErrorResponse
default: ErrorResponse
}
@@ -7050,6 +7154,32 @@ export type PostAppsByAppIdTraceResponses = {
export type PostAppsByAppIdTraceResponse =
PostAppsByAppIdTraceResponses[keyof PostAppsByAppIdTraceResponses]
+export type GetAppsByAppIdTraceCapabilitiesData = {
+ body?: never
+ path: {
+ app_id: string
+ }
+ query: {
+ tracing_provider: string
+ }
+ url: '/apps/{app_id}/trace-capabilities'
+}
+
+export type GetAppsByAppIdTraceCapabilitiesErrors = {
+ 400: UnsupportedTracingProviderError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type GetAppsByAppIdTraceCapabilitiesError =
+ GetAppsByAppIdTraceCapabilitiesErrors[keyof GetAppsByAppIdTraceCapabilitiesErrors]
+
+export type GetAppsByAppIdTraceCapabilitiesResponses = {
+ 200: TraceProviderCapabilitiesResponse
+}
+
+export type GetAppsByAppIdTraceCapabilitiesResponse =
+ GetAppsByAppIdTraceCapabilitiesResponses[keyof GetAppsByAppIdTraceCapabilitiesResponses]
+
export type DeleteAppsByAppIdTraceConfigData = {
body?: never
path: {
@@ -7125,6 +7255,7 @@ export type PatchAppsByAppIdTraceConfigErrors = {
| ErrorResponse
403: ErrorResponse
404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse
+ 409: TracingConfigConflictError | ErrorResponse
500: TracingConfigProcessingFailedError | ErrorResponse
default: ErrorResponse
}
@@ -7172,6 +7303,71 @@ export type PostAppsByAppIdTraceConfigResponses = {
export type PostAppsByAppIdTraceConfigResponse =
PostAppsByAppIdTraceConfigResponses[keyof PostAppsByAppIdTraceConfigResponses]
+export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdData = {
+ body?: never
+ path: {
+ app_id: string
+ source_id: string
+ source_kind: 'message' | 'node' | 'workflow'
+ }
+ query?: {
+ node_execution_id?: string
+ }
+ url: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}'
+}
+
+export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdErrors = {
+ 404: TracingNodeNotFoundError | TracingSourceNotFoundError | ErrorResponse
+ default: ErrorResponse
+}
+
+export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdError =
+ GetAppsByAppIdTraceRunsBySourceKindBySourceIdErrors[keyof GetAppsByAppIdTraceRunsBySourceKindBySourceIdErrors]
+
+export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponses = {
+ 200: TraceRunStatusResponse
+}
+
+export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse =
+ GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponses[keyof GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponses]
+
+export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryData = {
+ body: TraceRunRetryPayload
+ path: {
+ app_id: string
+ source_id: string
+ source_kind: 'message' | 'node' | 'workflow'
+ }
+ query?: {
+ node_execution_id?: string
+ }
+ url: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}/retry'
+}
+
+export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryErrors = {
+ 404:
+ | TracingNodeNotFoundError
+ | TracingRunNotFoundError
+ | TracingSourceNotFoundError
+ | ErrorResponse
+ 409:
+ | TracingAlreadyAcceptedError
+ | TracingDuplicateAcknowledgementRequiredError
+ | TracingRetryNotAllowedError
+ | ErrorResponse
+ default: ErrorResponse
+}
+
+export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryError =
+ PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryErrors[keyof PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryErrors]
+
+export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponses = {
+ 200: TraceRunStatusResponse
+}
+
+export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse =
+ PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponses[keyof PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponses]
+
export type PostAppsByAppIdTriggerEnableData = {
body: ParserEnable
path: {
diff --git a/packages/contracts/generated/api/console/apps/zod.gen.ts b/packages/contracts/generated/api/console/apps/zod.gen.ts
index e6cc027d761..cf994692b68 100644
--- a/packages/contracts/generated/api/console/apps/zod.gen.ts
+++ b/packages/contracts/generated/api/console/apps/zod.gen.ts
@@ -740,6 +740,8 @@ export const zProviderNotSupportTextToSpeechError = zErrorResponse.and(
* AppTraceResponse
*/
export const zAppTraceResponse = z.object({
+ capture_agent: z.boolean().optional().default(true),
+ capture_content: z.boolean().optional().default(false),
enabled: z.boolean().optional().default(false),
tracing_provider: z.string().nullish(),
})
@@ -748,10 +750,24 @@ export const zAppTraceResponse = z.object({
* AppTracePayload
*/
export const zAppTracePayload = z.object({
+ capture_agent: z.boolean().optional().default(true),
+ capture_content: z.boolean().optional().default(false),
enabled: z.boolean(),
tracing_provider: z.string().nullish(),
})
+/**
+ * InvalidTracingConfigError
+ *
+ * The tracing configuration is invalid.
+ */
+export const zInvalidTracingConfigError = zErrorResponse.and(
+ z.object({
+ code: z.literal('invalid_tracing_config'),
+ status: z.literal(400),
+ }),
+)
+
/**
* TracingProviderUnavailableError
*
@@ -788,6 +804,18 @@ export const zTraceConfigNotFoundError = zErrorResponse.and(
}),
)
+/**
+ * TracingConfigConflictError
+ *
+ * Tracing configuration changed. Refresh and try again.
+ */
+export const zTracingConfigConflictError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_config_conflict'),
+ status: z.literal(409),
+ }),
+)
+
/**
* TracingConfigProcessingFailedError
*
@@ -800,6 +828,21 @@ export const zTracingConfigProcessingFailedError = zErrorResponse.and(
}),
)
+/**
+ * TraceProviderCapabilitiesResponse
+ */
+export const zTraceProviderCapabilitiesResponse = z.object({
+ agent_spans: z.boolean(),
+ api_collection_enabled: z.boolean(),
+ export_ack: z.literal('accepted'),
+ native_cost: z.boolean(),
+ native_usage: z.boolean(),
+ provider: z.string(),
+ session: z.boolean(),
+ span_url: z.boolean(),
+ trace_url: z.boolean(),
+})
+
/**
* TraceAppConfigResponse
*/
@@ -820,22 +863,11 @@ export const zTraceAppConfigResponse = z.object({
* TraceConfigPayload
*/
export const zTraceConfigPayload = z.object({
+ activate: z.boolean().optional().default(false),
tracing_config: z.record(z.string(), z.string().nullable()),
tracing_provider: z.string(),
})
-/**
- * InvalidTracingConfigError
- *
- * The tracing configuration is invalid.
- */
-export const zInvalidTracingConfigError = zErrorResponse.and(
- z.object({
- code: z.literal('invalid_tracing_config'),
- status: z.literal(400),
- }),
-)
-
/**
* TracingConfigVerificationFailedError
*
@@ -860,6 +892,85 @@ export const zTraceConfigAlreadyExistsError = zErrorResponse.and(
}),
)
+/**
+ * TracingNodeNotFoundError
+ *
+ * The Agent node invocation does not belong to this trace.
+ */
+export const zTracingNodeNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_node_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TracingSourceNotFoundError
+ *
+ * The tracing source does not exist in this application.
+ */
+export const zTracingSourceNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_source_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TraceRunRetryPayload
+ */
+export const zTraceRunRetryPayload = z.object({
+ acknowledge_possible_duplicates: z.boolean().optional().default(false),
+})
+
+/**
+ * TracingRunNotFoundError
+ *
+ * No collected trace is available to retry.
+ */
+export const zTracingRunNotFoundError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_run_not_found'),
+ status: z.literal(404),
+ }),
+)
+
+/**
+ * TracingAlreadyAcceptedError
+ *
+ * An accepted trace cannot be exported again.
+ */
+export const zTracingAlreadyAcceptedError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_already_accepted'),
+ status: z.literal(409),
+ }),
+)
+
+/**
+ * TracingDuplicateAcknowledgementRequiredError
+ *
+ * Acknowledge possible duplicates before retrying an uncertain export.
+ */
+export const zTracingDuplicateAcknowledgementRequiredError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_duplicate_acknowledgement_required'),
+ status: z.literal(409),
+ }),
+)
+
+/**
+ * TracingRetryNotAllowedError
+ *
+ * This trace is not in a retryable state.
+ */
+export const zTracingRetryNotAllowedError = zErrorResponse.and(
+ z.object({
+ code: z.literal('tracing_retry_not_allowed'),
+ status: z.literal(409),
+ }),
+)
+
/**
* ParserEnable
*/
@@ -2013,6 +2124,56 @@ export const zTextToSpeechVoiceResponse = z.object({
*/
export const zTextToSpeechVoiceListResponse = z.array(zTextToSpeechVoiceResponse)
+/**
+ * TraceProviderSupportResponse
+ *
+ * Adapter support declarations, not successful-delivery or platform-verification results.
+ */
+export const zTraceProviderSupportResponse = z.object({
+ agent_spans: z.boolean(),
+ export_ack: z.literal('accepted'),
+ native_cost: z.boolean(),
+ native_usage: z.boolean(),
+ provider: z.string(),
+ session: z.boolean(),
+ span_url: z.boolean(),
+ trace_url: z.boolean(),
+})
+
+export const zTraceSourceKind = z.enum(['message', 'node', 'workflow'])
+
+/**
+ * TraceRunStatusResponse
+ */
+export const zTraceRunStatusResponse = z.object({
+ accepted_at: z.iso.datetime().nullish(),
+ attempts: z.int().nullish(),
+ capabilities: zTraceProviderSupportResponse.nullish(),
+ capture_agent: z.boolean().nullish(),
+ capture_content: z.boolean().nullish(),
+ config_revision: z.string().nullish(),
+ error_code: z.string().nullish(),
+ project_url: z.string().nullish(),
+ provider: z.string().nullish(),
+ queryable: z.boolean().nullish(),
+ source_id: z.string(),
+ source_kind: zTraceSourceKind,
+ span_count: z.int().nullish(),
+ span_url: z.string().nullish(),
+ status: z.enum([
+ 'accepted',
+ 'collecting',
+ 'exporting',
+ 'failed',
+ 'not_collected',
+ 'ready',
+ 'retrying',
+ 'uncertain',
+ ]),
+ trace_id: z.string().nullish(),
+ trace_url: z.string().nullish(),
+})
+
/**
* SimpleAccountResponse
*/
@@ -5428,7 +5589,7 @@ export const zAgentSoulModelSettings = z.object({
frequency_penalty: z.number().nullish(),
max_tokens: z.int().nullish(),
presence_penalty: z.number().nullish(),
- response_format: zAgentModelResponseFormatConfig.nullish(),
+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),
stop: z.array(z.string()).nullish(),
temperature: z.number().nullish(),
top_p: z.number().nullish(),
@@ -7318,6 +7479,19 @@ export const zPostAppsByAppIdTracePath = z.object({
*/
export const zPostAppsByAppIdTraceResponse = zSimpleResultResponse
+export const zGetAppsByAppIdTraceCapabilitiesPath = z.object({
+ app_id: z.uuid(),
+})
+
+export const zGetAppsByAppIdTraceCapabilitiesQuery = z.object({
+ tracing_provider: z.string(),
+})
+
+/**
+ * Provider support declarations
+ */
+export const zGetAppsByAppIdTraceCapabilitiesResponse = zTraceProviderCapabilitiesResponse
+
export const zDeleteAppsByAppIdTraceConfigPath = z.object({
app_id: z.uuid(),
})
@@ -7366,6 +7540,38 @@ export const zPostAppsByAppIdTraceConfigPath = z.object({
*/
export const zPostAppsByAppIdTraceConfigResponse = zTraceAppConfigResponse
+export const zGetAppsByAppIdTraceRunsBySourceKindBySourceIdPath = z.object({
+ app_id: z.uuid(),
+ source_id: z.uuid(),
+ source_kind: z.enum(['message', 'node', 'workflow']),
+})
+
+export const zGetAppsByAppIdTraceRunsBySourceKindBySourceIdQuery = z.object({
+ node_execution_id: z.uuid().optional(),
+})
+
+/**
+ * Frozen trace destination and export state
+ */
+export const zGetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse = zTraceRunStatusResponse
+
+export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryBody = zTraceRunRetryPayload
+
+export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryPath = z.object({
+ app_id: z.uuid(),
+ source_id: z.uuid(),
+ source_kind: z.enum(['message', 'node', 'workflow']),
+})
+
+export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryQuery = z.object({
+ node_execution_id: z.uuid().optional(),
+})
+
+/**
+ * Retry state for the same frozen destination
+ */
+export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse = zTraceRunStatusResponse
+
export const zPostAppsByAppIdTriggerEnableBody = zParserEnable
export const zPostAppsByAppIdTriggerEnablePath = z.object({
diff --git a/packages/contracts/generated/api/console/snippets/types.gen.ts b/packages/contracts/generated/api/console/snippets/types.gen.ts
index 162b61e1223..8c41ab7a0bd 100644
--- a/packages/contracts/generated/api/console/snippets/types.gen.ts
+++ b/packages/contracts/generated/api/console/snippets/types.gen.ts
@@ -960,7 +960,7 @@ export type AgentSoulModelSettings = {
frequency_penalty?: number | null
max_tokens?: number | null
presence_penalty?: number | null
- response_format?: AgentModelResponseFormatConfig | null
+ response_format?: string | AgentModelResponseFormatConfig | null
stop?: Array<string> | null
temperature?: number | null
top_p?: number | null
diff --git a/packages/contracts/generated/api/console/snippets/zod.gen.ts b/packages/contracts/generated/api/console/snippets/zod.gen.ts
index 91fc59f61c6..7e48b3e636f 100644
--- a/packages/contracts/generated/api/console/snippets/zod.gen.ts
+++ b/packages/contracts/generated/api/console/snippets/zod.gen.ts
@@ -1324,7 +1324,7 @@ export const zAgentSoulModelSettings = z.object({
frequency_penalty: z.number().nullish(),
max_tokens: z.int().nullish(),
presence_penalty: z.number().nullish(),
- response_format: zAgentModelResponseFormatConfig.nullish(),
+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),
stop: z.array(z.string()).nullish(),
temperature: z.number().nullish(),
top_p: z.number().nullish(),
diff --git a/packages/contracts/generated/api/console/system-features/types.gen.ts b/packages/contracts/generated/api/console/system-features/types.gen.ts
index e36a34b7b90..6fd2007682e 100644
--- a/packages/contracts/generated/api/console/system-features/types.gen.ts
+++ b/packages/contracts/generated/api/console/system-features/types.gen.ts
@@ -5,6 +5,7 @@ export type ClientOptions = {
}
export type SystemFeatureModel = {
+ agent_tracing_api_collection_enabled: boolean
branding: BrandingModel
deployment_edition: DeploymentEdition
enable_app_deploy: boolean
diff --git a/packages/contracts/generated/api/console/system-features/zod.gen.ts b/packages/contracts/generated/api/console/system-features/zod.gen.ts
index 6417c81f20f..a263864a563 100644
--- a/packages/contracts/generated/api/console/system-features/zod.gen.ts
+++ b/packages/contracts/generated/api/console/system-features/zod.gen.ts
@@ -128,6 +128,7 @@ export const zWebAppAuthModel = z.object({
* Non-sensitive bootstrap snapshot exposed before Console or Web authentication.
*/
export const zSystemFeatureModel = z.object({
+ agent_tracing_api_collection_enabled: z.boolean(),
branding: zBrandingModel.default({
application_title: '',
enabled: false,
diff --git a/packages/contracts/generated/api/console/trial-apps/types.gen.ts b/packages/contracts/generated/api/console/trial-apps/types.gen.ts
index 6b001b10387..8cf7b51fca2 100644
--- a/packages/contracts/generated/api/console/trial-apps/types.gen.ts
+++ b/packages/contracts/generated/api/console/trial-apps/types.gen.ts
@@ -1233,7 +1233,7 @@ export type AgentSoulModelSettings = {
frequency_penalty?: number | null
max_tokens?: number | null
presence_penalty?: number | null
- response_format?: AgentModelResponseFormatConfig | null
+ response_format?: string | AgentModelResponseFormatConfig | null
stop?: Array<string> | null
temperature?: number | null
top_p?: number | null
diff --git a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts b/packages/contracts/generated/api/console/trial-apps/zod.gen.ts
index 133b712c9de..8bdadfcfb6a 100644
--- a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts
+++ b/packages/contracts/generated/api/console/trial-apps/zod.gen.ts
@@ -1725,7 +1725,7 @@ export const zAgentSoulModelSettings = z.object({
frequency_penalty: z.number().nullish(),
max_tokens: z.int().nullish(),
presence_penalty: z.number().nullish(),
- response_format: zAgentModelResponseFormatConfig.nullish(),
+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),
stop: z.array(z.string()).nullish(),
temperature: z.number().nullish(),
top_p: z.number().nullish(),
diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.ts
index b7ed9dfc696..444a1c6a7db 100644
--- a/packages/contracts/generated/api/web/types.gen.ts
+++ b/packages/contracts/generated/api/web/types.gen.ts
@@ -1041,6 +1041,7 @@ export type SuggestedQuestionsResponse = {
}
export type SystemFeatureModel = {
+ agent_tracing_api_collection_enabled: boolean
branding: BrandingModel
deployment_edition: DeploymentEdition
enable_app_deploy: boolean
diff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.ts
index 30c3c4729c8..e4245e95da9 100644
--- a/packages/contracts/generated/api/web/zod.gen.ts
+++ b/packages/contracts/generated/api/web/zod.gen.ts
@@ -1666,6 +1666,7 @@ export const zWebAppAuthModel = z.object({
* Non-sensitive bootstrap snapshot exposed before Console or Web authentication.
*/
export const zSystemFeatureModel = z.object({
+ agent_tracing_api_collection_enabled: z.boolean(),
branding: zBrandingModel.default({
application_title: '',
enabled: false,
diff --git a/packages/contracts/network-access-current-ip.test.ts b/packages/contracts/network-access-current-ip.test.ts
index b68d06a4560..8673aeee186 100644
--- a/packages/contracts/network-access-current-ip.test.ts
+++ b/packages/contracts/network-access-current-ip.test.ts
@@ -1,10 +1,10 @@
import { describe, expect, it } from 'vite-plus/test'
-import { currentIp, workspaces } from './generated/api/console/workspaces/orpc.gen'
+import { currentIp, workspaces } from './generated/api/console/workspaces/orpc.gen.ts'
import {
zGetWorkspacesCurrentNetworkAccessGroupsByGroupIdCheckCurrentIpPath,
zGetWorkspacesCurrentNetworkAccessGroupsByGroupIdCheckCurrentIpResponse,
zGetWorkspacesCurrentNetworkAccessGroupsCurrentIpResponse,
-} from './generated/api/console/workspaces/zod.gen'
+} from './generated/api/console/workspaces/zod.gen.ts'
describe('generated network access current-IP contract', () => {
it('exposes the policy-independent operation alongside the group preflight', () => {
diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx
index f584d8fccd8..86066f3c79d 100644
--- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx
+++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx
@@ -196,7 +196,7 @@ describe('Tracing overview panel', () => {
act(() =>
client.setQueryData(
consoleQuery.apps.byAppId.trace.get.queryKey({ input: { params: { app_id: 'app-1' } } }),
- { enabled: false, tracing_provider: null },
+ { enabled: false, tracing_provider: null, capture_content: false, capture_agent: true },
),
)
const user = userEvent.setup()
@@ -232,7 +232,12 @@ describe('Tracing overview panel', () => {
act(() =>
client.setQueryData(
consoleQuery.apps.byAppId.trace.get.queryKey({ input: { params: { app_id: 'app-1' } } }),
- { enabled: true, tracing_provider: 'langsmith' },
+ {
+ enabled: true,
+ tracing_provider: 'langsmith',
+ capture_content: false,
+ capture_agent: true,
+ },
),
)
const user = await openConfiguredPanel()
diff --git a/web/test/console/system-features.ts b/web/test/console/system-features.ts
index dac07c20d21..0ebe648312a 100644
--- a/web/test/console/system-features.ts
+++ b/web/test/console/system-features.ts
@@ -15,6 +15,7 @@ export type DeepPartial<T> =
: T
const baseSystemFeatures = {
+ agent_tracing_api_collection_enabled: false,
deployment_edition: 'COMMUNITY',
enable_app_deploy: false,
sso_enforced_for_signin: false,