Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
feat(tracing): export Agent app and Workflow Agent execution trees (#44144)
zyssyz1230a620ddd95c2
diff --git a/api/clients/agent_backend/request_builder.py b/api/clients/agent_backend/request_builder.pyindex 75e8f060002..84584faafa8 100644--- a/api/clients/agent_backend/request_builder.py+++ b/api/clients/agent_backend/request_builder.py@@ -138,7 +138,8 @@ _AGENT_MODEL_SETTINGS_PASSTHROUGH_KEYS = ("frequency_penalty","max_tokens",)-_AGENT_MODEL_SETTINGS_KNOWN_KEYS = frozenset({*_AGENT_MODEL_SETTINGS_PASSTHROUGH_KEYS, "stop", "response_format"})+# response_format uses extra_body so both plugin strings and saved native objects reach the plugin unchanged.+_AGENT_MODEL_SETTINGS_KNOWN_KEYS = frozenset({*_AGENT_MODEL_SETTINGS_PASSTHROUGH_KEYS, "stop"})def _agent_model_settings(settings: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None:@@ -281,7 +282,9 @@ class AgentBackendRunRequestBuilder:),RunLayerSpec(name=DIFY_EXECUTION_CONTEXT_LAYER_ID,- config=(run_input.execution_context).model_dump(mode="json"),+ config=run_input.execution_context.model_dump(+ mode="json", exclude={"tracing"} if run_input.execution_context.tracing is None else set()+ ),),])@@ -415,7 +418,9 @@ class AgentBackendRunRequestBuilder:),RunLayerSpec(name=DIFY_EXECUTION_CONTEXT_LAYER_ID,- config=(run_input.execution_context).model_dump(mode="json"),+ config=run_input.execution_context.model_dump(+ mode="json", exclude={"tracing"} if run_input.execution_context.tracing is None else set()+ ),),])@@ -531,7 +536,11 @@ def redact_for_agent_backend_log(value: object) -> object:is_multimodal_file = value.get("type") == "image" and "filename" in value and "mime_type" in valuefor key, item in value.items():key_text = str(key).lower()- if any(part in key_text for part in _SENSITIVE_KEY_PARTS) or (is_multimodal_file and key_text == "url"):+ if (+ any(part in key_text for part in _SENSITIVE_KEY_PARTS)+ or (is_multimodal_file and key_text == "url")+ or (key_text == "tracing" and item is not None)+ ):redacted[key] = "[REDACTED]"else:redacted[key] = redact_for_agent_backend_log(item)diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.pyindex 54473bf9a1f..2b3f972c508 100644--- a/api/configs/feature/__init__.py+++ b/api/configs/feature/__init__.py@@ -1416,6 +1416,31 @@ class MultiModalTransferConfig(BaseSettings):class OpsTraceConfig(BaseSettings):+ OPS_AGENT_TRACE_ENABLED: bool = Field(+ default=False, description="Enable application-scoped Agent tracing collection and delivery."+ )+ OPS_AGENT_TRACE_MAX_SPANS: PositiveInt = Field(+ default=4096, description="Maximum runtime spans per application trace."+ )+ OPS_AGENT_TRACE_MAX_BYTES: PositiveInt = Field(+ default=16777216, description="Maximum runtime payload bytes per application trace."+ )+ OPS_AGENT_TRACE_BATCH_MAX_BYTES: PositiveInt = Field(+ default=2097152, description="Maximum inner tracing request body bytes."+ )+ OPS_AGENT_TRACE_COLLECTION_TIMEOUT: PositiveInt = Field(+ default=180, description="Seconds after terminal execution to wait for final Agent batches."+ )+ OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS: PositiveInt = Field(+ default=120, description="Export lease before unconfirmed delivery becomes uncertain."+ )+ OPS_AGENT_TRACE_MAX_ATTEMPTS: PositiveInt = Field(+ default=8, description="Attempts for known retryable trace export failures."+ )+ OPS_AGENT_TRACE_RETENTION_DAYS: PositiveInt = Field(+ default=7, description="Retention of trace payloads and encrypted destination snapshots."+ )+OPS_TRACE_UNIFIED_ENABLED: bool = Field(description="Enable unified ops tracing for providers registered in the unified registry.",default=False,@@ -1456,6 +1481,10 @@ class CeleryBeatConfig(BaseSettings):class CeleryScheduleTasksConfig(BaseSettings):+ ENABLE_APPLICATION_TRACE_RECONCILE_TASK: bool = Field(+ description="Enable periodic application trace recovery and retention cleanup when Agent tracing is enabled",+ default=True,+ )ENABLE_CONVERSATION_CLEANUP_TASK: bool = Field(description="Enable periodic recovery of soft-deleted conversation cleanup",default=True,diff --git a/api/controllers/API_SCHEMA_GUIDE.md b/api/controllers/API_SCHEMA_GUIDE.mdindex 1ef002cfbfe..81f556cf63a 100644--- a/api/controllers/API_SCHEMA_GUIDE.md+++ b/api/controllers/API_SCHEMA_GUIDE.md@@ -19,7 +19,7 @@ requested during root SSR, so treat this response as a minimal public bootstrapconfiguration endpoint, a feature registry, or a mirror of environment and Enterprise settings. Existing fields arelegacy inventory and do not establish precedent for new fields.-A new field is eligible only when all of the following are true:+Apart from the explicit Agent tracing exception below, a new field is eligible only when all of the following are true:1. Both Console and Web have named production consumers for the field.2. Both consumers need the value before authentication and tenant/workspace bootstrap to render initial state or@@ -44,9 +44,27 @@ Route excluded values to their actual owner:and failure cost explicitly.- Load large, slow, or page-specific data lazily through a consumer-owned query.+### Agent tracing deployment discovery exception++`agent_tracing_api_collection_enabled` is an explicit deployment-discovery exception to the pre-authentication+consumer requirements above. Both public Console and Web `/system-features` responses expose the same required+boolean from `OPS_AGENT_TRACE_ENABLED`; the authenticated Console tracing configuration and Workflow inheritance+surfaces consume it independently of workspace, application, or provider configuration. It has no schema default:+a missing field must not be mistaken for a confirmed disabled deployment.++This field reports only the API collection gate. It grants no configuration, credential, or log access and says+nothing about Agent runtime, worker, or provider health, export acknowledgement, or platform queryability. The+exception does not admit other environment settings or integration toggles, or broaden this response into a+configuration registry. Keep workspace policy and entitlements in their authenticated owners.++The existing Console and Web bootstrap requests gain one boolean and no additional configuration or provider I/O.+Their authentication, SSR request timing, and cache ownership are unchanged; consumers must retain an unknown+state when the response is missing or fails rather than infer a disabled or healthy deployment.+Every pull request that adds a System Features field must:-- Name both production consumer paths and explain why they require the value before authentication.+- Name both production consumer paths and explain why they require the value before authentication. For the named+ Agent tracing exception, document the authenticated Console consumers and intentional shared Web exposure instead.- Document the root SSR request, payload, caching, and failure-mode impact.- Update the Pydantic owner, regenerate OpenAPI Markdown and TypeScript/Zod contracts, and update shared fixtures.- Add Console and Web schema regression coverage. Do not hand-edit generated contracts or add compatibility defaults.diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.pyindex 43ff40c5098..47956be121e 100644--- a/api/controllers/console/__init__.py+++ b/api/controllers/console/__init__.py@@ -153,6 +153,8 @@ def _register_controllers() -> tuple[ModuleType, ...]:)from .agent import composer as agent_composerfrom .agent import roster as agent_roster+ from .agent import tracing as agent_tracing+ from .agent import tracing_runs as agent_tracing_runs# Import app controllersfrom .app import (@@ -270,6 +272,8 @@ def _register_controllers() -> tuple[ModuleType, ...]:workflow_run_archive,agent_composer,agent_roster,+ agent_tracing,+ agent_tracing_runs,advanced_prompt_template,agent,agent_app_access,@@ -365,6 +369,8 @@ def _register_controllers() -> tuple[ModuleType, ...]:workflow_run_archive,agent_composer,agent_roster,+ agent_tracing,+ agent_tracing_runs,advanced_prompt_template,agent,agent_app_access,@@ -458,6 +464,8 @@ __all__ = ["agent_config_inspector","agent_providers","agent_roster",+ "agent_tracing",+ "agent_tracing_runs","annotation","api","apikey",diff --git a/api/controllers/console/agent/tracing.py b/api/controllers/console/agent/tracing.pynew file mode 100644index 00000000000..32c775991c5--- /dev/null+++ b/api/controllers/console/agent/tracing.py@@ -0,0 +1,248 @@+"""Agent tracing settings address Agent resources, never hidden inline Apps."""++from collections.abc import Callable+from typing import Literal+from uuid import UUID++from flask import request+from flask_restx import Resource+from pydantic import Field+from werkzeug.exceptions import NotFound++from configs import dify_config+from controllers.common.rbac import AgentId, PlainApp, RBACCheck+from controllers.common.schema import query_params_from_model, register_response_schema_models+from controllers.console import console_ns+from controllers.console.app.app import AppTracePayload+from controllers.console.app.error import (+ AppNotFoundError,+ InvalidTracingConfigError,+ TracingConfigAlreadyExistsError,+ TracingConfigConflictError,+ TracingConfigNotFoundError,+ TracingConfigProcessingError,+ TracingConfigVerificationFailedError,+ TracingProviderUnavailableError,+ UnsupportedTracingProviderError,+)+from controllers.console.app.ops_trace import TraceAppConfigResponse, TraceConfigPayload, TraceProviderQuery+from controllers.console.flask_admission import console_account_admission+from controllers.console.wraps import RBACPermission+from extensions.ext_application_services import application_services+from fields.app_fields import AppTraceResponse+from fields.base import ResponseModel+from libs.external_api import error_responses+from libs.helper import dump_response+from machinery.context import RequestContext+from models.account import TenantAccountRole+from services.app.agent_app_contracts import AgentAppNotFoundError+from services.app.console_service import ConsoleAppNotFoundError+from services.app_tracing_config_service import (+ AppTraceSettings,+ AppTracingConfigAlreadyExistsError,+ AppTracingConfigAppNotFoundError,+ AppTracingConfigConflictError,+ AppTracingConfigInvalidConfigurationError,+ AppTracingConfigInvalidProviderError,+ AppTracingConfigNotFoundError,+ AppTracingConfigProcessingError,+ AppTracingConfigProviderUnavailableError,+ AppTracingConfigVerificationFailedError,+)++_EDIT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR})+++class TraceProviderSupportResponse(ResponseModel):+ """Adapter support declarations, not successful-delivery or platform-verification results."""++ provider: str+ agent_spans: bool+ session: bool+ native_usage: bool+ native_cost: bool+ trace_url: bool+ span_url: bool+ export_ack: Literal["accepted"] = Field(description="Exporter acknowledgement does not confirm query visibility")+++class TraceProviderCapabilitiesResponse(TraceProviderSupportResponse):+ api_collection_enabled: bool = Field(+ description="Current API collection gate only; does not probe Agent runtime or confirm delivery"+ )+++register_response_schema_models(console_ns, TraceProviderSupportResponse, TraceProviderCapabilitiesResponse)+++def _tracing_call[T](call: Callable[[], T]) -> T:+ """Translate configuration-domain failures without returning credentials or provider errors."""+ try:+ return call()+ except (AgentAppNotFoundError, ConsoleAppNotFoundError) as error:+ raise NotFound("Agent not found") from error+ except AppTracingConfigAppNotFoundError as error:+ raise AppNotFoundError() from error+ except AppTracingConfigAlreadyExistsError as error:+ raise TracingConfigAlreadyExistsError() from error+ except AppTracingConfigConflictError as error:+ raise TracingConfigConflictError() from error+ except AppTracingConfigNotFoundError as error:+ raise TracingConfigNotFoundError() from error+ except AppTracingConfigInvalidProviderError as error:+ raise UnsupportedTracingProviderError() from error+ except AppTracingConfigProviderUnavailableError as error:+ raise TracingProviderUnavailableError() from error+ except AppTracingConfigInvalidConfigurationError as error:+ raise InvalidTracingConfigError() from error+ except AppTracingConfigVerificationFailedError as error:+ raise TracingConfigVerificationFailedError() from error+ except AppTracingConfigProcessingError as error:+ raise TracingConfigProcessingError() from error+++@console_ns.route("/agent/<uuid:agent_id>/trace")+class AgentTraceApi(Resource):+ @console_ns.doc("get_agent_trace")+ @console_ns.response(200, "Agent tracing settings", console_ns.models[AppTraceResponse.__name__])+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.AGENT_MONITOR, AgentId())])+ def get(self, context: RequestContext, agent_id: UUID):+ settings = _tracing_call(lambda: application_services().agent_tracing_configs.get_trace(context, str(agent_id)))+ return dump_response(AppTraceResponse, settings)++ @error_responses(+ TracingProviderUnavailableError,+ TracingConfigNotFoundError,+ TracingConfigConflictError,+ InvalidTracingConfigError,+ TracingConfigProcessingError,+ UnsupportedTracingProviderError,+ )+ @console_ns.doc("update_agent_trace")+ @console_ns.expect(console_ns.models[AppTracePayload.__name__])+ @console_ns.response(200, "Updated Agent tracing settings", console_ns.models[AppTraceResponse.__name__])+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])+ def post(self, context: RequestContext, agent_id: UUID):+ payload = AppTracePayload.model_validate(console_ns.payload or {})+ settings = AppTraceSettings(**payload.model_dump())+ _tracing_call(lambda: application_services().agent_tracing_configs.set_trace(context, str(agent_id), settings))+ return dump_response(AppTraceResponse, settings)+++@console_ns.route("/agent/<uuid:agent_id>/trace-config")+class AgentTraceConfigApi(Resource):+ @error_responses(+ AppNotFoundError, TracingConfigProcessingError, TracingProviderUnavailableError, UnsupportedTracingProviderError+ )+ @console_ns.doc("get_agent_trace_config", params=query_params_from_model(TraceProviderQuery))+ @console_ns.response(200, "Masked Agent provider configuration", console_ns.models[TraceAppConfigResponse.__name__])+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])+ def get(self, context: RequestContext, agent_id: UUID):+ query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))+ record = _tracing_call(+ lambda: application_services().agent_tracing_configs.get(context, str(agent_id), query.tracing_provider)+ )+ if record is None:+ return dump_response(TraceAppConfigResponse, {"has_not_configured": True}, exclude_none=True)+ return dump_response(TraceAppConfigResponse, TraceAppConfigResponse.from_record(record), exclude_none=True)++ @error_responses(+ AppNotFoundError,+ InvalidTracingConfigError,+ TracingConfigAlreadyExistsError,+ TracingConfigProcessingError,+ TracingConfigVerificationFailedError,+ TracingProviderUnavailableError,+ UnsupportedTracingProviderError,+ )+ @console_ns.doc("create_agent_trace_config")+ @console_ns.expect(console_ns.models[TraceConfigPayload.__name__])+ @console_ns.response(201, "Agent provider configured", console_ns.models[TraceAppConfigResponse.__name__])+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])+ def post(self, context: RequestContext, agent_id: UUID):+ payload = TraceConfigPayload.model_validate(console_ns.payload or {})+ _tracing_call(+ lambda: application_services().agent_tracing_configs.create(+ context, str(agent_id), payload.tracing_provider, payload.tracing_config, activate=payload.activate+ )+ )+ return dump_response(TraceAppConfigResponse, {"result": "success"}, exclude_none=True), 201++ @error_responses(+ AppNotFoundError,+ InvalidTracingConfigError,+ TracingConfigNotFoundError,+ TracingConfigConflictError,+ TracingConfigProcessingError,+ TracingConfigVerificationFailedError,+ TracingProviderUnavailableError,+ UnsupportedTracingProviderError,+ )+ @console_ns.doc("update_agent_trace_config")+ @console_ns.expect(console_ns.models[TraceConfigPayload.__name__])+ @console_ns.response(200, "Agent provider updated", console_ns.models[TraceAppConfigResponse.__name__])+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])+ def patch(self, context: RequestContext, agent_id: UUID):+ payload = TraceConfigPayload.model_validate(console_ns.payload or {})+ _tracing_call(+ lambda: application_services().agent_tracing_configs.update(+ context, str(agent_id), payload.tracing_provider, payload.tracing_config, activate=payload.activate+ )+ )+ return dump_response(TraceAppConfigResponse, {"result": "success"}, exclude_none=True)++ @error_responses(+ AppNotFoundError, TracingConfigNotFoundError, TracingConfigProcessingError, UnsupportedTracingProviderError+ )+ @console_ns.doc("delete_agent_trace_config", params=query_params_from_model(TraceProviderQuery))+ @console_ns.response(204, "Agent provider configuration deleted")+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])+ def delete(self, context: RequestContext, agent_id: UUID):+ query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))+ _tracing_call(+ lambda: application_services().agent_tracing_configs.delete(context, str(agent_id), query.tracing_provider)+ )+ return "", 204+++def _provider_capabilities():+ from services.tracing.provider_dispatch import provider_capabilities++ query = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))+ try:+ capabilities = provider_capabilities(query.tracing_provider)+ except ValueError as error:+ raise UnsupportedTracingProviderError() from error+ return dump_response(+ TraceProviderCapabilitiesResponse,+ {**capabilities, "api_collection_enabled": dify_config.OPS_AGENT_TRACE_ENABLED},+ )+++@console_ns.route("/agent/<uuid:agent_id>/trace-capabilities")+class AgentTraceCapabilitiesApi(Resource):+ @error_responses(UnsupportedTracingProviderError)+ @console_ns.doc("get_agent_trace_capabilities", params=query_params_from_model(TraceProviderQuery))+ @console_ns.response(+ 200, "Provider support declarations", console_ns.models[TraceProviderCapabilitiesResponse.__name__]+ )+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.AGENT_MONITOR, AgentId())])+ def get(self, context: RequestContext, agent_id: UUID):+ _tracing_call(lambda: application_services().agent_tracing_configs.get_trace(context, str(agent_id)))+ return _provider_capabilities()+++@console_ns.route("/apps/<uuid:app_id>/trace-capabilities")+class AppTraceCapabilitiesApi(Resource):+ @error_responses(UnsupportedTracingProviderError)+ @console_ns.doc("get_app_trace_capabilities", params=query_params_from_model(TraceProviderQuery))+ @console_ns.response(+ 200, "Provider support declarations", console_ns.models[TraceProviderCapabilitiesResponse.__name__]+ )+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.APP_MONITOR, PlainApp())])+ def get(self, context: RequestContext, app_id: UUID):+ try:+ application_services().apps.console.get_trace(context, str(app_id))+ except ConsoleAppNotFoundError as error:+ raise NotFound("App not found") from error+ return _provider_capabilities()diff --git a/api/controllers/console/agent/tracing_runs.py b/api/controllers/console/agent/tracing_runs.pynew file mode 100644index 00000000000..60149273af5--- /dev/null+++ b/api/controllers/console/agent/tracing_runs.py@@ -0,0 +1,246 @@+"""Authorized trace delivery status and explicit retries of frozen destinations."""++from collections.abc import Callable+from datetime import UTC, datetime+from typing import Literal+from uuid import UUID++from flask import request+from flask_restx import Resource+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator+from werkzeug.exceptions import NotFound++from controllers.common.rbac import AgentId, PlainApp, RBACCheck+from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models+from controllers.console import console_ns+from controllers.console.agent.tracing import TraceProviderSupportResponse+from controllers.console.flask_admission import console_account_admission+from controllers.console.wraps import RBACPermission+from extensions.ext_application_services import application_services+from fields.base import ResponseModel+from libs.exception import BaseHTTPException+from libs.external_api import error_responses+from libs.helper import dump_response+from machinery.context import RequestContext+from models.account import TenantAccountRole+from repositories.application_trace_repository import TraceCollectionError+from services.app.agent_app_contracts import AgentAppNotFoundError+from services.tracing.service import application_tracing_service++type TraceSourceKind = Literal["message", "workflow", "node"]+_source_kind_adapter = TypeAdapter(TraceSourceKind)+_EDIT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR})+++class TraceRunQuery(BaseModel):+ node_execution_id: UUID | None = Field(default=None, description="Select an Agent node invocation within this run")+++class TraceRunRetryPayload(BaseModel):+ model_config = ConfigDict(extra="forbid")++ acknowledge_possible_duplicates: bool = Field(+ default=False,+ strict=True,+ description="Acknowledge possible duplicate records when retrying an uncertain export",+ )+++class TraceRunStatusResponse(ResponseModel):+ status: Literal["not_collected", "collecting", "ready", "exporting", "retrying", "accepted", "failed", "uncertain"]+ source_kind: TraceSourceKind+ source_id: str+ provider: str | None = None+ trace_id: str | None = None+ trace_url: str | None = None+ span_url: str | None = None+ project_url: str | None = Field(+ default=None,+ description="Frozen destination project URL; null when unavailable, never resolved from current config",+ )+ config_revision: str | None = Field(default=None, description="Frozen destination configuration revision")+ capture_content: bool | None = None+ capture_agent: bool | None = None+ span_count: int | None = None+ attempts: int | None = None+ error_code: str | None = None+ accepted_at: datetime | None = Field(+ default=None, description="Ingestion acknowledgement time, not query visibility"+ )+ queryable: bool | None = Field(default=None, description="Null when third-party query visibility is not verified")+ capabilities: TraceProviderSupportResponse | None = None++ @field_validator("accepted_at")+ @classmethod+ def serialize_database_time_as_utc(cls, value: datetime | None) -> datetime | None:+ # Database timestamps are naive UTC. Emit a timezone so the generated+ # ISO datetime contract and browser parsing agree on the same instant.+ if value is None:+ return None+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)+++class TraceSourceNotFoundError(BaseHTTPException):+ error_code = "tracing_source_not_found"+ code = 404+ description = "The tracing source does not exist in this application."+++class TraceNodeNotFoundError(BaseHTTPException):+ error_code = "tracing_node_not_found"+ code = 404+ description = "The Agent node invocation does not belong to this trace."+++class TraceRunNotFoundError(BaseHTTPException):+ error_code = "tracing_run_not_found"+ code = 404+ description = "No collected trace is available to retry."+++class TraceAlreadyAcceptedError(BaseHTTPException):+ error_code = "tracing_already_accepted"+ code = 409+ description = "An accepted trace cannot be exported again."+++class TraceRetryNotAllowedError(BaseHTTPException):+ error_code = "tracing_retry_not_allowed"+ code = 409+ description = "This trace is not in a retryable state."+++class TraceDuplicateAcknowledgementRequiredError(BaseHTTPException):+ error_code = "tracing_duplicate_acknowledgement_required"+ code = 409+ description = "Acknowledge possible duplicates before retrying an uncertain export."+++_TRACE_ERRORS: dict[str, type[BaseHTTPException]] = {+ cls.error_code: cls+ for cls in (+ TraceSourceNotFoundError,+ TraceNodeNotFoundError,+ TraceRunNotFoundError,+ TraceAlreadyAcceptedError,+ TraceRetryNotAllowedError,+ TraceDuplicateAcknowledgementRequiredError,+ )+}++register_schema_models(console_ns, TraceRunQuery, TraceRunRetryPayload)+register_response_schema_models(console_ns, TraceRunStatusResponse)+++def _run_call[T](call: Callable[[], T]) -> T:+ try:+ return call()+ except AgentAppNotFoundError as error:+ raise NotFound("Agent not found") from error+ except TraceCollectionError as error:+ if error_type := _TRACE_ERRORS.get(error.code):+ raise error_type() from error+ raise+++def _query_node_id() -> str | None:+ query = TraceRunQuery.model_validate(request.args.to_dict(flat=True))+ return str(query.node_execution_id) if query.node_execution_id else None+++def _agent_app_id(context: RequestContext, agent_id: UUID) -> str:+ return _run_call(lambda: application_services().agent_tracing_configs.resolve_app_id(context, str(agent_id)))+++def _status(context: RequestContext, app_id: str, kind: TraceSourceKind, source_id: str):+ node_id = _query_node_id()+ result = _run_call(+ lambda: application_tracing_service().status(+ tenant_id=context.active_workspace_id,+ app_id=app_id,+ source_kind=kind,+ source_id=source_id,+ node_execution_id=node_id,+ )+ )+ return dump_response(TraceRunStatusResponse, result)+++def _retry(context: RequestContext, app_id: str, kind: TraceSourceKind, source_id: str):+ node_id = _query_node_id()+ payload = TraceRunRetryPayload.model_validate(console_ns.payload or {})+ result = _run_call(+ lambda: application_tracing_service().retry(+ tenant_id=context.active_workspace_id,+ app_id=app_id,+ source_kind=kind,+ source_id=source_id,+ node_execution_id=node_id,+ acknowledge_possible_duplicates=payload.acknowledge_possible_duplicates,+ )+ )+ return dump_response(TraceRunStatusResponse, result)+++@console_ns.route("/agent/<uuid:agent_id>/trace-runs/<uuid:message_id>")+class AgentTraceRunApi(Resource):+ @error_responses(TraceSourceNotFoundError, TraceNodeNotFoundError)+ @console_ns.doc("get_agent_trace_run", params=query_params_from_model(TraceRunQuery))+ @console_ns.response(+ 200, "Frozen trace destination and export state", console_ns.models[TraceRunStatusResponse.__name__]+ )+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.AGENT_LOG_MANAGE, AgentId())])+ def get(self, context: RequestContext, agent_id: UUID, message_id: UUID):+ return _status(context, _agent_app_id(context, agent_id), "message", str(message_id))+++@console_ns.route("/agent/<uuid:agent_id>/trace-runs/<uuid:message_id>/retry")+class AgentTraceRunRetryApi(Resource):+ @error_responses(*_TRACE_ERRORS.values())+ @console_ns.doc("retry_agent_trace_run", params=query_params_from_model(TraceRunQuery))+ @console_ns.expect(console_ns.models[TraceRunRetryPayload.__name__])+ @console_ns.response(+ 200, "Retry state for the same frozen destination", console_ns.models[TraceRunStatusResponse.__name__]+ )+ @console_account_admission(allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.AGENT_EDIT, AgentId())])+ def post(self, context: RequestContext, agent_id: UUID, message_id: UUID):+ return _retry(context, _agent_app_id(context, agent_id), "message", str(message_id))+++@console_ns.route("/apps/<uuid:app_id>/trace-runs/<string:source_kind>/<uuid:source_id>")+class AppTraceRunApi(Resource):+ @error_responses(TraceSourceNotFoundError, TraceNodeNotFoundError)+ @console_ns.doc(+ "get_app_trace_run",+ params={+ **query_params_from_model(TraceRunQuery),+ "source_kind": {"enum": ["message", "workflow", "node"]},+ },+ )+ @console_ns.response(+ 200, "Frozen trace destination and export state", console_ns.models[TraceRunStatusResponse.__name__]+ )+ @console_account_admission(rbac_checks=[RBACCheck(RBACPermission.APP_LOG_AND_ANNOTATION, PlainApp())])+ def get(self, context: RequestContext, app_id: UUID, source_kind: str, source_id: UUID):+ return _status(context, str(app_id), _source_kind_adapter.validate_python(source_kind), str(source_id))+++@console_ns.route("/apps/<uuid:app_id>/trace-runs/<string:source_kind>/<uuid:source_id>/retry")+class AppTraceRunRetryApi(Resource):+ @error_responses(*_TRACE_ERRORS.values())+ @console_ns.doc(+ "retry_app_trace_run",+ params={+ **query_params_from_model(TraceRunQuery),+ "source_kind": {"enum": ["message", "workflow", "node"]},+ },+ )+ @console_ns.expect(console_ns.models[TraceRunRetryPayload.__name__])+ @console_ns.response(+ 200, "Retry state for the same frozen destination", console_ns.models[TraceRunStatusResponse.__name__]+ )+ @console_account_admission(+ allowed_roles=_EDIT_ROLES, rbac_checks=[RBACCheck(RBACPermission.APP_TRACING_CONFIG, PlainApp())]+ )+ def post(self, context: RequestContext, app_id: UUID, source_kind: str, source_id: UUID):+ return _retry(context, str(app_id), _source_kind_adapter.validate_python(source_kind), str(source_id))diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.pyindex 8e3faec0f3f..cb2f3e2e0c9 100644--- a/api/controllers/console/app/app.py+++ b/api/controllers/console/app/app.py@@ -19,7 +19,15 @@ from controllers.common.schema import (register_schema_models,)from controllers.console import console_ns-from controllers.console.app.error import AppNotFoundError, TracingProviderUnavailableError+from controllers.console.app.error import (+ AppNotFoundError,+ InvalidTracingConfigError,+ TracingConfigConflictError,+ TracingConfigNotFoundError,+ TracingConfigProcessingError,+ TracingProviderUnavailableError,+ UnsupportedTracingProviderError,+)from controllers.console.flask_admission import console_account_admissionfrom controllers.console.workspace.models import LoadBalancingPayloadfrom controllers.console.wraps import (@@ -64,7 +72,13 @@ from services.app.console_service import (InvalidAppExportError,)from services.app_tracing_config_service import (+ AppTraceSettings,+ AppTracingConfigAppNotFoundError,+ AppTracingConfigConflictError,+ AppTracingConfigInvalidConfigurationError,AppTracingConfigInvalidProviderError,+ AppTracingConfigNotFoundError,+ AppTracingConfigProcessingError,AppTracingConfigProviderUnavailableError,)from services.entities.app_entities import (@@ -72,7 +86,6 @@ from services.entities.app_entities import (AppListParams,AppListSortBy,AppRecord,- AppTraceSettings,CopyAppParams,CreateAppParams,StarredAppListParams,@@ -232,13 +245,14 @@ class AppApiStatusPayload(BaseModel):class AppTracePayload(BaseModel):enabled: bool = Field(..., description="Enable or disable tracing")tracing_provider: str | None = Field(default=None, description="Tracing provider")+ capture_content: bool = Field(default=False, description="Include prompt, output and tool content in new traces")+ capture_agent: bool = Field(default=True, description="Capture Agent internals when deployment support is enabled")- @field_validator("tracing_provider")- @classmethod- def validate_tracing_provider(cls, value: str | None, info) -> str | None:- if info.data.get("enabled") and not value:+ @model_validator(mode="after")+ def validate_tracing_provider(self) -> "AppTracePayload":+ if self.enabled and not self.tracing_provider:raise ValueError("tracing_provider is required when enabled is True")- return value+ return selfregister_enum_models(console_ns, RetrievalMethod, WorkflowExecutionStatus, DatasetPermissionEnum)@@ -657,7 +671,15 @@ class AppTraceApi(AppResource):def get(self, context: RequestContext, app_id: uuid.UUID):return dump_response(AppTraceResponse, application_services().apps.console.get_trace(context, str(app_id)))- @error_responses(TracingProviderUnavailableError)+ @error_responses(+ AppNotFoundError,+ TracingProviderUnavailableError,+ TracingConfigNotFoundError,+ TracingConfigConflictError,+ TracingConfigProcessingError,+ InvalidTracingConfigError,+ UnsupportedTracingProviderError,+ )@console_ns.doc("update_app_trace")@console_ns.doc(description="Update app tracing configuration")@console_ns.doc(params={"app_id": "Application ID"})@@ -679,6 +701,18 @@ class AppTraceApi(AppResource):str(app_id),AppTraceSettings(**payload.model_dump()),)+ except AppTracingConfigAppNotFoundError as error:+ raise AppNotFoundError() from error+ except AppTracingConfigConflictError as error:+ raise TracingConfigConflictError() from error+ except AppTracingConfigNotFoundError as error:+ raise TracingConfigNotFoundError() from error+ except AppTracingConfigProcessingError as error:+ raise TracingConfigProcessingError() from error+ except AppTracingConfigInvalidConfigurationError as error:+ raise InvalidTracingConfigError() from error+ except AppTracingConfigInvalidProviderError as error:+ raise UnsupportedTracingProviderError() from errorexcept AppTracingConfigProviderUnavailableError as error:raise TracingProviderUnavailableError() from errorreturn SimpleResultResponse(result="success").model_dump(mode="json")diff --git a/api/controllers/console/app/error.py b/api/controllers/console/app/error.pyindex adb0fbcf366..22e8bc7ed2a 100644--- a/api/controllers/console/app/error.py+++ b/api/controllers/console/app/error.py@@ -187,3 +187,9 @@ class AgentNotBoundError(BaseHTTPException):error_code = "agent_not_bound"description = "no agent is bound for this app/node"code = 400+++class TracingConfigConflictError(BaseHTTPException):+ error_code = "tracing_config_conflict"+ description = "Tracing configuration changed. Refresh and try again."+ code = 409diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.pyindex 6d41deb4d1f..ce1463d8d28 100644--- a/api/controllers/console/app/ops_trace.py+++ b/api/controllers/console/app/ops_trace.py@@ -10,6 +10,7 @@ from controllers.console.app.error import (AppNotFoundError,InvalidTracingConfigError,TracingConfigAlreadyExistsError,+ TracingConfigConflictError,TracingConfigNotFoundError,TracingConfigProcessingError,TracingConfigVerificationFailedError,@@ -30,6 +31,7 @@ from models.account import TenantAccountRolefrom services.app_tracing_config_service import (AppTracingConfigAlreadyExistsError,AppTracingConfigAppNotFoundError,+ AppTracingConfigConflictError,AppTracingConfigInvalidConfigurationError,AppTracingConfigInvalidProviderError,AppTracingConfigNotFoundError,@@ -53,6 +55,10 @@ class TraceProviderQuery(BaseModel):class TraceConfigPayload(BaseModel):+ activate: bool = Field(+ default=False,+ description="Save, select this provider, and enable tracing atomically; preserve capture settings",+ )tracing_provider: str = Field(..., description="Tracing provider name")tracing_config: dict[str, str | None] = Field(...,@@ -186,6 +192,7 @@ class TraceAppConfigApi(Resource):app_id=str(app_id),tracing_provider=req_data.tracing_provider,tracing_config=req_data.tracing_config,+ activate=req_data.activate,)except AppTracingConfigAppNotFoundError as error:raise AppNotFoundError() from error@@ -210,6 +217,7 @@ class TraceAppConfigApi(Resource):AppNotFoundError,InvalidTracingConfigError,TracingConfigNotFoundError,+ TracingConfigConflictError,TracingConfigProcessingError,TracingConfigVerificationFailedError,TracingProviderUnavailableError,@@ -246,9 +254,12 @@ class TraceAppConfigApi(Resource):app_id=str(app_id),tracing_provider=req_data.tracing_provider,tracing_config=req_data.tracing_config,+ activate=req_data.activate,)except AppTracingConfigAppNotFoundError as error:raise AppNotFoundError() from error+ except AppTracingConfigConflictError as error:+ raise TracingConfigConflictError() from errorexcept AppTracingConfigNotFoundError as error:raise TracingConfigNotFoundError() from errorexcept AppTracingConfigInvalidProviderError as error:diff --git a/api/controllers/inner_api/__init__.py b/api/controllers/inner_api/__init__.pyindex cc1752198fd..bc0610ee212 100644--- a/api/controllers/inner_api/__init__.py+++ b/api/controllers/inner_api/__init__.py@@ -31,7 +31,7 @@ def _register_controllers() -> tuple[ModuleType, ...]:"""Load route modules after their namespace exists."""from . import mail, runtime_credentials- from .agent import files, llm, runtime_usage, tools+ from .agent import files, llm, runtime_usage, tools, tracingfrom .app import app, dsl, file_grantsfrom .knowledge import retrievalfrom .plugin import agent_config, plugin, skills@@ -44,6 +44,7 @@ def _register_controllers() -> tuple[ModuleType, ...]:llm,runtime_usage,tools,+ tracing,app,dsl,file_grants,@@ -62,6 +63,7 @@ def _register_controllers() -> tuple[ModuleType, ...]:_agent_llm,_agent_runtime_usage,_agent_tools,+ _agent_tracing,_app_app,_app_dsl,_app_file_grants,diff --git a/api/controllers/inner_api/agent/tracing.py b/api/controllers/inner_api/agent/tracing.pynew file mode 100644index 00000000000..945f42eb09f--- /dev/null+++ b/api/controllers/inner_api/agent/tracing.py@@ -0,0 +1,49 @@+"""Authenticated, idempotent ingestion of Agent observations."""++from typing import Literal+from uuid import UUID++from dify_agent.protocol.tracing import AgentTraceBatch+from flask import request+from flask_restx import Resource+from pydantic import ValidationError+from werkzeug.exceptions import BadRequest, Conflict, NotFound, RequestEntityTooLarge++from configs import dify_config+from controllers.common.schema import register_response_schema_models, register_schema_models+from controllers.inner_api import inner_api_ns+from controllers.inner_api.wraps import agent_inner_api_only+from fields.base import ResponseModel+from repositories.application_trace_repository import TraceCollectionError+from services.tracing.service import application_tracing_service+++class AgentTraceBatchResponse(ResponseModel):+ result: Literal["success"]+ batch_id: UUID+ accepted_spans: int+++register_schema_models(inner_api_ns, AgentTraceBatch)+register_response_schema_models(inner_api_ns, AgentTraceBatchResponse)+++@inner_api_ns.route("/agent/tracing/batches")+class AgentTraceBatchesApi(Resource):+ @agent_inner_api_only+ @inner_api_ns.doc("inner_agent_trace_batches")+ @inner_api_ns.expect(inner_api_ns.models[AgentTraceBatch.__name__])+ @inner_api_ns.response(200, "Durably accepted", inner_api_ns.models[AgentTraceBatchResponse.__name__])+ def post(self):+ request.max_content_length = dify_config.OPS_AGENT_TRACE_BATCH_MAX_BYTES+ try:+ payload = AgentTraceBatch.model_validate(inner_api_ns.payload or {})+ accepted = application_tracing_service().ingest(payload)+ except ValidationError as error:+ raise BadRequest("Invalid Agent trace batch") from error+ except TraceCollectionError as error:+ error_type = {404: NotFound, 409: Conflict, 413: RequestEntityTooLarge}.get(error.status, BadRequest)+ raise error_type(error.code) from error+ return AgentTraceBatchResponse(result="success", batch_id=payload.batch_id, accepted_spans=accepted).model_dump(+ mode="json"+ )diff --git a/api/core/app/apps/agent_app/app_generator.py b/api/core/app/apps/agent_app/app_generator.pyindex 6cb48fa0497..e33fde46f77 100644--- a/api/core/app/apps/agent_app/app_generator.py+++ b/api/core/app/apps/agent_app/app_generator.py@@ -311,8 +311,21 @@ class AgentAppGenerator(MessageBasedAppGenerator):),)except GenerateTaskStoppedError:+ # The runner hands off only a runtime terminal it observed.+ # A local stop or failed cancellation wait is still just intent.passexcept Exception as e:+ from core.ops.base_trace_instance import application_tracing_hooks++ tracing_hooks = application_tracing_hooks()+ if tracing_hooks is not None:+ tracing_hooks.capture_agent_app_terminal(+ tenant_id=application_generate_entity.app_config.tenant_id,+ app_id=application_generate_entity.app_config.app_id,+ message_id=message_id,+ outcome="failed",+ output=None,+ )logger.exception("Unknown Error in Agent App generate worker")queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)finally:diff --git a/api/core/app/apps/agent_app/app_runner.py b/api/core/app/apps/agent_app/app_runner.pyindex c8315afee5b..cfe4a1b6250 100644--- a/api/core/app/apps/agent_app/app_runner.py+++ b/api/core/app/apps/agent_app/app_runner.py@@ -728,18 +728,41 @@ class AgentAppRunner:message_id,exc_info=True,)- self._publish_terminal_answer(- queue_manager=queue_manager,- model_name=model_name,- answer=answer,- query=query,- usage=terminal_usage,- )+ try:+ self._publish_terminal_answer(+ queue_manager=queue_manager,+ model_name=model_name,+ answer=answer,+ query=query,+ usage=terminal_usage,+ )+ except GenerateTaskStoppedError:+ # Stop can become visible after stream consumption returned its+ # success but before the final chunk/end reaches the consumer.+ self._finish_stopped_terminal(+ terminal,+ dify_context=dify_context,+ session_scope=scope,+ binding_id=runtime.binding_id,+ message_id=message_id,+ )+ raiseself._save_session(scope=scope,binding_id=runtime.binding_id,snapshot=terminal.session_snapshot,)+ from core.ops.base_trace_instance import application_tracing_hooks++ tracing_hooks = application_tracing_hooks()+ if tracing_hooks is not None:+ tracing_hooks.capture_agent_app_terminal(+ tenant_id=dify_context.tenant_id,+ app_id=dify_context.app_id,+ message_id=message_id,+ outcome="succeeded",+ output=answer,+ )def _build_session_scope(self,@@ -857,6 +880,20 @@ class AgentAppRunner:for public_event in public_events:if public_event.id is not None:last_event_id = public_event.id+ internal_events = self._event_adapter.adapt(public_event)+ for internal_event in internal_events:+ if isinstance(+ internal_event,+ AgentBackendRunSucceededInternalEvent+ | AgentBackendRunFailedInternalEvent+ | AgentBackendRunCancelledInternalEvent,+ ):+ # A terminal frame may already have arrived when stop+ # becomes visible. Keep the observed runtime result.+ terminal = internal_event+ break+ if terminal is not None:+ breakif queue_manager.is_stopped():flush_pending_agent_message_text()self._cancel_run(@@ -865,9 +902,11 @@ class AgentAppRunner:session_scope=session_scope,binding_id=binding_id,message_id=message_id,+ dify_context=dify_context,+ queue_manager=queue_manager,)raise GenerateTaskStoppedError()- for internal_event in self._event_adapter.adapt(public_event):+ for internal_event in internal_events:if queue_manager.is_stopped():flush_pending_agent_message_text()self._cancel_run(@@ -876,6 +915,8 @@ class AgentAppRunner:session_scope=session_scope,binding_id=binding_id,message_id=message_id,+ dify_context=dify_context,+ queue_manager=queue_manager,)raise GenerateTaskStoppedError()if internal_event.type in (@@ -904,11 +945,6 @@ class AgentAppRunner:)continuecontinue- flush_pending_agent_message_text()- terminal = internal_event- break- if terminal is not None:- breakexcept GenerateTaskStoppedError:raiseexcept Exception as error:@@ -919,19 +955,43 @@ class AgentAppRunner:session_scope=session_scope,binding_id=binding_id,message_id=message_id,+ dify_context=dify_context,+ queue_manager=queue_manager,)if queue_manager.is_stopped():raise GenerateTaskStoppedError() from errorraise- flush_pending_agent_message_text()+ try:+ flush_pending_agent_message_text()+ except GenerateTaskStoppedError:+ if terminal is not None:+ self._finish_stopped_terminal(+ terminal,+ dify_context=dify_context,+ session_scope=session_scope,+ binding_id=binding_id,+ message_id=message_id,+ )+ raiseif queue_manager.is_stopped():- self._cancel_run(- run_id,- after=last_event_id,- session_scope=session_scope,- binding_id=binding_id,- message_id=message_id,- )+ if terminal is not None:+ self._finish_stopped_terminal(+ terminal,+ dify_context=dify_context,+ session_scope=session_scope,+ binding_id=binding_id,+ message_id=message_id,+ )+ else:+ self._cancel_run(+ run_id,+ after=last_event_id,+ session_scope=session_scope,+ binding_id=binding_id,+ message_id=message_id,+ dify_context=dify_context,+ queue_manager=queue_manager,+ )raise GenerateTaskStoppedError()return terminal, process_recorder@@ -943,23 +1003,20 @@ class AgentAppRunner:session_scope: AgentAppSessionScope,binding_id: str,message_id: str,+ dify_context: DifyRunContext,+ queue_manager: AppQueueManager,) -> None:try:public_event = self._agent_backend_client.cancel_run_and_wait(run_id, after=after)for internal_event in self._event_adapter.adapt(public_event):- if (- isinstance(internal_event, AgentBackendRunCancelledInternalEvent)- and internal_event.session_snapshot is not None- ):- self._save_session(- scope=session_scope,- binding_id=binding_id,- snapshot=internal_event.session_snapshot,- )if isinstance(internal_event, AgentBackendRunCancelledInternalEvent):- self._persist_message_usage(+ self._finish_stopped_terminal(+ internal_event,+ dify_context=dify_context,+ session_scope=session_scope,+ binding_id=binding_id,message_id=message_id,- usage=_llm_usage_from_agent_backend(internal_event.usage),+ queue_manager=queue_manager,)except Exception:logger.warning(@@ -968,6 +1025,50 @@ class AgentAppRunner:exc_info=True,)+ def _finish_stopped_terminal(+ self,+ terminal: AgentBackendRunSucceededInternalEvent+ | AgentBackendRunFailedInternalEvent+ | AgentBackendRunCancelledInternalEvent,+ *,+ dify_context: DifyRunContext,+ session_scope: AgentAppSessionScope,+ binding_id: str,+ message_id: str,+ queue_manager: AppQueueManager | None = None,+ ) -> None:+ """Keep the observed runtime result when the client has requested stop."""+ if terminal.session_snapshot is not None:+ self._save_session(scope=session_scope, binding_id=binding_id, snapshot=terminal.session_snapshot)+ self._persist_message_usage(message_id=message_id, usage=_llm_usage_from_agent_backend(terminal.usage))+ if queue_manager is not None and not queue_manager.is_stopped():+ # A stream error can cancel its runtime only for cleanup. Keep its+ # snapshot/usage, but let the original business failure own the root.+ # Check after cleanup so a Stop observed during its wait still wins.+ return+ from core.ops.base_trace_instance import application_tracing_hooks++ tracing_hooks = application_tracing_hooks()+ if tracing_hooks is not None:+ outcome: Literal["succeeded", "failed", "cancelled"]+ if isinstance(terminal, AgentBackendRunSucceededInternalEvent):+ outcome = "succeeded"+ elif isinstance(terminal, AgentBackendRunFailedInternalEvent):+ outcome = "failed"+ else:+ outcome = "cancelled"+ tracing_hooks.capture_agent_app_terminal(+ tenant_id=dify_context.tenant_id,+ app_id=dify_context.app_id,+ message_id=message_id,+ outcome=outcome,+ output=(+ self._terminal_output_to_answer(terminal.output)+ if isinstance(terminal, AgentBackendRunSucceededInternalEvent)+ else None+ ),+ )+def _publish_terminal_answer(self,*,@@ -992,7 +1093,11 @@ class AgentAppRunner:if usage is None or (usage.total_tokens <= 0 and usage.total_price <= 0):returntry:- message = db.session.get(Message, message_id)+ # A stopped consumer may have committed its stop intent/partial+ # answer through another session. Merge usage with that latest row+ # under the same short lock used by its save, not an identity-cache+ # snapshot that can erase the intent or newer reported usage.+ message = db.session.get(Message, message_id, populate_existing=True, with_for_update=True)if message is None:logger.warning("Cannot persist Agent App usage: message not found: %s", message_id)returndiff --git a/api/core/app/apps/agent_app/runtime_request_builder.py b/api/core/app/apps/agent_app/runtime_request_builder.pyindex cc4a63d45d6..c373d96ffd3 100644--- a/api/core/app/apps/agent_app/runtime_request_builder.py+++ b/api/core/app/apps/agent_app/runtime_request_builder.py@@ -36,6 +36,7 @@ from clients.agent_backend import (from configs import dify_configfrom core.app.entities.app_invoke_entities import DifyRunContext, InvokeFromfrom core.app.llm.model_access import resolve_model_context_window, resolve_model_supports_vision+from core.ops.base_trace_instance import application_tracing_hooksfrom core.plugin.provider_identity import normalize_plugin_daemon_provider_identityfrom core.workflow.file_reference import build_file_reference, is_canonical_file_referencefrom core.workflow.nodes.agent_v2.dify_tools_builder import (@@ -167,6 +168,7 @@ class AgentAppRuntimeRequestBuilder:image_detail_config=context.image_detail_config,)+ tracing_hooks = application_tracing_hooks()request = self._request_builder.build_for_agent_app(AgentBackendAgentAppRunInput(model=AgentBackendModelConfig(@@ -188,6 +190,7 @@ class AgentAppRuntimeRequestBuilder:user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),agent_mode="agent_app",+ tracing=tracing_hooks.prepare_agent_app(context) if tracing_hooks is not None else None,),backend_binding_ref=context.backend_binding_ref,# ENG-616: expand slash-menu mention tokens to canonical names sodiff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.pyindex 8995e106a01..aa490fbf546 100644--- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py+++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py@@ -318,18 +318,28 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGeneratanswer=output_moderation_answer)+ stopped_agent_message: Message | None = Nonewith session_factory.create_session() as session:# A stopped Agent run may persist provider-reported usage after# cancellation completes. Do not replace it with local token estimates.if isinstance(event, QueueStopEvent):- self._save_message(+ saved_message = self._save_message(session=session,trace_manager=trace_manager,preserve_existing_usage=True,)+ if self._app_config.app_mode == AppMode.AGENT:+ stopped_agent_message = saved_messageelse:self._save_message(session=session, trace_manager=trace_manager)session.commit()+ if stopped_agent_message is not None:+ # Provider/billing subscribers may perform external I/O.+ # Release the stop-save row lock before notifying them.+ message_was_created.send(+ stopped_agent_message,+ application_generate_entity=self._application_generate_entity,+ )message_end_resp = self._message_end_to_stream_response()yield message_end_respcase QueueRetrieverResourcesEvent():@@ -428,7 +438,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGeneratsession: Session,trace_manager: TraceQueueManager | None = None,preserve_existing_usage: bool = False,- ):+ ) -> Message:"""Save message.:return:@@ -436,7 +446,10 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGeneratllm_result = self._task_state.llm_resultusage = llm_result.usage+ agent_stop_requested = preserve_existing_usage and self._app_config.app_mode == AppMode.AGENTmessage_stmt = select(Message).where(Message.id == self._message_id)+ if agent_stop_requested:+ message_stmt = message_stmt.with_for_update().execution_options(populate_existing=True)message = session.scalar(message_stmt)if not message:raise ValueError(f"message {self._message_id} not found")@@ -481,9 +494,13 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGeneratmetadata = self._task_state.metadata.model_dump(mode="json")if has_persisted_usage and "usage" in existing_metadata:metadata["usage"] = existing_metadata["usage"]+ if agent_stop_requested:+ # Closing the consumer records a request, not a runtime outcome.+ # The Agent worker owns the terminal after cancellation/cleanup.+ metadata["agent_stop_requested"] = Truemessage.message_metadata = json.dumps(metadata, ensure_ascii=False)- if trace_manager:+ if trace_manager and not agent_stop_requested:trace_manager.add_trace_task(TraceTask(TraceTaskName.MESSAGE_TRACE,@@ -493,10 +510,12 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline[EasyUIAppGenerat))- message_was_created.send(- message,- application_generate_entity=self._application_generate_entity,- )+ if not agent_stop_requested:+ message_was_created.send(+ message,+ application_generate_entity=self._application_generate_entity,+ )+ return messagedef _handle_stop(self, event: QueueStopEvent):"""diff --git a/api/core/ops/base_trace_instance.py b/api/core/ops/base_trace_instance.pyindex d0f27ad0c5f..bcd2a663584 100644--- a/api/core/ops/base_trace_instance.py+++ b/api/core/ops/base_trace_instance.py@@ -1,13 +1,91 @@+from __future__ import annotations+from abc import ABC, abstractmethod+from typing import TYPE_CHECKING, Literal, Protocol, cast+from flask import Flask, current_app, has_app_contextfrom sqlalchemy import selectfrom sqlalchemy.orm import Session+from configs import dify_configfrom core.ops.entities.config_entity import BaseTracingConfigfrom core.ops.entities.trace_entity import BaseTraceInfo+from core.ops.exceptions import RetryableTraceDispatchErrorfrom extensions.ext_database import dbfrom models import Account, App+if TYPE_CHECKING:+ from dify_agent.layers.execution_context import ApplicationTracingContext++ from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext+ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity+ from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext+++class ApplicationTracingHooks(Protocol):+ """Execution-owned port for the application tracing lifecycle.++ The composition root supplies an app-scoped implementation. Core callers+ carry trusted execution context and do not construct services or vendor SDKs.+ """++ def prepare_agent_app(self, context: AgentAppRuntimeBuildContext) -> ApplicationTracingContext | None: ...++ def prepare_workflow_agent(+ self, context: WorkflowAgentRuntimeBuildContext, conversation_id: str | None+ ) -> ApplicationTracingContext | None: ...++ def freeze_workflow_run(+ self,+ entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,+ *,+ workflow_id: str,+ workflow_run_id: str,+ conversation_id: object,+ graph_data: object,+ ) -> None: ...++ def queue_enabled_at_start(self, app_id: object) -> bool: ...++ def capture_terminal(self, info: BaseTraceInfo) -> bool: ...++ def capture_agent_app_terminal(+ self,+ *,+ tenant_id: str,+ app_id: str,+ message_id: str,+ outcome: Literal["succeeded", "failed", "cancelled"],+ output: str | None,+ ) -> None: ...+++class ApplicationTracingHooksNotRegisteredError(RetryableTraceDispatchError):+ """Enabled tracing must not silently fall back to a different export path."""++ def __init__(self) -> None:+ super().__init__("application_tracing_hooks_not_registered")+++_APPLICATION_TRACING_HOOKS_KEY = "application_tracing_hooks"+++def bind_application_tracing_hooks(app: Flask, hooks: ApplicationTracingHooks) -> None:+ """Bind the execution port to one Flask application, never a process singleton."""+ app.extensions[_APPLICATION_TRACING_HOOKS_KEY] = hooks+++def application_tracing_hooks() -> ApplicationTracingHooks | None:+ """Resolve the current app's port; a disabled deployment needs no Flask context."""+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return None+ if not has_app_context():+ raise ApplicationTracingHooksNotRegisteredError()+ hooks = current_app.extensions.get(_APPLICATION_TRACING_HOOKS_KEY)+ if hooks is None:+ raise ApplicationTracingHooksNotRegisteredError()+ return cast(ApplicationTracingHooks, hooks)+class BaseTraceInstance(ABC):"""diff --git a/api/core/ops/exceptions.py b/api/core/ops/exceptions.pyindex 35ceff6124d..e80eb3b9649 100644--- a/api/core/ops/exceptions.py+++ b/api/core/ops/exceptions.py@@ -13,6 +13,8 @@ class TraceProviderNotInstalledError(ImportError):class RetryableTraceDispatchError(RuntimeError):"""Base class for transient trace dispatch failures that Celery may retry."""+ acceptance_unknown: bool = False+class TraceParentContextAccessError(RetryableTraceDispatchError):"""Raised when unified parent context storage is temporarily unavailable."""diff --git a/api/core/ops/ops_trace_manager.py b/api/core/ops/ops_trace_manager.pyindex 94829ffede2..742a8dabf47 100644--- a/api/core/ops/ops_trace_manager.py+++ b/api/core/ops/ops_trace_manager.py@@ -20,6 +20,7 @@ from sqlalchemy.orm import Session, sessionmakerfrom configs import dify_configfrom core.helper.encrypter import batch_decrypt_token, encrypt_token, obfuscated_tokenfrom core.helper.trace_id_helper import ParentTraceContext+from core.ops.base_trace_instance import application_tracing_hooksfrom core.ops.entities.config_entity import (BaseTracingConfig,TracingProviderEnum,@@ -390,7 +391,11 @@ class OpsTraceManager:# Encrypt necessary keysfor key in secret_keys:if key in tracing_config:- if "*" in tracing_config[key]:+ if tracing_config[key] is None:+ # Preserve explicit clearing for optional credentials; the+ # provider model still rejects null on a required secret.+ new_config[key] = None+ elif "*" in tracing_config[key]:# If the key contains '*', retain the original value from the current configif current_trace_config:new_config[key] = current_trace_config.get(key, tracing_config[key])@@ -439,8 +444,10 @@ class OpsTraceManager:provider_config_map[tracing_provider]["secret_keys"],provider_config_map[tracing_provider]["other_keys"],)- new_config: dict[str, Any] = {}- keys_to_decrypt = [key for key in secret_keys if key in tracing_config]+ new_config: dict[str, Any] = {+ key: None for key in secret_keys if key in tracing_config and tracing_config[key] is None+ }+ keys_to_decrypt = [key for key in secret_keys if tracing_config.get(key) is not None]if keys_to_decrypt:decrypted_values = batch_decrypt_token(tenant_id, [tracing_config[key] for key in keys_to_decrypt])new_config.update(zip(keys_to_decrypt, decrypted_values, strict=False))@@ -1477,7 +1484,15 @@ class TraceQueueManager:self.app_id = app_idself.user_id = user_id- self.trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)+ self.trace_instance = None+ self._application_tracing_enabled = False+ if dify_config.OPS_AGENT_TRACE_ENABLED:+ tracing_hooks = application_tracing_hooks()+ self._application_tracing_enabled = (+ tracing_hooks.queue_enabled_at_start(app_id) if tracing_hooks is not None else False+ )+ else:+ self.trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)self.flask_app = current_app._get_current_object() # type: ignorefrom core.telemetry.gateway import is_enterprise_telemetry_enabled@@ -1489,7 +1504,7 @@ class TraceQueueManager:def add_trace_task(self, trace_task: TraceTask):global trace_manager_timer, trace_manager_queuetry:- if self._enterprise_telemetry_enabled or self.trace_instance:+ if self._enterprise_telemetry_enabled or self._application_tracing_enabled or self.trace_instance:trace_task.app_id = self.app_idtrace_manager_queue.put(trace_task)except Exception:diff --git a/api/core/workflow/nodes/agent_v2/runtime_request_builder.py b/api/core/workflow/nodes/agent_v2/runtime_request_builder.pyindex 0fb791fe874..424f46148de 100644--- a/api/core/workflow/nodes/agent_v2/runtime_request_builder.py+++ b/api/core/workflow/nodes/agent_v2/runtime_request_builder.py@@ -48,6 +48,7 @@ from clients.agent_backend import (from configs import dify_configfrom core.app.entities.app_invoke_entities import DifyRunContext, InvokeFromfrom core.app.llm.model_access import resolve_model_context_window+from core.ops.base_trace_instance import application_tracing_hooksfrom core.plugin.provider_identity import normalize_plugin_daemon_provider_identityfrom core.workflow.system_variables import SystemVariableKey, get_system_text, get_system_valuefrom graphon.file import File, FileTransferMethod@@ -227,6 +228,7 @@ class WorkflowAgentRuntimeRequestBuilder:agent_soul.model.plugin_id,)+ tracing_hooks = application_tracing_hooks()request = self._request_builder.build_for_workflow_node(AgentBackendWorkflowNodeRunInput(model=AgentBackendModelConfig(@@ -257,6 +259,13 @@ class WorkflowAgentRuntimeRequestBuilder:agent_config_version_kind="snapshot",agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),+ tracing=(+ tracing_hooks.prepare_workflow_agent(+ context, get_system_text(context.variable_pool, SystemVariableKey.CONVERSATION_ID)+ )+ if tracing_hooks is not None+ else None+ ),),backend_binding_ref=context.backend_binding_ref,agent_soul_prompt=soul_prompt or None,diff --git a/api/extensions/application_services/app.py b/api/extensions/application_services/app.pyindex 8459402af50..6eb6f2b5d9f 100644--- a/api/extensions/application_services/app.py+++ b/api/extensions/application_services/app.py@@ -28,7 +28,7 @@ from services.app.import_service import AppImportServicefrom services.app.query_service import AppDiscoveryServicefrom services.app_dsl_service import AppDslServicefrom services.app_package_service import AppPackageService-from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigServicefrom services.oauth_server_service import OAuthServerServicefrom services.rbac.members import MemberServicefrom services.recommended_app_package_service import RecommendedAppPackageService@@ -52,6 +52,7 @@ def build_app_services(recommended_packages: RecommendedAppPackageService,repository: ConsoleAppRepository,rbac_members: MemberService,+ tracing_configs: AppTracingConfigService,) -> AppServices:transfers = AppTransferGateway(session_factory=database_client,@@ -77,7 +78,7 @@ def build_app_services(access=EnterpriseConsoleAppAccess(members=rbac_members),transfers=transfers,creators=CreatorsPlatformGateway(oauth=oauth),- tracing=OpsTraceManagerGateway(),+ tracing=tracing_configs,lifecycle=AppLifecycleGateway(session_factory=database_client),),enterprise=EnterpriseAppService(diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.pyindex d4105d8db27..381099f5c1b 100644--- a/api/extensions/ext_application_services.py+++ b/api/extensions/ext_application_services.py@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session, sessionmakerfrom configs import dify_configfrom constants.dsl_version import CURRENT_APP_DSL_VERSIONfrom core.db.session_factory import get_session_maker+from core.ops.base_trace_instance import bind_application_tracing_hooksfrom core.schemas.schema_manager import SchemaManagerfrom core.tools.tool_file_manager import ToolFileManagerfrom enums import DeploymentEdition, WebAppAccessMode@@ -52,6 +53,7 @@ from repositories.account.repository import SQLAlchemyAccountRepositoryfrom repositories.account_activation_repository import SQLAlchemyAccountActivationRepositoryfrom repositories.account_integration_repository import SQLAlchemyAccountIntegrationRepositoryfrom repositories.api_based_extension_repository import APIBasedExtensionRepository+from repositories.app.agent_app_repository import AgentAppRepositoryfrom repositories.app.console_repository import ConsoleAppRepositoryfrom repositories.app.mcp_server_repository import AppMCPServerRepositoryfrom repositories.app.site_command_repository import AppSiteCommandRepository@@ -99,6 +101,7 @@ from services.account.login_adapters import RedisConsoleAuthSecurityGatewayfrom services.account.service import AccountSetupProvisionerfrom services.account_password_hasher import DefaultAccountPasswordHasherfrom services.agent.roster_package_exporter import RosterAgentPackageExporter+from services.agent_tracing_config_service import AgentTracingConfigServicefrom services.api_based_extension_adapters import APIBasedExtensionPingProbe, WorkspaceTokenCipherfrom services.api_based_extension_application_service import APIBasedExtensionApplicationServicefrom services.app.advanced_prompt_template_service import AdvancedPromptTemplateService@@ -187,6 +190,7 @@ from services.step_by_step_tour_service import StepByStepTourServicefrom services.system_feature_service import SystemFeatureServicefrom services.tag_application_service import TagApplicationServicefrom services.tool_file_download_service import ToolFileDownloadService+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooksfrom services.upload_file_delivery_service import UploadFileDeliveryServicefrom services.web_app_runtime_query_service import WebAppRuntimeQueryServicefrom services.web_authentication_adapters import (@@ -274,6 +278,7 @@ class ApplicationServices:conversation_variables: ConversationVariableQueryapp_statistics: AppStatisticQueryapp_tracing_configs: AppTracingConfigService+ agent_tracing_configs: AgentTracingConfigServicebilling_portal: BillingPortalServicecompliance_downloads: ComplianceDownloadServicedata_sources: DataSourceServices@@ -460,12 +465,17 @@ def build_application_services(apps=console_apps,datasets=dataset_dependencies.datasets,)+ tracing_configs = AppTracingConfigService(+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),+ provider=OpsTraceManagerGateway(),+ )apps = build_app_services(database_client=database_client,oauth=oauth_server,recommended_packages=recommended_app_packages,repository=console_apps,rbac_members=rbac.members,+ tracing_configs=tracing_configs,)tags = TagApplicationService(tags=TagRepository(session_factory=database_client))knowledge = build_knowledge_services(@@ -549,9 +559,11 @@ def build_application_services(),resource_access_tokens=build_resource_access_token_service(database_client=database_client),app_statistics=AppStatisticQueryRepository(session_factory=database_client),- app_tracing_configs=AppTracingConfigService(- configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),- provider=OpsTraceManagerGateway(),+ app_tracing_configs=tracing_configs,+ agent_tracing_configs=AgentTracingConfigService(+ owners=AgentAppRepository(session_factory=database_client),+ settings=apps.console,+ configs=tracing_configs,),billing_portal=BillingPortalService(accounts=accounts,@@ -749,6 +761,7 @@ def init_app(app: Flask) -> None:redis=redis_client,)app.extensions[_EXTENSION_KEY] = services+ bind_application_tracing_hooks(app, RuntimeApplicationTracingHooks())bind_account_loader(app, services.accounts.identity.load_user)diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.pyindex 6368a7fd1dd..4e58a0b0af0 100644--- a/api/extensions/ext_celery.py+++ b/api/extensions/ext_celery.py@@ -188,6 +188,14 @@ def init_app(app: DifyApp) -> Celery:# if you add a new task, please add the switch to CeleryScheduleTasksConfigbeat_schedule: dict[str, CeleryBeatScheduleEntry] = {}+ if dify_config.OPS_AGENT_TRACE_ENABLED:+ imports.append("tasks.application_trace_task")+ if dify_config.OPS_AGENT_TRACE_ENABLED and dify_config.ENABLE_APPLICATION_TRACE_RECONCILE_TASK:+ beat_schedule["reconcile_application_traces"] = {+ "task": "tasks.application_trace_task.reconcile_application_traces",+ "schedule": timedelta(seconds=30),+ "options": {"expires": 25},+ }if dify_config.AGENT_SANDBOX_METERING_ENABLED:try:interval = int(dify_config.AGENT_SANDBOX_METERING_INTERVAL_SECONDS)diff --git a/api/fields/app_fields.py b/api/fields/app_fields.pyindex 92408bac755..d1c3ffc45e3 100644--- a/api/fields/app_fields.py+++ b/api/fields/app_fields.py@@ -17,6 +17,8 @@ from services.entities.dsl_entities import DslImportWarning, ImportStatusclass AppTraceResponse(ResponseModel):enabled: bool = Falsetracing_provider: str | None = None+ capture_content: bool = False+ capture_agent: bool = Trueclass Tag(ResponseModel):diff --git a/api/migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py b/api/migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.pynew file mode 100644index 00000000000..a7cab08de98--- /dev/null+++ b/api/migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py@@ -0,0 +1,86 @@+"""Add durable application tracing collection and delivery state.++Revision ID: a17c2980f3e6+Revises: f1d3c5b7a902+Create Date: 2026-10-15 03:30:00+"""++import sqlalchemy as sa+from alembic import op++from models.types import StringUUID++revision = "a17c2980f3e6"+down_revision = "f1d3c5b7a902"+branch_labels = None+depends_on = None+++def upgrade() -> None:+ op.create_table(+ "application_trace_runs",+ sa.Column("id", StringUUID(), primary_key=True),+ sa.Column("tenant_id", StringUUID(), nullable=False),+ sa.Column("app_id", StringUUID(), nullable=False),+ sa.Column("source_kind", sa.String(16), nullable=False),+ sa.Column("source_id", StringUUID(), nullable=False),+ sa.Column("provider", sa.String(32), nullable=False),+ sa.Column("config_revision", sa.String(64), nullable=False),+ sa.Column("provider_config", sa.JSON(), nullable=False),+ sa.Column("trace_id", sa.String(32), nullable=False),+ sa.Column("root_span_id", sa.String(16), nullable=False),+ sa.Column("session_id", sa.String(255), nullable=False),+ sa.Column("capture_content", sa.Boolean(), nullable=False),+ sa.Column("capture_agent", sa.Boolean(), nullable=False),+ sa.Column("state", sa.String(24), nullable=False),+ sa.Column("outer_trace", sa.JSON(), nullable=True),+ sa.Column("span_count", sa.Integer(), nullable=False),+ sa.Column("payload_bytes", sa.BigInteger(), nullable=False),+ sa.Column("attempts", sa.Integer(), nullable=False),+ sa.Column("error_code", sa.String(96), nullable=True),+ sa.Column("trace_url", sa.Text(), nullable=True),+ sa.Column("provider_trace_id", sa.String(255), nullable=True),+ sa.Column("span_urls", sa.JSON(), nullable=False),+ sa.Column("export_token", StringUUID(), nullable=True),+ sa.Column("export_started_at", sa.DateTime(), nullable=True),+ sa.Column("export_heartbeat_at", sa.DateTime(), nullable=True),+ sa.Column("next_attempt_at", sa.DateTime(), nullable=True),+ sa.Column("accepted_at", sa.DateTime(), nullable=True),+ sa.Column("terminal_at", sa.DateTime(), nullable=True),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("updated_at", sa.DateTime(), nullable=False),+ sa.UniqueConstraint("tenant_id", "app_id", "source_kind", "source_id", name="application_trace_source_unique"),+ )+ op.create_index("application_trace_delivery_idx", "application_trace_runs", ["state", "next_attempt_at"])+ op.create_index("application_trace_retention_idx", "application_trace_runs", ["created_at"])+ op.create_table(+ "application_trace_invocations",+ sa.Column("id", StringUUID(), primary_key=True),+ sa.Column("run_id", StringUUID(), nullable=False),+ sa.Column("agent_id", StringUUID(), nullable=False),+ sa.Column("parent_span_id", sa.String(16), nullable=False),+ sa.Column("span_id", sa.String(16), nullable=False),+ sa.Column("node_execution_id", StringUUID(), nullable=True),+ sa.Column("attempt", sa.Integer(), nullable=False),+ sa.Column("finalized", sa.Boolean(), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("finalized_at", sa.DateTime(), nullable=True),+ )+ op.create_index("application_trace_invocation_run_idx", "application_trace_invocations", ["run_id"])+ op.create_table(+ "application_trace_batches",+ sa.Column("id", StringUUID(), primary_key=True),+ sa.Column("run_id", StringUUID(), nullable=False),+ sa.Column("invocation_id", StringUUID(), nullable=False),+ sa.Column("payload_hash", sa.String(64), nullable=False),+ sa.Column("spans", sa.JSON(), nullable=False),+ sa.Column("final", sa.Boolean(), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ )+ op.create_index("application_trace_batch_run_idx", "application_trace_batches", ["run_id", "created_at"])+++def downgrade() -> None:+ op.drop_table("application_trace_batches")+ op.drop_table("application_trace_invocations")+ op.drop_table("application_trace_runs")diff --git a/api/migrations/versions/2026_10_19_1340-f7e2c9a6d481_merge_tracing_and_workflow_pause_heads.py b/api/migrations/versions/2026_10_19_1340-f7e2c9a6d481_merge_tracing_and_workflow_pause_heads.pynew file mode 100644index 00000000000..04630caaf6a--- /dev/null+++ b/api/migrations/versions/2026_10_19_1340-f7e2c9a6d481_merge_tracing_and_workflow_pause_heads.py@@ -0,0 +1,19 @@+"""Merge application tracing and workflow pause migration heads.++Revision ID: f7e2c9a6d481+Revises: a17c2980f3e6, c6f4a90d8b31+Create Date: 2026-10-19 13:40:00+"""++revision = "f7e2c9a6d481"+down_revision = ("a17c2980f3e6", "c6f4a90d8b31")+branch_labels = None+depends_on = None+++def upgrade() -> None:+ pass+++def downgrade() -> None:+ passdiff --git a/api/migrations/versions/2026_10_21_1030-d6b9e2f1a407_merge_tracing_and_workflow_tool_heads.py b/api/migrations/versions/2026_10_21_1030-d6b9e2f1a407_merge_tracing_and_workflow_tool_heads.pynew file mode 100644index 00000000000..5d32480c057--- /dev/null+++ b/api/migrations/versions/2026_10_21_1030-d6b9e2f1a407_merge_tracing_and_workflow_tool_heads.py@@ -0,0 +1,19 @@+"""Merge application tracing and workflow tool migration heads.++Revision ID: d6b9e2f1a407+Revises: a8c3e7f4b912, f7e2c9a6d481+Create Date: 2026-10-21 10:30:00+"""++revision = "d6b9e2f1a407"+down_revision = ("a8c3e7f4b912", "f7e2c9a6d481")+branch_labels = None+depends_on = None+++def upgrade() -> None:+ pass+++def downgrade() -> None:+ passdiff --git a/api/models/__init__.py b/api/models/__init__.pyindex 21ff0d9fe65..a35befda04f 100644--- a/api/models/__init__.py+++ b/api/models/__init__.py@@ -32,6 +32,7 @@ from .agent import ()from .agent_sandbox_usage import AgentSandboxExecution, AgentSandboxUsageEventfrom .api_based_extension import APIBasedExtension, APIBasedExtensionPoint+from .application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRunfrom .comment import (WorkflowComment,WorkflowCommentMention,@@ -201,6 +202,9 @@ __all__ = ["AppTrigger","AppTriggerStatus","AppTriggerType",+ "ApplicationTraceBatch",+ "ApplicationTraceInvocation",+ "ApplicationTraceRun","BuiltinToolProvider","CeleryTask","CeleryTaskSet",diff --git a/api/models/agent_config_entities.py b/api/models/agent_config_entities.pyindex d3900fa3e77..8911a8888b4 100644--- a/api/models/agent_config_entities.py+++ b/api/models/agent_config_entities.py@@ -536,7 +536,8 @@ class AgentSoulModelSettings(AgentFlexibleConfig):frequency_penalty: float | None = Nonemax_tokens: int | None = Nonestop: list[str] | None = None- response_format: AgentModelResponseFormatConfig | None = None+ # Plugin parameter rules use string enums; existing saved configs may use native objects.+ response_format: str | AgentModelResponseFormatConfig | None = Noneclass AgentFeatureToggleConfig(AgentFlexibleConfig):diff --git a/api/models/application_trace.py b/api/models/application_trace.pynew file mode 100644index 00000000000..c95ced54ad7--- /dev/null+++ b/api/models/application_trace.py@@ -0,0 +1,84 @@+"""Durable application tracing envelopes, separate from business execution state.++Provider configuration snapshots contain the existing workspace-encrypted secret+values. They are never serialized by Console endpoints or sent to Agent runs.+"""++from datetime import datetime+from typing import Any++import sqlalchemy as sa+from sqlalchemy.orm import Mapped, mapped_column++from libs.datetime_utils import naive_utc_now+from models.base import TypeBase+from models.types import StringUUID+++class ApplicationTraceRun(TypeBase, kw_only=True):+ __tablename__ = "application_trace_runs"+ __table_args__ = (+ sa.UniqueConstraint("tenant_id", "app_id", "source_kind", "source_id", name="application_trace_source_unique"),+ sa.Index("application_trace_delivery_idx", "state", "next_attempt_at"),+ sa.Index("application_trace_retention_idx", "created_at"),+ )++ id: Mapped[str] = mapped_column(StringUUID, primary_key=True)+ tenant_id: Mapped[str] = mapped_column(StringUUID)+ app_id: Mapped[str] = mapped_column(StringUUID)+ source_kind: Mapped[str] = mapped_column(sa.String(16))+ source_id: Mapped[str] = mapped_column(StringUUID)+ provider: Mapped[str] = mapped_column(sa.String(32))+ config_revision: Mapped[str] = mapped_column(sa.String(64))+ provider_config: Mapped[dict[str, Any]] = mapped_column(sa.JSON)+ trace_id: Mapped[str] = mapped_column(sa.String(32))+ root_span_id: Mapped[str] = mapped_column(sa.String(16))+ session_id: Mapped[str] = mapped_column(sa.String(255))+ capture_content: Mapped[bool] = mapped_column(sa.Boolean)+ capture_agent: Mapped[bool] = mapped_column(sa.Boolean, default=True)+ state: Mapped[str] = mapped_column(sa.String(24), default="collecting")+ outer_trace: Mapped[dict[str, Any] | None] = mapped_column(sa.JSON, default=None)+ span_count: Mapped[int] = mapped_column(sa.Integer, default=0)+ payload_bytes: Mapped[int] = mapped_column(sa.BigInteger, default=0)+ attempts: Mapped[int] = mapped_column(sa.Integer, default=0)+ error_code: Mapped[str | None] = mapped_column(sa.String(96), default=None)+ trace_url: Mapped[str | None] = mapped_column(sa.Text, default=None)+ provider_trace_id: Mapped[str | None] = mapped_column(sa.String(255), default=None)+ span_urls: Mapped[dict[str, str]] = mapped_column(sa.JSON, default_factory=dict)+ export_token: Mapped[str | None] = mapped_column(StringUUID, default=None)+ export_started_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)+ export_heartbeat_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)+ next_attempt_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)+ accepted_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)+ terminal_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)+ created_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)+ updated_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)+++class ApplicationTraceInvocation(TypeBase, kw_only=True):+ __tablename__ = "application_trace_invocations"+ __table_args__ = (sa.Index("application_trace_invocation_run_idx", "run_id"),)++ id: Mapped[str] = mapped_column(StringUUID, primary_key=True)+ run_id: Mapped[str] = mapped_column(StringUUID)+ agent_id: Mapped[str] = mapped_column(StringUUID)+ parent_span_id: Mapped[str] = mapped_column(sa.String(16))+ span_id: Mapped[str] = mapped_column(sa.String(16))+ node_execution_id: Mapped[str | None] = mapped_column(StringUUID)+ attempt: Mapped[int] = mapped_column(sa.Integer)+ finalized: Mapped[bool] = mapped_column(sa.Boolean, default=False)+ created_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)+ finalized_at: Mapped[datetime | None] = mapped_column(sa.DateTime, default=None)+++class ApplicationTraceBatch(TypeBase, kw_only=True):+ __tablename__ = "application_trace_batches"+ __table_args__ = (sa.Index("application_trace_batch_run_idx", "run_id", "created_at"),)++ id: Mapped[str] = mapped_column(StringUUID, primary_key=True)+ run_id: Mapped[str] = mapped_column(StringUUID)+ invocation_id: Mapped[str] = mapped_column(StringUUID)+ payload_hash: Mapped[str] = mapped_column(sa.String(64))+ spans: Mapped[list[dict[str, Any]]] = mapped_column(sa.JSON)+ final: Mapped[bool] = mapped_column(sa.Boolean)+ created_at: Mapped[datetime] = mapped_column(sa.DateTime, default_factory=naive_utc_now)diff --git a/api/models/model.py b/api/models/model.pyindex 11c7ec4128c..c4dd6642230 100644--- a/api/models/model.py+++ b/api/models/model.py@@ -308,6 +308,7 @@ class MessageDict(TypedDict):id: strapp_id: strconversation_id: str+ model_provider: NotRequired[str | None]model_id: str | Noneinputs: dict[str, Any]query: str@@ -1786,6 +1787,7 @@ class Message(Base):"id": self.id,"app_id": self.app_id,"conversation_id": self.conversation_id,+ "model_provider": self.model_provider,"model_id": self.model_id,"inputs": self.inputs_with_session(session=session),"query": self.query,@@ -1810,6 +1812,7 @@ class Message(Base):id=data["id"],app_id=data["app_id"],conversation_id=data["conversation_id"],+ model_provider=data.get("model_provider"),model_id=data["model_id"],inputs=data["inputs"],total_price=data["total_price"],diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.mdindex 69ac746956d..09c87762c2c 100644--- a/api/openapi/markdown/console-openapi.md+++ b/api/openapi/markdown/console-openapi.md@@ -1471,6 +1471,185 @@ Get available TTS voices for an Agent and language| 404 | Agent not found | **application/json**: [AgentNotFoundError](#agentnotfounderror)<br>[ErrorResponse](#errorresponse)<br> || default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |+### [GET] /agent/{agent_id}/trace+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| agent_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Agent tracing settings | **application/json**: [AppTraceResponse](#apptraceresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [POST] /agent/{agent_id}/trace+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| agent_id | path | | Yes | string (uuid) |++#### Request Body++| Required | Schema |+| -------- | ------ |+| Yes | **application/json**: [AppTracePayload](#apptracepayload)<br> |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Updated Agent tracing settings | **application/json**: [AppTraceResponse](#apptraceresponse)<br> |+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |+| 404 | Not Found | **application/json**: [TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [GET] /agent/{agent_id}/trace-capabilities+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| tracing_provider | query | Tracing provider name | Yes | string |+| agent_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Provider support declarations | **application/json**: [TraceProviderCapabilitiesResponse](#traceprovidercapabilitiesresponse)<br> |+| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [DELETE] /agent/{agent_id}/trace-config+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| tracing_provider | query | Tracing provider name | Yes | string |+| agent_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 204 | Agent provider configuration deleted | |+| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [GET] /agent/{agent_id}/trace-config+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| tracing_provider | query | Tracing provider name | Yes | string |+| agent_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Masked Agent provider configuration | **application/json**: [TraceAppConfigResponse](#traceappconfigresponse)<br> |+| 400 | Bad Request | **application/json**: [TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [PATCH] /agent/{agent_id}/trace-config+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| agent_id | path | | Yes | string (uuid) |++#### Request Body++| Required | Schema |+| -------- | ------ |+| Yes | **application/json**: [TraceConfigPayload](#traceconfigpayload)<br> |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Agent provider updated | **application/json**: [TraceAppConfigResponse](#traceappconfigresponse)<br> |+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingConfigVerificationFailedError](#tracingconfigverificationfailederror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [POST] /agent/{agent_id}/trace-config+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| agent_id | path | | Yes | string (uuid) |++#### Request Body++| Required | Schema |+| -------- | ------ |+| Yes | **application/json**: [TraceConfigPayload](#traceconfigpayload)<br> |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 201 | Agent provider configured | **application/json**: [TraceAppConfigResponse](#traceappconfigresponse)<br> |+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingConfigVerificationFailedError](#tracingconfigverificationfailederror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> |+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TraceConfigAlreadyExistsError](#traceconfigalreadyexistserror)<br>[ErrorResponse](#errorresponse)<br> |+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [GET] /agent/{agent_id}/trace-runs/{message_id}+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |+| agent_id | path | | Yes | string (uuid) |+| message_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Frozen trace destination and export state | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [POST] /agent/{agent_id}/trace-runs/{message_id}/retry+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |+| agent_id | path | | Yes | string (uuid) |+| message_id | path | | Yes | string (uuid) |++#### Request Body++| Required | Schema |+| -------- | ------ |+| Yes | **application/json**: [TraceRunRetryPayload](#tracerunretrypayload)<br> |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Retry state for the same frozen destination | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingRunNotFoundError](#tracingrunnotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TracingAlreadyAcceptedError](#tracingalreadyacceptederror)<br>[TracingDuplicateAcknowledgementRequiredError](#tracingduplicateacknowledgementrequirederror)<br>[TracingRetryNotAllowedError](#tracingretrynotallowederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |+### [GET] /agent/{agent_id}/versions#### Parameters@@ -3611,8 +3790,27 @@ Update app tracing configuration| Code | Description | Schema || ---- | ----------- | ------ || 200 | Trace configuration updated successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)<br> |-| 400 | Bad Request | **application/json**: [TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[ErrorResponse](#errorresponse)<br> |+| 400 | Bad Request | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> || 403 | Insufficient permissions | **application/json**: [ErrorResponse](#errorresponse)<br> |+| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> |+| 500 | Internal Server Error | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [GET] /apps/{app_id}/trace-capabilities+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| tracing_provider | query | Tracing provider name | Yes | string |+| app_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Provider support declarations | **application/json**: [TraceProviderCapabilitiesResponse](#traceprovidercapabilitiesresponse)<br> |+| 400 | Bad Request | **application/json**: [UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> || default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |### [DELETE] /apps/{app_id}/trace-config@@ -3683,6 +3881,7 @@ Update an existing tracing configuration for an application| 400 | Invalid request parameters or tracing configuration | **application/json**: [InvalidTracingConfigError](#invalidtracingconfigerror)<br>[TracingConfigVerificationFailedError](#tracingconfigverificationfailederror)<br>[TracingProviderUnavailableError](#tracingproviderunavailableerror)<br>[UnsupportedTracingProviderError](#unsupportedtracingprovidererror)<br>[ErrorResponse](#errorresponse)<br> || 403 | Insufficient permissions | **application/json**: [ErrorResponse](#errorresponse)<br> || 404 | Application or tracing configuration not found | **application/json**: [AppNotFoundError](#appnotfounderror)<br>[TraceConfigNotFoundError](#traceconfignotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TracingConfigConflictError](#tracingconfigconflicterror)<br>[ErrorResponse](#errorresponse)<br> || 500 | Tracing configuration processing failed | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> || default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |@@ -3715,6 +3914,49 @@ Create a new tracing configuration for an application| 500 | Tracing configuration processing failed | **application/json**: [TracingConfigProcessingFailedError](#tracingconfigprocessingfailederror)<br>[ErrorResponse](#errorresponse)<br> || default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |+### [GET] /apps/{app_id}/trace-runs/{source_kind}/{source_id}+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| source_kind | path | | Yes | string, <br>**Available values:** "message", "node", "workflow" |+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |+| app_id | path | | Yes | string (uuid) |+| source_id | path | | Yes | string (uuid) |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Frozen trace destination and export state | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |++### [POST] /apps/{app_id}/trace-runs/{source_kind}/{source_id}/retry+#### Parameters++| Name | Located in | Description | Required | Schema |+| ---- | ---------- | ----------- | -------- | ------ |+| source_kind | path | | Yes | string, <br>**Available values:** "message", "node", "workflow" |+| node_execution_id | query | Select an Agent node invocation within this run | No | string (uuid) |+| app_id | path | | Yes | string (uuid) |+| source_id | path | | Yes | string (uuid) |++#### Request Body++| Required | Schema |+| -------- | ------ |+| Yes | **application/json**: [TraceRunRetryPayload](#tracerunretrypayload)<br> |++#### Responses++| Code | Description | Schema |+| ---- | ----------- | ------ |+| 200 | Retry state for the same frozen destination | **application/json**: [TraceRunStatusResponse](#tracerunstatusresponse)<br> |+| 404 | Not Found | **application/json**: [TracingNodeNotFoundError](#tracingnodenotfounderror)<br>[TracingRunNotFoundError](#tracingrunnotfounderror)<br>[TracingSourceNotFoundError](#tracingsourcenotfounderror)<br>[ErrorResponse](#errorresponse)<br> |+| 409 | Conflict | **application/json**: [TracingAlreadyAcceptedError](#tracingalreadyacceptederror)<br>[TracingDuplicateAcknowledgementRequiredError](#tracingduplicateacknowledgementrequirederror)<br>[TracingRetryNotAllowedError](#tracingretrynotallowederror)<br>[ErrorResponse](#errorresponse)<br> |+| default | Error | **application/json**: [ErrorResponse](#errorresponse)<br> |+### [POST] /apps/{app_id}/trigger-enable**Update app trigger (enable/disable)**@@ -17095,7 +17337,7 @@ rather than being dropped.| frequency_penalty | number | | No || max_tokens | integer | | No || presence_penalty | number | | No |-| response_format | [AgentModelResponseFormatConfig](#agentmodelresponseformatconfig) | | No |+| response_format | string<br>[AgentModelResponseFormatConfig](#agentmodelresponseformatconfig) | | No || stop | [ string ] | | No || temperature | number | | No || top_p | number | | No |@@ -18903,6 +19145,8 @@ Function Suggested questions after answer disabled.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- |+| capture_agent | boolean, <br>**Default:** true | Capture Agent internals when deployment support is enabled | No |+| capture_content | boolean | Include prompt, output and tool content in new traces | No || enabled | boolean | Enable or disable tracing | Yes || tracing_provider | string | Tracing provider | No |@@ -18910,6 +19154,8 @@ Function Suggested questions after answer disabled.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- |+| capture_agent | boolean, <br>**Default:** true | | No |+| capture_content | boolean | | No || enabled | boolean | | No || tracing_provider | string | | No |@@ -27371,6 +27617,7 @@ Non-sensitive bootstrap snapshot exposed before Console or Web authentication.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- |+| agent_tracing_api_collection_enabled | boolean | Deployment-level API gate for Agent trace collection, independent of workspace and provider configuration. Does not indicate Agent runtime, worker, or provider health or confirm trace delivery. | Yes || branding | [BrandingModel](#brandingmodel) | | Yes || deployment_edition | [DeploymentEdition](#deploymentedition) | | Yes || enable_app_deploy | boolean | | Yes |@@ -27869,15 +28116,101 @@ Tracing configuration not found.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- |+| activate | boolean | Save, select this provider, and enable tracing atomically; preserve capture settings | No || tracing_config | object | Tracing configuration data | Yes || tracing_provider | string | Tracing provider name | Yes |+#### TraceProviderCapabilitiesResponse++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| agent_spans | boolean | | Yes |+| api_collection_enabled | boolean | Current API collection gate only; does not probe Agent runtime or confirm delivery | Yes |+| export_ack | string | Exporter acknowledgement does not confirm query visibility | Yes |+| native_cost | boolean | | Yes |+| native_usage | boolean | | Yes |+| provider | string | | Yes |+| session | boolean | | Yes |+| span_url | boolean | | Yes |+| trace_url | boolean | | Yes |+#### TraceProviderQuery| Name | Type | Description | Required || ---- | ---- | ----------- | -------- || tracing_provider | string | Tracing provider name | Yes |+#### TraceProviderSupportResponse++Adapter support declarations, not successful-delivery or platform-verification results.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| agent_spans | boolean | | Yes |+| export_ack | string | Exporter acknowledgement does not confirm query visibility | Yes |+| native_cost | boolean | | Yes |+| native_usage | boolean | | Yes |+| provider | string | | Yes |+| session | boolean | | Yes |+| span_url | boolean | | Yes |+| trace_url | boolean | | Yes |++#### TraceRunQuery++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| node_execution_id | string | Select an Agent node invocation within this run | No |++#### TraceRunRetryPayload++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| acknowledge_possible_duplicates | boolean | Acknowledge possible duplicate records when retrying an uncertain export | No |++#### TraceRunStatusResponse++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| accepted_at | string | Ingestion acknowledgement time, not query visibility | No |+| attempts | integer | | No |+| capabilities | [TraceProviderSupportResponse](#traceprovidersupportresponse) | | No |+| capture_agent | boolean | | No |+| capture_content | boolean | | No |+| config_revision | string | Frozen destination configuration revision | No |+| error_code | string | | No |+| project_url | string | Frozen destination project URL; null when unavailable, never resolved from current config | No |+| provider | string | | No |+| queryable | boolean | Null when third-party query visibility is not verified | No |+| source_id | string | | Yes |+| source_kind | [TraceSourceKind](#tracesourcekind) | | Yes |+| span_count | integer | | No |+| span_url | string | | No |+| status | string, <br>**Available values:** "accepted", "collecting", "exporting", "failed", "not_collected", "ready", "retrying", "uncertain" | *Enum:* `"accepted"`, `"collecting"`, `"exporting"`, `"failed"`, `"not_collected"`, `"ready"`, `"retrying"`, `"uncertain"` | Yes |+| trace_id | string | | No |+| trace_url | string | | No |++#### TraceSourceKind++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TraceSourceKind | string | | |++#### TracingAlreadyAcceptedError++An accepted trace cannot be exported again.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingAlreadyAcceptedError | | An accepted trace cannot be exported again. | |++#### TracingConfigConflictError++Tracing configuration changed. Refresh and try again.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingConfigConflictError | | Tracing configuration changed. Refresh and try again. | |+#### TracingConfigProcessingFailedErrorThe tracing configuration could not be processed.@@ -27894,6 +28227,22 @@ The tracing configuration could not be verified.| ---- | ---- | ----------- | -------- || TracingConfigVerificationFailedError | | The tracing configuration could not be verified. | |+#### TracingDuplicateAcknowledgementRequiredError++Acknowledge possible duplicates before retrying an uncertain export.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingDuplicateAcknowledgementRequiredError | | Acknowledge possible duplicates before retrying an uncertain export. | |++#### TracingNodeNotFoundError++The Agent node invocation does not belong to this trace.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingNodeNotFoundError | | The Agent node invocation does not belong to this trace. | |+#### TracingProviderUnavailableErrorThe tracing provider is not available in this deployment.@@ -27902,6 +28251,30 @@ The tracing provider is not available in this deployment.| ---- | ---- | ----------- | -------- || TracingProviderUnavailableError | | The tracing provider is not available in this deployment. | |+#### TracingRetryNotAllowedError++This trace is not in a retryable state.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingRetryNotAllowedError | | This trace is not in a retryable state. | |++#### TracingRunNotFoundError++No collected trace is available to retry.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingRunNotFoundError | | No collected trace is available to retry. | |++#### TracingSourceNotFoundError++The tracing source does not exist in this application.++| Name | Type | Description | Required |+| ---- | ---- | ----------- | -------- |+| TracingSourceNotFoundError | | The tracing source does not exist in this application. | |+#### TrialAgentConfigQuery| Name | Type | Description | Required |diff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.mdindex b6ed652be66..ed64e186f96 100644--- a/api/openapi/markdown/web-openapi.md+++ b/api/openapi/markdown/web-openapi.md@@ -1950,6 +1950,7 @@ Non-sensitive bootstrap snapshot exposed before Console or Web authentication.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- |+| agent_tracing_api_collection_enabled | boolean | Deployment-level API gate for Agent trace collection, independent of workspace and provider configuration. Does not indicate Agent runtime, worker, or provider health or confirm trace delivery. | Yes || branding | [BrandingModel](#brandingmodel) | | Yes || deployment_edition | [DeploymentEdition](#deploymentedition) | | Yes || enable_app_deploy | boolean | | Yes |diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.pynew file mode 100644index 00000000000..ec1b1a39fb3--- /dev/null+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/agent_trace.py@@ -0,0 +1,27 @@+"""Aliyun LLM tracing export with synchronous OTLP/HTTP acknowledgment."""++from collections.abc import Mapping+from typing import Any++from core.ops.unified_trace.entities import CanonicalTrace+from dify_trace_aliyun.configuration import AliyunConfigurationError, ingestion_url+from services.tracing.genai_attributes import attributes+from services.tracing.otlp_export import build_request, export_http+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ try:+ url = ingestion_url(config)+ except AliyunConfigurationError:+ raise TraceExportRejectedError("Aliyun tracing configuration is invalid") from None+ payload = build_request(+ trace,+ {+ "service.name": str(config.get("app_name") or "dify_app"),+ "acs.arms.service.feature": "genai_app",+ },+ lambda span: attributes(span, trace, provider),+ )+ export_http(provider, url, {}, payload)+ return TraceExportResult(trace_id=trace.trace_id)diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.pyindex 717c1ac8765..5b7efaca491 100644--- a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/aliyun_trace.py@@ -18,9 +18,9 @@ from core.ops.entities.trace_entity import ()from core.repositories import DifyCoreRepositoryFactoryfrom dify_trace_aliyun.config import AliyunConfig+from dify_trace_aliyun.configuration import console_url, ingestion_url, verify_configurationfrom dify_trace_aliyun.data_exporter.traceclient import (TraceClient,- build_endpoint,convert_datetime_to_nanoseconds,convert_to_span_id,convert_to_trace_id,@@ -90,38 +90,38 @@ class AliyunDataTrace(BaseTraceInstance):aliyun_config: AliyunConfig,):super().__init__(aliyun_config)- endpoint = build_endpoint(aliyun_config.endpoint, aliyun_config.license_key)- self.trace_client = TraceClient(service_name=aliyun_config.app_name, endpoint=endpoint)+ self._config = aliyun_config.model_copy(deep=True)+ self.trace_client = TraceClient(+ service_name=self._config.app_name, endpoint=ingestion_url(self._config.model_dump())+ )@overridedef trace(self, trace_info: BaseTraceInfo):- match trace_info:- case WorkflowTraceInfo():- self.workflow_trace(trace_info)- case MessageTraceInfo():- self.message_trace(trace_info)- case ModerationTraceInfo():- pass- case SuggestedQuestionTraceInfo():- self.suggested_question_trace(trace_info)- case DatasetRetrievalTraceInfo():- self.dataset_retrieval_trace(trace_info)- case ToolTraceInfo():- self.tool_trace(trace_info)- case GenerateNameTraceInfo():- pass- case _:- pass-- def api_check(self):- return self.trace_client.api_check()+ with self.trace_client.batch():+ match trace_info:+ case WorkflowTraceInfo():+ self.workflow_trace(trace_info)+ case MessageTraceInfo():+ self.message_trace(trace_info)+ case ModerationTraceInfo():+ pass+ case SuggestedQuestionTraceInfo():+ self.suggested_question_trace(trace_info)+ case DatasetRetrievalTraceInfo():+ self.dataset_retrieval_trace(trace_info)+ case ToolTraceInfo():+ self.tool_trace(trace_info)+ case GenerateNameTraceInfo():+ pass+ case _:+ pass++ def api_check(self) -> bool:+ verify_configuration(self._config)+ return Truedef get_project_url(self):- try:- return self.trace_client.get_project_url()- except Exception as e:- logger.info("Aliyun get project url failed: %s", str(e), exc_info=True)- raise ValueError(f"Aliyun get project url failed: {str(e)}") from e+ return console_url()def _extract_app_id(self, trace_info: BaseTraceInfo) -> str:"""Extract app_id from trace_info, trying metadata first then message_data."""diff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.pyindex e0133e6cc92..bae1c06d16d 100644--- a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/config.py@@ -1,4 +1,4 @@-from pydantic import ValidationInfo, field_validator+from pydantic import ConfigDict, ValidationInfo, field_validatorfrom core.ops.entities.config_entity import BaseTracingConfigfrom core.ops.utils import validate_url_with_path@@ -9,6 +9,10 @@ class AliyunConfig(BaseTracingConfig):Model class for Aliyun tracing config."""+ # Endpoints may be pasted with an embedded LicenseKey. Even rejected input+ # must not appear in formatted validation exceptions or chained API errors.+ model_config = ConfigDict(hide_input_in_errors=True)+app_name: str = "dify_app"license_key: strendpoint: strdiff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.pynew file mode 100644index 00000000000..ca7c9bf689c--- /dev/null+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/configuration.py@@ -0,0 +1,158 @@+"""Aliyun destinations and a resource-only, zero-span collector access check.++GET and URL helpers are pure. Verification makes one explicit collector request;+it never constructs a tracer, creates span IDs, or exports business telemetry.+"""++import re+from collections.abc import Mapping+from typing import Literal+from urllib.parse import quote, unquote, urlsplit, urlunsplit++from dify_trace_aliyun.config import AliyunConfig++type ConfigurationCode = Literal["invalid_endpoint", "invalid_app_name", "invalid_license_key", "verification_failed"]+_MESSAGES: dict[ConfigurationCode, str] = {+ "invalid_endpoint": "Aliyun requires an HTTP(S) base URL without credentials or a collector suffix.",+ "invalid_app_name": "Aliyun requires an application name without control characters.",+ "invalid_license_key": "Aliyun requires a nonempty license key without whitespace or control characters.",+ "verification_failed": "Aliyun tracing configuration could not be verified.",+}+_DEFAULT_ENDPOINT = "https://tracing-analysis-dc-hz.aliyuncs.com"+++class AliyunConfigurationError(ValueError):+ """Diagnostics never include submitted URLs, credentials, or response bodies."""++ def __init__(self, code: ConfigurationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++def _endpoint(value: object) -> str:+ if not isinstance(value, str):+ raise AliyunConfigurationError("invalid_endpoint")+ value = value.strip(" ") or _DEFAULT_ENDPOINT+ try:+ parsed = urlsplit(value)+ host, port = parsed.hostname, parsed.port+ path = parsed.path.rstrip("/")+ # Decode only the public path to detect hidden collector routes. License+ # keys are opaque values and must never be unquoted or otherwise altered.+ decoded_path = unquote(path, errors="strict")+ valid = (+ parsed.scheme in {"http", "https"}+ and host+ and parsed.username is None+ and parsed.password is None+ and "?" not in value+ and "#" not in value+ and "\\" not in value+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)+ and not parsed.netloc.endswith(":")+ and (port is None or 1 <= port <= 65535)+ and "%" not in host+ and not re.search(r"%(?![0-9a-fA-F]{2})", path)+ and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)+ and not any(char in decoded_path for char in "\\?#%")+ and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)+ and "//" not in parsed.path+ and not any(part in {".", ".."} or part.lower().startswith("adapt_") for part in decoded_path.split("/"))+ and not decoded_path.lower().endswith(("/v1/traces", "/api/otlp/traces"))+ )+ if not valid or host is None:+ raise AliyunConfigurationError("invalid_endpoint")+ host = host.encode("idna").decode("ascii").lower()+ authority = f"[{host}]" if ":" in host else host+ if port is not None:+ authority = f"{authority}:{port}"+ return urlunsplit((parsed.scheme, authority, path, "", ""))+ except (UnicodeError, ValueError):+ raise AliyunConfigurationError("invalid_endpoint") from None+++def _license_key(value: object) -> str:+ if (+ not isinstance(value, str)+ or not value+ or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)+ ):+ raise AliyunConfigurationError("invalid_license_key")+ try:+ value.encode("utf-8")+ except UnicodeError:+ raise AliyunConfigurationError("invalid_license_key") from None+ return value+++def normalize_configuration(values: Mapping[str, object]) -> AliyunConfig:+ """Validate new writes without tightening the model used for historical GET.++ Masked keys survive this phase so the gateway can restore the original+ ciphertext on PATCH. Exporters reject masks after that restoration.+ """+ name = values.get("app_name", "dify_app")+ if not isinstance(name, str) or any(ord(char) < 32 or ord(char) == 127 for char in name):+ raise AliyunConfigurationError("invalid_app_name")+ return AliyunConfig(+ app_name=name.strip() or "dify_app",+ license_key=_license_key(values.get("license_key")),+ endpoint=_endpoint(values.get("endpoint")),+ )+++def ingestion_url(config: Mapping[str, object]) -> str:+ """Build one destination, encoding the raw license exactly once as a path segment."""+ normalized = normalize_configuration(config)+ if "*" in normalized.license_key:+ raise AliyunConfigurationError("invalid_license_key")+ hostname = (urlsplit(normalized.endpoint).hostname or "").removesuffix(".")+ is_sls = hostname == "log.aliyuncs.com" or hostname.endswith(".log.aliyuncs.com")+ suffix = "api/v1/traces" if is_sls else "api/otlp/traces"+ # Aliyun's documented path-form token contains literal '@'. RFC 3986 path+ # segments permit it, and this collector does not treat '%40' equivalently.+ # Preserve only that required reserved character; never unquote raw keys.+ return f"{normalized.endpoint}/adapt_{quote(normalized.license_key, safe='@')}/{suffix}"+++def console_url() -> str:+ """Return the platform console, not a destination-specific project or trace link."""+ return "https://arms.console.aliyun.com/#/llm"+++def verify_configuration(config: AliyunConfig) -> None:+ """Check access with one resource envelope and strictly zero scopes/spans.++ This collector distinguishes valid from invalid licenses for this request;+ HEAD 405 and a completely empty request are not verification. A successful+ acknowledgment does not establish business-span retention or queryability.+ """+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest++ from services.tracing.otlp_export import is_aliyun_http_success_ack+ from services.tracing.provider_common import request++ normalized = normalize_configuration(config.model_dump())+ url = ingestion_url(normalized.model_dump())+ payload = ExportTraceServiceRequest()+ resource = payload.resource_spans.add().resource+ resource.attributes.add(key="service.name").value.string_value = normalized.app_name+ try:+ response = request(+ "POST",+ url,+ "aliyun",+ headers={"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"},+ content=payload.SerializeToString(),+ )+ try:+ # Only the account-tested resource-only ACK establishes this check.+ # A generic empty OTLP ACK remains valid for ordinary export, but+ # cannot be substituted as evidence that this license was checked.+ if not is_aliyun_http_success_ack(response.status_code, response.content):+ raise AliyunConfigurationError("verification_failed")+ finally:+ response.close()+ except Exception:+ # No URL, LicenseKey, vendor body, or nested transport exception escapes.+ raise AliyunConfigurationError("verification_failed") from Nonediff --git a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.pyindex 451fe662315..adbb9aefee6 100644--- a/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py+++ b/api/providers/trace/trace-aliyun/src/dify_trace_aliyun/data_exporter/traceclient.py@@ -1,18 +1,15 @@import hashlib-import loggingimport randomimport socket-import threadingimport uuid-from collections import deque-from collections.abc import Sequence+from collections.abc import Generator, Sequence+from contextlib import contextmanager+from contextvars import ContextVarfrom datetime import datetimefrom typing import Final-from urllib.parse import urljoin-import httpxfrom opentelemetry import trace as trace_api-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter+from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spansfrom opentelemetry.sdk.resources import Resourcefrom opentelemetry.sdk.trace import ReadableSpanfrom opentelemetry.sdk.util.instrumentation import InstrumentationScope@@ -26,28 +23,25 @@ from opentelemetry.semconv.attributes import service_attributesfrom opentelemetry.trace import Link, SpanContext, TraceFlagsfrom configs import dify_config+from dify_trace_aliyun.configuration import ingestion_urlfrom dify_trace_aliyun.entities.aliyun_trace_entity import SpanDatafrom dify_trace_aliyun.entities.semconv import ACS_ARMS_SERVICE_FEATURE+from services.tracing.otlp_export import export_httpINVALID_SPAN_ID: Final[int] = 0x0000000000000000INVALID_TRACE_ID: Final[int] = 0x00000000000000000000000000000000-DEFAULT_TIMEOUT: Final[int] = 5-DEFAULT_MAX_QUEUE_SIZE: Final[int] = 1000-DEFAULT_SCHEDULE_DELAY_SEC: Final[int] = 5-DEFAULT_MAX_EXPORT_BATCH_SIZE: Final[int] = 50--logger = logging.getLogger(__name__)class TraceClient:- def __init__(- self,- service_name: str,- endpoint: str,- max_queue_size: int = DEFAULT_MAX_QUEUE_SIZE,- schedule_delay_sec: int = DEFAULT_SCHEDULE_DELAY_SEC,- max_export_batch_size: int = DEFAULT_MAX_EXPORT_BATCH_SIZE,- ):+ """Collect one dispatch locally and synchronously acknowledge its exact batch.++ Instances are cached, so pending spans belong to the current context rather+ than the client. A mapper failure discards only that dispatch; an uncertain+ response propagates to the durable task instead of silently retrying a batch.+ No SDK exporter, process-global configuration or background queue is created.+ """++ def __init__(self, service_name: str, endpoint: str) -> None:self.endpoint = endpointself.resource = Resource(attributes={@@ -59,80 +53,31 @@ class TraceClient:})self.span_builder = SpanBuilder(self.resource)- self.exporter = OTLPSpanExporter(endpoint=endpoint)-- self.max_queue_size = max_queue_size- self.schedule_delay_sec = schedule_delay_sec- self.max_export_batch_size = max_export_batch_size-- self.queue: deque = deque(maxlen=max_queue_size)- self.condition = threading.Condition(threading.Lock())- self.done = False+ self._dispatch_spans: ContextVar[list[ReadableSpan] | None] = ContextVar("aliyun_dispatch_spans", default=None)- self.worker_thread = threading.Thread(target=self._worker, daemon=True)- self.worker_thread.start()-- self._spans_dropped = False-- def export(self, spans: Sequence[ReadableSpan]):- self.exporter.export(spans)-- def api_check(self) -> bool:+ @contextmanager+ def batch(self) -> Generator[None]:+ """Export only after the whole mapper succeeds, and always restore context."""+ spans: list[ReadableSpan] = []+ token = self._dispatch_spans.set(spans)try:- response = httpx.head(self.endpoint, timeout=DEFAULT_TIMEOUT)- if response.status_code == 405:- return True- else:- logger.warning("AliyunTrace API check failed: Unexpected status code: %s", response.status_code)- return False- except httpx.RequestError as e:- logger.warning("AliyunTrace API check failed", exc_info=True)- raise ValueError(f"AliyunTrace API check failed: {str(e)}") from e-- def get_project_url(self) -> str:- return "https://arms.console.aliyun.com/#/llm"+ yield+ if spans:+ self.export(spans)+ finally:+ self._dispatch_spans.reset(token)++ def export(self, spans: Sequence[ReadableSpan]) -> None:+ if spans:+ export_http("aliyun", self.endpoint, {}, encode_spans(spans))def add_span(self, span_data: SpanData | None) -> None:if span_data is None:return-- span: ReadableSpan = self.span_builder.build_span(span_data)- with self.condition:- if len(self.queue) == self.max_queue_size:- if not self._spans_dropped:- logger.warning("Queue is full, likely spans will be dropped.")- self._spans_dropped = True-- self.queue.appendleft(span)- if len(self.queue) >= self.max_export_batch_size:- self.condition.notify()-- def _worker(self) -> None:- while not self.done:- with self.condition:- if len(self.queue) < self.max_export_batch_size and not self.done:- self.condition.wait(timeout=self.schedule_delay_sec)- self._export_batch()-- def _export_batch(self) -> None:- spans_to_export: list[ReadableSpan] = []- with self.condition:- while len(spans_to_export) < self.max_export_batch_size and self.queue:- spans_to_export.append(self.queue.pop())-- if spans_to_export:- try:- self.exporter.export(spans_to_export)- except Exception as e:- logger.warning("Error exporting spans: %s", e)-- def shutdown(self) -> None:- with self.condition:- self.done = True- self.condition.notify_all()- self.worker_thread.join()- self._export_batch()- self.exporter.shutdown()+ spans = self._dispatch_spans.get()+ if spans is None:+ raise RuntimeError("Aliyun spans require an active trace dispatch")+ spans.append(self.span_builder.build_span(span_data))class SpanBuilder:@@ -238,7 +183,5 @@ def convert_datetime_to_nanoseconds(start_time_a: datetime | None) -> int | Nonedef build_endpoint(base_url: str, license_key: str) -> str:- if "log.aliyuncs.com" in base_url: # cms2.0 endpoint- return urljoin(base_url, f"adapt_{license_key}/api/v1/traces")- else: # xtrace endpoint- return urljoin(base_url, f"adapt_{license_key}/api/otlp/traces")+ """Compatibility wrapper around the shared, credential-safe URL builder."""+ return ingestion_url({"endpoint": base_url, "license_key": license_key})diff --git a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.pyindex 8c3a4f3e4b2..2d520a644bd 100644--- a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py+++ b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/data_exporter/test_traceclient.py@@ -1,10 +1,7 @@-import logging-import timeimport uuidfrom datetime import datetime-from unittest.mock import MagicMock, patch+from unittest.mock import patch-import httpximport pytestfrom dify_trace_aliyun.data_exporter.traceclient import (INVALID_SPAN_ID,@@ -19,207 +16,29 @@ from dify_trace_aliyun.data_exporter.traceclient import (generate_span_id,)from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData+from opentelemetry.sdk.resources import Resourcefrom opentelemetry.sdk.trace import ReadableSpanfrom opentelemetry.trace import SpanKind, Status, StatusCode-def trace_client_factory():- """Factory fixture for creating TraceClient instances with automatic cleanup."""- clients_to_shutdown = []-- def _factory(**kwargs):- client = TraceClient(**kwargs)- clients_to_shutdown.append(client)- return client-- yield _factory-- # Cleanup: shutdown all created clients- for client in clients_to_shutdown:- client.shutdown()--class TestTraceClient:- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- @patch("dify_trace_aliyun.data_exporter.traceclient.socket.gethostname")- def test_init(- self, mock_gethostname: MagicMock, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]- ):- mock_gethostname.return_value = "test-host"- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")-- assert client.endpoint == "http://test-endpoint"- assert client.max_queue_size == 1000- assert client.schedule_delay_sec == 5- assert client.done is False- assert client.worker_thread.is_alive()-- client.shutdown()- # pyrefly: ignore [unnecessary-comparison]- assert client.done is True-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_export(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):- mock_exporter = mock_exporter_class.return_value- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")- spans = [ReadableSpan(name="test-span")]- client.export(spans)- mock_exporter.export.assert_called_once_with(spans)-- @patch("dify_trace_aliyun.data_exporter.traceclient.httpx.head")- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_api_check_success(- self, mock_exporter_class: MagicMock, mock_head: MagicMock, trace_client_factory: type[TraceClient]- ):- mock_response = MagicMock()- mock_response.status_code = 405- mock_head.return_value = mock_response-- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")- assert client.api_check() is True-- @patch("dify_trace_aliyun.data_exporter.traceclient.httpx.head")- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_api_check_failure_status(- self, mock_exporter_class: MagicMock, mock_head: MagicMock, trace_client_factory: type[TraceClient]- ):- mock_response = MagicMock()- mock_response.status_code = 500- mock_head.return_value = mock_response-- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")- assert client.api_check() is False-- @patch("dify_trace_aliyun.data_exporter.traceclient.httpx.head")- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_api_check_exception(- self, mock_exporter_class: MagicMock, mock_head: MagicMock, trace_client_factory: type[TraceClient]- ):- mock_head.side_effect = httpx.RequestError("Connection error")-- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")- with pytest.raises(ValueError, match="AliyunTrace API check failed: Connection error"):- client.api_check()-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_get_project_url(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")- assert client.get_project_url() == "https://arms.console.aliyun.com/#/llm"-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_add_span(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):- client = trace_client_factory(- service_name="test-service",- endpoint="http://test-endpoint",- max_export_batch_size=2,- )-- # Test add None+ def test_constructor_has_no_background_exporter(self) -> None:+ client = TraceClient(service_name="test-service", endpoint="https://collector.test")+ assert client.endpoint == "https://collector.test"+ assert not hasattr(client, "exporter")+ assert not hasattr(client, "worker_thread")+ assert not hasattr(client, "queue")++ def test_add_requires_dispatch(self) -> None:+ client = TraceClient(service_name="test-service", endpoint="https://collector.test")client.add_span(None)- assert len(client.queue) == 0-- # Test add valid SpanData- span_data = SpanData(- name="test-span",- trace_id=123,- span_id=456,- parent_span_id=None,- start_time=1000,- end_time=2000,- status=Status(StatusCode.OK),- span_kind=SpanKind.INTERNAL,- )-- span = ReadableSpan(name="test-span")- client.span_builder.build_span = MagicMock(return_value=span)-- with patch.object(client.condition, "notify") as mock_notify:- client.add_span(span_data)- assert len(client.queue) == 1- mock_notify.assert_not_called()-- client.add_span(span_data)- assert len(client.queue) == 2- mock_notify.assert_called_once()-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_add_span_queue_full(- self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient], caplog: pytest.LogCaptureFixture- ):- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint", max_queue_size=1)-- span_data = SpanData(- name="test-span",- trace_id=123,- span_id=456,- parent_span_id=None,- start_time=1000,- end_time=2000,- status=Status(StatusCode.OK),- span_kind=SpanKind.INTERNAL,- )- span = ReadableSpan(name="test-span")- client.span_builder.build_span = MagicMock(return_value=span)-- client.add_span(span_data)- assert len(client.queue) == 1-- with caplog.at_level(logging.WARNING):- client.add_span(span_data)- assert len(client.queue) == 1- assert "Queue is full, likely spans will be dropped." in caplog.text-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_export_batch_error(- self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient], caplog: pytest.LogCaptureFixture- ):- mock_exporter = mock_exporter_class.return_value- mock_exporter.export.side_effect = Exception("Export failed")-- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")- span = ReadableSpan(name="test-span")- client.queue.append(span)-- with caplog.at_level(logging.WARNING):- client._export_batch()- assert "Error exporting spans" in caplog.text-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_worker_loop(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):- # We need to test the wait timeout in _worker- # But _worker runs in a thread. Let's mock condition.wait.- client = trace_client_factory(- service_name="test-service",- endpoint="http://test-endpoint",- schedule_delay_sec=1,- )-- with patch.object(client.condition, "wait") as mock_wait:- # Let it run for a bit then shut down- time.sleep(0.2)- client.shutdown()- # mock_wait might have been called- assert mock_wait.called or client.done-- @patch("dify_trace_aliyun.data_exporter.traceclient.OTLPSpanExporter")- def test_shutdown_flushes(self, mock_exporter_class: MagicMock, trace_client_factory: type[TraceClient]):- mock_exporter = mock_exporter_class.return_value- client = trace_client_factory(service_name="test-service", endpoint="http://test-endpoint")-- span = ReadableSpan(name="test-span")- client.queue.append(span)-- client.shutdown()- # Should have called export twice (once in worker/export_batch, once in shutdown)- # or at least once if worker was waiting- assert mock_exporter.export.called- assert mock_exporter.shutdown.called+ with pytest.raises(RuntimeError, match="active trace dispatch"):+ client.add_span(SpanData(name="span", trace_id=123, span_id=456, start_time=1000, end_time=2000))class TestSpanBuilder:def test_build_span(self):- resource = MagicMock()+ resource = Resource({"service.name": "test"})builder = SpanBuilder(resource)span_data = SpanData(@@ -248,7 +67,7 @@ class TestSpanBuilder:assert span.attributes == {"attr1": "val1"}def test_build_span_no_parent(self):- resource = MagicMock()+ resource = Resource({"service.name": "test"})builder = SpanBuilder(resource)span_data = SpanData(diff --git a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.pyindex 4dce71a1cae..25ffc570c9c 100644--- a/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py+++ b/api/providers/trace/trace-aliyun/tests/unit_tests/aliyun_trace/test_aliyun_trace.py@@ -1,16 +1,18 @@from __future__ import annotationsimport json-import logging+from contextlib import nullcontextfrom datetime import UTC, datetimefrom types import SimpleNamespace-from typing import cast+from typing import Any, castfrom unittest.mock import MagicMockimport dify_trace_aliyun.aliyun_trace as aliyun_trace_module+import httpximport pytestfrom dify_trace_aliyun.aliyun_trace import AliyunDataTracefrom dify_trace_aliyun.config import AliyunConfig+from dify_trace_aliyun.configuration import AliyunConfigurationErrorfrom dify_trace_aliyun.entities.aliyun_trace_entity import SpanData, TraceMetadatafrom dify_trace_aliyun.entities.semconv import (DIFY_NODE_TYPE,@@ -44,8 +46,10 @@ from dify_trace_aliyun.entities.semconv import (TOOL_TYPE_FUNCTION,GenAISpanKind,)+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequestfrom opentelemetry.trace import Link, SpanContext, SpanKind, Status, StatusCode, TraceFlags+from core.helper import ssrf_proxyfrom core.ops.entities.trace_entity import (DatasetRetrievalTraceInfo,GenerateNameTraceInfo,@@ -65,6 +69,9 @@ class RecordingTraceClient:self.endpoint = endpointself.added_spans: list[object] = []+ def batch(self) -> nullcontext[None]:+ return nullcontext()+def add_span(self, span: object) -> None:self.added_spans.append(span)@@ -213,7 +220,6 @@ def _make_suggested_question_trace_info(**overrides) -> SuggestedQuestionTraceIn@pytest.fixturedef trace_instance(monkeypatch: pytest.MonkeyPatch) -> AliyunDataTrace:- monkeypatch.setattr(aliyun_trace_module, "build_endpoint", lambda base_url, license_key: "built-endpoint")monkeypatch.setattr(aliyun_trace_module, "TraceClient", RecordingTraceClient)# Mock get_service_account_with_tenant to avoid DB errorsmonkeypatch.setattr(AliyunDataTrace, "get_service_account_with_tenant", lambda self, app_id: MagicMock())@@ -223,17 +229,12 @@ def trace_instance(monkeypatch: pytest.MonkeyPatch) -> AliyunDataTrace:return trace-def test_init_builds_endpoint_and_client(monkeypatch: pytest.MonkeyPatch):- build_endpoint = MagicMock(return_value="built")- trace_client_cls = MagicMock()- monkeypatch.setattr(aliyun_trace_module, "build_endpoint", build_endpoint)- monkeypatch.setattr(aliyun_trace_module, "TraceClient", trace_client_cls)-- config = AliyunConfig(app_name="my-app", license_key="license", endpoint="https://example.com")+def test_init_builds_endpoint_and_client() -> None:+ config = AliyunConfig(app_name="my-app", license_key="license", endpoint="https://example.com/prefix")trace = AliyunDataTrace(config)-- build_endpoint.assert_called_once_with("https://example.com", "license")- trace_client_cls.assert_called_once_with(service_name="my-app", endpoint="built")+ config.license_key = "mutated"+ assert trace.trace_client.endpoint == "https://example.com/prefix/adapt_license/api/otlp/traces"+ assert trace.trace_client.resource.attributes["service.name"] == "my-app"assert trace.trace_config == config@@ -269,25 +270,48 @@ def test_trace_dispatches_to_correct_methods(trace_instance: AliyunDataTrace, motrace_instance.trace(GenerateNameTraceInfo(tenant_id="t", metadata={}))-def test_api_check_delegates(trace_instance: AliyunDataTrace):- trace_instance.trace_client.api_check = MagicMock(return_value=False)- assert trace_instance.api_check() is False+ ("status", "body", "verified"),+ [+ (200, b'"success"', True),+ (200, b"", False),+ (200, b"{}", False),+ (403, b"DO_NOT_EXPOSE_BODY", False),+ (405, b"", False),+ ],+)+def test_api_check_verifies_credentials_with_zero_spans(+ trace_instance: AliyunDataTrace, monkeypatch: pytest.MonkeyPatch, status: int, body: bytes, verified: bool+) -> None:+ sent: list[ExportTraceServiceRequest] = []++ def send(method: str, url: str, **options: Any) -> httpx.Response:+ assert method == "POST"+ assert url == "https://example.com/adapt_k/api/otlp/traces"+ assert options["max_retries"] == 0+ assert options["follow_redirects"] is False+ payload = ExportTraceServiceRequest()+ payload.ParseFromString(options["content"])+ sent.append(payload)+ return httpx.Response(status, content=body)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ if verified:+ assert trace_instance.api_check() is True+ else:+ with pytest.raises(AliyunConfigurationError) as error:+ trace_instance.api_check()+ assert "DO_NOT_EXPOSE_BODY" not in str(error.value)+ assert "adapt_" not in str(error.value)+ assert len(sent) == 1+ assert len(sent[0].resource_spans) == 1+ resource = sent[0].resource_spans[0]+ assert len(resource.scope_spans) == 0+ assert {item.key: item.value.string_value for item in resource.resource.attributes} == {"service.name": "app"}def test_get_project_url_success(trace_instance: AliyunDataTrace):- assert trace_instance.get_project_url() == "project-url"---def test_get_project_url_error(- trace_instance: AliyunDataTrace, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture-):- monkeypatch.setattr(trace_instance.trace_client, "get_project_url", MagicMock(side_effect=Exception("boom")))-- caplog.set_level(logging.INFO, logger=aliyun_trace_module.logger.name)- with pytest.raises(ValueError, match=r"Aliyun get project url failed: boom"):- trace_instance.get_project_url()-- assert "Aliyun get project url failed: boom" in caplog.text+ assert trace_instance.get_project_url() == "https://arms.console.aliyun.com/#/llm"def test_workflow_trace_adds_workflow_and_node_spans(trace_instance: AliyunDataTrace, monkeypatch: pytest.MonkeyPatch):diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.pynew file mode 100644index 00000000000..7100d76615e--- /dev/null+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/agent_trace.py@@ -0,0 +1,54 @@+"""Stable-ID OpenInference mapping for Phoenix HTTP and Arize gRPC ingestion."""++from collections.abc import Mapping+from typing import Any++from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalTrace+from services.tracing.otlp_export import build_request, export_grpc, export_http+from services.tracing.provider_common import endpoint, json_value, token_usage+from services.tracing.provider_dispatch import TraceExportResult+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ url = endpoint(+ config.get("endpoint", "https://otlp.arize.com" if provider == "arize" else "https://app.phoenix.arize.com")+ )+ api_key = str(config.get("api_key") or "")+ project = str(config.get("project") or "default")++ def attributes(span: CanonicalSpan) -> dict[str, object]:+ result: dict[str, object] = {+ "openinference.span.kind": span.kind.value.upper(),+ "input.value": json_value(span.inputs),+ "input.mime_type": "application/json",+ "output.value": json_value(span.outputs),+ "output.mime_type": "application/json",+ "metadata": json_value(span.metadata),+ "session.id": trace.session_id,+ "dify.span.id": span.id,+ }+ if span.kind is CanonicalSpanKind.LLM:+ model = span.metadata.get("model") or span.metadata.get("model_name")+ if model:+ result["llm.model_name"] = str(model)+ if span.metadata.get("model_provider"):+ result["llm.provider"] = str(span.metadata["model_provider"])+ names = {"input": "prompt", "output": "completion", "total": "total"}+ result.update({f"llm.token_count.{names[key]}": value for key, value in token_usage(span).items()})+ elif span.kind is CanonicalSpanKind.TOOL:+ result["tool.name"] = span.name+ result["tool.parameters"] = json_value(span.inputs)+ return result++ payload = build_request(+ trace,+ {"service.name": "dify-application", "openinference.project.name": project, "model_id": project},+ attributes,+ )+ headers = {"api_key": api_key, "authorization": f"Bearer {api_key}"} if api_key or provider == "arize" else {}+ if provider == "arize":+ headers["space_id"] = str(config.get("space_id") or "")+ export_grpc(provider, url, headers, payload)+ else:+ export_http(provider, f"{url}/v1/traces", headers, payload)+ return TraceExportResult(trace_id=trace.trace_id)diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.pynew file mode 100644index 00000000000..32053dc40f4--- /dev/null+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_configuration.py@@ -0,0 +1,304 @@+"""Optional Arize Cloud project metadata, resolved only on explicit config writes.++Ingestion verification is separate from metadata reads. A key without read access+may still ingest successfully, so an unavailable project link is never a reason+to reject that configuration. Readers use only the frozen destination cache.+"""++import hashlib+import json+import re+import time+from collections.abc import Mapping+from typing import Literal+from urllib.parse import quote, urlsplit++from pydantic import JsonValue, TypeAdapter++from core.helper import ssrf_proxy+from dify_trace_arize_phoenix.config import ArizeConfig++DESTINATION_KEY = "arize_project_destination"+_API_BASE = "https://api.arize.com/v2"+_UI_BASE = "https://app.arize.com"+_BUDGET_SECONDS = 5.0+_MAX_RESPONSE_BYTES = 1024 * 1024+_PAGE_LIMIT = 100+_MAX_PAGES = 3+_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])+_VERIFY_TIMEOUT_SECONDS = 5.0++type VerificationCode = Literal[+ "invalid_configuration", "authentication_failed", "permission_denied", "connection_failed", "unexpected_response"+]+++class ArizeConfigurationError(ValueError):+ """Fixed diagnostics never contain credentials, vendor details or endpoints."""++ def __init__(self, code: VerificationCode) -> None:+ self.code = code+ super().__init__(f"Arize ingestion verification failed: {code}")+++def verify_configuration(config: ArizeConfig) -> None:+ """Require one synchronous OTLP ACK, without creating any span or project.++ The default Arize collector was verified to authenticate this resource-only+ request for both valid and invalid credentials. It establishes this ingest+ protocol's access, not REST read permission or a project's eventual creation.+ No SDK exporter, span processor or global/environment credentials are used.+ """+ import grpc # type: ignore[import-untyped]+ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub++ try:+ parsed = urlsplit(config.endpoint)+ host, port = parsed.hostname, parsed.port+ if (+ parsed.scheme not in {"http", "https"}+ or not host+ or parsed.username is not None+ or parsed.password is not None+ or (port is not None and not 1 <= port <= 65535)+ ):+ raise ArizeConfigurationError("invalid_configuration")+ for value in (config.api_key, config.space_id):+ if (+ not isinstance(value, str)+ or not value+ or "*" in value+ or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value)+ ):+ raise ArizeConfigurationError("invalid_configuration")+ except ValueError:+ raise ArizeConfigurationError("invalid_configuration") from None++ payload = ExportTraceServiceRequest()+ resource = payload.resource_spans.add().resource+ for name in ("model_id", "openinference.project.name"):+ resource.attributes.add(key=name).value.string_value = config.project or "default"+ headers = (+ ("api_key", config.api_key),+ ("space_id", config.space_id),+ ("authorization", f"Bearer {config.api_key}"),+ )+ authority = f"[{host}]" if ":" in host else host+ target = f"{authority}:{port or 443}"+ options = (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", _MAX_RESPONSE_BYTES))+ channel: grpc.Channel | None = None+ context = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))+ try:+ try:+ channel = (+ grpc.insecure_channel(target, options=options)+ if parsed.scheme == "http"+ else grpc.secure_channel(target, grpc.ssl_channel_credentials(), options=options)+ )+ result = TraceServiceStub(channel).Export(+ payload, metadata=headers, timeout=_VERIFY_TIMEOUT_SECONDS, wait_for_ready=False+ )+ if result.partial_success.rejected_spans or result.partial_success.error_message:+ raise ArizeConfigurationError("unexpected_response")+ finally:+ if channel is not None:+ channel.close()+ except ArizeConfigurationError:+ raise+ except grpc.RpcError as error:+ code = error.code()+ if code is grpc.StatusCode.UNAUTHENTICATED:+ raise ArizeConfigurationError("authentication_failed") from None+ if code is grpc.StatusCode.PERMISSION_DENIED:+ raise ArizeConfigurationError("permission_denied") from None+ raise ArizeConfigurationError("connection_failed") from None+ except Exception:+ raise ArizeConfigurationError("connection_failed") from None+ finally:+ detach(context)+++def _opaque_id(value: object) -> bool:+ # IDs remain opaque. In particular, never derive organization IDs by+ # decoding the project or space IDs, or accept path/query metacharacters.+ return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9+/_=-]{1,1024}", value) is not None+++def _identity(values: Mapping[str, object]) -> dict[str, str] | None:+ try:+ config = ArizeConfig.model_validate(+ {key: values[key] for key in ("endpoint", "space_id", "project") if key in values}+ )+ endpoint = urlsplit(config.endpoint)+ # These fixed API/UI hosts belong to the documented default collector.+ # Custom or other regional collectors keep ingestion support, but must+ # not send their credentials to an assumed Cloud metadata destination.+ if (+ endpoint.scheme != "https"+ or endpoint.hostname != "otlp.arize.com"+ or endpoint.port not in (None, 443)+ or endpoint.username is not None+ or endpoint.password is not None+ or not _opaque_id(config.space_id)+ ):+ return None+ except ValueError:+ return None+ assert config.space_id is not None+ return {"endpoint": config.endpoint, "space_id": config.space_id, "project": config.project or "default"}+++def project_url(values: Mapping[str, object]) -> str | None:+ """Compute only a verified cached destination; never perform vendor I/O."""+ destination = values.get(DESTINATION_KEY)+ if not isinstance(destination, Mapping):+ return None+ identity = _identity(destination)+ # Legacy Arize encrypts space_id as well as api_key. Bind the public cache+ # to this exact persisted record without decrypting it on status reads.+ digest = _source_digest(values)+ if identity is None or digest is None or destination.get("source_digest") != digest:+ return None+ organization_id, project_id = destination.get("organization_id"), destination.get("project_id")+ if not _opaque_id(organization_id) or not _opaque_id(project_id):+ return None+ assert isinstance(organization_id, str)+ assert isinstance(project_id, str)+ return (+ f"{_UI_BASE}/organizations/{quote(organization_id, safe='')}"+ f"/spaces/{quote(identity['space_id'], safe='')}/projects/{quote(project_id, safe='')}"+ )+++def _source_digest(values: Mapping[str, object]) -> str | None:+ parts = [values.get(key) for key in ("endpoint", "space_id", "project")]+ if not all(isinstance(part, str) for part in parts):+ return None+ return hashlib.sha256(json.dumps(parts, ensure_ascii=True, separators=(",", ":")).encode()).hexdigest()+++def bind_destination(values: Mapping[str, object], destination: dict[str, str]) -> dict[str, str]:+ """Bind server-resolved metadata after legacy secret encryption has finished."""+ digest = _source_digest(values)+ if digest is None:+ raise ValueError("Arize persisted destination is incomplete")+ return {**destination, "source_digest": digest}+++def _get_json(+ path: str, headers: dict[str, str], deadline: float, *, params: dict[str, str | int] | None = None+) -> dict[str, JsonValue] | None:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ return None+ try:+ response = ssrf_proxy.get(+ _API_BASE + path,+ headers=headers,+ params=params,+ auth=None,+ timeout=remaining,+ max_retries=0,+ follow_redirects=False,+ stream_response=True,+ )+ try:+ if response.status_code != 200 or time.monotonic() >= deadline:+ return None+ encoding = response.headers.get("content-encoding", "identity").strip().lower()+ if encoding not in {"", "identity"}:+ return None+ content = bytearray()+ for chunk in response.iter_bytes():+ if time.monotonic() >= deadline or len(content) + len(chunk) > _MAX_RESPONSE_BYTES:+ return None+ content.extend(chunk)+ if time.monotonic() >= deadline:+ return None+ return _JSON_OBJECT.validate_json(content, strict=True)+ finally:+ response.close()+ except Exception:+ # Metadata is optional. Do not expose request headers, response bodies,+ # credential-bearing exceptions, or turn read permission into an ingest+ # permission requirement.+ return None+++def resolve_project_destination(values: Mapping[str, object]) -> dict[str, str] | None:+ """Resolve exact project/space ownership within one shared five-second budget.++ The documented name filter is a substring match. Read at most three scoped+ pages and require one exact name/space match. Missing organization IDs on+ older servers, denied reads, malformed responses and budget exhaustion all+ mean no optional link, without modifying ingestion configuration or history.+ """+ identity = _identity(values)+ key = values.get("api_key")+ if (+ identity is None+ or not isinstance(key, str)+ or not key+ or "*" in key+ or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in key)+ ):+ return None+ deadline = time.monotonic() + _BUDGET_SECONDS+ headers = {+ "Authorization": f"Bearer {key}",+ "Accept": "application/json",+ "Accept-Encoding": "identity",+ "Cookie": "",+ }+ space = _get_json("/spaces/" + quote(identity["space_id"], safe=""), headers, deadline)+ if space is None or space.get("id") != identity["space_id"] or not _opaque_id(space.get("organization_id")):+ return None+ organization_id = space["organization_id"]+ assert isinstance(organization_id, str)+ project_ids: set[str] = set()+ seen_cursors: set[str] = set()+ params: dict[str, str | int] = {+ "space_id": identity["space_id"],+ "name": identity["project"],+ "limit": _PAGE_LIMIT,+ }+ for _ in range(_MAX_PAGES):+ page = _get_json("/projects", headers, deadline, params=params)+ if page is None:+ return None+ projects, pagination = page.get("projects"), page.get("pagination")+ if not isinstance(projects, list) or len(projects) > _PAGE_LIMIT or not isinstance(pagination, dict):+ return None+ for project in projects:+ if (+ not isinstance(project, dict)+ or not _opaque_id(project.get("id"))+ or not isinstance(project.get("name"), str)+ or project.get("space_id") != identity["space_id"]+ ):+ return None+ if project["name"] == identity["project"]:+ project_id = project["id"]+ assert isinstance(project_id, str)+ project_ids.add(project_id)+ if len(project_ids) > 1:+ return None+ has_more = pagination.get("has_more")+ if has_more is False:+ if len(project_ids) != 1:+ return None+ return {**identity, "organization_id": organization_id, "project_id": next(iter(project_ids))}+ cursor = pagination.get("next_cursor")+ if (+ has_more is not True+ or not isinstance(cursor, str)+ or not cursor+ or len(cursor) > 4096+ or cursor in seen_cursors+ ):+ return None+ seen_cursors.add(cursor)+ params = {**params, "cursor": cursor}+ return Nonediff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.pyindex c9105574c0e..d5ab4a570a3 100644--- a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/arize_phoenix_trace.py@@ -209,10 +209,16 @@ def setup_tracer(arize_phoenix_config: ArizeConfig | PhoenixConfig) -> tuple[tra)else:phoenix_endpoint = f"{base_endpoint}{path}/v1/traces"- phoenix_headers = {- "api_key": arize_phoenix_config.api_key or "",- "authorization": f"Bearer {arize_phoenix_config.api_key or ''}",- }+ # Keep headers nonempty so the SDK does not fall back to ambient+ # OTEL exporter credentials for an anonymous Phoenix destination.+ phoenix_headers = {"Content-Type": "application/x-protobuf"}+ if arize_phoenix_config.api_key:+ phoenix_headers.update(+ {+ "api_key": arize_phoenix_config.api_key,+ "authorization": f"Bearer {arize_phoenix_config.api_key}",+ }+ )exporter = HttpOTLPSpanExporter(endpoint=phoenix_endpoint,headers=phoenix_headers,@@ -1484,6 +1490,11 @@ class ArizePhoenixDataTrace(BaseTraceInstance):return self.carrierdef api_check(self):+ if isinstance(self.arize_phoenix_config, ArizeConfig):+ from dify_trace_arize_phoenix.arize_configuration import verify_configuration++ verify_configuration(self.arize_phoenix_config)+ return Truetry:with self.tracer.start_span("api_check") as span:span.set_attribute("test", "true")@@ -1493,14 +1504,16 @@ class ArizePhoenixDataTrace(BaseTraceInstance):raise ValueError(f"[Arize/Phoenix] API check failed: {str(e)}") from edef get_project_url(self):- """Build a redirect URL that forwards the user to the correct project for Arize/Phoenix."""+ """Get a Phoenix redirect; Arize Cloud requires verified destination metadata."""try:project_name = self.arize_phoenix_config.projectendpoint = self.arize_phoenix_config.endpoint.rstrip("/")- # Arize+ # Arize Cloud links require verified organization/space/project IDs.+ # Only explicit config writes resolve and cache them; this legacy+ # instance has none of that metadata and cannot promise a project.if isinstance(self.arize_phoenix_config, ArizeConfig):- return f"https://app.arize.com/?redirect_project_name={project_name}"+ return None# Phoenixreturn f"{endpoint}/projects/?redirect_project_name={project_name}"diff --git a/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/configuration.py b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/configuration.pynew file mode 100644index 00000000000..8085c7462ea--- /dev/null+++ b/api/providers/trace/trace-arize-phoenix/src/dify_trace_arize_phoenix/configuration.py@@ -0,0 +1,182 @@+"""Phoenix configuration checks never create a tracer, project, or test span.++A bounded project-list GET establishes read access only, including on a keyless+self-hosted server. It does not establish ingestion or project write permission.+Legacy Arize/Phoenix runtime adapters remain separate owners.+"""++import re+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Literal+from urllib.parse import quote, unquote, urlsplit, urlunsplit++from pydantic import TypeAdapter++from core.helper import ssrf_proxy+from dify_trace_arize_phoenix.config import PhoenixConfig++type VerificationCode = Literal[+ "invalid_endpoint",+ "invalid_project",+ "invalid_api_key",+ "authentication_failed",+ "permission_denied",+ "endpoint_unavailable",+ "malformed_response",+ "connection_failed",+]+_MESSAGES: dict[VerificationCode, str] = {+ "invalid_endpoint": "Phoenix requires an HTTP(S) base URL, optionally with a path prefix, without /v1/traces.",+ "invalid_project": "Phoenix requires a valid project name.",+ "invalid_api_key": "Phoenix requires an API key without whitespace or mask placeholders, or no key.",+ "authentication_failed": "Phoenix authentication failed.",+ "permission_denied": "Phoenix project-list access was denied.",+ "endpoint_unavailable": "Phoenix endpoint could not be verified.",+ "malformed_response": "Phoenix verification returned an invalid response.",+ "connection_failed": "Phoenix verification request failed.",+}+_JSON_OBJECT = TypeAdapter(dict[str, object])+_MAX_RESPONSE_BYTES = 1024 * 1024+_DEFAULT_ENDPOINT = "https://app.phoenix.arize.com"+++class PhoenixConfigurationError(ValueError):+ """Safe diagnostics exclude credentials, request URLs, and vendor response bodies."""++ def __init__(self, code: VerificationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++@dataclass(frozen=True, slots=True)+class ReadablePhoenixEndpoint:+ endpoint: str+ project: str+++def _endpoint(value: object) -> str:+ if not isinstance(value, str):+ raise PhoenixConfigurationError("invalid_endpoint")+ value = value.strip(" ") or _DEFAULT_ENDPOINT+ try:+ parsed = urlsplit(value)+ host, port = parsed.hostname, parsed.port+ path = parsed.path.rstrip("/")+ decoded_path = unquote(path, errors="strict")+ valid = (+ parsed.scheme in {"http", "https"}+ and host+ and parsed.username is None+ and parsed.password is None+ and "?" not in value+ and "#" not in value+ and "\\" not in value+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)+ and not parsed.netloc.endswith(":")+ and (port is None or 1 <= port <= 65535)+ and "%" not in host+ and not re.search(r"%(?![0-9a-fA-F]{2})", path)+ and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)+ and not any(char in decoded_path for char in "\\?#%")+ and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)+ and "//" not in parsed.path+ and not any(part in {".", ".."} for part in decoded_path.split("/"))+ and not decoded_path.endswith("/v1/traces")+ )+ if not valid or host is None:+ raise PhoenixConfigurationError("invalid_endpoint")+ host = host.encode("idna").decode("ascii").lower()+ authority = f"[{host}]" if ":" in host else host+ if port is not None:+ authority = f"{authority}:{port}"+ return urlunsplit((parsed.scheme, authority, path, "", ""))+ except (UnicodeError, ValueError):+ raise PhoenixConfigurationError("invalid_endpoint") from None+++def _project(value: object) -> str:+ if value is None:+ return "default"+ if not isinstance(value, str) or any(ord(char) < 32 or ord(char) == 127 for char in value):+ raise PhoenixConfigurationError("invalid_project")+ result = value.strip() or "default"+ if result in {".", ".."}:+ raise PhoenixConfigurationError("invalid_project")+ return result+++def normalize_configuration(values: Mapping[str, object]) -> PhoenixConfig:+ """Normalize new writes without tightening the model used to read old records.++ Secret contents are checked only after PATCH restores masked credentials.+ The project redirect loader uses a GraphQL name variable, not REST's more+ restrictive project-identifier path. Do not impose that REST limit here.+ """+ key = values.get("api_key")+ if key is not None and not isinstance(key, str):+ raise PhoenixConfigurationError("invalid_api_key")+ return PhoenixConfig(+ endpoint=_endpoint(values.get("endpoint", _DEFAULT_ENDPOINT)),+ project=_project(values.get("project")),+ api_key=key,+ )+++def project_url(config: PhoenixConfig) -> str:+ """Compute the configured UI redirect without SDK initialization or network I/O."""+ return f"{_endpoint(config.endpoint)}/redirects/projects/{quote(_project(config.project), safe='')}"+++def verify_configuration(config: PhoenixConfig) -> ReadablePhoenixEndpoint:+ """Read at most one page; an empty project list is valid and creates nothing."""+ config = normalize_configuration(config.model_dump())+ key = config.api_key+ if key and ("*" in key or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in key)):+ raise PhoenixConfigurationError("invalid_api_key")+ headers = {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}+ if key:+ headers["Authorization"] = f"Bearer {key}"+ try:+ response = ssrf_proxy.get(+ config.endpoint + "/v1/projects",+ params={"limit": 1},+ headers=headers,+ auth=None,+ timeout=30.0,+ max_retries=0,+ follow_redirects=False,+ stream_response=True,+ )+ except Exception:+ raise PhoenixConfigurationError("connection_failed") from None+ try:+ if response.status_code == 401:+ raise PhoenixConfigurationError("authentication_failed")+ if response.status_code == 403:+ raise PhoenixConfigurationError("permission_denied")+ if response.status_code != 200:+ raise PhoenixConfigurationError("endpoint_unavailable")+ try:+ buffered = ssrf_proxy.buffer_response(response, max_response_bytes=_MAX_RESPONSE_BYTES)+ data = _JSON_OBJECT.validate_python(buffered.json(), strict=True)+ except (ValueError, TypeError):+ raise PhoenixConfigurationError("malformed_response") from None+ except Exception:+ raise PhoenixConfigurationError("connection_failed") from None+ finally:+ response.close()+ rows, cursor = data.get("data"), data.get("next_cursor")+ if not isinstance(rows, list) or "next_cursor" not in data or (cursor is not None and not isinstance(cursor, str)):+ raise PhoenixConfigurationError("malformed_response")+ for row in rows:+ if (+ not isinstance(row, dict)+ or not isinstance(row.get("id"), str)+ or not row["id"]+ or not isinstance(row.get("name"), str)+ or not row["name"]+ or (row.get("description") is not None and not isinstance(row["description"], str))+ ):+ raise PhoenixConfigurationError("malformed_response")+ return ReadablePhoenixEndpoint(config.endpoint, _project(config.project))diff --git a/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py b/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.pyindex 870a3df5b1e..6394aefafe5 100644--- a/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py+++ b/api/providers/trace/trace-arize-phoenix/tests/unit_tests/arize_phoenix_trace/test_arize_phoenix_trace.py@@ -2049,7 +2049,8 @@ def test_construct_llm_attributes_dict_branch(trace_instance):assert attrs["llm.input_messages.0.message.role"] == "user"-def test_api_check_success(trace_instance):+def test_phoenix_api_check_success(trace_instance):+ trace_instance.arize_phoenix_config = PhoenixConfig(project="test", endpoint="https://phoenix.test")assert trace_instance.api_check() is Truediff --git a/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/agent_trace.py b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/agent_trace.pynew file mode 100644index 00000000000..841bd1227c9--- /dev/null+++ b/api/providers/trace/trace-langfuse/src/dify_trace_langfuse/agent_trace.py@@ -0,0 +1,75 @@+"""Export complete Agent/Workflow trees through Langfuse v4 OTLP ingestion."""++import base64+from collections.abc import Mapping+from datetime import timedelta+from typing import Any+from urllib.parse import quote++from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.otlp_export import build_request, export_http+from services.tracing.provider_common import endpoint, json_value, required, token_usage, usd_cost, utc+from services.tracing.provider_dispatch import TraceExportResult+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ host = endpoint(config.get("host", "https://cloud.langfuse.com"))+ public_key, secret_key = required(config, "public_key"), required(config, "secret_key")+ auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode("ascii")+ root = trace.spans[0]+ project_url = config.get("project_url")+ trace_url = f"{endpoint(project_url)}/traces/{trace.trace_id}" if project_url else None++ def attributes(span: CanonicalSpan) -> dict[str, object]:+ values: dict[str, object] = {+ "langfuse.trace.name": root.name,+ "langfuse.session.id": trace.session_id,+ "langfuse.observation.type": "generation" if span.kind is CanonicalSpanKind.LLM else span.kind.value,+ "langfuse.observation.input": json_value(span.inputs),+ "langfuse.observation.output": json_value(span.outputs),+ "langfuse.observation.level": "ERROR" if span.status is CanonicalSpanStatus.ERROR else "DEFAULT",+ "dify.span.id": span.id,+ }+ for key, value in root.metadata.items():+ if value is not None:+ values[f"langfuse.trace.metadata.{key}"] = value if isinstance(value, str) else json_value(value)+ for key, value in span.metadata.items():+ if value is not None:+ values[f"langfuse.observation.metadata.{key}"] = value if isinstance(value, str) else json_value(value)+ user_id = span.metadata.get("user_id") or root.metadata.get("user_id")+ if user_id:+ values["langfuse.user.id"] = str(user_id)+ if span.error:+ values["langfuse.observation.status_message"] = span.error+ if span.kind is CanonicalSpanKind.LLM:+ model = span.metadata.get("model") or span.metadata.get("model_name")+ if model:+ values["langfuse.observation.model.name"] = str(model)+ usage = token_usage(span)+ if usage:+ values["langfuse.observation.usage_details"] = json_value(usage)+ cost = usd_cost(span)+ if cost is not None:+ values["langfuse.observation.cost_details"] = json_value({"total": cost})+ ttft = span.metadata.get("time_to_first_token", span.metadata.get("ttft"))+ if isinstance(ttft, (int, float)) and ttft >= 0:+ values["langfuse.observation.completion_start_time"] = (+ utc(span.start_time) + timedelta(seconds=ttft)+ ).isoformat()+ return values++ payload = build_request(trace, {"service.name": "dify-application"}, attributes)+ export_http(+ provider,+ f"{host}/api/public/otel/v1/traces",+ {+ "Authorization": f"Basic {auth}",+ "x-langfuse-ingestion-version": "4",+ },+ payload,+ )+ return TraceExportResult(+ trace_id=trace.trace_id,+ trace_url=trace_url,+ span_urls={span.id: f"{trace_url}?observation={quote(span.id)}" for span in trace.spans} if trace_url else {},+ )diff --git a/api/providers/trace/trace-langsmith/src/dify_trace_langsmith/agent_trace.py b/api/providers/trace/trace-langsmith/src/dify_trace_langsmith/agent_trace.pynew file mode 100644index 00000000000..0f0be6c5e64--- /dev/null+++ b/api/providers/trace/trace-langsmith/src/dify_trace_langsmith/agent_trace.py@@ -0,0 +1,105 @@+"""Canonical Agent trees mapped to synchronous, isolated LangSmith runs."""++from collections.abc import Mapping+from typing import Any+from urllib.parse import quote+from uuid import NAMESPACE_URL, uuid5++from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from core.ops.utils import generate_dotted_order+from services.tracing.provider_common import endpoint, finished_at, required, token_usage, usd_cost, utc+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError+from services.tracing.sdk_transport import SSRFSession+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ from langsmith import Client+ from urllib3.util.retry import Retry++ api_url = endpoint(config.get("endpoint", "https://api.smith.langchain.com"))+ project = required(config, "project")+ ids = {span.id: str(uuid5(NAMESPACE_URL, f"dify:{trace.trace_id}:{span.id}")) for span in trace.spans}+ root_id = ids[trace.root_span_id]+ orders: dict[str, str] = {}+ accepted: set[str] = set()+ client = Client(+ api_key=required(config, "api_key"),+ api_url=api_url,+ session=SSRFSession(provider),+ retry_config=Retry(total=0),+ auto_batch_tracing=False,+ tracing_sampling_rate=1.0,+ tracing_mode="langsmith",+ omit_traced_runtime_info=True,+ hide_inputs=False,+ hide_outputs=False,+ info={},+ )+ try:+ for span in trace.spans:+ orders[span.id] = generate_dotted_order(+ ids[span.id], utc(span.start_time), orders.get(span.parent_id or "")+ )+ metadata = {+ **span.metadata,+ "session_id": trace.session_id,+ "external_trace_id": trace.trace_id,+ "dify.span.id": span.id,+ "dify.span.kind": span.kind.value,+ }+ model = span.metadata.get("model") or span.metadata.get("model_name")+ if span.kind is CanonicalSpanKind.LLM and model:+ metadata["ls_model_name"] = model+ metadata["ls_provider"] = span.metadata.get("model_provider")+ values: dict[str, Any] = {}+ usage = token_usage(span)+ values.update(+ {f"{key}_tokens" if key != "total" else "total_tokens": value for key, value in usage.items()}+ )+ cost = usd_cost(span)+ if cost is not None:+ values["total_cost"] = cost+ outputs = dict(span.outputs) if isinstance(span.outputs, Mapping) else {"output": span.outputs}+ if usage:+ outputs["usage_metadata"] = {f"{key}_tokens": value for key, value in usage.items()}+ client.create_run(+ id=ids[span.id],+ name=span.name,+ inputs=dict(span.inputs) if isinstance(span.inputs, Mapping) else {"input": span.inputs},+ outputs=outputs,+ run_type="chain" if span.kind is CanonicalSpanKind.AGENT else span.kind.value,+ project_name=project,+ start_time=utc(span.start_time),+ end_time=finished_at(span),+ parent_run_id=ids.get(span.parent_id or ""),+ trace_id=root_id,+ dotted_order=orders[span.id],+ extra={"metadata": metadata},+ error=(span.error or "Trace operation failed") if span.status is CanonicalSpanStatus.ERROR else None,+ **values,+ )+ accepted.add(span.id)+ except Exception as error:+ if accepted:+ raise TraceExportUncertainError(+ "LangSmith accepted part of the trace; reconciliation is required"+ ) from None+ # LangSmith wraps injected transport exceptions; recover our sanitized outcome.+ cause: BaseException | None = error+ while cause is not None:+ if isinstance(cause, (RetryableTraceDispatchError, TraceExportRejectedError)):+ raise cause from None+ cause = cause.__cause__+ raise TraceExportRejectedError("LangSmith rejected the trace") from None+ finally:+ client.close()+ project_url = config.get("project_url")+ trace_url = f"{str(project_url).rstrip('/')}/r/{quote(root_id)}" if project_url else None+ return TraceExportResult(+ trace_id=root_id,+ trace_url=trace_url,+ span_urls={span.id: f"{str(project_url).rstrip('/')}/r/{ids[span.id]}" for span in trace.spans}+ if project_url+ else {},+ )diff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/agent_trace.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/agent_trace.pynew file mode 100644index 00000000000..3bb26942e9b--- /dev/null+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/agent_trace.py@@ -0,0 +1,259 @@+"""MLflow SDK span serialization with per-export credentials and bounded HTTP.++OSS uses its OTLP experiment endpoint (MLflow >= 3.4). Databricks retains the+existing experiment destination through the SDK's V3 trace and artifact APIs;+its UC-only OTLP endpoint is not interchangeable with an experiment ID.+"""++import base64+import json+from collections.abc import Mapping+from typing import Any, override+from urllib.parse import quote, urlencode, urlsplit++from google.protobuf.json_format import ParseDict, ParseError+from mlflow.entities import Span, TraceData, TraceInfo, TraceLocation, TraceState+from mlflow.protos.databricks_artifacts_pb2 import (+ ArtifactCredentialType,+ DatabricksMlflowArtifactsService,+ GetCredentialsForTraceDataUpload,+)+from mlflow.store.tracking.rest_store import RestStore+from mlflow.tracing.constant import SpanAttributeKey, TraceMetadataKey, TraceTagKey+from mlflow.tracing.utils.otlp import MLFLOW_EXPERIMENT_ID_HEADER, OTLP_TRACES_PATH+from mlflow.utils.rest_utils import MlflowHostCreds, extract_api_info_for_service+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+from opentelemetry.sdk.resources import Resource+from opentelemetry.sdk.trace import ReadableSpan+from opentelemetry.trace import SpanContext, Status, StatusCode++from core.ops.unified_trace.entities import CanonicalSpanStatus, CanonicalTrace+from services.tracing.otlp_export import export_http+from services.tracing.provider_common import endpoint, json_value, nanos, request, required, token_usage, usd_cost+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError+++def _chat_tools(value: object) -> list[dict[str, Any]] | None:+ """Map OTel's flat function definitions to MLflow's chat tool UI schema.++ Preserve the original gen_ai attribute separately. An explicit native field+ prevents MLflow's OTLP translator from copying the incompatible flat shape.+ Invalid/unsupported entries stay in the source metadata, not in chat tools.+ """+ if isinstance(value, str):+ try:+ value = json.loads(value)+ except ValueError:+ return None+ if not isinstance(value, list):+ return None+ tools = []+ for item in value:+ if not isinstance(item, dict) or item.get("type") != "function":+ continue+ function = item.get("function")+ if not isinstance(function, dict):+ function = {key: item[key] for key in ("name", "description", "parameters", "strict") if key in item}+ name = function.get("name")+ if not isinstance(name, str) or not name.strip():+ continue+ tool = {"type": "function", "function": function}+ parameters = function.get("parameters")+ if parameters is not None and not isinstance(parameters, dict):+ continue+ # MLflow's Python ChatTool model accepts only a subset of JSON Schema+ # (e.g. it requires properties and restricts enum values). Preserve the+ # full schema for the UI instead of silently dropping valid definitions.+ tools.append(tool)+ return tools+++def _spans(trace: CanonicalTrace, experiment_id: str, request_id: str) -> list[Span]:+ result = []+ for span in trace.spans:+ attributes: dict[str, Any] = {+ **span.metadata,+ SpanAttributeKey.REQUEST_ID: request_id,+ SpanAttributeKey.EXPERIMENT_ID: experiment_id,+ SpanAttributeKey.SPAN_TYPE: span.kind.value.upper(),+ SpanAttributeKey.SESSION_ID: trace.session_id,+ }+ if span.inputs is not None:+ attributes[SpanAttributeKey.INPUTS] = span.inputs+ if span.outputs is not None:+ attributes[SpanAttributeKey.OUTPUTS] = span.outputs+ if SpanAttributeKey.CHAT_TOOLS not in attributes:+ tools = _chat_tools(span.metadata.get("gen_ai.tool.definitions"))+ if tools is not None:+ attributes[SpanAttributeKey.CHAT_TOOLS] = tools+ if usage := token_usage(span):+ attributes[SpanAttributeKey.CHAT_USAGE] = {f"{key}_tokens": value for key, value in usage.items()}+ if (cost := usd_cost(span)) is not None:+ attributes[SpanAttributeKey.LLM_COST] = {"total_cost": cost}+ if model := span.metadata.get("model_name") or span.metadata.get("model"):+ attributes[SpanAttributeKey.MODEL] = model+ context = SpanContext(trace_id=int(trace.trace_id, 16), span_id=int(span.id, 16), is_remote=False)+ parent = (+ SpanContext(trace_id=context.trace_id, span_id=int(span.parent_id, 16), is_remote=False)+ if span.parent_id+ else None+ )+ result.append(+ Span(+ ReadableSpan(+ name=span.name,+ context=context,+ parent=parent,+ start_time=nanos(span.start_time),+ end_time=nanos(span.end_time) if span.end_time else None,+ attributes={key: json_value(value) for key, value in attributes.items()},+ status=Status(StatusCode.ERROR, span.error)+ if span.status == CanonicalSpanStatus.ERROR+ else Status(StatusCode.OK),+ resource=Resource.get_empty(),+ )+ )+ )+ return result+++def _auth(provider: str, config: Mapping[str, Any], host: str) -> dict[str, str]:+ if provider == "mlflow":+ if config.get("username") and config.get("password"):+ token = base64.b64encode(f"{config['username']}:{config['password']}".encode()).decode()+ return {"Authorization": f"Basic {token}"}+ return {}+ if config.get("client_id") and config.get("client_secret"):+ response = request(+ "POST",+ host + "/oidc/v1/token",+ provider,+ auth=(config["client_id"], config["client_secret"]),+ data={"grant_type": "client_credentials", "scope": "all-apis"},+ )+ token = response.json().get("access_token")+ if not isinstance(token, str) or not token:+ raise TraceExportRejectedError("Databricks OAuth token was not returned")+ else:+ token = required(config, "personal_access_token")+ return {"Authorization": f"Bearer {token}"}+++class _Store(RestStore):+ """Reuse native V3 request/response schemas without process-wide SDK credentials."""++ def __init__(self, host: str, headers: dict[str, str]) -> None:+ super().__init__(lambda: MlflowHostCreds(host))+ self._host = host+ self._headers = headers++ @override+ def _call_endpoint(self, api, json_body=None, endpoint=None, retry_timeout_seconds=None, response_proto=None):+ methods = self._V3_METHOD_TO_INFO if api in self._V3_APIS else self._METHOD_TO_INFO+ default_endpoint, method = methods[api]+ response = request(+ method,+ self._host + (endpoint or default_endpoint),+ "databricks",+ headers={"Content-Type": "application/json", **self._headers},+ content=json_body,+ )+ try:+ return ParseDict(response.json(), response_proto if response_proto is not None else api.Response())+ except (ValueError, ParseError):+ raise TraceExportUncertainError("Databricks returned an unreadable trace acknowledgment") from None+++def _upload_databricks(host: str, headers: dict[str, str], trace_id: str, payload: bytes) -> None:+ path, method = extract_api_info_for_service(DatabricksMlflowArtifactsService, "/api/2.0")[+ GetCredentialsForTraceDataUpload+ ]+ response = request(+ method, host + path.replace("{request_id}", quote(trace_id, safe="")), "databricks", headers=headers+ )+ credential = ParseDict(response.json(), GetCredentialsForTraceDataUpload.Response()).credential_info+ signed_url = credential.signed_uri+ parsed = urlsplit(signed_url)+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:+ raise TraceExportRejectedError("Invalid Databricks artifact upload destination")+ upload_headers = {header.name: header.value for header in credential.headers}+ if credential.type == ArtifactCredentialType.AZURE_ADLS_GEN2_SAS_URI:+ # The bounded trace payload fits a single append. Flush commits that append.+ separator = "&" if parsed.query else "?"+ create_headers = {key: value for key, value in upload_headers.items() if key.lower() != "x-ms-blob-type"}+ request("PUT", signed_url + separator + "resource=file", "databricks", headers=create_headers)+ request(+ "PATCH",+ signed_url + separator + "action=append&position=0&flush=true",+ "databricks",+ headers=create_headers,+ content=payload,+ )+ elif credential.type in {+ ArtifactCredentialType.AWS_PRESIGNED_URL,+ ArtifactCredentialType.GCP_SIGNED_URL,+ ArtifactCredentialType.AZURE_SAS_URI,+ }:+ if credential.type == ArtifactCredentialType.AZURE_SAS_URI:+ upload_headers["x-ms-blob-type"] = "BlockBlob"+ request("PUT", signed_url, "databricks", headers=upload_headers, content=payload)+ else:+ raise TraceExportRejectedError("Unsupported Databricks artifact credential type")+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ host = endpoint(required(config, "host" if provider == "databricks" else "tracking_uri"))+ experiment = str(config.get("experiment_id") or "0")+ if not experiment.isdigit():+ raise TraceExportRejectedError("MLflow experiment ID must be numeric")+ headers = _auth(provider, config, host)+ trace_id = "tr-" + trace.trace_id+ spans = _spans(trace, experiment, trace_id)+ if provider == "mlflow":+ payload = ExportTraceServiceRequest()+ scope = payload.resource_spans.add().scope_spans.add()+ scope.spans.extend(span.to_otel_proto() for span in spans)+ export_http(provider, host + OTLP_TRACES_PATH, {MLFLOW_EXPERIMENT_ID_HEADER: experiment, **headers}, payload)+ # MLflow 3.4 has classic (selectedTraceId) and GenAI-v3+ # (selectedEvaluationId) views; 3.17's v4 reads traceId. On close/change,+ # v4 removes its selectedEvaluationId fallback before updating traceId.+ # Each older view reads only its own key and ignores the other aliases.+ query = urlencode({"traceId": trace_id, "selectedTraceId": trace_id, "selectedEvaluationId": trace_id})+ url = f"{host}/#/experiments/{experiment}/traces?{query}"+ else:+ root = next(span for span in trace.spans if span.id == trace.root_span_id)+ metadata = {TraceMetadataKey.TRACE_SESSION: trace.session_id}+ usages = [token_usage(span) for span in trace.spans]+ if any(usages):+ metadata[TraceMetadataKey.TOKEN_USAGE] = json_value(+ {f"{key}_tokens": sum(usage.get(key, 0) for usage in usages) for key in ("input", "output", "total")}+ )+ costs = [cost for span in trace.spans if (cost := usd_cost(span)) is not None]+ if costs:+ metadata[TraceMetadataKey.COST] = json_value({"total_cost": sum(costs)})+ info = TraceInfo(+ trace_id=trace_id,+ trace_location=TraceLocation.from_experiment_id(experiment),+ request_time=nanos(root.start_time) // 1_000_000,+ execution_duration=(nanos(root.end_time) - nanos(root.start_time)) // 1_000_000 if root.end_time else None,+ state=TraceState.ERROR if root.status == CanonicalSpanStatus.ERROR else TraceState.OK,+ request_preview=json_value(root.inputs)[:10000] if root.inputs is not None else None,+ response_preview=json_value(root.outputs)[:10000] if root.outputs is not None else None,+ trace_metadata=metadata,+ tags={TraceTagKey.TRACE_NAME: root.name},+ client_request_id=trace.trace_id,+ )+ stored = _Store(host, headers).start_trace(info)+ try:+ trace_id = stored.trace_id+ if not trace_id:+ raise TraceExportUncertainError("Databricks did not acknowledge a trace identifier")+ data = TraceData(spans=_spans(trace, experiment, trace_id))+ _upload_databricks(host, headers, trace_id, json_value(data.to_dict()).encode())+ except Exception:+ raise TraceExportUncertainError(+ "Databricks accepted trace metadata but artifact delivery is incomplete"+ ) from None+ # Databricks opens its trace detail panel with a different key from OSS MLflow.+ url = f"{host}/ml/experiments/{experiment}/traces?selectedEvaluationId={quote(trace_id, safe='')}"+ return TraceExportResult(trace_id=trace_id, trace_url=url)diff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/configuration.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/configuration.pynew file mode 100644index 00000000000..326930318d2--- /dev/null+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/configuration.py@@ -0,0 +1,163 @@+"""Read Databricks experiment metadata without changing MLflow's global state.++This verifies authentication and read access to an existing active experiment.+It does not prove write permission, tracing API availability, or artifact delivery.+"""++from dataclasses import dataclass+from typing import Literal+from urllib.parse import urlsplit, urlunsplit++from google.protobuf.json_format import ParseDict, ParseError+from mlflow.protos.service_pb2 import GetExperiment+from pydantic import TypeAdapter++from core.helper import ssrf_proxy+from dify_trace_mlflow.config import DatabricksConfig++type VerificationCode = Literal[+ "invalid_workspace_url",+ "invalid_experiment_id",+ "authentication_failed",+ "permission_denied",+ "experiment_not_found_or_inaccessible",+ "experiment_not_active",+ "experiment_id_mismatch",+ "malformed_response",+ "endpoint_unavailable",+ "connection_failed",+]++_MESSAGES: dict[VerificationCode, str] = {+ "invalid_workspace_url": "Databricks requires an HTTPS workspace origin.",+ "invalid_experiment_id": "Databricks requires a numeric experiment ID.",+ "authentication_failed": "Databricks authentication failed.",+ "permission_denied": "Databricks experiment access was denied.",+ "experiment_not_found_or_inaccessible": "The selected Databricks experiment was not found or is inaccessible.",+ "experiment_not_active": "The selected Databricks experiment is not active.",+ "experiment_id_mismatch": "Databricks returned a different experiment.",+ "malformed_response": "Databricks verification returned an invalid response.",+ "endpoint_unavailable": "Databricks endpoint could not be verified.",+ "connection_failed": "Databricks verification request failed.",+}+_JSON_OBJECT = TypeAdapter(dict[str, object])+++class DatabricksConfigurationError(ValueError):+ """Fixed safe diagnostics; never include credentials or a vendor response."""++ def __init__(self, code: VerificationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++@dataclass(frozen=True, slots=True)+class ReadableDatabricksExperiment:+ host: str+ experiment_id: str+++def _target(config: DatabricksConfig) -> ReadableDatabricksExperiment:+ host = config.host.strip()+ try:+ parsed = urlsplit(host)+ valid = (+ parsed.scheme == "https"+ and parsed.hostname+ and parsed.username is None+ and parsed.password is None+ and parsed.path in {"", "/"}+ and not parsed.query+ and not parsed.fragment+ and "\\" not in host+ and not any(ord(char) <= 32 or ord(char) == 127 for char in host)+ )+ _ = parsed.port+ except ValueError:+ raise DatabricksConfigurationError("invalid_workspace_url") from None+ if not valid:+ raise DatabricksConfigurationError("invalid_workspace_url")+ experiment_id = config.experiment_id.strip()+ if not experiment_id or not experiment_id.isascii() or not experiment_id.isdigit():+ raise DatabricksConfigurationError("invalid_experiment_id")+ return ReadableDatabricksExperiment(+ host=urlunsplit((parsed.scheme, parsed.netloc, "", "", "")),+ experiment_id=experiment_id.lstrip("0") or "0",+ )+++def project_url(config: DatabricksConfig) -> str:+ """Pure URL presentation, including for historical configurations."""+ target = _target(config)+ return f"{target.host}/ml/experiments/{target.experiment_id}/traces"+++def _request(method: Literal["GET", "POST"], url: str, **options: object) -> dict[str, object]:+ request = ssrf_proxy.get if method == "GET" else ssrf_proxy.post+ try:+ response = request(url, max_retries=0, timeout=30.0, follow_redirects=False, **options)+ except Exception:+ raise DatabricksConfigurationError("connection_failed") from None+ try:+ if response.status_code == 401:+ raise DatabricksConfigurationError("authentication_failed")+ if response.status_code == 403:+ raise DatabricksConfigurationError("permission_denied")+ if response.status_code == 404 and method == "GET":+ raise DatabricksConfigurationError("experiment_not_found_or_inaccessible")+ if not 200 <= response.status_code < 300:+ raise DatabricksConfigurationError("endpoint_unavailable")+ try:+ return _JSON_OBJECT.validate_python(response.json(), strict=True)+ except ValueError:+ raise DatabricksConfigurationError("malformed_response") from None+ finally:+ response.close()+++def verify_configuration(config: DatabricksConfig) -> ReadableDatabricksExperiment:+ """Read the exact experiment with the exporter's existing PAT/M2M precedence.++ GetExperiment uses the official MLflow request/response schema. No permissions,+ SCIM or account API is needed, and no experiment, run or trace is created.+ Read access alone is intentionally not presented as write authorization.+ """+ target = _target(config)+ if config.client_id and config.client_secret:+ data = _request(+ "POST",+ target.host + "/oidc/v1/token",+ auth=(config.client_id, config.client_secret),+ data={"grant_type": "client_credentials", "scope": "all-apis"},+ )+ access_token = data.get("access_token")+ token_type = data.get("token_type")+ if (+ not isinstance(access_token, str)+ or not access_token.strip()+ or not isinstance(token_type, str)+ or token_type.lower() != "bearer"+ ):+ raise DatabricksConfigurationError("authentication_failed")+ else:+ access_token = config.personal_access_token+ if not access_token or not access_token.strip():+ raise DatabricksConfigurationError("authentication_failed")+ request = GetExperiment(experiment_id=target.experiment_id)+ data = _request(+ "GET",+ target.host + "/api/2.0/mlflow/experiments/get",+ params={"experiment_id": request.experiment_id},+ headers={"Authorization": f"Bearer {access_token}", "Accept": "application/json"},+ )+ try:+ result = ParseDict(data, GetExperiment.Response(), ignore_unknown_fields=True)+ except (ValueError, TypeError, ParseError):+ raise DatabricksConfigurationError("malformed_response") from None+ if not result.HasField("experiment") or not result.experiment.experiment_id:+ raise DatabricksConfigurationError("malformed_response")+ if result.experiment.experiment_id != target.experiment_id:+ raise DatabricksConfigurationError("experiment_id_mismatch")+ if result.experiment.lifecycle_stage != "active":+ raise DatabricksConfigurationError("experiment_not_active")+ return targetdiff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.pyindex c89a69b90f8..0f7ffb08de1 100644--- a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/mlflow_trace.py@@ -1,13 +1,28 @@+import jsonimport loggingimport os+from collections.abc import Callable+from contextvars import ContextVarfrom datetime import datetime, timedeltafrom typing import Any, cast, overrideimport mlflow-from mlflow.entities import Document, LiveSpan, Span, SpanEvent, SpanStatusCode, SpanType+from mlflow.entities import Document, LiveSpan, Span, SpanEvent, SpanStatusCode, SpanType, TraceData, TraceInfo+from mlflow.entities.span import NoOpSpan+from mlflow.exceptions import RestException+from mlflow.tracing.client import TracingClientfrom mlflow.tracing.constant import SpanAttributeKey, TokenUsageKey, TraceMetadataKey+from mlflow.tracing.export.mlflow_v3 import MlflowV3SpanExporterfrom mlflow.tracing.fluent import start_span_no_context, update_current_trace+from mlflow.tracing.processor.mlflow_v3 import MlflowV3SpanProcessorfrom mlflow.tracing.provider import detach_span_from_context, set_span_in_context+from mlflow.tracing.trace_manager import InMemoryTraceManager+from mlflow.tracing.utils import encode_span_id, get_otel_attribute+from opentelemetry import trace as otel_trace+from opentelemetry.context import Context+from opentelemetry.sdk.resources import Resource+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.sampling import ALWAYS_ONfrom sqlalchemy import selectfrom core.ops.base_trace_instance import BaseTraceInstance@@ -41,42 +56,77 @@ def datetime_to_nanoseconds(dt: datetime | None) -> int | None:return int(dt.timestamp() * 1_000_000_000)-def _start_span_no_context(- *,- name: str,- span_type: str,- parent_span: LiveSpan | None = None,- inputs: object | None = None,- attributes: SpanAttributes | None = None,- start_time_ns: int | None = None,-) -> LiveSpan:- """Start an MLflow span while preserving structured Dify attributes.-- MLflow 3.11 annotates `start_span_no_context(..., attributes=...)` as `dict[str, str]`,- but the implementation immediately calls `LiveSpan.set_attributes(dict[str, Any])`.- `LiveSpan` JSON-serializes arbitrary values before storing them in OpenTelemetry, and- reserved attributes like `mlflow.chat.tokenUsage` are expected to round-trip as dicts.- """- return start_span_no_context(- name=name,- span_type=span_type,- parent_span=parent_span,- inputs=inputs,- attributes=cast(dict[str, str] | None, attributes),- start_time_ns=start_time_ns,- )+class _CheckedTracingClient(TracingClient):+ """Remember errors hidden by SDK exporters without changing SDK serialization or I/O."""++ def __init__(self, tracking_uri: str, failed: ContextVar[bool]):+ super().__init__(tracking_uri)+ self._failed = failed++ def _checked[T](self, operation: Callable[[], T]) -> T:+ try:+ return operation()+ except Exception:+ self._failed.set(True)+ raise++ @override+ def start_trace(self, trace_info: TraceInfo) -> TraceInfo:+ return self._checked(lambda: super(_CheckedTracingClient, self).start_trace(trace_info))++ @override+ def _upload_trace_data(self, trace_info: TraceInfo, trace_data: TraceData) -> None:+ self._checked(lambda: super(_CheckedTracingClient, self)._upload_trace_data(trace_info, trace_data))++ @override+ def log_spans(self, location: str, spans: list[Span]) -> list[Span]:+ try:+ return super().log_spans(location, spans)+ except NotImplementedError:+ # SDK falls back to the full trace artifact on pre-3.4/file-store servers.+ raise+ except RestException as error:+ if "REST OTLP span logging is not supported" not in error.message:+ self._failed.set(True)+ raise+ except Exception:+ self._failed.set(True)+ raise+++class _SynchronousMlflowExporter(MlflowV3SpanExporter):+ """Celery owns asynchronous dispatch; no SDK queue may survive adapter LRU eviction."""++ def __init__(self, tracking_uri: str):+ self.failed: ContextVar[bool] = ContextVar("mlflow_legacy_export_failed", default=False)+ super().__init__(tracking_uri)+ self._client = _CheckedTracingClient(tracking_uri, self.failed)++ @override+ def _should_enable_async_logging(self) -> bool:+ return Falseclass MLflowDataTrace(BaseTraceInstance):def __init__(self, config: MLflowConfig | DatabricksConfig):super().__init__(config)+ self._tracer: otel_trace.Tracer | None = None+ self._exporter: _SynchronousMlflowExporter | None = None+ self._experiment_id = config.experiment_idif isinstance(config, DatabricksConfig):+ # Preserve the SDK's existing Databricks/UC destination selection.self._setup_databricks(config)- else:- self._setup_mlflow(config)+ os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] = "true"+ return+ self._setup_mlflow(config)- # Enable async logging to minimize performance overhead- os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] = "true"+ # The public MLflow destination API still replaces the SDK's shared provider+ # and can mutate OTEL_RESOURCE_ATTRIBUTES. Keep this legacy SDK pipeline private.+ # No SDK background queue or atexit callback may retain an evicted adapter.+ self._provider = TracerProvider(resource=Resource({}), sampler=ALWAYS_ON, shutdown_on_exit=False)+ self._exporter = _SynchronousMlflowExporter(tracking_uri=self._tracking_uri)+ self._provider.add_span_processor(MlflowV3SpanProcessor(self._exporter, export_metrics=False))+ self._tracer = self._provider.get_tracer(__name__)def _setup_databricks(self, config: DatabricksConfig):"""Setup connection to Databricks-managed MLflow instances"""@@ -95,6 +145,7 @@ class MLflowDataTrace(BaseTraceInstance):"See https://docs.databricks.com/aws/en/dev-tools/auth/#what-authorization-option-should-i-choose ""for more information about the authorization options.")+ self._tracking_uri = "databricks"mlflow.set_tracking_uri("databricks")mlflow.set_experiment(experiment_id=config.experiment_id)@@ -104,8 +155,7 @@ class MLflowDataTrace(BaseTraceInstance):def _setup_mlflow(self, config: MLflowConfig):"""Setup connection to MLflow instances"""- mlflow.set_tracking_uri(config.tracking_uri)- mlflow.set_experiment(experiment_id=config.experiment_id)+ self._tracking_uri = config.tracking_uri# Simple auth if providedif config.username and config.password:@@ -114,9 +164,73 @@ class MLflowDataTrace(BaseTraceInstance):self._project_url = f"{config.tracking_uri}/#/experiments/{config.experiment_id}/traces"+ def _start_span_no_context(+ self,+ *,+ name: str,+ span_type: str,+ parent_span: LiveSpan | None = None,+ inputs: object | None = None,+ attributes: SpanAttributes | None = None,+ start_time_ns: int | None = None,+ ) -> LiveSpan:+ """Use the real SDK processor without inheriting a process-wide destination.++ Every span needs the experiment attribute: the SDK's incremental exporter+ resolves each child independently, rather than inheriting the root's location.+ A missing/nonrecording span is an export failure, never successful dispatch.+ """+ if self._tracer is None:+ span = start_span_no_context(+ name=name,+ span_type=span_type,+ parent_span=parent_span,+ inputs=inputs,+ attributes=cast(dict[str, str] | None, attributes),+ start_time_ns=start_time_ns,+ )+ if isinstance(span, NoOpSpan):+ raise RuntimeError("mlflow_legacy_span_not_recording")+ return span+ context = Context()+ if parent_span is not None:+ parent = parent_span._span+ if not isinstance(parent, otel_trace.Span):+ raise RuntimeError("mlflow_legacy_parent_span_invalid")+ context = otel_trace.set_span_in_context(parent, context)+ try:+ otel_span = self._tracer.start_span(+ name,+ context=context,+ attributes={SpanAttributeKey.EXPERIMENT_ID: json.dumps(self._experiment_id)},+ start_time=start_time_ns,+ )+ except Exception:+ raise RuntimeError("mlflow_legacy_span_start_failed") from None+ if not otel_span.is_recording():+ raise RuntimeError("mlflow_legacy_span_not_recording")+ trace_id = get_otel_attribute(otel_span, SpanAttributeKey.REQUEST_ID)+ manager = InMemoryTraceManager.get_instance()+ span = (+ manager.get_span_from_id(trace_id, encode_span_id(otel_span.get_span_context().span_id))+ if trace_id+ else None+ )+ if not isinstance(span, LiveSpan):+ manager.pop_trace(otel_span.get_span_context().trace_id)+ otel_span.end()+ raise RuntimeError("mlflow_legacy_span_not_registered")+ span.set_span_type(span_type)+ if inputs is not None:+ span.set_inputs(inputs)+ span.set_attributes(attributes or {})+ span.set_attribute(SpanAttributeKey.EXPERIMENT_ID, self._experiment_id)+ return span+@overridedef trace(self, trace_info: BaseTraceInfo):- """Simple dispatch to trace methods"""+ """Dispatch legacy data; synchronous OSS export failures must reach the task owner."""+ failure_token = self._exporter.failed.set(False) if self._exporter is not None else Nonetry:match trace_info:case WorkflowTraceInfo():@@ -133,9 +247,14 @@ class MLflowDataTrace(BaseTraceInstance):self.suggested_question_trace(trace_info)case GenerateNameTraceInfo():self.generate_name_trace(trace_info)+ if self._exporter is not None and self._exporter.failed.get():+ raise RuntimeError("mlflow_legacy_export_failed")except Exception:logger.exception("[MLflow] Trace error")raise+ finally:+ if self._exporter is not None and failure_token is not None:+ self._exporter.failed.reset(failure_token)def workflow_trace(self, trace_info: WorkflowTraceInfo):"""Create workflow span as root, with node spans as children"""@@ -147,7 +266,7 @@ class MLflowDataTrace(BaseTraceInstance):if trace_info.query:workflow_inputs["query"] = trace_info.query- workflow_span = _start_span_no_context(+ workflow_span = self._start_span_no_context(name=TraceTaskName.WORKFLOW_TRACE.value,span_type=SpanType.CHAIN,inputs=workflow_inputs,@@ -185,7 +304,7 @@ class MLflowDataTrace(BaseTraceInstance):if not inputs:inputs = JSON_DICT_ADAPTER.validate_json(node.inputs) if node.inputs else {}- node_span = _start_span_no_context(+ node_span = self._start_span_no_context(name=node.title,span_type=self._get_node_span_type(node.node_type),parent_span=workflow_span,@@ -294,7 +413,7 @@ class MLflowDataTrace(BaseTraceInstance):base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001")file_list.append(f"{base_url}/{message_file_data.url}")- span = _start_span_no_context(+ span = self._start_span_no_context(name=TraceTaskName.MESSAGE_TRACE.value,span_type=SpanType.LLM,inputs=self._parse_prompts(trace_info.inputs),@@ -358,7 +477,7 @@ class MLflowDataTrace(BaseTraceInstance):return metadata.get("from_account_id") # type: ignore[return-value]def tool_trace(self, trace_info: ToolTraceInfo):- span = _start_span_no_context(+ span = self._start_span_no_context(name=trace_info.tool_name,span_type=SpanType.TOOL,inputs=trace_info.tool_inputs,@@ -395,7 +514,7 @@ class MLflowDataTrace(BaseTraceInstance):returnstart_time = trace_info.start_time or trace_info.message_data.created_at- span = _start_span_no_context(+ span = self._start_span_no_context(name=TraceTaskName.MODERATION_TRACE.value,span_type=SpanType.TOOL,inputs=trace_info.inputs or {},@@ -419,7 +538,7 @@ class MLflowDataTrace(BaseTraceInstance):if trace_info.message_data is None:return- span = _start_span_no_context(+ span = self._start_span_no_context(name=TraceTaskName.DATASET_RETRIEVAL_TRACE.value,span_type=SpanType.RETRIEVER,inputs=trace_info.inputs,@@ -438,7 +557,7 @@ class MLflowDataTrace(BaseTraceInstance):start_time = trace_info.start_time or trace_info.message_data.created_atend_time = trace_info.end_time or trace_info.message_data.updated_at- span = _start_span_no_context(+ span = self._start_span_no_context(name=TraceTaskName.SUGGESTED_QUESTION_TRACE.value,span_type=SpanType.TOOL,inputs=trace_info.inputs,@@ -467,7 +586,7 @@ class MLflowDataTrace(BaseTraceInstance):span.end(outputs=trace_info.suggested_question, end_time_ns=datetime_to_nanoseconds(end_time))def generate_name_trace(self, trace_info: GenerateNameTraceInfo):- span = _start_span_no_context(+ span = self._start_span_no_context(name=TraceTaskName.GENERATE_NAME_TRACE.value,span_type=SpanType.CHAIN,inputs=trace_info.inputs,@@ -566,7 +685,7 @@ class MLflowDataTrace(BaseTraceInstance):def api_check(self):"""Simple connection test"""try:- mlflow.search_experiments(max_results=1)+ mlflow.MlflowClient(tracking_uri=self._tracking_uri).search_experiments(max_results=1)return Trueexcept Exception as e:raise ValueError(f"MLflow connection failed: {str(e)}") from ediff --git a/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/oss_configuration.py b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/oss_configuration.pynew file mode 100644index 00000000000..20c201e00f0--- /dev/null+++ b/api/providers/trace/trace-mlflow/src/dify_trace_mlflow/oss_configuration.py@@ -0,0 +1,201 @@+"""Read one OSS MLflow experiment without changing SDK or authentication state.++Configuration verification proves only read access to an existing active experiment.+It does not establish trace ingestion, artifact delivery, or write permission. The+legacy exporter remains a separate owner; configuration never constructs it.+"""++import base64+import re+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Literal+from urllib.parse import unquote, urlsplit, urlunsplit++from google.protobuf.json_format import ParseDict, ParseError+from mlflow.protos.service_pb2 import GetExperiment+from pydantic import TypeAdapter++from core.helper import ssrf_proxy+from dify_trace_mlflow.config import MLflowConfig++type VerificationCode = Literal[+ "invalid_tracking_uri",+ "invalid_experiment_id",+ "incomplete_basic_auth",+ "authentication_failed",+ "permission_denied",+ "experiment_not_found_or_inaccessible",+ "experiment_not_active",+ "experiment_id_mismatch",+ "malformed_response",+ "endpoint_unavailable",+ "connection_failed",+]++_MESSAGES: dict[VerificationCode, str] = {+ "invalid_tracking_uri": "MLflow requires an HTTP(S) tracking server URL with an optional path prefix.",+ "invalid_experiment_id": "MLflow requires a numeric experiment ID.",+ "incomplete_basic_auth": "MLflow username and password must both be supplied or both be empty.",+ "authentication_failed": "MLflow authentication failed.",+ "permission_denied": "MLflow experiment access was denied.",+ "experiment_not_found_or_inaccessible": "The selected MLflow experiment was not found or is inaccessible.",+ "experiment_not_active": "The selected MLflow experiment is not active.",+ "experiment_id_mismatch": "MLflow returned a different experiment.",+ "malformed_response": "MLflow verification returned an invalid response.",+ "endpoint_unavailable": "MLflow endpoint could not be verified.",+ "connection_failed": "MLflow verification request failed.",+}+_JSON_OBJECT = TypeAdapter(dict[str, object])+_MAX_RESPONSE_BYTES = 1024 * 1024+++class MLflowConfigurationError(ValueError):+ """Fixed diagnostics that never include request values or vendor response text."""++ def __init__(self, code: VerificationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++@dataclass(frozen=True, slots=True)+class ReadableMLflowExperiment:+ tracking_uri: str+ experiment_id: str+++def _tracking_uri(value: object) -> str:+ if not isinstance(value, str):+ raise MLflowConfigurationError("invalid_tracking_uri")+ # Only surrounding ordinary spaces are normalization; URL parsers silently+ # discard some control characters, so inspect the original input first.+ value = value.strip(" ")+ try:+ parsed = urlsplit(value)+ host, port = parsed.hostname, parsed.port+ path = parsed.path.rstrip("/")+ decoded_path = unquote(path, errors="strict")+ valid = (+ parsed.scheme in {"http", "https"}+ and host+ and parsed.username is None+ and parsed.password is None+ and "?" not in value+ and "#" not in value+ and "\\" not in value+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)+ and not parsed.netloc.endswith(":")+ and (port is None or 1 <= port <= 65535)+ and "%" not in host+ and not re.search(r"%(?![0-9a-fA-F]{2})", path)+ and not re.search(r"%(?:2f|5c)", path, re.IGNORECASE)+ and not any(char in decoded_path for char in "\\?#%")+ and not any(ord(char) < 32 or ord(char) == 127 for char in decoded_path)+ and "//" not in parsed.path+ and not any(part in {".", ".."} for part in decoded_path.split("/"))+ )+ if not valid or host is None:+ raise MLflowConfigurationError("invalid_tracking_uri")+ host = host.encode("idna").decode("ascii").lower()+ authority = f"[{host}]" if ":" in host else host+ if port is not None:+ authority = f"{authority}:{port}"+ return urlunsplit((parsed.scheme, authority, path, "", ""))+ except (UnicodeError, ValueError):+ raise MLflowConfigurationError("invalid_tracking_uri") from None+++def _experiment_id(value: str) -> str:+ value = value.strip()+ if not value or not value.isascii() or not value.isdigit():+ raise MLflowConfigurationError("invalid_experiment_id")+ return value.lstrip("0") or "0"+++def normalize_configuration(values: Mapping[str, object]) -> MLflowConfig:+ """Validate new destinations while retaining masked/nullable password values.++ Pair validation belongs after masked credentials are restored and decrypted.+ Keeping it out of MLflowConfig also leaves historical records readable and+ avoids interpreting RSA ciphertext as a user-supplied password.+ """+ config = MLflowConfig.model_validate(values)+ return config.model_copy(+ update={+ "tracking_uri": _tracking_uri(values.get("tracking_uri", config.tracking_uri)),+ "experiment_id": _experiment_id(config.experiment_id),+ }+ )+++def project_url(config: MLflowConfig) -> str:+ """Pure link calculation, independent of authentication and server access."""+ host = _tracking_uri(config.tracking_uri)+ experiment_id = _experiment_id(config.experiment_id)+ return f"{host}/#/experiments/{experiment_id}/traces"+++def verify_configuration(config: MLflowConfig) -> ReadableMLflowExperiment:+ """GET the exact experiment with only this request's optional Basic credentials."""+ config = normalize_configuration(config.model_dump())+ username, password = config.username, config.password+ has_username, has_password = bool(username), bool(password)+ if has_username != has_password or (+ has_username+ and (+ username is None+ or password is None+ or not username.strip()+ or not password.strip()+ or ":" in username+ or any(ord(char) < 32 or ord(char) == 127 for char in username + password)+ )+ ):+ raise MLflowConfigurationError("incomplete_basic_auth")++ # Explicit Cookie suppresses the pooled HTTP client's ambient cookie jar.+ # No MLflow environment variables, SDK client, or process-wide auth is read.+ headers = {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}+ if username and password:+ encoded = base64.b64encode(f"{username}:{password}".encode()).decode()+ headers["Authorization"] = f"Basic {encoded}"+ request = GetExperiment(experiment_id=config.experiment_id)+ try:+ response = ssrf_proxy.get(+ config.tracking_uri + "/api/2.0/mlflow/experiments/get",+ params={"experiment_id": request.experiment_id},+ headers=headers,+ auth=None,+ max_retries=0,+ timeout=30.0,+ follow_redirects=False,+ stream_response=True,+ )+ except Exception:+ raise MLflowConfigurationError("connection_failed") from None+ try:+ if response.status_code == 401:+ raise MLflowConfigurationError("authentication_failed")+ if response.status_code == 403:+ raise MLflowConfigurationError("permission_denied")+ if response.status_code == 404:+ raise MLflowConfigurationError("experiment_not_found_or_inaccessible")+ if not 200 <= response.status_code < 300:+ raise MLflowConfigurationError("endpoint_unavailable")+ try:+ buffered = ssrf_proxy.buffer_response(response, max_response_bytes=_MAX_RESPONSE_BYTES)+ data = _JSON_OBJECT.validate_python(buffered.json(), strict=True)+ result = ParseDict(data, GetExperiment.Response(), ignore_unknown_fields=True)+ except (ValueError, TypeError, ParseError):+ raise MLflowConfigurationError("malformed_response") from None+ except Exception:+ raise MLflowConfigurationError("connection_failed") from None+ finally:+ response.close()+ if not result.HasField("experiment") or not result.experiment.experiment_id:+ raise MLflowConfigurationError("malformed_response")+ if result.experiment.experiment_id != config.experiment_id:+ raise MLflowConfigurationError("experiment_id_mismatch")+ if result.experiment.lifecycle_stage != "active":+ raise MLflowConfigurationError("experiment_not_active")+ return ReadableMLflowExperiment(config.tracking_uri, config.experiment_id)diff --git a/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py b/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.pyindex 7e1aa5f43f0..e4ecf4db806 100644--- a/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py+++ b/api/providers/trace/trace-mlflow/tests/unit_tests/mlflow_trace/test_mlflow_trace.py@@ -188,7 +188,7 @@ def mock_mlflow():def mock_tracing():"""Patch all MLflow tracing functions used by the module."""with (- patch("dify_trace_mlflow.mlflow_trace.start_span_no_context") as mock_start,+ patch("dify_trace_mlflow.mlflow_trace.MLflowDataTrace._start_span_no_context") as mock_start,patch("dify_trace_mlflow.mlflow_trace.update_current_trace") as mock_update,patch("dify_trace_mlflow.mlflow_trace.set_span_in_context") as mock_set,patch("dify_trace_mlflow.mlflow_trace.detach_span_from_context") as mock_detach,@@ -234,10 +234,10 @@ class TestInit:def test_mlflow_config_no_auth(self, mock_mlflow):config = MLflowConfig(tracking_uri="http://localhost:5000", experiment_id="0")trace = MLflowDataTrace(config)- mock_mlflow.set_tracking_uri.assert_called_with("http://localhost:5000")- mock_mlflow.set_experiment.assert_called_with(experiment_id="0")+ mock_mlflow.set_tracking_uri.assert_not_called()+ mock_mlflow.set_experiment.assert_not_called()assert trace.get_project_url() == "http://localhost:5000/#/experiments/0/traces"- assert os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] == "true"+ assert trace._tracking_uri == "http://localhost:5000"def test_mlflow_config_with_auth(self, mock_mlflow):config = MLflowConfig(@@ -262,6 +262,7 @@ class TestInit:assert os.environ["DATABRICKS_CLIENT_ID"] == "cid"assert os.environ["DATABRICKS_CLIENT_SECRET"] == "csec"mock_mlflow.set_tracking_uri.assert_called_with("databricks")+ assert trace._tracking_uri == "databricks"# Trailing slash strippedassert trace.get_project_url() == "https://db.com/ml/experiments/42/traces"@@ -1034,11 +1035,11 @@ class TestResolveToolCallIds:class TestApiCheck:def test_success(self, trace_instance, mock_mlflow):- mock_mlflow.search_experiments.return_value = []+ mock_mlflow.MlflowClient.return_value.search_experiments.return_value = []assert trace_instance.api_check() is Truedef test_failure(self, trace_instance, mock_mlflow):- mock_mlflow.search_experiments.side_effect = ConnectionError("refused")+ mock_mlflow.MlflowClient.return_value.search_experiments.side_effect = ConnectionError("refused")with pytest.raises(ValueError, match="MLflow connection failed"):trace_instance.api_check()diff --git a/api/providers/trace/trace-opik/src/dify_trace_opik/agent_trace.py b/api/providers/trace/trace-opik/src/dify_trace_opik/agent_trace.pynew file mode 100644index 00000000000..db03c1a8d36--- /dev/null+++ b/api/providers/trace/trace-opik/src/dify_trace_opik/agent_trace.py@@ -0,0 +1,129 @@+"""Synchronous Opik SDK ingestion with native UUIDv7, thread, and usage fields."""++from collections.abc import Mapping+from datetime import datetime+from typing import Any+from uuid import NAMESPACE_URL, UUID, uuid5++from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.provider_common import endpoint, finished_at, json_value, token_usage, usd_cost, utc+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError+from services.tracing.sdk_transport import SSRFTransport+++def _provider_id(trace_id: str, span_id: str, started_at: datetime) -> str:+ from opik.id_helpers import uuid4_to_uuid7++ seeded = UUID(str(uuid5(NAMESPACE_URL, f"dify:{trace_id}:{span_id}")), version=4)+ return str(uuid4_to_uuid7(utc(started_at), str(seeded)))+++def _content(value: Any) -> dict[str, Any] | list[dict[str, Any]] | str | None:+ """Opik accepts objects, object lists and strings, but not every JSON value."""+ if value is None or isinstance(value, (dict, str)):+ return value+ if isinstance(value, list) and all(isinstance(item, dict) for item in value):+ return value+ return json_value(value)+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ import httpx+ from opik.rest_api.client import OpikApi+ from opik.rest_api.types.error_info_write import ErrorInfoWrite+ from opik.rest_api.types.span_write import SpanWrite++ url = endpoint(config.get("url", "https://www.comet.com/opik/api"))+ project = str(config.get("project") or "Default Project")+ root = trace.spans[0]+ trace_id = _provider_id(trace.trace_id, "trace", root.start_time)+ ids = {span.id: _provider_id(trace.trace_id, span.id, span.start_time) for span in trace.spans}+ accepted = False+ with httpx.Client(transport=SSRFTransport(provider), timeout=30) as http_client:+ client = OpikApi(+ base_url=url,+ api_key=config.get("api_key"),+ workspace_name=str(config.get("workspace") or "default"),+ httpx_client=http_client,+ )+ spans = []+ for span in trace.spans:+ usage = token_usage(span)+ native_usage = {+ "prompt_tokens" if key == "input" else "completion_tokens" if key == "output" else "total_tokens": value+ for key, value in usage.items()+ }+ ttft = span.metadata.get("time_to_first_token", span.metadata.get("ttft"))+ spans.append(+ SpanWrite(+ id=ids[span.id],+ trace_id=trace_id,+ parent_span_id=ids.get(span.parent_id or ""),+ project_name=project,+ name=span.name,+ type="llm"+ if span.kind is CanonicalSpanKind.LLM+ else "tool"+ if span.kind is CanonicalSpanKind.TOOL+ else "general",+ start_time=utc(span.start_time),+ end_time=finished_at(span),+ input=_content(span.inputs),+ output=_content(span.outputs),+ metadata={+ **span.metadata,+ "dify.span.kind": span.kind.value,+ "dify.span.id": span.id,+ "dify.trace_id": trace.trace_id,+ "session_id": trace.session_id,+ },+ model=str(span.metadata.get("model") or span.metadata.get("model_name") or "")+ if span.kind is CanonicalSpanKind.LLM+ else None,+ provider=str(span.metadata.get("model_provider") or "")+ if span.kind is CanonicalSpanKind.LLM+ else None,+ usage=native_usage or None,+ total_estimated_cost=usd_cost(span),+ ttft=float(ttft) * 1000+ if span.kind is CanonicalSpanKind.LLM and isinstance(ttft, (int, float)) and ttft >= 0+ else None,+ error_info=ErrorInfoWrite(+ exception_type="DifyExecutionError",+ message=span.error or "Trace operation failed",+ traceback="",+ )+ if span.status is CanonicalSpanStatus.ERROR+ else None,+ )+ )+ try:+ client.traces.create_trace(+ id=trace_id,+ project_name=project,+ name=root.name,+ start_time=utc(root.start_time),+ end_time=finished_at(root),+ input=_content(root.inputs),+ output=_content(root.outputs),+ metadata={**root.metadata, "dify.trace_id": trace.trace_id},+ error_info=ErrorInfoWrite(+ exception_type="DifyExecutionError",+ message=root.error or "Trace operation failed",+ traceback="",+ )+ if root.status is CanonicalSpanStatus.ERROR+ else None,+ thread_id=trace.session_id,+ request_options={"max_retries": 0},+ )+ accepted = True+ client.spans.create_spans(spans=spans, request_options={"max_retries": 0})+ except Exception as error:+ if accepted:+ raise TraceExportUncertainError("Opik accepted part of the trace; reconciliation is required") from None+ if isinstance(error, (RetryableTraceDispatchError, TraceExportRejectedError)):+ raise+ raise TraceExportRejectedError("Opik rejected the trace") from None+ return TraceExportResult(trace_id=trace_id)diff --git a/api/providers/trace/trace-opik/src/dify_trace_opik/configuration.py b/api/providers/trace/trace-opik/src/dify_trace_opik/configuration.pynew file mode 100644index 00000000000..e2152b906fb--- /dev/null+++ b/api/providers/trace/trace-opik/src/dify_trace_opik/configuration.py@@ -0,0 +1,156 @@+"""Check Opik workspace access without SDK streamers or ambient credentials.++The auth endpoint is read-only despite using POST. Verification does not create+projects or traces and does not establish project write or ingestion permission.+"""++from dataclasses import dataclass+from typing import Literal+from urllib.parse import quote, urlencode, urlsplit++from pydantic import TypeAdapter++from core.helper import ssrf_proxy+from dify_trace_opik.config import OpikConfig++type VerificationCode = Literal[+ "invalid_endpoint",+ "invalid_workspace",+ "invalid_project",+ "authentication_failed",+ "permission_denied",+ "endpoint_unavailable",+ "malformed_response",+ "connection_failed",+]++_MESSAGES: dict[VerificationCode, str] = {+ "invalid_endpoint": "Opik requires an HTTP(S) API URL ending in /api/.",+ "invalid_workspace": "Opik requires a valid workspace name.",+ "invalid_project": "Opik requires a valid project name.",+ "authentication_failed": "Opik authentication failed.",+ "permission_denied": "Opik workspace access was denied.",+ "endpoint_unavailable": "Opik endpoint could not be verified.",+ "malformed_response": "Opik verification returned an invalid response.",+ "connection_failed": "Opik verification request failed.",+}+_JSON_OBJECT = TypeAdapter(dict[str, object])+++class OpikConfigurationError(ValueError):+ """Fixed safe codes; no credentials, vendor bodies or request details."""++ def __init__(self, code: VerificationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++@dataclass(frozen=True, slots=True)+class VerifiedOpikWorkspace:+ url: str+ workspace: str+ project: str+++def _api_url(config: OpikConfig) -> str:+ value = config.url.strip()+ try:+ parsed = urlsplit(value)+ valid = (+ parsed.scheme in {"http", "https"}+ and parsed.hostname+ and parsed.username is None+ and parsed.password is None+ and not parsed.query+ and not parsed.fragment+ and parsed.path.endswith("/api/")+ and "\\" not in value+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)+ )+ _ = parsed.port+ except ValueError:+ raise OpikConfigurationError("invalid_endpoint") from None+ if not valid:+ raise OpikConfigurationError("invalid_endpoint")+ return value+++def _name(value: str | None, default: str, code: Literal["invalid_workspace", "invalid_project"]) -> str:+ result = value.strip() if value else default+ result = result or default+ if any(ord(char) < 32 or ord(char) == 127 for char in result) or (+ code == "invalid_workspace" and result in {".", ".."}+ ):+ raise OpikConfigurationError(code)+ return result+++def console_url(config: OpikConfig) -> str:+ """Use the configured host and prefix, never the SDK's global OpikConfig."""+ return _api_url(config).removesuffix("api/")+++def project_url(config: OpikConfig) -> str | None:+ """Pure fallback; historical default workspaces cannot be resolved on GET."""+ base = console_url(config)+ workspace = _name(config.workspace, "default", "invalid_workspace")+ if workspace == "default":+ return None+ project = _name(config.project, "Default Project", "invalid_project")+ return f"{base}{quote(workspace, safe='')}/redirect/projects?{urlencode({'name': project})}"+++def _request(method: Literal["GET", "POST"], url: str, headers: dict[str, str]) -> dict[str, object] | None:+ try:+ response = ssrf_proxy.make_request(+ method,+ url,+ headers=headers,+ json={} if method == "POST" else None,+ max_retries=0,+ timeout=30.0,+ follow_redirects=False,+ )+ except Exception:+ raise OpikConfigurationError("connection_failed") from None+ try:+ if response.status_code == 401:+ raise OpikConfigurationError("authentication_failed")+ if response.status_code == 403:+ raise OpikConfigurationError("permission_denied")+ if not 200 <= response.status_code < 300:+ raise OpikConfigurationError("endpoint_unavailable")+ if method == "POST":+ # The official auth contract is No Content, not an arbitrary 2xx page.+ if response.content.strip():+ raise OpikConfigurationError("malformed_response")+ return None+ try:+ return _JSON_OBJECT.validate_python(response.json(), strict=True)+ except ValueError:+ raise OpikConfigurationError("malformed_response") from None+ finally:+ response.close()+++def verify_configuration(config: OpikConfig) -> VerifiedOpikWorkspace:+ """Check auth/workspace via the same endpoints as Opik 1.11.2's auth_check.++ /v1/private/auth checks workspace access and returns no content. Only the+ sentinel default workspace needs /v1/private/auth/workspace resolution.+ Project creation remains an ingestion concern, as in the legacy behavior.+ """+ url = _api_url(config)+ workspace = _name(config.workspace, "default", "invalid_workspace")+ project = _name(config.project, "Default Project", "invalid_project")+ headers = {"Comet-Workspace": workspace, "Accept": "application/json"}+ if config.api_key:+ headers["Authorization"] = config.api_key+ _request("POST", url + "v1/private/auth", headers)+ if workspace == "default":+ data = _request("GET", url + "v1/private/auth/workspace", headers)+ resolved = data.get("workspace_name") if data else None+ if not isinstance(resolved, str) or not resolved.strip():+ raise OpikConfigurationError("malformed_response")+ workspace = _name(resolved, "default", "invalid_workspace")+ return VerifiedOpikWorkspace(url=url, workspace=workspace, project=project)diff --git a/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py b/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.pyindex 710173ccdc8..7323542c0f6 100644--- a/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py+++ b/api/providers/trace/trace-opik/src/dify_trace_opik/opik_trace.py@@ -63,12 +63,12 @@ def _seed_to_uuid4(seed: str) -> str:return str(uuid.UUID(bytes=bytes(ba)))-def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None):- """Opik needs UUIDv7 while Dify uses UUIDv4 for identifier of most- messages and objects. The type-hints of BaseTraceInfo indicates that- objects start_time and message_id could be null which means we cannot map- it to a UUIDv7. Given that we have no way to identify that object- uniquely, generate a new random one UUIDv7 in that case.+def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None) -> uuid.UUID:+ """Preserve UUID4 mappings and deterministically adapt external trace IDs.++ OpenTelemetry trace IDs and other external identifiers need not be UUID4,+ which the Opik conversion helper requires. Reuse the existing stable seed+ mapping for those identifiers. Missing identifiers retain random-ID behavior."""if user_datetime is None:@@ -77,6 +77,13 @@ def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None):if user_uuid is None:user_uuid = str(uuid.uuid4())+ try:+ is_uuid4 = uuid.UUID(user_uuid).version == 4+ except ValueError:+ is_uuid4 = False+ if not is_uuid4:+ user_uuid = _seed_to_uuid4(user_uuid)+return uuid4_to_uuid7(user_datetime, user_uuid)diff --git a/api/providers/trace/trace-tencent/src/dify_trace_tencent/agent_trace.py b/api/providers/trace/trace-tencent/src/dify_trace_tencent/agent_trace.pynew file mode 100644index 00000000000..78c99f94e61--- /dev/null+++ b/api/providers/trace/trace-tencent/src/dify_trace_tencent/agent_trace.py@@ -0,0 +1,19 @@+"""Tencent APM Agent spans over its existing authenticated OTLP/gRPC protocol."""++from collections.abc import Mapping+from typing import Any++from core.ops.unified_trace.entities import CanonicalTrace+from services.tracing.genai_attributes import attributes+from services.tracing.otlp_export import build_request, export_grpc+from services.tracing.provider_common import endpoint, required+from services.tracing.provider_dispatch import TraceExportResult+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ url = endpoint(required(config, "endpoint"))+ payload = build_request(+ trace, {"service.name": required(config, "service_name")}, lambda span: attributes(span, trace, provider)+ )+ export_grpc(provider, url, {"authorization": f"Bearer {required(config, 'token')}"}, payload)+ return TraceExportResult(trace_id=trace.trace_id)diff --git a/api/providers/trace/trace-tencent/src/dify_trace_tencent/configuration.py b/api/providers/trace/trace-tencent/src/dify_trace_tencent/configuration.pynew file mode 100644index 00000000000..04146a6e450--- /dev/null+++ b/api/providers/trace/trace-tencent/src/dify_trace_tencent/configuration.py@@ -0,0 +1,142 @@+"""Tencent configuration syntax and console navigation without exporter state.++These checks do not establish token validity or ingestion permission.+Collectors may acknowledge zero-span OTLP requests without authenticating+them, so an empty Export acknowledgment must not be presented as authentication.+"""++import re+from collections.abc import Mapping+from typing import Literal+from urllib.parse import urlsplit, urlunsplit++from dify_trace_tencent.config import TencentConfig++type ConfigurationCode = Literal[+ "invalid_endpoint", "invalid_service_name", "invalid_token", "connection_failed", "unexpected_response"+]++_MESSAGES: dict[ConfigurationCode, str] = {+ "invalid_endpoint": "Tencent APM requires an HTTP(S) collector origin with a valid host and port.",+ "invalid_service_name": (+ "Tencent APM service names must contain 1 to 63 lowercase letters, digits or hyphens, "+ "start with a letter and end with a letter or digit."+ ),+ "invalid_token": "Tencent APM requires a nonempty token.",+ "connection_failed": "Tencent APM collector protocol connection failed.",+ "unexpected_response": "Tencent APM collector returned an unexpected protocol acknowledgment.",+}+_SERVICE_NAME = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?")+++class TencentConfigurationError(ValueError):+ """Fixed safe errors, without request values or credentials."""++ def __init__(self, code: ConfigurationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++def _endpoint(value: object) -> str:+ if not isinstance(value, str):+ raise TencentConfigurationError("invalid_endpoint")+ value = value.strip()+ try:+ parsed = urlsplit(value)+ host, port = parsed.hostname, parsed.port+ valid = (+ parsed.scheme in {"http", "https"}+ and host+ and parsed.username is None+ and parsed.password is None+ and parsed.path in {"", "/"}+ and not parsed.query+ and not parsed.fragment+ and "?" not in value+ and "#" not in value+ and "\\" not in value+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)+ and not parsed.netloc.endswith(":")+ and (port is None or 1 <= port <= 65535)+ )+ if not valid or host is None:+ raise TencentConfigurationError("invalid_endpoint")+ host = host.encode("idna").decode("ascii").lower()+ authority = f"[{host}]" if ":" in host else host+ if port is not None:+ authority = f"{authority}:{port}"+ return urlunsplit((parsed.scheme, authority, "", "", ""))+ except (UnicodeError, ValueError):+ raise TencentConfigurationError("invalid_endpoint") from None+++def normalize_configuration(values: Mapping[str, object]) -> TencentConfig:+ """Validate new writes before legacy model normalization drops URL parts.++ The permissive legacy model remains available to display existing stored+ configurations. Masked tokens are retained here so PATCH can restore the+ encrypted original through the shared configuration owner.+ """+ endpoint = _endpoint(values.get("endpoint"))+ service_name = values.get("service_name")+ if not isinstance(service_name, str) or not _SERVICE_NAME.fullmatch(service_name.strip()):+ raise TencentConfigurationError("invalid_service_name")+ token = values.get("token")+ if not isinstance(token, str) or not token.strip():+ raise TencentConfigurationError("invalid_token")+ return TencentConfig(endpoint=endpoint, service_name=service_name.strip(), token=token)+++def console_url() -> str:+ """Return the console entry, not a destination-specific project link."""+ return "https://console.cloud.tencent.com/apm"+++def verify_configuration(config: TencentConfig) -> TencentConfig:+ """Check syntax and one zero-span OTLP/gRPC exchange, not authentication.++ A collector may acknowledge invalid-token or unauthenticated empty+ requests. Actual credential validity and ingestion must therefore be+ assessed from a business trace, never inferred from this connection check.+ No tracer, span processor, metrics reader or global SDK state is created.+ """+ import grpc # type: ignore[import-untyped]+ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub++ config = normalize_configuration(config.model_dump())+ parsed = urlsplit(config.endpoint)+ host = parsed.hostname+ assert host is not None # Validated above, before constructing a channel.+ authority = f"[{host}]" if ":" in host else host+ target = f"{authority}:{parsed.port or 4317}"+ options = (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", 4 * 1024 * 1024))+ channel: grpc.Channel | None = None+ context = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))+ try:+ try:+ channel = (+ grpc.insecure_channel(target, options=options)+ if parsed.scheme == "http"+ else grpc.secure_channel(target, grpc.ssl_channel_credentials(), options=options)+ )+ response = TraceServiceStub(channel).Export(+ ExportTraceServiceRequest(),+ metadata=(("authorization", f"Bearer {config.token}"),),+ timeout=10.0,+ wait_for_ready=False,+ )+ if response.partial_success.rejected_spans or response.partial_success.error_message:+ raise TencentConfigurationError("unexpected_response")+ finally:+ if channel is not None:+ channel.close()+ except TencentConfigurationError:+ raise+ except Exception:+ # gRPC errors may contain metadata or vendor details. Expose neither.+ raise TencentConfigurationError("connection_failed") from None+ finally:+ detach(context)+ return configdiff --git a/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py b/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.pyindex 763a85ffd71..a473e9141af 100644--- a/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py+++ b/api/providers/trace/trace-tencent/src/dify_trace_tencent/span_builder.py@@ -154,8 +154,9 @@ class TencentSpanBuilder:outputs = node_execution.outputs or {}usage_data = process_data.get("usage", {}) if "usage" in process_data else outputs.get("usage", {})+ session_id = trace_info.metadata.get("conversation_id")attributes = {- GEN_AI_SESSION_ID: trace_info.metadata.get("conversation_id", ""),+ GEN_AI_SESSION_ID: "" if session_id is None else session_id,GEN_AI_SPAN_KIND: GenAISpanKind.GENERATION.value,GEN_AI_FRAMEWORK: "dify",GEN_AI_MODEL_NAME: process_data.get("model_name", ""),@@ -293,6 +294,7 @@ class TencentSpanBuilder:if node_execution.outputs:output_value = json.dumps(node_execution.outputs.get("result", []), ensure_ascii=False)+ session_id = trace_info.metadata.get("conversation_id")return SpanData(trace_id=trace_id,parent_span_id=workflow_span_id,@@ -301,7 +303,7 @@ class TencentSpanBuilder:start_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.created_at),end_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.finished_at),attributes={- GEN_AI_SESSION_ID: trace_info.metadata.get("conversation_id", ""),+ GEN_AI_SESSION_ID: "" if session_id is None else session_id,GEN_AI_SPAN_KIND: GenAISpanKind.RETRIEVER.value,GEN_AI_FRAMEWORK: "dify",RETRIEVAL_QUERY: input_value,@@ -345,6 +347,7 @@ class TencentSpanBuilder:trace_id: int, workflow_span_id: int, trace_info: WorkflowTraceInfo, node_execution: WorkflowNodeExecution) -> SpanData:"""Build generic task span for workflow nodes."""+ session_id = trace_info.metadata.get("conversation_id")return SpanData(trace_id=trace_id,parent_span_id=workflow_span_id,@@ -353,7 +356,7 @@ class TencentSpanBuilder:start_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.created_at),end_time=TencentSpanBuilder._get_time_nanoseconds(node_execution.finished_at),attributes={- GEN_AI_SESSION_ID: trace_info.metadata.get("conversation_id", ""),+ GEN_AI_SESSION_ID: "" if session_id is None else session_id,GEN_AI_SPAN_KIND: GenAISpanKind.TASK.value,GEN_AI_FRAMEWORK: "dify",INPUT_VALUE: json.dumps(node_execution.inputs, ensure_ascii=False),diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/agent_trace.py b/api/providers/trace/trace-weave/src/dify_trace_weave/agent_trace.pynew file mode 100644index 00000000000..8799e78f0b8--- /dev/null+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/agent_trace.py@@ -0,0 +1,89 @@+"""Completed Agent trees through Weave's typed call API and an isolated transport."""++from collections.abc import Mapping+from typing import Any, cast+from urllib.parse import quote+from uuid import UUID, uuid5++from weave.trace_server.trace_server_interface import (+ CallEndReq,+ CallStartReq,+ EndedCallSchemaForInsert,+ StartedCallSchemaForInsert,+ SummaryInsertMap,+)++from core.ops.unified_trace.entities import CanonicalTrace+from dify_trace_weave.transport import TraceServer, resolve_project+from services.tracing.provider_common import endpoint, required, token_usage, utc+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportResult, TraceExportUncertainError+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ api_key = required(config, "api_key")+ entity, project, ui_host = resolve_project(config, api_key)+ # Weave's concrete HTTP client leaves unrelated optional API methods as protocol stubs.+ server = TraceServer( # pyrefly: ignore[bad-instantiation]+ endpoint(config.get("endpoint") or "https://trace.wandb.ai"), auth=("api", api_key)+ )+ project_id = f"{entity}/{project}"+ trace_uuid = UUID(hex=trace.trace_id)+ span_ids = {span.id: str(uuid5(trace_uuid, span.id)) for span in trace.spans}+ # Weave counts calls whose id equals turn_id as conversation turns.+ turn_id = span_ids[trace.root_span_id] if trace.session_id else None+ wrote = False+ try:+ for span in trace.spans:+ started = server.call_start(+ CallStartReq(+ start=StartedCallSchemaForInsert(+ project_id=project_id,+ id=span_ids[span.id],+ op_name=span.name,+ display_name=span.name,+ trace_id=str(trace_uuid),+ parent_id=span_ids.get(span.parent_id) if span.parent_id else None,+ thread_id=trace.session_id or None,+ turn_id=turn_id,+ started_at=utc(span.start_time),+ attributes={**span.metadata, "dify.span_kind": span.kind.value, "dify.span_id": span.id},+ inputs=span.inputs if isinstance(span.inputs, dict) else {"input": span.inputs},+ )+ )+ )+ wrote = True+ if started.id != span_ids[span.id] or started.trace_id != str(trace_uuid):+ raise TraceExportUncertainError("Weave did not acknowledge the submitted identifiers")+ for span in reversed(trace.spans):+ usage = token_usage(span)+ summary: dict[str, Any] = {}+ if usage:+ model = str(span.metadata.get("model_name") or span.metadata.get("model") or "unknown")+ summary["usage"] = {+ model: {+ {"input": "prompt_tokens", "output": "completion_tokens", "total": "total_tokens"}[key]: value+ for key, value in usage.items()+ }+ | {"requests": 1}+ }+ if span.end_time is None:+ raise TraceExportRejectedError("Weave requires completed spans")+ server.call_end(+ CallEndReq(+ end=EndedCallSchemaForInsert(+ project_id=project_id,+ id=span_ids[span.id],+ ended_at=utc(span.end_time),+ output=span.outputs,+ exception=span.error or ("Agent span failed" if span.status.value == "error" else None),+ summary=cast(SummaryInsertMap, summary),+ )+ )+ )+ except Exception:+ if wrote:+ raise TraceExportUncertainError("Weave trace was partially accepted") from None+ raise+ base = f"{ui_host}/{quote(entity, safe='')}/{quote(project, safe='')}/r/call/"+ urls = {span_id: base + call_id for span_id, call_id in span_ids.items()}+ return TraceExportResult(trace_id=str(trace_uuid), trace_url=urls[trace.root_span_id], span_urls=urls)diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/configuration.py b/api/providers/trace/trace-weave/src/dify_trace_weave/configuration.pynew file mode 100644index 00000000000..e5f92f7055f--- /dev/null+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/configuration.py@@ -0,0 +1,250 @@+"""Read-only Weave configuration checks without SDK login or global client state.++Projects must already exist. Configuration verification never creates a project,+trace, or local credential file; actual legacy exports retain their SDK lifecycle.+"""++from dataclasses import dataclass+from typing import Literal+from urllib.parse import quote, urlsplit, urlunsplit++from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError+from weave.trace_server.trace_server_interface import CallsQueryStatsReq, CallsQueryStatsRes++from core.helper import ssrf_proxy+from dify_trace_weave.config import WeaveConfig++type VerificationCode = Literal[+ "invalid_project",+ "invalid_endpoint",+ "authentication_failed",+ "permission_denied",+ "project_not_found_or_inaccessible",+ "project_read_only",+ "project_permission_unknown",+ "graphql_error",+ "malformed_response",+ "endpoint_unavailable",+ "connection_failed",+]++_MESSAGES: dict[VerificationCode, str] = {+ "invalid_project": "Weave requires a valid entity and project name.",+ "invalid_endpoint": "Weave endpoint is invalid.",+ "authentication_failed": "Weave authentication failed.",+ "permission_denied": "Weave permission was denied.",+ "project_not_found_or_inaccessible": "Create the Weave project first or check project access.",+ "project_read_only": "Weave project is read-only for these credentials.",+ "project_permission_unknown": "Weave project write permission could not be verified.",+ "graphql_error": "Weave project verification returned an error.",+ "malformed_response": "Weave verification returned an invalid response.",+ "endpoint_unavailable": "Weave endpoint could not be verified.",+ "connection_failed": "Weave verification request failed.",+}++_VIEWER_QUERY = "query DifyWeaveViewer { viewer { id entity } }"+_PROJECT_QUERY = """query DifyWeaveProject($entity: String!, $project: String!) {+ viewer { id }+ project(name: $project, entityName: $entity) { id name entityName readOnly }+}"""+_JSON_OBJECT = TypeAdapter(dict[str, object])+++class WeaveConfigurationError(ValueError):+ """Fixed diagnostic codes: never include keys, request data, or server errors."""++ def __init__(self, code: VerificationCode) -> None:+ self.code = code+ super().__init__(_MESSAGES[code])+++@dataclass(frozen=True, slots=True)+class VerifiedWeaveProject:+ entity: str+ project: str+ project_url: str+++class _Viewer(BaseModel):+ id: str = Field(min_length=1)+ entity: str | None = None+ model_config = ConfigDict(strict=True)+++class _Project(BaseModel):+ id: str = Field(min_length=1)+ name: str = Field(min_length=1)+ entity_name: str = Field(alias="entityName", min_length=1)+ read_only: bool | None = Field(alias="readOnly")+ model_config = ConfigDict(strict=True)+++def _base_url(value: str, *, allow_http: bool) -> str:+ try:+ parsed = urlsplit(value)+ valid = (+ parsed.scheme in ({"https", "http"} if allow_http else {"https"})+ and parsed.hostname+ and parsed.username is None+ and parsed.password is None+ and not parsed.query+ and not parsed.fragment+ and "\\" not in value+ and not any(ord(char) <= 32 or ord(char) == 127 for char in value)+ )+ _ = parsed.port+ except ValueError:+ raise WeaveConfigurationError("invalid_endpoint") from None+ if not valid:+ raise WeaveConfigurationError("invalid_endpoint")+ return urlunsplit(parsed).rstrip("/")+++def _name(value: str) -> str:+ value = value.strip()+ if (+ not value+ or value in {".", ".."}+ or "/" in value+ or "\\" in value+ or any(ord(char) < 32 or ord(char) == 127 for char in value)+ ):+ raise WeaveConfigurationError("invalid_project")+ return value+++def _target(config: WeaveConfig) -> tuple[str | None, str]:+ entity = config.entity.strip() if config.entity else None+ project = config.project.strip()+ if "/" in project:+ parts = project.split("/")+ if entity or len(parts) != 2:+ raise WeaveConfigurationError("invalid_project")+ entity, project = _name(parts[0]), parts[1]+ return (_name(entity) if entity else None), _name(project)+++def console_url(config: WeaveConfig) -> str:+ """Map the public API origin to its UI; preserve private origins and prefixes."""+ host = _base_url(config.host or "https://api.wandb.ai", allow_http=True)+ parsed = urlsplit(host)+ if (+ parsed.scheme == "https"+ and parsed.hostname == "api.wandb.ai"+ and parsed.port in {None, 443}+ and not parsed.path+ ):+ return "https://wandb.ai"+ return host+++def project_url(config: WeaveConfig) -> str | None:+ """Pure presentation fallback; never resolve a default entity over the network."""+ entity, project = _target(config)+ if entity is None:+ return None+ return f"{console_url(config)}/{quote(entity, safe='')}/{quote(project, safe='')}"+++def _post(url: str, *, api_key: str, body: dict[str, object]) -> dict[str, object]:+ try:+ response = ssrf_proxy.post(+ url,+ auth=("api", api_key),+ json=body,+ max_retries=0,+ timeout=30.0,+ follow_redirects=False,+ )+ except Exception:+ raise WeaveConfigurationError("connection_failed") from None+ try:+ if response.status_code == 401:+ raise WeaveConfigurationError("authentication_failed")+ if response.status_code == 403:+ raise WeaveConfigurationError("permission_denied")+ if not 200 <= response.status_code < 300:+ raise WeaveConfigurationError("endpoint_unavailable")+ try:+ return _JSON_OBJECT.validate_python(response.json(), strict=True)+ except ValueError:+ raise WeaveConfigurationError("malformed_response") from None+ finally:+ response.close()+++def _graphql(host: str, *, api_key: str, body: dict[str, object]) -> dict[str, object]:+ response = _post(host + "/graphql", api_key=api_key, body=body)+ errors = response.get("errors")+ if errors is not None:+ if not isinstance(errors, list):+ raise WeaveConfigurationError("malformed_response")+ if errors:+ raise WeaveConfigurationError("graphql_error")+ try:+ return _JSON_OBJECT.validate_python(response.get("data"), strict=True)+ except ValidationError:+ raise WeaveConfigurationError("malformed_response") from None+++def verify_configuration(config: WeaveConfig) -> VerifiedWeaveProject:+ """Verify credentials, an existing writable project, and the trace read endpoint.++ The GraphQL fields match W&B's Viewer, GetProject and IsProjectReadOnly+ queries. CallsQueryStats uses the same typed read API as the Weave SDK.+ These checks do not write a test trace or replace ingestion authorization.+ """+ if not config.api_key.strip():+ raise WeaveConfigurationError("authentication_failed")+ entity, project = _target(config)+ host = _base_url(config.host or "https://api.wandb.ai", allow_http=True)+ trace_endpoint = _base_url(config.endpoint, allow_http=False)+ try:+ if entity is None:+ data = _graphql(host, api_key=config.api_key, body={"query": _VIEWER_QUERY})+ if "viewer" not in data:+ raise WeaveConfigurationError("malformed_response")+ if data.get("viewer") is None:+ raise WeaveConfigurationError("authentication_failed")+ viewer = _Viewer.model_validate(data["viewer"])+ if not viewer.entity:+ raise WeaveConfigurationError("invalid_project")+ entity = _name(viewer.entity)++ data = _graphql(+ host,+ api_key=config.api_key,+ body={"query": _PROJECT_QUERY, "variables": {"entity": entity, "project": project}},+ )+ if "viewer" not in data or "project" not in data:+ raise WeaveConfigurationError("malformed_response")+ if data.get("viewer") is None:+ raise WeaveConfigurationError("authentication_failed")+ _Viewer.model_validate(data["viewer"])+ if data.get("project") is None:+ raise WeaveConfigurationError("project_not_found_or_inaccessible")+ resolved = _Project.model_validate(data["project"])+ if resolved.name != project or resolved.entity_name != entity:+ raise WeaveConfigurationError("invalid_project")+ if resolved.read_only is True:+ raise WeaveConfigurationError("project_read_only")+ if resolved.read_only is None:+ raise WeaveConfigurationError("project_permission_unknown")++ stats_request = CallsQueryStatsReq(project_id=f"{entity}/{project}", limit=1)+ stats_response = _post(+ trace_endpoint + "/calls/query_stats",+ api_key=config.api_key,+ body=stats_request.model_dump(mode="json", exclude_none=True),+ )+ stats = CallsQueryStatsRes.model_validate(stats_response, strict=True)+ if stats.count < 0:+ raise WeaveConfigurationError("malformed_response")+ except ValidationError:+ raise WeaveConfigurationError("malformed_response") from None++ return VerifiedWeaveProject(+ entity=entity,+ project=project,+ project_url=f"{console_url(config)}/{quote(entity, safe='')}/{quote(project, safe='')}",+ )diff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/transport.py b/api/providers/trace/trace-weave/src/dify_trace_weave/transport.pynew file mode 100644index 00000000000..ed578b5d768--- /dev/null+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/transport.py@@ -0,0 +1,56 @@+"""Instance-authenticated Weave SDK transport shared by managed and legacy tracing."""++from collections.abc import Mapping+from typing import Any, override++from pydantic import BaseModel+from weave.trace_server_bindings.remote_http_trace_server import RemoteHTTPTraceServer++from dify_trace_weave.config import WeaveConfig+from dify_trace_weave.configuration import console_url+from services.tracing.provider_common import endpoint, request, required+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError+++class TraceServer(RemoteHTTPTraceServer):+ """Use SDK request/response validation without SDK retries or global W&B login."""++ @override+ def _post_request_executor(self, url: str, req: BaseModel, stream: bool = False):+ return request(+ "POST",+ self.trace_server_url + url,+ "weave",+ auth=self._auth,+ content=req.model_dump_json(by_alias=True).encode(),+ headers={"Content-Type": "application/json"},+ )++ @override+ def _generic_request(self, url, req, req_model, res_model, method="POST", params=None):+ response = self._post_request_executor(url, req)+ try:+ return res_model.model_validate(response.json())+ except ValueError:+ raise TraceExportUncertainError("Weave returned an unreadable ingestion acknowledgment") from None+++def resolve_project(config: Mapping[str, Any], api_key: str) -> tuple[str, str, str]:+ project = required(config, "project")+ host = endpoint(config.get("host") or "https://api.wandb.ai")+ entity = config.get("entity")+ if not entity and "/" in project:+ entity, project = project.split("/", 1)+ if not entity:+ response = request(+ "POST",+ host + "/graphql",+ "weave",+ auth=("api", api_key),+ json={"query": "query DifyTraceEntity { viewer { entity } }"},+ )+ entity = (response.json().get("data") or {}).get("viewer", {}).get("entity")+ if not isinstance(entity, str) or not entity or "/" in entity or not project or "/" in project:+ raise TraceExportRejectedError("Weave project or entity could not be resolved")+ ui_host = console_url(WeaveConfig(api_key=api_key, entity=entity, project=project, host=host))+ return entity, project, ui_hostdiff --git a/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py b/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.pyindex 8a9fbefe4ab..6766536203e 100644--- a/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py+++ b/api/providers/trace/trace-weave/src/dify_trace_weave/weave_trace.py@@ -1,11 +1,10 @@import loggingimport osimport uuid+from contextvars import ContextVarfrom datetime import UTC, datetime, timedeltafrom typing import Any, cast, override-import wandb-import weavefrom sqlalchemy.orm import sessionmakerfrom weave.trace_server.trace_server_interface import (CallEndReq,@@ -28,14 +27,30 @@ from core.ops.entities.trace_entity import (TraceTaskName,WorkflowTraceInfo,)+from core.ops.exceptions import RetryableTraceDispatchErrorfrom core.repositories import DifyCoreRepositoryFactoryfrom dify_trace_weave.config import WeaveConfig+from dify_trace_weave.configuration import console_url, project_url, verify_configurationfrom dify_trace_weave.entities.weave_trace_entity import WeaveTraceModel+from dify_trace_weave.transport import TraceServer, resolve_projectfrom extensions.ext_database import dbfrom graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKeyfrom models import EndUser, MessageFile, WorkflowNodeExecutionTriggeredFrom+from services.tracing.provider_common import endpoint+from services.tracing.provider_dispatch import TraceExportUncertainErrorlogger = logging.getLogger(__name__)+_dispatch_started_calls: ContextVar[dict[str, dict[str, Any]] | None] = ContextVar("weave_dispatch_calls", default=None)+++def _same_identifier(actual: str, expected: str) -> bool:+ """Accept UUID formatting differences without rewriting legacy wire identifiers."""+ if actual == expected:+ return True+ try:+ return uuid.UUID(actual) == uuid.UUID(expected)+ except ValueError:+ return Falseclass WeaveDataTrace(BaseTraceInstance):@@ -44,60 +59,65 @@ class WeaveDataTrace(BaseTraceInstance):weave_config: WeaveConfig,):super().__init__(weave_config)- self.weave_api_key = weave_config.api_key- self.project_name = weave_config.project- self.entity = weave_config.entity- self.host = weave_config.host-- # Login with API key first, including host if provided- if self.host:- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True, host=self.host)- else:- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True)-- if not login_status:- logger.error("Failed to login to Weights & Biases with the provided API key")- raise ValueError("Weave login failed")-- # Then initialize weave client- self.weave_client = weave.init(- project_name=(f"{self.entity}/{self.project_name}" if self.entity else self.project_name)+ # Construction is local: no global SDK client, login, netrc, or background queue.+ self._config = weave_config.model_copy(deep=True)+ self.weave_api_key = self._config.api_key+ self.project_name = self._config.project+ self.entity = self._config.entity+ self.host = self._config.host+ self.server = TraceServer( # pyrefly: ignore[bad-instantiation]+ endpoint(self._config.endpoint), should_batch=False, auth=("api", self.weave_api_key))+ self._project_id: str | None = Noneself.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001")self.calls: dict[str, Any] = {}- self.project_id = f"{self.weave_client.entity}/{self.weave_client.project}"- def get_project_url(- self,- ):- try:- project_identifier = f"{self.entity}/{self.project_name}" if self.entity else self.project_name- project_url = f"https://wandb.ai/{project_identifier}"- return project_url- except Exception as e:- logger.debug("Weave get run url failed", exc_info=True)- raise ValueError(f"Weave get run url failed: {str(e)}") from e+ @property+ def project_id(self) -> str:+ # Existing saved configurations may omit entity. Resolve it lazily with+ # this instance's credentials, never W&B's process-global defaults.+ if self._project_id is None:+ entity, project, _ = resolve_project(self._config.model_dump(), self.weave_api_key)+ self._project_id = f"{entity}/{project}"+ return self._project_id++ def get_project_url(self):+ return project_url(self._config) or console_url(self._config)@overridedef trace(self, trace_info: BaseTraceInfo):logger.debug("Trace info: %s", trace_info)- match trace_info:- case WorkflowTraceInfo():- self.workflow_trace(trace_info)- case MessageTraceInfo():- self.message_trace(trace_info)- case ModerationTraceInfo():- self.moderation_trace(trace_info)- case SuggestedQuestionTraceInfo():- self.suggested_question_trace(trace_info)- case DatasetRetrievalTraceInfo():- self.dataset_retrieval_trace(trace_info)- case ToolTraceInfo():- self.tool_trace(trace_info)- case GenerateNameTraceInfo():- self.generate_name_trace(trace_info)- case _:- pass+ accepted_calls: dict[str, dict[str, Any]] = {}+ token = _dispatch_started_calls.set(accepted_calls)+ try:+ match trace_info:+ case WorkflowTraceInfo():+ self.workflow_trace(trace_info)+ case MessageTraceInfo():+ self.message_trace(trace_info)+ case ModerationTraceInfo():+ self.moderation_trace(trace_info)+ case SuggestedQuestionTraceInfo():+ self.suggested_question_trace(trace_info)+ case DatasetRetrievalTraceInfo():+ self.dataset_retrieval_trace(trace_info)+ case ToolTraceInfo():+ self.tool_trace(trace_info)+ case GenerateNameTraceInfo():+ self.generate_name_trace(trace_info)+ case _:+ pass+ except RetryableTraceDispatchError as error:+ if accepted_calls and not error.acceptance_unknown:+ raise TraceExportUncertainError(+ "Weave trace was partially accepted; automatic retry is unsafe"+ ) from None+ raise+ finally:+ for call_id, call_meta in accepted_calls.items():+ if self.calls.get(call_id) is call_meta:+ self.calls.pop(call_id, None)+ _dispatch_started_calls.reset(token)def workflow_trace(self, trace_info: WorkflowTraceInfo):trace_id = trace_info.trace_id or trace_info.message_id or trace_info.workflow_run_id@@ -421,20 +441,9 @@ class WeaveDataTrace(BaseTraceInstance):self.finish_call(name_run)def api_check(self):- try:- if self.host:- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True, host=self.host)- else:- login_status = wandb.login(key=self.weave_api_key, verify=True, relogin=True)-- if not login_status:- raise ValueError("Weave login failed")- else:- logger.info("Weave login successful")- return True- except Exception as e:- logger.debug("Weave API check failed", exc_info=True)- raise ValueError(f"Weave API check failed: {str(e)}") from e+ # Read-only credential/project/endpoint verification; it creates no calls.+ verify_configuration(self._config)+ return Truedef _normalize_time(self, dt: datetime | None) -> datetime:if dt is None:@@ -475,8 +484,14 @@ class WeaveDataTrace(BaseTraceInstance):wb_user_id=None,))- self.weave_client.server.call_start(call_start_req)- self.calls[run_data.id] = {"trace_id": trace_id, "parent_id": parent_run_id}+ started = self.server.call_start(call_start_req)+ if not _same_identifier(started.id, run_data.id) or not _same_identifier(started.trace_id, trace_id):+ raise TraceExportUncertainError("Weave did not acknowledge the submitted identifiers")+ call_meta = {"trace_id": trace_id, "parent_id": parent_run_id}+ self.calls[run_data.id] = call_meta+ dispatch_calls = _dispatch_started_calls.get()+ if dispatch_calls is not None:+ dispatch_calls[run_data.id] = call_metadef finish_call(self, run_data: WeaveTraceModel):call_meta = self.calls.get(run_data.id)@@ -523,4 +538,7 @@ class WeaveDataTrace(BaseTraceInstance):summary=cast(SummaryInsertMap, summary),))- self.weave_client.server.call_end(call_end_req)+ try:+ self.server.call_end(call_end_req)+ finally:+ self.calls.pop(run_data.id, None)diff --git a/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py b/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.pyindex 4c0abbc8e4c..b836b06a861 100644--- a/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py+++ b/api/providers/trace/trace-weave/tests/unit_tests/weave_trace/test_weave_trace.py@@ -8,9 +8,10 @@ from unittest.mock import MagicMock, patchimport pytestfrom dify_trace_weave.config import WeaveConfig+from dify_trace_weave.configuration import WeaveConfigurationErrorfrom dify_trace_weave.entities.weave_trace_entity import WeaveTraceModelfrom dify_trace_weave.weave_trace import WeaveDataTrace-from weave.trace_server.trace_server_interface import TraceStatus+from weave.trace_server.trace_server_interface import CallStartRes, TraceStatusfrom core.ops.entities.trace_entity import (DatasetRetrievalTraceInfo,@@ -190,138 +191,65 @@ def _make_node(**overrides):@pytest.fixture-def mock_wandb():- with patch("dify_trace_weave.weave_trace.wandb") as mock:- mock.login.return_value = True- yield mock+def mock_server():+ with patch("dify_trace_weave.weave_trace.TraceServer") as factory:+ server = factory.return_value+ server.call_start.side_effect = lambda req: CallStartRes(id=req.start.id, trace_id=req.start.trace_id)+ yield factory@pytest.fixture-def mock_weave():- with patch("dify_trace_weave.weave_trace.weave") as mock:- client = MagicMock()- client.entity = "my-entity"- client.project = "my-project"- mock.init.return_value = client- yield mock, client+def trace_instance(mock_server):+ return WeaveDataTrace(_make_weave_config())@pytest.fixture-def trace_instance(mock_wandb, mock_weave):- """Create a WeaveDataTrace instance with mocked wandb/weave."""- _, weave_client = mock_weave- config = _make_weave_config()- instance = WeaveDataTrace(config)- return instance---def trace_instance_with_host(mock_wandb, mock_weave):- """Create a WeaveDataTrace instance with host configured."""- _, weave_client = mock_weave- config = _make_weave_config(host="https://my.wandb.host")- instance = WeaveDataTrace(config)- return instance---# ── TestInit ─────────────────────────────────────────────────────────────────+def trace_instance_with_host(mock_server):+ return WeaveDataTrace(_make_weave_config(host="https://my.wandb.host"))class TestInit:- def test_init_without_host(self, mock_wandb, mock_weave):- """Test __init__ calls wandb.login without host."""- mock_w, weave_client = mock_weave- config = _make_weave_config(host=None)- instance = WeaveDataTrace(config)-- mock_wandb.login.assert_called_once_with(key="wv-api-key", verify=True, relogin=True)- mock_w.init.assert_called_once_with(project_name="my-entity/my-project")+ def test_init_has_explicit_endpoint_and_auth_without_batching(self, mock_server):+ instance = WeaveDataTrace(_make_weave_config(endpoint="https://trace.test/prefix"))+ mock_server.assert_called_once_with("https://trace.test/prefix", should_batch=False, auth=("api", "wv-api-key"))assert instance.weave_api_key == "wv-api-key"- assert instance.project_name == "my-project"- assert instance.entity == "my-entity"+ assert instance.project_id == "my-entity/my-project"assert instance.calls == {}-- def test_init_with_host(self, mock_wandb, mock_weave):- """Test __init__ calls wandb.login with host."""- config = _make_weave_config(host="https://my.wandb.host")- instance = WeaveDataTrace(config)-- mock_wandb.login.assert_called_once_with(- key="wv-api-key", verify=True, relogin=True, host="https://my.wandb.host"- )- assert instance.host == "https://my.wandb.host"-- def test_init_with_host_path(self, mock_wandb, mock_weave):- """A self-hosted host keeps its path prefix all the way to wandb.login."""- config = _make_weave_config(host="https://wandb.internal/api")- instance = WeaveDataTrace(config)-- mock_wandb.login.assert_called_once_with(- key="wv-api-key", verify=True, relogin=True, host="https://wandb.internal/api"- )- assert instance.host == "https://wandb.internal/api"-- def test_init_without_entity(self, mock_wandb, mock_weave):- """Test __init__ initializes weave without entity prefix when entity is None."""- mock_w, weave_client = mock_weave- config = _make_weave_config(entity=None)- WeaveDataTrace(config)-- mock_w.init.assert_called_once_with(project_name="my-project")-- def test_init_login_failure_raises(self, mock_wandb, mock_weave):- """Test __init__ raises ValueError when wandb.login returns False."""- mock_wandb.login.return_value = False- config = _make_weave_config()-- with pytest.raises(ValueError, match="Weave login failed"):- WeaveDataTrace(config)-- def test_init_files_url_from_env(self, mock_wandb, mock_weave, monkeypatch: pytest.MonkeyPatch):- """Test FILES_URL is read from environment."""+ mock_server.return_value.call_start.assert_not_called()++ @pytest.mark.parametrize("host", [None, "https://my.wandb.host", "https://wandb.internal/api"])+ def test_init_preserves_host(self, mock_server, host):+ instance = WeaveDataTrace(_make_weave_config(host=host))+ assert instance.host == host++ def test_init_without_entity_does_not_resolve_over_network(self, mock_server):+ with patch("dify_trace_weave.weave_trace.resolve_project") as resolve:+ instance = WeaveDataTrace(_make_weave_config(entity=None))+ resolve.assert_not_called()+ assert instance.get_project_url() == "https://wandb.ai"+ resolve.assert_not_called()++ def test_qualified_project_does_not_require_default_entity(self, mock_server):+ instance = WeaveDataTrace(_make_weave_config(entity=None, project="team/project"))+ assert instance.project_id == "team/project"+ assert instance.get_project_url() == "https://wandb.ai/team/project"++ def test_init_files_url_from_env(self, mock_server, monkeypatch: pytest.MonkeyPatch):monkeypatch.setenv("FILES_URL", "http://files.example.com")- config = _make_weave_config()- instance = WeaveDataTrace(config)- assert instance.file_base_url == "http://files.example.com"+ assert WeaveDataTrace(_make_weave_config()).file_base_url == "http://files.example.com"- def test_init_files_url_default(self, mock_wandb, mock_weave, monkeypatch: pytest.MonkeyPatch):- """Test FILES_URL defaults to http://127.0.0.1:5001."""+ def test_init_files_url_default(self, mock_server, monkeypatch: pytest.MonkeyPatch):monkeypatch.delenv("FILES_URL", raising=False)- config = _make_weave_config()- instance = WeaveDataTrace(config)- assert instance.file_base_url == "http://127.0.0.1:5001"-- def test_project_id_set_correctly(self, trace_instance):- """Test that project_id is set from weave_client entity/project."""- assert trace_instance.project_id == "my-entity/my-project"---# ── TestGetProjectUrl ─────────────────────────────────────────────────────────+ assert WeaveDataTrace(_make_weave_config()).file_base_url == "http://127.0.0.1:5001"class TestGetProjectUrl:def test_get_project_url_with_entity(self, trace_instance):- """Returns wandb URL with entity/project."""- url = trace_instance.get_project_url()- assert url == "https://wandb.ai/my-entity/my-project"-- def test_get_project_url_without_entity(self, mock_wandb, mock_weave):- """Returns wandb URL with project only when entity is None."""- config = _make_weave_config(entity=None)- instance = WeaveDataTrace(config)- url = instance.get_project_url()- assert url == "https://wandb.ai/my-project"-- def test_get_project_url_exception_raises(self, trace_instance, monkeypatch: pytest.MonkeyPatch):- """Raises ValueError when exception occurs in get_project_url."""- monkeypatch.setattr(trace_instance, "entity", None)- monkeypatch.setattr(trace_instance, "project_name", None)- # Force an error by making string formatting fail- # Simulate exception via property- trace_instance.entity = None- trace_instance.project_name = None- url = trace_instance.get_project_url()- assert "https://wandb.ai/" in url+ assert trace_instance.get_project_url() == "https://wandb.ai/my-entity/my-project"++ def test_get_project_url_preserves_private_prefix(self, mock_server):+ instance = WeaveDataTrace(_make_weave_config(host="https://wandb.internal/prefix"))+ assert instance.get_project_url() == "https://wandb.internal/prefix/my-entity/my-project"# ── TestTraceDispatcher ─────────────────────────────────────────────────────@@ -408,7 +336,7 @@ class TestStartCall:assert "run-1" in trace_instance.callsassert trace_instance.calls["run-1"]["trace_id"] == "t-1"assert trace_instance.calls["run-1"]["parent_id"] is None- trace_instance.weave_client.server.call_start.assert_called_once()+ trace_instance.server.call_start.assert_called_once()def test_start_call_with_parent(self, trace_instance):"""Test start_call records parent_run_id."""@@ -431,7 +359,7 @@ class TestStartCall:attributes={"trace_id": "t-2", "start_time": _dt()},)trace_instance.start_call(run)- call_args = trace_instance.weave_client.server.call_start.call_args+ call_args = trace_instance.server.call_start.call_argsreq = call_args[0][0]assert req.start.inputs == {}@@ -444,7 +372,7 @@ class TestStartCall:attributes={"trace_id": "t-3", "start_time": _dt()},)trace_instance.start_call(run)- call_args = trace_instance.weave_client.server.call_start.call_args+ call_args = trace_instance.server.call_start.call_argsreq = call_args[0][0]# String inputs gets converted by validator to a dictassert isinstance(req.start.inputs, dict)@@ -472,7 +400,7 @@ class TestStartCall:# Manually override after constructionrun.attributes = "some-attr-string"trace_instance.start_call(run)- call_args = trace_instance.weave_client.server.call_start.call_args+ call_args = trace_instance.server.call_start.call_argsreq = call_args[0][0]assert isinstance(req.start.attributes, dict)assert req.start.attributes == {"attributes": "some-attr-string"}@@ -509,8 +437,8 @@ class TestFinishCall:exception=None,)trace_instance.finish_call(run)- trace_instance.weave_client.server.call_end.assert_called_once()- call_args = trace_instance.weave_client.server.call_end.call_args+ trace_instance.server.call_end.assert_called_once()+ call_args = trace_instance.server.call_end.call_argsreq = call_args[0][0]assert req.end.summary["status_counts"][TraceStatus.SUCCESS] == 1assert req.end.summary["status_counts"][TraceStatus.ERROR] == 0@@ -528,7 +456,7 @@ class TestFinishCall:exception="Something broke",)trace_instance.finish_call(run)- call_args = trace_instance.weave_client.server.call_end.call_args+ call_args = trace_instance.server.call_end.call_argsreq = call_args[0][0]assert req.end.summary["status_counts"][TraceStatus.ERROR] == 1assert req.end.summary["status_counts"][TraceStatus.SUCCESS] == 0@@ -557,7 +485,7 @@ class TestFinishCall:},)trace_instance.finish_call(run)- call_args = trace_instance.weave_client.server.call_end.call_args+ call_args = trace_instance.server.call_end.call_argsreq = call_args[0][0]assert req.end.summary["weave"]["latency_ms"] == 0@@ -571,7 +499,7 @@ class TestFinishCall:attributes=None,)trace_instance.finish_call(run)- trace_instance.weave_client.server.call_end.assert_called_once()+ trace_instance.server.call_end.assert_called_once()def test_finish_call_non_dict_attributes(self, trace_instance):"""Test finish_call handles non-dict attributes."""@@ -584,7 +512,7 @@ class TestFinishCall:)run.attributes = "some string attr"trace_instance.finish_call(run)- trace_instance.weave_client.server.call_end.assert_called_once()+ trace_instance.server.call_end.assert_called_once()# ── TestWorkflowTrace ─────────────────────────────────────────────────────────@@ -1165,40 +1093,17 @@ class TestGenerateNameTrace:class TestApiCheck:- def test_api_check_success_without_host(self, trace_instance, mock_wandb):- """api_check returns True on successful login without host."""- trace_instance.host = None- mock_wandb.login.return_value = True-- result = trace_instance.api_check()-- assert result is True- mock_wandb.login.assert_called_with(key=trace_instance.weave_api_key, verify=True, relogin=True)-- def test_api_check_success_with_host(self, trace_instance, mock_wandb):- """api_check returns True on successful login with host."""- trace_instance.host = "https://my.wandb.host"- mock_wandb.login.return_value = True-- result = trace_instance.api_check()-- assert result is True- mock_wandb.login.assert_called_with(- key=trace_instance.weave_api_key, verify=True, relogin=True, host="https://my.wandb.host"- )-- def test_api_check_login_failure_raises(self, trace_instance, mock_wandb):- """api_check raises ValueError when login returns False."""- trace_instance.host = None- mock_wandb.login.return_value = False-- with pytest.raises(ValueError, match="Weave API check failed"):- trace_instance.api_check()-- def test_api_check_exception_raises_value_error(self, trace_instance, mock_wandb):- """api_check raises ValueError when wandb.login raises exception."""- trace_instance.host = None- mock_wandb.login.side_effect = Exception("network error")-- with pytest.raises(ValueError, match="Weave API check failed: network error"):- trace_instance.api_check()+ def test_api_check_uses_read_only_verification(self, trace_instance):+ with patch("dify_trace_weave.weave_trace.verify_configuration") as verify:+ assert trace_instance.api_check() is True+ verify.assert_called_once_with(trace_instance._config)+ trace_instance.server.call_start.assert_not_called()+ trace_instance.server.call_end.assert_not_called()++ def test_api_check_propagates_safe_failure(self, trace_instance):+ with patch(+ "dify_trace_weave.weave_trace.verify_configuration",+ side_effect=WeaveConfigurationError("authentication_failed"),+ ):+ with pytest.raises(WeaveConfigurationError, match="Weave authentication failed"):+ trace_instance.api_check()diff --git a/api/repositories/app/agent_app_repository.py b/api/repositories/app/agent_app_repository.pyindex 328d0e32d7a..fef158cd315 100644--- a/api/repositories/app/agent_app_repository.py+++ b/api/repositories/app/agent_app_repository.py@@ -18,6 +18,14 @@ class AgentAppRepository:def __init__(self, *, session_factory: sessionmaker[Session]) -> None:self._session_factory = session_factory+ def resolve_roster_app_id(self, *, tenant_id: str, agent_id: str) -> str:+ """Resolve a roster Agent's configuration owner without materializing inline Apps."""+ with self._session_factory() as session:+ try:+ return AgentRosterService(session).get_agent_app_model(tenant_id=tenant_id, agent_id=agent_id).id+ except AgentNotFoundError as exc:+ raise AgentAppNotFoundError from exc+def resolve_runtime_app_id(self, *, tenant_id: str, agent_id: str) -> str:with self._session_factory() as session:try:diff --git a/api/repositories/app/console_repository.py b/api/repositories/app/console_repository.pyindex 64fa0844d09..08787fde139 100644--- a/api/repositories/app/console_repository.py+++ b/api/repositories/app/console_repository.py@@ -7,7 +7,6 @@ existing use cases that still need attached models.from __future__ import annotations-import jsonimport loggingfrom collections.abc import Sequencefrom dataclasses import replace@@ -50,6 +49,7 @@ from services.agent.errors import (from services.app.console_service import ConsoleAppNotFoundError, ConsoleAppsfrom services.app.query_service import AppDescription, DiscoveryApps, RelatedAppsfrom services.app_creation_records import create_installed_app_record, create_site_record+from services.app_tracing_config_service import AppTraceSettingsfrom services.entities.app_entities import (RECENT_APP_MODES,AppAgentBinding,@@ -65,7 +65,6 @@ from services.entities.app_entities import (AppRecord,AppReference,AppSummary,- AppTraceSettings,AppUpdateArguments,CreateAppParams,RecentAppListItem,@@ -326,12 +325,6 @@ class ConsoleAppRepository(ConsoleApps, DiscoveryApps, RelatedApps):app = require_console_app(session, context, app_id)return _app_trace_settings_adapter.validate_json(app.tracing) if app.tracing else AppTraceSettings()- @override- def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:- with self._session_factory.begin() as session:- app = require_console_app(session, context, app_id)- app.tracing = json.dumps({"enabled": settings.enabled, "tracing_provider": settings.tracing_provider})-@staticmethoddef _draft_trigger_app_ids(session: Session, workspace_id: str, apps: list[App]) -> set[str]:workflow_app_ids = [app.id for app in apps if app.mode in {"workflow", "advanced-chat"}]diff --git a/api/repositories/app/tracing_config_repository.py b/api/repositories/app/tracing_config_repository.pyindex 467380abdd9..afa4968b3b8 100644--- a/api/repositories/app/tracing_config_repository.py+++ b/api/repositories/app/tracing_config_repository.py@@ -1,7 +1,10 @@"""SQLAlchemy persistence adapter for app tracing provider configurations."""+import json+from dataclasses import asdict, replacefrom typing import Any, override+from pydantic import TypeAdapterfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmaker@@ -9,11 +12,16 @@ from models.enums import AppStatusfrom models.model import App, TraceAppConfigfrom repositories.app.console_visibility import console_visible_conditionfrom services.app_tracing_config_service import (+ AppTraceSettings,AppTracingConfigAppNotFoundError,+ AppTracingConfigConflictError,+ AppTracingConfigNotFoundError,AppTracingConfigRecord,AppTracingConfigStore,)+_trace_settings = TypeAdapter(AppTraceSettings)+class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):def __init__(self, *, session_factory: sessionmaker[Session]) -> None:@@ -40,9 +48,10 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):app_id: str,tracing_provider: str,tracing_config: dict[str, Any],+ activate: bool = False,) -> bool:with self._session_factory.begin() as session:- self._require_app(session, workspace_id, app_id)+ app = self._require_app(session, workspace_id, app_id, lock=True)if self._get_config(session, app_id, tracing_provider) is not None:return False@@ -53,6 +62,8 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):tracing_config=dict(tracing_config),))+ if activate:+ self._activate(app, tracing_provider)return True@override@@ -63,14 +74,20 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):app_id: str,tracing_provider: str,tracing_config: dict[str, Any],+ expected: AppTracingConfigRecord,+ activate: bool = False,) -> bool:with self._session_factory.begin() as session:- self._require_app(session, workspace_id, app_id)+ app = self._require_app(session, workspace_id, app_id, lock=True)config = self._get_config(session, app_id, tracing_provider)if config is None:- return False+ raise AppTracingConfigConflictError+ self._check_expected(config, expected)config.tracing_config = dict(tracing_config)+ if activate:+ config.is_active = True+ self._activate(app, tracing_provider)return True@override@@ -82,28 +99,65 @@ class SQLAlchemyAppTracingConfigRepository(AppTracingConfigStore):tracing_provider: str,) -> bool:with self._session_factory.begin() as session:- self._require_app(session, workspace_id, app_id)+ app = self._require_app(session, workspace_id, app_id, lock=True)config = self._get_config(session, app_id, tracing_provider)if config is None:return False+ settings = self._settings(app)+ if settings.tracing_provider == tracing_provider:+ app.tracing = json.dumps(asdict(replace(settings, enabled=False, tracing_provider=None)))session.delete(config)return True+ @override+ def set_trace(+ self, *, workspace_id: str, app_id: str, settings: AppTraceSettings, expected: AppTracingConfigRecord | None+ ) -> None:+ with self._session_factory.begin() as session:+ app = self._require_app(session, workspace_id, app_id, lock=True)+ if settings.enabled:+ if settings.tracing_provider is None or expected is None:+ raise AppTracingConfigNotFoundError+ config = self._get_config(session, app_id, settings.tracing_provider)+ if config is None:+ raise AppTracingConfigConflictError+ self._check_expected(config, expected)+ if not config.is_active or not config.tracing_config:+ raise AppTracingConfigNotFoundError+ app.tracing = json.dumps(asdict(settings))+@staticmethod- def _require_app(session: Session, workspace_id: str, app_id: str) -> None:- app_exists = session.scalar(- select(App.id)- .where(- App.id == app_id,- App.tenant_id == workspace_id,- App.status == AppStatus.NORMAL,- console_visible_condition(),- )- .limit(1)+ def _settings(app: App) -> AppTraceSettings:+ return _trace_settings.validate_json(app.tracing) if app.tracing else AppTraceSettings()++ @classmethod+ def _activate(cls, app: App, provider: str) -> None:+ app.tracing = json.dumps(asdict(replace(cls._settings(app), enabled=True, tracing_provider=provider)))++ @staticmethod+ def _check_expected(config: TraceAppConfig, expected: AppTracingConfigRecord) -> None:+ if (config.id, config.tracing_config, config.is_active) != (+ expected.id,+ expected.tracing_config,+ expected.is_active,+ ):+ raise AppTracingConfigConflictError++ @staticmethod+ def _require_app(session: Session, workspace_id: str, app_id: str, *, lock: bool = False) -> App:+ statement = select(App).where(+ App.id == app_id,+ App.tenant_id == workspace_id,+ App.status == AppStatus.NORMAL,+ console_visible_condition(),)- if app_exists is None:+ if lock:+ statement = statement.with_for_update()+ app = session.scalar(statement)+ if app is None:raise AppTracingConfigAppNotFoundError+ return app@staticmethoddef _get_config(session: Session, app_id: str, tracing_provider: str) -> TraceAppConfig | None:diff --git a/api/repositories/application_trace_repository.py b/api/repositories/application_trace_repository.pynew file mode 100644index 00000000000..653e66ba6e5--- /dev/null+++ b/api/repositories/application_trace_repository.py@@ -0,0 +1,505 @@+"""Bounded transactions for application trace collection and delivery leases."""++import hashlib+import json+from datetime import timedelta+from typing import Any, Literal+from uuid import UUID, uuid5++from dify_agent.protocol.tracing import AgentTraceBatch+from sqlalchemy import delete, or_, select+from sqlalchemy.orm import Session, sessionmaker++from libs.datetime_utils import naive_utc_now+from libs.uuid_utils import uuidv7+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun+from models.model import App, Message, TraceAppConfig+from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowRun+from repositories.factory import DifyAPIRepositoryFactory+from services.tracing.normalization import safe_attributes, span_id+++class TraceCollectionError(Exception):+ def __init__(self, code: str, status: int = 400) -> None:+ self.code = code+ self.status = status+ super().__init__(code)+++class ApplicationTraceRepository:+ def __init__(self, sessions: sessionmaker[Session]) -> None:+ self.sessions = sessions++ def _source(+ self, *, tenant_id: str, app_id: str, kind: str, source_id: str+ ) -> Message | WorkflowRun | WorkflowNodeExecutionModel | None:+ # Close the SQL owner read before consulting a configured backend: LogStore+ # can perform network I/O and need not have a corresponding SQL run/node row.+ with self.sessions() as session:+ if session.scalar(select(App.id).where(App.id == app_id, App.tenant_id == tenant_id)) is None:+ return None+ if kind == "message":+ return session.scalar(select(Message).where(Message.id == source_id, Message.app_id == app_id))+ source: WorkflowRun | WorkflowNodeExecutionModel | None+ if kind == "workflow":+ repository = DifyAPIRepositoryFactory.create_api_workflow_run_repository(self.sessions)+ source = repository.get_workflow_run_by_id(tenant_id=tenant_id, app_id=app_id, run_id=source_id)+ elif kind == "node":+ node_repository = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(self.sessions)+ source = node_repository.get_execution_by_id(execution_id=source_id, tenant_id=tenant_id)+ else:+ return None+ # The node reader has no app filter. Check the entire chain on either+ # backend's result before exposing a destination or authorizing a retry.+ if source is None or (source.id, source.tenant_id, source.app_id) != (source_id, tenant_id, app_id):+ return None+ return source++ def start(+ self,+ *,+ tenant_id: str,+ app_id: str,+ source_kind: Literal["message", "workflow", "node"],+ source_id: str,+ session_id: str,+ workflow_id: str | None = None,+ ) -> ApplicationTraceRun | None:+ # Re-entry uses the already validated, frozen source. It must not depend+ # on LogStore availability or indexing latency after collection began.+ with self.sessions() as session:+ if session.scalar(select(App.id).where(App.id == app_id, App.tenant_id == tenant_id)) is None:+ return None+ existing = session.scalar(+ select(ApplicationTraceRun).where(+ ApplicationTraceRun.tenant_id == tenant_id,+ ApplicationTraceRun.app_id == app_id,+ ApplicationTraceRun.source_kind == source_kind,+ ApplicationTraceRun.source_id == source_id,+ )+ )+ if existing is not None:+ return existing+ source = self._source(tenant_id=tenant_id, app_id=app_id, kind=source_kind, source_id=source_id)+ # The app row serializes parallel node starts, so one workflow freezes one+ # destination even when its first two Agent nodes start concurrently.+ with self.sessions.begin() as session:+ app = session.scalar(select(App).where(App.id == app_id, App.tenant_id == tenant_id).with_for_update())+ if app is None:+ return None+ existing = session.scalar(+ select(ApplicationTraceRun).where(+ ApplicationTraceRun.tenant_id == tenant_id,+ ApplicationTraceRun.app_id == app_id,+ ApplicationTraceRun.source_kind == source_kind,+ ApplicationTraceRun.source_id == source_id,+ )+ )+ if existing is not None:+ return existing+ if source is None:+ # A single-node preview allocates its execution ID before the+ # node row is saved. Only the trusted workflow builder may reserve+ # it, after checking its persisted owner workflow here.+ if source_kind not in {"node", "workflow"} or workflow_id is None:+ return None+ workflow = session.scalar(+ select(Workflow).where(+ Workflow.id == workflow_id, Workflow.app_id == app_id, Workflow.tenant_id == tenant_id+ )+ )+ if workflow is None:+ return None+ try:+ policy = json.loads(app.tracing) if app.tracing else {}+ except (TypeError, ValueError):+ policy = {}+ if not isinstance(policy, dict):+ policy = {}+ provider = policy.get("tracing_provider")+ config = (+ session.scalar(+ select(TraceAppConfig).where(+ TraceAppConfig.app_id == app_id,+ TraceAppConfig.tracing_provider == provider,+ )+ )+ if isinstance(provider, str) and policy.get("enabled") is True+ else None+ )+ enabled = config is not None and isinstance(config.tracing_config, dict)+ frozen = (+ dict(config.tracing_config) if config is not None and isinstance(config.tracing_config, dict) else {}+ )+ revision = hashlib.sha256(+ json.dumps({"config": frozen, "policy": policy}, sort_keys=True, default=str).encode()+ ).hexdigest()+ record = ApplicationTraceRun(+ id=str(uuidv7()),+ tenant_id=tenant_id,+ app_id=app_id,+ source_kind=source_kind,+ source_id=source_id,+ provider=provider if enabled and isinstance(provider, str) else "",+ config_revision=revision,+ provider_config=frozen,+ trace_id=uuidv7().hex,+ root_span_id=span_id(source_id),+ session_id=session_id[:255] or source_id,+ capture_content=enabled and policy.get("capture_content", False) is True,+ capture_agent=enabled and policy.get("capture_agent", True) is True,+ state="collecting" if enabled else "not_collected",+ )+ session.add(record)+ session.flush()+ return record++ def source_run(+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str+ ) -> ApplicationTraceRun | None:+ with self.sessions() as session:+ return session.scalar(+ select(ApplicationTraceRun).where(+ ApplicationTraceRun.tenant_id == tenant_id,+ ApplicationTraceRun.app_id == app_id,+ ApplicationTraceRun.source_kind == source_kind,+ ApplicationTraceRun.source_id == source_id,+ )+ )++ def record_initialization_failure(+ self,+ *,+ tenant_id: str,+ app_id: str,+ source_kind: str,+ source_id: str,+ ) -> ApplicationTraceRun | None:+ """A missing start snapshot must never fall back to an unmasked exporter."""+ source = self._source(tenant_id=tenant_id, app_id=app_id, kind=source_kind, source_id=source_id)+ if source is None:+ return None+ with self.sessions.begin() as session:+ app = session.scalar(select(App).where(App.id == app_id, App.tenant_id == tenant_id).with_for_update())+ if app is None:+ return None+ current = session.scalar(+ select(ApplicationTraceRun).where(+ ApplicationTraceRun.tenant_id == tenant_id,+ ApplicationTraceRun.app_id == app_id,+ ApplicationTraceRun.source_kind == source_kind,+ ApplicationTraceRun.source_id == source_id,+ )+ )+ if current is not None:+ return current+ current = ApplicationTraceRun(+ id=str(uuidv7()),+ tenant_id=tenant_id,+ app_id=app_id,+ source_kind=source_kind,+ source_id=source_id,+ provider="",+ config_revision="",+ provider_config={},+ trace_id=uuidv7().hex,+ root_span_id=span_id(source_id),+ session_id=source_id,+ capture_content=False,+ capture_agent=False,+ state="failed",+ error_code="tracing_initialization_failed",+ terminal_at=naive_utc_now(),+ )+ session.add(current)+ return current++ def invocation(+ self, *, run_id: str, agent_id: str, node_execution_id: str | None, attempt: int, call_key: str+ ) -> ApplicationTraceInvocation:+ identifier = str(uuid5(UUID(run_id), call_key))+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.state not in {"collecting", "ready"}:+ raise TraceCollectionError("tracing_run_not_collecting", 409)+ invocation = session.get(ApplicationTraceInvocation, identifier)+ if invocation is not None and (+ invocation.agent_id != agent_id+ or invocation.node_execution_id != node_execution_id+ or invocation.attempt != attempt+ ):+ raise TraceCollectionError("tracing_invocation_conflict", 409)+ if invocation is None:+ invocation = ApplicationTraceInvocation(+ id=identifier,+ run_id=run_id,+ agent_id=agent_id,+ parent_span_id=span_id(node_execution_id) if node_execution_id else run.root_span_id,+ span_id=span_id(f"agent-attempt:{identifier}"),+ node_execution_id=node_execution_id,+ attempt=attempt,+ )+ session.add(invocation)+ return invocation++ def ingest(self, batch: AgentTraceBatch, *, max_spans: int, max_bytes: int) -> tuple[int, str]:+ # Hash the original validated payload. Sanitization cannot turn two+ # conflicting producer payloads into a falsely successful replay.+ data = batch.model_dump(mode="json")+ canonical = json.dumps(data, sort_keys=True, separators=(",", ":")).encode()+ digest = hashlib.sha256(canonical).hexdigest()+ with self.sessions.begin() as session:+ run = session.scalar(+ select(ApplicationTraceRun).where(ApplicationTraceRun.id == str(batch.run_id)).with_for_update()+ )+ if run is None:+ raise TraceCollectionError("tracing_run_not_found", 404)+ app = session.scalar(select(App.id).where(App.id == run.app_id, App.tenant_id == run.tenant_id))+ if app is None:+ raise TraceCollectionError("tracing_owner_not_found", 404)+ invocation = session.scalar(+ select(ApplicationTraceInvocation).where(+ ApplicationTraceInvocation.id == str(batch.invocation_id),+ ApplicationTraceInvocation.run_id == run.id,+ )+ )+ if invocation is None:+ raise TraceCollectionError("tracing_invocation_not_found", 404)+ previous = session.get(ApplicationTraceBatch, str(batch.batch_id))+ if previous is not None:+ if (+ previous.run_id != run.id+ or previous.invocation_id != invocation.id+ or previous.payload_hash != digest+ ):+ raise TraceCollectionError("tracing_batch_conflict", 409)+ return len(previous.spans), run.id+ if run.state in {"accepted", "exporting", "uncertain", "failed"} or invocation.finalized:+ raise TraceCollectionError("tracing_collection_closed", 409)+ if any(span.trace_id != run.trace_id for span in batch.spans):+ raise TraceCollectionError("tracing_trace_id_mismatch", 409)+ if run.span_count + len(batch.spans) > max_spans or run.payload_bytes + len(canonical) > max_bytes:+ raise TraceCollectionError("tracing_payload_limit", 413)+ spans: list[dict[str, Any]] = []+ for value in batch.spans:+ span = value.model_dump(mode="json")+ span["attributes"] = safe_attributes(span["attributes"], capture_content=run.capture_content)+ if not run.capture_content:+ span["events"] = []+ span["status_message"] = "Agent operation failed" if span["status"] == "error" else None+ spans.append(span)+ session.add(+ ApplicationTraceBatch(+ id=str(batch.batch_id),+ run_id=run.id,+ invocation_id=invocation.id,+ payload_hash=digest,+ spans=spans,+ final=batch.final,+ )+ )+ run.span_count += len(spans)+ run.payload_bytes += len(canonical)+ run.updated_at = naive_utc_now()+ if batch.final:+ invocation.finalized = True+ invocation.finalized_at = naive_utc_now()+ return len(spans), run.id++ def terminal(self, run_id: str, trace: dict[str, Any], root_span_id: str) -> None:+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.state in {"accepted", "exporting", "uncertain", "failed"}:+ return+ if run.outer_trace is not None and run.outer_trace != trace:+ # A duplicate terminal task must not rewrite a completed immutable+ # span. The first persisted terminal envelope owns the export.+ return+ run.outer_trace = trace+ run.root_span_id = root_span_id+ run.terminal_at = run.terminal_at or naive_utc_now()+ run.updated_at = naive_utc_now()++ def get(self, run_id: str) -> ApplicationTraceRun | None:+ with self.sessions() as session:+ return session.get(ApplicationTraceRun, run_id)++ def collected(self, run_id: str) -> tuple[list[ApplicationTraceInvocation], list[ApplicationTraceBatch]]:+ with self.sessions() as session:+ invocations = list(+ session.scalars(+ select(ApplicationTraceInvocation)+ .where(+ ApplicationTraceInvocation.run_id == run_id,+ )+ .order_by(+ ApplicationTraceInvocation.attempt,+ ApplicationTraceInvocation.created_at,+ ApplicationTraceInvocation.id,+ )+ )+ )+ batches = list(+ session.scalars(+ select(ApplicationTraceBatch)+ .where(ApplicationTraceBatch.run_id == run_id)+ .order_by(ApplicationTraceBatch.created_at)+ )+ )+ return invocations, batches++ def claim(+ self, run_id: str, *, collection_timeout: int, lease_seconds: int+ ) -> tuple[ApplicationTraceRun, str] | None:+ now = naive_utc_now()+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.state in {"not_collected", "accepted", "failed", "uncertain"}:+ return None+ if run.state == "exporting":+ last_heartbeat = run.export_heartbeat_at or run.export_started_at+ if last_heartbeat and last_heartbeat + timedelta(seconds=lease_seconds) < now:+ run.state, run.error_code = "uncertain", "export_lease_expired"+ run.updated_at = now+ return None+ if run.outer_trace is None:+ return None+ incomplete = session.scalar(+ select(ApplicationTraceInvocation.id)+ .where(ApplicationTraceInvocation.run_id == run_id, ApplicationTraceInvocation.finalized.is_(False))+ .limit(1)+ )+ if incomplete:+ if run.terminal_at and run.terminal_at + timedelta(seconds=collection_timeout) < now:+ run.state, run.error_code = "failed", "agent_trace_collection_incomplete"+ run.updated_at = now+ return None+ if run.next_attempt_at and run.next_attempt_at > now:+ return None+ app = session.scalar(select(App.id).where(App.id == run.app_id, App.tenant_id == run.tenant_id))+ if app is None:+ run.state, run.error_code = "failed", "tracing_owner_not_found"+ return None+ token = str(uuidv7())+ run.state = "exporting"+ run.export_token = token+ run.export_started_at = now+ run.export_heartbeat_at = now+ run.updated_at = now+ run.attempts += 1+ return run, token++ def renew_export_lease(self, run_id: str, token: str) -> bool:+ """Renew only the current export attempt, in a short independent transaction."""+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.state != "exporting" or run.export_token != token:+ return False+ heartbeat_at = naive_utc_now()+ run.export_heartbeat_at = heartbeat_at+ run.updated_at = heartbeat_at+ return True++ def complete(+ self,+ run_id: str,+ token: str,+ *,+ state: str,+ error_code: str | None = None,+ result: Any = None,+ retry_delay: int | None = None,+ ) -> None:+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.export_token != token:+ return+ late_acceptance = (+ state == "accepted"+ and result is not None+ and run.state == "uncertain"+ and run.error_code == "export_lease_expired"+ )+ if run.state != "exporting" and not late_acceptance:+ return+ run.state, run.error_code = state, error_code+ run.updated_at = naive_utc_now()+ run.export_token = None+ if result is not None:+ run.provider_trace_id, run.trace_url, run.span_urls = (+ result.trace_id,+ result.trace_url,+ result.span_urls,+ )+ run.accepted_at = naive_utc_now()+ run.next_attempt_at = naive_utc_now() + timedelta(seconds=retry_delay) if retry_delay is not None else None++ def recovery_checked(self, run_id: str, *, error_code: str | None = None) -> None:+ """Rotate checked incomplete work without changing a concurrent export."""+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.outer_trace is not None or run.state not in {"collecting", "ready"}:+ return+ run.updated_at = naive_utc_now()+ if error_code is not None:+ run.state, run.error_code = "failed", error_code++ def pending_ids(self, limit: int = 100) -> list[str]:+ with self.sessions() as session:+ return list(+ session.scalars(+ select(ApplicationTraceRun.id)+ .where(+ ApplicationTraceRun.state.in_(["collecting", "ready", "retrying", "exporting"]),+ or_(+ ApplicationTraceRun.next_attempt_at.is_(None),+ ApplicationTraceRun.next_attempt_at <= naive_utc_now(),+ ),+ )+ .order_by(ApplicationTraceRun.terminal_at.is_(None), ApplicationTraceRun.updated_at)+ .limit(limit)+ )+ )++ def retry(self, *, run_id: str, acknowledge_possible_duplicates: bool) -> None:+ with self.sessions.begin() as session:+ run = session.scalar(select(ApplicationTraceRun).where(ApplicationTraceRun.id == run_id).with_for_update())+ if run is None or run.state == "not_collected":+ raise TraceCollectionError("tracing_run_not_found", 404)+ if run.state == "accepted":+ raise TraceCollectionError("tracing_already_accepted", 409)+ if run.state not in {"failed", "uncertain"} or run.outer_trace is None:+ raise TraceCollectionError("tracing_retry_not_allowed", 409)+ incomplete = session.scalar(+ select(ApplicationTraceInvocation.id)+ .where(+ ApplicationTraceInvocation.run_id == run.id,+ ApplicationTraceInvocation.finalized.is_(False),+ )+ .limit(1)+ )+ if incomplete or (run.error_code or "").startswith("tracing_"):+ raise TraceCollectionError("tracing_retry_not_allowed", 409)+ if run.state == "uncertain" and not acknowledge_possible_duplicates:+ raise TraceCollectionError("tracing_duplicate_acknowledgement_required", 409)+ run.state, run.error_code, run.export_token = "ready", None, None+ run.next_attempt_at = None+ run.updated_at = naive_utc_now()++ def purge(self, *, retention_days: int, limit: int = 100) -> int:+ cutoff = naive_utc_now() - timedelta(days=retention_days)+ with self.sessions.begin() as session:+ ids = list(+ session.scalars(+ select(ApplicationTraceRun.id)+ .where(ApplicationTraceRun.created_at < cutoff)+ .limit(limit)+ .with_for_update()+ )+ )+ if not ids:+ return 0+ session.execute(delete(ApplicationTraceBatch).where(ApplicationTraceBatch.run_id.in_(ids)))+ session.execute(delete(ApplicationTraceInvocation).where(ApplicationTraceInvocation.run_id.in_(ids)))+ session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id.in_(ids)))+ return len(ids)diff --git a/api/services/agent_tracing_config_service.py b/api/services/agent_tracing_config_service.pynew file mode 100644index 00000000000..c1dca1702c3--- /dev/null+++ b/api/services/agent_tracing_config_service.py@@ -0,0 +1,71 @@+"""Agent-scoped tracing configuration, backed by the existing App tracing owners.++Only roster Agent Apps own third-party credentials. Workflow-only Agents inherit+their caller's tracing policy and must not create a hidden configuration owner.+"""++from typing import Any, Protocol++from machinery.context import RequestContext+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigRecord, AppTracingConfigService+++class AgentTracingOwner(Protocol):+ def resolve_roster_app_id(self, *, tenant_id: str, agent_id: str) -> str: ...+++class AppTracingSettings(Protocol):+ def get_trace(self, context: RequestContext, app_id: str) -> AppTraceSettings: ...++ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None: ...+++class AgentTracingConfigService:+ def __init__(+ self, *, owners: AgentTracingOwner, settings: AppTracingSettings, configs: AppTracingConfigService+ ) -> None:+ self._owners = owners+ self._settings = settings+ self._configs = configs++ def resolve_app_id(self, context: RequestContext, agent_id: str) -> str:+ """Resolve the roster Agent's tenant-scoped App without creating runtime state."""+ return self._owners.resolve_roster_app_id(tenant_id=context.active_workspace_id, agent_id=agent_id)++ def get_trace(self, context: RequestContext, agent_id: str) -> AppTraceSettings:+ return self._settings.get_trace(context, self.resolve_app_id(context, agent_id))++ def set_trace(self, context: RequestContext, agent_id: str, settings: AppTraceSettings) -> None:+ self._settings.set_trace(context, self.resolve_app_id(context, agent_id), settings)++ def get(self, context: RequestContext, agent_id: str, tracing_provider: str) -> AppTracingConfigRecord | None:+ return self._configs.get(context, self.resolve_app_id(context, agent_id), tracing_provider)++ def create(+ self,+ context: RequestContext,+ agent_id: str,+ tracing_provider: str,+ tracing_config: dict[str, Any],+ *,+ activate: bool = False,+ ) -> None:+ self._configs.create(+ context, self.resolve_app_id(context, agent_id), tracing_provider, tracing_config, activate=activate+ )++ def update(+ self,+ context: RequestContext,+ agent_id: str,+ tracing_provider: str,+ tracing_config: dict[str, Any],+ *,+ activate: bool = False,+ ) -> None:+ self._configs.update(+ context, self.resolve_app_id(context, agent_id), tracing_provider, tracing_config, activate=activate+ )++ def delete(self, context: RequestContext, agent_id: str, tracing_provider: str) -> None:+ self._configs.delete(context, self.resolve_app_id(context, agent_id), tracing_provider)diff --git a/api/services/app/console_service.py b/api/services/app/console_service.pyindex 9ec2e62574e..c100f9aa6f9 100644--- a/api/services/app/console_service.py+++ b/api/services/app/console_service.py@@ -11,6 +11,7 @@ from machinery.context import RequestContextfrom models.model import AppModefrom services.agent.errors import InvalidRosterAgentPackageErrorfrom services.agent.roster_package_entities import RosterAgentPackageExport+from services.app_tracing_config_service import AppTraceSettingsfrom services.entities.app_entities import (AppChange,AppCreationSettings,@@ -21,7 +22,6 @@ from services.entities.app_entities import (AppPage,AppRecord,AppReference,- AppTraceSettings,CopyAppParams,CreateAppParams,ImportedAppPackage,@@ -121,8 +121,6 @@ class ConsoleApps(Protocol):def get_trace(self, context: RequestContext, app_id: str) -> AppTraceSettings: ...- def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None: ...-class AppTransfers(Protocol):def download_import(self, url: str) -> AbstractContextManager[BinaryIO]: ...@@ -171,10 +169,8 @@ class CreatorsPlatform(Protocol):def redirect_url(self, claim_code: str, oauth_code: str | None) -> str: ...-class AppTraceProvider(Protocol):- def validate_provider(self, tracing_provider: str) -> None: ...-- def require_provider_available(self, tracing_provider: str) -> None: ...+class AppTracingSettings(Protocol):+ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None: ...class AppLifecycle(Protocol):@@ -203,7 +199,7 @@ class ConsoleAppService:access: ConsoleAppAccess,transfers: AppTransfers,creators: CreatorsPlatform,- tracing: AppTraceProvider,+ tracing: AppTracingSettings,lifecycle: AppLifecycle,) -> None:self._apps = apps@@ -457,9 +453,4 @@ class ConsoleAppService:def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:self._apps.get_reference(context, app_id)- if settings.tracing_provider is not None:- if settings.enabled:- self._tracing.require_provider_available(settings.tracing_provider)- else:- self._tracing.validate_provider(settings.tracing_provider)- self._apps.set_trace(context, app_id, settings)+ self._tracing.set_trace(context, app_id, settings)diff --git a/api/services/app_tracing_config_gateway.py b/api/services/app_tracing_config_gateway.pyindex eb696f35afb..c513cb215f9 100644--- a/api/services/app_tracing_config_gateway.py+++ b/api/services/app_tracing_config_gateway.py@@ -20,7 +20,6 @@ from services.app_tracing_config_service import (logger = logging.getLogger(__name__)_PROJECT_URL_FALLBACKS = {- "arize": "https://app.arize.com/","phoenix": "https://app.phoenix.arize.com/projects/","langsmith": "https://smith.langchain.com/","opik": "https://www.comet.com/opik/",@@ -41,6 +40,17 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):def require_provider_available(self, tracing_provider: str) -> None:self._provider_config(tracing_provider)+ @override+ def validate_saved_config(+ self, *, workspace_id: str, tracing_provider: str, tracing_config: dict[str, Any]+ ) -> None:+ provider_config = self._provider_config(tracing_provider)+ try:+ decrypted = OpsTraceManager.decrypt_tracing_config(workspace_id, tracing_provider, tracing_config)+ except Exception as error:+ raise AppTracingConfigProcessingError from error+ self._validate_config(provider_config, decrypted)+@overridedef prepare_new_config(self,@@ -50,10 +60,21 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):tracing_config: dict[str, Any],) -> dict[str, Any]:provider_config = self._provider_config(tracing_provider)- normalized_config = self._normalize_config(provider_config, tracing_config)+ if tracing_provider == "tencent":+ normalized_config = self._normalize_tencent_config(tracing_config)+ elif tracing_provider == "mlflow":+ normalized_config = self._normalize_mlflow_config(tracing_config)+ elif tracing_provider == "phoenix":+ normalized_config = self._normalize_phoenix_config(tracing_config)+ elif tracing_provider == "aliyun":+ normalized_config = self._normalize_aliyun_config(tracing_config)+ else:+ normalized_config = self._normalize_config(provider_config, tracing_config)self._verify_config(normalized_config, tracing_provider)- project_url = self._get_project_url_for_create(normalized_config, tracing_provider)+ project_url = (+ None if tracing_provider == "arize" else self._get_project_url(normalized_config, tracing_provider)+ )try:encrypted_config = OpsTraceManager.encrypt_tracing_config(workspace_id,@@ -62,7 +83,9 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):)except Exception as error:raise AppTracingConfigProcessingError from error- if project_url:+ if tracing_provider == "arize":+ self._resolve_arize_destination(encrypted_config, normalized_config)+ elif project_url:encrypted_config["project_url"] = project_urlreturn encrypted_config@@ -76,6 +99,16 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):current_tracing_config: dict[str, Any] | None,) -> dict[str, Any]:provider_config = self._provider_config(tracing_provider)+ if tracing_provider == "tencent":+ # Check the original URL before the legacy model discards its path+ # and query. Masked tokens are restored by encryption below.+ tracing_config = self._normalize_tencent_config(tracing_config)+ elif tracing_provider == "mlflow":+ tracing_config = self._normalize_mlflow_config(tracing_config)+ elif tracing_provider == "phoenix":+ tracing_config = self._normalize_phoenix_config(tracing_config)+ elif tracing_provider == "aliyun":+ tracing_config = self._normalize_aliyun_config(tracing_config)self._validate_config(provider_config, tracing_config)try:encrypted_config = OpsTraceManager.encrypt_tracing_config(@@ -93,6 +126,30 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):raise AppTracingConfigProcessingError from errorself._verify_config(decrypted_config, tracing_provider)+ if tracing_provider == "weave":+ # Verification resolves an optional entity and qualified project name.+ # Encryption happened first to restore masked credentials; persist the+ # verified non-secret destination alongside that unchanged ciphertext.+ encrypted_config["entity"] = decrypted_config["entity"]+ encrypted_config["project"] = decrypted_config["project"]+ elif tracing_provider == "databricks":+ encrypted_config["host"] = decrypted_config["host"]+ encrypted_config["experiment_id"] = decrypted_config["experiment_id"]+ elif tracing_provider == "mlflow":+ encrypted_config["tracking_uri"] = decrypted_config["tracking_uri"]+ encrypted_config["experiment_id"] = decrypted_config["experiment_id"]+ elif tracing_provider == "phoenix":+ encrypted_config["endpoint"] = decrypted_config["endpoint"]+ encrypted_config["project"] = decrypted_config["project"]+ elif tracing_provider == "opik":+ for key in ("url", "workspace", "project"):+ encrypted_config[key] = decrypted_config[key]+ # Re-resolve after destination changes; never retain a previous project's link.+ project_url = None if tracing_provider == "arize" else self._get_project_url(decrypted_config, tracing_provider)+ if tracing_provider == "arize":+ self._resolve_arize_destination(encrypted_config, decrypted_config)+ elif project_url:+ encrypted_config["project_url"] = project_urlreturn encrypted_config@override@@ -118,7 +175,70 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):except Exception as error:raise AppTracingConfigProcessingError from error- if tracing_provider == "langfuse" and not decrypted_config.get("project_key"):+ if tracing_provider == "arize":+ from dify_trace_arize_phoenix.arize_configuration import project_url++ # The encrypted record carries server-owned non-secret metadata.+ # Legacy decryption intentionally retains only provider input fields.+ presented_config["project_url"] = project_url(tracing_config)+ elif tracing_provider == "aliyun":+ from dify_trace_aliyun.configuration import console_url++ # GET never constructs the legacy queue/exporter or probes credentials.+ presented_config["project_url"] = console_url()+ elif tracing_provider == "tencent":+ from dify_trace_tencent.configuration import console_url++ # Historical names remain readable. GET performs no connection+ # check and never constructs legacy tracing or metrics exporters.+ presented_config["project_url"] = console_url()+ elif tracing_provider == "opik":+ from dify_trace_opik.config import OpikConfig+ from dify_trace_opik.configuration import console_url, project_url++ try:+ config = OpikConfig.model_validate(decrypted_config)+ presented_config["project_url"] = project_url(config) or console_url(config)+ except ValueError:+ presented_config["project_url"] = _PROJECT_URL_FALLBACKS["opik"]+ elif tracing_provider == "phoenix":+ from dify_trace_arize_phoenix.config import PhoenixConfig+ from dify_trace_arize_phoenix.configuration import project_url++ try:+ presented_config["project_url"] = project_url(PhoenixConfig.model_validate(decrypted_config))+ except ValueError:+ # Historical destinations stay readable; never initialize a+ # legacy exporter or invent a link for an unsafe base URL.+ presented_config["project_url"] = None+ elif tracing_provider == "mlflow":+ from dify_trace_mlflow.config import MLflowConfig+ from dify_trace_mlflow.oss_configuration import project_url++ try:+ presented_config["project_url"] = project_url(MLflowConfig.model_validate(decrypted_config))+ except ValueError:+ # OSS deployments have no universal project destination. Never+ # initialize an SDK or substitute localhost for an invalid URL.+ presented_config["project_url"] = None+ elif tracing_provider == "databricks":+ from dify_trace_mlflow.config import DatabricksConfig+ from dify_trace_mlflow.configuration import project_url++ try:+ presented_config["project_url"] = project_url(DatabricksConfig.model_validate(decrypted_config))+ except ValueError:+ presented_config["project_url"] = _PROJECT_URL_FALLBACKS["databricks"]+ elif tracing_provider == "weave":+ from dify_trace_weave.config import WeaveConfig+ from dify_trace_weave.configuration import console_url, project_url++ try:+ config = WeaveConfig.model_validate(decrypted_config)+ presented_config["project_url"] = project_url(config) or console_url(config)+ except ValueError:+ presented_config["project_url"] = _PROJECT_URL_FALLBACKS["weave"]+ elif tracing_provider == "langfuse" and not decrypted_config.get("project_key"):try:project_key = OpsTraceManager.get_trace_config_project_key(decrypted_config, tracing_provider)presented_config["project_url"] = f"{decrypted_config.get('host')}/project/{project_key}"@@ -135,6 +255,24 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):return presented_config+ @staticmethod+ def _resolve_arize_destination(encrypted_config: dict[str, Any], verified_config: dict[str, Any]) -> None:+ from dify_trace_arize_phoenix.arize_configuration import (+ DESTINATION_KEY,+ bind_destination,+ project_url,+ resolve_project_destination,+ )++ # Only the explicit write path may replace this cache. Never trust IDs+ # or URLs submitted by a client, or retain a previous target on failure.+ encrypted_config.pop(DESTINATION_KEY, None)+ encrypted_config.pop("project_url", None)+ destination = resolve_project_destination(verified_config)+ if destination is not None:+ encrypted_config[DESTINATION_KEY] = bind_destination(encrypted_config, destination)+ encrypted_config["project_url"] = project_url(encrypted_config)+@staticmethoddef _provider_config(tracing_provider: str) -> TracingProviderConfigEntry:try:@@ -158,6 +296,42 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):normalized_config[key] = default_values.get(key)return normalized_config+ @staticmethod+ def _normalize_aliyun_config(tracing_config: dict[str, Any]) -> dict[str, Any]:+ from dify_trace_aliyun.configuration import normalize_configuration++ try:+ return normalize_configuration(tracing_config).model_dump()+ except ValueError:+ raise AppTracingConfigInvalidConfigurationError from None++ @staticmethod+ def _normalize_tencent_config(tracing_config: dict[str, Any]) -> dict[str, Any]:+ from dify_trace_tencent.configuration import TencentConfigurationError, normalize_configuration++ try:+ return normalize_configuration(tracing_config).model_dump()+ except TencentConfigurationError as error:+ raise AppTracingConfigInvalidConfigurationError from error++ @staticmethod+ def _normalize_mlflow_config(tracing_config: dict[str, Any]) -> dict[str, Any]:+ from dify_trace_mlflow.oss_configuration import normalize_configuration++ try:+ return normalize_configuration(tracing_config).model_dump()+ except ValueError as error:+ raise AppTracingConfigInvalidConfigurationError from error++ @staticmethod+ def _normalize_phoenix_config(tracing_config: dict[str, Any]) -> dict[str, Any]:+ from dify_trace_arize_phoenix.configuration import normalize_configuration++ try:+ return normalize_configuration(tracing_config).model_dump()+ except ValueError as error:+ raise AppTracingConfigInvalidConfigurationError from error+@staticmethoddef _validate_config(provider_config: TracingProviderConfigEntry,@@ -167,10 +341,105 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):try:return config_class.model_validate(tracing_config)except ValidationError as error:+ # Aliyun credentials can be embedded in a pasted endpoint. Do not+ # propagate a Pydantic error object whose errors() includes inputs.+ if config_class.__module__ == "dify_trace_aliyun.config":+ raise AppTracingConfigInvalidConfigurationError from Noneraise AppTracingConfigInvalidConfigurationError from error@staticmethoddef _verify_config(tracing_config: dict[str, Any], tracing_provider: str) -> None:+ if tracing_provider == "arize":+ from dify_trace_arize_phoenix.arize_configuration import ArizeConfigurationError, verify_configuration+ from dify_trace_arize_phoenix.config import ArizeConfig++ try:+ verify_configuration(ArizeConfig.model_validate(tracing_config))+ except ArizeConfigurationError as error:+ logger.warning("Arize ingestion verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from None+ return+ if tracing_provider == "aliyun":+ from dify_trace_aliyun.config import AliyunConfig+ from dify_trace_aliyun.configuration import AliyunConfigurationError, verify_configuration++ try:+ verify_configuration(AliyunConfig.model_validate(tracing_config))+ except AliyunConfigurationError as error:+ logger.warning("Aliyun configuration verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from None+ return+ if tracing_provider == "phoenix":+ from dify_trace_arize_phoenix.config import PhoenixConfig+ from dify_trace_arize_phoenix.configuration import PhoenixConfigurationError, verify_configuration++ try:+ resolved = verify_configuration(PhoenixConfig.model_validate(tracing_config))+ except PhoenixConfigurationError as error:+ logger.warning("Phoenix configuration verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from error+ tracing_config["endpoint"] = resolved.endpoint+ tracing_config["project"] = resolved.project+ return+ if tracing_provider == "mlflow":+ from dify_trace_mlflow.config import MLflowConfig+ from dify_trace_mlflow.oss_configuration import MLflowConfigurationError, verify_configuration++ try:+ resolved = verify_configuration(MLflowConfig.model_validate(tracing_config))+ except MLflowConfigurationError as error:+ logger.warning("MLflow configuration verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from error+ tracing_config["tracking_uri"] = resolved.tracking_uri+ tracing_config["experiment_id"] = resolved.experiment_id+ return+ if tracing_provider == "tencent":+ from dify_trace_tencent.config import TencentConfig+ from dify_trace_tencent.configuration import TencentConfigurationError, verify_configuration++ try:+ verify_configuration(TencentConfig.model_validate(tracing_config))+ except TencentConfigurationError as error:+ logger.warning("Tencent collector connection check failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from error+ return+ if tracing_provider == "opik":+ from dify_trace_opik.config import OpikConfig+ from dify_trace_opik.configuration import OpikConfigurationError, verify_configuration++ try:+ resolved = verify_configuration(OpikConfig.model_validate(tracing_config))+ except OpikConfigurationError as error:+ logger.warning("Opik configuration verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from error+ tracing_config["url"] = resolved.url+ tracing_config["workspace"] = resolved.workspace+ tracing_config["project"] = resolved.project+ return+ if tracing_provider == "databricks":+ from dify_trace_mlflow.config import DatabricksConfig+ from dify_trace_mlflow.configuration import DatabricksConfigurationError, verify_configuration++ try:+ resolved = verify_configuration(DatabricksConfig.model_validate(tracing_config))+ except DatabricksConfigurationError as error:+ logger.warning("Databricks configuration verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from error+ tracing_config["host"] = resolved.host+ tracing_config["experiment_id"] = resolved.experiment_id+ return+ if tracing_provider == "weave":+ from dify_trace_weave.config import WeaveConfig+ from dify_trace_weave.configuration import WeaveConfigurationError, verify_configuration++ try:+ resolved = verify_configuration(WeaveConfig.model_validate(tracing_config))+ except WeaveConfigurationError as error:+ logger.warning("Weave configuration verification failed: error_code=%s", error.code)+ raise AppTracingConfigVerificationFailedError from error+ tracing_config["entity"] = resolved.entity+ tracing_config["project"] = resolved.project+ returntry:is_effective = OpsTraceManager.check_trace_config_is_effective(tracing_config, tracing_provider)except ValueError as error:@@ -180,14 +449,50 @@ class OpsTraceManagerGateway(TracingConfigProviderGateway):raise AppTracingConfigVerificationFailedError@staticmethod- def _get_project_url_for_create(tracing_config: dict[str, Any], tracing_provider: str) -> str | None:+ def _get_project_url(tracing_config: dict[str, Any], tracing_provider: str) -> str | None:try:- if tracing_provider in ("arize", "phoenix"):- return OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider)+ if tracing_provider == "aliyun":+ from dify_trace_aliyun.configuration import console_url++ return console_url()+ if tracing_provider == "tencent":+ from dify_trace_tencent.configuration import console_url++ return console_url()+ if tracing_provider == "opik":+ from dify_trace_opik.config import OpikConfig+ from dify_trace_opik.configuration import console_url, project_url++ config = OpikConfig.model_validate(tracing_config)+ return project_url(config) or console_url(config)+ if tracing_provider == "databricks":+ from dify_trace_mlflow.config import DatabricksConfig+ from dify_trace_mlflow.configuration import project_url++ return project_url(DatabricksConfig.model_validate(tracing_config))+ if tracing_provider == "mlflow":+ from dify_trace_mlflow.config import MLflowConfig+ from dify_trace_mlflow.oss_configuration import project_url++ return project_url(MLflowConfig.model_validate(tracing_config))+ if tracing_provider == "weave":+ from dify_trace_weave.config import WeaveConfig+ from dify_trace_weave.configuration import project_url++ return project_url(WeaveConfig.model_validate(tracing_config))+ if tracing_provider == "phoenix":+ from dify_trace_arize_phoenix.config import PhoenixConfig+ from dify_trace_arize_phoenix.configuration import project_url++ return project_url(PhoenixConfig.model_validate(tracing_config))+ if tracing_provider == "arize":+ from dify_trace_arize_phoenix.arize_configuration import project_url++ return project_url(tracing_config)if tracing_provider == "langfuse":project_key = OpsTraceManager.get_trace_config_project_key(tracing_config, tracing_provider)return f"{tracing_config.get('host')}/project/{project_key}"- if tracing_provider in ("langsmith", "opik", "mlflow", "databricks", "tencent"):+ if tracing_provider in ("langsmith", "opik"):return OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider)except Exception:return Nonediff --git a/api/services/app_tracing_config_service.py b/api/services/app_tracing_config_service.pyindex 6d7867fa86b..11d3b1028ff 100644--- a/api/services/app_tracing_config_service.py+++ b/api/services/app_tracing_config_service.py@@ -7,6 +7,14 @@ from typing import Any, Protocolfrom machinery.context import RequestContext+@dataclass(frozen=True, slots=True)+class AppTraceSettings:+ enabled: bool = False+ tracing_provider: str | None = None+ capture_content: bool = False+ capture_agent: bool = True++@dataclass(frozen=True, slots=True)class AppTracingConfigRecord:id: str@@ -34,6 +42,7 @@ class AppTracingConfigStore(Protocol):app_id: str,tracing_provider: str,tracing_config: dict[str, Any],+ activate: bool = False,) -> bool: ...def update(@@ -43,6 +52,8 @@ class AppTracingConfigStore(Protocol):app_id: str,tracing_provider: str,tracing_config: dict[str, Any],+ expected: AppTracingConfigRecord,+ activate: bool = False,) -> bool: ...def delete(@@ -53,10 +64,18 @@ class AppTracingConfigStore(Protocol):tracing_provider: str,) -> bool: ...+ def set_trace(+ self, *, workspace_id: str, app_id: str, settings: AppTraceSettings, expected: AppTracingConfigRecord | None+ ) -> None: ...+class TracingConfigProviderGateway(Protocol):def validate_provider(self, tracing_provider: str) -> None: ...+ def validate_saved_config(+ self, *, workspace_id: str, tracing_provider: str, tracing_config: dict[str, Any]+ ) -> None: ...+def prepare_new_config(self,*,@@ -97,6 +116,11 @@ class AppTracingConfigAlreadyExistsError(AppTracingConfigError):super().__init__("Trace config is exist.")+class AppTracingConfigConflictError(AppTracingConfigError):+ def __init__(self) -> None:+ super().__init__("Tracing configuration changed. Refresh and try again.")++class AppTracingConfigNotFoundError(AppTracingConfigError):def __init__(self) -> None:super().__init__("Trace config not exist.")@@ -167,6 +191,8 @@ class AppTracingConfigService:app_id: str,tracing_provider: str,tracing_config: dict[str, Any],+ *,+ activate: bool = False,) -> None:workspace_id = context.active_workspace_idcurrent = self._configs.get(@@ -188,6 +214,7 @@ class AppTracingConfigService:app_id=app_id,tracing_provider=tracing_provider,tracing_config=encrypted_config,+ activate=activate,)if not created:raise AppTracingConfigAlreadyExistsError@@ -198,6 +225,8 @@ class AppTracingConfigService:app_id: str,tracing_provider: str,tracing_config: dict[str, Any],+ *,+ activate: bool = False,) -> None:workspace_id = context.active_workspace_idcurrent = self._configs.get(@@ -220,6 +249,8 @@ class AppTracingConfigService:app_id=app_id,tracing_provider=tracing_provider,tracing_config=encrypted_config,+ expected=current,+ activate=activate,)if not updated:raise AppTracingConfigNotFoundError@@ -233,3 +264,25 @@ class AppTracingConfigService:)if not deleted:raise AppTracingConfigNotFoundError++ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:+ expected = None+ provider = settings.tracing_provider+ if provider is not None:+ self._provider.validate_provider(provider)+ if settings.enabled:+ if provider is None:+ raise AppTracingConfigNotFoundError+ expected = self._configs.get(+ workspace_id=context.active_workspace_id, app_id=app_id, tracing_provider=provider+ )+ if expected is None or not expected.is_active or not expected.tracing_config:+ raise AppTracingConfigNotFoundError+ self._provider.validate_saved_config(+ workspace_id=context.active_workspace_id,+ tracing_provider=provider,+ tracing_config=expected.tracing_config,+ )+ self._configs.set_trace(+ workspace_id=context.active_workspace_id, app_id=app_id, settings=settings, expected=expected+ )diff --git a/api/services/entities/app_entities.py b/api/services/entities/app_entities.pyindex a14e3dc97d6..345f60505ef 100644--- a/api/services/entities/app_entities.py+++ b/api/services/entities/app_entities.py@@ -225,12 +225,6 @@ class AppReference:bound_agent_id: str | None-@dataclass(frozen=True, slots=True)-class AppTraceSettings:- enabled: bool = False- tracing_provider: str | None = None--@dataclass(frozen=True, slots=True)class AppCreationSettings:app: dict[str, Any]diff --git a/api/services/entities/feature_entities.py b/api/services/entities/feature_entities.pyindex 0bf39a385b8..20f9d149000 100644--- a/api/services/entities/feature_entities.py+++ b/api/services/entities/feature_entities.py@@ -179,6 +179,10 @@ class SystemFeatureModel(FeatureResponseModel):"""Non-sensitive bootstrap snapshot exposed before Console or Web authentication."""deployment_edition: DeploymentEdition+ agent_tracing_api_collection_enabled: bool = Field(+ description="Deployment-level API gate for Agent trace collection, independent of workspace and provider "+ "configuration. Does not indicate Agent runtime, worker, or provider health or confirm trace delivery.",+ )enable_app_deploy: bool = Falsesso_enforced_for_signin: bool = Falsesso_enforced_for_signin_protocol: SSOProtocol | None = Nonediff --git a/api/services/system_feature_service.py b/api/services/system_feature_service.pyindex d2056411907..58e3c7ab522 100644--- a/api/services/system_feature_service.py+++ b/api/services/system_feature_service.py@@ -25,8 +25,11 @@ class SystemFeatureService:@classmethoddef get_public_system_features(cls) -> feature_entities.SystemFeatureModel:- """Build the non-sensitive bootstrap snapshot shared by Console and Web."""- system_features = feature_entities.SystemFeatureModel(deployment_edition=dify_config.DEPLOYMENT_EDITION)+ """Build the shared public snapshot, including the Agent tracing deployment gate."""+ system_features = feature_entities.SystemFeatureModel(+ deployment_edition=dify_config.DEPLOYMENT_EDITION,+ agent_tracing_api_collection_enabled=dify_config.OPS_AGENT_TRACE_ENABLED,+ )system_features.rbac_enabled = dify_config.RBAC_ENABLEDcls._fulfill_system_params_from_env(system_features)diff --git a/api/services/tracing/__init__.py b/api/services/tracing/__init__.pynew file mode 100644index 00000000000..6aaaab24ebe--- /dev/null+++ b/api/services/tracing/__init__.py@@ -0,0 +1 @@+"""Application-owned tracing policy, collection, and delivery."""diff --git a/api/services/tracing/genai_attributes.py b/api/services/tracing/genai_attributes.pynew file mode 100644index 00000000000..1b1abcfe814--- /dev/null+++ b/api/services/tracing/genai_attributes.py@@ -0,0 +1,58 @@+"""GenAI semantic attributes shared by the Aliyun and Tencent adapters."""++from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalTrace+from services.tracing.provider_common import json_value, token_usage+++def attributes(span: CanonicalSpan, trace: CanonicalTrace, provider: str) -> dict[str, object]:+ kind = span.kind.value.upper()+ if provider == "tencent":+ kind = {"LLM": "GENERATION", "CHAIN": "WORKFLOW"}.get(kind, kind)+ result: dict[str, object] = {+ "gen_ai.span.kind": kind,+ "gen_ai.session.id": trace.session_id,+ "gen_ai.framework": "dify",+ "dify.span.id": span.id,+ "dify.metadata": json_value(span.metadata),+ "gen_ai.entity.input" if provider == "tencent" else "input.value": json_value(span.inputs),+ "gen_ai.entity.output" if provider == "tencent" else "output.value": json_value(span.outputs),+ }+ if provider == "tencent" and span.id == trace.root_span_id:+ result["gen_ai.is_entry"] = "true"+ for key in ("tenant_id", "app_id", "agent_id", "workflow_run_id", "node_execution_id", "message_id"):+ value = span.metadata.get(key) or trace.spans[0].metadata.get(key)+ if value:+ result[f"dify.{key}"] = str(value)+ user_id = span.metadata.get("user_id") or trace.spans[0].metadata.get("user_id")+ if user_id:+ result["gen_ai.user.id"] = str(user_id)+ if span.kind is CanonicalSpanKind.LLM:+ result["gen_ai.operation.name"] = "chat"+ result["gen_ai.prompt"] = json_value(span.inputs)+ result["gen_ai.completion"] = json_value(span.outputs)+ result["gen_ai.input.messages"] = json_value(span.inputs)+ result["gen_ai.output.messages"] = json_value(span.outputs)+ model = span.metadata.get("model") or span.metadata.get("model_name")+ if model:+ result["gen_ai.request.model"] = str(model)+ result["gen_ai.response.model"] = str(model)+ if span.metadata.get("model_provider"):+ result["gen_ai.provider.name"] = str(span.metadata["model_provider"])+ result.update({f"gen_ai.usage.{key}_tokens": value for key, value in token_usage(span).items()})+ ttft = span.metadata.get("time_to_first_token", span.metadata.get("ttft"))+ if isinstance(ttft, (int, float)) and ttft >= 0:+ result["gen_ai.response.time_to_first_token"] = int(ttft * 1_000_000_000)+ elif span.kind is CanonicalSpanKind.AGENT:+ result["gen_ai.operation.name"] = "invoke_agent"+ result["gen_ai.agent.name"] = span.name+ elif span.kind is CanonicalSpanKind.TOOL:+ result["gen_ai.operation.name"] = "execute_tool"+ result["gen_ai.tool.name"] = span.name+ result["gen_ai.tool.call.arguments"] = json_value(span.inputs)+ result["gen_ai.tool.call.result"] = json_value(span.outputs)+ result["tool.name"] = span.name+ result["tool.parameters"] = json_value(span.inputs)+ elif span.kind is CanonicalSpanKind.RETRIEVER:+ result["retrieval.query"] = json_value(span.inputs)+ result["retrieval.document"] = json_value(span.outputs)+ return resultdiff --git a/api/services/tracing/normalization.py b/api/services/tracing/normalization.pynew file mode 100644index 00000000000..0216e12431e--- /dev/null+++ b/api/services/tracing/normalization.py@@ -0,0 +1,346 @@+"""Normalize Agent SDK observations without conflating business and provider IDs."""++import hashlib+import json+from collections.abc import Mapping, Sequence+from datetime import UTC, datetime+from typing import Any++from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace++MAX_VALUE_CHARS = 16000+_SECRET_KEYS = frozenset(+ {+ "api_key",+ "apikey",+ "secret",+ "secret_key",+ "authorization",+ "password",+ "access_token",+ "refresh_token",+ "credentials",+ }+)+_CONTENT_ATTRIBUTES = frozenset(+ {+ "gen_ai.input.messages",+ "gen_ai.output.messages",+ "gen_ai.system_instructions",+ "gen_ai.prompt",+ "gen_ai.completion",+ "tool_arguments",+ "tool_response",+ "gen_ai.tool.call.arguments",+ "gen_ai.tool.call.result",+ "input",+ "output",+ "inputs",+ "outputs",+ }+)+++def span_id(logical_id: str) -> str:+ """Stable nonzero OTel span identity for Dify's persisted logical ID."""+ value = hashlib.sha256(f"dify-application-span:{logical_id}".encode()).hexdigest()[:16]+ return value if int(value, 16) else "0000000000000001"+++def sanitize(value: Any, *, depth: int = 0) -> Any:+ if depth > 12:+ return "[truncated]"+ if isinstance(value, Mapping):+ return {+ str(k): "[redacted]" if str(k).lower() in _SECRET_KEYS else sanitize(v, depth=depth + 1)+ for k, v in value.items()+ }+ if isinstance(value, (list, tuple)):+ return [sanitize(item, depth=depth + 1) for item in value[:256]]+ if isinstance(value, str):+ if value.lstrip().startswith(("{", "[")):+ try:+ decoded = json.loads(value)+ except ValueError:+ pass+ else:+ if isinstance(decoded, (dict, list)):+ return sanitize(decoded, depth=depth + 1)+ return value[:MAX_VALUE_CHARS] + ("[truncated]" if len(value) > MAX_VALUE_CHARS else "")+ if value is None or isinstance(value, (int, float, bool)):+ return value+ return str(value)[:MAX_VALUE_CHARS]+++def safe_attributes(attributes: Mapping[str, Any], *, capture_content: bool) -> dict[str, Any]:+ """Content-off is an allowlist, including metadata and error attributes."""+ if capture_content:+ result = sanitize(attributes)+ return dict(result) if isinstance(result, Mapping) else {}+ safe = {+ "gen_ai.system",+ "gen_ai.provider.name",+ "gen_ai.operation.name",+ "gen_ai.agent.name",+ "gen_ai.tool.name",+ "gen_ai.tool.call.id",+ "logfire.span_type",+ "llm.model_name",+ "openinference.span.kind",+ "dify.llm.total_price",+ "dify.llm.currency",+ "dify.agent.execution_state",+ "gen_ai.response.time_to_first_token",+ "dify.llm.time_to_first_token_unit",+ "dify.llm.time_to_first_token_source",+ "gen_ai.request.model",+ "gen_ai.response.model",+ "gen_ai.response.id",+ "gen_ai.request.max_tokens",+ "gen_ai.request.temperature",+ "gen_ai.request.top_p",+ "dify.trace_id",+ "dify.tenant_id",+ "dify.app_id",+ "dify.agent_id",+ "dify.invoke_from",+ "dify.conversation.id",+ "dify.workflow.id",+ "dify.workflow.run_id",+ "dify.node.id",+ "dify.node.execution_id",+ "dify.tracing.run_id",+ "dify.tracing.invocation_id",+ "gen_ai.user.id",+ }+ return {+ str(key): sanitize(value)+ for key, value in attributes.items()+ if key not in _CONTENT_ATTRIBUTES+ and (+ key in safe+ and isinstance(value, (str, int, float, bool))+ or str(key).startswith(("gen_ai.usage.", "llm.token_count."))+ and isinstance(value, (int, float))+ )+ }+++def _decoded(value: Any) -> Any:+ if isinstance(value, str):+ try:+ return json.loads(value)+ except (ValueError, TypeError):+ return value+ return value+++def _first(attributes: Mapping[str, Any], keys: Sequence[str]) -> Any:+ for key in keys:+ value = attributes.get(key)+ if value is not None:+ return value+ return None+++def normalize_agent_span(data: Mapping[str, Any], *, capture_content: bool) -> CanonicalSpan:+ attributes = safe_attributes(data.get("attributes") or {}, capture_content=capture_content)+ operation = str(attributes.get("gen_ai.operation.name", ""))+ declared = str(attributes.get("openinference.span.kind", "")).lower()+ if operation in {"invoke_agent", "agent"} or declared == "agent":+ kind = CanonicalSpanKind.AGENT+ elif operation in {"execute_tool", "tool"} or "gen_ai.tool.name" in attributes or declared == "tool":+ kind = CanonicalSpanKind.TOOL+ elif (+ operation in {"chat", "text_completion", "generate_content", "embeddings"}+ or "gen_ai.request.model" in attributes+ or declared == "llm"+ ):+ kind = CanonicalSpanKind.LLM+ elif declared == "retriever":+ kind = CanonicalSpanKind.RETRIEVER+ else:+ kind = CanonicalSpanKind.CHAIN+ metadata = {key: value for key, value in attributes.items() if key not in _CONTENT_ATTRIBUTES}+ mappings = {+ "model_name": ("gen_ai.response.model", "gen_ai.request.model", "llm.model_name"),+ "model_provider": ("gen_ai.provider.name", "gen_ai.system"),+ "prompt_tokens": ("gen_ai.usage.input_tokens", "llm.token_count.prompt"),+ "completion_tokens": ("gen_ai.usage.output_tokens", "llm.token_count.completion"),+ "cache_read_tokens": ("gen_ai.usage.cache_read.input_tokens", "gen_ai.usage.input_tokens.cached"),+ "total_price": ("dify.llm.total_price",),+ "currency": ("dify.llm.currency",),+ "time_to_first_token": ("gen_ai.response.time_to_first_token",),+ }+ for key, candidates in mappings.items():+ value = _first(attributes, candidates)+ if value is not None:+ metadata[key] = value+ if kind == CanonicalSpanKind.LLM:+ input_tokens, output_tokens = metadata.get("prompt_tokens"), metadata.get("completion_tokens")+ if isinstance(input_tokens, (int, float)) and isinstance(output_tokens, (int, float)):+ metadata["total_tokens"] = input_tokens + output_tokens+ else:+ # Agent/chain aggregate usage is useful context, never another billed generation.+ for key in ("prompt_tokens", "completion_tokens", "total_tokens", "total_price"):+ if key in metadata:+ metadata[f"aggregate_{key}"] = metadata.pop(key)+ error = str(data.get("status_message") or "Agent operation failed") if data.get("status") == "error" else None+ if error and not capture_content:+ error = "Agent operation failed"+ inputs = (+ _first(+ attributes,+ (+ "gen_ai.input.messages",+ "gen_ai.prompt",+ "gen_ai.tool.call.arguments",+ "tool_arguments",+ "input",+ "inputs",+ ),+ )+ if capture_content+ else None+ )+ outputs = (+ _first(+ attributes,+ (+ "gen_ai.output.messages",+ "gen_ai.completion",+ "gen_ai.tool.call.result",+ "tool_response",+ "output",+ "outputs",+ ),+ )+ if capture_content+ else None+ )+ return CanonicalSpan(+ id=str(data["span_id"]),+ parent_id=data.get("parent_span_id"),+ name=str(data["name"]),+ kind=kind,+ start_time=datetime.fromtimestamp(int(data["start_time_ns"]) / 1e9, UTC),+ end_time=datetime.fromtimestamp(int(data["end_time_ns"]) / 1e9, UTC),+ inputs=sanitize(_decoded(inputs)),+ outputs=sanitize(_decoded(outputs)),+ status=CanonicalSpanStatus.ERROR if error else CanonicalSpanStatus.OK,+ error=error,+ metadata=metadata,+ )+++def remap_outer_trace(+ trace: CanonicalTrace, *, trace_id: str, capture_content: bool, agent_message: bool+) -> CanonicalTrace:+ """Use the same IDs allocated before runtime and suppress synthetic Agent usage."""+ safe_metadata = {+ "workflow_id",+ "workflow_run_id",+ "workflow_app_log_id",+ "node_id",+ "node_execution_id",+ "node_type",+ "status",+ "model_provider",+ "model_name",+ "prompt_tokens",+ "completion_tokens",+ "total_tokens",+ "total_price",+ "currency",+ "trace_entity_type",+ "wrapper_type",+ "wrapper_index",+ "container_execution_id",+ "tenant_id",+ "app_id",+ "agent_id",+ "user_id",+ "conversation_id",+ "external_trace_id",+ "message_execution_state",+ "linked_parent_workflow_run_id",+ "linked_parent_node_execution_id",+ "linked_parent_application_trace_id",+ "linked_parent_trace_id",+ }+ spans = []+ for original in trace.spans:+ if agent_message and original.synthetic and original.kind == CanonicalSpanKind.LLM:+ continue+ metadata = (+ sanitize(original.metadata)+ if capture_content+ else {k: sanitize(v) for k, v in original.metadata.items() if k in safe_metadata}+ )+ if trace.trace_id != trace_id:+ metadata.setdefault("external_trace_id", trace.trace_id)+ if trace.external_parent is not None:+ # This pipeline exports a complete local tree. Cross-workflow+ # correlation is explicit until a durable cross-run merge exists.+ metadata["linked_parent_workflow_run_id"] = trace.external_parent.parent_workflow_run_id+ if trace.external_parent.parent_node_execution_id:+ metadata["linked_parent_node_execution_id"] = trace.external_parent.parent_node_execution_id+ metadata.pop("parent_trace_context", None)+ if original.kind != CanonicalSpanKind.LLM:+ for key in ("prompt_tokens", "completion_tokens", "total_tokens", "total_price"):+ if key in metadata:+ metadata[f"aggregate_{key}"] = metadata.pop(key)+ spans.append(+ original.model_copy(+ update={+ "id": span_id(original.id),+ "parent_id": span_id(original.parent_id) if original.parent_id else None,+ "inputs": sanitize(original.inputs) if capture_content else None,+ "outputs": sanitize(original.outputs) if capture_content else None,+ "metadata": metadata,+ "error": original.error+ if capture_content or not original.error+ else "Application execution failed",+ "can_parent_workflow": False,+ "publishes_parent_context": False,+ }+ )+ )+ return CanonicalTrace(+ trace_id=trace_id, session_id=trace.session_id, root_span_id=span_id(trace.root_span_id), spans=tuple(spans)+ )+++def order_spans(root_id: str, spans: Sequence[CanonicalSpan]) -> tuple[CanonicalSpan, ...]:+ by_id: dict[str, CanonicalSpan] = {}+ for span in spans:+ prior = by_id.get(span.id)+ if prior is not None and prior != span:+ raise ValueError("Conflicting trace span identity")+ by_id[span.id] = span+ if root_id not in by_id:+ raise ValueError("Trace root is missing")+ ordered: list[CanonicalSpan] = []+ complete: set[str] = set()+ visiting: set[str] = set()++ def visit(identifier: str) -> None:+ if identifier in complete:+ return+ if identifier in visiting:+ raise ValueError("Trace parent cycle")+ span = by_id.get(identifier)+ if span is None:+ raise ValueError("Trace parent is missing")+ visiting.add(identifier)+ if span.parent_id is not None:+ visit(span.parent_id)+ elif span.id != root_id:+ raise ValueError("Unexpected trace root")+ visiting.remove(identifier)+ complete.add(identifier)+ ordered.append(span)++ for span in spans:+ visit(span.id)+ return tuple(ordered)diff --git a/api/services/tracing/otlp_export.py b/api/services/tracing/otlp_export.pynew file mode 100644index 00000000000..9507e82f495--- /dev/null+++ b/api/services/tracing/otlp_export.py@@ -0,0 +1,202 @@+"""Encode completed spans with stable IDs and explicit provider ingestion ACKs.++No tracer or global SDK configuration is created: a replay must preserve the+original IDs and timestamps, and destination credentials belong to one export.+"""++import re+from collections.abc import Callable, Mapping+from datetime import datetime+from typing import Any+from urllib.parse import urlsplit+from uuid import UUID++from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanStatus, CanonicalTrace+from services.tracing.provider_common import json_value, nanos, request+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError+++def _id(value: str, length: int) -> bytes:+ if not re.fullmatch(rf"[0-9a-f]{{{length}}}", value) or int(value, 16) == 0:+ raise TraceExportRejectedError("Application trace contains an invalid OpenTelemetry identifier")+ return bytes.fromhex(value)+++def build_request(+ trace: CanonicalTrace,+ resource: Mapping[str, object],+ attributes: Callable[[CanonicalSpan], Mapping[str, object]],+) -> Any:+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+ from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue+ from opentelemetry.proto.trace.v1.trace_pb2 import Status++ def attribute(key: str, value: object) -> Any:+ if isinstance(value, bool):+ encoded = AnyValue(bool_value=value)+ elif isinstance(value, int):+ encoded = AnyValue(int_value=value)+ elif isinstance(value, float):+ encoded = AnyValue(double_value=value)+ elif isinstance(value, str):+ encoded = AnyValue(string_value=value)+ else:+ encoded = AnyValue(string_value=json_value(value))+ return KeyValue(key=key, value=encoded)++ trace_id = _id(trace.trace_id, 32)+ payload = ExportTraceServiceRequest()+ resource_spans = payload.resource_spans.add()+ resource_spans.resource.attributes.extend(attribute(k, v) for k, v in resource.items() if v is not None)+ scope = resource_spans.scope_spans.add()+ scope.scope.name = "dify.application.tracing"+ scope.scope.version = "1"+ for span in trace.spans:+ if span.end_time is None or nanos(span.end_time) < nanos(span.start_time):+ raise TraceExportRejectedError("Application trace contains incomplete or invalid timestamps")+ target = scope.spans.add(+ trace_id=trace_id,+ span_id=_id(span.id, 16),+ parent_span_id=_id(span.parent_id, 16) if span.parent_id else b"",+ name=span.name,+ kind=1,+ start_time_unix_nano=nanos(span.start_time),+ end_time_unix_nano=nanos(span.end_time),+ flags=1,+ )+ target.status.code = (+ Status.STATUS_CODE_ERROR if span.status is CanonicalSpanStatus.ERROR else Status.STATUS_CODE_OK+ )+ if span.error:+ target.status.message = span.error+ target.attributes.extend(attribute(k, v) for k, v in attributes(span).items() if v is not None)+ return payload+++def _check_response(response: Any, provider: str) -> None:+ if response.partial_success.rejected_spans:+ # Retrying the entire batch would duplicate the accepted subset.+ raise TraceExportUncertainError(f"{provider} partially accepted the trace; reconciliation is required")+++def is_aliyun_http_success_ack(status_code: int, content: bytes) -> bool:+ """Recognize the exact ACK observed from Aliyun's HTTP Trace collector.++ This literal response is an observed compatibility behavior, not the+ documented OTLP ExportTraceServiceResponse. Do not normalize other bodies+ or accept it for another status or provider.+ """+ return status_code == 200 and content == b'"success"'+++def _langfuse_enqueue_ack(body: object) -> bool:+ """Recognize Langfuse's queue acknowledgement without exposing its auth data.++ The v4 HTTP route returns the job after blob upload and awaited queue.add,+ rather than an OTLP ExportTraceServiceResponse. This confirms enqueue only.+ Upstream: langfuse/langfuse@bc1ade0aacca75104bad934e92d70a454ec16e67,+ web/src/server/otel/processOtelIngestion.ts:277 and+ packages/shared/src/server/otel/OtelIngestionProcessor.ts:350.+ """+ if not isinstance(body, Mapping) or body.get("name") != "otel-ingestion-job":+ return False+ if any(key in body for key in ("partialSuccess", "partial_success", "error", "errors")) or body.get("failedReason"):+ return False+ data = body.get("data")+ if not isinstance(data, Mapping) or data.get("name") != "otel-ingestion-job":+ return False+ identifier, timestamp = data.get("id"), data.get("timestamp")+ if not isinstance(identifier, str) or not isinstance(timestamp, str):+ return False+ try:+ if UUID(identifier).int == 0 or datetime.fromisoformat(timestamp).tzinfo is None:+ return False+ except ValueError:+ return False+ payload = data.get("payload")+ if not isinstance(payload, Mapping):+ return False+ stored_data, auth = payload.get("data"), payload.get("authCheck")+ if not isinstance(stored_data, Mapping) or not isinstance(auth, Mapping) or auth.get("validKey") is not True:+ return False+ scope = auth.get("scope")+ if not isinstance(scope, Mapping) or scope.get("accessLevel") != "project":+ return False+ return all(+ isinstance(value, str) and bool(value.strip())+ for value in (stored_data.get("fileKey"), stored_data.get("publicKey"), scope.get("projectId"))+ )+++def export_http(provider: str, url: str, headers: Mapping[str, str], payload: Any) -> None:+ from google.protobuf.json_format import ParseDict, ParseError+ from google.protobuf.message import DecodeError+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceResponse++ response = request(+ "POST",+ url,+ provider,+ headers={"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf", **headers},+ content=payload.SerializeToString(),+ )+ if provider == "aliyun" and is_aliyun_http_success_ack(response.status_code, response.content):+ return+ result = ExportTraceServiceResponse()+ if response.content:+ try:+ if "json" in response.headers.get("content-type", "").lower():+ body = response.json()+ if not isinstance(body, dict):+ raise ValueError("Invalid ingestion acknowledgment object")+ if (+ provider == "langfuse"+ and response.status_code == 200+ and any(+ key.lower() == "x-langfuse-ingestion-version" and value == "4" for key, value in headers.items()+ )+ and _langfuse_enqueue_ack(body)+ ):+ return+ ParseDict(body, result)+ else:+ result.ParseFromString(response.content)+ except (DecodeError, ParseError, ValueError):+ raise TraceExportUncertainError(f"{provider} returned an unreadable ingestion acknowledgment") from None+ _check_response(result, provider)+++def export_grpc(provider: str, url: str, headers: Mapping[str, str], payload: Any) -> None:+ """Use the provider's gRPC protocol; HTTP exports use the SSRF HTTP owner."""+ import grpc # type: ignore[import-untyped]+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import TraceServiceStub++ parsed = urlsplit(url)+ if not parsed.hostname or parsed.username or parsed.password or parsed.scheme not in {"http", "https"}:+ raise TraceExportRejectedError("Invalid OTLP gRPC endpoint")+ port = parsed.port or (4317 if provider == "tencent" else 443)+ host = f"[{parsed.hostname}]" if ":" in parsed.hostname else parsed.hostname+ target = f"{host}:{port}"+ channel = (+ grpc.insecure_channel(target)+ if parsed.scheme == "http"+ else grpc.secure_channel(target, grpc.ssl_channel_credentials())+ )+ try:+ result = TraceServiceStub(channel).Export(payload, metadata=tuple(headers.items()), timeout=30)+ except grpc.RpcError as error:+ code = error.code()+ if code in {+ grpc.StatusCode.DEADLINE_EXCEEDED,+ grpc.StatusCode.CANCELLED,+ grpc.StatusCode.UNKNOWN,+ grpc.StatusCode.UNAVAILABLE,+ }:+ raise TraceExportUncertainError(f"{provider} ingestion acknowledgment is unknown") from None+ if code in {grpc.StatusCode.RESOURCE_EXHAUSTED, grpc.StatusCode.ABORTED}:+ raise RetryableTraceDispatchError(f"{provider} ingestion temporarily unavailable") from None+ raise TraceExportRejectedError(f"{provider} ingestion rejected") from None+ finally:+ channel.close()+ _check_response(result, provider)diff --git a/api/services/tracing/provider_common.py b/api/services/tracing/provider_common.pynew file mode 100644index 00000000000..88fdef0636a--- /dev/null+++ b/api/services/tracing/provider_common.py@@ -0,0 +1,127 @@+"""Shared serialization and HTTP boundaries for application trace adapters."""++import json+import math+from collections.abc import Mapping+from datetime import UTC, datetime+from typing import Any+from urllib.parse import urlsplit, urlunsplit++from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError+from services.tracing.request_log_protection import protect_http_request_logs+++def json_value(value: object) -> str:+ return json.dumps(value, ensure_ascii=False, default=str, allow_nan=False)+++def utc(value: datetime) -> datetime:+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)+++def nanos(value: datetime) -> int:+ return int(utc(value).timestamp() * 1_000_000_000)+++def finished_at(span: CanonicalSpan) -> datetime:+ if span.end_time is None:+ raise TraceExportRejectedError("Application trace requires completed spans")+ return utc(span.end_time)+++def endpoint(value: object) -> str:+ """Validate a destination without allowing credentials in HTTP authority."""+ if not isinstance(value, str):+ raise TraceExportRejectedError("Tracing endpoint is required")+ parsed = urlsplit(value)+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:+ raise TraceExportRejectedError("Invalid tracing endpoint")+ if parsed.query or parsed.fragment:+ raise TraceExportRejectedError("Tracing endpoint must not contain query parameters or a fragment")+ return urlunsplit(parsed).rstrip("/")+++def required(config: Mapping[str, Any], name: str) -> str:+ value = config.get(name)+ if not isinstance(value, str) or not value.strip():+ raise TraceExportRejectedError(f"Tracing configuration requires {name}")+ return value+++def token_usage(span: CanonicalSpan) -> dict[str, int]:+ """Only model-call spans carry billable usage; wrappers retain metadata."""+ if span.kind is not CanonicalSpanKind.LLM:+ return {}+ result = {}+ for source, target in (("prompt_tokens", "input"), ("completion_tokens", "output"), ("total_tokens", "total")):+ value = span.metadata.get(source)+ if isinstance(value, int) and not isinstance(value, bool) and value >= 0:+ result[target] = value+ return result+++def usd_cost(span: CanonicalSpan) -> float | None:+ if span.kind is not CanonicalSpanKind.LLM or str(span.metadata.get("currency", "USD")).upper() != "USD":+ return None+ try:+ value = float(span.metadata["total_price"])+ except (KeyError, TypeError, ValueError):+ return None+ return value if math.isfinite(value) and value >= 0 else None+++def check_http_status(status: int, provider: str) -> None:+ if 200 <= status < 300:+ return+ if status in {408, 425, 429} or status >= 500:+ raise RetryableTraceDispatchError(f"{provider} ingestion temporarily unavailable (HTTP {status})")+ raise TraceExportRejectedError(f"{provider} ingestion rejected (HTTP {status})")+++def request(method: str, url: str, provider: str, **kwargs: Any):+ """Use the configured SSRF boundary, with retries owned by the durable task."""+ import httpx+ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, attach, detach, set_value++ from core.helper import ssrf_proxy++ token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))+ try:+ with protect_http_request_logs(provider == "aliyun"):+ if provider == "aliyun":+ response = _request_sensitive_aliyun(method, url, **kwargs)+ else:+ response = ssrf_proxy.make_request(+ method, url, max_retries=0, timeout=30.0, follow_redirects=False, **kwargs+ )+ except httpx.TransportError:+ raise TraceExportUncertainError(f"{provider} ingestion outcome is unknown after transport failure") from None+ finally:+ detach(token)+ check_http_status(response.status_code, provider)+ return response+++def _request_sensitive_aliyun(method: str, url: str, **kwargs: Any):+ """Discard private HTTP exception objects before raising a public failure."""+ import httpx++ from core.helper import ssrf_proxy++ response: httpx.Response | None = None+ failure: RuntimeError | None = None+ try:+ response = ssrf_proxy.make_request(method, url, max_retries=0, timeout=30.0, follow_redirects=False, **kwargs)+ except httpx.RequestError:+ failure = TraceExportUncertainError("aliyun ingestion outcome is unknown after transport failure")+ except Exception:+ failure = TraceExportRejectedError("aliyun HTTP request was rejected by the transport boundary")+ # Raise after leaving the original exception handler so neither its cause+ # nor implicit context retains a credential-bearing request or message.+ if failure is not None:+ raise failure from None+ if response is None:+ raise TraceExportUncertainError("aliyun HTTP transport returned no acknowledgment")+ return responsediff --git a/api/services/tracing/provider_dispatch.py b/api/services/tracing/provider_dispatch.pynew file mode 100644index 00000000000..06317c582e8--- /dev/null+++ b/api/services/tracing/provider_dispatch.py@@ -0,0 +1,122 @@+"""Dispatch a completed application trace without exposing destination credentials.++An accepted export acknowledges ingestion, not query availability. Completed spans+must not be exported again after acceptance; some providers do not deduplicate IDs.+The durable task owner decides whether an uncertain transport outcome is retried.+"""++from collections.abc import Mapping+from dataclasses import dataclass, field+from importlib import import_module+from typing import Any, Literal+from urllib.parse import urlsplit++from core.ops.exceptions import RetryableTraceDispatchError, TraceProviderNotInstalledError+from core.ops.unified_trace.entities import CanonicalTrace+++@dataclass(frozen=True)+class TraceExportResult:+ trace_id: str+ trace_url: str | None = None+ span_urls: dict[str, str] = field(default_factory=dict)+ status: Literal["accepted"] = "accepted"+++class TraceExportRejectedError(RuntimeError):+ """A terminal export rejection, containing no credential or response body."""+++class TraceExportUncertainError(RetryableTraceDispatchError):+ """The transport failed without proving whether ingestion accepted the data."""++ acceptance_unknown = True+++def frozen_project_url(provider: str, config: Mapping[str, Any]) -> str | None:+ """Read only persisted destination metadata, without credentials or vendor I/O."""+ if provider == "arize":+ try:+ from dify_trace_arize_phoenix.arize_configuration import project_url+ except ModuleNotFoundError:+ return None++ # Never substitute the current app destination for a historical run.+ # Unverified legacy redirects remain stored, but are not presented as+ # project links. A future config save does not backfill that snapshot.+ return project_url(config)+ # These adapters expose a platform console, not the run's destination project.+ if provider in {"aliyun", "tencent"}:+ return None+ value = config.get("project_url")+ if (+ not isinstance(value, str)+ or not value+ or any(ord(char) <= 32 or ord(char) == 127 for char in value)+ or "\\" in value+ ):+ return None+ try:+ parsed = urlsplit(value)+ if (+ parsed.scheme not in {"http", "https"}+ or not parsed.hostname+ or parsed.username is not None+ or parsed.password is not None+ ):+ return None+ _ = parsed.port+ except ValueError:+ return None+ return value+++_MODULES = {+ "langfuse": "dify_trace_langfuse.agent_trace",+ "langsmith": "dify_trace_langsmith.agent_trace",+ "phoenix": "dify_trace_arize_phoenix.agent_trace",+ "arize": "dify_trace_arize_phoenix.agent_trace",+ "opik": "dify_trace_opik.agent_trace",+ "weave": "dify_trace_weave.agent_trace",+ "mlflow": "dify_trace_mlflow.agent_trace",+ "databricks": "dify_trace_mlflow.agent_trace",+ "aliyun": "dify_trace_aliyun.agent_trace",+ "tencent": "dify_trace_tencent.agent_trace",+}+++def provider_capabilities(provider: str) -> dict[str, bool | str]:+ """Describe implemented mappings, independently of installation/account tests."""+ if provider not in _MODULES:+ raise ValueError("Unsupported tracing provider")+ return {+ "provider": provider,+ "agent_spans": True,+ "session": True,+ "native_usage": True,+ "native_cost": provider in {"langfuse", "langsmith", "opik", "mlflow", "databricks"},+ "trace_url": provider in {"langfuse", "langsmith", "weave", "mlflow", "databricks"},+ "span_url": provider in {"langfuse", "langsmith", "weave"},+ "export_ack": "accepted",+ }+++def export_trace(provider: str, config: Mapping[str, Any], trace: CanonicalTrace) -> TraceExportResult:+ """Export one complete tree to the caller's already-authorized destination."""+ module_name = _MODULES.get(provider)+ if module_name is None:+ raise TraceExportRejectedError("Unsupported tracing provider")+ if trace.external_parent is not None or trace.required_parent_context_id is not None:+ raise TraceExportRejectedError("Application trace export requires a complete resolved tree")+ if any(span.end_time is None for span in trace.spans):+ raise TraceExportRejectedError("Application trace export requires completed spans")+ try:+ module = import_module(module_name)+ return module.export_trace(provider, config, trace)+ except (TraceExportRejectedError, RetryableTraceDispatchError):+ raise+ except ImportError as error:+ raise TraceProviderNotInstalledError(provider, error.name or module_name) from None+ except Exception:+ # Vendor exceptions can contain full requests, headers, and endpoint secrets.+ raise TraceExportRejectedError(f"{provider} trace export failed") from Nonediff --git a/api/services/tracing/recovery.py b/api/services/tracing/recovery.pynew file mode 100644index 00000000000..4e9c8729edb--- /dev/null+++ b/api/services/tracing/recovery.py@@ -0,0 +1,200 @@+"""Recover dropped in-memory terminal events from committed business records.++Runtime finalization can precede the API's final answer transaction. In particular,+AgentAppRunner persists usage independently, so usage alone is not proof that a+Message is complete. Recovery never invents answers, usage, or success states.+"""++import json+import logging+import math+from datetime import datetime, timedelta+from typing import Protocol, cast++from sqlalchemy import select+from sqlalchemy.exc import SQLAlchemyError++from configs import dify_config+from core.ops.entities.trace_entity import BaseTraceInfo, MessageTraceInfo, TraceTaskName, WorkflowTraceInfo+from core.ops.ops_trace_manager import TraceTask+from graphon.enums import WorkflowExecutionStatus+from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceInvocation, ApplicationTraceRun+from models.enums import MessageStatus+from models.model import App, AppMode, Conversation, Message, MessageDict+from models.workflow import WorkflowRun+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError++logger = logging.getLogger(__name__)+_SETTLE_SECONDS = 5+_WORKFLOW_TERMINAL = {+ WorkflowExecutionStatus.SUCCEEDED,+ WorkflowExecutionStatus.PARTIAL_SUCCEEDED,+ WorkflowExecutionStatus.FAILED,+ WorkflowExecutionStatus.STOPPED,+}+++class RecoveryTarget(Protocol):+ repository: ApplicationTraceRepository++ def capture_terminal(self, info: BaseTraceInfo) -> bool: ...+++def agent_stop_requested(message: Message) -> bool:+ """Read the consumer's stop intent without inferring cancellation success."""+ try:+ metadata = json.loads(message.message_metadata) if message.message_metadata else {}+ except (ValueError, TypeError):+ return False+ return isinstance(metadata, dict) and metadata.get("agent_stop_requested") is True+++def _complete_message_metadata(message: Message) -> bool:+ """Recognize the EasyUI _save_message transaction, not an early usage flush."""+ try:+ metadata = json.loads(message.message_metadata) if message.message_metadata else {}+ except (ValueError, TypeError):+ return False+ if (+ not isinstance(metadata, dict)+ or not {"annotation_reply", "retriever_resources", "reasoning"} <= metadata.keys()+ ):+ return False+ usage = metadata.get("usage")+ if not isinstance(usage, dict):+ return False+ for name in ("prompt_tokens", "completion_tokens"):+ value = usage.get(name)+ if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value) or value < 0:+ return False+ latency = message.provider_response_latency+ return isinstance(latency, (int, float)) and math.isfinite(latency) and latency >= 0+++def _message_task(repository: ApplicationTraceRepository, run: ApplicationTraceRun, now: datetime) -> TraceTask | None:+ with repository.sessions() as session:+ source = session.execute(+ select(Message, App.mode)+ .join(App, App.id == Message.app_id)+ .join(Conversation, Conversation.id == Message.conversation_id)+ .where(+ Message.id == run.source_id,+ Message.app_id == run.app_id,+ App.tenant_id == run.tenant_id,+ Conversation.app_id == run.app_id,+ )+ ).one_or_none()+ if source is None:+ raise TraceCollectionError("tracing_source_not_found", 404)+ message, app_mode = source+ pending_agent_stop = app_mode == AppMode.AGENT and agent_stop_requested(message)+ if message.status is MessageStatus.PAUSED and not pending_agent_stop:+ return None+ invocations = list(+ session.scalars(select(ApplicationTraceInvocation).where(ApplicationTraceInvocation.run_id == run.id))+ )+ explicit_error = message.status is MessageStatus.ERROR and bool(message.error)+ if any(not invocation.finalized or invocation.finalized_at is None for invocation in invocations):+ if (explicit_error or pending_agent_stop) and message.updated_at + timedelta(+ seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT+ ) < now:+ raise TraceCollectionError("agent_trace_collection_incomplete")+ return None+ finalized = [invocation.finalized_at for invocation in invocations if invocation.finalized_at is not None]+ anchor = max([message.updated_at, *finalized])+ if pending_agent_stop:+ # Even a fully saved partial answer/usage cannot establish which+ # runtime terminal won the stop race. Wait for the existing worker+ # handoff, or converge to incomplete delivery after its deadline.+ if anchor + timedelta(seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT) < now:+ raise TraceCollectionError("business_terminal_not_persisted")+ return None+ if run.capture_agent and not invocations:+ return None+ if anchor + timedelta(seconds=_SETTLE_SECONDS) > now:+ return None+ complete = explicit_error or (+ message.status is MessageStatus.NORMAL+ and message.updated_at > message.created_at+ and _complete_message_metadata(message)+ )+ if not complete:+ if finalized and max(finalized) + timedelta(seconds=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT) < now:+ raise TraceCollectionError("business_terminal_not_persisted")+ return None+ return TraceTask(+ TraceTaskName.MESSAGE_TRACE,+ message_id=message.id,+ conversation_id=message.conversation_id,+ trace_session_id=run.session_id,+ )+++def _workflow_task(repository: ApplicationTraceRepository, run: ApplicationTraceRun, now: datetime) -> TraceTask | None:+ workflow = repository._source(tenant_id=run.tenant_id, app_id=run.app_id, kind="workflow", source_id=run.source_id)+ # An asynchronous workflow repository may not have saved the reserved run yet.+ if (+ not isinstance(workflow, WorkflowRun)+ or workflow.status not in _WORKFLOW_TERMINAL+ or workflow.finished_at is None+ ):+ return None+ if workflow.finished_at + timedelta(seconds=_SETTLE_SECONDS) > now:+ return None+ with repository.sessions() as session:+ conversation_id = session.scalar(+ select(Message.conversation_id)+ .join(Conversation, Conversation.id == Message.conversation_id)+ .where(+ Message.app_id == run.app_id, Message.workflow_run_id == workflow.id, Conversation.app_id == run.app_id+ )+ .limit(1)+ )+ return TraceTask(+ TraceTaskName.WORKFLOW_TRACE,+ workflow_run_id=workflow.id,+ conversation_id=conversation_id,+ trace_session_id=run.session_id,+ user_id=workflow.created_by,+ )+++def recover_terminal(target: RecoveryTarget, run: ApplicationTraceRun) -> bool:+ """Attempt one bounded recovery and rotate unresolved work behind other runs."""+ if (+ run.outer_trace is not None+ or run.state not in {"collecting", "ready"}+ or run.source_kind not in {"message", "workflow"}+ ):+ return False+ error_code: str | None = None+ try:+ task = (_message_task if run.source_kind == "message" else _workflow_task)(+ target.repository, run, naive_utc_now()+ )+ if task is None:+ return False+ info = task.execute()+ if isinstance(info, MessageTraceInfo) and isinstance(info.message_data, dict):+ info = info.model_copy(update={"message_data": Message.from_dict(cast(MessageDict, info.message_data))})+ if isinstance(info, WorkflowTraceInfo) and isinstance(info.workflow_data, dict):+ info = info.model_copy(update={"workflow_data": WorkflowRun.from_dict(info.workflow_data)})+ if not isinstance(info, (MessageTraceInfo, WorkflowTraceInfo)):+ raise TraceCollectionError("business_terminal_recovery_incomplete")+ if not target.capture_terminal(info):+ raise TraceCollectionError("business_terminal_recovery_unclaimed")+ return True+ except TraceCollectionError as error:+ error_code = error.code+ return False+ except SQLAlchemyError:+ # Leave the durable source intact. The next reconciler pass can retry.+ logger.warning("Application trace recovery database unavailable: run_id=%s", run.id)+ return False+ except Exception as error:+ error_code = "business_terminal_recovery_failed"+ logger.warning("Application trace recovery failed: run_id=%s error_type=%s", run.id, type(error).__name__)+ return False+ finally:+ target.repository.recovery_checked(run.id, error_code=error_code)diff --git a/api/services/tracing/request_log_protection.py b/api/services/tracing/request_log_protection.pynew file mode 100644index 00000000000..397290ab4f3--- /dev/null+++ b/api/services/tracing/request_log_protection.py@@ -0,0 +1,98 @@+"""Hide dynamic transport diagnostics only inside sensitive trace HTTP calls.++The Aliyun collector carries credentials in its path. Do not try to recognize+individual encodings of those credentials: replace dynamic content at the+standard Logger.makeRecord entry before any record factory or handler sees it.+The current record factory remains independently configurable. Other concurrent+requests retain their normal logging, levels and handlers. This does not defend+against arbitrary process code replacing the protected logging entry itself.+"""++import logging+from collections.abc import Callable, Generator+from contextlib import contextmanager+from contextvars import ContextVar+from threading import RLock++type RecordMaker = Callable[..., logging.LogRecord]++SAFE_HTTP_LOG_MESSAGE = "Sensitive tracing HTTP request; dynamic transport details redacted"+_sensitive_request: ContextVar[bool] = ContextVar("sensitive_trace_http_request", default=False)+++def _protected_record_maker(previous: RecordMaker) -> RecordMaker:+ def create_record(+ self: logging.Logger,+ name: str,+ level: int,+ fn: str,+ lno: int,+ msg: object,+ args: object,+ exc_info: object,+ func: str | None = None,+ extra: object = None,+ sinfo: str | None = None,+ ) -> logging.LogRecord:+ if not _sensitive_request.get():+ return previous(self, name, level, fn, lno, msg, args, exc_info, func, extra, sinfo)+ # Sanitize before delegating, including extras, chained exceptions and+ # stack information. Never stringify the original diagnostic values.+ # The original makeRecord still invokes the dynamic current factory.+ record = previous(self, name, level, fn, lno, SAFE_HTTP_LOG_MESSAGE, (), None, func, None, None)+ record.msg = SAFE_HTTP_LOG_MESSAGE+ record.args = ()+ record.exc_info = None+ record.exc_text = None+ record.stack_info = None+ return record++ return create_record+++class _RecordMakerLeases:+ def __init__(self) -> None:+ self.lock = RLock()+ self.users = 0+ self.previous: RecordMaker | None = None+ self.installed: RecordMaker | None = None++ def acquire(self) -> None:+ with self.lock:+ if self.users == 0:+ self.previous = logging.Logger.makeRecord+ self.installed = _protected_record_maker(self.previous)+ logging.Logger.makeRecord = self.installed+ self.users += 1++ def release(self) -> None:+ with self.lock:+ self.users -= 1+ if self.users == 0:+ # Preserve an independently installed method instead of+ # overwriting another logging integration during cleanup.+ if logging.Logger.makeRecord is self.installed and self.previous is not None:+ logging.Logger.makeRecord = self.previous+ self.previous = None+ self.installed = None+++_record_maker_leases = _RecordMakerLeases()+++@contextmanager+def protect_http_request_logs(sensitive: bool) -> Generator[None]:+ """Scope protection to this call; nested public calls restore their parent."""+ token = _sensitive_request.set(sensitive)+ leased = False+ try:+ if sensitive:+ _record_maker_leases.acquire()+ leased = True+ yield+ finally:+ try:+ if leased:+ _record_maker_leases.release()+ finally:+ _sensitive_request.reset(token)diff --git a/api/services/tracing/runtime_bridge.py b/api/services/tracing/runtime_bridge.pynew file mode 100644index 00000000000..f94aae3b192--- /dev/null+++ b/api/services/tracing/runtime_bridge.py@@ -0,0 +1,205 @@+"""Fail-open business hooks and durable terminal handoff for application tracing.++The deployment gate is checked before constructing any database-backed service.+Execution context carries opaque correlation and policy only, never provider keys.+"""++from __future__ import annotations++import json+import logging+from collections.abc import Mapping+from typing import TYPE_CHECKING, Literal++from dify_agent.layers.execution_context import ApplicationTracingContext++from configs import dify_config++if TYPE_CHECKING:+ from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext+ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity+ from core.ops.entities.trace_entity import BaseTraceInfo+ from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext+ from services.tracing.service import ApplicationTracingService++logger = logging.getLogger(__name__)+++def workflow_has_agent_v2(graph: object) -> bool:+ """Scope the new pipeline to graphs containing the Dify Agent v2 node."""+ from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data++ if isinstance(graph, str):+ try:+ graph = json.loads(graph)+ except ValueError:+ return False+ if not isinstance(graph, Mapping) or not isinstance(graph.get("nodes"), list):+ return False+ return any(+ isinstance(node, Mapping) and isinstance(node.get("data"), Mapping) and is_dify_agent_node_data(node["data"])+ for node in graph["nodes"]+ )+++def _service() -> ApplicationTracingService:+ from services.tracing.service import application_tracing_service++ return application_tracing_service()+++def _log_failure(stage: str, error: Exception) -> None:+ from repositories.application_trace_repository import TraceCollectionError++ code = error.code if isinstance(error, TraceCollectionError) else type(error).__name__+ logger.warning("Application tracing hook skipped: stage=%s error_code=%s", stage, code)+++def prepare_agent_app(context: AgentAppRuntimeBuildContext) -> ApplicationTracingContext | None:+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return None+ try:+ return _service().prepare_invocation(+ tenant_id=context.dify_context.tenant_id,+ app_id=context.dify_context.app_id,+ agent_id=context.agent_id,+ source_kind="message",+ source_id=context.idempotency_key,+ session_id=context.dify_context.trace_session_id or context.conversation_id,+ node_execution_id=None,+ attempt=0,+ call_key=context.idempotency_key,+ )+ except Exception as error:+ _log_failure("agent_app", error)+ return None+++def capture_agent_app_terminal(+ *,+ tenant_id: str,+ app_id: str,+ message_id: str,+ outcome: Literal["succeeded", "failed", "cancelled"],+ output: str | None,+) -> None:+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return+ try:+ _service().capture_agent_app_terminal(+ tenant_id=tenant_id, app_id=app_id, message_id=message_id, outcome=outcome, output=output+ )+ except Exception as error:+ _log_failure("agent_app_terminal", error)+++def prepare_workflow_agent(+ context: WorkflowAgentRuntimeBuildContext, conversation_id: str | None+) -> ApplicationTracingContext | None:+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return None+ try:+ # Hidden Tool nodes own a different app but share the caller's run and are+ # absent from its outer trace. Neither a source-app snapshot nor an orphan+ # parent is safe. Keep this new nested scope uncollected until it has an+ # explicit owner/parent contract; ordinary nodes and previews are unchanged.+ if context.dify_context.workflow_tool_invocation_id is not None:+ return None+ source_id = context.workflow_run_id or context.node_execution_id+ return _service().prepare_invocation(+ tenant_id=context.dify_context.tenant_id,+ app_id=context.dify_context.app_id,+ agent_id=context.agent.id,+ source_kind="workflow" if context.workflow_run_id else "node",+ source_id=source_id,+ session_id=context.dify_context.trace_session_id or conversation_id or source_id,+ node_execution_id=context.node_execution_id,+ attempt=context.attempt,+ call_key=f"{context.node_execution_id}:{context.attempt}",+ workflow_id=context.workflow_id,+ )+ except Exception as error:+ _log_failure("workflow_agent", error)+ return None+++def freeze_workflow_run(+ entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,+ *,+ workflow_id: str,+ workflow_run_id: str,+ conversation_id: object,+ graph_data: object,+) -> None:+ """Freeze policy immediately after the business workflow row is saved."""+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return+ if not workflow_has_agent_v2(graph_data):+ return+ try:+ explicit_session = entity.extras.get("trace_session_id")+ session_id = next(+ (+ value+ for value in (explicit_session, conversation_id, workflow_run_id)+ if isinstance(value, str) and value+ ),+ workflow_run_id,+ )+ _service().start(+ tenant_id=entity.app_config.tenant_id,+ app_id=entity.app_config.app_id,+ source_kind="workflow",+ source_id=workflow_run_id,+ session_id=session_id,+ workflow_id=workflow_id,+ )+ except Exception as error:+ _log_failure("workflow_start", error)+++def queue_enabled_at_start(app_id: object) -> bool:+ """Read the existing application switch without constructing a vendor SDK."""+ if not dify_config.OPS_AGENT_TRACE_ENABLED or app_id is None:+ return False+ from extensions.ext_database import db+ from models.model import App++ try:+ app = db.session.get(App, str(app_id))+ policy = json.loads(app.tracing) if app is not None and app.tracing else {}+ return isinstance(policy, dict) and policy.get("enabled") is True+ except Exception as error:+ _log_failure("queue_policy", error)+ return False+++def capture_terminal(info: BaseTraceInfo) -> bool:+ """Transfer ownership before legacy dispatch; transient DB failures retry.++ Unlike request construction, a worker must not fall back to a second export+ path when the durable collector is temporarily inaccessible.+ """+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return False+ from sqlalchemy.exc import SQLAlchemyError++ from core.ops.exceptions import RetryableTraceDispatchError++ try:+ return _service().capture_terminal(info)+ except SQLAlchemyError:+ raise RetryableTraceDispatchError("Application tracing persistence temporarily unavailable") from None+++class RuntimeApplicationTracingHooks:+ """Stateless app-scoped adapter; database services are created only on invocation."""++ __slots__ = ()++ prepare_agent_app = staticmethod(prepare_agent_app)+ prepare_workflow_agent = staticmethod(prepare_workflow_agent)+ freeze_workflow_run = staticmethod(freeze_workflow_run)+ queue_enabled_at_start = staticmethod(queue_enabled_at_start)+ capture_terminal = staticmethod(capture_terminal)+ capture_agent_app_terminal = staticmethod(capture_agent_app_terminal)diff --git a/api/services/tracing/sdk_transport.py b/api/services/tracing/sdk_transport.pynew file mode 100644index 00000000000..d78ee4bae9c--- /dev/null+++ b/api/services/tracing/sdk_transport.py@@ -0,0 +1,42 @@+"""Per-client SDK transports using Dify's HTTP egress policy, never global state."""++from typing import Any, override++import httpx+import requests++from services.tracing.provider_common import request as send_http+from services.tracing.provider_dispatch import TraceExportRejectedError+++class SSRFTransport(httpx.BaseTransport):+ def __init__(self, provider: str) -> None:+ self.provider = provider++ @override+ def handle_request(self, request: httpx.Request) -> httpx.Response:+ response = send_http(+ request.method, str(request.url), self.provider, headers=dict(request.headers), content=request.read()+ )+ return httpx.Response(response.status_code, headers=response.headers, content=response.content, request=request)+++class SSRFSession(requests.Session):+ def __init__(self, provider: str) -> None:+ super().__init__()+ self.provider = provider++ @override+ def send(self, request: requests.PreparedRequest, **kwargs: Any) -> requests.Response:+ if not request.method or not request.url or not isinstance(request.body, (str, bytes, type(None))):+ raise TraceExportRejectedError("Tracing SDK prepared an unsupported HTTP request")+ received = send_http(+ request.method, request.url, self.provider, headers=dict(request.headers), content=request.body+ )+ response = requests.Response()+ response.status_code = received.status_code+ response.headers.update(received.headers)+ response._content = received.content+ response.url = str(received.url)+ response.request = request+ return responsediff --git a/api/services/tracing/service.py b/api/services/tracing/service.pynew file mode 100644index 00000000000..acce7315b1d--- /dev/null+++ b/api/services/tracing/service.py@@ -0,0 +1,705 @@+"""Application tracing orchestration; secrets and vendor I/O stay in the API worker."""++import json+import logging+import threading+from collections.abc import Generator+from contextlib import contextmanager+from datetime import UTC, datetime, timedelta+from typing import Any, Literal, cast++from dify_agent.layers.execution_context import ApplicationTracingContext+from dify_agent.protocol.tracing import AgentTraceBatch+from sqlalchemy import select+from sqlalchemy.orm import sessionmaker++from configs import dify_config+from core.ops.entities.trace_entity import BaseTraceInfo, MessageTraceInfo, TraceTaskName, WorkflowTraceInfo+from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.ops_trace_manager import OpsTraceManager, TraceTask+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from core.ops.unified_trace.trace_builder import CanonicalTraceBuilder, RepositoryWorkflowExecutionLoader+from extensions.ext_database import db+from models import Account, App, AppMode, Conversation, Message, Workflow, WorkflowNodeExecutionModel, WorkflowRun+from models.application_trace import ApplicationTraceRun+from models.model import MessageDict+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError+from services.tracing.normalization import normalize_agent_span, order_spans, remap_outer_trace, sanitize+from services.tracing.provider_dispatch import (+ TraceExportUncertainError,+ export_trace,+ frozen_project_url,+ provider_capabilities,+)+from services.tracing.runtime_bridge import workflow_has_agent_v2++logger = logging.getLogger(__name__)+++def _message_inputs_are_empty(inputs: object) -> bool:+ if isinstance(inputs, str):+ if not inputs.strip():+ return True+ try:+ inputs = json.loads(inputs)+ except ValueError:+ return False+ return inputs is None or isinstance(inputs, (str, dict, list)) and not inputs+++class ApplicationTracingService:+ def __init__(self, repository: ApplicationTraceRepository) -> None:+ self.repository = repository++ def start(+ self,+ *,+ tenant_id: str,+ app_id: str,+ source_kind: Literal["message", "workflow", "node"],+ source_id: str,+ session_id: str,+ workflow_id: str | None = None,+ ) -> ApplicationTraceRun | None:+ if not dify_config.OPS_AGENT_TRACE_ENABLED:+ return None+ run = self.repository.start(+ tenant_id=tenant_id,+ app_id=app_id,+ source_kind=source_kind,+ source_id=source_id,+ session_id=session_id,+ workflow_id=workflow_id,+ )+ return run if run is not None and run.state != "not_collected" else None++ def prepare_invocation(+ self,+ *,+ tenant_id: str,+ app_id: str,+ agent_id: str,+ source_kind: Literal["message", "workflow", "node"],+ source_id: str,+ session_id: str,+ node_execution_id: str | None,+ attempt: int,+ call_key: str,+ workflow_id: str | None = None,+ ) -> ApplicationTracingContext | None:+ run = self.start(+ tenant_id=tenant_id,+ app_id=app_id,+ source_kind=source_kind,+ source_id=source_id,+ session_id=session_id,+ workflow_id=workflow_id,+ )+ if run is None or not run.capture_agent:+ return None+ invocation = self.repository.invocation(+ run_id=run.id, agent_id=agent_id, node_execution_id=node_execution_id, attempt=attempt, call_key=call_key+ )+ return ApplicationTracingContext(+ run_id=run.id,+ invocation_id=invocation.id,+ trace_id=run.trace_id,+ parent_span_id=invocation.span_id,+ session_id=run.session_id,+ capture_content=run.capture_content,+ )++ def ingest(self, batch: AgentTraceBatch) -> int:+ accepted, run_id = self.repository.ingest(+ batch,+ max_spans=dify_config.OPS_AGENT_TRACE_MAX_SPANS,+ max_bytes=dify_config.OPS_AGENT_TRACE_MAX_BYTES,+ )+ # Persistence commits before scheduling. Beat reconciles a broker outage.+ self.schedule(run_id)+ return accepted++ def capture_agent_app_terminal(+ self,+ *,+ tenant_id: str,+ app_id: str,+ message_id: str,+ outcome: Literal["succeeded", "failed", "cancelled"],+ output: str | None,+ ) -> None:+ """Persist a worker-observed terminal even after its SSE consumer closes.++ Usage-only Message writes are intentionally insufficient for recovery.+ This handoff is made only after the runner observes a terminal result or+ finishes handling an explicit stop, and it does not rewrite Message data.+ """+ source = self.repository._source(tenant_id=tenant_id, app_id=app_id, kind="message", source_id=message_id)+ if not isinstance(source, Message):+ raise TraceCollectionError("tracing_source_not_found", 404)+ conversation_id = source.conversation_id+ run = self.repository.source_run(+ tenant_id=tenant_id, app_id=app_id, source_kind="message", source_id=message_id+ )+ if run is not None and run.state == "not_collected":+ return+ info = TraceTask(+ TraceTaskName.MESSAGE_TRACE,+ message_id=message_id,+ conversation_id=conversation_id,+ trace_session_id=run.session_id if run else conversation_id,+ ).execute()+ if not isinstance(info, MessageTraceInfo):+ raise TraceCollectionError("tracing_terminal_not_supported")+ if isinstance(info.message_data, dict):+ info.message_data = Message.from_dict(cast(MessageDict, info.message_data))+ if (+ isinstance(info.message_data, Message)+ and _message_inputs_are_empty(info.inputs)+ and info.message_data.query+ ):+ # Early errors may precede formatted-message persistence. Preserve only+ # the known user query on the root; this is not an inferred LLM prompt.+ info.inputs = {"query": info.message_data.query}+ if output is not None and isinstance(info.message_data, Message):+ # This is a detached trace snapshot, not a write to the business row.+ info.message_data.answer = output+ info.outputs = output if output is not None else info.outputs+ info.end_time = datetime.now(UTC)+ info.metadata = {**info.metadata, "message_execution_state": outcome}+ if outcome != "succeeded":+ info.error = f"Agent message {outcome}"+ self.capture_terminal(info)++ @staticmethod+ def schedule(run_id: str) -> None:+ from tasks.application_trace_task import export_application_trace++ try:+ export_application_trace.delay(run_id)+ except Exception:+ logger.warning("Application trace is persisted; scheduling deferred: run_id=%s", run_id)++ def _service_account(self, run: ApplicationTraceRun, app_id: str) -> Account:+ with self.repository.sessions() as session:+ app = session.scalar(+ select(App).where(App.id == app_id, App.id == run.app_id, App.tenant_id == run.tenant_id)+ )+ if app is None or not app.created_by:+ raise TraceCollectionError("tracing_owner_not_found", 404)+ account = session.get(Account, app.created_by)+ if account is None:+ raise TraceCollectionError("tracing_owner_not_found", 404)+ account.set_tenant_id_with_session(run.tenant_id, session=session)+ return account++ def _run_for_source(+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str+ ) -> ApplicationTraceRun | None:+ run = self.repository.source_run(+ tenant_id=tenant_id, app_id=app_id, source_kind=source_kind, source_id=source_id+ )+ if run is None and source_kind == "message":+ with self.repository.sessions() as session:+ workflow_run_id = session.scalar(+ select(Message.workflow_run_id)+ .join(App, App.id == Message.app_id)+ .where(+ Message.id == source_id,+ Message.app_id == app_id,+ App.tenant_id == tenant_id,+ )+ )+ if workflow_run_id:+ run = self.repository.source_run(+ tenant_id=tenant_id, app_id=app_id, source_kind="workflow", source_id=workflow_run_id+ )+ return run++ def _managed_owner(self, app: App, message: Message | None = None, workflow_run_id: str | None = None) -> bool:+ """Honor frozen owners first; ordinary workflows retain legacy tracing."""+ if message is not None:+ if self._run_for_source(+ tenant_id=app.tenant_id, app_id=app.id, source_kind="message", source_id=message.id+ ):+ return True+ workflow_run_id = message.workflow_run_id or workflow_run_id+ if app.mode == AppMode.AGENT:+ return True+ if workflow_run_id:+ if self.repository.source_run(+ tenant_id=app.tenant_id, app_id=app.id, source_kind="workflow", source_id=workflow_run_id+ ):+ return True+ workflow = self.repository._source(+ tenant_id=app.tenant_id, app_id=app.id, kind="workflow", source_id=workflow_run_id+ )+ if isinstance(workflow, WorkflowRun):+ return workflow_has_agent_v2(workflow.graph)+ if app.mode == AppMode.ADVANCED_CHAT or (app.mode == AppMode.WORKFLOW and workflow_run_id):+ # Naming may run before Message.workflow_run_id is populated. Use+ # the owned app workflow; an unknown snapshot fails closed. The same+ # applies to a Workflow whose run is not yet indexed by LogStore.+ with self.repository.sessions() as session:+ definition = (+ session.scalar(+ select(Workflow).where(+ Workflow.id == app.workflow_id,+ Workflow.app_id == app.id,+ Workflow.tenant_id == app.tenant_id,+ )+ )+ if app.workflow_id+ else None+ )+ return definition is None or workflow_has_agent_v2(definition.graph)+ return False++ def _capture_auxiliary(self, info: BaseTraceInfo) -> bool:+ """Resolve real business owners even when message_id is a workflow-log ID."""+ actual_message_id = info.message_data.id if isinstance(info.message_data, Message) else None+ message_ids = [actual_message_id, info.metadata.get("message_id"), info.message_id]+ message: Message | None = None+ app: App | None = None+ with self.repository.sessions() as session:+ for message_id in message_ids:+ if not isinstance(message_id, str):+ continue+ owner = session.execute(+ select(Message, App).join(App, App.id == Message.app_id).where(Message.id == message_id)+ ).first()+ if owner is not None:+ message, app = owner+ break+ conversation_id = info.metadata.get("conversation_id")+ if app is None and isinstance(conversation_id, str):+ app = session.scalar(+ select(App)+ .join(Conversation, Conversation.app_id == App.id)+ .where(Conversation.id == conversation_id)+ )+ if app is not None:+ message = session.scalar(+ select(Message)+ .where(Message.conversation_id == conversation_id, Message.app_id == app.id)+ .order_by(Message.created_at.desc())+ .limit(1)+ )+ if app is not None:+ return self._managed_owner(app, message)+ workflow_run_id = info.metadata.get("workflow_run_id")+ if isinstance(workflow_run_id, str):+ with self.repository.sessions() as session:+ # A durable managed owner already passed the trusted start boundary.+ # Suppress auxiliary content even before LogStore indexing catches up.+ managed_app = session.scalar(+ select(App)+ .join(ApplicationTraceRun, ApplicationTraceRun.app_id == App.id)+ .where(+ ApplicationTraceRun.source_kind == "workflow",+ ApplicationTraceRun.source_id == workflow_run_id,+ ApplicationTraceRun.tenant_id == App.tenant_id,+ )+ )+ if managed_app is not None:+ return True+ app_id = info.metadata.get("app_id")+ app = (+ session.scalar(select(App).where(App.id == app_id))+ if isinstance(app_id, str)+ else session.scalar(+ select(App)+ .join(WorkflowRun, WorkflowRun.app_id == App.id)+ .where(WorkflowRun.id == workflow_run_id, WorkflowRun.tenant_id == App.tenant_id)+ )+ )+ if app is not None:+ return self._managed_owner(app, workflow_run_id=workflow_run_id)+ return False++ def _link_managed_parent(self, info: WorkflowTraceInfo, tenant_id: str) -> None:+ """A legacy child cannot await a parent exported by a different pipeline."""+ parent_run_id, parent_node_id = info.resolved_parent_context+ if not parent_run_id:+ return+ with self.repository.sessions() as session:+ parent = session.scalar(+ select(ApplicationTraceRun)+ .join(App, App.id == ApplicationTraceRun.app_id)+ .where(+ ApplicationTraceRun.source_kind == "workflow",+ ApplicationTraceRun.source_id == parent_run_id,+ ApplicationTraceRun.tenant_id == tenant_id,+ App.tenant_id == tenant_id,+ )+ )+ if parent is None:+ return+ if (+ self.repository._source(tenant_id=tenant_id, app_id=parent.app_id, kind="workflow", source_id=parent_run_id)+ is None+ ):+ return+ metadata = dict(info.metadata)+ metadata.pop("parent_trace_context", None)+ metadata["linked_parent_workflow_run_id"] = parent_run_id+ if parent_node_id:+ metadata["linked_parent_node_execution_id"] = parent_node_id+ metadata["linked_parent_application_trace_id"] = parent.id+ metadata["linked_parent_trace_id"] = parent.trace_id+ info.metadata = metadata++ def capture_terminal(self, info: BaseTraceInfo) -> bool:+ """Hand managed traces to durable collection while preserving legacy scope."""+ if isinstance(info, WorkflowTraceInfo):+ app_id = str(info.metadata.get("app_id", ""))+ source_id, source_kind = info.workflow_run_id, "workflow"+ elif isinstance(info, MessageTraceInfo) and isinstance(info.message_data, Message):+ app_id = info.message_data.app_id+ source_id, source_kind = info.message_data.id, "message"+ else:+ return self._capture_auxiliary(info)+ with self.repository.sessions() as session:+ app = session.scalar(select(App).where(App.id == app_id))+ if app is None:+ return False+ message = (+ session.scalar(select(Message).where(Message.id == source_id, Message.app_id == app_id))+ if source_kind == "message"+ else None+ )+ managed = self._managed_owner(app, message, source_id if source_kind == "workflow" else None)+ run = self._run_for_source(tenant_id=app.tenant_id, app_id=app.id, source_kind=source_kind, source_id=source_id)+ if run is not None and source_kind == "message" and run.source_kind == "workflow":+ # Chatflow emits both events; the message is already in the workflow tree.+ return True+ if run is None:+ if not managed:+ if isinstance(info, WorkflowTraceInfo):+ self._link_managed_parent(info, app.tenant_id)+ return False+ run = self.repository.record_initialization_failure(+ tenant_id=app.tenant_id,+ app_id=app.id,+ source_kind=source_kind,+ source_id=source_id,+ )+ logger.warning(+ "Application trace snapshot missing; legacy export suppressed: app_id=%s source_id=%s",+ app_id,+ source_id,+ )+ if run is None:+ return True+ if run.state in {"not_collected", "failed", "uncertain", "accepted", "exporting"}:+ return True+ if (+ app.mode == AppMode.AGENT+ and message is not None+ and info.metadata.get("message_execution_state") not in ("succeeded", "failed", "cancelled")+ ):+ from services.tracing.recovery import agent_stop_requested++ if agent_stop_requested(message):+ # An already queued consumer notification may arrive after a+ # stop. Its ordinary Message snapshot cannot own the runtime+ # outcome, even if span finalization has already completed.+ return True+ loader = RepositoryWorkflowExecutionLoader(lambda requested_app: self._service_account(run, requested_app))+ trace = CanonicalTraceBuilder(loader).build(info)+ if trace is None:+ raise TraceCollectionError("tracing_terminal_not_supported")+ trace = remap_outer_trace(+ trace, trace_id=run.trace_id, capture_content=run.capture_content, agent_message=source_kind == "message"+ )+ trace = trace.model_copy(update={"session_id": run.session_id})+ self.repository.terminal(run.id, trace.model_dump(mode="json"), trace.root_span_id)+ self.schedule(run.id)+ return True++ def settle_preview(self, run: ApplicationTraceRun) -> None:+ if run.source_kind != "node" or run.outer_trace is not None:+ return+ node = self.repository._source(tenant_id=run.tenant_id, app_id=run.app_id, kind="node", source_id=run.source_id)+ if not isinstance(node, WorkflowNodeExecutionModel) or str(node.status) in {"running", "pending", "paused"}:+ return+ error = node.error+ started_at = node.created_at.replace(tzinfo=UTC)+ ended_at = started_at + timedelta(seconds=node.elapsed_time or 0)+ span = CanonicalSpan(+ id=run.root_span_id,+ parent_id=None,+ name=node.title or "Agent preview",+ kind=CanonicalSpanKind.AGENT,+ start_time=started_at,+ end_time=ended_at,+ inputs=sanitize(node.inputs) if run.capture_content else None,+ outputs=sanitize(node.outputs) if run.capture_content else None,+ status=CanonicalSpanStatus.ERROR if error else CanonicalSpanStatus.OK,+ error=error if run.capture_content else "Node execution failed" if error else None,+ metadata={"node_execution_id": node.id, "node_id": node.node_id, "source": "single_step"},+ )+ trace = CanonicalTrace(+ trace_id=run.trace_id, session_id=run.session_id, root_span_id=run.root_span_id, spans=(span,)+ )+ self.repository.terminal(run.id, trace.model_dump(mode="json"), run.root_span_id)++ def assemble(self, run: ApplicationTraceRun) -> CanonicalTrace:+ if run.outer_trace is None:+ raise TraceCollectionError("tracing_terminal_missing")+ outer = CanonicalTrace.model_validate(run.outer_trace)+ invocations, batches = self.repository.collected(run.id)+ invocation_by_id = {inv.id: inv for inv in invocations}+ spans = list(outer.spans)+ internal_by_invocation: dict[str, dict[str, CanonicalSpan]] = {inv.id: {} for inv in invocations}+ for batch in batches:+ if batch.invocation_id not in invocation_by_id:+ raise TraceCollectionError("tracing_invocation_not_found")+ for data in batch.spans:+ span = normalize_agent_span(data, capture_content=run.capture_content)+ current = internal_by_invocation[batch.invocation_id].get(span.id)+ if current is not None and current != span:+ raise TraceCollectionError("tracing_span_conflict")+ internal_by_invocation[batch.invocation_id][span.id] = span+ for invocation in invocations:+ children = list(internal_by_invocation[invocation.id].values())+ if not children:+ raise TraceCollectionError("tracing_invocation_empty")+ valid_ids = {child.id for child in children}+ if any(child.parent_id not in valid_ids and child.parent_id != invocation.span_id for child in children):+ raise TraceCollectionError("tracing_invocation_parent_invalid")+ root_children = [child for child in children if child.parent_id == invocation.span_id]+ if not root_children:+ raise TraceCollectionError("tracing_invocation_root_missing")+ errors = [child.error for child in root_children if child.status == CanonicalSpanStatus.ERROR]+ spans.append(+ CanonicalSpan(+ id=invocation.span_id,+ parent_id=invocation.parent_span_id,+ name=f"Agent attempt {invocation.attempt + 1}",+ kind=CanonicalSpanKind.AGENT,+ start_time=min(child.start_time for child in children),+ end_time=max(child.end_time or child.start_time for child in children),+ status=CanonicalSpanStatus.ERROR if errors else CanonicalSpanStatus.OK,+ error=errors[0] if errors else None,+ metadata={+ "agent_id": invocation.agent_id,+ "attempt": invocation.attempt,+ "node_execution_id": invocation.node_execution_id,+ "invocation_id": invocation.id,+ },+ )+ )+ spans.extend(children)+ trusted = {+ "tenant_id": run.tenant_id,+ "app_id": run.app_id,+ "session_id": run.session_id,+ "dify.tenant_id": run.tenant_id,+ "dify.app_id": run.app_id,+ "dify.tracing.run_id": run.id,+ "dify.source.id": run.source_id,+ }+ spans = [span.model_copy(update={"metadata": {**span.metadata, **trusted}}) for span in spans]+ return CanonicalTrace(+ trace_id=run.trace_id,+ session_id=run.session_id,+ root_span_id=outer.root_span_id,+ spans=order_spans(outer.root_span_id, spans),+ )++ @contextmanager+ def _export_lease_heartbeat(self, run_id: str, token: str) -> Generator[None]:+ """Keep slow exporters leased without holding a transaction across vendor I/O.++ A lost database connection is not proof that the exporter stopped. The+ attempt continues; the token-checked completion can reconcile a late ACK+ after lease expiry unless a manual retry has already replaced its token.+ """+ stopped = threading.Event()+ interval = dify_config.OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS / 3++ def heartbeat() -> None:+ while not stopped.wait(interval):+ try:+ if not self.repository.renew_export_lease(run_id, token):+ return+ except Exception as error:+ logger.warning(+ "Application trace heartbeat failed: run_id=%s error_type=%s", run_id, type(error).__name__+ )++ worker = threading.Thread(target=heartbeat, name=f"application-trace-lease-{run_id}", daemon=True)+ started = False+ try:+ worker.start()+ started = True+ except Exception as error:+ logger.warning(+ "Application trace heartbeat unavailable: run_id=%s error_type=%s", run_id, type(error).__name__+ )+ try:+ yield+ finally:+ stopped.set()+ # A wedged DB request must not hold the vendor completion path open.+ # When it returns, the event stops the loop; renew is also token/state guarded.+ if started:+ worker.join(timeout=1.0)+ if worker.is_alive():+ logger.warning("Application trace heartbeat shutdown pending: run_id=%s", run_id)++ def deliver(self, run_id: str) -> None:+ record = self.repository.get(run_id)+ if record is None:+ return+ from services.tracing.recovery import recover_terminal++ recover_terminal(self, record)+ record = self.repository.get(run_id)+ if record is None:+ return+ self.settle_preview(record)+ claim = self.repository.claim(+ run_id,+ collection_timeout=dify_config.OPS_AGENT_TRACE_COLLECTION_TIMEOUT,+ lease_seconds=dify_config.OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS,+ )+ if claim is None:+ return+ run, token = claim+ with self._export_lease_heartbeat(run_id, token):+ try:+ try:+ trace = self.assemble(run)+ except ValueError as error:+ # Invalid persisted trees cannot be repaired by retrying a+ # provider request. ValidationError also inherits ValueError.+ raise TraceCollectionError("tracing_payload_invalid") from error+ config = dict(run.provider_config)+ config.update(OpsTraceManager.decrypt_tracing_config(run.tenant_id, run.provider, run.provider_config))+ result = export_trace(run.provider, config, trace)+ except TraceExportUncertainError:+ self.repository.complete(run_id, token, state="uncertain", error_code="provider_acceptance_unknown")+ except RetryableTraceDispatchError:+ if run.attempts >= dify_config.OPS_AGENT_TRACE_MAX_ATTEMPTS:+ self.repository.complete(run_id, token, state="failed", error_code="provider_retry_exhausted")+ else:+ self.repository.complete(+ run_id,+ token,+ state="retrying",+ error_code="provider_temporarily_unavailable",+ retry_delay=min(300, 5 * 2 ** min(run.attempts - 1, 6)),+ )+ except TraceCollectionError as error:+ self.repository.complete(run_id, token, state="failed", error_code=error.code)+ except Exception as error:+ logger.warning(+ "Application trace export failed: run_id=%s provider=%s error_type=%s",+ run_id,+ run.provider,+ type(error).__name__,+ )+ self.repository.complete(run_id, token, state="failed", error_code="provider_export_rejected")+ else:+ self.repository.complete(run_id, token, state="accepted", result=result)++ def status(+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str, node_execution_id: str | None = None+ ) -> dict[str, Any]:+ run, node_execution_id = self._resolve_status_source(+ tenant_id=tenant_id,+ app_id=app_id,+ source_kind=source_kind,+ source_id=source_id,+ node_execution_id=node_execution_id,+ )+ return self._status_for_run(run, source_kind, source_id, node_execution_id)++ def _resolve_status_source(+ self, *, tenant_id: str, app_id: str, source_kind: str, source_id: str, node_execution_id: str | None+ ) -> tuple[ApplicationTraceRun | None, str | None]:+ """Resolve node log aliases through their owned workflow, outside SQL transactions."""+ source = self.repository._source(tenant_id=tenant_id, app_id=app_id, kind=source_kind, source_id=source_id)+ if source is None:+ raise TraceCollectionError("tracing_source_not_found", 404)+ if source_kind == "node" and node_execution_id is not None and node_execution_id != source_id:+ raise TraceCollectionError("tracing_node_not_found", 404)+ run = self._run_for_source(tenant_id=tenant_id, app_id=app_id, source_kind=source_kind, source_id=source_id)+ # A standalone preview (including a frozen disabled decision) owns its+ # status even if its source later carries a workflow run reference.+ if run is None and isinstance(source, WorkflowNodeExecutionModel) and source.workflow_run_id:+ workflow = self.repository._source(+ tenant_id=tenant_id, app_id=app_id, kind="workflow", source_id=source.workflow_run_id+ )+ if (+ not isinstance(workflow, WorkflowRun)+ or not source.workflow_id+ or workflow.workflow_id != source.workflow_id+ ):+ raise TraceCollectionError("tracing_source_not_found", 404)+ run = self.repository.source_run(+ tenant_id=tenant_id, app_id=app_id, source_kind="workflow", source_id=workflow.id+ )+ node_execution_id = source.id+ return run, node_execution_id++ def _status_for_run(+ self, run: ApplicationTraceRun | None, source_kind: str, source_id: str, node_execution_id: str | None+ ) -> dict[str, Any]:+ if run is None or run.state == "not_collected":+ return {"status": "not_collected", "source_kind": source_kind, "source_id": source_id}+ invocations, _ = self.repository.collected(run.id)+ selected = (+ [inv for inv in invocations if inv.node_execution_id == node_execution_id] if node_execution_id else []+ )+ if node_execution_id and not selected and not (source_kind == "node" and run.source_kind == "workflow"):+ raise TraceCollectionError("tracing_node_not_found", 404)+ return {+ "status": run.state,+ "source_kind": source_kind,+ "source_id": source_id,+ "provider": run.provider or None,+ "trace_id": run.provider_trace_id or run.trace_id,+ "trace_url": run.trace_url,+ "span_url": run.span_urls.get(selected[-1].span_id) if selected else None,+ "project_url": frozen_project_url(run.provider, run.provider_config),+ "config_revision": run.config_revision or None,+ "capture_content": run.capture_content,+ "capture_agent": run.capture_agent,+ "span_count": run.span_count,+ "attempts": run.attempts,+ "error_code": run.error_code,+ "accepted_at": run.accepted_at,+ "queryable": None,+ "capabilities": provider_capabilities(run.provider) if run.provider else None,+ }++ def retry(+ self,+ *,+ tenant_id: str,+ app_id: str,+ source_kind: str,+ source_id: str,+ acknowledge_possible_duplicates: bool,+ node_execution_id: str | None = None,+ ) -> dict[str, Any]:+ run, node_execution_id = self._resolve_status_source(+ tenant_id=tenant_id,+ app_id=app_id,+ source_kind=source_kind,+ source_id=source_id,+ node_execution_id=node_execution_id,+ )+ if run is None:+ raise TraceCollectionError("tracing_run_not_found", 404)+ self._status_for_run(run, source_kind, source_id, node_execution_id)+ self.repository.retry(run_id=run.id, acknowledge_possible_duplicates=acknowledge_possible_duplicates)+ self.schedule(run.id)+ return self._status_for_run(self.repository.get(run.id), source_kind, source_id, node_execution_id)+++def application_tracing_service() -> ApplicationTracingService:+ return ApplicationTracingService(ApplicationTraceRepository(sessionmaker(bind=db.engine, expire_on_commit=False)))diff --git a/api/services/workflow_persistence.py b/api/services/workflow_persistence.pyindex 9244679933f..28bb6750b83 100644--- a/api/services/workflow_persistence.py+++ b/api/services/workflow_persistence.py@@ -12,6 +12,7 @@ from typing import Any, overridefrom core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntityfrom core.app.workflow.retry_history import RETRY_HISTORY_PROCESS_DATA_KEY, WorkflowNodeRetryAttemptfrom core.helper.trace_id_helper import ParentTraceContext+from core.ops.base_trace_instance import application_tracing_hooksfrom core.ops.entities.trace_entity import TraceTaskNamefrom core.ops.ops_trace_manager import TraceQueueManager, TraceTaskfrom core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository@@ -200,6 +201,15 @@ class WorkflowPersistenceLayer(Layer):self._workflow_execution_repository.save_synchronously(workflow_execution)self._workflow_execution = workflow_execution+ tracing_hooks = application_tracing_hooks()+ if tracing_hooks is not None:+ tracing_hooks.freeze_workflow_run(+ self._application_generate_entity,+ workflow_id=self._workflow_info.workflow_id,+ workflow_run_id=execution_id,+ conversation_id=self._system_variables().get(SystemVariableKey.CONVERSATION_ID.value),+ graph_data=self._workflow_info.graph_data,+ )def _handle_graph_run_succeeded(self, event: GraphRunSucceededEvent) -> None:execution = self._get_workflow_execution()diff --git a/api/services/workflow_service.py b/api/services/workflow_service.pyindex 74fef8c57c4..700d58e7273 100644--- a/api/services/workflow_service.py+++ b/api/services/workflow_service.py@@ -38,6 +38,7 @@ from core.workflow.node_factory import (from core.workflow.node_runtime import (apply_dify_debug_email_recipient,)+from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_datafrom core.workflow.nodes.human_input.callback import (DifyHITLCallback,render_form_content_before_submission,@@ -46,7 +47,13 @@ from core.workflow.nodes.human_input.callback import (from core.workflow.nodes.human_input.entities import FormInputConfig, HumanInputNodeDatafrom core.workflow.nodes.human_input.enums import HumanInputFormKindfrom core.workflow.nodes.human_input.pause_reason import HumanInputRequired-from core.workflow.system_variables import build_bootstrap_variables, build_system_variables, default_system_variables+from core.workflow.system_variables import (+ SystemVariableKey,+ build_bootstrap_variables,+ build_system_variables,+ default_system_variables,+ system_variable_selector,+)from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_poolfrom core.workflow.workflow_entry import WorkflowEntryfrom enterprise.telemetry.draft_trace import enqueue_draft_node_execution_trace@@ -1158,14 +1165,19 @@ class WorkflowService:Run draft workflow node"""files = files or []+ node_config = draft_workflow.get_node_config_by_id(node_id)+ node_type = Workflow.get_node_type_from_node_config(node_config)+ node_data = node_config["data"]+ is_agent_preview = is_dify_agent_node_data(node_data)+ if is_agent_preview and (+ draft_workflow.tenant_id != app_model.tenant_id or draft_workflow.app_id != app_model.id+ ):+ raise ValueError("Workflow does not belong to the application")with Session(bind=db.engine, expire_on_commit=False) as session, session.begin():draft_var_srv = WorkflowDraftVariableService(session)draft_var_srv.prefill_conversation_variable_default_values(draft_workflow, user_id=account.id)- node_config = draft_workflow.get_node_config_by_id(node_id)- node_type = Workflow.get_node_type_from_node_config(node_config)- node_data = node_config["data"]if is_start_node_type(node_type):with Session(bind=db.engine) as session, session.begin():draft_var_srv = WorkflowDraftVariableService(session)@@ -1234,37 +1246,59 @@ class WorkflowService:triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,)node, node_events = run+ if is_agent_preview:+ # Standalone previews have no WorkflowRun. The general debug bootstrap+ # inserts a synthetic run ID; do not use it as this caller's owner.+ variable_pool.remove(system_variable_selector(SystemVariableKey.WORKFLOW_EXECUTION_ID))+started_event: NodeRunStartedEvent | None = None+ started_execution: WorkflowNodeExecution | None = Nonedef persist_node_start() -> Generator[NodeEvent | ContainerAwaitRequest]:- nonlocal started_event+ nonlocal started_event, started_executionfor event in node_events:if isinstance(event, NodeRunStartedEvent) and event.node_id == node_id:started_event = event# Agent workspace allocation requires this caller row before the node body runs.- repository.save_synchronously(- WorkflowNodeExecution(- id=event.id,- node_execution_id=event.id,- workflow_id=draft_workflow.id,- index=1,- predecessor_node_id=event.predecessor_node_id,- node_id=event.node_id,- node_type=event.node_type,- title=event.node_title,- process_data=event.node_run_result.process_data or None,- metadata=event.node_run_result.metadata,- created_at=event.start_at,- )+ started_execution = WorkflowNodeExecution(+ id=event.id,+ node_execution_id=event.id,+ workflow_id=draft_workflow.id,+ index=1,+ predecessor_node_id=event.predecessor_node_id,+ node_id=event.node_id,+ node_type=event.node_type,+ title=event.node_title,+ process_data=event.node_run_result.process_data or None,+ metadata=event.node_run_result.metadata,+ created_at=event.start_at,)+ repository.save_synchronously(started_execution)yield eventstart_at = time.perf_counter()- node_execution = self._handle_single_step_result(- invoke_node_fn=lambda: (node, persist_node_start()),- start_at=start_at,- node_id=node_id,- )+ try:+ node_execution = self._handle_single_step_result(+ invoke_node_fn=lambda: (node, persist_node_start()),+ start_at=start_at,+ node_id=node_id,+ )+ except BaseException:+ if is_agent_preview and started_execution is not None:+ started_execution.status = WorkflowNodeExecutionStatus.FAILED+ started_execution.error = "Agent preview execution interrupted"+ started_execution.elapsed_time = time.perf_counter() - start_at+ started_execution.finished_at = naive_utc_now()+ try:+ repository.save_synchronously(started_execution)+ # Keep the terminal append for LogStore and other async repositories.+ repository.save(started_execution)+ except Exception as persistence_error:+ logging.getLogger(__name__).warning(+ "Agent preview terminal persistence failed (%s)", type(persistence_error).__name__+ )+ raise+node_execution.workflow_id = draft_workflow.idif started_event is not None:node_execution.id = started_event.id@@ -1706,7 +1740,6 @@ class WorkflowService:created_at=naive_utc_now(),finished_at=naive_utc_now(),)-# Populate execution result dataself._populate_execution_result(node_execution, node_run_result, run_succeeded, error)diff --git a/api/tasks/application_trace_task.py b/api/tasks/application_trace_task.pynew file mode 100644index 00000000000..a7d68d69b8d--- /dev/null+++ b/api/tasks/application_trace_task.py@@ -0,0 +1,22 @@+"""Durable application trace delivery on the existing ops_trace worker queue."""++from celery import shared_task++from configs import dify_config+++@shared_task(queue="ops_trace")+def export_application_trace(run_id: str) -> None:+ from services.tracing.service import application_tracing_service++ application_tracing_service().deliver(run_id)+++@shared_task(queue="ops_trace")+def reconcile_application_traces() -> None:+ from services.tracing.service import application_tracing_service++ service = application_tracing_service()+ for run_id in service.repository.pending_ids():+ service.schedule(run_id)+ service.repository.purge(retention_days=dify_config.OPS_AGENT_TRACE_RETENTION_DAYS)diff --git a/api/tasks/ops_trace_task.py b/api/tasks/ops_trace_task.pyindex acfa59df3f2..1295b3f97b4 100644--- a/api/tasks/ops_trace_task.py+++ b/api/tasks/ops_trace_task.py@@ -1,8 +1,9 @@"""Celery task for asynchronous ops trace dispatch.-Trace providers may report explicitly retryable dispatch failures through the-core retryable exception contract. The task preserves the payload file only+Trace providers may report retryable dispatch failures through the core+exception contract. Unknown acceptance is terminal to prevent automatic replay+of a possibly accepted prefix. The task preserves the payload file onlywhen Celery accepts the retry request; successful dispatches and terminalfailures clean up the stored payload.@@ -23,6 +24,7 @@ from celery.exceptions import Retryfrom flask import current_appfrom configs import dify_config+from core.ops.base_trace_instance import application_tracing_hooksfrom core.ops.entities.config_entity import OPS_FILE_PATH, OPS_TRACE_FAILED_KEYfrom core.ops.entities.trace_entity import trace_info_info_mapfrom core.ops.exceptions import RetryableTraceDispatchError@@ -58,7 +60,6 @@ def process_trace_tasks(self, file_info):trace_info = file_data.get("trace_info")trace_info_type = file_data.get("trace_info_type")enterprise_trace_dispatched = bool(file_data.get("_enterprise_trace_dispatched"))- trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)if trace_info.get("message_data"):trace_info["message_data"] = Message.from_dict(data=trace_info["message_data"])@@ -74,6 +75,13 @@ def process_trace_tasks(self, file_info):if trace_type:trace_info = trace_type(**trace_info)+ application_trace_captured = False+ if dify_config.OPS_AGENT_TRACE_ENABLED:+ with current_app.app_context():+ tracing_hooks = application_tracing_hooks()+ if tracing_hooks is not None:+ application_trace_captured = tracing_hooks.capture_terminal(trace_info)+from extensions.ext_enterprise_telemetry import is_enabled as is_ee_telemetry_enabledif is_ee_telemetry_enabled() and not enterprise_trace_dispatched:@@ -87,21 +95,29 @@ def process_trace_tasks(self, file_info):file_data["_enterprise_trace_dispatched"] = Trueenterprise_trace_dispatched = True- if trace_instance:- with current_app.app_context():- trace_instance.trace(trace_info)+ if not application_trace_captured:+ trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)+ if trace_instance:+ with current_app.app_context():+ trace_instance.trace(trace_info)logger.info("Processing trace tasks success, app_id: %s", app_id)except RetryableTraceDispatchError as e:- # Retryable dispatch failures represent a transient provider-side- # ordering gap, not corrupt payload data. Keep the payload only after+ # Only confirmed retryable failures, including transient parent ordering+ # gaps, may replay. Unknown acceptance must terminate without retry.+ # Keep the payload only after# Celery accepts the retry request; otherwise this attempt becomes a# terminal failure and the stored file is cleaned up in `finally`.## Enterprise telemetry runs before provider dispatch. If it already ran# and provider dispatch asks for a retry, persist that private flag so# the next attempt does not emit the same enterprise trace twice.- if self.request.retries >= _RETRYABLE_TRACE_DISPATCH_LIMIT:+ if e.acceptance_unknown:+ # Provider exception details can contain private request content.+ logger.exception("Trace acceptance unknown; automatic retry suppressed, app_id: %s", app_id, exc_info=False)+ failed_key = f"{OPS_TRACE_FAILED_KEY}_{app_id}"+ redis_client.incr(failed_key)+ elif self.request.retries >= _RETRYABLE_TRACE_DISPATCH_LIMIT:logger.exception("Retryable trace dispatch budget exhausted, app_id: %s", app_id)failed_key = f"{OPS_TRACE_FAILED_KEY}_{app_id}"redis_client.incr(failed_key)diff --git a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py b/api/tests/unit_tests/clients/agent_backend/test_request_builder.pyindex e11bdbe0eeb..169f6cf7a2c 100644--- a/api/tests/unit_tests/clients/agent_backend/test_request_builder.py+++ b/api/tests/unit_tests/clients/agent_backend/test_request_builder.py@@ -1,9 +1,11 @@+import jsonfrom typing import Any, castimport pytest+from dify_agent.adapters.llm.model import _get_stop_sequences, _map_model_settings_to_parametersfrom dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolsLayerConfigfrom dify_agent.layers.dify_plugin import DifyPluginLLMLayerConfig, DifyPluginToolConfig, DifyPluginToolsLayerConfig-from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig+from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfigfrom dify_agent.layers.knowledge import DifyKnowledgeBaseLayerConfigfrom dify_agent.layers.prompt import Config as PromptConfigfrom dify_agent.layers.shell import DifyShellEnvVarConfig, DifyShellLayerConfig@@ -14,7 +16,7 @@ from dify_agent.protocol import (DIFY_AGENT_OUTPUT_LAYER_ID,CreateRunRequest,)-from pydantic import ValidationError+from pydantic import JsonValue, ValidationErrorfrom clients.agent_backend import (AGENT_SOUL_PROMPT_LAYER_ID,@@ -32,6 +34,7 @@ from clients.agent_backend import (redact_for_agent_backend_log,)from clients.agent_backend.request_builder import DIFY_SHELL_LAYER_ID+from models.agent_config_entities import AgentSoulModelSettingsdef _run_input() -> AgentBackendWorkflowNodeRunInput:@@ -121,6 +124,70 @@ def test_request_builder_forwards_plugin_specific_model_settings_via_extra_body(}[email protected]("builder_kind", ["agent_app", "workflow_node"])+ "settings",+ [+ pytest.param({}, id="default"),+ pytest.param({"response_format": None}, id="none"),+ pytest.param({"response_format": "text"}, id="plugin-text"),+ pytest.param(+ {+ "response_format": "json_schema",+ "json_schema": json.dumps(+ {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}+ ),+ },+ id="plugin-json-schema",+ ),+ pytest.param({"response_format": {"type": "text"}}, id="legacy-text"),+ pytest.param(+ {+ "response_format": {+ "type": "json_schema",+ "json_schema": {"name": "response", "schema": {"type": "object"}},+ }+ },+ id="legacy-json-schema",+ ),+ ],+)+def test_response_format_reaches_plugin_parameters_through_real_request_serialization(+ builder_kind: str, settings: dict[str, JsonValue]+) -> None:+ soul_settings = AgentSoulModelSettings.model_validate(+ {**settings, "temperature": 0.2, "stop": ["END"], "enable_thinking": True}+ )+ model = AgentBackendModelConfig(+ plugin_id="langgenius/openai",+ model_provider="openai",+ model="gpt-test",+ model_settings=soul_settings.model_dump(mode="json", exclude_none=True),+ )+ builder = AgentBackendRunRequestBuilder()+ if builder_kind == "agent_app":+ request = builder.build_for_agent_app(_agent_app_input().model_copy(update={"model": model}))+ else:+ request = builder.build_for_workflow_node(_run_input().model_copy(update={"model": model}))++ request = CreateRunRequest.model_validate_json(request.model_dump_json())+ model_layer = next(layer for layer in request.composition.layers if layer.name == DIFY_AGENT_MODEL_LAYER_ID)+ model_config = DifyPluginLLMLayerConfig.model_validate(model_layer.config)+ plugin_settings = {key: value for key, value in settings.items() if value is not None}++ assert model_config.model_settings == {+ "temperature": 0.2,+ "stop_sequences": ["END"],+ "extra_body": {"enable_thinking": True, **plugin_settings},+ }+ assert _map_model_settings_to_parameters(model_config.model_settings) == {+ "temperature": 0.2,+ "enable_thinking": True,+ **plugin_settings,+ }+ assert _get_stop_sequences(model_config.model_settings) == ["END"]++@pytest.mark.parametrize("agent_config_version_kind", ["snapshot", "draft"])def test_agent_app_request_builder_keeps_agent_soul_prompt_for_snapshot_and_draft(agent_config_version_kind: str,@@ -192,6 +259,38 @@ def test_request_builder_sets_model_and_output_layer_contract_ids() -> None:assert layers[DIFY_AGENT_MODEL_LAYER_ID].config["execution_context"] == DIFY_EXECUTION_CONTEXT_LAYER_ID[email protected]("mode", ["agent_app", "workflow_node"])[email protected]("traced", [False, True])+def test_context_wire_contract_omits_tracing_for_legacy_runtime(mode: str, traced: bool) -> None:+ run_input = _agent_app_input() if mode == "agent_app" else _run_input()+ tracing = ApplicationTracingContext(+ run_id="00000000-0000-0000-0000-000000000001",+ invocation_id="00000000-0000-0000-0000-000000000002",+ trace_id="a" * 32,+ parent_span_id="b" * 16,+ session_id="session",+ capture_content=False,+ )+ if traced:+ run_input = run_input.model_copy(+ update={"execution_context": run_input.execution_context.model_copy(update={"tracing": tracing})}+ )+ builder = AgentBackendRunRequestBuilder()+ request = (+ builder.build_for_agent_app(run_input)+ if isinstance(run_input, AgentBackendAgentAppRunInput)+ else builder.build_for_workflow_node(run_input)+ )+ payload = next(+ layer.config for layer in request.composition.layers if layer.name == DIFY_EXECUTION_CONTEXT_LAYER_ID+ )+ if traced:+ assert payload["tracing"] == tracing.model_dump(mode="json")+ else:+ # Pre-tracing runtimes reject even a null unknown field (extra=forbid).+ assert "tracing" not in payload++def test_request_builder_adds_dify_plugin_tools_layer_when_configured() -> None:run_input = _run_input()run_input.tools = DifyPluginToolsLayerConfig(diff --git a/api/tests/unit_tests/conftest.py b/api/tests/unit_tests/conftest.pyindex 5bf81c3f0d3..d9a765d5746 100644--- a/api/tests/unit_tests/conftest.py+++ b/api/tests/unit_tests/conftest.py@@ -354,12 +354,19 @@ def app_services(sqlite_session_factory: sessionmaker[Session], rbac_domain: RBAfrom extensions.ext_application_services import _build_oauth_server_servicefrom extensions.ext_redis import redis_clientfrom repositories.app.console_repository import ConsoleAppRepository+ from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+ from services.app_tracing_config_gateway import OpsTraceManagerGateway+ from services.app_tracing_config_service import AppTracingConfigServicefrom services.recommended_app_package_service import RecommendedAppPackageServicereturn build_app_services(database_client=sqlite_session_factory,repository=ConsoleAppRepository(session_factory=sqlite_session_factory),rbac_members=rbac_domain.rbac.members,+ tracing_configs=AppTracingConfigService(+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory),+ provider=OpsTraceManagerGateway(),+ ),oauth=_build_oauth_server_service(database_client=sqlite_session_factory, redis=redis_client),recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()),)diff --git a/api/tests/unit_tests/controllers/console/agent/test_tracing.py b/api/tests/unit_tests/controllers/console/agent/test_tracing.pynew file mode 100644index 00000000000..ca1c59e3440--- /dev/null+++ b/api/tests/unit_tests/controllers/console/agent/test_tracing.py@@ -0,0 +1,272 @@+"""Agent tracing Console contracts and authorization stay separate from PlainApp."""++import inspect+from collections.abc import Callable+from dataclasses import dataclass, field+from uuid import UUID++import pytest+from flask import Flask+from pydantic import ValidationError+from werkzeug.exceptions import Forbidden, NotFound++from controllers.common.rbac import AgentId, RBACCheck+from controllers.console import flask_admission+from controllers.console.agent import tracing+from controllers.console.app.app import AppTracePayload+from extensions.ext_application_services import ApplicationServices+from libs.login import AccountWithTenant+from machinery.context import RequestContext+from models.account import Account, AccountStatus, TenantAccountRole+from services.app.agent_app_contracts import AgentAppNotFoundError+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigRecord+from tests.unit_tests.config_override import apply_config_overrides++AGENT_ID = UUID("1fbc2f02-f7e0-4910-9032-cbe5bf6c65d5")+CONTEXT = RequestContext("request", None, "account", "tenant")+++@dataclass+class ConfigCase:+ services: ApplicationServices+ calls: list[tuple[object, ...]] = field(default_factory=list)+ record: AppTracingConfigRecord | None = None+ failure: Exception | None = None+++def service(monkeypatch: pytest.MonkeyPatch, account_application_services: ApplicationServices) -> ConfigCase:+ case = ConfigCase(account_application_services)+ target = account_application_services.agent_tracing_configs++ def get_trace(context: RequestContext, agent_id: str) -> AppTraceSettings:+ case.calls.append(("get_trace", context, agent_id))+ if case.failure is not None:+ raise case.failure+ return AppTraceSettings()++ def set_trace(context: RequestContext, agent_id: str, settings: AppTraceSettings) -> None:+ case.calls.append(("set_trace", context, agent_id, settings))++ def get(context: RequestContext, agent_id: str, provider: str) -> AppTracingConfigRecord | None:+ case.calls.append(("get", context, agent_id, provider))+ return case.record++ def delete(context: RequestContext, agent_id: str, provider: str) -> None:+ case.calls.append(("delete", context, agent_id, provider))++ monkeypatch.setattr(target, "get_trace", get_trace)+ monkeypatch.setattr(target, "set_trace", set_trace)+ monkeypatch.setattr(target, "get", get)+ monkeypatch.setattr(target, "delete", delete)+ monkeypatch.setattr(tracing, "application_services", lambda: account_application_services)+ return case+++def test_status_round_trips_content_policy(app: Flask, service: ConfigCase) -> None:+ with app.test_request_context(+ json={"enabled": True, "tracing_provider": "langfuse", "capture_content": True, "capture_agent": False}+ ):+ result = inspect.unwrap(tracing.AgentTraceApi.post)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)+ assert result == {"enabled": True, "tracing_provider": "langfuse", "capture_content": True, "capture_agent": False}+ assert service.calls == [("set_trace", CONTEXT, str(AGENT_ID), AppTraceSettings(True, "langfuse", True, False))]+++def test_enable_requires_provider_even_when_field_omitted() -> None:+ with pytest.raises(ValidationError, match="tracing_provider is required"):+ AppTracePayload.model_validate({"enabled": True})+ payload = AppTracePayload.model_validate({"enabled": False})+ assert payload.capture_content is False+ assert payload.capture_agent is True++[email protected]("enabled", [False, True])+def test_capabilities_reports_current_api_gate_without_claiming_runtime_delivery(+ app: Flask, service: ConfigCase, monkeypatch: pytest.MonkeyPatch, enabled: bool+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=enabled)+ with app.test_request_context("/?tracing_provider=langfuse"):+ result = inspect.unwrap(tracing.AgentTraceCapabilitiesApi.get)(+ tracing.AgentTraceCapabilitiesApi(), CONTEXT, AGENT_ID+ )+ assert service.calls == [("get_trace", CONTEXT, str(AGENT_ID))]+ assert result["api_collection_enabled"] is enabled+ assert result["agent_spans"] is True+ assert result["export_ack"] == "accepted"+ assert "runtime_enabled" not in result+ assert "queryable" not in result++[email protected]("enabled", [False, True])+def test_app_capabilities_reports_api_gate(+ app: Flask, monkeypatch: pytest.MonkeyPatch, enabled: bool, service: ConfigCase+) -> None:+ calls: list[tuple[RequestContext, str]] = []++ def get_trace(context: RequestContext, app_id: str) -> AppTraceSettings:+ calls.append((context, app_id))+ return AppTraceSettings()++ monkeypatch.setattr(service.services.apps.console, "get_trace", get_trace)+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=enabled)+ with app.test_request_context("/?tracing_provider=mlflow"):+ result = inspect.unwrap(tracing.AppTraceCapabilitiesApi.get)(+ tracing.AppTraceCapabilitiesApi(), CONTEXT, AGENT_ID+ )+ assert result["api_collection_enabled"] is enabled+ assert result["provider"] == "mlflow"+ assert calls == [(CONTEXT, str(AGENT_ID))]+++def test_get_config_absent_and_delete_has_no_body(app: Flask, service: ConfigCase) -> None:+ with app.test_request_context("/?tracing_provider=langfuse"):+ assert inspect.unwrap(tracing.AgentTraceConfigApi.get)(tracing.AgentTraceConfigApi(), CONTEXT, AGENT_ID) == {+ "has_not_configured": True+ }+ assert inspect.unwrap(tracing.AgentTraceConfigApi.delete)(tracing.AgentTraceConfigApi(), CONTEXT, AGENT_ID) == (+ "",+ 204,+ )+ assert service.calls == [+ ("get", CONTEXT, str(AGENT_ID), "langfuse"),+ ("delete", CONTEXT, str(AGENT_ID), "langfuse"),+ ]+++def test_invalid_agent_returns_not_found(app: Flask, service: ConfigCase) -> None:+ service.failure = AgentAppNotFoundError()+ with app.test_request_context(), pytest.raises(NotFound):+ inspect.unwrap(tracing.AgentTraceApi.get)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)+++ "method",+ [+ tracing.AgentTraceApi.post,+ tracing.AgentTraceConfigApi.get,+ tracing.AgentTraceConfigApi.post,+ tracing.AgentTraceConfigApi.patch,+ tracing.AgentTraceConfigApi.delete,+ ],+)[email protected]("rbac_enabled", [False, True])+def test_config_requires_editor_role_or_agent_edit_permission(+ app: Flask, monkeypatch: pytest.MonkeyPatch, method: Callable[..., object], rbac_enabled: bool+) -> None:+ account = Account(name="Reader", email="[email protected]", status=AccountStatus.ACTIVE)+ account.id, account.role = "account", TenantAccountRole.NORMAL+ apply_config_overrides(monkeypatch, RBAC_ENABLED=rbac_enabled)+ monkeypatch.setattr(+ flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account=account, tenant_id="tenant")+ )+ denied_calls: list[dict[str, object]] = []++ def denied(**kwargs: object) -> None:+ denied_calls.append(kwargs)+ raise Forbidden()++ monkeypatch.setattr(flask_admission, "enforce_rbac_checks", denied)+ admission = inspect.unwrap(+ method, stop=lambda candidate: "allowed_roles" in inspect.getclosurevars(candidate).nonlocals+ )+ with app.test_request_context(), pytest.raises(Forbidden):+ admission(None, agent_id=AGENT_ID)+ if rbac_enabled:+ checks = denied_calls[0]["checks"]+ assert isinstance(checks, (list, tuple))+ (check,) = checks+ assert isinstance(check, RBACCheck)+ assert check.scene == tracing.RBACPermission.AGENT_EDIT+ assert isinstance(check.locator, AgentId)+ assert denied_calls[0]["path_args"] == {"agent_id": AGENT_ID}+ else:+ assert denied_calls == []+++def test_read_settings_and_masked_config_keep_agent_identity(app: Flask, service: ConfigCase) -> None:+ from datetime import UTC, datetime++ now = datetime(2026, 1, 1, tzinfo=UTC)+ service.record = AppTracingConfigRecord(+ id="config",+ app_id="backing-app",+ tracing_provider="langfuse",+ tracing_config={"secret_key": "[__HIDDEN__]", "project_url": "https://example.test/project"},+ is_active=True,+ created_at=now,+ updated_at=now,+ )+ with app.test_request_context("/?tracing_provider=langfuse"):+ settings = inspect.unwrap(tracing.AgentTraceApi.get)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)+ config = inspect.unwrap(tracing.AgentTraceConfigApi.get)(tracing.AgentTraceConfigApi(), CONTEXT, AGENT_ID)+ assert settings == {"enabled": False, "tracing_provider": None, "capture_content": False, "capture_agent": True}+ assert config["tracing_config"] == service.record.tracing_config+ assert config["app_id"] == "backing-app"+ assert service.calls == [("get_trace", CONTEXT, str(AGENT_ID)), ("get", CONTEXT, str(AGENT_ID), "langfuse")]++[email protected]("create", [True, False])[email protected]("activate", [False, True])+def test_config_write_forwards_credentials_but_response_does_not_echo_them(+ app: Flask, service: ConfigCase, monkeypatch: pytest.MonkeyPatch, create: bool, activate: bool+) -> None:+ calls: list[tuple[RequestContext, str, str, dict[str, object], bool]] = []++ def write(+ context: RequestContext, agent_id: str, provider: str, config: dict[str, object], *, activate: bool+ ) -> None:+ calls.append((context, agent_id, provider, config, activate))++ target = service.services.agent_tracing_configs+ monkeypatch.setattr(target, "create" if create else "update", write)+ credentials = {"secret_key": "never-echo-this-secret"}+ with app.test_request_context(+ method="POST" if create else "PATCH",+ json={"tracing_provider": "langfuse", "tracing_config": credentials, "activate": activate},+ ):+ api = tracing.AgentTraceConfigApi()+ result = inspect.unwrap(api.post if create else api.patch)(api, CONTEXT, AGENT_ID)+ assert calls == [(CONTEXT, str(AGENT_ID), "langfuse", credentials, activate)]+ assert result == (({"result": "success"}, 201) if create else {"result": "success"})+ assert "never-echo-this-secret" not in str(result)+++ ("error", "expected"),+ [+ (tracing.ConsoleAppNotFoundError(), NotFound),+ (tracing.AppTracingConfigAppNotFoundError(), tracing.AppNotFoundError),+ (tracing.AppTracingConfigAlreadyExistsError(), tracing.TracingConfigAlreadyExistsError),+ (tracing.AppTracingConfigNotFoundError(), tracing.TracingConfigNotFoundError),+ (tracing.AppTracingConfigConflictError(), tracing.TracingConfigConflictError),+ (+ tracing.AppTracingConfigInvalidProviderError("private-provider-secret"),+ tracing.UnsupportedTracingProviderError,+ ),+ (tracing.AppTracingConfigProviderUnavailableError(), tracing.TracingProviderUnavailableError),+ (tracing.AppTracingConfigInvalidConfigurationError(), tracing.InvalidTracingConfigError),+ (tracing.AppTracingConfigVerificationFailedError(), tracing.TracingConfigVerificationFailedError),+ (tracing.AppTracingConfigProcessingError(), tracing.TracingConfigProcessingError),+ ],+)+def test_config_domain_errors_use_public_contract_without_provider_details(+ app: Flask, service: ConfigCase, error: Exception, expected: type[Exception]+) -> None:+ service.failure = error+ with app.test_request_context(), pytest.raises(expected) as caught:+ inspect.unwrap(tracing.AgentTraceApi.get)(tracing.AgentTraceApi(), CONTEXT, AGENT_ID)+ assert "private-provider-secret" not in str(caught.value)+++def test_unknown_capability_and_missing_app_return_public_errors(+ app: Flask, service: ConfigCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ with app.test_request_context("/?tracing_provider=unknown"), pytest.raises(tracing.UnsupportedTracingProviderError):+ inspect.unwrap(tracing.AgentTraceCapabilitiesApi.get)(tracing.AgentTraceCapabilitiesApi(), CONTEXT, AGENT_ID)++ def missing(_context: RequestContext, _app_id: str) -> AppTraceSettings:+ raise tracing.ConsoleAppNotFoundError()++ monkeypatch.setattr(service.services.apps.console, "get_trace", missing)+ with app.test_request_context("/?tracing_provider=langfuse"), pytest.raises(NotFound):+ inspect.unwrap(tracing.AppTraceCapabilitiesApi.get)(tracing.AppTraceCapabilitiesApi(), CONTEXT, AGENT_ID)diff --git a/api/tests/unit_tests/controllers/console/agent/test_tracing_runs.py b/api/tests/unit_tests/controllers/console/agent/test_tracing_runs.pynew file mode 100644index 00000000000..7fb163b5bf0--- /dev/null+++ b/api/tests/unit_tests/controllers/console/agent/test_tracing_runs.py@@ -0,0 +1,294 @@+"""Run tracing endpoints preserve resource ownership, acknowledgement and error contracts."""++import inspect+from collections.abc import Callable+from dataclasses import dataclass, field+from datetime import UTC, datetime, timedelta, timezone+from functools import wraps+from uuid import UUID++import pytest+from flask import Flask+from flask_restx import Resource+from pydantic import ValidationError+from sqlalchemy.orm import Session, sessionmaker+from werkzeug.exceptions import Forbidden, NotFound++from controllers.common.rbac import AgentId, PlainApp, RBACCheck+from controllers.console import flask_admission+from controllers.console.agent import tracing_runs+from extensions.ext_application_services import ApplicationServices+from libs.external_api import ExternalApi+from libs.login import AccountWithTenant+from machinery.context import RequestContext+from models.account import Account, AccountStatus, TenantAccountRole+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError+from services.app.agent_app_contracts import AgentAppNotFoundError+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides++CONTEXT = RequestContext("request", None, "account", "tenant")+AGENT_ID = UUID("01a11111-1111-7111-8111-111111111111")+APP_ID = UUID("01a22222-2222-7222-8222-222222222222")+SOURCE_ID = UUID("01a33333-3333-7333-8333-333333333333")+NODE_ID = UUID("01a44444-4444-7444-8444-444444444444")+++@dataclass+class RunCase:+ owners: ApplicationServices+ runs: ApplicationTracingService+ owner_calls: list[tuple[RequestContext, str]] = field(default_factory=list)+ status_calls: list[dict[str, object]] = field(default_factory=list)+ retry_calls: list[dict[str, object]] = field(default_factory=list)+ response: dict[str, object] = field(+ default_factory=lambda: {"status": "not_collected", "source_kind": "message", "source_id": str(SOURCE_ID)}+ )+ owner_error: Exception | None = None+ retry_error: Exception | None = None+++def services(+ monkeypatch: pytest.MonkeyPatch,+ account_application_services: ApplicationServices,+ sqlite_session_factory: sessionmaker[Session],+) -> RunCase:+ runs = ApplicationTracingService(ApplicationTraceRepository(sqlite_session_factory))+ case = RunCase(account_application_services, runs)++ def resolve(context: RequestContext, agent_id: str) -> str:+ case.owner_calls.append((context, agent_id))+ if case.owner_error is not None:+ raise case.owner_error+ return str(APP_ID)++ def status(**kwargs: object) -> dict[str, object]:+ case.status_calls.append(kwargs)+ return case.response++ def retry(**kwargs: object) -> dict[str, object]:+ case.retry_calls.append(kwargs)+ if case.retry_error is not None:+ raise case.retry_error+ return {**case.response, "status": "retrying"}++ monkeypatch.setattr(account_application_services.agent_tracing_configs, "resolve_app_id", resolve)+ monkeypatch.setattr(runs, "status", status)+ monkeypatch.setattr(runs, "retry", retry)+ monkeypatch.setattr(tracing_runs, "application_services", lambda: account_application_services)+ monkeypatch.setattr(tracing_runs, "application_tracing_service", lambda: runs)+ return case+++ "accepted_at",+ [+ datetime(2026, 1, 1),+ datetime(2026, 1, 1, tzinfo=UTC),+ datetime(2026, 1, 1, 8, tzinfo=timezone(timedelta(hours=8))),+ ],+)+def test_agent_status_resolves_owner_and_never_exposes_destination_secrets(+ app: Flask, services: RunCase, accepted_at: datetime+) -> None:+ services.response.update(+ {+ "status": "accepted",+ "provider": "langfuse",+ "trace_id": "trace-id",+ "trace_url": "https://example.com/trace",+ "project_url": "https://example.com/project/frozen",+ "accepted_at": accepted_at,+ "queryable": None,+ "provider_config": {"secret_key": "do-not-return"},+ "config_revision": "frozen-revision",+ }+ )+ with app.test_request_context(f"/?node_execution_id={NODE_ID}"):+ result = inspect.unwrap(tracing_runs.AgentTraceRunApi.get)(+ tracing_runs.AgentTraceRunApi(), CONTEXT, AGENT_ID, SOURCE_ID+ )+ assert services.owner_calls == [(CONTEXT, str(AGENT_ID))]+ assert services.status_calls == [+ {+ "tenant_id": "tenant",+ "app_id": str(APP_ID),+ "source_kind": "message",+ "source_id": str(SOURCE_ID),+ "node_execution_id": str(NODE_ID),+ }+ ]+ assert result["accepted_at"] == "2026-01-01T00:00:00Z"+ assert result["queryable"] is None+ assert result["config_revision"] == "frozen-revision"+ assert result["project_url"] == "https://example.com/project/frozen"+ assert "provider_config" not in result+ assert "do-not-return" not in str(result)++[email protected]("kind", ["message", "workflow", "node"])+def test_app_status_source_scope_and_not_collected(app: Flask, services: RunCase, kind: str) -> None:+ services.response["source_kind"] = kind+ with app.test_request_context():+ result = inspect.unwrap(tracing_runs.AppTraceRunApi.get)(+ tracing_runs.AppTraceRunApi(), CONTEXT, APP_ID, kind, SOURCE_ID+ )+ assert services.owner_calls == []+ assert result["status"] == "not_collected"+ assert result["project_url"] is None+ assert services.status_calls == [+ {+ "tenant_id": "tenant",+ "app_id": str(APP_ID),+ "source_kind": kind,+ "source_id": str(SOURCE_ID),+ "node_execution_id": None,+ }+ ]++[email protected]("ack", [False, True])[email protected]("agent", [False, True])+def test_retry_forwards_only_frozen_run_identity_and_explicit_ack(+ app: Flask, services: RunCase, ack: bool, agent: bool+) -> None:+ with app.test_request_context(+ f"/?node_execution_id={NODE_ID}", method="POST", json={"acknowledge_possible_duplicates": ack} if ack else {}+ ):+ if agent:+ result = inspect.unwrap(tracing_runs.AgentTraceRunRetryApi.post)(+ tracing_runs.AgentTraceRunRetryApi(), CONTEXT, AGENT_ID, SOURCE_ID+ )+ else:+ result = inspect.unwrap(tracing_runs.AppTraceRunRetryApi.post)(+ tracing_runs.AppTraceRunRetryApi(), CONTEXT, APP_ID, "message", SOURCE_ID+ )+ assert result["status"] == "retrying"+ assert services.retry_calls == [+ {+ "tenant_id": "tenant",+ "app_id": str(APP_ID),+ "source_kind": "message",+ "source_id": str(SOURCE_ID),+ "node_execution_id": str(NODE_ID),+ "acknowledge_possible_duplicates": ack,+ }+ ]++[email protected]("payload", [{"provider": "different-provider"}, {"acknowledge_possible_duplicates": "true"}])+def test_retry_rejects_destination_changes_and_non_boolean_ack(+ app: Flask, services: RunCase, payload: dict[str, str]+) -> None:+ with app.test_request_context(method="POST", json=payload), pytest.raises(ValidationError):+ inspect.unwrap(tracing_runs.AppTraceRunRetryApi.post)(+ tracing_runs.AppTraceRunRetryApi(), CONTEXT, APP_ID, "message", SOURCE_ID+ )+ assert services.retry_calls == []+++def test_source_kind_is_validated_before_service_lookup(app: Flask, services: RunCase) -> None:+ with app.test_request_context(), pytest.raises(ValidationError):+ inspect.unwrap(tracing_runs.AppTraceRunApi.get)(+ tracing_runs.AppTraceRunApi(), CONTEXT, APP_ID, "unknown", SOURCE_ID+ )+ assert services.status_calls == []+++def test_agent_owner_rejection_prevents_run_lookup(app: Flask, services: RunCase) -> None:+ services.owner_error = AgentAppNotFoundError()+ with app.test_request_context(), pytest.raises(NotFound):+ inspect.unwrap(tracing_runs.AgentTraceRunApi.get)(tracing_runs.AgentTraceRunApi(), CONTEXT, AGENT_ID, SOURCE_ID)+ assert services.status_calls == []+++ ("code", "status"),+ [+ ("tracing_source_not_found", 404),+ ("tracing_node_not_found", 404),+ ("tracing_run_not_found", 404),+ ("tracing_already_accepted", 409),+ ("tracing_retry_not_allowed", 409),+ ("tracing_duplicate_acknowledgement_required", 409),+ ],+)+def test_retry_business_errors_match_served_error_contract(services: RunCase, code: str, status: int) -> None:+ services.retry_error = TraceCollectionError(code, status)+ target = tracing_runs.AppTraceRunRetryApi.post++ @wraps(target)+ def post(self: Resource) -> dict[str, object]:+ return inspect.unwrap(target)(self, CONTEXT, APP_ID, "message", SOURCE_ID)++ resource = type("RetryContractResource", (Resource,), {"post": post})+ app = Flask("trace-error-contract")+ api = ExternalApi(app)+ api.add_resource(resource, "/retry")+ with app.test_client() as client:+ response = client.post("/retry", json={})+ assert response.status_code == status+ body = response.json+ assert body is not None+ assert body["code"] == code+ assert body["status"] == status+++ ("method", "permission", "locator"),+ [+ (tracing_runs.AgentTraceRunApi.get, tracing_runs.RBACPermission.AGENT_LOG_MANAGE, AgentId),+ (tracing_runs.AppTraceRunApi.get, tracing_runs.RBACPermission.APP_LOG_AND_ANNOTATION, PlainApp),+ (tracing_runs.AgentTraceRunRetryApi.post, tracing_runs.RBACPermission.AGENT_EDIT, AgentId),+ (tracing_runs.AppTraceRunRetryApi.post, tracing_runs.RBACPermission.APP_TRACING_CONFIG, PlainApp),+ ],+)+def test_run_endpoints_enforce_resource_permissions(+ app: Flask,+ monkeypatch: pytest.MonkeyPatch,+ method: Callable[..., object],+ permission: tracing_runs.RBACPermission,+ locator: type[AgentId] | type[PlainApp],+) -> None:+ account = Account(name="Reader", email="[email protected]", status=AccountStatus.ACTIVE)+ account.id, account.role = "account", TenantAccountRole.NORMAL+ apply_config_overrides(monkeypatch, RBAC_ENABLED=True)+ monkeypatch.setattr(+ flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account=account, tenant_id="tenant")+ )+ denied_calls: list[dict[str, object]] = []++ def denied(**kwargs: object) -> None:+ denied_calls.append(kwargs)+ raise Forbidden()++ monkeypatch.setattr(flask_admission, "enforce_rbac_checks", denied)+ admission = inspect.unwrap(+ method, stop=lambda candidate: "allowed_roles" in inspect.getclosurevars(candidate).nonlocals+ )+ with app.test_request_context(), pytest.raises(Forbidden):+ admission(None, agent_id=AGENT_ID) if locator is AgentId else admission(None, app_id=APP_ID)+ checks = denied_calls[0]["checks"]+ assert isinstance(checks, (list, tuple))+ (check,) = checks+ assert isinstance(check, RBACCheck)+ assert check.scene == permission+ assert isinstance(check.locator, locator)++[email protected]("method", [tracing_runs.AgentTraceRunRetryApi.post, tracing_runs.AppTraceRunRetryApi.post])+def test_retry_rejects_readonly_members_without_rbac(+ app: Flask, monkeypatch: pytest.MonkeyPatch, method: Callable[..., object]+) -> None:+ account = Account(name="Reader", email="[email protected]", status=AccountStatus.ACTIVE)+ account.id, account.role = "account", TenantAccountRole.NORMAL+ apply_config_overrides(monkeypatch, RBAC_ENABLED=False)+ monkeypatch.setattr(+ flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account=account, tenant_id="tenant")+ )+ admission = inspect.unwrap(+ method, stop=lambda candidate: "allowed_roles" in inspect.getclosurevars(candidate).nonlocals+ )+ with app.test_request_context(), pytest.raises(Forbidden):+ admission(None)diff --git a/api/tests/unit_tests/controllers/console/app/test_console_app_api.py b/api/tests/unit_tests/controllers/console/app/test_console_app_api.pyindex 06f44f45955..8dd0edd30b4 100644--- a/api/tests/unit_tests/controllers/console/app/test_console_app_api.py+++ b/api/tests/unit_tests/controllers/console/app/test_console_app_api.py@@ -27,12 +27,12 @@ from services.app.console_service import (CreatorsPlatformDisabledError,InvalidAppExportError,)+from services.app_tracing_config_service import AppTraceSettingsfrom services.entities.app_entities import (AppExportOptions,AppListParams,AppPage,AppRecord,- AppTraceSettings,CopyAppParams,CreateAppParams,StarredAppListParams,diff --git a/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py b/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.pyindex c73e9a3b98b..dc28f6eee73 100644--- a/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py+++ b/api/tests/unit_tests/controllers/console/app/test_ops_trace_api.py@@ -19,6 +19,7 @@ from controllers.console.app.error import (AppNotFoundError,InvalidTracingConfigError,TracingConfigAlreadyExistsError,+ TracingConfigConflictError,TracingConfigNotFoundError,TracingConfigProcessingError,TracingConfigVerificationFailedError,@@ -33,6 +34,7 @@ from services.app_tracing_config_gateway import OpsTraceManagerGatewayfrom services.app_tracing_config_service import (AppTracingConfigAlreadyExistsError,AppTracingConfigAppNotFoundError,+ AppTracingConfigConflictError,AppTracingConfigInvalidConfigurationError,AppTracingConfigInvalidProviderError,AppTracingConfigNotFoundError,@@ -140,10 +142,21 @@ def test_trace_config_without_sdk_can_be_read_when_absent_or_deleted(configs.delete.assert_called_once_with(workspace_id=WORKSPACE_ID, app_id=APP_ID, tracing_provider="weave")-def test_enable_tracing_maps_missing_dependency_to_unavailable(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- unavailable = AppTracingConfigProviderUnavailableError()+ ("failure", "expected"),+ [+ (AppTracingConfigProviderUnavailableError(), TracingProviderUnavailableError),+ (AppTracingConfigNotFoundError(), TracingConfigNotFoundError),+ (AppTracingConfigConflictError(), TracingConfigConflictError),+ (AppTracingConfigInvalidConfigurationError(), InvalidTracingConfigError),+ (AppTracingConfigProcessingError(), TracingConfigProcessingError),+ ],+)+def test_enable_tracing_maps_configuration_failures(+ app: Flask, monkeypatch: pytest.MonkeyPatch, failure: Exception, expected: type[BaseHTTPException]+) -> None:console_apps = MagicMock()- console_apps.set_trace.side_effect = unavailable+ console_apps.set_trace.side_effect = failuremonkeypatch.setattr(app_module,"application_services",@@ -155,7 +168,7 @@ def test_enable_tracing_maps_missing_dependency_to_unavailable(app: Flask, monkemethod="POST",json={"enabled": True, "tracing_provider": "weave"},),- pytest.raises(TracingProviderUnavailableError) as caught,+ pytest.raises(expected) as caught,):_original(app_module.AppTraceApi.post)(app_module.AppTraceApi(),@@ -163,9 +176,9 @@ def test_enable_tracing_maps_missing_dependency_to_unavailable(app: Flask, monkeUUID(APP_ID),)- assert caught.value.code == 400- assert caught.value.error_code == "tracing_provider_unavailable"- assert caught.value.__cause__ is unavailable+ assert caught.value.code == expected.code+ assert caught.value.error_code == expected.error_code+ assert caught.value.__cause__ is failure@pytest.mark.parametrize("method", _MUTATION_METHODS)@@ -321,17 +334,20 @@ def test_trace_app_config_get_configured_returns_exact_legacy_body(("mlflow", {"tracking_uri": "", "experiment_id": "0", "username": None, "password": "******"}),],)[email protected]("activate", [False, True])def test_trace_app_config_write_returns_expected_response(app: Flask,tracing_configs: MagicMock,method_name: str,expected_result: object,+ activate: bool,provider: str,config: dict[str, str | None],) -> None:payload = ops_trace_module.TraceConfigPayload(tracing_provider=provider,tracing_config=config,+ activate=activate,)with app.test_request_context("/", method=method_name.upper()):@@ -348,6 +364,7 @@ def test_trace_app_config_write_returns_expected_response(app_id=APP_ID,tracing_provider=provider,tracing_config=config,+ activate=activate,)@@ -401,6 +418,14 @@ def test_trace_app_config_maps_missing_app_to_404(@pytest.mark.parametrize(("method_name", "service_error", "expected_http_error", "expected_status", "expected_code"),[+ pytest.param(+ "patch",+ AppTracingConfigConflictError(),+ TracingConfigConflictError,+ 409,+ "tracing_config_conflict",+ id="patch-concurrent-change",+ ),pytest.param("get",AppTracingConfigProviderUnavailableError(),diff --git a/api/tests/unit_tests/controllers/console/test_arize_tracing_ack_routes.py b/api/tests/unit_tests/controllers/console/test_arize_tracing_ack_routes.pynew file mode 100644index 00000000000..cf158494655--- /dev/null+++ b/api/tests/unit_tests/controllers/console/test_arize_tracing_ack_routes.py@@ -0,0 +1,193 @@+"""Both real config Resources reach the same ACK gate before atomic SQL updates.++HTTP dispatch, payload validation, owner resolution, DI services, encryption and+SQLite persistence are real. Login admission supplies a local owner account and+the existing RSA fixture supplies private-key storage; no external account is used.+"""++import json+from copy import deepcopy+from uuid import uuid4++import pytest+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace+from flask import Flask+from flask_restx import Api+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from controllers.console import flask_admission, wraps+from controllers.console.agent.tracing import AgentTraceConfigApi+from controllers.console.app.ops_trace import TraceAppConfigApi+from core.helper import ssrf_proxy+from core.ops.ops_trace_manager import OpsTraceManager+from enums import DeploymentEdition+from extensions.ext_application_services import ApplicationServices+from libs.login import AccountWithTenant+from models.account import Account, AccountStatus, TenantAccountRole+from models.agent import Agent, AgentScope, AgentSource, AgentStatus+from models.model import App, AppMode, TraceAppConfig+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_arize_configuration_ack import (+ collector as collector, # noqa: PLC0414 - Re-export the real local gRPC collector fixture.+)++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)++[email protected]("surface", ["app", "agent"])[email protected]("method", ["POST", "PATCH"])+def test_config_write_rejects_bad_ack_without_changing_old_rows_policy_or_cache(+ workspace: str,+ sqlite_session_factory: sessionmaker[Session],+ account_application_services: ApplicationServices,+ monkeypatch: pytest.MonkeyPatch,+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ surface: str,+ method: str,+) -> None:+ endpoint, calls, _ = collector+ app_id, agent_id, account_id = (str(uuid4()) for _ in range(3))+ old_policy = json.dumps(+ {"enabled": True, "tracing_provider": "langsmith", "capture_content": False, "capture_agent": False}+ )+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ mode=AppMode.AGENT if surface == "agent" else AppMode.WORKFLOW,+ tracing=old_policy,+ )+ )+ if surface == "agent":+ session.add(+ Agent(+ id=agent_id,+ tenant_id=workspace,+ name="ACK test",+ scope=AgentScope.ROSTER,+ source=AgentSource.AGENT_APP,+ app_id=app_id,+ status=AgentStatus.ACTIVE,+ created_by=account_id,+ updated_by=account_id,+ )+ )+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ original = OpsTraceManager.encrypt_tracing_config(+ workspace, "langsmith", {"api_key": "other-provider-key", "project": "other"}+ )+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=original)+ if method == "PATCH":+ old = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "arize",+ {+ "endpoint": endpoint,+ "project": "old-project",+ "api_key": "synthetic-good",+ "space_id": "synthetic-space",+ },+ )+ old.update(+ {+ "project_url": "https://app.arize.com/?redirect_project_name=old-project",+ "arize_project_destination": {"legacy": "unchanged"},+ }+ )+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="arize", tracing_config=old)+ with sqlite_session_factory() as session:+ before = deepcopy(+ list(+ session.execute(+ select(TraceAppConfig.__table__).where(TraceAppConfig.app_id == app_id).order_by(TraceAppConfig.id)+ ).mappings()+ )+ )++ account = Account(name="Owner", email="[email protected]", status=AccountStatus.ACTIVE)+ account.id, account.role = account_id, TenantAccountRole.OWNER+ identity = AccountWithTenant(account, workspace)+ monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: identity)+ monkeypatch.setattr(wraps, "current_account_with_tenant", lambda: identity)+ apply_config_overrides(+ monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, LOGIN_DISABLED=True, RBAC_ENABLED=False+ )++ def forbidden(*_args: object, **_kwargs: object) -> None:+ pytest.fail("Local custom collector verification must not perform optional Cloud HTTP reads")++ monkeypatch.setattr(ssrf_proxy, "get", forbidden)+ monkeypatch.setattr(ArizePhoenixDataTrace, "__init__", forbidden)+ app = Flask(__name__)+ app.extensions["application_services"] = account_application_services+ api = Api(app)+ api.add_resource(TraceAppConfigApi, "/console/api/apps/<uuid:app_id>/trace-config")+ api.add_resource(AgentTraceConfigApi, "/console/api/agent/<uuid:agent_id>/trace-config")+ url = (+ f"/console/api/apps/{app_id}/trace-config"+ if surface == "app"+ else f"/console/api/agent/{agent_id}/trace-config"+ )+ values = {+ "endpoint": endpoint,+ "project": "not-created-by-verification",+ "api_key": "synthetic-invalid",+ "space_id": "synthetic-space",+ }+ with app.test_client() as client:+ failed = client.open(+ url, method=method, json={"tracing_provider": "arize", "tracing_config": values, "activate": True}+ )+ assert failed.status_code == 400+ body = failed.json+ assert isinstance(body, dict)+ assert body["code"] == "tracing_config_verification_failed"+ assert "private-vendor-details" not in failed.get_data(as_text=True)+ assert "synthetic-invalid" not in failed.get_data(as_text=True)+ with sqlite_session_factory() as session:+ after = list(+ session.execute(+ select(TraceAppConfig.__table__).where(TraceAppConfig.app_id == app_id).order_by(TraceAppConfig.id)+ ).mappings()+ )+ assert after == before+ assert session.scalar(select(App.tracing).where(App.id == app_id)) == old_policy+ values["api_key"] = "synthetic-good"+ accepted = client.open(+ url, method=method, json={"tracing_provider": "arize", "tracing_config": values, "activate": True}+ )+ assert accepted.status_code == (201 if method == "POST" else 200)+ saved = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="arize")+ assert saved is not None+ assert saved.tracing_config is not None+ assert (+ OpsTraceManager.decrypt_tracing_config(workspace, "arize", saved.tracing_config)["api_key"] == "synthetic-good"+ )+ assert "project_url" not in saved.tracing_config+ assert "arize_project_destination" not in saved.tracing_config+ assert len(calls) == 2+ with sqlite_session_factory() as session:+ policy = json.loads(session.scalar(select(App.tracing).where(App.id == app_id)) or "{}")+ assert policy == {"enabled": True, "tracing_provider": "arize", "capture_content": False, "capture_agent": False}+ if method == "PATCH":+ with app.test_client() as client:+ masked = client.patch(+ url,+ json={+ "tracing_provider": "arize",+ "tracing_config": {**values, "project": "next-project", "api_key": "******", "space_id": "******"},+ },+ )+ assert masked.status_code == 200+ updated = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="arize")+ assert updated is not None+ assert updated.tracing_config is not None+ assert updated.tracing_config["api_key"] == saved.tracing_config["api_key"]+ assert updated.tracing_config["space_id"] == saved.tracing_config["space_id"]+ assert calls[-1][1]["api_key"] == "synthetic-good"+ assert calls[-1][1]["space_id"] == "synthetic-space"+ assert len(calls) == 3diff --git a/api/tests/unit_tests/controllers/console/test_feature.py b/api/tests/unit_tests/controllers/console/test_feature.pyindex bf5ec1fe354..df58c5efec7 100644--- a/api/tests/unit_tests/controllers/console/test_feature.py+++ b/api/tests/unit_tests/controllers/console/test_feature.py@@ -147,6 +147,7 @@ class TestSystemFeatureApi:from controllers.console.feature import SystemFeatureApisystem_features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False,deployment_edition=DeploymentEdition.COMMUNITY,is_allow_register=True,enable_learn_app=True,diff --git a/api/tests/unit_tests/controllers/console/test_feature_agent_tracing.py b/api/tests/unit_tests/controllers/console/test_feature_agent_tracing.pynew file mode 100644index 00000000000..3a65d7b8511--- /dev/null+++ b/api/tests/unit_tests/controllers/console/test_feature_agent_tracing.py@@ -0,0 +1,120 @@+"""Public deployment discovery is independent of workspace tracing configuration."""++from typing import Never++import pytest+from flask import Flask+from flask_restx import Api+from pydantic import ValidationError++from controllers.console import feature, wraps+from controllers.web.feature import SystemFeatureApi as WebSystemFeatureApi+from enums import DeploymentEdition+from extensions.ext_application_services import ApplicationServices+from extensions.ext_login import DifyLoginManager+from libs import login+from models.account import AccountStatus+from services.entities.feature_entities import FeatureModel, SystemFeatureModel+from services.feature_service import FeatureService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_account, make_tenant++[email protected]("enabled", [False, True])[email protected]("surface", ["console", "web"])+def test_public_system_features_exposes_gate_without_account_app_or_provider(+ monkeypatch: pytest.MonkeyPatch,+ account_application_services: ApplicationServices,+ enabled: bool,+ surface: str,+) -> None:+ apply_config_overrides(+ monkeypatch,+ DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY,+ OPS_AGENT_TRACE_ENABLED=enabled,+ LOGIN_DISABLED=False,+ )++ def forbidden_account() -> Never:+ pytest.fail("public deployment discovery must not resolve a Console account")++ def forbidden_workspace(_workspace_id: str, **_kwargs: object) -> Never:+ pytest.fail("public deployment discovery must not query workspace features")++ monkeypatch.setattr(login, "_resolve_current_user", forbidden_account)+ monkeypatch.setattr(FeatureService, "get_features", forbidden_workspace)+ app = Flask(__name__)+ app.extensions["application_services"] = account_application_services+ resource = feature.SystemFeatureApi if surface == "console" else WebSystemFeatureApi+ path = "/console/api/system-features" if surface == "console" else "/api/system-features"+ Api(app).add_resource(resource, path)++ response = app.test_client().get(path)++ assert response.status_code == 200+ payload = response.get_json()+ assert payload["agent_tracing_api_collection_enabled"] is enabled+ assert "tracing_provider" not in payload+ assert "runtime_enabled" not in payload+ assert "queryable" not in payload+ schema = SystemFeatureModel.model_json_schema(mode="serialization")+ gate = schema["properties"]["agent_tracing_api_collection_enabled"]+ assert gate["type"] == "boolean"+ assert "default" not in gate+ assert "agent_tracing_api_collection_enabled" in schema["required"]++[email protected]("enabled", [False, True])+def test_authenticated_workspace_features_omits_deployment_gate(+ monkeypatch: pytest.MonkeyPatch,+ account_application_services: ApplicationServices,+ enabled: bool,+) -> None:+ apply_config_overrides(+ monkeypatch,+ DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY,+ OPS_AGENT_TRACE_ENABLED=enabled,+ LOGIN_DISABLED=False,+ )+ account = make_account(status=AccountStatus.ACTIVE)+ account._current_tenant = make_tenant()+ monkeypatch.setattr(login, "_resolve_current_user", lambda: account)+ monkeypatch.setattr(login, "check_csrf_token", lambda _request, _account_id: None)+ monkeypatch.setattr(wraps, "_is_setup_completed", lambda: True)+ monkeypatch.setattr(feature, "application_services", lambda: account_application_services)+ app = Flask(__name__)+ Api(app).add_resource(feature.FeatureApi, "/console/api/features")++ response = app.test_client().get("/console/api/features")++ assert response.status_code == 200+ payload = response.get_json()+ assert "agent_tracing_api_collection_enabled" not in payload+ schema = FeatureModel.model_json_schema(mode="serialization")+ assert "agent_tracing_api_collection_enabled" not in schema["properties"]+++def test_workspace_features_still_require_authentication(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, LOGIN_DISABLED=False, OPS_AGENT_TRACE_ENABLED=True)+ monkeypatch.setattr(wraps, "_is_setup_completed", lambda: True)+ monkeypatch.setattr(login, "_resolve_current_user", lambda: None)++ def forbidden() -> ApplicationServices:+ pytest.fail("anonymous requests must not reach feature queries")++ monkeypatch.setattr(feature, "application_services", forbidden)+ app = Flask(__name__)+ DifyLoginManager(app)+ Api(app).add_resource(feature.FeatureApi, "/console/api/features")++ response = app.test_client().get("/console/api/features")++ assert response.status_code == 401+ assert b"agent_tracing_api_collection_enabled" not in response.data+++def test_missing_public_gate_cannot_default_to_disabled() -> None:+ with pytest.raises(ValidationError) as error:+ SystemFeatureModel.model_validate({"deployment_edition": DeploymentEdition.COMMUNITY})++ assert error.value.errors(include_input=False)[0]["loc"] == ("agent_tracing_api_collection_enabled",)diff --git a/api/tests/unit_tests/controllers/inner_api/test_agent_tracing.py b/api/tests/unit_tests/controllers/inner_api/test_agent_tracing.pynew file mode 100644index 00000000000..df1b4576ec0--- /dev/null+++ b/api/tests/unit_tests/controllers/inner_api/test_agent_tracing.py@@ -0,0 +1,96 @@+"""HTTP authentication and acknowledgement backed by the real SQLite collector."""++from collections.abc import Iterator+from uuid import uuid4++import pytest+from flask import Flask+from flask.testing import FlaskClient+from sqlalchemy import func, select+from sqlalchemy.orm import Session, sessionmaker++from controllers.inner_api import bp+from models.application_trace import ApplicationTraceBatch+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import config_overrides_context+from tests.unit_tests.services.tracing.test_application_trace_collection import batch_for, seed+++def client() -> Iterator[FlaskClient]:+ app = Flask(__name__)+ app.config["TESTING"] = True+ app.register_blueprint(bp)+ with config_overrides_context(+ PLUGIN_DAEMON_KEY="daemon", INNER_API_KEY_FOR_PLUGIN="inner", OPS_AGENT_TRACE_ENABLED=True+ ):+ yield app.test_client()+++def test_trace_http_ack_follows_commit_and_replay_is_idempotent(+ client: FlaskClient,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ collection = seed(sqlite_session_factory, "http", capture_content=False)+ invocation = collection.invocation()+ payload = batch_for(collection.run, invocation).model_dump(mode="json")+ service = ApplicationTracingService(collection.repository)+ monkeypatch.setattr("controllers.inner_api.agent.tracing.application_tracing_service", lambda: service)+ scheduled = []++ def schedule(run_id: str) -> None:+ # A fresh transaction can see the batch before the caller gets an ACK.+ with sqlite_session_factory() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 1+ scheduled.append(run_id)++ monkeypatch.setattr(service, "schedule", schedule)+ for _ in range(2):+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})+ assert response.status_code == 200+ assert response.get_json() == {"result": "success", "batch_id": payload["batch_id"], "accepted_spans": 1}+ assert scheduled == [collection.run.id, collection.run.id]++ payload["spans"][0]["name"] = "conflicting replay"+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})+ assert response.status_code == 409+ assert len(scheduled) == 2++[email protected]("header", [{}, {"X-Inner-Api-Key": "wrong"}])+def test_trace_http_requires_internal_auth(client: FlaskClient, header: dict[str, str]) -> None:+ response = client.post("/inner/api/agent/tracing/batches", json={}, headers=header)+ assert response.status_code == 404+++def test_trace_http_rejects_invalid_span_and_unknown_owner(+ client: FlaskClient,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ collection = seed(sqlite_session_factory, "http-validation")+ invocation = collection.invocation()+ payload = batch_for(collection.run, invocation).model_dump(mode="json")+ service = ApplicationTracingService(collection.repository)+ monkeypatch.setattr("controllers.inner_api.agent.tracing.application_tracing_service", lambda: service)+ payload["spans"][0]["end_time_ns"] = 0+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})+ assert response.status_code == 400+ payload["spans"][0]["end_time_ns"] = 2_000_000_000+ payload["invocation_id"] = str(uuid4())+ response = client.post("/inner/api/agent/tracing/batches", json=payload, headers={"X-Inner-Api-Key": "inner"})+ assert response.status_code == 404+ with sqlite_session_factory() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0+++def test_trace_http_enforces_body_limit_before_ingestion(client: FlaskClient, monkeypatch: pytest.MonkeyPatch) -> None:+ calls = []+ monkeypatch.setattr("controllers.inner_api.agent.tracing.application_tracing_service", lambda: calls.append(True))+ with config_overrides_context(OPS_AGENT_TRACE_BATCH_MAX_BYTES=64):+ response = client.post(+ "/inner/api/agent/tracing/batches", json={"padding": "x" * 256}, headers={"X-Inner-Api-Key": "inner"}+ )+ assert response.status_code == 413+ assert calls == []diff --git a/api/tests/unit_tests/controllers/openapi/auth/_world.py b/api/tests/unit_tests/controllers/openapi/auth/_world.pyindex 98985e191c8..9a50759f56d 100644--- a/api/tests/unit_tests/controllers/openapi/auth/_world.py+++ b/api/tests/unit_tests/controllers/openapi/auth/_world.py@@ -115,6 +115,7 @@ def system_features(webapp_auth: bool = False,) -> SystemFeatureModel:return SystemFeatureModel(+ agent_tracing_api_collection_enabled=False,deployment_edition=DeploymentEdition.ENTERPRISE,license=LicenseStatusModel(status=license_status),webapp_auth=WebAppAuthModel(enabled=webapp_auth),diff --git a/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py b/api/tests/unit_tests/controllers/openapi/test_auth_matrix.pyindex 6eafdd198f5..4e1a047a51b 100644--- a/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py+++ b/api/tests/unit_tests/controllers/openapi/test_auth_matrix.py@@ -1664,7 +1664,7 @@ def world(sqlite_session_factory: sessionmaker[Session], token_rows: dict[str, Rdef _system_features(*, edition: DeploymentEdition, webapp_auth: bool, license_status: LicenseStatus) -> SystemFeatureModel:- features = SystemFeatureModel(deployment_edition=edition)+ features = SystemFeatureModel(agent_tracing_api_collection_enabled=False, deployment_edition=edition)features.webapp_auth.enabled = webapp_authfeatures.license.status = license_statusreturn featuresdiff --git a/api/tests/unit_tests/controllers/web/test_feature.py b/api/tests/unit_tests/controllers/web/test_feature.pyindex 1a08908bc25..42b211aacb5 100644--- a/api/tests/unit_tests/controllers/web/test_feature.py+++ b/api/tests/unit_tests/controllers/web/test_feature.py@@ -29,7 +29,9 @@ class TestSystemFeatureApi:app: Flask,mocker: MockerFixture,) -> None:- system_features = SystemFeatureModel(deployment_edition=deployment_edition)+ system_features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=deployment_edition+ )feature_queries = _install_feature_queries(mocker)feature_queries.get_public_system_features.return_value = system_featuresdiff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.pyindex a4e2cc02ac4..7cec02c4fe3 100644--- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py@@ -479,6 +479,12 @@ class TestGenerateWorker:def test_generate_task_stopped_is_swallowed(self, generator, mocker: MockerFixture):self._wire(generator, mocker, run_side_effect=GenerateTaskStoppedError())+ # A locally stopped consumer does not prove runtime cancellation. The+ # runner must hand off the typed terminal it actually observed.+ mocker.patch(+ "core.ops.base_trace_instance.application_tracing_hooks",+ side_effect=AssertionError("stop intent must not invent a tracing terminal"),+ )queue_manager = mocker.MagicMock()self._call(generator, mocker, queue_manager)queue_manager.publish_error.assert_not_called()diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.pyindex f7a122427f4..4ef45d54e3c 100644--- a/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py+++ b/api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py@@ -14,6 +14,7 @@ from dify_agent.layers.knowledge import DifyKnowledgeBaseLayerConfigfrom dify_agent.layers.prompt import Config as PromptConfigfrom dify_agent.layers.user_prompt import DifyUserPromptLayerConfigfrom dify_agent.protocol.snapshot import SessionSnapshot+from flask import Flaskfrom clients.agent_backend import (DIFY_CONFIG_LAYER_ID,@@ -30,13 +31,21 @@ from core.app.apps.agent_app.runtime_request_builder import (AgentAppRuntimeRequestBuildError,)from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom+from core.ops.base_trace_instance import bind_application_tracing_hooksfrom core.workflow.file_reference import build_file_reference+from core.workflow.nodes.agent_v2.dify_tools_builder import WorkflowAgentDifyToolsBuilderfrom graphon.file import File, FileTransferMethod, FileTypefrom graphon.model_runtime.entities.message_entities import ImagePromptMessageContentfrom models.agent_config_entities import AgentSoulConfig+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooksfrom tests.unit_tests.config_override import apply_config_overrides[email protected](autouse=True)+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)++@pytest.fixture(autouse=True)def _no_runtime_agent_skills(monkeypatch: pytest.MonkeyPatch):monkeypatch.setattr(@@ -112,6 +121,43 @@ class TestBuildForAgentApp:assert [layer.name for layer in request.composition.layers][0] == "agent_app_user_prompt"+def test_builder_carries_application_trace_context_but_redacts_it_from_request_logs(monkeypatch: pytest.MonkeyPatch):+ from uuid import uuid4++ from dify_agent.layers.execution_context import ApplicationTracingContext++ tracing = ApplicationTracingContext(+ run_id=uuid4(),+ invocation_id=uuid4(),+ trace_id="1" * 32,+ parent_span_id="2" * 16,+ session_id="private-session",+ capture_content=False,+ )+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ context = _ctx(_soul_with_model())+ calls: list[AgentAppRuntimeBuildContext] = []++ def prepare(value: AgentAppRuntimeBuildContext) -> ApplicationTracingContext:+ calls.append(value)+ return tracing++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "prepare_agent_app", staticmethod(prepare))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)++ with app.app_context():+ result = AgentAppRuntimeRequestBuilder(dify_tools_builder=WorkflowAgentDifyToolsBuilder()).build(context)+ layer = next(item for item in result.request.composition.layers if item.name == "execution_context")+ assert DifyExecutionContextLayerConfig.model_validate(layer.config).tracing == tracing+ redacted = next(+ item for item in result.redacted_request["composition"]["layers"] if item["name"] == "execution_context"+ )+ assert redacted["config"]["tracing"] == "[REDACTED]"+ assert calls == [context]++class _NoToolsBuilder:def build_layers(self, **kwargs):del kwargsdiff --git a/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py b/api/tests/unit_tests/core/app/workflow/test_persistence_layer.pyindex 6046a70fabd..cdf3f984378 100644--- a/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py+++ b/api/tests/unit_tests/core/app/workflow/test_persistence_layer.py@@ -5,12 +5,14 @@ from datetime import UTC, datetime, timedeltafrom types import SimpleNamespaceimport pytest+from flask import Flaskfrom sqlalchemy import event as sqlalchemy_eventfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmaker-from core.app.entities.app_invoke_entities import WorkflowAppGenerateEntity+from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntityfrom core.app.workflow.retry_history import RETRY_HISTORY_PROCESS_DATA_KEY+from core.ops.base_trace_instance import bind_application_tracing_hooksfrom core.ops.ops_trace_manager import TraceTask, TraceTaskNamefrom core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepositoryfrom core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository@@ -62,13 +64,21 @@ from graphon.runtime.execution import ROOT_FRAME_IDfrom models import Account, WorkflowRunfrom models.enums import WorkflowRunTriggeredFromfrom models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from repositories.entities.workflow_execution import WorkflowExecutionfrom repositories.sqlalchemy_api_workflow_node_execution_repository import (DifyAPISQLAlchemyWorkflowNodeExecutionRepository,)+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooksfrom services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer+from tests.unit_tests.config_override import apply_config_overridesfrom tests.workflow_test_utils import build_test_graph_init_params[email protected](autouse=True)+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)++class _RepoRecorder:def __init__(self) -> None:self.saved: list[object] = []@@ -679,6 +689,37 @@ def test_terminal_failure_batches_only_unfinished_nodes_per_source(class TestWorkflowPersistenceLayer:+ def test_start_freezes_tracing_only_after_workflow_save(self, monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ layer, workflow_repo, _, _ = _make_layer(extras={"trace_session_id": "external-session"})+ calls: list[tuple[str, str, str, str, object]] = []++ def freeze(+ entity: AdvancedChatAppGenerateEntity | WorkflowAppGenerateEntity,+ *,+ workflow_id: str,+ workflow_run_id: str,+ conversation_id: object,+ graph_data: object,+ ) -> None:+ assert workflow_repo.synchronously_saved+ assert graph_data == {"nodes": [], "edges": []}+ saved_execution = workflow_repo.synchronously_saved[-1]+ assert isinstance(saved_execution, WorkflowExecution)+ assert saved_execution.id_ == workflow_run_id+ calls.append(+ (entity.app_config.app_id, entity.app_config.tenant_id, workflow_id, workflow_run_id, conversation_id)+ )++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "freeze_workflow_run", staticmethod(freeze))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)++ with app.app_context():+ layer._handle_graph_run_started()+ assert calls == [("app", "tenant", "workflow-id", "run-id", "conv-id")]+def test_get_execution_id_requires_system_variable(self):layer, _, _, _ = _make_layer(build_system_variables())diff --git a/api/tests/unit_tests/core/ops/test_application_tracing_hooks.py b/api/tests/unit_tests/core/ops/test_application_tracing_hooks.pynew file mode 100644index 00000000000..994d0cad57b--- /dev/null+++ b/api/tests/unit_tests/core/ops/test_application_tracing_hooks.py@@ -0,0 +1,52 @@+"""The execution port is gated before Flask lookup and bound per application."""++from contextvars import Context++import pytest+from flask import Flask, has_app_context++from core.ops.base_trace_instance import (+ ApplicationTracingHooksNotRegisteredError,+ application_tracing_hooks,+ bind_application_tracing_hooks,+)+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks+from tests.unit_tests.config_override import apply_config_overrides+++def test_disabled_port_needs_no_application_context(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)+ isolated_context = Context()+ assert not isolated_context.run(has_app_context)+ assert isolated_context.run(application_tracing_hooks) is None+++def test_enabled_port_reports_missing_registration_without_context(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ isolated_context = Context()+ assert not isolated_context.run(has_app_context)+ with pytest.raises(ApplicationTracingHooksNotRegisteredError, match="application_tracing_hooks_not_registered"):+ isolated_context.run(application_tracing_hooks)+++def test_enabled_port_reports_missing_registration_in_app(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ with Flask(__name__).app_context():+ with pytest.raises(ApplicationTracingHooksNotRegisteredError, match="application_tracing_hooks_not_registered"):+ application_tracing_hooks()+++def test_port_does_not_leak_between_flask_applications(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ first, second = Flask("first"), Flask("second")+ first_hooks = RuntimeApplicationTracingHooks()+ second_hooks = RuntimeApplicationTracingHooks()+ bind_application_tracing_hooks(first, first_hooks)+ bind_application_tracing_hooks(second, second_hooks)+ with first.app_context():+ assert application_tracing_hooks() is first_hooks+ with second.app_context():+ assert application_tracing_hooks() is second_hooks+ assert application_tracing_hooks() is first_hooks+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)+ assert application_tracing_hooks() is Nonediff --git a/api/tests/unit_tests/core/ops/test_ops_trace_manager.py b/api/tests/unit_tests/core/ops/test_ops_trace_manager.pyindex ba456261407..a7cc2ba847a 100644--- a/api/tests/unit_tests/core/ops/test_ops_trace_manager.py+++ b/api/tests/unit_tests/core/ops/test_ops_trace_manager.py@@ -467,14 +467,20 @@ def test_message_config_lookup_uses_real_conversation_and_model_config(database:assert OpsTraceManager.get_app_config_through_message_id("missing") is None[email protected]("module_name", ["dify_trace_weave", "wandb"])-def test_provider_loader_identifies_missing_optional_dependency(- monkeypatch: pytest.MonkeyPatch, module_name: str-) -> None:- monkeypatch.delitem(sys.modules, "dify_trace_weave.weave_trace", raising=False)- monkeypatch.setitem(sys.modules, module_name, None)- with pytest.raises(TraceProviderNotInstalledError, match=f"weave.*{module_name}") as caught:- module.OpsTraceProviderConfigMap()["weave"][email protected]("imports_cached", [False, True], ids=["cold", "warm"])[email protected]("module_name", ["dify_trace_weave", "weave"])+def test_provider_loader_identifies_missing_optional_dependency(module_name: str, imports_cached: bool) -> None:+ with patch.dict(sys.modules):+ if imports_cached:+ module.OpsTraceProviderConfigMap()["weave"]+ # Cached typed SDK submodules can bypass a missing top-level package.+ # Simulate its absence consistently and restore the complete cache after.+ for cached_name in list(sys.modules):+ if cached_name.partition(".")[0] in {"dify_trace_weave", "weave"}:+ del sys.modules[cached_name]+ with patch.dict(sys.modules, {module_name: None}):+ with pytest.raises(TraceProviderNotInstalledError, match=f"weave.*{module_name}") as caught:+ module.OpsTraceProviderConfigMap()["weave"]cause = caught.value.__cause__assert isinstance(cause, ModuleNotFoundError)diff --git a/api/tests/unit_tests/core/ops/test_trace_queue_manager.py b/api/tests/unit_tests/core/ops/test_trace_queue_manager.pyindex eb11d6197c4..51a924127ee 100644--- a/api/tests/unit_tests/core/ops/test_trace_queue_manager.py+++ b/api/tests/unit_tests/core/ops/test_trace_queue_manager.py@@ -14,6 +14,16 @@ import queuefrom unittest.mock import MagicMock, patchimport pytest+from flask import Flask++from core.ops.base_trace_instance import bind_application_tracing_hooks+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooks+from tests.unit_tests.config_override import apply_config_overrides++[email protected](autouse=True)+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)@pytest.fixture@@ -158,3 +168,37 @@ class TestTraceQueueManagerTelemetryGuard:assert called_task is trace_taskassert trace_queue.empty()assert called_task.app_id == "expected-app-id"++[email protected]("enabled_at_start", [False, True])+def test_application_pipeline_snapshots_switch_without_constructing_sdk(+ trace_queue_manager_and_task, monkeypatch: pytest.MonkeyPatch, enabled_at_start: bool+):+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ trace_queue_manager, trace_task_class, trace_task_name = trace_queue_manager_and_task+ trace_queue = queue.Queue()+ calls: list[object] = []+ current_enabled = enabled_at_start++ def queue_enabled(app_id: object) -> bool:+ calls.append(app_id)+ return current_enabled++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "queue_enabled_at_start", staticmethod(queue_enabled))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)+ with (+ app.app_context(),+ patch("core.telemetry.gateway.is_enterprise_telemetry_enabled", return_value=False),+ patch("core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance") as legacy,+ patch("core.ops.ops_trace_manager.trace_manager_queue", trace_queue),+ ):+ manager = trace_queue_manager(app_id="app-id")+ current_enabled = not enabled_at_start+ task = trace_task_class(trace_type=trace_task_name.WORKFLOW_TRACE)+ manager.add_trace_task(task)+ legacy.assert_not_called()+ assert calls == ["app-id"]+ assert manager.trace_instance is None+ assert trace_queue.qsize() == int(enabled_at_start)diff --git a/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py b/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.pyindex 1402b82c34b..81166c68e6a 100644--- a/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py+++ b/api/tests/unit_tests/core/ops/test_workflow_tool_export_boundary.py@@ -4,6 +4,7 @@ import jsonfrom datetime import datetimeimport pytest+import requestsfrom sqlalchemy.orm import Sessionfrom core.ops.entities.trace_entity import WorkflowTraceInfo@@ -12,11 +13,20 @@ from extensions.ext_database import dbfrom models import Accountfrom models.enums import CreatorUserRolefrom models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from tests.unit_tests.services.tracing.test_mlflow_legacy_destination import (+ exported_spans,+)+from tests.unit_tests.services.tracing.test_mlflow_legacy_destination import (+ transport as transport, # noqa: PLC0414 - Reuse the real SDK's isolated requests boundary.+)@pytest.mark.parametrize("exporter", ["canonical", "mlflow"])def test_external_workflow_trace_omits_source_internals(- exporter: str, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch+ exporter: str,+ sqlite_session: Session,+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],) -> None:started_at = datetime(2025, 1, 1)public_inputs = {"query": "public tool input"}@@ -94,31 +104,28 @@ def test_external_workflow_trace_omits_source_internals(from dify_trace_mlflow.config import MLflowConfigfrom dify_trace_mlflow.mlflow_trace import MLflowDataTracefrom mlflow.entities import Span- from mlflow.tracing import provider- from mlflow.tracing.processor.mlflow_v3 import MlflowV3SpanProcessorfrom mlflow.tracing.trace_manager import InMemoryTraceManager- from opentelemetry.sdk.trace import TracerProvider- from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter- monkeypatch.setattr("dify_trace_mlflow.mlflow_trace.mlflow.set_tracking_uri", lambda _uri: None)- monkeypatch.setattr("dify_trace_mlflow.mlflow_trace.mlflow.set_experiment", lambda **_kwargs: None)- monkeypatch.setattr("mlflow.tracking.fluent._active_experiment_id", "0")monkeypatch.setattr(InMemoryTraceManager, "_instance", InMemoryTraceManager())- monkeypatch.setenv("MLFLOW_ENABLE_ASYNC_TRACE_LOGGING", "true")- span_exporter = InMemorySpanExporter()- tracer_provider = TracerProvider()- tracer_provider.add_span_processor(MlflowV3SpanProcessor(span_exporter, export_metrics=False))- monkeypatch.setattr(provider, "_get_tracer", tracer_provider.get_tracer)+ # OSS owns a private synchronous SDK pipeline. Inspect its real wire spans+ # instead of replacing the unrelated process-wide MLflow tracer.+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://workflow-boundary.example", experiment_id="0"))try:- MLflowDataTrace(MLflowConfig()).workflow_trace(info)- spans = [Span(span) for span in span_exporter.get_finished_spans()]+ instance.trace(info)+ spans = [Span.from_otel_proto(span) for span in exported_spans(transport)]assert len(spans) == 2payload = json.dumps([span.to_dict() for span in spans])tool = next(span for span in spans if span.name == "outer-tool")assert tool.inputs == public_inputsassert tool.outputs == public_outputs+ assert transport+ assert {request.url for request in transport} <= {+ "https://workflow-boundary.example/version",+ "https://workflow-boundary.example/api/3.0/mlflow/traces",+ "https://workflow-boundary.example/v1/traces",+ }finally:- tracer_provider.shutdown()+ instance._provider.shutdown()assert "private source" not in payloadassert "source-llm" not in payloadassert "recursive-llm" not in payloaddiff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.pyindex 5651da46785..3ad29c87207 100644--- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py+++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py@@ -10,6 +10,7 @@ from dify_agent.layers.dify_plugin import DifyPluginToolConfig, DifyPluginToolsLfrom dify_agent.layers.prompt import Config as PromptConfigfrom dify_agent.protocol import DIFY_AGENT_HISTORY_LAYER_ID, DIFY_AGENT_MODEL_LAYER_ID, DIFY_AGENT_OUTPUT_LAYER_IDfrom dify_agent.protocol.snapshot import SessionSnapshot+from flask import Flaskfrom clients.agent_backend import (DIFY_CONFIG_LAYER_ID,@@ -19,6 +20,7 @@ from clients.agent_backend import ()from clients.agent_backend.request_builder import DIFY_SHELL_LAYER_IDfrom core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom+from core.ops.base_trace_instance import bind_application_tracing_hooksfrom core.workflow.file_reference import build_file_referencefrom core.workflow.nodes.agent_v2.dify_tools_builder import WorkflowAgentDifyToolsBuilderfrom core.workflow.nodes.agent_v2.runtime_request_builder import (@@ -47,9 +49,15 @@ from models.agent_config_entities import (DeclaredOutputType,WorkflowNodeJobConfig,)+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooksfrom tests.unit_tests.config_override import apply_config_overrides[email protected](autouse=True)+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)++@pytest.fixture(autouse=True)def _no_runtime_agent_skills(monkeypatch: pytest.MonkeyPatch):monkeypatch.setattr(@@ -232,6 +240,42 @@ def _context() -> WorkflowAgentRuntimeBuildContext:)[email protected]("preview", [False, True])+def test_builder_passes_inline_or_preview_context_to_application_tracing(+ monkeypatch: pytest.MonkeyPatch, preview: bool+):+ from uuid import uuid4++ from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfig++ tracing = ApplicationTracingContext(+ run_id=uuid4(),+ invocation_id=uuid4(),+ trace_id="1" * 32,+ parent_span_id="2" * 16,+ session_id="session",+ capture_content=False,+ )+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ context = replace(_context(), workflow_run_id=None if preview else "run-1", attempt=2)+ calls: list[tuple[WorkflowAgentRuntimeBuildContext, str | None]] = []++ def prepare(value: WorkflowAgentRuntimeBuildContext, conversation_id: str | None) -> ApplicationTracingContext:+ calls.append((value, conversation_id))+ return tracing++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "prepare_workflow_agent", staticmethod(prepare))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)++ with app.app_context():+ result = WorkflowAgentRuntimeRequestBuilder(dify_tools_builder=WorkflowAgentDifyToolsBuilder()).build(context)+ layer = next(item for item in result.request.composition.layers if item.name == DIFY_EXECUTION_CONTEXT_LAYER_ID)+ assert DifyExecutionContextLayerConfig.model_validate(layer.config).tracing == tracing+ assert calls == [(context, None)]++def _request_layers(result) -> dict[str, dict[str, object]]:dumped = result.request.model_dump(mode="json")return {layer["name"]: layer for layer in dumped["composition"]["layers"]}diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.pyindex 32de236ce22..3bafee7497b 100644--- a/api/tests/unit_tests/extensions/test_ext_application_services.py+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py@@ -6,7 +6,7 @@ from collections.abc import Mappingfrom decimal import Decimalfrom io import BytesIOfrom types import SimpleNamespace-from typing import cast+from typing import Never, castfrom unittest.mock import MagicMock, call, create_autospec, patchfrom uuid import uuid4@@ -20,6 +20,7 @@ from sqlalchemy.orm import Session, sessionmakerfrom core.credit_usage import CreditUsageAppTypefrom core.llm_generator.llm_generator import LLMGeneratorfrom core.model_context import get_credit_usage_metadata, use_credit_usage_metadata+from core.ops.base_trace_instance import application_tracing_hooksfrom core.tools.tool_file_manager import ToolFileManagerfrom enums import DeploymentEdition, WebAppAccessModefrom extensions import ext_application_services@@ -151,6 +152,7 @@ from services.retention.workflow_run.archive_download_task_cache import Workflowfrom services.retention.workflow_run.archive_log_service import WorkflowRunArchiveServicefrom services.tag_application_service import TagApplicationServicefrom services.tool_file_download_service import ToolFileDownloadService+from services.tracing import runtime_bridgefrom services.upload_file_delivery_service import UploadFileDeliveryServicefrom services.web_authentication_adapters import (AccountWebAuthenticationSecurityGateway,@@ -269,6 +271,40 @@ def test_init_app_registers_services_for_the_current_app(assert creators._oauth is services.oauth_server+def test_init_app_registers_distinct_tracing_ports_without_constructing_trace_services(+ monkeypatch: pytest.MonkeyPatch,+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ first_app, second_app = Flask("first"), Flask("second")+ monkeypatch.setattr(ext_application_services, "get_session_maker", lambda: sqlite_session_factory)+ apply_config_overrides(+ monkeypatch,+ DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY,+ INIT_PASSWORD="",+ OPS_AGENT_TRACE_ENABLED=True,+ )+ construction_calls: list[bool] = []++ def construct_trace_service() -> Never:+ construction_calls.append(True)+ raise AssertionError("Composition must not construct a trace service")++ monkeypatch.setattr(runtime_bridge, "_service", construct_trace_service)++ ext_application_services.init_app(first_app)+ ext_application_services.init_app(second_app)++ with first_app.app_context():+ first_hooks = application_tracing_hooks()+ assert isinstance(first_hooks, runtime_bridge.RuntimeApplicationTracingHooks)+ with second_app.app_context():+ second_hooks = application_tracing_hooks()+ assert isinstance(second_hooks, runtime_bridge.RuntimeApplicationTracingHooks)+ assert second_hooks is not first_hooks+ assert application_tracing_hooks() is first_hooks+ assert construction_calls == []++def test_build_application_services_preserves_composed_boundaries(sqlite_session_factory: sessionmaker[Session],) -> None:diff --git a/api/tests/unit_tests/migrations/test_application_agent_tracing.py b/api/tests/unit_tests/migrations/test_application_agent_tracing.pynew file mode 100644index 00000000000..571ed2afb73--- /dev/null+++ b/api/tests/unit_tests/migrations/test_application_agent_tracing.py@@ -0,0 +1,69 @@+"""The migration can create and remove the three tables without business tables."""++import importlib.util+from pathlib import Path++import pytest+import sqlalchemy as sa+from alembic.migration import MigrationContext+from alembic.operations import Operations+from alembic.script import ScriptDirectory++from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun+++ ("starting_revision", "required_revisions"),+ [+ ("f1d3c5b7a902", ("a17c2980f3e6", "c6f4a90d8b31", "f7e2c9a6d481")),+ ("a17c2980f3e6", ("c6f4a90d8b31", "f7e2c9a6d481")),+ ("c6f4a90d8b31", ("a17c2980f3e6", "f7e2c9a6d481")),+ ],+)+def test_tracing_and_workflow_pause_upgrade_paths_join(+ starting_revision: str, required_revisions: tuple[str, ...]+) -> None:+ scripts = ScriptDirectory(str(Path(__file__).resolve().parents[3] / "migrations"))+ merge = scripts.get_revision("f7e2c9a6d481")+ assert merge is not None+ assert isinstance(merge.down_revision, tuple)+ assert set(merge.down_revision) == {"a17c2980f3e6", "c6f4a90d8b31"}++ # Use Alembic's upgrade planner so either already-applied branch still runs+ # its missing sibling before the merge, without rewriting either migration.+ steps = scripts._upgrade_revs(merge.revision, starting_revision)+ assert {step.revision.revision for step in steps} == set(required_revisions)+ assert steps[-1].revision.revision == merge.revision+++def test_application_trace_migration_matches_models_and_roundtrips() -> None:+ path = (+ Path(__file__).resolve().parents[3]+ / "migrations/versions/2026_10_15_0330-a17c2980f3e6_add_application_agent_tracing.py"+ )+ spec = importlib.util.spec_from_file_location("application_trace_migration", path)+ assert spec is not None+ assert spec.loader is not None+ migration = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(migration)+ assert migration.down_revision == "f1d3c5b7a902"+ engine = sa.create_engine("sqlite://")+ try:+ with engine.begin() as connection:+ with Operations.context(MigrationContext.configure(connection)):+ migration.upgrade()+ inspector = sa.inspect(connection)+ for model in (ApplicationTraceRun, ApplicationTraceInvocation, ApplicationTraceBatch):+ actual = {column["name"]: column for column in inspector.get_columns(model.__tablename__)}+ assert set(actual) == set(model.__table__.columns.keys())+ for column in model.__table__.columns:+ assert actual[column.name]["nullable"] == column.nullable+ assert {index["name"] for index in inspector.get_indexes("application_trace_runs")} == {+ "application_trace_delivery_idx",+ "application_trace_retention_idx",+ }+ with Operations.context(MigrationContext.configure(connection)):+ migration.downgrade()+ assert sa.inspect(connection).get_table_names() == []+ finally:+ engine.dispose()diff --git a/api/tests/unit_tests/models/test_agent_config_entities.py b/api/tests/unit_tests/models/test_agent_config_entities.pyindex dbd6f84c30e..6a3b816887f 100644--- a/api/tests/unit_tests/models/test_agent_config_entities.py+++ b/api/tests/unit_tests/models/test_agent_config_entities.py@@ -1,4 +1,5 @@import pytest+from pydantic import ValidationErrorfrom core.workflow.file_reference import build_file_referencefrom models.agent_config_entities import (@@ -28,6 +29,30 @@ def test_agent_soul_model_settings_preserves_plugin_declared_parameters() -> Nonassert dumped["thinking_budget"] == 4096+ "response_format",+ [+ "text",+ "json_schema",+ {"type": "text"},+ {"type": "json_schema", "json_schema": {"name": "response", "schema": {"type": "object"}}},+ None,+ ],+)+def test_agent_model_response_format_preserves_plugin_and_legacy_shapes(response_format: object) -> None:+ settings = AgentSoulModelSettings.model_validate({"response_format": response_format})++ assert settings.model_dump(mode="json", exclude_none=True) == (+ {"response_format": response_format} if response_format is not None else {}+ )++[email protected]("response_format", [[], ["text"], 1, True])+def test_agent_model_response_format_rejects_non_string_or_object_values(response_format: object) -> None:+ with pytest.raises(ValidationError, match="response_format"):+ AgentSoulModelSettings.model_validate({"response_format": response_format})++def test_file_default_value_accepts_canonical_reference_mapping() -> None:reference = build_file_reference(record_id="tool-file-1")diff --git a/api/tests/unit_tests/models/test_message_trace_serialization.py b/api/tests/unit_tests/models/test_message_trace_serialization.pynew file mode 100644index 00000000000..755dbf158eb--- /dev/null+++ b/api/tests/unit_tests/models/test_message_trace_serialization.py@@ -0,0 +1,161 @@+"""Keep the persisted provider through private trace JSON and the real consumer."""++import json+from datetime import UTC, datetime, timedelta+from decimal import Decimal+from uuid import uuid4++import pytest+from dify_trace_arize_phoenix import arize_phoenix_trace+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace+from dify_trace_arize_phoenix.config import ArizeConfig+from openinference.semconv.trace import SpanAttributes+from opentelemetry.sdk.trace import Tracer, TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from pydantic import TypeAdapter+from sqlalchemy.orm import Session++from core.ops.entities.trace_entity import MessageTraceInfo+from core.ops.ops_trace_manager import OpsTraceManager, TaskData+from models.enums import ConversationFromSource+from models.model import Message, MessageDict+from tasks import ops_trace_task+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_message++_MESSAGE = TypeAdapter(MessageDict)+++def persisted_message(sqlite_session: Session) -> Message:+ now = datetime(2026, 10, 20, tzinfo=UTC)+ message = make_message(+ message_id=str(uuid4()),+ app_id=str(uuid4()),+ conversation_id=str(uuid4()),+ inputs={},+ query="Synthetic input",+ message={},+ answer="Synthetic output",+ message_unit_price=Decimal(0),+ answer_unit_price=Decimal(0),+ total_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.CONSOLE,+ created_at=now,+ updated_at=now + timedelta(seconds=1),+ )+ message.model_provider = "langgenius/openai/openai"+ message.model_id = "gpt-4o-mini"+ sqlite_session.add(message)+ sqlite_session.commit()+ sqlite_session.expire_all()+ restored = sqlite_session.get(Message, message.id)+ assert restored is not None+ return restored++[email protected]("provider", ["langgenius/openai/openai", "Custom/Provider:Exact", None])+def test_message_provider_survives_real_json_round_trip(+ sqlite_session: Session,+ persisted_message: Message,+ provider: str | None,+) -> None:+ persisted_message.model_provider = provider+ sqlite_session.commit()+ sqlite_session.refresh(persisted_message)+ wire = json.dumps(persisted_message.to_dict(session=sqlite_session), default=str)+ payload = _MESSAGE.validate_json(wire)+ restored = Message.from_dict(payload)+ assert "model_provider" in payload+ assert restored.model_provider == provider+ assert restored.model_id == persisted_message.model_id+ assert restored.id == persisted_message.id++[email protected]("shape", ["current", "legacy_missing", "explicit_null"])+def test_private_trace_task_preserves_provider_into_real_arize_span(+ sqlite_session: Session,+ persisted_message: Message,+ monkeypatch: pytest.MonkeyPatch,+ shape: str,+) -> None:+ message_data = persisted_message.to_dict(session=sqlite_session)+ if shape == "legacy_missing":+ message_data.pop("model_provider", None)+ elif shape == "explicit_null":+ message_data["model_provider"] = None+ start = datetime(2026, 10, 20, tzinfo=UTC)+ info = MessageTraceInfo(+ message_id=persisted_message.id,+ message_data=message_data,+ conversation_model="chat",+ conversation_mode="chat",+ inputs={"query": "Synthetic input"},+ outputs="Synthetic output",+ start_time=start,+ end_time=start + timedelta(seconds=1),+ metadata={},+ message_tokens=1,+ answer_tokens=1,+ total_tokens=2,+ )+ encoded = (+ TaskData(app_id=persisted_message.app_id, trace_info_type="MessageTraceInfo", trace_info=info.model_dump())+ .model_dump_json()+ .encode()+ )+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ processor = SimpleSpanProcessor(exporter)+ provider.add_span_processor(processor)++ def setup(_config: ArizeConfig) -> tuple[Tracer, SimpleSpanProcessor]:+ tracer = provider.get_tracer("message-provider-serialization")+ assert isinstance(tracer, Tracer)+ return tracer, processor++ monkeypatch.setattr(arize_phoenix_trace, "setup_tracer", setup)+ instance = ArizePhoenixDataTrace(ArizeConfig(api_key="unused-synthetic-key", space_id="synthetic-space"))+ paths: list[str] = []+ deleted: list[str] = []++ def load(path: str) -> bytes:+ paths.append(path)+ return encoded++ def delete(path: str) -> None:+ deleted.append(path)++ def lookup(app_id: str) -> ArizePhoenixDataTrace:+ assert app_id == persisted_message.app_id+ return instance++ def failed(_key: str) -> None:+ pytest.fail("The real trace consumer must not hide a failed dispatch")++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)+ monkeypatch.setattr(ops_trace_task.storage, "load", load)+ monkeypatch.setattr(ops_trace_task.storage, "delete", delete)+ monkeypatch.setattr(ops_trace_task.redis_client, "incr", failed)+ monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lookup)+ monkeypatch.setattr("extensions.ext_enterprise_telemetry.is_enabled", lambda: False)+ try:+ ops_trace_task.process_trace_tasks.apply(+ args=({"app_id": persisted_message.app_id, "file_id": "owned-synthetic"},), throw=True+ ).get()+ spans = exporter.get_finished_spans()+ llm = [span for span in spans if span.name == "llm"]+ assert len(llm) == 1+ attributes = llm[0].attributes+ assert attributes is not None+ assert attributes[SpanAttributes.LLM_MODEL_NAME] == persisted_message.model_id+ if shape == "current":+ assert attributes[SpanAttributes.LLM_PROVIDER] == persisted_message.model_provider+ else:+ assert SpanAttributes.LLM_PROVIDER not in attributes+ assert paths == [f"ops_trace/{persisted_message.app_id}/owned-synthetic.json"]+ assert deleted == paths+ finally:+ provider.shutdown()diff --git a/api/tests/unit_tests/repositories/app/test_agent_app_repository.py b/api/tests/unit_tests/repositories/app/test_agent_app_repository.pyindex f123e8a54c6..aa9300f2001 100644--- a/api/tests/unit_tests/repositories/app/test_agent_app_repository.py+++ b/api/tests/unit_tests/repositories/app/test_agent_app_repository.py@@ -11,10 +11,23 @@ from models.agent import Agent, AgentKind, AgentScope, AgentSource, AgentStatusfrom models.model import App, AppModelConfigfrom repositories.app.agent_app_repository import AgentAppRepositoryfrom services.agent.roster_service import AgentRosterService+from services.app.agent_app_contracts import AgentAppNotFoundErrortype Resolver = Literal["repository", "roster"][email protected]("_inline_agent")+def test_tracing_owner_rejects_inline_without_materializing_app(sqlite_session_factory: sessionmaker[Session]) -> None:+ repository = AgentAppRepository(session_factory=sqlite_session_factory)+ with pytest.raises(AgentAppNotFoundError):+ repository.resolve_roster_app_id(tenant_id="tenant-1", agent_id="inline-agent")+ with sqlite_session_factory() as session:+ assert session.scalar(select(App)) is None+ agent = session.get(Agent, "inline-agent")+ assert agent is not None+ assert agent.backing_app_id is None++@pytest.fixturedef _inline_agent(sqlite_session_factory: sessionmaker[Session]) -> None:with sqlite_session_factory.begin() as session:diff --git a/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py b/api/tests/unit_tests/repositories/app/test_tracing_config_repository.pyindex abb31de5b28..4c94bde58b1 100644--- a/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py+++ b/api/tests/unit_tests/repositories/app/test_tracing_config_repository.py@@ -1,4 +1,5 @@from collections.abc import Callable+from datetime import datetimeimport pytestfrom sqlalchemy import select@@ -6,7 +7,12 @@ from sqlalchemy.orm import Session, sessionmakerfrom models.model import App, AppMode, TraceAppConfigfrom repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository-from services.app_tracing_config_service import AppTracingConfigAppNotFoundError, AppTracingConfigRecord+from services.app_tracing_config_service import (+ AppTraceSettings,+ AppTracingConfigAppNotFoundError,+ AppTracingConfigConflictError,+ AppTracingConfigRecord,+)from tests.unit_tests.repositories.app.console_visibility import UNADDRESSABLE_IN_WORKSPACE, MakeUnaddressable_APP_ID = "11111111-1111-1111-1111-111111111111"@@ -69,6 +75,7 @@ def test_config_lifecycle_is_persisted_by_owned_transactions(app_id=_APP_ID,tracing_provider=_PROVIDER,tracing_config={"public_key": "updated"},+ expected=record,)with sqlite_session_factory() as session:config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == _APP_ID))@@ -76,12 +83,14 @@ def test_config_lifecycle_is_persisted_by_owned_transactions(assert config.tracing_config == {"public_key": "updated"}assert repository.delete(workspace_id=_WORKSPACE_ID, app_id=_APP_ID, tracing_provider=_PROVIDER)- assert not repository.update(- workspace_id=_WORKSPACE_ID,- app_id=_APP_ID,- tracing_provider=_PROVIDER,- tracing_config={"public_key": "missing"},- )+ with pytest.raises(AppTracingConfigConflictError):+ repository.update(+ workspace_id=_WORKSPACE_ID,+ app_id=_APP_ID,+ tracing_provider=_PROVIDER,+ tracing_config={"public_key": "missing"},+ expected=record,+ )assert not repository.delete(workspace_id=_WORKSPACE_ID, app_id=_APP_ID, tracing_provider=_PROVIDER)with sqlite_session_factory() as session:assert session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == _APP_ID)) is None@@ -106,6 +115,9 @@ def test_all_operations_reject_apps_console_cannot_address(repository = _repository(sqlite_session_factory)operations: tuple[Callable[[], object], ...] = (lambda: repository.get(workspace_id=workspace_id, app_id=_APP_ID, tracing_provider=_PROVIDER),+ lambda: repository.set_trace(+ workspace_id=workspace_id, app_id=_APP_ID, settings=AppTraceSettings(), expected=None+ ),lambda: repository.create(workspace_id=workspace_id,app_id=_APP_ID,@@ -117,6 +129,7 @@ def test_all_operations_reject_apps_console_cannot_address(app_id=_APP_ID,tracing_provider=_PROVIDER,tracing_config={},+ expected=AppTracingConfigRecord("config", _APP_ID, _PROVIDER, {}, True, datetime.now(), datetime.now()),),lambda: repository.delete(workspace_id=workspace_id, app_id=_APP_ID, tracing_provider=_PROVIDER),)diff --git a/api/tests/unit_tests/repositories/app/test_tracing_settings_repository.py b/api/tests/unit_tests/repositories/app/test_tracing_settings_repository.pynew file mode 100644index 00000000000..e76f69e4233--- /dev/null+++ b/api/tests/unit_tests/repositories/app/test_tracing_settings_repository.py@@ -0,0 +1,51 @@+"""Tracing policy round-trips without dropping content or Agent capture choices."""++import json++import pytest+from sqlalchemy.orm import Session, sessionmaker++from machinery.context import RequestContext+from models.model import App+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigAppNotFoundError+++def test_capture_policy_round_trip_and_legacy_defaults(sqlite_session_factory: sessionmaker[Session]) -> None:+ context = RequestContext("request", None, "account", "tenant")+ with sqlite_session_factory.begin() as session:+ session.add(+ App(+ id="app",+ tenant_id="tenant",+ name="Trace app",+ mode="workflow",+ enable_site=False,+ enable_api=False,+ tracing=json.dumps({"enabled": True, "tracing_provider": "langfuse"}),+ )+ )+ repository = ConsoleAppRepository(session_factory=sqlite_session_factory)+ assert repository.get_trace(context, "app") == AppTraceSettings(True, "langfuse", False, True)+ writer = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ writer.create(+ workspace_id="tenant", app_id="app", tracing_provider="langfuse", tracing_config={"secret": "encrypted"}+ )+ expected = writer.get(workspace_id="tenant", app_id="app", tracing_provider="langfuse")+ policy = AppTraceSettings(True, "langfuse", True, False)+ writer.set_trace(workspace_id="tenant", app_id="app", settings=policy, expected=expected)+ assert repository.get_trace(context, "app") == policy+ with sqlite_session_factory() as session:+ app = session.get(App, "app")+ assert app is not None+ assert app.tracing is not None+ assert json.loads(app.tracing) == {+ "enabled": True,+ "tracing_provider": "langfuse",+ "capture_content": True,+ "capture_agent": False,+ }+ with pytest.raises(AppTracingConfigAppNotFoundError):+ writer.set_trace(workspace_id="other-tenant", app_id="app", settings=AppTraceSettings(), expected=None)+ assert repository.get_trace(context, "app") == policydiff --git a/api/tests/unit_tests/repositories/test_console_app_repository.py b/api/tests/unit_tests/repositories/test_console_app_repository.pyindex ce6b3d6580b..44f12f5ccc6 100644--- a/api/tests/unit_tests/repositories/test_console_app_repository.py+++ b/api/tests/unit_tests/repositories/test_console_app_repository.py@@ -18,10 +18,10 @@ from models.model import App, AppMode, AppModelConfig, AppStar, IconType, Sitefrom models.workflow import Workflow, WorkflowTypefrom repositories.app.console_repository import ConsoleAppRepositoryfrom services.app.console_service import ConsoleAppNotFoundError+from services.app_tracing_config_service import AppTraceSettingsfrom services.entities.app_entities import (AppListParams,AppRecord,- AppTraceSettings,StarredAppListParams,UpdateAppParams,)@@ -160,7 +160,6 @@ def test_authorization_maintainer_requires_normal_app(lambda repo, ctx, app_id: repo.rename(ctx, app_id, "Renamed"),lambda repo, ctx, app_id: repo.set_starred(ctx, app_id, True),lambda repo, ctx, app_id: repo.get_trace(ctx, app_id),- lambda repo, ctx, app_id: repo.set_trace(ctx, app_id, AppTraceSettings(True, "langfuse")),lambda repo, ctx, app_id: repo.get_reference(ctx, app_id),],)@@ -332,14 +331,9 @@ def test_openapi_visible_app_list_only_includes_apps_with_api_enabled(assert page.total == 1-def test_trace_settings_commit_and_default(repository: ConsoleAppRepository, sqlite_session: Session) -> None:+def test_trace_settings_default(repository: ConsoleAppRepository, sqlite_session: Session) -> None:app = persist_app(sqlite_session)assert repository.get_trace(CONTEXT, app.id) == AppTraceSettings()- settings = AppTraceSettings(True, "langfuse")- repository.set_trace(CONTEXT, app.id, settings)- assert repository.get_trace(CONTEXT, app.id) == settings- repository.set_trace(CONTEXT, app.id, AppTraceSettings())- assert repository.get_trace(CONTEXT, app.id) == AppTraceSettings()def test_trace_settings_preserve_stored_json_validation(diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.pyindex afd0c71be99..7e0f00317f4 100644--- a/api/tests/unit_tests/services/app/test_console_service.py+++ b/api/tests/unit_tests/services/app/test_console_service.py@@ -39,6 +39,7 @@ from services.app.console_service import (CreatorsPlatformDisabledError,InvalidAppExportError,)+from services.app_tracing_config_service import AppTraceSettingsfrom services.entities.app_entities import (AppCreationSettings,AppDeletion,@@ -47,7 +48,6 @@ from services.entities.app_entities import (AppPage,AppRecord,AppReference,- AppTraceSettings,CopyAppParams,CreateAppParams,ImportedAppPackage,@@ -254,13 +254,11 @@ class Creators:class Tracing:- def validate_provider(self, tracing_provider: str) -> None:- if tracing_provider != "langfuse":+ def set_trace(self, context: RequestContext, app_id: str, settings: AppTraceSettings) -> None:+ del context, app_id+ if settings.tracing_provider not in (None, "langfuse"):raise ValueError("Invalid tracing provider")- def require_provider_available(self, tracing_provider: str) -> None:- self.validate_provider(tracing_provider)-Ports = tuple[ConsoleAppService, Apps, Access, Transfers, Creators]@@ -464,7 +462,7 @@ def test_trace_update_checks_ownership_and_provider_without_parsing_old_settingsassert apps.calls == [("get_reference", CONTEXT, "app")]else:service.set_trace(CONTEXT, "app", settings)- assert apps.calls == [("get_reference", CONTEXT, "app"), ("set_trace", CONTEXT, "app", settings)]+ assert apps.calls == [("get_reference", CONTEXT, "app")]class Lifecycle:diff --git a/api/tests/unit_tests/services/test_agent_tracing_config_service.py b/api/tests/unit_tests/services/test_agent_tracing_config_service.pynew file mode 100644index 00000000000..f1c4953bc04--- /dev/null+++ b/api/tests/unit_tests/services/test_agent_tracing_config_service.py@@ -0,0 +1,88 @@+"""Agent tracing resolves the complete owner chain before existing App operations."""++from collections.abc import Callable++import pytest+from sqlalchemy.orm import Session, sessionmaker++from extensions.application_services.app import AppServices+from machinery.context import RequestContext+from repositories.app.agent_app_repository import AgentAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.agent_tracing_config_service import AgentTracingConfigService+from services.app.agent_app_contracts import AgentAppNotFoundError+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTraceSettings, AppTracingConfigService++type TraceConfigArguments = tuple[()] | tuple[AppTraceSettings] | tuple[str] | tuple[str, dict[str, str]]+type RecordedCall = tuple[str, tuple[object, ...]]++CONTEXT = RequestContext("request", None, "account", "tenant")+++def invoke(service: AgentTracingConfigService, method: str) -> TraceConfigArguments:+ settings = AppTraceSettings(True, "langfuse", capture_content=True, capture_agent=False)+ config = {"secret_key": "masked"}+ if method == "get_trace":+ service.get_trace(CONTEXT, "agent")+ return ()+ if method == "set_trace":+ service.set_trace(CONTEXT, "agent", settings)+ return (settings,)+ if method == "get":+ service.get(CONTEXT, "agent", "langfuse")+ return ("langfuse",)+ if method == "delete":+ service.delete(CONTEXT, "agent", "langfuse")+ return ("langfuse",)+ if method == "create":+ service.create(CONTEXT, "agent", "langfuse", config)+ else:+ assert method == "update"+ service.update(CONTEXT, "agent", "langfuse", config)+ return ("langfuse", config)++[email protected]("method", ["get_trace", "set_trace", "get", "create", "update", "delete"])[email protected]("valid_owner", [True, False])+def test_agent_operations_resolve_tenant_owner_before_app_services(+ method: str,+ valid_owner: bool,+ sqlite_session_factory: sessionmaker[Session],+ app_services: AppServices,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ owners = AgentAppRepository(session_factory=sqlite_session_factory)+ settings = app_services.console+ configs = AppTracingConfigService(+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory),+ provider=OpsTraceManagerGateway(),+ )+ calls: list[RecordedCall] = []++ def resolve(*, tenant_id: str, agent_id: str) -> str:+ calls.append(("resolve", (tenant_id, agent_id)))+ if not valid_owner:+ raise AgentAppNotFoundError+ return "backing-app"++ def recorder(name: str) -> Callable[..., None]:+ def record(*args: object, **kwargs: object) -> None:+ assert kwargs == ({"activate": False} if name in ("create", "update") else {})+ calls.append((name, args))++ return record++ monkeypatch.setattr(owners, "resolve_roster_app_id", resolve)+ for name in ("get_trace", "set_trace"):+ monkeypatch.setattr(settings, name, recorder(name))+ for name in ("get", "create", "update", "delete"):+ monkeypatch.setattr(configs, name, recorder(name))+ service = AgentTracingConfigService(owners=owners, settings=settings, configs=configs)+ if not valid_owner:+ with pytest.raises(AgentAppNotFoundError):+ invoke(service, method)+ assert calls == [("resolve", ("tenant", "agent"))]+ return+ args = invoke(service, method)+ assert calls == [("resolve", ("tenant", "agent")), (method, (CONTEXT, "backing-app", *args))]diff --git a/api/tests/unit_tests/services/test_aliyun_tracing_config_gateway.py b/api/tests/unit_tests/services/test_aliyun_tracing_config_gateway.pynew file mode 100644index 00000000000..b8dc0fca5e1--- /dev/null+++ b/api/tests/unit_tests/services/test_aliyun_tracing_config_gateway.py@@ -0,0 +1,265 @@+"""Real RSA/SQL/controller Aliyun configuration through a recorded HTTP boundary."""++import inspect+import json+import traceback+from dataclasses import dataclass, field, replace+from typing import Never+from urllib.parse import quote+from uuid import UUID, uuid4++import httpx+import pytest+from dify_trace_aliyun.aliyun_trace import AliyunDataTrace+from flask import Flask+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+from sqlalchemy.orm import Session, sessionmaker++from controllers.console.app import ops_trace as controller+from controllers.console.app.error import InvalidTracingConfigError+from core.helper import ssrf_proxy+from core.ops.ops_trace_manager import OpsTraceManager+from extensions.ext_application_services import ApplicationServices+from machinery.context import RequestContext+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import (+ AppTracingConfigInvalidConfigurationError,+ AppTracingConfigRecord,+ AppTracingConfigService,+ AppTracingConfigVerificationFailedError,+)+from tests.unit_tests.model_factories import make_app++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++@dataclass+class AliyunCase:+ gateway: OpsTraceManagerGateway+ service: AppTracingConfigService+ store: SQLAlchemyAppTracingConfigRepository+ reader: ConsoleAppRepository+ context: RequestContext+ app_id: str+ verification_calls: list[tuple[str, dict[str, object]]] = field(default_factory=list)+ verification_succeeds: bool = True++ def read(self, provider: str = "aliyun") -> AppTracingConfigRecord | None:+ return self.store.get(+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider+ )+++def case(workspace: str, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch) -> AliyunCase:+ app_id = str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ tracing=json.dumps(+ {+ "enabled": True,+ "tracing_provider": "langsmith",+ "capture_content": True,+ "capture_agent": False,+ }+ ),+ )+ )+ gateway = OpsTraceManagerGateway()+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ old = OpsTraceManager.encrypt_tracing_config(+ workspace, "langsmith", {"api_key": "synthetic-other", "project": "other", "endpoint": "https://other.test"}+ )+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=old)+ result = AliyunCase(+ gateway,+ AppTracingConfigService(configs=store, provider=gateway),+ store,+ ConsoleAppRepository(session_factory=sqlite_session_factory),+ RequestContext("request", None, "account", workspace),+ app_id,+ )++ def verification_boundary(method: str, url: str, **options: object) -> httpx.Response:+ assert method == "POST"+ content = options["content"]+ assert isinstance(content, bytes)+ payload = ExportTraceServiceRequest.FromString(content)+ assert len(payload.resource_spans) == 1+ assert len(payload.resource_spans[0].scope_spans) == 0+ assert [attribute.key for attribute in payload.resource_spans[0].resource.attributes] == ["service.name"]+ result.verification_calls.append((url, dict(options)))+ status = 200 if result.verification_succeeds else 403+ return httpx.Response(status, content=b'"success"', request=httpx.Request(method, url))++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("Configuration must not initialize a legacy exporter or fetch its console link")++ monkeypatch.setattr(ssrf_proxy, "make_request", verification_boundary)+ monkeypatch.setattr(OpsTraceManager, "check_trace_config_is_effective", forbidden)+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)+ monkeypatch.setattr(AliyunDataTrace, "__init__", forbidden)+ return result+++def test_post_masked_patch_and_get_preserve_real_ciphertext_and_policy(case: AliyunCase) -> None:+ other = case.read("langsmith")+ submitted = {+ "endpoint": " HTTP://TRACING-ANALYSIS-DC-QD.ALIYUNCS.COM/reverse/ ",+ "app_name": " 中文 app_Name ",+ "license_key": "literal%2F@synthetic",+ }+ original = dict(submitted)+ case.service.create(case.context, case.app_id, "aliyun", submitted, activate=True)+ before = case.read()+ assert before is not None+ assert before.tracing_config is not None+ stored = before.tracing_config+ assert submitted == original+ assert stored["endpoint"] == "http://tracing-analysis-dc-qd.aliyuncs.com/reverse"+ assert stored["app_name"] == "中文 app_Name"+ assert stored["license_key"] != submitted["license_key"]+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "aliyun", stored)[+ "license_key"+ ] == ("literal%2F@synthetic")+ shown = case.service.get(case.context, case.app_id, "aliyun")+ assert shown is not None+ assert shown.tracing_config is not None+ masked = shown.tracing_config["license_key"]+ assert isinstance(masked, str)+ assert "*" in masked+ assert masked != submitted["license_key"]+ assert shown.tracing_config["project_url"] == "https://arms.console.aliyun.com/#/llm"+ assert len(case.verification_calls) == 1+ case.service.update(+ case.context,+ case.app_id,+ "aliyun",+ {+ "endpoint": "https://project.region.log.aliyuncs.com/prefix/",+ "app_name": " second_app ",+ "license_key": masked,+ },+ )+ after = case.read()+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["license_key"] == stored["license_key"]+ assert after.tracing_config["app_name"] == "second_app"+ assert after.tracing_config["endpoint"] == "https://project.region.log.aliyuncs.com/prefix"+ assert case.verification_calls[-1][0] == (+ "https://project.region.log.aliyuncs.com/prefix/adapt_literal%252F@synthetic/api/v1/traces"+ )+ assert case.service.get(case.context, case.app_id, "aliyun") is not None+ assert len(case.verification_calls) == 2+ assert case.read("langsmith") == other+ policy = case.reader.get_trace(case.context, case.app_id)+ assert policy.tracing_provider == "aliyun"+ assert policy.enabled+ assert policy.capture_content is True+ assert policy.capture_agent is False++[email protected]("update", [False, True])[email protected]("failure", ["invalid-schema", "verification-failed"])+def test_failed_save_preserves_rows_and_active_policy(case: AliyunCase, update: bool, failure: str) -> None:+ original = {"endpoint": "https://host.test", "app_name": "original", "license_key": "synthetic-secret"}+ if update:+ case.service.create(case.context, case.app_id, "aliyun", original)+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)+ submitted = {**original, "app_name": "changed"}+ if failure == "invalid-schema":+ submitted["endpoint"] = "https://host.test/adapt_DO-NOT-PERSIST/api/otlp/traces"+ expected = AppTracingConfigInvalidConfigurationError+ else:+ case.verification_succeeds = False+ expected = AppTracingConfigVerificationFailedError+ save = case.service.update if update else case.service.create+ if update:+ submitted["license_key"] = "******"+ with pytest.raises(expected):+ save(case.context, case.app_id, "aliyun", submitted, activate=True)+ assert case.read() == before+ assert case.read("langsmith") == other+ assert case.reader.get_trace(case.context, case.app_id) == policy++[email protected]("cached_url", [None, "https://wrong.test/stale"])+def test_historical_get_is_pure_even_for_a_now_invalid_ingestion_url(case: AliyunCase, cached_url: str | None) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "aliyun",+ {+ "endpoint": "https://host.test/adapt_old/api/otlp/traces",+ "app_name": "historical_name",+ "license_key": "synthetic-key",+ },+ )+ if cached_url:+ stored["project_url"] = cached_url+ before = dict(stored)+ result = case.gateway.present_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="aliyun", tracing_config=stored+ )+ assert result["project_url"] == "https://arms.console.aliyun.com/#/llm"+ assert isinstance(result["license_key"], str)+ assert "*" in result["license_key"]+ assert stored == before+ assert case.verification_calls == []++[email protected]("update", [False, True])[email protected]("encoded", [False, True])+def test_controller_rejects_embedded_license_without_echo_or_state_change(+ case: AliyunCase,+ app: Flask,+ account_application_services: ApplicationServices,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ update: bool,+ encoded: bool,+) -> None:+ old_key = "SYNTHETIC_OLD_LICENSE"+ original = {"endpoint": "https://host.test", "app_name": "old-app", "license_key": old_key}+ if update:+ case.service.create(case.context, case.app_id, "aliyun", original, activate=True)+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)+ before_checks = len(case.verification_calls)+ services = replace(account_application_services, app_tracing_configs=case.service)+ monkeypatch.setattr(controller, "application_services", lambda: services)+ key = "SYNTHETIC_PRIVATE_LICENSE@+/%2F"+ url_key = quote(key, safe="") if encoded else key+ payload = controller.TraceConfigPayload(+ tracing_provider="aliyun",+ tracing_config={+ "endpoint": f"https://host.test/adapt_{url_key}/api/otlp/traces",+ "app_name": "should-not-save",+ "license_key": "******" if update else key,+ },+ activate=True,+ )+ save = inspect.unwrap(controller.TraceAppConfigApi.patch if update else controller.TraceAppConfigApi.post)+ with app.test_request_context():+ with pytest.raises(InvalidTracingConfigError) as caught:+ save(controller.TraceAppConfigApi(), payload, case.context, UUID(case.app_id))+ # The real controller exposes this fixed BaseHTTPException body. Neither+ # Pydantic inputs nor a credential-bearing exception chain reaches it.+ assert caught.value.data == {+ "code": "invalid_tracing_config",+ "message": "The tracing configuration is invalid.",+ "status": 400,+ }+ rendered = json.dumps(caught.value.data) + str(caught.value) + repr(caught.value)+ rendered += "".join(traceback.format_exception(caught.value)) + caplog.text+ assert key not in rendered+ assert quote(key, safe="") not in rendered+ assert old_key not in rendered+ assert case.read() == before+ assert case.read("langsmith") == other+ assert case.reader.get_trace(case.context, case.app_id) == policy+ assert len(case.verification_calls) == before_checksdiff --git a/api/tests/unit_tests/services/test_app_tracing_atomic_operations.py b/api/tests/unit_tests/services/test_app_tracing_atomic_operations.pynew file mode 100644index 00000000000..092bec779b9--- /dev/null+++ b/api/tests/unit_tests/services/test_app_tracing_atomic_operations.py@@ -0,0 +1,225 @@+"""Public tracing operations keep credentials and policy in one transaction."""++import json+from dataclasses import dataclass++import pytest+from sqlalchemy import event, select+from sqlalchemy.orm import Session, sessionmaker++from machinery.context import RequestContext+from models.model import App, TraceAppConfig+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import (+ AppTraceSettings,+ AppTracingConfigConflictError,+ AppTracingConfigNotFoundError,+ AppTracingConfigRecord,+ AppTracingConfigService,+ AppTracingConfigVerificationFailedError,+)++CONTEXT = RequestContext("request", None, "account", "tenant")+++@dataclass+class Case:+ sessions: sessionmaker[Session]+ store: SQLAlchemyAppTracingConfigRepository+ service: AppTracingConfigService+ gateway: OpsTraceManagerGateway+ reader: ConsoleAppRepository++ def policy(self) -> AppTraceSettings:+ return self.reader.get_trace(CONTEXT, "app")++ def config(self, provider: str = "langfuse") -> AppTracingConfigRecord | None:+ return self.store.get(workspace_id="tenant", app_id="app", tracing_provider=provider)+++def case(sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch) -> Case:+ with sqlite_session_factory.begin() as session:+ session.add(+ App(+ id="app",+ tenant_id="tenant",+ name="Trace",+ mode="workflow",+ enable_site=False,+ enable_api=False,+ tracing=json.dumps(+ {"enabled": False, "tracing_provider": None, "capture_content": True, "capture_agent": False}+ ),+ )+ )+ gateway = OpsTraceManagerGateway()++ def prepare(*, tracing_config: dict[str, object], **_: object) -> dict[str, object]:+ return dict(tracing_config)++ monkeypatch.setattr(gateway, "prepare_new_config", prepare)+ monkeypatch.setattr(gateway, "prepare_updated_config", prepare)+ monkeypatch.setattr(gateway, "validate_saved_config", lambda **_: None)+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ return Case(+ sqlite_session_factory,+ store,+ AppTracingConfigService(configs=store, provider=gateway),+ gateway,+ ConsoleAppRepository(session_factory=sqlite_session_factory),+ )++[email protected]("activate", [False, True])+def test_create_and_update_preserve_capture_and_explicit_activation(case: Case, activate: bool) -> None:+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"}, activate=activate)+ assert case.policy() == AppTraceSettings(activate, "langfuse" if activate else None, True, False)+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "updated"}, activate=True)+ assert case.policy() == AppTraceSettings(True, "langfuse", True, False)+ record = case.config()+ assert record is not None+ assert record.tracing_config == {"secret": "updated"}+++def test_save_activation_uses_capture_policy_changed_during_vendor_validation(+ case: Case, monkeypatch: pytest.MonkeyPatch+) -> None:+ def prepare(**_: object) -> dict[str, object]:+ case.service.set_trace(CONTEXT, "app", AppTraceSettings(False, None, False, True))+ return {"secret": "validated"}++ monkeypatch.setattr(case.gateway, "prepare_new_config", prepare)+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "input"}, activate=True)+ assert case.policy() == AppTraceSettings(True, "langfuse", False, True)++[email protected]("operation", ["create", "update", "delete"])+def test_transaction_failure_rolls_back_credentials_and_policy(case: Case, operation: str) -> None:+ if operation != "create":+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"}, activate=True)+ before_policy, before_record = case.policy(), case.config()++ def fail_commit(session: Session) -> None:+ session.flush()+ raise RuntimeError("commit failed")++ event.listen(case.sessions, "before_commit", fail_commit)+ try:++ def mutate() -> None:+ if operation == "delete":+ case.service.delete(CONTEXT, "app", "langfuse")+ elif operation == "create":+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "new"}, activate=True)+ else:+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "new"}, activate=True)++ with pytest.raises(RuntimeError, match="commit failed"):+ mutate()+ finally:+ event.remove(case.sessions, "before_commit", fail_commit)+ assert case.policy() == before_policy+ assert case.config() == before_record++[email protected]("enabled", [False, True])+def test_delete_current_clears_selection_but_other_provider_preserves_policy(case: Case, enabled: bool) -> None:+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "selected"}, activate=True)+ case.service.create(CONTEXT, "app", "langsmith", {"secret": "other"})+ policy = AppTraceSettings(enabled, "langfuse", True, False)+ case.service.set_trace(CONTEXT, "app", policy)+ case.service.delete(CONTEXT, "app", "langsmith")+ assert case.policy() == policy+ case.service.delete(CONTEXT, "app", "langfuse")+ assert case.policy() == AppTraceSettings(False, None, True, False)+ assert case.config() is None+++def test_validation_failure_writes_neither_config_nor_policy(case: Case, monkeypatch: pytest.MonkeyPatch) -> None:+ def reject(**_: object) -> dict[str, object]:+ raise AppTracingConfigVerificationFailedError++ monkeypatch.setattr(case.gateway, "prepare_new_config", reject)+ with pytest.raises(AppTracingConfigVerificationFailedError):+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "bad"}, activate=True)+ assert case.config() is None+ assert case.policy() == AppTraceSettings(False, None, True, False)++[email protected]("state", ["missing", "empty", "inactive"])+def test_enable_requires_usable_owned_config(case: Case, state: str) -> None:+ if state != "missing":+ case.service.create(CONTEXT, "app", "langfuse", {} if state == "empty" else {"secret": "valid"})+ if state == "inactive":+ with case.sessions.begin() as session:+ record = session.scalar(select(TraceAppConfig))+ assert record is not None+ record.is_active = False+ with pytest.raises(AppTracingConfigNotFoundError):+ case.service.set_trace(CONTEXT, "app", AppTraceSettings(True, "langfuse"))+ assert case.policy() == AppTraceSettings(False, None, True, False)+++def test_update_rejects_credentials_replaced_while_validation_was_in_flight(+ case: Case, monkeypatch: pytest.MonkeyPatch+) -> None:+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})++ def prepare(**_: object) -> dict[str, object]:+ current = case.config()+ assert current is not None+ case.store.update(+ workspace_id="tenant",+ app_id="app",+ tracing_provider="langfuse",+ tracing_config={"secret": "concurrent"},+ expected=current,+ )+ return {"secret": "stale"}++ monkeypatch.setattr(case.gateway, "prepare_updated_config", prepare)+ with pytest.raises(AppTracingConfigConflictError):+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "masked"}, activate=True)+ saved = case.config()+ assert saved is not None+ assert saved.tracing_config == {"secret": "concurrent"}+ assert case.policy() == AppTraceSettings(False, None, True, False)+++def test_enable_rejects_configuration_deleted_during_validation(case: Case, monkeypatch: pytest.MonkeyPatch) -> None:+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})++ def validate(**_: object) -> None:+ case.service.delete(CONTEXT, "app", "langfuse")++ monkeypatch.setattr(case.gateway, "validate_saved_config", validate)+ with pytest.raises(AppTracingConfigConflictError):+ case.service.set_trace(CONTEXT, "app", AppTraceSettings(True, "langfuse"))+ assert case.policy() == AppTraceSettings(False, None, True, False)++[email protected]("recreate", [False, True])+def test_patch_rejects_configuration_deleted_during_validation(+ case: Case, monkeypatch: pytest.MonkeyPatch, recreate: bool+) -> None:+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})++ def prepare(**_: object) -> dict[str, object]:+ case.service.delete(CONTEXT, "app", "langfuse")+ if recreate:+ case.service.create(CONTEXT, "app", "langfuse", {"secret": "original"})+ return {"secret": "stale"}++ monkeypatch.setattr(case.gateway, "prepare_updated_config", prepare)+ with pytest.raises(AppTracingConfigConflictError):+ case.service.update(CONTEXT, "app", "langfuse", {"secret": "masked"}, activate=True)+ assert case.policy() == AppTraceSettings(False, None, True, False)+ saved = case.config()+ if recreate:+ assert saved is not None+ assert saved.tracing_config == {"secret": "original"}+ else:+ assert saved is Nonediff --git a/api/tests/unit_tests/services/test_app_tracing_config_gateway.py b/api/tests/unit_tests/services/test_app_tracing_config_gateway.pyindex df619b8e4e0..ba47e3a7dff 100644--- a/api/tests/unit_tests/services/test_app_tracing_config_gateway.py+++ b/api/tests/unit_tests/services/test_app_tracing_config_gateway.py@@ -109,7 +109,7 @@ def test_prepare_new_config_applies_defaults_validates_and_encrypts(monkeypatch:monkeypatch.setattr(gateway_module,"provider_config_map",- {"arize": _provider_entry(other_keys=["endpoint", "project"])},+ {"langsmith": _provider_entry(other_keys=["endpoint", "project"])},)submitted = {"api_key": "plain", "endpoint": "", "project": ""}@@ -120,7 +120,7 @@ def test_prepare_new_config_applies_defaults_validates_and_encrypts(monkeypatch:result = OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config=submitted,)@@ -131,14 +131,14 @@ def test_prepare_new_config_applies_defaults_validates_and_encrypts(monkeypatch:}assert submitted == {"api_key": "plain", "endpoint": "", "project": ""}assert result == {"api_key": "encrypted", "project_url": "https://project.example.com"}- manager.check_trace_config_is_effective.assert_called_once_with(normalized, "arize")- manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "arize", normalized)+ manager.check_trace_config_is_effective.assert_called_once_with(normalized, "langsmith")+ manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "langsmith", normalized)def test_prepare_new_config_reports_failed_verification_before_encryption(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})with patch.object(gateway_module, "OpsTraceManager") as manager:manager.check_trace_config_is_effective.return_value = False@@ -146,7 +146,7 @@ def test_prepare_new_config_reports_failed_verification_before_encryption(with pytest.raises(AppTracingConfigVerificationFailedError):OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={"api_key": "plain"},)@@ -156,7 +156,7 @@ def test_prepare_new_config_reports_failed_verification_before_encryption(def test_prepare_new_config_keeps_success_when_project_url_lookup_fails(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})with patch.object(gateway_module, "OpsTraceManager") as manager:manager.check_trace_config_is_effective.return_value = True@@ -165,7 +165,7 @@ def test_prepare_new_config_keeps_success_when_project_url_lookup_fails(result = OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={"api_key": "plain"},)@@ -192,7 +192,7 @@ def test_prepare_new_langfuse_config_builds_project_url(monkeypatch: pytest.Monkdef test_prepare_new_config_reports_provider_check_exception_as_failed_verification(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})with patch.object(gateway_module, "OpsTraceManager") as manager:manager.check_trace_config_is_effective.side_effect = ValueError("verification failed")@@ -200,7 +200,7 @@ def test_prepare_new_config_reports_provider_check_exception_as_failed_verificatwith pytest.raises(AppTracingConfigVerificationFailedError) as caught:OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={},)@@ -210,7 +210,7 @@ def test_prepare_new_config_reports_provider_check_exception_as_failed_verificatdef test_prepare_new_config_propagates_unexpected_verification_failure(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})failure = RuntimeError("unexpected provider bug")with patch.object(gateway_module, "OpsTraceManager") as manager:@@ -219,7 +219,7 @@ def test_prepare_new_config_propagates_unexpected_verification_failure(with pytest.raises(RuntimeError) as caught:OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={},)@@ -230,13 +230,13 @@ def test_prepare_new_config_propagates_unexpected_verification_failure(def test_prepare_new_config_rejects_invalid_schema_before_verification(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})with patch.object(gateway_module, "OpsTraceManager") as manager:with pytest.raises(AppTracingConfigInvalidConfigurationError):OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={"endpoint": {"invalid": "value"}},)@@ -245,7 +245,7 @@ def test_prepare_new_config_rejects_invalid_schema_before_verification(def test_prepare_new_config_reports_encryption_failure_as_processing_error(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})failure = RuntimeError("key provider unavailable")with patch.object(gateway_module, "OpsTraceManager") as manager:@@ -256,7 +256,7 @@ def test_prepare_new_config_reports_encryption_failure_as_processing_error(monkewith pytest.raises(AppTracingConfigProcessingError) as caught:OpsTraceManagerGateway().prepare_new_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={},)@@ -266,7 +266,7 @@ def test_prepare_new_config_reports_encryption_failure_as_processing_error(monkedef test_prepare_updated_config_preserves_masked_secret_from_current_config(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})submitted = {"api_key": "******", "project": "new-project"}current = {"api_key": "old-encrypted", "project": "old-project"}encrypted = {"api_key": "old-encrypted", "project": "new-project"}@@ -276,29 +276,48 @@ def test_prepare_updated_config_preserves_masked_secret_from_current_config(manager.encrypt_tracing_config.return_value = encryptedmanager.decrypt_tracing_config.return_value = decryptedmanager.check_trace_config_is_effective.return_value = True+ manager.get_trace_config_project_url.return_value = "https://example.com/project/new"result = OpsTraceManagerGateway().prepare_updated_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config=submitted,current_tracing_config=current,)assert result == encrypted+ assert result["project_url"] == "https://example.com/project/new"assert submitted == {"api_key": "******", "project": "new-project"}- manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "arize", submitted, current)- manager.decrypt_tracing_config.assert_called_once_with("workspace-1", "arize", encrypted)- manager.check_trace_config_is_effective.assert_called_once_with(decrypted, "arize")+ manager.encrypt_tracing_config.assert_called_once_with("workspace-1", "langsmith", submitted, current)+ manager.decrypt_tracing_config.assert_called_once_with("workspace-1", "langsmith", encrypted)+ manager.check_trace_config_is_effective.assert_called_once_with(decrypted, "langsmith")+ manager.get_trace_config_project_url.assert_called_once_with(decrypted, "langsmith")+++def test_updated_destination_drops_old_project_when_lookup_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})+ with patch.object(gateway_module, "OpsTraceManager") as manager:+ manager.encrypt_tracing_config.return_value = {"api_key": "encrypted", "project": "new-project"}+ manager.decrypt_tracing_config.return_value = {"api_key": "plain", "project": "new-project"}+ manager.check_trace_config_is_effective.return_value = True+ manager.get_trace_config_project_url.side_effect = RuntimeError("metadata unavailable")+ result = OpsTraceManagerGateway().prepare_updated_config(+ workspace_id="workspace-1",+ tracing_provider="langsmith",+ tracing_config={"project": "new-project"},+ current_tracing_config={"project_url": "https://example.com/project/old", "api_key": "encrypted"},+ )+ assert "project_url" not in resultdef test_prepare_updated_config_validates_schema_before_encryption(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})with patch.object(gateway_module, "OpsTraceManager") as manager:with pytest.raises(AppTracingConfigInvalidConfigurationError):OpsTraceManagerGateway().prepare_updated_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={"endpoint": {"invalid": "value"}},current_tracing_config={"api_key": "old-encrypted"},)@@ -311,7 +330,7 @@ def test_prepare_updated_config_validates_schema_before_encryption(monkeypatch:def test_prepare_updated_config_reports_decryption_failure_as_processing_error(monkeypatch: pytest.MonkeyPatch,) -> None:- monkeypatch.setattr(gateway_module, "provider_config_map", {"arize": _provider_entry()})+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})failure = RuntimeError("stored credential cannot be decrypted")with patch.object(gateway_module, "OpsTraceManager") as manager:@@ -321,7 +340,7 @@ def test_prepare_updated_config_reports_decryption_failure_as_processing_error(with pytest.raises(AppTracingConfigProcessingError) as caught:OpsTraceManagerGateway().prepare_updated_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={"project": "new-project"},current_tracing_config={"api_key": "old-encrypted"},)@@ -333,20 +352,20 @@ def test_prepare_updated_config_reports_decryption_failure_as_processing_error(@pytest.mark.parametrize(("provider", "fallback_url"),[- ("arize", "https://app.arize.com/"),- ("phoenix", "https://app.phoenix.arize.com/projects/"),+ ("arize", None),+ ("phoenix", "https://app.phoenix.arize.com/redirects/projects/default"),("langsmith", "https://smith.langchain.com/"),("opik", "https://www.comet.com/opik/"),("weave", "https://wandb.ai/"),- ("aliyun", "https://arms.console.aliyun.com/"),+ ("aliyun", "https://arms.console.aliyun.com/#/llm"),("tencent", "https://console.cloud.tencent.com/apm"),- ("mlflow", "http://localhost:5000/"),+ ("mlflow", "http://localhost:5000/#/experiments/0/traces"),("databricks", "https://www.databricks.com/"),],)def test_present_config_uses_provider_fallback_when_project_lookup_fails(provider: str,- fallback_url: str,+ fallback_url: str | None,) -> None:with patch.object(gateway_module, "OpsTraceManager") as manager:decrypted_config: dict[str, object] = {}@@ -361,7 +380,11 @@ def test_present_config_uses_provider_fallback_when_project_lookup_fails(tracing_config={"encrypted": "config"},)- assert result == {"project_url": fallback_url}+ assert result == {"project_url": fallback_url}+ if provider in {"arize", "aliyun", "mlflow", "phoenix"}:+ # Provider defaults produce a pure link; the failing legacy lookup is+ # deliberately not reached, even when no cached URL is present.+ manager.get_trace_config_project_url.assert_not_called()def test_present_langfuse_config_builds_project_url() -> None:@@ -398,7 +421,7 @@ def test_present_config_rejects_missing_stored_config() -> None:with pytest.raises(AppTracingConfigProcessingError, match="processing failed"):OpsTraceManagerGateway().present_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config=None,)@@ -412,9 +435,40 @@ def test_present_config_reports_decryption_failure_as_processing_error() -> Nonewith pytest.raises(AppTracingConfigProcessingError) as caught:OpsTraceManagerGateway().present_config(workspace_id="workspace-1",- tracing_provider="arize",+ tracing_provider="langsmith",tracing_config={"encrypted": "config"},)assert caught.value.__cause__ is failuremanager.obfuscated_decrypt_token.assert_not_called()++[email protected]("malformed", [False, True])+def test_validate_saved_config_decrypts_and_validates_without_vendor_io(+ monkeypatch: pytest.MonkeyPatch, malformed: bool+) -> None:+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})+ with patch.object(gateway_module, "OpsTraceManager") as manager:+ manager.decrypt_tracing_config.return_value = {"project": ["invalid-project"] if malformed else "saved-project"}+ gateway = OpsTraceManagerGateway()+ if malformed:+ with pytest.raises(AppTracingConfigInvalidConfigurationError):+ gateway.validate_saved_config(+ workspace_id="workspace", tracing_provider="langsmith", tracing_config={"api_key": "encrypted"}+ )+ else:+ gateway.validate_saved_config(+ workspace_id="workspace", tracing_provider="langsmith", tracing_config={"api_key": "encrypted"}+ )+ manager.decrypt_tracing_config.assert_called_once_with("workspace", "langsmith", {"api_key": "encrypted"})+ manager.check_trace_config_is_effective.assert_not_called()+++def test_validate_saved_config_rejects_undecryptable_credentials(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(gateway_module, "provider_config_map", {"langsmith": _provider_entry()})+ with patch.object(gateway_module, "OpsTraceManager") as manager:+ manager.decrypt_tracing_config.side_effect = ValueError("invalid ciphertext")+ with pytest.raises(AppTracingConfigProcessingError):+ OpsTraceManagerGateway().validate_saved_config(+ workspace_id="workspace", tracing_provider="langsmith", tracing_config={"api_key": "broken"}+ )diff --git a/api/tests/unit_tests/services/test_app_tracing_config_project_url.py b/api/tests/unit_tests/services/test_app_tracing_config_project_url.pynew file mode 100644index 00000000000..32915c4e0cd--- /dev/null+++ b/api/tests/unit_tests/services/test_app_tracing_config_project_url.py@@ -0,0 +1,238 @@+"""Real provider config serialization must retain verified project links on update."""++import json+from collections.abc import Mapping, Sequence+from dataclasses import dataclass, field+from datetime import UTC, datetime, timedelta+from decimal import Decimal+from uuid import uuid4++import httpx+import pytest+from sqlalchemy.orm import Session, sessionmaker++from core.helper import ssrf_proxy+from core.ops import ops_trace_manager+from core.ops.ops_trace_manager import OpsTraceManager+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from machinery.context import RequestContext+from models.enums import ConversationFromSource+from models.model import AppMode+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from repositories.application_trace_repository import ApplicationTraceRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError+from services.tracing.provider_dispatch import export_trace+from tests.unit_tests.model_factories import make_app, make_message+from tests.unit_tests.services.tracing.test_opik_configuration import AccessAPI+++@dataclass+class ProviderBoundary:+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)+ effective: bool = True+ lookup_fails: bool = False++ def verify(self, config: Mapping[str, object], provider: str) -> bool:+ self.calls.append(("verify", provider, dict(config)))+ return self.effective++ def project_url(self, config: Mapping[str, object], provider: str) -> str:+ self.calls.append(("url", provider, dict(config)))+ if self.lookup_fails:+ raise RuntimeError("Synthetic project lookup failure")+ return f"https://tracing.test/{provider}/{config['project']}"++ def project_key(self, config: Mapping[str, object], provider: str) -> str:+ self.calls.append(("key", provider, dict(config)))+ return "verified-project"+++def boundary(monkeypatch: pytest.MonkeyPatch) -> ProviderBoundary:+ boundary = ProviderBoundary()++ def encrypt(_tenant: str, value: str) -> str:+ return "test-cipher:" + value++ def decrypt(_tenant: str, values: Sequence[str]) -> list[str]:+ assert all(value.startswith("test-cipher:") for value in values)+ return [value.removeprefix("test-cipher:") for value in values]++ monkeypatch.setattr(ops_trace_manager, "encrypt_token", encrypt)+ monkeypatch.setattr(ops_trace_manager, "batch_decrypt_token", decrypt)+ monkeypatch.setattr(OpsTraceManager, "check_trace_config_is_effective", boundary.verify)+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", boundary.project_url)+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_key", boundary.project_key)+ return boundary++[email protected]("provider", ["langsmith", "opik"])[email protected]("project", ["original", "changed"])+def test_update_resolves_project_link_using_verified_unmasked_configuration(+ boundary: ProviderBoundary, provider: str, project: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ gateway = OpsTraceManagerGateway()+ workspace = str(uuid4())+ api = AccessAPI()+ if provider == "opik":+ monkeypatch.setattr(ssrf_proxy, "make_request", api.request)+ target = {"workspace": "team", "url": "https://tracing.test/opik/api/"} if provider == "opik" else {}+ current = gateway.prepare_new_config(+ workspace_id=workspace,+ tracing_provider=provider,+ tracing_config={"api_key": "synthetic-key", "project": "original", **target},+ )+ submitted = {"api_key": "******", "project": project, "project_url": "https://untrusted.test/stale", **target}+ updated = gateway.prepare_updated_config(+ workspace_id=workspace, tracing_provider=provider, tracing_config=submitted, current_tracing_config=current+ )++ prefix = f"https://tracing.test/{provider}/" + ("team/redirect/projects?name=" if provider == "opik" else "")+ assert updated["project_url"] == prefix + project+ assert updated["api_key"] == current["api_key"] == "test-cipher:synthetic-key"+ assert current["project_url"] == prefix + "original"+ assert submitted["api_key"] == "******"+ if provider == "opik":+ assert boundary.calls == []+ assert len(api.calls) == 2+ assert api.calls[-1][2]["headers"] == {+ "Authorization": "synthetic-key",+ "Comet-Workspace": "team",+ "Accept": "application/json",+ }+ else:+ assert [call[0] for call in boundary.calls] == ["verify", "url", "verify", "url"]+ assert boundary.calls[-1][2]["api_key"] == "synthetic-key"+ assert boundary.calls[-1][2]["project"] == project+++def test_langfuse_update_uses_current_host_and_resolved_project_key(boundary: ProviderBoundary) -> None:+ gateway = OpsTraceManagerGateway()+ workspace = str(uuid4())+ current = gateway.prepare_new_config(+ workspace_id=workspace,+ tracing_provider="langfuse",+ tracing_config={"public_key": "synthetic-public", "secret_key": "synthetic-secret", "host": "https://old.test"},+ )+ updated = gateway.prepare_updated_config(+ workspace_id=workspace,+ tracing_provider="langfuse",+ tracing_config={"public_key": "******", "secret_key": "******", "host": "https://new.test"},+ current_tracing_config=current,+ )+ assert updated["project_url"] == "https://new.test/project/verified-project"+ assert current["project_url"] == "https://old.test/project/verified-project"+ assert boundary.calls[-1][0] == "key"+ assert boundary.calls[-1][2]["secret_key"] == "synthetic-secret"+++def test_failed_optional_link_lookup_does_not_reuse_stale_project_url(boundary: ProviderBoundary) -> None:+ gateway = OpsTraceManagerGateway()+ workspace = str(uuid4())+ current = gateway.prepare_new_config(+ workspace_id=workspace,+ tracing_provider="langsmith",+ tracing_config={"api_key": "synthetic-key", "project": "original"},+ )+ boundary.lookup_fails = True+ updated = gateway.prepare_updated_config(+ workspace_id=workspace,+ tracing_provider="langsmith",+ tracing_config={"api_key": "******", "project": "changed"},+ current_tracing_config=current,+ )+ assert updated["project"] == "changed"+ assert "project_url" not in updated+ assert boundary.calls[-1][0] == "url"+++def test_invalid_updated_configuration_never_resolves_a_project_url(boundary: ProviderBoundary) -> None:+ boundary.effective = False+ with pytest.raises(AppTracingConfigVerificationFailedError):+ OpsTraceManagerGateway().prepare_updated_config(+ workspace_id=str(uuid4()),+ tracing_provider="langsmith",+ tracing_config={"api_key": "synthetic-key", "project": "changed"},+ current_tracing_config=None,+ )+ assert [call[0] for call in boundary.calls] == ["verify"]+++def test_updated_project_url_is_frozen_and_used_by_real_langsmith_serializer(+ boundary: ProviderBoundary,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ workspace, app_id, message_id = (str(uuid4()) for _ in range(3))+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ mode=AppMode.AGENT,+ tracing=json.dumps({"enabled": True, "tracing_provider": "langsmith"}),+ )+ )+ session.add(+ make_message(+ message_id=message_id,+ app_id=app_id,+ inputs={},+ query="test input",+ message={},+ answer="test output",+ message_unit_price=Decimal(0),+ answer_unit_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.CONSOLE,+ )+ )+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ configs = AppTracingConfigService(configs=store, provider=OpsTraceManagerGateway())+ context = RequestContext("request", None, "account", workspace)+ configs.create(context, app_id, "langsmith", {"api_key": "synthetic-key", "project": "original"})+ configs.update(context, app_id, "langsmith", {"api_key": "******", "project": "changed"})+ repository = ApplicationTraceRepository(sqlite_session_factory)+ run = repository.start(+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id, session_id="test-session"+ )+ assert run is not None+ assert run.provider_config["project_url"] == "https://tracing.test/langsmith/changed"+ configs.update(context, app_id, "langsmith", {"api_key": "******", "project": "later"})+ frozen = repository.source_run(tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id)+ assert frozen is not None+ assert frozen.provider_config["project_url"] == "https://tracing.test/langsmith/changed"++ requests: list[str] = []++ def send(method: str, url: str, **_kwargs: object) -> httpx.Response:+ requests.append(url)+ return httpx.Response(200, request=httpx.Request(method, url))++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ config = dict(frozen.provider_config)+ config.update(OpsTraceManager.decrypt_tracing_config(workspace, "langsmith", frozen.provider_config))+ start = datetime(2026, 10, 16, tzinfo=UTC)+ trace = CanonicalTrace(+ trace_id=frozen.trace_id,+ session_id=frozen.session_id,+ root_span_id=frozen.root_span_id,+ spans=(+ CanonicalSpan(+ id=frozen.root_span_id,+ name="Agent",+ kind=CanonicalSpanKind.AGENT,+ parent_id=None,+ status=CanonicalSpanStatus.OK,+ start_time=start,+ end_time=start + timedelta(seconds=1),+ ),+ ),+ )+ result = export_trace("langsmith", config, trace)+ assert result.status == "accepted"+ assert result.trace_url == f"https://tracing.test/langsmith/changed/r/{result.trace_id}"+ assert result.span_urls[frozen.root_span_id] == result.trace_url+ assert len(requests) == 1+ assert boundary.calls[-1][2]["project"] == "later"diff --git a/api/tests/unit_tests/services/test_app_tracing_config_service.py b/api/tests/unit_tests/services/test_app_tracing_config_service.pyindex 208b5d4a972..8325cf96bf1 100644--- a/api/tests/unit_tests/services/test_app_tracing_config_service.py+++ b/api/tests/unit_tests/services/test_app_tracing_config_service.py@@ -114,6 +114,7 @@ def test_create_calls_provider_between_separate_store_operations() -> None:app_id="app-1",tracing_provider="arize",tracing_config={"api_key": "encrypted"},+ activate=False,)@@ -170,6 +171,8 @@ def test_update_validates_reads_prepares_and_writes_in_order() -> None:app_id="app-1",tracing_provider="arize",tracing_config={"api_key": "new-encrypted"},+ expected=current,+ activate=False,)diff --git a/api/tests/unit_tests/services/test_arize_tracing_project_url.py b/api/tests/unit_tests/services/test_arize_tracing_project_url.pynew file mode 100644index 00000000000..f11cf7421a2--- /dev/null+++ b/api/tests/unit_tests/services/test_arize_tracing_project_url.py@@ -0,0 +1,292 @@+"""Arize project links use verified metadata without changing ingestion permissions."""++import json+from collections import deque+from copy import deepcopy+from decimal import Decimal+from typing import Never+from uuid import uuid4++import httpx+import pytest+from dify_trace_arize_phoenix import arize_configuration+from dify_trace_arize_phoenix.arize_configuration import DESTINATION_KEY+from dify_trace_arize_phoenix.config import ArizeConfig+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.helper import ssrf_proxy+from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from models.application_trace import ApplicationTraceRun+from models.enums import ConversationFromSource+from models.model import AppMode+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from repositories.application_trace_repository import ApplicationTraceRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError+from services.tracing.provider_dispatch import frozen_project_url+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.model_factories import make_app, make_message+from tests.unit_tests.services.tracing.test_arize_configuration import (+ DESTINATION,+ SPACE,+ URL,+ VALUES,+ page,+ project,+ response,+ space,+)+from tests.unit_tests.services.tracing.test_arize_configuration import (+ calls as calls, # noqa: PLC0414 - Re-export the HTTP boundary fixture for pytest.+)+from tests.unit_tests.services.tracing.test_arize_configuration import (+ responses as responses, # noqa: PLC0414 - Re-export the HTTP boundary fixture for pytest.+)++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++def verified(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]:+ calls: list[dict[str, object]] = []++ def verify(config: ArizeConfig) -> None:+ calls.append(config.model_dump())++ monkeypatch.setattr(arize_configuration, "verify_configuration", verify)+ return calls+++def test_legacy_arize_get_hides_unverified_redirect_without_network_or_rewriting(+ workspace: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "arize",+ {"api_key": "synthetic-ingestion-key", "space_id": "U3BhY2U6dGVzdA==", "project": "original"},+ )+ stored["project_url"] = "https://app.arize.com/?redirect_project_name=original"+ untouched = deepcopy(stored)+ calls: list[str] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ calls.append("legacy or network")+ raise AssertionError("Configuration GET must not initialize a tracer or resolve vendor metadata")++ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)+ monkeypatch.setattr(ssrf_proxy, "get", forbidden)+ shown = OpsTraceManagerGateway().present_config(+ workspace_id=workspace, tracing_provider="arize", tracing_config=stored+ )++ assert shown["project_url"] is None+ assert calls == []+ assert stored == untouched+++def test_real_rsa_create_masked_update_get_and_frozen_history(+ workspace: str,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ verified: list[dict[str, object]],+ calls: list[tuple[str, dict[str, object]]],+ responses: deque[httpx.Response | Exception],+) -> None:+ app_id, message_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ mode=AppMode.AGENT,+ tracing=json.dumps({"enabled": True, "tracing_provider": "arize"}),+ )+ )+ session.add(+ make_message(+ message_id=message_id,+ app_id=app_id,+ inputs={},+ query="test",+ message={},+ answer="ok",+ message_unit_price=Decimal(0),+ answer_unit_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.CONSOLE,+ )+ )+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ gateway = OpsTraceManagerGateway()+ service = AppTracingConfigService(configs=store, provider=gateway)+ context = RequestContext("request", None, "account", workspace)+ legacy = OpsTraceManager.encrypt_tracing_config(workspace, "arize", VALUES)+ legacy["project_url"] = "https://app.arize.com/?redirect_project_name=exact"+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="arize", tracing_config=legacy)+ repository = ApplicationTraceRepository(sqlite_session_factory)+ run = repository.start(+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id, session_id="session"+ )+ assert run is not None+ with sqlite_session_factory.begin() as session:+ accepted = session.get(ApplicationTraceRun, run.id)+ assert accepted is not None+ accepted.state = "accepted"+ accepted.attempts = 1+ with sqlite_session_factory() as session:+ accepted_row = deepcopy(+ dict(+ session.execute(select(ApplicationTraceRun.__table__).where(ApplicationTraceRun.id == run.id))+ .mappings()+ .one()+ )+ )+ frozen_before = deepcopy(run.provider_config)++ # Both legacy secret fields are really encrypted, rather than identity stubs.+ assert legacy["api_key"] != VALUES["api_key"]+ assert legacy["space_id"] != SPACE+ responses.extend([space(), page(project())])+ submitted = {+ **VALUES,+ "api_key": "******",+ "space_id": "******",+ DESTINATION_KEY: {"organization_id": "forged", "project_id": "forged"},+ "project_url": "https://untrusted.test/forged",+ }+ untouched = deepcopy(submitted)+ service.update(context, app_id, "arize", submitted)+ current = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="arize")+ assert current is not None+ assert current.tracing_config is not None+ saved = current.tracing_config+ assert saved["api_key"] == legacy["api_key"]+ assert saved["space_id"] == legacy["space_id"]+ assert saved[DESTINATION_KEY] == {**DESTINATION, "source_digest": saved[DESTINATION_KEY]["source_digest"]}+ assert saved["project_url"] == URL+ assert submitted == untouched+ assert len(calls) == 2+ assert verified[-1]["api_key"] == VALUES["api_key"]+ assert verified[-1]["space_id"] == SPACE++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("Reading current configuration or frozen history must not use vendor I/O")++ monkeypatch.setattr(ssrf_proxy, "get", forbidden)+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)+ shown = service.get(context, app_id, "arize")+ assert shown is not None+ assert shown.tracing_config is not None+ assert shown.tracing_config["project_url"] == URL+ assert DESTINATION_KEY not in shown.tracing_config+ frozen = repository.source_run(tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id)+ assert frozen is not None+ assert (frozen.state, frozen.attempts, frozen.provider_config) == ("accepted", 1, frozen_before)+ assert frozen_project_url("arize", frozen.provider_config) is None+ assert frozen.provider_config == frozen_before+ monkeypatch.setattr(OpsTraceManager, "decrypt_tracing_config", forbidden)+ status = ApplicationTracingService(repository).status(+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=message_id+ )+ assert status["project_url"] is None+ assert status["status"] == "accepted"+ with sqlite_session_factory() as session:+ after_row = dict(+ session.execute(select(ApplicationTraceRun.__table__).where(ApplicationTraceRun.id == run.id))+ .mappings()+ .one()+ )+ assert after_row == accepted_row++ # A future source freezes the verified destination; it never repairs the old run.+ later_id = str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(+ make_message(+ message_id=later_id,+ app_id=app_id,+ inputs={},+ query="later",+ message={},+ answer="ok",+ message_unit_price=Decimal(0),+ answer_unit_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.CONSOLE,+ )+ )+ later = repository.start(+ tenant_id=workspace, app_id=app_id, source_kind="message", source_id=later_id, session_id="session"+ )+ assert later is not None+ assert frozen_project_url("arize", later.provider_config) == URL++[email protected]("status", [401, 403, 404, 500])+def test_ingest_verified_key_without_metadata_read_permission_still_saves(+ workspace: str,+ verified: list[dict[str, object]],+ calls: list[tuple[str, dict[str, object]]],+ responses: deque[httpx.Response | Exception],+ status: int,+) -> None:+ responses.append(response({"message": "private-response"}, status))+ saved = OpsTraceManagerGateway().prepare_new_config(+ workspace_id=workspace, tracing_provider="arize", tracing_config=VALUES+ )+ assert "project_url" not in saved+ assert DESTINATION_KEY not in saved+ assert OpsTraceManager.decrypt_tracing_config(workspace, "arize", saved)["api_key"] == VALUES["api_key"]+ assert len(verified) == len(calls) == 1+++def test_changed_destination_cannot_keep_old_or_client_supplied_cache(+ workspace: str,+ verified: list[dict[str, object]],+ calls: list[tuple[str, dict[str, object]]],+ responses: deque[httpx.Response | Exception],+) -> None:+ gateway = OpsTraceManagerGateway()+ responses.extend([space(), page(project())])+ before = gateway.prepare_new_config(workspace_id=workspace, tracing_provider="arize", tracing_config=VALUES)+ assert before["project_url"] == URL+ responses.append(response({}, 403))+ after = gateway.prepare_updated_config(+ workspace_id=workspace,+ tracing_provider="arize",+ current_tracing_config=before,+ tracing_config={+ **VALUES,+ "api_key": "******",+ "space_id": "******",+ "project": "later",+ DESTINATION_KEY: before[DESTINATION_KEY],+ "project_url": URL,+ },+ )+ assert after["project"] == "later"+ assert after["api_key"] == before["api_key"]+ assert after["space_id"] == before["space_id"]+ assert "project_url" not in after+ assert DESTINATION_KEY not in after+ assert before["project_url"] == URL+ assert len(calls) == 3+ assert len(verified) == 2+++def test_failed_original_ingestion_verification_never_resolves_metadata(+ workspace: str,+ monkeypatch: pytest.MonkeyPatch,+ calls: list[tuple[str, dict[str, object]]],+) -> None:+ def rejected(_config: ArizeConfig) -> Never:+ raise arize_configuration.ArizeConfigurationError("permission_denied")++ monkeypatch.setattr(arize_configuration, "verify_configuration", rejected)+ with pytest.raises(AppTracingConfigVerificationFailedError):+ OpsTraceManagerGateway().prepare_new_config(+ workspace_id=workspace, tracing_provider="arize", tracing_config=VALUES+ )+ assert calls == []diff --git a/api/tests/unit_tests/services/test_databricks_tracing_config_gateway.py b/api/tests/unit_tests/services/test_databricks_tracing_config_gateway.pynew file mode 100644index 00000000000..4704659408a--- /dev/null+++ b/api/tests/unit_tests/services/test_databricks_tracing_config_gateway.py@@ -0,0 +1,218 @@+"""Databricks config persistence must not initialize a process-global MLflow client."""++from collections.abc import Callable, Iterator, Sequence+from dataclasses import dataclass, field+from typing import Never+from uuid import uuid4++import mlflow+import pytest+from dify_trace_mlflow.configuration import DatabricksConfigurationError+from dify_trace_mlflow.mlflow_trace import MLflowDataTrace+from sqlalchemy.orm import Session, sessionmaker++from core.helper import ssrf_proxy+from core.ops import ops_trace_manager+from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_databricks_configuration import ExperimentAPI, response+++@dataclass+class GatewayEnvironment:+ gateway: OpsTraceManagerGateway = field(default_factory=OpsTraceManagerGateway)+ api: ExperimentAPI = field(default_factory=ExperimentAPI)+ forbidden_calls: list[str] = field(default_factory=list)+++def environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[GatewayEnvironment]:+ environment = GatewayEnvironment()++ def encrypt(_tenant: str, value: str) -> str:+ return "test-cipher:" + value++ def decrypt(_tenant: str, values: Sequence[str]) -> list[str]:+ assert all(value.startswith("test-cipher:") for value in values)+ return [value.removeprefix("test-cipher:") for value in values]++ def forbidden(name: str) -> Callable[..., Never]:+ def block(*_args: object, **_kwargs: object) -> Never:+ environment.forbidden_calls.append(name)+ raise AssertionError("Configuration must not initialize a stateful exporter")++ return block++ monkeypatch.setattr(ops_trace_manager, "encrypt_token", encrypt)+ monkeypatch.setattr(ops_trace_manager, "batch_decrypt_token", decrypt)+ monkeypatch.setattr(ops_trace_manager, "obfuscated_token", lambda value: "******" if value else value)+ monkeypatch.setattr(ssrf_proxy, "get", environment.api.get)+ monkeypatch.setattr(ssrf_proxy, "post", environment.api.post)+ monkeypatch.setattr(MLflowDataTrace, "__init__", forbidden("legacy_constructor"))+ monkeypatch.setattr(mlflow, "set_tracking_uri", forbidden("tracking_uri"))+ monkeypatch.setattr(mlflow, "set_experiment", forbidden("experiment"))+ monkeypatch.setattr(mlflow, "search_experiments", forbidden("search"))+ monkeypatch.setattr(mlflow.config, "enable_async_logging", forbidden("async_logging"))+ yield environment+ assert environment.forbidden_calls == []++[email protected]("oauth", [False, True])+def test_create_and_patch_persist_verified_target_without_replacing_masked_credentials(+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], oauth: bool+) -> None:+ workspace, app_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id=app_id, tenant_id=workspace))+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ service = AppTracingConfigService(configs=store, provider=environment.gateway)+ context = RequestContext("request", None, "account", workspace)+ credentials = (+ {"client_id": "synthetic-client", "client_secret": "synthetic-secret"}+ if oauth+ else {"personal_access_token": "synthetic-pat"}+ )+ secret_field = "client_secret" if oauth else "personal_access_token"+ service.create(+ context, app_id, "databricks", {"host": " https://first.test/ ", "experiment_id": "00012", **credentials}+ )+ before = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")+ assert before is not None+ assert before.tracing_config is not None+ assert before.tracing_config["host"] == "https://first.test"+ assert before.tracing_config["experiment_id"] == "12"+ service.update(+ context,+ app_id,+ "databricks",+ {+ "host": " https://second.test/ ",+ "experiment_id": "00034",+ **credentials,+ secret_field: "******",+ },+ )+ after = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["host"] == "https://second.test"+ assert after.tracing_config["experiment_id"] == "34"+ assert after.tracing_config[secret_field] == before.tracing_config[secret_field]+ assert after.tracing_config["project_url"] == "https://second.test/ml/experiments/34/traces"+ assert before.tracing_config["project_url"] == "https://first.test/ml/experiments/12/traces"+ before_get = len(environment.api.calls)+ presented = service.get(context, app_id, "databricks")+ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config[secret_field] == "******"+ assert presented.tracing_config["project_url"] == after.tracing_config["project_url"]+ environment.gateway.validate_saved_config(+ workspace_id=workspace, tracing_provider="databricks", tracing_config=after.tracing_config+ )+ assert len(environment.api.calls) == before_get++[email protected]("saved_url", [None, "https://stale.test/old-project"])+def test_get_is_pure_for_historical_and_current_configs(environment: GatewayEnvironment, saved_url: str | None) -> None:+ workspace = str(uuid4())+ encrypted = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "databricks",+ {"host": "https://workspace.test/", "experiment_id": "12", "personal_access_token": "synthetic-pat"},+ )+ if saved_url is not None:+ encrypted["project_url"] = saved_url+ result = environment.gateway.present_config(+ workspace_id=workspace, tracing_provider="databricks", tracing_config=encrypted+ )+ assert result["project_url"] == "https://workspace.test/ml/experiments/12/traces"+ assert result["personal_access_token"] == "******"+ assert environment.api.calls == []+++def test_invalid_historical_host_uses_safe_pure_fallback(environment: GatewayEnvironment) -> None:+ workspace = str(uuid4())+ encrypted = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "databricks",+ {+ "host": "https://user:[email protected]",+ "experiment_id": "12",+ "personal_access_token": "synthetic-pat",+ },+ )+ result = environment.gateway.present_config(+ workspace_id=workspace, tracing_provider="databricks", tracing_config=encrypted+ )+ assert result["project_url"] == "https://www.databricks.com/"+ assert environment.api.calls == []+++def test_failed_patch_keeps_original_destination_and_logs_only_safe_code(+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], caplog: pytest.LogCaptureFixture+) -> None:+ workspace, app_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id=app_id, tenant_id=workspace))+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ service = AppTracingConfigService(configs=store, provider=environment.gateway)+ context = RequestContext("request", None, "account", workspace)+ original = {+ "host": "https://workspace.test",+ "experiment_id": "12",+ "personal_access_token": "synthetic-private-pat",+ }+ service.create(context, app_id, "databricks", original)+ before = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")+ assert before is not None+ environment.api.override = lambda _method, url, _options: response(url, {"message": "private-vendor-detail"}, 403)+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:+ service.update(+ context, app_id, "databricks", {**original, "experiment_id": "34", "personal_access_token": "******"}+ )+ assert isinstance(caught.value.__cause__, DatabricksConfigurationError)+ assert caught.value.__cause__.code == "permission_denied"+ after = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")+ assert after is not None+ assert after.tracing_config == before.tracing_config+ assert "permission_denied" in caplog.text+ assert "private-vendor-detail" not in caplog.text+ assert "synthetic-private-pat" not in caplog.text++[email protected]("start_with_oauth", [False, True])+def test_switching_auth_clears_unselected_credentials_in_persistence_and_requests(+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], start_with_oauth: bool+) -> None:+ workspace, app_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id=app_id, tenant_id=workspace))+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ service = AppTracingConfigService(configs=store, provider=environment.gateway)+ context = RequestContext("request", None, "account", workspace)+ pat = {"personal_access_token": "synthetic-pat", "client_id": None, "client_secret": None}+ oauth = {"personal_access_token": None, "client_id": "synthetic-client", "client_secret": "synthetic-secret"}+ target = {"host": "https://workspace.test", "experiment_id": "12"}+ service.create(context, app_id, "databricks", {**target, **(oauth if start_with_oauth else pat)})+ environment.api.calls.clear()+ service.update(context, app_id, "databricks", {**target, **(pat if start_with_oauth else oauth)})+ stored = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="databricks")+ assert stored is not None+ assert stored.tracing_config is not None+ assert stored.tracing_config["client_secret" if start_with_oauth else "personal_access_token"] is None+ assert [method for method, _url, _options in environment.api.calls] == (+ ["GET"] if start_with_oauth else ["POST", "GET"]+ )+ expected_token = "synthetic-pat" if start_with_oauth else "synthetic-oauth-token"+ assert environment.api.calls[-1][2]["headers"] == {+ "Authorization": f"Bearer {expected_token}",+ "Accept": "application/json",+ }+ presented = service.get(context, app_id, "databricks")+ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config["client_secret" if start_with_oauth else "personal_access_token"] is Nonediff --git a/api/tests/unit_tests/services/test_feature_query_service.py b/api/tests/unit_tests/services/test_feature_query_service.pyindex d66340e9a03..08ab5f740bb 100644--- a/api/tests/unit_tests/services/test_feature_query_service.py+++ b/api/tests/unit_tests/services/test_feature_query_service.py@@ -43,7 +43,9 @@ def test_workspace_queries_use_workspace_from_request_context() -> None:def test_deployment_queries_delegate_without_request_context() -> None:gateway = create_autospec(FeatureQueryGateway, instance=True, spec_set=True)- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)+ system_features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY+ )license_model = LicenseModel()gateway.get_public_system_features.return_value = system_featuresgateway.get_license.return_value = license_model@@ -83,6 +85,7 @@ def test_workspace_id_queries_delegate_without_request_context() -> None:def test_enterprise_license_admission(status: LicenseStatus, expected: bool) -> None:gateway = create_autospec(FeatureQueryGateway, instance=True, spec_set=True)gateway.get_public_system_features.return_value = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False,deployment_edition=DeploymentEdition.ENTERPRISE,license={"status": status},)diff --git a/api/tests/unit_tests/services/test_feature_service_gateway.py b/api/tests/unit_tests/services/test_feature_service_gateway.pyindex c4f8a2efcfd..fbfb02a1e56 100644--- a/api/tests/unit_tests/services/test_feature_service_gateway.py+++ b/api/tests/unit_tests/services/test_feature_service_gateway.py@@ -8,7 +8,9 @@ from services.system_feature_service import SystemFeatureServicedef test_public_system_features_delegate_to_existing_service(mocker: MockerFixture) -> None:- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)+ system_features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY+ )get_system_features = mocker.patch.object(SystemFeatureService,"get_public_system_features",diff --git a/api/tests/unit_tests/services/test_mlflow_tracing_config_gateway.py b/api/tests/unit_tests/services/test_mlflow_tracing_config_gateway.pynew file mode 100644index 00000000000..bd024bc1677--- /dev/null+++ b/api/tests/unit_tests/services/test_mlflow_tracing_config_gateway.py@@ -0,0 +1,273 @@+"""MLflow config persistence uses real RSA/SQLite without constructing an exporter."""++import json+from dataclasses import dataclass+from uuid import uuid4++import pytest+from dify_trace_mlflow.oss_configuration import MLflowConfigurationError+from sqlalchemy.orm import Session, sessionmaker++from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import (+ AppTracingConfigInvalidConfigurationError,+ AppTracingConfigRecord,+ AppTracingConfigService,+ AppTracingConfigVerificationFailedError,+)+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_databricks_configuration import ExperimentAPI, response+from tests.unit_tests.services.tracing.test_mlflow_configuration import (+ experiment_api as experiment_api, # noqa: PLC0414 - Re-export pytest fixtures only in this module.+)+from tests.unit_tests.services.tracing.test_mlflow_configuration import (+ isolated_sdk_state as isolated_sdk_state, # noqa: PLC0414 - Re-export pytest fixtures only in this module.+)++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++@dataclass+class ConfigurationCase:+ api: ExperimentAPI+ gateway: OpsTraceManagerGateway+ service: AppTracingConfigService+ store: SQLAlchemyAppTracingConfigRepository+ reader: ConsoleAppRepository+ context: RequestContext+ app_id: str++ def read(self, provider: str = "mlflow") -> AppTracingConfigRecord | None:+ return self.store.get(+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider+ )+++def case(+ workspace: str, sqlite_session_factory: sessionmaker[Session], experiment_api: ExperimentAPI+) -> ConfigurationCase:+ app_id = str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ tracing=json.dumps(+ {"enabled": True, "tracing_provider": "langsmith", "capture_content": True, "capture_agent": False}+ ),+ )+ )+ gateway = OpsTraceManagerGateway()+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ previous = OpsTraceManager.encrypt_tracing_config(+ workspace, "langsmith", {"api_key": "synthetic-other-key", "project": "other", "endpoint": "https://other.test"}+ )+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=previous)+ return ConfigurationCase(+ experiment_api,+ gateway,+ AppTracingConfigService(configs=store, provider=gateway),+ store,+ ConsoleAppRepository(session_factory=sqlite_session_factory),+ RequestContext("request", None, "account", workspace),+ app_id,+ )+++def test_post_masked_patch_and_get_preserve_ciphertext_and_exact_destination(case: ConfigurationCase) -> None:+ original = case.read("langsmith")+ submitted = {+ "tracking_uri": " HTTP://FIRST.Example:5000/team/ ",+ "experiment_id": "00012",+ "username": "test-user",+ "password": "synthetic-password",+ }+ untouched = dict(submitted)+ case.service.create(case.context, case.app_id, "mlflow", submitted, activate=True)+ before = case.read()+ assert before is not None+ assert before.tracing_config is not None+ config = before.tracing_config+ assert config["tracking_uri"] == "http://first.example:5000/team"+ assert config["experiment_id"] == "12"+ assert config["password"] != "synthetic-password"+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "mlflow", config)["password"] == (+ "synthetic-password"+ )+ assert submitted == untouched+ shown = case.service.get(case.context, case.app_id, "mlflow")+ assert shown is not None+ assert shown.tracing_config is not None+ masked = shown.tracing_config["password"]+ assert isinstance(masked, str)+ assert "*" in masked+ assert masked != "synthetic-password"+ assert len(case.api.calls) == 1+ case.service.update(+ case.context,+ case.app_id,+ "mlflow",+ {+ "tracking_uri": "https://SECOND.test/new-prefix/",+ "experiment_id": "00034",+ "username": "test-user",+ "password": masked,+ },+ )+ after = case.read()+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["password"] == config["password"]+ assert after.tracing_config["tracking_uri"] == "https://second.test/new-prefix"+ assert after.tracing_config["experiment_id"] == "34"+ assert after.tracing_config["project_url"] == "https://second.test/new-prefix/#/experiments/34/traces"+ assert len(case.api.calls) == 2+ assert case.api.calls[-1][1] == "https://second.test/new-prefix/api/2.0/mlflow/experiments/get"+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=after.tracing_config+ )+ assert case.service.get(case.context, case.app_id, "mlflow") is not None+ assert len(case.api.calls) == 2+ assert case.read("langsmith") == original+ policy = case.reader.get_trace(case.context, case.app_id)+ assert policy.enabled+ assert policy.tracing_provider == "mlflow"+ assert policy.capture_content is True+ assert policy.capture_agent is False++[email protected]("clear_with_null", [False, True])+def test_basic_to_anonymous_and_back_does_not_reuse_credentials(case: ConfigurationCase, clear_with_null: bool) -> None:+ destination = {"tracking_uri": "https://same.test/prefix", "experiment_id": "12"}+ case.service.create(+ case.context, case.app_id, "mlflow", {**destination, "username": "first", "password": "first-secret"}+ )+ empty = None if clear_with_null else ""+ case.service.update(case.context, case.app_id, "mlflow", {**destination, "username": empty, "password": empty})+ assert case.api.calls[-1][2]["headers"] == {+ "Accept": "application/json",+ "Accept-Encoding": "identity",+ "Cookie": "",+ }+ stored = case.read()+ assert stored is not None+ assert stored.tracing_config is not None+ assert (+ OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "mlflow", stored.tracing_config)[+ "password"+ ]+ == empty+ )+ case.service.update(+ case.context, case.app_id, "mlflow", {**destination, "username": "second", "password": "second-secret"}+ )+ assert case.api.calls[-1][2]["headers"] != case.api.calls[0][2]["headers"]+++def test_failed_patch_keeps_config_policy_and_other_provider(+ case: ConfigurationCase, caplog: pytest.LogCaptureFixture+) -> None:+ original = {+ "tracking_uri": "https://first.test",+ "experiment_id": "12",+ "username": "user",+ "password": "private-password",+ }+ case.service.create(case.context, case.app_id, "mlflow", original)+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)+ case.api.override = lambda _method, url, _options: response(url, {"message": "private-vendor-response"}, 403)+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:+ case.service.update(+ case.context,+ case.app_id,+ "mlflow",+ {**original, "tracking_uri": "https://second.test", "password": "******"},+ activate=True,+ )+ assert isinstance(caught.value.__cause__, MLflowConfigurationError)+ assert caught.value.__cause__.code == "permission_denied"+ assert case.read() == before+ assert case.read("langsmith") == other+ assert case.reader.get_trace(case.context, case.app_id) == policy+ assert "permission_denied" in caplog.text+ assert "private-vendor-response" not in caplog.text+ assert "private-password" not in caplog.text+++def test_failed_auth_pair_patch_does_not_erase_a_password(case: ConfigurationCase) -> None:+ original = {"tracking_uri": "https://first.test", "experiment_id": "12", "username": "user", "password": "password"}+ case.service.create(case.context, case.app_id, "mlflow", original)+ before = case.read()+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:+ case.service.update(case.context, case.app_id, "mlflow", {**original, "password": None})+ assert isinstance(caught.value.__cause__, MLflowConfigurationError)+ assert caught.value.__cause__.code == "incomplete_basic_auth"+ assert len(case.api.calls) == 1+ assert case.read() == before++[email protected]("saved_url", [None, "https://stale.test/wrong-experiment"])+def test_historical_get_is_pure_even_with_partial_auth(case: ConfigurationCase, saved_url: str | None) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "mlflow",+ {+ "tracking_uri": "https://server.test/prefix/",+ "experiment_id": "00012",+ "username": "historical-user",+ "password": None,+ },+ )+ if saved_url is not None:+ stored["project_url"] = saved_url+ shown = case.gateway.present_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=stored+ )+ assert shown["project_url"] == "https://server.test/prefix/#/experiments/12/traces"+ assert shown["password"] is None+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=stored+ )+ assert case.api.calls == []+++def test_invalid_historical_url_has_no_fabricated_link_or_network(case: ConfigurationCase) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "mlflow",+ {+ "tracking_uri": "https://user:[email protected]?token=private",+ "experiment_id": "12",+ "username": None,+ "password": None,+ },+ )+ shown = case.gateway.present_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="mlflow", tracing_config=stored+ )+ assert shown["project_url"] is None+ assert case.api.calls == []+++def test_invalid_new_destination_never_reaches_storage_or_network(case: ConfigurationCase) -> None:+ with pytest.raises(AppTracingConfigInvalidConfigurationError):+ case.service.create(+ case.context,+ case.app_id,+ "mlflow",+ {+ "tracking_uri": "https://server.test/prefix/../other",+ "experiment_id": "12",+ "username": "user",+ "password": "secret",+ },+ activate=True,+ )+ assert case.read() is None+ assert case.api.calls == []+ assert case.reader.get_trace(case.context, case.app_id).tracing_provider == "langsmith"diff --git a/api/tests/unit_tests/services/test_opik_tracing_config_gateway.py b/api/tests/unit_tests/services/test_opik_tracing_config_gateway.pynew file mode 100644index 00000000000..e72e22706b1--- /dev/null+++ b/api/tests/unit_tests/services/test_opik_tracing_config_gateway.py@@ -0,0 +1,211 @@+"""Opik configuration transactions use real RSA and never initialize an SDK streamer."""++import json+from collections.abc import Iterator+from dataclasses import dataclass+from typing import Never+from uuid import uuid4++import httpx+import opik+import pytest+from dify_trace_opik.configuration import OpikConfigurationError+from dify_trace_opik.opik_trace import OpikDataTrace+from opik.config import OpikConfig as SDKOpikConfig+from sqlalchemy.orm import Session, sessionmaker++from core.helper import ssrf_proxy+from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from models.model import App+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_opik_configuration import AccessAPI++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++@dataclass+class ConfigurationCase:+ api: AccessAPI+ gateway: OpsTraceManagerGateway+ service: AppTracingConfigService+ store: SQLAlchemyAppTracingConfigRepository+ reader: ConsoleAppRepository+ context: RequestContext+ app_id: str+ sessions: sessionmaker[Session]+++def case(+ workspace: str, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> Iterator[ConfigurationCase]:+ app_id = str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id=app_id, tenant_id=workspace))+ api = AccessAPI()+ gateway = OpsTraceManagerGateway()+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ forbidden_calls: list[str] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ forbidden_calls.append("sdk_constructor")+ raise AssertionError("Configuration must not initialize an SDK client or ambient configuration")++ monkeypatch.setattr(ssrf_proxy, "make_request", api.request)+ monkeypatch.setattr(OpikDataTrace, "__init__", forbidden)+ monkeypatch.setattr(opik.Opik, "__init__", forbidden)+ monkeypatch.setattr(SDKOpikConfig, "__init__", forbidden)+ yield ConfigurationCase(+ api,+ gateway,+ AppTracingConfigService(configs=store, provider=gateway),+ store,+ ConsoleAppRepository(session_factory=sqlite_session_factory),+ RequestContext("request", None, "account", workspace),+ app_id,+ sqlite_session_factory,+ )+ assert forbidden_calls == []+++def test_post_get_patch_preserve_real_ciphertext_and_verified_destination(case: ConfigurationCase) -> None:+ secret = "synthetic-private-opik-key"+ case.service.create(+ case.context,+ case.app_id,+ "opik",+ {"api_key": secret, "workspace": "", "project": " first ", "url": "https://first.test/prefix/api/"},+ activate=True,+ )+ before = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")+ assert before is not None+ assert before.tracing_config is not None+ config = before.tracing_config+ assert config["api_key"] != secret+ assert config["workspace"] == "resolved-team"+ assert config["project"] == "first"+ assert config["project_url"] == "https://first.test/prefix/resolved-team/redirect/projects?name=first"+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "opik", config)["api_key"] == secret+ assert case.reader.get_trace(case.context, case.app_id).tracing_provider == "opik"++ before_get = len(case.api.calls)+ presented = case.service.get(case.context, case.app_id, "opik")+ assert presented is not None+ assert presented.tracing_config is not None+ masked = presented.tracing_config["api_key"]+ assert masked != secret+ assert "*" in masked+ assert len(case.api.calls) == before_get+ case.api.default_workspace = "new-team"+ case.service.update(+ case.context,+ case.app_id,+ "opik",+ {"api_key": masked, "workspace": "default", "project": "A & B", "url": "https://second.test/api/"},+ )+ after = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["api_key"] == config["api_key"]+ assert after.tracing_config["workspace"] == "new-team"+ assert after.tracing_config["project"] == "A & B"+ assert after.tracing_config["url"] == "https://second.test/api/"+ assert after.tracing_config["project_url"] == "https://second.test/new-team/redirect/projects?name=A+%26+B"+ assert case.api.calls[-1][2]["headers"] == {+ "Authorization": secret,+ "Comet-Workspace": "default",+ "Accept": "application/json",+ }+ before_get = len(case.api.calls)+ presented = case.service.get(case.context, case.app_id, "opik")+ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config["project_url"] == after.tracing_config["project_url"]+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="opik", tracing_config=after.tracing_config+ )+ assert len(case.api.calls) == before_get++[email protected]("saved_url", [None, "https://wrong-global.test/old-project"])[email protected]("workspace_name", ["team", "default", None])+def test_historical_get_uses_only_configuration_host_without_network(+ case: ConfigurationCase, saved_url: str | None, workspace_name: str | None+) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "opik",+ {+ "api_key": "synthetic-key",+ "workspace": workspace_name,+ "project": "A & B",+ "url": "https://private.test/opik/api/",+ },+ )+ if saved_url:+ stored["project_url"] = saved_url+ result = case.gateway.present_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="opik", tracing_config=stored+ )+ expected = (+ "https://private.test/opik/team/redirect/projects?name=A+%26+B"+ if workspace_name == "team"+ else "https://private.test/opik/"+ )+ assert result["project_url"] == expected+ assert result["api_key"] != "synthetic-key"+ assert case.api.calls == []++[email protected]("status", [401, 403])[email protected]("existing", [False, True])+def test_bad_key_or_workspace_does_not_save_or_change_active_provider(+ case: ConfigurationCase, status: int, existing: bool, caplog: pytest.LogCaptureFixture+) -> None:+ original = {"api_key": "synthetic-private-key", "workspace": "team", "project": "original"}+ if existing:+ case.service.create(case.context, case.app_id, "opik", original)+ with case.sessions.begin() as session:+ app = session.get(App, case.app_id)+ assert app is not None+ app.tracing = json.dumps({"enabled": True, "tracing_provider": "langsmith", "capture_content": False})+ before_policy = case.reader.get_trace(case.context, case.app_id)+ before = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")+ case.api.override = lambda method, url, _options: httpx.Response(+ status, text="private-vendor-details", request=httpx.Request(method, url)+ )+ operation = case.service.update if existing else case.service.create+ with pytest.raises(AppTracingConfigVerificationFailedError) as error:+ operation(case.context, case.app_id, "opik", {**original, "workspace": "inaccessible"}, activate=True)+ assert isinstance(error.value.__cause__, OpikConfigurationError)+ assert error.value.__cause__.code == ("authentication_failed" if status == 401 else "permission_denied")+ after = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")+ assert after == before+ assert case.reader.get_trace(case.context, case.app_id) == before_policy+ assert "private-vendor-details" not in caplog.text+ assert "synthetic-private-key" not in caplog.text+++def test_clear_optional_api_key_keeps_keyless_self_host_valid(+ case: ConfigurationCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ target = {"url": "http://local.test:5173/api/", "workspace": "team", "project": "sample"}+ case.service.create(case.context, case.app_id, "opik", {**target, "api_key": "synthetic-key"})+ monkeypatch.setenv("OPIK_API_KEY", "unrelated-key")+ case.api.calls.clear()+ case.service.update(case.context, case.app_id, "opik", {**target, "api_key": None})+ stored = case.store.get(workspace_id=case.context.active_workspace_id, app_id=case.app_id, tracing_provider="opik")+ assert stored is not None+ assert stored.tracing_config is not None+ assert stored.tracing_config["api_key"] is None+ presented = case.service.get(case.context, case.app_id, "opik")+ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config["api_key"] is None+ assert case.api.calls[0][2]["headers"] == {"Comet-Workspace": "team", "Accept": "application/json"}+ assert len(case.api.calls) == 1diff --git a/api/tests/unit_tests/services/test_phoenix_tracing_config_gateway.py b/api/tests/unit_tests/services/test_phoenix_tracing_config_gateway.pynew file mode 100644index 00000000000..3f07363c34f--- /dev/null+++ b/api/tests/unit_tests/services/test_phoenix_tracing_config_gateway.py@@ -0,0 +1,285 @@+"""Persist verified Phoenix configuration while preserving owner policy and masked keys."""++import json+from collections import deque+from dataclasses import dataclass+from uuid import uuid4++import httpx+import pytest+from dify_trace_arize_phoenix.configuration import PhoenixConfigurationError+from sqlalchemy.orm import Session, sessionmaker++from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import (+ AppTracingConfigInvalidConfigurationError,+ AppTracingConfigRecord,+ AppTracingConfigService,+ AppTracingConfigVerificationFailedError,+)+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_phoenix_configuration import (+ api_calls as api_calls, # noqa: PLC0414 - Re-export pytest fixtures only in this module.+)+from tests.unit_tests.services.tracing.test_phoenix_configuration import (+ api_responses as api_responses, # noqa: PLC0414 - Re-export pytest fixtures only in this module.+)+from tests.unit_tests.services.tracing.test_phoenix_configuration import (+ no_legacy_or_environment_auth as no_legacy_or_environment_auth, # noqa: PLC0414 - Re-export pytest fixtures.+)+from tests.unit_tests.services.tracing.test_phoenix_configuration import response++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++@dataclass+class PhoenixCase:+ gateway: OpsTraceManagerGateway+ service: AppTracingConfigService+ store: SQLAlchemyAppTracingConfigRepository+ reader: ConsoleAppRepository+ context: RequestContext+ app_id: str++ def read(self, provider: str = "phoenix") -> AppTracingConfigRecord | None:+ return self.store.get(+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider+ )+++def case(workspace: str, sqlite_session_factory: sessionmaker[Session]) -> PhoenixCase:+ app_id = str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ tracing=json.dumps(+ {+ "enabled": True,+ "tracing_provider": "langsmith",+ "capture_content": True,+ "capture_agent": False,+ }+ ),+ )+ )+ gateway = OpsTraceManagerGateway()+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ old = OpsTraceManager.encrypt_tracing_config(+ workspace, "langsmith", {"api_key": "synthetic-other-key", "project": "other", "endpoint": "https://other.test"}+ )+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=old)+ return PhoenixCase(+ gateway,+ AppTracingConfigService(configs=store, provider=gateway),+ store,+ ConsoleAppRepository(session_factory=sqlite_session_factory),+ RequestContext("request", None, "account", workspace),+ app_id,+ )+++def test_post_masked_patch_get_keep_normalized_destination_and_ciphertext(+ case: PhoenixCase,+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ other = case.read("langsmith")+ submitted = {+ "endpoint": " HTTP://FIRST.Example:6006/reverse/ ",+ "project": " first project ",+ "api_key": "synthetic-secret",+ }+ untouched = dict(submitted)+ case.service.create(case.context, case.app_id, "phoenix", submitted, activate=True)+ before = case.read()+ assert before is not None+ assert before.tracing_config is not None+ stored = before.tracing_config+ assert stored["endpoint"] == "http://first.example:6006/reverse"+ assert stored["project"] == "first project"+ assert stored["api_key"] != "synthetic-secret"+ assert submitted == untouched+ assert OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "phoenix", stored)["api_key"] == (+ "synthetic-secret"+ )+ shown = case.service.get(case.context, case.app_id, "phoenix")+ assert shown is not None+ assert shown.tracing_config is not None+ masked = shown.tracing_config["api_key"]+ assert isinstance(masked, str)+ assert "*" in masked+ assert masked != "synthetic-secret"+ assert shown.tracing_config["project_url"] == "http://first.example:6006/reverse/redirects/projects/first%20project"+ assert len(api_calls) == 1+ case.service.update(+ case.context,+ case.app_id,+ "phoenix",+ {+ "endpoint": " https://SECOND.test/s/space/ ",+ "project": " second & 项目 ",+ "api_key": masked,+ },+ )+ after = case.read()+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["api_key"] == stored["api_key"]+ assert after.tracing_config["endpoint"] == "https://second.test/s/space"+ assert after.tracing_config["project"] == "second & 项目"+ assert after.tracing_config["project_url"] == (+ "https://second.test/s/space/redirects/projects/second%20%26%20%E9%A1%B9%E7%9B%AE"+ )+ assert api_calls[-1][0] == "https://second.test/s/space/v1/projects"+ headers = api_calls[-1][1]["headers"]+ assert isinstance(headers, dict)+ assert headers["Authorization"] == "Bearer synthetic-secret"+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=after.tracing_config+ )+ assert case.service.get(case.context, case.app_id, "phoenix") is not None+ assert len(api_calls) == 2+ assert case.read("langsmith") == other+ policy = case.reader.get_trace(case.context, case.app_id)+ assert policy.enabled+ assert policy.tracing_provider == "phoenix"+ assert policy.capture_content is True+ assert policy.capture_agent is False++[email protected]("empty", [None, ""])+def test_bearer_to_keyless_and_back_is_explicit(+ case: PhoenixCase,+ api_calls: list[tuple[str, dict[str, object]]],+ empty: str | None,+) -> None:+ destination = {"endpoint": "https://host.test", "project": "project"}+ case.service.create(case.context, case.app_id, "phoenix", {**destination, "api_key": "first-key"})+ case.service.update(case.context, case.app_id, "phoenix", {**destination, "api_key": empty})+ assert api_calls[-1][1]["headers"] == {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}+ stored = case.read()+ assert stored is not None+ assert stored.tracing_config is not None+ assert (+ OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "phoenix", stored.tracing_config)[+ "api_key"+ ]+ == empty+ )+ case.service.update(case.context, case.app_id, "phoenix", {**destination, "api_key": "second-key"})+ assert api_calls[-1][1]["headers"] != api_calls[0][1]["headers"]+ assert len(api_calls) == 3++[email protected]("failure", ["denied", "wrong-path-html"])+def test_failed_patch_preserves_policy_config_and_other_provider(+ case: PhoenixCase,+ api_calls: list[tuple[str, dict[str, object]]],+ api_responses: deque[httpx.Response | Exception],+ caplog: pytest.LogCaptureFixture,+ failure: str,+) -> None:+ original = {"endpoint": "https://host.test", "project": "project", "api_key": "private-secret"}+ case.service.create(case.context, case.app_id, "phoenix", original)+ before, other, policy = case.read(), case.read("langsmith"), case.reader.get_trace(case.context, case.app_id)+ if failure == "denied":+ api_responses.append(response({"secret": "private-vendor-response"}, 403))+ else:+ api_responses.append(+ httpx.Response(+ 200,+ text="<html>private-vendor-response</html>",+ request=httpx.Request("GET", "https://host.test/not-a-collector/v1/projects"),+ )+ )+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:+ case.service.update(+ case.context,+ case.app_id,+ "phoenix",+ {+ **original,+ "endpoint": "https://host.test/not-a-collector",+ "api_key": "******",+ },+ activate=True,+ )+ assert isinstance(caught.value.__cause__, PhoenixConfigurationError)+ assert caught.value.__cause__.code == ("permission_denied" if failure == "denied" else "malformed_response")+ assert case.read() == before+ assert case.read("langsmith") == other+ assert case.reader.get_trace(case.context, case.app_id) == policy+ assert "private-vendor-response" not in caplog.text+ assert "private-secret" not in caplog.text+ assert len(api_calls) == 2++[email protected]("saved_url", [None, "https://stale.test/wrong-project"])+def test_historical_get_calculates_url_without_verification_or_sdk(+ case: PhoenixCase,+ api_calls: list[tuple[str, dict[str, object]]],+ saved_url: str | None,+) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "phoenix",+ {+ "endpoint": "https://host.test/s/space/",+ "project": "historical project",+ "api_key": None,+ },+ )+ if saved_url:+ stored["project_url"] = saved_url+ shown = case.gateway.present_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=stored+ )+ assert shown["project_url"] == "https://host.test/s/space/redirects/projects/historical%20project"+ assert shown["api_key"] is None+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=stored+ )+ assert api_calls == []++[email protected]("endpoint", ["https://user:[email protected]?token=private", "https://host.test/v1/traces"])+def test_unsafe_historical_destination_is_readable_without_link_or_io(+ case: PhoenixCase,+ api_calls: list[tuple[str, dict[str, object]]],+ endpoint: str,+) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "phoenix",+ {+ "endpoint": endpoint,+ "project": "old-project",+ "api_key": None,+ },+ )+ shown = case.gateway.present_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="phoenix", tracing_config=stored+ )+ assert shown["endpoint"] == endpoint+ assert shown["project_url"] is None+ assert api_calls == []++[email protected]("values", [{"endpoint": "https://host.test/v1/traces"}, {"project": "bad\nproject"}])+def test_invalid_new_destination_has_no_io_or_storage_effect(+ case: PhoenixCase,+ api_calls: list[tuple[str, dict[str, object]]],+ values: dict[str, str],+) -> None:+ policy = case.reader.get_trace(case.context, case.app_id)+ with pytest.raises(AppTracingConfigInvalidConfigurationError):+ case.service.create(case.context, case.app_id, "phoenix", {"api_key": "synthetic-key", **values}, activate=True)+ assert case.read() is None+ assert case.reader.get_trace(case.context, case.app_id) == policy+ assert api_calls == []diff --git a/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py b/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.pyindex ddf4ff03538..c8ee83d8668 100644--- a/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py+++ b/api/tests/unit_tests/services/test_system_feature_service_deployment_edition.py@@ -13,7 +13,27 @@ from services.system_feature_service import SystemFeatureServicedef test_system_feature_model_requires_deployment_edition() -> None:with pytest.raises(ValidationError):- SystemFeatureModel.model_validate({})+ SystemFeatureModel.model_validate({"agent_tracing_api_collection_enabled": False})++[email protected]("edition", list(DeploymentEdition))[email protected]("enabled", [False, True])+def test_public_agent_tracing_gate_uses_configured_value_in_every_edition(+ monkeypatch: pytest.MonkeyPatch,+ config_overrides: Callable[..., None],+ edition: DeploymentEdition,+ enabled: bool,+) -> None:+ config_overrides(DEPLOYMENT_EDITION=edition, OPS_AGENT_TRACE_ENABLED=enabled)+ monkeypatch.setattr(+ "services.system_feature_service.EnterpriseService.get_info",+ staticmethod(lambda: {}),+ )++ result = SystemFeatureService.get_public_system_features()++ assert result.agent_tracing_api_collection_enabled is enabled+ assert result.model_dump(mode="json")["agent_tracing_api_collection_enabled"] is enabled@pytest.mark.parametrize(diff --git a/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py b/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.pyindex 0741f6a6f32..8fbf8cc674e 100644--- a/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py+++ b/api/tests/unit_tests/services/test_system_feature_service_enable_app_deploy.py@@ -33,7 +33,9 @@ def test_fulfill_params_from_enterprise_enable_app_deploy(staticmethod(lambda: enterprise_info),)- features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)+ features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY+ )features.enable_app_deploy = initialSystemFeatureService._fulfill_params_from_enterprise(features)diff --git a/api/tests/unit_tests/services/test_system_feature_service_learn_app.py b/api/tests/unit_tests/services/test_system_feature_service_learn_app.pyindex 95e53bb51aa..370db54a8bf 100644--- a/api/tests/unit_tests/services/test_system_feature_service_learn_app.py+++ b/api/tests/unit_tests/services/test_system_feature_service_learn_app.py@@ -10,7 +10,9 @@ from services.system_feature_service import SystemFeatureServicedef test_system_feature_model_defaults_enable_learn_app() -> None:- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)+ system_features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY+ )assert system_features.enable_learn_app is Trueassert system_features.enable_step_by_step_tour is Falsediff --git a/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py b/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.pyindex cc2490b6983..9ed70688d3e 100644--- a/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py+++ b/api/tests/unit_tests/services/test_system_feature_service_plugin_installation_permission.py@@ -89,7 +89,9 @@ def test_system_features_exposes_only_validated_plugin_installation_policy(}),)- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)+ features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE+ )SystemFeatureService._fulfill_params_from_enterprise(features)diff --git a/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py b/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.pyindex b889282e24b..48e42cfe8b3 100644--- a/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py+++ b/api/tests/unit_tests/services/test_system_feature_service_sso_protocol.py@@ -24,7 +24,9 @@ def test_system_features_exposes_valid_enterprise_sso_protocols(}),)- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)+ features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE+ )SystemFeatureService._fulfill_params_from_enterprise(features)@@ -48,7 +50,9 @@ def test_system_features_normalizes_empty_enterprise_sso_protocols_to_none(}),)- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)+ features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE+ )SystemFeatureService._fulfill_params_from_enterprise(features)@@ -73,7 +77,9 @@ def test_system_features_rejects_invalid_enterprise_sso_protocols(}),)- features = SystemFeatureModel(deployment_edition=DeploymentEdition.ENTERPRISE)+ features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.ENTERPRISE+ )with caplog.at_level(logging.ERROR, logger="services.system_feature_service"):SystemFeatureService._fulfill_params_from_enterprise(features)@@ -84,7 +90,9 @@ def test_system_features_rejects_invalid_enterprise_sso_protocols(def test_system_features_defaults_sso_protocols_to_none() -> None:- features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)+ features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY+ )assert features.sso_enforced_for_signin_protocol is Noneassert features.webapp_auth.sso_config.protocol is Nonediff --git a/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py b/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.pyindex 69757ebd7cd..6764dbf39d8 100644--- a/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py+++ b/api/tests/unit_tests/services/test_system_feature_service_webapp_public_access.py@@ -24,7 +24,9 @@ def test_fulfill_system_params_from_env_sets_allow_public_access() -> None:config_overrides(WEBAPP_PUBLIC_ACCESS_ENABLED=env_value)- system_features = SystemFeatureModel(deployment_edition=DeploymentEdition.COMMUNITY)+ system_features = SystemFeatureModel(+ agent_tracing_api_collection_enabled=False, deployment_edition=DeploymentEdition.COMMUNITY+ )SystemFeatureService._fulfill_system_params_from_env(system_features)assert system_features.webapp_auth.allow_public_access is expecteddiff --git a/api/tests/unit_tests/services/test_tencent_tracing_config_gateway.py b/api/tests/unit_tests/services/test_tencent_tracing_config_gateway.pynew file mode 100644index 00000000000..ff5ab51464e--- /dev/null+++ b/api/tests/unit_tests/services/test_tencent_tracing_config_gateway.py@@ -0,0 +1,324 @@+"""Tencent configuration persistence uses real RSA and bounded protocol checks."""++import json+import os+from dataclasses import dataclass+from typing import Never+from uuid import uuid4++import grpc # type: ignore[import-untyped]+import pytest+from dify_trace_tencent.configuration import TencentConfigurationError+from sqlalchemy.orm import Session, sessionmaker++from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from repositories.app.console_repository import ConsoleAppRepository+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import (+ AppTracingConfigInvalidConfigurationError,+ AppTracingConfigRecord,+ AppTracingConfigService,+ AppTracingConfigVerificationFailedError,+)+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_tencent_configuration import CollectorBoundary+from tests.unit_tests.services.tracing.test_tencent_configuration import (+ _forbid_provider_io as _forbid_provider_io, # noqa: PLC0414 - Re-export pytest fixtures only in this module.+)+from tests.unit_tests.services.tracing.test_tencent_configuration import (+ collector as collector, # noqa: PLC0414 - Re-export pytest fixtures only in this module.+)++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++@dataclass+class ConfigurationCase:+ collector: CollectorBoundary+ gateway: OpsTraceManagerGateway+ service: AppTracingConfigService+ store: SQLAlchemyAppTracingConfigRepository+ reader: ConsoleAppRepository+ context: RequestContext+ app_id: str++ def read(self, provider: str = "tencent") -> AppTracingConfigRecord | None:+ return self.store.get(+ workspace_id=self.context.active_workspace_id, app_id=self.app_id, tracing_provider=provider+ )+++def case(+ workspace: str, sqlite_session_factory: sessionmaker[Session], collector: CollectorBoundary+) -> ConfigurationCase:+ app_id = str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=workspace,+ tracing=json.dumps(+ {+ "enabled": True,+ "tracing_provider": "langsmith",+ "capture_content": True,+ "capture_agent": False,+ }+ ),+ )+ )+ gateway = OpsTraceManagerGateway()+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ old_config = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "langsmith",+ {"api_key": "synthetic-existing-key", "project": "existing-project", "endpoint": "https://existing.example"},+ )+ assert store.create(workspace_id=workspace, app_id=app_id, tracing_provider="langsmith", tracing_config=old_config)+ return ConfigurationCase(+ collector,+ gateway,+ AppTracingConfigService(configs=store, provider=gateway),+ store,+ ConsoleAppRepository(session_factory=sqlite_session_factory),+ RequestContext("request", None, "account", workspace),+ app_id,+ )+++def test_post_get_masked_patch_persist_normalization_and_preserve_real_ciphertext(case: ConfigurationCase) -> None:+ secret = "synthetic-private-tencent-token"+ old_provider = case.read("langsmith")+ submitted = {"token": secret, "endpoint": " HTTPS://FIRST.Example:4320/ ", "service_name": " first-app "}+ original_submission = dict(submitted)++ case.service.create(case.context, case.app_id, "tencent", submitted, activate=True)++ before = case.read()+ assert before is not None+ assert before.tracing_config is not None+ config = before.tracing_config+ assert config["token"] != secret+ assert config["endpoint"] == "https://first.example:4320"+ assert config["service_name"] == "first-app"+ assert config["project_url"] == "https://console.cloud.tencent.com/apm"+ assert (+ OpsTraceManager.decrypt_tracing_config(case.context.active_workspace_id, "tencent", config)["token"] == secret+ )+ assert submitted == original_submission+ assert len(case.collector.channels) == 1+ assert case.collector.channels[0].calls[0].metadata == (("authorization", f"Bearer {secret}"),)+ policy = case.reader.get_trace(case.context, case.app_id)+ assert policy.enabled is True+ assert policy.tracing_provider == "tencent"+ assert policy.capture_content is True+ assert policy.capture_agent is False+ assert case.read("langsmith") == old_provider++ presented = case.service.get(case.context, case.app_id, "tencent")+ assert presented is not None+ assert presented.tracing_config is not None+ masked = presented.tracing_config["token"]+ assert isinstance(masked, str)+ assert masked != secret+ assert "*" in masked+ assert presented.tracing_config["project_url"] == "https://console.cloud.tencent.com/apm"+ assert len(case.collector.channels) == 1++ replacement = {"token": masked, "endpoint": " http://SECOND.Example:4319/ ", "service_name": " second-app "}+ original_replacement = dict(replacement)+ case.service.update(case.context, case.app_id, "tencent", replacement, activate=True)++ after = case.read()+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["token"] == config["token"]+ assert after.tracing_config["endpoint"] == "http://second.example:4319"+ assert after.tracing_config["service_name"] == "second-app"+ assert after.tracing_config["project_url"] == "https://console.cloud.tencent.com/apm"+ assert replacement == original_replacement+ assert len(case.collector.channels) == 2+ channel = case.collector.channels[-1]+ assert channel.target == "second.example:4319"+ assert channel.secure is False+ assert channel.calls[0].metadata == (("authorization", f"Bearer {secret}"),)+ assert all(channel.close_count == 1 for channel in case.collector.channels)+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="tencent", tracing_config=after.tracing_config+ )+ assert case.service.get(case.context, case.app_id, "tencent") is not None+ assert len(case.collector.channels) == 2+ assert case.read("langsmith") == old_provider++[email protected]("saved_url", [None, "https://wrong.example/previous-project"])+def test_historical_name_remains_readable_and_saved_validation_does_not_connect(+ case: ConfigurationCase, saved_url: str | None+) -> None:+ stored = OpsTraceManager.encrypt_tracing_config(+ case.context.active_workspace_id,+ "tencent",+ {"token": "synthetic-historical-token", "endpoint": "https://legacy.example:4320", "service_name": "dify_app"},+ )+ if saved_url is not None:+ stored["project_url"] = saved_url+ assert case.store.create(+ workspace_id=case.context.active_workspace_id,+ app_id=case.app_id,+ tracing_provider="tencent",+ tracing_config=stored,+ )+ before = case.read()+ before_policy = case.reader.get_trace(case.context, case.app_id)++ presented = case.service.get(case.context, case.app_id, "tencent")+ case.gateway.validate_saved_config(+ workspace_id=case.context.active_workspace_id, tracing_provider="tencent", tracing_config=stored+ )++ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config["service_name"] == "dify_app"+ assert presented.tracing_config["project_url"] == "https://console.cloud.tencent.com/apm"+ assert presented.tracing_config["token"] != "synthetic-historical-token"+ assert "*" in presented.tracing_config["token"]+ assert case.collector.channels == []+ assert case.read() == before+ assert case.reader.get_trace(case.context, case.app_id) == before_policy++[email protected]("existing", [False, True])+ ("field_name", "invalid_value", "error_code"),+ [+ ("service_name", "dify_app", "invalid_service_name"),+ ("service_name", "", "invalid_service_name"),+ ("endpoint", "https://collector.example:4320/v1/traces", "invalid_endpoint"),+ ("endpoint", "https://collector.example:4320?token=private-endpoint-detail", "invalid_endpoint"),+ ("endpoint", "https://", "invalid_endpoint"),+ ],+)+def test_invalid_create_or_patch_rejects_before_rpc_encryption_or_persistence(+ case: ConfigurationCase,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ existing: bool,+ field_name: str,+ invalid_value: str,+ error_code: str,+) -> None:+ original = {+ "token": "synthetic-private-token",+ "endpoint": "https://original.example:4320",+ "service_name": "original-app",+ }+ if existing:+ case.service.create(case.context, case.app_id, "tencent", original)+ case.collector.channels.clear()+ before = case.read()+ old_provider = case.read("langsmith")+ before_policy = case.reader.get_trace(case.context, case.app_id)+ forbidden_attempts: list[str] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ forbidden_attempts.append("encryption_or_write")+ raise AssertionError("Invalid new input must fail before encryption or persistence")++ monkeypatch.setattr(OpsTraceManager, "encrypt_tracing_config", forbidden)+ monkeypatch.setattr(case.store, "create", forbidden)+ monkeypatch.setattr(case.store, "update", forbidden)+ operation = case.service.update if existing else case.service.create+ with pytest.raises(AppTracingConfigInvalidConfigurationError) as caught:+ operation(case.context, case.app_id, "tencent", {**original, field_name: invalid_value}, activate=True)++ assert isinstance(caught.value.__cause__, TencentConfigurationError)+ assert caught.value.__cause__.code == error_code+ assert forbidden_attempts == []+ assert case.collector.channels == []+ assert case.read() == before+ assert case.read("langsmith") == old_provider+ assert case.reader.get_trace(case.context, case.app_id) == before_policy+ assert "synthetic-private-token" not in caplog.text+ assert "private-endpoint-detail" not in caplog.text++[email protected]("existing", [False, True])[email protected]("endpoint", ["https://collector.example:4320", "http://localhost:4317"])+def test_rpc_failure_never_saves_or_activates_even_for_localhost(+ case: ConfigurationCase, caplog: pytest.LogCaptureFixture, existing: bool, endpoint: str+) -> None:+ secret = "synthetic-private-token"+ original = {"token": secret, "endpoint": "https://original.example:4320", "service_name": "original-app"}+ if existing:+ case.service.create(case.context, case.app_id, "tencent", original)+ case.collector.channels.clear()+ before = case.read()+ old_provider = case.read("langsmith")+ before_policy = case.reader.get_trace(case.context, case.app_id)+ case.collector.failure = grpc.RpcError("private-provider-details synthetic-private-token")+ operation = case.service.update if existing else case.service.create++ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:+ operation(+ case.context,+ case.app_id,+ "tencent",+ {"token": "******" if existing else secret, "endpoint": endpoint, "service_name": "changed-app"},+ activate=True,+ )++ assert isinstance(caught.value.__cause__, TencentConfigurationError)+ assert caught.value.__cause__.code == "connection_failed"+ assert len(case.collector.channels) == 1+ channel = case.collector.channels[0]+ assert channel.close_count == 1+ assert len(channel.calls) == 1+ assert channel.calls[0].metadata == (("authorization", f"Bearer {secret}"),)+ assert case.read() == before+ assert case.read("langsmith") == old_provider+ assert case.reader.get_trace(case.context, case.app_id) == before_policy+ assert "connection_failed" in caplog.text+ assert "private-provider-details" not in caplog.text+ assert secret not in caplog.text+ assert secret not in str(caught.value)+ assert secret not in str(caught.value.__cause__)+++def test_explicit_configuration_ignores_and_preserves_otel_environment(+ case: ConfigurationCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ environment = {+ "OTEL_EXPORTER_OTLP_ENDPOINT": "http://ambient.example:4317",+ "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://ambient-traces.example:4317",+ "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer unrelated-token",+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "authorization=Bearer unrelated-traces-token",+ "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",+ "OTEL_SERVICE_NAME": "unrelated-service",+ "OTEL_RESOURCE_ATTRIBUTES": "token=unrelated-resource-token,service.name=ambient-app",+ }+ for key, value in environment.items():+ monkeypatch.setenv(key, value)++ case.service.create(+ case.context,+ case.app_id,+ "tencent",+ {"token": "explicit-token", "endpoint": "https://explicit.example:4320", "service_name": "explicit-app"},+ )++ assert len(case.collector.channels) == 1+ channel = case.collector.channels[0]+ assert channel.target == "explicit.example:4320"+ assert channel.secure is True+ assert channel.calls[0].metadata == (("authorization", "Bearer explicit-token"),)+ assert channel.calls[0].request.SerializeToString() == b""+ assert channel.close_count == 1+ presented = case.service.get(case.context, case.app_id, "tencent")+ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config["service_name"] == "explicit-app"+ assert len(case.collector.channels) == 1+ assert {key: os.environ.get(key) for key in environment} == environmentdiff --git a/api/tests/unit_tests/services/test_weave_tracing_config_gateway.py b/api/tests/unit_tests/services/test_weave_tracing_config_gateway.pynew file mode 100644index 00000000000..918d30bc8cc--- /dev/null+++ b/api/tests/unit_tests/services/test_weave_tracing_config_gateway.py@@ -0,0 +1,167 @@+"""Config reads and writes must not construct Weave's stateful legacy exporter."""++from collections.abc import Callable, Iterator, Sequence+from dataclasses import dataclass, field+from typing import Never+from uuid import uuid4++import pytest+import wandb+import weave+from dify_trace_weave.configuration import WeaveConfigurationError+from dify_trace_weave.weave_trace import WeaveDataTrace+from sqlalchemy.orm import Session, sessionmaker++from core.helper import ssrf_proxy+from core.ops import ops_trace_manager+from core.ops.ops_trace_manager import OpsTraceManager+from machinery.context import RequestContext+from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigService, AppTracingConfigVerificationFailedError+from tests.unit_tests.model_factories import make_app+from tests.unit_tests.services.tracing.test_weave_configuration import ReadAPI+++@dataclass+class GatewayEnvironment:+ gateway: OpsTraceManagerGateway = field(default_factory=OpsTraceManagerGateway)+ api: ReadAPI = field(default_factory=ReadAPI)+ forbidden_calls: list[str] = field(default_factory=list)+++def environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[GatewayEnvironment]:+ environment = GatewayEnvironment()++ def encrypt(_tenant: str, value: str) -> str:+ return "test-cipher:" + value++ def decrypt(_tenant: str, values: Sequence[str]) -> list[str]:+ assert all(value.startswith("test-cipher:") for value in values)+ return [value.removeprefix("test-cipher:") for value in values]++ def mask(_value: str) -> str:+ return "******"++ def forbidden(name: str) -> Callable[..., Never]:+ def block(*_args: object, **_kwargs: object) -> Never:+ environment.forbidden_calls.append(name)+ raise AssertionError("Configuration must not initialize a stateful exporter")++ return block++ monkeypatch.setattr(ops_trace_manager, "encrypt_token", encrypt)+ monkeypatch.setattr(ops_trace_manager, "batch_decrypt_token", decrypt)+ monkeypatch.setattr(ops_trace_manager, "obfuscated_token", mask)+ monkeypatch.setattr(ssrf_proxy, "post", environment.api.post)+ monkeypatch.setattr(WeaveDataTrace, "__init__", forbidden("legacy_constructor"))+ monkeypatch.setattr(wandb, "login", forbidden("wandb_login"))+ monkeypatch.setattr(weave, "init", forbidden("weave_init"))+ yield environment+ assert environment.forbidden_calls == []+++def test_create_persists_resolved_default_entity_and_canonical_project(environment: GatewayEnvironment) -> None:+ submitted = {"api_key": "synthetic-key", "project": " demo "}+ stored = environment.gateway.prepare_new_config(+ workspace_id=str(uuid4()), tracing_provider="weave", tracing_config=submitted+ )+ assert stored["entity"] == "default-team"+ assert stored["project"] == "demo"+ assert stored["project_url"] == "https://wandb.ai/default-team/demo"+ assert stored["api_key"] == "test-cipher:synthetic-key"+ assert submitted == {"api_key": "synthetic-key", "project": " demo "}+ assert len(environment.api.calls) == 3+++def test_patch_persists_verified_destination_with_the_existing_ciphertext(+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session]+) -> None:+ workspace, app_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id=app_id, tenant_id=workspace))+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ service = AppTracingConfigService(configs=store, provider=environment.gateway)+ context = RequestContext("request", None, "account", workspace)+ service.create(context, app_id, "weave", {"api_key": "synthetic-key", "entity": "old", "project": "original"})+ before = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="weave")+ assert before is not None+ assert before.tracing_config is not None++ submitted = {"api_key": "******", "project": "new-team/changed"}+ service.update(context, app_id, "weave", submitted)+ after = store.get(workspace_id=workspace, app_id=app_id, tracing_provider="weave")+ assert after is not None+ assert after.tracing_config is not None+ assert after.tracing_config["entity"] == "new-team"+ assert after.tracing_config["project"] == "changed"+ assert after.tracing_config["project_url"] == "https://wandb.ai/new-team/changed"+ assert after.tracing_config["api_key"] == before.tracing_config["api_key"] == "test-cipher:synthetic-key"+ assert before.tracing_config["project_url"] == "https://wandb.ai/old/original"+ assert submitted == {"api_key": "******", "project": "new-team/changed"}+ assert len(environment.api.calls) == 4++ presented = service.get(context, app_id, "weave")+ assert presented is not None+ assert presented.tracing_config is not None+ assert presented.tracing_config["api_key"] == "******"+ assert presented.tracing_config["project_url"] == after.tracing_config["project_url"]+ environment.gateway.validate_saved_config(+ workspace_id=workspace, tracing_provider="weave", tracing_config=after.tracing_config+ )+ assert len(environment.api.calls) == 4+++ ("entity", "project", "host", "expected"),+ [+ ("team", "demo", None, "https://wandb.ai/team/demo"),+ (None, "team/demo", None, "https://wandb.ai/team/demo"),+ (None, "demo", None, "https://wandb.ai"),+ (None, "demo", "http://private.test/wandb", "http://private.test/wandb"),+ ("team", "a & b", "https://private.test/wandb", "https://private.test/wandb/team/a%20%26%20b"),+ ],+)+def test_get_computes_project_or_console_link_without_vendor_requests(+ environment: GatewayEnvironment, entity: str | None, project: str, host: str | None, expected: str+) -> None:+ workspace = str(uuid4())+ encrypted = OpsTraceManager.encrypt_tracing_config(+ workspace, "weave", {"api_key": "synthetic-key", "entity": entity, "project": project, "host": host}+ )+ encrypted["project_url"] = "https://stale.test/previous-project"+ result = environment.gateway.present_config(+ workspace_id=workspace, tracing_provider="weave", tracing_config=encrypted+ )+ assert result["project_url"] == expected+ assert result["api_key"] == "******"+ assert environment.api.calls == []+++def test_inaccessible_project_is_a_sanitized_domain_error_and_is_not_saved(+ environment: GatewayEnvironment, sqlite_session_factory: sessionmaker[Session], caplog: pytest.LogCaptureFixture+) -> None:+ workspace, app_id = str(uuid4()), str(uuid4())+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id=app_id, tenant_id=workspace))+ environment.api.project_exists = False+ store = SQLAlchemyAppTracingConfigRepository(session_factory=sqlite_session_factory)+ service = AppTracingConfigService(configs=store, provider=environment.gateway)+ with pytest.raises(AppTracingConfigVerificationFailedError) as caught:+ service.create(+ RequestContext("request", None, "account", workspace),+ app_id,+ "weave",+ {+ "api_key": "synthetic-private-key",+ "entity": "team",+ "project": "missing",+ },+ )+ assert isinstance(caught.value.__cause__, WeaveConfigurationError)+ assert caught.value.__cause__.code == "project_not_found_or_inaccessible"+ assert store.get(workspace_id=workspace, app_id=app_id, tracing_provider="weave") is None+ assert "synthetic-private-key" not in caplog.text+ assert "project_not_found_or_inaccessible" in caplog.text+ assert len(environment.api.calls) == 1diff --git a/api/tests/unit_tests/services/test_workflow_agent_preview_identity.py b/api/tests/unit_tests/services/test_workflow_agent_preview_identity.pynew file mode 100644index 00000000000..dd4c5c56375--- /dev/null+++ b/api/tests/unit_tests/services/test_workflow_agent_preview_identity.py@@ -0,0 +1,195 @@+"""Agent preview callers are committed before execution and retain their identity."""++import json+from collections.abc import Generator, Iterator+from unittest.mock import patch+from uuid import uuid4++import pytest+from flask import Flask+from sqlalchemy import Engine, func, select+from sqlalchemy.orm import sessionmaker++from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.nodes.agent_v2 import DifyAgentNode+from core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStore+from core.workflow.system_variables import SystemVariableKey, get_system_text+from extensions.ext_database import db+from graphon.engine_events import NodeRunFailedEvent, NodeRunStartedEvent, NodeRunSucceededEvent+from graphon.enums import WorkflowNodeExecutionStatus+from graphon.node_events import NodeRunResult+from libs.datetime_utils import naive_utc_now+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from services.workflow_service import WorkflowService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.services.test_workflow_service import TestWorkflowAssociatedDataFactory as Factory+++class PreviewCancelled(BaseException):+ """Cancellation can bypass Exception handlers, like a stopped greenlet."""+++def preview_flask_app(sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch) -> Iterator[Flask]:+ apply_config_overrides(monkeypatch, AGENT_BACKEND_BASE_URL="https://agent.test")+ app = Flask(__name__)+ app.config["SQLALCHEMY_DATABASE_URI"] = sqlite_engine.url.render_as_string(hide_password=False)+ db.init_app(app)+ with app.app_context():+ yield app++[email protected]("outcome", ["success", "failure", "exception", "cancel"])[email protected]("preview_flask_app")+def test_agent_preview_commits_runtime_identity_and_updates_same_row(sqlite_engine: Engine, outcome: str) -> None:+ tenant_id, app_id, workflow_id, account_id = (str(uuid4()) for _ in range(4))+ node_id = "agent-preview"+ app = Factory.create_app(app_id=app_id, tenant_id=tenant_id)+ account = Factory.create_account(account_id=account_id)+ workflow = Factory.create_workflow(+ workflow_id=workflow_id,+ tenant_id=tenant_id,+ app_id=app_id,+ graph={+ "nodes": [+ {+ "id": node_id,+ "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Agent"},+ }+ ],+ "edges": [],+ },+ )+ sessions = sessionmaker(sqlite_engine, expire_on_commit=False)+ with sessions.begin() as session:+ session.add_all([app, account, workflow])+ repository = SQLAlchemyWorkflowNodeExecutionRepository(+ sessions, tenant_id, account, app_id, WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP+ )+ observed_created_at = []+ execution_ids: list[str] = []++ def events(node: DifyAgentNode) -> Generator[NodeRunStartedEvent | NodeRunSucceededEvent | NodeRunFailedEvent]:+ execution_id = node.execution_id+ execution_ids.append(execution_id)+ started_at = naive_utc_now()+ # Node.run now emits its typed start before entering the node body. The+ # service observer must commit that event before resuming this generator.+ yield NodeRunStartedEvent(+ id=execution_id,+ node_id=node_id,+ node_type="agent",+ node_version="2",+ node_title="Agent",+ start_at=started_at,+ )+ # A separate transaction must see the committed caller before any+ # Agent workspace or tracing request can be dispatched.+ variable_pool = node.runtime_state.variable_pool+ workflow_run_id = get_system_text(variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID)+ assert workflow_run_id is None+ with sessions.begin() as session:+ caller = WorkflowAgentWorkspaceStore._load_execution_by_identity(+ session=session,+ tenant_id=tenant_id,+ app_id=app_id,+ workflow_id=workflow_id,+ workflow_run_id=workflow_run_id,+ node_id=node_id,+ node_execution_id=execution_id,+ )+ assert caller.id == caller.node_execution_id == execution_id+ assert caller.status == WorkflowNodeExecutionStatus.RUNNING+ assert caller.finished_at is None+ observed_created_at.append(caller.created_at)+ # Session materialization writes these independently; final persistence+ # must not discard either piece of caller ownership.+ caller.agent_workspace_binding_id = "workspace-binding"+ caller.process_data = json.dumps({"workflow_agent_binding_id": "workflow-binding"})+ if outcome == "exception":+ raise RuntimeError("private runtime error details")+ if outcome == "cancel":+ raise PreviewCancelled()+ status = WorkflowNodeExecutionStatus.SUCCEEDED if outcome == "success" else WorkflowNodeExecutionStatus.FAILED+ result = NodeRunResult(+ status=status, outputs={"answer": "done"}, error="" if outcome == "success" else "failed"+ )+ if outcome == "success":+ yield NodeRunSucceededEvent(+ id=execution_id, node_id=node_id, node_type="agent", node_run_result=result, start_at=started_at+ )+ else:+ yield NodeRunFailedEvent(+ id=execution_id,+ node_id=node_id,+ node_type="agent",+ node_run_result=result,+ start_at=started_at,+ error="failed",+ )++ service = WorkflowService(sessions)+ with (+ patch("services.workflow_service.WorkflowDraftVariableService.prefill_conversation_variable_default_values"),+ patch.object(DifyAgentNode, "run", events),+ patch(+ "services.workflow_service.DifyCoreRepositoryFactory.create_workflow_node_execution_repository",+ return_value=repository,+ ),+ patch("services.workflow_service.DraftVariableSaver.save"),+ patch("services.workflow_service.enqueue_draft_node_execution_trace"),+ ):+ if outcome in {"exception", "cancel"}:+ with pytest.raises(RuntimeError if outcome == "exception" else PreviewCancelled):+ service.run_draft_workflow_node(app, workflow, node_id, {}, account)+ else:+ response = service.run_draft_workflow_node(app, workflow, node_id, {}, account)+ assert response.id == response.node_execution_id == execution_ids[0]++ assert len(execution_ids) == 1+ execution_id = execution_ids[0]+ with sessions() as session:+ assert session.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 1+ row = session.get(WorkflowNodeExecutionModel, execution_id)+ assert row is not None+ assert row.id == row.node_execution_id == execution_id+ assert row.workflow_run_id is None+ assert row.workflow_id == workflow_id+ assert row.tenant_id == tenant_id+ assert row.app_id == app_id+ assert row.created_at == observed_created_at[0]+ assert row.finished_at is not None+ assert row.elapsed_time >= 0+ assert row.agent_workspace_binding_id == "workspace-binding"+ assert row.process_data_dict is not None+ assert row.process_data_dict["workflow_agent_binding_id"] == "workflow-binding"+ assert row.status == (+ WorkflowNodeExecutionStatus.SUCCEEDED if outcome == "success" else WorkflowNodeExecutionStatus.FAILED+ )+ if outcome in {"exception", "cancel"}:+ assert row.error == "Agent preview execution interrupted"++[email protected]("preview_flask_app")+def test_agent_preview_rejects_mismatched_workflow_before_creating_caller(sqlite_engine: Engine) -> None:+ app = Factory.create_app(app_id="app", tenant_id="tenant")+ account = Factory.create_account()+ workflow = Factory.create_workflow(+ app_id="another-app",+ tenant_id="another-tenant",+ graph={+ "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}],+ "edges": [],+ },+ )+ sessions = sessionmaker(sqlite_engine, expire_on_commit=False)+ with (+ patch(+ "services.workflow_service.WorkflowDraftVariableService.prefill_conversation_variable_default_values"+ ) as prefill,+ pytest.raises(ValueError, match="does not belong"),+ ):+ WorkflowService(sessions).run_draft_workflow_node(app, workflow, "agent", {}, account)+ prefill.assert_not_called()+ with sessions() as session:+ assert session.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 0diff --git a/api/tests/unit_tests/services/tracing/test_aliyun_configuration.py b/api/tests/unit_tests/services/tracing/test_aliyun_configuration.pynew file mode 100644index 00000000000..e679dc996b8--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_aliyun_configuration.py@@ -0,0 +1,276 @@+"""Pure Aliyun destination rules; these tests do not claim account verification."""++import traceback+from dataclasses import dataclass, field+from urllib.parse import unquote, urlsplit++import httpx+import pytest+from dify_trace_aliyun.aliyun_trace import AliyunDataTrace+from dify_trace_aliyun.config import AliyunConfig+from dify_trace_aliyun.configuration import (+ AliyunConfigurationError,+ console_url,+ ingestion_url,+ normalize_configuration,+ verify_configuration,+)+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+from pydantic import ValidationError++from core.helper import ssrf_proxy+++def values(**overrides: object) -> dict[str, object]:+ return {"endpoint": "http://tracing-analysis-dc-qd.aliyuncs.com", "license_key": "synthetic-key", **overrides}+++def test_normalization_preserves_opaque_license_prefix_and_application_name() -> None:+ submitted = values(+ endpoint=" HTTPS://Project.CN-QINGDAO.LOG.ALIYUNCS.COM:443/apm/trace/opentelemetry/ ",+ app_name=" 中文 agent_Name 01 ",+ license_key="literal%2F@a+b/c",+ )+ original = dict(submitted)+ result = normalize_configuration(submitted)+ assert result.endpoint == "https://project.cn-qingdao.log.aliyuncs.com:443/apm/trace/opentelemetry"+ assert result.app_name == "中文 agent_Name 01"+ assert result.license_key == "literal%2F@a+b/c"+ assert submitted == original+ assert ingestion_url(result.model_dump()) == (+ "https://project.cn-qingdao.log.aliyuncs.com:443/apm/trace/opentelemetry/"+ "adapt_literal%252F@a%2Bb%2Fc/api/v1/traces"+ )++[email protected]("key", ["raw%2F", "raw%252F", "a/b?c#d", "left@right_one+two", "密钥"])+def test_ingestion_path_round_trips_the_raw_license_exactly_once(key: str) -> None:+ url = ingestion_url(values(license_key=key))+ path = urlsplit(url).path+ assert path.startswith("/adapt_")+ assert path.endswith("/api/otlp/traces")+ encoded = path.removeprefix("/adapt_").removesuffix("/api/otlp/traces")+ assert "/" not in encoded+ assert unquote(encoded) == key++[email protected]("name", ["", " ", "dify_app", "应用 & Space"])+def test_application_names_keep_aliyun_semantics(name: str) -> None:+ assert normalize_configuration(values(app_name=name)).app_name == (name.strip() or "dify_app")+++def test_legacy_default_endpoint_and_app_name_remain_available() -> None:+ result = normalize_configuration(values(endpoint=""))+ assert result.endpoint == "https://tracing-analysis-dc-hz.aliyuncs.com"+ assert result.app_name == "dify_app"+++ ("base", "expected"),+ [+ ("https://log.aliyuncs.com", "https://log.aliyuncs.com/adapt_synthetic-key/api/v1/traces"),+ (+ "https://CN-QINGDAO.LOG.ALIYUNCS.COM.:443/prefix/",+ "https://cn-qingdao.log.aliyuncs.com.:443/prefix/adapt_synthetic-key/api/v1/traces",+ ),+ (+ "https://log.aliyuncs.com.evil.test",+ "https://log.aliyuncs.com.evil.test/adapt_synthetic-key/api/otlp/traces",+ ),+ ("https://evil-log.aliyuncs.com", "https://evil-log.aliyuncs.com/adapt_synthetic-key/api/otlp/traces"),+ (+ "https://collector.test/log.aliyuncs.com",+ "https://collector.test/log.aliyuncs.com/adapt_synthetic-key/api/otlp/traces",+ ),+ ("http://[::1]:4318/reverse/", "http://[::1]:4318/reverse/adapt_synthetic-key/api/otlp/traces"),+ ],+)+def test_ingestion_family_uses_only_the_hostname(base: str, expected: str) -> None:+ assert ingestion_url(values(endpoint=base)) == expected+++ "endpoint",+ [+ None,+ 1,+ "ftp://host.test",+ "https:///missing",+ "https://user:[email protected]",+ "https://host.test?",+ "https://host.test#",+ "https://host.test/path?key=secret",+ "https://host.test:bad",+ "https://host.test:0",+ "https://host.test:65536",+ "https://host.test:",+ "https://[invalid",+ "\nhttps://host.test",+ "https://ho\nst.test",+ "https://host.test\\@elsewhere",+ "https://host.test/a/../b",+ "https://host.test/a/%2e%2e/b",+ "https://host.test/a%2fb",+ "https://host.test/%252e%252e",+ "https://host.test/%zz",+ "https://host.test/double//path",+ "https://host.test/adapt_DO-NOT-PRINT/api/otlp/traces",+ "https://host.test/reverse/adapt_DO-NOT-PRINT/api/v1/traces/",+ "https://host.test/%61dapt_DO-NOT-PRINT",+ "https://host.test/api/otlp/traces",+ "https://host.test/v1/traces",+ ],+)+def test_new_endpoints_reject_unsafe_or_already_completed_routes(endpoint: object) -> None:+ with pytest.raises(AliyunConfigurationError) as error:+ normalize_configuration(values(endpoint=endpoint))+ assert error.value.code == "invalid_endpoint"+ assert "DO-NOT-PRINT" not in str(error.value)+ assert "host.test" not in str(error.value)++[email protected]("field", ["endpoint", "license_key"])+def test_required_fields_are_not_invented(field: str) -> None:+ submitted = values()+ del submitted[field]+ with pytest.raises(AliyunConfigurationError):+ normalize_configuration(submitted)++[email protected]("key", [None, 1, "", " ", " secret", "secret ", "secret\n", "secret\x00", "secret\ud800"])+def test_invalid_license_is_rejected_without_echoing_it(key: object) -> None:+ with pytest.raises(AliyunConfigurationError) as error:+ normalize_configuration(values(license_key=key))+ assert error.value.code == "invalid_license_key"+ assert "secret" not in str(error.value)++[email protected]("name", [None, 1, "app\nname", "app\x00name"])+def test_application_control_characters_are_rejected(name: object) -> None:+ with pytest.raises(AliyunConfigurationError) as error:+ normalize_configuration(values(app_name=name))+ assert error.value.code == "invalid_app_name"+++def test_masks_survive_patch_normalization_but_cannot_be_sent() -> None:+ result = normalize_configuration(values(license_key="***masked"))+ assert result.license_key == "***masked"+ with pytest.raises(AliyunConfigurationError):+ ingestion_url(result.model_dump())+++def test_console_link_and_historical_model_do_not_require_new_write_validation() -> None:+ old = AliyunConfig(license_key="synthetic-key", endpoint="https://host.test/adapt_old/api/otlp/traces")+ assert old.endpoint == "https://host.test/adapt_old/api/otlp/traces"+ assert console_url() == "https://arms.console.aliyun.com/#/llm"+++def test_historical_model_validation_does_not_format_sensitive_endpoint_inputs() -> None:+ secret = "SYNTHETIC_LICENSE_NOT_IN_ERRORS"+ with pytest.raises(ValidationError) as caught:+ AliyunConfig(license_key=secret, endpoint=f"ftp://host.test/adapt_{secret}/api/otlp/traces")+ rendered = str(caught.value) + repr(caught.value) + "".join(traceback.format_exception(caught.value))+ assert secret not in rendered+++@dataclass+class CollectorBoundary:+ status: int = 200+ content: bytes = b'"success"'+ failure: Exception | None = None+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)+ responses: list[httpx.Response] = field(default_factory=list)+++def collector(monkeypatch: pytest.MonkeyPatch) -> CollectorBoundary:+ boundary = CollectorBoundary()++ def send(method: str, url: str, **options: object) -> httpx.Response:+ boundary.calls.append((method, url, dict(options)))+ if boundary.failure is not None:+ raise boundary.failure+ response = httpx.Response(boundary.status, content=boundary.content, request=httpx.Request(method, url))+ boundary.responses.append(response)+ return response++ def forbidden(*_args: object, **_kwargs: object) -> None:+ raise AssertionError("Resource-only verification must not initialize a legacy exporter")++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ monkeypatch.setattr(AliyunDataTrace, "__init__", forbidden)+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://ambient.test")+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=ambient-credential")+ return boundary+++def test_verification_emits_one_resource_and_zero_business_spans(collector: CollectorBoundary) -> None:+ config = normalize_configuration(values(app_name="test_app", license_key="raw%2F@a+b/c"))+ verify_configuration(config)+ assert len(collector.calls) == 1+ method, url, options = collector.calls[0]+ assert method == "POST"+ assert url == "http://tracing-analysis-dc-qd.aliyuncs.com/adapt_raw%252F@a%2Bb%2Fc/api/otlp/traces"+ assert options["headers"] == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}+ assert options["max_retries"] == 0+ assert options["follow_redirects"] is False+ body = options["content"]+ assert isinstance(body, bytes)+ assert body+ payload = ExportTraceServiceRequest.FromString(body)+ assert len(payload.resource_spans) == 1+ resource = payload.resource_spans[0]+ assert len(resource.scope_spans) == 0+ assert len(resource.resource.attributes) == 1+ assert resource.resource.attributes[0].key == "service.name"+ assert resource.resource.attributes[0].value.string_value == "test_app"+ assert collector.responses[0].is_closed+++ ("status", "body"),+ [+ (200, b""),+ (200, b"{}"),+ (200, b"success"),+ (200, b'"success"\n'),+ (200, b'{"ret":0}'),+ (200, b'"failure"'),+ (200, b"<html>private-response</html>"),+ (201, b'"success"'),+ (204, b""),+ (400, b"private-response"),+ (401, b"private-response"),+ (403, b"private-response"),+ (404, b"private-response"),+ (405, b"private-response"),+ (429, b"private-response"),+ (503, b"private-response"),+ ],+)+def test_only_the_observed_resource_ack_verifies_access(+ collector: CollectorBoundary, status: int, body: bytes, caplog: pytest.LogCaptureFixture+) -> None:+ collector.status = status+ collector.content = body+ config = normalize_configuration(values(license_key="SYNTHETIC_PRIVATE_LICENSE@key"))+ with pytest.raises(AliyunConfigurationError) as error:+ verify_configuration(config)+ assert error.value.code == "verification_failed"+ rendered = str(error.value) + repr(error.value) + "".join(traceback.format_exception(error.value)) + caplog.text+ assert "SYNTHETIC_PRIVATE_LICENSE" not in rendered+ assert "private-response" not in rendered+ assert len(collector.calls) == 1+ assert all(response.is_closed for response in collector.responses)+++def test_transport_failure_does_not_escape_with_private_url(collector: CollectorBoundary) -> None:+ collector.failure = httpx.ConnectError("request failed at /adapt_PRIVATE%40KEY/api/otlp/traces")+ config = normalize_configuration(values(license_key="PRIVATE@KEY"))+ with pytest.raises(AliyunConfigurationError) as error:+ verify_configuration(config)+ assert error.value.code == "verification_failed"+ rendered = "".join(traceback.format_exception(error.value))+ assert "PRIVATE@KEY" not in rendered+ assert "PRIVATE%40KEY" not in rendered+ assert len(collector.calls) == 1diff --git a/api/tests/unit_tests/services/tracing/test_aliyun_otlp_ack.py b/api/tests/unit_tests/services/tracing/test_aliyun_otlp_ack.pynew file mode 100644index 00000000000..d1c028d324f--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_aliyun_otlp_ack.py@@ -0,0 +1,241 @@+"""Exercise Aliyun's observed HTTP ACK through both real protobuf owners."""++import logging+from collections.abc import Mapping+from datetime import UTC, datetime, timedelta+from typing import Literal, TypedDict, Unpack++import httpx+import pytest+from dify_trace_aliyun.data_exporter.traceclient import TraceClient+from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (+ ExportTraceServiceRequest,+ ExportTraceServiceResponse,+)++from core.helper import ssrf_proxy+from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.otlp_export import export_http, is_aliyun_http_success_ack+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError, export_trace++ENDPOINT = "https://collector.test/adapt_synthetic-license/api/otlp/traces"+START = datetime(2026, 10, 19, tzinfo=UTC)+TRACE_ID = "1" * 32+SPAN_ID = "2" * 16++type Owner = Literal["managed", "legacy"]+++class RequestOptions(TypedDict, total=False):+ content: bytes+ headers: Mapping[str, str]+ max_retries: int+ timeout: float+ follow_redirects: bool+++def boundary(+ monkeypatch: pytest.MonkeyPatch, status: int, content: bytes, content_type: str+) -> list[ExportTraceServiceRequest]:+ received: list[ExportTraceServiceRequest] = []++ def send(method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ assert method == "POST"+ assert url == ENDPOINT+ assert options["max_retries"] == 0+ assert options["follow_redirects"] is False+ assert options["headers"] == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True+ received.append(ExportTraceServiceRequest.FromString(options["content"]))+ headers = {"Content-Type": content_type} if content_type else {}+ return httpx.Response(status, content=content, headers=headers)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ return received+++def dispatch(owner: Owner) -> None:+ if owner == "managed":+ trace = CanonicalTrace(+ trace_id=TRACE_ID,+ root_span_id=SPAN_ID,+ session_id="synthetic-conversation",+ spans=(+ CanonicalSpan(+ id=SPAN_ID,+ parent_id=None,+ name="synthetic root",+ kind=CanonicalSpanKind.AGENT,+ start_time=START,+ end_time=START + timedelta(seconds=1),+ status=CanonicalSpanStatus.OK,+ ),+ ),+ )+ result = export_trace(+ "aliyun",+ {"endpoint": "https://collector.test", "license_key": "synthetic-license", "app_name": "synthetic"},+ trace,+ )+ assert result.status == "accepted"+ else:+ client = TraceClient("synthetic", ENDPOINT)+ with client.batch():+ client.add_span(+ SpanData(+ trace_id=int(TRACE_ID, 16),+ span_id=int(SPAN_ID, 16),+ name="synthetic root",+ start_time=int(START.timestamp()) * 1_000_000_000,+ end_time=int((START + timedelta(seconds=1)).timestamp()) * 1_000_000_000,+ )+ )+ assert client._dispatch_spans.get() is None+++def assert_one_batch(received: list[ExportTraceServiceRequest]) -> None:+ assert len(received) == 1+ spans = received[0].resource_spans[0].scope_spans[0].spans+ assert len(spans) == 1+ assert spans[0].trace_id.hex() == TRACE_ID+ assert spans[0].span_id.hex() == SPAN_ID++[email protected]("owner", ["managed", "legacy"])[email protected]("content_type", ["", "application/json", "application/x-protobuf"])+def test_exact_observed_ack_accepts_one_batch_without_changing_its_ids(+ monkeypatch: pytest.MonkeyPatch, owner: Owner, content_type: str+) -> None:+ received = boundary(monkeypatch, 200, b'"success"', content_type)+ dispatch(owner)+ assert_one_batch(received)++[email protected]("owner", ["managed", "legacy"])+ ("content", "content_type"),+ [+ (b"success", ""),+ (b'"SUCCESS"', ""),+ (b' "success"', "application/json"),+ (b'"success"\n', "application/json"),+ (b'{"success":true}', "application/json"),+ (b'{"message":"success"}', "application/json"),+ (b"true", "application/json"),+ (b"not protobuf", ""),+ ],+)+def test_other_ack_bodies_are_uncertain_and_not_replayed(+ monkeypatch: pytest.MonkeyPatch, owner: Owner, content: bytes, content_type: str+) -> None:+ received = boundary(monkeypatch, 200, content, content_type)+ with pytest.raises(TraceExportUncertainError, match="unreadable ingestion acknowledgment") as error:+ dispatch(owner)+ assert error.value.acceptance_unknown is True+ assert_one_batch(received)++[email protected]("owner", ["managed", "legacy"])[email protected]("status", [201, 202, 204])+def test_literal_ack_does_not_accept_another_success_status(+ monkeypatch: pytest.MonkeyPatch, owner: Owner, status: int+) -> None:+ received = boundary(monkeypatch, status, b'"success"', "")+ assert is_aliyun_http_success_ack(status, b'"success"') is False+ with pytest.raises(TraceExportUncertainError) as error:+ dispatch(owner)+ assert error.value.acceptance_unknown is True+ assert_one_batch(received)++[email protected]("owner", ["managed", "legacy"])[email protected]("status", [403, 429, 503])+def test_literal_ack_does_not_override_http_rejection_or_known_retry(+ monkeypatch: pytest.MonkeyPatch, owner: Owner, status: int+) -> None:+ received = boundary(monkeypatch, status, b'"success"', "")+ expected = TraceExportRejectedError if status == 403 else RetryableTraceDispatchError+ with pytest.raises(expected) as error:+ dispatch(owner)+ if isinstance(error.value, RetryableTraceDispatchError):+ assert error.value.acceptance_unknown is False+ assert_one_batch(received)++[email protected]("owner", ["managed", "legacy"])[email protected]("encoding", ["protobuf", "json"])[email protected]("rejected_spans", [0, 1])+def test_standard_otlp_ack_and_partial_success_remain_validated(+ monkeypatch: pytest.MonkeyPatch, owner: Owner, encoding: str, rejected_spans: int+) -> None:+ if encoding == "protobuf":+ response = ExportTraceServiceResponse()+ response.partial_success.rejected_spans = rejected_spans+ response.partial_success.error_message = "synthetic warning"+ content = response.SerializeToString()+ content_type = "application/x-protobuf"+ else:+ content = b'{"partialSuccess":{"rejectedSpans":"%d","errorMessage":"synthetic warning"}}' % rejected_spans+ content_type = "application/json"+ received = boundary(monkeypatch, 200, content, content_type)+ if rejected_spans:+ with pytest.raises(TraceExportUncertainError, match="partially accepted") as error:+ dispatch(owner)+ assert error.value.acceptance_unknown is True+ else:+ dispatch(owner)+ assert_one_batch(received)++[email protected]("owner", ["managed", "legacy"])+def test_damaged_compressed_ack_is_uncertain_after_real_http_submission(+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, owner: Owner+) -> None:+ received: list[ExportTraceServiceRequest] = []+ previous_suppression = get_value(_SUPPRESS_INSTRUMENTATION_KEY)++ def send(wire: httpx.Request) -> httpx.Response:+ assert wire.method == "POST"+ assert str(wire.url) == ENDPOINT+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True+ received.append(ExportTraceServiceRequest.FromString(wire.content))+ # Reading a malformed compressed response raises DecodingError, which+ # is a RequestError but not a TransportError. Submission already ran.+ return httpx.Response(+ 200,+ headers={"Content-Encoding": "gzip"},+ stream=httpx.ByteStream(b"invalid gzip acknowledgment"),+ request=wire,+ )++ with httpx.Client(transport=httpx.MockTransport(send)) as client:++ def get_client(ssl_verify_enabled: bool) -> httpx.Client:+ assert isinstance(ssl_verify_enabled, bool)+ return client++ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", get_client)+ with caplog.at_level(logging.INFO, logger="httpx"):+ with pytest.raises(TraceExportUncertainError) as error:+ dispatch(owner)+ assert error.value.acceptance_unknown is True+ assert error.value.__cause__ is None+ assert error.value.__context__ is None+ assert "synthetic-license" not in str(error.value) + caplog.text+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == previous_suppression+ assert_one_batch(received)++[email protected]("provider", ["langfuse", "phoenix", "tencent"])[email protected]("content_type", ["", "application/json"])+def test_literal_success_ack_does_not_expand_other_providers(+ monkeypatch: pytest.MonkeyPatch, provider: str, content_type: str+) -> None:+ received = boundary(monkeypatch, 200, b'"success"', content_type)+ with pytest.raises(TraceExportUncertainError, match="unreadable ingestion acknowledgment") as error:+ export_http(provider, ENDPOINT, {}, ExportTraceServiceRequest())+ assert error.value.acceptance_unknown is True+ assert len(received) == 1diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_collection.py b/api/tests/unit_tests/services/tracing/test_application_trace_collection.pynew file mode 100644index 00000000000..84f421cc9f2--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_collection.py@@ -0,0 +1,519 @@+"""Exercise collection transactions and delivery state against real SQLite rows."""++import json+from dataclasses import dataclass+from datetime import UTC, datetime, timedelta+from decimal import Decimal+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5++import pytest+from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceEvent, AgentTraceSpan+from sqlalchemy import func, select+from sqlalchemy.orm import Session, sessionmaker++from core.ops.entities.trace_entity import MessageTraceInfo+from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun+from models.enums import ConversationFromSource+from models.model import App, AppMode, Message, TraceAppConfig+from repositories.application_trace_repository import ApplicationTraceRepository, TraceCollectionError+from services.tracing import service as service_module+from services.tracing.provider_dispatch import TraceExportResult, TraceExportUncertainError+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_app, make_message+++def identifier(value: str) -> str:+ return str(uuid5(NAMESPACE_URL, value))+++@dataclass+class Collection:+ repository: ApplicationTraceRepository+ sessions: sessionmaker[Session]+ run: ApplicationTraceRun++ def invocation(self, name: str = "first") -> ApplicationTraceInvocation:+ return self.repository.invocation(+ run_id=self.run.id, agent_id=identifier("agent"), node_execution_id=None, attempt=0, call_key=name+ )+++def seed(+ sessions: sessionmaker[Session],+ name: str,+ *,+ capture_content: bool = False,+ enabled: bool = True,+ capture_agent: bool = True,+) -> Collection:+ app_id, tenant_id, message_id = identifier(f"app-{name}"), identifier("tenant"), identifier(f"message-{name}")+ with sessions.begin() as session:+ session.add(+ make_app(+ app_id=app_id,+ tenant_id=tenant_id,+ mode=AppMode.AGENT,+ tracing=json.dumps(+ {+ "enabled": enabled,+ "tracing_provider": "langfuse",+ "capture_content": capture_content,+ "capture_agent": capture_agent,+ }+ ),+ )+ )+ session.add(+ make_message(+ message_id=message_id,+ app_id=app_id,+ conversation_id=identifier(f"conversation-{name}"),+ inputs={},+ query="hello",+ message={},+ answer="done",+ message_unit_price=Decimal(0),+ answer_unit_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.CONSOLE,+ )+ )+ session.add(+ TraceAppConfig(+ app_id=app_id,+ tracing_provider="langfuse",+ tracing_config={+ "host": "https://langfuse.test",+ "secret_key": "encrypted-original",+ "project_url": "https://langfuse.test/project/original",+ },+ )+ )+ repository = ApplicationTraceRepository(sessions)+ run = repository.start(+ tenant_id=tenant_id, app_id=app_id, source_kind="message", source_id=message_id, session_id="session-original"+ )+ assert run is not None+ return Collection(repository, sessions, run)+++def collection(sqlite_session_factory: sessionmaker[Session]) -> Collection:+ return seed(sqlite_session_factory, "first")+++def batch_for(+ run: ApplicationTraceRun,+ invocation: ApplicationTraceInvocation,+ *,+ final: bool = True,+ batch_id: UUID | None = None,+ span_id: str = "a" * 16,+) -> AgentTraceBatch:+ return AgentTraceBatch(+ run_id=UUID(run.id),+ invocation_id=UUID(invocation.id),+ batch_id=batch_id or uuid4(),+ final=final,+ spans=[+ AgentTraceSpan(+ trace_id=run.trace_id,+ span_id=span_id,+ parent_span_id=invocation.span_id,+ name="Agent execution",+ start_time_ns=1_000_000_000,+ end_time_ns=2_000_000_000,+ status="ok",+ attributes={"gen_ai.operation.name": "invoke_agent"},+ )+ ],+ )+++def ingest(collection: Collection, batch: AgentTraceBatch) -> tuple[int, str]:+ return collection.repository.ingest(batch, max_spans=100, max_bytes=1_000_000)+++def terminal(collection: Collection) -> None:+ run = collection.run+ root = CanonicalSpan(+ id=run.root_span_id,+ parent_id=None,+ name="Agent turn",+ kind=CanonicalSpanKind.CHAIN,+ start_time=datetime.fromtimestamp(0, UTC),+ end_time=datetime.fromtimestamp(3, UTC),+ status=CanonicalSpanStatus.OK,+ )+ trace = CanonicalTrace(trace_id=run.trace_id, session_id=run.session_id, root_span_id=root.id, spans=(root,))+ collection.repository.terminal(run.id, trace.model_dump(mode="json"), root.id)+++def claim(collection: Collection) -> tuple[ApplicationTraceRun, str] | None:+ return collection.repository.claim(collection.run.id, collection_timeout=60, lease_seconds=30)+++def test_batch_duplicate_is_idempotent_after_final_and_acceptance(collection: Collection) -> None:+ invocation = collection.invocation()+ batch = batch_for(collection.run, invocation)+ assert ingest(collection, batch) == (1, collection.run.id)+ assert ingest(collection, batch) == (1, collection.run.id)+ terminal(collection)+ leased = claim(collection)+ assert leased is not None+ collection.repository.complete(+ collection.run.id, leased[1], state="accepted", result=TraceExportResult("provider-id")+ )+ assert ingest(collection, batch) == (1, collection.run.id)+ stored = collection.repository.get(collection.run.id)+ assert stored is not None+ assert stored.span_count == 1+ with collection.sessions() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 1+++def test_conflicting_batch_id_rolls_back_without_replacing_payload(collection: Collection) -> None:+ batch = batch_for(collection.run, collection.invocation(), final=False)+ ingest(collection, batch)+ changed = batch.model_copy(update={"final": True})+ with pytest.raises(TraceCollectionError, match="tracing_batch_conflict"):+ ingest(collection, changed)+ invocations, batches = collection.repository.collected(collection.run.id)+ assert invocations[0].finalized is False+ assert batches[0].final is False+++def test_run_cannot_ingest_another_runs_invocation(collection: Collection) -> None:+ other = seed(collection.sessions, "other")+ foreign = other.invocation()+ forged = batch_for(collection.run, foreign)+ with pytest.raises(TraceCollectionError, match="tracing_invocation_not_found"):+ ingest(collection, forged)+ with collection.sessions() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0+++def test_payload_limits_reject_whole_transaction(collection: Collection) -> None:+ batch = batch_for(collection.run, collection.invocation())+ with pytest.raises(TraceCollectionError, match="tracing_payload_limit"):+ collection.repository.ingest(batch, max_spans=100, max_bytes=1)+ invocations, batches = collection.repository.collected(collection.run.id)+ assert not batches+ assert invocations[0].finalized is False+ stored = collection.repository.get(collection.run.id)+ assert stored is not None+ assert stored.span_count == 0+++def test_wrong_trace_id_is_rejected_before_persistence(collection: Collection) -> None:+ batch = batch_for(collection.run, collection.invocation())+ altered = batch.model_copy(update={"spans": [batch.spans[0].model_copy(update={"trace_id": "f" * 32})]})+ with pytest.raises(TraceCollectionError, match="tracing_trace_id_mismatch"):+ ingest(collection, altered)+ _, batches = collection.repository.collected(collection.run.id)+ assert not batches+++def test_content_off_cleans_the_durable_batch_not_only_the_export(collection: Collection) -> None:+ original = batch_for(collection.run, collection.invocation(), final=False)+ observation = original.spans[0].model_copy(+ update={+ "status": "error",+ "status_message": "private error",+ "attributes": {+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": "model-one",+ "gen_ai.input.messages": "private prompt",+ "gen_ai.tool.call.arguments": "private arguments",+ "dify.payload": "private metadata",+ "authorization": "private credential",+ },+ "events": [+ AgentTraceEvent(+ name="exception", timestamp_ns=1_500_000_000, attributes={"exception.message": "private event"}+ )+ ],+ }+ )+ batch = original.model_copy(update={"spans": [observation]})+ ingest(collection, batch)+ _, batches = collection.repository.collected(collection.run.id)+ assert "private" not in json.dumps(batches[0].spans)+ assert batches[0].spans[0]["events"] == []+ assert batches[0].spans[0]["status"] == "error"+ assert batches[0].spans[0]["attributes"]["gen_ai.request.model"] == "model-one"+ # A changed original payload is a conflict even if content-off sanitization+ # would produce exactly the same persisted span.+ changed = batch.model_copy(+ update={"spans": [observation.model_copy(update={"status_message": "different private error"})]}+ )+ with pytest.raises(TraceCollectionError, match="tracing_batch_conflict"):+ ingest(collection, changed)+++def test_invocation_key_replay_must_not_change_agent_binding(collection: Collection) -> None:+ original = collection.invocation()+ repeated = collection.invocation()+ assert original.id == repeated.id+ with pytest.raises(TraceCollectionError, match="tracing_invocation_conflict"):+ collection.repository.invocation(+ run_id=collection.run.id,+ agent_id=identifier("different-agent"),+ node_execution_id=identifier("different-node"),+ attempt=2,+ call_key="first",+ )+++def test_claim_waits_for_all_invocations_final_and_then_only_one_lease(collection: Collection) -> None:+ first, second = collection.invocation("first"), collection.invocation("second")+ terminal(collection)+ ingest(collection, batch_for(collection.run, first))+ assert claim(collection) is None+ ingest(collection, batch_for(collection.run, second, span_id="b" * 16))+ leased = claim(collection)+ assert leased is not None+ assert claim(collection) is None+ assert leased[0].attempts == 1+++def test_expired_delivery_lease_accepts_confirmed_late_ack_for_same_attempt(collection: Collection) -> None:+ ingest(collection, batch_for(collection.run, collection.invocation()))+ terminal(collection)+ leased = claim(collection)+ assert leased is not None+ with collection.sessions.begin() as session:+ stored = session.get(ApplicationTraceRun, collection.run.id)+ assert stored is not None+ stored.export_started_at = naive_utc_now() - timedelta(seconds=60)+ stored.export_heartbeat_at = stored.export_started_at+ assert claim(collection) is None+ current = collection.repository.get(collection.run.id)+ assert current is not None+ assert current.state == "uncertain"+ collection.repository.complete(collection.run.id, leased[1], state="accepted", result=TraceExportResult("late"))+ stored = collection.repository.get(collection.run.id)+ assert stored is not None+ assert stored.state == "accepted"+ assert stored.error_code is None+ assert stored.accepted_at is not None+ assert stored.provider_trace_id == "late"+ assert claim(collection) is None+++def test_incomplete_final_times_out_without_export(collection: Collection) -> None:+ collection.invocation()+ terminal(collection)+ with collection.sessions.begin() as session:+ stored = session.get(ApplicationTraceRun, collection.run.id)+ assert stored is not None+ stored.terminal_at = naive_utc_now() - timedelta(seconds=120)+ assert claim(collection) is None+ stored = collection.repository.get(collection.run.id)+ assert stored is not None+ assert stored.state == "failed"+ assert stored.error_code == "agent_trace_collection_incomplete"+++def test_running_trace_freezes_credentials_content_policy_and_session(collection: Collection) -> None:+ original = collection.run+ with collection.sessions.begin() as session:+ app = session.get(App, original.app_id)+ assert app is not None+ app.tracing = json.dumps({"enabled": False, "tracing_provider": "opik", "capture_content": True})+ config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == original.app_id))+ assert config is not None+ config.tracing_config = {"secret_key": "encrypted-new"}+ repeated = collection.repository.start(+ tenant_id=original.tenant_id,+ app_id=original.app_id,+ source_kind="message",+ source_id=original.source_id,+ session_id="changed",+ )+ assert repeated is not None+ assert repeated.provider == "langfuse"+ assert repeated.provider_config["secret_key"] == "encrypted-original"+ assert repeated.capture_content is False+ assert repeated.config_revision == original.config_revision+ assert repeated.session_id == "session-original"++[email protected]("outcome", ["accepted", "uncertain", "retrying"])+def test_delivery_checkpoint_controls_replays(+ collection: Collection, monkeypatch: pytest.MonkeyPatch, outcome: str+) -> None:+ ingest(collection, batch_for(collection.run, collection.invocation()))+ terminal(collection)+ exports: list[tuple[str, dict[str, object], CanonicalTrace]] = []+ decryptions: list[dict[str, object]] = []++ def decrypt(_tenant: str, _provider: str, config: dict[str, object]) -> dict[str, object]:+ decryptions.append(config)+ return {"secret_key": "plaintext-only-in-worker"}++ def export(provider: str, config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:+ exports.append((provider, config, trace))+ if outcome == "uncertain":+ raise TraceExportUncertainError("ack lost")+ if outcome == "retrying":+ raise RetryableTraceDispatchError("busy")+ return TraceExportResult("vendor-trace", "https://trace.test/example")++ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", decrypt)+ monkeypatch.setattr(service_module, "export_trace", export)+ service = ApplicationTracingService(collection.repository)+ service.deliver(collection.run.id)+ service.deliver(collection.run.id)+ assert len(exports) == 1+ assert decryptions[0]["secret_key"] == "encrypted-original"+ stored = collection.repository.get(collection.run.id)+ assert stored is not None+ assert stored.state == outcome+ assert stored.provider_config["secret_key"] == "encrypted-original"+ assert "plaintext-only-in-worker" not in json.dumps(stored.outer_trace)+ assert exports[0][1]["host"] == "https://langfuse.test"+++def test_source_owner_checks_app_and_tenant(collection: Collection) -> None:+ run = collection.run+ assert (+ collection.repository.start(+ tenant_id=identifier("foreign-tenant"),+ app_id=run.app_id,+ source_kind="message",+ source_id=run.source_id,+ session_id="x",+ )+ is None+ )+ other = seed(collection.sessions, "second-app")+ assert (+ collection.repository.start(+ tenant_id=run.tenant_id,+ app_id=other.run.app_id,+ source_kind="message",+ source_id=run.source_id,+ session_id="x",+ )+ is None+ )+++def test_disabled_source_decision_stays_disabled_after_config_enable(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ case = seed(sqlite_session_factory, "initially-disabled", enabled=False)+ service = ApplicationTracingService(case.repository)+ assert case.run.state == "not_collected"+ with case.sessions.begin() as session:+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.tracing = json.dumps({"enabled": True, "tracing_provider": "langfuse", "capture_content": True})+ message = session.get(Message, case.run.source_id)+ assert message is not None+ assert (+ service.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="message",+ source_id=case.run.source_id,+ session_id="later-session",+ )+ is None+ )+ info = MessageTraceInfo(+ metadata={},+ message_data=message,+ message_id=message.id,+ conversation_model="agent",+ conversation_mode="agent",+ message_tokens=1,+ answer_tokens=1,+ total_tokens=2,+ )+ assert service.capture_terminal(info) is True+ stored = case.repository.get(case.run.id)+ assert stored is not None+ assert stored.state == "not_collected"+ assert stored.provider_config == {}+ assert stored.outer_trace is None+++def test_capture_agent_disabled_preserves_outer_only_content_policy(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ case = seed(sqlite_session_factory, "outer-only", capture_agent=False)+ service = ApplicationTracingService(case.repository)+ scheduled: list[str] = []+ monkeypatch.setattr(service, "schedule", scheduled.append)+ assert (+ service.prepare_invocation(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ agent_id=identifier("agent"),+ source_kind="message",+ source_id=case.run.source_id,+ session_id="session",+ node_execution_id=None,+ attempt=0,+ call_key="call",+ )+ is None+ )+ with case.sessions() as session:+ message = session.get(Message, case.run.source_id)+ assert message is not None+ info = MessageTraceInfo(+ metadata={},+ message_data=message,+ message_id=message.id,+ conversation_model="agent",+ conversation_mode="agent",+ message_tokens=1,+ answer_tokens=1,+ total_tokens=2,+ inputs="private input",+ outputs="private output",+ )+ assert service.capture_terminal(info) is True+ assert scheduled == [case.run.id]+ stored = case.repository.get(case.run.id)+ assert stored is not None+ assert stored.outer_trace is not None+ assert "private" not in json.dumps(stored.outer_trace)+ assert len(stored.outer_trace["spans"]) == 1+ invocations, _ = case.repository.collected(case.run.id)+ assert invocations == []+++def test_deleting_selected_configuration_keeps_historical_destination(collection: Collection) -> None:+ from machinery.context import RequestContext+ from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository+ from services.app_tracing_config_gateway import OpsTraceManagerGateway+ from services.app_tracing_config_service import AppTracingConfigService++ original = collection.run+ service = AppTracingConfigService(+ configs=SQLAlchemyAppTracingConfigRepository(session_factory=collection.sessions),+ provider=OpsTraceManagerGateway(),+ )+ service.delete(RequestContext("request", None, "account", original.tenant_id), original.app_id, "langfuse")+ stored = collection.repository.get(original.id)+ assert stored is not None+ assert stored.provider_config == original.provider_config+ assert stored.provider_config["project_url"] == "https://langfuse.test/project/original"+ assert stored.config_revision == original.config_revision+ with collection.sessions() as session:+ assert session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == original.app_id)) is None+ app = session.get(App, original.app_id)+ assert app is not None+ assert app.tracing is not None+ assert json.loads(app.tracing)["enabled"] is False+ assert json.loads(app.tracing)["tracing_provider"] is Nonediff --git a/api/tests/unit_tests/services/tracing/test_application_trace_handoff.py b/api/tests/unit_tests/services/tracing/test_application_trace_handoff.pynew file mode 100644index 00000000000..55ed619edfa--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_handoff.py@@ -0,0 +1,468 @@+"""SQLite ownership regressions at the actual legacy-task handoff boundary."""++import json+from dataclasses import dataclass+from datetime import datetime, timedelta+from uuid import UUID, uuid4++import httpx+import pytest+from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceSpan+from flask import Flask+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+from sqlalchemy import delete, func, select+from sqlalchemy.exc import OperationalError+from sqlalchemy.orm import Session, sessionmaker++from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext+from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom+from core.ops.base_trace_instance import bind_application_tracing_hooks+from core.ops.entities.trace_entity import (+ BaseTraceInfo,+ GenerateNameTraceInfo,+ MessageTraceInfo,+ ModerationTraceInfo,+ WorkflowTraceInfo,+)+from core.ops.ops_trace_manager import OpsTraceManager+from extensions.ext_database import db+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus+from models.agent_config_entities import AgentSoulConfig+from models.application_trace import ApplicationTraceRun+from models.enums import ConversationFromSource, CreatorUserRole, WorkflowRunTriggeredFrom+from models.model import App, AppMode, Message+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun, WorkflowType+from services.tracing import runtime_bridge+from services.tracing.provider_dispatch import export_trace+from services.tracing.service import ApplicationTracingService+from tasks import ops_trace_task+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_account, make_conversation, make_workflow+from tests.unit_tests.services.tracing.test_application_trace_collection import Collection, identifier, seed, terminal++AGENT_GRAPH = {+ "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}],+ "edges": list[object](),+}+++@dataclass+class Case:+ collection: Collection+ service: ApplicationTracingService+ scheduled: list[str]+++def setup(+ sessions: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, name: str, mode: AppMode, *, snapshot: bool+) -> Case:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ collection = seed(sessions, name)+ with sessions.begin() as session:+ app = session.get(App, collection.run.app_id)+ message = session.get(Message, collection.run.source_id)+ assert app is not None+ assert message is not None+ app.mode = mode+ session.add(+ make_conversation(+ conversation_id=message.conversation_id,+ app_id=app.id,+ mode=mode,+ inputs={},+ from_source=ConversationFromSource.CONSOLE,+ )+ )+ if not snapshot:+ session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id == collection.run.id))+ service = ApplicationTracingService(collection.repository)+ scheduled: list[str] = []+ monkeypatch.setattr(service, "schedule", scheduled.append)+ monkeypatch.setattr(runtime_bridge, "_service", lambda: service)+ return Case(collection, service, scheduled)+++def message_info(case: Case) -> MessageTraceInfo:+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ return MessageTraceInfo(+ metadata={"app_id": message.app_id},+ message_id=message.id,+ message_data=message.to_dict(session=session),+ conversation_model="agent",+ conversation_mode="agent",+ message_tokens=0,+ answer_tokens=0,+ total_tokens=0,+ inputs="private input",+ outputs="private output",+ )+++def dispatch(case: Case, info: BaseTraceInfo, monkeypatch: pytest.MonkeyPatch) -> list[BaseTraceInfo]:+ calls: list[BaseTraceInfo] = []+ failures: list[str] = []++ class Legacy:+ def trace(self, restored: BaseTraceInfo) -> None:+ calls.append(restored)++ payload = json.dumps({"trace_info_type": type(info).__name__, "trace_info": info.model_dump(mode="json")})+ monkeypatch.setattr(ops_trace_task.storage, "load", lambda _path: payload)+ monkeypatch.setattr(ops_trace_task.storage, "delete", lambda _path: None)+ monkeypatch.setattr(ops_trace_task.redis_client, "incr", failures.append)+ monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lambda _app: Legacy())+ monkeypatch.setattr("extensions.ext_enterprise_telemetry.is_enabled", lambda: False)+ app = Flask(__name__)+ bind_application_tracing_hooks(app, runtime_bridge.RuntimeApplicationTracingHooks())+ with app.app_context():+ ops_trace_task.process_trace_tasks.run({"app_id": case.collection.run.app_id, "file_id": "fixture"})+ assert failures == []+ return calls+++def workflow(+ case: Case, name: str, *, has_agent: bool, snapshot: bool+) -> tuple[WorkflowRun, ApplicationTraceRun | None]:+ graph = AGENT_GRAPH if has_agent else {"nodes": [{"data": {"type": "llm"}}], "edges": list[object]()}+ now = datetime(2026, 10, 15)+ with case.collection.sessions.begin() as session:+ definition = make_workflow(+ workflow_id=identifier(f"workflow-{name}"),+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ graph=graph,+ )+ session.add(definition)+ app = session.get(App, case.collection.run.app_id)+ assert app is not None+ app.workflow_id = definition.id+ row = WorkflowRun(+ id=identifier(f"workflow-run-{name}"),+ app_id=app.id,+ tenant_id=app.tenant_id,+ workflow_id=definition.id,+ type=WorkflowType.WORKFLOW,+ version="1",+ graph=json.dumps(graph),+ inputs="{}",+ outputs="{}",+ status=WorkflowExecutionStatus.SUCCEEDED,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=identifier("owner"),+ created_at=now,+ finished_at=now + timedelta(seconds=2),+ elapsed_time=2,+ total_tokens=0,+ total_steps=0,+ )+ session.add(row)+ trace_run = (+ case.collection.repository.start(+ tenant_id=row.tenant_id,+ app_id=row.app_id,+ source_kind="workflow",+ source_id=row.id,+ session_id="workflow-session",+ )+ if snapshot+ else None+ )+ return row, trace_run+++def workflow_info(row: WorkflowRun, parent: WorkflowRun | None = None) -> WorkflowTraceInfo:+ metadata: dict[str, object] = {"app_id": row.app_id}+ if parent is not None:+ metadata["parent_trace_context"] = {+ "parent_workflow_run_id": parent.id,+ "parent_node_execution_id": identifier("parent-tool-node"),+ }+ return WorkflowTraceInfo(+ workflow_data=row,+ workflow_id=row.workflow_id,+ tenant_id=row.tenant_id,+ workflow_run_id=row.id,+ workflow_run_elapsed_time=2,+ workflow_run_status="succeeded",+ workflow_run_inputs={},+ workflow_run_outputs={},+ workflow_run_version="1",+ total_tokens=0,+ file_list=[],+ query="",+ metadata=metadata,+ start_time=row.created_at,+ end_time=row.finished_at,+ )+++def test_chatflow_message_event_cannot_export_again_via_legacy(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ case = setup(sqlite_session_factory, monkeypatch, "chatflow", AppMode.ADVANCED_CHAT, snapshot=False)+ row, trace_run = workflow(case, "chatflow", has_agent=True, snapshot=True)+ assert trace_run is not None+ terminal(Collection(case.collection.repository, case.collection.sessions, trace_run))+ before = case.collection.repository.get(trace_run.id)+ assert before is not None+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.workflow_run_id = row.id+ assert dispatch(case, message_info(case), monkeypatch) == []+ after = case.collection.repository.get(trace_run.id)+ assert after is not None+ assert after.outer_trace == before.outer_trace+ assert case.scheduled == []+ with case.collection.sessions() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceRun)) == 1++[email protected]("mode", [AppMode.AGENT, AppMode.ADVANCED_CHAT])+def test_snapshot_failure_is_persisted_and_never_falls_back(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, mode: AppMode+) -> None:+ case = setup(sqlite_session_factory, monkeypatch, "failure", mode, snapshot=False)++ def fail_start(**_kwargs: object) -> None:+ raise OperationalError("snapshot", {}, Exception("temporary database unavailable"))++ monkeypatch.setattr(case.collection.repository, "start", fail_start)+ context = AgentAppRuntimeBuildContext(+ dify_context=DifyRunContext(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ user_id=identifier("user"),+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ ),+ agent_id=identifier("agent"),+ idempotency_key=case.collection.run.source_id,+ conversation_id="conversation",+ agent_config_snapshot_id=identifier("snapshot"),+ agent_soul=AgentSoulConfig(),+ user_query="query",+ binding_id=identifier("binding"),+ backend_binding_ref="backend",+ )+ assert runtime_bridge.prepare_agent_app(context) is None+ assert dispatch(case, message_info(case), monkeypatch) == []+ stored = case.collection.repository.source_run(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ source_kind="message",+ source_id=case.collection.run.source_id,+ )+ assert stored is not None+ assert stored.state == "failed"+ assert stored.error_code == "tracing_initialization_failed"+ assert stored.provider_config == {}++[email protected]("mode", [AppMode.AGENT, AppMode.ADVANCED_CHAT, AppMode.CHAT])[email protected]("kind", ["title", "moderation", "aliased_moderation"])+def test_auxiliary_events_resolve_owner_without_bypassing_content_policy(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, mode: AppMode, kind: str+) -> None:+ case = setup(sqlite_session_factory, monkeypatch, "aux", mode, snapshot=False)+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ if kind == "title":+ info: BaseTraceInfo = GenerateNameTraceInfo(+ tenant_id=case.collection.run.tenant_id,+ conversation_id=message.conversation_id,+ metadata={"conversation_id": message.conversation_id},+ inputs="private conversation",+ outputs="private generated title",+ )+ else:+ info = ModerationTraceInfo(+ message_id=identifier("workflow-app-log") if kind == "aliased_moderation" else message.id,+ message_data=message.to_dict(session=session),+ metadata={"message_id": message.id},+ flagged=False,+ action="direct_output",+ preset_response="",+ query="private prompt",+ inputs="private prompt",+ outputs="private output",+ )+ legacy = dispatch(case, info, monkeypatch)+ assert len(legacy) == (1 if mode is AppMode.CHAT else 0)+ assert case.scheduled == []+++def test_plain_workflow_preserves_legacy_parent_context(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ parent_case = setup(sqlite_session_factory, monkeypatch, "plain-parent", AppMode.WORKFLOW, snapshot=False)+ parent, _ = workflow(parent_case, "plain-parent", has_agent=False, snapshot=False)+ child_case = setup(sqlite_session_factory, monkeypatch, "plain-child", AppMode.WORKFLOW, snapshot=False)+ child, _ = workflow(child_case, "plain-child", has_agent=False, snapshot=False)+ info = workflow_info(child, parent)+ original = dict(info.metadata)+ assert child_case.service.capture_terminal(info) is False+ assert info.metadata == original+ with child_case.collection.sessions() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceRun)) == 0+++def test_plain_chatflow_auxiliary_events_keep_the_legacy_route(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ case = setup(sqlite_session_factory, monkeypatch, "plain-chatflow", AppMode.ADVANCED_CHAT, snapshot=False)+ row, _ = workflow(case, "plain-chatflow", has_agent=False, snapshot=False)+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.workflow_run_id = row.id+ info = GenerateNameTraceInfo(+ tenant_id=row.tenant_id,+ conversation_id=message.conversation_id,+ metadata={"conversation_id": message.conversation_id},+ inputs="legacy input",+ outputs="legacy title",+ )+ assert len(dispatch(case, info, monkeypatch)) == 1+++def test_legacy_child_of_managed_parent_uses_explicit_link_not_unavailable_context(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ parent_case = setup(sqlite_session_factory, monkeypatch, "managed-parent", AppMode.WORKFLOW, snapshot=False)+ parent, parent_trace = workflow(parent_case, "managed-parent", has_agent=True, snapshot=True)+ assert parent_trace is not None+ child_case = setup(sqlite_session_factory, monkeypatch, "legacy-child", AppMode.WORKFLOW, snapshot=False)+ child, _ = workflow(child_case, "legacy-child", has_agent=False, snapshot=False)+ info = workflow_info(child, parent)+ assert child_case.service.capture_terminal(info) is False+ assert info.resolved_parent_context == (None, None)+ assert info.metadata["linked_parent_workflow_run_id"] == parent.id+ assert info.metadata["linked_parent_node_execution_id"] == identifier("parent-tool-node")+ assert info.metadata["linked_parent_application_trace_id"] == parent_trace.id+ assert info.metadata["linked_parent_trace_id"] == parent_trace.trace_id+++def test_mixed_workflow_exports_normal_llm_and_agent_calls_without_double_usage(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ case = setup(sqlite_session_factory, monkeypatch, "mixed", AppMode.WORKFLOW, snapshot=False)+ row, managed = workflow(case, "mixed", has_agent=True, snapshot=True)+ assert managed is not None+ agent_node_id = identifier("agent-execution")+ owner = identifier("mixed-owner")+ with case.collection.sessions.begin() as session:+ app = session.get(App, row.app_id)+ assert app is not None+ app.created_by = owner+ session.add(make_account(account_id=owner))+ for index, kind in enumerate(("llm", "agent")):+ node_id = identifier("normal-execution") if kind == "llm" else agent_node_id+ usage: dict[str, int] = (+ {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} if kind == "llm" else {}+ )+ session.add(+ WorkflowNodeExecutionModel(+ id=node_id,+ node_execution_id=node_id,+ tenant_id=row.tenant_id,+ app_id=row.app_id,+ workflow_id=row.workflow_id,+ workflow_run_id=row.id,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ index=index,+ node_id=kind,+ node_type=kind,+ title=kind,+ inputs="{}",+ outputs="{}",+ process_data=json.dumps(+ {+ "model_mode": "chat",+ "model_name": "ordinary-model",+ "model_provider": "openai",+ "usage": usage,+ }+ )+ if kind == "llm"+ else "{}",+ status=WorkflowNodeExecutionStatus.SUCCEEDED,+ elapsed_time=1,+ execution_metadata=json.dumps(+ {"total_tokens": 15 if kind == "llm" else 10, "total_price": "0.02", "currency": "USD"}+ ),+ created_at=row.created_at,+ finished_at=row.finished_at,+ created_by=owner,+ created_by_role=CreatorUserRole.ACCOUNT,+ )+ )+ invocation = case.collection.repository.invocation(+ run_id=managed.id, agent_id=identifier("agent"), node_execution_id=agent_node_id, attempt=0, call_key="agent:0"+ )+ spans = [+ AgentTraceSpan(+ trace_id=managed.trace_id,+ span_id="a" * 16,+ parent_span_id=invocation.span_id,+ name="agent run",+ start_time_ns=1_000_000_000,+ end_time_ns=3_000_000_000,+ status="ok",+ attributes={"gen_ai.operation.name": "invoke_agent"},+ )+ ]+ spans.extend(+ AgentTraceSpan(+ trace_id=managed.trace_id,+ span_id=key * 16,+ parent_span_id="a" * 16,+ name="model call",+ start_time_ns=1_000_000_000,+ end_time_ns=2_000_000_000,+ status="ok",+ attributes={+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": "agent-model",+ "gen_ai.usage.input_tokens": 3,+ "gen_ai.usage.output_tokens": 2,+ "dify.llm.total_price": "0.01",+ "dify.llm.currency": "USD",+ },+ )+ for key in ("b", "c")+ )+ case.collection.repository.ingest(+ AgentTraceBatch(+ run_id=UUID(managed.id), invocation_id=UUID(invocation.id), batch_id=uuid4(), final=True, spans=spans+ ),+ max_spans=100,+ max_bytes=100000,+ )+ monkeypatch.setattr(type(db), "engine", property(lambda _db: sqlite_session_factory.kw["bind"]))+ assert case.service.capture_terminal(workflow_info(row)) is True+ stored = case.collection.repository.get(managed.id)+ assert stored is not None+ trace = case.service.assemble(stored)+ payloads: list[ExportTraceServiceRequest] = []++ def send(method: str, url: str, **kwargs: object) -> httpx.Response:+ body = kwargs["content"]+ assert isinstance(body, bytes)+ payloads.append(ExportTraceServiceRequest.FromString(body))+ return httpx.Response(200, content=b"", request=httpx.Request(method, url))++ monkeypatch.setattr("core.helper.ssrf_proxy.make_request", send)+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)+ observations = payloads[0].resource_spans[0].scope_spans[0].spans+ attributes = [{item.key: item.value.string_value for item in span.attributes} for span in observations]+ generations = [item for item in attributes if item["langfuse.observation.type"] == "generation"]+ assert len(generations) == 3+ assert sum(json.loads(item["langfuse.observation.usage_details"])["total"] for item in generations) == 25+ assert sum(json.loads(item["langfuse.observation.cost_details"])["total"] for item in generations) == pytest.approx(+ 0.04+ )diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_logstore.py b/api/tests/unit_tests/services/tracing/test_application_trace_logstore.pynew file mode 100644index 00000000000..22bc2132e8f--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_logstore.py@@ -0,0 +1,422 @@+"""Configured LogStore readers work without SQL mirrors and retain source ownership."""++import json+from collections.abc import Iterator+from dataclasses import dataclass+from datetime import timedelta+from typing import Literal++import pytest+from aliyun.log.logclient import LogClient+from sqlalchemy import Connection, Engine, event, select+from sqlalchemy.orm import Session, sessionmaker++from core.ops.entities.trace_entity import ModerationTraceInfo+from extensions.logstore.aliyun_logstore import AliyunLogStore+from extensions.logstore.repositories.logstore_api_workflow_run_repository import _dict_to_workflow_run+from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceRun+from models.model import App, AppMode+from models.workflow import WorkflowNodeExecutionModel, WorkflowRun+from repositories.application_trace_repository import TraceCollectionError+from services.tracing.provider_dispatch import TraceExportResult+from services.tracing.recovery import _workflow_task+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_workflow+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ Collection,+ batch_for,+ claim,+ identifier,+ ingest,+ seed,+ terminal,+)+from tests.unit_tests.services.tracing.test_application_trace_handoff import AGENT_GRAPH, workflow_info+++@dataclass+class StoredSource:+ case: Collection+ kind: Literal["workflow", "node"]+ row: dict[str, object]+ requests: list[str]++ def start(self) -> ApplicationTraceRun:+ run = self.case.repository.start(+ tenant_id=self.case.run.tenant_id,+ app_id=self.case.run.app_id,+ source_kind=self.kind,+ source_id=str(self.row["id"]),+ session_id="logstore-session",+ )+ assert run is not None+ return run++ def service(self) -> ApplicationTracingService:+ return ApplicationTracingService(self.case.repository)++ def status(self, node_execution_id: str | None = None) -> dict[str, object]:+ return self.service().status(+ tenant_id=self.case.run.tenant_id,+ app_id=self.case.run.app_id,+ source_kind=self.kind,+ source_id=str(self.row["id"]),+ node_execution_id=node_execution_id,+ )++[email protected](params=["workflow", "node"])+def stored(+ request: pytest.FixtureRequest,+ sqlite_engine: Engine,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> Iterator[StoredSource]:+ kind = request.param+ assert kind in {"workflow", "node"}+ case = seed(sqlite_session_factory, "logstore")+ with case.sessions.begin() as session:+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.mode = AppMode.WORKFLOW+ now = naive_utc_now()+ row: dict[str, object] = {+ "id": identifier("logstore-source"),+ "tenant_id": case.run.tenant_id,+ "app_id": case.run.app_id,+ "workflow_id": identifier("definition"),+ "created_by": identifier("owner"),+ "created_at": (now - timedelta(seconds=30)).isoformat(),+ "finished_at": (now - timedelta(seconds=10)).isoformat(),+ "status": "succeeded",+ "elapsed_time": 20,+ "inputs": json.dumps({"query": "private input"}),+ "outputs": json.dumps({"result": "private output"}),+ "node_id": "agent-node",+ "node_type": "agent",+ "title": "Agent preview",+ "graph": json.dumps(AGENT_GRAPH),+ }+ apply_config_overrides(+ monkeypatch,+ LOGSTORE_DUAL_WRITE_ENABLED=False,+ API_WORKFLOW_RUN_REPOSITORY=(+ "extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository"+ ),+ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=(+ "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository."+ "LogstoreAPIWorkflowNodeExecutionRepository"+ ),+ )+ monkeypatch.setenv("LOGSTORE_DUAL_READ_ENABLED", "false")+ monkeypatch.setenv("LOGSTORE_PG_MODE_ENABLED", "false")+ monkeypatch.setenv("ALIYUN_SLS_ACCESS_KEY_ID", "test-access-key")+ monkeypatch.setenv("ALIYUN_SLS_ACCESS_KEY_SECRET", "test-secret")+ monkeypatch.setenv("ALIYUN_SLS_ENDPOINT", "https://sls.example.test")+ monkeypatch.setenv("ALIYUN_SLS_REGION", "test-region")+ monkeypatch.setattr(AliyunLogStore, "_instance", None)+ monkeypatch.setattr(AliyunLogStore, "_initialized", False)+ monkeypatch.setattr(AliyunLogStore, "_check_endpoint_connectivity", staticmethod(lambda _endpoint, _timeout: None))++ def reject_http_transport(*_args: object, **_kwargs: object) -> None:+ pytest.fail("Configured LogStore test escaped its isolated transport boundary")++ monkeypatch.setattr(LogClient, "_getHttpResponse", reject_http_transport)+ transactions: set[Connection] = set()++ def begun(connection: Connection) -> None:+ transactions.add(connection)++ def ended(connection: Connection) -> None:+ transactions.discard(connection)++ requests: list[str] = []++ def get_logs(_client: AliyunLogStore, *, query: str, logstore: str, **_kwargs: object) -> list[dict[str, object]]:+ # Exercise real configured repository construction, scoped query generation,+ # and model decoding; replace only the remote transport boundary.+ assert not transactions, "LogStore I/O must not hold a SQL transaction"+ assert f'tenant_id:"{case.run.tenant_id}"' in query.replace(": ", ":")+ assert logstore == (+ AliyunLogStore.workflow_execution_logstore+ if kind == "workflow"+ else AliyunLogStore.workflow_node_execution_logstore+ )+ if kind == "workflow":+ assert f'app_id:"{case.run.app_id}"' in query+ requests.append(query)+ return [dict(row)] if row else []++ monkeypatch.setattr(AliyunLogStore, "get_logs", get_logs)++ def execute_sql(+ client: AliyunLogStore, *, sql: str, query: str, logstore: str, **_kwargs: object+ ) -> list[dict[str, object]]:+ # The node reader now uses SQL in both SDK and PG modes. Preserve the+ # real repository's query/decoding path while isolating this transport.+ assert kind == "node"+ assert f'id:"{identifier("logstore-source")}"' in query.replace(": ", ":")+ assert f"tenant_id = '{case.run.tenant_id}'" in sql+ assert f"id = '{identifier('logstore-source')}'" in sql+ assert f'FROM "{logstore}"' in sql+ assert "ORDER BY log_version DESC LIMIT 1" in sql+ return get_logs(client, query=query, logstore=logstore)++ monkeypatch.setattr(AliyunLogStore, "execute_sql", execute_sql)+ event.listen(sqlite_engine, "begin", begun)+ event.listen(sqlite_engine, "commit", ended)+ event.listen(sqlite_engine, "rollback", ended)+ try:+ yield StoredSource(case, kind, row, requests)+ finally:+ event.remove(sqlite_engine, "begin", begun)+ event.remove(sqlite_engine, "commit", ended)+ event.remove(sqlite_engine, "rollback", ended)+++def test_status_and_retry_use_logstore_without_sql_source_rows(+ stored: StoredSource, monkeypatch: pytest.MonkeyPatch+) -> None:+ run = stored.start()+ with stored.case.sessions() as session:+ assert session.scalar(select(WorkflowRun)) is None+ assert session.scalar(select(WorkflowNodeExecutionModel)) is None+ with stored.case.sessions.begin() as session:+ current = session.get(ApplicationTraceRun, run.id)+ assert current is not None+ current.state = "failed"+ current.error_code = "provider_export_rejected"+ current.outer_trace = {"persisted": True}+ assert stored.status()["status"] == "failed"+ scheduled: list[str] = []+ service = stored.service()+ monkeypatch.setattr(service, "schedule", scheduled.append)+ stored.requests.clear()+ result = service.retry(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ source_kind=stored.kind,+ source_id=run.source_id,+ acknowledge_possible_duplicates=False,+ )+ assert result["status"] == "ready"+ assert scheduled == [run.id]+ assert len(stored.requests) == 1++[email protected]("field", ["id", "tenant_id", "app_id"])+def test_backend_result_must_match_complete_source_owner(stored: StoredSource, field: str) -> None:+ run = stored.start()+ stored.row[field] = identifier("foreign")+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found") as caught:+ stored.service().status(+ tenant_id=run.tenant_id, app_id=run.app_id, source_kind=stored.kind, source_id=run.source_id+ )+ assert caught.value.status == 404+ assert stored.case.repository.get(run.id) is not None+++def test_foreign_app_owner_is_rejected_before_logstore_request(stored: StoredSource) -> None:+ stored.start()+ stored.requests.clear()+ with stored.case.sessions.begin() as session:+ app = session.get(App, stored.case.run.app_id)+ assert app is not None+ app.tenant_id = identifier("other-tenant")+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found"):+ stored.status()+ assert stored.requests == []+++def test_initialization_failure_uses_configured_source_reader(stored: StoredSource) -> None:+ run = stored.case.repository.record_initialization_failure(+ tenant_id=stored.case.run.tenant_id,+ app_id=stored.case.run.app_id,+ source_kind=stored.kind,+ source_id=str(stored.row["id"]),+ )+ assert run is not None+ assert run.state == "failed"+ assert run.error_code == "tracing_initialization_failed"+++def test_existing_snapshot_reentry_does_not_consult_logstore(stored: StoredSource) -> None:+ original = stored.start()+ stored.requests.clear()+ stored.row.clear()+ current = stored.case.repository.start(+ tenant_id=original.tenant_id,+ app_id=original.app_id,+ source_kind=stored.kind,+ source_id=original.source_id,+ session_id="must-not-replace",+ )+ assert current is not None+ assert current.id == original.id+ assert current.session_id == original.session_id+ assert stored.requests == []++[email protected]("stored", ["workflow"], indirect=True)+def test_workflow_node_deep_link_is_scoped_to_durable_invocation(stored: StoredSource) -> None:+ run = stored.start()+ case = Collection(stored.case.repository, stored.case.sessions, run)+ node_id = identifier("logstore-agent-node")+ invocation = case.repository.invocation(+ run_id=run.id, agent_id=identifier("agent"), node_execution_id=node_id, attempt=0, call_key="node"+ )+ ingest(case, batch_for(run, invocation))+ terminal(case)+ lease = claim(case)+ assert lease is not None+ case.repository.complete(+ run.id,+ lease[1],+ state="accepted",+ result=TraceExportResult("provider-id", span_urls={invocation.span_id: "https://trace.test/span"}),+ )+ assert stored.status(node_id)["span_url"] == "https://trace.test/span"+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found"):+ stored.status(identifier("another-run-node"))+++def test_preview_settlement_and_workflow_recovery_use_configured_source(stored: StoredSource) -> None:+ run = stored.start()+ if stored.kind == "workflow":+ task = _workflow_task(stored.case.repository, run, naive_utc_now())+ assert task is not None+ assert task.workflow_run_id == run.source_id+ assert task.kwargs["trace_session_id"] == run.session_id+ stored.row["status"] = "running"+ assert _workflow_task(stored.case.repository, run, naive_utc_now()) is None+ else:+ stored.service().settle_preview(run)+ current = stored.case.repository.get(run.id)+ assert current is not None+ assert current.outer_trace is not None+ root = current.outer_trace["spans"][0]+ assert root["status"] == "ok"+ assert root["inputs"] is None+ assert root["outputs"] is None+ assert root["metadata"]["node_execution_id"] == run.source_id++[email protected]("stored", ["workflow"], indirect=True)+def test_missing_snapshot_classifies_logstore_agent_workflow_and_persists_failure(stored: StoredSource) -> None:+ info = workflow_info(_dict_to_workflow_run(stored.row))+ assert stored.service().capture_terminal(info) is True+ result = stored.status()+ assert result["status"] == "failed"+ assert result["error_code"] == "tracing_initialization_failed"++[email protected]("stored", ["workflow"], indirect=True)[email protected]("has_agent", [True, False])+def test_unindexed_workflow_uses_owned_definition_for_safe_legacy_decision(+ stored: StoredSource, has_agent: bool+) -> None:+ info = workflow_info(_dict_to_workflow_run(stored.row))+ with stored.case.sessions.begin() as session:+ app = session.get(App, stored.case.run.app_id)+ assert app is not None+ definition = make_workflow(+ workflow_id=str(stored.row["workflow_id"]),+ tenant_id=app.tenant_id,+ app_id=app.id,+ graph=AGENT_GRAPH if has_agent else {"nodes": [{"data": {"type": "llm"}}]},+ )+ app.workflow_id = definition.id+ session.add(definition)+ stored.row.clear()+ assert stored.service().capture_terminal(info) is has_agent++[email protected]("stored", ["workflow"], indirect=True)[email protected]("snapshot", [True, False])+def test_auxiliary_content_is_suppressed_for_logstore_workflow(stored: StoredSource, snapshot: bool) -> None:+ if snapshot:+ stored.start()+ info = ModerationTraceInfo(+ message_id=identifier("workflow-log"),+ metadata={+ "workflow_run_id": stored.row["id"],+ **({} if snapshot else {"app_id": stored.case.run.app_id}),+ },+ flagged=False,+ action="direct_output",+ preset_response="",+ query="private query",+ inputs="private input",+ outputs="private output",+ )+ assert stored.service().capture_terminal(info) is True+ if snapshot:+ # Durable ownership is sufficient to suppress auxiliary payload even when+ # the remote run has not been indexed yet.+ stored.row.clear()+ assert stored.service().capture_terminal(info) is True+ else:+ stored.row["graph"] = json.dumps({"nodes": [{"data": {"type": "llm"}}]})+ assert stored.service().capture_terminal(info) is False++[email protected]("stored", ["workflow"], indirect=True)[email protected]("mismatch", [None, "tenant_id", "app_id"])+def test_linked_parent_requires_logstore_run_and_durable_owner_to_agree(+ stored: StoredSource, mismatch: str | None+) -> None:+ parent = stored.start()+ info = workflow_info(_dict_to_workflow_run(stored.row), _dict_to_workflow_run(stored.row))+ original = dict(info.metadata)+ if mismatch:+ stored.row[mismatch] = identifier("foreign")+ stored.service()._link_managed_parent(info, parent.tenant_id)+ if mismatch:+ assert info.metadata == original+ else:+ assert info.resolved_parent_context == (None, None)+ assert info.metadata["linked_parent_trace_id"] == parent.trace_id+ assert info.metadata["linked_parent_workflow_run_id"] == parent.source_id++[email protected]("stored", ["workflow"], indirect=True)+def test_auxiliary_durable_owner_rejects_app_reassigned_to_other_tenant(stored: StoredSource) -> None:+ run = stored.start()+ with stored.case.sessions.begin() as session:+ app = session.get(App, run.app_id)+ assert app is not None+ app.tenant_id = identifier("foreign")+ info = ModerationTraceInfo(+ metadata={"workflow_run_id": run.source_id}, query="", flagged=False, action="direct_output", preset_response=""+ )+ assert stored.service().capture_terminal(info) is False+++def test_sql_workflow_source_still_uses_scoped_configured_reader(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ case = seed(sqlite_session_factory, "sql-reader")+ row = _dict_to_workflow_run(+ {+ "id": identifier("sql-run"),+ "tenant_id": case.run.tenant_id,+ "app_id": case.run.app_id,+ "workflow_id": identifier("definition"),+ "created_by": identifier("owner"),+ }+ )+ with case.sessions.begin() as session:+ session.add(row)+ source = case.repository._source(+ tenant_id=case.run.tenant_id, app_id=case.run.app_id, kind="workflow", source_id=row.id+ )+ assert isinstance(source, WorkflowRun)+ assert source.id == row.id+ assert (+ case.repository._source(+ tenant_id=identifier("wrong-tenant"), app_id=case.run.app_id, kind="workflow", source_id=row.id+ )+ is None+ )diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_node_alias.py b/api/tests/unit_tests/services/tracing/test_application_trace_node_alias.pynew file mode 100644index 00000000000..59f9116f177--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_node_alias.py@@ -0,0 +1,386 @@+"""Node logs resolve the caller's trace using SQL or configured LogStore ownership."""++import json+from collections.abc import Iterator+from dataclasses import dataclass++import pytest+from aliyun.log.logclient import LogClient+from sqlalchemy import Connection, Engine, event, select+from sqlalchemy.orm import Session, sessionmaker++from extensions.logstore.aliyun_logstore import AliyunLogStore+from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (+ _dict_to_workflow_node_execution_model,+)+from extensions.logstore.repositories.logstore_api_workflow_run_repository import _dict_to_workflow_run+from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceRun+from models.model import App, AppMode+from models.workflow import WorkflowNodeExecutionModel, WorkflowRun+from repositories.application_trace_repository import TraceCollectionError+from services.tracing.provider_dispatch import TraceExportResult+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ Collection,+ batch_for,+ claim,+ identifier,+ ingest,+ seed,+ terminal,+)+++@dataclass+class NodeAlias:+ case: Collection+ node: dict[str, object]+ workflow: dict[str, object]+ requests: list[str]+ logstore: bool++ def service(self) -> ApplicationTracingService:+ return ApplicationTracingService(self.case.repository)++ def status(self, node_execution_id: str | None = None) -> dict[str, object]:+ return self.service().status(+ tenant_id=self.case.run.tenant_id,+ app_id=self.case.run.app_id,+ source_kind="node",+ source_id=str(self.node["id"]),+ node_execution_id=node_execution_id,+ )++ def accept(self, *, spans: bool = True) -> tuple[str, str]:+ span_urls: dict[str, str] = {}+ for attempt in (0, 1):+ invocation = self.case.repository.invocation(+ run_id=self.case.run.id,+ agent_id=identifier("referenced-agent"),+ node_execution_id=str(self.node["id"]),+ attempt=attempt,+ call_key=f"attempt-{attempt}",+ )+ ingest(self.case, batch_for(self.case.run, invocation, span_id=str(attempt + 1) * 16))+ span_urls[invocation.span_id] = f"https://provider.test/span/{attempt}"+ terminal(self.case)+ lease = claim(self.case)+ assert lease is not None+ self.case.repository.complete(+ self.case.run.id,+ lease[1],+ state="accepted",+ result=TraceExportResult(+ "provider-trace", "https://provider.test/trace", span_urls=span_urls if spans else {}+ ),+ )+ return "https://provider.test/trace", "https://provider.test/span/1"++[email protected](params=["sql", "logstore"])+def alias(+ request: pytest.FixtureRequest,+ sqlite_engine: Engine,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> Iterator[NodeAlias]:+ case = seed(sqlite_session_factory, "node-alias")+ with case.sessions.begin() as session:+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.mode = AppMode.WORKFLOW+ common: dict[str, object] = {+ "tenant_id": case.run.tenant_id,+ "app_id": case.run.app_id,+ "workflow_id": identifier("definition"),+ "created_by": identifier("actor"),+ "created_at": naive_utc_now().isoformat(),+ "status": "succeeded",+ }+ workflow = {**common, "id": identifier("workflow-source"), "graph": json.dumps({"nodes": []})}+ node = {+ **common,+ "id": identifier("node-source"),+ "workflow_run_id": workflow["id"],+ "node_id": "reference-agent",+ "node_type": "agent",+ "title": "Reference Agent",+ }+ requests: list[str] = []+ transactions: set[Connection] = set()++ def begun(connection: Connection) -> None:+ transactions.add(connection)++ def ended(connection: Connection) -> None:+ transactions.discard(connection)++ if request.param == "sql":+ with case.sessions.begin() as session:+ session.add(_dict_to_workflow_run(workflow))+ session.add(_dict_to_workflow_node_execution_model(node))+ else:+ apply_config_overrides(+ monkeypatch,+ LOGSTORE_DUAL_WRITE_ENABLED=False,+ API_WORKFLOW_RUN_REPOSITORY=(+ "extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository"+ ),+ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=(+ "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository."+ "LogstoreAPIWorkflowNodeExecutionRepository"+ ),+ )+ for key, value in {+ "LOGSTORE_DUAL_READ_ENABLED": "false",+ "LOGSTORE_PG_MODE_ENABLED": "false",+ "ALIYUN_SLS_ACCESS_KEY_ID": "test-access-key",+ "ALIYUN_SLS_ACCESS_KEY_SECRET": "test-secret",+ "ALIYUN_SLS_ENDPOINT": "https://sls.example.test",+ "ALIYUN_SLS_REGION": "test-region",+ }.items():+ monkeypatch.setenv(key, value)+ monkeypatch.setattr(AliyunLogStore, "_instance", None)+ monkeypatch.setattr(AliyunLogStore, "_initialized", False)+ monkeypatch.setattr(+ AliyunLogStore, "_check_endpoint_connectivity", staticmethod(lambda _endpoint, _timeout: None)+ )++ def reject_http_transport(*_args: object, **_kwargs: object) -> None:+ pytest.fail("Configured LogStore test escaped its isolated transport boundary")++ monkeypatch.setattr(LogClient, "_getHttpResponse", reject_http_transport)++ def get_logs(+ _client: AliyunLogStore, *, query: str, logstore: str, **_kwargs: object+ ) -> list[dict[str, object]]:+ assert not transactions, "Configured source reads must not hold a SQL transaction"+ assert f'tenant_id:"{case.run.tenant_id}"' in query.replace(": ", ":")+ assert logstore in {+ AliyunLogStore.workflow_execution_logstore,+ AliyunLogStore.workflow_node_execution_logstore,+ }+ is_workflow = logstore == AliyunLogStore.workflow_execution_logstore+ if is_workflow:+ assert f'app_id:"{case.run.app_id}"' in query+ requests.append("workflow" if is_workflow else "node")+ row = workflow if is_workflow else node+ return [dict(row)] if row else []++ monkeypatch.setattr(AliyunLogStore, "get_logs", get_logs)++ def execute_sql(+ client: AliyunLogStore, *, sql: str, query: str, logstore: str, **_kwargs: object+ ) -> list[dict[str, object]]:+ # Node lookups use the SQL transport in SDK mode too; the real+ # service still validates the returned node-to-workflow owner chain.+ assert logstore == AliyunLogStore.workflow_node_execution_logstore+ assert f'id:"{identifier("node-source")}"' in query.replace(": ", ":")+ assert f"tenant_id = '{case.run.tenant_id}'" in sql+ assert f"id = '{identifier('node-source')}'" in sql+ assert f'FROM "{logstore}"' in sql+ assert "ORDER BY log_version DESC LIMIT 1" in sql+ return get_logs(client, query=query, logstore=logstore)++ monkeypatch.setattr(AliyunLogStore, "execute_sql", execute_sql)+ with case.sessions() as session:+ assert session.scalar(select(WorkflowRun)) is None+ assert session.scalar(select(WorkflowNodeExecutionModel)) is None+ event.listen(sqlite_engine, "begin", begun)+ event.listen(sqlite_engine, "commit", ended)+ event.listen(sqlite_engine, "rollback", ended)+ try:+ run = case.repository.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="workflow",+ source_id=str(workflow["id"]),+ session_id="caller-session",+ )+ assert run is not None+ requests.clear()+ yield NodeAlias(+ Collection(case.repository, case.sessions, run), node, workflow, requests, request.param == "logstore"+ )+ finally:+ event.remove(sqlite_engine, "begin", begun)+ event.remove(sqlite_engine, "commit", ended)+ event.remove(sqlite_engine, "rollback", ended)++[email protected]("spans", [True, False])+def test_alias_returns_parent_trace_and_latest_owned_invocation(alias: NodeAlias, spans: bool) -> None:+ trace_url, span_url = alias.accept(spans=spans)+ status = alias.status()+ assert status["status"] == "accepted"+ assert status["source_kind"] == "node"+ assert status["source_id"] == alias.node["id"]+ assert status["trace_id"] == "provider-trace"+ assert status["trace_url"] == trace_url+ assert status["span_url"] == (span_url if spans else None)+ assert status["config_revision"] == alias.case.run.config_revision+ if alias.logstore:+ assert alias.requests == ["node", "workflow"]++[email protected]("capture_agent", [True, False])[email protected]("accepted", [True, False])+def test_alias_without_internal_invocation_returns_parent_status(+ alias: NodeAlias, capture_agent: bool, accepted: bool+) -> None:+ with alias.case.sessions.begin() as session:+ run = session.get(ApplicationTraceRun, alias.case.run.id)+ assert run is not None+ run.capture_agent = capture_agent+ if accepted:+ terminal(alias.case)+ lease = claim(alias.case)+ assert lease is not None+ alias.case.repository.complete(+ alias.case.run.id,+ lease[1],+ state="accepted",+ result=TraceExportResult("outer-only", "https://provider.test/outer"),+ )+ status = alias.status()+ assert status["status"] == ("accepted" if accepted else "collecting")+ assert status["span_url"] is None+ assert status["capture_agent"] is capture_agent+ assert status["trace_url"] == ("https://provider.test/outer" if accepted else None)++[email protected]("state", ["collecting", "not_collected"])+def test_standalone_node_snapshot_wins_over_parent(alias: NodeAlias, state: str) -> None:+ alias.accept()+ node_run = alias.case.repository.start(+ tenant_id=alias.case.run.tenant_id,+ app_id=alias.case.run.app_id,+ source_kind="node",+ source_id=str(alias.node["id"]),+ session_id="preview-session",+ )+ assert node_run is not None+ with alias.case.sessions.begin() as session:+ run = session.get(ApplicationTraceRun, node_run.id)+ assert run is not None+ run.state = state+ alias.requests.clear()+ status = alias.status()+ assert status["status"] == state+ assert status.get("trace_url") is None+ if alias.logstore:+ assert alias.requests == ["node"]+++def test_explicit_other_node_filter_cannot_select_sibling_span(+ alias: NodeAlias, monkeypatch: pytest.MonkeyPatch+) -> None:+ sibling = alias.case.repository.invocation(+ run_id=alias.case.run.id,+ agent_id=identifier("referenced-agent"),+ node_execution_id=identifier("sibling-node"),+ attempt=0,+ call_key="sibling",+ )+ ingest(alias.case, batch_for(alias.case.run, sibling, span_id="f" * 16))+ alias.accept()+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found") as error:+ alias.status(identifier("sibling-node"))+ assert error.value.status == 404+ service = alias.service()+ scheduled: list[str] = []+ monkeypatch.setattr(service, "schedule", scheduled.append)+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found"):+ service.retry(+ tenant_id=alias.case.run.tenant_id,+ app_id=alias.case.run.app_id,+ source_kind="node",+ source_id=str(alias.node["id"]),+ node_execution_id=identifier("sibling-node"),+ acknowledge_possible_duplicates=True,+ )+ assert not scheduled++[email protected]("state", ["failed", "uncertain", "accepted"])+def test_alias_retry_uses_parent_guards_and_resolves_sources_once(+ alias: NodeAlias, state: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ alias.accept()+ with alias.case.sessions.begin() as session:+ run = session.get(ApplicationTraceRun, alias.case.run.id)+ assert run is not None+ run.state = state+ service = alias.service()+ scheduled: list[str] = []+ monkeypatch.setattr(service, "schedule", scheduled.append)++ def retry(acknowledge: bool) -> dict[str, object]:+ alias.requests.clear()+ return service.retry(+ tenant_id=alias.case.run.tenant_id,+ app_id=alias.case.run.app_id,+ source_kind="node",+ source_id=str(alias.node["id"]),+ acknowledge_possible_duplicates=acknowledge,+ )++ if state == "accepted":+ with pytest.raises(TraceCollectionError, match="tracing_already_accepted") as error:+ retry(True)+ assert error.value.status == 409+ assert not scheduled+ else:+ if state == "uncertain":+ with pytest.raises(TraceCollectionError, match="tracing_duplicate_acknowledgement_required"):+ retry(False)+ assert not scheduled+ assert retry(state == "uncertain")["status"] == "ready"+ assert scheduled == [alias.case.run.id]+ if alias.logstore:+ assert alias.requests == ["node", "workflow"]++[email protected]("source", ["node", "workflow"])[email protected]("field", ["tenant_id", "app_id", "workflow_id"])+def test_alias_rejects_broken_owner_chain(alias: NodeAlias, source: str, field: str) -> None:+ alias.accept()+ row = alias.node if source == "node" else alias.workflow+ if alias.logstore:+ row[field] = identifier("foreign")+ else:+ with alias.case.sessions.begin() as session:+ model = session.get(WorkflowNodeExecutionModel if source == "node" else WorkflowRun, str(row["id"]))+ assert model is not None+ # Keep field assignments explicit so ownership failures are visible.+ if field == "tenant_id":+ model.tenant_id = identifier("foreign")+ elif field == "app_id":+ model.app_id = identifier("foreign")+ else:+ model.workflow_id = identifier("foreign")+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found") as error:+ alias.status()+ assert error.value.status == 404+++def test_alias_missing_parent_business_source_is_not_a_durable_owner_bypass(alias: NodeAlias) -> None:+ if alias.logstore:+ alias.workflow.clear()+ else:+ with alias.case.sessions.begin() as session:+ parent = session.get(WorkflowRun, str(alias.workflow["id"]))+ assert parent is not None+ session.delete(parent)+ with pytest.raises(TraceCollectionError, match="tracing_source_not_found"):+ alias.status()+++def test_owned_alias_without_parent_trace_is_not_collected(alias: NodeAlias) -> None:+ with alias.case.sessions.begin() as session:+ run = session.get(ApplicationTraceRun, alias.case.run.id)+ assert run is not None+ session.delete(run)+ assert alias.status() == {"status": "not_collected", "source_kind": "node", "source_id": alias.node["id"]}diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_normalization.py b/api/tests/unit_tests/services/tracing/test_application_trace_normalization.pynew file mode 100644index 00000000000..870b812dc2c--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_normalization.py@@ -0,0 +1,218 @@+"""Content policy and model-call accounting across all canonical tree layers."""++import json+from datetime import UTC, datetime++import pytest++from core.helper.trace_id_helper import ParentTraceContext+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.normalization import normalize_agent_span, order_spans, remap_outer_trace, safe_attributes+from services.tracing.provider_common import token_usage, usd_cost+++def agent_data(+ attributes: dict[str, object], *, span_id: str = "2" * 16, parent_id: str = "1" * 16+) -> dict[str, object]:+ return {+ "span_id": span_id,+ "parent_span_id": parent_id,+ "name": "operation",+ "start_time_ns": 1_000_000_000,+ "end_time_ns": 2_000_000_000,+ "status": "error",+ "status_message": "sensitive error output",+ "attributes": attributes,+ }+++def test_content_off_is_safe_for_all_metadata_namespaces() -> None:+ normalized = normalize_agent_span(+ agent_data(+ {+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": "model-one",+ "gen_ai.usage.input_tokens": 3,+ "gen_ai.usage.output_tokens": 2,+ "gen_ai.input.messages": [{"content": "private input"}],+ "gen_ai.output.messages": [{"content": "private output"}],+ "exception.message": "private exception",+ "dify.app_id": "app-one",+ "dify.custom_payload": {"prompt": "private metadata"},+ "gen_ai.request.messages": "private request metadata",+ "gen_ai.response.output": "private response metadata",+ "gen_ai.usage.custom_prompt": "private usage metadata",+ }+ ),+ capture_content=False,+ )+ assert normalized.inputs is None+ assert normalized.outputs is None+ assert "private" not in normalized.model_dump_json()+ assert "sensitive error" not in normalized.model_dump_json()+ assert normalized.metadata["dify.app_id"] == "app-one"+ assert normalized.metadata["model_name"] == "model-one"+ assert normalized.metadata["total_tokens"] == 5+++def test_json_encoded_tool_credentials_are_redacted_from_inputs_and_metadata() -> None:+ normalized = normalize_agent_span(+ agent_data(+ {+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.name": "search",+ "gen_ai.tool.call.arguments": json.dumps({"api_key": "tool-secret-123", "query": "public query"}),+ "gen_ai.tool.call.result": json.dumps(+ {"nested": {"authorization": "bearer-secret-123"}, "result": "okay"}+ ),+ }+ ),+ capture_content=True,+ )+ serialized = normalized.model_dump_json()+ assert "tool-secret-123" not in serialized+ assert "bearer-secret-123" not in serialized+ assert "public query" in serialized+ assert normalized.inputs["api_key"] == "[redacted]"+++def test_content_off_drops_error_event_payload_and_keeps_numeric_usage() -> None:+ attrs = safe_attributes(+ {+ "input": "user text",+ "authorization": "secret",+ "exception.stacktrace": "sensitive",+ "gen_ai.usage.input_tokens": 10,+ "gen_ai.tool.name": "lookup",+ },+ capture_content=False,+ )+ assert attrs == {"gen_ai.usage.input_tokens": 10, "gen_ai.tool.name": "lookup"}+++def test_real_runtime_accounting_attributes_survive_content_off() -> None:+ normalized = normalize_agent_span(+ agent_data(+ {+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": "gpt-4o-mini",+ "gen_ai.usage.input_tokens": 8,+ "gen_ai.usage.output_tokens": 4,+ "dify.llm.total_price": "0.03",+ "dify.llm.currency": "USD",+ "gen_ai.response.time_to_first_token": 0.2,+ "dify.llm.time_to_first_token_unit": "s",+ "dify.llm.time_to_first_token_source": "gateway_usage",+ "gen_ai.input.messages": "private input",+ }+ ),+ capture_content=False,+ )+ assert usd_cost(normalized) == 0.03+ assert normalized.metadata["currency"] == "USD"+ assert normalized.metadata["time_to_first_token"] == 0.2+ assert token_usage(normalized) == {"input": 8, "output": 4, "total": 12}+ assert normalized.inputs is None+++def span(identifier: str, parent: str | None, kind: CanonicalSpanKind, *, synthetic: bool = False) -> CanonicalSpan:+ return CanonicalSpan(+ id=identifier,+ parent_id=parent,+ name=identifier,+ kind=kind,+ start_time=datetime.fromtimestamp(0, UTC),+ end_time=datetime.fromtimestamp(3, UTC),+ status=CanonicalSpanStatus.OK,+ inputs={"content": "private"},+ outputs="private",+ metadata={+ "prompt_tokens": 6,+ "completion_tokens": 4,+ "total_tokens": 10,+ "total_price": "0.03",+ "prompt": "private",+ },+ synthetic=synthetic,+ )+++def test_two_native_model_calls_replace_synthetic_aggregate_generation() -> None:+ outer = CanonicalTrace(+ trace_id="old",+ session_id="session",+ root_span_id="message",+ spans=(+ span("message", None, CanonicalSpanKind.CHAIN),+ span("message:llm", "message", CanonicalSpanKind.LLM, synthetic=True),+ ),+ )+ remapped = remap_outer_trace(outer, trace_id="1" * 32, capture_content=False, agent_message=True)+ calls = [+ normalize_agent_span(+ agent_data(+ {+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": "model",+ "gen_ai.usage.input_tokens": 3,+ "gen_ai.usage.output_tokens": 2,+ },+ span_id=identifier,+ parent_id=remapped.root_span_id,+ ),+ capture_content=False,+ )+ for identifier in ("2" * 16, "3" * 16)+ ]+ combined = order_spans(remapped.root_span_id, [calls[1], *remapped.spans, calls[0]])+ assert combined[0].id == remapped.root_span_id+ generations = [item for item in combined if item.kind is CanonicalSpanKind.LLM]+ assert len(generations) == 2+ assert sum(token_usage(item)["total"] for item in generations) == 10+ assert token_usage(combined[0]) == {}+ assert combined[0].metadata["aggregate_total_tokens"] == 10+ assert "private" not in json.dumps([item.model_dump(mode="json") for item in combined])+++def test_non_agent_workflow_retains_its_real_llm_node() -> None:+ trace = CanonicalTrace(+ trace_id="old",+ session_id="session",+ root_span_id="workflow",+ spans=(+ span("workflow", None, CanonicalSpanKind.CHAIN),+ span("llm-node", "workflow", CanonicalSpanKind.LLM),+ ),+ )+ remapped = remap_outer_trace(trace, trace_id="1" * 32, capture_content=True, agent_message=False)+ assert len(remapped.spans) == 2+ assert remapped.spans[1].kind is CanonicalSpanKind.LLM+ assert token_usage(remapped.spans[1])["total"] == 10+++def test_order_spans_rejects_orphans_cycles_and_conflicting_ids() -> None:+ root = span("root", None, CanonicalSpanKind.CHAIN)+ with pytest.raises(ValueError, match="parent is missing"):+ order_spans("root", [root, span("orphan", "missing", CanonicalSpanKind.TOOL)])+ with pytest.raises(ValueError, match="parent cycle"):+ order_spans(+ "root", [root, span("one", "two", CanonicalSpanKind.TOOL), span("two", "one", CanonicalSpanKind.TOOL)]+ )+ with pytest.raises(ValueError, match="Conflicting"):+ order_spans("root", [root, root.model_copy(update={"name": "other"})])+++def test_managed_external_parent_is_preserved_as_explicit_link_when_content_off() -> None:+ trace = CanonicalTrace(+ trace_id="old",+ session_id="session",+ root_span_id="workflow",+ spans=(span("workflow", None, CanonicalSpanKind.CHAIN),),+ external_parent=ParentTraceContext(+ parent_workflow_run_id="outer-workflow", parent_node_execution_id="outer-tool"+ ),+ )+ remapped = remap_outer_trace(trace, trace_id="1" * 32, capture_content=False, agent_message=False)+ assert remapped.external_parent is None+ assert remapped.spans[0].metadata["linked_parent_workflow_run_id"] == "outer-workflow"+ assert remapped.spans[0].metadata["linked_parent_node_execution_id"] == "outer-tool"diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_recovery.py b/api/tests/unit_tests/services/tracing/test_application_trace_recovery.pynew file mode 100644index 00000000000..471366713c5--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_recovery.py@@ -0,0 +1,1215 @@+"""Recover lost terminal notifications using real business rows and TraceTask."""++import json+from collections.abc import Callable+from dataclasses import dataclass+from datetime import datetime, timedelta+from decimal import Decimal+from typing import Literal++import pytest+from sqlalchemy import Engine+from sqlalchemy.orm import Session, scoped_session, sessionmaker++from core.app.entities.app_invoke_entities import ChatAppGenerateEntity+from core.ops.ops_trace_manager import TraceTask+from core.ops.unified_trace.entities import CanonicalTrace+from extensions.ext_database import db+from graphon.enums import WorkflowExecutionStatus+from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceInvocation+from models.enums import ConversationFromSource, CreatorUserRole, MessageStatus, WorkflowRunTriggeredFrom+from models.model import App, AppMode, Message+from models.workflow import WorkflowRun, WorkflowType+from services.tracing import recovery+from services.tracing import service as service_module+from services.tracing.normalization import span_id+from services.tracing.provider_dispatch import TraceExportResult, TraceExportUncertainError+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.model_factories import make_account, make_conversation+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ Collection,+ batch_for,+ identifier,+ ingest,+ seed,+)+++@dataclass+class RecoveryCase:+ collection: Collection+ service: ApplicationTracingService+ now: datetime+ scheduled: list[str]+++def attach_conversation(case: Collection) -> None:+ with case.sessions.begin() as session:+ message = session.get(Message, case.run.source_id)+ assert message is not None+ session.add(+ make_conversation(+ conversation_id=message.conversation_id,+ app_id=case.run.app_id,+ mode=AppMode.AGENT,+ inputs={},+ from_source=ConversationFromSource.CONSOLE,+ )+ )+++def case(+ sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> RecoveryCase:+ collection = seed(sqlite_session_factory, "recover", capture_content=True)+ attach_conversation(collection)+ invocation = collection.invocation()+ ingest(collection, batch_for(collection.run, invocation))+ now = naive_utc_now()+ with sqlite_session_factory.begin() as session:+ message = session.get(Message, collection.run.source_id)+ assert message is not None+ stored_invocation = session.get(ApplicationTraceInvocation, invocation.id)+ assert message is not None+ assert stored_invocation is not None+ message.created_at = now - timedelta(seconds=30)+ message.updated_at = now - timedelta(seconds=10)+ message.answer = "persisted final answer"+ message.provider_response_latency = 1.0+ message.message_tokens = 2+ message.answer_tokens = 1+ stored_invocation.finalized_at = now - timedelta(seconds=10)+ monkeypatch.setattr(type(db), "engine", property(lambda _db: sqlite_engine))+ monkeypatch.setattr(db, "session", scoped_session(sqlite_session_factory))+ monkeypatch.setattr(TraceTask, "_workflow_run_repo", None)+ monkeypatch.setattr("core.telemetry.gateway.is_enterprise_telemetry_enabled", lambda: False)+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: now)+ service = ApplicationTracingService(collection.repository)+ scheduled: list[str] = []+ monkeypatch.setattr(service, "schedule", scheduled.append)+ return RecoveryCase(collection, service, now, scheduled)+++def complete_metadata() -> str:+ return json.dumps(+ {+ "annotation_reply": None,+ "retriever_resources": [],+ "reasoning": {},+ "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},+ }+ )+++def save_agent_stop(+ case: RecoveryCase,+ monkeypatch: pytest.MonkeyPatch,+ answer: str,+ *,+ mode: AppMode = AppMode.AGENT,+ consumer_session: Session | None = None,+ process_stream: bool = False,+ notify: Callable[[Message, ChatAppGenerateEntity], None] | None = None,+) -> list[TraceTask]:+ """Run the real consumer save and tracing queue against the owned SQL rows."""+ import queue++ from flask import Flask++ from core.app.app_config.entities import (+ EasyUIBasedAppConfig,+ EasyUIBasedAppModelConfigFrom,+ ModelConfigEntity,+ PromptTemplateEntity,+ )+ from core.app.entities.app_invoke_entities import InvokeFrom+ from core.app.entities.queue_entities import QueueStopEvent+ from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline+ from core.ops import ops_trace_manager+ from core.ops.base_trace_instance import bind_application_tracing_hooks+ from services.tracing import runtime_bridge+ from tests.unit_tests.config_override import apply_config_overrides+ from tests.unit_tests.core.app.task_pipeline.test_easy_ui_based_generate_task_pipeline import (+ _QueueManager,+ queue_message,+ )+ from tests.unit_tests.core.model_fixtures import make_model_config++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ monkeypatch.setattr(runtime_bridge, "_service", lambda: case.service)+ monkeypatch.setattr(ops_trace_manager, "trace_manager_queue", queue.Queue())+ monkeypatch.setattr(ops_trace_manager.TraceQueueManager, "start_timer", lambda _manager: None)++ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity) -> None:+ if notify is not None:+ notify(message, application_generate_entity)++ monkeypatch.setattr("core.app.task_pipeline.easy_ui_based_generate_task_pipeline.message_was_created.send", send)+ monkeypatch.setattr(+ "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.naive_utc_now",+ lambda: case.now - timedelta(seconds=10),+ )+ with case.collection.sessions.begin() as session:+ persisted = session.get(Message, case.collection.run.source_id)+ assert persisted is not None+ if persisted.message_metadata is None:+ persisted.message_metadata = complete_metadata()+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ conversation = session.get(service_module.Conversation, message.conversation_id)+ assert conversation is not None+ entity = ChatAppGenerateEntity(+ task_id=identifier("stop-race-task"),+ app_config=EasyUIBasedAppConfig(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ app_mode=mode,+ app_model_config_from=EasyUIBasedAppModelConfigFrom.APP_LATEST_CONFIG,+ app_model_config_dict={},+ model=ModelConfigEntity(provider="test-provider", model="test-model"),+ prompt_template=PromptTemplateEntity(prompt_type=PromptTemplateEntity.PromptType.SIMPLE),+ ),+ model_conf=make_model_config(provider="test-provider", model="test-model", mode="chat"),+ inputs={},+ files=[],+ user_id=identifier("stop-race-user"),+ stream=True,+ invoke_from=InvokeFrom.DEBUGGER,+ )+ queue_manager = _QueueManager()+ pipeline = EasyUIBasedGenerateTaskPipeline(entity, queue_manager, conversation, message, stream=True)+ pipeline._task_state.llm_result.message.content = answer+ application = Flask(__name__)+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())+ with application.app_context():+ trace_manager = ops_trace_manager.TraceQueueManager(app_id=case.collection.run.app_id)+ if process_stream:+ queue_manager.messages = [queue_message(QueueStopEvent(stopped_by=QueueStopEvent.StopBy.USER_MANUAL))]+ # Token estimates and moderation are separate external boundaries;+ # exercise real stream dispatch, SQL save, commit and notification.+ monkeypatch.setattr(pipeline, "_handle_stop", lambda _event: None)+ monkeypatch.setattr(pipeline, "handle_output_moderation_when_task_finished", lambda _answer: None)+ list(pipeline._process_stream_response(publisher=None, trace_manager=trace_manager))+ elif consumer_session is None:+ with case.collection.sessions.begin() as session:+ pipeline._save_message(session=session, trace_manager=trace_manager, preserve_existing_usage=True)+ else:+ pipeline._save_message(session=consumer_session, trace_manager=trace_manager, preserve_existing_usage=True)+ consumer_session.commit()+ return trace_manager.collect_tasks()++[email protected]("receiver_fails", [False, True])+def test_stopped_agent_notifies_the_real_provider_receiver_once_after_commit(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, receiver_fails: bool+) -> None:+ from sqlalchemy import inspect++ from core.entities.provider_entities import ProviderQuotaType, QuotaConfiguration, QuotaUnit, SystemConfiguration+ from events.event_handlers import update_provider_when_message_created as receiver++ calls: list[str] = []+ provider_updates: list[str] = []+ monkeypatch.setattr(+ receiver,+ "_execute_provider_updates",+ lambda updates: provider_updates.extend(operation.description for operation in updates),+ )++ def notify(message: Message, entity: ChatAppGenerateEntity) -> None:+ # The production session factory does not expire on commit. Check real+ # detached fields used by the actual receiver, not a mocked send order.+ assert inspect(message).detached is True+ assert not inspect(message).expired_attributes+ assert message.answer == "partial answer"+ with case.collection.sessions() as session:+ saved = session.get(Message, message.id)+ assert saved is not None+ assert recovery.agent_stop_requested(saved) is True+ assert saved.answer == message.answer+ system = SystemConfiguration(+ enabled=True,+ current_quota_type=ProviderQuotaType.TRIAL,+ quota_configurations=[+ QuotaConfiguration(+ quota_type=ProviderQuotaType.TRIAL,+ quota_unit=QuotaUnit.TOKENS,+ quota_limit=100,+ quota_used=0,+ is_valid=True,+ )+ ],+ )+ assert receiver._calculate_quota_usage(message=message, system_configuration=system, model_name="test") == 3+ receiver.handle(message, application_generate_entity=entity)+ calls.append(message.id)+ if receiver_fails:+ raise RuntimeError("receiver failed after commit")++ if receiver_fails:+ with pytest.raises(RuntimeError, match="receiver failed after commit"):+ save_agent_stop(case, monkeypatch, "partial answer", process_stream=True, notify=notify)+ else:+ assert save_agent_stop(case, monkeypatch, "partial answer", process_stream=True, notify=notify) == []+ assert calls == [case.collection.run.source_id]+ assert provider_updates == ["basic_last_used_update"]+ with case.collection.sessions() as session:+ saved = session.get(Message, case.collection.run.source_id)+ assert saved is not None+ assert recovery.agent_stop_requested(saved) is True++[email protected]("answer", ["", "partial answer before stop"])[email protected]("outcome", ["succeeded", "failed", "cancelled"])[email protected]("delivery", ["accepted", "uncertain"])+def test_stopped_consumer_cannot_freeze_a_terminal_before_the_runtime_worker(+ case: RecoveryCase,+ monkeypatch: pytest.MonkeyPatch,+ answer: str,+ outcome: Literal["succeeded", "failed", "cancelled"],+ delivery: str,+) -> None:+ from typing import cast++ from core.ops.entities.trace_entity import MessageTraceInfo, TraceTaskName+ from models.model import MessageDict++ # Simulate an old queued notification as well as the current consumer save:+ # persisted state, not a detached event payload, must decide who owns the end.+ notification = TraceTask(+ TraceTaskName.MESSAGE_TRACE,+ message_id=case.collection.run.source_id,+ conversation_id=identifier("conversation-recover"),+ )+ info = notification.execute()+ assert isinstance(info, MessageTraceInfo)+ assert isinstance(info.message_data, dict)+ info.message_data = Message.from_dict(cast(MessageDict, info.message_data))+ exported: list[CanonicalTrace] = []++ def export(_provider: str, _config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:+ exported.append(trace)+ if delivery == "uncertain":+ raise TraceExportUncertainError("acknowledgment lost")+ return TraceExportResult(trace_id=trace.trace_id)++ monkeypatch.setattr(service_module, "export_trace", export)+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ queued = save_agent_stop(case, monkeypatch, answer)+ assert case.service.capture_terminal(info) is True+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.outer_trace is None, "a stopped SSE save is not the runtime terminal"+ assert queued == []+ assert recovery.recover_terminal(case.service, stored) is False+ assert case.collection.repository.claim(stored.id, collection_timeout=60, lease_seconds=30) is None+ case.service.deliver(stored.id)+ assert exported == []++ output = "real runtime answer" if outcome == "succeeded" else None+ case.service.capture_agent_app_terminal(+ tenant_id=stored.tenant_id,+ app_id=stored.app_id,+ message_id=stored.source_id,+ outcome=outcome,+ output=output,+ )+ terminal = case.collection.repository.get(stored.id)+ assert terminal is not None+ trace = case.service.assemble(terminal)+ root = next(span for span in trace.spans if span.id == trace.root_span_id)+ assert root.metadata["message_execution_state"] == outcome+ assert root.status.value == ("ok" if outcome == "succeeded" else "error")+ assert len(trace.spans) == 3++ # A duplicate worker result and late consumer notification cannot rewrite+ # the frozen end or create a second ingestion attempt after acceptance.+ case.service.deliver(stored.id)+ case.service.deliver(stored.id)+ assert len(exported) == 1+ assert exported[0] == trace+ frozen = case.collection.repository.get(stored.id)+ assert frozen is not None+ assert frozen.state == delivery+ case.service.capture_agent_app_terminal(+ tenant_id=stored.tenant_id,+ app_id=stored.app_id,+ message_id=stored.source_id,+ outcome="succeeded" if outcome != "succeeded" else "cancelled",+ output="late result must not replace the accepted envelope",+ )+ assert case.service.capture_terminal(info) is True+ unchanged = case.collection.repository.get(stored.id)+ assert unchanged is not None+ assert unchanged.outer_trace == frozen.outer_trace+ assert unchanged.attempts == frozen.attempts == 1+ case.service.deliver(stored.id)+ assert len(exported) == 1+ with case.collection.sessions() as session:+ message = session.get(Message, stored.source_id)+ assert message is not None+ assert message.status is MessageStatus.NORMAL+ assert message.answer == answer+ assert message.message_tokens == 2+ assert message.answer_tokens == 1++[email protected]("mode", [AppMode.CHAT, AppMode.AGENT_CHAT])+def test_classic_stopped_consumer_keeps_its_existing_trace_notification(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, mode: AppMode+) -> None:+ queued = save_agent_stop(case, monkeypatch, "classic partial answer", mode=mode)+ assert len(queued) == 1+ assert queued[0].trace_type is service_module.TraceTaskName.MESSAGE_TRACE+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ assert "agent_stop_requested" not in json.loads(message.message_metadata or "{}")+++def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ from core.app.apps.agent_app.app_runner import AgentAppRunner+ from graphon.model_runtime.entities.llm_entities import LLMUsage++ # The worker's scoped session may already have loaded this Message before+ # the consumer commits its independent stop transaction.+ cached_message = db.session.get(Message, case.collection.run.source_id)+ assert cached_message is not None+ assert cached_message.message_metadata is None+ save_agent_stop(case, monkeypatch, "partial answer")+ AgentAppRunner._persist_message_usage(+ message_id=cached_message.id,+ usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}),+ )+ with case.collection.sessions() as session:+ message = session.get(Message, cached_message.id)+ assert message is not None+ assert recovery.agent_stop_requested(message) is True+ assert message.answer == "partial answer"+ assert message.message_tokens == 13+ assert message.answer_tokens == 8+ assert json.loads(message.message_metadata or "{}")["usage"]["total_tokens"] == 21+++def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_read(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ from core.app.apps.agent_app.app_runner import AgentAppRunner+ from graphon.model_runtime.entities.llm_entities import LLMUsage++ with case.collection.sessions.begin() as session:+ initial = session.get(Message, case.collection.run.source_id)+ assert initial is not None+ initial.message_tokens = 0+ initial.answer_tokens = 0+ with case.collection.sessions() as consumer:+ cached = consumer.get(Message, case.collection.run.source_id)+ assert cached is not None+ assert cached.message_tokens == 0+ AgentAppRunner._persist_message_usage(+ message_id=cached.id,+ usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}),+ )+ save_agent_stop(case, monkeypatch, "partial answer", consumer_session=consumer)+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ assert recovery.agent_stop_requested(message) is True+ assert message.message_tokens == 13+ assert message.answer_tokens == 8+ assert message.total_price == Decimal("0.21")+ assert json.loads(message.message_metadata or "{}")["usage"]["total_tokens"] == 21++[email protected]("runtime_outcome", ["succeeded", "failed", "cancelled", "cancel_wait_failed"])[email protected]("final_batch_arrived", [False, True])[email protected]("snapshot_save_fails", [False, True])+def test_stopped_runner_hands_off_only_an_observed_runtime_terminal(+ case: RecoveryCase,+ monkeypatch: pytest.MonkeyPatch,+ runtime_outcome: str,+ final_batch_arrived: bool,+ snapshot_save_fails: bool,+) -> None:+ from typing import cast++ from flask import Flask++ from clients.agent_backend import FakeAgentBackendRunClient, FakeAgentBackendScenario+ from core.app.apps.agent_app.session_store import AgentAppSessionScope+ from core.app.apps.base_app_queue_manager import AppQueueManager+ from core.app.apps.exc import GenerateTaskStoppedError+ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom+ from core.ops.base_trace_instance import bind_application_tracing_hooks+ from services.tracing import runtime_bridge+ from tests.unit_tests.config_override import apply_config_overrides+ from tests.unit_tests.core.app.apps.agent_app.test_app_runner import (+ _CancelAndWaitFailingClient,+ _ExplodingSessionStore,+ _FakeSessionStore,+ _runner,+ _StoppedQueueManager,+ )++ save_agent_stop(case, monkeypatch, "partial answer")+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_COLLECTION_TIMEOUT=60)+ pending_invocation = None+ if not final_batch_arrived:+ pending_invocation = case.collection.invocation("awaiting-stop-final-batch")+ ingest(case.collection, batch_for(case.collection.run, pending_invocation, final=False, span_id="b" * 16))+ client = (+ _CancelAndWaitFailingClient()+ if runtime_outcome == "cancel_wait_failed"+ else FakeAgentBackendRunClient(+ scenario=FakeAgentBackendScenario.FAILED+ if runtime_outcome == "failed"+ else FakeAgentBackendScenario.SUCCESS+ )+ )+ if runtime_outcome in {"succeeded", "failed"}:+ # The terminal frame has arrived, but stop is already visible at the+ # next check. Do not discard this known result and ask SDK cancellation+ # to reclassify it (the SDK rejects non-cancelled terminal events).+ arrived_terminal = client._events(client.run_id)[-1]+ monkeypatch.setattr(client, "stream_events", lambda _run_id, **_kwargs: iter([arrived_terminal]))++ def unexpected_cancel(*_args: object, **_kwargs: object) -> None:+ raise AssertionError("a known runtime terminal must not be cancelled again")++ monkeypatch.setattr(client, "cancel_run_and_wait", unexpected_cancel)+ store = _ExplodingSessionStore() if snapshot_save_fails else _FakeSessionStore()+ runner = _runner(client, store)+ run = case.collection.run+ context = DifyRunContext(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ user_id=identifier("stop-race-user"),+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ )+ scope = AgentAppSessionScope(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ conversation_id=identifier("conversation-recover"),+ agent_id=identifier("stop-agent"),+ agent_config_snapshot_id=identifier("stop-snapshot"),+ home_snapshot_id=None,+ )+ application = Flask(__name__)+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())+ with application.app_context(), pytest.raises(GenerateTaskStoppedError):+ runner._consume_stream(+ client.run_id,+ dify_context=context,+ message_id=run.source_id,+ queue_manager=cast(AppQueueManager, _StoppedQueueManager()),+ model_name="test-model",+ query="test query",+ session_scope=scope,+ binding_id=identifier("stop-binding"),+ )+ frozen = case.collection.repository.get(run.id)+ assert frozen is not None+ if runtime_outcome == "cancel_wait_failed":+ assert frozen.outer_trace is None+ assert store.saved == []+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: case.now + timedelta(seconds=120))+ assert recovery.recover_terminal(case.service, frozen) is False+ expired = case.collection.repository.get(run.id)+ assert expired is not None+ assert expired.state == "failed"+ assert expired.error_code == (+ "business_terminal_not_persisted" if final_batch_arrived else "agent_trace_collection_incomplete"+ )+ return+ assert frozen.outer_trace is not None+ trace = case.service.assemble(frozen)+ root = next(span for span in trace.spans if span.id == trace.root_span_id)+ assert root.metadata["message_execution_state"] == runtime_outcome+ assert root.status.value == ("ok" if runtime_outcome == "succeeded" else "error")+ assert len(store.saved) == (0 if snapshot_save_fails else 1)+ exported: list[CanonicalTrace] = []++ def export(_provider: str, _config: dict[str, object], payload: CanonicalTrace) -> TraceExportResult:+ exported.append(payload)+ return TraceExportResult(trace_id=payload.trace_id)++ monkeypatch.setattr(service_module, "export_trace", export)+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ if pending_invocation is not None:+ assert case.collection.repository.claim(run.id, collection_timeout=60, lease_seconds=30) is None+ case.service.deliver(run.id)+ assert exported == []+ ingest(case.collection, batch_for(run, pending_invocation, final=True, span_id="b" * 16))+ case.service.deliver(run.id)+ case.service.deliver(run.id)+ assert len(exported) == 1+ assert len(exported[0].spans) == (3 if final_batch_arrived else 5)+ accepted = case.collection.repository.get(run.id)+ assert accepted is not None+ assert accepted.state == "accepted"+ assert accepted.attempts == 1+ with case.collection.sessions() as session:+ message = session.get(Message, run.source_id)+ assert message is not None+ assert message.answer == "partial answer"+ assert recovery.agent_stop_requested(message) is True++[email protected]("incomplete", [False, True])+def test_missing_runtime_stop_terminal_converges_without_inventing_success(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, incomplete: bool+) -> None:+ from tests.unit_tests.config_override import apply_config_overrides++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_COLLECTION_TIMEOUT=60)+ save_agent_stop(case, monkeypatch, "partial answer")+ if incomplete:+ invocation = case.collection.invocation("still-collecting")+ ingest(case.collection, batch_for(case.collection.run, invocation, final=False, span_id="b" * 16))+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: case.now + timedelta(seconds=120))+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.state == "failed"+ assert stored.error_code == (+ "agent_trace_collection_incomplete" if incomplete else "business_terminal_not_persisted"+ )+ assert stored.outer_trace is None+ assert case.collection.repository.claim(stored.id, collection_timeout=60, lease_seconds=30) is None++[email protected]("stop_on_end", [False, True])+def test_stop_during_final_queue_publication_preserves_the_already_observed_success(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, stop_on_end: bool+) -> None:+ from cachetools import TTLCache+ from flask import Flask++ from clients.agent_backend import FakeAgentBackendRunClient+ from core.app.apps import base_app_queue_manager+ from core.app.apps.base_app_queue_manager import PublishFrom+ from core.app.apps.exc import GenerateTaskStoppedError+ from core.app.apps.execution_coordinator import set_app_task_stop_flag+ from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager+ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom+ from core.app.entities.queue_entities import (+ AppQueueEvent,+ MessageQueueMessage,+ QueueLLMChunkEvent,+ QueueMessageEndEvent,+ )+ from core.ops.base_trace_instance import bind_application_tracing_hooks+ from services.tracing import runtime_bridge+ from tests.unit_tests.core.app.apps.agent_app.test_app_runner import _FakeSessionStore, _runner, _soul++ save_agent_stop(case, monkeypatch, "partial answer")+ # Only Redis storage is in memory. Use the real queue's publish stop check+ # rather than an already-stopped fake consumer or an exception-only stub.+ flags: dict[str, object] = {}+ monkeypatch.setattr(base_app_queue_manager.redis_client, "get", flags.get)+ monkeypatch.setattr(+ base_app_queue_manager.redis_client, "setex", lambda key, _ttl, value: flags.__setitem__(key, value)+ )+ monkeypatch.setattr(+ "core.app.apps.agent_app.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None+ )+ task_id = identifier("stop-at-final-publish")+ queue_manager = MessageBasedAppQueueManager(+ task_id=task_id,+ user_id=identifier("stop-race-user"),+ invoke_from=InvokeFrom.DEBUGGER,+ conversation_id=identifier("conversation-recover"),+ app_mode=AppMode.AGENT,+ message_id=case.collection.run.source_id,+ )+ clock = [0.0]+ queue_manager._stopped_cache = TTLCache(maxsize=1, ttl=1, timer=lambda: clock[0])+ store = _FakeSessionStore()+ runner = _runner(FakeAgentBackendRunClient(), store)+ original_publish = queue_manager._publish++ def publish_after_stop(event: AppQueueEvent, pub_from: PublishFrom) -> None:+ if isinstance(event, QueueMessageEndEvent if stop_on_end else QueueLLMChunkEvent):+ set_app_task_stop_flag(task_id)+ clock[0] = 2.0 # Expire the real queue's earlier cached false result.+ original_publish(event, pub_from)++ monkeypatch.setattr(queue_manager, "_publish", publish_after_stop)+ run = case.collection.run+ context = DifyRunContext(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ user_id=identifier("stop-race-user"),+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ )+ application = Flask(__name__)+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())+ with application.app_context(), pytest.raises(GenerateTaskStoppedError):+ runner.run(+ dify_context=context,+ agent_id=identifier("stop-agent"),+ agent_config_snapshot_id=identifier("stop-snapshot"),+ agent_soul=_soul(),+ home_snapshot_id=None,+ conversation_id=identifier("conversation-recover"),+ query="test query",+ message_id=run.source_id,+ model_name="test-model",+ queue_manager=queue_manager,+ )+ published = queue_manager._q.get_nowait()+ assert isinstance(published, MessageQueueMessage)+ assert isinstance(published.event, QueueLLMChunkEvent)+ if stop_on_end:+ published_end = queue_manager._q.get_nowait()+ assert isinstance(published_end, MessageQueueMessage)+ assert isinstance(published_end.event, QueueMessageEndEvent)+ frozen = case.collection.repository.get(run.id)+ assert frozen is not None+ assert frozen.outer_trace is not None, "success arrived before the final queue publication saw Stop"+ trace = CanonicalTrace.model_validate(frozen.outer_trace)+ root = next(span for span in trace.spans if span.id == trace.root_span_id)+ assert root.metadata["message_execution_state"] == "succeeded"+ assert root.status.value == "ok"+ assert len(store.saved) == 1++[email protected]("stop_during_cleanup", [False, True])+def test_runtime_cleanup_cancellation_does_not_reclassify_a_business_stream_failure(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, stop_during_cleanup: bool+) -> None:+ from collections.abc import Iterator+ from typing import cast++ from dify_agent.protocol import RunEvent+ from flask import Flask++ from core.app.apps.agent_app.session_store import AgentAppSessionScope+ from core.app.apps.base_app_queue_manager import AppQueueManager+ from core.app.apps.exc import GenerateTaskStoppedError+ from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom+ from core.ops.base_trace_instance import bind_application_tracing_hooks+ from services.tracing import runtime_bridge+ from tests.unit_tests.config_override import apply_config_overrides+ from tests.unit_tests.core.app.apps.agent_app.test_app_runner import (+ _FakeQueueManager,+ _FakeSessionStore,+ _runner,+ _UsageCancellationClient,+ )++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ monkeypatch.setattr(runtime_bridge, "_service", lambda: case.service)+ client = _UsageCancellationClient()+ queue_manager = _FakeQueueManager()++ def disconnected_stream(*_args: object, **_kwargs: object) -> Iterator[RunEvent]:+ raise RuntimeError("stream transport failed")++ monkeypatch.setattr(client, "stream_events", disconnected_stream)+ if stop_during_cleanup:+ original_cancel = client.cancel_run_and_wait++ def cancel_then_stop(*_args: object, **_kwargs: object) -> RunEvent:+ terminal = original_cancel(client.run_id)+ queue_manager.request_stop()+ return terminal++ monkeypatch.setattr(client, "cancel_run_and_wait", cancel_then_stop)+ store = _FakeSessionStore()+ runner = _runner(client, store)+ run = case.collection.run+ context = DifyRunContext(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ user_id=identifier("cleanup-user"),+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ )+ scope = AgentAppSessionScope(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ conversation_id=identifier("conversation-recover"),+ agent_id=identifier("cleanup-agent"),+ agent_config_snapshot_id=identifier("cleanup-snapshot"),+ home_snapshot_id=None,+ )+ application = Flask(__name__)+ bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks())+ with application.app_context():+ with pytest.raises(GenerateTaskStoppedError if stop_during_cleanup else RuntimeError):+ runner._consume_stream(+ client.run_id,+ dify_context=context,+ message_id=run.source_id,+ queue_manager=cast(AppQueueManager, queue_manager),+ model_name="test-model",+ query="test query",+ session_scope=scope,+ binding_id=identifier("cleanup-binding"),+ )+ if not stop_during_cleanup:+ # This is the generator's existing generic exception handoff. A+ # cleanup child-run cancellation must not win the application root.+ runtime_bridge.capture_agent_app_terminal(+ tenant_id=run.tenant_id,+ app_id=run.app_id,+ message_id=run.source_id,+ outcome="failed",+ output=None,+ )+ frozen = case.collection.repository.get(run.id)+ assert frozen is not None+ assert frozen.outer_trace is not None+ trace = CanonicalTrace.model_validate(frozen.outer_trace)+ root = next(span for span in trace.spans if span.id == trace.root_span_id)+ assert root.metadata["message_execution_state"] == ("cancelled" if stop_during_cleanup else "failed")+ assert root.status.value == "error"+ assert len(store.saved) == 1+ with case.collection.sessions() as session:+ message = session.get(Message, run.source_id)+ assert message is not None+ assert message.message_tokens == 13+ assert message.answer_tokens == 8+ assert message.total_price == Decimal("0.21")+ exported: list[CanonicalTrace] = []++ def export(_provider: str, _config: dict[str, object], payload: CanonicalTrace) -> TraceExportResult:+ exported.append(payload)+ return TraceExportResult(trace_id=payload.trace_id)++ monkeypatch.setattr(service_module, "export_trace", export)+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ case.service.deliver(run.id)+ case.service.deliver(run.id)+ accepted = case.collection.repository.get(run.id)+ assert accepted is not None+ assert accepted.state == "accepted"+ assert accepted.attempts == 1+ assert len(exported) == 1++[email protected]("capture_agent", [False, True])+def test_missing_stop_worker_without_runtime_spans_also_reaches_a_delivery_failure(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, capture_agent: bool+) -> None:+ from tests.unit_tests.config_override import apply_config_overrides++ collection = seed(case.collection.sessions, f"stop-without-spans-{capture_agent}", capture_agent=capture_agent)+ attach_conversation(collection)+ stopped = RecoveryCase(collection, ApplicationTracingService(collection.repository), case.now, [])+ monkeypatch.setattr(stopped.service, "schedule", stopped.scheduled.append)+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_COLLECTION_TIMEOUT=60)+ save_agent_stop(stopped, monkeypatch, "")+ monkeypatch.setattr(recovery, "naive_utc_now", lambda: case.now + timedelta(seconds=120))+ assert recovery.recover_terminal(stopped.service, collection.run) is False+ stored = collection.repository.get(collection.run.id)+ assert stored is not None+ assert stored.state == "failed"+ assert stored.error_code == "business_terminal_not_persisted"+ assert stored.outer_trace is None++[email protected]("outcome", ["succeeded", "cancelled", "failed"])+def test_worker_terminal_survives_closed_sse_without_inventing_a_message_write(+ case: RecoveryCase, outcome: Literal["succeeded", "failed", "cancelled"]+) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.message_metadata = json.dumps({"usage": {"prompt_tokens": 2, "completion_tokens": 1}})+ message.answer = ""+ # A runtime final/usage flush still does not imply that an API message ended.+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ output = "actual worker result" if outcome == "succeeded" else None+ case.service.capture_agent_app_terminal(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ message_id=case.collection.run.source_id,+ outcome=outcome,+ output=output,+ )+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ trace = CanonicalTrace.model_validate(stored.outer_trace)+ root = trace.spans[0]+ assert root.metadata["message_execution_state"] == outcome+ assert root.status.value == ("ok" if outcome == "succeeded" else "error")+ assert root.outputs == (output or "")+ assert case.scheduled == [stored.id]+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ assert message.answer == ""+ assert message.message_metadata is not None+ assert set(json.loads(message.message_metadata)) == {"usage"}+++def test_worker_terminal_revalidates_message_tenant(case: RecoveryCase) -> None:+ from repositories.application_trace_repository import TraceCollectionError++ with pytest.raises(TraceCollectionError, match="tracing_source_not_found"):+ case.service.capture_agent_app_terminal(+ tenant_id=identifier("foreign-tenant"),+ app_id=case.collection.run.app_id,+ message_id=case.collection.run.source_id,+ outcome="succeeded",+ output="must not export",+ )+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.outer_trace is None+++def test_recovers_real_message_terminal_once_through_existing_trace_task(case: RecoveryCase) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.message_metadata = complete_metadata()+ message.updated_at = case.now - timedelta(seconds=11)+ assert recovery.recover_terminal(case.service, case.collection.run) is True+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.outer_trace is not None+ assert stored.outer_trace["session_id"] == "session-original"+ assert stored.outer_trace["spans"][0]["outputs"] == "persisted final answer"+ assert recovery.recover_terminal(case.service, stored) is False+ assert case.scheduled == [stored.id]+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ assert message.message_tokens == 2+ assert message.answer_tokens == 1+ assert message.answer == "persisted final answer"+++def test_deliver_recovers_and_exports_once_after_terminal_queue_loss(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.message_metadata = complete_metadata()+ message.updated_at = case.now - timedelta(seconds=11)+ exported: list[CanonicalTrace] = []++ def export(_provider: str, _config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:+ exported.append(trace)+ return TraceExportResult(trace_id="provider-id")++ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ monkeypatch.setattr(service_module, "export_trace", export)+ case.service.deliver(case.collection.run.id)+ case.service.deliver(case.collection.run.id)+ assert len(exported) == 1+ assert exported[0].spans[0].outputs == "persisted final answer"+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.state == "accepted"+ assert stored.attempts == 1+++def test_early_usage_commit_is_not_treated_as_a_final_answer(case: RecoveryCase) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.message_metadata = json.dumps({"usage": {"prompt_tokens": 2, "completion_tokens": 1}})+ message.answer = ""+ message.updated_at = case.now - timedelta(seconds=11)+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.outer_trace is None+ assert stored.state == "collecting"+ assert not case.scheduled+++def test_final_without_business_commit_times_out_as_failed_not_success(case: RecoveryCase) -> None:+ invocations, _ = case.collection.repository.collected(case.collection.run.id)+ with case.collection.sessions.begin() as session:+ invocation = session.get(ApplicationTraceInvocation, invocations[0].id)+ assert invocation is not None+ invocation.finalized_at = case.now - timedelta(hours=1)+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.state == "failed"+ assert stored.error_code == "business_terminal_not_persisted"+ assert stored.outer_trace is None+++def test_final_settle_window_and_all_invocations_must_finish(case: RecoveryCase) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.message_metadata = complete_metadata()+ message.updated_at = case.now - timedelta(seconds=1)+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ second = case.collection.invocation("pending-second")+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.updated_at = case.now - timedelta(seconds=10)+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ with case.collection.sessions() as session:+ invocation = session.get(ApplicationTraceInvocation, second.id)+ assert invocation is not None+ assert invocation.finalized is False+++def test_explicit_message_failure_can_recover_without_fabricating_usage(case: RecoveryCase) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.status = MessageStatus.ERROR+ message.error = "model initialization failed"+ message.answer = ""+ message.message_metadata = None+ message.message_tokens = 0+ message.answer_tokens = 0+ message.updated_at = case.now - timedelta(seconds=11)+ assert recovery.recover_terminal(case.service, case.collection.run) is True+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.outer_trace is not None+ assert stored.outer_trace["spans"][0]["status"] == "error"+ assert stored.outer_trace["spans"][0]["outputs"] == ""+ assert stored.outer_trace["spans"][0]["metadata"]["aggregate_total_tokens"] == 0+++def test_cross_tenant_source_is_rejected_before_trace_task_executes(+ case: RecoveryCase, monkeypatch: pytest.MonkeyPatch+) -> None:+ with case.collection.sessions.begin() as session:+ app = session.get(App, case.collection.run.app_id)+ assert app is not None+ app.tenant_id = identifier("other-tenant")+ called: list[bool] = []++ def forbidden(_task: TraceTask) -> None:+ called.append(True)++ monkeypatch.setattr(TraceTask, "execute", forbidden)+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ assert called == []+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.error_code == "tracing_source_not_found"+++def test_unresolved_recovery_rotates_behind_unchecked_sources(case: RecoveryCase) -> None:+ other = seed(case.collection.sessions, "other-recovery")+ assert case.collection.repository.pending_ids(limit=1) == [case.collection.run.id]+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ assert case.collection.repository.pending_ids(limit=1) == [other.run.id]+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert stored.state == "collecting"++[email protected]("status", [MessageStatus.PAUSED, MessageStatus.NORMAL])+def test_no_final_marker_or_paused_message_is_not_recovered(case: RecoveryCase, status: MessageStatus) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.status = status+ message.updated_at = case.now - timedelta(seconds=10)+ assert recovery.recover_terminal(case.service, case.collection.run) is False+ assert not case.scheduled+++ ("status", "finished"),+ [+ (WorkflowExecutionStatus.RUNNING, False),+ (WorkflowExecutionStatus.PAUSED, True),+ (WorkflowExecutionStatus.SUCCEEDED, False),+ (WorkflowExecutionStatus.SUCCEEDED, True),+ (WorkflowExecutionStatus.PARTIAL_SUCCEEDED, True),+ (WorkflowExecutionStatus.FAILED, True),+ (WorkflowExecutionStatus.STOPPED, True),+ ],+)+def test_workflow_recovery_requires_committed_terminal_state(+ case: RecoveryCase, status: WorkflowExecutionStatus, finished: bool+) -> None:+ owner_id, workflow_run_id = identifier("workflow-owner"), identifier("recover-workflow-run")+ with case.collection.sessions.begin() as session:+ app = session.get(App, case.collection.run.app_id)+ assert app is not None+ app.created_by = owner_id+ session.add(make_account(account_id=owner_id))+ session.add(+ WorkflowRun(+ id=workflow_run_id,+ tenant_id=case.collection.run.tenant_id,+ app_id=app.id,+ workflow_id=identifier("workflow"),+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ graph="{}",+ inputs="{}",+ outputs=json.dumps({"result": "persisted workflow output"}),+ status=status,+ error="actual workflow failure"+ if status in {WorkflowExecutionStatus.FAILED, WorkflowExecutionStatus.STOPPED}+ else None,+ created_by=owner_id,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_at=case.now - timedelta(seconds=30),+ finished_at=case.now - timedelta(seconds=10) if finished else None,+ elapsed_time=20,+ total_tokens=0,+ total_steps=0,+ )+ )+ run = case.collection.repository.start(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ source_kind="workflow",+ source_id=workflow_run_id,+ session_id="workflow-session",+ )+ assert run is not None+ expected = finished and status in {+ WorkflowExecutionStatus.SUCCEEDED,+ WorkflowExecutionStatus.PARTIAL_SUCCEEDED,+ WorkflowExecutionStatus.FAILED,+ WorkflowExecutionStatus.STOPPED,+ }+ assert recovery.recover_terminal(case.service, run) is expected+ stored = case.collection.repository.get(run.id)+ assert stored is not None+ if expected:+ assert stored.outer_trace is not None+ assert stored.outer_trace["root_span_id"] == span_id(workflow_run_id)+ assert stored.outer_trace["spans"][0]["outputs"] == {"result": "persisted workflow output"}+ else:+ assert stored.outer_trace is None++[email protected]("formatted", [None, "", " \n\t", "{}", "[]", "null", " { } ", "[\n]", {}, []])[email protected]("capture_content", [False, True])+def test_early_error_root_preserves_known_query_only_when_content_is_enabled(+ case: RecoveryCase, formatted: object, capture_content: bool+) -> None:+ from sqlalchemy import update++ from models.application_trace import ApplicationTraceRun++ query = "KNOWN_USER_QUERY_BEFORE_FORMATTED_MESSAGE"+ with case.collection.sessions.begin() as session:+ # Historical rows can contain JSON strings/lists despite the ORM's dict annotation.+ session.execute(+ update(Message.metadata.tables["messages"])+ .where(Message.id == case.collection.run.source_id)+ .values(message=formatted, query=query, answer="")+ )+ run = session.get(ApplicationTraceRun, case.collection.run.id)+ assert run is not None+ run.capture_content = capture_content+ case.service.capture_agent_app_terminal(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ message_id=case.collection.run.source_id,+ outcome="failed",+ output=None,+ )+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ trace = CanonicalTrace.model_validate(stored.outer_trace)+ root = trace.spans[0]+ assert root.status.value == "error"+ assert root.metadata["message_execution_state"] == "failed"+ assert root.inputs == ({"query": query} if capture_content else None)+ assert root.outputs == ("" if capture_content else None)+ assembled = case.service.assemble(stored)+ assert (query in assembled.model_dump_json()) is capture_content+ assert all(span.inputs is None for span in assembled.spans if span.id != assembled.root_span_id)+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ assert message.message == formatted+ assert message.query == query+ assert message.answer == ""+++ ("formatted", "expected"),+ [+ (+ {"messages": [{"role": "user", "content": "actual formatted input"}]},+ {"messages": [{"role": "user", "content": "actual formatted input"}]},+ ),+ (+ '{"messages":[{"role":"user","content":"actual formatted input"}]}',+ {"messages": [{"role": "user", "content": "actual formatted input"}]},+ ),+ (+ [{"role": "user", "content": "actual formatted input"}],+ [{"role": "user", "content": "actual formatted input"}],+ ),+ (+ '[{"role":"user","content":"actual formatted input"}]',+ [{"role": "user", "content": "actual formatted input"}],+ ),+ ("opaque non-JSON formatted input", "opaque non-JSON formatted input"),+ ("0", "0"),+ ("false", "false"),+ ],+)+def test_worker_handoff_never_replaces_existing_formatted_inputs(+ case: RecoveryCase, formatted: object, expected: object+) -> None:+ from sqlalchemy import update++ query = "QUERY_MUST_NOT_REPLACE_FORMATTED_INPUTS"+ with case.collection.sessions.begin() as session:+ session.execute(+ update(Message.metadata.tables["messages"])+ .where(Message.id == case.collection.run.source_id)+ .values(message=formatted, query=query, answer="")+ )+ case.service.capture_agent_app_terminal(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ message_id=case.collection.run.source_id,+ outcome="failed",+ output=None,+ )+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ trace = CanonicalTrace.model_validate(stored.outer_trace)+ assert trace.spans[0].inputs == expected+ assert query not in trace.model_dump_json()+ with case.collection.sessions() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ assert message.message == formatted+++def test_worker_handoff_does_not_invent_input_when_query_is_empty(case: RecoveryCase) -> None:+ with case.collection.sessions.begin() as session:+ message = session.get(Message, case.collection.run.source_id)+ assert message is not None+ message.message = {}+ message.query = ""+ message.answer = ""+ case.service.capture_agent_app_terminal(+ tenant_id=case.collection.run.tenant_id,+ app_id=case.collection.run.app_id,+ message_id=case.collection.run.source_id,+ outcome="failed",+ output=None,+ )+ stored = case.collection.repository.get(case.collection.run.id)+ assert stored is not None+ assert CanonicalTrace.model_validate(stored.outer_trace).spans[0].inputs == {}diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_repository_boundaries.py b/api/tests/unit_tests/services/tracing/test_application_trace_repository_boundaries.pynew file mode 100644index 00000000000..7122829140b--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_repository_boundaries.py@@ -0,0 +1,209 @@+"""Persistence boundaries that must fail closed or remain immutable across retries."""++from datetime import timedelta+from uuid import UUID++import pytest+from sqlalchemy import delete, func, select+from sqlalchemy.orm import Session, sessionmaker++from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceBatch, ApplicationTraceInvocation, ApplicationTraceRun+from models.model import App+from repositories.application_trace_repository import TraceCollectionError+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ batch_for,+ claim,+ identifier,+ ingest,+ seed,+ terminal,+)++[email protected]("policy", ["invalid-json", "null", "[]"])+def test_malformed_policy_freezes_disabled_decision(sqlite_session_factory: sessionmaker[Session], policy: str) -> None:+ case = seed(sqlite_session_factory, "malformed")+ with case.sessions.begin() as session:+ session.execute(delete(ApplicationTraceRun).where(ApplicationTraceRun.id == case.run.id))+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.tracing = policy+ run = case.repository.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="message",+ source_id=case.run.source_id,+ session_id="",+ )+ assert run is not None+ assert run.state == "not_collected"+ assert run.provider_config == {}+ assert run.session_id == case.run.source_id+ assert not run.capture_content+ assert not run.capture_agent+++def test_reserved_preview_requires_persisted_owner_workflow(sqlite_session_factory: sessionmaker[Session]) -> None:+ case = seed(sqlite_session_factory, "missing-preview-owner")+ assert (+ case.repository.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="node",+ source_id=identifier("preview"),+ session_id="preview",+ workflow_id=identifier("missing-workflow"),+ )+ is None+ )+++def test_initialization_failure_cannot_replace_snapshot_or_create_foreign_row(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ case = seed(sqlite_session_factory, "failure-snapshot")+ same = case.repository.record_initialization_failure(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="message",+ source_id=case.run.source_id,+ )+ assert same is not None+ assert same.id == case.run.id+ assert same.state == "collecting"+ assert same.provider_config == case.run.provider_config+ assert (+ case.repository.record_initialization_failure(+ tenant_id=identifier("foreign-tenant"),+ app_id=case.run.app_id,+ source_kind="message",+ source_id=case.run.source_id,+ )+ is None+ )++[email protected]("target", ["missing_run", "revoked_owner", "closed_invocation"])+def test_ingestion_rejects_missing_or_closed_owner_without_writing(+ sqlite_session_factory: sessionmaker[Session], target: str+) -> None:+ case = seed(sqlite_session_factory, "ingestion-owner")+ invocation = case.invocation()+ batch = batch_for(case.run, invocation)+ if target == "missing_run":+ batch = batch.model_copy(update={"run_id": UUID(identifier("missing-run"))})+ error = "tracing_run_not_found"+ elif target == "revoked_owner":+ with case.sessions.begin() as session:+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.tenant_id = identifier("another-tenant")+ error = "tracing_owner_not_found"+ else:+ with case.sessions.begin() as session:+ stored = session.get(ApplicationTraceInvocation, invocation.id)+ assert stored is not None+ stored.finalized = True+ error = "tracing_collection_closed"+ with pytest.raises(TraceCollectionError, match=error):+ ingest(case, batch)+ with case.sessions() as session:+ assert session.scalar(select(func.count()).select_from(ApplicationTraceBatch)) == 0+ stored_run = case.repository.get(case.run.id)+ assert stored_run is not None+ assert stored_run.span_count == 0+++def test_first_terminal_is_immutable_and_revoked_owner_is_never_exported(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ case = seed(sqlite_session_factory, "terminal-owner")+ terminal(case)+ before = case.repository.get(case.run.id)+ assert before is not None+ case.repository.terminal(case.run.id, {"replacement": True}, "f" * 16)+ after = case.repository.get(case.run.id)+ assert after is not None+ assert after.outer_trace == before.outer_trace+ assert after.root_span_id == before.root_span_id+ with case.sessions.begin() as session:+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.tenant_id = identifier("moved-owner")+ assert claim(case) is None+ failed = case.repository.get(case.run.id)+ assert failed is not None+ assert failed.error_code == "tracing_owner_not_found"+ assert failed.state == "failed"+ case.repository.terminal(case.run.id, {"replacement": True}, "f" * 16)+ frozen = case.repository.get(case.run.id)+ assert frozen is not None+ assert frozen.outer_trace == before.outer_trace+++ ("state", "error"),+ [+ ("not_collected", "tracing_run_not_found"),+ ("accepted", "tracing_already_accepted"),+ ("collecting", "tracing_retry_not_allowed"),+ ],+)+def test_retry_rejects_ineligible_states(sqlite_session_factory: sessionmaker[Session], state: str, error: str) -> None:+ case = seed(sqlite_session_factory, "retry-state")+ terminal(case)+ with case.sessions.begin() as session:+ row = session.get(ApplicationTraceRun, case.run.id)+ assert row is not None+ row.state = state+ with pytest.raises(TraceCollectionError, match=error):+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)+ row = case.repository.get(case.run.id)+ assert row is not None+ assert row.state == state+++def test_retry_unknown_run_and_invocation_reject_before_creation(sqlite_session_factory: sessionmaker[Session]) -> None:+ case = seed(sqlite_session_factory, "retry-unknown")+ with pytest.raises(TraceCollectionError, match="tracing_run_not_found"):+ case.repository.retry(run_id=identifier("missing"), acknowledge_possible_duplicates=True)+ with pytest.raises(TraceCollectionError, match="tracing_run_not_collecting"):+ case.repository.invocation(+ run_id=identifier("missing"),+ agent_id=identifier("agent"),+ node_execution_id=None,+ attempt=0,+ call_key="x",+ )+ assert case.repository.claim(identifier("missing"), collection_timeout=1, lease_seconds=1) is None+++def test_collection_without_terminal_is_not_exportable(sqlite_session_factory: sessionmaker[Session]) -> None:+ case = seed(sqlite_session_factory, "no-terminal")+ ingest(case, batch_for(case.run, case.invocation()))+ assert claim(case) is None+ row = case.repository.get(case.run.id)+ assert row is not None+ assert row.attempts == 0+ assert row.state == "collecting"+++def test_retention_removes_all_payload_rows_and_preserves_newer_runs(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ expired = seed(sqlite_session_factory, "expired")+ fresh = seed(sqlite_session_factory, "fresh")+ for case in (expired, fresh):+ ingest(case, batch_for(case.run, case.invocation()))+ with expired.sessions.begin() as session:+ row = session.get(ApplicationTraceRun, expired.run.id)+ assert row is not None+ row.created_at = naive_utc_now() - timedelta(days=8)+ assert expired.repository.purge(retention_days=7) == 1+ assert expired.repository.get(expired.run.id) is None+ assert expired.repository.collected(expired.run.id) == ([], [])+ assert fresh.repository.get(fresh.run.id) is not None+ invocations, batches = fresh.repository.collected(fresh.run.id)+ assert len(invocations) == len(batches) == 1+ assert expired.repository.purge(retention_days=7) == 0diff --git a/api/tests/unit_tests/services/tracing/test_application_trace_service_boundaries.py b/api/tests/unit_tests/services/tracing/test_application_trace_service_boundaries.pynew file mode 100644index 00000000000..52baaeaf778--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_application_trace_service_boundaries.py@@ -0,0 +1,425 @@+"""Exercise public orchestration, privacy, and delivery decisions against durable rows."""++from datetime import datetime+from uuid import UUID++import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.ops.exceptions import RetryableTraceDispatchError+from graphon.enums import WorkflowNodeExecutionStatus+from models.application_trace import ApplicationTraceRun+from models.enums import CreatorUserRole+from models.model import Message, TraceAppConfig+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from repositories.application_trace_repository import TraceCollectionError+from services.tracing import service as service_module+from services.tracing.provider_dispatch import TraceExportResult+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_message, make_workflow+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ Collection,+ batch_for,+ claim,+ identifier,+ ingest,+ seed,+ terminal,+)+++def source(case: Collection) -> dict[str, str]:+ return {+ "tenant_id": case.run.tenant_id,+ "app_id": case.run.app_id,+ "source_kind": "message",+ "source_id": case.run.source_id,+ }+++def test_prepare_invocation_returns_only_opaque_context_and_respects_capture_gate(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ case = seed(sqlite_session_factory, "prepare", capture_content=True)+ service = ApplicationTracingService(case.repository)+ context = service.prepare_invocation(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ agent_id=identifier("agent"),+ source_kind="message",+ source_id=case.run.source_id,+ session_id="ignored-new-session",+ node_execution_id=None,+ attempt=0,+ call_key="call",+ )+ assert context is not None+ assert context.run_id == UUID(case.run.id)+ assert context.capture_content is True+ assert context.session_id == "session-original"+ assert "encrypted-original" not in context.model_dump_json()+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)+ assert (+ service.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="message",+ source_id=case.run.source_id,+ session_id="new",+ )+ is None+ )++[email protected]("missing", ["tenant", "app", "message"])+def test_status_denies_foreign_or_absent_source_without_destination_details(+ sqlite_session_factory: sessionmaker[Session], missing: str+) -> None:+ case = seed(sqlite_session_factory, "scope")+ identity = source(case)+ identity[{"tenant": "tenant_id", "app": "app_id", "message": "source_id"}[missing]] = identifier("foreign")+ with pytest.raises(TraceCollectionError) as caught:+ ApplicationTracingService(case.repository).status(**identity)+ assert caught.value.code == "tracing_source_not_found"+ assert caught.value.status == 404+ assert "encrypted-original" not in str(caught.value)+++def test_status_uses_frozen_destination_and_selects_latest_node_attempt(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ case = seed(sqlite_session_factory, "status")+ with case.sessions.begin() as session:+ config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == case.run.app_id))+ assert config is not None+ config.tracing_config = {"project_url": "https://new.example/project/new", "secret_key": "encrypted-new"}++ def forbidden(*_args: object, **_kwargs: object) -> None:+ pytest.fail("status must not decrypt credentials or resolve current provider metadata")++ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", forbidden)+ monkeypatch.setattr(service_module.OpsTraceManager, "get_trace_config_project_url", forbidden)+ node_id = identifier("selected-node")+ invocation = case.repository.invocation(+ run_id=case.run.id, agent_id=identifier("agent"), node_execution_id=node_id, attempt=1, call_key="node:1"+ )+ ingest(case, batch_for(case.run, invocation))+ terminal(case)+ lease = claim(case)+ assert lease is not None+ case.repository.complete(+ case.run.id,+ lease[1],+ state="accepted",+ result=TraceExportResult(+ "provider-trace",+ trace_url="https://frozen.example/trace",+ span_urls={invocation.span_id: "https://frozen.example/span"},+ ),+ )+ service = ApplicationTracingService(case.repository)+ result = service.status(**source(case), node_execution_id=node_id)+ assert result["status"] == "accepted"+ assert result["trace_id"] == "provider-trace"+ assert result["span_url"] == "https://frozen.example/span"+ assert result["project_url"] == "https://langfuse.test/project/original"+ assert result["config_revision"] == case.run.config_revision+ assert result["queryable"] is None+ assert result["capture_content"] is False+ assert "provider_config" not in result+ assert "encrypted-original" not in str(result)+ with pytest.raises(TraceCollectionError, match="tracing_node_not_found"):+ service.status(**source(case), node_execution_id=identifier("another-node"))+++def test_new_run_freezes_updated_project_without_changing_previous_run(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ case = seed(sqlite_session_factory, "updated-project")+ new_id = identifier("new-message")+ with case.sessions.begin() as session:+ config = session.scalar(select(TraceAppConfig).where(TraceAppConfig.app_id == case.run.app_id))+ assert config is not None+ config.tracing_config = {"project_url": "https://new.example/project/new", "secret_key": "encrypted-new"}+ message = session.get(Message, case.run.source_id)+ assert message is not None+ session.add(+ make_message(+ message_id=new_id,+ app_id=case.run.app_id,+ conversation_id=message.conversation_id,+ inputs={},+ query="new run",+ message={},+ answer="done",+ message_unit_price=message.message_unit_price,+ answer_unit_price=message.answer_unit_price,+ currency=message.currency,+ from_source=message.from_source,+ )+ )+ new_run = case.repository.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="message",+ source_id=new_id,+ session_id="new-session",+ )+ assert new_run is not None+ service = ApplicationTracingService(case.repository)+ assert service.status(**source(case))["project_url"] == "https://langfuse.test/project/original"+ assert service.status(**{**source(case), "source_id": new_id})["project_url"] == "https://new.example/project/new"+++ "project_url",+ [+ None,+ "",+ 1,+ "javascript:alert(1)",+ "//example.com/project",+ "https://user:[email protected]/project",+ "https://@example.com",+ "https://example.com\\@other.test",+ "https://example.com:invalid/project",+ "https://[invalid/project",+ "https://example.com/\nproject",+ ],+)+def test_status_hides_unavailable_or_unsafe_frozen_project_urls(+ sqlite_session_factory: sessionmaker[Session], project_url: object+) -> None:+ case = seed(sqlite_session_factory, "invalid-project")+ with case.sessions.begin() as session:+ run = session.get(ApplicationTraceRun, case.run.id)+ assert run is not None+ run.provider_config = {+ "secret_key": "encrypted-original",+ **({"project_url": project_url} if project_url is not None else {}),+ }+ result = ApplicationTracingService(case.repository).status(**source(case))+ assert result["project_url"] is None+ assert "encrypted-original" not in str(result)++[email protected]("provider", ["aliyun", "tencent"])+def test_status_does_not_present_platform_console_as_project(+ sqlite_session_factory: sessionmaker[Session], provider: str+) -> None:+ case = seed(sqlite_session_factory, "platform-console")+ with case.sessions.begin() as session:+ run = session.get(ApplicationTraceRun, case.run.id)+ assert run is not None+ run.provider = provider+ run.provider_config = {"project_url": "https://console.example.com"}+ assert ApplicationTracingService(case.repository).status(**source(case))["project_url"] is None++[email protected]("disabled", [True, False])+def test_uncollected_status_and_missing_run_retry_do_not_construct_provider(+ sqlite_session_factory: sessionmaker[Session], disabled: bool+) -> None:+ case = seed(sqlite_session_factory, "missing", enabled=not disabled)+ if not disabled:+ with case.sessions.begin() as session:+ stored = session.get(ApplicationTraceRun, case.run.id)+ assert stored is not None+ session.delete(stored)+ service = ApplicationTracingService(case.repository)+ assert service.status(**source(case)) == {+ "status": "not_collected",+ "source_kind": "message",+ "source_id": case.run.source_id,+ }+ if not disabled:+ with pytest.raises(TraceCollectionError, match="tracing_run_not_found"):+ service.retry(**source(case), acknowledge_possible_duplicates=True)+++def test_retry_acknowledgement_guards_schedule_and_keeps_frozen_snapshot(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ case = seed(sqlite_session_factory, "retry")+ ingest(case, batch_for(case.run, case.invocation()))+ terminal(case)+ lease = claim(case)+ assert lease is not None+ case.repository.complete(case.run.id, lease[1], state="uncertain", error_code="provider_acceptance_unknown")+ scheduled: list[str] = []+ service = ApplicationTracingService(case.repository)+ monkeypatch.setattr(service, "schedule", scheduled.append)+ with pytest.raises(TraceCollectionError, match="tracing_duplicate_acknowledgement_required"):+ service.retry(**source(case), acknowledge_possible_duplicates=False)+ assert scheduled == []+ result = service.retry(**source(case), acknowledge_possible_duplicates=True)+ assert scheduled == [case.run.id]+ assert result["status"] == "ready"+ assert result["config_revision"] == case.run.config_revision+ current = case.repository.get(case.run.id)+ assert current is not None+ assert current.provider_config == case.run.provider_config+++ ("failure", "code"),+ [+ (RetryableTraceDispatchError("private-transport-error"), "provider_retry_exhausted"),+ (ValueError("private-credential-error"), "provider_export_rejected"),+ ],+)+def test_delivery_exhaustion_and_rejection_never_log_vendor_details(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ failure: Exception,+ code: str,+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_MAX_ATTEMPTS=1)+ case = seed(sqlite_session_factory, "delivery")+ ingest(case, batch_for(case.run, case.invocation()))+ terminal(case)++ def export(*_args: object) -> TraceExportResult:+ raise failure++ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ monkeypatch.setattr(service_module, "export_trace", export)+ service = ApplicationTracingService(case.repository)+ service.deliver(case.run.id)+ current = case.repository.get(case.run.id)+ assert current is not None+ assert current.state == "failed"+ assert current.error_code == code+ assert "private-" not in caplog.text+ service.deliver(identifier("missing-run"))++[email protected]("capture_content", [False, True])+ "status",+ [WorkflowNodeExecutionStatus.RUNNING, WorkflowNodeExecutionStatus.SUCCEEDED, WorkflowNodeExecutionStatus.FAILED],+)+def test_preview_terminal_uses_persisted_node_state_and_frozen_privacy(+ sqlite_session_factory: sessionmaker[Session], status: WorkflowNodeExecutionStatus, capture_content: bool+) -> None:+ case = seed(sqlite_session_factory, "preview", capture_content=capture_content)+ node_id = identifier("preview-node")+ workflow_id = identifier("preview-definition")+ with case.sessions.begin() as session:+ session.add(make_workflow(workflow_id=workflow_id, tenant_id=case.run.tenant_id, app_id=case.run.app_id))+ session.add(+ WorkflowNodeExecutionModel(+ id=node_id,+ node_execution_id=node_id,+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ workflow_id=workflow_id,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,+ index=0,+ node_id="agent",+ node_type="agent",+ title="Agent preview",+ inputs='{"secret":"private-input"}',+ outputs='{"answer":"private-output"}',+ process_data="{}",+ status=status,+ elapsed_time=2,+ error="private-model-error" if status == WorkflowNodeExecutionStatus.FAILED else None,+ created_at=datetime(2026, 1, 1),+ created_by=identifier("account"),+ created_by_role=CreatorUserRole.ACCOUNT,+ )+ )+ run = case.repository.start(+ tenant_id=case.run.tenant_id,+ app_id=case.run.app_id,+ source_kind="node",+ source_id=node_id,+ workflow_id=workflow_id,+ session_id="preview-session",+ )+ assert run is not None+ ApplicationTracingService(case.repository).settle_preview(run)+ current = case.repository.get(run.id)+ assert current is not None+ if status == WorkflowNodeExecutionStatus.RUNNING:+ assert current.outer_trace is None+ else:+ assert current.outer_trace is not None+ trace = service_module.CanonicalTrace.model_validate(current.outer_trace)+ root = trace.spans[0]+ assert root.status.value == ("error" if status == WorkflowNodeExecutionStatus.FAILED else "ok")+ assert ("private-" in trace.model_dump_json()) is capture_content+ assert root.metadata["node_execution_id"] == node_id+ if not capture_content and status == WorkflowNodeExecutionStatus.FAILED:+ assert root.error == "Node execution failed"++[email protected]("owner_state", ["no_creator", "missing_account", "wrong_app"])+def test_missing_owner_identity_cannot_authorize_trace_expansion(+ sqlite_session_factory: sessionmaker[Session], owner_state: str+) -> None:+ from models.model import App++ case = seed(sqlite_session_factory, "owner")+ with case.sessions.begin() as session:+ app = session.get(App, case.run.app_id)+ assert app is not None+ app.created_by = None if owner_state == "no_creator" else identifier("missing-owner")+ app_id = identifier("another-app") if owner_state == "wrong_app" else case.run.app_id+ with pytest.raises(TraceCollectionError, match="tracing_owner_not_found"):+ ApplicationTracingService(case.repository)._service_account(case.run, app_id)+++ ("invalid", "code"),+ [+ ("empty", "tracing_invocation_empty"),+ ("foreign_parent", "tracing_invocation_parent_invalid"),+ ("no_invocation_root", "tracing_invocation_root_missing"),+ ("conflicting_span", "tracing_span_conflict"),+ ],+)+def test_invalid_runtime_tree_is_rejected_before_provider_credentials_or_io(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, invalid: str, code: str+) -> None:+ case = seed(sqlite_session_factory, "invalid-runtime")+ batch = batch_for(case.run, case.invocation())+ if invalid == "empty":+ batch.spans = []+ elif invalid == "foreign_parent":+ batch.spans[0] = batch.spans[0].model_copy(update={"parent_span_id": "f" * 16})+ elif invalid == "no_invocation_root":+ batch.spans[0] = batch.spans[0].model_copy(update={"parent_span_id": batch.spans[0].span_id})+ else:+ ingest(case, batch.model_copy(update={"final": False}))+ from uuid import uuid4++ batch = batch.model_copy(+ update={"batch_id": uuid4(), "spans": [batch.spans[0].model_copy(update={"name": "conflicting-version"})]}+ )+ ingest(case, batch)+ terminal(case)++ def forbidden(*_args: object, **_kwargs: object) -> None:+ pytest.fail("invalid trees must fail before credentials or provider requests")++ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", forbidden)+ monkeypatch.setattr(service_module, "export_trace", forbidden)+ service = ApplicationTracingService(case.repository)+ service.deliver(case.run.id)+ current = case.repository.get(case.run.id)+ assert current is not None+ assert current.state == "failed"+ assert current.error_code == code+++def test_assemble_requires_a_persisted_business_terminal(sqlite_session_factory: sessionmaker[Session]) -> None:+ case = seed(sqlite_session_factory, "no-terminal")+ with pytest.raises(TraceCollectionError, match="tracing_terminal_missing"):+ ApplicationTracingService(case.repository).assemble(case.run)diff --git a/api/tests/unit_tests/services/tracing/test_arize_configuration.py b/api/tests/unit_tests/services/tracing/test_arize_configuration.pynew file mode 100644index 00000000000..7e6a1e587d1--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_arize_configuration.py@@ -0,0 +1,260 @@+"""Arize metadata reads are optional, bounded and pinned to the exact destination."""++from collections import deque+from collections.abc import Iterator+from typing import Never++import httpx+import pytest+from dify_trace_arize_phoenix import arize_configuration as configuration+from dify_trace_arize_phoenix.arize_configuration import (+ DESTINATION_KEY,+ bind_destination,+ project_url,+ resolve_project_destination,+)++from core.helper import ssrf_proxy++SPACE = "U3BhY2U6dGVzdA=="+ORG = "T3JnOnRlc3Q="+PROJECT = "UHJvamVjdDovKz0="+VALUES = {"endpoint": "https://otlp.arize.com", "space_id": SPACE, "project": "exact", "api_key": "synthetic-key"}+DESTINATION = {+ "endpoint": VALUES["endpoint"],+ "space_id": SPACE,+ "project": "exact",+ "organization_id": ORG,+ "project_id": PROJECT,+}+URL = "https://app.arize.com/organizations/T3JnOnRlc3Q%3D/spaces/U3BhY2U6dGVzdA%3D%3D/projects/UHJvamVjdDovKz0%3D"+++def response(body: object, status: int = 200) -> httpx.Response:+ return httpx.Response(status, json=body, request=httpx.Request("GET", "https://api.arize.com/v2/projects"))+++def space() -> httpx.Response:+ return response({"id": SPACE, "organization_id": ORG})+++def page(*projects: dict[str, str], more: bool = False, cursor: str | None = None) -> httpx.Response:+ return response({"projects": list(projects), "pagination": {"has_more": more, "next_cursor": cursor}})+++def project(name: str = "exact", project_id: str = PROJECT) -> dict[str, str]:+ return {"id": project_id, "name": name, "space_id": SPACE}+++def responses() -> deque[httpx.Response | Exception]:+ return deque()+++def calls(+ monkeypatch: pytest.MonkeyPatch, responses: deque[httpx.Response | Exception]+) -> list[tuple[str, dict[str, object]]]:+ calls: list[tuple[str, dict[str, object]]] = []++ def get(url: str, **options: object) -> httpx.Response:+ calls.append((url, options))+ assert responses, "Unplanned metadata request"+ item = responses.popleft()+ if isinstance(item, Exception):+ raise item+ return item++ monkeypatch.setattr(ssrf_proxy, "get", get)+ return calls+++def test_exact_scoped_project_and_opaque_ids_use_official_route(+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception]+) -> None:+ received = [space(), page(project("exact-substring"), more=True, cursor="next"), page(project())]+ responses.extend(received)+ assert resolve_project_destination(VALUES) == DESTINATION+ assert (+ project_url(+ {**VALUES, DESTINATION_KEY: bind_destination(VALUES, DESTINATION), "project_url": "https://untrusted.test"}+ )+ == URL+ )+ assert len(calls) == 3+ assert calls[0][0] == "https://api.arize.com/v2/spaces/U3BhY2U6dGVzdA%3D%3D"+ assert calls[1][1]["params"] == {"space_id": SPACE, "name": "exact", "limit": 100}+ assert calls[2][1]["params"] == {"space_id": SPACE, "name": "exact", "limit": 100, "cursor": "next"}+ for url, options in calls:+ assert url.startswith("https://api.arize.com/v2/")+ assert options["headers"] == {+ "Authorization": "Bearer synthetic-key",+ "Accept": "application/json",+ "Accept-Encoding": "identity",+ "Cookie": "",+ }+ assert options["auth"] is None+ assert options["follow_redirects"] is False+ assert options["max_retries"] == 0+ assert options["stream_response"] is True+ timeout = options["timeout"]+ assert isinstance(timeout, float)+ assert 0 < timeout <= 5+ assert all(item.is_closed for item in received)++[email protected]("status", [301, 401, 403, 404, 429, 500])+def test_unreadable_metadata_has_no_link_and_closes_response(+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], status: int+) -> None:+ denied = response({"error": "private-response"}, status)+ responses.append(denied)+ assert resolve_project_destination(VALUES) is None+ assert len(calls) == 1+ assert denied.is_closed+++ "body", [{"id": SPACE}, {"id": "other", "organization_id": ORG}, {"id": SPACE, "organization_id": "../bad"}]+)+def test_missing_or_mismatched_space_metadata_is_not_a_project(+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], body: dict[str, str]+) -> None:+ responses.append(response(body))+ assert resolve_project_destination(VALUES) is None+ assert len(calls) == 1+++ "body",+ [+ {"projects": [project("EXACT")], "pagination": {"has_more": False}},+ {"projects": [project(), project(project_id="other")], "pagination": {"has_more": False}},+ {"projects": [{**project(), "space_id": "other"}], "pagination": {"has_more": False}},+ {"projects": [{**project(), "id": "../bad"}], "pagination": {"has_more": False}},+ {"projects": [project()] * 101, "pagination": {"has_more": False}},+ {"projects": [project()], "pagination": {"has_more": True}},+ {"projects": [project()], "pagination": {"has_more": "false"}},+ {"projects": [project()]},+ {"data": [project()], "pagination": {"has_more": False}},+ ],+)+def test_incomplete_ambiguous_or_non_exact_projects_do_not_fabricate_links(+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], body: object+) -> None:+ responses.extend([space(), response(body)])+ assert resolve_project_destination(VALUES) is None+ assert len(calls) == 2++[email protected]("repeat", [False, True])+def test_pagination_is_bounded_and_never_accepts_a_partial_match(+ calls: list[tuple[str, dict[str, object]]], responses: deque[httpx.Response | Exception], repeat: bool+) -> None:+ responses.append(space())+ responses.extend(page(project(), more=True, cursor="same" if repeat else str(index)) for index in range(3))+ assert resolve_project_destination(VALUES) is None+ assert len(calls) == (3 if repeat else 4)+++ "item",+ [+ httpx.ReadTimeout("synthetic-timeout"),+ httpx.Response(200, content=b"not-json"),+ httpx.Response(200, json=[]),+ httpx.Response(200, content=b" " * (1024 * 1024 + 1)),+ httpx.Response(200, json={}, headers={"content-encoding": "unsupported"}),+ ],+)+def test_transport_and_bounded_decode_failures_remain_optional(+ calls: list[tuple[str, dict[str, object]]],+ responses: deque[httpx.Response | Exception],+ item: httpx.Response | Exception,+) -> None:+ responses.append(item)+ assert resolve_project_destination(VALUES) is None+ assert len(calls) == 1+ if isinstance(item, httpx.Response):+ assert item.is_closed+++def test_all_pages_share_one_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ clock = [100.0]+ calls: list[float] = []+ received = [space(), page(project(), more=True, cursor="next")]++ def get(_url: str, **options: object) -> httpx.Response:+ timeout = options["timeout"]+ assert isinstance(timeout, float)+ calls.append(timeout)+ clock[0] += 2.5+ return received[len(calls) - 1]++ monkeypatch.setattr(configuration.time, "monotonic", lambda: clock[0])+ monkeypatch.setattr(ssrf_proxy, "get", get)+ assert resolve_project_destination(VALUES) is None+ assert calls == [5.0, 2.5]+ assert all(item.is_closed for item in received)+++def test_slow_response_chunks_stop_at_the_shared_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ clock = [100.0]+ chunks: list[int] = []+ received = httpx.Response(200)++ def stream() -> Iterator[bytes]:+ for index in range(10):+ clock[0] += 2.0+ chunks.append(index)+ yield b" "++ monkeypatch.setattr(configuration.time, "monotonic", lambda: clock[0])+ monkeypatch.setattr(received, "iter_bytes", stream)+ monkeypatch.setattr(ssrf_proxy, "get", lambda _url, **_options: received)+ assert resolve_project_destination(VALUES) is None+ assert chunks == [0, 1, 2]+ assert received.is_closed+++ "change",+ [+ {"endpoint": "https://custom.test"},+ {"endpoint": "https://otlp.eu-west-1a.arize.com"},+ {"endpoint": "http://otlp.arize.com"},+ {"endpoint": "https://user:[email protected]"},+ {"endpoint": "https://otlp.arize.com:444"},+ {"space_id": "../bad"},+ {"api_key": None},+ {"api_key": ""},+ {"api_key": "******"},+ {"api_key": "key\nheader"},+ ],+)+def test_unsupported_destination_or_credentials_do_not_send_to_an_assumed_host(+ calls: list[tuple[str, dict[str, object]]], change: dict[str, str | None]+) -> None:+ assert resolve_project_destination({**VALUES, **change}) is None+ assert calls == []+++ "change",+ [+ {"endpoint": "https://otlp.arize.com:443"},+ {"space_id": "other"},+ {"project": "later"},+ {DESTINATION_KEY: {**DESTINATION, "organization_id": "bad?query"}},+ {DESTINATION_KEY: {**DESTINATION, "project_id": "../bad"}},+ {DESTINATION_KEY: None},+ ],+)+def test_get_formatter_rejects_stale_or_forged_metadata_without_io(+ monkeypatch: pytest.MonkeyPatch, change: dict[str, object]+) -> None:+ def forbidden(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("GET must not call the vendor")++ monkeypatch.setattr(ssrf_proxy, "get", forbidden)+ assert project_url({**VALUES, DESTINATION_KEY: bind_destination(VALUES, DESTINATION), **change}) is Nonediff --git a/api/tests/unit_tests/services/tracing/test_arize_configuration_ack.py b/api/tests/unit_tests/services/tracing/test_arize_configuration_ack.pynew file mode 100644index 00000000000..1bf6b77dcfc--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_arize_configuration_ack.py@@ -0,0 +1,315 @@+"""A real gRPC collector must acknowledge Arize credentials before config persistence."""++from collections.abc import Iterator+from concurrent.futures import ThreadPoolExecutor+from threading import Event+from typing import Never++import grpc # type: ignore[import-untyped]+import httpx+import pytest+from dify_trace_arize_phoenix import arize_configuration as configuration+from dify_trace_arize_phoenix.arize_configuration import ArizeConfigurationError, verify_configuration+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace+from dify_trace_arize_phoenix.config import ArizeConfig+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (+ ExportTraceServiceRequest,+ ExportTraceServiceResponse,+)++from core.helper import ssrf_proxy+from core.ops.ops_trace_manager import OpsTraceManager+from services.app_tracing_config_gateway import OpsTraceManagerGateway+from services.app_tracing_config_service import AppTracingConfigVerificationFailedError++pytest_plugins = ("tests.unit_tests.services.tracing.test_nullable_tracing_credentials",)+++def collector() -> Iterator[tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]]]:+ calls: list[tuple[bytes, dict[str, str]]] = []+ state: dict[str, object] = {"status": grpc.StatusCode.OK}+ release = Event()+ state["release"] = release++ def export(request: ExportTraceServiceRequest, context: grpc.ServicerContext) -> ExportTraceServiceResponse:+ headers = {+ key: value.decode() if isinstance(value, bytes) else value for key, value in context.invocation_metadata()+ }+ calls.append((request.SerializeToString(), headers))+ if headers.get("api_key") != "synthetic-good" or headers.get("space_id") != "synthetic-space":+ context.abort(grpc.StatusCode.PERMISSION_DENIED, "private-vendor-details")+ code = state["status"]+ if isinstance(code, grpc.StatusCode) and code is not grpc.StatusCode.OK:+ context.abort(code, "private-vendor-details")+ if state.get("delay"):+ release.wait(2)+ result = ExportTraceServiceResponse()+ if state.get("partial"):+ result.partial_success.rejected_spans = 1+ if state.get("message"):+ result.partial_success.error_message = "private-vendor-details"+ return result++ def dispatch(*args: object, **kwargs: object) -> ExportTraceServiceResponse:+ assert kwargs == {}+ request, context = args+ assert isinstance(request, ExportTraceServiceRequest)+ assert isinstance(context, grpc.ServicerContext)+ return export(request, context)++ handler = grpc.unary_unary_rpc_method_handler(+ dispatch,+ request_deserializer=ExportTraceServiceRequest.FromString,+ response_serializer=ExportTraceServiceResponse.SerializeToString,+ )+ with ThreadPoolExecutor(max_workers=2) as pool:+ server = grpc.server(pool)+ server.add_generic_rpc_handlers(+ (+ grpc.method_handlers_generic_handler(+ "opentelemetry.proto.collector.trace.v1.TraceService", {"Export": handler}+ ),+ )+ )+ port = server.add_insecure_port("127.0.0.1:0")+ assert port > 0+ server.start()+ try:+ yield f"http://127.0.0.1:{port}", calls, state+ finally:+ release.set()+ server.stop(0).wait(2)+++def test_rejected_real_collector_ack_cannot_be_saved(+ workspace: str,+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+) -> None:+ endpoint, calls, _ = collector+ with pytest.raises(AppTracingConfigVerificationFailedError):+ OpsTraceManagerGateway().prepare_new_config(+ workspace_id=workspace,+ tracing_provider="arize",+ tracing_config={+ "endpoint": endpoint,+ "project": "synthetic-project",+ "space_id": "synthetic-space",+ "api_key": "synthetic-invalid",+ },+ )+ assert len(calls) == 1+++def test_synchronous_ack_is_resource_only_and_has_no_sdk_or_environment_side_effects(+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ endpoint, calls, _ = collector+ attempted: list[bool] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ attempted.append(True)+ raise AssertionError("Arize verification must not construct the SDK tracer")++ monkeypatch.setattr(ArizePhoenixDataTrace, "__init__", forbidden)+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "api_key=ambient-invalid,space_id=ambient-other")+ verify_configuration(+ ArizeConfig(+ endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space", project="synthetic-project"+ )+ )+ assert attempted == []+ assert len(calls) == 1+ payload = ExportTraceServiceRequest.FromString(calls[0][0])+ assert len(payload.resource_spans) == 1+ assert {a.key: a.value.string_value for a in payload.resource_spans[0].resource.attributes} == {+ "model_id": "synthetic-project",+ "openinference.project.name": "synthetic-project",+ }+ assert len(payload.resource_spans[0].scope_spans) == 0+ assert calls[0][1]["api_key"] == "synthetic-good"+ assert calls[0][1]["space_id"] == "synthetic-space"+ assert calls[0][1]["authorization"] == "Bearer synthetic-good"+ assert "ambient" not in str(calls)+++ ("code", "expected"),+ [+ (grpc.StatusCode.UNAUTHENTICATED, "authentication_failed"),+ (grpc.StatusCode.PERMISSION_DENIED, "permission_denied"),+ (grpc.StatusCode.UNAVAILABLE, "connection_failed"),+ (grpc.StatusCode.RESOURCE_EXHAUSTED, "connection_failed"),+ ],+)+def test_real_rpc_rejection_uses_only_safe_fixed_diagnostics(+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ code: grpc.StatusCode,+ expected: str,+ caplog: pytest.LogCaptureFixture,+) -> None:+ endpoint, calls, state = collector+ state["status"] = code+ with pytest.raises(ArizeConfigurationError) as caught:+ verify_configuration(ArizeConfig(endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space"))+ assert caught.value.code == expected+ assert caught.value.__suppress_context__+ assert "synthetic-good" not in str(caught.value)+ assert "private-vendor-details" not in str(caught.value)+ assert "private-vendor-details" not in caplog.text+ assert len(calls) == 1++[email protected]("partial", ["partial", "message"])+def test_partial_or_error_ack_is_not_success(+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ partial: str,+) -> None:+ endpoint, calls, state = collector+ state[partial] = True+ with pytest.raises(ArizeConfigurationError, match="unexpected_response"):+ verify_configuration(ArizeConfig(endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space"))+ assert len(calls) == 1+++def test_deadline_is_finite_and_does_not_retry(+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ endpoint, calls, state = collector+ state["delay"] = True+ monkeypatch.setattr(configuration, "_VERIFY_TIMEOUT_SECONDS", 0.05)+ with pytest.raises(ArizeConfigurationError, match="connection_failed"):+ verify_configuration(ArizeConfig(endpoint=endpoint, api_key="synthetic-good", space_id="synthetic-space"))+ assert len(calls) == 1++[email protected]("reject", [False, True])+def test_tls_channel_is_explicit_suppressed_and_closed_on_all_outcomes(+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ monkeypatch: pytest.MonkeyPatch,+ reject: bool,+) -> None:+ endpoint, _, state = collector+ if reject:+ state["status"] = grpc.StatusCode.PERMISSION_DENIED+ original = grpc.insecure_channel+ options_seen: list[tuple[str, object, object]] = []+ closed: list[bool] = []++ def secure(target: str, credentials: grpc.ChannelCredentials, options: object) -> grpc.Channel:+ assert isinstance(credentials, grpc.ChannelCredentials)+ options_seen.append((target, options, get_value(_SUPPRESS_INSTRUMENTATION_KEY)))+ channel = original(endpoint.removeprefix("http://"))+ close = channel.close++ def close_recording() -> None:+ closed.append(True)+ close()++ monkeypatch.setattr(channel, "close", close_recording)+ return channel++ monkeypatch.setattr(grpc, "secure_channel", secure)+ before = get_value(_SUPPRESS_INSTRUMENTATION_KEY)+ config = ArizeConfig(api_key="synthetic-good", space_id="synthetic-space")+ if reject:+ with pytest.raises(ArizeConfigurationError):+ verify_configuration(config)+ else:+ verify_configuration(config)+ assert options_seen == [+ ("otlp.arize.com:443", (("grpc.enable_retries", 0), ("grpc.max_receive_message_length", 1024 * 1024)), True)+ ]+ assert closed == [True]+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == before+++ ("field", "value"),+ [+ ("api_key", None),+ ("api_key", ""),+ ("api_key", "******"),+ ("api_key", "key\nheader"),+ ("space_id", None),+ ("space_id", " "),+ ("space_id", "******"),+ ("endpoint", "https://user:[email protected]"),+ ("endpoint", "https://otlp.arize.com:0"),+ ],+)+def test_invalid_auth_values_never_construct_a_channel(+ monkeypatch: pytest.MonkeyPatch,+ field: str,+ value: str | None,+) -> None:+ attempted: list[bool] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ attempted.append(True)+ raise AssertionError("Invalid configuration must be rejected before transport construction")++ monkeypatch.setattr(grpc, "secure_channel", forbidden)+ monkeypatch.setattr(grpc, "insecure_channel", forbidden)+ config = ArizeConfig.model_validate({"api_key": "synthetic-good", "space_id": "synthetic-space", field: value})+ with pytest.raises(ArizeConfigurationError, match="invalid_configuration"):+ verify_configuration(config)+ assert attempted == []++[email protected]("accepted", [False, True])+def test_legacy_api_check_also_requires_the_real_ack(+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ accepted: bool,+) -> None:+ endpoint, calls, _ = collector+ instance = ArizePhoenixDataTrace(+ ArizeConfig(+ endpoint=endpoint, api_key="synthetic-good" if accepted else "synthetic-invalid", space_id="synthetic-space"+ )+ )+ try:+ if accepted:+ assert instance.api_check()+ else:+ with pytest.raises(ArizeConfigurationError, match="permission_denied"):+ instance.api_check()+ finally:+ instance.processor.shutdown()+ assert len(calls) == 1+ assert len(ExportTraceServiceRequest.FromString(calls[0][0]).resource_spans[0].scope_spans) == 0+++def test_real_ingest_ack_without_rest_read_permission_can_still_save(+ workspace: str,+ collector: tuple[str, list[tuple[bytes, dict[str, str]]], dict[str, object]],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ endpoint, rpc_calls, _ = collector+ original = grpc.insecure_channel+ reads: list[str] = []++ def secure(_target: str, _credentials: grpc.ChannelCredentials, **_options: object) -> grpc.Channel:+ return original(endpoint.removeprefix("http://"))++ def get(url: str, **_options: object) -> httpx.Response:+ reads.append(url)+ return httpx.Response(403, json={"message": "private-vendor-details"})++ monkeypatch.setattr(grpc, "secure_channel", secure)+ monkeypatch.setattr(ssrf_proxy, "get", get)+ saved = OpsTraceManagerGateway().prepare_new_config(+ workspace_id=workspace,+ tracing_provider="arize",+ tracing_config={"api_key": "synthetic-good", "space_id": "synthetic-space", "project": "not-created"},+ )+ assert len(rpc_calls) == len(reads) == 1+ assert reads[0] == "https://api.arize.com/v2/spaces/synthetic-space"+ assert "project_url" not in saved+ assert "arize_project_destination" not in saved+ assert OpsTraceManager.decrypt_tracing_config(workspace, "arize", saved)["api_key"] == "synthetic-good"+ payload = ExportTraceServiceRequest.FromString(rpc_calls[0][0])+ assert len(payload.resource_spans[0].scope_spans) == 0diff --git a/api/tests/unit_tests/services/tracing/test_databricks_configuration.py b/api/tests/unit_tests/services/tracing/test_databricks_configuration.pynew file mode 100644index 00000000000..e16ff2891e9--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_databricks_configuration.py@@ -0,0 +1,230 @@+"""Official experiment schemas at a request-scoped, read-only HTTP boundary."""++from collections.abc import Callable+from dataclasses import dataclass, field++import httpx+import pytest+from dify_trace_mlflow.config import DatabricksConfig+from dify_trace_mlflow.configuration import DatabricksConfigurationError, project_url, verify_configuration+from google.protobuf.json_format import MessageToDict+from mlflow.protos.service_pb2 import Experiment, GetExperiment++from core.helper import ssrf_proxy+++def response(url: str, body: object, status: int = 200) -> httpx.Response:+ return httpx.Response(status, json=body, request=httpx.Request("GET", url))+++@dataclass+class ExperimentAPI:+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)+ override: Callable[[str, str, dict[str, object]], httpx.Response] | None = None++ def get(self, url: str, **options: object) -> httpx.Response:+ self.calls.append(("GET", url, options))+ if self.override:+ return self.override("GET", url, options)+ assert url.endswith("/api/2.0/mlflow/experiments/get")+ params = options["params"]+ assert isinstance(params, dict)+ requested = GetExperiment(experiment_id=params["experiment_id"])+ return response(+ url,+ MessageToDict(+ GetExperiment.Response(+ experiment=Experiment(experiment_id=requested.experiment_id, lifecycle_stage="active")+ ),+ preserving_proto_field_name=True,+ ),+ )++ def post(self, url: str, **options: object) -> httpx.Response:+ self.calls.append(("POST", url, options))+ assert url.endswith("/oidc/v1/token"), "configuration must never create experiments, runs or traces"+ if self.override:+ return self.override("POST", url, options)+ return response(url, {"access_token": "synthetic-oauth-token", "token_type": "Bearer", "expires_in": 3600})+++def experiment_api(monkeypatch: pytest.MonkeyPatch) -> ExperimentAPI:+ api = ExperimentAPI()+ monkeypatch.setattr(ssrf_proxy, "get", api.get)+ monkeypatch.setattr(ssrf_proxy, "post", api.post)+ return api+++def test_pat_requests_are_isolated_and_read_only(experiment_api: ExperimentAPI) -> None:+ for host, token, experiment_id in [+ ("https://first.test/", "first-pat", "000123"),+ ("https://second.test", "second-pat", "456"),+ ]:+ config = DatabricksConfig(host=host, experiment_id=experiment_id, personal_access_token=token)+ result = verify_configuration(config)+ assert result.host == host.rstrip("/")+ assert result.experiment_id == experiment_id.lstrip("0")+ method, url, options = experiment_api.calls[-1]+ assert method == "GET"+ assert url == host.rstrip("/") + "/api/2.0/mlflow/experiments/get"+ assert options["headers"] == {"Authorization": f"Bearer {token}", "Accept": "application/json"}+ assert options["max_retries"] == 0+ assert options["timeout"] == 30.0+ assert options["follow_redirects"] is False+ assert len(experiment_api.calls) == 2+++def test_oauth_precedence_matches_exporter_without_reusing_pat(experiment_api: ExperimentAPI) -> None:+ config = DatabricksConfig(+ host="https://workspace.test",+ experiment_id="12",+ personal_access_token="unused-pat",+ client_id="synthetic-client",+ client_secret="synthetic-secret",+ )+ assert verify_configuration(config).experiment_id == "12"+ method, url, options = experiment_api.calls[0]+ assert method == "POST"+ assert url == "https://workspace.test/oidc/v1/token"+ assert options["auth"] == ("synthetic-client", "synthetic-secret")+ assert options["data"] == {"grant_type": "client_credentials", "scope": "all-apis"}+ assert "headers" not in options+ assert options["follow_redirects"] is False+ assert experiment_api.calls[1][2]["headers"] == {+ "Authorization": "Bearer synthetic-oauth-token",+ "Accept": "application/json",+ }+++def test_incomplete_oauth_uses_explicit_pat_as_existing_exporter_does(experiment_api: ExperimentAPI) -> None:+ verify_configuration(+ DatabricksConfig(+ host="https://workspace.test", experiment_id="12", client_id="unused-client", personal_access_token="pat"+ )+ )+ assert len(experiment_api.calls) == 1+ assert experiment_api.calls[0][0] == "GET"+ assert experiment_api.calls[0][2]["headers"] == {"Authorization": "Bearer pat", "Accept": "application/json"}+++def test_non_ascii_experiment_ids_are_not_sent(experiment_api: ExperimentAPI) -> None:+ with pytest.raises(DatabricksConfigurationError) as caught:+ verify_configuration(+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="pat")+ )+ assert caught.value.code == "invalid_experiment_id"+ assert experiment_api.calls == []+++ "credentials",+ [+ {},+ {"client_id": "client"},+ {"client_secret": "secret"},+ {"personal_access_token": " "},+ ],+)+def test_missing_credentials_fail_before_vendor_io(experiment_api: ExperimentAPI, credentials: dict[str, str]) -> None:+ with pytest.raises(DatabricksConfigurationError) as caught:+ verify_configuration(DatabricksConfig(host="https://workspace.test", experiment_id="12", **credentials))+ assert caught.value.code == "authentication_failed"+ assert experiment_api.calls == []+++ "host",+ [+ "http://workspace.test",+ "https://user:[email protected]",+ "https://workspace.test/path",+ "https://workspace.test?o=1",+ "https://workspace.test/#/experiments/12",+ "https://workspace.test\\@evil.test",+ "https://workspace.test:\n443",+ "https://workspace.test:bad",+ "https://[invalid",+ ],+)+def test_invalid_workspace_origins_do_not_send_credentials(experiment_api: ExperimentAPI, host: str) -> None:+ with pytest.raises(DatabricksConfigurationError) as caught:+ verify_configuration(DatabricksConfig(host=host, experiment_id="12", personal_access_token="synthetic-key"))+ assert caught.value.code == "invalid_workspace_url"+ assert experiment_api.calls == []+++def test_project_url_is_pure_and_supports_explicit_https_port(experiment_api: ExperimentAPI) -> None:+ config = DatabricksConfig(host=" https://Workspace.Example:443/ ", experiment_id="00012")+ assert project_url(config) == "https://Workspace.Example:443/ml/experiments/12/traces"+ assert experiment_api.calls == []+++ ("status", "code"),+ [+ (401, "authentication_failed"),+ (403, "permission_denied"),+ (404, "experiment_not_found_or_inaccessible"),+ (302, "endpoint_unavailable"),+ (500, "endpoint_unavailable"),+ ],+)+def test_http_failures_are_safe_and_responses_are_closed(experiment_api: ExperimentAPI, status: int, code: str) -> None:+ result = response("https://workspace.test", {"message": "private-key-vendor-response"}, status)+ experiment_api.override = lambda _method, _url, _options: result+ with pytest.raises(DatabricksConfigurationError) as caught:+ verify_configuration(+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="synthetic-key")+ )+ assert caught.value.code == code+ assert "private-key" not in str(caught.value)+ assert "synthetic-key" not in str(caught.value)+ assert result.is_closed+++ ("body", "code"),+ [+ ({}, "malformed_response"),+ ([], "malformed_response"),+ ({"experiment": "invalid"}, "malformed_response"),+ ({"experiment": {"experiment_id": "other", "lifecycle_stage": "active"}}, "experiment_id_mismatch"),+ ({"experiment": {"experiment_id": "12", "lifecycle_stage": "deleted"}}, "experiment_not_active"),+ ({"experiment": {"experiment_id": "12"}}, "experiment_not_active"),+ ],+)+def test_response_must_identify_selected_active_experiment(+ experiment_api: ExperimentAPI, body: object, code: str+) -> None:+ experiment_api.override = lambda _method, url, _options: response(url, body)+ with pytest.raises(DatabricksConfigurationError) as caught:+ verify_configuration(+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="synthetic-key")+ )+ assert caught.value.code == code++[email protected]("body", [{}, {"access_token": ""}, {"access_token": "token", "token_type": "other"}])+def test_invalid_oauth_response_never_reaches_experiment_api(experiment_api: ExperimentAPI, body: object) -> None:+ experiment_api.override = lambda _method, url, _options: response(url, body)+ with pytest.raises(DatabricksConfigurationError, match="authentication failed"):+ verify_configuration(+ DatabricksConfig(+ host="https://workspace.test", experiment_id="12", client_id="client", client_secret="secret"+ )+ )+ assert len(experiment_api.calls) == 1+++def test_transport_errors_do_not_disclose_request_details(experiment_api: ExperimentAPI) -> None:+ def fail(_method: str, _url: str, _options: dict[str, object]) -> httpx.Response:+ raise httpx.ConnectError("private-key-vendor-response")++ experiment_api.override = fail+ with pytest.raises(DatabricksConfigurationError) as caught:+ verify_configuration(+ DatabricksConfig(host="https://workspace.test", experiment_id="12", personal_access_token="synthetic-key")+ )+ assert caught.value.code == "connection_failed"+ assert "private-key" not in str(caught.value)diff --git a/api/tests/unit_tests/services/tracing/test_export_lease.py b/api/tests/unit_tests/services/tracing/test_export_lease.pynew file mode 100644index 00000000000..7d742207d6b--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_export_lease.py@@ -0,0 +1,223 @@+"""Real SQLite state transitions and background renewal for delivery leases.++PostgreSQL row-lock scheduling and the deployed Celery pool still need dev+validation; these tests establish token and acknowledgement semantics.+"""++import threading+from datetime import timedelta++import pytest+from sqlalchemy.orm import Session, sessionmaker++from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceRun+from repositories import application_trace_repository as repository_module+from repositories.application_trace_repository import TraceCollectionError+from services.tracing import service as service_module+from services.tracing.provider_dispatch import TraceExportResult, TraceExportUncertainError+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ Collection,+ batch_for,+ claim,+ ingest,+ seed,+ terminal,+)+++def case(sqlite_session_factory: sessionmaker[Session]) -> Collection:+ result = seed(sqlite_session_factory, "lease-test")+ ingest(result, batch_for(result.run, result.invocation()))+ terminal(result)+ return result+++def _stored(case: Collection) -> ApplicationTraceRun:+ row = case.repository.get(case.run.id)+ assert row is not None+ return row+++def _expire(case: Collection) -> str:+ leased = claim(case)+ assert leased is not None+ with case.sessions.begin() as session:+ row = session.get(ApplicationTraceRun, case.run.id)+ assert row is not None+ row.export_started_at = naive_utc_now() - timedelta(seconds=60)+ row.export_heartbeat_at = row.export_started_at+ assert claim(case) is None+ assert _stored(case).state == "uncertain"+ return leased[1]+++def test_renewal_keeps_long_export_alive_then_missing_heartbeat_expires(+ case: Collection, monkeypatch: pytest.MonkeyPatch+) -> None:+ clock = [naive_utc_now()]+ monkeypatch.setattr(repository_module, "naive_utc_now", lambda: clock[0])+ leased = claim(case)+ assert leased is not None+ token = leased[1]+ clock[0] += timedelta(seconds=20)+ assert case.repository.renew_export_lease(case.run.id, token)+ clock[0] += timedelta(seconds=20)+ assert claim(case) is None+ current = _stored(case)+ assert current.state == "exporting"+ assert current.attempts == 1+ assert current.export_started_at is not None+ assert current.export_started_at + timedelta(seconds=30) < clock[0]+ clock[0] += timedelta(seconds=11)+ assert claim(case) is None+ expired = _stored(case)+ assert expired.state == "uncertain"+ assert expired.error_code == "export_lease_expired"+ assert expired.export_token == token+ assert not case.repository.renew_export_lease(case.run.id, token)+++def test_old_rows_without_heartbeat_use_started_at(case: Collection) -> None:+ leased = claim(case)+ assert leased is not None+ with case.sessions.begin() as session:+ row = session.get(ApplicationTraceRun, case.run.id)+ assert row is not None+ row.export_started_at = naive_utc_now() - timedelta(seconds=60)+ row.export_heartbeat_at = None+ assert claim(case) is None+ current = _stored(case)+ assert current.state == "uncertain"+ assert current.export_token == leased[1]+++def test_wrong_token_cannot_renew_or_complete(case: Collection) -> None:+ leased = claim(case)+ assert leased is not None+ before = _stored(case)+ assert not case.repository.renew_export_lease(case.run.id, "other-token")+ case.repository.complete(case.run.id, "other-token", state="accepted", result=TraceExportResult("wrong"))+ current = _stored(case)+ assert current.export_token == leased[1]+ assert current.export_heartbeat_at == before.export_heartbeat_at+ assert current.state == "exporting"+ assert current.provider_trace_id is None+++def test_late_success_wins_before_manual_retry_and_is_idempotent(case: Collection) -> None:+ token = _expire(case)+ result = TraceExportResult("confirmed", "https://example.com/trace")+ case.repository.complete(case.run.id, token, state="accepted", result=result)+ current = _stored(case)+ assert current.state == "accepted"+ assert current.error_code is None+ assert current.export_token is None+ assert current.accepted_at is not None+ assert current.trace_url == result.trace_url+ case.repository.complete(case.run.id, token, state="accepted", result=TraceExportResult("duplicate-ack"))+ assert _stored(case).provider_trace_id == "confirmed"+ with pytest.raises(TraceCollectionError, match="tracing_already_accepted"):+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)+++def test_manual_retry_revokes_old_heartbeat_and_ack_before_and_after_new_claim(case: Collection) -> None:+ old_token = _expire(case)+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)+ assert not case.repository.renew_export_lease(case.run.id, old_token)+ case.repository.complete(case.run.id, old_token, state="accepted", result=TraceExportResult("old"))+ assert _stored(case).state == "ready"+ second = claim(case)+ assert second is not None+ assert second[1] != old_token+ case.repository.complete(case.run.id, old_token, state="accepted", result=TraceExportResult("old"))+ current = _stored(case)+ assert current.state == "exporting"+ assert current.export_token == second[1]+ assert current.provider_trace_id is None+ case.repository.complete(case.run.id, second[1], state="accepted", result=TraceExportResult("new"))+ assert _stored(case).provider_trace_id == "new"++[email protected]("late_state", ["failed", "retrying", "uncertain"])+def test_late_non_success_cannot_overwrite_lease_expiry(case: Collection, late_state: str) -> None:+ token = _expire(case)+ case.repository.complete(case.run.id, token, state=late_state, error_code="late-failure")+ current = _stored(case)+ assert current.state == "uncertain"+ assert current.error_code == "export_lease_expired"+++def test_other_uncertain_reason_cannot_be_promoted_by_late_ack(case: Collection) -> None:+ token = _expire(case)+ with case.sessions.begin() as session:+ row = session.get(ApplicationTraceRun, case.run.id)+ assert row is not None+ row.error_code = "provider_acceptance_unknown"+ case.repository.complete(case.run.id, token, state="accepted", result=TraceExportResult("late"))+ assert _stored(case).state == "uncertain"++[email protected]("transient_failure", [False, True])[email protected]("outcome", ["accepted", "uncertain"])+def test_service_heartbeats_during_export_and_stops_on_every_outcome(+ case: Collection,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ transient_failure: bool,+ outcome: str,+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_EXPORT_LEASE_SECONDS=1)+ clock = [naive_utc_now()]+ monkeypatch.setattr(repository_module, "naive_utc_now", lambda: clock[0])+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ renewed = threading.Event()+ calls = []+ original = case.repository.renew_export_lease++ def renew(run_id: str, token: str) -> bool:+ calls.append(threading.get_ident())+ if transient_failure and len(calls) == 1:+ raise RuntimeError("private-provider-value")+ result = original(run_id, token)+ renewed.set()+ return result++ monkeypatch.setattr(case.repository, "renew_export_lease", renew)++ def export(*_args: object) -> TraceExportResult:+ # The original start time is now beyond the lease; only a real background+ # transaction can prevent a second claimant from marking it uncertain.+ clock[0] += timedelta(seconds=3)+ assert renewed.wait(3), "background heartbeat did not commit"+ assert case.repository.claim(case.run.id, collection_timeout=60, lease_seconds=1) is None+ assert _stored(case).state == "exporting"+ if outcome == "uncertain":+ raise TraceExportUncertainError("lost acknowledgement")+ return TraceExportResult("ack")++ monkeypatch.setattr(service_module, "export_trace", export)+ ApplicationTracingService(case.repository).deliver(case.run.id)+ assert _stored(case).state == outcome+ assert calls+ assert all(thread_id != threading.get_ident() for thread_id in calls)+ assert not any(thread.name == f"application-trace-lease-{case.run.id}" for thread in threading.enumerate())+ assert "private-provider-value" not in caplog.text+++def test_heartbeat_stops_even_when_completion_database_write_raises(+ case: Collection, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ monkeypatch.setattr(service_module, "export_trace", lambda *_args: TraceExportResult("ack"))++ def fail_complete(*_args: object, **_kwargs: object) -> None:+ raise RuntimeError("database unavailable")++ monkeypatch.setattr(case.repository, "complete", fail_complete)+ with pytest.raises(RuntimeError, match="database unavailable"):+ ApplicationTracingService(case.repository).deliver(case.run.id)+ assert not any(thread.name == f"application-trace-lease-{case.run.id}" for thread in threading.enumerate())diff --git a/api/tests/unit_tests/services/tracing/test_invalid_trace_delivery.py b/api/tests/unit_tests/services/tracing/test_invalid_trace_delivery.pynew file mode 100644index 00000000000..fbfd62dd790--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_invalid_trace_delivery.py@@ -0,0 +1,55 @@+"""Malformed durable traces must never be sent or offered for provider retry."""++from uuid import uuid4++import pytest+from dify_agent.protocol.tracing import AgentTraceSpan+from sqlalchemy.orm import Session, sessionmaker++from models.application_trace import ApplicationTraceRun+from repositories.application_trace_repository import TraceCollectionError+from services.tracing import service as service_module+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.services.tracing.test_application_trace_collection import batch_for, ingest, seed, terminal++[email protected]("invalid", ["cycle", "invalid_envelope"])+def test_invalid_tree_is_permanent_before_decryption_or_provider_io(+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, invalid: str+) -> None:+ case = seed(sqlite_session_factory, str(uuid4()))+ batch = batch_for(case.run, case.invocation())+ if invalid == "cycle":+ for span_id, parent_id in [("b" * 16, "c" * 16), ("c" * 16, "b" * 16)]:+ batch.spans.append(+ AgentTraceSpan(+ trace_id=case.run.trace_id,+ span_id=span_id,+ parent_span_id=parent_id,+ name="cyclic tool",+ start_time_ns=1_000_000_000,+ end_time_ns=2_000_000_000,+ status="ok",+ )+ )+ ingest(case, batch)+ terminal(case)+ if invalid == "invalid_envelope":+ with case.sessions.begin() as session:+ row = session.get(ApplicationTraceRun, case.run.id)+ assert row is not None+ row.outer_trace = {"corrupt": True}++ def forbidden(*_args: object, **_kwargs: object) -> None:+ pytest.fail("invalid traces must fail before credentials or provider I/O")++ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", forbidden)+ monkeypatch.setattr(service_module, "export_trace", forbidden)+ ApplicationTracingService(case.repository).deliver(case.run.id)+ current = case.repository.get(case.run.id)+ assert current is not None+ assert current.state == "failed"+ assert current.error_code == "tracing_payload_invalid"+ assert current.attempts == 1+ with pytest.raises(TraceCollectionError, match="tracing_retry_not_allowed"):+ case.repository.retry(run_id=case.run.id, acknowledge_possible_duplicates=True)diff --git a/api/tests/unit_tests/services/tracing/test_legacy_aliyun_transport.py b/api/tests/unit_tests/services/tracing/test_legacy_aliyun_transport.pynew file mode 100644index 00000000000..811cf01feb4--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_legacy_aliyun_transport.py@@ -0,0 +1,344 @@+"""Persisted legacy sources through real OTel protobuf, replacing only HTTP I/O."""++import json+import os+from collections.abc import Mapping+from concurrent.futures import ThreadPoolExecutor+from datetime import UTC, datetime, timedelta+from decimal import Decimal+from threading import Barrier, Thread+from typing import TypedDict, Unpack+from uuid import UUID++import httpx+import pytest+from dify_trace_aliyun.aliyun_trace import AliyunDataTrace+from dify_trace_aliyun.config import AliyunConfig+from dify_trace_aliyun.data_exporter.traceclient import TraceClient, convert_to_span_id+from dify_trace_aliyun.entities.aliyun_trace_entity import SpanData+from dify_trace_aliyun.entities.semconv import GEN_AI_USAGE_TOTAL_TOKENS+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (+ ExportTraceServiceRequest,+ ExportTraceServiceResponse,+)+from sqlalchemy.orm import Session++from core.helper import ssrf_proxy+from core.ops.entities.trace_entity import MessageTraceInfo, ModerationTraceInfo, WorkflowTraceInfo+from core.ops.exceptions import RetryableTraceDispatchError+from graphon.enums import BuiltinNodeTypes, WorkflowExecutionStatus, WorkflowNodeExecutionStatus+from models.account import TenantAccountJoin, TenantAccountRole+from models.enums import ConversationFromSource, CreatorUserRole, MessageStatus+from models.model import AppMode+from models.workflow import (+ WorkflowNodeExecutionModel,+ WorkflowNodeExecutionTriggeredFrom,+ WorkflowRun,+ WorkflowRunTriggeredFrom,+ WorkflowType,+)+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError+from tests.unit_tests.model_factories import make_account, make_app, make_message, make_tenant++START = datetime(2026, 10, 19, tzinfo=UTC)+RUN_ID = "a0000000-0000-4000-8000-000000000001"+MESSAGE_ID = "a0000000-0000-4000-8000-000000000002"+NODE_IDS = [f"b0000000-0000-4000-8000-00000000000{i}" for i in range(3)]+++class RequestOptions(TypedDict, total=False):+ content: bytes+ headers: Mapping[str, str]+ max_retries: int+ follow_redirects: bool+ timeout: float+++def config(**overrides: object) -> AliyunConfig:+ return AliyunConfig.model_validate(+ {+ "app_name": "owned-service",+ "license_key": "owned-key",+ "endpoint": "https://collector.test/prefix",+ **overrides,+ }+ )+++def decoded(options: RequestOptions) -> ExportTraceServiceRequest:+ payload = ExportTraceServiceRequest()+ payload.ParseFromString(options["content"])+ return payload+++def test_constructor_and_presentation_have_no_network_sdk_queue_or_environment_mutation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def forbidden(*_args: object, **_kwargs: object) -> None:+ pytest.fail("Constructor and presentation must not create I/O, exporters or worker threads")++ monkeypatch.setattr(ssrf_proxy, "make_request", forbidden)+ monkeypatch.setattr(OTLPSpanExporter, "__init__", forbidden)+ monkeypatch.setattr(Thread, "start", forbidden)+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://ambient.test")+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=ambient")+ before = dict(os.environ)+ original = config()+ adapter = AliyunDataTrace(original)+ original.license_key = "mutated-after-construction"+ assert adapter.trace_client.endpoint == "https://collector.test/prefix/adapt_owned-key/api/otlp/traces"+ assert adapter.get_project_url() == "https://arms.console.aliyun.com/#/llm"+ assert not hasattr(adapter.trace_client, "exporter")+ assert not hasattr(adapter.trace_client, "queue")+ assert not hasattr(adapter.trace_client, "worker_thread")+ assert dict(os.environ) == before++[email protected]("agent_runtime_app")[email protected]("failure", [None, 400, 401, 403, 429, 503, "timeout", "partial", "malformed", "redirect"])+def test_persisted_workflow_uses_one_acknowledged_protobuf_batch_and_preserves_business_state(+ sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, failure: int | str | None+) -> None:+ tenant = make_tenant(tenant_id="tenant")+ account = make_account(account_id="owner", tenant=tenant)+ app = make_app(app_id="app", tenant_id=tenant.id, mode=AppMode.WORKFLOW, created_by=account.id)+ membership = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole.OWNER)+ run = WorkflowRun(+ id=RUN_ID,+ tenant_id=tenant.id,+ app_id=app.id,+ workflow_id="workflow",+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,+ version="v1",+ inputs=json.dumps({"query": "hello"}),+ outputs=json.dumps({"answer": "ok"}),+ status=WorkflowExecutionStatus.SUCCEEDED,+ total_tokens=32,+ created_by=account.id,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_at=START,+ finished_at=START + timedelta(seconds=1),+ elapsed_time=1,+ )+ sqlite_session.add_all([tenant, account, app, membership, run])+ nodes = [+ WorkflowNodeExecutionModel(+ id=NODE_IDS[i],+ tenant_id=tenant.id,+ app_id=app.id,+ workflow_id="workflow",+ workflow_run_id=RUN_ID,+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ index=i,+ node_id=f"node-{i}",+ title=title,+ node_type=kind,+ status=WorkflowNodeExecutionStatus.SUCCEEDED,+ created_by=account.id,+ created_by_role=CreatorUserRole.ACCOUNT,+ inputs=json.dumps({"synthetic": True}),+ outputs=json.dumps(+ {"text": "ok", "usage": {"prompt_tokens": 21, "completion_tokens": 11, "total_tokens": 32}}+ ),+ created_at=START + timedelta(milliseconds=i),+ finished_at=START + timedelta(milliseconds=i + 100),+ elapsed_time=0.1,+ process_data=json.dumps(+ {"prompts": [{"role": "user", "text": "hello"}], "model_provider": "test", "model_name": "test-model"}+ )+ if i == 1+ else None,+ )+ for i, (title, kind) in enumerate(+ [("Start", BuiltinNodeTypes.START), ("LLM", BuiltinNodeTypes.LLM), ("End", BuiltinNodeTypes.END)]+ )+ ]+ sqlite_session.add_all(nodes)+ sqlite_session.commit()+ info = WorkflowTraceInfo(+ query="hello",+ workflow_id="workflow",+ tenant_id=tenant.id,+ workflow_run_id=RUN_ID,+ workflow_run_elapsed_time=1.0,+ workflow_run_status="succeeded",+ workflow_run_inputs={"query": "hello"},+ workflow_run_outputs={"answer": "ok"},+ workflow_run_version="v1",+ total_tokens=32,+ file_list=[],+ metadata={"app_id": app.id, "conversation_id": "owned-conversation"},+ start_time=START,+ end_time=START + timedelta(seconds=1),+ )+ sent: list[tuple[str, RequestOptions, ExportTraceServiceRequest]] = []++ def send(method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ assert method == "POST"+ assert url == "https://collector.test/prefix/adapt_owned-key/api/otlp/traces"+ sent.append((url, options, decoded(options)))+ if len(sent) == 1:+ if failure == "timeout":+ raise httpx.ReadTimeout("synthetic lost ACK", request=httpx.Request("POST", url))+ if isinstance(failure, int):+ return httpx.Response(failure, content=b"DO_NOT_EXPOSE_BODY")+ if failure == "malformed":+ return httpx.Response(200, content=b"not protobuf")+ if failure == "partial":+ response = ExportTraceServiceResponse()+ response.partial_success.rejected_spans = 1+ response.partial_success.error_message = "DO_NOT_EXPOSE_BODY"+ return httpx.Response(200, content=response.SerializeToString())+ if failure == "redirect":+ return httpx.Response(307, headers={"Location": "https://unrelated.test"})+ return httpx.Response(200, content=ExportTraceServiceResponse().SerializeToString())++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ adapter = AliyunDataTrace(config())+ if failure is None:+ adapter.trace(info)+ else:+ expected = (+ RetryableTraceDispatchError+ if failure in {429, 503}+ else TraceExportUncertainError+ if failure in {"timeout", "partial", "malformed"}+ else TraceExportRejectedError+ )+ with pytest.raises(expected) as error:+ adapter.trace(info)+ assert "DO_NOT_EXPOSE_BODY" not in str(error.value)+ if isinstance(error.value, RetryableTraceDispatchError):+ assert error.value.acceptance_unknown is (expected is TraceExportUncertainError)+ assert len(sent) == 1 # No SDK retry, per-span request, batching thread or silent queue success.+ _, options, payload = sent[0]+ assert options["max_retries"] == 0+ assert options["follow_redirects"] is False+ assert options["headers"] == {"Content-Type": "application/x-protobuf", "Accept": "application/x-protobuf"}+ resource = payload.resource_spans[0]+ attributes = {item.key: item.value for item in resource.resource.attributes}+ assert attributes["service.name"].string_value == "owned-service"+ spans = list(resource.scope_spans[0].spans)+ assert len(spans) == 4+ assert [s.name for s in spans] == ["workflow", "Start", "LLM", "End"]+ root_id = convert_to_span_id(RUN_ID, "workflow").to_bytes(8)+ assert [s.span_id for s in spans] == [root_id, *[convert_to_span_id(i, "node").to_bytes(8) for i in NODE_IDS]]+ assert all(s.trace_id == UUID(RUN_ID).bytes for s in spans)+ assert [s.parent_span_id for s in spans] == [b"", root_id, root_id, root_id]+ assert spans[0].start_time_unix_nano == int(START.timestamp() * 1_000_000_000)+ assert spans[0].end_time_unix_nano == int((START + timedelta(seconds=1)).timestamp() * 1_000_000_000)+ model_attributes = {item.key: item.value for item in spans[2].attributes}+ assert model_attributes[GEN_AI_USAGE_TOTAL_TOKENS].string_value == "32"+ assert all(s.status.code == 1 for s in spans)+ assert adapter.trace_client._dispatch_spans.get() is None+ assert info.workflow_run_status == "succeeded"+ sqlite_session.refresh(run)+ assert run.status == WorkflowExecutionStatus.SUCCEEDED+ assert run.total_tokens == 32+ assert all(node.status == WorkflowNodeExecutionStatus.SUCCEEDED for node in nodes)+ # A fresh business operation does not inherit any old batch after success or error.+ adapter.trace(ModerationTraceInfo(flagged=False, action="allow", preset_response="", query="", metadata={}))+ assert len(sent) == 1+ adapter.trace(info.model_copy(update={"workflow_run_id": "a0000000-0000-4000-8000-000000000003"}))+ assert len(sent) == 2+ recovered_spans = list(sent[1][2].resource_spans[0].scope_spans[0].spans)+ assert [s.name for s in recovered_spans] == ["workflow"]++[email protected]("failed", [False, True])+def test_message_mapping_keeps_original_ids_raw_usage_error_and_business_state(+ sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, failed: bool+) -> None:+ message = make_message(+ message_id=MESSAGE_ID,+ app_id="app",+ conversation_id="conversation",+ query="hello",+ answer="ok",+ status=MessageStatus.ERROR if failed else MessageStatus.NORMAL,+ inputs={},+ message={},+ message_unit_price=Decimal(0),+ answer_unit_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.CONSOLE,+ from_account_id="owner",+ message_tokens=21,+ answer_tokens=11,+ )+ sqlite_session.add(message)+ sqlite_session.commit()+ info = MessageTraceInfo(+ conversation_model="chat",+ conversation_mode="chat",+ message_id=MESSAGE_ID,+ message_data=message,+ inputs={"prompt": "hello"},+ outputs="ok",+ message_tokens=21,+ answer_tokens=11,+ total_tokens=32,+ error="synthetic model failure" if failed else None,+ metadata={+ "app_id": "app",+ "conversation_id": "conversation",+ "ls_model_name": "test-model",+ "ls_provider": "test",+ },+ start_time=START,+ end_time=START + timedelta(seconds=1),+ )+ sent: list[ExportTraceServiceRequest] = []++ def send(_method: str, _url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ sent.append(decoded(options))+ return httpx.Response(200, content=b"")++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ adapter = AliyunDataTrace(config())+ adapter.trace(info)+ assert len(sent) == 1+ spans = list(sent[0].resource_spans[0].scope_spans[0].spans)+ assert [s.name for s in spans] == ["message", "llm"]+ root_id = convert_to_span_id(MESSAGE_ID, "message").to_bytes(8)+ assert [s.parent_span_id for s in spans] == [b"", root_id]+ assert [s.span_id for s in spans] == [root_id, convert_to_span_id(MESSAGE_ID, "llm").to_bytes(8)]+ assert all(s.trace_id == UUID(MESSAGE_ID).bytes and s.status.code == (2 if failed else 1) for s in spans)+ assert {a.key: a.value for a in spans[1].attributes}[GEN_AI_USAGE_TOTAL_TOKENS].string_value == "32"+ assert message.status == (MessageStatus.ERROR if failed else MessageStatus.NORMAL)+++def test_one_cached_client_keeps_interleaved_batches_and_failure_cleanup_isolated(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ client = TraceClient(service_name="owned-service", endpoint="https://collector.test/adapt_key/api/otlp/traces")+ barrier = Barrier(2)+ observed: list[list[str]] = []++ def send(_method: str, _url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ payload = decoded(options)+ observed.append([s.name for rs in payload.resource_spans for scope in rs.scope_spans for s in scope.spans])+ return httpx.Response(200, content=b"")++ monkeypatch.setattr(ssrf_proxy, "make_request", send)++ def run(label: str) -> None:+ try:+ with client.batch():+ client.add_span(SpanData(name=label, trace_id=1, span_id=2, start_time=1, end_time=2))+ barrier.wait(timeout=5)+ if label == "failed":+ raise ValueError("mapper failed before submission")+ client.add_span(SpanData(name=label + "-child", trace_id=1, span_id=3, start_time=1, end_time=2))+ except ValueError:+ assert label == "failed"+ assert client._dispatch_spans.get() is None++ with ThreadPoolExecutor(max_workers=2) as pool:+ list(pool.map(run, ["accepted", "failed"]))+ assert observed == [["accepted", "accepted-child"]]+ with client.batch():+ client.add_span(SpanData(name="next", trace_id=1, span_id=4, start_time=1, end_time=2))+ assert observed == [["accepted", "accepted-child"], ["next"]]diff --git a/api/tests/unit_tests/services/tracing/test_legacy_weave_transport.py b/api/tests/unit_tests/services/tracing/test_legacy_weave_transport.pynew file mode 100644index 00000000000..6ab4749fd89--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_legacy_weave_transport.py@@ -0,0 +1,521 @@+"""Real Weave SDK serialization with only the outbound HTTP boundary replaced."""++import atexit+import json+import os+from collections.abc import Mapping+from concurrent.futures import ThreadPoolExecutor+from datetime import UTC, datetime, timedelta+from pathlib import Path+from threading import Barrier+from typing import TypedDict, Unpack++import httpx+import pytest+import wandb+import weave+from dify_trace_weave.config import WeaveConfig+from dify_trace_weave.entities.weave_trace_entity import WeaveTraceModel+from dify_trace_weave.weave_trace import WeaveDataTrace+from sqlalchemy.orm import Session+from weave.trace_server.trace_server_interface import CallEndReq, CallStartReq, TraceStatus++from core.helper import ssrf_proxy+from core.ops.entities.trace_entity import GenerateNameTraceInfo, WorkflowTraceInfo+from core.ops.exceptions import RetryableTraceDispatchError+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus+from models.account import TenantAccountJoin, TenantAccountRole+from models.enums import CreatorUserRole+from models.model import AppMode+from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError+from tests.unit_tests.model_factories import make_account, make_app, make_tenant++START = datetime(2026, 10, 19, tzinfo=UTC)+++class RequestOptions(TypedDict, total=False):+ content: bytes+ json: dict[str, object]+ auth: tuple[str, str]+ headers: Mapping[str, str]+ max_retries: int+ follow_redirects: bool+ timeout: float+++def config(**overrides: object) -> WeaveConfig:+ return WeaveConfig.model_validate({"api_key": "fixture-key", "entity": "team", "project": "project", **overrides})+++def call(call_id: str = "call-1", **overrides: object) -> WeaveTraceModel:+ return WeaveTraceModel.model_validate(+ {+ "id": call_id,+ "op": "tool",+ "inputs": {"synthetic": True},+ "outputs": {"result": "ok"},+ "attributes": {"trace_id": call_id, "start_time": START, "end_time": START + timedelta(seconds=1)},+ **overrides,+ }+ )+++def response(url: str, body: object, *, status: int = 200) -> httpx.Response:+ return httpx.Response(status, request=httpx.Request("POST", url), json=body)+++def wire(+ monkeypatch: pytest.MonkeyPatch, failure: tuple[str, str, int | str] | None+) -> list[tuple[str, RequestOptions, dict[str, object]]]:+ sent: list[tuple[str, RequestOptions, dict[str, object]]] = []+ fault_pending = True++ def send(method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ nonlocal fault_pending+ assert method == "POST"+ assert url in {"https://trace.wandb.ai/call/start", "https://trace.wandb.ai/call/end"}+ body = json.loads(options["content"])+ sent.append((url, options, body))+ stage = "start" if "start" in body else "end"+ record = body[stage]+ if fault_pending and failure is not None and (stage, record["id"]) == failure[:2]:+ fault_pending = False+ outcome = failure[2]+ if outcome == "timeout":+ raise httpx.ReadTimeout("synthetic lost acknowledgment", request=httpx.Request("POST", url))+ if outcome == "malformed":+ return httpx.Response(200, request=httpx.Request("POST", url), content=b"invalid-json")+ assert isinstance(outcome, int)+ return response(url, {"error": "DO_NOT_EXPOSE_BODY"}, status=outcome)+ if url.endswith("/call/start"):+ req = CallStartReq.model_validate(body)+ return response(url, {"id": req.start.id, "trace_id": req.start.trace_id})+ CallEndReq.model_validate(body)+ return response(url, {})++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ return sent+++def test_constructor_and_presentation_have_no_login_network_files_or_background_state(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def forbidden(*_args: object, **_kwargs: object) -> None:+ pytest.fail("Constructor must not initialize global credentials, files, I/O or background work")++ monkeypatch.setattr(Path, "touch", forbidden)+ monkeypatch.setattr(wandb, "login", forbidden)+ monkeypatch.setattr(weave, "init", forbidden)+ monkeypatch.setattr("wandb.sdk.lib.wbauth.wbnetrc.write_netrc_auth", forbidden)+ monkeypatch.setattr("weave.trace_server_bindings.remote_http_trace_server.AsyncBatchProcessor", forbidden)+ monkeypatch.setattr("weave.trace_server_bindings.remote_http_trace_server.CallBatchProcessor", forbidden)+ monkeypatch.setattr(ssrf_proxy, "make_request", forbidden)+ monkeypatch.setattr(atexit, "register", forbidden)+ monkeypatch.setenv("WANDB_API_KEY", "unrelated-key")+ monkeypatch.setenv("WANDB_BASE_URL", "https://unrelated.test")+ monkeypatch.setenv("WEAVE_TRACE_SERVER", "https://unrelated-traces.test")+ before = dict(os.environ)+ original = config(entity=None)+ adapter = WeaveDataTrace(original)+ original.api_key = "changed-after-construction"+ assert adapter.weave_api_key == "fixture-key"+ assert adapter.get_project_url() == "https://wandb.ai"+ assert adapter.server.trace_server_url == "https://trace.wandb.ai"+ assert adapter.server._auth == ("api", "fixture-key")+ assert not adapter.server.should_batch+ assert adapter.server.call_processor is None+ assert adapter.server.feedback_processor is None+ assert dict(os.environ) == before++[email protected]("agent_runtime_app")+ "failure",+ [None]+ + [+ (stage, call_id, status)+ for stage, call_id in [("start", "legacy-run"), ("end", "legacy-run"), ("start", "node-0"), ("end", "node-0")]+ for status in [429, 503]+ ]+ + [+ ("start", "legacy-run", "timeout"),+ ("start", "legacy-run", "malformed"),+ ("end", "node-0", "malformed"),+ ("end", "legacy-run", "timeout"),+ ]+ + [("start", "legacy-run", 401), ("end", "legacy-run", 401)],+)+def test_workflow_emits_four_real_sdk_calls_with_original_ids_payload_and_status(+ sqlite_session: Session,+ wire: list[tuple[str, RequestOptions, dict[str, object]]],+ failure: tuple[str, str, int | str] | None,+) -> None:+ # The normal Flask/SQLite fixtures bind the actual service-account lookup and+ # production repository to persisted rows. Only outbound HTTP is substituted.+ tenant = make_tenant(tenant_id="tenant")+ account = make_account(account_id="owner", tenant=tenant)+ app = make_app(app_id="app", tenant_id=tenant.id, mode=AppMode.WORKFLOW, created_by=account.id)+ membership = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole.OWNER)+ sqlite_session.add_all([tenant, account, app, membership])+ nodes = [+ WorkflowNodeExecutionModel(+ id=f"node-{i}",+ tenant_id=tenant.id,+ app_id=app.id,+ workflow_id="workflow",+ workflow_run_id="legacy-run",+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ index=i,+ node_id=f"node-{i}",+ title=title,+ node_type=kind,+ status=WorkflowNodeExecutionStatus.SUCCEEDED,+ created_by=account.id,+ created_by_role=CreatorUserRole.ACCOUNT,+ inputs=json.dumps({"synthetic": True}),+ outputs=json.dumps({"answer": "ok"}),+ created_at=START + timedelta(milliseconds=i),+ finished_at=START + timedelta(milliseconds=i + 100),+ elapsed_time=0.1,+ process_data=json.dumps(+ {+ "prompts": [{"role": "user", "text": "hello"}],+ "model_mode": "chat",+ "model_provider": "test",+ "model_name": "test-model",+ }+ )+ if i == 1+ else None,+ execution_metadata=json.dumps(+ {"total_tokens": 32, "total_price": "0.0000098", "currency": "USD"} if i == 1 else {}+ ),+ )+ for i, (title, kind) in enumerate(+ [("Start", BuiltinNodeTypes.START), ("LLM", BuiltinNodeTypes.LLM), ("End", BuiltinNodeTypes.END)]+ )+ ]+ sqlite_session.add_all(nodes)+ sqlite_session.commit()+ adapter = WeaveDataTrace(config())+ info = WorkflowTraceInfo(+ query="hello",+ workflow_id="workflow",+ tenant_id="tenant",+ workflow_run_id="legacy-run",+ workflow_run_elapsed_time=1.0,+ workflow_run_status="succeeded",+ workflow_run_inputs={"query": "hello"},+ workflow_run_outputs={"answer": "ok"},+ workflow_run_version="v1",+ total_tokens=32,+ file_list=[],+ metadata={"app_id": "app", "conversation_id": None},+ trace_id="external-correlation",+ start_time=START,+ end_time=START + timedelta(seconds=1),+ )++ if failure is not None:+ stage, call_id, outcome = failure+ first_start = (stage, call_id) == ("start", "legacy-run")+ expected = (+ TraceExportRejectedError+ if outcome == 401+ else RetryableTraceDispatchError+ if first_start and outcome in {429, 503}+ else TraceExportUncertainError+ )+ with pytest.raises(expected) as error:+ adapter.trace(info)+ if isinstance(error.value, RetryableTraceDispatchError):+ assert error.value.acceptance_unknown is (expected is TraceExportUncertainError)+ assert len(wire) == (1 if first_start else 8 if call_id == "legacy-run" else 2 if stage == "start" else 3)+ assert "DO_NOT_EXPOSE" not in str(error.value)+ assert adapter.calls == {}+ assert info.workflow_run_status == "succeeded"+ assert all(node.status == WorkflowNodeExecutionStatus.SUCCEEDED for node in nodes)+ before_recovery = len(wire)+ adapter.trace(+ GenerateNameTraceInfo(+ tenant_id=tenant.id,+ inputs="new business operation",+ outputs="new title",+ metadata={},+ start_time=START,+ end_time=START + timedelta(seconds=1),+ )+ )+ assert len(wire) == before_recovery + 2+ assert adapter.calls == {}+ return++ adapter.trace(info)++ starts = [CallStartReq.model_validate(body).start for _, _, body in wire if "start" in body]+ ends = [CallEndReq.model_validate(body).end for _, _, body in wire if "end" in body]+ assert len(starts) == len(ends) == 4+ assert [s.id for s in starts] == ["legacy-run", "node-0", "node-1", "node-2"]+ assert [s.parent_id for s in starts] == [None, "legacy-run", "legacy-run", "legacy-run"]+ assert all(s.trace_id == "external-correlation" and s.project_id == "team/project" for s in starts)+ # Legacy has no native thread/turn contract: keep the existing raw metadata.+ assert all(s.thread_id is None and s.turn_id is None for s in starts)+ assert starts[0].attributes["conversation_id"] is None+ model = starts[2]+ assert model.attributes["total_tokens"] == 32+ assert model.attributes["total_price"] == "0.0000098"+ assert model.inputs["messages"][0]["content"] == "hello"+ model_end = next(e for e in ends if e.id == "node-1")+ assert isinstance(model_end.output, dict)+ assert model_end.output["usage_metadata"]["total_tokens"] == 32+ assert all(e.exception is None and e.summary["status_counts"][TraceStatus.SUCCESS] == 1 for e in ends)+ assert all("usage" not in e.summary for e in ends) # Do not invent native aggregation for the old mapper.+ assert all(options["auth"] == ("api", "fixture-key") and options["max_retries"] == 0 for _, options, _ in wire)+ assert adapter.calls == {}+++def test_interleaved_instances_resolve_only_their_own_credentials_and_targets(monkeypatch: pytest.MonkeyPatch) -> None:+ barrier = Barrier(2)+ observed: list[tuple[str, tuple[str, str], str]] = []+ monkeypatch.setenv("WANDB_API_KEY", "ambient-key")+ monkeypatch.setenv("WANDB_ENTITY", "ambient-entity")+ monkeypatch.setenv("WANDB_PROJECT", "ambient-project")+ before = dict(os.environ)++ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ label = (+ "a"+ if url+ in {+ "https://control-a.test/prefix/graphql",+ "https://trace-a.test/prefix/call/start",+ "https://trace-a.test/prefix/call/end",+ }+ else "b"+ )+ assert url in {+ f"https://control-{label}.test/prefix/graphql",+ f"https://trace-{label}.test/prefix/call/start",+ f"https://trace-{label}.test/prefix/call/end",+ }+ assert options["auth"] == ("api", f"key-{label}")+ if url.endswith("/graphql"):+ return response(url, {"data": {"viewer": {"entity": f"team-{label}"}}})+ body = json.loads(options["content"])+ record = body.get("start", body.get("end"))+ assert record["project_id"] == f"team-{label}/project-{label}"+ observed.append((url, options["auth"], record["project_id"]))+ if "start" in body:+ barrier.wait(timeout=5)+ return response(url, {"id": record["id"], "trace_id": record["trace_id"]})+ return response(url, {})++ monkeypatch.setattr(ssrf_proxy, "make_request", send)++ def run(label: str) -> None:+ adapter = WeaveDataTrace(+ config(+ api_key=f"key-{label}",+ entity=None,+ project=f"project-{label}",+ host=f"https://control-{label}.test/prefix",+ endpoint=f"https://trace-{label}.test/prefix/",+ )+ )+ data = call(f"call-{label}")+ adapter.start_call(data)+ adapter.finish_call(data)++ with ThreadPoolExecutor(max_workers=2) as pool:+ list(pool.map(run, ["a", "b"]))+ assert len(observed) == 4+ assert dict(os.environ) == before+++def test_concurrent_dispatches_on_one_cached_instance_keep_prefix_state_isolated(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ barrier = Barrier(2)++ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ body = json.loads(options["content"])+ if "start" in body:+ start = CallStartReq.model_validate(body).start+ if start.inputs["label"] == "unaccepted":+ barrier.wait(timeout=5)+ return response(url, {}, status=503)+ return response(url, {"id": start.id, "trace_id": start.trace_id})+ CallEndReq.model_validate(body)+ barrier.wait(timeout=5)+ return response(url, {}, status=503)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ adapter = WeaveDataTrace(config())++ def run(label: str) -> RetryableTraceDispatchError:+ info = GenerateNameTraceInfo(+ tenant_id="tenant",+ inputs={"label": label},+ outputs={"label": label},+ metadata={},+ start_time=START,+ end_time=START + timedelta(seconds=1),+ )+ with pytest.raises(RetryableTraceDispatchError) as error:+ adapter.trace(info)+ return error.value++ with ThreadPoolExecutor(max_workers=2) as pool:+ accepted, unaccepted = list(pool.map(run, ["accepted", "unaccepted"]))+ assert isinstance(accepted, TraceExportUncertainError)+ assert accepted.acceptance_unknown+ assert not unaccepted.acceptance_unknown+ assert adapter.calls == {}+++ ("stage", "failure"),+ [+ (stage, failure)+ for stage in ("start", "end")+ for failure in ("rejected", "retryable", "malformed", "mismatched_id")+ if not (stage == "end" and failure == "mismatched_id")+ ],+)+def test_synchronous_failures_propagate_and_next_call_recovers(+ monkeypatch: pytest.MonkeyPatch, stage: str, failure: str+) -> None:+ broken = True+ requests = []++ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ body = json.loads(options["content"])+ requests.append(url)+ if broken and url.endswith("/call/" + stage):+ if failure == "rejected":+ return response(url, {"error": "DO_NOT_EXPOSE_KEY_OR_BODY"}, status=401)+ if failure == "retryable":+ return response(url, {}, status=503)+ if failure == "malformed":+ return httpx.Response(200, request=httpx.Request("POST", url), content=b"invalid-json")+ return response(url, {"id": "wrong-id", "trace_id": "wrong-trace"})+ if "start" in body:+ return response(url, {"id": body["start"]["id"], "trace_id": body["start"]["trace_id"]})+ return response(url, {})++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ adapter = WeaveDataTrace(config())+ data = call()+ expected = {+ "rejected": TraceExportRejectedError,+ "retryable": RetryableTraceDispatchError,+ "malformed": TraceExportUncertainError,+ "mismatched_id": TraceExportUncertainError,+ }[failure]+ if stage == "end":+ adapter.start_call(data)+ with pytest.raises(expected) as error:+ adapter.start_call(data) if stage == "start" else adapter.finish_call(data)+ assert "DO_NOT_EXPOSE" not in str(error.value)+ assert len(requests) == (1 if stage == "start" else 2)+ assert adapter.calls == {}+ broken = False+ recovered = call("next-call", exception="synthetic tool error")+ adapter.start_call(recovered)+ adapter.finish_call(recovered)+ assert adapter.calls == {}+++def test_legacy_api_check_uses_read_only_project_and_stats_requests(monkeypatch: pytest.MonkeyPatch) -> None:+ sent = []++ def post(url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ assert options["auth"] == ("api", "fixture-key")+ assert options["follow_redirects"] is False+ sent.append(url)+ if url == "https://api.wandb.ai/graphql":+ return response(+ url,+ {+ "data": {+ "viewer": {"id": "viewer"},+ "project": {"id": "project-id", "name": "project", "entityName": "team", "readOnly": False},+ }+ },+ )+ assert url == "https://trace.wandb.ai/calls/query_stats"+ assert options["json"]["project_id"] == "team/project"+ return response(url, {"count": 0})++ monkeypatch.setattr(ssrf_proxy, "post", post)+ adapter = WeaveDataTrace(config())+ assert sent == []+ assert adapter.api_check()+ assert sent == ["https://api.wandb.ai/graphql", "https://trace.wandb.ai/calls/query_stats"]+ assert adapter.calls == {}+++ ("sent_id", "ack_id"),+ [+ ("1234567812345678123456781234abcd", "12345678-1234-5678-1234-56781234abcd"),+ ("12345678-1234-5678-1234-56781234abcd", "1234567812345678123456781234ABCD"),+ ("12345678-1234-5678-1234-56781234ABCD", "1234567812345678123456781234abcd"),+ ],+)+def test_uuid_equivalent_start_ack_preserves_sent_ids(+ monkeypatch: pytest.MonkeyPatch, sent_id: str, ack_id: str+) -> None:+ sent: list[dict[str, object]] = []++ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ body = json.loads(options["content"])+ sent.append(body)+ if "start" in body:+ start = CallStartReq.model_validate(body).start+ assert start.id == sent_id+ assert start.trace_id == sent_id+ return response(url, {"id": ack_id, "trace_id": ack_id})+ end = CallEndReq.model_validate(body).end+ assert end.id == sent_id+ return response(url, {})++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ adapter = WeaveDataTrace(config())+ data = call(sent_id)+ adapter.start_call(data)+ assert adapter.calls[sent_id]["trace_id"] == sent_id+ adapter.finish_call(data)+ assert len(sent) == 2+ assert adapter.calls == {}++[email protected]("field", ["id", "trace_id"])+ ("sent_id", "wrong_ack"),+ [+ ("1234567812345678123456781234abcd", "12345678-1234-5678-1234-56781234abce"),+ ("legacy-correlation", "LEGACY-CORRELATION"),+ ("1234567812345678123456781234abcd", "not-a-uuid"),+ ],+)+def test_start_ack_rejects_different_uuid_or_opaque_identifiers(+ monkeypatch: pytest.MonkeyPatch, field: str, sent_id: str, wrong_ack: str+) -> None:+ sent = []++ def send(_method: str, url: str, **options: Unpack[RequestOptions]) -> httpx.Response:+ start = CallStartReq.model_validate_json(options["content"]).start+ sent.append(start)+ ack = {"id": start.id, "trace_id": start.trace_id}+ ack[field] = wrong_ack+ return response(url, ack)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ adapter = WeaveDataTrace(config())+ with pytest.raises(TraceExportUncertainError, match="did not acknowledge the submitted identifiers"):+ adapter.start_call(call(sent_id))+ assert len(sent) == 1+ assert adapter.calls == {}diff --git a/api/tests/unit_tests/services/tracing/test_mlflow_configuration.py b/api/tests/unit_tests/services/tracing/test_mlflow_configuration.pynew file mode 100644index 00000000000..d8f1bd5e5ab--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_mlflow_configuration.py@@ -0,0 +1,317 @@+"""OSS MLflow verification reads one experiment with isolated Basic credentials."""++import base64+import os+from collections.abc import Callable, Iterator+from concurrent.futures import ThreadPoolExecutor+from threading import Barrier+from typing import Never++import httpx+import mlflow+import pytest+from dify_trace_mlflow.config import MLflowConfig+from dify_trace_mlflow.mlflow_trace import MLflowDataTrace+from dify_trace_mlflow.oss_configuration import (+ MLflowConfigurationError,+ normalize_configuration,+ project_url,+ verify_configuration,+)+from pydantic import ValidationError++from core.helper import ssrf_proxy+from tests.unit_tests.services.tracing.test_databricks_configuration import ExperimentAPI, response++[email protected](autouse=True)+def isolated_sdk_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:+ names = (+ "MLFLOW_TRACKING_URI",+ "MLFLOW_TRACKING_USERNAME",+ "MLFLOW_TRACKING_PASSWORD",+ "MLFLOW_TRACKING_TOKEN",+ "MLFLOW_EXPERIMENT_ID",+ "MLFLOW_ENABLE_ASYNC_TRACE_LOGGING",+ "DATABRICKS_HOST",+ "DATABRICKS_TOKEN",+ )+ for name in names:+ monkeypatch.setenv(name, "synthetic-ambient-value")+ forbidden_calls: list[str] = []++ def forbidden(name: str) -> Callable[..., Never]:+ def block(*_args: object, **_kwargs: object) -> Never:+ forbidden_calls.append(name)+ raise AssertionError("Configuration must not initialize an exporter or global SDK")++ return block++ monkeypatch.setattr(MLflowDataTrace, "__init__", forbidden("legacy_constructor"))+ for name in ("set_tracking_uri", "set_experiment", "search_experiments"):+ monkeypatch.setattr(mlflow, name, forbidden(name))+ monkeypatch.setattr(mlflow.config, "enable_async_logging", forbidden("async_logging"))+ yield+ assert forbidden_calls == []+ assert all(os.environ[name] == "synthetic-ambient-value" for name in names)+++def experiment_api(monkeypatch: pytest.MonkeyPatch) -> ExperimentAPI:+ api = ExperimentAPI()+ monkeypatch.setattr(ssrf_proxy, "get", api.get)++ def forbid_post(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("OSS verification must not create an experiment, run, trace, or session")++ monkeypatch.setattr(ssrf_proxy, "post", forbid_post)+ return api+++def test_exact_experiment_and_path_prefix_use_only_explicit_basic_auth(experiment_api: ExperimentAPI) -> None:+ values = {+ "tracking_uri": " HTTP://FIRST.Example:5000/team/mlflow/ ",+ "experiment_id": "00012",+ "username": "first-user",+ "password": "first-password",+ }+ config = normalize_configuration(values)+ target = verify_configuration(config)+ assert target.tracking_uri == "http://first.example:5000/team/mlflow"+ assert target.experiment_id == "12"+ assert project_url(config) == "http://first.example:5000/team/mlflow/#/experiments/12/traces"+ method, url, options = experiment_api.calls[0]+ assert method == "GET"+ assert url == "http://first.example:5000/team/mlflow/api/2.0/mlflow/experiments/get"+ assert options["params"] == {"experiment_id": "12"}+ assert options["headers"] == {+ "Accept": "application/json",+ "Accept-Encoding": "identity",+ "Cookie": "",+ "Authorization": "Basic " + base64.b64encode(b"first-user:first-password").decode(),+ }+ assert options["follow_redirects"] is False+ assert options["stream_response"] is True+ assert options["timeout"] == 30.0+ assert options["max_retries"] == 0+ assert values["tracking_uri"].startswith(" HTTP")+++def test_anonymous_request_does_not_inherit_previous_or_environment_credentials(experiment_api: ExperimentAPI) -> None:+ verify_configuration(+ MLflowConfig(tracking_uri="https://same.test", experiment_id="12", username="user", password="password")+ )+ verify_configuration(MLflowConfig(tracking_uri="https://same.test", experiment_id="34"))+ assert len(experiment_api.calls) == 2+ assert experiment_api.calls[1][2]["headers"] == {+ "Accept": "application/json",+ "Accept-Encoding": "identity",+ "Cookie": "",+ }+++def test_concurrent_destinations_keep_experiment_and_auth_together(experiment_api: ExperimentAPI) -> None:+ barrier = Barrier(2)+ expected_destinations = {+ "https://first.test/prefix/api/2.0/mlflow/experiments/get": ("first", "12"),+ "https://second.test/prefix/api/2.0/mlflow/experiments/get": ("second", "34"),+ }++ def read(_method: str, url: str, options: dict[str, object]) -> httpx.Response:+ params = options["params"]+ headers = options["headers"]+ assert isinstance(params, dict)+ assert isinstance(headers, dict)+ label, experiment_id = expected_destinations[url]+ assert params["experiment_id"] == experiment_id+ assert headers["Authorization"] == "Basic " + base64.b64encode(f"{label}:{label}-secret".encode()).decode()+ barrier.wait(timeout=5)+ return response(url, {"experiment": {"experiment_id": params["experiment_id"], "lifecycle_stage": "active"}})++ experiment_api.override = read+ with ThreadPoolExecutor(max_workers=2) as pool:+ pending = [+ pool.submit(+ verify_configuration,+ MLflowConfig(+ tracking_uri=f"https://{label}.test/prefix",+ experiment_id=experiment,+ username=label,+ password=f"{label}-secret",+ ),+ )+ for label, experiment in (("first", "12"), ("second", "34"))+ ]+ assert [item.result().experiment_id for item in pending] == ["12", "34"]+ assert sorted(url for _method, url, _options in experiment_api.calls) == sorted(expected_destinations)+++ "uri",+ [+ "",+ "ftp://server.test",+ "https:///missing",+ "https://user:[email protected]",+ "https://@server.test",+ "https://server.test?",+ "https://server.test#",+ "https://server.test?key=secret",+ "https://server.test/#/experiments/1",+ "https://server.test:bad",+ "https://server.test:0",+ "https://server.test:",+ "https://[invalid",+ "https://server.test\\@other.test",+ "\nhttps://server.test",+ "https://ser\nver.test",+ "https://server.test/./prefix",+ "https://server.test/prefix/../other",+ "https://server.test/%2e%2e/other",+ "https://server.test/prefix%2fother",+ "https://server.test/%252fother",+ "https://server.test/%00",+ "https://server.test/%zz",+ "https://server.test//prefix",+ ],+)+def test_ambiguous_destinations_fail_before_request(experiment_api: ExperimentAPI, uri: str) -> None:+ with pytest.raises((MLflowConfigurationError, ValidationError)):+ verify_configuration(normalize_configuration({"tracking_uri": uri, "experiment_id": "12"}))+ assert experiment_api.calls == []++[email protected]("experiment", ["", "-1", "1.5", "12", "abc"])+def test_experiment_ids_are_ascii_integer_identifiers(experiment_api: ExperimentAPI, experiment: str) -> None:+ with pytest.raises((MLflowConfigurationError, ValidationError)):+ normalize_configuration({"tracking_uri": "https://server.test", "experiment_id": experiment})+ assert experiment_api.calls == []+++ ("username", "password"),+ [("user", None), (None, "password"), ("user", ""), (" ", "password"), ("user", " "), ("user:name", "password")],+)+def test_partial_or_ambiguous_basic_auth_never_sends_credentials(+ experiment_api: ExperimentAPI, username: str | None, password: str | None+) -> None:+ with pytest.raises(MLflowConfigurationError) as caught:+ verify_configuration(MLflowConfig(tracking_uri="https://server.test", username=username, password=password))+ assert caught.value.code == "incomplete_basic_auth"+ assert experiment_api.calls == []+++ ("status", "code"),+ [+ (401, "authentication_failed"),+ (403, "permission_denied"),+ (404, "experiment_not_found_or_inaccessible"),+ (302, "endpoint_unavailable"),+ (307, "endpoint_unavailable"),+ (500, "endpoint_unavailable"),+ ],+)+def test_http_failures_do_not_follow_redirect_or_disclose_body(+ experiment_api: ExperimentAPI, status: int, code: str+) -> None:+ result = response("https://server.test", {"message": "private-vendor-body"}, status)+ result.headers["Location"] = "https://other.test/secret"+ experiment_api.override = lambda _method, _url, _options: result+ with pytest.raises(MLflowConfigurationError) as caught:+ verify_configuration(+ MLflowConfig(tracking_uri="https://server.test", username="user", password="private-password")+ )+ assert caught.value.code == code+ assert "private" not in str(caught.value)+ assert result.is_closed+ assert len(experiment_api.calls) == 1+++ ("body", "code"),+ [+ ({}, "malformed_response"),+ ([], "malformed_response"),+ ({"experiment": "invalid"}, "malformed_response"),+ ({"experiment": {"experiment_id": "34", "lifecycle_stage": "active"}}, "experiment_id_mismatch"),+ ({"experiment": {"experiment_id": "12", "lifecycle_stage": "deleted"}}, "experiment_not_active"),+ ({"experiment": {"experiment_id": "12"}}, "experiment_not_active"),+ ],+)+def test_response_must_match_exact_active_experiment(experiment_api: ExperimentAPI, body: object, code: str) -> None:+ experiment_api.override = lambda _method, url, _options: response(url, body)+ with pytest.raises(MLflowConfigurationError) as caught:+ verify_configuration(MLflowConfig(tracking_uri="https://server.test", experiment_id="12"))+ assert caught.value.code == code+++def test_transport_failure_and_oversized_response_are_bounded_and_safe(experiment_api: ExperimentAPI) -> None:+ def fail(_method: str, _url: str, _options: dict[str, object]) -> httpx.Response:+ raise httpx.ConnectError("private-url-and-password")++ experiment_api.override = fail+ with pytest.raises(MLflowConfigurationError, match="MLflow verification request failed"):+ verify_configuration(MLflowConfig(tracking_uri="https://server.test"))+ oversized = response("https://server.test", {"message": "x" * (1024 * 1024)})+ experiment_api.override = lambda _method, _url, _options: oversized+ with pytest.raises(MLflowConfigurationError) as caught:+ verify_configuration(MLflowConfig(tracking_uri="https://server.test"))+ assert caught.value.code == "malformed_response"+ assert oversized.is_closed+++def test_project_link_requires_no_auth_or_io_and_preserves_valid_prefix(experiment_api: ExperimentAPI) -> None:+ assert project_url(MLflowConfig(tracking_uri="https://Server.test/team/mlflow/", experiment_id="000")) == (+ "https://server.test/team/mlflow/#/experiments/0/traces"+ )+ assert experiment_api.calls == []+++def test_real_http_transport_overrides_ambient_auth_and_cookies(monkeypatch: pytest.MonkeyPatch) -> None:+ requests: list[httpx.Request] = []++ def handle(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ assert request.method == "GET"+ assert request.url.path == "/prefix/api/2.0/mlflow/experiments/get"+ assert request.headers.get("cookie") == ""+ return httpx.Response(+ 200,+ headers={"Set-Cookie": "session=synthetic-vendor-session; Path=/"},+ json={"experiment": {"experiment_id": request.url.params["experiment_id"], "lifecycle_stage": "active"}},+ )++ with httpx.Client(+ transport=httpx.MockTransport(handle),+ auth=httpx.BasicAuth("ambient-user", "ambient-password"),+ cookies={"session": "synthetic-ambient-session"},+ trust_env=False,+ ) as client:++ def get_client(_ssl_verify_enabled: bool) -> httpx.Client:+ return client++ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", get_client)+ verify_configuration(+ MLflowConfig(+ tracking_uri="https://server.test/prefix", experiment_id="12", username="owner", password="secret"+ )+ )+ verify_configuration(MLflowConfig(tracking_uri="https://server.test/prefix", experiment_id="34"))+ assert len(requests) == 2+ assert requests[0].headers["authorization"] == "Basic " + base64.b64encode(b"owner:secret").decode()+ assert "authorization" not in requests[1].headers++[email protected]("compressed", [False, True])+def test_invalid_encoding_or_json_response_is_closed(experiment_api: ExperimentAPI, compressed: bool) -> None:+ result = httpx.Response(200, content=b"private-vendor-text", request=httpx.Request("GET", "https://server.test"))+ if compressed:+ result.headers["Content-Encoding"] = "gzip"+ experiment_api.override = lambda _method, _url, _options: result+ with pytest.raises(MLflowConfigurationError) as caught:+ verify_configuration(MLflowConfig(tracking_uri="https://server.test"))+ assert caught.value.code == "malformed_response"+ assert "private-vendor-text" not in str(caught.value)+ assert result.is_closeddiff --git a/api/tests/unit_tests/services/tracing/test_mlflow_legacy_destination.py b/api/tests/unit_tests/services/tracing/test_mlflow_legacy_destination.pynew file mode 100644index 00000000000..9a2ab3b6bea--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_mlflow_legacy_destination.py@@ -0,0 +1,637 @@+"""Real MLflow SDK export must not select the application's ambient OTLP destination."""++import json+import os+from concurrent.futures import ThreadPoolExecutor+from datetime import UTC, datetime, timedelta+from threading import Barrier+from typing import Never++import mlflow+import pytest+import requests+from dify_trace_mlflow.config import DatabricksConfig, MLflowConfig+from dify_trace_mlflow.mlflow_trace import MLflowDataTrace+from google.protobuf.json_format import Parse+from mlflow.entities import SpanType, Trace+from mlflow.entities.trace_location import MlflowExperimentLocation+from mlflow.protos.service_pb2 import StartTraceV3+from mlflow.tracing import provider as sdk_provider+from mlflow.tracing.constant import SpanAttributeKey, TraceMetadataKey, TraceTagKey+from mlflow.tracing.trace_manager import InMemoryTraceManager+from mlflow.tracing.utils.otlp import _decode_otel_proto_anyvalue+from opentelemetry import trace+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest+from opentelemetry.proto.trace.v1.trace_pb2 import Span as ProtoSpan+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.trace import INVALID_SPAN_CONTEXT, NonRecordingSpan++from core.ops.entities.trace_entity import (+ DatasetRetrievalTraceInfo,+ GenerateNameTraceInfo,+ MessageTraceInfo,+ ModerationTraceInfo,+ SuggestedQuestionTraceInfo,+ ToolTraceInfo,+ WorkflowTraceInfo,+)+from graphon.enums import BuiltinNodeTypes+from models.model import Message+from models.workflow import WorkflowNodeExecutionModel+++def transport(monkeypatch: pytest.MonkeyPatch) -> list[requests.PreparedRequest]:+ calls: list[requests.PreparedRequest] = []++ def send(_session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:+ calls.append(request)+ response = requests.Response()+ response.status_code = 200+ response.request = request+ response.url = request.url or ""+ if response.url.endswith("/version"):+ response._content = b"3.17.1"+ elif response.url.endswith("/v1/traces"):+ response._content = b"{}"+ elif response.url.endswith("/api/3.0/mlflow/traces"):+ assert isinstance(request.body, (str, bytes))+ body = json.loads(request.body)+ info = body["trace"]["trace_info"]+ info.setdefault("tags", {})[TraceTagKey.SPANS_LOCATION] = "TRACKING_STORE"+ response._content = json.dumps(body).encode()+ else:+ raise AssertionError(f"Unexpected SDK endpoint: {request.method} {request.path_url}")+ return response++ monkeypatch.setattr(requests.Session, "send", send)+ monkeypatch.setenv("MLFLOW_DISABLE_TELEMETRY", "true")+ monkeypatch.delenv("_MLFLOW_TELEMETRY_SESSION_ID", raising=False)+ monkeypatch.setenv("MLFLOW_ENABLE_ASYNC_TRACE_LOGGING", "false")+ monkeypatch.setenv("MLFLOW_ENABLE_OTLP_EXPORTER", "true")+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://dify-internal.example:4318")+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http")+ monkeypatch.setenv("OTEL_SERVICE_NAME", "dify-api")+ monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=unit-test")+ return calls+++def tool_info() -> ToolTraceInfo:+ started = datetime(2026, 1, 1, tzinfo=UTC)+ return ToolTraceInfo(+ tool_name="code",+ tool_inputs={"a": 17},+ tool_outputs="323",+ tool_config={},+ tool_parameters={},+ time_cost=1,+ metadata={},+ inputs={"a": 17},+ outputs="323",+ start_time=started,+ end_time=started + timedelta(seconds=1),+ )++[email protected]("default_provider", ["true", "false"])+def test_real_sdk_uses_private_mlflow_exporter_without_touching_shared_state(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+ default_provider: str,+) -> None:+ monkeypatch.setenv("MLFLOW_USE_DEFAULT_TRACER_PROVIDER", default_provider)+ main_provider = TracerProvider()+ monkeypatch.setattr(trace, "_TRACER_PROVIDER", main_provider)+ sdk_before = sdk_provider.provider.get()+ destination_before = sdk_provider._MLFLOW_TRACE_USER_DESTINATION.get()+ unrelated = MlflowExperimentLocation("999")+ monkeypatch.setattr(sdk_provider._MLFLOW_TRACE_USER_DESTINATION, "_global_value", unrelated)+ destination_before = sdk_provider._MLFLOW_TRACE_USER_DESTINATION.get()+ environment_before = dict(os.environ)++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("Legacy exporter must not initialize or reset the process SDK")++ for name in ("set_tracking_uri", "set_experiment"):+ monkeypatch.setattr(mlflow, name, forbidden)+ monkeypatch.setattr(sdk_provider, "set_destination", forbidden)+ monkeypatch.setattr(sdk_provider, "reset", forbidden)+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://legacy.example", experiment_id="12"))+ assert transport == []+ instance.trace(tool_info())+ assert sdk_provider.provider.get() is sdk_before+ assert sdk_provider._MLFLOW_TRACE_USER_DESTINATION.get() is destination_before+ assert trace.get_tracer_provider() is main_provider+ assert dict(os.environ) == environment_before+ otlp_requests = [request for request in transport if (request.url or "").endswith("/v1/traces")]+ assert len(otlp_requests) == 1+ assert otlp_requests[0].url == "https://legacy.example/v1/traces"+ assert otlp_requests[0].headers["x-mlflow-experiment-id"] == "12"+ body = otlp_requests[0].body+ assert isinstance(body, bytes)+ spans = ExportTraceServiceRequest.FromString(body).resource_spans[0].scope_spans[0].spans+ assert len(spans) == 1+ assert spans[0].name == "code"+ assert len(spans[0].trace_id) == 16+ assert any(spans[0].trace_id)+ assert any((request.url or "").endswith("/api/3.0/mlflow/traces") for request in transport)+ main_provider.shutdown()+++def exported_spans(calls: list[requests.PreparedRequest]) -> list[ProtoSpan]:+ spans: list[ProtoSpan] = []+ for request in calls:+ if (request.url or "").endswith("/v1/traces"):+ assert isinstance(request.body, bytes)+ spans.extend(ExportTraceServiceRequest.FromString(request.body).resource_spans[0].scope_spans[0].spans)+ return spans+++def exported_traces(calls: list[requests.PreparedRequest]) -> list[Trace]:+ results: list[Trace] = []+ for request in calls:+ if (request.url or "").endswith("/api/3.0/mlflow/traces"):+ assert isinstance(request.body, (bytes, str))+ results.append(Trace.from_proto(Parse(request.body, StartTraceV3()).trace))+ return results+++def test_workflow_parent_children_usage_metadata_and_explicit_experiment(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+) -> None:+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://workflow.example", experiment_id="42"))+ started = datetime(2026, 1, 1, tzinfo=UTC)+ nodes = [+ WorkflowNodeExecutionModel(+ id=f"node-{index}",+ tenant_id="tenant",+ app_id="app",+ title=name,+ node_type=node_type,+ status="succeeded",+ inputs='{"question":"synthetic"}',+ outputs='{"text":"answer"}',+ created_at=started + timedelta(milliseconds=index),+ elapsed_time=0.1,+ process_data=json.dumps(+ {+ "prompts": [{"role": "user", "text": "synthetic"}],+ "model_name": "test-model",+ "model_provider": "test",+ "usage": {"prompt_tokens": 25, "completion_tokens": 7, "total_tokens": 32},+ }+ )+ if node_type == BuiltinNodeTypes.LLM+ else None,+ )+ for index, (name, node_type) in enumerate(+ [+ ("Start", BuiltinNodeTypes.START),+ ("LLM", BuiltinNodeTypes.LLM),+ ("End", BuiltinNodeTypes.END),+ ]+ )+ ]+ monkeypatch.setattr(instance, "_get_workflow_nodes", lambda _run_id: nodes)+ info = WorkflowTraceInfo(+ workflow_id="workflow",+ tenant_id="tenant",+ workflow_run_id="run",+ workflow_run_elapsed_time=1,+ workflow_run_status="succeeded",+ workflow_run_inputs={"q": "synthetic", "sys.hidden": "drop"},+ workflow_run_outputs={"answer": "answer"},+ workflow_run_version="1",+ total_tokens=32,+ file_list=[],+ query="",+ conversation_id="session",+ metadata={"user_id": "user"},+ start_time=started,+ end_time=started + timedelta(seconds=1),+ )+ ambient_provider = TracerProvider()+ with ambient_provider.get_tracer("ambient").start_as_current_span("unrelated"):+ instance.trace(info)+ spans = exported_spans(transport)+ assert len(spans) == 4+ root = next(span for span in spans if span.name == "workflow")+ assert not root.parent_span_id+ assert all(span.trace_id == root.trace_id for span in spans)+ assert all(span.parent_span_id == root.span_id for span in spans if span is not root)+ assert all(+ request.headers["x-mlflow-experiment-id"] == "42"+ for request in transport+ if (request.url or "").endswith("/v1/traces")+ )+ native = exported_traces(transport)+ assert len(native) == 1+ location = native[0].info.trace_location.mlflow_experiment+ assert location is not None+ assert location.experiment_id == "42"+ assert native[0].info.trace_metadata[TraceMetadataKey.TRACE_SESSION] == "session"+ assert native[0].info.trace_metadata[TraceMetadataKey.TRACE_USER] == "user"+ assert json.loads(native[0].info.trace_metadata[TraceMetadataKey.TOKEN_USAGE]) == {+ "input_tokens": 25,+ "output_tokens": 7,+ "total_tokens": 32,+ }+ attributes = {+ attribute.key: _decode_otel_proto_anyvalue(attribute.value)+ for attribute in root.attributes+ if attribute.key in (SpanAttributeKey.INPUTS, SpanAttributeKey.OUTPUTS)+ }+ assert attributes[SpanAttributeKey.INPUTS] == {"q": "synthetic"}+ assert attributes[SpanAttributeKey.OUTPUTS] == {"answer": "answer"}+ ambient_provider.shutdown()+++def test_interleaved_instances_keep_children_on_their_own_uri_and_experiment(+ transport: list[requests.PreparedRequest],+) -> None:+ barrier = Barrier(2)+ instances = [+ MLflowDataTrace(MLflowConfig(tracking_uri=f"https://{name}.example", experiment_id=experiment))+ for name, experiment in [("first", "12"), ("second", "34")]+ ]++ def emit(instance: MLflowDataTrace) -> tuple[str, str]:+ root = instance._start_span_no_context(name="workflow", span_type=SpanType.CHAIN)+ barrier.wait(timeout=5)+ child = instance._start_span_no_context(name="node", span_type=SpanType.TOOL, parent_span=root)+ child.end(outputs="ok")+ barrier.wait(timeout=5)+ root.end(outputs="ok")+ return root.trace_id, root.span_id++ with ThreadPoolExecutor(max_workers=2) as pool:+ ids = list(pool.map(emit, instances))+ assert ids[0][0] != ids[1][0]+ for host, experiment in [("first", "12"), ("second", "34")]:+ requests_for_host = [item for item in transport if (item.url or "").startswith(f"https://{host}.example/")]+ spans = exported_spans(requests_for_host)+ assert len(spans) == 2+ assert spans[0].trace_id == spans[1].trace_id+ root = next(span for span in spans if not span.parent_span_id)+ child = next(span for span in spans if span.parent_span_id)+ assert child.parent_span_id == root.span_id+ assert all(+ item.headers["x-mlflow-experiment-id"] == experiment+ for item in requests_for_host+ if (item.url or "").endswith("/v1/traces")+ )+ location = exported_traces(requests_for_host)[0].info.trace_location.mlflow_experiment+ assert location is not None+ assert location.experiment_id == experiment+++ "method", ["message", "moderation", "dataset_retrieval", "suggested_question", "generate_name"]+)+def test_other_legacy_entrypoints_use_the_same_private_sdk(+ method: str,+ transport: list[requests.PreparedRequest],+) -> None:+ started = datetime(2026, 1, 1, tzinfo=UTC)+ message = Message(+ model_provider="test",+ model_id="model",+ answer="answer",+ total_price=0,+ created_at=started,+ updated_at=started + timedelta(seconds=1),+ )+ common = {+ "metadata": {},+ "message_data": message,+ "inputs": "synthetic",+ "outputs": "answer",+ "start_time": started,+ "end_time": started + timedelta(seconds=1),+ }+ match method:+ case "message":+ info = MessageTraceInfo(+ **common,+ conversation_model="chat",+ conversation_mode="chat",+ message_tokens=3,+ answer_tokens=2,+ total_tokens=5,+ )+ case "moderation":+ info = ModerationTraceInfo(**common, flagged=False, action="allow", preset_response="", query="synthetic")+ case "dataset_retrieval":+ info = DatasetRetrievalTraceInfo(**common, documents=[{"content": "synthetic"}])+ case "suggested_question":+ info = SuggestedQuestionTraceInfo(**common, total_tokens=5, suggested_question=["synthetic"], level="info")+ case _:+ info = GenerateNameTraceInfo(**common, tenant_id="tenant")+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://other.example", experiment_id="16"))+ instance.trace(info)+ assert len(exported_spans(transport)) == 1+ location = exported_traces(transport)[0].info.trace_location.mlflow_experiment+ assert location is not None+ assert location.experiment_id == "16"++[email protected]("failure", ["exception", "noop", "unregistered"])+def test_span_creation_failure_is_explicit_and_does_not_export_success(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+ failure: str,+) -> None:+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://failure.example", experiment_id="10"))+ if failure == "exception":++ def fail(*_args: object, **_kwargs: object) -> Never:+ raise ValueError("must-not-leak-vendor-details")++ monkeypatch.setattr(instance._tracer, "start_span", fail)+ expected = "mlflow_legacy_span_start_failed"+ elif failure == "noop":++ def noop(*_args: object, **_kwargs: object) -> NonRecordingSpan:+ return NonRecordingSpan(INVALID_SPAN_CONTEXT)++ monkeypatch.setattr(instance._tracer, "start_span", noop)+ expected = "mlflow_legacy_span_not_recording"+ else:+ manager = InMemoryTraceManager.get_instance()+ monkeypatch.setattr(manager, "get_span_from_id", lambda *_args: None)+ expected = "mlflow_legacy_span_not_registered"+ with pytest.raises(RuntimeError, match=f"^{expected}$"):+ instance.trace(tool_info())+ assert not exported_spans(transport)+ assert not exported_traces(transport)+++def test_legacy_basic_credentials_remain_compatible_at_transport_boundary(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+) -> None:+ # This preserves legacy process-level credentials; it is not an isolation claim.+ for key in ("MLFLOW_TRACKING_USERNAME", "MLFLOW_TRACKING_PASSWORD"):+ monkeypatch.delenv(key, raising=False)+ instance = MLflowDataTrace(+ MLflowConfig(+ tracking_uri="https://basic.example",+ experiment_id="12",+ username="synthetic-user",+ password="synthetic-secret",+ )+ )+ instance.trace(tool_info())+ import base64++ expected = "Basic " + base64.b64encode(b"synthetic-user:synthetic-secret").decode()+ assert all(request.headers["Authorization"] == expected for request in transport)++[email protected]("noop", [False, True])+def test_databricks_preserves_legacy_sdk_setup_and_span_selection(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+ noop: bool,+) -> None:+ import dify_trace_mlflow.mlflow_trace as legacy+ from mlflow.entities import LiveSpan+ from mlflow.entities.span import NoOpSpan++ setup: list[tuple[str, str]] = []+ provider = TracerProvider()+ raw = provider.get_tracer("databricks-legacy").start_span("code")+ span = LiveSpan(raw, "tr-synthetic-databricks")+ options: list[dict[str, object]] = []++ def start(**kwargs: object) -> LiveSpan | NoOpSpan:+ options.append(kwargs)+ return NoOpSpan() if noop else span++ monkeypatch.setattr(mlflow, "set_tracking_uri", lambda uri: setup.append(("uri", uri)))+ monkeypatch.setattr(mlflow, "set_experiment", lambda *, experiment_id: setup.append(("experiment", experiment_id)))+ monkeypatch.setattr(legacy, "start_span_no_context", start)+ for name in ("DATABRICKS_HOST", "DATABRICKS_TOKEN", "MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"):+ monkeypatch.delenv(name, raising=False)+ instance = MLflowDataTrace(+ DatabricksConfig(host="https://workspace.example", experiment_id="123", personal_access_token="synthetic-pat")+ )+ assert setup == [("uri", "databricks"), ("experiment", "123")]+ assert instance._tracer is None+ assert instance._exporter is None+ if noop:+ with pytest.raises(RuntimeError, match="^mlflow_legacy_span_not_recording$"):+ instance.trace(tool_info())+ raw.end()+ else:+ instance.trace(tool_info())+ assert span.outputs == "323"+ assert options[0]["name"] == "code"+ assert options[0]["inputs"] == {"a": 17}+ assert transport == []+ provider.shutdown()+++def test_private_exporter_is_synchronous_and_does_not_retain_evicted_adapters(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+) -> None:+ import atexit+ import gc+ import weakref++ from mlflow.tracing.export import mlflow_v3++ monkeypatch.setenv("MLFLOW_ENABLE_ASYNC_TRACE_LOGGING", "true")++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ raise AssertionError("A private legacy exporter must not retain an SDK queue or exit callback")++ monkeypatch.setattr(mlflow_v3, "AsyncTraceExportQueue", forbidden)+ monkeypatch.setattr(atexit, "register", forbidden)+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://synchronous.example", experiment_id="77"))+ reference = weakref.ref(instance)+ provider_reference = weakref.ref(instance._provider)+ instance.trace(tool_info())+ instance.trace(tool_info())+ assert len(exported_traces(transport)) == 2 # Complete before return, no global flush call.+ assert os.environ["MLFLOW_ENABLE_ASYNC_TRACE_LOGGING"] == "true"+ del instance+ gc.collect()+ assert reference() is None+ assert provider_reference() is None++[email protected]("endpoint", ["/v1/traces", "/api/3.0/mlflow/traces"])+def test_synchronous_http_failure_reaches_task_boundary_and_does_not_poison_next_trace(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+ endpoint: str,+) -> None:+ original_send = requests.Session.send+ fail = True++ def send(session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:+ if fail and (request.url or "").endswith(endpoint):+ response = requests.Response()+ response.status_code = 403+ response._content = b'{"error_code":"PERMISSION_DENIED","message":"synthetic denial"}'+ response.request = request+ response.url = request.url or ""+ return response+ return original_send(session, request)++ monkeypatch.setattr(requests.Session, "send", send)+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://failure-transport.example", experiment_id="70"))+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):+ instance.trace(tool_info())+ assert instance._exporter is not None+ assert instance._exporter.failed.get() is False+ fail = False+ transport.clear()+ instance.trace(tool_info())+ assert len(exported_spans(transport)) == len(exported_traces(transport)) == 1++[email protected]("artifact_fails", [False, True])+def test_expected_501_falls_back_to_sdk_artifact_and_only_final_failure_is_reported(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+ artifact_fails: bool,+) -> None:+ from mlflow.entities import TraceData, TraceInfo+ from mlflow.tracing.client import TracingClient++ assert transport == []+ original_send = requests.Session.send+ uploaded: list[TraceData] = []++ def send(session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:+ if (request.url or "").endswith("/v1/traces"):+ response = requests.Response()+ response.status_code = 501+ response._content = b'{"error_code":"NOT_IMPLEMENTED","message":"REST OTLP span logging is not supported"}'+ response.request = request+ response.url = request.url or ""+ return response+ response = original_send(session, request)+ if (request.url or "").endswith("/api/3.0/mlflow/traces"):+ body = response.json()+ body["trace"]["trace_info"]["tags"].pop(TraceTagKey.SPANS_LOCATION)+ response._content = json.dumps(body).encode()+ return response++ def upload(_client: TracingClient, _info: TraceInfo, data: TraceData) -> None:+ if artifact_fails:+ raise requests.HTTPError("synthetic artifact failure")+ uploaded.append(data)++ monkeypatch.setattr(requests.Session, "send", send)+ monkeypatch.setattr(TracingClient, "_upload_trace_data", upload)+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://file-store.example", experiment_id="80"))+ if artifact_fails:+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):+ instance.trace(tool_info())+ else:+ instance.trace(tool_info())+ assert len(uploaded) == 1+ assert len(uploaded[0].spans) == 1+ assert uploaded[0].spans[0].outputs == "323"++[email protected]("fallback_fails", [False, True])+def test_v3_endpoint_not_found_keeps_real_sdk_v2_fallback(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+ fallback_fails: bool,+) -> None:+ from mlflow.entities.trace_info_v2 import TraceInfoV2+ from mlflow.exceptions import MlflowException+ from mlflow.protos.databricks_pb2 import ENDPOINT_NOT_FOUND+ from mlflow.store.tracking.rest_store import RestStore++ assert transport == []+ original_endpoint = RestStore._call_endpoint+ actions: list[str] = []++ def endpoint(store: RestStore, api: object, json_body: str, **kwargs: object) -> object:+ if api is StartTraceV3:+ actions.append("v3-not-found")+ raise MlflowException("old server", error_code=ENDPOINT_NOT_FOUND)+ return original_endpoint(store, api, json_body, **kwargs)++ def start_v2(_store: RestStore, **kwargs: object) -> TraceInfoV2:+ actions.append("v2-start")+ if fallback_fails:+ raise requests.HTTPError("synthetic v2 failure")+ return TraceInfoV2.from_dict(+ {+ "request_id": "tr-v2",+ "experiment_id": str(kwargs["experiment_id"]),+ "timestamp_ms": 1,+ "execution_time_ms": 1,+ "status": "OK",+ "request_metadata": {},+ "tags": {TraceTagKey.SPANS_LOCATION: "TRACKING_STORE"},+ }+ )++ def end_v2(_store: RestStore, **_kwargs: object) -> None:+ actions.append("v2-end")++ monkeypatch.setattr(RestStore, "_call_endpoint", endpoint)+ monkeypatch.setattr(RestStore, "deprecated_start_trace_v2", start_v2)+ monkeypatch.setattr(RestStore, "deprecated_end_trace_v2", end_v2)+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://old-v2.example", experiment_id="90"))+ if fallback_fails:+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):+ instance.trace(tool_info())+ assert actions == ["v3-not-found", "v2-start"]+ else:+ instance.trace(tool_info())+ assert actions == ["v3-not-found", "v2-start", "v2-end"]+++def test_concurrent_dispatches_on_cached_adapter_do_not_share_failure_state(+ monkeypatch: pytest.MonkeyPatch,+ transport: list[requests.PreparedRequest],+) -> None:+ from contextvars import ContextVar++ original_send = requests.Session.send+ fail_this_dispatch: ContextVar[bool] = ContextVar("fail_this_dispatch", default=False)+ barrier = Barrier(2)++ def send(session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:+ if (request.url or "").endswith("/v1/traces"):+ barrier.wait(timeout=5)+ if fail_this_dispatch.get():+ response = requests.Response()+ response.status_code = 403+ response._content = b'{"error_code":"PERMISSION_DENIED","message":"synthetic denial"}'+ response.request = request+ response.url = request.url or ""+ return response+ return original_send(session, request)++ monkeypatch.setattr(requests.Session, "send", send)+ instance = MLflowDataTrace(MLflowConfig(tracking_uri="https://concurrent.example", experiment_id="71"))++ def emit(fail: bool) -> bool:+ token = fail_this_dispatch.set(fail)+ try:+ if fail:+ with pytest.raises(RuntimeError, match="^mlflow_legacy_export_failed$"):+ instance.trace(tool_info())+ return False+ instance.trace(tool_info())+ return True+ finally:+ fail_this_dispatch.reset(token)+ assert instance._exporter is not None+ assert not instance._exporter.failed.get()++ with ThreadPoolExecutor(max_workers=2) as pool:+ assert list(pool.map(emit, [True, False])) == [False, True]+ assert len(exported_spans(transport)) == 1diff --git a/api/tests/unit_tests/services/tracing/test_native_weave_mlflow.py b/api/tests/unit_tests/services/tracing/test_native_weave_mlflow.pynew file mode 100644index 00000000000..14a7b9964ad--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_native_weave_mlflow.py@@ -0,0 +1,516 @@+"""Real SDK schema/serialization boundaries with HTTP transport substituted.++These tests do not claim acceptance by a live third-party account.+"""++import base64+import json+import os+from collections.abc import Mapping+from copy import deepcopy+from datetime import UTC, datetime, timedelta+from typing import TypedDict, Unpack+from urllib.parse import parse_qs, urlsplit++import httpx+import pytest+from mlflow.entities import Span, TraceData+from mlflow.tracing.constant import SpanAttributeKey+from mlflow.tracing.otel.translation import translate_span_when_storing+from mlflow.types.chat import ChatTool+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (+ ExportTraceServiceRequest,+ ExportTraceServiceResponse,+)+from weave.trace_server.trace_server_interface import StartedCallSchemaForInsert++from core.helper import ssrf_proxy+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.normalization import remap_outer_trace+from services.tracing.provider_dispatch import TraceExportUncertainError, export_trace+++class TransportOptions(TypedDict, total=False):+ content: bytes | str+ headers: Mapping[str, str]+ auth: tuple[str, str]+ max_retries: int+++def trace() -> CanonicalTrace:+ start = datetime(2026, 1, 1, tzinfo=UTC)+ root = CanonicalSpan(+ id="0000000000000001",+ parent_id=None,+ name="Agent",+ kind=CanonicalSpanKind.AGENT,+ start_time=start,+ end_time=start + timedelta(seconds=2),+ inputs={"query": "hello"},+ outputs="answer",+ status=CanonicalSpanStatus.OK,+ metadata={"total_price": 999, "total_tokens": 999},+ )+ child = CanonicalSpan(+ id="0000000000000002",+ parent_id=root.id,+ name="Model",+ kind=CanonicalSpanKind.LLM,+ start_time=start,+ end_time=start + timedelta(seconds=1),+ inputs={"prompt": "hello"},+ outputs="answer",+ status=CanonicalSpanStatus.OK,+ metadata={+ "model_name": "test-model",+ "prompt_tokens": 2,+ "completion_tokens": 3,+ "total_tokens": 5,+ "total_price": "0.01",+ "currency": "USD",+ },+ )+ return CanonicalTrace(+ trace_id="1234567890abcdef1234567890abcdef", session_id="session-1", root_span_id=root.id, spans=(root, child)+ )+++def _response(url: str, *, body: object = None, content: bytes | None = None, status: int = 200) -> httpx.Response:+ request = httpx.Request("POST", url)+ if content is not None:+ return httpx.Response(status, request=request, content=content)+ return httpx.Response(status, request=request, json=body or {})+++def weave_started_calls(monkeypatch: pytest.MonkeyPatch) -> list[StartedCallSchemaForInsert]:+ starts: list[StartedCallSchemaForInsert] = []++ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ body = json.loads(kwargs["content"])+ if start := body.get("start"):+ started = StartedCallSchemaForInsert.model_validate(start)+ starts.append(started)+ return _response(url, body={"id": started.id, "trace_id": started.trace_id})+ return _response(url)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ return starts++[email protected]("host", ["https://api.wandb.ai", "https://API.WANDB.AI:443"])+def test_weave_sdk_exports_tree_session_usage_without_global_credentials(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, host: str+) -> None:+ calls = []+ before = dict(os.environ)++ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ body = json.loads(kwargs["content"])+ calls.append((_method, url, kwargs, body))+ start = body.get("start")+ return _response(url, body={"id": start["id"], "trace_id": start["trace_id"]} if start else {})++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ result = export_trace("weave", {"api_key": "tenant-key", "entity": "team", "project": "demo", "host": host}, trace)+ assert len(calls) == 4+ root, child = calls[0][3]["start"], calls[1][3]["start"]+ assert child["parent_id"] == root["id"]+ assert child["thread_id"] == root["thread_id"] == "session-1"+ assert child["turn_id"] == root["turn_id"] == root["id"]+ assert calls[2][3]["end"]["summary"]["usage"]["test-model"]["total_tokens"] == 5+ assert calls[3][3]["end"]["summary"] == {}+ assert result.trace_url == f"https://wandb.ai/team/demo/r/call/{root['id']}"+ assert result.span_urls[trace.spans[1].id].endswith(child["id"])+ assert all(call[2]["auth"] == ("api", "tenant-key") and call[2]["max_retries"] == 0 for call in calls)+ assert dict(os.environ) == before++[email protected]("root_kind", [CanonicalSpanKind.CHAIN, CanonicalSpanKind.AGENT])+def test_weave_nested_agent_calls_form_one_outer_turn(+ trace: CanonicalTrace, weave_started_calls: list[StartedCallSchemaForInsert], root_kind: CanonicalSpanKind+) -> None:+ root = trace.spans[0].model_copy(update={"kind": root_kind})+ agent = trace.spans[1].model_copy(update={"kind": CanonicalSpanKind.AGENT})+ model = trace.spans[1].model_copy(update={"id": "0000000000000003", "parent_id": agent.id})+ nested = trace.model_copy(update={"spans": (root, agent, model)})++ export_trace("weave", {"api_key": "tenant-key", "entity": "team", "project": "demo"}, nested)++ outer, inner_agent, inner_model = weave_started_calls+ assert inner_agent.parent_id == outer.id+ assert inner_model.parent_id == inner_agent.id+ assert all(call.thread_id == trace.session_id and call.turn_id == outer.id for call in weave_started_calls)+ # Native Threads queries count only call.id == call.turn_id, including for+ # Workflow and single-step preview outer roots with nested Agent details.+ assert [call.id for call in weave_started_calls if call.id == call.turn_id] == [outer.id]+++def test_weave_same_session_has_distinct_stable_turns(+ trace: CanonicalTrace, weave_started_calls: list[StartedCallSchemaForInsert]+) -> None:+ config = {"api_key": "tenant-key", "entity": "team", "project": "demo"}+ first = export_trace("weave", config, trace)+ repeated = export_trace("weave", config, trace)+ next_trace = trace.model_copy(update={"trace_id": "abcdef1234567890abcdef1234567890"})+ second = export_trace("weave", config, next_trace)++ first_calls, repeated_calls, second_calls = (+ weave_started_calls[:2],+ weave_started_calls[2:4],+ weave_started_calls[4:],+ )+ assert repeated_calls == first_calls+ assert repeated == first+ assert second.trace_url != first.trace_url+ assert all(call.thread_id == trace.session_id for call in weave_started_calls)+ assert first_calls[0].turn_id == first_calls[0].id+ assert second_calls[0].turn_id == second_calls[0].id+ assert second_calls[0].turn_id != first_calls[0].turn_id+ assert all(call.turn_id == first_calls[0].id for call in first_calls)+ assert all(call.turn_id == second_calls[0].id for call in second_calls)+++def test_weave_without_session_does_not_fabricate_a_thread_or_turn(+ trace: CanonicalTrace, weave_started_calls: list[StartedCallSchemaForInsert]+) -> None:+ trace = trace.model_copy(update={"session_id": ""})++ export_trace("weave", {"api_key": "tenant-key", "entity": "team", "project": "demo"}, trace)++ assert len(weave_started_calls) == len(trace.spans)+ assert all(call.thread_id is None and call.turn_id is None for call in weave_started_calls)+++def test_weave_resolves_optional_entity_using_only_supplied_credentials(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace+) -> None:+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ assert kwargs["auth"] == ("api", "tenant-key")+ if url.endswith("/graphql"):+ return _response(url, body={"data": {"viewer": {"entity": "default-team"}}})+ body = json.loads(kwargs["content"])+ if start := body.get("start"):+ assert start["project_id"] == "default-team/demo"+ return _response(url, body={"id": start["id"], "trace_id": start["trace_id"]})+ return _response(url)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ result = export_trace("weave", {"api_key": "tenant-key", "project": "demo"}, trace)+ assert result.trace_url is not None+ assert "/default-team/demo/" in result.trace_url+++def test_weave_partial_tree_is_uncertain_and_not_retried(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace+) -> None:+ calls = []++ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ calls.append(url)+ if len(calls) > 1:+ return _response(url, status=503)+ start = json.loads(kwargs["content"])["start"]+ return _response(url, body={"id": start["id"], "trace_id": start["trace_id"]})++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ with pytest.raises(TraceExportUncertainError, match="partially accepted"):+ export_trace("weave", {"api_key": "secret", "entity": "team", "project": "demo"}, trace)+ assert len(calls) == 2++[email protected]("tracking_uri", ["https://mlflow.example", "https://mlflow.example/proxy/mlflow"])+def test_mlflow_sdk_span_roundtrip_and_credential_isolation(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, tracking_uri: str+) -> None:+ calls = []+ before = dict(os.environ)++ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ calls.append((url, kwargs))+ return _response(url, content=b"")++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ for username in ("tenant-a", "tenant-b"):+ result = export_trace(+ "mlflow",+ {+ "tracking_uri": tracking_uri,+ "experiment_id": "42",+ "username": username,+ "password": "password",+ },+ trace,+ )+ assert result.trace_id == "tr-" + trace.trace_id+ assert result.trace_url is not None+ fragment_path, _, query = urlsplit(result.trace_url).fragment.partition("?")+ assert result.trace_url.startswith(tracking_uri + "/#/")+ assert fragment_path == "/experiments/42/traces"+ assert parse_qs(query) == {+ "traceId": [result.trace_id],+ "selectedTraceId": [result.trace_id],+ "selectedEvaluationId": [result.trace_id],+ }+ for index, (url, options) in enumerate(calls):+ assert url == tracking_uri + "/v1/traces"+ assert options["headers"]["x-mlflow-experiment-id"] == "42"+ assert (+ options["headers"]["Authorization"]+ == "Basic " + base64.b64encode(f"tenant-{'a' if index == 0 else 'b'}:password".encode()).decode()+ )+ content = options["content"]+ assert isinstance(content, bytes)+ payload = ExportTraceServiceRequest.FromString(content)+ spans = [Span.from_otel_proto(span) for span in payload.resource_spans[0].scope_spans[0].spans]+ assert spans[1].parent_id == spans[0].span_id+ assert spans[1].get_attribute(SpanAttributeKey.CHAT_USAGE) == {+ "input_tokens": 2,+ "output_tokens": 3,+ "total_tokens": 5,+ }+ assert spans[1].get_attribute(SpanAttributeKey.LLM_COST) == {"total_cost": 0.01}+ assert spans[0].get_attribute(SpanAttributeKey.CHAT_USAGE) is None+ assert spans[0].get_attribute(SpanAttributeKey.SESSION_ID) == "session-1"+ assert dict(os.environ) == before+++def _tool_definitions() -> list[dict[str, object]]:+ return [+ {+ "type": "function",+ "name": "calculate",+ "description": "TOOL_CONTENT_CANARY",+ "parameters": {+ "type": "object",+ "properties": {"value": {"type": "integer"}},+ "required": ["value"],+ "additionalProperties": False,+ "$comment": "Preserve schema extension fields",+ },+ "strict": True,+ },+ {"type": "function", "function": {"name": "already_nested", "description": "Nested definition"}},+ ]++[email protected]("serialized", [False, True])+def test_mlflow_tool_definitions_have_native_chat_schema_without_losing_source(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, serialized: bool+) -> None:+ definitions = _tool_definitions()+ source = json.dumps(definitions) if serialized else definitions+ trace.spans[1].metadata["gen_ai.tool.definitions"] = source+ before = deepcopy(trace.model_dump())+ native: list[Span] = []++ def send(_method: str, url: str, **options: Unpack[TransportOptions]) -> httpx.Response:+ content = options["content"]+ assert isinstance(content, bytes)+ payload = ExportTraceServiceRequest.FromString(content)+ native.extend(Span.from_otel_proto(item) for item in payload.resource_spans[0].scope_spans[0].spans)+ return _response(url, content=b"")++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ export_trace("mlflow", {"tracking_uri": "https://mlflow.example", "experiment_id": "42"}, trace)+ tools = native[1].get_attribute(SpanAttributeKey.CHAT_TOOLS)+ assert tools == [+ {"type": "function", "function": {key: value for key, value in definitions[0].items() if key != "type"}},+ definitions[1],+ ]+ assert isinstance(tools, list)+ for tool in tools:+ ChatTool.model_validate(tool, strict=True)+ assert native[1].get_attribute("gen_ai.tool.definitions") == source+ stored = translate_span_when_storing(native[1])+ assert json.loads(stored["attributes"][SpanAttributeKey.CHAT_TOOLS]) == tools+ assert trace.model_dump() == before+++def test_mlflow_redacted_trace_does_not_recreate_tool_definition_content(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace+) -> None:+ trace.spans[1].metadata["gen_ai.tool.definitions"] = _tool_definitions()+ trace.spans[1].metadata[SpanAttributeKey.CHAT_TOOLS] = [+ {"type": "function", "function": {"name": "TOOL_CONTENT_CANARY"}}+ ]+ redacted = remap_outer_trace(trace, trace_id=trace.trace_id, capture_content=False, agent_message=False)+ native: list[Span] = []++ def send(_method: str, url: str, **options: Unpack[TransportOptions]) -> httpx.Response:+ content = options["content"]+ assert isinstance(content, bytes)+ assert b"TOOL_CONTENT_CANARY" not in content+ payload = ExportTraceServiceRequest.FromString(content)+ native.extend(Span.from_otel_proto(item) for item in payload.resource_spans[0].scope_spans[0].spans)+ return _response(url, content=b"")++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ export_trace("mlflow", {"tracking_uri": "https://mlflow.example", "experiment_id": "42"}, redacted)+ for span in native:+ assert span.get_attribute(SpanAttributeKey.CHAT_TOOLS) is None+ assert span.get_attribute("gen_ai.tool.definitions") is None+ assert span.inputs is None+ assert span.outputs is None++[email protected]("definitions", [[None, 12, {"type": "other"}, {"type": "function"}], "invalid-json"])+def test_mlflow_unsupported_tool_shapes_remain_in_metadata_without_fake_functions(+ trace: CanonicalTrace, definitions: object+) -> None:+ from dify_trace_mlflow.agent_trace import _spans++ trace.spans[1].metadata["gen_ai.tool.definitions"] = definitions+ before = deepcopy(trace.model_dump())+ result = _spans(trace, "42", "tr-" + trace.trace_id)+ assert result[1].get_attribute("gen_ai.tool.definitions") == definitions+ assert result[1].get_attribute(SpanAttributeKey.CHAT_TOOLS) == ([] if isinstance(definitions, list) else None)+ assert trace.model_dump() == before+++def test_mlflow_existing_native_chat_tools_take_precedence(trace: CanonicalTrace) -> None:+ from dify_trace_mlflow.agent_trace import _spans++ native_tools = [{"type": "function", "function": {"name": "already_native"}}]+ trace.spans[1].metadata[SpanAttributeKey.CHAT_TOOLS] = native_tools+ trace.spans[1].metadata["gen_ai.tool.definitions"] = _tool_definitions()+ result = _spans(trace, "42", "tr-" + trace.trace_id)+ assert result[1].get_attribute(SpanAttributeKey.CHAT_TOOLS) == native_tools+++def test_mlflow_preserves_json_schema_beyond_the_sdk_chat_tool_model(trace: CanonicalTrace) -> None:+ from dify_trace_mlflow.agent_trace import _spans++ parameters = {+ "$ref": "#/$defs/Arguments",+ "$defs": {+ "Arguments": {+ "type": "object",+ "properties": {"choice": {"enum": [None, {"mode": "auto"}]}},+ "additionalProperties": {"type": "string"},+ }+ },+ }+ source = [{"type": "function", "name": "schema_extension", "parameters": parameters}]+ trace.spans[1].metadata["gen_ai.tool.definitions"] = source+ result = _spans(trace, "42", "tr-" + trace.trace_id)+ expected = [{"type": "function", "function": {"name": "schema_extension", "parameters": parameters}}]+ assert result[1].get_attribute(SpanAttributeKey.CHAT_TOOLS) == expected+ assert result[1].get_attribute("gen_ai.tool.definitions") == source+ stored = translate_span_when_storing(result[1])+ assert json.loads(stored["attributes"][SpanAttributeKey.CHAT_TOOLS]) == expected+++def test_mlflow_partial_otlp_ack_is_uncertain(monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace) -> None:+ ack = ExportTraceServiceResponse()+ ack.partial_success.rejected_spans = 1+ monkeypatch.setattr(+ ssrf_proxy, "make_request", lambda _method, url, **_kwargs: _response(url, content=ack.SerializeToString())+ )+ with pytest.raises(TraceExportUncertainError, match="partially accepted"):+ export_trace("mlflow", {"tracking_uri": "https://mlflow.example", "experiment_id": "42"}, trace)+++ "credential_type", ["AWS_PRESIGNED_URL", "GCP_SIGNED_URL", "AZURE_SAS_URI", "AZURE_ADLS_GEN2_SAS_URI"]+)[email protected]("oauth", [False, True])+def test_databricks_sdk_v3_metadata_and_native_artifact(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, credential_type: str, oauth: bool+) -> None:+ calls: list[tuple[str, str, TransportOptions]] = []+ uploaded: list[TraceData] = []+ before = dict(os.environ)+ server_trace_id = "tr-abcdef0123456789abcdef0123456789"+ definitions = _tool_definitions()+ trace.spans[1].metadata["gen_ai.tool.definitions"] = definitions++ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ calls.append((_method, url, kwargs))+ if url.endswith("/oidc/v1/token"):+ assert kwargs["auth"] == ("client", "client-secret")+ return _response(url, body={"access_token": "oauth-token"})+ if url.endswith("/api/3.0/mlflow/traces"):+ assert kwargs["headers"]["Authorization"] == ("Bearer oauth-token" if oauth else "Bearer pat-token")+ body = json.loads(kwargs["content"])+ assert body["trace"]["trace_info"]["trace_metadata"]["mlflow.trace.session"] == "session-1"+ body["trace"]["trace_info"]["trace_id"] = server_trace_id+ return _response(url, body=body)+ if url.endswith("/credentials-for-data-upload"):+ assert f"/traces/{server_trace_id}/" in url+ return _response(+ url,+ body={+ "credential_info": {+ "signed_uri": "https://storage.example/trace?sig=fake",+ "type": credential_type,+ "headers": [],+ }+ },+ )+ assert "Authorization" not in kwargs["headers"]+ if kwargs.get("content"):+ uploaded.append(TraceData.from_dict(json.loads(kwargs["content"])))+ return _response(url)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ config = {"host": "https://databricks.example", "experiment_id": "42"}+ config.update(+ {"client_id": "client", "client_secret": "client-secret"} if oauth else {"personal_access_token": "pat-token"}+ )+ result = export_trace("databricks", config, trace)+ assert result.trace_id == server_trace_id+ assert result.trace_url == (+ f"https://databricks.example/ml/experiments/42/traces?selectedEvaluationId={server_trace_id}"+ )+ data = uploaded[0]+ assert len(data.spans) == 2+ assert all(span.trace_id == server_trace_id for span in data.spans)+ assert data.spans[1].parent_id == data.spans[0].span_id+ usage = data.spans[1].get_attribute(SpanAttributeKey.CHAT_USAGE)+ assert usage is not None+ assert usage["total_tokens"] == 5+ assert data.spans[1].get_attribute(SpanAttributeKey.LLM_COST) == {"total_cost": 0.01}+ tools = data.spans[1].get_attribute(SpanAttributeKey.CHAT_TOOLS)+ assert isinstance(tools, list)+ assert len(tools) == 2+ for tool in tools:+ ChatTool.model_validate(tool, strict=True)+ assert data.spans[1].get_attribute("gen_ai.tool.definitions") == definitions+ assert dict(os.environ) == before+ assert len(calls) == 3 + int(oauth) + int(credential_type == "AZURE_ADLS_GEN2_SAS_URI")+++def test_databricks_partial_metadata_delivery_is_uncertain(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace+) -> None:+ def send(_method: str, url: str, **kwargs: Unpack[TransportOptions]) -> httpx.Response:+ if url.endswith("/api/3.0/mlflow/traces"):+ return _response(url, body=json.loads(kwargs["content"]))+ return _response(url, status=503)++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ with pytest.raises(TraceExportUncertainError, match="metadata"):+ export_trace(+ "databricks",+ {"host": "https://databricks.example", "experiment_id": "42", "personal_access_token": "secret"},+ trace,+ )++[email protected]("provider", ["weave", "databricks"])+def test_success_status_with_unreadable_ack_is_uncertain(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, provider: str+) -> None:+ monkeypatch.setattr(ssrf_proxy, "make_request", lambda _method, url, **_kwargs: _response(url, content=b"not-json"))+ config = {+ "api_key": "secret",+ "entity": "team",+ "project": "demo",+ "host": "https://example.com",+ "experiment_id": "42",+ "personal_access_token": "secret",+ }+ with pytest.raises(TraceExportUncertainError, match="unreadable"):+ export_trace(provider, config, trace)diff --git a/api/tests/unit_tests/services/tracing/test_nullable_tracing_credentials.py b/api/tests/unit_tests/services/tracing/test_nullable_tracing_credentials.pynew file mode 100644index 00000000000..fa8d4215b31--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_nullable_tracing_credentials.py@@ -0,0 +1,92 @@+"""Nullable provider credentials never reach RSA/base64 as non-string tokens."""++from uuid import uuid4++import pytest+from Crypto.PublicKey import RSA+from pydantic import ValidationError+from sqlalchemy.orm import Session++import models.engine as model_engine+from core.ops.ops_trace_manager import OpsTraceManager+from extensions.ext_key_provider import key_provider_manager+from libs import gmpy2_pkcs10aep_cipher, rsa+from libs.key_providers.rsa_key_provider import RSAKeyProvider+from tests.unit_tests.model_factories import make_tenant+++def workspace(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> str:+ key = RSA.generate(2048)+ tenant_id = str(uuid4())+ sqlite_session.add(make_tenant(tenant_id=tenant_id, encrypt_public_key=key.publickey().export_key().decode()))+ sqlite_session.commit()+ monkeypatch.setattr(model_engine.db, "session", sqlite_session)+ monkeypatch.setattr(key_provider_manager, "_provider", RSAKeyProvider())+ monkeypatch.setattr(rsa, "get_decrypt_decoding", lambda _tenant: (key, gmpy2_pkcs10aep_cipher.new(key)))+ return tenant_id+++def test_databricks_switches_auth_with_real_encryption_and_explicit_null(workspace: str) -> None:+ pat = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "databricks",+ {+ "host": "https://workspace.test",+ "experiment_id": "12",+ "personal_access_token": "synthetic-pat",+ "client_id": None,+ "client_secret": None,+ },+ )+ assert pat["personal_access_token"] != "synthetic-pat"+ decrypted_pat = OpsTraceManager.decrypt_tracing_config(workspace, "databricks", pat)+ assert decrypted_pat["personal_access_token"] == "synthetic-pat"+ assert decrypted_pat["client_secret"] is None++ oauth = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "databricks",+ {+ "host": "https://workspace.test",+ "experiment_id": "12",+ "personal_access_token": None,+ "client_id": "synthetic-client",+ "client_secret": "synthetic-secret",+ },+ pat,+ )+ assert oauth["personal_access_token"] is None+ assert oauth["client_secret"] != "synthetic-secret"+ decrypted_oauth = OpsTraceManager.decrypt_tracing_config(workspace, "databricks", oauth)+ assert decrypted_oauth["personal_access_token"] is None+ assert decrypted_oauth["client_secret"] == "synthetic-secret"+++def test_other_optional_secret_model_can_be_cleared_without_losing_validation(workspace: str) -> None:+ password = OpsTraceManager.encrypt_tracing_config(+ workspace,+ "mlflow",+ {+ "tracking_uri": "https://mlflow.test",+ "experiment_id": "0",+ "username": "user",+ "password": "synthetic-password",+ },+ )+ assert OpsTraceManager.decrypt_tracing_config(workspace, "mlflow", password)["password"] == "synthetic-password"+ cleared = OpsTraceManager.encrypt_tracing_config(workspace, "mlflow", {**password, "password": None}, password)+ assert cleared["password"] is None+ assert OpsTraceManager.decrypt_tracing_config(workspace, "mlflow", cleared)["password"] is None+++def test_non_nullable_secret_is_still_rejected_by_provider_model(workspace: str) -> None:+ with pytest.raises(ValidationError, match="secret_key"):+ OpsTraceManager.encrypt_tracing_config(+ workspace, "langfuse", {"public_key": "synthetic-public", "secret_key": None}+ )+ encrypted = OpsTraceManager.encrypt_tracing_config(+ workspace, "langfuse", {"public_key": "synthetic-public", "secret_key": "synthetic-secret"}+ )+ with pytest.raises(ValidationError, match="secret_key"):+ OpsTraceManager.decrypt_tracing_config(workspace, "langfuse", {**encrypted, "secret_key": None})diff --git a/api/tests/unit_tests/services/tracing/test_opik_configuration.py b/api/tests/unit_tests/services/tracing/test_opik_configuration.pynew file mode 100644index 00000000000..073b5bb8233--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_opik_configuration.py@@ -0,0 +1,188 @@+"""Opik auth contracts at the read-only HTTP boundary, without SDK streamers."""++from collections.abc import Callable+from dataclasses import dataclass, field+from urllib.parse import parse_qs, urlsplit++import httpx+import pytest+from dify_trace_opik.config import OpikConfig+from dify_trace_opik.configuration import OpikConfigurationError, console_url, project_url, verify_configuration++from core.helper import ssrf_proxy+++@dataclass+class AccessAPI:+ calls: list[tuple[str, str, dict[str, object]]] = field(default_factory=list)+ default_workspace: object = "resolved-team"+ override: Callable[[str, str, dict[str, object]], httpx.Response] | None = None++ def request(self, method: str, url: str, **options: object) -> httpx.Response:+ self.calls.append((method, url, options))+ if self.override:+ return self.override(method, url, options)+ request = httpx.Request(method, url)+ if method == "POST":+ assert url.endswith("/v1/private/auth"), "configuration must never create projects, spans or traces"+ assert options["json"] == {}+ return httpx.Response(204, request=request)+ assert method == "GET"+ assert url.endswith("/v1/private/auth/workspace")+ return httpx.Response(200, json={"workspace_name": self.default_workspace}, request=request)+++def access_api(monkeypatch: pytest.MonkeyPatch) -> AccessAPI:+ api = AccessAPI()+ monkeypatch.setattr(ssrf_proxy, "make_request", api.request)+ return api+++def test_named_workspaces_use_isolated_auth_without_default_lookup(access_api: AccessAPI) -> None:+ for host, key, workspace in [+ ("https://first.test/opik/api/", "first-key", "first-team"),+ ("https://second.test/api/", "second-key", "second-team"),+ ]:+ result = verify_configuration(OpikConfig(api_key=key, workspace=workspace, project="not-created", url=host))+ assert (result.url, result.workspace, result.project) == (host, workspace, "not-created")+ method, url, options = access_api.calls[-1]+ assert method == "POST"+ assert url == host + "v1/private/auth"+ assert options["headers"] == {+ "Authorization": key,+ "Comet-Workspace": workspace,+ "Accept": "application/json",+ }+ assert options["json"] == {}+ assert options["max_retries"] == 0+ assert options["follow_redirects"] is False+ assert options["timeout"] == 30.0+ assert len(access_api.calls) == 2++[email protected]("workspace", [None, "", "default", " default "])+def test_default_workspace_is_resolved_only_during_verification(access_api: AccessAPI, workspace: str | None) -> None:+ result = verify_configuration(OpikConfig(api_key="synthetic-key", workspace=workspace))+ assert result.workspace == "resolved-team"+ assert result.project == "Default Project"+ assert [method for method, _, _ in access_api.calls] == ["POST", "GET"]+ assert access_api.calls[1][2]["headers"] == access_api.calls[0][2]["headers"]++[email protected]("key", [None, ""])+def test_keyless_self_host_does_not_fall_back_to_environment(+ access_api: AccessAPI, monkeypatch: pytest.MonkeyPatch, key: str | None+) -> None:+ monkeypatch.setenv("OPIK_API_KEY", "unrelated-global-key")+ monkeypatch.setenv("OPIK_WORKSPACE", "unrelated-global-team")+ monkeypatch.setenv("OPIK_URL_OVERRIDE", "https://unrelated.test/api/")+ result = verify_configuration(OpikConfig(api_key=key, workspace="local", url="http://selfhost.test:5173/api/"))+ assert result.url == "http://selfhost.test:5173/api/"+ assert access_api.calls[0][2]["headers"] == {"Comet-Workspace": "local", "Accept": "application/json"}+ assert len(access_api.calls) == 1+++ "url",+ [+ "https://user:[email protected]/api/",+ "https://private.test:invalid/api/",+ "http:///api/",+ "https://private.test/api/?next=/api/",+ "https://private.test/api/#/api/",+ "https://private.test/with space/api/",+ "https://private.test/back\\slash/api/",+ ],+)+def test_invalid_destination_is_rejected_before_sending_credentials(access_api: AccessAPI, url: str) -> None:+ with pytest.raises(OpikConfigurationError) as error:+ verify_configuration(OpikConfig(api_key="private-key", url=url))+ assert error.value.code == "invalid_endpoint"+ assert access_api.calls == []+ assert "private-key" not in str(error.value)+++ ("status", "code"),+ [+ (401, "authentication_failed"),+ (403, "permission_denied"),+ (302, "endpoint_unavailable"),+ (500, "endpoint_unavailable"),+ ],+)+def test_auth_failures_are_sanitized(access_api: AccessAPI, status: int, code: str) -> None:+ access_api.override = lambda method, url, _options: httpx.Response(+ status, text="private-vendor-error", request=httpx.Request(method, url)+ )+ with pytest.raises(OpikConfigurationError) as error:+ verify_configuration(OpikConfig(api_key="private-key", workspace="team"))+ assert error.value.code == code+ assert "private" not in str(error.value)+ assert len(access_api.calls) == 1+++def test_auth_html_success_is_not_a_verified_workspace(access_api: AccessAPI) -> None:+ access_api.override = lambda method, url, _options: httpx.Response(+ 200, text="<html>Login</html>", request=httpx.Request(method, url)+ )+ with pytest.raises(OpikConfigurationError, match="invalid response"):+ verify_configuration(OpikConfig(workspace="team"))++[email protected]("workspace", [None, "", " ", 123, []])+def test_default_workspace_response_requires_a_name(access_api: AccessAPI, workspace: object) -> None:+ access_api.default_workspace = workspace+ with pytest.raises(OpikConfigurationError) as error:+ verify_configuration(OpikConfig())+ assert error.value.code == "malformed_response"+++def test_connection_error_never_exposes_request_or_key(access_api: AccessAPI) -> None:+ def fail(method: str, url: str, _options: dict[str, object]) -> httpx.Response:+ raise httpx.ReadTimeout("private-key in vendor detail", request=httpx.Request(method, url))++ access_api.override = fail+ with pytest.raises(OpikConfigurationError) as error:+ verify_configuration(OpikConfig(api_key="private-key"))+ assert error.value.code == "connection_failed"+ assert "private-key" not in str(error.value)+++def test_project_url_uses_configured_host_and_encodes_names_without_network(access_api: AccessAPI) -> None:+ config = OpikConfig(url="https://private.test:7443/prefix/api/", workspace="team/name", project="A & B/? 中文")+ url = project_url(config)+ assert url is not None+ parsed = urlsplit(url)+ assert parsed.netloc == "private.test:7443"+ assert parsed.path == "/prefix/team%2Fname/redirect/projects"+ assert parse_qs(parsed.query) == {"name": ["A & B/? 中文"]}+ assert access_api.calls == []++[email protected]("workspace", [None, "", "default"])+def test_historical_default_workspace_get_has_a_pure_console_fallback(+ access_api: AccessAPI, workspace: str | None+) -> None:+ config = OpikConfig(url="http://selfhost.test:5173/prefix/api/", workspace=workspace)+ assert project_url(config) is None+ assert console_url(config) == "http://selfhost.test:5173/prefix/"+ assert access_api.calls == []+++ ("workspace", "project", "code"),+ [+ ("..", "ok", "invalid_workspace"),+ ("bad\nworkspace", "ok", "invalid_workspace"),+ ("team", "bad\nproject", "invalid_project"),+ ],+)+def test_invalid_names_cannot_escape_the_console_path_or_headers(+ access_api: AccessAPI, workspace: str, project: str, code: str+) -> None:+ with pytest.raises(OpikConfigurationError) as error:+ verify_configuration(OpikConfig(workspace=workspace, project=project))+ assert error.value.code == code+ assert access_api.calls == []diff --git a/api/tests/unit_tests/services/tracing/test_opik_legacy_identifiers.py b/api/tests/unit_tests/services/tracing/test_opik_legacy_identifiers.pynew file mode 100644index 00000000000..af48f2234e9--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_opik_legacy_identifiers.py@@ -0,0 +1,214 @@+"""Legacy external IDs retain stable SDK batches and workflow parent relations."""++import json+from datetime import datetime, timedelta+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5++import pytest+from dify_trace_opik.opik_trace import OpikDataTrace, prepare_opik_uuid+from opik import Opik+from opik.config import OpikConfig as SDKConfig+from opik.id_helpers import uuid4_to_uuid7+from opik.message_processing.batching.batchers import CreateSpanMessageBatcher, CreateTraceMessageBatcher+from opik.message_processing.messages import (+ BaseMessage,+ CreateSpanMessage,+ CreateSpansBatchMessage,+ CreateTraceBatchMessage,+ CreateTraceMessage,+)+from opik.message_processing.streamer import Streamer+from sqlalchemy import Engine+from sqlalchemy.orm import Session, sessionmaker++from core.ops.entities.trace_entity import WorkflowTraceInfo+from extensions.ext_database import db+from extensions.logstore.repositories.logstore_api_workflow_node_execution_repository import (+ _dict_to_workflow_node_execution_model,+)+from models.account import TenantAccountJoin, TenantAccountRole+from tests.unit_tests.model_factories import make_account, make_app, make_tenant++START = datetime(2026, 10, 16, 15, 18, 10, 792586)+WORKFLOW_ID = "11111111-2222-4333-8444-555555555555"+EXTERNAL_IDS = [+ "0123456789abcdef0123456789abcdef",+ "019140fb-6972-7d74-a219-6cba9eb543b1",+ str(uuid5(NAMESPACE_URL, "external-trace")),+ "external-trace_correlation",+]++[email protected]("identifier", [WORKFLOW_ID, WORKFLOW_ID.upper(), UUID(WORKFLOW_ID).hex])+def test_valid_uuid4_keeps_the_original_sdk_mapping(identifier: str) -> None:+ assert prepare_opik_uuid(START, identifier) == uuid4_to_uuid7(START, identifier)++[email protected]("identifier", EXTERNAL_IDS)+def test_external_identifiers_become_stable_uuid7_with_unchanged_time(identifier: str) -> None:+ mapped = prepare_opik_uuid(START, identifier)+ assert mapped.version == 7+ assert mapped == prepare_opik_uuid(START, identifier)+ assert int.from_bytes(mapped.bytes[:6]) == int(START.timestamp() * 1000)+ assert mapped != prepare_opik_uuid(START, identifier + "-different")+++def test_missing_identifier_keeps_random_fallback() -> None:+ first, second = prepare_opik_uuid(START, None), prepare_opik_uuid(START, None)+ assert first.version == second.version == 7+ assert first != second+ assert int.from_bytes(first.bytes[:6]) == int(START.timestamp() * 1000)+ assert prepare_opik_uuid(None, WORKFLOW_ID).version == 7++[email protected]("external_id", [*EXTERNAL_IDS, WORKFLOW_ID, None])+def test_workflow_external_ids_reach_real_sdk_batches_with_complete_owned_tree(+ external_id: str | None,+ sqlite_engine: Engine,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ tenant_id, app_id, account_id, definition_id = (str(uuid4()) for _ in range(4))+ node_ids = [str(uuid4()) for _ in range(3)]+ with sqlite_session_factory.begin() as session:+ session.add(make_tenant(tenant_id=tenant_id))+ session.add(make_account(account_id=account_id))+ session.add(+ TenantAccountJoin(tenant_id=tenant_id, account_id=account_id, role=TenantAccountRole.OWNER, current=True)+ )+ session.add(make_app(app_id=app_id, tenant_id=tenant_id, created_by=account_id))+ for index, kind in enumerate(("start", "llm", "end")):+ usage = {"prompt_tokens": 27, "completion_tokens": 5, "total_tokens": 32}+ process: dict[str, object] = (+ {+ "model_mode": "chat",+ "model_name": "test-model",+ "model_provider": "provider/test",+ "usage": usage,+ "prompts": [{"role": "user", "content": "synthetic"}],+ }+ if kind == "llm"+ else {}+ )+ session.add(+ _dict_to_workflow_node_execution_model(+ {+ "id": node_ids[index],+ "tenant_id": tenant_id,+ "app_id": app_id,+ "workflow_id": definition_id,+ "workflow_run_id": WORKFLOW_ID,+ "node_id": kind,+ "node_type": kind,+ "title": kind,+ "created_by": account_id,+ "created_at": (START + timedelta(milliseconds=index * 100)).isoformat(),+ "status": "succeeded",+ "index": index,+ "elapsed_time": 0.1,+ "inputs": json.dumps({}),+ "outputs": json.dumps({"answer": "synthetic"}),+ "process_data": json.dumps(process),+ "execution_metadata": json.dumps({"total_tokens": 32} if kind == "llm" else {}),+ }+ )+ )+ # A node from another run must not enter this trace's SDK batch.+ session.add(+ _dict_to_workflow_node_execution_model(+ {+ "id": str(uuid4()),+ "tenant_id": tenant_id,+ "app_id": app_id,+ "workflow_id": definition_id,+ "workflow_run_id": str(uuid4()),+ "node_id": "foreign",+ "node_type": "llm",+ "title": "foreign",+ "created_by": account_id,+ "created_at": START.isoformat(),+ "status": "succeeded",+ "elapsed_time": 0.1,+ }+ )+ )++ # Bind the real SQLAlchemy owner to the test engine; no Session/repository double.+ monkeypatch.setattr(type(db), "engine", property(lambda _owner: sqlite_engine))+ queued: list[BaseMessage] = []+ streamer = object.__new__(Streamer)+ monkeypatch.setattr(streamer, "put", queued.append)+ client = object.__new__(Opik)+ client._streamer = streamer+ client._config = SDKConfig.model_construct(url_override="https://offline.invalid/api", project_name="test-project")+ client._project_name = "test-project"+ client._project_name_most_recent_trace = None+ instance = object.__new__(OpikDataTrace)+ instance.opik_client = client+ instance.project = "test-project"+ info = WorkflowTraceInfo(+ workflow_id=definition_id,+ tenant_id=tenant_id,+ workflow_run_id=WORKFLOW_ID,+ workflow_run_elapsed_time=1,+ workflow_run_status="succeeded",+ workflow_run_inputs={"query": "synthetic"},+ workflow_run_outputs={"answer": "synthetic"},+ workflow_run_version="draft",+ total_tokens=32,+ file_list=[],+ query="synthetic",+ metadata={"app_id": app_id},+ start_time=START,+ end_time=START + timedelta(seconds=1),+ trace_id=external_id,+ )+ instance.workflow_trace(info)+ trace_batches: list[BaseMessage] = []+ span_batches: list[BaseMessage] = []+ trace_batcher = CreateTraceMessageBatcher(trace_batches.append, 1000, 2)+ span_batcher = CreateSpanMessageBatcher(span_batches.append, 1000, 2)+ for message in queued:+ if isinstance(message, CreateTraceMessage):+ trace_batcher.add(message)+ elif isinstance(message, CreateSpanMessage):+ span_batcher.add(message)+ else:+ pytest.fail("Unexpected SDK message type")+ trace_batcher.flush()+ span_batcher.flush()+ assert len(trace_batches) == len(span_batches) == 1+ trace_batch, span_batch = trace_batches[0], span_batches[0]+ assert isinstance(trace_batch, CreateTraceBatchMessage)+ assert isinstance(span_batch, CreateSpansBatchMessage)+ assert len(trace_batch.batch) == 1+ assert len(span_batch.batch) == 4+ trace = trace_batch.batch[0]+ root = next(span for span in span_batch.batch if span.parent_span_id is None)+ assert trace.id == str(prepare_opik_uuid(START, external_id or WORKFLOW_ID))+ assert trace.start_time.timestamp() == START.timestamp()+ assert root.id != trace.id+ assert {span.trace_id for span in span_batch.batch} == {trace.id}+ assert len({span.id for span in span_batch.batch}) == 4+ assert all(UUID(str(span.id)).version == 7 for span in span_batch.batch)+ children = [span for span in span_batch.batch if span.parent_span_id is not None]+ assert all(span.parent_span_id == root.id for span in children)+ assert {span.metadata["node_execution_id"] for span in children if isinstance(span.metadata, dict)} == set(node_ids)+ for index, node_id in enumerate(node_ids):+ child = next(+ span+ for span in children+ if isinstance(span.metadata, dict) and span.metadata["node_execution_id"] == node_id+ )+ assert child.start_time.timestamp() == (START + timedelta(milliseconds=index * 100)).timestamp()+ assert child.end_time is not None+ assert (child.end_time - child.start_time).total_seconds() == pytest.approx(0.1)+ llm = next(span for span in children if span.type == "llm")+ assert llm.usage is not None+ assert {key: llm.usage[key] for key in ("prompt_tokens", "completion_tokens", "total_tokens")} == {+ "prompt_tokens": 27,+ "completion_tokens": 5,+ "total_tokens": 32,+ }+ assert sum(span.usage.get("total_tokens", 0) for span in span_batch.batch if span.usage) == 32+ assert trace.output == {"answer": "synthetic"}diff --git a/api/tests/unit_tests/services/tracing/test_phoenix_configuration.py b/api/tests/unit_tests/services/tracing/test_phoenix_configuration.pynew file mode 100644index 00000000000..032b511aa25--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_phoenix_configuration.py@@ -0,0 +1,336 @@+"""Phoenix configuration verifies only a bounded project-list read, with isolated auth."""++import os+from collections import deque+from collections.abc import Iterator+from concurrent.futures import ThreadPoolExecutor+from threading import Barrier+from typing import Never++import httpx+import pytest+from dify_trace_arize_phoenix.arize_phoenix_trace import ArizePhoenixDataTrace+from dify_trace_arize_phoenix.config import PhoenixConfig+from dify_trace_arize_phoenix.configuration import (+ PhoenixConfigurationError,+ normalize_configuration,+ project_url,+ verify_configuration,+)++from core.helper import ssrf_proxy+from core.ops.ops_trace_manager import OpsTraceManager+++def response(body: object, status: int = 200) -> httpx.Response:+ return httpx.Response(status, json=body, request=httpx.Request("GET", "https://phoenix.test/v1/projects"))++[email protected](autouse=True)+def no_legacy_or_environment_auth(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:+ names = ("PHOENIX_API_KEY", "PHOENIX_PROJECT_NAME", "PHOENIX_COLLECTOR_ENDPOINT", "OTEL_EXPORTER_OTLP_HEADERS")+ for name in names:+ monkeypatch.setenv(name, "synthetic-ambient-value")+ called: list[bool] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ called.append(True)+ raise AssertionError("Phoenix configuration must not initialize a tracer or write test spans")++ monkeypatch.setattr(ArizePhoenixDataTrace, "__init__", forbidden)+ monkeypatch.setattr(OpsTraceManager, "check_trace_config_is_effective", forbidden)+ monkeypatch.setattr(OpsTraceManager, "get_trace_config_project_url", forbidden)+ for method in ("post", "put", "patch", "delete"):+ monkeypatch.setattr(ssrf_proxy, method, forbidden)+ yield+ assert called == []+ assert all(os.environ[name] == "synthetic-ambient-value" for name in names)+++def api_responses() -> deque[httpx.Response | Exception]:+ return deque()+++def api_calls(+ monkeypatch: pytest.MonkeyPatch, api_responses: deque[httpx.Response | Exception]+) -> list[tuple[str, dict[str, object]]]:+ calls: list[tuple[str, dict[str, object]]] = []++ def get(url: str, **options: object) -> httpx.Response:+ calls.append((url, options))+ result = api_responses.popleft() if api_responses else response({"data": [], "next_cursor": None})+ if isinstance(result, Exception):+ raise result+ return result++ monkeypatch.setattr(ssrf_proxy, "get", get)+ return calls+++def test_normalization_preserves_prefix_and_does_not_mutate_input(+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ values = {"endpoint": " HTTP://Phoenix.TEST:6006/proxy/s/space/ ", "project": " project name ", "api_key": None}+ untouched = dict(values)+ config = normalize_configuration(values)+ result = verify_configuration(config)+ assert (result.endpoint, result.project) == ("http://phoenix.test:6006/proxy/s/space", "project name")+ assert values == untouched+ assert api_calls == [+ (+ "http://phoenix.test:6006/proxy/s/space/v1/projects",+ {+ "params": {"limit": 1},+ "headers": {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""},+ "auth": None,+ "timeout": 30.0,+ "max_retries": 0,+ "follow_redirects": False,+ "stream_response": True,+ },+ )+ ]++[email protected]("key", [None, "", "synthetic-key"])+def test_optional_key_does_not_inherit_environment(+ api_calls: list[tuple[str, dict[str, object]]],+ key: str | None,+) -> None:+ result = verify_configuration(PhoenixConfig(endpoint="https://phoenix.test", project="not-created", api_key=key))+ assert result.project == "not-created"+ expected = {"Accept": "application/json", "Accept-Encoding": "identity", "Cookie": ""}+ if key:+ expected["Authorization"] = "Bearer synthetic-key"+ assert api_calls[0][1]["headers"] == expected+ assert len(api_calls) == 1++[email protected]("project", [None, "", " "])+def test_empty_project_uses_default(project: str | None, api_calls: list[tuple[str, dict[str, object]]]) -> None:+ config = normalize_configuration({"project": project})+ assert config.project == "default"+ assert project_url(config) == "https://app.phoenix.arize.com/redirects/projects/default"+ assert api_calls == []+++def test_project_redirect_encodes_name_for_graphql_loader_without_io(+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ config = PhoenixConfig(endpoint="https://host.test:8443/s/space/", project="A /? & #中文")+ assert (+ project_url(config)+ == "https://host.test:8443/s/space/redirects/projects/A%20%2F%3F%20%26%20%23%E4%B8%AD%E6%96%87"+ )+ assert api_calls == []+++ "endpoint",+ [+ "ftp://host.test",+ "https:///missing",+ "https://user:[email protected]",+ "https://host.test?",+ "https://host.test#",+ "https://host.test/path?token=secret",+ "https://host.test/path#fragment",+ "https://host.test:bad",+ "https://host.test:0",+ "https://host.test:65536",+ "https://host.test:",+ "https://[invalid",+ "https://host.test\\@elsewhere",+ "\nhttps://host.test",+ "https://ho\nst.test",+ "https://host.test/a/../b",+ "https://host.test/a/%2e%2e/b",+ "https://host.test/a%2fb",+ "https://host.test/a%5cb",+ "https://host.test/a%252fb",+ "https://host.test//proxy",+ "https://host.test/%00",+ "https://host.test/%zz",+ "https://host.test/v1/traces",+ "https://host.test/s/space/v1/traces/",+ None,+ ],+)+def test_invalid_base_is_rejected_before_io(endpoint: object, api_calls: list[tuple[str, dict[str, object]]]) -> None:+ with pytest.raises(PhoenixConfigurationError) as caught:+ normalize_configuration({"endpoint": endpoint, "api_key": "private-secret"})+ assert caught.value.code == "invalid_endpoint"+ assert "private-secret" not in str(caught.value)+ assert api_calls == []++[email protected]("project", ["bad\nname", "\tname", ".", "..", 123])+def test_invalid_project_is_rejected_before_io(project: object, api_calls: list[tuple[str, dict[str, object]]]) -> None:+ with pytest.raises(PhoenixConfigurationError) as caught:+ normalize_configuration({"project": project})+ assert caught.value.code == "invalid_project"+ assert api_calls == []++[email protected]("key", [" ", "line\nbreak", "invalid token", "******", "ab***cd"])+def test_header_or_mask_key_is_rejected_after_restore(key: str, api_calls: list[tuple[str, dict[str, object]]]) -> None:+ config = normalize_configuration({"api_key": key})+ with pytest.raises(PhoenixConfigurationError) as caught:+ verify_configuration(config)+ assert caught.value.code == "invalid_api_key"+ assert api_calls == []+++ ("status", "code"),+ [+ (401, "authentication_failed"),+ (403, "permission_denied"),+ (302, "endpoint_unavailable"),+ (404, "endpoint_unavailable"),+ (500, "endpoint_unavailable"),+ (204, "endpoint_unavailable"),+ ],+)+def test_http_failures_are_safe_and_close_response(+ status: int,+ code: str,+ api_responses: deque[httpx.Response | Exception],+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ item = response({"private-vendor-detail": "private-secret"}, status)+ api_responses.append(item)+ with pytest.raises(PhoenixConfigurationError) as caught:+ verify_configuration(PhoenixConfig(api_key="private-secret"))+ assert caught.value.code == code+ assert "private-secret" not in str(caught.value)+ assert item.is_closed+ assert len(api_calls) == 1+++ "body",+ [+ None,+ [],+ {},+ {"data": []},+ {"data": {}, "next_cursor": None},+ {"data": [], "next_cursor": 3},+ {"data": [None], "next_cursor": None},+ {"data": [{"id": "1"}], "next_cursor": None},+ {"data": [{"id": 1, "name": "x"}], "next_cursor": None},+ {"data": [{"id": "1", "name": ""}], "next_cursor": None},+ {"data": [{"id": "1", "name": "x", "description": {}}], "next_cursor": None},+ ],+)+def test_project_list_shape_is_validated(+ body: object,+ api_responses: deque[httpx.Response | Exception],+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ item = response(body)+ api_responses.append(item)+ with pytest.raises(PhoenixConfigurationError) as caught:+ verify_configuration(PhoenixConfig())+ assert caught.value.code == "malformed_response"+ assert item.is_closed+ assert len(api_calls) == 1+++def test_existing_project_page_does_not_require_selected_project_or_follow_cursor(+ api_responses: deque[httpx.Response | Exception],+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ api_responses.append(+ response({"data": [{"id": "Project:1", "name": "other", "description": None}], "next_cursor": "opaque-cursor"})+ )+ assert verify_configuration(PhoenixConfig(project="new-project")).project == "new-project"+ assert len(api_calls) == 1++[email protected]("kind", ["html", "gzip", "oversized"])+def test_wrong_path_html_compressed_and_oversized_responses_are_not_success(+ kind: str,+ api_responses: deque[httpx.Response | Exception],+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ content = b"x" * (1024 * 1024 + 1) if kind == "oversized" else b"<html>private-login-page</html>"+ item = httpx.Response(+ 200, content=content, request=httpx.Request("GET", "https://host.test/not-a-collector/v1/projects")+ )+ if kind == "gzip":+ item.headers["Content-Encoding"] = "gzip"+ api_responses.append(item)+ with pytest.raises(PhoenixConfigurationError) as caught:+ verify_configuration(PhoenixConfig(endpoint="https://host.test/not-a-collector"))+ assert caught.value.code == "malformed_response"+ assert "private-login-page" not in str(caught.value)+ assert item.is_closed+ assert len(api_calls) == 1+++def test_transport_error_does_not_leak_url_or_credentials(+ api_responses: deque[httpx.Response | Exception],+ api_calls: list[tuple[str, dict[str, object]]],+) -> None:+ api_responses.append(httpx.ReadTimeout("private-key in vendor response"))+ with pytest.raises(PhoenixConfigurationError) as caught:+ verify_configuration(PhoenixConfig(api_key="private-key"))+ assert caught.value.code == "connection_failed"+ assert "private-key" not in str(caught.value)+ assert caught.value.__suppress_context__+ assert len(api_calls) == 1+++def test_concurrent_targets_and_credentials_remain_isolated(monkeypatch: pytest.MonkeyPatch) -> None:+ barrier = Barrier(2)+ requests: list[httpx.Request] = []+ targets = {+ "https://first.test/s/one/v1/projects?limit=1": "first-key",+ "https://second.test/reverse/v1/projects?limit=1": "second-key",+ }++ def handle(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ assert request.method == "GET"+ assert request.headers["Authorization"] == "Bearer " + targets[str(request.url)]+ barrier.wait(timeout=5)+ return httpx.Response(200, json={"data": [], "next_cursor": None})++ with httpx.Client(transport=httpx.MockTransport(handle), trust_env=False) as client:+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _ssl_verify_enabled: client)+ configs = [+ PhoenixConfig(endpoint="https://first.test/s/one", api_key="first-key", project="one"),+ PhoenixConfig(endpoint="https://second.test/reverse", api_key="second-key", project="two"),+ ]+ with ThreadPoolExecutor(max_workers=2) as pool:+ assert [item.project for item in pool.map(verify_configuration, configs)] == ["one", "two"]+ assert sorted(str(request.url) for request in requests) == sorted(targets)+++def test_real_proxy_client_has_no_ambient_auth_or_cookie_fallback(monkeypatch: pytest.MonkeyPatch) -> None:+ requests: list[httpx.Request] = []++ def handle(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ assert str(request.url) == "https://host.test/s/space/v1/projects?limit=1"+ assert request.headers["Cookie"] == ""+ return httpx.Response(+ 200, headers={"Set-Cookie": "session=vendor; Path=/"}, json={"data": [], "next_cursor": None}+ )++ with httpx.Client(+ transport=httpx.MockTransport(handle),+ auth=httpx.BasicAuth("ambient", "secret"),+ cookies={"session": "ambient"},+ trust_env=False,+ ) as client:+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _ssl_verify_enabled: client)+ verify_configuration(PhoenixConfig(endpoint="https://host.test/s/space", api_key="explicit-key"))+ verify_configuration(PhoenixConfig(endpoint="https://host.test/s/space", api_key=None))+ assert requests[0].headers["Authorization"] == "Bearer explicit-key"+ assert "Authorization" not in requests[1].headers+ assert len(requests) == 2diff --git a/api/tests/unit_tests/services/tracing/test_phoenix_export_auth.py b/api/tests/unit_tests/services/tracing/test_phoenix_export_auth.pynew file mode 100644index 00000000000..ed5f8c1d3ac--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_phoenix_export_auth.py@@ -0,0 +1,142 @@+"""Exercise anonymous Phoenix auth at the HTTP/1.1 and legacy SDK boundaries."""++from collections.abc import Iterator+from datetime import UTC, datetime, timedelta++import httpcore+import httpx+import pytest+import requests+from dify_trace_arize_phoenix.arize_phoenix_trace import setup_tracer+from dify_trace_arize_phoenix.config import PhoenixConfig+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest++from core.helper import ssrf_proxy+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.provider_dispatch import export_trace+++def trace() -> CanonicalTrace:+ start = datetime(2026, 10, 20, tzinfo=UTC)+ return CanonicalTrace(+ trace_id="1" * 32,+ session_id="phoenix-auth-session",+ root_span_id="2" * 16,+ spans=(+ CanonicalSpan(+ id="2" * 16,+ parent_id=None,+ name="Phoenix auth regression",+ kind=CanonicalSpanKind.AGENT,+ start_time=start,+ end_time=start + timedelta(seconds=1),+ status=CanonicalSpanStatus.OK,+ ),+ ),+ )+++def http11_wire(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[bytes]]:+ """Keep real HTTPX/httpcore/h11 serialization, replacing only socket I/O."""+ chunks: list[bytes] = []++ def write(_stream: httpcore.MockStream, buffer: bytes, timeout: float | None = None) -> None:+ del timeout+ chunks.append(buffer)++ monkeypatch.setattr(httpcore.MockStream, "write", write)+ transport = httpx.HTTPTransport()+ transport.close()+ transport._pool = httpcore.ConnectionPool(+ network_backend=httpcore.MockBackend([b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"])+ )+ with httpx.Client(transport=transport, trust_env=False) as client:+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _verify: client)+ yield chunks+++def test_managed_exports_serialize_optional_auth_without_cross_destination_credentials(+ monkeypatch: pytest.MonkeyPatch, trace: CanonicalTrace, http11_wire: list[bytes]+) -> None:+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer%20ambient-general,api_key=ambient-general")+ monkeypatch.setenv(+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer%20ambient-traces,api_key=ambient-traces"+ )++ for index, key in enumerate(("first-synthetic-key", "", None, "second-synthetic-key", None, "")):+ previous_chunks = len(http11_wire)+ result = export_trace(+ "phoenix", {"endpoint": "http://phoenix.test/proxy", "api_key": key, "project": f"project-{index}"}, trace+ )+ assert result.status == "accepted"+ assert result.trace_id == trace.trace_id+ head, separator, body = b"".join(http11_wire[previous_chunks:]).partition(b"\r\n\r\n")+ assert separator+ lines = head.split(b"\r\n")+ assert lines[0] == b"POST /proxy/v1/traces HTTP/1.1"+ header_values: list[tuple[bytes, bytes]] = []+ for line in lines[1:]:+ name, value = line.split(b": ", 1)+ header_values.append((name, value))+ headers = httpx.Headers(header_values)+ assert headers["Content-Type"] == "application/x-protobuf"+ if key:+ assert headers["api_key"] == key+ assert headers["authorization"] == f"Bearer {key}"+ else:+ assert "api_key" not in headers+ assert "authorization" not in headers+ payload = ExportTraceServiceRequest.FromString(body)+ resource = payload.resource_spans[0]+ assert {item.key: item.value.string_value for item in resource.resource.attributes}[+ "openinference.project.name"+ ] == f"project-{index}"+ assert resource.scope_spans[0].spans[0].trace_id.hex() == trace.trace_id+++def test_legacy_sdk_prepares_anonymous_requests_without_ambient_or_previous_auth(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "authorization=Bearer%20ambient-general,api_key=ambient-general")+ monkeypatch.setenv(+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer%20ambient-traces,api_key=ambient-traces"+ )+ monkeypatch.delenv("OTEL_EXPORTER_OTLP_COMPRESSION", raising=False)+ monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", raising=False)+ prepared: list[requests.PreparedRequest] = []++ def send(_session: requests.Session, request: requests.PreparedRequest, **_kwargs: object) -> requests.Response:+ prepared.append(request)+ response = requests.Response()+ response.status_code = 200+ response._content = b""+ response.request = request+ return response++ monkeypatch.setattr(requests.Session, "send", send)+ for index, key in enumerate(("first-synthetic-key", "", None, "second-synthetic-key", None, "")):+ tracer, processor = setup_tracer(+ PhoenixConfig(endpoint="http://phoenix.test/proxy", api_key=key, project=f"project-{index}")+ )+ try:+ tracer.start_span("Legacy Phoenix auth regression").end()+ finally:+ processor.shutdown()+ assert len(prepared) == index + 1+ request = prepared[-1]+ assert request.method == "POST"+ assert request.url == "http://phoenix.test/proxy/v1/traces"+ assert request.headers["Content-Type"] == "application/x-protobuf"+ if key:+ assert request.headers["api_key"] == key+ assert request.headers["authorization"] == f"Bearer {key}"+ else:+ assert "api_key" not in request.headers+ assert "authorization" not in request.headers+ assert isinstance(request.body, bytes)+ payload = ExportTraceServiceRequest.FromString(request.body)+ assert {item.key: item.value.string_value for item in payload.resource_spans[0].resource.attributes}[+ "openinference.project.name"+ ] == f"project-{index}"diff --git a/api/tests/unit_tests/services/tracing/test_provider_dispatch.py b/api/tests/unit_tests/services/tracing/test_provider_dispatch.pynew file mode 100644index 00000000000..32ffd59b8e7--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_provider_dispatch.py@@ -0,0 +1,715 @@+"""Exercise real vendor serializers and OTLP stubs at their network boundary."""++import json+from collections.abc import Callable, Mapping, Sequence+from datetime import UTC, datetime, timedelta+from typing import Never, NotRequired, TypedDict, cast+from uuid import UUID++import httpx+import pytest+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (+ ExportTraceServiceRequest,+ ExportTraceServiceResponse,+)+from opentelemetry.proto.trace.v1.trace_pb2 import Span++from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.unified_trace.entities import CanonicalSpan, CanonicalSpanKind, CanonicalSpanStatus, CanonicalTrace+from services.tracing.normalization import remap_outer_trace+from services.tracing.provider_dispatch import (+ TraceExportRejectedError,+ TraceExportUncertainError,+ export_trace,+ provider_capabilities,+)+++class HTTPArguments(TypedDict):+ content: bytes+ headers: Mapping[str, str]+ max_retries: NotRequired[int]+ follow_redirects: NotRequired[bool]+++type HTTPCall = tuple[str, str, HTTPArguments]+++def trace() -> CanonicalTrace:+ start = datetime(2026, 10, 15, 1, tzinfo=UTC)+ return CanonicalTrace(+ trace_id="1" * 32,+ session_id="conversation-1",+ root_span_id="1" * 16,+ spans=(+ CanonicalSpan(+ id="1" * 16,+ parent_id=None,+ name="Agent turn",+ kind=CanonicalSpanKind.AGENT,+ start_time=start,+ end_time=start + timedelta(seconds=3),+ status=CanonicalSpanStatus.OK,+ inputs={"question": "hi"},+ outputs={"answer": "done"},+ metadata={"app_id": "app-one", "tenant_id": "tenant-one", "total_tokens": 12, "total_price": "0.03"},+ ),+ CanonicalSpan(+ id="2" * 16,+ parent_id="1" * 16,+ name="model call",+ kind=CanonicalSpanKind.LLM,+ start_time=start,+ end_time=start + timedelta(seconds=1),+ status=CanonicalSpanStatus.OK,+ inputs=[{"role": "user", "content": "hi"}],+ outputs="done",+ metadata={+ "model": "gpt-4o-mini",+ "model_provider": "openai",+ "prompt_tokens": 8,+ "completion_tokens": 4,+ "total_tokens": 12,+ "total_price": "0.03",+ "currency": "USD",+ "ttft": 0.2,+ },+ ),+ CanonicalSpan(+ id="3" * 16,+ parent_id="1" * 16,+ name="search",+ kind=CanonicalSpanKind.TOOL,+ start_time=start + timedelta(seconds=1),+ end_time=start + timedelta(seconds=2),+ status=CanonicalSpanStatus.ERROR,+ inputs={"q": "x"},+ outputs=None,+ error="tool failed",+ ),+ ),+ )+++def http_calls(monkeypatch: pytest.MonkeyPatch) -> list[HTTPCall]:+ from core.helper import ssrf_proxy++ calls: list[HTTPCall] = []++ def send(method: str, url: str, **kwargs: object) -> httpx.Response:+ calls.append((method, url, cast(HTTPArguments, kwargs)))+ return httpx.Response(200, content=b"", request=httpx.Request(method, url))++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ return calls+++def spans_from_call(call: HTTPCall) -> list[Span]:+ request = ExportTraceServiceRequest.FromString(call[2]["content"])+ return list(request.resource_spans[0].scope_spans[0].spans)+++def attributes(span: Span) -> dict[str, object]:+ result: dict[str, object] = {}+ for item in span.attributes:+ kind = item.value.WhichOneof("value")+ assert kind is not None+ result[item.key] = {+ "string_value": item.value.string_value,+ "bool_value": item.value.bool_value,+ "int_value": item.value.int_value,+ "double_value": item.value.double_value,+ "array_value": item.value.array_value,+ "kvlist_value": item.value.kvlist_value,+ "bytes_value": item.value.bytes_value,+ }[kind]+ return result+++def test_langfuse_v4_tree_and_native_usage(trace: CanonicalTrace, http_calls: list[HTTPCall]) -> None:+ config = {+ "host": "https://us.cloud.langfuse.com",+ "public_key": "pk-test",+ "secret_key": "sk-test",+ "project_url": "https://us.cloud.langfuse.com/project/example",+ }+ result = export_trace("langfuse", config, trace)+ assert result.status == "accepted"+ assert result.trace_id == trace.trace_id+ assert len(http_calls) == 1+ _, url, args = http_calls[0]+ assert url.endswith("/api/public/otel/v1/traces")+ assert args["headers"]["x-langfuse-ingestion-version"] == "4"+ assert args["max_retries"] == 0+ assert args["follow_redirects"] is False+ spans = spans_from_call(http_calls[0])+ assert [span.span_id.hex() for span in spans] == [s.id for s in trace.spans]+ assert spans[1].parent_span_id == spans[0].span_id+ assert spans[2].status.code == 2+ root, llm = attributes(spans[0]), attributes(spans[1])+ assert root["langfuse.observation.type"] == "agent"+ assert "langfuse.observation.usage_details" not in root+ usage_details, cost_details = llm["langfuse.observation.usage_details"], llm["langfuse.observation.cost_details"]+ assert isinstance(usage_details, str)+ assert isinstance(cost_details, str)+ assert json.loads(usage_details) == {"input": 8, "output": 4, "total": 12}+ assert json.loads(cost_details) == {"total": 0.03}+ assert all(attributes(span)["langfuse.session.id"] == trace.session_id for span in spans)+ assert result.span_urls[trace.spans[1].id].endswith(trace.spans[1].id)+++ ("provider", "config", "suffix", "kind_key", "kind"),+ [+ (+ "phoenix",+ {"endpoint": "https://phoenix.test/proxy", "api_key": "key", "project": "one"},+ "/proxy/v1/traces",+ "openinference.span.kind",+ "AGENT",+ ),+ (+ "aliyun",+ {"endpoint": "https://region.log.aliyuncs.com", "license_key": "license", "app_name": "one"},+ "/adapt_license/api/v1/traces",+ "gen_ai.span.kind",+ "AGENT",+ ),+ ],+)+def test_other_http_providers_have_native_semantics(+ trace: CanonicalTrace,+ http_calls: list[HTTPCall],+ provider: str,+ config: dict[str, str],+ suffix: str,+ kind_key: str,+ kind: str,+) -> None:+ result = export_trace(provider, config, trace)+ assert result.status == "accepted"+ assert http_calls[0][1].endswith(suffix)+ spans = spans_from_call(http_calls[0])+ assert attributes(spans[0])[kind_key] == kind+ token_key = "llm.token_count.prompt" if provider == "phoenix" else "gen_ai.usage.input_tokens"+ assert attributes(spans[1])[token_key] == 8+ assert token_key not in attributes(spans[0])+++ ("endpoint_url", "suffix"),+ [+ ("https://log.aliyuncs.com", "api/v1/traces"),+ ("https://cn-hangzhou.log.aliyuncs.com", "api/v1/traces"),+ ("https://project.cn-heyuan.log.aliyuncs.com/apm/trace/opentelemetry", "api/v1/traces"),+ ("https://project.cn-heyuan-intranet.log.aliyuncs.com/apm/trace/opentelemetry", "api/v1/traces"),+ ("https://cn-shanghai-share.log.aliyuncs.com", "api/v1/traces"),+ ("https://cn-beijing.dualstack.log.aliyuncs.com", "api/v1/traces"),+ ("https://cn-hangzhou.log.aliyuncs.com.:443", "api/v1/traces"),+ ("https://tracing-analysis-dc-hz.aliyuncs.com", "api/otlp/traces"),+ ("https://collector.example.test:4318/custom-prefix", "api/otlp/traces"),+ ("https://evil-log.aliyuncs.com.evil.test", "api/otlp/traces"),+ ("https://log.aliyuncs.com.evil.test", "api/otlp/traces"),+ ("https://evil-log.aliyuncs.com", "api/otlp/traces"),+ ("https://collector.example.test/log.aliyuncs.com", "api/otlp/traces"),+ ],+)+def test_aliyun_endpoint_family_uses_dns_labels_and_preserves_custom_destinations(+ trace: CanonicalTrace, http_calls: list[HTTPCall], endpoint_url: str, suffix: str+) -> None:+ result = export_trace(+ "aliyun", {"endpoint": endpoint_url, "license_key": "synthetic-license", "app_name": "test-app"}, trace+ )+ assert result.status == "accepted"+ assert len(http_calls) == 1+ assert http_calls[0][1] == f"{endpoint_url}/adapt_synthetic-license/{suffix}"+ spans = spans_from_call(http_calls[0])+ assert [span.span_id.hex() for span in spans] == [span.id for span in trace.spans]+ assert spans[1].parent_span_id == spans[0].span_id+++def test_aliyun_raw_license_is_encoded_once_at_the_real_otlp_boundary(+ trace: CanonicalTrace, http_calls: list[HTTPCall]+) -> None:+ result = export_trace(+ "aliyun",+ {"endpoint": "https://COLLECTOR.test:8443/prefix/", "license_key": "raw%2F@a+b/c", "app_name": "test_app"},+ trace,+ )+ assert result.status == "accepted"+ assert len(http_calls) == 1+ assert http_calls[0][1] == "https://collector.test:8443/prefix/adapt_raw%252F@a%2Bb%2Fc/api/otlp/traces"+ assert [span.span_id.hex() for span in spans_from_call(http_calls[0])] == [span.id for span in trace.spans]+++def test_aliyun_complete_token_url_is_rejected_without_any_ingestion(+ trace: CanonicalTrace, http_calls: list[HTTPCall]+) -> None:+ with pytest.raises(TraceExportRejectedError, match="Aliyun tracing configuration is invalid"):+ export_trace(+ "aliyun",+ {"endpoint": "https://collector.test/adapt_private/api/otlp/traces", "license_key": "private"},+ trace,+ )+ assert http_calls == []+++ ("provider", "config", "header"),+ [+ (+ "arize",+ {"endpoint": "https://otlp.arize.com", "api_key": "key", "space_id": "space", "project": "one"},+ ("space_id", "space"),+ ),+ (+ "tencent",+ {"endpoint": "https://apm.test:4317", "token": "token", "service_name": "one"},+ ("authorization", "Bearer token"),+ ),+ ],+)+def test_grpc_real_stub_serialization(+ trace: CanonicalTrace,+ monkeypatch: pytest.MonkeyPatch,+ provider: str,+ config: dict[str, str],+ header: tuple[str, str],+) -> None:+ calls: list[tuple[ExportTraceServiceRequest, Sequence[tuple[str, str]]]] = []++ class Channel:+ def unary_unary(+ self,+ method: str,+ request_serializer: Callable[[ExportTraceServiceRequest], bytes],+ response_deserializer: Callable[[bytes], ExportTraceServiceResponse],+ **_kwargs: object,+ ) -> Callable[..., ExportTraceServiceResponse]:+ assert method == "/opentelemetry.proto.collector.trace.v1.TraceService/Export"++ def invoke(+ payload: ExportTraceServiceRequest, metadata: Sequence[tuple[str, str]], timeout: float+ ) -> ExportTraceServiceResponse:+ assert timeout == 30+ calls.append((ExportTraceServiceRequest.FromString(request_serializer(payload)), metadata))+ return response_deserializer(ExportTraceServiceResponse().SerializeToString())++ return invoke++ def close(self) -> None:+ pass++ monkeypatch.setattr("grpc.secure_channel", lambda *_args, **_kwargs: Channel())+ result = export_trace(provider, config, trace)+ assert result.status == "accepted"+ assert header in calls[0][1]+ spans = calls[0][0].resource_spans[0].scope_spans[0].spans+ assert spans[1].parent_span_id == spans[0].span_id+++def test_langsmith_real_sdk_uses_isolated_sync_runs(trace: CanonicalTrace, http_calls: list[HTTPCall]) -> None:+ result = export_trace(+ "langsmith", {"api_key": "ls-test", "endpoint": "https://langsmith.test", "project": "one"}, trace+ )+ writes = [json.loads(call[2]["content"]) for call in http_calls if call[0] == "POST"]+ assert len(writes) == 3+ root, llm, tool = writes+ assert root["run_type"] == "chain"+ assert llm["run_type"] == "llm"+ assert llm["parent_run_id"] == root["id"]+ assert llm["dotted_order"].startswith(root["dotted_order"])+ assert llm["input_tokens"] == 8+ assert llm["output_tokens"] == 4+ assert llm["total_cost"] == 0.03+ assert "input_tokens" not in root+ assert all(write["extra"]["metadata"]["session_id"] == trace.session_id for write in writes)+ assert tool["error"] == "tool failed"+ assert UUID(result.trace_id)+++def test_opik_real_sdk_preserves_tree_thread_and_native_usage(+ trace: CanonicalTrace, http_calls: list[HTTPCall]+) -> None:+ result = export_trace(+ "opik",+ {"api_key": "opik-test", "url": "https://opik.test/api", "workspace": "org-one", "project": "one"},+ trace,+ )+ assert len(http_calls) == 2+ root = json.loads(http_calls[0][2]["content"])+ batch = json.loads(http_calls[1][2]["content"])+ spans = batch["spans"]+ assert UUID(result.trace_id).version == 7+ assert root["thread_id"] == trace.session_id+ assert spans[1]["parent_span_id"] == spans[0]["id"]+ assert spans[1]["usage"]["prompt_tokens"] == 8+ assert spans[1]["total_estimated_cost"] == 0.03+ assert not spans[0].get("usage")+ assert spans[2]["error_info"]["message"] == "tool failed"+ assert http_calls[0][2]["headers"]["comet-workspace"] == "org-one"+++ ("value", "wire_value"),+ [+ (None, None),+ ("323", "323"),+ ({"result": 323}, {"result": 323}),+ ([{"result": 323}], [{"result": 323}]),+ ([], []),+ (323, "323"),+ (0, "0"),+ (-5, "-5"),+ (3.5, "3.5"),+ (True, "true"),+ (False, "false"),+ ([1, 2], "[1, 2]"),+ (["one", "two"], '["one", "two"]'),+ ([{"result": 1}, "text"], '[{"result": 1}, "text"]'),+ ([[1], [2]], "[[1], [2]]"),+ ],+)+def test_opik_real_sdk_preserves_json_content_across_trace_and_span_boundaries(+ trace: CanonicalTrace, http_calls: list[HTTPCall], value: object, wire_value: object+) -> None:+ content_trace = trace.model_copy(+ update={"spans": tuple(span.model_copy(update={"inputs": value, "outputs": value}) for span in trace.spans)}+ )+ result = export_trace(+ "opik",+ {"api_key": "synthetic-key", "url": "https://opik.test/api", "workspace": "team", "project": "one"},+ content_trace,+ )+ assert result.status == "accepted"+ assert len(http_calls) == 2+ root = json.loads(http_calls[0][2]["content"])+ spans = json.loads(http_calls[1][2]["content"])["spans"]+ for payload in [root, *spans]:+ assert payload.get("input") == wire_value+ assert payload.get("output") == wire_value+ assert spans[1]["parent_span_id"] == spans[0]["id"]+ assert spans[2]["parent_span_id"] == spans[0]["id"]+ assert root["thread_id"] == trace.session_id+ assert spans[1]["usage"]["total_tokens"] == 12+ assert not spans[0].get("usage")+ assert not spans[2].get("usage")++[email protected]("capture_content", [True, False])[email protected]("root_error", [True, False])+def test_opik_trace_error_matches_sanitized_root_and_not_handled_tool_failure(+ trace: CanonicalTrace, http_calls: list[HTTPCall], capture_content: bool, root_error: bool+) -> None:+ root = trace.spans[0].model_copy(+ update={+ "status": CanonicalSpanStatus.ERROR if root_error else CanonicalSpanStatus.OK,+ "error": "private-model-error" if root_error else None,+ "inputs": {"query": "private-error-input"},+ }+ )+ source = trace.model_copy(update={"spans": (root, *trace.spans[1:])})+ canonical = remap_outer_trace(source, trace_id=trace.trace_id, capture_content=capture_content, agent_message=True)+ export_trace(+ "opik",+ {"api_key": "synthetic-key", "url": "https://opik.test/api", "workspace": "team", "project": "one"},+ canonical,+ )+ root_payload = json.loads(http_calls[0][2]["content"])+ spans = json.loads(http_calls[1][2]["content"])["spans"]+ assert spans[2]["error_info"] is not None+ if root_error:+ expected = {+ "exception_type": "DifyExecutionError",+ "message": "private-model-error" if capture_content else "Application execution failed",+ "traceback": "",+ }+ assert root_payload["error_info"] == spans[0]["error_info"] == expected+ else:+ assert root_payload.get("error_info") is None+ assert spans[0].get("error_info") is None+ if not capture_content:+ assert all(b"private-model-error" not in call[2]["content"] for call in http_calls)+ assert all(b"private-error-input" not in call[2]["content"] for call in http_calls)+++ ("status", "error"),+ [(401, TraceExportRejectedError), (429, RetryableTraceDispatchError), (503, RetryableTraceDispatchError)],+)+def test_explicit_http_rejections_are_classified_without_response_body(+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch, status: int, error: type[Exception]+) -> None:+ from core.helper import ssrf_proxy++ monkeypatch.setattr(+ ssrf_proxy,+ "make_request",+ lambda method, url, **_kwargs: httpx.Response(status, text="secret-token", request=httpx.Request(method, url)),+ )+ with pytest.raises(error) as caught:+ export_trace(+ "langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "secret-token"}, trace+ )+ assert "secret-token" not in str(caught.value)+++def test_partial_otlp_acceptance_requires_reconciliation(+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch+) -> None:+ from core.helper import ssrf_proxy++ response = ExportTraceServiceResponse()+ response.partial_success.rejected_spans = 1+ monkeypatch.setattr(+ ssrf_proxy,+ "make_request",+ lambda method, url, **_kwargs: httpx.Response(+ 200, content=response.SerializeToString(), request=httpx.Request(method, url)+ ),+ )+ with pytest.raises(TraceExportUncertainError):+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)+++def _langfuse_queue_job_ack() -> dict[str, object]:+ # Synthetic values only. Real acknowledgements contain auth context and must+ # never be logged or persisted as acceptance-test evidence.+ return {+ "name": "otel-ingestion-job",+ "data": {+ "id": "11111111-2222-4333-8444-555555555555",+ "timestamp": "2026-10-15T01:00:00.000Z",+ "name": "otel-ingestion-job",+ "payload": {+ "data": {"fileKey": "otel/test/payload.json", "publicKey": "never-log-public-key"},+ "authCheck": {+ "validKey": True,+ "scope": {"projectId": "never-log-project", "accessLevel": "project", "orgId": "test-org"},+ },+ "sdkName": "test-sdk",+ "ingestionVersion": "4",+ },+ },+ }+++def _replace_ack_value(ack: dict[str, object], path: tuple[str, ...], value: object) -> None:+ for field in path[:-1]:+ nested = ack[field]+ assert isinstance(nested, dict)+ ack = cast(dict[str, object], nested)+ ack[path[-1]] = value+++def test_langfuse_v4_queue_ack_confirms_ingestion_acceptance_only(+ trace: CanonicalTrace,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ capsys: pytest.CaptureFixture[str],+) -> None:+ from core.helper import ssrf_proxy++ monkeypatch.setattr(+ ssrf_proxy,+ "make_request",+ lambda method, url, **_kwargs: httpx.Response(+ 200, json=_langfuse_queue_job_ack(), request=httpx.Request(method, url)+ ),+ )+ result = export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)+ assert result.status == "accepted"+ assert result.trace_id == trace.trace_id+ assert result.trace_url is None+ captured = capsys.readouterr()+ evidence = caplog.text + captured.out + captured.err + repr(result)+ assert "never-log-public-key" not in evidence+ assert "never-log-project" not in evidence+ assert "authCheck" not in evidence+++ "scenario",+ [+ "wrong_name",+ "wrong_nested_name",+ "invalid_id",+ "invalid_timestamp",+ "invalid_auth",+ "missing_scope",+ "missing_file",+ "partial_success",+ "error",+ "failed_job",+ "unknown",+ "null",+ "array",+ ],+)+def test_langfuse_ambiguous_json_ack_remains_uncertain(+ trace: CanonicalTrace,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ capsys: pytest.CaptureFixture[str],+ scenario: str,+) -> None:+ from core.helper import ssrf_proxy++ ack = _langfuse_queue_job_ack()+ body: object = ack+ if scenario == "wrong_name":+ ack["name"] = "different-job"+ elif scenario == "wrong_nested_name":+ _replace_ack_value(ack, ("data", "name"), "different-job")+ elif scenario == "invalid_id":+ _replace_ack_value(ack, ("data", "id"), "not-a-uuid")+ elif scenario == "invalid_timestamp":+ _replace_ack_value(ack, ("data", "timestamp"), "2026-10-15T01:00:00")+ elif scenario == "invalid_auth":+ _replace_ack_value(ack, ("data", "payload", "authCheck", "validKey"), False)+ elif scenario == "missing_scope":+ _replace_ack_value(ack, ("data", "payload", "authCheck", "scope"), dict[str, object]())+ elif scenario == "missing_file":+ _replace_ack_value(ack, ("data", "payload", "data", "fileKey"), "")+ elif scenario == "partial_success":+ ack["partialSuccess"] = {"rejectedSpans": "1"}+ elif scenario == "error":+ ack["error"] = "never-log-public-key"+ elif scenario == "failed_job":+ ack["failedReason"] = "never-log-public-key"+ elif scenario == "unknown":+ body = {"accepted": True, "publicKey": "never-log-public-key"}+ elif scenario == "null":+ body = None+ elif scenario == "array":+ body = list[object]()+ monkeypatch.setattr(+ ssrf_proxy,+ "make_request",+ lambda method, url, **_kwargs: httpx.Response(+ 200,+ content=json.dumps(body).encode(),+ headers={"Content-Type": "application/json"},+ request=httpx.Request(method, url),+ ),+ )+ with pytest.raises(TraceExportUncertainError) as caught:+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)+ captured = capsys.readouterr()+ evidence = caplog.text + captured.out + captured.err + str(caught.value)+ assert "never-log-public-key" not in evidence+ assert "never-log-project" not in evidence+ assert "authCheck" not in evidence+++ ("provider", "version", "status"), [("phoenix", "4", 200), ("langfuse", "3", 200), ("langfuse", "4", 202)]+)+def test_langfuse_queue_ack_is_not_a_generic_otlp_success(+ monkeypatch: pytest.MonkeyPatch, provider: str, version: str, status: int+) -> None:+ from core.helper import ssrf_proxy+ from services.tracing.otlp_export import export_http++ monkeypatch.setattr(+ ssrf_proxy,+ "make_request",+ lambda method, url, **_kwargs: httpx.Response(+ status, json=_langfuse_queue_job_ack(), request=httpx.Request(method, url)+ ),+ )+ with pytest.raises(TraceExportUncertainError):+ export_http(+ provider,+ "https://collector.test/v1/traces",+ {"x-langfuse-ingestion-version": version},+ ExportTraceServiceRequest(),+ )++[email protected]("body", [{}, {"partialSuccess": {}}, {"partialSuccess": {"rejectedSpans": "1"}}])+def test_langfuse_standard_otlp_json_ack_remains_supported(+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch, body: dict[str, dict[str, str]]+) -> None:+ from core.helper import ssrf_proxy++ monkeypatch.setattr(+ ssrf_proxy,+ "make_request",+ lambda method, url, **_kwargs: httpx.Response(200, json=body, request=httpx.Request(method, url)),+ )+ config = {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}+ if body.get("partialSuccess", {}).get("rejectedSpans"):+ with pytest.raises(TraceExportUncertainError):+ export_trace("langfuse", config, trace)+ else:+ assert export_trace("langfuse", config, trace).status == "accepted"+++def test_lost_http_ack_is_uncertain_not_rejected(trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch) -> None:+ from core.helper import ssrf_proxy++ def timeout(*_args: object, **_kwargs: object) -> Never:+ raise httpx.ReadTimeout("response lost")++ monkeypatch.setattr(ssrf_proxy, "make_request", timeout)+ with pytest.raises(TraceExportUncertainError):+ export_trace("langfuse", {"host": "https://langfuse.test", "public_key": "pk", "secret_key": "sk"}, trace)++[email protected]("provider", ["langsmith", "opik"])[email protected]("accepted_prefix", [False, True])+def test_native_sdk_preserves_first_failure_or_partial_acceptance(+ trace: CanonicalTrace, monkeypatch: pytest.MonkeyPatch, provider: str, accepted_prefix: bool+) -> None:+ from core.helper import ssrf_proxy++ calls: list[str] = []++ def send(method: str, url: str, **_kwargs: object) -> httpx.Response:+ calls.append(url)+ status = 200 if accepted_prefix and len(calls) == 1 else 429+ return httpx.Response(status, content=b"", request=httpx.Request(method, url))++ monkeypatch.setattr(ssrf_proxy, "make_request", send)+ config = {"api_key": "test-key", "project": "one", "endpoint": "https://smith.test", "url": "https://opik.test/api"}+ expected = TraceExportUncertainError if accepted_prefix else RetryableTraceDispatchError+ with pytest.raises(expected) as caught:+ export_trace(provider, config, trace)+ assert isinstance(caught.value, TraceExportUncertainError) is accepted_prefix+ assert len(calls) == (2 if accepted_prefix else 1)+++def test_http_exports_keep_destination_credentials_local(trace: CanonicalTrace, http_calls: list[HTTPCall]) -> None:+ for project in ("a", "b"):+ export_trace(+ "langfuse", {"host": f"https://{project}.test", "public_key": project, "secret_key": project}, trace+ )+ assert http_calls[0][1].startswith("https://a.test/")+ assert http_calls[1][1].startswith("https://b.test/")+ assert http_calls[0][2]["headers"]["Authorization"] != http_calls[1][2]["headers"]["Authorization"]+ assert http_calls[0][2]["content"] == http_calls[1][2]["content"]+++ "provider",+ ["langfuse", "langsmith", "phoenix", "arize", "opik", "weave", "mlflow", "databricks", "aliyun", "tencent"],+)+def test_capabilities_state_implemented_contract_not_platform_validation(provider: str) -> None:+ result = provider_capabilities(provider)+ assert result["agent_spans"] is True+ assert result["export_ack"] == "accepted"+ assert "platform_validated" not in result+++def test_rejects_unfinished_and_unknown_provider_before_sending(+ trace: CanonicalTrace, http_calls: list[HTTPCall]+) -> None:+ unfinished = trace.model_copy(update={"spans": (trace.spans[0].model_copy(update={"end_time": None}),)})+ with pytest.raises(TraceExportRejectedError):+ export_trace("langfuse", {}, unfinished)+ with pytest.raises(TraceExportRejectedError):+ export_trace("unknown", {}, trace)+ assert not http_callsdiff --git a/api/tests/unit_tests/services/tracing/test_request_log_protection.py b/api/tests/unit_tests/services/tracing/test_request_log_protection.pynew file mode 100644index 00000000000..ebff6db3cac--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_request_log_protection.py@@ -0,0 +1,292 @@+"""Real HTTPX/SSRF diagnostics with synthetic credential-bearing destinations."""++import logging+import traceback+from collections.abc import Callable, Iterator+from concurrent.futures import ThreadPoolExecutor+from contextlib import ExitStack+from threading import Barrier, Event+from urllib.parse import quote++import httpx+import pytest+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value++from core.helper import ssrf_proxy+from core.logging.structured_formatter import StructuredJSONFormatter+from core.ops.exceptions import RetryableTraceDispatchError+from services.tracing.provider_common import request+from services.tracing.provider_dispatch import TraceExportRejectedError, TraceExportUncertainError+from services.tracing.request_log_protection import SAFE_HTTP_LOG_MESSAGE, protect_http_request_logs++SYNTHETIC_KEY = "SYNTHETIC-NOT-A-REAL-LICENSE/+?=测试"+ENCODED_KEY = quote(SYNTHETIC_KEY, safe="")+PRIVATE_URL = f"https://collector.test/prefix/adapt_{ENCODED_KEY}/api/otlp/traces"+type HTTPHandler = Callable[[httpx.Request], httpx.Response]+type InstallClient = Callable[[HTTPHandler], None]++[email protected](autouse=True)+def enable_http_diagnostics(caplog: pytest.LogCaptureFixture) -> Iterator[None]:+ """Exercise INFO records even if an imported SDK changed HTTPX's level."""+ with caplog.at_level(logging.INFO, logger="httpx"):+ yield+++def install_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[InstallClient]:+ with ExitStack() as stack:++ def install(handler: HTTPHandler) -> None:+ client = stack.enter_context(httpx.Client(transport=httpx.MockTransport(handler)))++ def get_client(ssl_verify_enabled: bool) -> httpx.Client:+ assert isinstance(ssl_verify_enabled, bool)+ return client++ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", get_client)++ yield install+++def evidence(caplog: pytest.LogCaptureFixture) -> str:+ formatter = StructuredJSONFormatter()+ return caplog.text + "".join(repr(record.__dict__) + formatter.format(record) for record in caplog.records)++[email protected]("outcome", [200, 503, 302, "squid", "timeout", "invalid"])[email protected]("scheme", ["http", "https"])+def test_aliyun_keeps_real_wire_url_but_protects_transport_logs_and_exception_chains(+ install_client: InstallClient, caplog: pytest.LogCaptureFixture, outcome: int | str, scheme: str+) -> None:+ caplog.set_level(logging.INFO)+ original_factory = logging.getLogRecordFactory()+ original_make_record = logging.Logger.makeRecord+ previous_suppression = get_value(_SUPPRESS_INSTRUMENTATION_KEY)+ sent: list[httpx.Request] = []+ private_url = PRIVATE_URL.replace("https://", scheme + "://", 1)++ def handler(wire: httpx.Request) -> httpx.Response:+ sent.append(wire)+ assert str(wire.url) == private_url+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True+ assert set(wire.extensions["timeout"].values()) == {30.0}+ logger = logging.getLogger("core.helper.ssrf_proxy")+ logger.warning(+ "Synthetic URL %s raw %s encoded %s",+ wire.url,+ SYNTHETIC_KEY,+ quote(ENCODED_KEY, safe=""),+ extra={"private_url": str(wire.url), "private_payload": {"license": SYNTHETIC_KEY}},+ )+ assert logging.getLogRecordFactory() is original_factory+ if outcome in {"timeout", "invalid"}:+ try:+ raise ValueError(SYNTHETIC_KEY + ENCODED_KEY)+ except ValueError as cause:+ logger.exception("Synthetic transport exception %s", wire.url)+ if outcome == "invalid":+ raise ValueError(SYNTHETIC_KEY) from cause+ raise httpx.ReadError("lost acknowledgment " + str(wire.url), request=wire) from cause+ if outcome == "squid":+ return httpx.Response(403, headers={"Server": "squid"})+ assert isinstance(outcome, int)+ return httpx.Response(outcome, headers={"Location": "https://redirect.test/"})++ install_client(handler)+ if outcome == 200:+ result = request("POST", private_url, "aliyun", content=b"synthetic")+ assert result.status_code == 200+ rendered_exception = ""+ else:+ expected = (+ TraceExportUncertainError+ if outcome == "timeout"+ else RetryableTraceDispatchError+ if outcome == 503+ else TraceExportRejectedError+ )+ with pytest.raises(expected) as caught:+ request("POST", private_url, "aliyun", content=b"synthetic")+ assert caught.value.__cause__ is None+ assert caught.value.__context__ is None+ rendered_exception = "".join(traceback.format_exception(caught.value))+ assert len(sent) == 1 # Real SSRF max_retries=0 and no redirects remain in force.+ assert logging.getLogRecordFactory() is original_factory+ assert logging.Logger.makeRecord is original_make_record+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == previous_suppression+ assert any(+ record.name == "core.helper.ssrf_proxy" and record.levelno == logging.WARNING for record in caplog.records+ )+ if outcome in {200, 503, 302, "squid"}:+ assert any(record.name == "httpx" and record.levelno == logging.INFO for record in caplog.records)+ assert all(record.getMessage() == SAFE_HTTP_LOG_MESSAGE for record in caplog.records)+ rendered = evidence(caplog) + rendered_exception+ for private in [SYNTHETIC_KEY, ENCODED_KEY, quote(ENCODED_KEY, safe="")]:+ assert private.casefold() not in rendered.casefold()+ logging.getLogger("httpx").info("AFTER-REQUEST-PUBLIC-LOG")+ assert "AFTER-REQUEST-PUBLIC-LOG" in caplog.text+++def test_overlapping_private_requests_keep_public_request_logs_and_restore_factory(+ install_client: InstallClient, caplog: pytest.LogCaptureFixture+) -> None:+ caplog.set_level(logging.INFO)+ original_factory = logging.getLogRecordFactory()+ original_make_record = logging.Logger.makeRecord+ levels = (logging.root.level, logging.getLogger("httpx").level, logging.getLogger("core.helper.ssrf_proxy").level)+ barrier = Barrier(3)+ first_finished = Event()+ keys = {label: f"SYNTHETIC-PRIVATE-{label}/+?" for label in ["a", "b"]}+ urls = {label: f"https://{label}.test/adapt_{quote(key, safe='')}/api/otlp/traces" for label, key in keys.items()}+ urls["public"] = "https://public.test/PUBLIC-DESTINATION-VISIBLE"++ def handler(wire: httpx.Request) -> httpx.Response:+ label = wire.url.host.split(".")[0]+ barrier.wait(timeout=5)+ if label != "a":+ assert first_finished.wait(timeout=5)+ logging.getLogger("core.helper.ssrf_proxy").warning("diagnostic URL %s", wire.url)+ return httpx.Response(200)++ install_client(handler)++ def run(label: str) -> int:+ result = request("POST", urls[label], "aliyun" if label in keys else "weave")+ if label == "a":+ first_finished.set()+ return result.status_code++ with ThreadPoolExecutor(max_workers=3) as pool:+ assert list(pool.map(run, ["a", "b", "public"])) == [200, 200, 200]+ rendered = evidence(caplog)+ for key in keys.values():+ assert key not in rendered+ assert quote(key, safe="") not in rendered+ assert "PUBLIC-DESTINATION-VISIBLE" in rendered+ assert logging.getLogRecordFactory() is original_factory+ assert logging.Logger.makeRecord is original_make_record+ assert levels == (+ logging.root.level,+ logging.getLogger("httpx").level,+ logging.getLogger("core.helper.ssrf_proxy").level,+ )+++def test_nested_scopes_sanitize_before_existing_factory_and_restore_public_logging(+ caplog: pytest.LogCaptureFixture,+) -> None:+ caplog.set_level(logging.INFO)+ original = logging.getLogRecordFactory()+ observed: list[str] = []++ def existing_factory(*args: object, **kwargs: object) -> logging.LogRecord:+ record = original(*args, **kwargs)+ observed.append(record.getMessage())+ return record++ logging.setLogRecordFactory(existing_factory)+ try:+ with protect_http_request_logs(True):+ assert logging.getLogRecordFactory() is existing_factory+ logging.getLogger("httpcore.connection").info("outer %s", SYNTHETIC_KEY)+ with protect_http_request_logs(False):+ logging.getLogger("httpx").info("NESTED-PUBLIC-LOG")+ with protect_http_request_logs(True):+ logging.getLogger("httpx").info("inner %s", ENCODED_KEY)+ logging.getLogger("core.helper.ssrf_proxy").info("outer restored %s", SYNTHETIC_KEY)+ assert logging.getLogRecordFactory() is existing_factory+ assert observed == [SAFE_HTTP_LOG_MESSAGE, "NESTED-PUBLIC-LOG", SAFE_HTTP_LOG_MESSAGE, SAFE_HTTP_LOG_MESSAGE]+ finally:+ logging.setLogRecordFactory(original)+++def test_non_chaining_factory_replacement_during_request_receives_no_private_fields(+ caplog: pytest.LogCaptureFixture,+) -> None:+ caplog.set_level(logging.INFO)+ original = logging.getLogRecordFactory()+ observed: list[tuple[object, ...]] = []+ try:+ with protect_http_request_logs(True):+ assert logging.getLogRecordFactory() is original++ def externally_installed(*args: object, **kwargs: object) -> logging.LogRecord:+ observed.append((args[4], args[5], args[6], args[8]))+ return original(*args, **kwargs)++ logging.setLogRecordFactory(externally_installed)+ logging.getLogger("httpx").info(+ "private %s", SYNTHETIC_KEY, extra={"url": PRIVATE_URL, "license": ENCODED_KEY}+ )+ assert logging.getLogRecordFactory() is externally_installed+ assert observed == [(SAFE_HTTP_LOG_MESSAGE, (), None, None)]+ assert "url" not in caplog.records[0].__dict__+ assert "license" not in caplog.records[0].__dict__+ logging.getLogger("httpx").info("EXTERNAL-FACTORY-PUBLIC-LOG")+ assert SYNTHETIC_KEY not in evidence(caplog)+ assert "EXTERNAL-FACTORY-PUBLIC-LOG" in caplog.text+ finally:+ logging.setLogRecordFactory(original)+++def test_protection_never_sets_the_record_factory(monkeypatch: pytest.MonkeyPatch) -> None:+ original = logging.getLogRecordFactory()++ def forbidden(_factory: object) -> None:+ pytest.fail("Request protection must not configure the LogRecordFactory")++ monkeypatch.setattr(logging, "setLogRecordFactory", forbidden)+ with protect_http_request_logs(True):+ logging.getLogger("httpx").warning("private %s", SYNTHETIC_KEY)+ assert logging.getLogRecordFactory() is original+ assert logging.getLogRecordFactory() is original+++def test_public_make_record_signature_preserves_public_fields_and_redacts_private_fields() -> None:+ logger = logging.getLogger("httpx")+ error_info = (ValueError, ValueError(SYNTHETIC_KEY), None)+ original_factory = logging.getLogRecordFactory()+ for sensitive in [True, False]:+ with protect_http_request_logs(sensitive):+ record = logger.makeRecord(+ name="httpx",+ level=logging.WARNING,+ fn="fixture.py",+ lno=42,+ msg="message %s",+ args=(SYNTHETIC_KEY,),+ exc_info=error_info,+ func="request",+ extra={"license": ENCODED_KEY},+ sinfo=SYNTHETIC_KEY,+ )+ assert logging.getLogRecordFactory() is original_factory+ assert record.name == "httpx"+ assert record.levelno == logging.WARNING+ if sensitive:+ assert record.getMessage() == SAFE_HTTP_LOG_MESSAGE+ assert record.exc_info is None+ assert record.stack_info is None+ assert "license" not in record.__dict__+ else:+ assert record.getMessage() == "message " + SYNTHETIC_KEY+ assert record.exc_info is error_info+ assert record.stack_info == SYNTHETIC_KEY+ assert record.__dict__["license"] == ENCODED_KEY+++def test_exit_does_not_overwrite_a_foreign_make_record_method() -> None:+ original_method: Callable[..., logging.LogRecord] = logging.Logger.makeRecord+ original_factory = logging.getLogRecordFactory()+ try:+ with protect_http_request_logs(True):++ def externally_installed(*args: object, **kwargs: object) -> logging.LogRecord:+ return original_method(*args, **kwargs)++ logging.Logger.makeRecord = externally_installed+ assert logging.Logger.makeRecord is externally_installed+ assert logging.getLogRecordFactory() is original_factory+ finally:+ logging.Logger.makeRecord = original_methoddiff --git a/api/tests/unit_tests/services/tracing/test_runtime_bridge.py b/api/tests/unit_tests/services/tracing/test_runtime_bridge.pynew file mode 100644index 00000000000..bb2ae7ca369--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_runtime_bridge.py@@ -0,0 +1,367 @@+"""Verify execution hook ownership, policy gating, and fail-open business behavior."""++from dataclasses import dataclass, field+from typing import cast+from uuid import uuid4++import pytest+from dify_agent.layers.execution_context import ApplicationTracingContext+from sqlalchemy.exc import OperationalError+from sqlalchemy.orm import Session, sessionmaker++from core.app.app_config.entities import WorkflowUIBasedAppConfig+from core.app.apps.agent_app.runtime_request_builder import AgentAppRuntimeBuildContext+from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom, WorkflowAppGenerateEntity+from core.ops.entities.trace_entity import BaseTraceInfo+from core.ops.exceptions import RetryableTraceDispatchError+from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext+from graphon.runtime import VariablePool+from models.agent import (+ Agent,+ AgentConfigSnapshot,+ AgentScope,+ AgentSource,+ WorkflowAgentBindingType,+ WorkflowAgentNodeBinding,+)+from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig+from models.model import AppMode+from repositories.application_trace_repository import ApplicationTraceRepository+from services.tracing import runtime_bridge+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+++@dataclass+class BridgeCase:+ target: ApplicationTracingService+ calls: list[tuple[str, dict[str, object]]] = field(default_factory=list)+ failure: Exception | None = None+ context: ApplicationTracingContext = field(+ default_factory=lambda: ApplicationTracingContext(+ run_id=uuid4(),+ invocation_id=uuid4(),+ trace_id="1" * 32,+ parent_span_id="2" * 16,+ session_id="session",+ capture_content=False,+ )+ )+++def service(monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session]) -> BridgeCase:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ target = ApplicationTracingService(ApplicationTraceRepository(sqlite_session_factory))+ case = BridgeCase(target)++ def prepare(**values: object) -> ApplicationTracingContext:+ if case.failure is not None:+ raise case.failure+ case.calls.append(("prepare", values))+ return case.context++ def start(**values: object) -> None:+ if case.failure is not None:+ raise case.failure+ case.calls.append(("start", values))++ def terminal(_info: BaseTraceInfo) -> bool:+ if case.failure is not None:+ raise case.failure+ return True++ def agent_terminal(**values: object) -> None:+ if case.failure is not None:+ raise case.failure+ case.calls.append(("terminal", values))++ monkeypatch.setattr(target, "prepare_invocation", prepare)+ monkeypatch.setattr(target, "start", start)+ monkeypatch.setattr(target, "capture_terminal", terminal)+ monkeypatch.setattr(target, "capture_agent_app_terminal", agent_terminal)+ monkeypatch.setattr(runtime_bridge, "_service", lambda: target)+ return case+++def run_context(app_id: str) -> DifyRunContext:+ return DifyRunContext(+ tenant_id="tenant",+ app_id=app_id,+ user_id="account",+ user_from=UserFrom.ACCOUNT,+ invoke_from=InvokeFrom.DEBUGGER,+ )+++def agent_context() -> AgentAppRuntimeBuildContext:+ return AgentAppRuntimeBuildContext(+ dify_context=run_context("agent-app"),+ agent_id="agent",+ agent_config_snapshot_id="snapshot",+ agent_soul=AgentSoulConfig(),+ conversation_id="conversation",+ user_query="query",+ idempotency_key="message-id",+ binding_id="binding",+ backend_binding_ref="reference",+ )+++def workflow_context(preview: bool) -> WorkflowAgentRuntimeBuildContext:+ return WorkflowAgentRuntimeBuildContext(+ dify_context=run_context("workflow-app"),+ agent=Agent(+ id="referenced-agent",+ tenant_id="tenant",+ name="Agent",+ scope=AgentScope.ROSTER,+ source=AgentSource.ROSTER,+ ),+ snapshot=AgentConfigSnapshot(+ id="snapshot", tenant_id="tenant", agent_id="referenced-agent", version=1, config_snapshot=AgentSoulConfig()+ ),+ binding=WorkflowAgentNodeBinding(+ id="binding",+ tenant_id="tenant",+ app_id="workflow-app",+ workflow_id="workflow",+ workflow_version="draft",+ node_id="node",+ binding_type=WorkflowAgentBindingType.ROSTER_AGENT,+ agent_id="referenced-agent",+ node_job_config=WorkflowNodeJobConfig(),+ ),+ workflow_id="workflow",+ workflow_run_id=None if preview else "run",+ node_id="node",+ node_execution_id="node-execution",+ attempt=2,+ variable_pool=VariablePool(),+ binding_id="binding",+ backend_binding_ref="reference",+ )+++def workflow_entity(extras: dict[str, object] | None = None) -> WorkflowAppGenerateEntity:+ return WorkflowAppGenerateEntity.model_construct(+ app_config=WorkflowUIBasedAppConfig(+ app_id="app", tenant_id="tenant", app_mode=AppMode.WORKFLOW, workflow_id="workflow"+ ),+ extras=extras or {},+ )+++def test_disabled_gate_never_constructs_database_service_or_reads_context(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)+ calls = []++ def forbidden() -> ApplicationTracingService:+ calls.append(True)+ raise AssertionError("disabled tracing must not construct a service")++ monkeypatch.setattr(runtime_bridge, "_service", forbidden)+ assert runtime_bridge.prepare_agent_app(cast(AgentAppRuntimeBuildContext, object())) is None+ assert runtime_bridge.prepare_workflow_agent(cast(WorkflowAgentRuntimeBuildContext, object()), None) is None+ runtime_bridge.freeze_workflow_run(+ cast(WorkflowAppGenerateEntity, object()),+ workflow_id="workflow",+ workflow_run_id="run",+ conversation_id=None,+ graph_data={},+ )+ assert runtime_bridge.capture_terminal(cast(BaseTraceInfo, object())) is False+ assert runtime_bridge.queue_enabled_at_start("app") is False+ assert calls == []+++def test_agent_app_uses_message_identity_and_stable_conversation(service: BridgeCase) -> None:+ assert runtime_bridge.prepare_agent_app(agent_context()) is service.context+ assert service.calls == [+ (+ "prepare",+ {+ "tenant_id": "tenant",+ "app_id": "agent-app",+ "agent_id": "agent",+ "source_kind": "message",+ "source_id": "message-id",+ "session_id": "conversation",+ "node_execution_id": None,+ "attempt": 0,+ "call_key": "message-id",+ },+ )+ ]++[email protected]("preview", [False, True])+def test_workflow_invocation_uses_owner_app_and_execution_attempt(service: BridgeCase, preview: bool) -> None:+ assert runtime_bridge.prepare_workflow_agent(workflow_context(preview), None) is service.context+ values = service.calls[0][1]+ assert values["app_id"] == "workflow-app"+ assert values["agent_id"] == "referenced-agent"+ assert values["source_kind"] == ("node" if preview else "workflow")+ assert values["source_id"] == ("node-execution" if preview else "run")+ assert values["session_id"] == values["source_id"]+ assert values["call_key"] == "node-execution:2"+ assert values["workflow_id"] == "workflow"+++def test_workflow_start_prefers_explicit_trace_session(service: BridgeCase) -> None:+ entity = workflow_entity({"trace_session_id": "external-session"})+ runtime_bridge.freeze_workflow_run(+ entity,+ workflow_id="workflow",+ workflow_run_id="run",+ conversation_id="conversation",+ graph_data={"nodes": [{"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}]},+ )+ assert service.calls == [+ (+ "start",+ {+ "tenant_id": "tenant",+ "app_id": "app",+ "source_kind": "workflow",+ "source_id": "run",+ "session_id": "external-session",+ "workflow_id": "workflow",+ },+ )+ ]+++def test_business_hook_failure_logs_type_without_sensitive_exception_message(+ service: BridgeCase, caplog: pytest.LogCaptureFixture+) -> None:+ service.failure = ValueError("password=private-secret")+ assert runtime_bridge.prepare_agent_app(agent_context()) is None+ assert "ValueError" in caplog.text+ assert "private-secret" not in caplog.text+++def test_worker_database_failure_retries_without_legacy_fallback(service: BridgeCase) -> None:+ service.failure = OperationalError("statement", {}, Exception("sensitive database error"))+ with pytest.raises(RetryableTraceDispatchError, match="persistence temporarily unavailable"):+ runtime_bridge.capture_terminal(cast(BaseTraceInfo, object()))+++def test_ordinary_workflow_does_not_create_managed_snapshot(service: BridgeCase) -> None:+ runtime_bridge.freeze_workflow_run(+ cast(WorkflowAppGenerateEntity, object()),+ workflow_id="workflow",+ workflow_run_id="run",+ conversation_id=None,+ graph_data={"nodes": [{"data": {"type": "llm"}}, {"data": {"type": "agent", "version": "1"}}]},+ )+ assert service.calls == []+++ "graph", ["not-json", "null", "[]", {}, {"nodes": {}}, {"nodes": [None, {"data": {"type": "llm"}}]}]+)+def test_invalid_or_ordinary_graph_cannot_enable_agent_collection(graph: object) -> None:+ assert runtime_bridge.workflow_has_agent_v2(graph) is False+++def test_graph_string_with_agent_is_recognized() -> None:+ assert (+ runtime_bridge.workflow_has_agent_v2(+ '{"nodes":[{"data":{"type":"agent","version":"2","agent_node_kind":"dify_agent"}}]}'+ )+ is True+ )++[email protected]("outcome", ["succeeded", "failed", "cancelled"])+def test_worker_terminal_handoff_preserves_outcome_without_sse(service: BridgeCase, outcome: str) -> None:+ from typing import Literal, cast++ runtime_bridge.capture_agent_app_terminal(+ tenant_id="tenant",+ app_id="agent-app",+ message_id="message",+ output="answer",+ outcome=cast(Literal["succeeded", "failed", "cancelled"], outcome),+ )+ assert service.calls == [+ (+ "terminal",+ {+ "tenant_id": "tenant",+ "app_id": "agent-app",+ "message_id": "message",+ "outcome": outcome,+ "output": "answer",+ },+ )+ ]+++def test_terminal_handoff_gate_and_fail_open_log_redact_sensitive_details(+ service: BridgeCase, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:+ from repositories.application_trace_repository import TraceCollectionError++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)+ runtime_bridge.capture_agent_app_terminal(+ tenant_id="tenant", app_id="app", message_id="message", outcome="failed", output=None+ )+ assert service.calls == []+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ service.failure = TraceCollectionError("safe_error_code")+ runtime_bridge.capture_agent_app_terminal(+ tenant_id="tenant", app_id="app", message_id="message", outcome="failed", output="private-content"+ )+ assert "safe_error_code" in caplog.text+ assert "private-content" not in caplog.text+++def test_workflow_fail_open_logs_error_kind_without_content(+ service: BridgeCase, caplog: pytest.LogCaptureFixture+) -> None:+ service.failure = ValueError("private-prompt")+ assert runtime_bridge.prepare_workflow_agent(workflow_context(False), "conversation") is None+ runtime_bridge.freeze_workflow_run(+ workflow_entity(),+ workflow_id="workflow",+ workflow_run_id="run",+ conversation_id="conversation",+ graph_data={"nodes": [{"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}]},+ )+ assert "workflow_agent" in caplog.text+ assert "workflow_start" in caplog.text+ assert "ValueError" in caplog.text+ assert "private-prompt" not in caplog.text+++ ("policy", "expected"),+ [+ (None, False),+ ('{"enabled":true}', True),+ ('{"enabled":"true"}', False),+ ("[]", False),+ ("invalid-private", False),+ ],+)+def test_queue_policy_reads_real_app_rows_without_constructing_vendor_sdk(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ policy: str | None,+ expected: bool,+) -> None:+ from extensions.ext_database import db+ from tests.unit_tests.model_factories import make_app++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ with sqlite_session_factory.begin() as session:+ session.add(make_app(app_id="queue-app", tenant_id="tenant", tracing=policy))+ with sqlite_session_factory() as session:+ monkeypatch.setattr(db, "session", session)+ assert runtime_bridge.queue_enabled_at_start("queue-app") is expected+ assert runtime_bridge.queue_enabled_at_start("missing") is False+ assert runtime_bridge.queue_enabled_at_start(None) is False+ assert "invalid-private" not in caplog.textdiff --git a/api/tests/unit_tests/services/tracing/test_tencent_configuration.py b/api/tests/unit_tests/services/tracing/test_tencent_configuration.pynew file mode 100644index 00000000000..c3cd8d08cb2--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_tencent_configuration.py@@ -0,0 +1,390 @@+"""Tencent configuration checks must reject malformed input without provider I/O."""++from collections.abc import Callable, Iterator, Sequence+from dataclasses import dataclass, field+from types import MappingProxyType+from typing import Never++import grpc # type: ignore[import-untyped]+import pytest+from dify_trace_tencent.configuration import (+ TencentConfigurationError,+ console_url,+ normalize_configuration,+ verify_configuration,+)+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (+ ExportTraceServiceRequest,+ ExportTraceServiceResponse,+)+++def configuration_values(**overrides: object) -> dict[str, object]:+ return {+ "endpoint": "https://collector.example:4320",+ "service_name": "dify-agent",+ "token": "synthetic-token",+ **overrides,+ }++[email protected](autouse=True)+def _forbid_provider_io(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:+ attempts: list[str] = []++ def forbidden(*_args: object, **_kwargs: object) -> Never:+ attempts.append("provider_io")+ raise AssertionError("Local configuration validation must not construct clients or perform network I/O")++ for target in (+ "dify_trace_tencent.tencent_trace.TencentDataTrace.__init__",+ "dify_trace_tencent.client.TencentTraceClient.__init__",+ "socket.socket",+ "socket.create_connection",+ "socket.getaddrinfo",+ "grpc.secure_channel",+ "grpc.insecure_channel",+ ):+ monkeypatch.setattr(target, forbidden)+ yield+ assert attempts == []+++def test_normalization_accepts_read_only_mapping_and_preserves_opaque_token() -> None:+ values = configuration_values(+ endpoint=" https://COLLECTOR.Example:4320/ ",+ service_name=" dify-agent ",+ token=" opaque token/+== ",+ )+ original = dict(values)++ result = normalize_configuration(MappingProxyType(values))++ assert result.model_dump() == {+ "endpoint": "https://collector.example:4320",+ "service_name": "dify-agent",+ "token": " opaque token/+== ",+ }+ assert values == original++[email protected]("service_name", ["a", "a" * 63, "app-1", "a--b"])+def test_accepts_documented_service_name_boundaries(service_name: str) -> None:+ result = normalize_configuration(configuration_values(service_name=service_name))++ assert result.service_name == service_name+++ "service_name",+ [None, 42, "", " ", "dify_app", "Dify", "1app", "-app", "app-", "a" * 64, "应用", "a.b", "a/b", "a b", "a\nb"],+)+def test_rejects_invalid_service_names_before_provider_io(service_name: object) -> None:+ with pytest.raises(TencentConfigurationError) as caught:+ normalize_configuration(configuration_values(service_name=service_name))++ assert caught.value.code == "invalid_service_name"+++ ("endpoint", "expected"),+ [+ ("https://COLLECTOR.Example:4320/", "https://collector.example:4320"),+ ("http://collector.example:4319", "http://collector.example:4319"),+ ("https://collector.example/", "https://collector.example"),+ ("http://collector.example", "http://collector.example"),+ ("https://collector.example:443", "https://collector.example:443"),+ ("http://127.0.0.1:1/", "http://127.0.0.1:1"),+ ("https://[2001:DB8::1]:65535/", "https://[2001:db8::1]:65535"),+ ("https://BÜCHER.example:4320/", "https://xn--bcher-kva.example:4320"),+ ],+)+def test_normalizes_origins_without_changing_explicit_port(endpoint: str, expected: str) -> None:+ result = normalize_configuration(configuration_values(endpoint=endpoint))++ assert result.endpoint == expected+++ "endpoint",+ [+ None,+ 4320,+ "",+ " ",+ "collector.example:4320",+ "ftp://collector.example:4320",+ "https://",+ "https:///",+ "https://user:[email protected]:4320",+ "https://@collector.example:4320",+ "https://collector.example:4320/v1/traces",+ "https://collector.example:4320//",+ "https://collector.example:4320?token=synthetic-secret",+ "https://collector.example:4320?",+ "https://collector.example:4320#project",+ "https://collector.example:4320#",+ "https://collector.example\\@other.example:4320",+ "https://col lector.example:4320",+ "https://collector.example:\n4320",+ "https://collector.example:\t4320",+ "https://collector\x00.example:4320",+ "https://collector.example:0",+ "https://collector.example:65536",+ "https://collector.example:-1",+ "https://collector.example:invalid",+ "https://collector.example:",+ "https://[2001:db8::1",+ "https://[not-an-ipv6-address]:4320",+ ],+)+def test_rejects_invalid_endpoints_before_provider_io(endpoint: object) -> None:+ with pytest.raises(TencentConfigurationError) as caught:+ normalize_configuration(configuration_values(endpoint=endpoint))++ assert caught.value.code == "invalid_endpoint"+ assert "synthetic-secret" not in str(caught.value)+ assert "synthetic-token" not in str(caught.value)++[email protected]("token", [None, 42, "", " ", "\t\n"])+def test_rejects_missing_or_blank_tokens_without_provider_io(token: object) -> None:+ with pytest.raises(TencentConfigurationError) as caught:+ normalize_configuration(configuration_values(token=token))++ assert caught.value.code == "invalid_token"+++def test_masked_token_survives_local_validation_for_patch_restoration() -> None:+ result = normalize_configuration(configuration_values(token="***"))++ assert result.token == "***"+++ ("field", "error_code"),+ [("endpoint", "invalid_endpoint"), ("service_name", "invalid_service_name"), ("token", "invalid_token")],+)+def test_missing_fields_do_not_receive_legacy_defaults(field: str, error_code: str) -> None:+ values = configuration_values()+ del values[field]++ with pytest.raises(TencentConfigurationError) as caught:+ normalize_configuration(values)++ assert caught.value.code == error_code+++def test_console_url_is_constant_without_configuration_or_provider_io() -> None:+ assert console_url() == "https://console.cloud.tencent.com/apm"+++@dataclass+class ExportCall:+ request: ExportTraceServiceRequest+ metadata: tuple[tuple[str, str], ...]+ timeout: float+ wait_for_ready: bool+++@dataclass+class CollectorChannel:+ target: str+ secure: bool+ options: tuple[tuple[str, int], ...]+ response_bytes: bytes+ failure: Exception | None = None+ fail_stub_construction: bool = False+ calls: list[ExportCall] = field(default_factory=list)+ close_count: int = 0++ def unary_unary(+ self,+ method: str,+ request_serializer: Callable[[ExportTraceServiceRequest], bytes],+ response_deserializer: Callable[[bytes], ExportTraceServiceResponse],+ **_kwargs: object,+ ) -> Callable[..., ExportTraceServiceResponse]:+ assert method == "/opentelemetry.proto.collector.trace.v1.TraceService/Export"+ if self.fail_stub_construction:+ raise RuntimeError("private-stub-detail synthetic-token")++ def invoke(+ payload: ExportTraceServiceRequest,+ metadata: Sequence[tuple[str, str]],+ timeout: float,+ wait_for_ready: bool,+ ) -> ExportTraceServiceResponse:+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) is True+ request = ExportTraceServiceRequest.FromString(request_serializer(payload))+ self.calls.append(+ ExportCall(request=request, metadata=tuple(metadata), timeout=timeout, wait_for_ready=wait_for_ready)+ )+ if self.failure is not None:+ raise self.failure+ return response_deserializer(self.response_bytes)++ return invoke++ def close(self) -> None:+ self.close_count += 1+++@dataclass+class CollectorBoundary:+ response_bytes: bytes = b""+ failure: Exception | None = None+ fail_stub_construction: bool = False+ fail_channel_construction: bool = False+ channels: list[CollectorChannel] = field(default_factory=list)++ def open_channel(self, target: str, secure: bool, options: Sequence[tuple[str, int]]) -> CollectorChannel:+ if self.fail_channel_construction:+ raise RuntimeError("private-channel-detail synthetic-token")+ channel = CollectorChannel(+ target=target,+ secure=secure,+ options=tuple(options),+ response_bytes=self.response_bytes,+ failure=self.failure,+ fail_stub_construction=self.fail_stub_construction,+ )+ self.channels.append(channel)+ return channel++ def secure_channel(self, target: str, credentials: object, options: Sequence[tuple[str, int]]) -> CollectorChannel:+ assert credentials is not None+ return self.open_channel(target, secure=True, options=options)++ def insecure_channel(self, target: str, options: Sequence[tuple[str, int]]) -> CollectorChannel:+ return self.open_channel(target, secure=False, options=options)+++def collector(monkeypatch: pytest.MonkeyPatch, _forbid_provider_io: None) -> Iterator[CollectorBoundary]:+ prior_suppression = get_value(_SUPPRESS_INSTRUMENTATION_KEY)+ boundary = CollectorBoundary()+ monkeypatch.setattr("grpc.secure_channel", boundary.secure_channel)+ monkeypatch.setattr("grpc.insecure_channel", boundary.insecure_channel)+ yield boundary+ assert get_value(_SUPPRESS_INSTRUMENTATION_KEY) == prior_suppression+++ ("endpoint", "target", "secure"),+ [+ ("https://collector.example", "collector.example:4317", True),+ ("http://collector.example", "collector.example:4317", False),+ ("https://collector.example:4320", "collector.example:4320", True),+ ("http://collector.example:4319", "collector.example:4319", False),+ ("https://[2001:db8::1]:4320", "[2001:db8::1]:4320", True),+ ],+)+def test_probe_serializes_one_empty_export_and_closes_channel(+ collector: CollectorBoundary, endpoint: str, target: str, secure: bool+) -> None:+ config = normalize_configuration(configuration_values(endpoint=endpoint))++ assert verify_configuration(config) == config++ assert len(collector.channels) == 1+ channel = collector.channels[0]+ assert (channel.target, channel.secure) == (target, secure)+ assert dict(channel.options) == {"grpc.enable_retries": 0, "grpc.max_receive_message_length": 4 * 1024 * 1024}+ assert len(channel.calls) == 1+ call = channel.calls[0]+ assert list(call.request.resource_spans) == []+ assert call.request.SerializeToString() == b""+ assert call.metadata == (("authorization", "Bearer synthetic-token"),)+ assert call.timeout == 10+ assert call.wait_for_ready is False+ assert channel.close_count == 1+++def test_probe_credentials_belong_to_each_request(collector: CollectorBoundary) -> None:+ for token in ("first-synthetic-token", "second-synthetic-token"):+ verify_configuration(normalize_configuration(configuration_values(token=token)))++ assert len(collector.channels) == 2+ assert [channel.calls[0].metadata for channel in collector.channels] == [+ (("authorization", "Bearer first-synthetic-token"),),+ (("authorization", "Bearer second-synthetic-token"),),+ ]+ assert all(channel.close_count == 1 for channel in collector.channels)+++ ("rejected_spans", "error_message"),+ [(1, ""), (0, "private-collector-detail synthetic-token")],+)+def test_probe_rejects_partial_response_and_closes_channel(+ collector: CollectorBoundary, rejected_spans: int, error_message: str+) -> None:+ response = ExportTraceServiceResponse()+ response.partial_success.rejected_spans = rejected_spans+ response.partial_success.error_message = error_message+ collector.response_bytes = response.SerializeToString()++ with pytest.raises(TencentConfigurationError) as caught:+ verify_configuration(normalize_configuration(configuration_values()))++ assert caught.value.code == "unexpected_response"+ assert "private-collector-detail" not in str(caught.value)+ assert "synthetic-token" not in str(caught.value)+ assert collector.channels[0].close_count == 1+++ "failure",+ [+ grpc.RpcError("private-rpc-detail synthetic-token"),+ ValueError("private-serialization-detail synthetic-token"),+ ],+)+def test_probe_transport_failures_are_safe_and_close_channel(collector: CollectorBoundary, failure: Exception) -> None:+ collector.failure = failure++ with pytest.raises(TencentConfigurationError) as caught:+ verify_configuration(normalize_configuration(configuration_values()))++ assert caught.value.code == "connection_failed"+ assert "private-" not in str(caught.value)+ assert "synthetic-token" not in str(caught.value)+ assert len(collector.channels[0].calls) == 1+ assert collector.channels[0].close_count == 1+++def test_probe_unreadable_response_is_safe_and_closes_channel(collector: CollectorBoundary) -> None:+ collector.response_bytes = b"\xff"++ with pytest.raises(TencentConfigurationError) as caught:+ verify_configuration(normalize_configuration(configuration_values()))++ assert caught.value.code == "connection_failed"+ assert collector.channels[0].close_count == 1+++def test_probe_stub_construction_failure_is_safe_and_closes_channel(collector: CollectorBoundary) -> None:+ collector.fail_stub_construction = True++ with pytest.raises(TencentConfigurationError) as caught:+ verify_configuration(normalize_configuration(configuration_values()))++ assert caught.value.code == "connection_failed"+ assert "private-stub-detail" not in str(caught.value)+ assert "synthetic-token" not in str(caught.value)+ assert collector.channels[0].calls == []+ assert collector.channels[0].close_count == 1+++def test_probe_channel_construction_failure_has_fixed_safe_error(collector: CollectorBoundary) -> None:+ collector.fail_channel_construction = True++ with pytest.raises(TencentConfigurationError) as caught:+ verify_configuration(normalize_configuration(configuration_values()))++ assert caught.value.code == "connection_failed"+ assert "private-channel-detail" not in str(caught.value)+ assert "synthetic-token" not in str(caught.value)+ assert collector.channels == []diff --git a/api/tests/unit_tests/services/tracing/test_tencent_legacy_sessions.py b/api/tests/unit_tests/services/tracing/test_tencent_legacy_sessions.pynew file mode 100644index 00000000000..7ab192b9903--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_tencent_legacy_sessions.py@@ -0,0 +1,194 @@+"""Pure legacy Tencent session regressions; IDs here are builder-local IDs."""++from datetime import UTC, datetime, timedelta++import pytest+from dify_trace_tencent.entities.tencent_trace_entity import SpanData+from dify_trace_tencent.span_builder import TencentSpanBuilder+from dify_trace_tencent.utils import TencentTraceUtils+from opentelemetry.trace import StatusCode+from pydantic import ValidationError++from core.ops.entities.trace_entity import WorkflowTraceInfo+from graphon.entities import WorkflowNodeExecution+from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus++RUN_ID = "00000000-0000-4000-8000-000000000001"+WORKFLOW_ID = "00000000-0000-4000-8000-000000000002"+TENANT_ID = "00000000-0000-4000-8000-000000000003"+STARTED = datetime(2026, 10, 17, tzinfo=UTC)+++def workflow_trace(metadata: dict[str, object]) -> WorkflowTraceInfo:+ session_id = metadata.get("conversation_id")+ return WorkflowTraceInfo(+ workflow_id=WORKFLOW_ID,+ tenant_id=TENANT_ID,+ workflow_run_id=RUN_ID,+ workflow_run_elapsed_time=3.0,+ workflow_run_status="succeeded",+ workflow_run_inputs={"query": "synthetic input"},+ workflow_run_outputs={"answer": "synthetic answer"},+ workflow_run_version="1",+ error="",+ total_tokens=32,+ prompt_tokens=13,+ completion_tokens=19,+ file_list=[],+ query="synthetic input",+ metadata=metadata,+ conversation_id=session_id if isinstance(session_id, str) else None,+ start_time=STARTED,+ end_time=STARTED + timedelta(seconds=3),+ )+++def workflow_node(node_type: str, index: int = 1) -> WorkflowNodeExecution:+ return WorkflowNodeExecution(+ id=f"00000000-0000-4000-8000-{index + 10:012d}",+ workflow_id=WORKFLOW_ID,+ workflow_execution_id=RUN_ID,+ index=index,+ node_id=f"node-{index}",+ node_type=node_type,+ title={BuiltinNodeTypes.START: "Start", BuiltinNodeTypes.END: "End"}.get(node_type, "Synthetic node"),+ inputs={"query": "synthetic input"},+ process_data={"model_name": "synthetic-model", "model_provider": "synthetic-provider"},+ outputs={+ "text": "synthetic answer",+ "usage": {"prompt_tokens": 13, "completion_tokens": 19, "total_tokens": 32},+ "finish_reason": "stop",+ },+ status=WorkflowNodeExecutionStatus.SUCCEEDED,+ created_at=STARTED + timedelta(seconds=index - 1),+ finished_at=STARTED + timedelta(seconds=index),+ )+++def build_node_span(trace: WorkflowTraceInfo, node: WorkflowNodeExecution) -> SpanData:+ trace_id = TencentTraceUtils.convert_to_trace_id(trace.workflow_run_id)+ workflow_span_id = TencentTraceUtils.convert_to_span_id(trace.workflow_run_id, "workflow")+ if node.node_type == BuiltinNodeTypes.LLM:+ return TencentSpanBuilder.build_workflow_llm_span(trace_id, workflow_span_id, trace, node)+ if node.node_type == BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL:+ return TencentSpanBuilder.build_workflow_retrieval_span(trace_id, workflow_span_id, trace, node)+ return TencentSpanBuilder.build_workflow_task_span(trace_id, workflow_span_id, trace, node)+++ "node_type", [BuiltinNodeTypes.START, BuiltinNodeTypes.LLM, BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL]+)+ ("metadata", "expected_session"),+ [+ pytest.param({}, "", id="missing"),+ pytest.param({"conversation_id": None}, "", id="none"),+ pytest.param({"conversation_id": ""}, "", id="empty"),+ pytest.param({"conversation_id": "owned-conversation"}, "owned-conversation", id="conversation"),+ ],+)+def test_workflow_builders_accept_optional_sessions_without_rewriting_metadata(+ node_type: str, metadata: dict[str, object], expected_session: str+) -> None:+ trace = workflow_trace(metadata)+ before = dict(trace.metadata)++ span = build_node_span(trace, workflow_node(node_type))++ assert span.attributes["gen_ai.session.id"] == expected_session+ assert trace.metadata == before+++def complete_workflow_spans(trace: WorkflowTraceInfo) -> list[SpanData]:+ spans = TencentSpanBuilder.build_workflow_spans(+ trace, TencentTraceUtils.convert_to_trace_id(trace.workflow_run_id), "synthetic-user"+ )+ spans.extend(+ build_node_span(trace, workflow_node(node_type, index))+ for index, node_type in enumerate((BuiltinNodeTypes.START, BuiltinNodeTypes.LLM, BuiltinNodeTypes.END), start=1)+ )+ return spans+++def assert_usage_is_not_duplicated(spans: list[SpanData]) -> None:+ generations = [span for span in spans if span.attributes.get("gen_ai.span.kind") == "GENERATION"]+ assert len(generations) == 1+ generation = generations[0]+ assert generation.attributes["gen_ai.usage.input_tokens"] == "13"+ assert generation.attributes["gen_ai.usage.output_tokens"] == "19"+ assert generation.attributes["gen_ai.usage.total_tokens"] == "32"+ assert sum(int(span.attributes.get("gen_ai.usage.total_tokens", "0")) for span in spans) == 32+++def test_workflow_without_conversation_keeps_start_llm_end_under_one_root() -> None:+ trace = workflow_trace({"conversation_id": None})++ spans = complete_workflow_spans(trace)++ assert [span.name for span in spans] == ["workflow", "Start", "GENERATION", "End"]+ root = spans[0]+ assert root.parent_span_id is None+ assert all(span.parent_span_id == root.span_id for span in spans[1:])+ assert all(span.trace_id == root.trace_id for span in spans)+ assert len({span.span_id for span in spans}) == 4+ assert all(span.attributes["gen_ai.session.id"] == "" for span in spans[1:])+ assert trace.metadata["conversation_id"] is None+ assert_usage_is_not_duplicated(spans)+++def test_chatflow_keeps_conversation_and_message_workflow_parent_layer() -> None:+ trace = workflow_trace({"conversation_id": "owned-conversation"})++ spans = complete_workflow_spans(trace)++ assert [span.name for span in spans] == ["message", "workflow", "Start", "GENERATION", "End"]+ message, workflow = spans[:2]+ assert message.parent_span_id is None+ assert workflow.parent_span_id == message.span_id+ assert all(span.parent_span_id == workflow.span_id for span in spans[2:])+ assert all(span.trace_id == message.trace_id for span in spans)+ assert len({span.span_id for span in spans}) == 5+ assert message.attributes["gen_ai.session.id"] == "owned-conversation"+ assert all(span.attributes["gen_ai.session.id"] == "owned-conversation" for span in spans[2:])+ assert_usage_is_not_duplicated(spans)++[email protected]("status", [WorkflowNodeExecutionStatus.FAILED, WorkflowNodeExecutionStatus.EXCEPTION])+def test_error_status_and_description_survive_missing_session(status: WorkflowNodeExecutionStatus) -> None:+ trace = workflow_trace({"conversation_id": None})+ trace.error = "synthetic workflow failure"+ trace.workflow_run_status = "failed"+ node = workflow_node(BuiltinNodeTypes.LLM)+ node.status = status+ node.error = "synthetic node failure"++ root = TencentSpanBuilder.build_workflow_spans(+ trace, TencentTraceUtils.convert_to_trace_id(trace.workflow_run_id), "synthetic-user"+ )[0]+ span = build_node_span(trace, node)++ assert root.status.status_code == StatusCode.ERROR+ assert root.status.description == "synthetic workflow failure"+ assert span.status.status_code == StatusCode.ERROR+ assert span.status.description == "synthetic node failure"+ assert span.parent_span_id == root.span_id+ assert span.attributes["gen_ai.session.id"] == ""+ assert_usage_is_not_duplicated([root, span])+++ "node_type", [BuiltinNodeTypes.START, BuiltinNodeTypes.LLM, BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL]+)+ "invalid_session", [0, False, [], {}, 123], ids=["zero", "false", "empty-list", "empty-map", "int"]+)+def test_non_string_sessions_remain_validation_errors(node_type: str, invalid_session: object) -> None:+ trace = workflow_trace({"conversation_id": invalid_session})++ with pytest.raises(ValidationError) as caught:+ build_node_span(trace, workflow_node(node_type))++ errors = caught.value.errors(include_input=False, include_url=False)+ assert len(errors) == 1+ assert errors[0]["loc"] == ("attributes", "gen_ai.session.id")+ assert errors[0]["type"] == "string_type"diff --git a/api/tests/unit_tests/services/tracing/test_weave_configuration.py b/api/tests/unit_tests/services/tracing/test_weave_configuration.pynew file mode 100644index 00000000000..63d4983119b--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_weave_configuration.py@@ -0,0 +1,258 @@+"""Read-only Weave validation at the real config/schema and HTTP boundaries."""++from collections.abc import Callable+from dataclasses import dataclass, field+from typing import cast++import httpx+import pytest+from dify_trace_weave.config import WeaveConfig+from dify_trace_weave.configuration import (+ WeaveConfigurationError,+ console_url,+ project_url,+ verify_configuration,+)+from weave.trace_server.trace_server_interface import CallsQueryStatsReq++from core.helper import ssrf_proxy+++def object_dict(value: object) -> dict[str, object]:+ assert isinstance(value, dict)+ return cast(dict[str, object], value)+++def response(url: str, body: object, *, status: int = 200) -> httpx.Response:+ return httpx.Response(status, json=body, request=httpx.Request("POST", url))+++@dataclass+class ReadAPI:+ calls: list[tuple[str, dict[str, object]]] = field(default_factory=list)+ override: Callable[[str, dict[str, object]], httpx.Response] | None = None+ project_fields: dict[str, object] = field(default_factory=dict)+ project_exists: bool = True+ viewer_exists: bool = True+ count: object = 0++ def post(self, url: str, **options: object) -> httpx.Response:+ self.calls.append((url, options))+ body = object_dict(options["json"])+ if self.override is not None:+ return self.override(url, body)+ if url.endswith("/graphql"):+ query = body["query"]+ assert isinstance(query, str)+ assert "mutation" not in query+ if "DifyWeaveViewer" in query:+ return response(url, {"data": {"viewer": {"id": "viewer-id", "entity": "default-team"}}})+ variables = object_dict(body["variables"])+ project: dict[str, object] = {+ "id": "project-id",+ "name": variables["project"],+ "entityName": variables["entity"],+ "readOnly": False,+ }+ project.update(self.project_fields)+ return response(+ url,+ {+ "data": {+ "viewer": {"id": "viewer-id"} if self.viewer_exists else None,+ "project": project if self.project_exists else None,+ }+ },+ )+ assert url.endswith("/calls/query_stats")+ query_request = CallsQueryStatsReq.model_validate(body)+ assert query_request.limit == 1+ return response(url, {"count": self.count})+++def read_api(monkeypatch: pytest.MonkeyPatch) -> ReadAPI:+ api = ReadAPI()+ monkeypatch.setattr(ssrf_proxy, "post", api.post)+ return api+++ ("entity", "project", "expected_entity", "requests"),+ [("team", "demo", "team", 2), (None, "team/demo", "team", 2), (None, "demo", "default-team", 3)],+)+def test_verify_resolves_project_with_request_scoped_authentication(+ read_api: ReadAPI, entity: str | None, project: str, expected_entity: str, requests: int+) -> None:+ result = verify_configuration(WeaveConfig(api_key="synthetic-key", entity=entity, project=project))+ assert result.entity == expected_entity+ assert result.project == "demo"+ assert result.project_url == f"https://wandb.ai/{expected_entity}/demo"+ assert len(read_api.calls) == requests+ for _url, options in read_api.calls:+ assert options["auth"] == ("api", "synthetic-key")+ assert options["max_retries"] == 0+ assert options["follow_redirects"] is False+ assert options["timeout"] == 30.0+ assert object_dict(read_api.calls[-1][1]["json"])["project_id"] == f"{expected_entity}/demo"+++ ("entity", "project"),+ [+ ("team", ""),+ ("team", ".."),+ ("/", "demo"),+ (None, "/demo"),+ (None, "team/"),+ (None, "team/sub/demo"),+ ("other", "team/demo"),+ ("team", "de\\mo"),+ ],+)+def test_invalid_target_is_rejected_before_network(read_api: ReadAPI, entity: str | None, project: str) -> None:+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity=entity, project=project))+ assert caught.value.code == "invalid_project"+ assert read_api.calls == []+++def test_private_endpoints_preserve_paths_and_encode_project_links(read_api: ReadAPI) -> None:+ config = WeaveConfig(+ api_key="synthetic-key",+ entity="team one",+ project="demo & notes#",+ host="http://wandb.internal/wandb/",+ endpoint="https://trace.internal/proxy/",+ )+ result = verify_configuration(config)+ assert result.project_url == "http://wandb.internal/wandb/team%20one/demo%20%26%20notes%23"+ assert [url for url, _options in read_api.calls] == [+ "http://wandb.internal/wandb/graphql",+ "https://trace.internal/proxy/calls/query_stats",+ ]+++ "host", ["https://user:[email protected]", "https://host.test/?secret=x", "https://host.test/#x"]+)+def test_unsafe_host_is_rejected_without_authentication_request(read_api: ReadAPI, host: str) -> None:+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo", host=host))+ assert caught.value.code == "invalid_endpoint"+ assert read_api.calls == []+++ ("status", "code"),+ [+ (401, "authentication_failed"),+ (403, "permission_denied"),+ (302, "endpoint_unavailable"),+ (404, "endpoint_unavailable"),+ (429, "endpoint_unavailable"),+ (503, "endpoint_unavailable"),+ ],+)+def test_http_errors_are_sanitized_and_never_retried(read_api: ReadAPI, status: int, code: str) -> None:+ read_api.override = lambda url, _body: response(url, {"error": "never-log-secret"}, status=status)+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="never-log-secret", entity="team", project="demo"))+ assert caught.value.code == code+ assert "never-log-secret" not in str(caught.value)+ assert len(read_api.calls) == 1+++ ("fields", "code"),+ [+ ({"readOnly": True}, "project_read_only"),+ ({"readOnly": None}, "project_permission_unknown"),+ ({"readOnly": "false"}, "malformed_response"),+ ({"id": ""}, "malformed_response"),+ ({"name": "other"}, "invalid_project"),+ ({"entityName": "other"}, "invalid_project"),+ ],+)+def test_project_permission_and_identity_are_checked(read_api: ReadAPI, fields: dict[str, object], code: str) -> None:+ read_api.project_fields = fields+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))+ assert caught.value.code == code+ assert len(read_api.calls) == 1+++def test_missing_project_is_not_created(read_api: ReadAPI) -> None:+ read_api.project_exists = False+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))+ assert caught.value.code == "project_not_found_or_inaccessible"+ assert len(read_api.calls) == 1+++def test_anonymous_viewer_cannot_validate_a_public_project(read_api: ReadAPI) -> None:+ read_api.viewer_exists = False+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))+ assert caught.value.code == "authentication_failed"++[email protected]("count", [-1, True, "0", None])+def test_invalid_trace_endpoint_response_is_rejected(read_api: ReadAPI, count: object) -> None:+ read_api.count = count+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))+ assert caught.value.code == "malformed_response"+ assert len(read_api.calls) == 2+++ ("body", "code"),+ [+ ({"errors": [{"message": "never-log-secret"}]}, "graphql_error"),+ ({"errors": "never-log-secret"}, "malformed_response"),+ ({"data": []}, "malformed_response"),+ ({"data": {}}, "malformed_response"),+ ],+)+def test_graphql_errors_and_invalid_envelopes_are_rejected(+ read_api: ReadAPI, body: dict[str, object], code: str+) -> None:+ read_api.override = lambda url, _body: response(url, body)+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="never-log-secret", entity="team", project="demo"))+ assert caught.value.code == code+ assert "never-log-secret" not in str(caught.value)+++def test_connection_failure_has_no_raw_exception_or_secret(read_api: ReadAPI) -> None:+ def fail(_url: str, _body: dict[str, object]) -> httpx.Response:+ raise httpx.ReadTimeout("never-log-secret")++ read_api.override = fail+ with pytest.raises(WeaveConfigurationError) as caught:+ verify_configuration(WeaveConfig(api_key="synthetic-key", entity="team", project="demo"))+ assert caught.value.code == "connection_failed"+ assert "never-log-secret" not in str(caught.value)+ assert caught.value.__cause__ is None+++def test_sequential_tenants_use_only_their_supplied_credentials(read_api: ReadAPI) -> None:+ for tenant in ("a", "b"):+ verify_configuration(WeaveConfig(api_key=f"synthetic-{tenant}", entity=f"team-{tenant}", project="demo"))+ assert [options["auth"] for _url, options in read_api.calls] == [+ ("api", "synthetic-a"),+ ("api", "synthetic-a"),+ ("api", "synthetic-b"),+ ("api", "synthetic-b"),+ ]+++def test_project_url_and_console_fallback_are_pure(read_api: ReadAPI) -> None:+ assert project_url(WeaveConfig(api_key="synthetic", project="demo")) is None+ assert project_url(WeaveConfig(api_key="synthetic", project="team/demo")) == "https://wandb.ai/team/demo"+ assert console_url(WeaveConfig(api_key="synthetic", project="demo")) == "https://wandb.ai"+ assert console_url(WeaveConfig(api_key="synthetic", project="demo", host="https://private.test/wandb")) == (+ "https://private.test/wandb"+ )+ assert read_api.calls == []diff --git a/api/tests/unit_tests/services/tracing/test_workflow_tool_tracing_boundary.py b/api/tests/unit_tests/services/tracing/test_workflow_tool_tracing_boundary.pynew file mode 100644index 00000000000..4c0a881ba12--- /dev/null+++ b/api/tests/unit_tests/services/tracing/test_workflow_tool_tracing_boundary.py@@ -0,0 +1,292 @@+"""Hidden Workflow Tool Agents must not invent a trace owned by the source app."""++import json+from copy import deepcopy+from dataclasses import replace+from datetime import UTC, datetime, timedelta++import pytest+from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from core.ops.entities.trace_entity import WorkflowTraceInfo+from core.ops.unified_trace.trace_builder import CanonicalTraceBuilder+from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository+from core.workflow.nodes.agent_v2.runtime_request_builder import WorkflowAgentRuntimeBuildContext+from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus, WorkflowType+from models.application_trace import ApplicationTraceRun+from models.enums import CreatorUserRole+from models.model import App, AppMode, TraceAppConfig+from models.workflow import (+ WorkflowNodeExecutionModel,+ WorkflowNodeExecutionTriggeredFrom,+ WorkflowRun,+ WorkflowRunTriggeredFrom,+)+from repositories.application_trace_repository import ApplicationTraceRepository+from services.tracing import runtime_bridge+from services.tracing.normalization import remap_outer_trace+from services.tracing.service import ApplicationTracingService+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.model_factories import make_account, make_app, make_workflow+from tests.unit_tests.services.tracing.test_application_trace_collection import (+ Collection,+ batch_for,+ identifier,+ ingest,+)+from tests.unit_tests.services.tracing.test_runtime_bridge import workflow_context, workflow_entity+++def seed(+ sessions: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ *,+ root_has_agent: bool,+ source_enabled: bool = True,+) -> tuple[ApplicationTracingService, WorkflowAgentRuntimeBuildContext, dict[str, str]]:+ ids = {+ name: identifier("tool-boundary-" + name)+ for name in [+ "tenant",+ "root",+ "source",+ "root_workflow",+ "source_workflow",+ "run",+ "root_agent",+ "root_tool",+ "child",+ "actor",+ ]+ }+ root_graph: dict[str, list[dict[str, object]]] = {+ "nodes": [{"id": "tool", "data": {"type": "tool", "title": "Root tool"}}],+ "edges": [],+ }+ if root_has_agent:+ root_graph["nodes"].append(+ {+ "id": "root-agent",+ "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Root Agent"},+ }+ )+ now = datetime(2026, 10, 21, tzinfo=UTC)+ with sessions.begin() as session:+ for app, provider, enabled, content in [+ ("root", "langfuse", True, False),+ ("source", "langsmith", source_enabled, True),+ ]:+ session.add(+ make_app(+ app_id=ids[app],+ tenant_id=ids["tenant"],+ mode=AppMode.WORKFLOW,+ tracing=json.dumps({"enabled": enabled, "tracing_provider": provider, "capture_content": content}),+ )+ )+ session.add(+ make_workflow(+ workflow_id=ids[app + "_workflow"],+ app_id=ids[app],+ tenant_id=ids["tenant"],+ graph=root_graph if app == "root" else {"nodes": [], "edges": []},+ )+ )+ session.add(+ TraceAppConfig(+ app_id=ids[app],+ tracing_provider=provider,+ tracing_config={"project_url": "https://tracing.test/" + app},+ )+ )+ session.add(+ WorkflowRun(+ id=ids["run"],+ tenant_id=ids["tenant"],+ app_id=ids["root"],+ workflow_id=ids["root_workflow"],+ type=WorkflowType.WORKFLOW,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ graph=json.dumps(root_graph),+ inputs="{}",+ status=WorkflowExecutionStatus.SUCCEEDED,+ created_by=ids["actor"],+ created_by_role=CreatorUserRole.ACCOUNT,+ created_at=now,+ finished_at=now + timedelta(seconds=3),+ )+ )+ for key, node_type in [("root_agent", "agent"), ("root_tool", "tool"), ("child", "agent")]:+ child = key == "child"+ app = "source" if child else "root"+ session.add(+ WorkflowNodeExecutionModel(+ id=ids[key],+ tenant_id=ids["tenant"],+ app_id=ids[app],+ workflow_id=ids[app + "_workflow"],+ workflow_run_id=ids["run"],+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_TOOL+ if child+ else WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ triggered_from_workflow_id=ids["root_workflow"] if child else None,+ triggered_from_node_execution_id=ids["root_tool"] if child else None,+ node_execution_id=ids[key],+ index=1,+ node_id=key,+ node_type=node_type,+ title=key,+ status=WorkflowNodeExecutionStatus.SUCCEEDED,+ created_by=ids["actor"],+ created_by_role=CreatorUserRole.ACCOUNT,+ created_at=now,+ finished_at=now + timedelta(seconds=2),+ inputs="{}",+ outputs="{}",+ process_data=json.dumps(+ {"workflow_tool_root_app_id": ids["root"], "workflow_tool_invocation_id": "tool-call"}+ )+ if child+ else "{}",+ )+ )+ service = ApplicationTracingService(ApplicationTraceRepository(sessions))+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ monkeypatch.setattr(runtime_bridge, "_service", lambda: service)+ entity = workflow_entity()+ entity.app_config = entity.app_config.model_copy(+ update={"app_id": ids["root"], "tenant_id": ids["tenant"], "workflow_id": ids["root_workflow"]}+ )+ runtime_bridge.freeze_workflow_run(+ entity,+ workflow_id=ids["root_workflow"],+ workflow_run_id=ids["run"],+ conversation_id=None,+ graph_data=root_graph,+ )+ original = workflow_context(False)+ context = replace(+ original,+ dify_context=original.dify_context.model_copy(+ update={"tenant_id": ids["tenant"], "app_id": ids["source"], "workflow_tool_invocation_id": "tool-call"}+ ),+ workflow_id=ids["source_workflow"],+ workflow_run_id=ids["run"],+ node_execution_id=ids["child"],+ )+ return service, context, ids++[email protected]("root_has_agent", [False, True])[email protected]("source_enabled", [False, True])+def test_hidden_child_never_reserves_source_policy_or_changes_root_freeze(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+ root_has_agent: bool,+ source_enabled: bool,+) -> None:+ service, context, ids = seed(+ sqlite_session_factory, monkeypatch, root_has_agent=root_has_agent, source_enabled=source_enabled+ )+ with sqlite_session_factory() as session:+ before = deepcopy([dict(row) for row in session.execute(select(ApplicationTraceRun.__table__)).mappings()])+ service_lookups: list[bool] = []++ def lookup() -> ApplicationTracingService:+ service_lookups.append(True)+ return service++ monkeypatch.setattr(runtime_bridge, "_service", lookup)+ # Marked children never consult policy or ownership, even for malformed+ # empty markers or a foreign tenant. They cannot create an orphan B run.+ for tenant in (ids["tenant"], identifier("foreign-tool-tenant")):+ for marker in ("tool-call", ""):+ child = replace(+ context,+ dify_context=context.dify_context.model_copy(+ update={+ "tenant_id": tenant,+ "workflow_tool_invocation_id": marker,+ }+ ),+ )+ assert runtime_bridge.prepare_workflow_agent(child, None) is None+ assert service_lookups == []+ with sqlite_session_factory() as session:+ observed = [+ (r.app_id, r.provider, r.capture_content, r.source_id) for r in session.scalars(select(ApplicationTraceRun))+ ]+ assert observed == ([(ids["root"], "langfuse", False, ids["run"])] if root_has_agent else [])+ after = [dict(row) for row in session.execute(select(ApplicationTraceRun.__table__)).mappings()]+ assert after == before+ assert (+ service.repository.source_run(+ tenant_id=ids["tenant"], app_id=ids["source"], source_kind="workflow", source_id=ids["run"]+ )+ is None+ )+++def test_hidden_child_skip_keeps_normal_root_agent_trace_assemblable(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ service, child, ids = seed(sqlite_session_factory, monkeypatch, root_has_agent=True)+ assert runtime_bridge.prepare_workflow_agent(child, None) is None+ root_context = replace(+ child,+ dify_context=child.dify_context.model_copy(update={"app_id": ids["root"], "workflow_tool_invocation_id": None}),+ workflow_id=ids["root_workflow"],+ node_execution_id=ids["root_agent"],+ )+ runtime = runtime_bridge.prepare_workflow_agent(root_context, None)+ assert runtime is not None+ run = service.repository.get(str(runtime.run_id))+ assert run is not None+ assert run.app_id == ids["root"]+ frozen = deepcopy(run.provider_config)+ with sqlite_session_factory.begin() as session:+ owner = session.get(App, ids["root"])+ assert owner is not None+ owner.tracing = json.dumps({"enabled": True, "tracing_provider": "langsmith", "capture_content": True})+ workflow = session.get(WorkflowRun, ids["run"])+ assert workflow is not None+ actor = make_account(account_id=ids["actor"])+ nodes = SQLAlchemyWorkflowNodeExecutionRepository(+ session_factory=sqlite_session_factory,+ tenant_id=ids["tenant"],+ user=actor,+ app_id=ids["root"],+ triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,+ )+ info = WorkflowTraceInfo(+ workflow_data=workflow,+ workflow_id=ids["root_workflow"],+ tenant_id=ids["tenant"],+ workflow_run_id=ids["run"],+ workflow_run_elapsed_time=3,+ workflow_run_status="succeeded",+ workflow_run_inputs={},+ workflow_run_outputs={},+ workflow_run_version="1",+ total_tokens=0,+ file_list=[],+ query="",+ metadata={"app_id": ids["root"]},+ )+ outer = CanonicalTraceBuilder(lambda _info: nodes.get_by_workflow_execution(ids["run"])).build(info)+ assert outer is not None+ outer = remap_outer_trace(outer, trace_id=run.trace_id, capture_content=False, agent_message=False)+ service.repository.terminal(run.id, outer.model_dump(mode="json"), outer.root_span_id)+ invocation = service.repository.collected(run.id)[0][0]+ ingest(Collection(service.repository, sqlite_session_factory, run), batch_for(run, invocation))+ current = service.repository.get(run.id)+ assert current is not None+ trace = service.assemble(current)+ span_ids = {span.id for span in trace.spans}+ assert all(span.parent_id is None or span.parent_id in span_ids for span in trace.spans)+ assert current.provider_config == frozen+ assert current.provider == "langfuse"+ assert current.capture_content is Falsediff --git a/api/tests/unit_tests/tasks/test_application_trace_task.py b/api/tests/unit_tests/tasks/test_application_trace_task.pynew file mode 100644index 00000000000..f9713f5be4a--- /dev/null+++ b/api/tests/unit_tests/tasks/test_application_trace_task.py@@ -0,0 +1,72 @@+"""Exercise Celery entrypoints against durable application trace state."""++from datetime import timedelta++import pytest+from sqlalchemy.orm import Session, sessionmaker++from core.ops.unified_trace.entities import CanonicalTrace+from libs.datetime_utils import naive_utc_now+from models.application_trace import ApplicationTraceRun+from services.tracing import service as service_module+from services.tracing.provider_dispatch import TraceExportResult+from services.tracing.service import ApplicationTracingService+from tasks.application_trace_task import export_application_trace, reconcile_application_traces+from tests.unit_tests.config_override import apply_config_overrides+from tests.unit_tests.services.tracing.test_application_trace_collection import batch_for, ingest, seed, terminal+++def test_worker_task_exports_once_and_replayed_task_observes_checkpoint(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ case = seed(sqlite_session_factory, "worker-task")+ ingest(case, batch_for(case.run, case.invocation()))+ terminal(case)+ service = ApplicationTracingService(case.repository)+ monkeypatch.setattr(service_module, "application_tracing_service", lambda: service)+ monkeypatch.setattr(service_module.OpsTraceManager, "decrypt_tracing_config", lambda *_args: {})+ exports: list[CanonicalTrace] = []++ def export(_provider: str, _config: dict[str, object], trace: CanonicalTrace) -> TraceExportResult:+ exports.append(trace)+ return TraceExportResult("vendor-id", "https://example.test/trace/vendor-id")++ monkeypatch.setattr(service_module, "export_trace", export)+ export_application_trace.run(case.run.id)+ export_application_trace.run(case.run.id)+ assert len(exports) == 1+ assert exports[0].trace_id == case.run.trace_id+ result = case.repository.get(case.run.id)+ assert result is not None+ assert result.state == "accepted"+ assert result.attempts == 1+ assert result.trace_url == "https://example.test/trace/vendor-id"+++def test_reconcile_only_schedules_due_pending_rows_and_applies_retention(+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_RETENTION_DAYS=3)+ due = seed(sqlite_session_factory, "due")+ deferred = seed(sqlite_session_factory, "future")+ accepted = seed(sqlite_session_factory, "accepted-expired")+ with sqlite_session_factory.begin() as session:+ future = session.get(ApplicationTraceRun, deferred.run.id)+ expired = session.get(ApplicationTraceRun, accepted.run.id)+ assert future is not None+ assert expired is not None+ future.state = "retrying"+ future.next_attempt_at = naive_utc_now() + timedelta(hours=1)+ expired.state = "accepted"+ expired.created_at = naive_utc_now() - timedelta(days=4)+ service = ApplicationTracingService(due.repository)+ scheduled: list[str] = []+ monkeypatch.setattr(service, "schedule", scheduled.append)+ monkeypatch.setattr(service_module, "application_tracing_service", lambda: service)+ reconcile_application_traces.run()+ assert scheduled == [due.run.id]+ assert due.repository.get(accepted.run.id) is None+ assert due.repository.get(deferred.run.id) is not None+ assert due.repository.get(due.run.id) is not Nonediff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.pyindex 33e000371dc..79dc983f267 100644--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py@@ -22,7 +22,6 @@ from unittest.mock import MagicMock, patchimport pytestfrom celery.exceptions import Retry-from flask import Flaskfrom sqlalchemy import Engine, eventfrom sqlalchemy.exc import OperationalErrorfrom sqlalchemy.orm import Session, sessionmaker@@ -134,10 +133,14 @@ def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:@pytest.fixture-def elasticsearch_delete_endpoint() -> Iterator[tuple[int, list[int], list[tuple[str, int]]]]:+def elasticsearch_delete_endpoint(+ monkeypatch: pytest.MonkeyPatch,+) -> Iterator[tuple[int, list[int], list[tuple[str, int]]]]:"""Serve real SDK requests with Elasticsearch's missing-index response shape."""pytest.importorskip("elasticsearch")- pytest.importorskip("dify_vdb_elasticsearch.elasticsearch_vector")+ elasticsearch_vector = pytest.importorskip("dify_vdb_elasticsearch.elasticsearch_vector")+ from elasticsearch import Elasticsearch+responses: list[int] = []requests: list[tuple[str, int]] = []@@ -174,9 +177,23 @@ def elasticsearch_delete_endpoint() -> Iterator[tuple[int, list[int], list[tupleserver = HTTPServer(("127.0.0.1", 0), Handler)thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)thread.start()+ client = Elasticsearch(+ [f"http://127.0.0.1:{server.server_port}"],+ request_timeout=2,+ max_retries=0,+ retry_on_timeout=False,+ )++ def local_client(**_options: object) -> Elasticsearch:+ return client++ # Celery can enter a different Flask app from the test fixture's app.+ # Bind only SDK construction; keep real ping/info/delete and error decoding.+ monkeypatch.setattr(elasticsearch_vector, "Elasticsearch", local_client)try:yield server.server_port, responses, requestsfinally:+ client.close()server.shutdown()thread.join(timeout=1)server.server_close()@@ -662,8 +679,6 @@ class TestIndexProcessorParameters:def test_missing_elasticsearch_index_allows_parent_child_cleanup_after_retry(self,commit_fails: bool,- app: Flask,- monkeypatch: pytest.MonkeyPatch,elasticsearch_delete_endpoint: tuple[int, list[int], list[tuple[str, int]]],dataset_id: str,tenant_id: str,@@ -672,20 +687,9 @@ class TestIndexProcessorParameters:mock_storage: MagicMock,caplog: pytest.LogCaptureFixture,) -> None:- port, responses, requests = elasticsearch_delete_endpoint+ _port, responses, requests = elasticsearch_delete_endpointexpected_statuses = [200, 404] if commit_fails else [404]responses.extend(expected_statuses)- for key, value in {- "ELASTICSEARCH_USE_CLOUD": False,- "ELASTICSEARCH_HOST": "http://127.0.0.1",- "ELASTICSEARCH_PORT": port,- "ELASTICSEARCH_USERNAME": "elastic",- "ELASTICSEARCH_PASSWORD": "test",- "ELASTICSEARCH_REQUEST_TIMEOUT": 2,- "ELASTICSEARCH_MAX_RETRIES": 0,- "ELASTICSEARCH_RETRY_ON_TIMEOUT": False,- }.items():- monkeypatch.setitem(app.config, key, value)document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)segment = _persist_segment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.iddiff --git a/api/tests/unit_tests/tasks/test_ops_trace_task.py b/api/tests/unit_tests/tasks/test_ops_trace_task.pyindex c23372250ae..95960c69239 100644--- a/api/tests/unit_tests/tasks/test_ops_trace_task.py+++ b/api/tests/unit_tests/tasks/test_ops_trace_task.py@@ -1,16 +1,30 @@import jsonimport sys-from contextlib import contextmanager+from contextlib import contextmanager, nullcontextfrom types import ModuleTypefrom unittest.mock import MagicMock, patchimport pytestfrom celery.exceptions import Retry+from flask import Flaskfrom configs import dify_config+from core.ops.base_trace_instance import (+ ApplicationTracingHooksNotRegisteredError,+ bind_application_tracing_hooks,+)from core.ops.entities.config_entity import OPS_TRACE_FAILED_KEY-from core.ops.exceptions import RetryableTraceDispatchError+from core.ops.entities.trace_entity import BaseTraceInfo+from core.ops.exceptions import PendingTraceParentContextError, RetryableTraceDispatchError+from services.tracing.provider_dispatch import TraceExportUncertainError+from services.tracing.runtime_bridge import RuntimeApplicationTracingHooksfrom tasks.ops_trace_task import process_trace_tasks+from tests.unit_tests.config_override import apply_config_overrides++[email protected](autouse=True)+def _application_tracing_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=False)@contextmanager@@ -81,10 +95,20 @@ def _run_task(file_info: dict[str, str], retries: int = 0) -> None:process_trace_tasks.pop_request()-def test_process_trace_tasks_retries_retryable_dispatch_failure_and_preserves_payload():+ "pending_error",+ [+ _retryable_dispatch_error(),+ PendingTraceParentContextError("parent-node"),+ TraceExportUncertainError("DO_NOT_EXPOSE_PROVIDER_BODY"),+ ],+)+def test_process_trace_tasks_retries_only_confirmed_retryable_dispatch_failures(+ pending_error: RetryableTraceDispatchError,+ caplog: pytest.LogCaptureFixture,+) -> None:file_info = {"app_id": "app-id", "file_id": "file-id"}trace_instance = MagicMock()- pending_error = _retryable_dispatch_error()trace_instance.trace.side_effect = pending_errorretry_error = Retry()@@ -95,16 +119,23 @@ def test_process_trace_tasks_retries_retryable_dispatch_failure_and_preserves_papatch("tasks.ops_trace_task.storage.delete") as mock_delete,patch("tasks.ops_trace_task.redis_client.incr") as mock_incr,patch.object(process_trace_tasks, "retry", side_effect=retry_error) as mock_retry,- pytest.raises(Retry),+ pytest.raises(Retry) if not pending_error.acceptance_unknown else nullcontext(),):_run_task(file_info)- mock_retry.assert_called_once_with(- exc=pending_error,- countdown=process_trace_tasks.default_retry_delay,- )- mock_delete.assert_not_called()- mock_incr.assert_not_called()+ trace_instance.trace.assert_called_once_with({})+ if pending_error.acceptance_unknown:+ mock_retry.assert_not_called()+ mock_delete.assert_called_once_with("ops_trace/app-id/file-id.json")+ mock_incr.assert_called_once_with(f"{OPS_TRACE_FAILED_KEY}_app-id")+ assert "DO_NOT_EXPOSE_PROVIDER_BODY" not in caplog.text+ else:+ mock_retry.assert_called_once_with(+ exc=pending_error,+ countdown=process_trace_tasks.default_retry_delay,+ )+ mock_delete.assert_not_called()+ mock_incr.assert_not_called()def test_process_trace_tasks_marks_enterprise_trace_dispatched_before_retryable_dispatch_retry():@@ -303,3 +334,150 @@ def test_process_trace_tasks_deletes_payload_and_counts_exhausted_retryable_dispmock_retry.assert_not_called()mock_delete.assert_called_once_with("ops_trace/app-id/file-id.json")mock_incr.assert_called_once_with(f"{OPS_TRACE_FAILED_KEY}_app-id")+++def test_managed_application_capture_receives_restored_info_and_preserves_enterprise(monkeypatch: pytest.MonkeyPatch):+ from core.ops.entities.trace_entity import MessageTraceInfo++ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ calls: list[BaseTraceInfo] = []++ def capture(info: BaseTraceInfo) -> bool:+ calls.append(info)+ return True++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "capture_terminal", staticmethod(capture))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)+ enterprise_calls: list[BaseTraceInfo] = []++ class RecordingEnterpriseTrace:+ def trace(self, info: BaseTraceInfo) -> None:+ enterprise_calls.append(info)++ payload = json.dumps(+ {+ "trace_info_type": "MessageTraceInfo",+ "trace_info": {+ "metadata": {},+ "message_id": "message-id",+ "conversation_model": "agent",+ "conversation_mode": "agent",+ "message_tokens": 0,+ "answer_tokens": 0,+ "total_tokens": 0,+ },+ }+ )+ with (+ app.app_context(),+ patch("extensions.ext_enterprise_telemetry.is_enabled", return_value=True),+ patch("enterprise.telemetry.enterprise_trace.EnterpriseOtelTrace", new=RecordingEnterpriseTrace),+ patch(+ "core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance",+ side_effect=AssertionError("managed traces must not instantiate legacy providers"),+ ) as lookup,+ patch("tasks.ops_trace_task.storage.load", return_value=payload),+ patch("tasks.ops_trace_task.storage.delete") as deleted,+ ):+ _run_task({"app_id": "app-id", "file_id": "file-id"})+ assert len(calls) == 1+ restored = calls[0]+ assert isinstance(restored, MessageTraceInfo)+ assert restored.message_id == "message-id"+ assert enterprise_calls == [restored]+ lookup.assert_not_called()+ deleted.assert_called_once()++[email protected]("enabled", [False, True])+def test_nonmanaged_task_uses_legacy_and_disabled_gate_never_calls_collector(+ monkeypatch: pytest.MonkeyPatch, enabled: bool+):+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=enabled)+ legacy_calls: list[object] = []++ class RecordingLegacyTrace:+ def trace(self, info: object) -> None:+ legacy_calls.append(info)++ legacy = RecordingLegacyTrace()+ calls: list[BaseTraceInfo] = []++ def capture(info: BaseTraceInfo) -> bool:+ calls.append(info)+ return False++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "capture_terminal", staticmethod(capture))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)+ with (+ app.app_context(),+ patch("extensions.ext_enterprise_telemetry.is_enabled", return_value=False),+ patch("core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance", return_value=legacy) as lookup,+ patch("tasks.ops_trace_task.storage.load", return_value=_make_payload()),+ patch("tasks.ops_trace_task.storage.delete"),+ ):+ _run_task({"app_id": "app-id", "file_id": "file-id"})+ assert calls == ([{}] if enabled else [])+ assert legacy_calls == [{}]+ lookup.assert_called_once_with("app-id")+++def test_capture_persistence_failure_preserves_payload_without_legacy_export(monkeypatch: pytest.MonkeyPatch):+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ pending_error = RetryableTraceDispatchError("database busy")+ calls: list[BaseTraceInfo] = []++ def capture(info: BaseTraceInfo) -> bool:+ calls.append(info)+ raise pending_error++ monkeypatch.setattr(RuntimeApplicationTracingHooks, "capture_terminal", staticmethod(capture))+ hooks = RuntimeApplicationTracingHooks()+ app = Flask(__name__)+ bind_application_tracing_hooks(app, hooks)+ with (+ app.app_context(),+ patch(+ "core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance",+ side_effect=AssertionError("capture failures must not instantiate legacy providers"),+ ) as lookup,+ patch("tasks.ops_trace_task.storage.load", return_value=_make_payload()),+ patch("tasks.ops_trace_task.storage.delete") as deleted,+ patch.object(process_trace_tasks, "retry", side_effect=Retry()) as retry,+ pytest.raises(Retry),+ ):+ _run_task({"app_id": "app-id", "file_id": "file-id"})+ lookup.assert_not_called()+ deleted.assert_not_called()++ assert calls == [{}]+ retry.assert_called_once_with(exc=pending_error, countdown=process_trace_tasks.default_retry_delay)+++def test_unregistered_application_hooks_preserve_payload_without_legacy_export(monkeypatch: pytest.MonkeyPatch) -> None:+ apply_config_overrides(monkeypatch, OPS_AGENT_TRACE_ENABLED=True)+ app = Flask(__name__)+ with (+ app.app_context(),+ patch(+ "core.ops.ops_trace_manager.OpsTraceManager.get_ops_trace_instance",+ side_effect=AssertionError("unregistered hooks must not instantiate legacy providers"),+ ) as lookup,+ patch("tasks.ops_trace_task.storage.load", return_value=_make_payload()),+ patch("tasks.ops_trace_task.storage.delete") as deleted,+ patch("tasks.ops_trace_task.redis_client.incr") as failed,+ patch.object(process_trace_tasks, "retry", side_effect=Retry()) as retry,+ pytest.raises(Retry),+ ):+ _run_task({"app_id": "app-id", "file_id": "file-id"})++ deleted.assert_not_called()+ failed.assert_not_called()+ retry.assert_called_once()+ lookup.assert_not_called()+ assert isinstance(retry.call_args.kwargs["exc"], ApplicationTracingHooksNotRegisteredError)+ assert retry.call_args.kwargs["countdown"] == process_trace_tasks.default_retry_delaydiff --git a/dify-agent/.example.env b/dify-agent/.example.envindex 00afe2698aa..cf45162451e 100644--- a/dify-agent/.example.env+++ b/dify-agent/.example.env@@ -108,3 +108,5 @@ DIFY_AGENT_LOCAL_SANDBOX_MATERIALIZED_HOME_ROOT=/home/difyDIFY_AGENT_LOCAL_SANDBOX_WORKSPACE_ROOT=/workspaceDIFY_AGENT_LOCAL_SANDBOX_HOME_SNAPSHOT_ROOT=/home/dify/.snapshotsLOGFIRE_CONSOLE=false+# Application tracing uses only API-issued run policy and the inner API above.+DIFY_AGENT_APPLICATION_TRACING_ENABLED=falsediff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.mdindex 812f6971b63..bcfd6cf9eb3 100644--- a/dify-agent/docs/dify-agent/guide/index.md+++ b/dify-agent/docs/dify-agent/guide/index.md@@ -82,6 +82,7 @@ in [E2B runtime metering](runtime-metering.md).| `DIFY_AGENT_OUTBOUND_HTTP_MAX_KEEPALIVE_CONNECTIONS` | `20` | Maximum idle shared outbound HTTP connections. || `DIFY_AGENT_OUTBOUND_HTTP_KEEPALIVE_EXPIRY` | `30` | Idle keep-alive expiry in seconds. || `DIFY_AGENT_TRAJECTORY_ENABLED` | `false` | Opt-in switch for the separate Agent trajectory export pipeline. |+| `DIFY_AGENT_APPLICATION_TRACING_ENABLED` | `false` | Collect API-authorized application spans into the Dify inner API tracing inbox. Requires the server inner API URL and key. || `DIFY_AGENT_TRAJECTORY_OTLP_TRACES_ENDPOINT` | empty | Full HTTP(S) OTLP traces endpoint required when trajectory export is enabled. || `DIFY_AGENT_TRAJECTORY_OTLP_HEADERS` | `{}` | JSON object of request headers sent with Agent OTLP trace exports. || `DIFY_AGENT_TRAJECTORY_SERVICE_NAME` | `dify-agent-trajectory` | Resource service name attached to Agent trajectory spans. |@@ -119,6 +120,48 @@ separate. The mode is fixed at process initialization and requires a restartto change. No extra shared root span is added; incomplete traces may stillappear temporarily while a run or export is in progress.+Application tracing is a third, independent pipeline. When enabled, a run's+API-issued `execution_context.tracing` supplies its durable run/invocation IDs,+OpenTelemetry parent, session and content/sampling policy. Its spans are sent as+JSON to `POST /inner/api/agent/tracing/batches`, authenticated with the server's+`DIFY_AGENT_INNER_API_KEY` in `X-Inner-Api-Key`. Provider credentials and routing+remain in the Dify API. Application spans do not inherit platform or deployment+trajectory destinations, and one Agent is never instrumented by both application+and deployment pipelines. Binary content is excluded.++The actual model-call span also carries the Dify gateway's billed amount as+`dify.llm.total_price` (a decimal string) and `dify.llm.currency` when a usage+payload is returned. `gen_ai.response.time_to_first_token` is measured in seconds+(`dify.llm.time_to_first_token_unit=s`): gateway-provided TTFT takes precedence;+otherwise it measures request issue to the first content/tool-call chunk.+`dify.llm.time_to_first_token_source` distinguishes those two sources. Missing+accounting is left absent. The current Graphon LLM gateway does not expose cache+token counters, so the Agent adapter does not manufacture them.++Each batch has a stable `batch_id`. Retries reuse the same payload, and a 2xx+response is accepted only with `{result: "success", batch_id, accepted_spans}`+matching that batch. An invocation emits an empty `final: true` batch after its+ended spans are flushed and acknowledged, before its normal terminal event when+delivery completes within five seconds. Failure or cancellation still flushes.+Collection failures do not fail the Agent run; outstanding delivery can finish+in the background after the terminal-event deadline. A full in-memory queue or+unacknowledged data suppresses completion instead of claiming a complete trace.+The queue is bounded (2048 spans, batches of at most 128), with up to 128 unacknowledged+batches retained until invocation finalization. Batches are split below 1 MiB+of UTF-8 JSON. A single oversized span is dropped and marks its invocation+incomplete rather than silently truncating content or sending a success marker.+This is not a durable local+spool: process loss can leave an incomplete trace in the API. Shutdown drains the+collector after active runs finish.++If setup fails or cancellation arrives before Pydantic AI starts its Agent span,+the collector emits one failed `invoke_agent` span with the measured setup+interval, then finalizes normally. It records `dify.agent.execution_state` as+`initialization_failed` or `cancelled`. Runs that already started their native+Agent span keep that span without an extra wrapper. Sampling-off invocations+remain span-free; successful but unexplained empty collections are not converted+to synthetic success.+Example `.env`:```envdiff --git a/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md b/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.mdindex c17b4626a19..a81958c10a2 100644--- a/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md+++ b/dify-agent/docs/dify-agent/user-manual/execution-context-layer/index.md@@ -5,3 +5,33 @@ Capability. Config carries tenant/user/app/invoke identity and correlation IDs.Other modules refer to its registered name in their Config. Server-only daemonsettings and clients are borrowed from Deps.services. Knowledge retrievalrequires its complete caller identity before any external request.++The optional `tracing` field is an API-issued `ApplicationTracingContext`:++```json+{+ "version": 1,+ "run_id": "429cab49-8663-42b4-a701-ac1038ee4fa7",+ "invocation_id": "95bdeed9-4652-4708-8322-b866f159ad62",+ "trace_id": "81b617d638eb419ba2b209c14984d908",+ "parent_span_id": "716db0b6c736b395",+ "session_id": "conversation-id",+ "capture_content": false,+ "sampled": true+}+```++`run_id` identifies the API's persisted application trace. `invocation_id`+identifies this particular Agent execution; multiple nodes or attempts can+belong to one application trace. Both are UUIDs. The trace and parent span IDs+are nonzero, lowercase OpenTelemetry hex IDs (32 and 16 characters). The parent+is the API-owned message or Agent-node invocation, and is restored as the actual+OpenTelemetry parent rather than attached only as metadata. Tenant/app identity+continues to come from the containing execution context.++Application collection requires `DIFY_AGENT_APPLICATION_TRACING_ENABLED=true`+on the Agent server. The API owns sampling and content policy per invocation;+no third-party destination or credential is accepted in a run request. When an+application context is selected it replaces deployment trajectory instrumentation+for that invocation, including when `sampled` is false. Without an application+context, the existing deployment trajectory behavior is unchanged.diff --git a/dify-agent/src/dify_agent/adapters/llm/model.py b/dify-agent/src/dify_agent/adapters/llm/model.pyindex 1f4c93a161e..98d1ede1bef 100644--- a/dify-agent/src/dify_agent/adapters/llm/model.py+++ b/dify-agent/src/dify_agent/adapters/llm/model.py@@ -10,13 +10,18 @@ and reads that accumulated usage after all model/tool rounds finish.from __future__ import annotationsimport base64+import logging+import mathimport re+import timefrom collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequencefrom contextlib import asynccontextmanagerfrom dataclasses import KW_ONLY, InitVar, dataclass, fieldfrom datetime import datetime, timezonefrom typing import cast+from opentelemetry.trace import Span, StatusCode, get_current_span+from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMUsagefrom graphon.model_runtime.entities.message_entities import (AssistantPromptMessage,@@ -78,6 +83,7 @@ _THINK_OPEN_TAG = "<think>"_THINK_CLOSE_TAG = "</think>"_THINK_TAG_PATTERN = re.compile(r"<think>(.*?)</think>", re.DOTALL)_DETAIL_HIGH = "high"+logger = logging.getLogger(__name__)@dataclass(slots=True)@@ -233,11 +239,34 @@ class DifyStreamedResponse(StreamedResponse):_timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))_embedded_thinking_parser: "_EmbeddedThinkingParser" = field(default_factory=lambda: _EmbeddedThinkingParser())_dify_usage: LLMUsage | None = field(default=None, init=False, repr=False)+ # Pydantic AI opens its actual chat span before calling the wrapped model.+ # Capture that span here; HTTP transport spans may be current while iterating.+ _model_span: Span = field(default_factory=get_current_span, init=False, repr=False)+ _request_started_at: float = field(default_factory=time.perf_counter, init=False, repr=False)+ _first_content_seconds: float | None = field(default=None, init=False, repr=False)+ _usage_ttft_seconds: float | None = field(default=None, init=False, repr=False)@overrideasync def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]:+ try:+ async for event in self._iterate_gateway_events():+ yield event+ except BaseException as error:+ # Some instrumentation versions end the chat span without marking+ # errors raised while consuming an asynchronous response. Keep the+ # actual model call consistent with the failed Agent envelope.+ try:+ if self._model_span.is_recording():+ self._model_span.set_status(StatusCode.ERROR, "Model request did not complete")+ self._model_span.set_attribute("error.type", type(error).__name__)+ except Exception:+ logger.debug("Unable to record model trace failure")+ raise++ async def _iterate_gateway_events(self) -> AsyncIterator[ModelResponseStreamEvent]:chunk_sequence = 0async for chunk in self.chunks:+ self._observe_trace_usage(chunk)if chunk.delta.usage is not None:self._dify_usage = chunk.delta.usageself._usage: RequestUsage = _map_usage(chunk.delta.usage)@@ -257,6 +286,52 @@ class DifyStreamedResponse(StreamedResponse):for event in self._embedded_thinking_parser.flush(self._parts_manager, self.provider_name_value):yield event+ def _observe_trace_usage(self, chunk: LLMResultChunk) -> None:+ """Attach gateway accounting to the real model span, without inventing usage.++ Dify's usage TTFT is in seconds. If absent, measure from gateway request+ issue to the first content/tool-call chunk, excluding empty usage chunks.+ Price is the gateway's billed decimal amount in its accompanying currency.+ Graphon's current LLMUsage has no cache counters, so none are synthesized.+ """+ try:+ if not self._model_span.is_recording():+ return+ if self._first_content_seconds is None and (chunk.delta.message.content or chunk.delta.message.tool_calls):+ self._first_content_seconds = time.perf_counter() - self._request_started_at+ usage = chunk.delta.usage+ attributes: dict[str, str | int | float] = {}+ ttft = self._first_content_seconds+ source = "gateway_first_content"+ if usage is not None:+ attributes.update(+ {+ "gen_ai.usage.input_tokens": usage.prompt_tokens,+ "gen_ai.usage.output_tokens": usage.completion_tokens,+ "gen_ai.usage.total_tokens": usage.total_tokens,+ }+ )+ if usage.total_price.is_finite() and usage.total_price >= 0 and usage.currency:+ attributes["dify.llm.total_price"] = str(usage.total_price)+ attributes["dify.llm.currency"] = usage.currency+ if (+ usage.time_to_first_token is not None+ and math.isfinite(usage.time_to_first_token)+ and usage.time_to_first_token >= 0+ ):+ self._usage_ttft_seconds = usage.time_to_first_token+ if self._usage_ttft_seconds is not None:+ ttft = self._usage_ttft_seconds+ source = "gateway_usage"+ if ttft is not None and math.isfinite(ttft) and ttft >= 0:+ attributes["gen_ai.response.time_to_first_token"] = ttft+ attributes["dify.llm.time_to_first_token_unit"] = "s"+ attributes["dify.llm.time_to_first_token_source"] = source+ self._model_span.set_attributes(attributes)+ except Exception:+ # Observability must never change a successful/failed model outcome.+ logger.warning("Failed to annotate Dify LLM accounting span", exc_info=True)+@propertydef dify_usage(self) -> LLMUsage | None:"""Return Dify's complete usage for this model request."""diff --git a/dify-agent/src/dify_agent/adapters/llm/provider.py b/dify-agent/src/dify_agent/adapters/llm/provider.pyindex 7c5176a232e..9143b15a78f 100644--- a/dify-agent/src/dify_agent/adapters/llm/provider.py+++ b/dify-agent/src/dify_agent/adapters/llm/provider.py@@ -97,7 +97,9 @@ class DifyApiLLMClient:if missing:raise UserError(f"Agent LLM Gateway requires execution context fields: {', '.join(missing)}")- caller = context.model_dump(mode="json")+ # Application tracing policy belongs to the collector, not the strict+ # metering-gateway caller DTO. Keep it out even when its value is None.+ caller = context.model_dump(mode="json", exclude={"tracing"})caller.update({"invocation_id": invocation_id,diff --git a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.pyindex 0b8c5725e48..b2a4b232a93 100644--- a/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py+++ b/dify-agent/src/dify_agent/layers/dify_core_tools/layer.py@@ -13,6 +13,8 @@ from copy import deepcopyfrom dataclasses import replacefrom typing import cast, override+from opentelemetry.sdk.trace import ReadableSpan+from opentelemetry.trace import StatusCode, get_current_spanfrom pydantic import JsonValuefrom pydantic_ai import RunContext, Toolfrom pydantic_ai.tools import ToolDefinition@@ -93,9 +95,11 @@ class Toolset(FunctionToolset[Deps]):tool_parameters=cast(dict[str, JsonValue], tool_arguments),)return response.observation- except DifyCoreToolsClientConfigurationError:+ except DifyCoreToolsClientConfigurationError as exc:+ _mark_tool_span_failed(_ctx, exc)return "Tool is unavailable because required execution context is missing."except DifyCoreToolsClientError as exc:+ _mark_tool_span_failed(_ctx, exc)return _tool_error_text(tool_name=tool_name, error=exc)async def prepare_tool_definition(_ctx: RunContext[Deps], tool_def: ToolDefinition) -> ToolDefinition:@@ -110,6 +114,30 @@ class Toolset(FunctionToolset[Deps]):)+def _mark_tool_span_failed(ctx: RunContext[Deps], error: DifyCoreToolsClientError) -> None:+ """Keep handled API errors on this native tool span without failing the Agent.++ Pydantic AI wraps tool execution in the current ``execute_tool`` span. When+ instrumentation is disabled, an ambient Agent/platform span must stay intact.+ Only record a safe exception type; the observation retains its existing text+ and the instrumentation's content policy controls whether that text is traced.+ """+ span = get_current_span()+ attributes = span.attributes if isinstance(span, ReadableSpan) else None+ if (+ not span.is_recording()+ or not attributes+ or attributes.get("gen_ai.operation.name") != "execute_tool"+ or ctx.tool_call_id is None+ or attributes.get("gen_ai.tool.call.id") != ctx.tool_call_id+ or attributes.get("gen_ai.tool.name") != ctx.tool_name+ ):+ return+ error_type = type(error).__name__+ span.set_attribute("error.type", error_type)+ span.set_status(StatusCode.ERROR, error_type)++def _tool_error_text(*, tool_name: str, error: DifyCoreToolsClientError) -> str:if error.retryable:return TEMPORARY_UNAVAILABLE_OBSERVATIONdiff --git a/dify-agent/src/dify_agent/layers/execution_context/__init__.py b/dify-agent/src/dify_agent/layers/execution_context/__init__.pyindex 356653ae65c..44060ce0f85 100644--- a/dify-agent/src/dify_agent/layers/execution_context/__init__.py+++ b/dify-agent/src/dify_agent/layers/execution_context/__init__.py@@ -7,6 +7,7 @@ DTO, but that runtime implementation still lives in sibling modules."""from dify_agent.layers.execution_context.configs import (+ ApplicationTracingContext,DifyExecutionContextAgentConfigVersionKind,DifyExecutionContextAgentMode,DifyExecutionContextInvokeFrom,@@ -15,6 +16,7 @@ from dify_agent.layers.execution_context.configs import ()__all__ = [+ "ApplicationTracingContext","DifyExecutionContextAgentConfigVersionKind","DifyExecutionContextAgentMode","DifyExecutionContextInvokeFrom",diff --git a/dify-agent/src/dify_agent/layers/execution_context/configs.py b/dify-agent/src/dify_agent/layers/execution_context/configs.pyindex 1f8d1d51453..1e5242bf2f7 100644--- a/dify-agent/src/dify_agent/layers/execution_context/configs.py+++ b/dify-agent/src/dify_agent/layers/execution_context/configs.py@@ -11,9 +11,10 @@ Server-only plugin-daemon settings are injected by the runtime runtime servicesand therefore do not appear in this public schema."""-from typing import ClassVar, Literal, TypeAlias+from typing import Annotated, ClassVar, Literal, TypeAlias+from uuid import UUID-from pydantic import BaseModel, ConfigDict+from pydantic import BaseModel, ConfigDict, Field, field_validatorDifyExecutionContextAgentMode: TypeAlias = Literal[@@ -37,6 +38,32 @@ DifyExecutionContextInvokeFrom: TypeAlias = Literal[]+class ApplicationTracingContext(BaseModel):+ """API-issued application tracing policy; never contains exporter credentials.++ ``run_id`` refers to the API's durable tracing run, not the Agent backend's+ execution id. The parent identifies the owning message or workflow node.+ """++ version: Literal[1] = 1+ run_id: UUID+ invocation_id: UUID+ trace_id: Annotated[str, Field(pattern=r"^[0-9a-f]{32}$")]+ parent_span_id: Annotated[str, Field(pattern=r"^[0-9a-f]{16}$")]+ session_id: str+ capture_content: bool+ sampled: bool = True++ model_config = ConfigDict(extra="forbid", frozen=True)++ @field_validator("trace_id", "parent_span_id")+ @classmethod+ def validate_nonzero_id(cls, value: str) -> str:+ if int(value, 16) == 0:+ raise ValueError("OpenTelemetry ids must be nonzero")+ return value++class DifyExecutionContextLayerConfig(BaseModel):"""Public config for Dify execution identity and daemon transport context."""@@ -55,11 +82,13 @@ class DifyExecutionContextLayerConfig(BaseModel):agent_mode: DifyExecutionContextAgentModeinvoke_from: DifyExecutionContextInvokeFromtrace_id: str | None = None+ tracing: ApplicationTracingContext | None = Nonemodel_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid", arbitrary_types_allowed=True)__all__ = [+ "ApplicationTracingContext","DifyExecutionContextAgentConfigVersionKind","DifyExecutionContextAgentMode","DifyExecutionContextInvokeFrom",diff --git a/dify-agent/src/dify_agent/protocol/tracing.py b/dify-agent/src/dify_agent/protocol/tracing.pynew file mode 100644index 00000000000..7cdd6ca28cb--- /dev/null+++ b/dify-agent/src/dify_agent/protocol/tracing.py@@ -0,0 +1,59 @@+"""JSON delivery contract between the Agent collector and the Dify inner API."""++from typing import Annotated, Literal+from uuid import UUID++from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator+++class AgentTraceEvent(BaseModel):+ name: str+ timestamp_ns: int = Field(ge=0)+ attributes: dict[str, JsonValue] = Field(default_factory=dict)++ model_config = ConfigDict(extra="forbid")+++class AgentTraceSpan(BaseModel):+ trace_id: Annotated[str, Field(pattern=r"^[0-9a-f]{32}$")]+ span_id: Annotated[str, Field(pattern=r"^[0-9a-f]{16}$")]+ parent_span_id: Annotated[str, Field(pattern=r"^[0-9a-f]{16}$")] | None = None+ name: str+ start_time_ns: int = Field(ge=0)+ end_time_ns: int = Field(ge=0)+ attributes: dict[str, JsonValue] = Field(default_factory=dict)+ status: Literal["ok", "error"]+ status_message: str | None = None+ events: list[AgentTraceEvent] = Field(default_factory=list)++ model_config = ConfigDict(extra="forbid")++ @model_validator(mode="after")+ def validate_span(self) -> "AgentTraceSpan":+ for value in (self.trace_id, self.span_id, self.parent_span_id):+ if value is not None and int(value, 16) == 0:+ raise ValueError("OpenTelemetry ids must be nonzero")+ if self.end_time_ns < self.start_time_ns:+ raise ValueError("span end time must not precede start time")+ return self+++class AgentTraceBatch(BaseModel):+ run_id: UUID+ invocation_id: UUID+ batch_id: UUID+ spans: list[AgentTraceSpan]+ final: bool = False++ model_config = ConfigDict(extra="forbid")+++class AgentTraceBatchAcknowledgement(BaseModel):+ result: Literal["success"]+ batch_id: UUID+ accepted_spans: int = Field(ge=0)++ model_config = ConfigDict(extra="forbid")+++__all__ = ["AgentTraceBatch", "AgentTraceBatchAcknowledgement", "AgentTraceEvent", "AgentTraceSpan"]diff --git a/dify-agent/src/dify_agent/runtime/application_trace_exporter.py b/dify-agent/src/dify_agent/runtime/application_trace_exporter.pynew file mode 100644index 00000000000..585b227431d--- /dev/null+++ b/dify-agent/src/dify_agent/runtime/application_trace_exporter.py@@ -0,0 +1,352 @@+"""Bounded application-span delivery to the API-owned, durable tracing inbox.++The Agent backend never holds third-party credentials. A successful export means+the inner API acknowledged this exact batch. Exhausted retries are retained until+run finalization; a run with dropped or unacknowledged spans never sends ``final``.+"""++from __future__ import annotations++import hashlib+import json+import logging+import threading+import time+from collections import defaultdict, deque+from collections.abc import Sequence+from typing import Final, override+from uuid import UUID, uuid5++import httpx+from opentelemetry.context import Context+from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult+from opentelemetry.trace import StatusCode+from pydantic import JsonValue, TypeAdapter++from dify_agent.protocol.tracing import AgentTraceBatch, AgentTraceBatchAcknowledgement, AgentTraceEvent, AgentTraceSpan++DIFY_TRACING_RUN_ID_ATTRIBUTE: Final[str] = "dify.tracing.run_id"+DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: Final[str] = "dify.tracing.invocation_id"+_JSON_ATTRIBUTES = TypeAdapter(dict[str, JsonValue])+logger = logging.getLogger(__name__)+++def _scope(span: ReadableSpan) -> tuple[UUID, UUID] | None:+ attributes = span.attributes or {}+ try:+ return (+ UUID(str(attributes[DIFY_TRACING_RUN_ID_ATTRIBUTE])),+ UUID(str(attributes[DIFY_TRACING_INVOCATION_ID_ATTRIBUTE])),+ )+ except (KeyError, ValueError):+ return None+++def _serialize_span(span: ReadableSpan) -> AgentTraceSpan:+ context = span.get_span_context()+ if context is None or span.start_time is None or span.end_time is None:+ raise ValueError("Application tracing requires a completed span")+ return AgentTraceSpan(+ trace_id=f"{context.trace_id:032x}",+ span_id=f"{context.span_id:016x}",+ parent_span_id=f"{span.parent.span_id:016x}" if span.parent else None,+ name=span.name,+ start_time_ns=span.start_time,+ end_time_ns=span.end_time,+ attributes=_JSON_ATTRIBUTES.validate_json(json.dumps(dict(span.attributes or {}), allow_nan=False)),+ status="error" if span.status.status_code == StatusCode.ERROR else "ok",+ status_message=span.status.description,+ events=[+ AgentTraceEvent(+ name=event.name,+ timestamp_ns=event.timestamp,+ attributes=_JSON_ATTRIBUTES.validate_json(json.dumps(dict(event.attributes or {}), allow_nan=False)),+ )+ for event in span.events+ ],+ )+++class JSONSpanExporter(SpanExporter):+ """Send idempotent JSON batches using the server-owned inner API credential."""++ def __init__(+ self,+ *,+ inner_api_url: str,+ inner_api_key: str,+ client: httpx.Client | None = None,+ max_attempts: int = 3,+ retry_delay_seconds: float = 0.1,+ max_pending_batches: int = 128,+ max_batch_bytes: int = 1024 * 1024,+ ) -> None:+ self._url = f"{inner_api_url.rstrip('/')}/inner/api/agent/tracing/batches"+ self._headers = {"X-Inner-Api-Key": inner_api_key}+ self._client = client or httpx.Client(timeout=2.0, trust_env=False)+ self._owns_client = client is None+ self._max_attempts = max(1, max_attempts)+ self._retry_delay_seconds = retry_delay_seconds+ self._max_pending_batches = max_pending_batches+ self._max_batch_bytes = max_batch_bytes+ self._pending: dict[UUID, AgentTraceBatch] = {}+ self._incomplete_runs: set[tuple[UUID, UUID]] = set()+ self._state_lock = threading.Lock()+ self._delivery_lock = threading.RLock()++ def mark_incomplete(self, scope: tuple[UUID, UUID]) -> None:+ with self._state_lock:+ self._incomplete_runs.add(scope)++ @override+ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:+ grouped: dict[tuple[UUID, UUID], list[AgentTraceSpan]] = defaultdict(list)+ successful = True+ for span in spans:+ scope = _scope(span)+ if scope is None:+ successful = False+ continue+ try:+ grouped[scope].append(_serialize_span(span))+ except Exception:+ self.mark_incomplete(scope)+ successful = False+ logger.warning("Invalid application trace span; run_id=%s invocation_id=%s", *scope)+ with self._delivery_lock:+ for (run_id, invocation_id), run_spans in grouped.items():+ batches, oversized = self._batches(run_id, invocation_id, run_spans)+ successful = successful and not oversized+ for batch in batches:+ if not self._deliver(batch):+ successful = False+ if len(self._pending) < self._max_pending_batches:+ self._pending[batch.batch_id] = batch+ else:+ self.mark_incomplete((run_id, invocation_id))+ return SpanExportResult.SUCCESS if successful else SpanExportResult.FAILURE++ def _batches(+ self, run_id: UUID, invocation_id: UUID, spans: list[AgentTraceSpan]+ ) -> tuple[list[AgentTraceBatch], bool]:+ batches: list[AgentTraceBatch] = []+ current: list[AgentTraceSpan] = []+ # Reserve more than the fixed envelope, commas and list punctuation.+ size = 1024+ oversized = False++ def append_batch() -> None:+ encoded = json.dumps(+ [span.model_dump(mode="json") for span in current],+ sort_keys=True,+ ensure_ascii=False,+ separators=(",", ":"),+ )+ batches.append(+ AgentTraceBatch(+ run_id=run_id,+ invocation_id=invocation_id,+ batch_id=uuid5(invocation_id, hashlib.sha256(encoded.encode()).hexdigest()),+ spans=list(current),+ )+ )++ for span in spans:+ span_size = len(span.model_dump_json().encode()) + 1+ if span_size + 1024 > self._max_batch_bytes:+ oversized = True+ self.mark_incomplete((run_id, invocation_id))+ logger.warning("Application trace span exceeds delivery limit; invocation_id=%s", invocation_id)+ continue+ if size + span_size > self._max_batch_bytes:+ append_batch()+ current = []+ size = 1024+ current.append(span)+ size += span_size+ if current:+ append_batch()+ return batches, oversized++ def _deliver(self, batch: AgentTraceBatch) -> bool:+ payload = batch.model_dump(mode="json")+ if len(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()) > self._max_batch_bytes:+ self.mark_incomplete((batch.run_id, batch.invocation_id))+ return False+ for attempt in range(self._max_attempts):+ retryable = True+ try:+ response = self._client.post(self._url, headers=self._headers, json=payload)+ if response.is_success:+ acknowledgement = AgentTraceBatchAcknowledgement.model_validate(response.json())+ if acknowledgement.batch_id == batch.batch_id and acknowledgement.accepted_spans == len(+ batch.spans+ ):+ return True+ retryable = response.status_code >= 500 or response.status_code in (408, 429)+ except (httpx.TransportError, ValueError):+ pass+ except Exception:+ retryable = False+ if not retryable or attempt + 1 == self._max_attempts:+ break+ time.sleep(self._retry_delay_seconds * (2**attempt))+ logger.warning(+ "Application trace batch was not acknowledged; run_id=%s batch_id=%s", batch.run_id, batch.batch_id+ )+ return False++ def finish_invocation(self, run_id: UUID, invocation_id: UUID) -> bool:+ """Retry pending data, then acknowledge completion only after all data."""+ with self._delivery_lock:+ successful = True+ for batch_id, batch in tuple(self._pending.items()):+ if (batch.run_id, batch.invocation_id) != (run_id, invocation_id):+ continue+ if not self._deliver(batch):+ successful = False+ del self._pending[batch_id]+ with self._state_lock:+ incomplete = (run_id, invocation_id) in self._incomplete_runs+ if not successful or incomplete:+ self.mark_incomplete((run_id, invocation_id))+ return False+ return self._deliver(+ AgentTraceBatch(+ run_id=run_id,+ invocation_id=invocation_id,+ batch_id=uuid5(invocation_id, "final"),+ spans=[],+ final=True,+ )+ )++ @override+ def shutdown(self) -> None:+ with self._delivery_lock:+ for batch in self._pending.values():+ self._deliver(batch)+ self._pending.clear()+ with self._state_lock:+ self._incomplete_runs.clear()+ if self._owns_client:+ self._client.close()++ @override+ def force_flush(self, timeout_millis: int = 30000) -> bool:+ # Delivery is synchronous; the processor owns buffering and flush order.+ return True+++class ApplicationBatchSpanProcessor(SpanProcessor):+ """Batch spans without silently declaring a queue-overflowed run complete.++ OTel's default batch processor silently drops oldest spans on overflow. This+ collector must tell the API whether a run is complete, so a drop suppresses+ its final marker. Network I/O is confined to the worker/explicit flush thread.+ """++ def __init__(+ self,+ exporter: JSONSpanExporter,+ *,+ max_queue_size: int = 2048,+ max_export_batch_size: int = 128,+ schedule_delay_seconds: float = 1.0,+ ) -> None:+ self.exporter = exporter+ self._max_queue_size = max_queue_size+ self._max_export_batch_size = max_export_batch_size+ self._schedule_delay_seconds = schedule_delay_seconds+ self._queue: deque[ReadableSpan] = deque()+ self._started_invocations: set[tuple[UUID, UUID]] = set()+ self._condition = threading.Condition()+ self._export_lock = threading.Lock()+ self._stopping = False+ self._worker_thread = threading.Thread(target=self._worker, name="dify-application-traces", daemon=True)+ self._worker_thread.start()++ @override+ def on_start(self, span: Span, parent_context: Context | None = None) -> None:+ if (span.attributes or {}).get("gen_ai.operation.name") == "invoke_agent":+ if scope := _scope(span):+ with self._condition:+ self._started_invocations.add(scope)++ def invocation_started(self, scope: tuple[UUID, UUID]) -> bool:+ with self._condition:+ return scope in self._started_invocations++ def release_invocation(self, scope: tuple[UUID, UUID]) -> None:+ with self._condition:+ self._started_invocations.discard(scope)++ @override+ def on_end(self, span: ReadableSpan) -> None:+ with self._condition:+ if self._stopping or len(self._queue) >= self._max_queue_size:+ if scope := _scope(span):+ self.exporter.mark_incomplete(scope)+ return+ self._queue.append(span)+ if len(self._queue) >= self._max_export_batch_size:+ self._condition.notify()++ def _drain_locked(self) -> None:+ # Flush a finite snapshot: unrelated busy runs must not indefinitely+ # extend a completed invocation's finalization barrier.+ with self._condition:+ remaining = len(self._queue)+ while remaining:+ with self._condition:+ batch = [self._queue.popleft() for _ in range(min(remaining, self._max_export_batch_size))]+ remaining -= len(batch)+ if not batch:+ return+ try:+ self.exporter.export(batch)+ except Exception:+ for span in batch:+ if scope := _scope(span):+ self.exporter.mark_incomplete(scope)+ logger.warning("Application trace batch export failed", exc_info=True)++ def _worker(self) -> None:+ try:+ while True:+ with self._condition:+ if not self._stopping and len(self._queue) < self._max_export_batch_size:+ self._condition.wait(self._schedule_delay_seconds)+ stopping = self._stopping+ with self._export_lock:+ self._drain_locked()+ if stopping:+ return+ finally:+ self.exporter.shutdown()++ @override+ def force_flush(self, timeout_millis: int = 30000) -> bool:+ if not self._export_lock.acquire(timeout=timeout_millis / 1000):+ return False+ try:+ self._drain_locked()+ return True+ finally:+ self._export_lock.release()++ @override+ def shutdown(self) -> None:+ with self._condition:+ self._stopping = True+ self._condition.notify_all()+ self._worker_thread.join(timeout=10)+++__all__ = [+ "ApplicationBatchSpanProcessor",+ "DIFY_TRACING_RUN_ID_ATTRIBUTE",+ "DIFY_TRACING_INVOCATION_ID_ATTRIBUTE",+ "JSONSpanExporter",+]diff --git a/dify-agent/src/dify_agent/runtime/observability.py b/dify-agent/src/dify_agent/runtime/observability.pyindex b7b4d0de3e6..58de42b4ee8 100644--- a/dify-agent/src/dify_agent/runtime/observability.py+++ b/dify-agent/src/dify_agent/runtime/observability.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import loggingfrom collections.abc import Sequencefrom dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, Final, Literal, override@@ -10,8 +11,13 @@ from opentelemetry.sdk.trace import ReadableSpanfrom opentelemetry.trace import (INVALID_SPAN,Link,+ NonRecordingSpan,Span,+ SpanContext,SpanKind,+ Status,+ StatusCode,+ TraceFlags,Tracer,TracerProvider,get_current_span,@@ -19,7 +25,13 @@ from opentelemetry.trace import ()from opentelemetry.util.types import Attributes-from dify_agent.layers.execution_context.configs import DifyExecutionContextLayerConfig+from dify_agent.layers.execution_context.configs import ApplicationTracingContext, DifyExecutionContextLayerConfig+from dify_agent.runtime.application_trace_exporter import (+ ApplicationBatchSpanProcessor,+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE,+ DIFY_TRACING_RUN_ID_ATTRIBUTE,+ JSONSpanExporter,+)if TYPE_CHECKING:from logfire import Logfire@@ -36,6 +48,7 @@ DIFY_WORKFLOW_RUN_ID_ATTRIBUTE: Final[str] = "dify.workflow.run_id"DIFY_NODE_ID_ATTRIBUTE: Final[str] = "dify.node.id"DIFY_NODE_EXECUTION_ID_ATTRIBUTE: Final[str] = "dify.node.execution_id"GEN_AI_USER_ID_ATTRIBUTE: Final[str] = "gen_ai.user.id"+logger = logging.getLogger(__name__)def dify_run_attributes(execution_context: DifyExecutionContextLayerConfig) -> tuple[tuple[str, str], ...]:@@ -179,6 +192,9 @@ class _RunScopedTracer(Tracer):def _run_attributes(self, attributes: Attributes) -> Attributes:return {**(attributes or {}), **dict(self.attributes)}+ def _parent_context(self, context: Context | None) -> Context | None:+ return context+@overridedef start_span(self,@@ -193,7 +209,7 @@ class _RunScopedTracer(Tracer):) -> Span:return self.delegate.start_span(name,- context=context,+ context=self._parent_context(context),kind=kind,attributes=self._run_attributes(attributes),links=links,@@ -217,7 +233,7 @@ class _RunScopedTracer(Tracer):):return self.delegate.start_as_current_span(name,- context=context,+ context=self._parent_context(context),kind=kind,attributes=self._run_attributes(attributes),links=links,@@ -239,6 +255,7 @@ class AgentObservability:agent: Agent[Any, Any],*,execution_context: DifyExecutionContextLayerConfig | None = None,+ include_content: bool | None = None,) -> None:"""Instrument one run's agent and attribute its spans to its Dify context.@@ -249,7 +266,7 @@ class AgentObservability:"""self.client.instrument_pydantic_ai(agent,- include_content=self.include_content,+ include_content=self.include_content if include_content is None else include_content,include_binary_content=False,tracer_provider=self._tracer_provider(execution_context),)@@ -277,6 +294,163 @@ class AgentObservability:await asyncio.to_thread(self.client.shutdown, timeout_millis=5000)+@dataclass(frozen=True)+class _ApplicationRunTracer(_RunScopedTracer):+ tracing: ApplicationTracingContext++ @override+ def _parent_context(self, context: Context | None) -> Context:+ current = get_current_span(context)+ # Preserve only this invocation's descendants. Platform spans, a previous+ # invocation, or caller-supplied ambient context must not choose the root.+ if isinstance(current, ReadableSpan):+ attributes = current.attributes or {}+ current_context = current.get_span_context()+ if (+ attributes.get(DIFY_TRACING_RUN_ID_ATTRIBUTE) == str(self.tracing.run_id)+ and attributes.get(DIFY_TRACING_INVOCATION_ID_ATTRIBUTE) == str(self.tracing.invocation_id)+ and current_context is not None+ and current_context.trace_id == int(self.tracing.trace_id, 16)+ ):+ return context if context is not None else set_span_in_context(current)+ parent = SpanContext(+ trace_id=int(self.tracing.trace_id, 16),+ span_id=int(self.tracing.parent_span_id, 16),+ is_remote=True,+ trace_flags=TraceFlags(TraceFlags.SAMPLED if self.tracing.sampled else TraceFlags.DEFAULT),+ )+ return set_span_in_context(NonRecordingSpan(parent), Context())+++@dataclass(frozen=True)+class _ApplicationRunTracerProvider(RunScopedTracerProvider):+ tracing: ApplicationTracingContext++ @override+ def get_tracer(+ self,+ instrumenting_module_name: str,+ instrumenting_library_version: str | None = None,+ schema_url: str | None = None,+ attributes: Attributes = None,+ ) -> Tracer:+ delegate = self.delegate.get_tracer(+ instrumenting_module_name,+ instrumenting_library_version,+ schema_url=schema_url,+ attributes=attributes,+ )+ return _ApplicationRunTracer(delegate, self.attributes, self.tracing)+++@dataclass(frozen=True, slots=True, kw_only=True)+class ApplicationAgentObservability(AgentObservability):+ """Per-invocation policy over one process-owned internal export pipeline."""++ exporter: JSONSpanExporter+ processor: ApplicationBatchSpanProcessor++ @override+ def instrument(+ self,+ agent: Agent[Any, Any],+ *,+ execution_context: DifyExecutionContextLayerConfig | None = None,+ include_content: bool | None = None,+ ) -> None:+ tracing = execution_context.tracing if execution_context else None+ if tracing is None or not tracing.sampled:+ agent.instrument = False+ return+ # The immutable API policy always wins over process defaults/callers.+ AgentObservability.instrument(+ self, agent, execution_context=execution_context, include_content=tracing.capture_content+ )++ @override+ def _tracer_provider(self, execution_context: DifyExecutionContextLayerConfig | None) -> TracerProvider:+ if execution_context is None or execution_context.tracing is None:+ raise ValueError("Application tracing requires API-issued execution context")+ tracing = execution_context.tracing+ attributes = dict(dify_run_attributes(execution_context))+ attributes.update(+ {+ DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),+ DIFY_TRACE_ID_ATTRIBUTE: tracing.trace_id,+ "gen_ai.session.id": tracing.session_id,+ }+ )+ return _ApplicationRunTracerProvider(+ self.client.config.get_tracer_provider(), tuple(attributes.items()), tracing+ )++ async def finish_invocation(self, tracing: ApplicationTracingContext) -> bool:+ """Flush ended spans before the terminal event exposes the completed run."""++ def flush_and_finish() -> bool:+ if not self.processor.force_flush(timeout_millis=30000):+ self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))+ return False+ try:+ return self.exporter.finish_invocation(tracing.run_id, tracing.invocation_id)+ finally:+ self.processor.release_invocation((tracing.run_id, tracing.invocation_id))++ try:+ # The worker may continue its bounded retries after this deadline;+ # collection must not hold the Agent's terminal event indefinitely.+ return await asyncio.wait_for(asyncio.to_thread(flush_and_finish), timeout=5)+ except TimeoutError:+ logger.warning("Application trace finalization timed out; invocation_id=%s", tracing.invocation_id)+ return False+ except Exception:+ logger.warning(+ "Application trace finalization failed; invocation_id=%s", tracing.invocation_id, exc_info=True+ )+ return False++ def record_unstarted_invocation(+ self,+ execution_context: DifyExecutionContextLayerConfig,+ *,+ start_time_ns: int,+ end_time_ns: int,+ error: BaseException | None,+ cancelled: bool,+ ) -> None:+ """Represent setup failure/cancellation only when no native run span began.++ Successful invocations keep the native tree. An unexplained empty+ collector stays incomplete rather than being invented as success.+ """+ tracing = execution_context.tracing+ if tracing is None or not tracing.sampled or (error is None and not cancelled):+ return+ if self.processor.invocation_started((tracing.run_id, tracing.invocation_id)):+ return+ error_type = type(error).__name__ if error is not None else "CancelledError"+ message = "Agent cancelled before execution" if cancelled else error_type+ if not cancelled and error is not None and tracing.capture_content:+ message = str(error) or error_type+ try:+ tracer = self._tracer_provider(execution_context).get_tracer("dify-agent.initialization")+ span = tracer.start_span(+ "invoke_agent dify-agent",+ start_time=start_time_ns,+ attributes={+ "gen_ai.operation.name": "invoke_agent",+ "dify.agent.execution_state": "cancelled" if cancelled else "initialization_failed",+ "error.type": error_type,+ },+ )+ span.set_status(Status(StatusCode.ERROR, message))+ span.end(end_time=max(start_time_ns, end_time_ns))+ except Exception:+ self.exporter.mark_incomplete((tracing.run_id, tracing.invocation_id))+ logger.warning("Unable to record Agent initialization failure; invocation_id=%s", tracing.invocation_id)++__all__ = ["DIFY_AGENT_ID_ATTRIBUTE","DIFY_APP_ID_ATTRIBUTE",diff --git a/dify-agent/src/dify_agent/runtime/run_scheduler.py b/dify-agent/src/dify_agent/runtime/run_scheduler.pyindex dfebfdadade..dfdae91c432 100644--- a/dify-agent/src/dify_agent/runtime/run_scheduler.py+++ b/dify-agent/src/dify_agent/runtime/run_scheduler.py@@ -28,7 +28,7 @@ from dify_agent.runtime.event_coalescer import (DEFAULT_TEXT_DELTA_MAX_CHARS,)from dify_agent.runtime.event_sink import RunEventSink, RunFinalizationResult, emit_run_failed-from dify_agent.runtime.observability import AgentObservability+from dify_agent.runtime.observability import AgentObservability, ApplicationAgentObservabilityfrom dify_agent.runtime.runner import DEFAULT_AGENT_RUN_TIMEOUT_SECONDS, AgentRunRunnerfrom dify_agent.server.schemas import RunRecord@@ -118,6 +118,7 @@ class RunScheduler:runner_factory: RunRunnerFactory | Noneservices: Servicesagent_observability: AgentObservability | None+ application_observability: ApplicationAgentObservability | None_lifecycle_lock: asyncio.Lockdef __init__(@@ -132,6 +133,7 @@ class RunScheduler:stream_text_delta_max_chars: int = DEFAULT_TEXT_DELTA_MAX_CHARS,runner_factory: RunRunnerFactory | None = None,agent_observability: AgentObservability | None = None,+ application_observability: ApplicationAgentObservability | None = None,) -> None:self.store = storeself.shutdown_grace_seconds = shutdown_grace_seconds@@ -144,6 +146,7 @@ class RunScheduler:self.services = servicesself.runner_factory = runner_factoryself.agent_observability = agent_observability+ self.application_observability = application_observabilityself._lifecycle_lock = asyncio.Lock()async def create_run(self, request: CreateRunRequest) -> RunRecord:@@ -316,6 +319,7 @@ class RunScheduler:stream_text_delta_flush_interval_seconds=self.stream_text_delta_flush_interval_seconds,stream_text_delta_max_chars=self.stream_text_delta_max_chars,agent_observability=self.agent_observability,+ application_observability=self.application_observability,)def _discard_active_run(self, run_id: str) -> None:diff --git a/dify-agent/src/dify_agent/runtime/runner.py b/dify-agent/src/dify_agent/runtime/runner.pyindex 0ad20ca75f0..3e13fd73a7e 100644--- a/dify-agent/src/dify_agent/runtime/runner.py+++ b/dify-agent/src/dify_agent/runtime/runner.py@@ -8,6 +8,8 @@ scopes. Workspace and Binding retirement remain API-owned."""import asyncio+import logging+import timefrom collections.abc import AsyncIterable, Callable, Mappingfrom dataclasses import dataclassfrom typing import Protocol, cast, runtime_checkable@@ -53,13 +55,15 @@ from dify_agent.runtime.event_sink import (emit_run_started,emit_run_succeeded,)-from dify_agent.runtime.observability import AgentObservability+from dify_agent.layers.execution_context.configs import ApplicationTracingContext, DifyExecutionContextLayerConfig+from dify_agent.runtime.observability import AgentObservability, ApplicationAgentObservabilityfrom dify_agent.runtime.user_prompt_validation import EMPTY_USER_PROMPTS_ERROR, has_non_blank_user_prompt_AGENT_OUTPUT_ADAPTER = TypeAdapter(object)_MAX_AGENT_STEPS_PER_RUN = 500DEFAULT_AGENT_RUN_TIMEOUT_SECONDS = 60 * 60+logger = logging.getLogger(__name__)@runtime_checkable@@ -136,6 +140,7 @@ class AgentRunRunner:stream_text_delta_flush_interval_seconds: floatstream_text_delta_max_chars: intagent_observability: AgentObservability | None+ application_observability: ApplicationAgentObservability | None_terminal_session_snapshot: SessionSnapshot | None_terminal_usage: AgentRunUsage | None@@ -153,6 +158,7 @@ class AgentRunRunner:stream_text_delta_flush_interval_seconds: float = DEFAULT_TEXT_DELTA_FLUSH_INTERVAL_SECONDS,stream_text_delta_max_chars: int = DEFAULT_TEXT_DELTA_MAX_CHARS,agent_observability: AgentObservability | None = None,+ application_observability: ApplicationAgentObservability | None = None,) -> None:if stream_text_delta_flush_interval_seconds <= 0:raise ValueError("stream_text_delta_flush_interval_seconds must be positive")@@ -169,6 +175,7 @@ class AgentRunRunner:self.stream_text_delta_flush_interval_seconds = stream_text_delta_flush_interval_secondsself.stream_text_delta_max_chars = stream_text_delta_max_charsself.agent_observability = agent_observability+ self.application_observability = application_observabilityself._terminal_session_snapshot = Noneself._terminal_usage = None@@ -186,12 +193,27 @@ class AgentRunRunner:"""Execute the run and emit the documented event sequence."""self._terminal_session_snapshot = Noneself._terminal_usage = None+ execution_context = self._application_execution_context()+ started_at_ns = time.time_ns()if self.is_cancelled():+ await self._finalize_application_tracing(execution_context, started_at_ns, error=None, cancelled=True)return+ failure: BaseException | None = Nonetry:- _ = await emit_run_started(self.sink, run_id=self.run_id)- outcome = await self._run_agent()+ try:+ _ = await emit_run_started(self.sink, run_id=self.run_id)+ outcome = await self._run_agent()+ except BaseException as exc:+ failure = exc+ raise+ finally:+ await self._finalize_application_tracing(+ execution_context,+ started_at_ns,+ error=failure,+ cancelled=isinstance(failure, asyncio.CancelledError) or self.is_cancelled(),+ )except Exception as exc:if self.is_cancelled():return@@ -219,7 +241,38 @@ class AgentRunRunner:usage=outcome.usage,)+ def _application_execution_context(self) -> DifyExecutionContextLayerConfig | None:+ """Resolve valid correlation even if later model/resource setup fails."""+ if self.application_observability is None:+ return None+ try:+ layers = {layer.name: layer for layer in self.request.composition.layers}+ llm_config = llm_module.Config.model_validate(layers[DIFY_AGENT_MODEL_LAYER_ID].config)+ return DifyExecutionContextLayerConfig.model_validate(layers[llm_config.execution_context].config)+ except (KeyError, TypeError, ValueError):+ return None++ async def _finalize_application_tracing(+ self,+ execution_context: DifyExecutionContextLayerConfig | None,+ start_time_ns: int,+ *,+ error: BaseException | None,+ cancelled: bool,+ ) -> None:+ if execution_context is None or execution_context.tracing is None or self.application_observability is None:+ return+ self.application_observability.record_unstarted_invocation(+ execution_context,+ start_time_ns=start_time_ns,+ end_time_ns=time.time_ns(),+ error=error,+ cancelled=cancelled,+ )+ await self.application_observability.finish_invocation(execution_context.tracing)+async def _run_agent(self) -> RunSuccessOutcome:+ application_tracing: ApplicationTracingContext | None = Nonetry:deps = load_modules(self.request, self.services, self.run_id, registry=self.registry)modules = create_modules(deps, registry=self.registry)@@ -260,15 +313,23 @@ class AgentRunRunner:capabilities=capabilities,output_type=output_type,)- if self.agent_observability is not None:- from dify_agent.layers.execution_context.configs import DifyExecutionContextLayerConfig-- self.agent_observability.instrument(- agent,- execution_context=DifyExecutionContextLayerConfig.model_validate(- deps.layers[llm_config.execution_context]["config"]- ),- )+ execution_context = DifyExecutionContextLayerConfig.model_validate(+ deps.layers[llm_config.execution_context]["config"]+ )+ observability: AgentObservability | None = self.agent_observability+ if execution_context.tracing is not None and self.application_observability is not None:+ application_tracing = execution_context.tracing+ observability = self.application_observability+ if observability is not None:+ try:+ observability.instrument(agent, execution_context=execution_context)+ except Exception:+ agent.instrument = False+ if application_tracing is not None and self.application_observability is not None:+ self.application_observability.exporter.mark_incomplete(+ (application_tracing.run_id, application_tracing.invocation_id)+ )+ logger.warning("Agent tracing instrumentation failed; run_id=%s", self.run_id, exc_info=True)except (KeyError, TypeError, ValueError) as exc:raise AgentRunValidationError(str(exc)) from excdiff --git a/dify-agent/src/dify_agent/server/app.py b/dify-agent/src/dify_agent/server/app.pyindex abae36a5e0a..ef1be498bef 100644--- a/dify-agent/src/dify_agent/server/app.py+++ b/dify-agent/src/dify_agent/server/app.py@@ -30,7 +30,11 @@ from dify_agent.layers.execution_context import DifyExecutionContextLayerConfigfrom dify_agent.runtime.context import Servicesfrom dify_agent.runtime.run_scheduler import RunSchedulerfrom dify_agent.server.auth import create_bearer_token_dependency-from dify_agent.server.observability import configure_agent_observability, configure_server_observability+from dify_agent.server.observability import (+ configure_agent_observability,+ configure_application_observability,+ configure_server_observability,+)from dify_agent.server.routes.e2b_usage import create_e2b_usage_routerfrom dify_agent.server.routes.runs import create_runs_routerfrom dify_agent.server.routes.execution_bindings import create_execution_bindings_router@@ -93,7 +97,9 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:@asynccontextmanagerasync def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:agent_observability = configure_agent_observability(resolved_settings)+ application_observability = configure_application_observability(resolved_settings)_app.state.agent_observability = agent_observability+ _app.state.application_observability = application_observabilityredis = Redis.from_url(resolved_settings.redis_url)plugin_daemon_http_client = create_plugin_daemon_http_client(resolved_settings)dify_api_inner_http_client = create_dify_api_inner_http_client(resolved_settings)@@ -123,6 +129,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:stream_text_delta_flush_interval_seconds=(resolved_settings.stream_text_delta_flush_interval_ms / 1000),stream_text_delta_max_chars=resolved_settings.stream_text_delta_max_chars,agent_observability=agent_observability,+ application_observability=application_observability,)state["store"] = storestate["scheduler"] = scheduler@@ -136,6 +143,8 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:# Closed last so runs draining above still reach the exporter.if agent_observability is not None:await agent_observability.aclose()+ if application_observability is not None:+ await application_observability.aclose()app = FastAPI(title="Dify Agent Run Server", version="0.1.0", lifespan=lifespan)app.state.platform_observability = configure_server_observability(app, settings=resolved_settings)diff --git a/dify-agent/src/dify_agent/server/observability.py b/dify-agent/src/dify_agent/server/observability.pyindex 01fecfd0cee..97ffcf94a86 100644--- a/dify-agent/src/dify_agent/server/observability.py+++ b/dify-agent/src/dify_agent/server/observability.py@@ -20,6 +20,7 @@ code that mutates ``os.environ`` concurrently.from __future__ import annotationsimport os+import loggingfrom collections.abc import Iteratorfrom contextlib import contextmanagerfrom threading import RLock@@ -31,12 +32,14 @@ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExportfrom opentelemetry.sdk.trace.export import BatchSpanProcessorfrom opentelemetry.sdk.trace.sampling import ALWAYS_ON, ParentBased-from dify_agent.runtime.observability import AgentObservability, IsolatedTracerProvider+from dify_agent.runtime.application_trace_exporter import ApplicationBatchSpanProcessor, JSONSpanExporter+from dify_agent.runtime.observability import AgentObservability, ApplicationAgentObservability, IsolatedTracerProviderfrom dify_agent.server.settings import ServerSettings_global_instrumentation_ready = False_global_trace_context_mode: Literal["isolated", "shared"] | None = None_observability_configuration_lock = RLock()+logger = logging.getLogger(__name__)@contextmanager@@ -174,8 +177,42 @@ def configure_agent_observability(settings: ServerSettings) -> AgentObservabilit)+def configure_application_observability(settings: ServerSettings) -> ApplicationAgentObservability | None:+ """Configure application collection once, separately from platform/trajectory.++ Application policy is API-issued per invocation; destinations and provider+ credentials remain inside the Dify API. No SDK environment is inherited.+ """+ if not settings.application_tracing_enabled:+ return None+ if not settings.inner_api_key:+ logger.warning("Application tracing is enabled without an inner API credential; collector disabled")+ return None+ exporter = JSONSpanExporter(inner_api_url=settings.inner_api_url, inner_api_key=settings.inner_api_key)+ processor = ApplicationBatchSpanProcessor(exporter)+ try:+ with _isolated_agent_environment():+ client = logfire.configure(+ local=True,+ send_to_logfire=False,+ service_name="dify-agent-application",+ console=False,+ metrics=False,+ inspect_arguments=False,+ add_baggage_to_attributes=False,+ sampling=logfire.SamplingOptions(head=ParentBased(ALWAYS_ON)),+ additional_span_processors=[processor],+ )+ except Exception:+ processor.shutdown()+ logger.warning("Application tracing setup failed; collector disabled", exc_info=True)+ return None+ return ApplicationAgentObservability(client=client, exporter=exporter, processor=processor)++__all__ = ["AgentObservability","configure_agent_observability",+ "configure_application_observability","configure_server_observability",]diff --git a/dify-agent/src/dify_agent/server/settings.py b/dify-agent/src/dify_agent/server/settings.pyindex a5b5623a9ba..9bab69c5ea7 100644--- a/dify-agent/src/dify_agent/server/settings.py+++ b/dify-agent/src/dify_agent/server/settings.py@@ -131,6 +131,7 @@ class ServerSettings(BaseSettings):server_secret_key: str | None = Noneapi_token: str | None = Nonetrajectory_enabled: bool = False+ application_tracing_enabled: bool = Falsetrajectory_otlp_traces_endpoint: AnyHttpUrl | None = Nonetrajectory_otlp_headers: dict[str, SecretStr] = Field(default_factory=dict, repr=False)trajectory_service_name: str = Field(default="dify-agent-trajectory", min_length=1)diff --git a/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py b/dify-agent/tests/local/dify_agent/adapters/llm/test_model.pyindex eb6df382054..83bc1319467 100644--- a/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py+++ b/dify-agent/tests/local/dify_agent/adapters/llm/test_model.py@@ -5,6 +5,7 @@ from contextlib import asynccontextmanagerfrom decimal import Decimalfrom typing import castfrom unittest.mock import patch+from uuid import uuid4import httpximport pytest@@ -25,10 +26,15 @@ from pydantic_ai.messages import (UserPromptPart,)from pydantic_ai.models import ModelRequestParameters+from pydantic_ai.models.instrumented import InstrumentationSettings, InstrumentedModelfrom pydantic_ai.tools import ToolDefinition+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import StatusCodefrom dify_agent.adapters.llm import DifyApiLLMProvider, DifyLLMAdapterModel-from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig+from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfigfrom ._test_support import (AssistantPromptMessage,@@ -158,6 +164,7 @@ class DifyLLMAdapterModelTests(unittest.IsolatedAsyncioTestCase):payload = json.loads(request.content.decode("utf-8"))self.assertEqual(payload["caller"]["user_id"], "user-123")+ self.assertNotIn("tracing", payload["caller"])data = payload["target"]self.assertEqual(data["provider"], "langgenius/openai/openai")self.assertEqual(data["model"], "demo-model")@@ -209,6 +216,152 @@ class DifyLLMAdapterModelTests(unittest.IsolatedAsyncioTestCase):self.assertEqual(response.parts[0].part_kind, "text")self.assertEqual(cast(TextPart, response.parts[0]).content, "adapter response")+ async def test_application_tracing_policy_is_excluded_from_gateway_caller(self) -> None:+ provider = self.make_provider()+ provider.execution_context.tracing = ApplicationTracingContext(+ run_id=uuid4(),+ invocation_id=uuid4(),+ trace_id=uuid4().hex,+ parent_span_id=uuid4().hex[:16],+ session_id="test-session",+ capture_content=True,+ )++ def handler(request: httpx.Request) -> httpx.Response:+ payload = json.loads(request.content)+ self.assertNotIn("tracing", payload["caller"])+ self.assertEqual(payload["caller"]["tenant_id"], "tenant-1")+ return build_stream_response(*single_text_chunk("hello"))++ async with self.mock_gateway_stream(httpx.MockTransport(handler)):+ adapter = DifyLLMAdapterModel("demo-model", provider, model_provider="openai")+ await adapter.request([ModelRequest(parts=[UserPromptPart("hello")])], None, ModelRequestParameters())++ async def test_real_model_span_records_gateway_accounting_on_streaming_and_nonstreaming_requests(self) -> None:+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ usage = make_usage(+ prompt_tokens=17,+ completion_tokens=9,+ total_price=Decimal("0.001234"),+ currency="CNY",+ time_to_first_token=0.125,+ )++ def handler(_request: httpx.Request) -> httpx.Response:+ return build_stream_response(+ LLMResultChunk(+ model="demo-model",+ delta=LLMResultChunkDelta(+ index=0,+ message=AssistantPromptMessage(content="hello"),+ usage=usage,+ ),+ ),+ LLMResultChunk(+ model="demo-model",+ delta=LLMResultChunkDelta(+ index=1,+ message=AssistantPromptMessage(content=" world"),+ ),+ ),+ )++ try:+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):+ adapter = InstrumentedModel(+ DifyLLMAdapterModel("demo-model", self.make_provider(), model_provider="openai"),+ InstrumentationSettings(tracer_provider=tracer_provider),+ )+ messages = [ModelRequest(parts=[UserPromptPart("test")])]+ await adapter.request(messages, None, ModelRequestParameters())+ async with adapter.request_stream(messages, None, ModelRequestParameters()) as stream:+ async for _ in stream:+ pass+ spans = exporter.get_finished_spans()+ self.assertEqual(len(spans), 2)+ for span in spans:+ attributes = span.attributes or {}+ self.assertEqual(attributes["gen_ai.operation.name"], "chat")+ self.assertEqual(attributes["dify.llm.total_price"], "0.001234")+ self.assertEqual(attributes["dify.llm.currency"], "CNY")+ self.assertEqual(attributes["gen_ai.response.time_to_first_token"], 0.125)+ self.assertEqual(attributes["dify.llm.time_to_first_token_unit"], "s")+ self.assertEqual(attributes["dify.llm.time_to_first_token_source"], "gateway_usage")+ self.assertEqual(attributes["gen_ai.usage.input_tokens"], 17)+ self.assertEqual(attributes["gen_ai.usage.output_tokens"], 9)+ self.assertEqual(attributes["gen_ai.usage.total_tokens"], 26)+ self.assertNotIn("gen_ai.usage.cache_read.input_tokens", attributes)+ finally:+ tracer_provider.shutdown()++ async def test_streaming_gateway_failure_marks_the_real_chat_span_as_error(self) -> None:+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))++ def handler(_request: httpx.Request) -> httpx.Response:+ return build_stream_error("EndpointSetupFailedError", "sensitive-provider-detail")++ try:+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):+ adapter = InstrumentedModel(+ DifyLLMAdapterModel("demo-model", self.make_provider(), model_provider="openai"),+ InstrumentationSettings(tracer_provider=tracer_provider),+ )+ with self.assertRaises(UserError):+ async with adapter.request_stream(+ [ModelRequest(parts=[UserPromptPart("test")])], None, ModelRequestParameters()+ ) as stream:+ async for _ in stream:+ pass+ spans = exporter.get_finished_spans()+ self.assertEqual(len(spans), 1)+ self.assertEqual(spans[0].status.status_code, StatusCode.ERROR)+ self.assertEqual((spans[0].attributes or {}).get("error.type"), "UserError")+ self.assertNotIn("dify.llm.total_price", spans[0].attributes or {})+ finally:+ tracer_provider.shutdown()++ async def test_model_span_measures_first_content_without_inventing_missing_cost(self) -> None:+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))++ def handler(_request: httpx.Request) -> httpx.Response:+ return build_stream_response(+ LLMResultChunk(+ model="demo-model",+ delta=LLMResultChunkDelta(+ index=0,+ message=AssistantPromptMessage(content=""),+ ),+ ),+ LLMResultChunk(+ model="demo-model",+ delta=LLMResultChunkDelta(+ index=1,+ message=AssistantPromptMessage(content="hello"),+ ),+ ),+ )++ try:+ async with self.mock_gateway_stream(httpx.MockTransport(handler)):+ adapter = InstrumentedModel(+ DifyLLMAdapterModel("demo-model", self.make_provider(), model_provider="openai"),+ InstrumentationSettings(tracer_provider=tracer_provider),+ )+ await adapter.request([ModelRequest(parts=[UserPromptPart("test")])], None, ModelRequestParameters())+ attributes = exporter.get_finished_spans()[0].attributes or {}+ self.assertGreaterEqual(attributes["gen_ai.response.time_to_first_token"], 0)+ self.assertEqual(attributes["dify.llm.time_to_first_token_source"], "gateway_first_content")+ self.assertNotIn("dify.llm.total_price", attributes)+ self.assertNotIn("dify.llm.currency", attributes)+ finally:+ tracer_provider.shutdown()+async def test_request_accumulates_complete_dify_usage_across_model_rounds(self) -> None:usages = [make_usage(diff --git a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.pynew file mode 100644index 00000000000..dc0f7716253--- /dev/null+++ b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_tracing.py@@ -0,0 +1,188 @@+"""Real Pydantic AI tool spans retain handled API failure status without failing the run."""++import json+from collections.abc import Iterator++import httpx+import logfire+import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import StatusCode+from pydantic_ai import Agent+from pydantic_ai.messages import ModelMessage, ModelRequest, ModelResponse, TextPart, ToolCallPart, ToolReturnPart+from pydantic_ai.models.function import AgentInfo, FunctionModel++from dify_agent.layers.dify_core_tools.layer import Config, Toolset+from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig+from dify_agent.runtime.application_trace_exporter import _serialize_span+from dify_agent.runtime.context import Deps, Services+from dify_agent.runtime.observability import AgentObservability+++def observability(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[AgentObservability, InMemorySpanExporter]]:+ monkeypatch.setattr(Agent, "_instrument_default", False)+ exporter = InMemorySpanExporter()+ client = logfire.configure(+ local=True,+ send_to_logfire=False,+ console=False,+ metrics=False,+ inspect_arguments=False,+ additional_span_processors=[SimpleSpanProcessor(exporter)],+ )+ yield AgentObservability(client), exporter+ client.shutdown()+++def tool_model(messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse:+ returned = [+ part+ for message in messages+ if isinstance(message, ModelRequest)+ for part in message.parts+ if isinstance(part, ToolReturnPart)+ ]+ if not returned:+ return ModelResponse(parts=[ToolCallPart("calculate", {}, tool_call_id="owned-call")])+ return ModelResponse(parts=[TextPart(f"Handled: {returned[-1].content}")])+++def dependencies(client: httpx.AsyncClient, *, missing_context: bool = False) -> Deps:+ config = Config(+ tools=[+ {+ "provider_type": "builtin",+ "provider_id": "code",+ "tool_name": "calculate",+ "parameters_json_schema": {"type": "object", "properties": {}},+ }+ ]+ )+ context = DifyExecutionContextLayerConfig(+ tenant_id="tenant",+ user_id=None if missing_context else "user",+ user_from="account",+ app_id="app",+ agent_mode="agent_app",+ invoke_from="web-app",+ )+ return Deps(+ layers={+ "core_tools": {"config": config.model_dump(mode="json"), "state": {}},+ "execution_context": {"config": context.model_dump(mode="json"), "state": {}},+ },+ services=Services(client, client, inner_api_key="synthetic-key"),+ run_id="test",+ )++[email protected]("include_content", [False, True])[email protected]("failure", ["invocation", "timeout", "configuration"])+async def test_handled_failure_marks_only_its_native_tool_span(+ observability: tuple[AgentObservability, InMemorySpanExporter],+ include_content: bool,+ failure: str,+) -> None:+ telemetry, exporter = observability+ calls: list[str] = []++ def handle(request: httpx.Request) -> httpx.Response:+ calls.append(request.url.path)+ if failure == "timeout":+ raise httpx.ReadTimeout("private-transport-content", request=request)+ return httpx.Response(422, json={"code": "agent_tool_invoke_failed", "message": "private-error-content"})++ async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client:+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])+ telemetry.instrument(agent, include_content=include_content)+ result = await agent.run("Call the tool", deps=dependencies(client, missing_context=failure == "configuration"))+ assert result.output.startswith("Handled:")+ if failure == "invocation":+ assert "tool invoke error: private-error-content" in result.output+ elif failure == "timeout":+ assert "temporarily unavailable" in result.output+ else:+ assert "required execution context is missing" in result.output+ assert len(calls) == (0 if failure == "configuration" else 1)+ telemetry.client.force_flush()+ spans = exporter.get_finished_spans()+ tools = [span for span in spans if (span.attributes or {}).get("gen_ai.operation.name") == "execute_tool"]+ roots = [span for span in spans if (span.attributes or {}).get("gen_ai.operation.name") == "invoke_agent"]+ assert len(tools) == 1+ assert len(roots) == 1+ assert tools[0].status.status_code is StatusCode.ERROR+ assert roots[0].status.status_code is not StatusCode.ERROR+ serialized = _serialize_span(tools[0])+ assert serialized.status == "error"+ assert serialized.status_message in {"DifyCoreToolsClientError", "DifyCoreToolsClientConfigurationError"}+ if not include_content:+ assert "private-error-content" not in json.dumps(+ [_serialize_span(span).model_dump(mode="json") for span in spans]+ )+ assert "private-transport-content" not in serialized.model_dump_json()+++async def test_successful_observation_is_not_classified_by_its_text(+ observability: tuple[AgentObservability, InMemorySpanExporter],+) -> None:+ telemetry, exporter = observability+ observation = "tool invoke error timeout is only sample text in a successful result"+ async with httpx.AsyncClient(+ transport=httpx.MockTransport(lambda _request: httpx.Response(200, json={"observation": observation}))+ ) as client:+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])+ telemetry.instrument(agent)+ result = await agent.run("Call the tool", deps=dependencies(client))+ assert observation in result.output+ telemetry.client.force_flush()+ tool = next(+ span+ for span in exporter.get_finished_spans()+ if (span.attributes or {}).get("gen_ai.operation.name") == "execute_tool"+ )+ assert tool.status.status_code is not StatusCode.ERROR+ assert _serialize_span(tool).status == "ok"+++ "attributes",+ [+ {"gen_ai.operation.name": "invoke_agent"},+ {"gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "calculate", "gen_ai.tool.call.id": "other-call"},+ {+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.name": "other-tool",+ "gen_ai.tool.call.id": "owned-call",+ },+ ],+)+async def test_uninstrumented_tool_failure_does_not_mark_an_unrelated_ambient_span(+ attributes: dict[str, str],+) -> None:+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ try:+ async with httpx.AsyncClient(+ transport=httpx.MockTransport(+ lambda _request: httpx.Response(+ 422, json={"code": "agent_tool_invoke_failed", "message": "handled failure"}+ )+ )+ ) as client:+ agent = Agent(FunctionModel(tool_model), deps_type=Deps, toolsets=[Toolset("core_tools")])+ agent.instrument = False+ with provider.get_tracer("ambient").start_as_current_span("Ambient", attributes=attributes):+ result = await agent.run("Call the tool", deps=dependencies(client))+ assert "tool invoke error: handled failure" in result.output+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code is not StatusCode.ERROR+ finally:+ provider.shutdown()diff --git a/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py b/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.pyindex 34aa332c5d8..3048560b0cf 100644--- a/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py+++ b/dify-agent/tests/local/dify_agent/layers/execution_context/test_configs.py@@ -6,6 +6,7 @@ from dify_agent.layers.execution_context import DifyExecutionContextLayerConfigdef test_execution_context_package_exports_client_safe_config_symbols_only() -> None:assert execution_context_exports.__all__ == [+ "ApplicationTracingContext","DifyExecutionContextAgentConfigVersionKind","DifyExecutionContextAgentMode","DifyExecutionContextInvokeFrom",diff --git a/dify-agent/tests/local/dify_agent/runtime/test_application_tracing.py b/dify-agent/tests/local/dify_agent/runtime/test_application_tracing.pynew file mode 100644index 00000000000..e9d37cbaa00--- /dev/null+++ b/dify-agent/tests/local/dify_agent/runtime/test_application_tracing.py@@ -0,0 +1,511 @@+"""Exercise the actual Pydantic AI/Logfire tree and JSON delivery over local HTTP."""++import asyncio+import json+import os+import threading+import time+from collections.abc import Iterator+from dataclasses import dataclass, field+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from uuid import uuid4++import httpx+import logfire+import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export import SpanExportResult+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags, use_span+from pydantic import ValidationError+from pydantic_ai import Tool+from pydantic_ai.models.test import TestModel+from pydantic_ai.models.function import FunctionModel+from pydantic_ai.toolsets import FunctionToolset++from dify_agent.layers.dify_plugin import llm_layer+from dify_agent.layers.execution_context import ApplicationTracingContext, DifyExecutionContextLayerConfig+from dify_agent.protocol.schemas import CreateRunRequest, RunComposition, RunLayerSpec, RunSucceededEvent+from dify_agent.protocol.tracing import AgentTraceBatch+from dify_agent.runtime.agent_factory import create_agent+from dify_agent.runtime.application_trace_exporter import (+ ApplicationBatchSpanProcessor,+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE,+ DIFY_TRACING_RUN_ID_ATTRIBUTE,+ JSONSpanExporter,+)+from dify_agent.runtime.context import Services+from dify_agent.runtime.event_sink import InMemoryRunEventSink+from dify_agent.runtime.observability import AgentObservability+from dify_agent.runtime.runner import AgentRunRunner+from dify_agent.server.observability import configure_application_observability+from dify_agent.server.settings import ServerSettings+++@dataclass+class TraceInbox:+ batches: list[dict[str, object]] = field(default_factory=list)+ headers: list[str | None] = field(default_factory=list)+ paths: list[str] = field(default_factory=list)+ failures: int = 0+ status: int | None = None+ url: str = ""++[email protected](autouse=True)+def isolated_environment(monkeypatch, tmp_path) -> None:+ monkeypatch.chdir(tmp_path)+ for key in tuple(os.environ):+ if key.startswith(("OTEL_", "LOGFIRE_", "DIFY_AGENT_")):+ monkeypatch.delenv(key)+++def inbox() -> Iterator[TraceInbox]:+ state = TraceInbox()++ class Handler(BaseHTTPRequestHandler):+ def do_POST(self):+ payload = json.loads(self.rfile.read(int(self.headers["Content-Length"])))+ batch = AgentTraceBatch.model_validate(payload)+ state.batches.append(payload)+ state.headers.append(self.headers.get("X-Inner-Api-Key"))+ state.paths.append(self.path)+ status = state.status or (503 if state.failures else 200)+ state.failures = max(0, state.failures - 1)+ response = json.dumps(+ {"result": "success", "batch_id": str(batch.batch_id), "accepted_spans": len(batch.spans)}+ ).encode()+ self.send_response(status)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(response)))+ self.end_headers()+ self.wfile.write(response)++ def log_message(self, *_args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)+ thread.start()+ state.url = f"http://127.0.0.1:{server.server_port}"+ try:+ yield state+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=1)+++def tracing_context(*, capture_content=False, sampled=True, run_id=None):+ return ApplicationTracingContext(+ run_id=run_id or uuid4(),+ invocation_id=uuid4(),+ trace_id=uuid4().hex,+ parent_span_id=uuid4().hex[:16],+ session_id="test-session",+ capture_content=capture_content,+ sampled=sampled,+ )+++def execution_context(tracing):+ return DifyExecutionContextLayerConfig(+ tenant_id="test-tenant",+ app_id="test-app",+ agent_id="test-agent",+ agent_mode="agent_app",+ invoke_from="web-app",+ tracing=tracing,+ )+++def configured_collector(inbox):+ collector = configure_application_observability(+ ServerSettings(+ application_tracing_enabled=True,+ inner_api_url=inbox.url,+ inner_api_key="test-inner-key",+ )+ )+ assert collector is not None+ return collector+++def _tool() -> str:+ return "tool-private-content"++[email protected]("capture_content", [False, True])+def test_actual_agent_tree_uses_api_parent_and_per_run_content_policy(inbox, capture_content):+ collector = configured_collector(inbox)+ tracing = tracing_context(capture_content=capture_content)+ agent = create_agent(+ TestModel(custom_output_text="answer-private-content"),+ toolsets=[FunctionToolset(tools=[Tool(_tool)])],+ capabilities=[],+ )+ collector.instrument(agent, execution_context=execution_context(tracing))+ unrelated = NonRecordingSpan(SpanContext(trace_id=1, span_id=2, is_remote=True, trace_flags=TraceFlags(1)))+ try:+ with use_span(unrelated):+ assert agent.run_sync("prompt-private-content").output == "answer-private-content"+ assert asyncio.run(collector.finish_invocation(tracing))+ spans = [span for batch in inbox.batches for span in batch["spans"]]+ assert spans+ assert any("execute_tool" in span["name"] or "_tool" in span["name"] for span in spans)+ roots = [span for span in spans if span["parent_span_id"] == tracing.parent_span_id]+ assert len(roots) == 1+ assert roots[0]["name"] == "invoke_agent dify-agent"+ ids = {span["span_id"] for span in spans}+ for span in spans:+ assert span["trace_id"] == tracing.trace_id+ assert span["parent_span_id"] == tracing.parent_span_id or span["parent_span_id"] in ids+ assert span["attributes"]["dify.tenant_id"] == "test-tenant"+ assert span["attributes"][DIFY_TRACING_INVOCATION_ID_ATTRIBUTE] == str(tracing.invocation_id)+ serialized = json.dumps(spans)+ assert ("prompt-private-content" in serialized) is capture_content+ assert ("tool-private-content" in serialized) is capture_content+ assert inbox.batches[-1]["final"] is True+ assert inbox.batches[-1]["spans"] == []+ assert set(inbox.paths) == {"/inner/api/agent/tracing/batches"}+ assert set(inbox.headers) == {"test-inner-key"}+ finally:+ asyncio.run(collector.aclose())+++def test_retry_reuses_exact_batch_and_completion_waits_for_ack(inbox):+ collector = configured_collector(inbox)+ tracing = tracing_context()+ inbox.failures = 1+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])+ collector.instrument(agent, execution_context=execution_context(tracing))+ try:+ agent.run_sync("hello")+ assert asyncio.run(collector.finish_invocation(tracing))+ assert len(inbox.batches) >= 3+ assert inbox.batches[0] == inbox.batches[1]+ assert inbox.batches[-1]["final"] is True+ finally:+ asyncio.run(collector.aclose())+++def test_unacknowledged_invocation_never_emits_final_or_blocks_another_invocation(inbox):+ collector = configured_collector(inbox)+ first = tracing_context()+ second = tracing_context(run_id=first.run_id)+ try:+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])+ collector.instrument(agent, execution_context=execution_context(first))+ agent.run_sync("hello")+ inbox.status = 503+ assert not asyncio.run(collector.finish_invocation(first))+ assert not any(batch["final"] for batch in inbox.batches)+ inbox.status = None+ collector.instrument(agent, execution_context=execution_context(second))+ agent.run_sync("second")+ assert asyncio.run(collector.finish_invocation(second))+ finals = [batch for batch in inbox.batches if batch["final"]]+ assert [batch["invocation_id"] for batch in finals] == [str(second.invocation_id)]+ finally:+ asyncio.run(collector.aclose())+++def test_sampling_off_sends_only_completion(inbox):+ collector = configured_collector(inbox)+ tracing = tracing_context(sampled=False)+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])+ collector.instrument(agent, execution_context=execution_context(tracing))+ try:+ agent.run_sync("hello")+ assert asyncio.run(collector.finish_invocation(tracing))+ assert len(inbox.batches) == 1+ assert inbox.batches[0]["final"] is True+ assert inbox.batches[0]["spans"] == []+ finally:+ asyncio.run(collector.aclose())+++def test_shutdown_drains_ended_spans_without_inventing_completion(inbox):+ collector = configured_collector(inbox)+ tracing = tracing_context()+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])+ collector.instrument(agent, execution_context=execution_context(tracing))+ agent.run_sync("hello")+ asyncio.run(collector.aclose())+ assert any(batch["spans"] for batch in inbox.batches)+ assert not any(batch["final"] for batch in inbox.batches)+++def test_concurrent_invocations_keep_parent_and_content_policy_separate(inbox):+ collector = configured_collector(inbox)+ first = tracing_context(capture_content=False)+ second = tracing_context(capture_content=True, run_id=first.run_id)++ async def invoke(tracing, prompt):+ agent = create_agent(TestModel(), toolsets=[], capabilities=[])+ collector.instrument(agent, execution_context=execution_context(tracing))+ await agent.run(prompt)+ assert await collector.finish_invocation(tracing)++ try:++ async def both():+ await asyncio.gather(invoke(first, "private-first"), invoke(second, "visible-second"))++ asyncio.run(both())+ for tracing in (first, second):+ batches = [batch for batch in inbox.batches if batch["invocation_id"] == str(tracing.invocation_id)]+ assert batches[-1]["final"] is True+ spans = [span for batch in batches for span in batch["spans"]]+ roots = [span for span in spans if span["parent_span_id"] == tracing.parent_span_id]+ assert len(roots) == 1+ assert {span["trace_id"] for span in spans} == {tracing.trace_id}+ assert "private-first" not in json.dumps(spans)+ assert ("visible-second" in json.dumps(spans)) is tracing.capture_content+ finally:+ asyncio.run(collector.aclose())+++def test_queue_overflow_does_not_acknowledge_complete_run(inbox):+ tracing = tracing_context()+ exporter = JSONSpanExporter(inner_api_url=inbox.url, inner_api_key="test-inner-key")+ processor = ApplicationBatchSpanProcessor(+ exporter, max_queue_size=1, max_export_batch_size=10, schedule_delay_seconds=3600+ )+ provider = TracerProvider()+ provider.add_span_processor(processor)+ tracer = provider.get_tracer("test")+ attributes = {+ DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),+ "choices": ("a", "b"),+ }+ try:+ for name in ("one", "two"):+ with tracer.start_as_current_span(name, attributes=attributes):+ pass+ assert processor.force_flush()+ assert not exporter.finish_invocation(tracing.run_id, tracing.invocation_id)+ assert not exporter.finish_invocation(tracing.run_id, tracing.invocation_id)+ assert not any(batch["final"] for batch in inbox.batches)+ assert inbox.batches[0]["spans"][0]["attributes"]["choices"] == ["a", "b"]+ finally:+ provider.shutdown()++[email protected]("delivery_status", [None, 503])+def test_runner_prefers_application_pipeline_and_finishes_before_success(inbox, monkeypatch, delivery_status):+ collector = configured_collector(inbox)+ inbox.status = delivery_status+ tracing = tracing_context()+ deployment_exporter = InMemorySpanExporter()+ deployment_client = logfire.configure(+ local=True,+ send_to_logfire=False,+ console=False,+ metrics=False,+ additional_span_processors=[SimpleSpanProcessor(deployment_exporter)],+ )+ monkeypatch.setattr(llm_layer.Capability, "build_model", lambda self, deps: TestModel())+ incoming = CreateRunRequest(+ composition=RunComposition(+ layers=[+ RunLayerSpec(name="execution_context", config=execution_context(tracing).model_dump(mode="json")),+ RunLayerSpec(name="llm", config={"plugin_id": "test", "model_provider": "test", "model": "test"}),+ RunLayerSpec(name="prompt", config={"user": "hello"}),+ ]+ )+ )++ async def run():+ async with httpx.AsyncClient(trust_env=False) as client:+ sink = InMemoryRunEventSink()+ runner = AgentRunRunner(+ sink=sink,+ request=incoming,+ run_id="backend-run",+ services=Services(client, client),+ agent_observability=AgentObservability(client=deployment_client),+ application_observability=collector,+ )+ await runner.run()+ assert isinstance(sink.events["backend-run"][-1], RunSucceededEvent)+ assert inbox.batches[-1]["final"] is (delivery_status is None)++ try:+ asyncio.run(run())+ deployment_client.force_flush()+ assert not deployment_exporter.get_finished_spans()+ finally:+ asyncio.run(collector.aclose())+ deployment_client.shutdown()++[email protected]("oversized", [False, True])+def test_batches_respect_utf8_byte_limit_and_oversized_spans_prevent_final(inbox, oversized):+ tracing = tracing_context()+ memory = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(memory))+ tracer = provider.get_tracer("test")+ attributes = {+ DIFY_TRACING_RUN_ID_ATTRIBUTE: str(tracing.run_id),+ DIFY_TRACING_INVOCATION_ID_ATTRIBUTE: str(tracing.invocation_id),+ "unicode-content": "汉" * (5000 if oversized else 600),+ }+ for index in range(3):+ with tracer.start_as_current_span(str(index), attributes=attributes):+ pass+ exporter = JSONSpanExporter(inner_api_url=inbox.url, inner_api_key="test-inner-key", max_batch_bytes=4096)+ try:+ result = exporter.export(memory.get_finished_spans())+ assert result == (SpanExportResult.FAILURE if oversized else SpanExportResult.SUCCESS)+ assert exporter.finish_invocation(tracing.run_id, tracing.invocation_id) is not oversized+ if oversized:+ assert not inbox.batches+ else:+ assert len([batch for batch in inbox.batches if batch["spans"]]) == 3+ assert all(+ len(json.dumps(batch, ensure_ascii=False, separators=(",", ":")).encode()) <= 4096+ for batch in inbox.batches+ )+ finally:+ provider.shutdown()+ exporter.shutdown()++[email protected]("cancel", [False, True])+def test_runner_flushes_on_model_error_and_cancellation(inbox, monkeypatch, cancel):+ collector = configured_collector(inbox)+ tracing = tracing_context()++ async def run():+ entered = asyncio.Event()++ async def stream(_messages, _info):+ entered.set()+ if cancel:+ await asyncio.Event().wait()+ raise RuntimeError("test-only model failure")+ yield "unreachable"++ monkeypatch.setattr(+ llm_layer.Capability, "build_model", lambda self, deps: FunctionModel(stream_function=stream)+ )+ incoming = CreateRunRequest(+ composition=RunComposition(+ layers=[+ RunLayerSpec(name="execution_context", config=execution_context(tracing).model_dump(mode="json")),+ RunLayerSpec(name="llm", config={"plugin_id": "test", "model_provider": "test", "model": "test"}),+ RunLayerSpec(name="prompt", config={"user": "hello"}),+ ]+ )+ )+ async with httpx.AsyncClient(trust_env=False) as client:+ runner = AgentRunRunner(+ sink=InMemoryRunEventSink(),+ request=incoming,+ run_id="backend-run",+ services=Services(client, client),+ application_observability=collector,+ )+ task = asyncio.create_task(runner.run())+ await entered.wait()+ if cancel:+ task.cancel()+ with pytest.raises(asyncio.CancelledError if cancel else RuntimeError):+ await task+ assert inbox.batches[-1]["final"] is True+ assert any(batch["spans"] for batch in inbox.batches)+ roots = [+ span+ for batch in inbox.batches+ for span in batch["spans"]+ if span["attributes"].get("gen_ai.operation.name") == "invoke_agent"+ ]+ assert len(roots) == 1+ assert "dify.agent.execution_state" not in roots[0]["attributes"]++ try:+ asyncio.run(run())+ finally:+ asyncio.run(collector.aclose())++[email protected]("early_cancel", [False, True])[email protected]("sampled", [False, True])+def test_setup_failure_or_early_cancel_has_one_timed_fallback_only_when_sampled(+ inbox, monkeypatch, early_cancel, sampled+):+ collector = configured_collector(inbox)+ tracing = tracing_context(sampled=sampled, capture_content=True)++ def fail_model_setup(self, deps):+ raise ValueError("test-only invalid model configuration")++ monkeypatch.setattr(llm_layer.Capability, "build_model", fail_model_setup)+ incoming = CreateRunRequest(+ composition=RunComposition(+ layers=[+ RunLayerSpec(name="execution_context", config=execution_context(tracing).model_dump(mode="json")),+ RunLayerSpec(name="llm", config={"plugin_id": "test", "model_provider": "test", "model": "test"}),+ RunLayerSpec(name="prompt", config={"user": "hello"}),+ ]+ )+ )+ before = time.time_ns()++ async def run():+ async with httpx.AsyncClient(trust_env=False) as client:+ runner = AgentRunRunner(+ sink=InMemoryRunEventSink(),+ request=incoming,+ run_id="backend-run",+ services=Services(client, client),+ application_observability=collector,+ is_cancelled=lambda: early_cancel,+ )+ if early_cancel:+ await runner.run()+ else:+ with pytest.raises(ValueError, match="invalid model configuration"):+ await runner.run()++ try:+ asyncio.run(run())+ after = time.time_ns()+ assert inbox.batches[-1]["final"] is True+ spans = [span for batch in inbox.batches for span in batch["spans"]]+ if not sampled:+ assert spans == []+ return+ assert len(spans) == 1+ span = spans[0]+ assert span["trace_id"] == tracing.trace_id+ assert span["parent_span_id"] == tracing.parent_span_id+ assert span["attributes"]["gen_ai.operation.name"] == "invoke_agent"+ assert span["attributes"]["dify.agent.execution_state"] == (+ "cancelled" if early_cancel else "initialization_failed"+ )+ assert span["status"] == "error"+ assert before <= span["start_time_ns"] <= span["end_time_ns"] <= after+ assert span["status_message"] == (+ "Agent cancelled before execution" if early_cancel else "test-only invalid model configuration"+ )+ finally:+ asyncio.run(collector.aclose())++[email protected]("field,value", [("version", 2), ("trace_id", "0" * 32), ("parent_span_id", "x" * 16)])+def test_invalid_api_trace_context_is_rejected(field, value):+ payload = tracing_context().model_dump(mode="json")+ payload[field] = value+ with pytest.raises(ValidationError):+ ApplicationTracingContext.model_validate(payload)+++def test_application_collection_is_disabled_by_default():+ assert ServerSettings().application_tracing_enabled is False+ assert configure_application_observability(ServerSettings()) is Nonediff --git a/dify-agent/tests/local/dify_agent/server/test_app.py b/dify-agent/tests/local/dify_agent/server/test_app.pyindex 3841cd7c662..d97b24a348c 100644--- a/dify-agent/tests/local/dify_agent/server/test_app.py+++ b/dify-agent/tests/local/dify_agent/server/test_app.py@@ -99,6 +99,7 @@ class FakeRunScheduler:stream_text_delta_flush_interval_seconds: float,stream_text_delta_max_chars: int,agent_observability: object | None = None,+ application_observability: object | None = None,) -> None:self.store = storeself.shutdown_grace_seconds = shutdown_grace_seconds@@ -110,6 +111,7 @@ class FakeRunScheduler:self.plugin_daemon_http_client = services.plugin_daemon_http_clientself.dify_api_http_client = services.dify_api_http_clientself.agent_observability = agent_observability+ self.application_observability = application_observabilityself.shutdown_called = Falseself.created.append(self)diff --git a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py b/dify-agent/tests/local/dify_agent/test_client_safe_exports.pyindex 4cff38d7f13..65f6753457c 100644--- a/dify-agent/tests/local/dify_agent/test_client_safe_exports.py+++ b/dify-agent/tests/local/dify_agent/test_client_safe_exports.py@@ -79,6 +79,15 @@ def test_client_public_exports_work_with_default_dependencies_only(tmp_path: Patassert agent_stub_shell_env_module.build_shell_agent_stub_env is not Noneassert shell_module.DifyShellLayerConfig is not Noneassert execution_context_module.DifyExecutionContextLayerConfig is not None+ tracing = execution_context_module.ApplicationTracingContext(+ run_id="11111111-1111-4111-8111-111111111111",+ invocation_id="22222222-2222-4222-8222-222222222222",+ trace_id="1" * 32,+ parent_span_id="2" * 16,+ session_id="client-session",+ capture_content=False,+ )+ assert execution_context_module.ApplicationTracingContext.model_validate_json(tracing.model_dump_json()) == tracingassert plugin_module.DifyPluginLLMLayerConfig is not Noneassert output_module.DifyOutputLayerConfig is not Noneassert user_prompt_module.DifyUserPromptLayerConfig is not Nonediff --git a/docker/envs/core-services/dify-agent.env.example b/docker/envs/core-services/dify-agent.env.exampleindex 42094c3a1c5..1c202a4f266 100644--- a/docker/envs/core-services/dify-agent.env.example+++ b/docker/envs/core-services/dify-agent.env.example@@ -30,6 +30,8 @@ DIFY_AGENT_OUTBOUND_HTTP_MAX_KEEPALIVE_CONNECTIONS=20DIFY_AGENT_OUTBOUND_HTTP_KEEPALIVE_EXPIRY=30DIFY_AGENT_TRAJECTORY_ENABLED=false+# Collect API-issued application traces into the Dify inner API inbox.+DIFY_AGENT_APPLICATION_TRACING_ENABLED=falseDIFY_AGENT_TRAJECTORY_OTLP_TRACES_ENDPOINT=DIFY_AGENT_TRAJECTORY_OTLP_HEADERS={}DIFY_AGENT_TRAJECTORY_SERVICE_NAME=dify-agent-trajectorydiff --git a/docker/envs/core-services/shared.env.example b/docker/envs/core-services/shared.env.exampleindex cedce027c4d..2b3250e0cc3 100644--- a/docker/envs/core-services/shared.env.example+++ b/docker/envs/core-services/shared.env.example@@ -104,6 +104,9 @@ DEBUG=falseFLASK_DEBUG=falseENABLE_REQUEST_LOGGING=FalseOPS_TRACE_UNIFIED_ENABLED=false+# Application-level Agent tracing; also enable the collector in dify-agent.env.+OPS_AGENT_TRACE_ENABLED=false+ENABLE_APPLICATION_TRACE_RECONCILE_TASK=trueOPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES=780OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS=5# OPS_TRACE_PARENT_CONTEXT_TTL_SECONDS should cover the retry windowdiff --git a/packages/contracts/generated/api/console/agent/orpc.gen.ts b/packages/contracts/generated/api/console/agent/orpc.gen.tsindex 2c3d85a2ce4..5ae49bd98bd 100644--- a/packages/contracts/generated/api/console/agent/orpc.gen.ts+++ b/packages/contracts/generated/api/console/agent/orpc.gen.ts@@ -13,6 +13,7 @@ import {zAgentNotPublishedError,zAgentSoulLockedError,zAgentVersionNotFoundError,+ zAppNotFoundError,zAppSuggestedQuestionsAfterAnswerDisabledError,zAppUnavailableError,zAudioTooLargeError,@@ -20,6 +21,7 @@ import {zCompletionRequestError,zInspectorUnavailableError,zInvalidComposerConfigError,+ zInvalidTracingConfigError,zMaxKeysExceededError,zModelCurrentlyNotSupportError,zNoActiveBindingError,@@ -31,7 +33,20 @@ import {zProviderQuotaExceededError,zRateLimitError,zSpeechToTextDisabledError,+ zTraceConfigAlreadyExistsError,+ zTraceConfigNotFoundError,+ zTracingAlreadyAcceptedError,+ zTracingConfigConflictError,+ zTracingConfigProcessingFailedError,+ zTracingConfigVerificationFailedError,+ zTracingDuplicateAcknowledgementRequiredError,+ zTracingNodeNotFoundError,+ zTracingProviderUnavailableError,+ zTracingRetryNotAllowedError,+ zTracingRunNotFoundError,+ zTracingSourceNotFoundError,zUnsupportedAudioTypeError,+ zUnsupportedTracingProviderError,} from './zod.gen.ts'import {zDeleteAgentByAgentIdApiKeysByApiKeyIdPath,@@ -46,6 +61,9 @@ import {zDeleteAgentByAgentIdConfigSkillsByNameResponse,zDeleteAgentByAgentIdPath,zDeleteAgentByAgentIdResponse,+ zDeleteAgentByAgentIdTraceConfigPath,+ zDeleteAgentByAgentIdTraceConfigQuery,+ zDeleteAgentByAgentIdTraceConfigResponse,zGetAgentByAgentIdApiAccessPath,zGetAgentByAgentIdApiAccessResponse,zGetAgentByAgentIdApiKeysPath,@@ -119,6 +137,17 @@ import {zGetAgentByAgentIdTextToAudioVoicesPath,zGetAgentByAgentIdTextToAudioVoicesQuery,zGetAgentByAgentIdTextToAudioVoicesResponse,+ zGetAgentByAgentIdTraceCapabilitiesPath,+ zGetAgentByAgentIdTraceCapabilitiesQuery,+ zGetAgentByAgentIdTraceCapabilitiesResponse,+ zGetAgentByAgentIdTraceConfigPath,+ zGetAgentByAgentIdTraceConfigQuery,+ zGetAgentByAgentIdTraceConfigResponse,+ zGetAgentByAgentIdTracePath,+ zGetAgentByAgentIdTraceResponse,+ zGetAgentByAgentIdTraceRunsByMessageIdPath,+ zGetAgentByAgentIdTraceRunsByMessageIdQuery,+ zGetAgentByAgentIdTraceRunsByMessageIdResponse,zGetAgentByAgentIdVersionsByVersionIdPath,zGetAgentByAgentIdVersionsByVersionIdResponse,zGetAgentByAgentIdVersionsPath,@@ -127,6 +156,9 @@ import {zGetAgentInviteOptionsResponse,zGetAgentQuery,zGetAgentResponse,+ zPatchAgentByAgentIdTraceConfigBody,+ zPatchAgentByAgentIdTraceConfigPath,+ zPatchAgentByAgentIdTraceConfigResponse,zPostAgentBody,zPostAgentByAgentIdApiEnableBody,zPostAgentByAgentIdApiEnablePath,@@ -176,6 +208,16 @@ import {zPostAgentByAgentIdTextToAudioBody,zPostAgentByAgentIdTextToAudioPath,zPostAgentByAgentIdTextToAudioResponse,+ zPostAgentByAgentIdTraceBody,+ zPostAgentByAgentIdTraceConfigBody,+ zPostAgentByAgentIdTraceConfigPath,+ zPostAgentByAgentIdTraceConfigResponse,+ zPostAgentByAgentIdTracePath,+ zPostAgentByAgentIdTraceResponse,+ zPostAgentByAgentIdTraceRunsByMessageIdRetryBody,+ zPostAgentByAgentIdTraceRunsByMessageIdRetryPath,+ zPostAgentByAgentIdTraceRunsByMessageIdRetryQuery,+ zPostAgentByAgentIdTraceRunsByMessageIdRetryResponse,zPostAgentByAgentIdVersionsByVersionIdRestorePath,zPostAgentByAgentIdVersionsByVersionIdRestoreResponse,zPostAgentResponse,@@ -1370,7 +1412,234 @@ export const textToAudio = {voices,}+export const get29 = oc+ .route({+ inputStructure: 'detailed',+ method: 'GET',+ operationId: 'getAgentByAgentIdTrace',+ path: '/agent/{agent_id}/trace',+ tags: ['console'],+ })+ .input(z.object({ params: zGetAgentByAgentIdTracePath }))+ .output(zGetAgentByAgentIdTraceResponse)+export const post18 = oc+ .route({+ inputStructure: 'detailed',+ method: 'POST',+ operationId: 'postAgentByAgentIdTrace',+ path: '/agent/{agent_id}/trace',+ tags: ['console'],+ })+ .input(z.object({ body: zPostAgentByAgentIdTraceBody, params: zPostAgentByAgentIdTracePath }))+ .output(zPostAgentByAgentIdTraceResponse)+ .errors({+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },+ })++export const trace = {+ get: get29,+ post: post18,+}++export const get30 = oc+ .route({+ inputStructure: 'detailed',+ method: 'GET',+ operationId: 'getAgentByAgentIdTraceCapabilities',+ path: '/agent/{agent_id}/trace-capabilities',+ tags: ['console'],+ })+ .input(+ z.object({+ params: zGetAgentByAgentIdTraceCapabilitiesPath,+ query: zGetAgentByAgentIdTraceCapabilitiesQuery,+ }),+ )+ .output(zGetAgentByAgentIdTraceCapabilitiesResponse)+ .errors({ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError } })++export const traceCapabilities = {+ get: get30,+}++export const delete5 = oc+ .route({+ inputStructure: 'detailed',+ method: 'DELETE',+ operationId: 'deleteAgentByAgentIdTraceConfig',+ path: '/agent/{agent_id}/trace-config',+ successStatus: 204,+ tags: ['console'],+ })+ .input(+ z.object({+ params: zDeleteAgentByAgentIdTraceConfigPath,+ query: zDeleteAgentByAgentIdTraceConfigQuery,+ }),+ )+ .output(zDeleteAgentByAgentIdTraceConfigResponse)+ .errors({+ app_not_found: { status: 404, data: zAppNotFoundError },+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },+ })++export const get31 = oc+ .route({+ inputStructure: 'detailed',+ method: 'GET',+ operationId: 'getAgentByAgentIdTraceConfig',+ path: '/agent/{agent_id}/trace-config',+ tags: ['console'],+ })+ .input(+ z.object({+ params: zGetAgentByAgentIdTraceConfigPath,+ query: zGetAgentByAgentIdTraceConfigQuery,+ }),+ )+ .output(zGetAgentByAgentIdTraceConfigResponse)+ .errors({+ app_not_found: { status: 404, data: zAppNotFoundError },+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },+ })++export const patch = oc+ .route({+ inputStructure: 'detailed',+ method: 'PATCH',+ operationId: 'patchAgentByAgentIdTraceConfig',+ path: '/agent/{agent_id}/trace-config',+ tags: ['console'],+ })+ .input(+ z.object({+ body: zPatchAgentByAgentIdTraceConfigBody,+ params: zPatchAgentByAgentIdTraceConfigPath,+ }),+ )+ .output(zPatchAgentByAgentIdTraceConfigResponse)+ .errors({+ app_not_found: { status: 404, data: zAppNotFoundError },+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },+ tracing_config_verification_failed: {+ status: 400,+ data: zTracingConfigVerificationFailedError,+ },+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },+ })++export const post19 = oc+ .route({+ inputStructure: 'detailed',+ method: 'POST',+ operationId: 'postAgentByAgentIdTraceConfig',+ path: '/agent/{agent_id}/trace-config',+ successStatus: 201,+ tags: ['console'],+ })+ .input(+ z.object({+ body: zPostAgentByAgentIdTraceConfigBody,+ params: zPostAgentByAgentIdTraceConfigPath,+ }),+ )+ .output(zPostAgentByAgentIdTraceConfigResponse)+ .errors({+ app_not_found: { status: 404, data: zAppNotFoundError },+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },+ trace_config_already_exists: { status: 409, data: zTraceConfigAlreadyExistsError },+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },+ tracing_config_verification_failed: {+ status: 400,+ data: zTracingConfigVerificationFailedError,+ },+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },+ })++export const traceConfig = {+ delete: delete5,+ get: get31,+ patch,+ post: post19,+}++export const post20 = oc+ .route({+ inputStructure: 'detailed',+ method: 'POST',+ operationId: 'postAgentByAgentIdTraceRunsByMessageIdRetry',+ path: '/agent/{agent_id}/trace-runs/{message_id}/retry',+ tags: ['console'],+ })+ .input(+ z.object({+ body: zPostAgentByAgentIdTraceRunsByMessageIdRetryBody,+ params: zPostAgentByAgentIdTraceRunsByMessageIdRetryPath,+ query: zPostAgentByAgentIdTraceRunsByMessageIdRetryQuery.optional(),+ }),+ )+ .output(zPostAgentByAgentIdTraceRunsByMessageIdRetryResponse)+ .errors({+ tracing_already_accepted: { status: 409, data: zTracingAlreadyAcceptedError },+ tracing_duplicate_acknowledgement_required: {+ status: 409,+ data: zTracingDuplicateAcknowledgementRequiredError,+ },+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },+ tracing_retry_not_allowed: { status: 409, data: zTracingRetryNotAllowedError },+ tracing_run_not_found: { status: 404, data: zTracingRunNotFoundError },+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },+ })++export const retry = {+ post: post20,+}++export const get32 = oc+ .route({+ inputStructure: 'detailed',+ method: 'GET',+ operationId: 'getAgentByAgentIdTraceRunsByMessageId',+ path: '/agent/{agent_id}/trace-runs/{message_id}',+ tags: ['console'],+ })+ .input(+ z.object({+ params: zGetAgentByAgentIdTraceRunsByMessageIdPath,+ query: zGetAgentByAgentIdTraceRunsByMessageIdQuery.optional(),+ }),+ )+ .output(zGetAgentByAgentIdTraceRunsByMessageIdResponse)+ .errors({+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },+ })++export const byMessageId3 = {+ get: get32,+ retry,+}++export const traceRuns = {+ byMessageId: byMessageId3,+}++export const post21 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1386,10 +1655,10 @@ export const post18 = oc})export const restore = {- post: post18,+ post: post21,}-export const get29 = oc+export const get33 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1405,11 +1674,11 @@ export const get29 = oc})export const byVersionId = {- get: get29,+ get: get33,restore,}-export const get30 = oc+export const get34 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1422,11 +1691,11 @@ export const get30 = oc.errors({ agent_not_found_error: { status: 404, data: zAgentNotFoundError } })export const versions = {- get: get30,+ get: get34,byVersionId,}-export const delete5 = oc+export const delete6 = oc.route({inputStructure: 'detailed',method: 'DELETE',@@ -1439,7 +1708,7 @@ export const delete5 = oc.output(zDeleteAgentByAgentIdResponse).errors({ agent_not_found_error: { status: 404, data: zAgentNotFoundError } })-export const get31 = oc+export const get35 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1467,8 +1736,8 @@ export const put3 = oc})export const byAgentId = {- delete: delete5,- get: get31,+ delete: delete6,+ get: get35,put: put3,apiAccess,apiEnable,@@ -1491,10 +1760,14 @@ export const byAgentId = {sandbox,statistics,textToAudio,+ trace,+ traceCapabilities,+ traceConfig,+ traceRuns,versions,}-export const get32 = oc+export const get36 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -1506,7 +1779,7 @@ export const get32 = oc.output(zGetAgentResponse).errors({ agent_not_found_error: { status: 404, data: zAgentNotFoundError } })-export const post19 = oc+export const post22 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -1523,8 +1796,8 @@ export const post19 = oc})export const agent = {- get: get32,- post: post19,+ get: get36,+ post: post22,inviteOptions,byAgentId,}diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.tsindex d01e6f92cb8..23eb12f6779 100644--- a/packages/contracts/generated/api/console/agent/types.gen.ts+++ b/packages/contracts/generated/api/console/agent/types.gen.ts@@ -572,6 +572,162 @@ export type ProviderNotSupportTextToSpeechError = ErrorResponse & {export type TextToSpeechVoiceListResponse = Array<TextToSpeechVoiceResponse>+export type AppTraceResponse = {+ capture_agent?: boolean+ capture_content?: boolean+ enabled?: boolean+ tracing_provider?: string | null+}++export type AppTracePayload = {+ capture_agent?: boolean+ capture_content?: boolean+ enabled: boolean+ tracing_provider?: string | null+}++export type InvalidTracingConfigError = ErrorResponse & {+ code: 'invalid_tracing_config'+ status: 400+}++export type TracingProviderUnavailableError = ErrorResponse & {+ code: 'tracing_provider_unavailable'+ status: 400+}++export type UnsupportedTracingProviderError = ErrorResponse & {+ code: 'unsupported_tracing_provider'+ status: 400+}++export type TraceConfigNotFoundError = ErrorResponse & {+ code: 'trace_config_not_found'+ status: 404+}++export type TracingConfigConflictError = ErrorResponse & {+ code: 'tracing_config_conflict'+ status: 409+}++export type TracingConfigProcessingFailedError = ErrorResponse & {+ code: 'tracing_config_processing_failed'+ status: 500+}++export type TraceProviderCapabilitiesResponse = {+ agent_spans: boolean+ api_collection_enabled: boolean+ export_ack: 'accepted'+ native_cost: boolean+ native_usage: boolean+ provider: string+ session: boolean+ span_url: boolean+ trace_url: boolean+}++export type AppNotFoundError = ErrorResponse & {+ code: 'app_not_found'+ status: 404+}++export type TraceAppConfigResponse = {+ app_id?: string | null+ created_at?: string | null+ error?: string | null+ has_not_configured?: boolean | null+ id?: string | null+ is_active?: boolean | null+ result?: string | null+ tracing_config?: {+ [key: string]: string | null+ } | null+ tracing_provider?: string | null+ updated_at?: string | null+}++export type TraceConfigPayload = {+ activate?: boolean+ tracing_config: {+ [key: string]: string | null+ }+ tracing_provider: string+}++export type TracingConfigVerificationFailedError = ErrorResponse & {+ code: 'tracing_config_verification_failed'+ status: 400+}++export type TraceConfigAlreadyExistsError = ErrorResponse & {+ code: 'trace_config_already_exists'+ status: 409+}++export type TraceRunStatusResponse = {+ accepted_at?: string | null+ attempts?: number | null+ capabilities?: TraceProviderSupportResponse | null+ capture_agent?: boolean | null+ capture_content?: boolean | null+ config_revision?: string | null+ error_code?: string | null+ project_url?: string | null+ provider?: string | null+ queryable?: boolean | null+ source_id: string+ source_kind: TraceSourceKind+ span_count?: number | null+ span_url?: string | null+ status:+ | 'accepted'+ | 'collecting'+ | 'exporting'+ | 'failed'+ | 'not_collected'+ | 'ready'+ | 'retrying'+ | 'uncertain'+ trace_id?: string | null+ trace_url?: string | null+}++export type TracingNodeNotFoundError = ErrorResponse & {+ code: 'tracing_node_not_found'+ status: 404+}++export type TracingSourceNotFoundError = ErrorResponse & {+ code: 'tracing_source_not_found'+ status: 404+}++export type TraceRunRetryPayload = {+ acknowledge_possible_duplicates?: boolean+}++export type TracingRunNotFoundError = ErrorResponse & {+ code: 'tracing_run_not_found'+ status: 404+}++export type TracingAlreadyAcceptedError = ErrorResponse & {+ code: 'tracing_already_accepted'+ status: 409+}++export type TracingDuplicateAcknowledgementRequiredError = ErrorResponse & {+ code: 'tracing_duplicate_acknowledgement_required'+ status: 409+}++export type TracingRetryNotAllowedError = ErrorResponse & {+ code: 'tracing_retry_not_allowed'+ status: 409+}+export type AgentConfigSnapshotListResponse = {data: Array<AgentConfigSnapshotSummaryResponse>}@@ -1144,6 +1300,19 @@ export type TextToSpeechVoiceResponse = {value: string}+export type TraceProviderSupportResponse = {+ agent_spans: boolean+ export_ack: 'accepted'+ native_cost: boolean+ native_usage: boolean+ provider: string+ session: boolean+ span_url: boolean+ trace_url: boolean+}++export type TraceSourceKind = 'message' | 'node' | 'workflow'+export type AgentConfigRevisionResponse = {created_at?: number | nullcreated_by?: string | null@@ -1921,7 +2090,7 @@ export type AgentSoulModelSettings = {frequency_penalty?: number | nullmax_tokens?: number | nullpresence_penalty?: number | null- response_format?: AgentModelResponseFormatConfig | null+ response_format?: string | AgentModelResponseFormatConfig | nullstop?: Array<string> | nulltemperature?: number | nulltop_p?: number | null@@ -3961,6 +4130,269 @@ export type GetAgentByAgentIdTextToAudioVoicesResponses = {export type GetAgentByAgentIdTextToAudioVoicesResponse =GetAgentByAgentIdTextToAudioVoicesResponses[keyof GetAgentByAgentIdTextToAudioVoicesResponses]+export type GetAgentByAgentIdTraceData = {+ body?: never+ path: {+ agent_id: string+ }+ query?: never+ url: '/agent/{agent_id}/trace'+}++export type GetAgentByAgentIdTraceErrors = {+ default: ErrorResponse+}++export type GetAgentByAgentIdTraceError =+ GetAgentByAgentIdTraceErrors[keyof GetAgentByAgentIdTraceErrors]++export type GetAgentByAgentIdTraceResponses = {+ 200: AppTraceResponse+}++export type GetAgentByAgentIdTraceResponse =+ GetAgentByAgentIdTraceResponses[keyof GetAgentByAgentIdTraceResponses]++export type PostAgentByAgentIdTraceData = {+ body: AppTracePayload+ path: {+ agent_id: string+ }+ query?: never+ url: '/agent/{agent_id}/trace'+}++export type PostAgentByAgentIdTraceErrors = {+ 400:+ | InvalidTracingConfigError+ | TracingProviderUnavailableError+ | UnsupportedTracingProviderError+ | ErrorResponse+ 404: TraceConfigNotFoundError | ErrorResponse+ 409: TracingConfigConflictError | ErrorResponse+ 500: TracingConfigProcessingFailedError | ErrorResponse+ default: ErrorResponse+}++export type PostAgentByAgentIdTraceError =+ PostAgentByAgentIdTraceErrors[keyof PostAgentByAgentIdTraceErrors]++export type PostAgentByAgentIdTraceResponses = {+ 200: AppTraceResponse+}++export type PostAgentByAgentIdTraceResponse =+ PostAgentByAgentIdTraceResponses[keyof PostAgentByAgentIdTraceResponses]++export type GetAgentByAgentIdTraceCapabilitiesData = {+ body?: never+ path: {+ agent_id: string+ }+ query: {+ tracing_provider: string+ }+ url: '/agent/{agent_id}/trace-capabilities'+}++export type GetAgentByAgentIdTraceCapabilitiesErrors = {+ 400: UnsupportedTracingProviderError | ErrorResponse+ default: ErrorResponse+}++export type GetAgentByAgentIdTraceCapabilitiesError =+ GetAgentByAgentIdTraceCapabilitiesErrors[keyof GetAgentByAgentIdTraceCapabilitiesErrors]++export type GetAgentByAgentIdTraceCapabilitiesResponses = {+ 200: TraceProviderCapabilitiesResponse+}++export type GetAgentByAgentIdTraceCapabilitiesResponse =+ GetAgentByAgentIdTraceCapabilitiesResponses[keyof GetAgentByAgentIdTraceCapabilitiesResponses]++export type DeleteAgentByAgentIdTraceConfigData = {+ body?: never+ path: {+ agent_id: string+ }+ query: {+ tracing_provider: string+ }+ url: '/agent/{agent_id}/trace-config'+}++export type DeleteAgentByAgentIdTraceConfigErrors = {+ 400: UnsupportedTracingProviderError | ErrorResponse+ 404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse+ 500: TracingConfigProcessingFailedError | ErrorResponse+ default: ErrorResponse+}++export type DeleteAgentByAgentIdTraceConfigError =+ DeleteAgentByAgentIdTraceConfigErrors[keyof DeleteAgentByAgentIdTraceConfigErrors]++export type DeleteAgentByAgentIdTraceConfigResponses = {+ 204: void+}++export type DeleteAgentByAgentIdTraceConfigResponse =+ DeleteAgentByAgentIdTraceConfigResponses[keyof DeleteAgentByAgentIdTraceConfigResponses]++export type GetAgentByAgentIdTraceConfigData = {+ body?: never+ path: {+ agent_id: string+ }+ query: {+ tracing_provider: string+ }+ url: '/agent/{agent_id}/trace-config'+}++export type GetAgentByAgentIdTraceConfigErrors = {+ 400: TracingProviderUnavailableError | UnsupportedTracingProviderError | ErrorResponse+ 404: AppNotFoundError | ErrorResponse+ 500: TracingConfigProcessingFailedError | ErrorResponse+ default: ErrorResponse+}++export type GetAgentByAgentIdTraceConfigError =+ GetAgentByAgentIdTraceConfigErrors[keyof GetAgentByAgentIdTraceConfigErrors]++export type GetAgentByAgentIdTraceConfigResponses = {+ 200: TraceAppConfigResponse+}++export type GetAgentByAgentIdTraceConfigResponse =+ GetAgentByAgentIdTraceConfigResponses[keyof GetAgentByAgentIdTraceConfigResponses]++export type PatchAgentByAgentIdTraceConfigData = {+ body: TraceConfigPayload+ path: {+ agent_id: string+ }+ query?: never+ url: '/agent/{agent_id}/trace-config'+}++export type PatchAgentByAgentIdTraceConfigErrors = {+ 400:+ | InvalidTracingConfigError+ | TracingConfigVerificationFailedError+ | TracingProviderUnavailableError+ | UnsupportedTracingProviderError+ | ErrorResponse+ 404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse+ 409: TracingConfigConflictError | ErrorResponse+ 500: TracingConfigProcessingFailedError | ErrorResponse+ default: ErrorResponse+}++export type PatchAgentByAgentIdTraceConfigError =+ PatchAgentByAgentIdTraceConfigErrors[keyof PatchAgentByAgentIdTraceConfigErrors]++export type PatchAgentByAgentIdTraceConfigResponses = {+ 200: TraceAppConfigResponse+}++export type PatchAgentByAgentIdTraceConfigResponse =+ PatchAgentByAgentIdTraceConfigResponses[keyof PatchAgentByAgentIdTraceConfigResponses]++export type PostAgentByAgentIdTraceConfigData = {+ body: TraceConfigPayload+ path: {+ agent_id: string+ }+ query?: never+ url: '/agent/{agent_id}/trace-config'+}++export type PostAgentByAgentIdTraceConfigErrors = {+ 400:+ | InvalidTracingConfigError+ | TracingConfigVerificationFailedError+ | TracingProviderUnavailableError+ | UnsupportedTracingProviderError+ | ErrorResponse+ 404: AppNotFoundError | ErrorResponse+ 409: TraceConfigAlreadyExistsError | ErrorResponse+ 500: TracingConfigProcessingFailedError | ErrorResponse+ default: ErrorResponse+}++export type PostAgentByAgentIdTraceConfigError =+ PostAgentByAgentIdTraceConfigErrors[keyof PostAgentByAgentIdTraceConfigErrors]++export type PostAgentByAgentIdTraceConfigResponses = {+ 201: TraceAppConfigResponse+}++export type PostAgentByAgentIdTraceConfigResponse =+ PostAgentByAgentIdTraceConfigResponses[keyof PostAgentByAgentIdTraceConfigResponses]++export type GetAgentByAgentIdTraceRunsByMessageIdData = {+ body?: never+ path: {+ agent_id: string+ message_id: string+ }+ query?: {+ node_execution_id?: string+ }+ url: '/agent/{agent_id}/trace-runs/{message_id}'+}++export type GetAgentByAgentIdTraceRunsByMessageIdErrors = {+ 404: TracingNodeNotFoundError | TracingSourceNotFoundError | ErrorResponse+ default: ErrorResponse+}++export type GetAgentByAgentIdTraceRunsByMessageIdError =+ GetAgentByAgentIdTraceRunsByMessageIdErrors[keyof GetAgentByAgentIdTraceRunsByMessageIdErrors]++export type GetAgentByAgentIdTraceRunsByMessageIdResponses = {+ 200: TraceRunStatusResponse+}++export type GetAgentByAgentIdTraceRunsByMessageIdResponse =+ GetAgentByAgentIdTraceRunsByMessageIdResponses[keyof GetAgentByAgentIdTraceRunsByMessageIdResponses]++export type PostAgentByAgentIdTraceRunsByMessageIdRetryData = {+ body: TraceRunRetryPayload+ path: {+ agent_id: string+ message_id: string+ }+ query?: {+ node_execution_id?: string+ }+ url: '/agent/{agent_id}/trace-runs/{message_id}/retry'+}++export type PostAgentByAgentIdTraceRunsByMessageIdRetryErrors = {+ 404:+ | TracingNodeNotFoundError+ | TracingRunNotFoundError+ | TracingSourceNotFoundError+ | ErrorResponse+ 409:+ | TracingAlreadyAcceptedError+ | TracingDuplicateAcknowledgementRequiredError+ | TracingRetryNotAllowedError+ | ErrorResponse+ default: ErrorResponse+}++export type PostAgentByAgentIdTraceRunsByMessageIdRetryError =+ PostAgentByAgentIdTraceRunsByMessageIdRetryErrors[keyof PostAgentByAgentIdTraceRunsByMessageIdRetryErrors]++export type PostAgentByAgentIdTraceRunsByMessageIdRetryResponses = {+ 200: TraceRunStatusResponse+}++export type PostAgentByAgentIdTraceRunsByMessageIdRetryResponse =+ PostAgentByAgentIdTraceRunsByMessageIdRetryResponses[keyof PostAgentByAgentIdTraceRunsByMessageIdRetryResponses]+export type GetAgentByAgentIdVersionsData = {body?: neverpath: {diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.tsindex cf27ddb2219..f6d8eb36780 100644--- a/packages/contracts/generated/api/console/agent/zod.gen.ts+++ b/packages/contracts/generated/api/console/agent/zod.gen.ts@@ -569,6 +569,253 @@ export const zProviderNotSupportTextToSpeechError = zErrorResponse.and(}),)+/**+ * AppTraceResponse+ */+export const zAppTraceResponse = z.object({+ capture_agent: z.boolean().optional().default(true),+ capture_content: z.boolean().optional().default(false),+ enabled: z.boolean().optional().default(false),+ tracing_provider: z.string().nullish(),+})++/**+ * AppTracePayload+ */+export const zAppTracePayload = z.object({+ capture_agent: z.boolean().optional().default(true),+ capture_content: z.boolean().optional().default(false),+ enabled: z.boolean(),+ tracing_provider: z.string().nullish(),+})++/**+ * InvalidTracingConfigError+ *+ * The tracing configuration is invalid.+ */+export const zInvalidTracingConfigError = zErrorResponse.and(+ z.object({+ code: z.literal('invalid_tracing_config'),+ status: z.literal(400),+ }),+)++/**+ * TracingProviderUnavailableError+ *+ * The tracing provider is not available in this deployment.+ */+export const zTracingProviderUnavailableError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_provider_unavailable'),+ status: z.literal(400),+ }),+)++/**+ * UnsupportedTracingProviderError+ *+ * The tracing provider is not supported.+ */+export const zUnsupportedTracingProviderError = zErrorResponse.and(+ z.object({+ code: z.literal('unsupported_tracing_provider'),+ status: z.literal(400),+ }),+)++/**+ * TraceConfigNotFoundError+ *+ * Tracing configuration not found.+ */+export const zTraceConfigNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('trace_config_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TracingConfigConflictError+ *+ * Tracing configuration changed. Refresh and try again.+ */+export const zTracingConfigConflictError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_config_conflict'),+ status: z.literal(409),+ }),+)++/**+ * TracingConfigProcessingFailedError+ *+ * The tracing configuration could not be processed.+ */+export const zTracingConfigProcessingFailedError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_config_processing_failed'),+ status: z.literal(500),+ }),+)++/**+ * TraceProviderCapabilitiesResponse+ */+export const zTraceProviderCapabilitiesResponse = z.object({+ agent_spans: z.boolean(),+ api_collection_enabled: z.boolean(),+ export_ack: z.literal('accepted'),+ native_cost: z.boolean(),+ native_usage: z.boolean(),+ provider: z.string(),+ session: z.boolean(),+ span_url: z.boolean(),+ trace_url: z.boolean(),+})++/**+ * AppNotFoundError+ *+ * App not found.+ */+export const zAppNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('app_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TraceAppConfigResponse+ */+export const zTraceAppConfigResponse = z.object({+ app_id: z.string().nullish(),+ created_at: z.string().nullish(),+ error: z.string().nullish(),+ has_not_configured: z.boolean().nullish(),+ id: z.string().nullish(),+ is_active: z.boolean().nullish(),+ result: z.string().nullish(),+ tracing_config: z.record(z.string(), z.string().nullable()).nullish(),+ tracing_provider: z.string().nullish(),+ updated_at: z.string().nullish(),+})++/**+ * TraceConfigPayload+ */+export const zTraceConfigPayload = z.object({+ activate: z.boolean().optional().default(false),+ tracing_config: z.record(z.string(), z.string().nullable()),+ tracing_provider: z.string(),+})++/**+ * TracingConfigVerificationFailedError+ *+ * The tracing configuration could not be verified.+ */+export const zTracingConfigVerificationFailedError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_config_verification_failed'),+ status: z.literal(400),+ }),+)++/**+ * TraceConfigAlreadyExistsError+ *+ * A tracing configuration already exists for this provider.+ */+export const zTraceConfigAlreadyExistsError = zErrorResponse.and(+ z.object({+ code: z.literal('trace_config_already_exists'),+ status: z.literal(409),+ }),+)++/**+ * TracingNodeNotFoundError+ *+ * The Agent node invocation does not belong to this trace.+ */+export const zTracingNodeNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_node_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TracingSourceNotFoundError+ *+ * The tracing source does not exist in this application.+ */+export const zTracingSourceNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_source_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TraceRunRetryPayload+ */+export const zTraceRunRetryPayload = z.object({+ acknowledge_possible_duplicates: z.boolean().optional().default(false),+})++/**+ * TracingRunNotFoundError+ *+ * No collected trace is available to retry.+ */+export const zTracingRunNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_run_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TracingAlreadyAcceptedError+ *+ * An accepted trace cannot be exported again.+ */+export const zTracingAlreadyAcceptedError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_already_accepted'),+ status: z.literal(409),+ }),+)++/**+ * TracingDuplicateAcknowledgementRequiredError+ *+ * Acknowledge possible duplicates before retrying an uncertain export.+ */+export const zTracingDuplicateAcknowledgementRequiredError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_duplicate_acknowledgement_required'),+ status: z.literal(409),+ }),+)++/**+ * TracingRetryNotAllowedError+ *+ * This trace is not in a retryable state.+ */+export const zTracingRetryNotAllowedError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_retry_not_allowed'),+ status: z.literal(409),+ }),+)+/*** AgentConfigSnapshotRestoreResponse*/@@ -1106,6 +1353,56 @@ export const zTextToSpeechVoiceResponse = z.object({*/export const zTextToSpeechVoiceListResponse = z.array(zTextToSpeechVoiceResponse)+/**+ * TraceProviderSupportResponse+ *+ * Adapter support declarations, not successful-delivery or platform-verification results.+ */+export const zTraceProviderSupportResponse = z.object({+ agent_spans: z.boolean(),+ export_ack: z.literal('accepted'),+ native_cost: z.boolean(),+ native_usage: z.boolean(),+ provider: z.string(),+ session: z.boolean(),+ span_url: z.boolean(),+ trace_url: z.boolean(),+})++export const zTraceSourceKind = z.enum(['message', 'node', 'workflow'])++/**+ * TraceRunStatusResponse+ */+export const zTraceRunStatusResponse = z.object({+ accepted_at: z.iso.datetime().nullish(),+ attempts: z.int().nullish(),+ capabilities: zTraceProviderSupportResponse.nullish(),+ capture_agent: z.boolean().nullish(),+ capture_content: z.boolean().nullish(),+ config_revision: z.string().nullish(),+ error_code: z.string().nullish(),+ project_url: z.string().nullish(),+ provider: z.string().nullish(),+ queryable: z.boolean().nullish(),+ source_id: z.string(),+ source_kind: zTraceSourceKind,+ span_count: z.int().nullish(),+ span_url: z.string().nullish(),+ status: z.enum([+ 'accepted',+ 'collecting',+ 'exporting',+ 'failed',+ 'not_collected',+ 'ready',+ 'retrying',+ 'uncertain',+ ]),+ trace_id: z.string().nullish(),+ trace_url: z.string().nullish(),+})+/*** AgentAppPublishedReferenceResponse*/@@ -2774,7 +3071,7 @@ export const zAgentSoulModelSettings = z.object({frequency_penalty: z.number().nullish(),max_tokens: z.int().nullish(),presence_penalty: z.number().nullish(),- response_format: zAgentModelResponseFormatConfig.nullish(),+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),stop: z.array(z.string()).nullish(),temperature: z.number().nullish(),top_p: z.number().nullish(),@@ -4217,6 +4514,117 @@ export const zGetAgentByAgentIdTextToAudioVoicesQuery = z.object({*/export const zGetAgentByAgentIdTextToAudioVoicesResponse = zTextToSpeechVoiceListResponse+export const zGetAgentByAgentIdTracePath = z.object({+ agent_id: z.uuid(),+})++/**+ * Agent tracing settings+ */+export const zGetAgentByAgentIdTraceResponse = zAppTraceResponse++export const zPostAgentByAgentIdTraceBody = zAppTracePayload++export const zPostAgentByAgentIdTracePath = z.object({+ agent_id: z.uuid(),+})++/**+ * Updated Agent tracing settings+ */+export const zPostAgentByAgentIdTraceResponse = zAppTraceResponse++export const zGetAgentByAgentIdTraceCapabilitiesPath = z.object({+ agent_id: z.uuid(),+})++export const zGetAgentByAgentIdTraceCapabilitiesQuery = z.object({+ tracing_provider: z.string(),+})++/**+ * Provider support declarations+ */+export const zGetAgentByAgentIdTraceCapabilitiesResponse = zTraceProviderCapabilitiesResponse++export const zDeleteAgentByAgentIdTraceConfigPath = z.object({+ agent_id: z.uuid(),+})++export const zDeleteAgentByAgentIdTraceConfigQuery = z.object({+ tracing_provider: z.string(),+})++/**+ * Agent provider configuration deleted+ */+export const zDeleteAgentByAgentIdTraceConfigResponse = z.void()++export const zGetAgentByAgentIdTraceConfigPath = z.object({+ agent_id: z.uuid(),+})++export const zGetAgentByAgentIdTraceConfigQuery = z.object({+ tracing_provider: z.string(),+})++/**+ * Masked Agent provider configuration+ */+export const zGetAgentByAgentIdTraceConfigResponse = zTraceAppConfigResponse++export const zPatchAgentByAgentIdTraceConfigBody = zTraceConfigPayload++export const zPatchAgentByAgentIdTraceConfigPath = z.object({+ agent_id: z.uuid(),+})++/**+ * Agent provider updated+ */+export const zPatchAgentByAgentIdTraceConfigResponse = zTraceAppConfigResponse++export const zPostAgentByAgentIdTraceConfigBody = zTraceConfigPayload++export const zPostAgentByAgentIdTraceConfigPath = z.object({+ agent_id: z.uuid(),+})++/**+ * Agent provider configured+ */+export const zPostAgentByAgentIdTraceConfigResponse = zTraceAppConfigResponse++export const zGetAgentByAgentIdTraceRunsByMessageIdPath = z.object({+ agent_id: z.uuid(),+ message_id: z.uuid(),+})++export const zGetAgentByAgentIdTraceRunsByMessageIdQuery = z.object({+ node_execution_id: z.uuid().optional(),+})++/**+ * Frozen trace destination and export state+ */+export const zGetAgentByAgentIdTraceRunsByMessageIdResponse = zTraceRunStatusResponse++export const zPostAgentByAgentIdTraceRunsByMessageIdRetryBody = zTraceRunRetryPayload++export const zPostAgentByAgentIdTraceRunsByMessageIdRetryPath = z.object({+ agent_id: z.uuid(),+ message_id: z.uuid(),+})++export const zPostAgentByAgentIdTraceRunsByMessageIdRetryQuery = z.object({+ node_execution_id: z.uuid().optional(),+})++/**+ * Retry state for the same frozen destination+ */+export const zPostAgentByAgentIdTraceRunsByMessageIdRetryResponse = zTraceRunStatusResponse+export const zGetAgentByAgentIdVersionsPath = z.object({agent_id: z.uuid(),})diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.tsindex b19f277e3c2..43e89f94d39 100644--- a/packages/contracts/generated/api/console/apps/orpc.gen.ts+++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts@@ -46,9 +46,16 @@ import {zTooManyFilesError,zTraceConfigAlreadyExistsError,zTraceConfigNotFoundError,+ zTracingAlreadyAcceptedError,+ zTracingConfigConflictError,zTracingConfigProcessingFailedError,zTracingConfigVerificationFailedError,+ zTracingDuplicateAcknowledgementRequiredError,+ zTracingNodeNotFoundError,zTracingProviderUnavailableError,+ zTracingRetryNotAllowedError,+ zTracingRunNotFoundError,+ zTracingSourceNotFoundError,zUnsupportedAudioTypeError,zUnsupportedTracingProviderError,zWorkflowRunNotFoundError,@@ -202,11 +209,17 @@ import {zGetAppsByAppIdTextToAudioVoicesPath,zGetAppsByAppIdTextToAudioVoicesQuery,zGetAppsByAppIdTextToAudioVoicesResponse,+ zGetAppsByAppIdTraceCapabilitiesPath,+ zGetAppsByAppIdTraceCapabilitiesQuery,+ zGetAppsByAppIdTraceCapabilitiesResponse,zGetAppsByAppIdTraceConfigPath,zGetAppsByAppIdTraceConfigQuery,zGetAppsByAppIdTraceConfigResponse,zGetAppsByAppIdTracePath,zGetAppsByAppIdTraceResponse,+ zGetAppsByAppIdTraceRunsBySourceKindBySourceIdPath,+ zGetAppsByAppIdTraceRunsBySourceKindBySourceIdQuery,+ zGetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse,zGetAppsByAppIdTriggersPath,zGetAppsByAppIdTriggersResponse,zGetAppsByAppIdWorkflowAppLogsPath,@@ -413,6 +426,10 @@ import {zPostAppsByAppIdTraceConfigResponse,zPostAppsByAppIdTracePath,zPostAppsByAppIdTraceResponse,+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryBody,+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryPath,+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryQuery,+ zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse,zPostAppsByAppIdTriggerEnableBody,zPostAppsByAppIdTriggerEnablePath,zPostAppsByAppIdTriggerEnableResponse,@@ -2683,13 +2700,42 @@ export const post35 = oc}).input(z.object({ body: zPostAppsByAppIdTraceBody, params: zPostAppsByAppIdTracePath })).output(zPostAppsByAppIdTraceResponse)- .errors({ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError } })+ .errors({+ app_not_found: { status: 404, data: zAppNotFoundError },+ invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },+ trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },+ tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },+ tracing_provider_unavailable: { status: 400, data: zTracingProviderUnavailableError },+ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError },+ })export const trace = {get: get45,post: post35,}+export const get46 = oc+ .route({+ inputStructure: 'detailed',+ method: 'GET',+ operationId: 'getAppsByAppIdTraceCapabilities',+ path: '/apps/{app_id}/trace-capabilities',+ tags: ['console'],+ })+ .input(+ z.object({+ params: zGetAppsByAppIdTraceCapabilitiesPath,+ query: zGetAppsByAppIdTraceCapabilitiesQuery,+ }),+ )+ .output(zGetAppsByAppIdTraceCapabilitiesResponse)+ .errors({ unsupported_tracing_provider: { status: 400, data: zUnsupportedTracingProviderError } })++export const traceCapabilities = {+ get: get46,+}+/*** Delete an existing trace app configuration*@@ -2723,7 +2769,7 @@ export const delete8 = oc/*** Get tracing configuration for an application*/-export const get46 = oc+export const get47 = oc.route({description: 'Get tracing configuration for an application',inputStructure: 'detailed',@@ -2766,6 +2812,7 @@ export const patch = ocapp_not_found: { status: 404, data: zAppNotFoundError },invalid_tracing_config: { status: 400, data: zInvalidTracingConfigError },trace_config_not_found: { status: 404, data: zTraceConfigNotFoundError },+ tracing_config_conflict: { status: 409, data: zTracingConfigConflictError },tracing_config_processing_failed: { status: 500, data: zTracingConfigProcessingFailedError },tracing_config_verification_failed: {status: 400,@@ -2810,15 +2857,80 @@ export const post36 = ocexport const traceConfig = {delete: delete8,- get: get46,+ get: get47,patch,post: post36,}+export const post37 = oc+ .route({+ inputStructure: 'detailed',+ method: 'POST',+ operationId: 'postAppsByAppIdTraceRunsBySourceKindBySourceIdRetry',+ path: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}/retry',+ tags: ['console'],+ })+ .input(+ z.object({+ body: zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryBody,+ params: zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryPath,+ query: zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryQuery.optional(),+ }),+ )+ .output(zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse)+ .errors({+ tracing_already_accepted: { status: 409, data: zTracingAlreadyAcceptedError },+ tracing_duplicate_acknowledgement_required: {+ status: 409,+ data: zTracingDuplicateAcknowledgementRequiredError,+ },+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },+ tracing_retry_not_allowed: { status: 409, data: zTracingRetryNotAllowedError },+ tracing_run_not_found: { status: 404, data: zTracingRunNotFoundError },+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },+ })++export const retry = {+ post: post37,+}++export const get48 = oc+ .route({+ inputStructure: 'detailed',+ method: 'GET',+ operationId: 'getAppsByAppIdTraceRunsBySourceKindBySourceId',+ path: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}',+ tags: ['console'],+ })+ .input(+ z.object({+ params: zGetAppsByAppIdTraceRunsBySourceKindBySourceIdPath,+ query: zGetAppsByAppIdTraceRunsBySourceKindBySourceIdQuery.optional(),+ }),+ )+ .output(zGetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse)+ .errors({+ tracing_node_not_found: { status: 404, data: zTracingNodeNotFoundError },+ tracing_source_not_found: { status: 404, data: zTracingSourceNotFoundError },+ })++export const bySourceId = {+ get: get48,+ retry,+}++export const bySourceKind = {+ bySourceId,+}++export const traceRuns = {+ bySourceKind,+}+/*** Update app trigger (enable/disable)*/-export const post37 = oc+export const post38 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -2837,13 +2949,13 @@ export const post37 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const triggerEnable = {- post: post37,+ post: post38,}/*** Get app triggers list*/-export const get47 = oc+export const get49 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -2857,7 +2969,7 @@ export const get47 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const triggers = {- get: get47,+ get: get49,}/**@@ -2865,7 +2977,7 @@ export const triggers = {** Get workflow application execution logs*/-export const get48 = oc+export const get50 = oc.route({description: 'Get workflow application execution logs',inputStructure: 'detailed',@@ -2885,7 +2997,7 @@ export const get48 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const workflowAppLogs = {- get: get48,+ get: get50,}/**@@ -2893,7 +3005,7 @@ export const workflowAppLogs = {** Get workflow runs count statistics*/-export const get49 = oc+export const get51 = oc.route({description: 'Get workflow runs count statistics',inputStructure: 'detailed',@@ -2913,7 +3025,7 @@ export const get49 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const count3 = {- get: get49,+ get: get51,}/**@@ -2921,7 +3033,7 @@ export const count3 = {** Stop running workflow task*/-export const post38 = oc+export const post39 = oc.route({description: 'Stop running workflow task',inputStructure: 'detailed',@@ -2936,7 +3048,7 @@ export const post38 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const stop3 = {- post: post38,+ post: post39,}export const byTaskId3 = {@@ -2950,7 +3062,7 @@ export const tasks = {/*** Get the internal node executions of one workflow tool invocation*/-export const get50 = oc+export const get52 = oc.route({description: 'Get the internal node executions of one workflow tool invocation',inputStructure: 'detailed',@@ -2968,7 +3080,7 @@ export const get50 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const children = {- get: get50,+ get: get52,}export const byNodeExecutionId = {@@ -2980,7 +3092,7 @@ export const byNodeExecutionId = {** Get workflow run node execution list*/-export const get51 = oc+export const get53 = oc.route({description: 'Get workflow run node execution list',inputStructure: 'detailed',@@ -2995,7 +3107,7 @@ export const get51 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const nodeExecutions = {- get: get51,+ get: get53,byNodeExecutionId,}@@ -3004,7 +3116,7 @@ export const nodeExecutions = {** Get workflow run detail*/-export const get52 = oc+export const get54 = oc.route({description: 'Get workflow run detail',inputStructure: 'detailed',@@ -3019,14 +3131,14 @@ export const get52 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const byRunId = {- get: get52,+ get: get54,nodeExecutions,}/*** Create a ToolFile from one workflow Agent Binding file and return its download URL*/-export const post39 = oc+export const post40 = oc.route({description:'Create a ToolFile from one workflow Agent Binding file and return its download URL',@@ -3052,13 +3164,13 @@ export const post39 = oc})export const download4 = {- post: post39,+ post: post40,}/*** Read a text/binary preview file in a workflow Agent node sandbox*/-export const get53 = oc+export const get55 = oc.route({description: 'Read a text/binary preview file in a workflow Agent node sandbox',inputStructure: 'detailed',@@ -3082,13 +3194,13 @@ export const get53 = oc})export const read = {- get: get53,+ get: get55,}/*** List a directory in a workflow Agent node sandbox*/-export const get54 = oc+export const get56 = oc.route({description: 'List a directory in a workflow Agent node sandbox',inputStructure: 'detailed',@@ -3112,7 +3224,7 @@ export const get54 = oc})export const files3 = {- get: get54,+ get: get56,download: download4,read,}@@ -3138,7 +3250,7 @@ export const byWorkflowRunId = {** Get workflow run list*/-export const get55 = oc+export const get57 = oc.route({description: 'Get workflow run list',inputStructure: 'detailed',@@ -3158,7 +3270,7 @@ export const get55 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const workflowRuns2 = {- get: get55,+ get: get57,count: count3,tasks,byRunId,@@ -3170,7 +3282,7 @@ export const workflowRuns2 = {** Get all users in current tenant for mentions*/-export const get56 = oc+export const get58 = oc.route({description: 'Get all users in current tenant for mentions',inputStructure: 'detailed',@@ -3188,7 +3300,7 @@ export const get56 = oc})export const mentionUsers = {- get: get56,+ get: get58,}/**@@ -3251,7 +3363,7 @@ export const byReplyId = {** Add a reply to a workflow comment*/-export const post40 = oc+export const post41 = oc.route({description: 'Add a reply to a workflow comment',inputStructure: 'detailed',@@ -3275,7 +3387,7 @@ export const post40 = oc})export const replies = {- post: post40,+ post: post41,byReplyId,}@@ -3284,7 +3396,7 @@ export const replies = {** Resolve a workflow comment*/-export const post41 = oc+export const post42 = oc.route({description: 'Resolve a workflow comment',inputStructure: 'detailed',@@ -3302,7 +3414,7 @@ export const post41 = oc})export const resolve = {- post: post41,+ post: post42,}/**@@ -3333,7 +3445,7 @@ export const delete10 = oc** Get a specific workflow comment*/-export const get57 = oc+export const get59 = oc.route({description: 'Get a specific workflow comment',inputStructure: 'detailed',@@ -3379,7 +3491,7 @@ export const put4 = ocexport const byCommentId = {delete: delete10,- get: get57,+ get: get59,put: put4,replies,resolve,@@ -3390,7 +3502,7 @@ export const byCommentId = {** Get all comments for a workflow*/-export const get58 = oc+export const get60 = oc.route({description: 'Get all comments for a workflow',inputStructure: 'detailed',@@ -3412,7 +3524,7 @@ export const get58 = oc** Create a new workflow comment*/-export const post42 = oc+export const post43 = oc.route({description: 'Create a new workflow comment',inputStructure: 'detailed',@@ -3436,8 +3548,8 @@ export const post42 = oc})export const comments = {- get: get58,- post: post42,+ get: get60,+ post: post43,mentionUsers,byCommentId,}@@ -3445,7 +3557,7 @@ export const comments = {/*** Get workflow average app interaction statistics*/-export const get59 = oc+export const get61 = oc.route({description: 'Get workflow average app interaction statistics',inputStructure: 'detailed',@@ -3464,13 +3576,13 @@ export const get59 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const averageAppInteractions = {- get: get59,+ get: get61,}/*** Get workflow daily runs statistics*/-export const get60 = oc+export const get62 = oc.route({description: 'Get workflow daily runs statistics',inputStructure: 'detailed',@@ -3489,13 +3601,13 @@ export const get60 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const dailyConversations2 = {- get: get60,+ get: get62,}/*** Get workflow daily terminals statistics*/-export const get61 = oc+export const get63 = oc.route({description: 'Get workflow daily terminals statistics',inputStructure: 'detailed',@@ -3514,13 +3626,13 @@ export const get61 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const dailyTerminals = {- get: get61,+ get: get63,}/*** Get workflow daily token cost statistics*/-export const get62 = oc+export const get64 = oc.route({description: 'Get workflow daily token cost statistics',inputStructure: 'detailed',@@ -3539,7 +3651,7 @@ export const get62 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const tokenCosts2 = {- get: get62,+ get: get64,}export const statistics2 = {@@ -3559,7 +3671,7 @@ export const workflow = {** Get default block configuration by type*/-export const get63 = oc+export const get65 = oc.route({description: 'Get default block configuration by type',inputStructure: 'detailed',@@ -3579,7 +3691,7 @@ export const get63 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const byBlockType = {- get: get63,+ get: get65,}/**@@ -3587,7 +3699,7 @@ export const byBlockType = {** Get default block configurations for workflow*/-export const get64 = oc+export const get66 = oc.route({description: 'Get default block configurations for workflow',inputStructure: 'detailed',@@ -3602,14 +3714,14 @@ export const get64 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const defaultWorkflowBlockConfigs = {- get: get64,+ get: get66,byBlockType,}/*** Get conversation variables for workflow*/-export const get65 = oc+export const get67 = oc.route({description: 'Get conversation variables for workflow',inputStructure: 'detailed',@@ -3625,7 +3737,7 @@ export const get65 = oc/*** Update conversation variables for workflow draft*/-export const post43 = oc+export const post44 = oc.route({description: 'Update conversation variables for workflow draft',inputStructure: 'detailed',@@ -3644,8 +3756,8 @@ export const post43 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const conversationVariables2 = {- get: get65,- post: post43,+ get: get67,+ post: post44,}/**@@ -3653,7 +3765,7 @@ export const conversationVariables2 = {** Get environment variables for workflow*/-export const get66 = oc+export const get68 = oc.route({description: 'Get environment variables for workflow',inputStructure: 'detailed',@@ -3673,7 +3785,7 @@ export const get66 = oc/*** Update environment variables for workflow draft*/-export const post44 = oc+export const post45 = oc.route({description: 'Update environment variables for workflow draft',inputStructure: 'detailed',@@ -3692,14 +3804,14 @@ export const post44 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const environmentVariables = {- get: get66,- post: post44,+ get: get68,+ post: post45,}/*** Update draft workflow features*/-export const post45 = oc+export const post46 = oc.route({description: 'Update draft workflow features',inputStructure: 'detailed',@@ -3718,7 +3830,7 @@ export const post45 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const features = {- post: post45,+ post: post46,}/**@@ -3726,7 +3838,7 @@ export const features = {** Test human input delivery for workflow*/-export const post46 = oc+export const post47 = oc.route({description: 'Test human input delivery for workflow',inputStructure: 'detailed',@@ -3746,7 +3858,7 @@ export const post46 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const deliveryTest = {- post: post46,+ post: post47,}/**@@ -3754,7 +3866,7 @@ export const deliveryTest = {** Get human input form preview for workflow*/-export const post47 = oc+export const post48 = oc.route({description: 'Get human input form preview for workflow',inputStructure: 'detailed',@@ -3774,7 +3886,7 @@ export const post47 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const preview4 = {- post: post47,+ post: post48,}/**@@ -3782,7 +3894,7 @@ export const preview4 = {** Submit human input form preview for workflow*/-export const post48 = oc+export const post49 = oc.route({description: 'Submit human input form preview for workflow',inputStructure: 'detailed',@@ -3802,7 +3914,7 @@ export const post48 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const run5 = {- post: post48,+ post: post49,}export const form2 = {@@ -3828,7 +3940,7 @@ export const humanInput2 = {** Run draft workflow iteration node*/-export const post49 = oc+export const post50 = oc.route({description: 'Run draft workflow iteration node',inputStructure: 'detailed',@@ -3848,7 +3960,7 @@ export const post49 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const run6 = {- post: post49,+ post: post50,}export const byNodeId6 = {@@ -3868,7 +3980,7 @@ export const iteration2 = {** Run draft workflow loop node*/-export const post50 = oc+export const post51 = oc.route({description: 'Run draft workflow loop node',inputStructure: 'detailed',@@ -3888,7 +4000,7 @@ export const post50 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const run7 = {- post: post50,+ post: post51,}export const byNodeId7 = {@@ -3903,7 +4015,7 @@ export const loop2 = {nodes: nodes6,}-export const get67 = oc+export const get69 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -3918,10 +4030,10 @@ export const get67 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const candidates = {- get: get67,+ get: get69,}-export const post51 = oc+export const post52 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -3945,10 +4057,10 @@ export const post51 = oc})export const copyFromRoster = {- post: post51,+ post: post52,}-export const post52 = oc+export const post53 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -3966,10 +4078,10 @@ export const post52 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const impact = {- post: post52,+ post: post53,}-export const post53 = oc+export const post54 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -3995,10 +4107,10 @@ export const post53 = oc})export const saveToRoster = {- post: post53,+ post: post54,}-export const post54 = oc+export const post55 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -4021,10 +4133,10 @@ export const post54 = oc})export const validate = {- post: post54,+ post: post55,}-export const get68 = oc+export const get70 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -4071,7 +4183,7 @@ export const put5 = oc})export const agentComposer = {- get: get68,+ get: get70,put: put5,candidates,copyFromRoster,@@ -4083,7 +4195,7 @@ export const agentComposer = {/*** Get last run result for draft workflow node*/-export const get69 = oc+export const get71 = oc.route({description: 'Get last run result for draft workflow node',inputStructure: 'detailed',@@ -4097,7 +4209,7 @@ export const get69 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const lastRun = {- get: get69,+ get: get71,}/**@@ -4105,7 +4217,7 @@ export const lastRun = {** Run draft workflow node*/-export const post55 = oc+export const post56 = oc.route({description: 'Run draft workflow node',inputStructure: 'detailed',@@ -4125,7 +4237,7 @@ export const post55 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const run8 = {- post: post55,+ post: post56,}/**@@ -4133,7 +4245,7 @@ export const run8 = {** Poll for trigger events and execute single node when event arrives*/-export const post56 = oc+export const post57 = oc.route({description: 'Poll for trigger events and execute single node when event arrives',inputStructure: 'detailed',@@ -4148,7 +4260,7 @@ export const post56 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const run9 = {- post: post56,+ post: post57,}export const trigger = {@@ -4175,7 +4287,7 @@ export const delete11 = oc/*** Get variables for a specific node*/-export const get70 = oc+export const get72 = oc.route({description: 'Get variables for a specific node',inputStructure: 'detailed',@@ -4190,7 +4302,7 @@ export const get70 = ocexport const variables = {delete: delete11,- get: get70,+ get: get72,}export const byNodeId8 = {@@ -4210,7 +4322,7 @@ export const nodes7 = {** Run draft workflow*/-export const post57 = oc+export const post58 = oc.route({description: 'Run draft workflow',inputStructure: 'detailed',@@ -4233,13 +4345,13 @@ export const post57 = oc})export const run10 = {- post: post57,+ post: post58,}/*** Server-Sent Events stream of inspector deltas for a draft workflow run.*/-export const get71 = oc+export const get73 = oc.route({description: 'Server-Sent Events stream of inspector deltas for a draft workflow run.',inputStructure: 'detailed',@@ -4256,13 +4368,13 @@ export const get71 = oc})export const events = {- get: get71,+ get: get73,}/*** Full value for one declared output, including signed download URL for files.*/-export const get72 = oc+export const get74 = oc.route({description: 'Full value for one declared output, including signed download URL for files.',inputStructure: 'detailed',@@ -4285,7 +4397,7 @@ export const get72 = oc})export const preview5 = {- get: get72,+ get: get74,}export const byOutputName = {@@ -4295,7 +4407,7 @@ export const byOutputName = {/*** One node's declared outputs for a draft workflow run.*/-export const get73 = oc+export const get75 = oc.route({description: "One node's declared outputs for a draft workflow run.",inputStructure: 'detailed',@@ -4313,14 +4425,14 @@ export const get73 = oc})export const byNodeId9 = {- get: get73,+ get: get75,byOutputName,}/*** Snapshot of every node's declared outputs for a draft workflow run.*/-export const get74 = oc+export const get76 = oc.route({description: "Snapshot of every node's declared outputs for a draft workflow run.",inputStructure: 'detailed',@@ -4337,7 +4449,7 @@ export const get74 = oc})export const nodeOutputs = {- get: get74,+ get: get76,events,byNodeId: byNodeId9,}@@ -4353,7 +4465,7 @@ export const runs = {/*** Get system variables for workflow*/-export const get75 = oc+export const get77 = oc.route({description: 'Get system variables for workflow',inputStructure: 'detailed',@@ -4367,7 +4479,7 @@ export const get75 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const systemVariables = {- get: get75,+ get: get77,}/**@@ -4375,7 +4487,7 @@ export const systemVariables = {** Poll for trigger events and execute full workflow when event arrives*/-export const post58 = oc+export const post59 = oc.route({description: 'Poll for trigger events and execute full workflow when event arrives',inputStructure: 'detailed',@@ -4398,7 +4510,7 @@ export const post58 = oc})export const run11 = {- post: post58,+ post: post59,}/**@@ -4406,7 +4518,7 @@ export const run11 = {** Full workflow debug when the start node is a trigger*/-export const post59 = oc+export const post60 = oc.route({description: 'Full workflow debug when the start node is a trigger',inputStructure: 'detailed',@@ -4429,7 +4541,7 @@ export const post59 = oc})export const runAll = {- post: post59,+ post: post60,}export const trigger2 = {@@ -4477,7 +4589,7 @@ export const delete12 = oc/*** Get a specific workflow variable*/-export const get76 = oc+export const get78 = oc.route({description: 'Get a specific workflow variable',inputStructure: 'detailed',@@ -4513,7 +4625,7 @@ export const patch2 = ocexport const byVariableId = {delete: delete12,- get: get76,+ get: get78,patch: patch2,reset,}@@ -4540,7 +4652,7 @@ export const delete13 = oc** Get draft workflow variables*/-export const get77 = oc+export const get79 = oc.route({description: 'Get draft workflow variables',inputStructure: 'detailed',@@ -4564,7 +4676,7 @@ export const get77 = ocexport const variables2 = {delete: delete13,- get: get77,+ get: get79,byVariableId,}@@ -4573,7 +4685,7 @@ export const variables2 = {** Get draft workflow for an application*/-export const get78 = oc+export const get80 = oc.route({description: 'Get draft workflow for an application',inputStructure: 'detailed',@@ -4595,7 +4707,7 @@ export const get78 = oc** Sync draft workflow configuration*/-export const post60 = oc+export const post61 = oc.route({description: 'Sync draft workflow configuration',inputStructure: 'detailed',@@ -4618,8 +4730,8 @@ export const post60 = oc})export const draft2 = {- get: get78,- post: post60,+ get: get80,+ post: post61,conversationVariables: conversationVariables2,environmentVariables,features,@@ -4639,7 +4751,7 @@ export const draft2 = {** Get published workflow for an application*/-export const get79 = oc+export const get81 = oc.route({description: 'Get published workflow for an application',inputStructure: 'detailed',@@ -4656,7 +4768,7 @@ export const get79 = oc/*** Publish workflow*/-export const post61 = oc+export const post62 = oc.route({inputStructure: 'detailed',method: 'POST',@@ -4675,14 +4787,14 @@ export const post61 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const publish = {- get: get79,- post: post61,+ get: get81,+ post: post62,}/*** Server-Sent Events stream of inspector deltas for a published workflow run.*/-export const get80 = oc+export const get82 = oc.route({description: 'Server-Sent Events stream of inspector deltas for a published workflow run.',inputStructure: 'detailed',@@ -4699,13 +4811,13 @@ export const get80 = oc})export const events2 = {- get: get80,+ get: get82,}/*** Full value for one declared output of a published run.*/-export const get81 = oc+export const get83 = oc.route({description: 'Full value for one declared output of a published run.',inputStructure: 'detailed',@@ -4732,7 +4844,7 @@ export const get81 = oc})export const preview6 = {- get: get81,+ get: get83,}export const byOutputName2 = {@@ -4742,7 +4854,7 @@ export const byOutputName2 = {/*** One node's declared outputs for a published workflow run.*/-export const get82 = oc+export const get84 = oc.route({description: "One node's declared outputs for a published workflow run.",inputStructure: 'detailed',@@ -4760,14 +4872,14 @@ export const get82 = oc})export const byNodeId10 = {- get: get82,+ get: get84,byOutputName: byOutputName2,}/*** Snapshot of every node's declared outputs for a published workflow run.*/-export const get83 = oc+export const get85 = oc.route({description: "Snapshot of every node's declared outputs for a published workflow run.",inputStructure: 'detailed',@@ -4784,7 +4896,7 @@ export const get83 = oc})export const nodeOutputs2 = {- get: get83,+ get: get85,events: events2,byNodeId: byNodeId10,}@@ -4804,7 +4916,7 @@ export const published = {/*** Get webhook trigger for a node*/-export const get84 = oc+export const get86 = oc.route({inputStructure: 'detailed',method: 'GET',@@ -4823,7 +4935,7 @@ export const get84 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const webhook = {- get: get84,+ get: get86,}export const triggers2 = {@@ -4833,7 +4945,7 @@ export const triggers2 = {/*** Restore a published workflow version into the draft workflow*/-export const post62 = oc+export const post63 = oc.route({description: 'Restore a published workflow version into the draft workflow',inputStructure: 'detailed',@@ -4847,7 +4959,7 @@ export const post62 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const restore = {- post: post62,+ post: post63,}/**@@ -4902,7 +5014,7 @@ export const byWorkflowId = {** Get all published workflows for an application*/-export const get85 = oc+export const get87 = oc.route({description: 'Get all published workflows for an application',inputStructure: 'detailed',@@ -4922,7 +5034,7 @@ export const get85 = oc.errors({ app_not_found: { status: 404, data: zAppNotFoundError } })export const workflows3 = {- get: get85,+ get: get87,defaultWorkflowBlockConfigs,draft: draft2,publish,@@ -4950,7 +5062,7 @@ export const delete15 = oc/*** Get application details*/-export const get86 = oc+export const get88 = oc.route({description: 'Get application details',inputStructure: 'detailed',@@ -4979,7 +5091,7 @@ export const put7 = ocexport const byAppId2 = {delete: delete15,- get: get86,+ get: get88,put: put7,advancedChat,agent,@@ -5011,7 +5123,9 @@ export const byAppId2 = {statistics,textToAudio,trace,+ traceCapabilities,traceConfig,+ traceRuns,triggerEnable,triggers,workflowAppLogs,@@ -5043,7 +5157,7 @@ export const byApiKeyId = {/*** Get all API keys for an app*/-export const get87 = oc+export const get89 = oc.route({description: 'Get all API keys for an app',inputStructure: 'detailed',@@ -5058,7 +5172,7 @@ export const get87 = oc/*** Create a new API key for an app*/-export const post63 = oc+export const post64 = oc.route({description: 'Create a new API key for an app',inputStructure: 'detailed',@@ -5076,8 +5190,8 @@ export const post63 = oc})export const apiKeys = {- get: get87,- post: post63,+ get: get89,+ post: post64,byApiKeyId,}@@ -5088,7 +5202,7 @@ export const byResourceId = {/*** Get list of applications with pagination and filtering*/-export const get88 = oc+export const get90 = oc.route({description: 'Get list of applications with pagination and filtering',inputStructure: 'detailed',@@ -5103,7 +5217,7 @@ export const get88 = oc/*** Create a new application*/-export const post64 = oc+export const post65 = oc.route({description: 'Create a new application',inputStructure: 'detailed',@@ -5117,8 +5231,8 @@ export const post64 = oc.output(zPostAppsResponse)export const apps = {- get: get88,- post: post64,+ get: get90,+ post: post65,imports,recent,starred,diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.tsindex 96355856cfe..d2ba8be55c7 100644--- a/packages/contracts/generated/api/console/apps/types.gen.ts+++ b/packages/contracts/generated/api/console/apps/types.gen.ts@@ -851,15 +851,24 @@ export type ProviderNotSupportTextToSpeechError = ErrorResponse & {export type TextToSpeechVoiceListResponse = Array<TextToSpeechVoiceResponse>export type AppTraceResponse = {+ capture_agent?: boolean+ capture_content?: booleanenabled?: booleantracing_provider?: string | null}export type AppTracePayload = {+ capture_agent?: boolean+ capture_content?: booleanenabled: booleantracing_provider?: string | null}+export type InvalidTracingConfigError = ErrorResponse & {+ code: 'invalid_tracing_config'+ status: 400+}+export type TracingProviderUnavailableError = ErrorResponse & {code: 'tracing_provider_unavailable'status: 400@@ -875,11 +884,28 @@ export type TraceConfigNotFoundError = ErrorResponse & {status: 404}+export type TracingConfigConflictError = ErrorResponse & {+ code: 'tracing_config_conflict'+ status: 409+}+export type TracingConfigProcessingFailedError = ErrorResponse & {code: 'tracing_config_processing_failed'status: 500}+export type TraceProviderCapabilitiesResponse = {+ agent_spans: boolean+ api_collection_enabled: boolean+ export_ack: 'accepted'+ native_cost: boolean+ native_usage: boolean+ provider: string+ session: boolean+ span_url: boolean+ trace_url: boolean+}+export type TraceAppConfigResponse = {app_id?: string | nullcreated_at?: string | null@@ -896,17 +922,13 @@ export type TraceAppConfigResponse = {}export type TraceConfigPayload = {+ activate?: booleantracing_config: {[key: string]: string | null}tracing_provider: string}-export type InvalidTracingConfigError = ErrorResponse & {- code: 'invalid_tracing_config'- status: 400-}-export type TracingConfigVerificationFailedError = ErrorResponse & {code: 'tracing_config_verification_failed'status: 400@@ -917,6 +939,68 @@ export type TraceConfigAlreadyExistsError = ErrorResponse & {status: 409}+export type TraceRunStatusResponse = {+ accepted_at?: string | null+ attempts?: number | null+ capabilities?: TraceProviderSupportResponse | null+ capture_agent?: boolean | null+ capture_content?: boolean | null+ config_revision?: string | null+ error_code?: string | null+ project_url?: string | null+ provider?: string | null+ queryable?: boolean | null+ source_id: string+ source_kind: TraceSourceKind+ span_count?: number | null+ span_url?: string | null+ status:+ | 'accepted'+ | 'collecting'+ | 'exporting'+ | 'failed'+ | 'not_collected'+ | 'ready'+ | 'retrying'+ | 'uncertain'+ trace_id?: string | null+ trace_url?: string | null+}++export type TracingNodeNotFoundError = ErrorResponse & {+ code: 'tracing_node_not_found'+ status: 404+}++export type TracingSourceNotFoundError = ErrorResponse & {+ code: 'tracing_source_not_found'+ status: 404+}++export type TraceRunRetryPayload = {+ acknowledge_possible_duplicates?: boolean+}++export type TracingRunNotFoundError = ErrorResponse & {+ code: 'tracing_run_not_found'+ status: 404+}++export type TracingAlreadyAcceptedError = ErrorResponse & {+ code: 'tracing_already_accepted'+ status: 409+}++export type TracingDuplicateAcknowledgementRequiredError = ErrorResponse & {+ code: 'tracing_duplicate_acknowledgement_required'+ status: 409+}++export type TracingRetryNotAllowedError = ErrorResponse & {+ code: 'tracing_retry_not_allowed'+ status: 409+}+export type ParserEnable = {enable_trigger: booleantrigger_id: string@@ -2186,6 +2270,19 @@ export type TextToSpeechVoiceResponse = {value: string}+export type TraceProviderSupportResponse = {+ agent_spans: boolean+ export_ack: 'accepted'+ native_cost: boolean+ native_usage: boolean+ provider: string+ session: boolean+ span_url: boolean+ trace_url: boolean+}++export type TraceSourceKind = 'message' | 'node' | 'workflow'+export type WorkflowAppLogPartialResponse = {created_at?: number | nullcreated_by_account?: SimpleAccountResponse | null@@ -4007,7 +4104,7 @@ export type AgentSoulModelSettings = {frequency_penalty?: number | nullmax_tokens?: number | nullpresence_penalty?: number | null- response_format?: AgentModelResponseFormatConfig | null+ response_format?: string | AgentModelResponseFormatConfig | nullstop?: Array<string> | nulltemperature?: number | nulltop_p?: number | null@@ -7036,8 +7133,15 @@ export type PostAppsByAppIdTraceData = {}export type PostAppsByAppIdTraceErrors = {- 400: TracingProviderUnavailableError | ErrorResponse+ 400:+ | InvalidTracingConfigError+ | TracingProviderUnavailableError+ | UnsupportedTracingProviderError+ | ErrorResponse403: ErrorResponse+ 404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse+ 409: TracingConfigConflictError | ErrorResponse+ 500: TracingConfigProcessingFailedError | ErrorResponsedefault: ErrorResponse}@@ -7050,6 +7154,32 @@ export type PostAppsByAppIdTraceResponses = {export type PostAppsByAppIdTraceResponse =PostAppsByAppIdTraceResponses[keyof PostAppsByAppIdTraceResponses]+export type GetAppsByAppIdTraceCapabilitiesData = {+ body?: never+ path: {+ app_id: string+ }+ query: {+ tracing_provider: string+ }+ url: '/apps/{app_id}/trace-capabilities'+}++export type GetAppsByAppIdTraceCapabilitiesErrors = {+ 400: UnsupportedTracingProviderError | ErrorResponse+ default: ErrorResponse+}++export type GetAppsByAppIdTraceCapabilitiesError =+ GetAppsByAppIdTraceCapabilitiesErrors[keyof GetAppsByAppIdTraceCapabilitiesErrors]++export type GetAppsByAppIdTraceCapabilitiesResponses = {+ 200: TraceProviderCapabilitiesResponse+}++export type GetAppsByAppIdTraceCapabilitiesResponse =+ GetAppsByAppIdTraceCapabilitiesResponses[keyof GetAppsByAppIdTraceCapabilitiesResponses]+export type DeleteAppsByAppIdTraceConfigData = {body?: neverpath: {@@ -7125,6 +7255,7 @@ export type PatchAppsByAppIdTraceConfigErrors = {| ErrorResponse403: ErrorResponse404: AppNotFoundError | TraceConfigNotFoundError | ErrorResponse+ 409: TracingConfigConflictError | ErrorResponse500: TracingConfigProcessingFailedError | ErrorResponsedefault: ErrorResponse}@@ -7172,6 +7303,71 @@ export type PostAppsByAppIdTraceConfigResponses = {export type PostAppsByAppIdTraceConfigResponse =PostAppsByAppIdTraceConfigResponses[keyof PostAppsByAppIdTraceConfigResponses]+export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdData = {+ body?: never+ path: {+ app_id: string+ source_id: string+ source_kind: 'message' | 'node' | 'workflow'+ }+ query?: {+ node_execution_id?: string+ }+ url: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}'+}++export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdErrors = {+ 404: TracingNodeNotFoundError | TracingSourceNotFoundError | ErrorResponse+ default: ErrorResponse+}++export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdError =+ GetAppsByAppIdTraceRunsBySourceKindBySourceIdErrors[keyof GetAppsByAppIdTraceRunsBySourceKindBySourceIdErrors]++export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponses = {+ 200: TraceRunStatusResponse+}++export type GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse =+ GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponses[keyof GetAppsByAppIdTraceRunsBySourceKindBySourceIdResponses]++export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryData = {+ body: TraceRunRetryPayload+ path: {+ app_id: string+ source_id: string+ source_kind: 'message' | 'node' | 'workflow'+ }+ query?: {+ node_execution_id?: string+ }+ url: '/apps/{app_id}/trace-runs/{source_kind}/{source_id}/retry'+}++export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryErrors = {+ 404:+ | TracingNodeNotFoundError+ | TracingRunNotFoundError+ | TracingSourceNotFoundError+ | ErrorResponse+ 409:+ | TracingAlreadyAcceptedError+ | TracingDuplicateAcknowledgementRequiredError+ | TracingRetryNotAllowedError+ | ErrorResponse+ default: ErrorResponse+}++export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryError =+ PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryErrors[keyof PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryErrors]++export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponses = {+ 200: TraceRunStatusResponse+}++export type PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse =+ PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponses[keyof PostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponses]+export type PostAppsByAppIdTriggerEnableData = {body: ParserEnablepath: {diff --git a/packages/contracts/generated/api/console/apps/zod.gen.ts b/packages/contracts/generated/api/console/apps/zod.gen.tsindex e6cc027d761..cf994692b68 100644--- a/packages/contracts/generated/api/console/apps/zod.gen.ts+++ b/packages/contracts/generated/api/console/apps/zod.gen.ts@@ -740,6 +740,8 @@ export const zProviderNotSupportTextToSpeechError = zErrorResponse.and(* AppTraceResponse*/export const zAppTraceResponse = z.object({+ capture_agent: z.boolean().optional().default(true),+ capture_content: z.boolean().optional().default(false),enabled: z.boolean().optional().default(false),tracing_provider: z.string().nullish(),})@@ -748,10 +750,24 @@ export const zAppTraceResponse = z.object({* AppTracePayload*/export const zAppTracePayload = z.object({+ capture_agent: z.boolean().optional().default(true),+ capture_content: z.boolean().optional().default(false),enabled: z.boolean(),tracing_provider: z.string().nullish(),})+/**+ * InvalidTracingConfigError+ *+ * The tracing configuration is invalid.+ */+export const zInvalidTracingConfigError = zErrorResponse.and(+ z.object({+ code: z.literal('invalid_tracing_config'),+ status: z.literal(400),+ }),+)+/*** TracingProviderUnavailableError*@@ -788,6 +804,18 @@ export const zTraceConfigNotFoundError = zErrorResponse.and(}),)+/**+ * TracingConfigConflictError+ *+ * Tracing configuration changed. Refresh and try again.+ */+export const zTracingConfigConflictError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_config_conflict'),+ status: z.literal(409),+ }),+)+/*** TracingConfigProcessingFailedError*@@ -800,6 +828,21 @@ export const zTracingConfigProcessingFailedError = zErrorResponse.and(}),)+/**+ * TraceProviderCapabilitiesResponse+ */+export const zTraceProviderCapabilitiesResponse = z.object({+ agent_spans: z.boolean(),+ api_collection_enabled: z.boolean(),+ export_ack: z.literal('accepted'),+ native_cost: z.boolean(),+ native_usage: z.boolean(),+ provider: z.string(),+ session: z.boolean(),+ span_url: z.boolean(),+ trace_url: z.boolean(),+})+/*** TraceAppConfigResponse*/@@ -820,22 +863,11 @@ export const zTraceAppConfigResponse = z.object({* TraceConfigPayload*/export const zTraceConfigPayload = z.object({+ activate: z.boolean().optional().default(false),tracing_config: z.record(z.string(), z.string().nullable()),tracing_provider: z.string(),})-/**- * InvalidTracingConfigError- *- * The tracing configuration is invalid.- */-export const zInvalidTracingConfigError = zErrorResponse.and(- z.object({- code: z.literal('invalid_tracing_config'),- status: z.literal(400),- }),-)-/*** TracingConfigVerificationFailedError*@@ -860,6 +892,85 @@ export const zTraceConfigAlreadyExistsError = zErrorResponse.and(}),)+/**+ * TracingNodeNotFoundError+ *+ * The Agent node invocation does not belong to this trace.+ */+export const zTracingNodeNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_node_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TracingSourceNotFoundError+ *+ * The tracing source does not exist in this application.+ */+export const zTracingSourceNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_source_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TraceRunRetryPayload+ */+export const zTraceRunRetryPayload = z.object({+ acknowledge_possible_duplicates: z.boolean().optional().default(false),+})++/**+ * TracingRunNotFoundError+ *+ * No collected trace is available to retry.+ */+export const zTracingRunNotFoundError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_run_not_found'),+ status: z.literal(404),+ }),+)++/**+ * TracingAlreadyAcceptedError+ *+ * An accepted trace cannot be exported again.+ */+export const zTracingAlreadyAcceptedError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_already_accepted'),+ status: z.literal(409),+ }),+)++/**+ * TracingDuplicateAcknowledgementRequiredError+ *+ * Acknowledge possible duplicates before retrying an uncertain export.+ */+export const zTracingDuplicateAcknowledgementRequiredError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_duplicate_acknowledgement_required'),+ status: z.literal(409),+ }),+)++/**+ * TracingRetryNotAllowedError+ *+ * This trace is not in a retryable state.+ */+export const zTracingRetryNotAllowedError = zErrorResponse.and(+ z.object({+ code: z.literal('tracing_retry_not_allowed'),+ status: z.literal(409),+ }),+)+/*** ParserEnable*/@@ -2013,6 +2124,56 @@ export const zTextToSpeechVoiceResponse = z.object({*/export const zTextToSpeechVoiceListResponse = z.array(zTextToSpeechVoiceResponse)+/**+ * TraceProviderSupportResponse+ *+ * Adapter support declarations, not successful-delivery or platform-verification results.+ */+export const zTraceProviderSupportResponse = z.object({+ agent_spans: z.boolean(),+ export_ack: z.literal('accepted'),+ native_cost: z.boolean(),+ native_usage: z.boolean(),+ provider: z.string(),+ session: z.boolean(),+ span_url: z.boolean(),+ trace_url: z.boolean(),+})++export const zTraceSourceKind = z.enum(['message', 'node', 'workflow'])++/**+ * TraceRunStatusResponse+ */+export const zTraceRunStatusResponse = z.object({+ accepted_at: z.iso.datetime().nullish(),+ attempts: z.int().nullish(),+ capabilities: zTraceProviderSupportResponse.nullish(),+ capture_agent: z.boolean().nullish(),+ capture_content: z.boolean().nullish(),+ config_revision: z.string().nullish(),+ error_code: z.string().nullish(),+ project_url: z.string().nullish(),+ provider: z.string().nullish(),+ queryable: z.boolean().nullish(),+ source_id: z.string(),+ source_kind: zTraceSourceKind,+ span_count: z.int().nullish(),+ span_url: z.string().nullish(),+ status: z.enum([+ 'accepted',+ 'collecting',+ 'exporting',+ 'failed',+ 'not_collected',+ 'ready',+ 'retrying',+ 'uncertain',+ ]),+ trace_id: z.string().nullish(),+ trace_url: z.string().nullish(),+})+/*** SimpleAccountResponse*/@@ -5428,7 +5589,7 @@ export const zAgentSoulModelSettings = z.object({frequency_penalty: z.number().nullish(),max_tokens: z.int().nullish(),presence_penalty: z.number().nullish(),- response_format: zAgentModelResponseFormatConfig.nullish(),+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),stop: z.array(z.string()).nullish(),temperature: z.number().nullish(),top_p: z.number().nullish(),@@ -7318,6 +7479,19 @@ export const zPostAppsByAppIdTracePath = z.object({*/export const zPostAppsByAppIdTraceResponse = zSimpleResultResponse+export const zGetAppsByAppIdTraceCapabilitiesPath = z.object({+ app_id: z.uuid(),+})++export const zGetAppsByAppIdTraceCapabilitiesQuery = z.object({+ tracing_provider: z.string(),+})++/**+ * Provider support declarations+ */+export const zGetAppsByAppIdTraceCapabilitiesResponse = zTraceProviderCapabilitiesResponse+export const zDeleteAppsByAppIdTraceConfigPath = z.object({app_id: z.uuid(),})@@ -7366,6 +7540,38 @@ export const zPostAppsByAppIdTraceConfigPath = z.object({*/export const zPostAppsByAppIdTraceConfigResponse = zTraceAppConfigResponse+export const zGetAppsByAppIdTraceRunsBySourceKindBySourceIdPath = z.object({+ app_id: z.uuid(),+ source_id: z.uuid(),+ source_kind: z.enum(['message', 'node', 'workflow']),+})++export const zGetAppsByAppIdTraceRunsBySourceKindBySourceIdQuery = z.object({+ node_execution_id: z.uuid().optional(),+})++/**+ * Frozen trace destination and export state+ */+export const zGetAppsByAppIdTraceRunsBySourceKindBySourceIdResponse = zTraceRunStatusResponse++export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryBody = zTraceRunRetryPayload++export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryPath = z.object({+ app_id: z.uuid(),+ source_id: z.uuid(),+ source_kind: z.enum(['message', 'node', 'workflow']),+})++export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryQuery = z.object({+ node_execution_id: z.uuid().optional(),+})++/**+ * Retry state for the same frozen destination+ */+export const zPostAppsByAppIdTraceRunsBySourceKindBySourceIdRetryResponse = zTraceRunStatusResponse+export const zPostAppsByAppIdTriggerEnableBody = zParserEnableexport const zPostAppsByAppIdTriggerEnablePath = z.object({diff --git a/packages/contracts/generated/api/console/snippets/types.gen.ts b/packages/contracts/generated/api/console/snippets/types.gen.tsindex 162b61e1223..8c41ab7a0bd 100644--- a/packages/contracts/generated/api/console/snippets/types.gen.ts+++ b/packages/contracts/generated/api/console/snippets/types.gen.ts@@ -960,7 +960,7 @@ export type AgentSoulModelSettings = {frequency_penalty?: number | nullmax_tokens?: number | nullpresence_penalty?: number | null- response_format?: AgentModelResponseFormatConfig | null+ response_format?: string | AgentModelResponseFormatConfig | nullstop?: Array<string> | nulltemperature?: number | nulltop_p?: number | nulldiff --git a/packages/contracts/generated/api/console/snippets/zod.gen.ts b/packages/contracts/generated/api/console/snippets/zod.gen.tsindex 91fc59f61c6..7e48b3e636f 100644--- a/packages/contracts/generated/api/console/snippets/zod.gen.ts+++ b/packages/contracts/generated/api/console/snippets/zod.gen.ts@@ -1324,7 +1324,7 @@ export const zAgentSoulModelSettings = z.object({frequency_penalty: z.number().nullish(),max_tokens: z.int().nullish(),presence_penalty: z.number().nullish(),- response_format: zAgentModelResponseFormatConfig.nullish(),+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),stop: z.array(z.string()).nullish(),temperature: z.number().nullish(),top_p: z.number().nullish(),diff --git a/packages/contracts/generated/api/console/system-features/types.gen.ts b/packages/contracts/generated/api/console/system-features/types.gen.tsindex e36a34b7b90..6fd2007682e 100644--- a/packages/contracts/generated/api/console/system-features/types.gen.ts+++ b/packages/contracts/generated/api/console/system-features/types.gen.ts@@ -5,6 +5,7 @@ export type ClientOptions = {}export type SystemFeatureModel = {+ agent_tracing_api_collection_enabled: booleanbranding: BrandingModeldeployment_edition: DeploymentEditionenable_app_deploy: booleandiff --git a/packages/contracts/generated/api/console/system-features/zod.gen.ts b/packages/contracts/generated/api/console/system-features/zod.gen.tsindex 6417c81f20f..a263864a563 100644--- a/packages/contracts/generated/api/console/system-features/zod.gen.ts+++ b/packages/contracts/generated/api/console/system-features/zod.gen.ts@@ -128,6 +128,7 @@ export const zWebAppAuthModel = z.object({* Non-sensitive bootstrap snapshot exposed before Console or Web authentication.*/export const zSystemFeatureModel = z.object({+ agent_tracing_api_collection_enabled: z.boolean(),branding: zBrandingModel.default({application_title: '',enabled: false,diff --git a/packages/contracts/generated/api/console/trial-apps/types.gen.ts b/packages/contracts/generated/api/console/trial-apps/types.gen.tsindex 6b001b10387..8cf7b51fca2 100644--- a/packages/contracts/generated/api/console/trial-apps/types.gen.ts+++ b/packages/contracts/generated/api/console/trial-apps/types.gen.ts@@ -1233,7 +1233,7 @@ export type AgentSoulModelSettings = {frequency_penalty?: number | nullmax_tokens?: number | nullpresence_penalty?: number | null- response_format?: AgentModelResponseFormatConfig | null+ response_format?: string | AgentModelResponseFormatConfig | nullstop?: Array<string> | nulltemperature?: number | nulltop_p?: number | nulldiff --git a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts b/packages/contracts/generated/api/console/trial-apps/zod.gen.tsindex 133b712c9de..8bdadfcfb6a 100644--- a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts+++ b/packages/contracts/generated/api/console/trial-apps/zod.gen.ts@@ -1725,7 +1725,7 @@ export const zAgentSoulModelSettings = z.object({frequency_penalty: z.number().nullish(),max_tokens: z.int().nullish(),presence_penalty: z.number().nullish(),- response_format: zAgentModelResponseFormatConfig.nullish(),+ response_format: z.union([z.string(), zAgentModelResponseFormatConfig]).nullish(),stop: z.array(z.string()).nullish(),temperature: z.number().nullish(),top_p: z.number().nullish(),diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.tsindex b7ed9dfc696..444a1c6a7db 100644--- a/packages/contracts/generated/api/web/types.gen.ts+++ b/packages/contracts/generated/api/web/types.gen.ts@@ -1041,6 +1041,7 @@ export type SuggestedQuestionsResponse = {}export type SystemFeatureModel = {+ agent_tracing_api_collection_enabled: booleanbranding: BrandingModeldeployment_edition: DeploymentEditionenable_app_deploy: booleandiff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.tsindex 30c3c4729c8..e4245e95da9 100644--- a/packages/contracts/generated/api/web/zod.gen.ts+++ b/packages/contracts/generated/api/web/zod.gen.ts@@ -1666,6 +1666,7 @@ export const zWebAppAuthModel = z.object({* Non-sensitive bootstrap snapshot exposed before Console or Web authentication.*/export const zSystemFeatureModel = z.object({+ agent_tracing_api_collection_enabled: z.boolean(),branding: zBrandingModel.default({application_title: '',enabled: false,diff --git a/packages/contracts/network-access-current-ip.test.ts b/packages/contracts/network-access-current-ip.test.tsindex b68d06a4560..8673aeee186 100644--- a/packages/contracts/network-access-current-ip.test.ts+++ b/packages/contracts/network-access-current-ip.test.ts@@ -1,10 +1,10 @@import { describe, expect, it } from 'vite-plus/test'-import { currentIp, workspaces } from './generated/api/console/workspaces/orpc.gen'+import { currentIp, workspaces } from './generated/api/console/workspaces/orpc.gen.ts'import {zGetWorkspacesCurrentNetworkAccessGroupsByGroupIdCheckCurrentIpPath,zGetWorkspacesCurrentNetworkAccessGroupsByGroupIdCheckCurrentIpResponse,zGetWorkspacesCurrentNetworkAccessGroupsCurrentIpResponse,-} from './generated/api/console/workspaces/zod.gen'+} from './generated/api/console/workspaces/zod.gen.ts'describe('generated network access current-IP contract', () => {it('exposes the policy-independent operation alongside the group preflight', () => {diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsxindex f584d8fccd8..86066f3c79d 100644--- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx+++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/__tests__/panel.spec.tsx@@ -196,7 +196,7 @@ describe('Tracing overview panel', () => {act(() =>client.setQueryData(consoleQuery.apps.byAppId.trace.get.queryKey({ input: { params: { app_id: 'app-1' } } }),- { enabled: false, tracing_provider: null },+ { enabled: false, tracing_provider: null, capture_content: false, capture_agent: true },),)const user = userEvent.setup()@@ -232,7 +232,12 @@ describe('Tracing overview panel', () => {act(() =>client.setQueryData(consoleQuery.apps.byAppId.trace.get.queryKey({ input: { params: { app_id: 'app-1' } } }),- { enabled: true, tracing_provider: 'langsmith' },+ {+ enabled: true,+ tracing_provider: 'langsmith',+ capture_content: false,+ capture_agent: true,+ },),)const user = await openConfiguredPanel()diff --git a/web/test/console/system-features.ts b/web/test/console/system-features.tsindex dac07c20d21..0ebe648312a 100644--- a/web/test/console/system-features.ts+++ b/web/test/console/system-features.ts@@ -15,6 +15,7 @@ export type DeepPartial<T> =: Tconst baseSystemFeatures = {+ agent_tracing_api_collection_enabled: false,deployment_edition: 'COMMUNITY',enable_app_deploy: false,sso_enforced_for_signin: false,