Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

14.1 KB
Raw Blame
"""Content policy and model-call accounting across all canonical tree layers."""

import json
from datetime import UTC, datetime

import pytest
from dify_trace_sdk.ids import span_id_for
from dify_trace_sdk.model import Cost, Span, SpanKind, SpanStatus, TokenUsage, Trace
from pydantic import ValidationError

from domain.tracing.normalization import (
    legacy_outer_trace,
    normalize_agent_span,
    order_spans,
    remap_outer_trace,
    safe_attributes,
)


def agent_data(
    attributes: dict[str, object], *, span_id: str = "2" * 16, parent_id: str = "1" * 16
) -> dict[str, object]:
    return {
        "span_id": span_id,
        "parent_span_id": parent_id,
        "name": "operation",
        "start_time_ns": 1_000_000_000,
        "end_time_ns": 2_000_000_000,
        "status": "error",
        "status_message": "sensitive error output",
        "attributes": attributes,
    }


def test_content_off_is_safe_for_all_metadata_namespaces() -> None:
    normalized = normalize_agent_span(
        agent_data(
            {
                "gen_ai.operation.name": "chat",
                "gen_ai.request.model": "model-one",
                "gen_ai.usage.input_tokens": 3,
                "gen_ai.usage.output_tokens": 2,
                "gen_ai.input.messages": [{"content": "private input"}],
                "gen_ai.output.messages": [{"content": "private output"}],
                "exception.message": "private exception",
                "dify.app_id": "app-one",
                "dify.custom_payload": {"prompt": "private metadata"},
                "gen_ai.request.messages": "private request metadata",
                "gen_ai.response.output": "private response metadata",
                "gen_ai.usage.custom_prompt": "private usage metadata",
            }
        ),
        capture_content=False,
    )
    assert normalized.inputs is None
    assert normalized.outputs is None
    assert "private" not in normalized.model_dump_json()
    assert "sensitive error" not in normalized.model_dump_json()
    assert normalized.error == "Agent operation failed"
    assert normalized.attributes["dify.app_id"] == "app-one"
    assert normalized.model_name == "model-one"
    assert normalized.usage == TokenUsage(prompt_tokens=3, completion_tokens=2, total_tokens=5)


def test_json_encoded_tool_credentials_are_redacted_from_inputs_and_metadata() -> None:
    normalized = normalize_agent_span(
        agent_data(
            {
                "gen_ai.operation.name": "execute_tool",
                "gen_ai.tool.name": "search",
                "gen_ai.tool.call.arguments": json.dumps({"api_key": "tool-secret-123", "query": "public query"}),
                "gen_ai.tool.call.result": json.dumps(
                    {"nested": {"authorization": "bearer-secret-123"}, "result": "okay"}
                ),
            }
        ),
        capture_content=True,
    )
    serialized = normalized.model_dump_json()
    assert "tool-secret-123" not in serialized
    assert "bearer-secret-123" not in serialized
    assert "public query" in serialized
    assert normalized.kind is SpanKind.TOOL
    assert normalized.inputs == {"api_key": "[redacted]", "query": "public query"}
    assert normalized.outputs == {"nested": {"authorization": "[redacted]"}, "result": "okay"}


def test_content_off_drops_error_event_payload_and_keeps_numeric_usage() -> None:
    attrs = safe_attributes(
        {
            "input": "user text",
            "authorization": "secret",
            "exception.stacktrace": "sensitive",
            "gen_ai.usage.input_tokens": 10,
            "gen_ai.tool.name": "lookup",
        },
        capture_content=False,
    )
    assert attrs == {"gen_ai.usage.input_tokens": 10, "gen_ai.tool.name": "lookup"}


def test_real_runtime_accounting_attributes_survive_content_off() -> None:
    normalized = normalize_agent_span(
        agent_data(
            {
                "gen_ai.operation.name": "chat",
                "gen_ai.request.model": "gpt-4o-mini",
                "gen_ai.usage.input_tokens": 8,
                "gen_ai.usage.output_tokens": 4,
                # The runtime reports its decimal price as a string.
                "dify.llm.total_price": "0.03",
                "dify.llm.currency": "USD",
                "gen_ai.response.time_to_first_token": 0.2,
                "dify.llm.time_to_first_token_unit": "s",
                "dify.llm.time_to_first_token_source": "gateway_usage",
                "gen_ai.input.messages": "private input",
            }
        ),
        capture_content=False,
    )
    assert normalized.cost == Cost(amount=0.03, currency="USD")
    assert normalized.time_to_first_token == 0.2
    assert normalized.usage == TokenUsage(prompt_tokens=8, completion_tokens=4, total_tokens=12)
    assert normalized.attributes["dify.llm.time_to_first_token_source"] == "gateway_usage"
    assert normalized.inputs is None


@pytest.mark.parametrize("price", ["not-a-price", "-0.01", "nan", "inf", True])
def test_unusable_runtime_price_is_not_reported_as_cost(price: object) -> None:
    normalized = normalize_agent_span(
        agent_data({"gen_ai.operation.name": "chat", "dify.llm.total_price": price}), capture_content=True
    )
    assert normalized.kind is SpanKind.LLM
    assert normalized.cost is None


def test_agent_aggregate_usage_is_context_not_another_generation() -> None:
    normalized = normalize_agent_span(
        agent_data(
            {
                "gen_ai.operation.name": "invoke_agent",
                "gen_ai.request.model": "model",
                "gen_ai.usage.input_tokens": 30,
                "gen_ai.usage.output_tokens": 20,
                "dify.llm.total_price": "0.5",
            }
        ),
        capture_content=False,
    )
    assert normalized.kind is SpanKind.AGENT
    assert normalized.usage is None
    assert normalized.cost is None
    assert normalized.model_name is None
    assert normalized.attributes["aggregate_prompt_tokens"] == 30
    assert normalized.attributes["aggregate_completion_tokens"] == 20


def span(name: str, parent: str | None, kind: SpanKind) -> Span:
    return Span(
        span_id=span_id_for(name),
        parent_span_id=span_id_for(parent) if parent else None,
        name=name,
        kind=kind,
        start_time=datetime.fromtimestamp(0, UTC),
        end_time=datetime.fromtimestamp(3, UTC),
        status=SpanStatus.OK,
        inputs={"content": "private"},
        outputs="private",
        usage=TokenUsage(prompt_tokens=6, completion_tokens=4, total_tokens=10) if kind is SpanKind.LLM else None,
        cost=Cost(amount=0.03, currency="USD") if kind is SpanKind.LLM else None,
        attributes={"prompt": "private", "node_type": kind.value},
    )


def trace_of(*spans: Span) -> Trace:
    return Trace(
        trace_id=span_id_for("old") * 2,
        external_trace_id=None,
        session_id="session",
        user_id="user",
        parent=None,
        spans=spans,
    )


def test_two_native_model_calls_replace_synthetic_aggregate_generation() -> None:
    outer = trace_of(span("message", None, SpanKind.CHAIN), span("message:llm", "message", SpanKind.LLM))
    remapped = remap_outer_trace(
        outer, trace_id="1" * 32, capture_content=False, synthetic_span_ids=frozenset({span_id_for("message:llm")})
    )
    assert [item.span_id for item in remapped.spans] == [span_id_for("message")]
    assert remapped.trace_id == "1" * 32
    assert (remapped.session_id, remapped.user_id) == ("session", "user")
    calls = [
        normalize_agent_span(
            agent_data(
                {
                    "gen_ai.operation.name": "chat",
                    "gen_ai.request.model": "model",
                    "gen_ai.usage.input_tokens": 3,
                    "gen_ai.usage.output_tokens": 2,
                },
                span_id=identifier,
                parent_id=remapped.root.span_id,
            ),
            capture_content=False,
        )
        for identifier in ("2" * 16, "3" * 16)
    ]
    combined = order_spans(remapped.root.span_id, [calls[1], *remapped.spans, calls[0]])
    assert combined[0].span_id == remapped.root.span_id
    generations = [item for item in combined if item.kind is SpanKind.LLM]
    assert len(generations) == 2
    assert sum(item.usage.total_tokens for item in generations if item.usage is not None) == 10
    assert combined[0].usage is None
    assert "private" not in json.dumps([item.model_dump(mode="json") for item in combined])


def test_non_agent_workflow_retains_its_real_llm_node() -> None:
    trace = trace_of(span("workflow", None, SpanKind.CHAIN), span("llm-node", "workflow", SpanKind.LLM))
    remapped = remap_outer_trace(trace, trace_id="1" * 32, capture_content=True, synthetic_span_ids=frozenset())
    assert len(remapped.spans) == 2
    llm = remapped.spans[1]
    assert llm.kind is SpanKind.LLM
    assert llm.usage == TokenUsage(prompt_tokens=6, completion_tokens=4, total_tokens=10)
    assert llm.cost == Cost(amount=0.03, currency="USD")
    assert llm.inputs == {"content": "private"}
    assert llm.attributes["prompt"] == "private"


def test_content_off_outer_trace_keeps_only_identifiers_and_masks_errors() -> None:
    failed = span("workflow", None, SpanKind.CHAIN).model_copy(
        update={"status": SpanStatus.ERROR, "error": "private failure"}
    )
    remapped = remap_outer_trace(
        trace_of(failed), trace_id="1" * 32, capture_content=False, synthetic_span_ids=frozenset()
    )
    root = remapped.root
    assert root.error == "Application execution failed"
    assert root.status is SpanStatus.ERROR
    assert (root.inputs, root.outputs) == (None, None)
    assert root.attributes == {"node_type": "chain"}


def test_order_spans_rejects_orphans_cycles_and_conflicting_ids() -> None:
    root = span("root", None, SpanKind.CHAIN)
    with pytest.raises(ValueError, match="parent is missing"):
        order_spans(root.span_id, [root, span("orphan", "missing", SpanKind.TOOL)])
    with pytest.raises(ValueError, match="parent cycle"):
        order_spans(root.span_id, [root, span("one", "two", SpanKind.TOOL), span("two", "one", SpanKind.TOOL)])
    with pytest.raises(ValueError, match="Conflicting"):
        order_spans(root.span_id, [root, root.model_copy(update={"name": "other"})])
    with pytest.raises(ValueError, match="root is missing"):
        order_spans(root.span_id, [span("one", "root", SpanKind.TOOL)])
    with pytest.raises(ValueError, match="Unexpected trace root"):
        order_spans(root.span_id, [root, span("second-root", None, SpanKind.CHAIN)])


def test_managed_external_parent_is_preserved_as_explicit_link_when_content_off() -> None:
    root = span("workflow", None, SpanKind.CHAIN).model_copy(
        update={
            "attributes": {
                "parent_workflow_run_id": "outer-workflow",
                "parent_node_execution_id": "outer-tool",
                "prompt": "private",
            }
        }
    )
    outer = Trace(
        trace_id="2" * 32,
        external_trace_id="caller-trace",
        session_id="session",
        user_id=None,
        parent=None,
        spans=(root,),
    )
    remapped = remap_outer_trace(outer, trace_id="1" * 32, capture_content=False, synthetic_span_ids=frozenset())
    assert remapped.parent is None
    assert remapped.external_trace_id is None
    assert remapped.root.attributes == {
        "parent_workflow_run_id": "outer-workflow",
        "parent_node_execution_id": "outer-tool",
        "external_trace_id": "caller-trace",
    }


TRACE_ID = "0af7651916cd43dd8448eb211c80319c"


def _legacy(**span_overrides: object) -> dict[str, object]:
    root, llm = span_id_for("message-1"), span_id_for("message-1:llm")
    return {
        "trace_id": TRACE_ID,
        "session_id": "conversation-1",
        "root_span_id": root,
        "spans": [
            {
                "id": root,
                "parent_id": None,
                "name": "message",
                "kind": "chain",
                "start_time": "2026-01-01T08:00:00",
                "end_time": "2026-01-01T08:00:02",
                "inputs": {"query": "hi"},
                "outputs": "hello",
                "status": "ok",
                "error": None,
                "metadata": {"aggregate_total_tokens": 20},
                "synthetic": False,
                "can_parent_workflow": False,
                "publishes_parent_context": False,
                "links": [],
            },
            {
                "id": llm,
                "parent_id": root,
                "name": "llm",
                "kind": "llm",
                "start_time": "2026-01-01T08:00:00",
                "end_time": None,
                "inputs": None,
                "outputs": None,
                "status": "ok",
                "error": None,
                "metadata": {
                    "model_name": "gpt-4o",
                    "model_provider": "openai",
                    "prompt_tokens": 12,
                    "completion_tokens": 8,
                    "total_tokens": 20,
                    "total_price": "0.0042",
                    "currency": "USD",
                },
                "synthetic": False,
                "can_parent_workflow": False,
                "publishes_parent_context": False,
                "links": [],
                **span_overrides,
            },
        ],
        "external_parent": None,
        "required_parent_context_id": None,
    }


def test_outer_traces_stored_by_the_previous_release_still_export() -> None:
    trace = legacy_outer_trace(_legacy())

    assert (trace.trace_id, trace.session_id, trace.parent) == (TRACE_ID, "conversation-1", None)
    root, llm = trace.spans
    assert root.attributes == {"aggregate_total_tokens": 20}
    assert llm.kind is SpanKind.LLM
    assert (llm.model_name, llm.model_provider) == ("gpt-4o", "openai")
    assert llm.usage is not None
    assert (llm.usage.prompt_tokens, llm.usage.total_tokens) == (12, 20)
    assert llm.cost is not None
    assert (llm.cost.amount, llm.cost.currency) == (0.0042, "USD")
    assert llm.end_time == datetime(2026, 1, 1, 8, 0, tzinfo=UTC)


def test_malformed_legacy_traces_still_fail_validation() -> None:
    with pytest.raises(ValidationError, match="must follow its parent"):
        legacy_outer_trace(_legacy(parent_id="0000000000000abc"))