Skip to content

Automate browser based workflows with AI

22.5 KB
Raw Blame
"""Tests for the copilot tracing_setup module's initialization sequence.

Covers the disabled path, the logfire-available path, and the logfire-missing
path. Stubs logfire's and agents' public surface so no real logfire internals
are mutated. The private-API patches in `_patch_agent_span_attributes` are
covered by an integration test once the copilot runtime wiring lands.
"""

from __future__ import annotations

import contextlib
import json
import sys
import threading
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock

import pytest

from skyvern.config import settings
from skyvern.forge.sdk.copilot import tracing_setup

_SYNTHETIC_PROXY_CREDENTIAL = "synthetic-proxy-secret"
_SYNTHETIC_PROXY_HOST = "internalproxy"
_SYNTHETIC_PROXY_URL = f"http://user:{_SYNTHETIC_PROXY_CREDENTIAL}@{_SYNTHETIC_PROXY_HOST}:8080"


def _nested_mapping(value: object, depth: int = 22) -> object:
    for _ in range(depth):
        value = {"child": value}
    return value


@pytest.fixture(autouse=True)
def _reset_module_state(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(tracing_setup, "_TRACING_INITIALIZED", False)
    monkeypatch.setattr(tracing_setup, "_SPAN_RENAME_WARNED", threading.Event())
    monkeypatch.delenv("COPILOT_TRACING_ENABLED", raising=False)


@pytest.fixture
def agents_stub(monkeypatch: pytest.MonkeyPatch) -> tuple[list[bool], list[Any]]:
    disabled_calls: list[bool] = []
    processors_calls: list[Any] = []
    monkeypatch.setitem(
        sys.modules,
        "agents",
        SimpleNamespace(
            set_tracing_disabled=lambda flag: disabled_calls.append(flag),
            set_trace_processors=lambda procs: processors_calls.append(procs),
        ),
    )
    return disabled_calls, processors_calls


@pytest.fixture
def stub_logger(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, Any]]]:
    logged: list[tuple[str, dict[str, Any]]] = []
    monkeypatch.setattr(
        tracing_setup,
        "LOG",
        SimpleNamespace(
            warning=lambda msg, **kwargs: logged.append((msg, kwargs)),
            info=lambda msg, **kwargs: None,
        ),
    )
    return logged


def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any) -> list[Any]:
    usages: list[Any] = []

    class FakeUsage:
        def __init__(
            self,
            *,
            input_tokens: int,
            output_tokens: int,
            cache_read_tokens: int | None = None,
            cache_write_tokens: int | None = None,
        ) -> None:
            self.input_tokens = input_tokens
            self.output_tokens = output_tokens
            self.cache_read_tokens = cache_read_tokens
            self.cache_write_tokens = cache_write_tokens
            usages.append(self)

    monkeypatch.setitem(
        sys.modules,
        "genai_prices",
        SimpleNamespace(Usage=FakeUsage, calc_price=calc_price),
    )
    return usages


def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.MonkeyPatch) -> None:
    from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry

    calls: list[dict[str, Any]] = []

    def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
        calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})
        return SimpleNamespace(total_price=0.123)

    _install_genai_prices_stub(monkeypatch, fake_calc_price)

    attrs: dict[str, Any] = {}
    usage = CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20)
    tracing_setup._attach_cost_attr(attrs, usage, "gpt-5.5")

    assert attrs["operation.cost"] == 0.123
    assert len(calls) == 1
    assert calls[0]["model_ref"] == "gpt-5.5"
    assert calls[0]["provider_id"] is None


def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry

    calls: list[dict[str, Any]] = []

    def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
        calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})
        if provider_id is None:
            raise LookupError(f"unknown {model_ref}")
        return SimpleNamespace(total_price=0.456)

    usages = _install_genai_prices_stub(monkeypatch, fake_calc_price)

    attrs: dict[str, Any] = {}
    usage = CopilotModelCallTelemetry(
        model_call_index=1,
        input_tokens=384702,
        output_tokens=4666,
        cache_read_tokens=123,
        cache_write_tokens=456,
    )
    tracing_setup._attach_cost_attr(attrs, usage, "anthropic/claude-opus-4-7")

    assert attrs["operation.cost"] == 0.456
    assert calls == [
        {"usage": usages[0], "model_ref": "anthropic/claude-opus-4-7", "provider_id": None},
        {"usage": usages[0], "model_ref": "claude-opus-4-7", "provider_id": "anthropic"},
    ]
    assert usages[0].cache_read_tokens == 123
    assert usages[0].cache_write_tokens == 456


def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.MonkeyPatch) -> None:
    from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry

    calls: list[dict[str, Any]] = []

    def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
        calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})
        raise LookupError(f"unknown {provider_id}/{model_ref}")

    _install_genai_prices_stub(monkeypatch, fake_calc_price)

    attrs: dict[str, Any] = {}
    usage = CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20)
    tracing_setup._attach_cost_attr(attrs, usage, "unknown/model")

    assert "operation.cost" not in attrs
    assert len(calls) == 2
    assert calls[0]["model_ref"] == "unknown/model"
    assert calls[0]["provider_id"] is None
    assert calls[1]["model_ref"] == "model"
    assert calls[1]["provider_id"] == "unknown"


def test_attach_cost_attr_uses_runtime_pricing_without_genai_prices(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry

    monkeypatch.setitem(sys.modules, "genai_prices", None)
    monkeypatch.setattr(tracing_setup, "_model_call_cost", lambda telemetry, model: 0.789)
    attrs: dict[str, Any] = {}

    tracing_setup._attach_cost_attr(
        attrs,
        CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20),
        "gpt-5.6-sol",
    )

    assert attrs["operation.cost"] == 0.789


def test_attach_cost_attr_omits_cost_without_raw_usage(monkeypatch: pytest.MonkeyPatch) -> None:
    from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry

    def fail_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
        pytest.fail("pricing should not run without raw token usage")

    _install_genai_prices_stub(monkeypatch, fail_calc_price)
    attrs: dict[str, Any] = {}

    tracing_setup._attach_cost_attr(attrs, CopilotModelCallTelemetry(model_call_index=1), "gpt-5.6")

    assert "operation.cost" not in attrs


def test_disabled_path(agents_stub: tuple[list[bool], list[Any]]) -> None:
    disabled_calls, processors_calls = agents_stub

    span = tracing_setup.copilot_span("name")

    assert isinstance(span, contextlib.nullcontext)
    assert tracing_setup._TRACING_INITIALIZED is False

    tracing_setup.ensure_tracing_initialized()
    assert disabled_calls == [True]
    assert processors_calls == []
    assert tracing_setup._TRACING_INITIALIZED is True


def test_enabled_with_logfire(
    monkeypatch: pytest.MonkeyPatch,
    agents_stub: tuple[list[bool], list[Any]],
) -> None:
    monkeypatch.setenv("COPILOT_TRACING_ENABLED", "1")
    _, processors_calls = agents_stub

    configure_calls: list[dict[str, Any]] = []
    instrument_calls: list[None] = []
    patch_calls: list[None] = []

    monkeypatch.setitem(
        sys.modules,
        "logfire",
        SimpleNamespace(
            configure=lambda **kw: configure_calls.append(kw),
            instrument_openai_agents=lambda: instrument_calls.append(None),
        ),
    )
    monkeypatch.setattr(tracing_setup, "_patch_agent_span_attributes", lambda: patch_calls.append(None))

    tracing_setup.ensure_tracing_initialized()
    tracing_setup.ensure_tracing_initialized()

    assert configure_calls == [
        {
            "send_to_logfire": "if-token-present",
            "service_name": settings.OTEL_SERVICE_NAME,
            "environment": settings.ENV,
        }
    ]
    assert instrument_calls == [None]
    assert patch_calls == [None]
    assert processors_calls == [[]]


def test_enabled_without_logfire(
    monkeypatch: pytest.MonkeyPatch,
    agents_stub: tuple[list[bool], list[Any]],
    stub_logger: list[tuple[str, dict[str, Any]]],
) -> None:
    monkeypatch.setenv("COPILOT_TRACING_ENABLED", "1")
    _, processors_calls = agents_stub

    # Setting sys.modules[name] = None makes subsequent `import name` raise ImportError.
    monkeypatch.setitem(sys.modules, "logfire", None)
    patch_calls: list[None] = []
    monkeypatch.setattr(tracing_setup, "_patch_agent_span_attributes", lambda: patch_calls.append(None))

    tracing_setup.ensure_tracing_initialized()

    assert any("logfire is not installed" in msg for msg, _ in stub_logger)
    assert processors_calls == [[]]
    assert patch_calls == []


def test_warn_span_rename_once(stub_logger: list[tuple[str, dict[str, Any]]]) -> None:
    tracing_setup._warn_span_rename_once(RuntimeError("first"))
    tracing_setup._warn_span_rename_once(RuntimeError("second"))

    assert len(stub_logger) == 1
    assert "first" in stub_logger[0][1]["error"]


class TestTracingSetup:
    @pytest.mark.parametrize("value", ["1", "true", "TRUE", " yes "])
    def test_is_tracing_enabled_truthy_values(
        self,
        monkeypatch: pytest.MonkeyPatch,
        value: str,
    ) -> None:
        monkeypatch.setenv("COPILOT_TRACING_ENABLED", value)

        from skyvern.forge.sdk.copilot.tracing_setup import is_tracing_enabled

        assert is_tracing_enabled() is True

    def test_is_tracing_enabled_defaults_off(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.delenv("COPILOT_TRACING_ENABLED", raising=False)

        from skyvern.forge.sdk.copilot.tracing_setup import is_tracing_enabled

        assert is_tracing_enabled() is False

    def test_copilot_span_returns_nullcontext_when_disabled(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        import skyvern.forge.sdk.copilot.tracing_setup as tracing_setup

        monkeypatch.delenv("COPILOT_TRACING_ENABLED", raising=False)
        monkeypatch.setattr(
            tracing_setup,
            "ensure_tracing_initialized",
            lambda: pytest.fail("ensure_tracing_initialized should not be called"),
        )

        span = tracing_setup.copilot_span("test_span", {"value": 1})

        with span:
            pass

    def test_copilot_span_uses_custom_span_when_enabled(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        import skyvern.forge.sdk.copilot.tracing_setup as tracing_setup

        monkeypatch.setenv("COPILOT_TRACING_ENABLED", "1")
        monkeypatch.setattr(tracing_setup, "ensure_tracing_initialized", lambda: None)

        captured: dict[str, Any] = {}

        class FakeSpan:
            def __enter__(self) -> FakeSpan:
                return self

            def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
                return False

        fake_span = FakeSpan()

        def fake_custom_span(name: str, data: dict[str, Any] | None = None) -> FakeSpan:
            captured["name"] = name
            captured["data"] = data
            return fake_span

        monkeypatch.setattr("agents.tracing.custom_span", fake_custom_span)

        span = tracing_setup.copilot_span("run_blocks", {"block_count": 2})

        assert span is fake_span
        assert captured == {"name": "run_blocks", "data": {"block_count": 2}}

    def test_copilot_span_redacts_proxy_data_when_enabled(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        monkeypatch.setenv("COPILOT_TRACING_ENABLED", "1")
        monkeypatch.setattr(tracing_setup, "ensure_tracing_initialized", lambda: None)
        captured: dict[str, Any] = {}
        fake_span = object()

        def fake_custom_span(name: str, data: dict[str, Any] | None = None) -> object:
            captured["name"] = name
            captured["data"] = data
            return fake_span

        monkeypatch.setattr("agents.tracing.custom_span", fake_custom_span)

        span = tracing_setup.copilot_span(
            "workflow_proxy_location_normalized",
            {
                "input_proxy_location": {
                    "url": "http://user:[email protected]:8080",
                },
                "effective_proxy_location": "RESIDENTIAL",
                "input_proxy_location_present": True,
            },
        )

        assert span is fake_span
        serialized = json.dumps(captured["data"])
        assert "synthetic-secret" not in serialized
        assert "token.proxy.example" not in serialized
        assert captured["data"]["input_proxy_location"].startswith("custom_url:")
        assert captured["data"]["effective_proxy_location"] == "RESIDENTIAL"
        assert captured["data"]["input_proxy_location_present"] is True

    def test_copilot_span_fails_closed_beyond_the_depth_cap(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        monkeypatch.setenv("COPILOT_TRACING_ENABLED", "1")
        monkeypatch.setattr(tracing_setup, "ensure_tracing_initialized", lambda: None)
        captured: dict[str, Any] = {}

        def fake_custom_span(name: str, data: dict[str, Any] | None = None) -> object:
            captured["name"] = name
            captured["data"] = data
            return object()

        monkeypatch.setattr("agents.tracing.custom_span", fake_custom_span)

        tracing_setup.copilot_span(
            "deep_proxy_data",
            {"payload": _nested_mapping({"proxy_url": _SYNTHETIC_PROXY_URL})},
        )

        serialized = json.dumps(captured["data"])
        assert _SYNTHETIC_PROXY_CREDENTIAL not in serialized
        assert _SYNTHETIC_PROXY_HOST not in serialized
        assert "****" in serialized

    def test_copilot_span_drops_data_when_redaction_fails(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        monkeypatch.setenv("COPILOT_TRACING_ENABLED", "1")
        monkeypatch.setattr(tracing_setup, "ensure_tracing_initialized", lambda: None)
        captured: dict[str, Any] = {}
        fake_span = object()

        def fail_redaction(value: object) -> object:
            del value
            raise RuntimeError("synthetic redaction failure")

        def fake_custom_span(name: str, data: dict[str, Any] | None = None) -> object:
            captured["name"] = name
            captured["data"] = data
            return fake_span

        monkeypatch.setattr(tracing_setup, "redact_sensitive_fields", fail_redaction)
        monkeypatch.setattr("agents.tracing.custom_span", fake_custom_span)

        span = tracing_setup.copilot_span("redaction_failure", {"proxy_url": _SYNTHETIC_PROXY_URL})

        assert span is fake_span
        assert captured == {"name": "redaction_failure", "data": None}


class TestPatchAgentSpanAttributes:
    """Tests that invoke `_patch_agent_span_attributes`. The autouse fixture
    snapshots and restores both mutated logfire callables so tests never leak
    patch state to the rest of the suite — `_patch_agent_span_attributes` is
    not idempotent.

    These tests exercise the real logfire integration and are skipped when
    logfire is not installed (it's an optional runtime dependency).
    """

    @pytest.fixture(autouse=True)
    def _restore_oai_mod(self) -> Any:
        pytest.importorskip("logfire")
        import logfire._internal.integrations.openai_agents as _oai_mod

        orig_attrs = _oai_mod.attributes_from_span_data
        orig_create = _oai_mod.LogfireTraceProviderWrapper.create_span
        try:
            yield
        finally:
            _oai_mod.attributes_from_span_data = orig_attrs
            _oai_mod.LogfireTraceProviderWrapper.create_span = orig_create

    def test_patch_adds_otel_agent_attributes(self) -> None:
        from agents import AgentSpanData

        from skyvern.forge.sdk.copilot.tracing_setup import (
            _copilot_model_name,
            _patch_agent_span_attributes,
        )

        _patch_agent_span_attributes()

        import logfire._internal.integrations.openai_agents as _oai_mod

        token = _copilot_model_name.set("gpt-4o")
        try:
            span_data = AgentSpanData(name="workflow-copilot", handoffs=[], tools=[], output_type="str")
            attrs = _oai_mod.attributes_from_span_data(span_data, "Agent run: {name!r}")
        finally:
            _copilot_model_name.reset(token)

        assert attrs["gen_ai.agent.name"] == "workflow-copilot"
        assert attrs["gen_ai.operation.name"] == "invoke_agent"
        assert attrs["gen_ai.provider.name"] == "openai"
        assert attrs["gen_ai.request.model"] == "gpt-4o"
        assert attrs["name"] == "workflow-copilot"

    def test_patch_sets_otel_span_name_on_create_span(self) -> None:
        import logfire._internal.integrations.openai_agents as _oai_mod
        from agents import AgentSpanData

        from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes

        mock_logfire_span = MagicMock()
        mock_logfire_span._span_name = "Agent run: {name!r}"
        mock_result = MagicMock()
        mock_result.span_helper.span = mock_logfire_span

        def fake_create(
            self: Any, span_data: Any, span_id: Any = None, parent: Any = None, disabled: bool = False
        ) -> Any:
            return mock_result

        _oai_mod.LogfireTraceProviderWrapper.create_span = fake_create

        _patch_agent_span_attributes()

        span_data = AgentSpanData(
            name="workflow-copilot",
            handoffs=[],
            tools=[],
            output_type="str",
        )
        _oai_mod.LogfireTraceProviderWrapper.create_span(MagicMock(), span_data)

        assert mock_logfire_span._span_name == "invoke_agent workflow-copilot"

    def test_patch_adds_raw_cache_usage_aliases_and_call_metadata(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        import logfire._internal.integrations.openai_agents as _oai_mod
        from agents import GenerationSpanData
        from litellm.types.utils import Usage

        from skyvern.forge.sdk.copilot.model_telemetry import _model_call_telemetry_scope
        from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes

        monkeypatch.setattr(tracing_setup, "_attach_cost_attr", lambda attrs, telemetry, model: None)
        _patch_agent_span_attributes()

        with _model_call_telemetry_scope(6) as telemetry:
            telemetry.response_model = "gpt-5.6-sol-2026-07-09"
            telemetry.capture(
                Usage(
                    prompt_tokens=100,
                    completion_tokens=7,
                    total_tokens=107,
                    prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},
                )
            )
            attrs = _oai_mod.attributes_from_span_data(
                GenerationSpanData(
                    model="openai/gpt-5.6",
                    usage={
                        "requests": 1,
                        "input_tokens": 100,
                        "output_tokens": 7,
                        "total_tokens": 107,
                        "input_tokens_details": {"cached_tokens": 31},
                    },
                ),
                "Chat completion with {model!r}",
            )

        assert attrs["gen_ai.usage.input_tokens"] == 100
        assert attrs["gen_ai.usage.output_tokens"] == 7
        assert attrs["gen_ai.usage.cache_read_tokens"] == 31
        assert attrs["gen_ai.usage.cached_tokens"] == 31
        assert attrs["gen_ai.usage.cache_write_tokens"] == 47
        assert attrs["gen_ai.usage.cache_read.input_tokens"] == 31
        assert attrs["gen_ai.usage.cache_creation.input_tokens"] == 47
        assert attrs["copilot.model_call_index"] == 6
        assert attrs["copilot.cache.mode"] == "implicit"
        assert attrs["copilot.cache.breakpoint_count"] == 0
        assert attrs["gen_ai.response.model"] == "gpt-5.6-sol-2026-07-09"
        assert attrs["usage"]["total_tokens"] == 107
        assert attrs["usage"]["input_tokens_details"] == {"cached_tokens": 31}

    def test_patch_reports_explicit_cache_envelope_metadata(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        import logfire._internal.integrations.openai_agents as _oai_mod
        from agents import GenerationSpanData

        from skyvern.forge.sdk.copilot.model_telemetry import _model_call_telemetry_scope
        from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes

        monkeypatch.setattr(tracing_setup, "_attach_cost_attr", lambda attrs, telemetry, model: None)
        _patch_agent_span_attributes()

        with _model_call_telemetry_scope(2) as telemetry:
            telemetry.cache_mode = "explicit"
            telemetry.cache_breakpoint_count = 1
            telemetry.cache_stable_prefix_chars = 12000
            attrs = _oai_mod.attributes_from_span_data(
                GenerationSpanData(model="gpt-5.6-sol"),
                "Chat completion with {model!r}",
            )

        assert attrs["copilot.cache.mode"] == "explicit"
        assert attrs["copilot.cache.breakpoint_count"] == 1
        assert attrs["copilot.cache.stable_prefix_chars"] == 12000

    def test_patch_redacts_function_span_input_and_sets_tool_semconv(self) -> None:
        import json

        import logfire._internal.integrations.openai_agents as _oai_mod
        from agents import FunctionSpanData

        from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes

        _patch_agent_span_attributes()

        raw_input = json.dumps({"username": "alice", "password": "hunter2"})
        span_data = FunctionSpanData(name="update_workflow", input=raw_input, output=None)
        attrs = _oai_mod.attributes_from_span_data(span_data, "Function: {name!r}")

        assert attrs["gen_ai.operation.name"] == "execute_tool"
        assert attrs["gen_ai.tool.name"] == "update_workflow"
        # Tool-call secrets must not leak to the trace backend.
        assert "hunter2" not in str(attrs.get("input", ""))