Skip to content

Automate browser based workflows with AI

fix:(SKY-9633) Fix copilot otel cost attribution for non-openai providers (#5859)

Andrew Neilson6784c68dc16f
diff --git a/skyvern/forge/sdk/copilot/tracing_setup.py b/skyvern/forge/sdk/copilot/tracing_setup.py
index a53e4c977..dd6c86f2c 100644
--- a/skyvern/forge/sdk/copilot/tracing_setup.py
+++ b/skyvern/forge/sdk/copilot/tracing_setup.py
@@ -125,6 +125,16 @@ def _usage_field(obj: Any, *keys: str) -> Any:
return None
+def _calculate_price(calc_price: Any, usage: Any, model: str) -> Any:
+ try:
+ return calc_price(usage, model_ref=model)
+ except Exception:
+ provider, separator, model_ref = model.partition("/")
+ if not separator or not provider or not model_ref:
+ raise
+ return calc_price(usage, model_ref=model_ref, provider_id=provider)
+
+
def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> None:
"""Stamp ``operation.cost`` (USD) on a GenerationSpanData span.
@@ -144,19 +154,18 @@ def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> N
_usage_field(usage, "input_tokens_details", "prompt_tokens_details"),
"cached_tokens",
)
+ if cached is None:
+ cached = _usage_field(usage, "cache_read_input_tokens")
cache_read = int(cached) if isinstance(cached, (int, float)) and cached > 0 else None
try:
from genai_prices import Usage, calc_price
- price = calc_price(
- Usage(
- input_tokens=int(input_tokens),
- output_tokens=int(output_tokens),
- cache_read_tokens=cache_read,
- ),
- model_ref=model,
- provider_id="openai",
+ price_usage = Usage(
+ input_tokens=int(input_tokens),
+ output_tokens=int(output_tokens),
+ cache_read_tokens=cache_read,
)
+ price = _calculate_price(calc_price, price_usage, model)
except Exception:
return
attrs["operation.cost"] = float(price.total_price)
diff --git a/tests/unit/test_copilot_tracing_setup.py b/tests/unit/test_copilot_tracing_setup.py
index 4961da3eb..277f425e2 100644
--- a/tests/unit/test_copilot_tracing_setup.py
+++ b/tests/unit/test_copilot_tracing_setup.py
@@ -57,6 +57,99 @@ def stub_logger(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, An
return logged
+def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any) -> list[Any]:
+ usages: list[Any] = []
+
+ class FakeUsage:
+ def __init__(
+ self,
+ *,
+ input_tokens: int,
+ output_tokens: int,
+ cache_read_tokens: int | None = None,
+ ) -> None:
+ self.input_tokens = input_tokens
+ self.output_tokens = output_tokens
+ self.cache_read_tokens = cache_read_tokens
+ usages.append(self)
+
+ monkeypatch.setitem(
+ sys.modules,
+ "genai_prices",
+ SimpleNamespace(Usage=FakeUsage, calc_price=calc_price),
+ )
+ return usages
+
+
+def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[dict[str, Any]] = []
+
+ def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
+ calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})
+ return SimpleNamespace(total_price=0.123)
+
+ _install_genai_prices_stub(monkeypatch, fake_calc_price)
+
+ attrs: dict[str, Any] = {}
+ usage = SimpleNamespace(input_tokens=10, output_tokens=20)
+ tracing_setup._attach_cost_attr(attrs, usage, "gpt-5.5")
+
+ assert attrs["operation.cost"] == 0.123
+ assert len(calls) == 1
+ assert calls[0]["model_ref"] == "gpt-5.5"
+ assert calls[0]["provider_id"] is None
+
+
+def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls: list[dict[str, Any]] = []
+
+ def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
+ calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})
+ if provider_id is None:
+ raise LookupError(f"unknown {model_ref}")
+ return SimpleNamespace(total_price=0.456)
+
+ usages = _install_genai_prices_stub(monkeypatch, fake_calc_price)
+
+ attrs: dict[str, Any] = {}
+ usage = SimpleNamespace(
+ input_tokens=384702,
+ output_tokens=4666,
+ input_tokens_details=SimpleNamespace(cached_tokens=123),
+ )
+ tracing_setup._attach_cost_attr(attrs, usage, "anthropic/claude-opus-4-7")
+
+ assert attrs["operation.cost"] == 0.456
+ assert calls == [
+ {"usage": usages[0], "model_ref": "anthropic/claude-opus-4-7", "provider_id": None},
+ {"usage": usages[0], "model_ref": "claude-opus-4-7", "provider_id": "anthropic"},
+ ]
+ assert usages[0].cache_read_tokens == 123
+
+
+def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[dict[str, Any]] = []
+
+ def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
+ calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})
+ raise LookupError(f"unknown {provider_id}/{model_ref}")
+
+ _install_genai_prices_stub(monkeypatch, fake_calc_price)
+
+ attrs: dict[str, Any] = {}
+ usage = SimpleNamespace(input_tokens=10, output_tokens=20)
+ tracing_setup._attach_cost_attr(attrs, usage, "unknown/model")
+
+ assert "operation.cost" not in attrs
+ assert len(calls) == 2
+ assert calls[0]["model_ref"] == "unknown/model"
+ assert calls[0]["provider_id"] is None
+ assert calls[1]["model_ref"] == "model"
+ assert calls[1]["provider_id"] == "unknown"
+
+
def test_disabled_path(agents_stub: tuple[list[bool], list[Any]]) -> None:
disabled_calls, processors_calls = agents_stub