Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix:(SKY-9633) Fix copilot otel cost attribution for non-openai providers (#5859)
Andrew Neilson6784c68dc16f
diff --git a/skyvern/forge/sdk/copilot/tracing_setup.py b/skyvern/forge/sdk/copilot/tracing_setup.pyindex a53e4c977..dd6c86f2c 100644--- a/skyvern/forge/sdk/copilot/tracing_setup.py+++ b/skyvern/forge/sdk/copilot/tracing_setup.py@@ -125,6 +125,16 @@ def _usage_field(obj: Any, *keys: str) -> Any:return None+def _calculate_price(calc_price: Any, usage: Any, model: str) -> Any:+ try:+ return calc_price(usage, model_ref=model)+ except Exception:+ provider, separator, model_ref = model.partition("/")+ if not separator or not provider or not model_ref:+ raise+ return calc_price(usage, model_ref=model_ref, provider_id=provider)++def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> None:"""Stamp ``operation.cost`` (USD) on a GenerationSpanData span.@@ -144,19 +154,18 @@ def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> N_usage_field(usage, "input_tokens_details", "prompt_tokens_details"),"cached_tokens",)+ if cached is None:+ cached = _usage_field(usage, "cache_read_input_tokens")cache_read = int(cached) if isinstance(cached, (int, float)) and cached > 0 else Nonetry:from genai_prices import Usage, calc_price- price = calc_price(- Usage(- input_tokens=int(input_tokens),- output_tokens=int(output_tokens),- cache_read_tokens=cache_read,- ),- model_ref=model,- provider_id="openai",+ price_usage = Usage(+ input_tokens=int(input_tokens),+ output_tokens=int(output_tokens),+ cache_read_tokens=cache_read,)+ price = _calculate_price(calc_price, price_usage, model)except Exception:returnattrs["operation.cost"] = float(price.total_price)diff --git a/tests/unit/test_copilot_tracing_setup.py b/tests/unit/test_copilot_tracing_setup.pyindex 4961da3eb..277f425e2 100644--- a/tests/unit/test_copilot_tracing_setup.py+++ b/tests/unit/test_copilot_tracing_setup.py@@ -57,6 +57,99 @@ def stub_logger(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, Anreturn logged+def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any) -> list[Any]:+ usages: list[Any] = []++ class FakeUsage:+ def __init__(+ self,+ *,+ input_tokens: int,+ output_tokens: int,+ cache_read_tokens: int | None = None,+ ) -> None:+ self.input_tokens = input_tokens+ self.output_tokens = output_tokens+ self.cache_read_tokens = cache_read_tokens+ usages.append(self)++ monkeypatch.setitem(+ sys.modules,+ "genai_prices",+ SimpleNamespace(Usage=FakeUsage, calc_price=calc_price),+ )+ return usages+++def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.MonkeyPatch) -> None:+ calls: list[dict[str, Any]] = []++ def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:+ calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})+ return SimpleNamespace(total_price=0.123)++ _install_genai_prices_stub(monkeypatch, fake_calc_price)++ attrs: dict[str, Any] = {}+ usage = SimpleNamespace(input_tokens=10, output_tokens=20)+ tracing_setup._attach_cost_attr(attrs, usage, "gpt-5.5")++ assert attrs["operation.cost"] == 0.123+ assert len(calls) == 1+ assert calls[0]["model_ref"] == "gpt-5.5"+ assert calls[0]["provider_id"] is None+++def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ calls: list[dict[str, Any]] = []++ def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:+ calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})+ if provider_id is None:+ raise LookupError(f"unknown {model_ref}")+ return SimpleNamespace(total_price=0.456)++ usages = _install_genai_prices_stub(monkeypatch, fake_calc_price)++ attrs: dict[str, Any] = {}+ usage = SimpleNamespace(+ input_tokens=384702,+ output_tokens=4666,+ input_tokens_details=SimpleNamespace(cached_tokens=123),+ )+ tracing_setup._attach_cost_attr(attrs, usage, "anthropic/claude-opus-4-7")++ assert attrs["operation.cost"] == 0.456+ assert calls == [+ {"usage": usages[0], "model_ref": "anthropic/claude-opus-4-7", "provider_id": None},+ {"usage": usages[0], "model_ref": "claude-opus-4-7", "provider_id": "anthropic"},+ ]+ assert usages[0].cache_read_tokens == 123+++def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.MonkeyPatch) -> None:+ calls: list[dict[str, Any]] = []++ def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:+ calls.append({"usage": usage, "model_ref": model_ref, "provider_id": provider_id})+ raise LookupError(f"unknown {provider_id}/{model_ref}")++ _install_genai_prices_stub(monkeypatch, fake_calc_price)++ attrs: dict[str, Any] = {}+ usage = SimpleNamespace(input_tokens=10, output_tokens=20)+ tracing_setup._attach_cost_attr(attrs, usage, "unknown/model")++ assert "operation.cost" not in attrs+ assert len(calls) == 2+ assert calls[0]["model_ref"] == "unknown/model"+ assert calls[0]["provider_id"] is None+ assert calls[1]["model_ref"] == "model"+ assert calls[1]["provider_id"] == "unknown"++def test_disabled_path(agents_stub: tuple[list[bool], list[Any]]) -> None:disabled_calls, processors_calls = agents_stub