Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log the provider response id on LLM duration metrics (SKY-16508) (#8606)
pedrohsdb26eb9effe88f
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 184b9d3b7..214425f39 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -612,6 +612,22 @@ def _effective_service_tier(response: object) -> str | None:return reported if isinstance(reported, str) else _recovered_service_tier(response)+# litellm stamps "chatcmpl-<uuid4>" on a response whose provider sent no id; logging it would+# invent a correlation key that matches nothing on the provider side.+_LITELLM_SYNTHESIZED_RESPONSE_ID = re.compile(+ r"chatcmpl-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"+)+++def _response_id_log_fields(response: object) -> dict[str, str]:+ response_id = getattr(response, "id", None)+ if not isinstance(response_id, str) or not response_id:+ return {}+ if _LITELLM_SYNTHESIZED_RESPONSE_ID.fullmatch(response_id):+ return {}+ return {"response_id": response_id}++def _response_provider(response: object) -> str | None:provider = getattr(response, "provider", None)if provider is None:@@ -2350,6 +2366,7 @@ class LLMAPIHandlerFactory:service_tier=service_tier,llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),+ **_response_id_log_fields(response),**_enrich_tree_log_fields(context, step),**_consume_prompt_breakdown(context),**_recording_correlation_fields(recording_attempt_id, interpretation_session_id),@@ -2975,6 +2992,7 @@ class LLMAPIHandlerFactory:service_tier_source=service_tier_source,llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),+ **_response_id_log_fields(response),**_enrich_tree_log_fields(context, step),**_consume_prompt_breakdown(context),**_recording_correlation_fields(recording_attempt_id, interpretation_session_id),@@ -3745,6 +3763,7 @@ class LLMCaller:service_tier_source=service_tier_source,llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),+ **_response_id_log_fields(response),**_enrich_tree_log_fields(context, step),**_consume_prompt_breakdown(context),**_recording_correlation_fields(recording_attempt_id, interpretation_session_id),@@ -4220,7 +4239,7 @@ class LLMCaller:LOG.warning("OpenRouter response missing usage.cost; cost will be reported as 0",llm_key=self.original_llm_key,- response_id=getattr(response, "id", None),+ **_response_id_log_fields(response),)else:computed_cost = LLMAPIHandlerFactory.completion_cost_or_none(response)diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex e1d1e343e..22474838d 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -2332,6 +2332,31 @@ async def test_llm_caller_does_not_mark_parse_failure_completed(assert enrich_span.call_args.kwargs["mark_completed"] is False+ "response_id, expected",+ [+ ("gen-provider-issued-1", {"response_id": "gen-provider-issued-1"}),+ # litellm synthesizes an id when the provider sent none.+ (None, {}),+ ],+ ids=["provider-id", "litellm-synthesized"],+)+async def test_llm_caller_metrics_log_carries_only_a_provider_issued_response_id(+ monkeypatch: pytest.MonkeyPatch, response_id: str | None, expected: dict[str, str]+) -> None:+ caller, logger = _stub_successful_llm_caller(monkeypatch)+ response = _bridge_response(model_id=None)+ if response_id is not None:+ response.id = response_id+ monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))++ await caller.call(prompt="test", prompt_name="extract-actions")++ metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")+ assert {k: v for k, v in metrics.items() if k == "response_id"} == expected++def test_recovered_tier_stays_out_of_the_persisted_response_artifact() -> None:"""The LLM_RESPONSE artifact is dumped from this object. A tier we recovered must not appearthere, or a later investigation reads our own inference as something the provider reported —diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.pyindex b3ca8020b..95b130a80 100644--- a/tests/unit/test_llm_handler_parity_characterization.py+++ b/tests/unit/test_llm_handler_parity_characterization.py@@ -631,6 +631,34 @@ async def test_copilot_model_usage_survives_response_parse_failure(assert router.usage_events[0]["gen_ai.response.model"] == "openai/gpt-4.1"+ "response_id, expected",+ [+ ("gen-provider-issued-1", {"response_id": "gen-provider-issued-1"}),+ # What litellm stamps when the provider sent no id.+ (litellm.ModelResponse().id, {}),+ (None, {}),+ ],+ ids=["provider-id", "litellm-synthesized", "no-id"],+)+async def test_metrics_log_carries_only_a_provider_issued_response_id(+ span_exporter: InMemorySpanExporter, response_id: str | None, expected: dict[str, str]+) -> None:+ def response() -> ParityResponse:+ r = ParityResponse("gpt-4")+ r.id = response_id # type: ignore[attr-defined]+ return r++ direct = await _run_direct(span_exporter, responses=[response()], parameters={})+ router = await _run_router(span_exporter, responses=[response()], parameters={})++ for outcome in (direct, router):+ assert outcome.error is None+ (metrics,) = outcome.metrics_events+ assert {k: v for k, v in metrics.items() if k == "response_id"} == expected++@pytest.mark.asyncioasync def test_zero_usage_records_none_token_fields_on_both(span_exporter: InMemorySpanExporter) -> None:def zero_response() -> ParityResponse: