Skip to content

Automate browser based workflows with AI

Log the provider response id on LLM duration metrics (SKY-16508) (#8606)

pedrohsdb26eb9effe88f
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 184b9d3b7..214425f39 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -612,6 +612,22 @@ def _effective_service_tier(response: object) -> str | None:
return reported if isinstance(reported, str) else _recovered_service_tier(response)
+# litellm stamps "chatcmpl-<uuid4>" on a response whose provider sent no id; logging it would
+# invent a correlation key that matches nothing on the provider side.
+_LITELLM_SYNTHESIZED_RESPONSE_ID = re.compile(
+ r"chatcmpl-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
+)
+
+
+def _response_id_log_fields(response: object) -> dict[str, str]:
+ response_id = getattr(response, "id", None)
+ if not isinstance(response_id, str) or not response_id:
+ return {}
+ if _LITELLM_SYNTHESIZED_RESPONSE_ID.fullmatch(response_id):
+ return {}
+ return {"response_id": response_id}
+
+
def _response_provider(response: object) -> str | None:
provider = getattr(response, "provider", None)
if provider is None:
@@ -2350,6 +2366,7 @@ class LLMAPIHandlerFactory:
service_tier=service_tier,
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
+ **_response_id_log_fields(response),
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
**_recording_correlation_fields(recording_attempt_id, interpretation_session_id),
@@ -2975,6 +2992,7 @@ class LLMAPIHandlerFactory:
service_tier_source=service_tier_source,
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
+ **_response_id_log_fields(response),
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
**_recording_correlation_fields(recording_attempt_id, interpretation_session_id),
@@ -3745,6 +3763,7 @@ class LLMCaller:
service_tier_source=service_tier_source,
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
+ **_response_id_log_fields(response),
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
**_recording_correlation_fields(recording_attempt_id, interpretation_session_id),
@@ -4220,7 +4239,7 @@ class LLMCaller:
LOG.warning(
"OpenRouter response missing usage.cost; cost will be reported as 0",
llm_key=self.original_llm_key,
- response_id=getattr(response, "id", None),
+ **_response_id_log_fields(response),
)
else:
computed_cost = LLMAPIHandlerFactory.completion_cost_or_none(response)
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index e1d1e343e..22474838d 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -2332,6 +2332,31 @@ async def test_llm_caller_does_not_mark_parse_failure_completed(
assert enrich_span.call_args.kwargs["mark_completed"] is False
+ "response_id, expected",
+ [
+ ("gen-provider-issued-1", {"response_id": "gen-provider-issued-1"}),
+ # litellm synthesizes an id when the provider sent none.
+ (None, {}),
+ ],
+ ids=["provider-id", "litellm-synthesized"],
+)
+async def test_llm_caller_metrics_log_carries_only_a_provider_issued_response_id(
+ monkeypatch: pytest.MonkeyPatch, response_id: str | None, expected: dict[str, str]
+) -> None:
+ caller, logger = _stub_successful_llm_caller(monkeypatch)
+ response = _bridge_response(model_id=None)
+ if response_id is not None:
+ response.id = response_id
+ monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))
+
+ await caller.call(prompt="test", prompt_name="extract-actions")
+
+ metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
+ assert {k: v for k, v in metrics.items() if k == "response_id"} == expected
+
+
def test_recovered_tier_stays_out_of_the_persisted_response_artifact() -> None:
"""The LLM_RESPONSE artifact is dumped from this object. A tier we recovered must not appear
there, or a later investigation reads our own inference as something the provider reported —
diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.py
index b3ca8020b..95b130a80 100644
--- a/tests/unit/test_llm_handler_parity_characterization.py
+++ b/tests/unit/test_llm_handler_parity_characterization.py
@@ -631,6 +631,34 @@ async def test_copilot_model_usage_survives_response_parse_failure(
assert router.usage_events[0]["gen_ai.response.model"] == "openai/gpt-4.1"
+ "response_id, expected",
+ [
+ ("gen-provider-issued-1", {"response_id": "gen-provider-issued-1"}),
+ # What litellm stamps when the provider sent no id.
+ (litellm.ModelResponse().id, {}),
+ (None, {}),
+ ],
+ ids=["provider-id", "litellm-synthesized", "no-id"],
+)
+async def test_metrics_log_carries_only_a_provider_issued_response_id(
+ span_exporter: InMemorySpanExporter, response_id: str | None, expected: dict[str, str]
+) -> None:
+ def response() -> ParityResponse:
+ r = ParityResponse("gpt-4")
+ r.id = response_id # type: ignore[attr-defined]
+ return r
+
+ direct = await _run_direct(span_exporter, responses=[response()], parameters={})
+ router = await _run_router(span_exporter, responses=[response()], parameters={})
+
+ for outcome in (direct, router):
+ assert outcome.error is None
+ (metrics,) = outcome.metrics_events
+ assert {k: v for k, v in metrics.items() if k == "response_id"} == expected
+
+
@pytest.mark.asyncio
async def test_zero_usage_records_none_token_fields_on_both(span_exporter: InMemorySpanExporter) -> None:
def zero_response() -> ParityResponse: