Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Log LLM provider request/response IDs in SDK usage logs (#2035)

Chandrasekharan Mcafcca7d3c3a
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index c64923dea..b45b85623 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -36,6 +36,49 @@ logger = logging.getLogger(__name__)
# global mutation in concurrent environments.
litellm.drop_params = True
+# Request-id response headers across providers, checked in order:
+# OpenAI/Azure OpenAI, Anthropic, AWS Bedrock, Azure API Management.
+# litellm forwards these in _hidden_params["additional_headers"]
+# prefixed with "llm_provider-".
+_PROVIDER_REQUEST_ID_HEADERS = (
+ "x-request-id",
+ "request-id",
+ "x-amzn-requestid",
+ "apim-request-id",
+)
+
+
+def extract_provider_ids(response: object) -> tuple[str | None, str | None]:
+ """Extract the provider's response id and request id from a litellm response.
+
+ Returns (response_id, request_id) where response_id is the body-level id
+ (e.g. OpenAI "chatcmpl-...", Anthropic "msg_...") and request_id is the
+ provider's request-id response header — the value providers ask for when
+ troubleshooting a specific API call.
+
+ Either value may be None; some providers (e.g. VertexAI/Gemini) expose
+ neither, in which case litellm generates a synthetic response id.
+ """
+ if response is None:
+ return None, None
+ try:
+ response_id = response.get("id")
+ except (AttributeError, TypeError):
+ response_id = None
+ if response_id is None:
+ response_id = getattr(response, "id", None)
+
+ hidden_params = getattr(response, "_hidden_params", None) or {}
+ headers = hidden_params.get("additional_headers") or {}
+ normalized = {
+ key.lower().removeprefix("llm_provider-"): value for key, value in headers.items()
+ }
+ request_id = next(
+ (normalized[h] for h in _PROVIDER_REQUEST_ID_HEADERS if normalized.get(h)),
+ None,
+ )
+ return response_id, request_id
+
# ── Emulated llama-index types ───────────────────────────────────────────────
# These types emulate the llama-index interface without requiring the dependency.
@@ -309,6 +352,7 @@ class LLM:
messages,
response.get("usage"),
"complete",
+ response=response,
)
# Handle refusal or empty content from the LLM provider
@@ -421,6 +465,7 @@ class LLM:
messages,
response.get("usage"),
"complete_vision",
+ response=response,
)
if response_text is None:
@@ -495,6 +540,7 @@ class LLM:
messages,
chunk.get("usage"),
"stream_complete",
+ response=chunk,
)
response = self._process_stream_chunk(
@@ -561,6 +607,7 @@ class LLM:
messages,
response.get("usage"),
"acomplete",
+ response=response,
)
# Handle refusal or empty content from the LLM provider
@@ -680,6 +727,7 @@ class LLM:
messages: list[dict[str, str]],
usage: Mapping[str, int] | None,
llm_api: str,
+ response: object | None = None,
) -> None:
usage_data: Mapping[str, int] = usage or {}
prompt_tokens = usage_data.get("prompt_tokens", 0)
@@ -700,13 +748,23 @@ class LLM:
exc_info=True,
)
+ # Provider ids ride on the existing per-call usage line to aid
+ # troubleshooting (shareable with the provider) without extra log noise.
+ # Absent ids are omitted so providers without them don't add clutter.
+ response_id, request_id = extract_provider_ids(response)
+ id_suffix = ""
+ if response_id is not None:
+ id_suffix += f" response_id={response_id}"
+ if request_id is not None:
+ id_suffix += f" request_id={request_id}"
logger.info(
- "[sdk1][LLM][%s][%s] Usage: prompt=%d completion=%d total=%d",
+ "[sdk1][LLM][%s][%s] Usage: prompt=%d completion=%d total=%d%s",
model,
llm_api,
prompt_tokens,
completion_tokens,
total_tokens,
+ id_suffix,
)
try:
diff --git a/unstract/sdk1/tests/test_llm_provider_ids.py b/unstract/sdk1/tests/test_llm_provider_ids.py
new file mode 100644
index 000000000..be1c0d545
--- /dev/null
+++ b/unstract/sdk1/tests/test_llm_provider_ids.py
@@ -0,0 +1,181 @@
+"""Tests for provider response/request id extraction and logging."""
+
+import logging
+from functools import lru_cache
+from importlib import import_module
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+@lru_cache(maxsize=1)
+def _load_llm_module() -> object:
+ import sys
+ from types import ModuleType
+
+ # Stub python-magic so importing LLM does not depend on libmagic
+ # being available in the test environment.
+ sys.modules.setdefault("magic", ModuleType("magic"))
+ return import_module("unstract.sdk1.llm")
+
+
+class _FakeModelResponse(dict):
+ """Dict-like litellm ModelResponse stand-in with _hidden_params."""
+
+ def __init__(
+ self,
+ *args: object,
+ hidden_params: dict | None = None,
+ **kwargs: object,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ if hidden_params is not None:
+ self._hidden_params = hidden_params
+
+
+def test_extract_provider_ids_openai_headers() -> None:
+ llm_module = _load_llm_module()
+ response = _FakeModelResponse(
+ {"id": "chatcmpl-abc123"},
+ hidden_params={
+ "additional_headers": {
+ "llm_provider-x-request-id": "req_openai_1",
+ "llm_provider-content-type": "application/json",
+ }
+ },
+ )
+
+ assert llm_module.extract_provider_ids(response) == (
+ "chatcmpl-abc123",
+ "req_openai_1",
+ )
+
+
+def test_extract_provider_ids_anthropic_headers() -> None:
+ llm_module = _load_llm_module()
+ response = _FakeModelResponse(
+ {"id": "msg_01XYZ"},
+ hidden_params={
+ "additional_headers": {"llm_provider-request-id": "req_anthropic_1"}
+ },
+ )
+
+ assert llm_module.extract_provider_ids(response) == ("msg_01XYZ", "req_anthropic_1")
+
+
+def test_extract_provider_ids_bedrock_headers() -> None:
+ llm_module = _load_llm_module()
+ response = _FakeModelResponse(
+ {"id": "msg_bdrk_01"},
+ hidden_params={
+ "additional_headers": {"llm_provider-x-amzn-requestid": "req_aws_1"}
+ },
+ )
+
+ assert llm_module.extract_provider_ids(response) == ("msg_bdrk_01", "req_aws_1")
+
+
+def test_extract_provider_ids_prefers_x_request_id() -> None:
+ llm_module = _load_llm_module()
+ response = _FakeModelResponse(
+ {"id": "chatcmpl-1"},
+ hidden_params={
+ "additional_headers": {
+ "llm_provider-apim-request-id": "apim_1",
+ "llm_provider-x-request-id": "req_1",
+ }
+ },
+ )
+
+ assert llm_module.extract_provider_ids(response) == ("chatcmpl-1", "req_1")
+
+
+def test_extract_provider_ids_without_hidden_params() -> None:
+ llm_module = _load_llm_module()
+
+ # Plain dicts (and streaming chunks without headers) yield no request id.
+ assert llm_module.extract_provider_ids({"id": "chatcmpl-2"}) == (
+ "chatcmpl-2",
+ None,
+ )
+
+
+def test_extract_provider_ids_handles_none_and_empty() -> None:
+ llm_module = _load_llm_module()
+
+ assert llm_module.extract_provider_ids(None) == (None, None)
+ assert llm_module.extract_provider_ids({}) == (None, None)
+
+
+def test_extract_provider_ids_falls_back_to_id_attribute() -> None:
+ llm_module = _load_llm_module()
+
+ class _AttrOnlyResponse:
+ id = "chatcmpl-attr"
+
+ def get(self, key: str) -> None:
+ return None
+
+ assert llm_module.extract_provider_ids(_AttrOnlyResponse()) == (
+ "chatcmpl-attr",
+ None,
+ )
+
+
+def _build_llm_for_record_usage(llm_cls: type) -> object:
+ llm = llm_cls.__new__(llm_cls)
+ llm._platform_api_key = "platform-key"
+ llm.platform_kwargs = {"run_id": "run-1"}
+ llm._usage_kwargs = {}
+ llm._pending_usage = []
+ llm.adapter = MagicMock()
+ llm.adapter.get_provider.return_value = "openai"
+ return llm
+
+
+def test_record_usage_logs_provider_ids(caplog: pytest.LogCaptureFixture) -> None:
+ llm_module = _load_llm_module()
+ llm = _build_llm_for_record_usage(llm_module.LLM)
+ response = _FakeModelResponse(
+ {"id": "chatcmpl-log1"},
+ hidden_params={"additional_headers": {"llm_provider-x-request-id": "req_log_1"}},
+ )
+
+ with (
+ patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)),
+ caplog.at_level(logging.INFO, logger=llm_module.logger.name),
+ ):
+ llm._record_usage(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "hello"}],
+ usage={"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7},
+ llm_api="complete",
+ response=response,
+ )
+
+ usage_logs = [r.message for r in caplog.records if "Usage:" in r.message]
+ assert len(usage_logs) == 1
+ assert "response_id=chatcmpl-log1" in usage_logs[0]
+ assert "request_id=req_log_1" in usage_logs[0]
+
+
+def test_record_usage_logs_without_response(caplog: pytest.LogCaptureFixture) -> None:
+ llm_module = _load_llm_module()
+ llm = _build_llm_for_record_usage(llm_module.LLM)
+
+ with (
+ patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)),
+ caplog.at_level(logging.INFO, logger=llm_module.logger.name),
+ ):
+ llm._record_usage(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "hello"}],
+ usage={"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7},
+ llm_api="complete",
+ )
+
+ usage_logs = [r.message for r in caplog.records if "Usage:" in r.message]
+ assert len(usage_logs) == 1
+ # Absent ids are omitted to keep the line compact.
+ assert "response_id" not in usage_logs[0]
+ assert "request_id" not in usage_logs[0]