Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-4137 [FIX] Stream LLM completions under the hood so long generations complete instead of timing out (#2294)

Praveen Kumar0a2c3e31c66c
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json
index 52381d6cf..434d9b571 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json
@@ -45,7 +45,13 @@
"multipleOf": 1,
"title": "Timeout",
"default": 900,
- "description": "Timeout in seconds"
+ "description": "Timeout in seconds. Replies are streamed, so this bounds the wait for the first token and any silence between chunks, not the length of the whole reply."
+ },
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
},
"enable_thinking": {
"type": "boolean",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json
index a19e4e09d..547f8c2f3 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json
@@ -49,6 +49,12 @@
"title": "Max Retries",
"default": 5,
"description": "Maximum number of retries to attempt when a request fails."
+ },
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
}
}
}
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json
index 45ed0dfa5..45b74d24a 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json
@@ -69,6 +69,12 @@
"default": 900,
"description": "Timeout in seconds"
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_reasoning": {
"type": "boolean",
"title": "Enable Reasoning",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json
index 54eabf8ad..e93de4e0b 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json
@@ -54,6 +54,12 @@
"title": "Timeout",
"default": 900,
"description": "Request timeout in seconds."
+ },
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
}
}
}
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json
index 0f96008c7..6de8e023c 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json
@@ -111,6 +111,12 @@
"title": "Timeout",
"default": 900,
"description": "Timeout in seconds"
+ },
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
}
},
"dependencies": {
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json
index 670616258..2b2aff351 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json
@@ -56,6 +56,12 @@
"default": 900,
"description": "Timeout in seconds."
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_reasoning": {
"type": "boolean",
"title": "Enable Reasoning",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json
index b03dd9bdf..d92bf2763 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json
@@ -46,6 +46,12 @@
"default": 600,
"description": "Timeout in seconds"
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"max_retries": {
"type": "number",
"minimum": 0,
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json
index 3075271e9..68555bd49 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json
@@ -70,6 +70,12 @@
"default": 900,
"description": "Timeout in seconds."
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_thinking": {
"type": "boolean",
"title": "Enable Thinking",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json
index 78c0def9b..deaa54e22 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json
@@ -49,6 +49,12 @@
"default": 900,
"description": "Timeout in seconds"
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_reasoning": {
"type": "boolean",
"title": "Enable Reasoning",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json
index 2277516f2..f33ec0cc5 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json
@@ -55,6 +55,12 @@
"default": 900,
"description": "Timeout in seconds."
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_reasoning": {
"type": "boolean",
"title": "Enable Reasoning",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json
index 8f2b9db7a..8f8ecebb2 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json
@@ -56,6 +56,12 @@
"default": 3,
"description": "The maximum number of times to retry a request if it fails."
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"request_timeout": {
"type": "number",
"minimum": 0,
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json
index 35de705a5..7b9327f88 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json
@@ -62,6 +62,12 @@
"default": 900,
"description": "Timeout in seconds"
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_reasoning": {
"type": "boolean",
"title": "Enable Reasoning",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json
index a7eede115..dfb3a6d27 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json
@@ -55,6 +55,12 @@
"default": 900,
"description": "Timeout in seconds."
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"enable_reasoning": {
"type": "boolean",
"title": "Enable Reasoning",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json
index bd385ae90..2d15b08fc 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json
@@ -46,6 +46,12 @@
"default": 5,
"description": "Maximum number of retries to attempt when a request fails."
},
+ "enable_streaming": {
+ "type": "boolean",
+ "title": "Enable Streaming",
+ "default": true,
+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."
+ },
"max_tokens": {
"type": "number",
"minimum": 0,
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index e78068518..2082c24d9 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -25,6 +25,7 @@ from unstract.sdk1.utils.common import (
from unstract.sdk1.utils.retry_utils import (
acall_with_retry,
call_with_retry,
+ collect_with_retry,
is_retryable_litellm_error,
iter_with_retry,
pop_litellm_retry_kwargs,
@@ -302,6 +303,12 @@ class LLM:
or enable_prompt_caching
or is_prompt_caching_enabled()
)
+ # Per-adapter streaming switch. Read from the raw metadata because
+ # the provider validators drop keys they do not model, so it never
+ # reaches litellm as a completion param.
+ self._enable_streaming = self._resolve_enable_streaming(
+ self._adapter_metadata
+ )
# REF: https://docs.litellm.ai/docs/completion/input#translated-openai-params
# supported = get_supported_openai_params(model=self.kwargs["model"],
@@ -496,6 +503,86 @@ class LLM:
{"role": "user", "content": user_content},
]
+ # Streaming under the hood is on by default and a per-adapter choice
+ # (``enable_streaming`` in the adapter metadata, surfaced as "Enable
+ # Streaming" in every LLM adapter form). Adapters stored before the field
+ # existed carry no value and stream too; the switch exists to opt an
+ # endpoint out when it cannot stream.
+ #
+ # Why streaming: a non-streaming endpoint keeps the socket silent until the
+ # last token, so a reply that takes longer than the read timeout, or than
+ # an intermediary tolerates on an idle connection, never arrives (observed
+ # on Anthropic as ``litellm.Timeout`` after 900 s and 1800 s on generations
+ # the Anthropic console finishes in 16 minutes; Anthropic's own SDK refuses
+ # non-streaming requests that may exceed 10 minutes). With streaming,
+ # ``complete()`` collects the chunks and rebuilds the full response, so
+ # callers see the same shape as before.
+ _STREAM_BY_DEFAULT = True
+
+ @classmethod
+ def _resolve_enable_streaming(cls, adapter_metadata: Mapping[str, object]) -> bool:
+ """Adapter flag if stored, else the platform default (on)."""
+ flag = adapter_metadata.get("enable_streaming")
+ if flag is None:
+ return cls._STREAM_BY_DEFAULT
+ return bool(flag)
+
+ def _streams_under_the_hood(self) -> bool:
+ return self._enable_streaming
+
+ @staticmethod
+ def _chunk_has_content(chunk: object) -> bool:
+ """Whether a stream chunk carries generated text or reasoning.
+
+ Bookkeeping chunks (``message_start``, usage-only tails) do not count:
+ a failure before the first content chunk is still a failed request.
+ """
+ get = getattr(chunk, "get", None)
+ if not callable(get):
+ return False
+ choices = get("choices")
+ if not choices:
+ return False
+ delta = choices[0].get("delta") or {}
+ return bool(delta.get("content") or delta.get("reasoning_content"))
+
+ def _complete_via_stream(
+ self,
+ messages: list[dict[str, object]],
+ completion_kwargs: dict[str, object],
+ max_retries: int,
+ ) -> dict[str, object]:
+ """Run a streamed completion and rebuild the non-streaming response.
+
+ Retries follow ``collect_with_retry``: a failure before the first
+ content chunk is retried like a non-streaming call; a failure after
+ content started is raised as-is so a long generation is never
+ replayed. ``litellm.stream_chunk_builder`` reassembles content,
+ thinking blocks, ``finish_reason``, usage (including cache tokens)
+ and the provider response headers.
+ """
+ chunks = collect_with_retry(
+ lambda: litellm.completion(
+ messages=messages,
+ stream=True,
+ stream_options={"include_usage": True},
+ **completion_kwargs,
+ ),
+ max_retries=max_retries,
+ retry_predicate=is_retryable_litellm_error,
+ is_content=self._chunk_has_content,
+ description=self._get_adapter_info(),
+ )
+ response = litellm.stream_chunk_builder(chunks, messages=messages)
+ if response is None:
+ raise LLMError(
+ message=(
+ f"Error from LLM adapter '{self._get_adapter_info()}': "
+ "provider returned an empty stream"
+ )
+ )
+ return cast("dict[str, object]", response)
+
@capture_metrics
def complete(
self,
@@ -541,12 +628,23 @@ class LLM:
max_retries = pop_litellm_retry_kwargs(
completion_kwargs, self._get_adapter_info()
)
- response: dict[str, object] = call_with_retry(
- lambda: litellm.completion(messages=messages, **completion_kwargs),
- max_retries=max_retries,
- retry_predicate=is_retryable_litellm_error,
- description=self._get_adapter_info(),
- )
+ # A mocked completion (UNSTRACT_LLM_MOCK_RESPONSE, used by the e2e
+ # rig) never touches the network, so streaming buys nothing there,
+ # and litellm's mock reports fixed usage (10/20/30) only on the
+ # non-streaming path. The rig counts calls with that contract.
+ mocked = "mock_response" in completion_kwargs
+ response: dict[str, object]
+ if self._streams_under_the_hood() and not mocked:
+ response = self._complete_via_stream(
+ messages, completion_kwargs, max_retries
+ )
+ else:
+ response = call_with_retry(
+ lambda: litellm.completion(messages=messages, **completion_kwargs),
+ max_retries=max_retries,
+ retry_predicate=is_retryable_litellm_error,
+ description=self._get_adapter_info(),
+ )
response_text = response["choices"][0]["message"]["content"]
finish_reason = response["choices"][0].get("finish_reason")
diff --git a/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py b/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py
index 1f04026bb..794082e21 100644
--- a/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py
+++ b/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py
@@ -274,6 +274,57 @@ def iter_with_retry[T](
time.sleep(delay)
+def collect_with_retry[T](
+ fn: Callable[[], Iterable[T]],
+ *,
+ max_retries: int,
+ retry_predicate: Callable[[Exception], bool],
+ is_content: Callable[[T], bool],
+ description: str = "",
+ logger_instance: logging.Logger | None = None,
+) -> list[T]:
+ """Drain fn() into a list with retry. Only retries before content.
+
+ Streamed completions emit bookkeeping items (e.g. Anthropic
+ ``message_start``) before any content. A failure at that point is a
+ failed *request*, retried exactly as a non-streaming call would be. A
+ failure after ``is_content`` matched an item is a failed *generation*
+ and is raised immediately: replaying it would re-run a generation that
+ may already have consumed minutes. Items from a failed attempt are
+ discarded so a retry never duplicates them.
+ """
+ _validate_max_retries(max_retries)
+ log = logger_instance or logger
+ for attempt in range(max_retries + 1):
+ items: list[T] = []
+ has_content = False
+ gen: Iterable[T] | None = None
+ try:
+ # ``fn()`` itself can fail: litellm's streaming ``completion()``
+ # sends the request when called, so a 429/5xx/connection error
+ # is raised here, before any chunk is yielded. That is a failed
+ # request and must retry like the non-streaming path.
+ gen = fn()
+ for item in gen:
+ items.append(item)
+ has_content = has_content or is_content(item)
+ return items
+ except Exception as e:
+ # Release the in-flight HTTP/socket resources before retrying.
+ close = getattr(gen, "close", None)
+ if callable(close):
+ close()
+ if has_content:
+ raise
+ delay = _get_retry_delay(
+ e, attempt, max_retries, retry_predicate, description, log
+ )
+ if delay is None:
+ raise
+ time.sleep(delay)
+ raise RuntimeError("unreachable") # for type-checker: loop always returns or raises
+
+
def is_retryable_error(error: Exception) -> bool:
"""Check if a requests-library HTTP error should trigger a retry.
diff --git a/unstract/sdk1/tests/test_anthropic_stream_complete.py b/unstract/sdk1/tests/test_anthropic_stream_complete.py
new file mode 100644
index 000000000..6fa2234a1
--- /dev/null
+++ b/unstract/sdk1/tests/test_anthropic_stream_complete.py
@@ -0,0 +1,352 @@
+"""``LLM.complete()`` streams under the hood for direct Anthropic.
+
+A non-streaming Anthropic request keeps the socket silent for the whole
+generation. For long replies the response never arrives: the client's read
+timeout fires (900 s / 1800 s observed in staging and production) and the
+identical request was replayed by the retry helper. Anthropic's own SDK
+refuses non-streaming requests expected to exceed 10 minutes for this reason.
+
+These tests pin the contract: for the ``anthropic`` provider ``complete()``
+issues ``stream=True``, collects the chunks and rebuilds the same response
+shape callers already consume. Every provider streams unless its adapter
+opts out.
+
+LiteLLM's ``mock_response`` streaming path produces real ``ModelResponseStream``
+chunks, so the rebuild goes through ``litellm.stream_chunk_builder`` for real.
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import lru_cache
+from importlib import import_module
+from typing import TYPE_CHECKING
+from unittest.mock import patch
+
+import litellm
+import pytest
+from unstract.sdk1.utils import retry_utils
+
+if TYPE_CHECKING:
+ from collections.abc import Callable, Iterator
+
+ANTHROPIC_ADAPTER_ID = "anthropic|90ebd4cd-2f19-4cef-a884-9eeb6ac0f203"
+OPENAI_ADAPTER_ID = "openai|502ecf49-e47c-445c-9907-6d4b90c5cd17"
+MOCK_TEXT = "hello streamed world"
+# Bound before any test patches ``litellm.completion`` so the spy can delegate
+# to the real mock path instead of recursing into itself.
+_REAL_COMPLETION = litellm.completion
+
+
+@lru_cache(maxsize=1)
+def _load_llm_module() -> object:
+ import sys
+ from types import ModuleType
+
+ sys.modules.setdefault("magic", ModuleType("magic"))
+ return import_module("unstract.sdk1.llm")
+
+
+def _make_llm(adapter_id: str, model: str, **metadata: object) -> object:
+ llm_module = _load_llm_module()
+ return llm_module.LLM(
+ adapter_id=adapter_id,
+ adapter_metadata={"model": model, "api_key": "test-key", **metadata},
+ )
+
+
+def _mock_chunks(model: str) -> list[object]:
+ """Real litellm stream chunks for MOCK_TEXT, produced without network."""
+ return list(
+ _REAL_COMPLETION(
+ model=model,
+ messages=[{"role": "user", "content": "hi"}],
+ stream=True,
+ mock_response=MOCK_TEXT,
+ api_key="test-key",
+ )
+ )
+
+
+class _Spy:
+ """Record litellm.completion kwargs and delegate to the mock path."""
+
+ def __init__(self) -> None:
+ self.calls: list[dict[str, object]] = []
+
+ def __call__(self, **kwargs: object) -> object:
+ self.calls.append(kwargs)
+ return _REAL_COMPLETION(**{**kwargs, "mock_response": MOCK_TEXT})
+
+
+def no_cost() -> Iterator[None]:
+ llm_module = _load_llm_module()
+ with patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)):
+ yield
+
+
+# ── Anthropic streams under the hood ─────────────────────────────────────────
+
+
+def test_anthropic_complete_streams_under_the_hood(no_cost: None) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")
+ spy = _Spy()
+
+ with patch.object(llm_module.litellm, "completion", spy):
+ result = llm.complete("hi")
+
+ assert len(spy.calls) == 1
+ assert spy.calls[0]["stream"] is True
+ assert result["response"].text == MOCK_TEXT
+
+
+def test_anthropic_rebuilt_response_keeps_consumer_facing_fields(
+ no_cost: None,
+) -> None:
+ """Rebuilt raw response keeps finish_reason and usage.
+
+ Callers read ``raw.choices[0].finish_reason`` and usage off the raw
+ response (cloud line-item extractor, usage accounting).
+ """
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")
+
+ with patch.object(llm_module.litellm, "completion", _Spy()):
+ result = llm.complete("hi")
+
+ raw = result["response"].raw
+ assert raw.choices[0].finish_reason == "stop"
+ assert raw["choices"][0]["message"]["content"] == MOCK_TEXT
+ assert raw.get("usage") is not None
+ assert raw["usage"]["completion_tokens"] > 0
+
+
+def test_anthropic_stream_records_usage_once(
+ no_cost: None, caplog: pytest.LogCaptureFixture
+) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")
+
+ with (
+ patch.object(llm_module.litellm, "completion", _Spy()),
+ caplog.at_level(logging.INFO, logger=llm_module.logger.name),
+ ):
+ llm.complete("hi")
+
+ usage_logs = [r.message for r in caplog.records if "Usage:" in r.message]
+ assert len(usage_logs) == 1
+ assert "[complete]" in usage_logs[0]
+
+
+def test_adapter_opted_out_stays_non_streaming(no_cost: None) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(
+ OPENAI_ADAPTER_ID,
+ "gpt-4o",
+ api_base="https://api.openai.com/v1",
+ enable_streaming=False,
+ )
+ spy = _Spy()
+
+ with patch.object(llm_module.litellm, "completion", spy):
+ result = llm.complete("hi")
+
+ assert len(spy.calls) == 1
+ assert not spy.calls[0].get("stream")
+ assert result["response"].text == MOCK_TEXT
+
+
+# ── Retry semantics on the streamed path ─────────────────────────────────────
+
+
+def _failing_then_ok(
+ chunks: list[object], fail_after: int, error: Exception
+) -> tuple[Callable[..., Iterator[object]], list[int]]:
+ """Build a litellm.completion stand-in that fails once, then succeeds.
+
+ The first call raises ``error`` after yielding ``fail_after`` chunks; the
+ second call yields every chunk.
+ """
+ calls: list[int] = []
+
+ def fake(**_: object) -> Iterator[object]:
+ calls.append(1)
+ if len(calls) == 1:
+ yield from chunks[:fail_after]
+ raise error
+ yield from chunks
+
+ return fake, calls
+
+
+def _timeout() -> Exception:
+ return litellm.Timeout(
+ message="Connection timed out after 900.0 seconds.",
+ model="claude-sonnet-4-6",
+ llm_provider="anthropic",
+ )
+
+
+def test_stream_failure_before_any_content_is_retried(no_cost: None) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)
+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")
+ fake, calls = _failing_then_ok(chunks, fail_after=0, error=_timeout())
+
+ with (
+ patch.object(llm_module.litellm, "completion", fake),
+ patch.object(retry_utils.time, "sleep") as fake_sleep,
+ ):
+ result = llm.complete("hi")
+
+ assert len(calls) == 2
+ assert result["response"].text == MOCK_TEXT
+ fake_sleep.assert_called_once()
+
+
+def test_request_failure_before_stream_exists_is_retried(no_cost: None) -> None:
+ """Litellm raises from ``completion()`` itself on request errors.
+
+ A rate limit or connection error happens before a stream object exists,
+ so it must be retried like a non-streaming call, not escape on the first
+ attempt.
+ """
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)
+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")
+ calls: list[int] = []
+
+ def fake(**_: object) -> Iterator[object]:
+ calls.append(1)
+ if len(calls) == 1:
+ raise litellm.RateLimitError(
+ message="rate limited",
+ model="claude-sonnet-4-6",
+ llm_provider="anthropic",
+ )
+ return iter(chunks)
+
+ with (
+ patch.object(llm_module.litellm, "completion", fake),
+ patch.object(retry_utils.time, "sleep") as fake_sleep,
+ ):
+ result = llm.complete("hi")
+
+ assert len(calls) == 2
+ assert result["response"].text == MOCK_TEXT
+ fake_sleep.assert_called_once()
+
+
+def test_stream_failure_after_content_is_not_replayed(no_cost: None) -> None:
+ """A drop after content started surfaces immediately.
+
+ Replaying would re-run a generation that may already have run for minutes.
+ """
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)
+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")
+ fake, calls = _failing_then_ok(chunks, fail_after=2, error=_timeout())
+
+ with (
+ patch.object(llm_module.litellm, "completion", fake),
+ pytest.raises(llm_module.LLMError) as exc_info,
+ ):
+ llm.complete("hi")
+
+ assert len(calls) == 1
+ assert "timed out" in str(exc_info.value)
+
+
+def test_stream_non_retryable_error_is_not_retried(no_cost: None) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)
+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")
+ error = litellm.AuthenticationError(
+ message="invalid x-api-key", model="claude-sonnet-4-6", llm_provider="anthropic"
+ )
+ fake, calls = _failing_then_ok(chunks, fail_after=0, error=error)
+
+ with (
+ patch.object(llm_module.litellm, "completion", fake),
+ pytest.raises(llm_module.LLMError),
+ ):
+ llm.complete("hi")
+
+ assert len(calls) == 1
+
+
+def test_empty_stream_raises_llm_error(no_cost: None) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")
+
+ def empty(**_: object) -> Iterator[object]:
+ yield from ()
+
+ with (
+ patch.object(llm_module.litellm, "completion", empty),
+ pytest.raises(llm_module.LLMError),
+ ):
+ llm.complete("hi")
+
+
+# ── Per-adapter ``enable_streaming`` switch ──────────────────────────────────
+
+
+def _stream_flag_sent(llm: object) -> bool:
+ llm_module = _load_llm_module()
+ spy = _Spy()
+ with patch.object(llm_module.litellm, "completion", spy):
+ llm.complete("hi")
+ return bool(spy.calls[0].get("stream"))
+
+
+def test_anthropic_adapter_can_opt_out_of_streaming(no_cost: None) -> None:
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", enable_streaming=False)
+ assert _stream_flag_sent(llm) is False
+
+
+def test_openai_adapter_streams_by_default(no_cost: None) -> None:
+ llm = _make_llm(OPENAI_ADAPTER_ID, "gpt-4o", api_base="https://api.openai.com/v1")
+ assert _stream_flag_sent(llm) is True
+
+
+def test_missing_flag_streams_for_every_provider(no_cost: None) -> None:
+ """Adapters stored before the field existed stream like new ones."""
+ anthropic = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")
+ openai = _make_llm(OPENAI_ADAPTER_ID, "gpt-4o", api_base="https://api.openai.com/v1")
+ assert _stream_flag_sent(anthropic) is True
+ assert _stream_flag_sent(openai) is True
+
+
+def test_streaming_flag_never_reaches_litellm(no_cost: None) -> None:
+ llm_module = _load_llm_module()
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", enable_streaming=True)
+ spy = _Spy()
+ with patch.object(llm_module.litellm, "completion", spy):
+ llm.complete("hi")
+ assert "enable_streaming" not in spy.calls[0]
+
+
+def test_mocked_completion_keeps_non_streaming_path(
+ no_cost: None, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """The e2e rig counts LLM calls via litellm's fixed mock usage (10/20/30).
+
+ That usage is only reported on the non-streaming mock path, and a mock
+ never touches the network, so an injected mock response must bypass
+ streaming even when the adapter streams.
+ """
+ llm_module = _load_llm_module()
+ monkeypatch.setenv(llm_module._MOCK_RESPONSE_ENV, "canned answer")
+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")
+ spy = _Spy()
+
+ with patch.object(llm_module.litellm, "completion", spy):
+ result = llm.complete("hi")
+
+ assert spy.calls[0]["mock_response"] == "canned answer"
+ assert not spy.calls[0].get("stream")
+ assert result["response"].raw["usage"]["prompt_tokens"] == 10
+ assert result["response"].raw["usage"]["completion_tokens"] == 20
diff --git a/unstract/sdk1/tests/test_llm_schema_streaming_flag.py b/unstract/sdk1/tests/test_llm_schema_streaming_flag.py
new file mode 100644
index 000000000..2e1b85558
--- /dev/null
+++ b/unstract/sdk1/tests/test_llm_schema_streaming_flag.py
@@ -0,0 +1,39 @@
+"""Every LLM adapter form exposes the per-adapter ``enable_streaming`` switch.
+
+The switch is read by ``LLM`` from the raw adapter metadata (see
+``LLM._resolve_enable_streaming``). It defaults to on everywhere; the form
+field exists so an endpoint that cannot stream can be opted out on its own
+adapter.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+STATIC_DIR = (
+ Path(__file__).resolve().parents[1]
+ / "src"
+ / "unstract"
+ / "sdk1"
+ / "adapters"
+ / "llm1"
+ / "static"
+)
+SCHEMAS = sorted(STATIC_DIR.glob("*.json"))
+
+
+def test_static_dir_has_schemas() -> None:
+ assert len(SCHEMAS) >= 14
+
+
[email protected]("schema_path", SCHEMAS, ids=[p.name for p in SCHEMAS])
+def test_schema_exposes_enable_streaming(schema_path: Path) -> None:
+ props = json.loads(schema_path.read_text())["properties"]
+ field = props.get("enable_streaming")
+ assert field is not None, f"{schema_path.name} lacks enable_streaming"
+ assert field["type"] == "boolean"
+ assert field["title"] == "Enable Streaming"
+ assert field["default"] is True
diff --git a/unstract/sdk1/tests/utils/test_collect_with_retry.py b/unstract/sdk1/tests/utils/test_collect_with_retry.py
new file mode 100644
index 000000000..974a6e6e0
--- /dev/null
+++ b/unstract/sdk1/tests/utils/test_collect_with_retry.py
@@ -0,0 +1,152 @@
+"""``collect_with_retry``: drain an iterable, retrying only before content.
+
+Used by the streamed completion path. ``iter_with_retry`` stops retrying at
+the first yielded item, but a provider stream emits bookkeeping chunks (e.g.
+Anthropic ``message_start``) before any content. A failure at that point is
+still a failed *request*, so it should be retried; a failure after content
+started is a failed *generation* and must not be replayed.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest.mock import patch
+
+import pytest
+from unstract.sdk1.utils import retry_utils
+from unstract.sdk1.utils.retry_utils import collect_with_retry
+
+if TYPE_CHECKING:
+ from collections.abc import Callable, Iterator
+
+
+def _is_content(item: str) -> bool:
+ return item.startswith("c")
+
+
+def _stream_factory(
+ script: list[list[str | Exception]],
+) -> tuple[Callable[[], Iterator[str]], list[int]]:
+ """Each call replays the next script entry; an Exception entry is raised."""
+ calls: list[int] = []
+
+ def fn() -> Iterator[str]:
+ idx = len(calls)
+ calls.append(1)
+ for item in script[idx]:
+ if isinstance(item, Exception):
+ raise item
+ yield item
+
+ return fn, calls
+
+
+def test_returns_all_items_when_stream_succeeds() -> None:
+ fn, calls = _stream_factory([["meta", "c1", "c2"]])
+ result = collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content
+ )
+ assert result == ["meta", "c1", "c2"]
+ assert len(calls) == 1
+
+
+def test_retries_when_failure_precedes_content() -> None:
+ fn, calls = _stream_factory([["meta", TimeoutError()], ["meta", "c1"]])
+ with patch.object(retry_utils.time, "sleep"):
+ result = collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content
+ )
+ # Bookkeeping items from the failed attempt are discarded, not duplicated.
+ assert result == ["meta", "c1"]
+ assert len(calls) == 2
+
+
+def test_retries_when_stream_creation_itself_raises() -> None:
+ """``fn()`` failing before returning an iterable is a failed request.
+
+ litellm's streaming ``completion()`` sends the HTTP request when called,
+ so a 429/5xx/connection error surfaces from ``fn()`` itself rather than
+ from iteration.
+ """
+ calls: list[int] = []
+
+ def fn() -> Iterator[str]:
+ calls.append(1)
+ if len(calls) == 1:
+ raise TimeoutError("request failed before any chunk")
+ return iter(["meta", "c1"])
+
+ with patch.object(retry_utils.time, "sleep"):
+ result = collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content
+ )
+ assert result == ["meta", "c1"]
+ assert len(calls) == 2
+
+
+def test_stream_creation_failure_that_is_not_retryable_is_raised() -> None:
+ calls: list[int] = []
+
+ def fn() -> Iterator[str]:
+ calls.append(1)
+ raise ValueError("bad request")
+
+ with pytest.raises(ValueError):
+ collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: False, is_content=_is_content
+ )
+ assert len(calls) == 1
+
+
+def test_does_not_retry_once_content_was_received() -> None:
+ fn, calls = _stream_factory([["meta", "c1", TimeoutError()], ["meta", "c1"]])
+ with pytest.raises(TimeoutError):
+ collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content
+ )
+ assert len(calls) == 1
+
+
+def test_does_not_retry_non_retryable_error() -> None:
+ fn, calls = _stream_factory([[ValueError("bad key")], ["c1"]])
+ with pytest.raises(ValueError):
+ collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: False, is_content=_is_content
+ )
+ assert len(calls) == 1
+
+
+def test_gives_up_after_max_retries() -> None:
+ fn, calls = _stream_factory([[TimeoutError()], [TimeoutError()], [TimeoutError()]])
+ with patch.object(retry_utils.time, "sleep"), pytest.raises(TimeoutError):
+ collect_with_retry(
+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content
+ )
+ assert len(calls) == 3
+
+
+def test_closes_failed_generator_before_retrying() -> None:
+ closed: list[bool] = []
+
+ class _Gen:
+ def __init__(self, fail: bool) -> None:
+ self._fail = fail
+
+ def __iter__(self) -> Iterator[str]:
+ if self._fail:
+ raise TimeoutError()
+ yield "c1"
+
+ def close(self) -> None:
+ closed.append(True)
+
+ attempts = iter([_Gen(fail=True), _Gen(fail=False)])
+ with patch.object(retry_utils.time, "sleep"):
+ result = collect_with_retry(
+ lambda: next(attempts),
+ max_retries=1,
+ retry_predicate=lambda _: True,
+ is_content=_is_content,
+ )
+ assert result == ["c1"]
+ assert closed == [True]