LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4137 [FIX] Stream LLM completions under the hood so long generations complete instead of timing out (#2294)
Praveen Kumar0a2c3e31c66c
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.jsonindex 52381d6cf..434d9b571 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anthropic.json@@ -45,7 +45,13 @@"multipleOf": 1,"title": "Timeout","default": 900,- "description": "Timeout in seconds"+ "description": "Timeout in seconds. Replies are streamed, so this bounds the wait for the first token and any silence between chunks, not the length of the whole reply."+ },+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."},"enable_thinking": {"type": "boolean",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.jsonindex a19e4e09d..547f8c2f3 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/anyscale.json@@ -49,6 +49,12 @@"title": "Max Retries","default": 5,"description": "Maximum number of retries to attempt when a request fails."+ },+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."}}}diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.jsonindex 45ed0dfa5..45b74d24a 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure.json@@ -69,6 +69,12 @@"default": 900,"description": "Timeout in seconds"},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_reasoning": {"type": "boolean","title": "Enable Reasoning",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.jsonindex 54eabf8ad..e93de4e0b 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/azure_ai.json@@ -54,6 +54,12 @@"title": "Timeout","default": 900,"description": "Request timeout in seconds."+ },+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."}}}diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.jsonindex 0f96008c7..6de8e023c 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/bedrock.json@@ -111,6 +111,12 @@"title": "Timeout","default": 900,"description": "Timeout in seconds"+ },+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."}},"dependencies": {diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.jsonindex 670616258..2b2aff351 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json@@ -56,6 +56,12 @@"default": 900,"description": "Timeout in seconds."},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_reasoning": {"type": "boolean","title": "Enable Reasoning",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.jsonindex b03dd9bdf..d92bf2763 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/gemini.json@@ -46,6 +46,12 @@"default": 600,"description": "Timeout in seconds"},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"max_retries": {"type": "number","minimum": 0,diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.jsonindex 3075271e9..68555bd49 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/minimax.json@@ -70,6 +70,12 @@"default": 900,"description": "Timeout in seconds."},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_thinking": {"type": "boolean","title": "Enable Thinking",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.jsonindex 78c0def9b..deaa54e22 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/mistral.json@@ -49,6 +49,12 @@"default": 900,"description": "Timeout in seconds"},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_reasoning": {"type": "boolean","title": "Enable Reasoning",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.jsonindex 2277516f2..f33ec0cc5 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/nvidia_build.json@@ -55,6 +55,12 @@"default": 900,"description": "Timeout in seconds."},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_reasoning": {"type": "boolean","title": "Enable Reasoning",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.jsonindex 8f2b9db7a..8f8ecebb2 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json@@ -56,6 +56,12 @@"default": 3,"description": "The maximum number of times to retry a request if it fails."},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"request_timeout": {"type": "number","minimum": 0,diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.jsonindex 35de705a5..7b9327f88 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json@@ -62,6 +62,12 @@"default": 900,"description": "Timeout in seconds"},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_reasoning": {"type": "boolean","title": "Enable Reasoning",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.jsonindex a7eede115..dfb3a6d27 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openrouter.json@@ -55,6 +55,12 @@"default": 900,"description": "Timeout in seconds."},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"enable_reasoning": {"type": "boolean","title": "Enable Reasoning",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.jsonindex bd385ae90..2d15b08fc 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/vertex_ai.json@@ -46,6 +46,12 @@"default": 5,"description": "Maximum number of retries to attempt when a request fails."},+ "enable_streaming": {+ "type": "boolean",+ "title": "Enable Streaming",+ "default": true,+ "description": "Stream the model's reply and assemble it in the platform. Keeps the connection alive on long replies so a provider read timeout is not hit. Turn it off only for an endpoint that cannot stream."+ },"max_tokens": {"type": "number","minimum": 0,diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.pyindex e78068518..2082c24d9 100644--- a/unstract/sdk1/src/unstract/sdk1/llm.py+++ b/unstract/sdk1/src/unstract/sdk1/llm.py@@ -25,6 +25,7 @@ from unstract.sdk1.utils.common import (from unstract.sdk1.utils.retry_utils import (acall_with_retry,call_with_retry,+ collect_with_retry,is_retryable_litellm_error,iter_with_retry,pop_litellm_retry_kwargs,@@ -302,6 +303,12 @@ class LLM:or enable_prompt_cachingor is_prompt_caching_enabled())+ # Per-adapter streaming switch. Read from the raw metadata because+ # the provider validators drop keys they do not model, so it never+ # reaches litellm as a completion param.+ self._enable_streaming = self._resolve_enable_streaming(+ self._adapter_metadata+ )# REF: https://docs.litellm.ai/docs/completion/input#translated-openai-params# supported = get_supported_openai_params(model=self.kwargs["model"],@@ -496,6 +503,86 @@ class LLM:{"role": "user", "content": user_content},]+ # Streaming under the hood is on by default and a per-adapter choice+ # (``enable_streaming`` in the adapter metadata, surfaced as "Enable+ # Streaming" in every LLM adapter form). Adapters stored before the field+ # existed carry no value and stream too; the switch exists to opt an+ # endpoint out when it cannot stream.+ #+ # Why streaming: a non-streaming endpoint keeps the socket silent until the+ # last token, so a reply that takes longer than the read timeout, or than+ # an intermediary tolerates on an idle connection, never arrives (observed+ # on Anthropic as ``litellm.Timeout`` after 900 s and 1800 s on generations+ # the Anthropic console finishes in 16 minutes; Anthropic's own SDK refuses+ # non-streaming requests that may exceed 10 minutes). With streaming,+ # ``complete()`` collects the chunks and rebuilds the full response, so+ # callers see the same shape as before.+ _STREAM_BY_DEFAULT = True++ @classmethod+ def _resolve_enable_streaming(cls, adapter_metadata: Mapping[str, object]) -> bool:+ """Adapter flag if stored, else the platform default (on)."""+ flag = adapter_metadata.get("enable_streaming")+ if flag is None:+ return cls._STREAM_BY_DEFAULT+ return bool(flag)++ def _streams_under_the_hood(self) -> bool:+ return self._enable_streaming++ @staticmethod+ def _chunk_has_content(chunk: object) -> bool:+ """Whether a stream chunk carries generated text or reasoning.++ Bookkeeping chunks (``message_start``, usage-only tails) do not count:+ a failure before the first content chunk is still a failed request.+ """+ get = getattr(chunk, "get", None)+ if not callable(get):+ return False+ choices = get("choices")+ if not choices:+ return False+ delta = choices[0].get("delta") or {}+ return bool(delta.get("content") or delta.get("reasoning_content"))++ def _complete_via_stream(+ self,+ messages: list[dict[str, object]],+ completion_kwargs: dict[str, object],+ max_retries: int,+ ) -> dict[str, object]:+ """Run a streamed completion and rebuild the non-streaming response.++ Retries follow ``collect_with_retry``: a failure before the first+ content chunk is retried like a non-streaming call; a failure after+ content started is raised as-is so a long generation is never+ replayed. ``litellm.stream_chunk_builder`` reassembles content,+ thinking blocks, ``finish_reason``, usage (including cache tokens)+ and the provider response headers.+ """+ chunks = collect_with_retry(+ lambda: litellm.completion(+ messages=messages,+ stream=True,+ stream_options={"include_usage": True},+ **completion_kwargs,+ ),+ max_retries=max_retries,+ retry_predicate=is_retryable_litellm_error,+ is_content=self._chunk_has_content,+ description=self._get_adapter_info(),+ )+ response = litellm.stream_chunk_builder(chunks, messages=messages)+ if response is None:+ raise LLMError(+ message=(+ f"Error from LLM adapter '{self._get_adapter_info()}': "+ "provider returned an empty stream"+ )+ )+ return cast("dict[str, object]", response)+@capture_metricsdef complete(self,@@ -541,12 +628,23 @@ class LLM:max_retries = pop_litellm_retry_kwargs(completion_kwargs, self._get_adapter_info())- response: dict[str, object] = call_with_retry(- lambda: litellm.completion(messages=messages, **completion_kwargs),- max_retries=max_retries,- retry_predicate=is_retryable_litellm_error,- description=self._get_adapter_info(),- )+ # A mocked completion (UNSTRACT_LLM_MOCK_RESPONSE, used by the e2e+ # rig) never touches the network, so streaming buys nothing there,+ # and litellm's mock reports fixed usage (10/20/30) only on the+ # non-streaming path. The rig counts calls with that contract.+ mocked = "mock_response" in completion_kwargs+ response: dict[str, object]+ if self._streams_under_the_hood() and not mocked:+ response = self._complete_via_stream(+ messages, completion_kwargs, max_retries+ )+ else:+ response = call_with_retry(+ lambda: litellm.completion(messages=messages, **completion_kwargs),+ max_retries=max_retries,+ retry_predicate=is_retryable_litellm_error,+ description=self._get_adapter_info(),+ )response_text = response["choices"][0]["message"]["content"]finish_reason = response["choices"][0].get("finish_reason")diff --git a/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py b/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.pyindex 1f04026bb..794082e21 100644--- a/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py+++ b/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py@@ -274,6 +274,57 @@ def iter_with_retry[T](time.sleep(delay)+def collect_with_retry[T](+ fn: Callable[[], Iterable[T]],+ *,+ max_retries: int,+ retry_predicate: Callable[[Exception], bool],+ is_content: Callable[[T], bool],+ description: str = "",+ logger_instance: logging.Logger | None = None,+) -> list[T]:+ """Drain fn() into a list with retry. Only retries before content.++ Streamed completions emit bookkeeping items (e.g. Anthropic+ ``message_start``) before any content. A failure at that point is a+ failed *request*, retried exactly as a non-streaming call would be. A+ failure after ``is_content`` matched an item is a failed *generation*+ and is raised immediately: replaying it would re-run a generation that+ may already have consumed minutes. Items from a failed attempt are+ discarded so a retry never duplicates them.+ """+ _validate_max_retries(max_retries)+ log = logger_instance or logger+ for attempt in range(max_retries + 1):+ items: list[T] = []+ has_content = False+ gen: Iterable[T] | None = None+ try:+ # ``fn()`` itself can fail: litellm's streaming ``completion()``+ # sends the request when called, so a 429/5xx/connection error+ # is raised here, before any chunk is yielded. That is a failed+ # request and must retry like the non-streaming path.+ gen = fn()+ for item in gen:+ items.append(item)+ has_content = has_content or is_content(item)+ return items+ except Exception as e:+ # Release the in-flight HTTP/socket resources before retrying.+ close = getattr(gen, "close", None)+ if callable(close):+ close()+ if has_content:+ raise+ delay = _get_retry_delay(+ e, attempt, max_retries, retry_predicate, description, log+ )+ if delay is None:+ raise+ time.sleep(delay)+ raise RuntimeError("unreachable") # for type-checker: loop always returns or raises++def is_retryable_error(error: Exception) -> bool:"""Check if a requests-library HTTP error should trigger a retry.diff --git a/unstract/sdk1/tests/test_anthropic_stream_complete.py b/unstract/sdk1/tests/test_anthropic_stream_complete.pynew file mode 100644index 000000000..6fa2234a1--- /dev/null+++ b/unstract/sdk1/tests/test_anthropic_stream_complete.py@@ -0,0 +1,352 @@+"""``LLM.complete()`` streams under the hood for direct Anthropic.++A non-streaming Anthropic request keeps the socket silent for the whole+generation. For long replies the response never arrives: the client's read+timeout fires (900 s / 1800 s observed in staging and production) and the+identical request was replayed by the retry helper. Anthropic's own SDK+refuses non-streaming requests expected to exceed 10 minutes for this reason.++These tests pin the contract: for the ``anthropic`` provider ``complete()``+issues ``stream=True``, collects the chunks and rebuilds the same response+shape callers already consume. Every provider streams unless its adapter+opts out.++LiteLLM's ``mock_response`` streaming path produces real ``ModelResponseStream``+chunks, so the rebuild goes through ``litellm.stream_chunk_builder`` for real.+"""++from __future__ import annotations++import logging+from functools import lru_cache+from importlib import import_module+from typing import TYPE_CHECKING+from unittest.mock import patch++import litellm+import pytest+from unstract.sdk1.utils import retry_utils++if TYPE_CHECKING:+ from collections.abc import Callable, Iterator++ANTHROPIC_ADAPTER_ID = "anthropic|90ebd4cd-2f19-4cef-a884-9eeb6ac0f203"+OPENAI_ADAPTER_ID = "openai|502ecf49-e47c-445c-9907-6d4b90c5cd17"+MOCK_TEXT = "hello streamed world"+# Bound before any test patches ``litellm.completion`` so the spy can delegate+# to the real mock path instead of recursing into itself.+_REAL_COMPLETION = litellm.completion+++@lru_cache(maxsize=1)+def _load_llm_module() -> object:+ import sys+ from types import ModuleType++ sys.modules.setdefault("magic", ModuleType("magic"))+ return import_module("unstract.sdk1.llm")+++def _make_llm(adapter_id: str, model: str, **metadata: object) -> object:+ llm_module = _load_llm_module()+ return llm_module.LLM(+ adapter_id=adapter_id,+ adapter_metadata={"model": model, "api_key": "test-key", **metadata},+ )+++def _mock_chunks(model: str) -> list[object]:+ """Real litellm stream chunks for MOCK_TEXT, produced without network."""+ return list(+ _REAL_COMPLETION(+ model=model,+ messages=[{"role": "user", "content": "hi"}],+ stream=True,+ mock_response=MOCK_TEXT,+ api_key="test-key",+ )+ )+++class _Spy:+ """Record litellm.completion kwargs and delegate to the mock path."""++ def __init__(self) -> None:+ self.calls: list[dict[str, object]] = []++ def __call__(self, **kwargs: object) -> object:+ self.calls.append(kwargs)+ return _REAL_COMPLETION(**{**kwargs, "mock_response": MOCK_TEXT})+++def no_cost() -> Iterator[None]:+ llm_module = _load_llm_module()+ with patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)):+ yield+++# ── Anthropic streams under the hood ─────────────────────────────────────────+++def test_anthropic_complete_streams_under_the_hood(no_cost: None) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")+ spy = _Spy()++ with patch.object(llm_module.litellm, "completion", spy):+ result = llm.complete("hi")++ assert len(spy.calls) == 1+ assert spy.calls[0]["stream"] is True+ assert result["response"].text == MOCK_TEXT+++def test_anthropic_rebuilt_response_keeps_consumer_facing_fields(+ no_cost: None,+) -> None:+ """Rebuilt raw response keeps finish_reason and usage.++ Callers read ``raw.choices[0].finish_reason`` and usage off the raw+ response (cloud line-item extractor, usage accounting).+ """+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")++ with patch.object(llm_module.litellm, "completion", _Spy()):+ result = llm.complete("hi")++ raw = result["response"].raw+ assert raw.choices[0].finish_reason == "stop"+ assert raw["choices"][0]["message"]["content"] == MOCK_TEXT+ assert raw.get("usage") is not None+ assert raw["usage"]["completion_tokens"] > 0+++def test_anthropic_stream_records_usage_once(+ no_cost: None, caplog: pytest.LogCaptureFixture+) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")++ with (+ patch.object(llm_module.litellm, "completion", _Spy()),+ caplog.at_level(logging.INFO, logger=llm_module.logger.name),+ ):+ llm.complete("hi")++ usage_logs = [r.message for r in caplog.records if "Usage:" in r.message]+ assert len(usage_logs) == 1+ assert "[complete]" in usage_logs[0]+++def test_adapter_opted_out_stays_non_streaming(no_cost: None) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(+ OPENAI_ADAPTER_ID,+ "gpt-4o",+ api_base="https://api.openai.com/v1",+ enable_streaming=False,+ )+ spy = _Spy()++ with patch.object(llm_module.litellm, "completion", spy):+ result = llm.complete("hi")++ assert len(spy.calls) == 1+ assert not spy.calls[0].get("stream")+ assert result["response"].text == MOCK_TEXT+++# ── Retry semantics on the streamed path ─────────────────────────────────────+++def _failing_then_ok(+ chunks: list[object], fail_after: int, error: Exception+) -> tuple[Callable[..., Iterator[object]], list[int]]:+ """Build a litellm.completion stand-in that fails once, then succeeds.++ The first call raises ``error`` after yielding ``fail_after`` chunks; the+ second call yields every chunk.+ """+ calls: list[int] = []++ def fake(**_: object) -> Iterator[object]:+ calls.append(1)+ if len(calls) == 1:+ yield from chunks[:fail_after]+ raise error+ yield from chunks++ return fake, calls+++def _timeout() -> Exception:+ return litellm.Timeout(+ message="Connection timed out after 900.0 seconds.",+ model="claude-sonnet-4-6",+ llm_provider="anthropic",+ )+++def test_stream_failure_before_any_content_is_retried(no_cost: None) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")+ fake, calls = _failing_then_ok(chunks, fail_after=0, error=_timeout())++ with (+ patch.object(llm_module.litellm, "completion", fake),+ patch.object(retry_utils.time, "sleep") as fake_sleep,+ ):+ result = llm.complete("hi")++ assert len(calls) == 2+ assert result["response"].text == MOCK_TEXT+ fake_sleep.assert_called_once()+++def test_request_failure_before_stream_exists_is_retried(no_cost: None) -> None:+ """Litellm raises from ``completion()`` itself on request errors.++ A rate limit or connection error happens before a stream object exists,+ so it must be retried like a non-streaming call, not escape on the first+ attempt.+ """+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")+ calls: list[int] = []++ def fake(**_: object) -> Iterator[object]:+ calls.append(1)+ if len(calls) == 1:+ raise litellm.RateLimitError(+ message="rate limited",+ model="claude-sonnet-4-6",+ llm_provider="anthropic",+ )+ return iter(chunks)++ with (+ patch.object(llm_module.litellm, "completion", fake),+ patch.object(retry_utils.time, "sleep") as fake_sleep,+ ):+ result = llm.complete("hi")++ assert len(calls) == 2+ assert result["response"].text == MOCK_TEXT+ fake_sleep.assert_called_once()+++def test_stream_failure_after_content_is_not_replayed(no_cost: None) -> None:+ """A drop after content started surfaces immediately.++ Replaying would re-run a generation that may already have run for minutes.+ """+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")+ fake, calls = _failing_then_ok(chunks, fail_after=2, error=_timeout())++ with (+ patch.object(llm_module.litellm, "completion", fake),+ pytest.raises(llm_module.LLMError) as exc_info,+ ):+ llm.complete("hi")++ assert len(calls) == 1+ assert "timed out" in str(exc_info.value)+++def test_stream_non_retryable_error_is_not_retried(no_cost: None) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", max_retries=2)+ chunks = _mock_chunks("anthropic/claude-sonnet-4-6")+ error = litellm.AuthenticationError(+ message="invalid x-api-key", model="claude-sonnet-4-6", llm_provider="anthropic"+ )+ fake, calls = _failing_then_ok(chunks, fail_after=0, error=error)++ with (+ patch.object(llm_module.litellm, "completion", fake),+ pytest.raises(llm_module.LLMError),+ ):+ llm.complete("hi")++ assert len(calls) == 1+++def test_empty_stream_raises_llm_error(no_cost: None) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")++ def empty(**_: object) -> Iterator[object]:+ yield from ()++ with (+ patch.object(llm_module.litellm, "completion", empty),+ pytest.raises(llm_module.LLMError),+ ):+ llm.complete("hi")+++# ── Per-adapter ``enable_streaming`` switch ──────────────────────────────────+++def _stream_flag_sent(llm: object) -> bool:+ llm_module = _load_llm_module()+ spy = _Spy()+ with patch.object(llm_module.litellm, "completion", spy):+ llm.complete("hi")+ return bool(spy.calls[0].get("stream"))+++def test_anthropic_adapter_can_opt_out_of_streaming(no_cost: None) -> None:+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", enable_streaming=False)+ assert _stream_flag_sent(llm) is False+++def test_openai_adapter_streams_by_default(no_cost: None) -> None:+ llm = _make_llm(OPENAI_ADAPTER_ID, "gpt-4o", api_base="https://api.openai.com/v1")+ assert _stream_flag_sent(llm) is True+++def test_missing_flag_streams_for_every_provider(no_cost: None) -> None:+ """Adapters stored before the field existed stream like new ones."""+ anthropic = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")+ openai = _make_llm(OPENAI_ADAPTER_ID, "gpt-4o", api_base="https://api.openai.com/v1")+ assert _stream_flag_sent(anthropic) is True+ assert _stream_flag_sent(openai) is True+++def test_streaming_flag_never_reaches_litellm(no_cost: None) -> None:+ llm_module = _load_llm_module()+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6", enable_streaming=True)+ spy = _Spy()+ with patch.object(llm_module.litellm, "completion", spy):+ llm.complete("hi")+ assert "enable_streaming" not in spy.calls[0]+++def test_mocked_completion_keeps_non_streaming_path(+ no_cost: None, monkeypatch: pytest.MonkeyPatch+) -> None:+ """The e2e rig counts LLM calls via litellm's fixed mock usage (10/20/30).++ That usage is only reported on the non-streaming mock path, and a mock+ never touches the network, so an injected mock response must bypass+ streaming even when the adapter streams.+ """+ llm_module = _load_llm_module()+ monkeypatch.setenv(llm_module._MOCK_RESPONSE_ENV, "canned answer")+ llm = _make_llm(ANTHROPIC_ADAPTER_ID, "claude-sonnet-4-6")+ spy = _Spy()++ with patch.object(llm_module.litellm, "completion", spy):+ result = llm.complete("hi")++ assert spy.calls[0]["mock_response"] == "canned answer"+ assert not spy.calls[0].get("stream")+ assert result["response"].raw["usage"]["prompt_tokens"] == 10+ assert result["response"].raw["usage"]["completion_tokens"] == 20diff --git a/unstract/sdk1/tests/test_llm_schema_streaming_flag.py b/unstract/sdk1/tests/test_llm_schema_streaming_flag.pynew file mode 100644index 000000000..2e1b85558--- /dev/null+++ b/unstract/sdk1/tests/test_llm_schema_streaming_flag.py@@ -0,0 +1,39 @@+"""Every LLM adapter form exposes the per-adapter ``enable_streaming`` switch.++The switch is read by ``LLM`` from the raw adapter metadata (see+``LLM._resolve_enable_streaming``). It defaults to on everywhere; the form+field exists so an endpoint that cannot stream can be opted out on its own+adapter.+"""++from __future__ import annotations++import json+from pathlib import Path++import pytest++STATIC_DIR = (+ Path(__file__).resolve().parents[1]+ / "src"+ / "unstract"+ / "sdk1"+ / "adapters"+ / "llm1"+ / "static"+)+SCHEMAS = sorted(STATIC_DIR.glob("*.json"))+++def test_static_dir_has_schemas() -> None:+ assert len(SCHEMAS) >= 14++[email protected]("schema_path", SCHEMAS, ids=[p.name for p in SCHEMAS])+def test_schema_exposes_enable_streaming(schema_path: Path) -> None:+ props = json.loads(schema_path.read_text())["properties"]+ field = props.get("enable_streaming")+ assert field is not None, f"{schema_path.name} lacks enable_streaming"+ assert field["type"] == "boolean"+ assert field["title"] == "Enable Streaming"+ assert field["default"] is Truediff --git a/unstract/sdk1/tests/utils/test_collect_with_retry.py b/unstract/sdk1/tests/utils/test_collect_with_retry.pynew file mode 100644index 000000000..974a6e6e0--- /dev/null+++ b/unstract/sdk1/tests/utils/test_collect_with_retry.py@@ -0,0 +1,152 @@+"""``collect_with_retry``: drain an iterable, retrying only before content.++Used by the streamed completion path. ``iter_with_retry`` stops retrying at+the first yielded item, but a provider stream emits bookkeeping chunks (e.g.+Anthropic ``message_start``) before any content. A failure at that point is+still a failed *request*, so it should be retried; a failure after content+started is a failed *generation* and must not be replayed.+"""++from __future__ import annotations++from typing import TYPE_CHECKING+from unittest.mock import patch++import pytest+from unstract.sdk1.utils import retry_utils+from unstract.sdk1.utils.retry_utils import collect_with_retry++if TYPE_CHECKING:+ from collections.abc import Callable, Iterator+++def _is_content(item: str) -> bool:+ return item.startswith("c")+++def _stream_factory(+ script: list[list[str | Exception]],+) -> tuple[Callable[[], Iterator[str]], list[int]]:+ """Each call replays the next script entry; an Exception entry is raised."""+ calls: list[int] = []++ def fn() -> Iterator[str]:+ idx = len(calls)+ calls.append(1)+ for item in script[idx]:+ if isinstance(item, Exception):+ raise item+ yield item++ return fn, calls+++def test_returns_all_items_when_stream_succeeds() -> None:+ fn, calls = _stream_factory([["meta", "c1", "c2"]])+ result = collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content+ )+ assert result == ["meta", "c1", "c2"]+ assert len(calls) == 1+++def test_retries_when_failure_precedes_content() -> None:+ fn, calls = _stream_factory([["meta", TimeoutError()], ["meta", "c1"]])+ with patch.object(retry_utils.time, "sleep"):+ result = collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content+ )+ # Bookkeeping items from the failed attempt are discarded, not duplicated.+ assert result == ["meta", "c1"]+ assert len(calls) == 2+++def test_retries_when_stream_creation_itself_raises() -> None:+ """``fn()`` failing before returning an iterable is a failed request.++ litellm's streaming ``completion()`` sends the HTTP request when called,+ so a 429/5xx/connection error surfaces from ``fn()`` itself rather than+ from iteration.+ """+ calls: list[int] = []++ def fn() -> Iterator[str]:+ calls.append(1)+ if len(calls) == 1:+ raise TimeoutError("request failed before any chunk")+ return iter(["meta", "c1"])++ with patch.object(retry_utils.time, "sleep"):+ result = collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content+ )+ assert result == ["meta", "c1"]+ assert len(calls) == 2+++def test_stream_creation_failure_that_is_not_retryable_is_raised() -> None:+ calls: list[int] = []++ def fn() -> Iterator[str]:+ calls.append(1)+ raise ValueError("bad request")++ with pytest.raises(ValueError):+ collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: False, is_content=_is_content+ )+ assert len(calls) == 1+++def test_does_not_retry_once_content_was_received() -> None:+ fn, calls = _stream_factory([["meta", "c1", TimeoutError()], ["meta", "c1"]])+ with pytest.raises(TimeoutError):+ collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content+ )+ assert len(calls) == 1+++def test_does_not_retry_non_retryable_error() -> None:+ fn, calls = _stream_factory([[ValueError("bad key")], ["c1"]])+ with pytest.raises(ValueError):+ collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: False, is_content=_is_content+ )+ assert len(calls) == 1+++def test_gives_up_after_max_retries() -> None:+ fn, calls = _stream_factory([[TimeoutError()], [TimeoutError()], [TimeoutError()]])+ with patch.object(retry_utils.time, "sleep"), pytest.raises(TimeoutError):+ collect_with_retry(+ fn, max_retries=2, retry_predicate=lambda _: True, is_content=_is_content+ )+ assert len(calls) == 3+++def test_closes_failed_generator_before_retrying() -> None:+ closed: list[bool] = []++ class _Gen:+ def __init__(self, fail: bool) -> None:+ self._fail = fail++ def __iter__(self) -> Iterator[str]:+ if self._fail:+ raise TimeoutError()+ yield "c1"++ def close(self) -> None:+ closed.append(True)++ attempts = iter([_Gen(fail=True), _Gen(fail=False)])+ with patch.object(retry_utils.time, "sleep"):+ result = collect_with_retry(+ lambda: next(attempts),+ max_retries=1,+ retry_predicate=lambda _: True,+ is_content=_is_content,+ )+ assert result == ["c1"]+ assert closed == [True]