LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3810 [FEAT] Prompt caching: SDK master switch + document-context caching (answer_prompt) (#2199)
Praveen Kumar90415861699a
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py b/unstract/sdk1/src/unstract/sdk1/adapters/base1.pyindex 366c404f3..f396bd618 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py@@ -1140,6 +1140,24 @@ class AWSBedrockLLMParameters(BaseChatCompletionParameters):result_metadata["thinking"] = thinking_configresult_metadata["temperature"] = 1+ # Prompt caching is opt-in and applied on the message payload (a+ # `cache_control` block on the stable system prompt), not as a LiteLLM+ # completion param, so it is excluded from Pydantic validation and+ # carried through on the validated dict for the LLM layer to read.+ # Only Anthropic/Claude models on Bedrock support prompt caching, so+ # don't advertise the flag for other Bedrock families (Titan, Llama,+ # etc.). The LLM layer enforces the same model gate; this just keeps the+ # validated metadata honest.+ # Check both ``model`` and ``model_id`` so callers routing through a+ # Bedrock Application Inference Profile (opaque ARN in ``model``, Claude+ # id in ``model_id``) still qualify.+ bedrock_model_ids = " ".join(+ str(result_metadata.get(field, "")) for field in _MODEL_ID_FIELDS+ ).lower()+ enable_prompt_caching = bool(+ adapter_metadata.get("enable_prompt_caching", False)+ ) and ("anthropic" in bedrock_model_ids or "claude" in bedrock_model_ids)+_pack_bedrock_guardrail_config(result_metadata)# Create validation metadata excluding control fields. `auth_type` is@@ -1156,6 +1174,7 @@ class AWSBedrockLLMParameters(BaseChatCompletionParameters):"guardrail_identifier","guardrail_version","guardrail_trace",+ "enable_prompt_caching",)}@@ -1174,6 +1193,7 @@ class AWSBedrockLLMParameters(BaseChatCompletionParameters):# lenient. Reads auth_type from result_metadata since validation_# metadata strips it before Pydantic.validated = _resolve_bedrock_aws_credentials(result_metadata, validated)+ validated["enable_prompt_caching"] = enable_prompt_cachingreturn _strip_deprecated_sampling_params(validated)@staticmethod@@ -1241,6 +1261,12 @@ class AnthropicLLMParameters(BaseChatCompletionParameters):result_metadata["thinking"] = thinking_configresult_metadata["temperature"] = 1+ # Prompt caching is opt-in and applied on the message payload (a+ # `cache_control` block on the stable system prompt), not as a LiteLLM+ # completion param, so it is excluded from Pydantic validation and+ # carried through on the validated dict for the LLM layer to read.+ enable_prompt_caching = bool(adapter_metadata.get("enable_prompt_caching", False))+# Create validation metadata excluding control fieldsexclude_fields = ("enable_thinking",@@ -1248,6 +1274,7 @@ class AnthropicLLMParameters(BaseChatCompletionParameters):"thinking","enable_extended_context","extra_headers",+ "enable_prompt_caching",)validation_metadata = {k: v for k, v in result_metadata.items() if k not in exclude_fields@@ -1263,6 +1290,8 @@ class AnthropicLLMParameters(BaseChatCompletionParameters):if enable_extended_context:validated["extra_headers"] = {"anthropic-beta": "context-1m-2025-08-07"}+ validated["enable_prompt_caching"] = enable_prompt_caching+return _strip_deprecated_sampling_params(validated)@staticmethoddiff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.pyindex b0a712b49..e78068518 100644--- a/unstract/sdk1/src/unstract/sdk1/llm.py+++ b/unstract/sdk1/src/unstract/sdk1/llm.py@@ -4,7 +4,7 @@ import refrom collections.abc import Callable, Generator, Mapping, Sequencefrom dataclasses import dataclass, fieldfrom enum import Enum-from functools import lru_cache+from functools import cache, lru_cachefrom typing import Any, NoReturn, castimport litellm@@ -32,6 +32,44 @@ from unstract.sdk1.utils.retry_utils import (logger = logging.getLogger(__name__)++# Truthy-looking values that people commonly set expecting a boolean flag to+# turn on — but only "true" enables caching. We warn (once each) so a stray+# ENABLE_PROMPT_CACHING=1 doesn't leave caching silently off.+_PROMPT_CACHING_TRUTHY_LOOKALIKES = frozenset(+ {"1", "yes", "y", "on", "t", "enable", "enabled"}+)+++@cache+def _warn_prompt_caching_lookalike(value: str) -> None:+ logger.warning(+ "ENABLE_PROMPT_CACHING=%r is not recognized as enabled; only 'true' "+ "(case-insensitive) turns prompt caching on — caching stays OFF.",+ value,+ )+++def is_prompt_caching_enabled() -> bool:+ """Whether LLM prompt caching is enabled platform-wide (opt-in, default off).++ A single master switch (``ENABLE_PROMPT_CACHING`` env var) that turns the+ caching capability on for every ``LLM`` on a supported provider, so callers+ don't each have to pass ``enable_prompt_caching``. Consumers still decide+ *what* to cache by passing ``cache_prefix``. Exposed for consumers that gate+ their own prompt-restructuring on the same flag.++ Only ``"true"`` (case-insensitive) enables it; a truthy-looking value like+ ``"1"``/``"yes"``/``"on"`` logs a one-time warning and stays off.+ """+ raw = os.environ.get("ENABLE_PROMPT_CACHING", "").strip().lower()+ if raw == "true":+ return True+ if raw in _PROMPT_CACHING_TRUTHY_LOOKALIKES:+ _warn_prompt_caching_lookalike(raw)+ return False++# Lets tests force a deterministic completion without a provider or a secret.# Unset in production, where this is a no-op._MOCK_RESPONSE_ENV = "UNSTRACT_LLM_MOCK_RESPONSE"@@ -187,6 +225,7 @@ class LLM:system_prompt: str = "",kwargs: dict[str, object] | None = None,capture_metrics: bool = False,+ enable_prompt_caching: bool = False,) -> None:"""Initialize the LLM interface.@@ -199,6 +238,9 @@ class LLM:system_prompt: System prompt for the LLMkwargs: Additional keyword arguments for configurationcapture_metrics: Whether to capture performance metrics+ enable_prompt_caching: Force provider prompt caching on for+ supported providers (Anthropic / Bedrock-Anthropic), regardless+ of the stored adapter metadata. Ignored for other providers."""if adapter_metadata is None:adapter_metadata = {}@@ -250,6 +292,16 @@ class LLM:self.kwargs = self.adapter.validate(self._adapter_metadata)self._cost_model = self.kwargs.pop("cost_model", None)self.kwargs.pop("context_window", None)+ # Opt-in provider prompt caching (Anthropic / Bedrock-Anthropic).+ # Enabled either via adapter metadata (from the stored adapter+ # config) or the explicit constructor arg (for callers that build+ # the LLM by ``adapter_instance_id`` and can't edit stored metadata).+ # Popped so it never reaches litellm; applied on the message payload.+ self._enable_prompt_caching = (+ bool(self.kwargs.pop("enable_prompt_caching", False))+ or enable_prompt_caching+ or is_prompt_caching_enabled()+ )# REF: https://docs.litellm.ai/docs/completion/input#translated-openai-params# supported = get_supported_openai_params(model=self.kwargs["model"],@@ -328,8 +380,129 @@ class LLM:actual_err=e,) from e+ # Providers for which we emit explicit ``cache_control`` blocks. Anthropic+ # and Bedrock-Anthropic support message-level prompt caching this way;+ # OpenAI / Azure auto-cache server-side (no marker needed) and other+ # providers don't support it, so we never tag their payloads.+ _PROMPT_CACHE_PROVIDERS = frozenset({"anthropic", "bedrock"})+ # Bedrock hosts many model families (Anthropic Claude, Amazon Titan/Nova,+ # Meta Llama, Cohere, Mistral, AI21). Only Anthropic/Claude models on+ # Bedrock honor ``cache_control``; emitting the blocks for other families+ # would be ineffective and could produce unsupported message shapes. These+ # substrings identify the cache-capable Bedrock models by their model id.+ _BEDROCK_CACHE_MODEL_MARKERS = ("anthropic", "claude")++ def _prompt_caching_active(self) -> bool:+ """Whether to emit ``cache_control`` blocks for this call."""+ if not self._enable_prompt_caching:+ return False+ provider = self.adapter.get_provider()+ if provider not in self._PROMPT_CACHE_PROVIDERS:+ return False+ if provider == "bedrock":+ # Gate on the underlying model, not just the provider: only+ # Anthropic/Claude models on Bedrock support cache_control. Check+ # both ``model`` and ``model_id`` — when a caller routes through a+ # Bedrock Application Inference Profile, the ARN in ``model`` is+ # opaque and the Claude id appears only in ``model_id``.+ recognized = any(+ marker in str(self.kwargs.get(field, "")).lower()+ for field in ("model", "model_id")+ for marker in self._BEDROCK_CACHE_MODEL_MARKERS+ )+ if not recognized:+ # Enabled but the model can't be confirmed as Anthropic/Claude+ # (e.g. a fully opaque Application Inference Profile ARN with no+ # Claude id in model/model_id). Skipping is safe; leave a+ # breadcrumb so operators can diagnose a Claude-on-Bedrock call+ # that unexpectedly isn't caching.+ logger.debug(+ "Prompt caching enabled but skipped for Bedrock: "+ "model=%r model_id=%r not recognized as Anthropic/Claude",+ self.kwargs.get("model"),+ self.kwargs.get("model_id"),+ )+ return recognized+ return True++ def is_prompt_caching_active(self) -> bool:+ """Public: whether ``cache_control`` blocks are emitted for this LLM.++ True only when caching is enabled (adapter flag, constructor arg, or the+ ``ENABLE_PROMPT_CACHING`` master switch) *and* the provider/model+ supports it. Callers use this to decide whether reordering a prompt into+ a cached prefix is worthwhile — reordering for a non-caching provider+ changes prompt structure with no benefit.+ """+ return self._prompt_caching_active()++ def _build_messages(+ self, prompt: str, cache_prefix: str | None = None+ ) -> list[dict[str, object]]:+ """Build the system + user message list for a chat completion.++ When prompt caching is active (opt-in flag + a supported provider), a+ stable prefix is tagged with ``cache_control`` so providers that support+ prefix caching (Anthropic, Bedrock-Anthropic) reuse it across calls.+ LiteLLM forwards ``cache_control`` blocks to the provider unchanged.++ - non-empty ``cache_prefix`` given: the user turn is split into a cached+ stable prefix block followed by the per-request volatile block. The+ text the model sees is ``cache_prefix + prompt`` — identical to+ passing the concatenation as a single prompt, so no prompt semantics+ change.+ - otherwise: the stable system prompt is cached.++ Only the stable portion is tagged; per-request content is never cached.+ An empty ``cache_prefix`` is treated as absent — Anthropic rejects empty+ text content blocks, and an empty prefix carries no caching benefit.+ """+ if self._prompt_caching_active() and cache_prefix:+ return [+ {"role": "system", "content": self._system_prompt},+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": cache_prefix,+ "cache_control": {"type": "ephemeral"},+ },+ {"type": "text", "text": prompt},+ ],+ },+ ]+ if self._prompt_caching_active():+ return [+ {+ "role": "system",+ "content": [+ {+ "type": "text",+ "text": self._system_prompt,+ "cache_control": {"type": "ephemeral"},+ }+ ],+ },+ {"role": "user", "content": prompt},+ ]+ # Caching inactive (opt-in off, or an unsupported provider): emit no+ # cache_control, but if the caller split the prompt into+ # (cache_prefix, prompt) the model must still see the full text —+ # concatenate rather than drop the prefix.+ user_content = cache_prefix + prompt if cache_prefix else prompt+ return [+ {"role": "system", "content": self._system_prompt},+ {"role": "user", "content": user_content},+ ]+@capture_metrics- def complete(self, prompt: str, **kwargs: object) -> dict[str, object]:+ def complete(+ self,+ prompt: str,+ cache_prefix: str | None = None,+ **kwargs: object,+ ) -> dict[str, object]:"""Return a standard chat completion dict with optional metrics capture.Return a standard chat completion dict and optionally captures metrics if run@@ -337,6 +510,11 @@ class LLM:Args:prompt (str) The input text prompt for generating the completion.+ cache_prefix (str | None) Stable text to cache ahead of ``prompt``.+ When prompt caching is active for a supported provider, this is+ emitted as a ``cache_control`` block so repeated calls sharing+ the same prefix reuse it. The model sees ``cache_prefix + prompt``+ unchanged. Ignored when caching is off or unsupported.**kwargs (Any) Additional arguments passed to the completion function.Returns:@@ -344,10 +522,7 @@ class LLM:any processed output, and the captured metrics (if applicable)."""try:- messages: list[dict[str, str]] = [- {"role": "system", "content": self._system_prompt},- {"role": "user", "content": prompt},- ]+ messages = self._build_messages(prompt, cache_prefix=cache_prefix)logger.debug(f"[sdk1][LLM]Invoking {self.adapter.get_provider()} completion API")@@ -355,6 +530,7 @@ class LLM:completion_kwargs = self.adapter.validate({**self.kwargs, **kwargs})_inject_mock_response(completion_kwargs)completion_kwargs.pop("cost_model", None)+ completion_kwargs.pop("enable_prompt_caching", None)completion_kwargs.pop("context_window", None)# if hasattr(self, "model") and self.model not in O1_MODELS:@@ -479,6 +655,7 @@ class LLM:completion_kwargs = self.adapter.validate({**self.kwargs, **kwargs})_inject_mock_response(completion_kwargs)completion_kwargs.pop("cost_model", None)+ completion_kwargs.pop("enable_prompt_caching", None)completion_kwargs.pop("context_window", None)response: dict[str, object] = litellm.completion(@@ -530,17 +707,17 @@ class LLM:self,prompt: str,callback_manager: object | None = None,+ cache_prefix: str | None = None,**kwargs: object,) -> Generator[LLMResponseCompat, None, None]:"""Yield LLMResponseCompat objects with text chunks.- Chunks arrive as they stream from the provider.+ Chunks arrive as they stream from the provider. ``cache_prefix`` behaves+ as in :meth:`complete` — a stable prefix cached ahead of ``prompt`` when+ prompt caching is active for a supported provider."""try:- messages = [- {"role": "system", "content": self._system_prompt},- {"role": "user", "content": prompt},- ]+ messages = self._build_messages(prompt, cache_prefix=cache_prefix)logger.debug(f"[sdk1][LLM]Invoking {self.adapter.get_provider()} stream completion API")@@ -548,6 +725,7 @@ class LLM:completion_kwargs = self.adapter.validate({**self.kwargs, **kwargs})_inject_mock_response(completion_kwargs)completion_kwargs.pop("cost_model", None)+ completion_kwargs.pop("enable_prompt_caching", None)completion_kwargs.pop("context_window", None)max_retries = pop_litellm_retry_kwargs(@@ -607,13 +785,20 @@ class LLM:message=error_msg, status_code=status_code, actual_err=e) from e- async def acomplete(self, prompt: str, **kwargs: object) -> dict[str, object]:- """Asynchronous chat completion (wrapper around ``litellm.acompletion``)."""+ async def acomplete(+ self,+ prompt: str,+ cache_prefix: str | None = None,+ **kwargs: object,+ ) -> dict[str, object]:+ """Asynchronous chat completion (wrapper around ``litellm.acompletion``).++ ``cache_prefix`` mirrors :meth:`complete` / :meth:`stream_complete`: when+ prompt caching is active it is emitted as a cached stable prefix ahead of+ ``prompt``; otherwise it is concatenated so the model sees the same text.+ """try:- messages = [- {"role": "system", "content": self._system_prompt},- {"role": "user", "content": prompt},- ]+ messages = self._build_messages(prompt, cache_prefix=cache_prefix)logger.debug(f"[sdk1][LLM]Invoking {self.adapter.get_provider()} async completion API")@@ -621,6 +806,7 @@ class LLM:completion_kwargs = self.adapter.validate({**self.kwargs, **kwargs})_inject_mock_response(completion_kwargs)completion_kwargs.pop("cost_model", None)+ completion_kwargs.pop("enable_prompt_caching", None)completion_kwargs.pop("context_window", None)max_retries = pop_litellm_retry_kwargs(@@ -767,6 +953,57 @@ class LLM:self._pending_usage = []return records+ def _compute_call_cost(+ self,+ model: str,+ prompt_tokens: int,+ completion_tokens: int,+ has_cache_tokens: bool,+ response: object | None,+ ) -> float:+ """Compute the dollar cost of a single call.++ When caching is active, cache-read tokens are billed at ~0.1x and+ cache-write at ~1.25x of the base input rate. ``litellm.cost_per_token``+ prices every prompt token at the full input rate, so it over-reports+ cost on cache hits. In that case let litellm read the cache token counts+ off the response for an accurate figure, falling back to the per-token+ path (which is exact when no caching is involved).+ """+ if has_cache_tokens and response is not None:+ try:+ # Pass ``model`` so cached calls price against the same model as+ # the ``cost_per_token`` fallback below. Without it,+ # ``completion_cost`` derives the model from the response and+ # ignores any ``cost_model`` override.+ return litellm.completion_cost(completion_response=response, model=model)+ except Exception:+ # Warn (not debug): the cost_per_token fallback prices every+ # prompt token at the full input rate, so it over-reports cost+ # by up to ~10x on cache hits. Operators watching spend need to+ # see that a recorded cost may be inflated.+ logger.warning(+ "completion_cost() failed for model=%s; falling back to "+ "cost_per_token — recorded cost may be OVER-reported for "+ "this cached call",+ model,+ exc_info=True,+ )+ try:+ prompt_cost, compl_cost = litellm.cost_per_token(+ model=model,+ prompt_tokens=prompt_tokens,+ completion_tokens=completion_tokens,+ )+ return prompt_cost + compl_cost+ except Exception:+ logger.warning(+ "Failed to compute cost for model=%s; recording as 0.0",+ model,+ exc_info=True,+ )+ return 0.0+def _record_usage(self,model: str,@@ -779,6 +1016,10 @@ class LLM:prompt_tokens = usage_data.get("prompt_tokens", 0)completion_tokens = usage_data.get("completion_tokens", 0)total_tokens = usage_data.get("total_tokens", 0)+ # Prompt-caching token counts (populated by Anthropic / Bedrock-Anthropic+ # when caching is enabled; 0 for every other provider/call).+ cache_creation_tokens = usage_data.get("cache_creation_input_tokens", 0) or 0+ cache_read_tokens = usage_data.get("cache_read_input_tokens", 0) or 0# Fall back to litellm when providers omit prompt tokens — avoids 0-token billing.if prompt_tokens == 0 and messages:@@ -803,30 +1044,29 @@ class LLM:id_suffix += f" response_id={response_id}"if request_id is not None:id_suffix += f" request_id={request_id}"+ cache_suffix = ""+ if cache_creation_tokens or cache_read_tokens:+ cache_suffix = (+ f" cache_write={cache_creation_tokens} cache_read={cache_read_tokens}"+ )logger.info(- "[sdk1][LLM][%s][%s] Usage: prompt=%d completion=%d total=%d%s",+ "[sdk1][LLM][%s][%s] Usage: prompt=%d completion=%d total=%d%s%s",model,llm_api,prompt_tokens,completion_tokens,total_tokens,+ cache_suffix,id_suffix,)- try:- prompt_cost, compl_cost = litellm.cost_per_token(- model=model,- prompt_tokens=prompt_tokens,- completion_tokens=completion_tokens,- )- cost = prompt_cost + compl_cost- except Exception:- logger.warning(- "Failed to compute cost for model=%s; recording as 0.0",- model,- exc_info=True,- )- cost = 0.0+ cost = self._compute_call_cost(+ model=model,+ prompt_tokens=prompt_tokens,+ completion_tokens=completion_tokens,+ has_cache_tokens=bool(cache_creation_tokens or cache_read_tokens),+ response=response,+ )# Trailing segment matches legacy Audit semantics (e.g. bedrock/anthropic/claude).display_model = model.rsplit("/", 1)[-1] if model else modeldiff --git a/unstract/sdk1/tests/test_prompt_caching.py b/unstract/sdk1/tests/test_prompt_caching.pynew file mode 100644index 000000000..22912525e--- /dev/null+++ b/unstract/sdk1/tests/test_prompt_caching.py@@ -0,0 +1,551 @@+"""Tests for opt-in provider prompt caching.++Covers the two halves of the feature:++1. Adapter ``validate()`` (Anthropic + Bedrock-Anthropic) carries the+ ``enable_prompt_caching`` control flag through on the validated dict, but+ never leaks it into the LiteLLM completion kwargs Pydantic validates.+2. ``LLM._build_messages()`` tags only the stable system prompt with+ ``cache_control`` when caching is enabled, and leaves the string form+ untouched otherwise.+"""++from typing import Any++import pytest+from unstract.sdk1.adapters.base1 import (+ AnthropicLLMParameters,+ AWSBedrockLLMParameters,+)++# ── validate(): flag carried through, absent by default ─────────────────────++VALIDATE_CASES = [+ ("anthropic", AnthropicLLMParameters, "claude-opus-4-8", {"api_key": "k"}),+ (+ "bedrock",+ AWSBedrockLLMParameters,+ "anthropic.claude-opus-4-8-20260101-v1:0",+ {"aws_region_name": "us-east-1"},+ ),+]+++ "name,cls,model,extra", VALIDATE_CASES, ids=[c[0] for c in VALIDATE_CASES]+)+def test_validate_defaults_prompt_caching_off(+ name: str, cls: type, model: str, extra: dict[str, Any]+) -> None:+ result = cls.validate({"model": model, **extra})+ assert result["enable_prompt_caching"] is False+++ "name,cls,model,extra", VALIDATE_CASES, ids=[c[0] for c in VALIDATE_CASES]+)+def test_validate_carries_prompt_caching_flag(+ name: str, cls: type, model: str, extra: dict[str, Any]+) -> None:+ result = cls.validate({"model": model, "enable_prompt_caching": True, **extra})+ assert result["enable_prompt_caching"] is True+++ "name,cls,model,extra", VALIDATE_CASES, ids=[c[0] for c in VALIDATE_CASES]+)+def test_validate_is_idempotent_on_prompt_caching(+ name: str, cls: type, model: str, extra: dict[str, Any]+) -> None:+ """Re-validating a validated dict must preserve the flag (round-trip)."""+ once = cls.validate({"model": model, "enable_prompt_caching": True, **extra})+ twice = cls.validate({**once})+ assert twice["enable_prompt_caching"] is True+++# ── _build_messages(): cache_control only on the system prefix ──────────────+++class _StubAdapter:+ def __init__(self, provider: str) -> None:+ self._provider = provider++ def get_provider(self) -> str:+ return self._provider+++class _StubLLM:+ """Bind the real caching helpers to a stub carrying just the state they read."""++ from unstract.sdk1.llm import LLM++ _build_messages = LLM._build_messages+ _prompt_caching_active = LLM._prompt_caching_active+ is_prompt_caching_active = LLM.is_prompt_caching_active+ _PROMPT_CACHE_PROVIDERS = LLM._PROMPT_CACHE_PROVIDERS+ _BEDROCK_CACHE_MODEL_MARKERS = LLM._BEDROCK_CACHE_MODEL_MARKERS++ def __init__(+ self,+ system_prompt: str,+ enable_prompt_caching: bool,+ provider: str = "anthropic",+ model: str = "claude-opus-4-8",+ model_id: str | None = None,+ ) -> None:+ self._system_prompt = system_prompt+ self._enable_prompt_caching = enable_prompt_caching+ self.adapter = _StubAdapter(provider)+ # Only read by the Bedrock model gate; harmless for other providers.+ self.kwargs = {"model": model}+ if model_id is not None:+ self.kwargs["model_id"] = model_id+++def test_build_messages_plain_string_when_caching_off() -> None:+ llm = _StubLLM("SYSTEM", enable_prompt_caching=False)+ messages = llm._build_messages("USER")+ assert messages == [+ {"role": "system", "content": "SYSTEM"},+ {"role": "user", "content": "USER"},+ ]+++def test_build_messages_tags_only_system_when_caching_on() -> None:+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True)+ messages = llm._build_messages("USER")++ system, user = messages+ assert system["role"] == "system"+ assert system["content"] == [+ {+ "type": "text",+ "text": "SYSTEM",+ "cache_control": {"type": "ephemeral"},+ }+ ]+ # The per-request user prompt is never tagged for caching.+ assert user == {"role": "user", "content": "USER"}++[email protected]("provider", ["openai", "azure", "gemini", "mistral"])+def test_build_messages_not_tagged_for_unsupported_provider(provider: str) -> None:+ """Caching flag on, but a provider we don't emit cache_control for -> plain."""+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True, provider=provider)+ assert llm._build_messages("USER") == [+ {"role": "system", "content": "SYSTEM"},+ {"role": "user", "content": "USER"},+ ]+++def test_build_messages_cache_prefix_splits_user_turn() -> None:+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True)+ messages = llm._build_messages("VOLATILE", cache_prefix="STABLE")++ system, user = messages+ # System stays a plain string; the stable prefix is cached in the user turn.+ assert system == {"role": "system", "content": "SYSTEM"}+ assert user["role"] == "user"+ assert user["content"] == [+ {"type": "text", "text": "STABLE", "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": "VOLATILE"},+ ]+ # Text-equivalence invariant: the model sees prefix + prompt, unchanged.+ seen = "".join(block["text"] for block in user["content"])+ assert seen == "STABLE" + "VOLATILE"+++def test_build_messages_empty_cache_prefix_treated_as_absent() -> None:+ """An empty cache_prefix must NOT create an empty (Anthropic-rejected) block.++ Caching is active, but ``cache_prefix=""`` should fall back to the plain+ single-block user turn rather than emitting ``{"type": "text", "text": ""}``.+ """+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True)+ assert llm._build_messages("VOLATILE", cache_prefix="") == [+ {+ "role": "system",+ "content": [+ {+ "type": "text",+ "text": "SYSTEM",+ "cache_control": {"type": "ephemeral"},+ }+ ],+ },+ {"role": "user", "content": "VOLATILE"},+ ]+++def test_build_messages_cache_prefix_preserved_when_caching_off() -> None:+ """Caching off must NOT drop the prefix — the model still sees prefix+prompt.++ Regression guard: a consumer that splits its prompt into (cache_prefix,+ prompt) must produce the full text even on an unsupported provider / with+ caching disabled, otherwise the prefix (e.g. instructions/context) is lost.+ """+ llm = _StubLLM("SYSTEM", enable_prompt_caching=False)+ assert llm._build_messages("VOLATILE", cache_prefix="STABLE") == [+ {"role": "system", "content": "SYSTEM"},+ {"role": "user", "content": "STABLE" + "VOLATILE"},+ ]+++def test_build_messages_cache_prefix_preserved_for_unsupported_provider() -> None:+ """Flag on, but a provider we don't emit cache_control for -> full prompt, no tag."""+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True, provider="openai")+ assert llm._build_messages("VOLATILE", cache_prefix="STABLE") == [+ {"role": "system", "content": "SYSTEM"},+ {"role": "user", "content": "STABLE" + "VOLATILE"},+ ]+++# ── Bedrock model gate: cache_control only for Anthropic/Claude on Bedrock ───++# Bedrock hosts many families; only Anthropic/Claude honor cache_control.+_BEDROCK_ANTHROPIC_MODELS = [+ "bedrock/anthropic.claude-opus-4-8-20260101-v1:0",+ "bedrock/us.anthropic.claude-sonnet-4-6-20250101-v1:0",+ "anthropic.claude-3-5-sonnet-20241022-v2:0",+]+_BEDROCK_NON_ANTHROPIC_MODELS = [+ "bedrock/amazon.titan-text-premier-v1:0",+ "bedrock/meta.llama3-70b-instruct-v1:0",+ "bedrock/cohere.command-r-plus-v1:0",+ "bedrock/mistral.mistral-large-2407-v1:0",+]++[email protected]("model", _BEDROCK_ANTHROPIC_MODELS)+def test_bedrock_anthropic_model_caches(model: str) -> None:+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True, provider="bedrock", model=model)+ assert llm._prompt_caching_active() is True+ # cache_control block is emitted on the split user turn.+ user = llm._build_messages("VOLATILE", cache_prefix="STABLE")[1]+ assert user["content"][0]["cache_control"] == {"type": "ephemeral"}++[email protected]("model", _BEDROCK_NON_ANTHROPIC_MODELS)+def test_bedrock_non_anthropic_model_does_not_cache(model: str) -> None:+ """Titan/Llama/Cohere/Mistral on Bedrock must not get cache_control blocks."""+ llm = _StubLLM("SYSTEM", enable_prompt_caching=True, provider="bedrock", model=model)+ assert llm._prompt_caching_active() is False+ # Falls back to the plain string form; prefix still preserved as full text.+ assert llm._build_messages("VOLATILE", cache_prefix="STABLE") == [+ {"role": "system", "content": "SYSTEM"},+ {"role": "user", "content": "STABLE" + "VOLATILE"},+ ]+++def test_bedrock_opaque_inference_profile_uses_model_id() -> None:+ """AIP ARN in ``model`` is opaque; the Claude id in ``model_id`` still caches."""+ llm = _StubLLM(+ "SYSTEM",+ enable_prompt_caching=True,+ provider="bedrock",+ model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcd1234",+ model_id="anthropic.claude-opus-4-8-20260101-v1:0",+ )+ assert llm._prompt_caching_active() is True+++def test_bedrock_opaque_profile_without_claude_id_does_not_cache() -> None:+ """No Anthropic id in either field -> can't confirm Claude -> no caching."""+ llm = _StubLLM(+ "SYSTEM",+ enable_prompt_caching=True,+ provider="bedrock",+ model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcd1234",+ )+ assert llm._prompt_caching_active() is False+++def test_is_prompt_caching_active_matches_private() -> None:+ """The public probe mirrors the internal gate for callers (e.g. answer_prompt)."""+ on = _StubLLM("SYSTEM", enable_prompt_caching=True, provider="anthropic")+ assert on.is_prompt_caching_active() is True+ off_provider = _StubLLM("SYSTEM", enable_prompt_caching=True, provider="openai")+ assert off_provider.is_prompt_caching_active() is False+ off_bedrock = _StubLLM(+ "SYSTEM",+ enable_prompt_caching=True,+ provider="bedrock",+ model="bedrock/amazon.titan-text-premier-v1:0",+ )+ assert off_bedrock.is_prompt_caching_active() is False+++def test_validate_bedrock_non_anthropic_never_enables_caching() -> None:+ """base1 keeps the validated flag honest: non-Anthropic Bedrock -> False."""+ result = AWSBedrockLLMParameters.validate(+ {+ "model": "amazon.titan-text-premier-v1:0",+ "enable_prompt_caching": True,+ "aws_region_name": "us-east-1",+ }+ )+ assert result["enable_prompt_caching"] is False+++# ── cost accounting: cached calls price against the cost_model override ──────+++def test_cost_override_passed_to_completion_cost_on_cached_call(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Cached calls price against the ``cost_model`` override.++ ``completion_cost`` must receive ``model=`` so cached and non-cached calls+ price against the same model, matching the ``cost_per_token`` fallback path.+ """+ import types++ import unstract.sdk1.llm as llm_mod+ from unstract.sdk1.llm import LLM++ captured: dict[str, Any] = {}++ def _fake_completion_cost(**kwargs: object) -> float:+ captured.update(kwargs)+ return 0.42++ monkeypatch.setattr(llm_mod.litellm, "completion_cost", _fake_completion_cost)++ # Minimal stub instance for ``self`` so a future ``self.`` reference fails+ # loudly instead of silently working (unlike a bare ``object()``).+ cost = LLM._compute_call_cost(+ types.SimpleNamespace(),+ model="anthropic/claude-opus-4-8",+ prompt_tokens=100,+ completion_tokens=10,+ has_cache_tokens=True,+ response={"id": "resp"},+ )+ assert cost == 0.42+ assert captured.get("model") == "anthropic/claude-opus-4-8"+ assert captured.get("completion_response") == {"id": "resp"}+++def test_compute_cost_falls_back_when_completion_cost_raises(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """completion_cost failure on a cached call falls back to cost_per_token."""+ import types++ import unstract.sdk1.llm as llm_mod+ from unstract.sdk1.llm import LLM++ def _boom(**kwargs: object) -> float:+ raise RuntimeError("no price map")++ monkeypatch.setattr(llm_mod.litellm, "completion_cost", _boom)+ monkeypatch.setattr(llm_mod.litellm, "cost_per_token", lambda **k: (0.1, 0.2))++ cost = LLM._compute_call_cost(+ types.SimpleNamespace(),+ model="m",+ prompt_tokens=100,+ completion_tokens=10,+ has_cache_tokens=True,+ response={"id": "r"},+ )+ assert cost == pytest.approx(0.3)+++def test_compute_cost_returns_zero_when_both_paths_fail(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """If both completion_cost and cost_per_token fail, cost is recorded as 0.0."""+ import types++ import unstract.sdk1.llm as llm_mod+ from unstract.sdk1.llm import LLM++ def _boom(**kwargs: object) -> float:+ raise RuntimeError("nope")++ monkeypatch.setattr(llm_mod.litellm, "completion_cost", _boom)+ monkeypatch.setattr(llm_mod.litellm, "cost_per_token", _boom)++ cost = LLM._compute_call_cost(+ types.SimpleNamespace(),+ model="m",+ prompt_tokens=1,+ completion_tokens=1,+ has_cache_tokens=True,+ response={"id": "r"},+ )+ assert cost == 0.0+++def test_compute_cost_skips_completion_cost_when_response_is_none(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """has_cache_tokens True but no response -> skip completion_cost, use per-token."""+ import types++ import unstract.sdk1.llm as llm_mod+ from unstract.sdk1.llm import LLM++ called = {"completion_cost": False}++ def _cc(**kwargs: object) -> float:+ called["completion_cost"] = True+ return 9.9++ monkeypatch.setattr(llm_mod.litellm, "completion_cost", _cc)+ monkeypatch.setattr(llm_mod.litellm, "cost_per_token", lambda **k: (0.4, 0.6))++ cost = LLM._compute_call_cost(+ types.SimpleNamespace(),+ model="m",+ prompt_tokens=1,+ completion_tokens=1,+ has_cache_tokens=True,+ response=None,+ )+ assert called["completion_cost"] is False+ assert cost == pytest.approx(1.0)+++# ── control flags never leak into the litellm completion payload ─────────────+++def test_control_flags_not_forwarded_to_litellm_completion(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """enable_prompt_caching / cost_model / context_window never reach litellm.++ They are applied on our side (cache_control blocks, cost accounting) and must+ be popped before ``litellm.completion``; otherwise litellm's Pydantic+ validation rejects the unknown params.+ """+ import unstract.sdk1.llm as llm_mod+ from unstract.sdk1.exceptions import LLMError+ from unstract.sdk1.llm import LLM++ captured: dict[str, Any] = {}++ def _capture_completion(**kwargs: object) -> dict[str, object]:+ captured.update(kwargs)+ raise RuntimeError("stop after capture")++ monkeypatch.setattr(llm_mod.litellm, "completion", _capture_completion)+ monkeypatch.delenv("ENABLE_PROMPT_CACHING", raising=False)++ meta = {+ "model": "claude-opus-4-8",+ "api_key": "sk-test",+ "enable_prompt_caching": True,+ }+ llm = LLM(adapter_id=_ANTHROPIC_ADAPTER_ID, adapter_metadata=meta, system_prompt="S")+ with pytest.raises(LLMError):+ llm.complete("hi")++ assert captured.get("messages"), "payload should have been built and forwarded"+ assert "enable_prompt_caching" not in captured+ assert "cost_model" not in captured+ assert "context_window" not in captured+++def test_control_flags_not_forwarded_to_litellm_acompletion(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Same strip guarantee on the async surface (regression for acomplete)."""+ import asyncio++ import unstract.sdk1.llm as llm_mod+ from unstract.sdk1.exceptions import LLMError+ from unstract.sdk1.llm import LLM++ captured: dict[str, Any] = {}++ async def _capture_acompletion(**kwargs: object) -> dict[str, object]:+ captured.update(kwargs)+ raise RuntimeError("stop after capture")++ monkeypatch.setattr(llm_mod.litellm, "acompletion", _capture_acompletion)+ monkeypatch.delenv("ENABLE_PROMPT_CACHING", raising=False)++ meta = {+ "model": "claude-opus-4-8",+ "api_key": "sk-test",+ "enable_prompt_caching": True,+ }+ llm = LLM(adapter_id=_ANTHROPIC_ADAPTER_ID, adapter_metadata=meta, system_prompt="S")+ with pytest.raises(LLMError):+ asyncio.run(llm.acomplete("hi", cache_prefix="STABLE"))++ assert captured.get("messages"), "async payload should have been forwarded"+ assert "enable_prompt_caching" not in captured+ assert "cost_model" not in captured+++# ── constructor / env opt-in (real LLM, no flag in adapter metadata) ────────++_ANTHROPIC_ADAPTER_ID = "anthropic|90ebd4cd-2f19-4cef-a884-9eeb6ac0f203"+++def test_constructor_flag_forces_caching_without_metadata(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A caller that builds by adapter without the stored flag can still opt in."""+ monkeypatch.delenv("ENABLE_PROMPT_CACHING", raising=False)+ from unstract.sdk1.llm import LLM++ meta = {"model": "claude-opus-4-8", "api_key": "sk-test"}+ llm = LLM(+ adapter_id=_ANTHROPIC_ADAPTER_ID,+ adapter_metadata=meta,+ system_prompt="S",+ enable_prompt_caching=True,+ )+ assert llm._enable_prompt_caching is True+ # cache_prefix path produces the split user turn end to end.+ messages = llm._build_messages("VOLATILE", cache_prefix="STABLE")+ assert messages[1]["content"][0]["cache_control"] == {"type": "ephemeral"}+++def test_constructor_flag_defaults_off(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("ENABLE_PROMPT_CACHING", raising=False)+ from unstract.sdk1.llm import LLM++ meta = {"model": "claude-opus-4-8", "api_key": "sk-test"}+ llm = LLM(adapter_id=_ANTHROPIC_ADAPTER_ID, adapter_metadata=meta, system_prompt="S")+ assert llm._enable_prompt_caching is False+++def test_env_var_enables_caching_platform_wide(monkeypatch: pytest.MonkeyPatch) -> None:+ """The ENABLE_PROMPT_CACHING master switch turns caching on with no per-call flag."""+ monkeypatch.setenv("ENABLE_PROMPT_CACHING", "true")+ from unstract.sdk1.llm import LLM, is_prompt_caching_enabled++ assert is_prompt_caching_enabled() is True+ meta = {"model": "claude-opus-4-8", "api_key": "sk-test"}+ llm = LLM(adapter_id=_ANTHROPIC_ADAPTER_ID, adapter_metadata=meta, system_prompt="S")+ assert llm._enable_prompt_caching is True+++def test_env_var_true_is_case_insensitive(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("ENABLE_PROMPT_CACHING", "TRUE")+ from unstract.sdk1.llm import is_prompt_caching_enabled++ assert is_prompt_caching_enabled() is True+++def test_env_var_truthy_lookalike_stays_off_and_warns(+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:+ """A value like '1'/'yes'/'on' does NOT enable caching, but warns once."""+ import logging++ from unstract.sdk1.llm import (+ _warn_prompt_caching_lookalike,+ is_prompt_caching_enabled,+ )++ _warn_prompt_caching_lookalike.cache_clear()+ monkeypatch.setenv("ENABLE_PROMPT_CACHING", "1")+ with caplog.at_level(logging.WARNING):+ assert is_prompt_caching_enabled() is False+ assert any("not recognized as enabled" in r.message for r in caplog.records)diff --git a/workers/executor/executors/answer_prompt.py b/workers/executor/executors/answer_prompt.pyindex d1eef5b3b..4184b78b7 100644--- a/workers/executor/executors/answer_prompt.py+++ b/workers/executor/executors/answer_prompt.py@@ -104,6 +104,41 @@ class AnswerPromptService:)return promptx+ @staticmethod+ def _llm_caches_prompts(llm: Any) -> bool:+ """Whether reordering into a cached prefix pays off for this LLM.++ True only when the LLM reports prompt caching is active for it — caching+ enabled (adapter flag / constructor arg / ``ENABLE_PROMPT_CACHING``+ master switch) AND a supported provider/model. LLM objects that don't+ expose the capability default to False, so their prompt order is left+ unchanged.+ """+ checker = getattr(llm, "is_prompt_caching_active", None)+ if not callable(checker):+ return False+ # Fail closed: if the probe raises, keep the original (context-last)+ # prompt order rather than risk a reorder for an LLM whose caching+ # support is unknown. Hence the deliberately broad excepts below.+ try:+ return bool(checker())+ except Exception: # noqa: BLE001 - fail closed, see comment above+ provider = getattr(getattr(llm, "adapter", None), "get_provider", None)+ provider_name = None+ if callable(provider):+ try:+ provider_name = provider()+ except Exception: # noqa: BLE001 - provider is best-effort log context+ provider_name = None+ logger.warning(+ "Failed to determine prompt-caching support for LLM "+ "(type=%s, provider=%s); using original prompt order",+ type(llm).__name__,+ provider_name,+ exc_info=True,+ )+ return False+@staticmethoddef construct_and_run_prompt(tool_settings: dict[str, Any],@@ -148,20 +183,36 @@ class AnswerPromptService:if not enable_word_confidence or summarize_as_source:word_confidence_postamble = ""- prompt = AnswerPromptService.construct_prompt(- preamble=tool_settings.get(PSKeys.PREAMBLE, ""),- prompt=output[prompt],- postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),- grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),- context=context,- platform_postamble=platform_postamble,- word_confidence_postamble=word_confidence_postamble,- prompt_type=prompt_type,- )- output[PSKeys.COMBINED_PROMPT] = prompt+ prompt_args = {+ "preamble": tool_settings.get(PSKeys.PREAMBLE, ""),+ "prompt": output[prompt],+ "postamble": tool_settings.get(PSKeys.POSTAMBLE, ""),+ "grammar_list": tool_settings.get(PSKeys.GRAMMAR, []),+ "context": context,+ "platform_postamble": platform_postamble,+ "word_confidence_postamble": word_confidence_postamble,+ "prompt_type": prompt_type,+ }+ cache_prefix: str | None = None+ # Only reorder into a cached prefix when this LLM actually caches+ # (caching enabled AND a supported provider/model). Reordering for an+ # unsupported provider would change the prompt structure — moving the+ # document context ahead of the instructions — with no caching benefit,+ # so those providers keep the original prompt order.+ if AnswerPromptService._llm_caches_prompts(llm):+ # Reorder so the reused document context becomes a cached prefix.+ # The text the model sees is ``cache_prefix + prompt_str``.+ cache_prefix, prompt_str = AnswerPromptService.construct_cached_prompt(+ **prompt_args+ )+ output[PSKeys.COMBINED_PROMPT] = cache_prefix + prompt_str+ else:+ prompt_str = AnswerPromptService.construct_prompt(**prompt_args)+ output[PSKeys.COMBINED_PROMPT] = prompt_strreturn AnswerPromptService.run_completion(llm=llm,- prompt=prompt,+ prompt=prompt_str,+ cache_prefix=cache_prefix,metadata=metadata,prompt_key=output[PSKeys.NAME],prompt_type=prompt_type,@@ -189,6 +240,31 @@ class AnswerPromptService:)return notes+ @staticmethod+ def _prepare_postambles(+ postamble: str,+ platform_postamble: str,+ word_confidence_postamble: str,+ prompt_type: str,+ ) -> tuple[str, str]:+ """Apply JSON + platform/word-confidence formatting to the postambles.++ Shared by :meth:`construct_prompt` and :meth:`construct_cached_prompt` so+ the cached and non-cached prompts stay byte-identical apart from the+ context/question ordering. Returns the ``(postamble, platform_postamble)``+ pair to interpolate.+ """+ if prompt_type == PSKeys.JSON:+ json_postamble = os.environ.get(+ PSKeys.JSON_POSTAMBLE, PSKeys.DEFAULT_JSON_POSTAMBLE+ )+ postamble += f"\n{json_postamble}"+ if platform_postamble:+ platform_postamble += "\n\n"+ if word_confidence_postamble:+ platform_postamble += f"{word_confidence_postamble}\n\n"+ return postamble, platform_postamble+@staticmethoddef construct_prompt(preamble: str,@@ -203,25 +279,51 @@ class AnswerPromptService:"""Build the full prompt string with preamble, grammar, postamble, context."""prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}"prompt += AnswerPromptService._build_grammar_notes(grammar_list)- if prompt_type == PSKeys.JSON:- json_postamble = os.environ.get(- PSKeys.JSON_POSTAMBLE, PSKeys.DEFAULT_JSON_POSTAMBLE- )- postamble += f"\n{json_postamble}"- if platform_postamble:- platform_postamble += "\n\n"- if word_confidence_postamble:- platform_postamble += f"{word_confidence_postamble}\n\n"+ postamble, platform_postamble = AnswerPromptService._prepare_postambles(+ postamble, platform_postamble, word_confidence_postamble, prompt_type+ )prompt += (f"\n\n{postamble}\n\nContext:\n---------------\n{context}\n"f"-----------------\n\n{platform_postamble}Answer:")return prompt+ @staticmethod+ def construct_cached_prompt(+ preamble: str,+ prompt: str,+ postamble: str,+ grammar_list: list[dict[str, Any]],+ context: str,+ platform_postamble: str,+ word_confidence_postamble: str,+ prompt_type: str = "text",+ ) -> tuple[str, str]:+ """Build ``(cache_prefix, volatile)`` with the document context first.++ Same content as :meth:`construct_prompt`, but the reused ``context`` is+ moved to the front so it forms a cacheable prefix that repeats across+ every prompt run against the same document, while the per-prompt+ question is the volatile suffix. The model sees ``cache_prefix ++ volatile``. This reorders the prompt (context before question instead of+ after), which is why it is flag-gated and A/B-evaluated.+ """+ postamble, platform_postamble = AnswerPromptService._prepare_postambles(+ postamble, platform_postamble, word_confidence_postamble, prompt_type+ )+ cache_prefix = f"Context:\n---------------\n{context}\n-----------------\n\n"+ volatile = (+ f"{preamble}\n\nQuestion or Instruction: {prompt}"+ + AnswerPromptService._build_grammar_notes(grammar_list)+ + f"\n\n{postamble}\n\n{platform_postamble}Answer:"+ )+ return cache_prefix, volatile+@staticmethoddef run_completion(llm: Any,prompt: str,+ cache_prefix: str | None = None,metadata: dict[str, str] | None = None,prompt_key: str | None = None,prompt_type: str | None = "text",@@ -249,6 +351,7 @@ class AnswerPromptService:try:completion = llm.complete(prompt=prompt,+ cache_prefix=cache_prefix,process_text=process_text,extract_json=prompt_type.lower() != PSKeys.TEXT,)diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.pyindex ce7fbea0d..b672d56ab 100644--- a/workers/executor/executors/legacy_executor.py+++ b/workers/executor/executors/legacy_executor.py@@ -1928,6 +1928,8 @@ class LegacyExecutor(BaseExecutor):from executor.executors.constants import PromptServiceConstants as PSKeystry:+ # Caching is enabled globally via the ENABLE_PROMPT_CACHING master+ # switch (honored inside the SDK ``LLM``); no per-call flag needed.llm = llm_cls(adapter_instance_id=output[PSKeys.LLM],tool=shim,diff --git a/workers/tests/test_answer_prompt_caching.py b/workers/tests/test_answer_prompt_caching.pynew file mode 100644index 000000000..28fa00662--- /dev/null+++ b/workers/tests/test_answer_prompt_caching.py@@ -0,0 +1,235 @@+"""Guardrail tests for flag-gated prompt-caching in answer_prompt.++When ``ENABLE_PROMPT_CACHING`` is on, ``construct_cached_prompt`` reorders the+prompt so the reused document context becomes a cacheable prefix (context+first) instead of a suffix (context last). These tests lock in that:++- the default (flag off) prompt is unchanged (context last),+- the cached variant is a pure *reorder* — every piece of the original prompt+ is preserved, only the context moves to the front,+- ``cache_prefix`` is exactly the context block (no per-prompt question), so+ it repeats byte-for-byte across prompts on the same document,+- the reorder only happens when the selected LLM actually caches+ (``is_prompt_caching_active()``); unsupported LLMs keep the original order.++The executor package's ``__init__`` pulls the full celery stack, so we load the+module with stubbed parent packages — the methods under test are pure strings.+"""++import importlib+import os+import sys+import types++import pytest++_WORKERS = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+++def _load_answer_prompt():+ """Import answer_prompt without triggering the executor package's celery stack.++ The real ``executor`` / ``executor.executors`` package ``__init__``s pull the+ full celery worker stack, so we temporarily register lightweight namespace+ stubs pointing at the real source dirs, import the module, then remove any+ stub we added. The imported module (and its already-resolved ``constants`` /+ ``exceptions`` imports) stay cached under their own names, so the stubs are+ unneeded afterwards — and removing them keeps ``sys.modules`` clean for other+ tests in the same process instead of leaving synthetic packages behind.+ """+ injected = []+ for pkg, rel in [+ ("executor", "executor"),+ ("executor.executors", "executor/executors"),+ ]:+ if pkg not in sys.modules:+ mod = types.ModuleType(pkg)+ mod.__path__ = [os.path.join(_WORKERS, rel)]+ sys.modules[pkg] = mod+ injected.append(pkg)+ try:+ return importlib.import_module("executor.executors.answer_prompt")+ finally:+ for pkg in injected:+ sys.modules.pop(pkg, None)+++_mod = _load_answer_prompt()+A = _mod.AnswerPromptService++_ARGS = dict(+ preamble="You are an extractor.",+ prompt="What is the tenant name?",+ postamble="Answer concisely.",+ grammar_list=[],+ context="UNIT 101 John Smith $1,450\nUNIT 102 Maria Davis $1,525",+ platform_postamble="",+ word_confidence_postamble="",+ prompt_type="text",+)+++def test_default_prompt_is_context_last():+ off = A.construct_prompt(**_ARGS)+ assert off.index("Question or Instruction") < off.index("UNIT 101")+++def test_cached_prompt_is_context_first():+ prefix, volatile = A.construct_cached_prompt(**_ARGS)+ full = prefix + volatile+ assert full.index("UNIT 101") < full.index("Question or Instruction")+++def test_cache_prefix_is_context_block_only():+ prefix, _volatile = A.construct_cached_prompt(**_ARGS)+ assert prefix.startswith("Context:")+ assert "UNIT 101" in prefix+ # The volatile per-prompt question must NOT leak into the cached prefix,+ # or the prefix would differ per prompt and never hit the cache.+ assert "Question or Instruction" not in prefix+++def test_cached_variant_is_a_pure_reorder_no_content_lost():+ prefix, volatile = A.construct_cached_prompt(**_ARGS)+ full = prefix + volatile+ for piece in (+ "You are an extractor.",+ "Question or Instruction: What is the tenant name?",+ "Answer concisely.",+ "UNIT 101 John Smith $1,450",+ "Answer:",+ ):+ assert piece in full, f"missing from cached prompt: {piece!r}"+++# --- shared postamble formatting (cached and uncached must not diverge) ------+++def test_prepare_postambles_applies_json_and_platform_formatting():+ json_postamble = os.environ.get(+ _mod.PSKeys.JSON_POSTAMBLE, _mod.PSKeys.DEFAULT_JSON_POSTAMBLE+ )+ post, plat = A._prepare_postambles("BASE", "PLATFORM", "WORDCONF", _mod.PSKeys.JSON)+ assert post == f"BASE\n{json_postamble}"+ assert plat == "PLATFORM\n\nWORDCONF\n\n"+++def test_prepare_postambles_noop_for_text_without_platform():+ post, plat = A._prepare_postambles("BASE", "", "", "text")+ assert post == "BASE"+ assert plat == ""+++def test_cached_and_uncached_share_postamble_formatting():+ """Both builders route postambles through the shared helper, so a JSON ++ platform postamble is formatted identically — guards against silent+ divergence between cached and non-cached prompts (breaks A/B comparison)."""+ args = dict(_ARGS)+ args.update(+ prompt_type=_mod.PSKeys.JSON,+ postamble="BASE_POST",+ platform_postamble="PLATFORM",+ word_confidence_postamble="WORDCONF",+ )+ flat = A.construct_prompt(**args)+ prefix, volatile = A.construct_cached_prompt(**args)+ cached = prefix + volatile+ json_postamble = os.environ.get(+ _mod.PSKeys.JSON_POSTAMBLE, _mod.PSKeys.DEFAULT_JSON_POSTAMBLE+ )+ for piece in ("BASE_POST", "PLATFORM", "WORDCONF", json_postamble):+ assert piece in flat, f"missing from construct_prompt: {piece!r}"+ assert piece in cached, f"missing from construct_cached_prompt: {piece!r}"+++# NOTE: the ENABLE_PROMPT_CACHING env-var master switch is owned by the SDK+# (``unstract.sdk1.llm.is_prompt_caching_enabled``) and covered by its tests;+# answer_prompt gates purely on the LLM's ``is_prompt_caching_active()`` probe+# (see ``_llm_caches_prompts`` tests below), so there is no local env flag to+# test here.+++# --- gate: only reorder into a cached prefix when the LLM actually caches -----+++class _FakeLLM:+ """Minimal LLM stub exposing the SDK's caching-capability probe."""++ def __init__(self, active: bool):+ self._active = active++ def is_prompt_caching_active(self) -> bool:+ return self._active+++class _LegacyLLM:+ """LLM stub without the capability probe (older SDK / mock)."""+++def test_llm_caches_prompts_probe():+ assert A._llm_caches_prompts(_FakeLLM(True)) is True+ assert A._llm_caches_prompts(_FakeLLM(False)) is False+ # An LLM that doesn't expose the probe must default to "no caching".+ assert A._llm_caches_prompts(_LegacyLLM()) is False+++def test_llm_caches_prompts_probe_swallows_errors():+ class _BoomLLM:+ def is_prompt_caching_active(self):+ raise RuntimeError("boom")++ assert A._llm_caches_prompts(_BoomLLM()) is False+++def _run_and_capture(monkeypatch, llm):+ """Run construct_and_run_prompt with run_completion stubbed to capture args."""+ captured: dict = {}++ def _fake_run_completion(**kwargs):+ captured.update(kwargs)+ return "answer"++ monkeypatch.setattr(A, "run_completion", staticmethod(_fake_run_completion))++ tool_settings = {+ _mod.PSKeys.PREAMBLE: "You are an extractor.",+ _mod.PSKeys.POSTAMBLE: "Answer concisely.",+ _mod.PSKeys.GRAMMAR: [],+ }+ output = {+ "promptx": "What is the tenant name?",+ _mod.PSKeys.NAME: "q1",+ _mod.PSKeys.TYPE: "text",+ }+ A.construct_and_run_prompt(+ tool_settings=tool_settings,+ output=output,+ llm=llm,+ context="UNIT 101 John Smith $1,450",+ prompt="promptx",+ metadata={},+ )+ return captured, output+++def test_supported_llm_reorders_and_sends_cache_prefix(monkeypatch):+ captured, output = _run_and_capture(monkeypatch, _FakeLLM(True))+ # Cached path: context-first prompt + a cache_prefix that is the context.+ assert captured["cache_prefix"] is not None+ assert captured["cache_prefix"].startswith("Context:")+ combined = output[_mod.PSKeys.COMBINED_PROMPT]+ assert combined.index("UNIT 101") < combined.index("Question or Instruction")+++def test_unsupported_llm_keeps_original_order_and_no_cache_prefix(monkeypatch):+ # Global flag ON, but the LLM's provider/model does not support caching.+ monkeypatch.setenv("ENABLE_PROMPT_CACHING", "true")+ captured, output = _run_and_capture(monkeypatch, _FakeLLM(False))+ # No cache prefix, and the original context-last prompt order is preserved.+ assert captured["cache_prefix"] is None+ combined = output[_mod.PSKeys.COMBINED_PROMPT]+ assert combined.index("Question or Instruction") < combined.index("UNIT 101")+++if __name__ == "__main__":+ raise SystemExit(pytest.main([__file__, "-q"]))