OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: report an over-length embedding input as what it is (#2474)
Rico Furtado5084cf34ecd1
diff --git a/frontend/hooks/useChatStreaming.test.tsx b/frontend/hooks/useChatStreaming.test.tsxindex 1a0dc4526..2f5f8b908 100644--- a/frontend/hooks/useChatStreaming.test.tsx+++ b/frontend/hooks/useChatStreaming.test.tsx@@ -356,6 +356,23 @@ describe("useChatStreaming", () => {expect(returned?.content).not.toContain("{");});+ /** Surface the relayed provider diagnosis as a chat error instead of assistant text. */+ it("treats a gateway error relayed as a Python dict as one", async () => {+ serveStream([+ textChunk(+ `Error code: 400 - {'error': {'message': "This model's maximum context length is 512 tokens.", 'code': 'context_length_exceeded'}}`,+ ),+ ]);+ const { result } = setup();++ const returned = await send(result, { prompt: "p" });++ expect(returned?.error).toBe(true);+ expect(returned?.content).toBe(+ "This model's maximum context length is 512 tokens.",+ );+ });+it("stops loading after an error", async () => {server.use(http.post(ENDPOINT, () => HttpResponse.text("nope", { status: 500 })),diff --git a/frontend/lib/chat-stream-errors.test.ts b/frontend/lib/chat-stream-errors.test.tsindex 1bf4d42f0..7b8f410f6 100644--- a/frontend/lib/chat-stream-errors.test.ts+++ b/frontend/lib/chat-stream-errors.test.ts@@ -102,6 +102,26 @@ describe("looksLikeProviderErrorContent", () => {);});+ /** Recognize relayed SDK dictionaries as errors even without credential markers. */+ it("detects an OpenAI SDK error printed as a Python dict", () => {+ assert.equal(+ looksLikeProviderErrorContent(+ `Error code: 400 - {'error': {'message': "This model's maximum context length is 512 tokens.", 'code': 'context_length_exceeded'}}`,+ ),+ true,+ );+ });++ /** Allow ordinary assistant replies to contain dictionary examples. */+ it("does not flag a reply that shows a dict with no error key", () => {+ assert.equal(+ looksLikeProviderErrorContent(+ "Send {'message': 'hello', 'user': 'ana'} as the request body.",+ ),+ false,+ );+ });+it("flags the exact generic stream fallback message", () => {assert.equal(looksLikeProviderErrorContent(@@ -188,4 +208,52 @@ describe("formatProviderErrorMessage", () => {"Invalid API key",);});++ // The OpenAI SDK prints the body as a Python dict, which is how a gateway+ // error reads once Langflow relays it. It used to collapse to "400 -".+ const overflow =+ "watsonx/intfloat/multilingual-e5-large: Invalid input argument for Model " ++ "'intfloat/multilingual-e5-large': This model's maximum context length is 512 " ++ "tokens. However, you requested 548 tokens in the input for embedding generation.";++ /** Preserve the provider's diagnosis through SDK and Langflow wrappers. */+ it("reads the message out of an OpenAI SDK error printed as a Python dict", () => {+ assert.equal(+ formatProviderErrorMessage(+ `Error code: 400 - {'error': {'message': "${overflow}", 'type': 'invalid_request_error', 'code': 'context_length_exceeded'}}`,+ ),+ overflow,+ );+ assert.equal(+ formatProviderErrorMessage(+ `Error building Component OpenSearch: \n\nError code: 400 - {'error': {'message': "${overflow}", 'type': 'invalid_request_error'}}`,+ ),+ overflow,+ );+ });++ /** Restore escaped apostrophes without losing embedded double quotes. */+ it("unescapes a single-quoted Python message", () => {+ assert.equal(+ formatProviderErrorMessage(+ "Error code: 404 - {'error': {'message': 'The model `x` doesn\\'t exist, said \"the API\"', 'type': 'api_error'}}",+ ),+ 'The model `x` doesn\'t exist, said "the API"',+ );+ });++ /** Keep an unreadable payload rather than reducing it to a bare status code. */+ it("keeps the whole string when the prefix is only a status code", () => {+ const raw = "Error code: 502 - {'error': <object at 0x1>}";+ assert.equal(+ formatProviderErrorMessage(raw),+ "502 - {'error': <object at 0x1>}",+ );+ });++ /** Leave prose mentioning a message field intact when it is not an error object. */+ it("does not read a message out of prose that merely mentions one", () => {+ const raw = "The reply had 'message': 'hello' in it";+ assert.equal(formatProviderErrorMessage(raw), raw);+ });});diff --git a/frontend/lib/chat-stream-errors.ts b/frontend/lib/chat-stream-errors.tsindex 09d73618f..768c6f06d 100644--- a/frontend/lib/chat-stream-errors.ts+++ b/frontend/lib/chat-stream-errors.ts@@ -19,6 +19,23 @@ function stripErrorLabelPrefixes(text: string): string {}}+/**+ * The OpenAI SDK prints an error body as a Python dict —+ * `Error code: 400 - {'error': {'message': "...", ...}}` — which JSON.parse+ * rejects. That is the shape a gateway error has once Langflow relays it.+ * Mirrors _load_error_body in src/api/provider_validation.py.+ */+const PYTHON_DICT_MESSAGE =+ /^\s*\{[\s\S]*?'message':\s*(?:"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)')/;++/** Extract a nonempty SDK dictionary message and unescape its quotes and backslashes. */+function tryParsePythonDictMessage(text: string): string | null {+ const match = PYTHON_DICT_MESSAGE.exec(text);+ const message = (match?.[1] ?? match?.[2])?.replace(/\\(['"\\])/g, "$1");+ return message?.trim() ? message : null;+}++/** Read a provider message from JSON, falling back to the SDK's Python dictionary form. */function tryParseJsonMessage(text: string): string | null {try {const data = JSON.parse(text) as Record<string, unknown>;@@ -55,33 +72,35 @@ function tryParseJsonMessage(text: string): string | null {return data.detail;}} catch {- return null;+ return tryParsePythonDictMessage(text);}return null;}+/** Return the first balanced object, ignoring braces inside either style of quoted string. */function extractBalancedJsonObject(text: string): string | null {const start = text.indexOf("{");if (start < 0) {return null;}let depth = 0;- let inString = false;+ // Either quote opens a string: a Python dict repr uses `'` where it can.+ let quote = "";let escaped = false;for (let i = start; i < text.length; i++) {const ch = text[i];- if (inString) {+ if (quote) {if (escaped) {escaped = false;} else if (ch === "\\") {escaped = true;- } else if (ch === '"') {- inString = false;+ } else if (ch === quote) {+ quote = "";}continue;}- if (ch === '"') {- inString = true;+ if (ch === '"' || ch === "'") {+ quote = ch;} else if (ch === "{") {depth += 1;} else if (ch === "}") {@@ -153,12 +172,11 @@ export function looksLikeProviderErrorContent(text: string): boolean {) {return true;}+ // Either quote: the OpenAI SDK prints an error body as a Python dict,+ // `Error code: 400 - {'error': {...}}`, not as JSON.if (trimmed.includes("{") &&- (lowered.includes('"error"') ||- lowered.includes('"errormessage"') ||- lowered.includes('"errorcode"') ||- lowered.includes('"errors"'))+ /["'](?:error|errormessage|errorcode|errors)["']/.test(lowered)) {return true;}@@ -180,11 +198,15 @@ function humanizeProviderErrorMessage(message: string): string {* transport error, and the payload after that quote is the entire diagnosis — so* keeping only the prefix discards everything useful. A real prefix ("Invalid API* key {...") ends on a word character and is still preferred.+ *+ * Nor is a prefix with no words in it: `Error code: 502 - {...}` leaves "502 -",+ * which names no cause at all.*/const SYNTACTIC_FRAGMENT_TAIL = /[([{'"`,=:]$/;+/** Require a letter and a complete ending before using a prefix as an error message. */function isReadablePrefix(prefix: string): boolean {- return prefix.length > 0 && !SYNTACTIC_FRAGMENT_TAIL.test(prefix);+ return /\p{L}/u.test(prefix) && !SYNTACTIC_FRAGMENT_TAIL.test(prefix);}/**diff --git a/src/api/provider_validation.py b/src/api/provider_validation.pyindex d753bce6a..026d379e7 100644--- a/src/api/provider_validation.py+++ b/src/api/provider_validation.py@@ -1,5 +1,6 @@"""Provider validation utilities for testing API keys and models during onboarding."""+import astimport asyncioimport jsonimport re@@ -65,25 +66,29 @@ _PROVIDER_ERROR_CONTENT_MARKERS = _PROVIDER_CREDENTIAL_ERROR_MARKERS + (def _extract_balanced_json_object(text: str) -> str | None:- """Return the first top-level `{...}` substring, ignoring trailing junk."""+ """Return the first top-level `{...}` substring, ignoring trailing junk.++ Either quote opens a string: the OpenAI SDK prints an error body as a+ Python dict, which quotes with `'` wherever the text allows.+ """start = text.find("{")if start < 0:return Nonedepth = 0- in_string = False+ quote = ""escape = Falsefor i in range(start, len(text)):ch = text[i]- if in_string:+ if quote:if escape:escape = Falseelif ch == "\\":escape = True- elif ch == '"':- in_string = False+ elif ch == quote:+ quote = ""continue- if ch == '"':- in_string = True+ if ch in "\"'":+ quote = chelif ch == "{":depth += 1elif ch == "}":@@ -219,13 +224,23 @@ def is_provider_stale_connection_error(exc: BaseException | None) -> bool:#: Markers for a prompt that does not fit the model's context window. That is a-#: property of one request — a long conversation, a lot of retrieved text — and-#: says nothing about whether the provider is serving.+#: property of one request — a long conversation, a lot of retrieved text, an+#: oversized chunk sent for embedding — and says nothing about whether the+#: provider is serving._CONTEXT_WINDOW_ERROR_MARKERS = ("maximum context length","context_length_exceeded","contextwindowexceeded","prompt is too long",+ # The wordings LiteLLM itself recognises (ExceptionCheckers+ # .is_error_str_context_window_exceeded), for the providers whose errors it+ # does not already raise as ContextWindowExceededError.+ "exceed context limit",+ "maximum context limit",+ "longer than the model's context length",+ "input tokens exceed the configured limit",+ "exceeds the available context size",+ "exceeds the maximum number of tokens allowed",)@@ -237,6 +252,44 @@ def is_context_window_error(text: str | BaseException | None) -> bool:return any(marker in lowered for marker in _CONTEXT_WINDOW_ERROR_MARKERS)+_CONTEXT_LIMIT_RE = re.compile(r"maximum context length is (\d+)", re.IGNORECASE)+#: Tried in order: vLLM's chat wording also says "requested 0 output tokens",+#: which is not the size of the prompt.+_CONTEXT_REQUESTED_RES = (+ re.compile(r"prompt contains (?:at least )?(\d+) input tokens", re.IGNORECASE),+ re.compile(r"resulted in (\d+) tokens", re.IGNORECASE),+ re.compile(r"requested (\d+) tokens", re.IGNORECASE),+)+#: Anthropic: "prompt is too long: 210000 tokens > 200000 maximum".+_CONTEXT_COMPARISON_RE = re.compile(r"(\d+) tokens > (\d+) maximum", re.IGNORECASE)+++def context_window_token_counts(+ text: str | BaseException | None,+) -> tuple[int | None, int | None]:+ """The `(limit, requested)` token counts a context-window error states.++ Either is None when the provider did not say. Callers use the pair to tell+ how far over the limit an input was, so a wrong number is worse than none:+ only wordings that name the quantity outright are read.+ """+ if text is None:+ return None, None+ message = str(text) if not isinstance(text, str) else text+ comparison = _CONTEXT_COMPARISON_RE.search(message)+ if comparison:+ return int(comparison.group(2)), int(comparison.group(1))+ limit = _CONTEXT_LIMIT_RE.search(message)+ requested = next(+ (found for pattern in _CONTEXT_REQUESTED_RES if (found := pattern.search(message))),+ None,+ )+ return (+ int(limit.group(1)) if limit else None,+ int(requested.group(1)) if requested else None,+ )++_GENERIC_UPSTREAM_ERROR_MARKERS = ("an unknown error occurred","an error occurred while generating a response",@@ -256,6 +309,9 @@ def is_generic_upstream_error(text: str | BaseException | None) -> bool:return lowered in generics+_ERROR_OBJECT_KEY_RE = re.compile(r"[\"'](?:error|errormessage|errorcode|errors)[\"']")++def looks_like_provider_error_content(text: str | None) -> bool:"""True when assistant text looks like a provider/auth failure rather than a normal reply."""if not text or not text.strip():@@ -268,13 +324,9 @@ def looks_like_provider_error_content(text: str | None) -> bool:lowered = content.lower()if any(marker in lowered for marker in _PROVIDER_ERROR_CONTENT_MARKERS):return True- # Raw provider payloads often embed JSON error objects.- if "{" in content and (- '"error"' in lowered- or '"errormessage"' in lowered- or '"errorcode"' in lowered- or '"errors"' in lowered- ):+ # Raw provider payloads often embed an error object: as JSON, or as the+ # Python dict the OpenAI SDK prints (`Error code: 400 - {'error': {...}}`).+ if "{" in content and _ERROR_OBJECT_KEY_RE.search(lowered):return Truereturn False@@ -297,7 +349,10 @@ def sanitize_provider_error_content(text: str | BaseException | None) -> str:# Strip the first JSON object as a last resort.json_start = cleaned.find("{")prefix = cleaned[:json_start].rstrip(": ").strip()- return prefix or "An error occurred while generating a response."+ # "Error code: 502 - {...}" leaves "502 -", which names no cause at all.+ if not any(ch.isalpha() for ch in prefix):+ return "An error occurred while generating a response."+ return prefixreturn cleaned@@ -626,11 +681,25 @@ async def resolve_chat_stream_error_message_async(return cleaned+def _load_error_body(error_text: str) -> Any:+ """Parse an error body sent as JSON or printed as a Python literal.++ The OpenAI SDK formats its exceptions as `Error code: 400 - {'error': ...}`,+ a dict repr that `json.loads` rejects. That is the form a gateway error+ takes by the time Langflow relays it.+ """+ try:+ return json.loads(error_text)+ except ValueError:+ if not error_text.lstrip().startswith("{"):+ raise+ return ast.literal_eval(error_text)++def _parse_json_error_message(error_text: str) -> str:"""Parse JSON error message and extract just the message field."""try:- # Try to parse as JSON- error_data = json.loads(error_text)+ error_data = _load_error_body(error_text)if isinstance(error_data, dict):# WatsonX format: {"errors": [{"code": "...", "message": "..."}], ...}@@ -663,7 +732,7 @@ def _parse_json_error_message(error_text: str) -> str:# Generic format: {"detail": "..."}if "detail" in error_data:return error_data["detail"]- except (json.JSONDecodeError, ValueError, TypeError):+ except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):pass# Return original text if not JSON or can't parsediff --git a/src/api/v1/llm.py b/src/api/v1/llm.pyindex 3b1c56ed1..7e9d2aeda 100644--- a/src/api/v1/llm.py+++ b/src/api/v1/llm.py@@ -29,18 +29,25 @@ _embeddings_use = require_llm_proxy_any_permission(("chat:use", "knowledge:uploa_chat_use = require_llm_proxy_permission("chat:use")-def _openai_error(message: str, status_code: int, error_type: str = "invalid_request_error"):- return JSONResponse(- {"error": {"message": message, "type": error_type}},- status_code=status_code,- )+def _openai_error(+ message: str,+ status_code: int,+ error_type: str = "invalid_request_error",+ code: str | None = None,+):+ """Build an OpenAI error response, including a code only when supplied."""+ error = {"message": message, "type": error_type}+ if code:+ error["code"] = code+ return JSONResponse({"error": error}, status_code=status_code)def _gateway_error(exc: LlmGatewayError):+ """Log internal gateway details and return the caller-safe error and code."""error_type = "invalid_request_error" if exc.status_code < 500 else "api_error"if exc.detail != exc.message:logger.error("LLM gateway error", status_code=exc.status_code, error=exc.detail)- return _openai_error(exc.message, exc.status_code, error_type)+ return _openai_error(exc.message, exc.status_code, error_type, exc.code)async def list_openai_models_endpoint(diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.pyindex 7e40fcc49..b81e86c35 100644--- a/src/services/llm_gateway.py+++ b/src/services/llm_gateway.py@@ -14,7 +14,7 @@ import jsonimport refrom collections import OrderedDict, dequefrom collections.abc import AsyncIterator, Awaitable, Callable, Mapping-from typing import Any, Literal+from typing import Any, Literal, NamedTuplefrom uuid import uuid4from services import provider_error_log@@ -48,14 +48,49 @@ class LlmGatewayError(Exception):`message` is safe to return to the caller. `detail` carries the internaltext (upstream exception type and body) and is for logs only — returning it- is stack-trace exposure (CodeQL py/stack-trace-exposure).+ is stack-trace exposure (CodeQL py/stack-trace-exposure). `code` is the+ OpenAI `error.code` for the failure, when it has been classified."""- def __init__(self, message: str, status_code: int = 400, detail: str | None = None):+ def __init__(+ self,+ message: str,+ status_code: int = 400,+ detail: str | None = None,+ code: str | None = None,+ ):+ """Store caller-safe text, HTTP status, internal details, and an optional code."""super().__init__(message)self.message = messageself.status_code = status_codeself.detail = detail or message+ self.code = code+++CONTEXT_LENGTH_ERROR_CODE = "context_length_exceeded"+++class LlmContextLengthError(LlmGatewayError):+ """The input does not fit the model's context window.++ The provider is serving and a shorter input succeeds, so a caller that can+ shorten or split what it sent catches this instead of the base class.+ `max_tokens` and `requested_tokens` are None when the provider's error did+ not state them.+ """++ def __init__(+ self,+ message: str,+ detail: str | None = None,+ *,+ max_tokens: int | None = None,+ requested_tokens: int | None = None,+ ):+ """Create a context-length 400 with the token counts the provider supplied."""+ super().__init__(message, 400, detail, code=CONTEXT_LENGTH_ERROR_CODE)+ self.max_tokens = max_tokens+ self.requested_tokens = requested_tokensdef _get_config():@@ -834,18 +869,12 @@ def _provider_error_text(detail: str, exc: BaseException | None) -> str | None:#: Upstream statuses worth passing through: a client can act on these itself.#: Everything else becomes 502 — an upstream 401/403 must not read as an OpenRAG-#: auth failure, and an upstream 400 must not read as a bad request to us.+#: auth failure, and an upstream 400 must not read as a bad request to us. The+#: one exception is an input that overflows the context window, which *is* a bad+#: request to us; see `_classify_upstream_failure`._PASSTHROUGH_UPSTREAM_STATUSES = frozenset({408, 429, 503, 504})-def _upstream_status_code(exc: BaseException) -> int:- """HTTP status for a failed upstream call, from the provider's own where useful."""- status = getattr(exc, "status_code", None)- if isinstance(status, int) and status in _PASSTHROUGH_UPSTREAM_STATUSES:- return status- return 502--def _upstream_client_message(detail: str, provider: str = "", model: str = "", exc: BaseException | None = None) -> str:@@ -908,6 +937,81 @@ def _is_request_scoped_failure(detail: str, exc: BaseException | None) -> bool:return is_context_window_error(detail)+#: Exception classes, anywhere in the MRO, that mean the provider was never+#: reached. LiteLLM gives its own APIConnectionError a 500, so on the status+#: alone it would read as the provider failing.+_UNREACHABLE_EXCEPTION_NAMES = frozenset(+ {"APIConnectionError", "ConnectionError", "ConnectError", "ClientConnectionError"}+)+++class _UpstreamFailure(NamedTuple):+ """How one failed upstream call is reported to the caller and to the banner."""++ code: str+ status_code: int+ counts_against_provider: bool+++def _classify_upstream_failure(detail: str, exc: BaseException | None) -> _UpstreamFailure:+ """Name a failed upstream call: its `error.code`, its HTTP status, its blame.++ The status keeps the passthrough rule above. The code says what the status+ cannot: a 502 covers a rejected key, a model that does not exist and a+ provider that is down, and a client deciding whether to retry needs to know+ which. Only an input that overflows the context window is the caller's own+ doing, so only that becomes a 400 and stays out of the provider's health.+ """+ from api.provider_validation import (+ is_provider_credential_error,+ is_provider_tls_error,+ is_provider_truncated_response_error,+ )++ if _is_request_scoped_failure(detail, exc):+ return _UpstreamFailure(CONTEXT_LENGTH_ERROR_CODE, 400, counts_against_provider=False)++ status = getattr(exc, "status_code", None)+ if not isinstance(status, int):+ status = None+ # Same precedence as `_upstream_client_message`, so the code and the text+ # never describe two different failures.+ if is_provider_tls_error(detail) or is_provider_truncated_response_error(detail):+ code = "upstream_unreachable"+ elif status in (401, 403) or is_provider_credential_error(detail):+ code = "upstream_auth_error"+ elif status == 429:+ code = "rate_limit_exceeded"+ elif status in (408, 504):+ code = "upstream_timeout"+ elif exc is not None and any(+ cls.__name__ in _UNREACHABLE_EXCEPTION_NAMES for cls in type(exc).__mro__+ ):+ code = "upstream_unreachable"+ elif status in (500, 502, 503):+ code = "upstream_unavailable"+ elif status is not None and 400 <= status < 500:+ code = "upstream_invalid_request"+ else:+ code = "upstream_error"+ http_status = status if status in _PASSTHROUGH_UPSTREAM_STATUSES else 502+ return _UpstreamFailure(code, http_status, counts_against_provider=True)+++def _upstream_gateway_error(+ message: str, detail: str, failure: _UpstreamFailure+) -> LlmGatewayError:+ """The exception a classified upstream failure is raised as."""+ if failure.code == CONTEXT_LENGTH_ERROR_CODE:+ from api.provider_validation import context_window_token_counts++ max_tokens, requested_tokens = context_window_token_counts(detail)+ return LlmContextLengthError(+ message, detail, max_tokens=max_tokens, requested_tokens=requested_tokens+ )+ return LlmGatewayError(message, failure.status_code, detail=detail, code=failure.code)++def _redact(message: str, credentials: Mapping[str, Any]) -> str:redacted = messagefor value in credentials.values():@@ -1332,13 +1436,10 @@ async def chat_completions(# The health banner otherwise reports whatever its own probe hit, which# is a different request and so often a different error. Hand it the# text this caller is being shown.- if not _is_request_scoped_failure(detail, exc):+ failure = _classify_upstream_failure(detail, exc)+ if failure.counts_against_provider:provider_error_log.record_failure(provider, "chat", message)- raise LlmGatewayError(- message,- _upstream_status_code(exc),- detail=detail,- ) from exc+ raise _upstream_gateway_error(message, detail, failure) from excif stream:# A stream can still fail mid-flight, so _stream_sse clears the record@@ -1553,7 +1654,7 @@ def _empty_stream_message(provider: str, model: str, finish_reason: str | None))-def _error_frame(message: str, provider: str, model: str) -> str:+def _error_frame(message: str, provider: str, model: str, code: str = "upstream_error") -> str:"""SSE frame an OpenAI client turns back into a raised APIError."""return ("data: "@@ -1562,7 +1663,7 @@ def _error_frame(message: str, provider: str, model: str) -> str:"error": {"message": message,"type": "api_error",- "code": "upstream_error",+ "code": code,"provider": provider,"model": model,}@@ -1587,6 +1688,11 @@ async def _stream_sse(model: str = "",credentials: Mapping[str, Any] | None = None,) -> AsyncIterator[str]:+ """Yield OpenAI SSE frames, repair tool calls, and report classified failures.++ Context-window failures emit an error frame without affecting provider+ health. Close the upstream stream and finish with a ``[DONE]`` frame.+ """tally = _StreamTally()buffer = _ToolCallBuffer()template: dict[str, Any] = {}@@ -1675,9 +1781,10 @@ async def _stream_sse(tally.error = detaillogger.error("LLM chat stream failed", provider=provider, model=model, error=detail)message = _upstream_client_message(detail, provider, model, exc)- if not _is_request_scoped_failure(detail, exc):+ failure = _classify_upstream_failure(detail, exc)+ if failure.counts_against_provider:provider_error_log.record_failure(provider, "chat", message)- yield _error_frame(message, provider, model)+ yield _error_frame(message, provider, model, failure.code)finally:close = getattr(stream, "aclose", None) or getattr(stream, "close", None)if close is not None:@@ -1809,11 +1916,11 @@ async def embeddings(detail = _redact(f"{type(exc).__name__}: {exc}", credentials)logger.error("LLM embeddings failed", provider=provider, model=litellm_model, error=detail)message = _upstream_client_message(detail, provider, litellm_model, exc)- provider_error_log.record_failure(provider, "embedding", message)- raise LlmGatewayError(- message,- _upstream_status_code(exc),- detail=detail,- ) from exc+ # An over-length input is one chunk's problem. Recording it latched the+ # banner against a provider that was embedding every other chunk fine.+ failure = _classify_upstream_failure(detail, exc)+ if failure.counts_against_provider:+ provider_error_log.record_failure(provider, "embedding", message)+ raise _upstream_gateway_error(message, detail, failure) from excprovider_error_log.record_success(provider, "embedding")return responsediff --git a/src/services/task_service.py b/src/services/task_service.pyindex de0cd1be8..9cea924ad 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -181,6 +181,32 @@ def _provider_credential_failure_metadata(error: str) -> dict | None:}+def _context_window_failure_metadata(error: str) -> dict | None:+ """Classify a chunk the embedding model rejected as longer than its input limit."""+ from api.provider_validation import context_window_token_counts, is_context_window_error++ if not is_context_window_error(error):+ return None++ limit, requested = context_window_token_counts(error)+ if limit and requested:+ size = f" ({requested} tokens; the limit is {limit})"+ elif limit:+ size = f" (the limit is {limit} tokens)"+ else:+ size = ""+ return {+ "component": "openrag",+ "failure_phase": "embedding",+ "user_facing_message": (+ f"A section of this document is longer than the embedding model accepts{size}. "+ "Try a smaller chunk size or an embedding model with a larger input limit, "+ "then retry ingestion."+ ),+ "actionable_by": "USER_ACTIONABLE",+ }++def _is_transient_connectivity_error(error: str) -> bool:lowered = error.lower()return any(marker in lowered for marker in _TRANSIENT_CONNECTIVITY_ERROR_MARKERS)@@ -1415,6 +1441,12 @@ class TaskService:if credential_meta:return credential_meta+ # Before the Langflow fallbacks, which would report the provider's raw+ # sentence as a retryable failure: retrying the same chunk fails again.+ context_window_meta = _context_window_failure_metadata(error)+ if context_window_meta:+ return context_window_meta+if _is_transient_connectivity_error(error) or _is_langflow_transport_failure(error):return {"component": "langflow",diff --git a/tests/unit/api/test_v1_llm.py b/tests/unit/api/test_v1_llm.pyindex 278a39c37..b2c2b4052 100644--- a/tests/unit/api/test_v1_llm.py+++ b/tests/unit/api/test_v1_llm.py@@ -10,7 +10,7 @@ from fastapi import params as fastapi_paramsfrom fastapi.responses import JSONResponse, StreamingResponsefrom api.v1 import llm as v1_llm-from services.llm_gateway import LlmGatewayError+from services.llm_gateway import LlmContextLengthError, LlmGatewayErrorfrom services.model_catalog import CATALOG_UNAVAILABLE_MESSAGE, CatalogUnavailableErrorfrom session_manager import User@@ -98,6 +98,7 @@ async def test_chat_completions_streams(monkeypatch):@pytest.mark.asyncioasync def test_chat_completions_maps_gateway_errors(monkeypatch):+ """Return gateway failures without inventing a code for an unclassified error."""monkeypatch.setattr(v1_llm,"chat_completions",@@ -109,6 +110,51 @@ async def test_chat_completions_maps_gateway_errors(monkeypatch):assert response.status_code == 400data = json.loads(response.body)assert data["error"]["message"] == "OpenAI API key is not configured"+ # No code is invented for a failure the gateway did not classify.+ assert "code" not in data["error"]+++async def test_embeddings_reports_an_over_length_input_as_a_coded_400(monkeypatch):+ """What an OpenAI client expects for it, so it can branch on `error.code`."""+ message = "watsonx/e5: This model's maximum context length is 512 tokens."+ monkeypatch.setattr(+ v1_llm,+ "embeddings",+ AsyncMock(side_effect=LlmContextLengthError(message, detail="BadRequestError: ...")),+ )+ request = MagicMock()+ request.json = AsyncMock(return_value={"model": "watsonx:e5", "input": ["x"]})++ response = await v1_llm.embeddings_endpoint(request, user=_user())++ assert response.status_code == 400+ assert json.loads(response.body) == {+ "error": {+ "message": message,+ "type": "invalid_request_error",+ "code": "context_length_exceeded",+ }+ }+++async def test_a_classified_upstream_failure_carries_its_code(monkeypatch):+ """Preserve the classified upstream status and code in an embeddings response."""+ monkeypatch.setattr(+ v1_llm,+ "embeddings",+ AsyncMock(+ side_effect=LlmGatewayError("rate limited", 429, code="rate_limit_exceeded"),+ ),+ )+ request = MagicMock()+ request.json = AsyncMock(return_value={"model": "watsonx:e5", "input": ["x"]})++ response = await v1_llm.embeddings_endpoint(request, user=_user())++ assert response.status_code == 429+ assert json.loads(response.body)["error"]["code"] == "rate_limit_exceeded"@pytest.mark.asynciodiff --git a/tests/unit/services/test_llm_gateway.py b/tests/unit/services/test_llm_gateway.pyindex b0eb86175..4576912d4 100644--- a/tests/unit/services/test_llm_gateway.py+++ b/tests/unit/services/test_llm_gateway.py@@ -14,6 +14,7 @@ from config.config_manager import (WatsonXConfig,)from services.llm_gateway import (+ LlmContextLengthError,LlmGatewayError,chat_completions,embeddings,@@ -1911,6 +1912,7 @@ class TestRequestScopedFailuresStayOutOfTheBanner:@pytest.mark.asyncioasync def test_a_context_window_overflow_is_not_recorded(self, monkeypatch):+ """Report chat overflow as a coded 400 without recording a provider failure."""from services import llm_gateway, provider_error_logclass ContextWindowExceededError(Exception):@@ -1923,6 +1925,11 @@ class TestRequestScopedFailuresStayOutOfTheBanner:assert "maximum context length" in str(raised.value)assert provider_error_log.latest_failure("rhoai", "chat") is None+ # The caller's own doing, so it is told so: OpenAI's status and code.+ assert isinstance(raised.value, llm_gateway.LlmContextLengthError)+ assert raised.value.status_code == 400+ assert raised.value.code == "context_length_exceeded"+ assert (raised.value.max_tokens, raised.value.requested_tokens) == (32768, 32769)@pytest.mark.asyncioasync def test_an_overflow_does_not_erase_a_real_failure(self, monkeypatch):@@ -1952,6 +1959,7 @@ class TestRequestScopedFailuresStayOutOfTheBanner:@pytest.mark.asyncioasync def test_a_mid_stream_overflow_is_reported_but_not_recorded(self):+ """Emit a context-length SSE error without adding a provider health failure."""from services import llm_gateway, provider_error_logoverflow = self._OVERFLOW@@ -1967,9 +1975,185 @@ class TestRequestScopedFailuresStayOutOfTheBanner:error = json.loads(lines[-2][len("data: ") :])["error"]assert "maximum context length" in error["message"]+ assert error["code"] == "context_length_exceeded"assert provider_error_log.latest_failure("rhoai", "chat") is None+#: The provider's sentence from tracker issue 93001, in a watsonx error body.+_EMBEDDING_OVERFLOW = (+ 'watsonxException - {"errors":[{"message":"Invalid input argument for Model '+ "'intfloat/multilingual-e5-large': This model's maximum context length is 512 tokens. "+ "However, you requested 548 tokens in the input for embedding generation. Please reduce "+ 'the length of the input."}]}'+)+_E5 = {"model": "watsonx:intfloat/multilingual-e5-large", "input": ["one long chunk"]}+++class TestAnOverLengthEmbeddingInput:+ """One chunk too long for the model is that chunk's problem, not the provider's.++ It used to come back as a 502 and latch the health banner against a+ provider that was embedding every other chunk of the document fine.+ """++ @pytest.fixture(autouse=True)+ def _clean(self):+ """Clear provider failures before and after each embedding overflow test."""+ from services import provider_error_log++ provider_error_log.clear()+ yield+ provider_error_log.clear()++ def _raise(self, monkeypatch, exc: BaseException):+ """Make the embedding provider raise the supplied exception."""++ async def _boom(**_kwargs):+ """Simulate a failed upstream embedding call."""+ raise exc++ monkeypatch.setattr("litellm.aembedding", _boom)++ @pytest.mark.asyncio+ async def test_it_is_a_400_carrying_the_openai_code_and_the_providers_words(self, monkeypatch):+ """Keep the provider's message and token counts in a typed context-length 400."""+ self._raise(monkeypatch, _FakeLiteLLMError(_EMBEDDING_OVERFLOW, status_code=400))++ with pytest.raises(LlmContextLengthError) as raised:+ await embeddings(_E5, config=_config())++ assert raised.value.status_code == 400+ assert raised.value.code == "context_length_exceeded"+ assert (raised.value.max_tokens, raised.value.requested_tokens) == (512, 548)+ assert raised.value.message.startswith("watsonx/intfloat/multilingual-e5-large: ")+ assert "maximum context length is 512 tokens" in raised.value.message++ @pytest.mark.asyncio+ async def test_it_is_not_recorded_against_the_provider(self, monkeypatch):+ """Keep an oversized embedding input out of provider health failures."""+ from services import provider_error_log++ self._raise(monkeypatch, _FakeLiteLLMError(_EMBEDDING_OVERFLOW, status_code=400))++ with pytest.raises(LlmGatewayError):+ await embeddings(_E5, config=_config())++ assert provider_error_log.latest_failure("watsonx", "embedding") is None++ @pytest.mark.asyncio+ async def test_it_does_not_erase_a_real_failure(self, monkeypatch):+ """Preserve an earlier provider failure when an embedding input is too long."""+ from services import provider_error_log++ provider_error_log.record_failure("watsonx", "embedding", "earlier real failure")+ self._raise(monkeypatch, _FakeLiteLLMError(_EMBEDDING_OVERFLOW, status_code=400))++ with pytest.raises(LlmGatewayError):+ await embeddings(_E5, config=_config())++ assert provider_error_log.latest_failure("watsonx", "embedding") == "earlier real failure"++ @pytest.mark.asyncio+ async def test_an_overflow_without_counts_still_raises_the_typed_error(self, monkeypatch):+ """Classify overflow without guessing token counts the provider omitted."""+ self._raise(monkeypatch, RuntimeError('{"error": {"code": "context_length_exceeded"}}'))++ with pytest.raises(LlmContextLengthError) as raised:+ await embeddings(_E5, config=_config())++ assert (raised.value.max_tokens, raised.value.requested_tokens) == (None, None)++ @pytest.mark.asyncio+ async def test_other_embedding_failures_are_still_recorded(self, monkeypatch):+ """Keep non-context embedding failures as provider failures with a 502 status."""+ from services import provider_error_log++ self._raise(+ monkeypatch,+ _FakeLiteLLMError(+ '{"error": {"message": "The model `e5` does not exist."}}', status_code=404+ ),+ )++ with pytest.raises(LlmGatewayError) as raised:+ await embeddings(_E5, config=_config())++ assert not isinstance(raised.value, LlmContextLengthError)+ assert raised.value.status_code == 502+ assert "does not exist" in provider_error_log.latest_failure("watsonx", "embedding")+++ ("upstream_status", "upstream_text", "status_code", "code"),+ [+ # Passed through: a client can act on these itself.+ (429, "rate limit exceeded, retry in 30s", 429, "rate_limit_exceeded"),+ (408, "the request took too long", 408, "upstream_timeout"),+ (504, "no response from the model server", 504, "upstream_timeout"),+ (503, "the model is still loading", 503, "upstream_unavailable"),+ # Still a 502, so an upstream 4xx never reads as a bad request to us,+ # but no longer indistinguishable from one another.+ (500, "internal error", 502, "upstream_unavailable"),+ (401, "Incorrect API key provided", 502, "upstream_auth_error"),+ (403, "this project may not call the model", 502, "upstream_auth_error"),+ (404, "the model `e5` does not exist", 502, "upstream_invalid_request"),+ (400, "encoding_format is not supported for this model", 502, "upstream_invalid_request"),+ ],+)+async def test_embeddings_names_an_upstream_failure_and_keeps_its_status(+ monkeypatch, upstream_status, upstream_text, status_code, code+):+ """Map upstream embedding failures to the expected HTTP status and OpenAI code."""++ async def boom(**kwargs):+ """Raise an embedding failure with the parameterized upstream status."""+ raise _FakeLiteLLMError(upstream_text, status_code=upstream_status)++ monkeypatch.setattr("litellm.aembedding", boom)+ with pytest.raises(LlmGatewayError) as exc:+ await embeddings(_E5, config=_config())++ assert (exc.value.status_code, exc.value.code) == (status_code, code)+++class _FakeTimeout(ConnectionError):+ """LiteLLM's Timeout: a connection error by ancestry, a 408 by status."""++ status_code = 408+++class APIConnectionError(Exception):+ """Named as LiteLLM's is: the gateway matches on the class name, not the import."""++ status_code = 500+++ ("exc", "expected"),+ [+ # LiteLLM stamps its connection error with a 500; the provider never answered.+ (APIConnectionError("Cannot connect to host"), ("upstream_unreachable", 502)),+ (ConnectionRefusedError("[Errno 111] Connection refused"), ("upstream_unreachable", 502)),+ (+ RuntimeError("SSLCertVerificationError: certificate verify failed"),+ ("upstream_unreachable", 502),+ ),+ # A timeout is one too, by ancestry, but its own status says more.+ (_FakeTimeout("timed out"), ("upstream_timeout", 408)),+ (RuntimeError("something nobody anticipated"), ("upstream_error", 502)),+ ],+)+def test_a_failure_with_no_useful_status_is_named_from_what_raised_it(exc, expected):+ """Classify transport failures using exception ancestry and text as well as status."""+ from services.llm_gateway import _classify_upstream_failure++ failure = _classify_upstream_failure(f"{type(exc).__name__}: {exc}", exc)++ assert (failure.code, failure.status_code) == expected+ assert failure.counts_against_provider++class TestToolsBesideReasoningEffort:"""gpt-5.x refuses function tools next to its own default reasoning effort.diff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.pyindex dbd63ff2e..77f9c7799 100644--- a/tests/unit/test_provider_error_formatting.py+++ b/tests/unit/test_provider_error_formatting.py@@ -7,6 +7,7 @@ import jsonimport pytestfrom api.provider_validation import (+ context_window_token_counts,format_provider_error_message,is_context_window_error,is_generic_upstream_error,@@ -97,6 +98,17 @@ def test_looks_like_provider_error_content():assert is_generic_upstream_error("An unknown error occurred.")+def test_looks_like_provider_error_content_reads_the_sdk_python_dict():+ """The OpenAI SDK prints the body as a Python dict, which is how Langflow relays it."""+ assert looks_like_provider_error_content(+ "Error code: 400 - {'error': {'message': \"This model's maximum context length is "+ "512 tokens.\", 'code': 'context_length_exceeded'}}"+ )+ assert not looks_like_provider_error_content(+ "Send {'message': 'hello', 'user': 'ana'} as the request body."+ )++def test_resolve_chat_stream_error_message_keeps_generic_without_probing():# Sync helper must not probe — that stays in the async path.assert (@@ -526,20 +538,134 @@ async def test_probe_checks_non_selected_providers_with_keys(monkeypatch):'{"error": {"code": "context_length_exceeded"}}',"ContextWindowExceededError: litellm.ContextWindowExceededError: ...","prompt is too long: 210000 tokens > 200000 maximum",+ # An oversized chunk sent for embedding (tracker issue 93001).+ "This model's maximum context length is 512 tokens. However, you requested 548 "+ "tokens in the input for embedding generation.",+ # Wordings LiteLLM recognises without always raising its typed error.+ "the request exceeds the available context size, try increasing it",+ "Input tokens exceed the configured limit of 272000 tokens.",+ "The input token count exceeds the maximum number of tokens allowed (1048575).",],)def test_is_context_window_error(text):+ """Recognize the context-window error wordings used by supported providers."""assert is_context_window_error(text)@pytest.mark.parametrize("text",- [None, "", "Incorrect API key provided", "The model `x` does not exist", "rate limit"],+ [+ None,+ "",+ "Incorrect API key provided",+ "The model `x` does not exist",+ "rate limit",+ # A limit on tokens per minute is the provider's, not this request's.+ "Rate limit reached: limit 30000 tokens per min, requested 31000",+ ],)def test_is_context_window_error_rejects_other_failures(text):+ """Distinguish context-window overflow from credentials and rate-limit failures."""assert not is_context_window_error(text)+ ("text", "expected"),+ [+ # watsonx.ai / vLLM embeddings, as reported in tracker issue 93001.+ (+ "This model's maximum context length is 512 tokens. However, you requested 548 "+ "tokens in the input for embedding generation.",+ (512, 548),+ ),+ # vLLM chat also says "requested 0 output tokens": not the prompt's size.+ (+ "This model's maximum context length is 32768 tokens. However, you requested 0 "+ "output tokens and your prompt contains at least 32769 input tokens",+ (32768, 32769),+ ),+ (+ "This model's maximum context length is 8192 tokens, however you requested 9000 "+ "tokens (9000 in your prompt; 0 for the completion).",+ (8192, 9000),+ ),+ (+ "This model's maximum context length is 4097 tokens. However, your messages "+ "resulted in 4275 tokens. Please reduce the length of the messages.",+ (4097, 4275),+ ),+ ("prompt is too long: 210000 tokens > 200000 maximum", (200000, 210000)),+ # Stated limit, unstated size: half an answer beats a guessed one.+ ("This model's maximum context length is 512 tokens.", (512, None)),+ ('{"error": {"code": "context_length_exceeded"}}', (None, None)),+ (None, (None, None)),+ ],+)+def test_context_window_token_counts(text, expected):+ """Extract stated input limits and counts without mistaking output tokens for input."""+ assert context_window_token_counts(text) == expected+++_EMBEDDING_OVERFLOW = (+ "watsonx/intfloat/multilingual-e5-large: Invalid input argument for Model "+ "'intfloat/multilingual-e5-large': This model's maximum context length is 512 tokens. "+ "However, you requested 548 tokens in the input for embedding generation. Please reduce "+ "the length of the input."+)+#: How the OpenAI SDK prints a gateway error: a Python dict, not JSON.+_SDK_ERROR = (+ f"Error code: 400 - {{'error': {{'message': \"{_EMBEDDING_OVERFLOW}\", "+ "'type': 'invalid_request_error', 'code': 'context_length_exceeded'}}"+)+++ "raw",+ [+ _SDK_ERROR,+ "Error building Component OpenSearch (Multi-Model Multi-Embedding): \n\n" + _SDK_ERROR,+ "Error code: 400 - " + json.dumps({"error": {"message": _EMBEDDING_OVERFLOW}}),+ _EMBEDDING_OVERFLOW,+ ],+ ids=["sdk-dict", "langflow-wrapped", "json", "bare"],+)+def test_sanitize_keeps_the_providers_sentence_whatever_wraps_it(raw):+ """The dict form used to collapse to "502 -", which is all the file's error showed."""+ assert sanitize_provider_error_content(raw) == _EMBEDDING_OVERFLOW+++def test_sanitize_reads_a_python_dict_quoted_either_way():+ """Preserve apostrophes and quotes when extracting an SDK dictionary message."""+ raw = (+ "Error code: 404 - {'error': {'message': 'The model `e5` doesn\\'t exist, "+ "said \"the API\"', 'type': 'api_error'}}"+ )+ assert sanitize_provider_error_content(raw) == 'The model `e5` doesn\'t exist, said "the API"'+++ "raw",+ [+ "Error code: 502 - {'error': <Response [502]>}",+ "502 - {not json",+ "Error code: 502 - {'error': {'type': 'api_error'}}",+ ],+)+def test_sanitize_never_passes_off_a_bare_status_code_as_the_message(raw):+ """Use the fallback when an unreadable error body leaves only a status code."""+ assert sanitize_provider_error_content(raw) == "An error occurred while generating a response."+++async def test_resolve_ingest_error_message_keeps_the_gateway_error_langflow_relayed():+ """No probe is needed for it: the provider already said exactly what is wrong."""+ from api.provider_validation import resolve_ingest_error_message++ raw = "Error building Component OpenSearch (Multi-Model Multi-Embedding): \n\n" + _SDK_ERROR++ assert await resolve_ingest_error_message(raw) == _EMBEDDING_OVERFLOW++_KUBE_RBAC_PROXY_CUTOFF = ("MidStreamFallbackError: litellm.MidStreamFallbackError: litellm.APIConnectionError: ""APIConnectionError: Hosted_vllmException - Response payload is not completed: "diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.pyindex 5b4effb83..8af68728a 100644--- a/tests/unit/test_task_service_get_task_status2.py+++ b/tests/unit/test_task_service_get_task_status2.py@@ -312,6 +312,49 @@ class TestInferFailureMetadata:assert "error running graph" not in meta["user_facing_message"].lower()assert "{" not in meta["user_facing_message"]+ def test_chunk_too_long_for_the_embedding_model_is_user_actionable(self, task_service):+ """Retrying resends the same chunk, so RETRYABLE would be a false promise."""+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error=(+ "watsonx/intfloat/multilingual-e5-large: Invalid input argument for Model "+ "'intfloat/multilingual-e5-large': This model's maximum context length is 512 "+ "tokens. However, you requested 548 tokens in the input for embedding generation."+ ),+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta == {+ "component": "openrag",+ "failure_phase": "embedding",+ "user_facing_message": (+ "A section of this document is longer than the embedding model accepts "+ "(548 tokens; the limit is 512). Try a smaller chunk size or an embedding "+ "model with a larger input limit, then retry ingestion."+ ),+ "actionable_by": "USER_ACTIONABLE",+ }++ @pytest.mark.parametrize(+ ("error", "size"),+ [+ ("This model's maximum context length is 512 tokens.", " (the limit is 512 tokens)"),+ ("Error code: 400 - {'error': {'code': 'context_length_exceeded'}}", ""),+ ],+ )+ def test_chunk_too_long_states_only_the_counts_the_provider_gave(+ self, task_service, error, size+ ):+ """Report embedding overflow as actionable using only known token counts."""+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW, docling_status=DoclingPhaseStatus.SUCCESS, error=error+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta["user_facing_message"].startswith(+ f"A section of this document is longer than the embedding model accepts{size}. "+ )+ assert meta["actionable_by"] == "USER_ACTIONABLE"+def test_langflow_disconnect_with_api_key_text_prefers_credentials(self, task_service):ft = _make_file_task(phase=IngestionPhase.LANGFLOW,