Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: embed an over-length input in pieces instead of failing it (#2477)

Rico Furtado9740dbb6c815
diff --git a/.env.example b/.env.example
index 3dcc119ed..d383ecfc3 100644
--- a/.env.example
+++ b/.env.example
@@ -3,6 +3,13 @@
# If unset or false, Langflow pipeline will be used (default: upload -> ingest -> delete)
DISABLE_INGEST_WITH_LANGFLOW=false
+# What to do when the embedding model rejects a chunk or query as longer than
+# its input limit (for example 512 tokens).
+# average (default): embed the text in smaller pieces and use the
+# length-weighted average of their vectors, so ingestion continues.
+# error: fail the request; the file reports that a section is too long.
+# OPENRAG_EMBEDDING_OVERFLOW_POLICY=average
+
# Set to false to skip ingesting OpenRAG sample docs during onboarding
# Default: true
INGEST_SAMPLE_DATA=true
diff --git a/docker-compose.yml b/docker-compose.yml
index aba3fd087..76b92006f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -67,6 +67,7 @@ services:
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
- LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID}
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
+ - OPENRAG_EMBEDDING_OVERFLOW_POLICY=${OPENRAG_EMBEDDING_OVERFLOW_POLICY:-}
- INGEST_SAMPLE_DATA=${INGEST_SAMPLE_DATA:-true}
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
- OPENSEARCH_PORT=${OPENSEARCH_INTERNAL_PORT:-9200}
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index e8ab24f34..f294e2cc4 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -244,6 +244,7 @@ For Langflow flow IDs and Langflow timeout settings, see [Langflow settings](#la
| `HOST_DOCKER_INTERNAL` | `host.docker.internal` | Host address for accessing services running on the host machine from within Docker containers. This is used to connect to the local Docling Serve instance started by OpenRAG. If your system uses a different hostname for this purpose, set this variable accordingly. |
| `INGESTION_TIMEOUT` | `3600` | Document ingestion timeout limit in seconds for each file. Increase this value if you experience timeouts when ingesting very large documents. Must be greater than or equal to [`LANGFLOW_TIMEOUT`](#langflow-settings). |
| `UPLOAD_BATCH_SIZE` | `25` | When ingesting folders, set the maximum number of files to ingest per batch. Each batch is an [ingestion task](/ingestion#monitor-ingestion). Increase this value to ingest more files per batch. If this value is too high, performance issues can occur. |
+| `OPENRAG_EMBEDDING_OVERFLOW_POLICY` | `average` | What OpenRAG does when the embedding model rejects a chunk or query as longer than its input limit. With `average`, OpenRAG embeds the text in smaller pieces and stores the length-weighted average of their vectors, so ingestion continues; the full text is still stored and searchable by keyword. With `error`, the request fails and the file reports that a section is longer than the model accepts. |
## Langflow settings {#langflow-settings}
diff --git a/src/config/settings.py b/src/config/settings.py
index b7509ecea..89a2bca92 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -658,6 +658,18 @@ DISABLE_INGEST_WITH_LANGFLOW = os.getenv("DISABLE_INGEST_WITH_LANGFLOW", "false"
"yes",
)
+
+def embedding_overflow_policy() -> str:
+ """What the gateway does with an embedding input the model rejects as too long.
+
+ ``average`` (the default) embeds the input in pieces and returns their
+ averaged vector, so the caller still gets one vector per input. ``error``
+ returns the provider's rejection as a 400 instead.
+ """
+ raw = os.getenv("OPENRAG_EMBEDDING_OVERFLOW_POLICY", "").strip().lower()
+ return "error" if raw == "error" else "average"
+
+
# Show the "+" file upload button in the chat input
OPENRAG_INGEST_VIA_CHAT = os.getenv("OPENRAG_INGEST_VIA_CHAT", "false").lower() in (
"true",
diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.py
index b81e86c35..5140a439e 100644
--- a/src/services/llm_gateway.py
+++ b/src/services/llm_gateway.py
@@ -9,11 +9,12 @@ from __future__ import annotations
import asyncio
import contextlib
-import functools
import json
+import math
import re
from collections import OrderedDict, deque
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
+from itertools import pairwise
from typing import Any, Literal, NamedTuple
from uuid import uuid4
@@ -1832,6 +1833,154 @@ def _embedding_input(value: Any) -> Any:
return [value] if isinstance(value, str) else value
+#: How many times one input is cut smaller before the gateway gives up on it.
+_MAX_OVERFLOW_SPLIT_ROUNDS = 3
+#: Pieces are sized under the proportion the provider reported: tokens are not
+#: spread evenly through a text, and a cut may land a little short of its mark.
+_OVERFLOW_HEADROOM = 0.85
+
+
+def _is_text_list(value: Any) -> bool:
+ """Whether `value` is a batch of strings, the only input that can be cut into pieces."""
+ return isinstance(value, list) and bool(value) and all(isinstance(item, str) for item in value)
+
+
+def _split_into_pieces(
+ text: str, max_tokens: int | None, requested_tokens: int | None
+) -> list[str]:
+ """Cut `text` into equal pieces expected to fit the model, at whitespace where it can.
+
+ The provider's own counts say how far over the input was, so the number of
+ pieces follows that proportion; with no counts, the text is halved. Equal
+ pieces rather than "as much as fits, then the rest": a short tail would
+ carry little context of its own into the average.
+ """
+ count = 2
+ if max_tokens and requested_tokens and requested_tokens > max_tokens:
+ count = max(2, math.ceil(requested_tokens / (max_tokens * _OVERFLOW_HEADROOM)))
+ size = len(text) / count
+ slack = int(size // 10)
+ cuts = [0]
+ for number in range(1, count):
+ ideal = round(number * size)
+ earliest = max(ideal - slack, cuts[-1] + 1)
+ cuts.append(
+ next((at for at in range(ideal, earliest - 1, -1) if text[at - 1].isspace()), ideal)
+ )
+ cuts.append(len(text))
+ return [piece for start, end in pairwise(cuts) if (piece := text[start:end]).strip()]
+
+
+def _weighted_mean_vector(vectors: list[list[float]], weights: list[int]) -> list[float]:
+ """The mean of `vectors` weighted by `weights`, at the length the model's vectors have.
+
+ A plain mean of vectors that point different ways is shorter than any of
+ them, which would understate the chunk wherever the index compares raw
+ lengths (l2, inner product). Scaling it back to the pieces' weighted mean
+ length keeps it comparable to every other vector from the same model: unit
+ length when the model normalises its output, its usual magnitude when not.
+ """
+ total = sum(weights)
+ mean = [
+ sum(vector[axis] * weight for vector, weight in zip(vectors, weights, strict=True)) / total
+ for axis in range(len(vectors[0]))
+ ]
+ length = math.hypot(*mean)
+ if not length:
+ return mean
+ target = (
+ sum(math.hypot(*vector) * weight for vector, weight in zip(vectors, weights, strict=True))
+ / total
+ )
+ return [value * target / length for value in mean]
+
+
+async def _embed_averaging_overflow(
+ send: Callable[[list[str]], Awaitable[dict[str, Any]]],
+ texts: list[str],
+ overflow: LlmContextLengthError,
+ *,
+ provider: str,
+ model: str,
+ max_batch: int | None = None,
+) -> dict[str, Any]:
+ """Embed a batch the provider rejected as too long, still one vector per input.
+
+ An embeddings call owes its caller exactly one vector per input, so an
+ over-length input cannot become several chunks here. It is embedded in
+ pieces and the piece vectors are averaged by length instead, so every part
+ of the text still contributes to the vector. The provider says only that
+ something in the batch was too long, so the batch is halved until the
+ offending input is alone. `send` raises `LlmContextLengthError` for a
+ rejection of that kind and `LlmGatewayError` for anything else. With
+ `max_batch`, no request carries more inputs than that, even when one input
+ is cut into more pieces.
+ """
+ envelope: dict[str, Any] = {}
+ usage: dict[str, int | float] = {}
+
+ async def embed(
+ batch: list[str], rounds: int, rejected: LlmContextLengthError | None = None
+ ) -> list[list[float]]:
+ if rejected is None and max_batch and len(batch) > max_batch:
+ vectors: list[list[float]] = []
+ for start in range(0, len(batch), max_batch):
+ vectors += await embed(batch[start : start + max_batch], rounds)
+ return vectors
+ if rejected is None:
+ try:
+ payload = await send(batch)
+ except LlmContextLengthError as error:
+ rejected = error
+ else:
+ if not envelope:
+ envelope.update(
+ {
+ key: value
+ for key, value in payload.items()
+ if key not in {"data", "usage"}
+ }
+ )
+ for key, value in (payload.get("usage") or {}).items():
+ if isinstance(value, (int, float)):
+ usage[key] = usage.get(key, 0) + value
+ items = sorted(payload.get("data", []), key=lambda item: item.get("index", 0))
+ return [item["embedding"] for item in items]
+ if len(batch) > 1:
+ middle = len(batch) // 2
+ return await embed(batch[:middle], rounds) + await embed(batch[middle:], rounds)
+ return [await average(batch[0], rounds, rejected)]
+
+ async def average(text: str, rounds: int, rejected: LlmContextLengthError) -> list[float]:
+ pieces = _split_into_pieces(text, rejected.max_tokens, rejected.requested_tokens)
+ if rounds >= _MAX_OVERFLOW_SPLIT_ROUNDS or len(pieces) < 2:
+ raise rejected
+ vectors = await embed(pieces, rounds + 1)
+ # Sizes only: the text itself is the customer's document.
+ logger.warning(
+ "Embedding input exceeded the model's limit; embedded in pieces and averaged",
+ provider=provider,
+ model=model,
+ max_tokens=rejected.max_tokens,
+ requested_tokens=rejected.requested_tokens,
+ chars=len(text),
+ pieces=len(pieces),
+ )
+ return _weighted_mean_vector(vectors, [len(piece) for piece in pieces])
+
+ vectors = await embed(texts, 0, overflow)
+ response = {
+ **envelope,
+ "data": [
+ {"object": "embedding", "index": index, "embedding": vector}
+ for index, vector in enumerate(vectors)
+ ],
+ }
+ if usage:
+ response["usage"] = usage
+ return response
+
+
async def embeddings(
body: Mapping[str, Any], *, config=None, interactive: bool = False
) -> dict[str, Any]:
@@ -1840,7 +1989,13 @@ async def embeddings(
`interactive` marks a query embedding someone is waiting on (search, chat
retrieval). At a provider with a concurrency limit it waits in its own
lane instead of behind bulk ingestion; see `_embedding_limiter`.
+
+ An input the model rejects as too long is embedded in pieces and averaged
+ rather than failed, unless `OPENRAG_EMBEDDING_OVERFLOW_POLICY=error`; see
+ `_embed_averaging_overflow`.
"""
+ from config.settings import embedding_overflow_policy
+
cfg = config or _get_config()
litellm_model, provider, credentials = resolve_call(
body.get("model"), kind="embedding", config=cfg
@@ -1849,53 +2004,73 @@ async def embeddings(
limiter = _embedding_limiter(provider, cfg)
lane = _INTERACTIVE_LANE if interactive else _BULK_LANE
embedding_input = _embedding_input(body.get("input"))
+
+ batch_size = _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE if provider == "watsonx_onprem" else None
+
+ async def call_provider(inputs: Any) -> dict[str, Any]:
+ import litellm
+
+ async with _embedding_slot(limiter, provider, litellm_model, lane):
+ result = await with_stale_connection_retry(
+ lambda: litellm.aembedding(
+ model=litellm_model,
+ input=inputs,
+ **credentials,
+ **runtime_kwargs,
+ ),
+ provider=provider,
+ model=litellm_model,
+ kind="embedding",
+ )
+ return _to_openai_dict(result)
+
+ async def send(inputs: Any) -> dict[str, Any]:
+ try:
+ return await call_provider(inputs)
+ except LlmGatewayError:
+ raise
+ except Exception as exc:
+ detail = _redact(f"{type(exc).__name__}: {exc}", credentials)
+ message = _upstream_client_message(detail, provider, litellm_model, exc)
+ # An over-length input is one chunk's problem. Recording it latched the
+ # banner against a provider that was embedding every other chunk fine.
+ failure = _classify_upstream_failure(detail, exc)
+ if failure.counts_against_provider:
+ provider_error_log.record_failure(provider, "embedding", message)
+ raise _upstream_gateway_error(message, detail, failure) from exc
+
+ async def embed(inputs: Any) -> dict[str, Any]:
+ try:
+ return await send(inputs)
+ except LlmContextLengthError as overflow:
+ if embedding_overflow_policy() != "average" or not _is_text_list(inputs):
+ raise
+ return await _embed_averaging_overflow(
+ send,
+ inputs,
+ overflow,
+ provider=provider,
+ model=litellm_model,
+ max_batch=batch_size,
+ )
+
should_batch = (
- provider == "watsonx_onprem"
+ batch_size is not None
and isinstance(embedding_input, list)
- and len(embedding_input) > _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE
+ and len(embedding_input) > batch_size
and not all(isinstance(item, int) for item in embedding_input)
)
try:
- import litellm
-
if not should_batch:
- async with _embedding_slot(limiter, provider, litellm_model, lane):
- result = await with_stale_connection_retry(
- lambda: litellm.aembedding(
- model=litellm_model,
- input=embedding_input,
- **credentials,
- **runtime_kwargs,
- ),
- provider=provider,
- model=litellm_model,
- kind="embedding",
- )
- response = _to_openai_dict(result)
+ response = await embed(embedding_input)
else:
+ # Each batch recovers on its own, so an over-length input in a later
+ # batch does not send the earlier, already embedded ones again.
response = {}
data: list[dict[str, Any]] = []
usage: dict[str, int | float] = {}
- for offset in range(
- 0,
- len(embedding_input),
- _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE,
- ):
- batch = embedding_input[offset : offset + _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE]
- async with _embedding_slot(limiter, provider, litellm_model, lane):
- result = await with_stale_connection_retry(
- functools.partial(
- litellm.aembedding,
- model=litellm_model,
- input=batch,
- **credentials,
- **runtime_kwargs,
- ),
- provider=provider,
- model=litellm_model,
- kind="embedding",
- )
- payload = _to_openai_dict(result)
+ for offset in range(0, len(embedding_input), batch_size):
+ payload = await embed(embedding_input[offset : offset + batch_size])
if not response:
response = {
key: value for key, value in payload.items() if key not in {"data", "usage"}
@@ -1910,17 +2085,10 @@ async def embeddings(
response["data"] = data
if usage:
response["usage"] = usage
- except LlmGatewayError:
+ except LlmGatewayError as error:
+ logger.error(
+ "LLM embeddings failed", provider=provider, model=litellm_model, error=error.detail
+ )
raise
- except Exception as exc:
- detail = _redact(f"{type(exc).__name__}: {exc}", credentials)
- logger.error("LLM embeddings failed", provider=provider, model=litellm_model, error=detail)
- message = _upstream_client_message(detail, provider, litellm_model, exc)
- # An over-length input is one chunk's problem. Recording it latched the
- # banner against a provider that was embedding every other chunk fine.
- failure = _classify_upstream_failure(detail, exc)
- if failure.counts_against_provider:
- provider_error_log.record_failure(provider, "embedding", message)
- raise _upstream_gateway_error(message, detail, failure) from exc
provider_error_log.record_success(provider, "embedding")
return response
diff --git a/tests/unit/services/test_llm_gateway.py b/tests/unit/services/test_llm_gateway.py
index 4576912d4..bcad5e5cc 100644
--- a/tests/unit/services/test_llm_gateway.py
+++ b/tests/unit/services/test_llm_gateway.py
@@ -1,6 +1,7 @@
"""LLM gateway: route OpenAI-shaped requests through LiteLLM using OpenRAG config."""
import json
+import math
from types import SimpleNamespace
import pytest
@@ -2154,6 +2155,390 @@ def test_a_failure_with_no_useful_status_is_named_from_what_raised_it(exc, expec
assert failure.counts_against_provider
+def _direction(text: str) -> list[float]:
+ """A unit vector that leans towards "a" or "b" by how much of each the text has."""
+ a, b = text.count("a"), text.count("b")
+ length = math.hypot(a, b) or 1.0
+ return [a / length, b / length]
+
+
+class _LimitedProvider:
+ """Stands in for `litellm.aembedding` at a model with an input limit.
+
+ One character is one token, plus `dense` more for every "z", so a test can
+ pack more tokens into one part of a text than another.
+ """
+
+ def __init__(self, limit: int, *, state_counts: bool = True, dense: int = 0, vector=_direction):
+ self.limit = limit
+ self.state_counts = state_counts
+ self.dense = dense
+ self.vector = vector
+ self.calls: list[list] = []
+ self.rejections = 0
+
+ def tokens(self, text: str) -> int:
+ return len(text) + self.dense * text.count("z")
+
+ async def __call__(self, **kwargs):
+ inputs = kwargs["input"]
+ self.calls.append(list(inputs))
+ longest = max(self.tokens(text) for text in inputs)
+ if longest > self.limit:
+ self.rejections += 1
+ raise _FakeLiteLLMError(
+ f"This model's maximum context length is {self.limit} tokens. However, you "
+ f"requested {longest} tokens in the input for embedding generation."
+ if self.state_counts
+ else '{"error": {"code": "context_length_exceeded"}}',
+ status_code=400,
+ )
+ spent = sum(self.tokens(text) for text in inputs)
+ return {
+ "object": "list",
+ "model": "intfloat/multilingual-e5-large",
+ "data": [
+ {"object": "embedding", "index": index, "embedding": self.vector(text)}
+ for index, text in enumerate(inputs)
+ ],
+ "usage": {"prompt_tokens": spent, "total_tokens": spent},
+ }
+
+
+#: 100 characters against a limit of 80: two pieces of 50, cut at the space.
+_OVER_LIMIT = "a" * 49 + " " + "b" * 50
+_E5_MODEL = "watsonx:intfloat/multilingual-e5-large"
+
+
+class TestAnOverLengthEmbeddingInputIsAveraged:
+ """The gateway embeds an over-length input in pieces and returns their average.
+
+ `/v1/embeddings` owes one vector per input, so the caller cannot be handed
+ more chunks; failing instead stopped whole documents ingesting through
+ flows that have no way to split the chunk themselves.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _clean(self, monkeypatch):
+ from services import provider_error_log
+
+ monkeypatch.delenv("OPENRAG_EMBEDDING_OVERFLOW_POLICY", raising=False)
+ provider_error_log.clear()
+ yield
+ provider_error_log.clear()
+
+ def _provider(self, monkeypatch, provider):
+ monkeypatch.setattr("litellm.aembedding", provider)
+ return provider
+
+ @pytest.mark.asyncio
+ async def test_one_vector_comes_back_for_the_input(self, monkeypatch):
+ from services import provider_error_log
+
+ provider = self._provider(monkeypatch, _LimitedProvider(80))
+
+ result = await embeddings({"model": _E5_MODEL, "input": [_OVER_LIMIT]}, config=_config())
+
+ assert provider.calls == [[_OVER_LIMIT], [_OVER_LIMIT[:50], _OVER_LIMIT[50:]]]
+ # Half "a" and half "b" by length, at the unit length the pieces have.
+ assert result["data"] == [
+ {"object": "embedding", "index": 0, "embedding": pytest.approx([0.5**0.5, 0.5**0.5])}
+ ]
+ assert result["model"] == "intfloat/multilingual-e5-large"
+ assert result["usage"] == {"prompt_tokens": 100, "total_tokens": 100}
+ assert provider_error_log.latest_failure("watsonx", "embedding") is None
+
+ @pytest.mark.asyncio
+ async def test_a_bare_string_input_is_handled_the_same_way(self, monkeypatch):
+ self._provider(monkeypatch, _LimitedProvider(80))
+
+ result = await embeddings({"model": _E5_MODEL, "input": _OVER_LIMIT}, config=_config())
+
+ assert len(result["data"]) == 1
+
+ @pytest.mark.asyncio
+ async def test_the_warning_gives_the_sizes_and_never_the_text(self, monkeypatch):
+ from unittest.mock import MagicMock
+
+ from services import llm_gateway
+
+ log = MagicMock()
+ monkeypatch.setattr(llm_gateway, "logger", log)
+ self._provider(monkeypatch, _LimitedProvider(80))
+
+ await embeddings({"model": _E5_MODEL, "input": [_OVER_LIMIT]}, config=_config())
+
+ assert [call.kwargs for call in log.warning.call_args_list] == [
+ {
+ "provider": "watsonx",
+ "model": "watsonx/intfloat/multilingual-e5-large",
+ "max_tokens": 80,
+ "requested_tokens": 100,
+ "chars": 100,
+ "pieces": 2,
+ }
+ ]
+ assert "aaaa" not in repr(log.mock_calls)
+ log.error.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_only_the_offending_input_in_a_batch_is_cut(self, monkeypatch):
+ provider = self._provider(monkeypatch, _LimitedProvider(80))
+ batch = ["aaa", _OVER_LIMIT, "bbb", "aab"]
+
+ result = await embeddings({"model": _E5_MODEL, "input": batch}, config=_config())
+
+ # The provider does not say which input was too long, so the batch is
+ # halved until that input is alone; every other input is sent whole.
+ assert provider.calls == [
+ batch,
+ ["aaa", _OVER_LIMIT],
+ ["aaa"],
+ [_OVER_LIMIT],
+ [_OVER_LIMIT[:50], _OVER_LIMIT[50:]],
+ ["bbb", "aab"],
+ ]
+ assert [item["index"] for item in result["data"]] == [0, 1, 2, 3]
+ assert [item["embedding"] for item in result["data"]] == [
+ _direction("aaa"),
+ pytest.approx([0.5**0.5, 0.5**0.5]),
+ _direction("bbb"),
+ _direction("aab"),
+ ]
+
+ @pytest.mark.asyncio
+ async def test_a_rejection_without_counts_halves_the_input(self, monkeypatch):
+ provider = self._provider(monkeypatch, _LimitedProvider(80, state_counts=False))
+
+ result = await embeddings({"model": _E5_MODEL, "input": [_OVER_LIMIT]}, config=_config())
+
+ assert provider.calls[1] == [_OVER_LIMIT[:50], _OVER_LIMIT[50:]]
+ assert len(result["data"]) == 1
+
+ @pytest.mark.asyncio
+ async def test_a_piece_that_is_still_too_long_is_cut_again(self, monkeypatch):
+ # Every token-heavy "z" sits in the second half, so pieces sized by the
+ # overall proportion fit in the first half and overflow in the second.
+ text = ("a" * 9 + " ") * 5 + ("z" * 9 + " ") * 5
+ provider = self._provider(
+ monkeypatch, _LimitedProvider(100, dense=9, vector=lambda _text: [1.0, 0.0])
+ )
+
+ result = await embeddings({"model": _E5_MODEL, "input": [text]}, config=_config())
+
+ assert result["data"][0]["embedding"] == pytest.approx([1.0, 0.0])
+ assert len(result["data"]) == 1
+ # The whole text, then the first set of pieces, then pieces of pieces.
+ assert provider.rejections > 2
+
+ @pytest.mark.asyncio
+ async def test_it_gives_up_after_three_rounds_of_cutting(self, monkeypatch):
+ from services import provider_error_log
+
+ # A limit nothing can meet, so every piece is rejected however small.
+ provider = self._provider(monkeypatch, _LimitedProvider(0))
+
+ with pytest.raises(LlmContextLengthError) as raised:
+ await embeddings({"model": _E5_MODEL, "input": ["one long chunk"]}, config=_config())
+
+ assert raised.value.status_code == 400
+ # One call for the input, then two per round: the pieces, and one of them alone.
+ assert len(provider.calls) == 7
+ assert provider_error_log.latest_failure("watsonx", "embedding") is None
+
+ @pytest.mark.asyncio
+ async def test_the_error_policy_returns_the_rejection_untouched(self, monkeypatch):
+ monkeypatch.setenv("OPENRAG_EMBEDDING_OVERFLOW_POLICY", "error")
+ provider = self._provider(monkeypatch, _LimitedProvider(80))
+
+ with pytest.raises(LlmContextLengthError) as raised:
+ await embeddings({"model": _E5_MODEL, "input": [_OVER_LIMIT]}, config=_config())
+
+ assert provider.calls == [[_OVER_LIMIT]]
+ assert (raised.value.max_tokens, raised.value.requested_tokens) == (80, 100)
+
+ @pytest.mark.asyncio
+ async def test_token_array_inputs_are_not_cut(self, monkeypatch):
+ calls = []
+
+ async def reject(**kwargs):
+ calls.append(kwargs["input"])
+ raise _FakeLiteLLMError(_EMBEDDING_OVERFLOW, status_code=400)
+
+ monkeypatch.setattr("litellm.aembedding", reject)
+
+ with pytest.raises(LlmContextLengthError):
+ await embeddings({"model": _E5_MODEL, "input": [[101, 2023, 102]]}, config=_config())
+
+ assert calls == [[[101, 2023, 102]]]
+
+ @pytest.mark.asyncio
+ async def test_another_failure_among_the_pieces_is_reported_as_itself(self, monkeypatch):
+ from services import provider_error_log
+
+ calls = []
+
+ async def overflow_then_rate_limit(**kwargs):
+ calls.append(kwargs["input"])
+ if len(calls) == 1:
+ raise _FakeLiteLLMError(_EMBEDDING_OVERFLOW, status_code=400)
+ raise _FakeLiteLLMError("rate limit exceeded, retry in 30s", status_code=429)
+
+ monkeypatch.setattr("litellm.aembedding", overflow_then_rate_limit)
+
+ with pytest.raises(LlmGatewayError) as raised:
+ await embeddings({"model": _E5_MODEL, "input": [_OVER_LIMIT]}, config=_config())
+
+ assert (raised.value.status_code, raised.value.code) == (429, "rate_limit_exceeded")
+ assert len(calls) == 2
+ assert "rate limit" in provider_error_log.latest_failure("watsonx", "embedding")
+
+ @pytest.mark.asyncio
+ async def test_a_rejection_it_does_not_recognise_is_left_alone(self, monkeypatch):
+ calls = []
+
+ async def reject(**kwargs):
+ calls.append(kwargs["input"])
+ raise _FakeLiteLLMError("the input is too large for this deployment", status_code=400)
+
+ monkeypatch.setattr("litellm.aembedding", reject)
+
+ with pytest.raises(LlmGatewayError) as raised:
+ await embeddings({"model": _E5_MODEL, "input": [_OVER_LIMIT]}, config=_config())
+
+ assert (raised.value.status_code, raised.value.code) == (502, "upstream_invalid_request")
+ assert len(calls) == 1
+
+ def _onprem(self, monkeypatch, provider):
+ """`provider` behind watsonx on-prem, which takes at most 32 inputs a request."""
+ self._provider(monkeypatch, provider)
+ monkeypatch.setattr(
+ "services.llm_gateway._provider_runtime_kwargs", lambda provider, config: {}
+ )
+ return SimpleNamespace(
+ providers=ProvidersConfig(
+ openai=OpenAIConfig(),
+ anthropic=AnthropicConfig(),
+ watsonx=WatsonXConfig(),
+ ollama=OllamaConfig(),
+ custom={
+ "watsonx_onprem": GenericProviderConfig(
+ credentials={
+ "api_base": "https://cpd.example.com",
+ "username": "cpduser",
+ "api_key": "secret",
+ },
+ configured=True,
+ )
+ },
+ )
+ )
+
+ @pytest.mark.asyncio
+ async def test_an_overflow_in_a_later_batch_does_not_resend_the_earlier_ones(self, monkeypatch):
+ provider = _LimitedProvider(80)
+ config = self._onprem(monkeypatch, provider)
+ batch = [f"a{index}" for index in range(32)] + [_OVER_LIMIT]
+
+ result = await embeddings(
+ {"model": "watsonx_onprem:ibm/slate-30m-english-rtrvr", "input": batch},
+ config=config,
+ )
+
+ assert provider.calls == [
+ batch[:32],
+ [_OVER_LIMIT],
+ [_OVER_LIMIT[:50], _OVER_LIMIT[50:]],
+ ]
+ assert [item["index"] for item in result["data"]] == list(range(33))
+ assert result["data"][32]["embedding"] == pytest.approx([0.5**0.5, 0.5**0.5])
+ # The 32 that fitted were embedded and billed once.
+ assert result["usage"]["prompt_tokens"] == sum(len(text) for text in batch[:32]) + 100
+
+ @pytest.mark.asyncio
+ async def test_an_input_cut_into_many_pieces_keeps_to_the_batch_size(self, monkeypatch):
+ provider = _LimitedProvider(10)
+ config = self._onprem(monkeypatch, provider)
+ text = "ab " * 133 + "a"
+
+ result = await embeddings(
+ {"model": "watsonx_onprem:ibm/slate-30m-english-rtrvr", "input": [text]},
+ config=config,
+ )
+
+ # 400 tokens against 10 means 48 pieces: sent as 32 and 16, never as 48.
+ assert [len(call) for call in provider.calls] == [1, 32, 16]
+ assert len(result["data"]) == 1
+
+
+ ("value", "policy"),
+ [(None, "average"), ("", "average"), ("average", "average"), (" Error ", "error")],
+)
+def test_embedding_overflow_policy_defaults_to_averaging(monkeypatch, value, policy):
+ from config.settings import embedding_overflow_policy
+
+ if value is None:
+ monkeypatch.delenv("OPENRAG_EMBEDDING_OVERFLOW_POLICY", raising=False)
+ else:
+ monkeypatch.setenv("OPENRAG_EMBEDDING_OVERFLOW_POLICY", value)
+
+ assert embedding_overflow_policy() == policy
+
+
+class TestCuttingAnInputIntoPieces:
+ def test_pieces_are_equal_and_cut_at_whitespace(self):
+ from services.llm_gateway import _split_into_pieces
+
+ text = "word " * 200
+
+ pieces = _split_into_pieces(text, 100, 700)
+
+ # 700 tokens against 100, with headroom: nine pieces, not seven.
+ assert len(pieces) == 9
+ assert "".join(pieces) == text
+ assert all(piece.endswith(" ") for piece in pieces)
+ assert max(map(len, pieces)) - min(map(len, pieces)) <= len(text) // 9 // 5
+
+ def test_a_long_unbroken_run_is_cut_where_it_falls(self):
+ from services.llm_gateway import _split_into_pieces
+
+ assert _split_into_pieces("x" * 100, 80, 100) == ["x" * 50, "x" * 50]
+
+ def test_without_counts_the_text_is_halved(self):
+ from services.llm_gateway import _split_into_pieces
+
+ assert _split_into_pieces(_OVER_LIMIT, None, None) == [_OVER_LIMIT[:50], _OVER_LIMIT[50:]]
+
+ def test_a_text_too_short_to_cut_comes_back_whole(self):
+ from services.llm_gateway import _split_into_pieces
+
+ assert _split_into_pieces("x", 512, 548) == ["x"]
+
+
+class TestAveragingPieceVectors:
+ def test_longer_pieces_weigh_more(self):
+ from services.llm_gateway import _weighted_mean_vector
+
+ mean = _weighted_mean_vector([[1.0, 0.0], [0.0, 1.0]], [3, 1])
+
+ assert mean == pytest.approx([0.9486833, 0.3162278])
+ assert math.hypot(*mean) == pytest.approx(1.0)
+
+ def test_vectors_that_are_not_unit_length_keep_their_scale(self):
+ from services.llm_gateway import _weighted_mean_vector
+
+ # Lengths 3 and 4: the mean is scaled to 3.5, not left at 2.5 or forced to 1.
+ mean = _weighted_mean_vector([[3.0, 0.0], [0.0, 4.0]], [1, 1])
+
+ assert mean == pytest.approx([2.1, 2.8])
+
+ def test_pieces_that_cancel_out_are_left_as_the_zero_vector(self):
+ from services.llm_gateway import _weighted_mean_vector
+
+ assert _weighted_mean_vector([[1.0, 0.0], [-1.0, 0.0]], [1, 1]) == [0.0, 0.0]
+
+
class TestToolsBesideReasoningEffort:
"""gpt-5.x refuses function tools next to its own default reasoning effort.