The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(openai): Respect 300k token limit for embeddings API requests (#33668)
Kaparthy Reddy2d4f00a45130
diff --git a/libs/partners/openai/langchain_openai/embeddings/base.py b/libs/partners/openai/langchain_openai/embeddings/base.pyindex c2a87fe06b..1ca475c8e5 100644--- a/libs/partners/openai/langchain_openai/embeddings/base.py+++ b/libs/partners/openai/langchain_openai/embeddings/base.py@@ -19,6 +19,9 @@ from langchain_openai.chat_models._client_utils import _resolve_sync_and_async_alogger = logging.getLogger(__name__)+MAX_TOKENS_PER_REQUEST = 300000+"""API limit per request for embedding tokens."""+def _process_batched_chunked_embeddings(num_texts: int,@@ -524,9 +527,9 @@ class OpenAIEmbeddings(BaseModel, Embeddings):) -> list[list[float]]:"""Generate length-safe embeddings for a list of texts.- This method handles tokenization and embedding generation, respecting the- set embedding context length and chunk size. It supports both tiktoken- and HuggingFace tokenizer based on the tiktoken_enabled flag.+ This method handles tokenization and embedding generation, respecting the set+ embedding context length and chunk size. It supports both tiktoken and+ HuggingFace tokenizer based on the tiktoken_enabled flag.Args:texts: A list of texts to embed.@@ -540,14 +543,38 @@ class OpenAIEmbeddings(BaseModel, Embeddings):client_kwargs = {**self._invocation_params, **kwargs}_iter, tokens, indices = self._tokenize(texts, _chunk_size)batched_embeddings: list[list[float]] = []- for i in _iter:- response = self.client.create(- input=tokens[i : i + _chunk_size], **client_kwargs- )+ # Calculate token counts per chunk+ token_counts = [+ len(t) if isinstance(t, list) else len(t.split()) for t in tokens+ ]++ # Process in batches respecting the token limit+ i = 0+ while i < len(tokens):+ # Determine how many chunks we can include in this batch+ batch_token_count = 0+ batch_end = i++ for j in range(i, min(i + _chunk_size, len(tokens))):+ chunk_tokens = token_counts[j]+ # Check if adding this chunk would exceed the limit+ if batch_token_count + chunk_tokens > MAX_TOKENS_PER_REQUEST:+ if batch_end == i:+ # Single chunk exceeds limit - handle it anyway+ batch_end = j + 1+ break+ batch_token_count += chunk_tokens+ batch_end = j + 1++ # Make API call with this batch+ batch_tokens = tokens[i:batch_end]+ response = self.client.create(input=batch_tokens, **client_kwargs)if not isinstance(response, dict):response = response.model_dump()batched_embeddings.extend(r["embedding"] for r in response["data"])+ i = batch_end+embeddings = _process_batched_chunked_embeddings(len(texts), tokens, batched_embeddings, indices, self.skip_empty)@@ -594,15 +621,40 @@ class OpenAIEmbeddings(BaseModel, Embeddings):None, self._tokenize, texts, _chunk_size)batched_embeddings: list[list[float]] = []- for i in range(0, len(tokens), _chunk_size):+ # Calculate token counts per chunk+ token_counts = [+ len(t) if isinstance(t, list) else len(t.split()) for t in tokens+ ]++ # Process in batches respecting the token limit+ i = 0+ while i < len(tokens):+ # Determine how many chunks we can include in this batch+ batch_token_count = 0+ batch_end = i++ for j in range(i, min(i + _chunk_size, len(tokens))):+ chunk_tokens = token_counts[j]+ # Check if adding this chunk would exceed the limit+ if batch_token_count + chunk_tokens > MAX_TOKENS_PER_REQUEST:+ if batch_end == i:+ # Single chunk exceeds limit - handle it anyway+ batch_end = j + 1+ break+ batch_token_count += chunk_tokens+ batch_end = j + 1++ # Make API call with this batch+ batch_tokens = tokens[i:batch_end]response = await self.async_client.create(- input=tokens[i : i + _chunk_size], **client_kwargs+ input=batch_tokens, **client_kwargs)-if not isinstance(response, dict):response = response.model_dump()batched_embeddings.extend(r["embedding"] for r in response["data"])+ i = batch_end+embeddings = _process_batched_chunked_embeddings(len(texts), tokens, batched_embeddings, indices, self.skip_empty)diff --git a/libs/partners/openai/tests/unit_tests/embeddings/test_base.py b/libs/partners/openai/tests/unit_tests/embeddings/test_base.pyindex f87dc181a3..15d534b04c 100644--- a/libs/partners/openai/tests/unit_tests/embeddings/test_base.py+++ b/libs/partners/openai/tests/unit_tests/embeddings/test_base.py@@ -1,7 +1,9 @@import os-from unittest.mock import patch+from typing import Any+from unittest.mock import Mock, patchimport pytest+from pydantic import SecretStrfrom langchain_openai import OpenAIEmbeddings@@ -96,3 +98,53 @@ async def test_embed_with_kwargs_async() -> None:mock_create.assert_any_call(input=texts, **client_kwargs)assert result == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]+++def test_embeddings_respects_token_limit() -> None:+ """Test that embeddings respect the 300k token per request limit."""+ # Create embeddings instance+ embeddings = OpenAIEmbeddings(+ model="text-embedding-ada-002", api_key=SecretStr("test-key")+ )++ call_counts = []++ def mock_create(**kwargs: Any) -> Mock:+ input_ = kwargs["input"]+ # Track how many tokens in this call+ if isinstance(input_, list):+ total_tokens = sum(+ len(t) if isinstance(t, list) else len(t.split()) for t in input_+ )+ call_counts.append(total_tokens)+ # Verify this call doesn't exceed limit+ assert total_tokens <= 300000, (+ f"Batch exceeded token limit: {total_tokens} tokens"+ )++ # Return mock response+ mock_response = Mock()+ mock_response.model_dump.return_value = {+ "data": [+ {"embedding": [0.1] * 1536}+ for _ in range(len(input_) if isinstance(input_, list) else 1)+ ]+ }+ return mock_response++ embeddings.client.create = mock_create++ # Create a scenario that would exceed 300k tokens in a single batch+ # with default chunk_size=1000+ # Simulate 500 texts with ~1000 tokens each = 500k tokens total+ large_texts = ["word " * 1000 for _ in range(500)]++ # This should not raise an error anymore+ embeddings.embed_documents(large_texts)++ # Verify we made multiple API calls to respect the limit+ assert len(call_counts) > 1, "Should have split into multiple batches"++ # Verify each call respected the limit+ for count in call_counts:+ assert count <= 300000, f"Batch exceeded limit: {count}"