Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

Fix Gemini OpenAI-compat embedding batch size limit (#214)

Matthias Humt599757f805c8
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.py
index 04bb15c..e655917 100644
--- a/packages/leann-core/src/leann/embedding_compute.py
+++ b/packages/leann-core/src/leann/embedding_compute.py
@@ -774,7 +774,6 @@ def compute_embeddings_openai(
logger.info(
f"Computing embeddings for {len(texts)} texts using OpenAI API, model: '{model_name}'"
)
- print(f"len of texts: {len(texts)}")
# Apply prompt template if provided
# Priority: build_prompt_template (new format) > prompt_template (old format)
@@ -796,11 +795,20 @@ def compute_embeddings_openai(
all_embeddings = []
# get the avg len of texts
avg_len = sum(len(text) for text in texts) / len(texts)
- print(f"avg len of texts: {avg_len}")
# if avg len is less than 1000, use the max batch size
if avg_len > 300:
max_batch_size = 500
+ # Gemini's OpenAI-compatible endpoint hard-limits embedding batches to 100 inputs per request.
+ # If we exceed this, the API returns:
+ # "BatchEmbedContentsRequest.requests: at most 100 requests can be in one batch"
+ if "generativelanguage.googleapis.com" in (resolved_base_url or ""):
+ max_batch_size = min(max_batch_size, 100)
+ logger.info(
+ "Detected Gemini OpenAI-compatible base_url; capping embedding batch_size to %d.",
+ max_batch_size,
+ )
+
# if avg len is less than 1000, use the max batch size
try:
@@ -836,7 +844,6 @@ def compute_embeddings_openai(
embeddings = np.array(all_embeddings, dtype=np.float32)
logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")
- print(f"len of embeddings: {len(embeddings)}")
return embeddings
diff --git a/tests/test_embedding_prompt_template.py b/tests/test_embedding_prompt_template.py
index 22a1fef..b678ff5 100644
--- a/tests/test_embedding_prompt_template.py
+++ b/tests/test_embedding_prompt_template.py
@@ -197,6 +197,31 @@ class TestPromptTemplatePrepending:
# Verify result shape
assert result.shape[0] == 1000, "Should return embeddings for all texts"
+ def test_gemini_openai_compat_caps_batch_size_to_100(
+ self, mock_openai_module, mock_openai_client
+ ):
+ texts = [f"Doc {i}" for i in range(250)]
+ provider_options = {"base_url": "https://generativelanguage.googleapis.com/v1beta/openai"}
+
+ mock_response = Mock()
+ mock_response.data = [Mock(embedding=[0.1, 0.2, 0.3]) for _ in range(250)]
+ mock_openai_client.embeddings.create.return_value = mock_response
+
+ result = compute_embeddings_openai(
+ texts=texts,
+ model_name="gemini-embedding-001",
+ provider_options=provider_options,
+ )
+
+ # Should chunk into <=100 inputs per request.
+ assert mock_openai_client.embeddings.create.call_count == 3
+ batch_sizes = [
+ len(call.kwargs["input"])
+ for call in mock_openai_client.embeddings.create.call_args_list
+ ]
+ assert batch_sizes == [100, 100, 50]
+ assert result.shape[0] == 250
+
def test_prompt_template_with_special_characters(self, mock_openai_module, mock_openai_client):
"""Verify template with special characters is handled correctly.