[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Fix Gemini OpenAI-compat embedding batch size limit (#214)
Matthias Humt599757f805c8
diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex 04bb15c..e655917 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -774,7 +774,6 @@ def compute_embeddings_openai(logger.info(f"Computing embeddings for {len(texts)} texts using OpenAI API, model: '{model_name}'")- print(f"len of texts: {len(texts)}")# Apply prompt template if provided# Priority: build_prompt_template (new format) > prompt_template (old format)@@ -796,11 +795,20 @@ def compute_embeddings_openai(all_embeddings = []# get the avg len of textsavg_len = sum(len(text) for text in texts) / len(texts)- print(f"avg len of texts: {avg_len}")# if avg len is less than 1000, use the max batch sizeif avg_len > 300:max_batch_size = 500+ # Gemini's OpenAI-compatible endpoint hard-limits embedding batches to 100 inputs per request.+ # If we exceed this, the API returns:+ # "BatchEmbedContentsRequest.requests: at most 100 requests can be in one batch"+ if "generativelanguage.googleapis.com" in (resolved_base_url or ""):+ max_batch_size = min(max_batch_size, 100)+ logger.info(+ "Detected Gemini OpenAI-compatible base_url; capping embedding batch_size to %d.",+ max_batch_size,+ )+# if avg len is less than 1000, use the max batch sizetry:@@ -836,7 +844,6 @@ def compute_embeddings_openai(embeddings = np.array(all_embeddings, dtype=np.float32)logger.info(f"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}")- print(f"len of embeddings: {len(embeddings)}")return embeddingsdiff --git a/tests/test_embedding_prompt_template.py b/tests/test_embedding_prompt_template.pyindex 22a1fef..b678ff5 100644--- a/tests/test_embedding_prompt_template.py+++ b/tests/test_embedding_prompt_template.py@@ -197,6 +197,31 @@ class TestPromptTemplatePrepending:# Verify result shapeassert result.shape[0] == 1000, "Should return embeddings for all texts"+ def test_gemini_openai_compat_caps_batch_size_to_100(+ self, mock_openai_module, mock_openai_client+ ):+ texts = [f"Doc {i}" for i in range(250)]+ provider_options = {"base_url": "https://generativelanguage.googleapis.com/v1beta/openai"}++ mock_response = Mock()+ mock_response.data = [Mock(embedding=[0.1, 0.2, 0.3]) for _ in range(250)]+ mock_openai_client.embeddings.create.return_value = mock_response++ result = compute_embeddings_openai(+ texts=texts,+ model_name="gemini-embedding-001",+ provider_options=provider_options,+ )++ # Should chunk into <=100 inputs per request.+ assert mock_openai_client.embeddings.create.call_count == 3+ batch_sizes = [+ len(call.kwargs["input"])+ for call in mock_openai_client.embeddings.create.call_args_list+ ]+ assert batch_sizes == [100, 100, 50]+ assert result.shape[0] == 250+def test_prompt_template_with_special_characters(self, mock_openai_module, mock_openai_client):"""Verify template with special characters is handled correctly.