[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(core): skip empty/invalid chunks before embedding; guard OpenAI embeddings (#55)
Andy Lee14e84d9e2db6
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 144e858..cfcf1ad 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -306,6 +306,23 @@ class LeannBuilder:def build_index(self, index_path: str):if not self.chunks:raise ValueError("No chunks added.")++ # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned+ valid_chunks: list[dict[str, Any]] = []+ skipped = 0+ for chunk in self.chunks:+ text = chunk.get("text", "")+ if isinstance(text, str) and text.strip():+ valid_chunks.append(chunk)+ else:+ skipped += 1+ if skipped > 0:+ print(+ f"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks"+ )+ self.chunks = valid_chunks+ if not self.chunks:+ raise ValueError("All provided chunks are empty or invalid. Nothing to index.")if self.dimensions is None:self.dimensions = len(compute_embeddings(diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex 9cce58c..ee038d5 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -244,6 +244,16 @@ def compute_embeddings_openai(texts: list[str], model_name: str) -> np.ndarray:except ImportError as e:raise ImportError(f"OpenAI package not installed: {e}")+ # Validate input list+ if not texts:+ raise ValueError("Cannot compute embeddings for empty text list")+ # Extra validation: abort early if any item is empty/whitespace+ invalid_count = sum(1 for t in texts if not isinstance(t, str) or not t.strip())+ if invalid_count > 0:+ raise ValueError(+ f"Found {invalid_count} empty/invalid text(s) in input. Upstream should filter before calling OpenAI."+ )+api_key = os.getenv("OPENAI_API_KEY")if not api_key:raise RuntimeError("OPENAI_API_KEY environment variable not set")