Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

[Feature] Batch uploading in chromadb (#814)

Rupesh Bansald8a7d7134494
diff --git a/embedchain/vectordb/chroma.py b/embedchain/vectordb/chroma.py
index be4e8604b..c77e83c1a 100644
--- a/embedchain/vectordb/chroma.py
+++ b/embedchain/vectordb/chroma.py
@@ -25,6 +25,8 @@ except RuntimeError:
class ChromaDB(BaseVectorDB):
"""Vector database using ChromaDB."""
+ BATCH_SIZE = 100
+
def __init__(self, config: Optional[ChromaDbConfig] = None):
"""Initialize a new ChromaDB instance
@@ -123,10 +125,6 @@ class ChromaDB(BaseVectorDB):
args["limit"] = limit
return self.collection.get(**args)
- def get_advanced(self, where):
- where_clause = self._generate_where_clause(where)
- return self.collection.get(where=where_clause, limit=1)
-
def add(
self,
embeddings: List[List[float]],
@@ -149,10 +147,31 @@ class ChromaDB(BaseVectorDB):
:param skip_embedding: Optional. If True, then the embeddings are assumed to be already generated.
:type skip_embedding: bool
"""
- if skip_embedding:
- self.collection.add(embeddings=embeddings, documents=documents, metadatas=metadatas, ids=ids)
- else:
- self.collection.add(documents=documents, metadatas=metadatas, ids=ids)
+ size = len(documents)
+ if skip_embedding and (embeddings is None or len(embeddings) != len(documents)):
+ raise ValueError("Cannot add documents to chromadb with inconsistent embeddings")
+
+ if len(documents) != size or len(metadatas) != size or len(ids) != size:
+ raise ValueError(
+ "Cannot add documents to chromadb with inconsistent sizes. Documents size: {}, Metadata size: {},"
+ " Ids size: {}".format(len(documents), len(metadatas), len(ids))
+ )
+
+ for i in range(0, len(documents), self.BATCH_SIZE):
+ print("Inserting batches from {} to {} in chromadb".format(i, min(len(documents), i + self.BATCH_SIZE)))
+ if skip_embedding:
+ self.collection.add(
+ embeddings=embeddings[i : i + self.BATCH_SIZE],
+ documents=documents[i : i + self.BATCH_SIZE],
+ metadatas=metadatas[i : i + self.BATCH_SIZE],
+ ids=ids[i : i + self.BATCH_SIZE],
+ )
+ else:
+ self.collection.add(
+ documents=documents[i : i + self.BATCH_SIZE],
+ metadatas=metadatas[i : i + self.BATCH_SIZE],
+ ids=ids[i : i + self.BATCH_SIZE],
+ )
def _format_result(self, results: QueryResult) -> list[tuple[Document, float]]:
"""
@@ -208,7 +227,8 @@ class ChromaDB(BaseVectorDB):
except InvalidDimensionException as e:
raise InvalidDimensionException(
e.message()
- + ". This is commonly a side-effect when an embedding function, different from the one used to add the embeddings, is used to retrieve an embedding from the database." # noqa E501
+ + ". This is commonly a side-effect when an embedding function, different from the one used to add the"
+ " embeddings, is used to retrieve an embedding from the database."
) from None
results_formatted = self._format_result(result)
contents = [result[0].page_content for result in results_formatted]
diff --git a/tests/vectordb/test_chroma_db.py b/tests/vectordb/test_chroma_db.py
index 0472ec0f1..8caf14ac6 100644
--- a/tests/vectordb/test_chroma_db.py
+++ b/tests/vectordb/test_chroma_db.py
@@ -228,6 +228,40 @@ class TestChromaDbCollection(unittest.TestCase):
expected_value = ["document"]
self.assertEqual(data, expected_value)
+ def test_add_with_invalid_inputs(self):
+ """
+ Test add fails with invalid inputs
+ """
+ # Start with a clean app
+ self.app_with_settings.reset()
+ # app = App(config=AppConfig(collect_metrics=False), db=db)
+
+ # Collection should be empty when created
+ self.assertEqual(self.app_with_settings.db.count(), 0)
+
+ with self.assertRaises(ValueError):
+ self.app_with_settings.db.add(
+ embeddings=[[0, 0, 0]],
+ documents=["document", "document2"],
+ metadatas=[{"value": "somevalue"}],
+ ids=["id"],
+ skip_embedding=True,
+ )
+ # After adding, should contain no item
+ self.assertEqual(self.app_with_settings.db.count(), 0)
+
+ with self.assertRaises(ValueError):
+ self.app_with_settings.db.add(
+ embeddings=None,
+ documents=["document", "document2"],
+ metadatas=[{"value": "somevalue"}],
+ ids=["id"],
+ skip_embedding=True,
+ )
+
+ # After adding, should contain no item
+ self.assertEqual(self.app_with_settings.db.count(), 0)
+
def test_collections_are_persistent(self):
"""
Test that a collection can be picked up later.