The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
Add metadata support to added data sources (#253)
Rayhan Patel8f42ced9b590
diff --git a/embedchain/embedchain.py b/embedchain/embedchain.pyindex 1ea79ace6..3cd2f4179 100644--- a/embedchain/embedchain.py+++ b/embedchain/embedchain.py@@ -36,7 +36,7 @@ class EmbedChain:self.collection = self.config.db.collectionself.user_asks = []- def add(self, data_type, url, config: AddConfig = None):+ def add(self, data_type, url, metadata=None, config: AddConfig = None):"""Adds the data from the given URL to the vector db.Loads the data, chunks it, create embedding for each chunk@@ -44,6 +44,7 @@ class EmbedChain::param data_type: The type of the data to add.:param url: The URL where the data is located.+ :param metadata: Optional. Metadata associated with the data source.:param config: Optional. The `AddConfig` instance to use as configurationoptions."""@@ -51,10 +52,10 @@ class EmbedChain:config = AddConfig()data_formatter = DataFormatter(data_type, config)- self.user_asks.append([data_type, url])- self.load_and_embed(data_formatter.loader, data_formatter.chunker, url)+ self.user_asks.append([data_type, url, metadata])+ self.load_and_embed(data_formatter.loader, data_formatter.chunker, url, metadata)- def add_local(self, data_type, content, config: AddConfig = None):+ def add_local(self, data_type, content, metadata=None, config: AddConfig = None):"""Adds the data you supply to the vector db.Loads the data, chunks it, create embedding for each chunk@@ -62,6 +63,7 @@ class EmbedChain::param data_type: The type of the data to add.:param content: The local data. Refer to the `README` for formatting.+ :param metadata: Optional. Metadata associated with the data source.:param config: Optional. The `AddConfig` instance to use asconfiguration options."""@@ -74,9 +76,10 @@ class EmbedChain:data_formatter.loader,data_formatter.chunker,content,+ metadata,)- def load_and_embed(self, loader, chunker, src):+ def load_and_embed(self, loader, chunker, src, metadata=None):"""Loads the data from the given URL, chunks it, and adds it to database.@@ -84,6 +87,7 @@ class EmbedChain::param chunker: The chunker to use to chunk the data.:param src: The data to be handled by the loader. Can be a URL forremote sources or local content for local loaders.+ :param metadata: Optional. Metadata associated with the data source."""embeddings_data = chunker.create_chunks(loader, src)documents = embeddings_data["documents"]@@ -112,7 +116,11 @@ class EmbedChain:documents, metadatas = zip(*data_dict.values())chunks_before_addition = self.count()- self.collection.add(documents=documents, metadatas=list(metadatas), ids=ids)++ # Add metadata to each document+ metadatas_with_metadata = [meta or metadata for meta in metadatas]++ self.collection.add(documents=documents, metadatas=list(metadatas_with_metadata), ids=ids)print((f"Successfully saved {src}. New chunks count: "