Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

feat: added data format to metadata internally (#314)

aaishikduttac12362486ff8
diff --git a/embedchain/chunkers/base_chunker.py b/embedchain/chunkers/base_chunker.py
index e2334488a..e44366b10 100644
--- a/embedchain/chunkers/base_chunker.py
+++ b/embedchain/chunkers/base_chunker.py
@@ -5,6 +5,7 @@ class BaseChunker:
def __init__(self, text_splitter):
"""Initialize the chunker."""
self.text_splitter = text_splitter
+ self.data_type = None
def create_chunks(self, loader, src):
"""
@@ -22,7 +23,10 @@ class BaseChunker:
metadatas = []
for data in datas:
content = data["content"]
+
meta_data = data["meta_data"]
+ # add data type to meta data to allow query using data type
+ meta_data["data_type"] = self.data_type
url = meta_data["url"]
chunks = self.get_chunks(content)
@@ -47,3 +51,9 @@ class BaseChunker:
Override in child class if custom logic.
"""
return self.text_splitter.split_text(content)
+
+ def set_data_type(self, data_type):
+ """
+ set the data type of chunker
+ """
+ self.data_type = data_type
diff --git a/embedchain/data_formatter/data_formatter.py b/embedchain/data_formatter/data_formatter.py
index 3e941feae..fb878e54b 100644
--- a/embedchain/data_formatter/data_formatter.py
+++ b/embedchain/data_formatter/data_formatter.py
@@ -69,6 +69,8 @@ class DataFormatter:
"docs_site": DocsSiteChunker(config),
}
if data_type in chunkers:
- return chunkers[data_type]
+ chunker = chunkers[data_type]
+ chunker.set_data_type(data_type)
+ return chunker
else:
raise ValueError(f"Unsupported data type: {data_type}")