The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
feat: added data format to metadata internally (#314)
aaishikduttac12362486ff8
diff --git a/embedchain/chunkers/base_chunker.py b/embedchain/chunkers/base_chunker.pyindex e2334488a..e44366b10 100644--- a/embedchain/chunkers/base_chunker.py+++ b/embedchain/chunkers/base_chunker.py@@ -5,6 +5,7 @@ class BaseChunker:def __init__(self, text_splitter):"""Initialize the chunker."""self.text_splitter = text_splitter+ self.data_type = Nonedef create_chunks(self, loader, src):"""@@ -22,7 +23,10 @@ class BaseChunker:metadatas = []for data in datas:content = data["content"]+meta_data = data["meta_data"]+ # add data type to meta data to allow query using data type+ meta_data["data_type"] = self.data_typeurl = meta_data["url"]chunks = self.get_chunks(content)@@ -47,3 +51,9 @@ class BaseChunker:Override in child class if custom logic."""return self.text_splitter.split_text(content)++ def set_data_type(self, data_type):+ """+ set the data type of chunker+ """+ self.data_type = data_typediff --git a/embedchain/data_formatter/data_formatter.py b/embedchain/data_formatter/data_formatter.pyindex 3e941feae..fb878e54b 100644--- a/embedchain/data_formatter/data_formatter.py+++ b/embedchain/data_formatter/data_formatter.py@@ -69,6 +69,8 @@ class DataFormatter:"docs_site": DocsSiteChunker(config),}if data_type in chunkers:- return chunkers[data_type]+ chunker = chunkers[data_type]+ chunker.set_data_type(data_type)+ return chunkerelse:raise ValueError(f"Unsupported data type: {data_type}")