The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
Chunkers: Refactor each chunker & add base class
Taranjeet Singh4329caa17c63
diff --git a/embedchain/chunkers/base_chunker.py b/embedchain/chunkers/base_chunker.pynew file mode 100644index 000000000..bcd5a86da--- /dev/null+++ b/embedchain/chunkers/base_chunker.py@@ -0,0 +1,27 @@+import hashlib+++class BaseChunker:+ def __init__(self, text_splitter):+ self.text_splitter = text_splitter++ def create_chunks(self, loader, url):+ documents = []+ ids = []+ datas = loader.load_data(url)+ metadatas = []+ for data in datas:+ content = data["content"]+ meta_data = data["meta_data"]+ chunks = self.text_splitter.split_text(content)+ url = meta_data["url"]+ for chunk in chunks:+ chunk_id = hashlib.sha256((chunk + url).encode()).hexdigest()+ ids.append(chunk_id)+ documents.append(chunk)+ metadatas.append(meta_data)+ return {+ "documents": documents,+ "ids": ids,+ "metadatas": metadatas,+ }diff --git a/embedchain/chunkers/pdf_file.py b/embedchain/chunkers/pdf_file.pyindex ad760594a..47a23c7af 100644--- a/embedchain/chunkers/pdf_file.py+++ b/embedchain/chunkers/pdf_file.py@@ -1,4 +1,4 @@-import hashlib+from embedchain.chunkers.base_chunker import BaseChunkerfrom langchain.text_splitter import RecursiveCharacterTextSplitter@@ -9,28 +9,8 @@ TEXT_SPLITTER_CHUNK_PARAMS = {"length_function": len,}-TEXT_SPLITTER = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)--class PdfFileChunker:-- def create_chunks(self, loader, url):- documents = []- ids = []- datas = loader.load_data(url)- metadatas = []- for data in datas:- content = data["content"]- meta_data = data["meta_data"]- chunks = TEXT_SPLITTER.split_text(content)- url = meta_data["url"]- for chunk in chunks:- chunk_id = hashlib.sha256((chunk + url).encode()).hexdigest()- ids.append(chunk_id)- documents.append(chunk)- metadatas.append(meta_data)- return {- "documents": documents,- "ids": ids,- "metadatas": metadatas,- }\ No newline at end of file+class PdfFileChunker(BaseChunker):+ def __init__(self):+ text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)+ super().__init__(text_splitter)\ No newline at end of filediff --git a/embedchain/chunkers/website.py b/embedchain/chunkers/website.pyindex 6d70131d9..090eb730b 100644--- a/embedchain/chunkers/website.py+++ b/embedchain/chunkers/website.py@@ -1,4 +1,4 @@-import hashlib+from embedchain.chunkers.base_chunker import BaseChunkerfrom langchain.text_splitter import RecursiveCharacterTextSplitter@@ -9,28 +9,8 @@ TEXT_SPLITTER_CHUNK_PARAMS = {"length_function": len,}-TEXT_SPLITTER = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)--class WebsiteChunker:-- def create_chunks(self, loader, url):- documents = []- ids = []- datas = loader.load_data(url)- metadatas = []- for data in datas:- content = data["content"]- meta_data = data["meta_data"]- chunks = TEXT_SPLITTER.split_text(content)- url = meta_data["url"]- for chunk in chunks:- chunk_id = hashlib.sha256((chunk + url).encode()).hexdigest()- ids.append(chunk_id)- documents.append(chunk)- metadatas.append(meta_data)- return {- "documents": documents,- "ids": ids,- "metadatas": metadatas,- }\ No newline at end of file+class WebsiteChunker(BaseChunker):+ def __init__(self):+ text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)+ super().__init__(text_splitter)diff --git a/embedchain/chunkers/youtube_video.py b/embedchain/chunkers/youtube_video.pyindex 6c4b1ae1b..7435c02df 100644--- a/embedchain/chunkers/youtube_video.py+++ b/embedchain/chunkers/youtube_video.py@@ -1,4 +1,4 @@-import hashlib+from embedchain.chunkers.base_chunker import BaseChunkerfrom langchain.text_splitter import RecursiveCharacterTextSplitter@@ -9,28 +9,8 @@ TEXT_SPLITTER_CHUNK_PARAMS = {"length_function": len,}-TEXT_SPLITTER = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)--class YoutubeVideoChunker:-- def create_chunks(self, loader, url):- documents = []- ids = []- datas = loader.load_data(url)- metadatas = []- for data in datas:- content = data["content"]- meta_data = data["meta_data"]- chunks = TEXT_SPLITTER.split_text(content)- url = meta_data["url"]- for chunk in chunks:- chunk_id = hashlib.sha256((chunk + url).encode()).hexdigest()- ids.append(chunk_id)- documents.append(chunk)- metadatas.append(meta_data)- return {- "documents": documents,- "ids": ids,- "metadatas": metadatas,- }\ No newline at end of file+class YoutubeVideoChunker(BaseChunker):+ def __init__(self):+ text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)+ super().__init__(text_splitter)\ No newline at end of file