The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
feat: add local qna pair
cachhoff2d5ce7facb
diff --git a/embedchain/chunkers/qna_pair.py b/embedchain/chunkers/qna_pair.pynew file mode 100644index 000000000..7fe9e57be--- /dev/null+++ b/embedchain/chunkers/qna_pair.py@@ -0,0 +1,16 @@+from embedchain.chunkers.base_chunker import BaseChunker++from langchain.text_splitter import RecursiveCharacterTextSplitter+++TEXT_SPLITTER_CHUNK_PARAMS = {+ "chunk_size": 300,+ "chunk_overlap": 0,+ "length_function": len,+}+++class QnaPairChunker(BaseChunker):+ def __init__(self):+ text_splitter = RecursiveCharacterTextSplitter(**TEXT_SPLITTER_CHUNK_PARAMS)+ super().__init__(text_splitter)diff --git a/embedchain/embedchain.py b/embedchain/embedchain.pyindex d68c73945..033efc0b7 100644--- a/embedchain/embedchain.py+++ b/embedchain/embedchain.py@@ -8,9 +8,11 @@ from langchain.embeddings.openai import OpenAIEmbeddingsfrom embedchain.loaders.youtube_video import YoutubeVideoLoaderfrom embedchain.loaders.pdf_file import PdfFileLoaderfrom embedchain.loaders.web_page import WebPageLoader+from embedchain.loaders_local.qna_pair import QnaPairLoaderfrom embedchain.chunkers.youtube_video import YoutubeVideoChunkerfrom embedchain.chunkers.pdf_file import PdfFileChunkerfrom embedchain.chunkers.web_page import WebPageChunker+from embedchain.chunkers.qna_pair import QnaPairChunkerfrom embedchain.vectordb.chroma_db import ChromaDBload_dotenv()@@ -46,7 +48,8 @@ class EmbedChain:loaders = {'youtube_video': YoutubeVideoLoader(),'pdf_file': PdfFileLoader(),- 'web_page': WebPageLoader()+ 'web_page': WebPageLoader(),+ 'qna_pair': QnaPairLoader()}if data_type in loaders:return loaders[data_type]@@ -64,7 +67,8 @@ class EmbedChain:chunkers = {'youtube_video': YoutubeVideoChunker(),'pdf_file': PdfFileChunker(),- 'web_page': WebPageChunker()+ 'web_page': WebPageChunker(),+ 'qna_pair': QnaPairChunker(),}if data_type in chunkers:return chunkers[data_type]@@ -85,6 +89,20 @@ class EmbedChain:self.user_asks.append([data_type, url])self.load_and_embed(loader, chunker, url)+ def add_local(self, data_type, content):+ """+ Adds the data you supply to the vector db.+ Loads the data, chunks it, create embedding for each chunk+ and then stores the embedding to vector database.++ :param data_type: The type of the data to add.+ :param content: The local data. Refer to the `README` for formatting.+ """+ loader = self._get_loader(data_type)+ chunker = self._get_chunker(data_type)+ self.user_asks.append([data_type, content])+ self.load_and_embed(loader, chunker, content)+def load_and_embed(self, loader, chunker, url):"""Loads the data from the given URL, chunks it, and adds it to the database.diff --git a/embedchain/loaders_local/__init__.py b/embedchain/loaders_local/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/embedchain/loaders_local/qna_pair.py b/embedchain/loaders_local/qna_pair.pynew file mode 100644index 000000000..2fc57159a--- /dev/null+++ b/embedchain/loaders_local/qna_pair.py@@ -0,0 +1,16 @@+from embedchain.utils import markdown_to_plaintext+++class QnaPairLoader:++ def load_data(self, content):+ question, answer = content+ answer = markdown_to_plaintext(answer)+ content = f"Q: {question}\nA: {answer}"+ meta_data = {+ "url": "local",+ }+ return [{+ "content": content,+ "meta_data": meta_data,+ }]\ No newline at end of file