The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
feat: add SitemapLoader (#68)
aaishikduttad2e8f796ca60
diff --git a/README.md b/README.mdindex e72a92b8c..be88d23a3 100644--- a/README.md+++ b/README.md@@ -294,6 +294,13 @@ To supply your own QnA pair, use the data_type as `qna_pair` and enter a tuple.```pythonapp.add_local('qna_pair', ("Question", "Answer"))```+### Sitemap++To add a XML site map containing list of all urls, use the data_type as `sitemap` and enter the sitemap url. Eg:++```python+app.add('sitemap', 'a_valid_sitemap_url/sitemap.xml')+```### Reusing a Vector DBdiff --git a/embedchain/config/InitConfig.py b/embedchain/config/InitConfig.pyindex fb4daf2e5..923ab879a 100644--- a/embedchain/config/InitConfig.py+++ b/embedchain/config/InitConfig.py@@ -62,6 +62,7 @@ class InitConfig(BaseConfig):Sets database to default (`ChromaDb`)."""from embedchain.vectordb.chroma_db import ChromaDB+self.db = ChromaDB(ef=self.ef, host=self.host, port=self.port)def _setup_logging(self, debug_level):diff --git a/embedchain/data_formatter/data_formatter.py b/embedchain/data_formatter/data_formatter.pyindex bff1b4f26..6e6e18f76 100644--- a/embedchain/data_formatter/data_formatter.py+++ b/embedchain/data_formatter/data_formatter.py@@ -9,6 +9,7 @@ from embedchain.loaders.docx_file import DocxFileLoaderfrom embedchain.loaders.local_qna_pair import LocalQnaPairLoaderfrom embedchain.loaders.local_text import LocalTextLoaderfrom embedchain.loaders.pdf_file import PdfFileLoader+from embedchain.loaders.sitemap import SitemapLoaderfrom embedchain.loaders.web_page import WebPageLoaderfrom embedchain.loaders.youtube_video import YoutubeVideoLoader@@ -39,6 +40,7 @@ class DataFormatter:"qna_pair": LocalQnaPairLoader(),"text": LocalTextLoader(),"docx": DocxFileLoader(),+ "sitemap": SitemapLoader(),}if data_type in loaders:return loaders[data_type]@@ -60,6 +62,7 @@ class DataFormatter:"qna_pair": QnaPairChunker(config),"text": TextChunker(config),"docx": DocxFileChunker(config),+ "sitemap": WebPageChunker(config),}if data_type in chunkers:return chunkers[data_type]diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.pynew file mode 100644index 000000000..e38a2be8f--- /dev/null+++ b/embedchain/loaders/sitemap.py@@ -0,0 +1,24 @@+import requests+from bs4 import BeautifulSoup++from embedchain.loaders.web_page import WebPageLoader+++class SitemapLoader:+ def load_data(self, sitemap_url):+ """+ This method takes a sitemap URL as input and retrieves+ all the URLs to use the WebPageLoader to load content+ of each page.+ """+ output = []+ web_page_loader = WebPageLoader()+ response = requests.get(sitemap_url)+ response.raise_for_status()++ soup = BeautifulSoup(response.text, "xml")+ links = [link.text for link in soup.find_all("loc")]+ for link in links:+ each_load_data = web_page_loader.load_data(link)+ output.append(each_load_data)+ return [data[0] for data in output]diff --git a/embedchain/vectordb/chroma_db.py b/embedchain/vectordb/chroma_db.pyindex 2142f9d3c..72408104a 100644--- a/embedchain/vectordb/chroma_db.py+++ b/embedchain/vectordb/chroma_db.py@@ -1,5 +1,5 @@-import osimport logging+import osimport chromadbfrom chromadb.utils import embedding_functionsdiff --git a/setup.py b/setup.pyindex 6469e3347..c9dee2117 100644--- a/setup.py+++ b/setup.py@@ -29,6 +29,7 @@ setuptools.setup("beautifulsoup4","pypdf","pytube",+ "lxml","gpt4all","sentence_transformers","docx2txt",