The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
feat: filter sitemap (#304)
cachho3da57248538c
diff --git a/docs/advanced/data_types.mdx b/docs/advanced/data_types.mdxindex 7e37efd1e..b2ad8eb5f 100644--- a/docs/advanced/data_types.mdx+++ b/docs/advanced/data_types.mdx@@ -30,6 +30,14 @@ To add any web page, use the data_type as `web_page`. Eg:app.add('web_page', 'a_valid_web_page_url')```+### Sitemap++Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg:++```python+app.add('sitemap', 'https://example.com/sitemap.xml')+```+### Doc fileTo add any doc/docx file, use the data_type as `docx`. Eg:diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.pyindex 516581e0e..09dd5f53a 100644--- a/embedchain/loaders/sitemap.py+++ b/embedchain/loaders/sitemap.py@@ -5,6 +5,7 @@ from bs4 import BeautifulSoupfrom bs4.builder import ParserRejectedMarkupfrom embedchain.loaders.web_page import WebPageLoader+from embedchain.utils import is_readableclass SitemapLoader:@@ -20,11 +21,20 @@ class SitemapLoader:response.raise_for_status()soup = BeautifulSoup(response.text, "xml")- links = [link.text for link in soup.find_all("loc")]++ links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]+ if len(links) == 0:+ # Get all <loc> tags as a fallback. This might include images.+ links = [link.text for link in soup.find_all("loc")]+for link in links:try:each_load_data = web_page_loader.load_data(link)- output.append(each_load_data)++ if is_readable(each_load_data[0].get("content")):+ output.append(each_load_data)+ else:+ logging.warning(f"Page is not readable (too many invalid characters): {link}")except ParserRejectedMarkup as e:logging.error(f"Failed to parse {link}: {e}")return [data[0] for data in output]diff --git a/embedchain/utils.py b/embedchain/utils.pyindex f2b04490c..02ee2debc 100644--- a/embedchain/utils.py+++ b/embedchain/utils.py@@ -1,4 +1,5 @@import re+import stringdef clean_string(text):@@ -33,3 +34,14 @@ def clean_string(text):cleaned_text = re.sub(r"([^\w\s])\1*", r"\1", cleaned_text)return cleaned_text+++def is_readable(s):+ """+ Heuristic to determine if a string is "readable" (mostly contains printable characters and forms meaningful words)++ :param s: string+ :return: True if the string is more than 95% printable.+ """+ printable_ratio = sum(c in string.printable for c in s) / len(s)+ return printable_ratio > 0.95 # 95% of characters are printable