Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

feat: filter sitemap (#304)

cachho3da57248538c
diff --git a/docs/advanced/data_types.mdx b/docs/advanced/data_types.mdx
index 7e37efd1e..b2ad8eb5f 100644
--- a/docs/advanced/data_types.mdx
+++ b/docs/advanced/data_types.mdx
@@ -30,6 +30,14 @@ To add any web page, use the data_type as `web_page`. Eg:
app.add('web_page', 'a_valid_web_page_url')
```
+### Sitemap
+
+Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg:
+
+```python
+app.add('sitemap', 'https://example.com/sitemap.xml')
+```
+
### Doc file
To add any doc/docx file, use the data_type as `docx`. Eg:
diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.py
index 516581e0e..09dd5f53a 100644
--- a/embedchain/loaders/sitemap.py
+++ b/embedchain/loaders/sitemap.py
@@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
from bs4.builder import ParserRejectedMarkup
from embedchain.loaders.web_page import WebPageLoader
+from embedchain.utils import is_readable
class SitemapLoader:
@@ -20,11 +21,20 @@ class SitemapLoader:
response.raise_for_status()
soup = BeautifulSoup(response.text, "xml")
- links = [link.text for link in soup.find_all("loc")]
+
+ links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]
+ if len(links) == 0:
+ # Get all <loc> tags as a fallback. This might include images.
+ links = [link.text for link in soup.find_all("loc")]
+
for link in links:
try:
each_load_data = web_page_loader.load_data(link)
- output.append(each_load_data)
+
+ if is_readable(each_load_data[0].get("content")):
+ output.append(each_load_data)
+ else:
+ logging.warning(f"Page is not readable (too many invalid characters): {link}")
except ParserRejectedMarkup as e:
logging.error(f"Failed to parse {link}: {e}")
return [data[0] for data in output]
diff --git a/embedchain/utils.py b/embedchain/utils.py
index f2b04490c..02ee2debc 100644
--- a/embedchain/utils.py
+++ b/embedchain/utils.py
@@ -1,4 +1,5 @@
import re
+import string
def clean_string(text):
@@ -33,3 +34,14 @@ def clean_string(text):
cleaned_text = re.sub(r"([^\w\s])\1*", r"\1", cleaned_text)
return cleaned_text
+
+
+def is_readable(s):
+ """
+ Heuristic to determine if a string is "readable" (mostly contains printable characters and forms meaningful words)
+
+ :param s: string
+ :return: True if the string is more than 95% printable.
+ """
+ printable_ratio = sum(c in string.printable for c in s) / len(s)
+ return printable_ratio > 0.95 # 95% of characters are printable