Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

fix: escape bs4 parsing error (#271)

cachhocd0c7bc971f5
diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.py
index e38a2be8f..516581e0e 100644
--- a/embedchain/loaders/sitemap.py
+++ b/embedchain/loaders/sitemap.py
@@ -1,5 +1,8 @@
+import logging
+
import requests
from bs4 import BeautifulSoup
+from bs4.builder import ParserRejectedMarkup
from embedchain.loaders.web_page import WebPageLoader
@@ -19,6 +22,9 @@ class SitemapLoader:
soup = BeautifulSoup(response.text, "xml")
links = [link.text for link in soup.find_all("loc")]
for link in links:
- each_load_data = web_page_loader.load_data(link)
- output.append(each_load_data)
+ try:
+ each_load_data = web_page_loader.load_data(link)
+ output.append(each_load_data)
+ except ParserRejectedMarkup as e:
+ logging.error(f"Failed to parse {link}: {e}")
return [data[0] for data in output]