The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
fix sitemap loader (#986)
Deven Patelc9fbc2e7d6fb
diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.pyindex 06e7e2390..b95f854cf 100644--- a/embedchain/loaders/sitemap.py+++ b/embedchain/loaders/sitemap.py@@ -16,7 +16,6 @@ except ImportError:from embedchain.helpers.json_serializable import register_deserializablefrom embedchain.loaders.base_loader import BaseLoaderfrom embedchain.loaders.web_page import WebPageLoader-from embedchain.utils import is_readable@register_deserializable@@ -40,19 +39,16 @@ class SitemapLoader(BaseLoader):doc_id = hashlib.sha256((" ".join(links) + sitemap_url).encode()).hexdigest()- def load_link(link):+ def load_web_page(link):try:- each_load_data = web_page_loader.load_data(link)- if is_readable(each_load_data.get("data")[0].get("content")):- return each_load_data.get("data")- else:- logging.warning(f"Page is not readable (too many invalid characters): {link}")+ loader_data = web_page_loader.load_data(link)+ return loader_data.get("data")except ParserRejectedMarkup as e:logging.error(f"Failed to parse {link}: {e}")return Nonewith concurrent.futures.ThreadPoolExecutor() as executor:- future_to_link = {executor.submit(load_link, link): link for link in links}+ future_to_link = {executor.submit(load_web_page, link): link for link in links}for future in tqdm(concurrent.futures.as_completed(future_to_link), total=len(links), desc="Loading pages"):link = future_to_link[future]try: