Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

[Bugfix] fix sitemap loader (#1000)

Deven Patelff4a333be7ee
diff --git a/embedchain/loaders/beehiiv.py b/embedchain/loaders/beehiiv.py
index 66ccfa24d..f9cc920f3 100644
--- a/embedchain/loaders/beehiiv.py
+++ b/embedchain/loaders/beehiiv.py
@@ -1,9 +1,10 @@
import hashlib
import logging
import time
-import requests
from xml.etree import ElementTree
+import requests
+
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.utils import is_readable
diff --git a/embedchain/loaders/rss_feed.py b/embedchain/loaders/rss_feed.py
index 033602ea7..ba44e3c90 100644
--- a/embedchain/loaders/rss_feed.py
+++ b/embedchain/loaders/rss_feed.py
@@ -28,7 +28,8 @@ class RSSFeedLoader(BaseLoader):
@staticmethod
def get_rss_content(url: str):
try:
- from langchain.document_loaders import RSSFeedLoader as LangchainRSSFeedLoader
+ from langchain.document_loaders import \
+ RSSFeedLoader as LangchainRSSFeedLoader
except ImportError:
raise ImportError(
"""RSSFeedLoader file requires extra dependencies.
diff --git a/embedchain/loaders/sitemap.py b/embedchain/loaders/sitemap.py
index 77fc90f28..bb6e33e8f 100644
--- a/embedchain/loaders/sitemap.py
+++ b/embedchain/loaders/sitemap.py
@@ -37,9 +37,11 @@ class SitemapLoader(BaseLoader):
if urlparse(sitemap_url).scheme in ["http", "https"]:
response = requests.get(sitemap_url)
response.raise_for_status()
+ soup = BeautifulSoup(response.text, "xml")
else:
with open(sitemap_url, "r") as file:
soup = BeautifulSoup(file, "xml")
+
links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]
if len(links) == 0:
links = [link.text for link in soup.find_all("loc")]
diff --git a/embedchain/loaders/substack.py b/embedchain/loaders/substack.py
index a5fd0b6f6..40278a6b4 100644
--- a/embedchain/loaders/substack.py
+++ b/embedchain/loaders/substack.py
@@ -1,9 +1,10 @@
import hashlib
import logging
import time
+from xml.etree import ElementTree
import requests
-from xml.etree import ElementTree
+
from embedchain.helpers.json_serializable import register_deserializable
from embedchain.loaders.base_loader import BaseLoader
from embedchain.utils import is_readable
diff --git a/embedchain/utils.py b/embedchain/utils.py
index 8daacda5c..ba6efd217 100644
--- a/embedchain/utils.py
+++ b/embedchain/utils.py
@@ -196,7 +196,8 @@ def detect_datatype(source: Any) -> DataType:
formatted_source = format_source(str(source), 30)
if url:
- from langchain.document_loaders.youtube import ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCS
+ from langchain.document_loaders.youtube import \
+ ALLOWED_NETLOCK as YOUTUBE_ALLOWED_NETLOCS
if url.netloc in YOUTUBE_ALLOWED_NETLOCS:
logging.debug(f"Source of `{formatted_source}` detected as `youtube_video`.")
diff --git a/pyproject.toml b/pyproject.toml
index b2dfa5bb7..f550b907e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "embedchain"
-version = "0.1.30"
+version = "0.1.31"
description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
authors = [
"Taranjeet Singh <[email protected]>",