The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
Feat: Improve test coverage of DocsSiteLoader (#758)
SerSamgy431f8c2c6a7a
diff --git a/pyproject.toml b/pyproject.tomlindex 7bfc09d24..bc9c47cfd 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -123,6 +123,7 @@ pytest-env = "^0.8.1"click = "^8.1.3"isort = "^5.12.0"pytest-cov = "^4.1.0"+responses = "^0.23.3"[tool.poetry.extras]streamlit = ["streamlit"]diff --git a/tests/loaders/test_docs_site_loader.py b/tests/loaders/test_docs_site_loader.pynew file mode 100644index 000000000..16b503b9f--- /dev/null+++ b/tests/loaders/test_docs_site_loader.py@@ -0,0 +1,218 @@+import pytest+import responses+from bs4 import BeautifulSoup+++ "ignored_tag",+ [+ "<nav>This is a navigation bar.</nav>",+ "<aside>This is an aside.</aside>",+ "<form>This is a form.</form>",+ "<header>This is a header.</header>",+ "<noscript>This is a noscript.</noscript>",+ "<svg>This is an SVG.</svg>",+ "<canvas>This is a canvas.</canvas>",+ "<footer>This is a footer.</footer>",+ "<script>This is a script.</script>",+ "<style>This is a style.</style>",+ ],+ ids=["nav", "aside", "form", "header", "noscript", "svg", "canvas", "footer", "script", "style"],+)+ "selectee",+ [+ """+<article class="bd-article">+ <h2>Article Title</h2>+ <p>Article content goes here.</p>+ {ignored_tag}+</article>+""",+ """+<article role="main">+ <h2>Main Article Title</h2>+ <p>Main article content goes here.</p>+ {ignored_tag}+</article>+""",+ """+<div class="md-content">+ <h2>Markdown Content</h2>+ <p>Markdown content goes here.</p>+ {ignored_tag}+</div>+""",+ """+<div role="main">+ <h2>Main Content</h2>+ <p>Main content goes here.</p>+ {ignored_tag}+</div>+""",+ """+<div class="container">+ <h2>Container</h2>+ <p>Container content goes here.</p>+ {ignored_tag}+</div>+ """,+ """+<div class="section">+ <h2>Section</h2>+ <p>Section content goes here.</p>+ {ignored_tag}+</div>+ """,+ """+<article>+ <h2>Generic Article</h2>+ <p>Generic article content goes here.</p>+ {ignored_tag}+</article>+ """,+ """+<main>+ <h2>Main Content</h2>+ <p>Main content goes here.</p>+ {ignored_tag}+</main>+""",+ ],+ ids=[+ "article.bd-article",+ 'article[role="main"]',+ "div.md-content",+ 'div[role="main"]',+ "div.container",+ "div.section",+ "article",+ "main",+ ],+)+def test_load_data_gets_by_selectors_and_ignored_tags(selectee, ignored_tag, loader, mocked_responses, mocker):+ child_url = "https://docs.embedchain.ai/quickstart"+ selectee = selectee.format(ignored_tag=ignored_tag)+ html_body = """+<!DOCTYPE html>+<html lang="en">+<body>+ {selectee}+</body>+</html>+"""+ html_body = html_body.format(selectee=selectee)+ mocked_responses.get(child_url, body=html_body, status=200, content_type="text/html")++ url = "https://docs.embedchain.ai/"+ html_body = """+<!DOCTYPE html>+<html lang="en">+<body>+ <li><a href="/quickstart">Quickstart</a></li>+</body>+</html>+"""+ mocked_responses.get(url, body=html_body, status=200, content_type="text/html")++ mock_sha256 = mocker.patch("embedchain.loaders.docs_site_loader.hashlib.sha256")+ doc_id = "mocked_hash"+ mock_sha256.return_value.hexdigest.return_value = doc_id++ result = loader.load_data(url)+ selector_soup = BeautifulSoup(selectee, "html.parser")+ expected_content = " ".join((selector_soup.select_one("h2").get_text(), selector_soup.select_one("p").get_text()))+ assert result["doc_id"] == doc_id+ assert result["data"] == [+ {+ "content": expected_content,+ "meta_data": {"url": "https://docs.embedchain.ai/quickstart"},+ }+ ]+++def test_load_data_gets_child_links_recursively(loader, mocked_responses, mocker):+ child_url = "https://docs.embedchain.ai/quickstart"+ html_body = """+<!DOCTYPE html>+<html lang="en">+<body>+ <li><a href="/">..</a></li>+ <li><a href="/quickstart">.</a></li>+</body>+</html>+"""+ mocked_responses.get(child_url, body=html_body, status=200, content_type="text/html")++ child_url = "https://docs.embedchain.ai/introduction"+ html_body = """+<!DOCTYPE html>+<html lang="en">+<body>+ <li><a href="/">..</a></li>+ <li><a href="/introduction">.</a></li>+</body>+</html>+"""+ mocked_responses.get(child_url, body=html_body, status=200, content_type="text/html")++ url = "https://docs.embedchain.ai/"+ html_body = """+<!DOCTYPE html>+<html lang="en">+<body>+ <li><a href="/quickstart">Quickstart</a></li>+ <li><a href="/introduction">Introduction</a></li>+</body>+</html>+"""+ mocked_responses.get(url, body=html_body, status=200, content_type="text/html")++ mock_sha256 = mocker.patch("embedchain.loaders.docs_site_loader.hashlib.sha256")+ doc_id = "mocked_hash"+ mock_sha256.return_value.hexdigest.return_value = doc_id++ result = loader.load_data(url)+ assert result["doc_id"] == doc_id+ expected_data = [+ {"content": "..\n.", "meta_data": {"url": "https://docs.embedchain.ai/quickstart"}},+ {"content": "..\n.", "meta_data": {"url": "https://docs.embedchain.ai/introduction"}},+ ]+ assert all(item in expected_data for item in result["data"])+++def test_load_data_fails_to_fetch_website(loader, mocked_responses, mocker):+ child_url = "https://docs.embedchain.ai/introduction"+ mocked_responses.get(child_url, status=404)++ url = "https://docs.embedchain.ai/"+ html_body = """+<!DOCTYPE html>+<html lang="en">+<body>+ <li><a href="/introduction">Introduction</a></li>+</body>+</html>+"""+ mocked_responses.get(url, body=html_body, status=200, content_type="text/html")++ mock_sha256 = mocker.patch("embedchain.loaders.docs_site_loader.hashlib.sha256")+ doc_id = "mocked_hash"+ mock_sha256.return_value.hexdigest.return_value = doc_id++ result = loader.load_data(url)+ assert result["doc_id"] is doc_id+ assert result["data"] == []+++def loader():+ from embedchain.loaders.docs_site_loader import DocsSiteLoader++ return DocsSiteLoader()+++def mocked_responses():+ with responses.RequestsMock() as rsps:+ yield rsps