Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: add an opt-in Parallel MCP web document reader (#433)

George Pickett035a92c64a7c
diff --git a/README.md b/README.md
index c79eaa9..3305bb6 100755
--- a/README.md
+++ b/README.md
@@ -909,6 +909,10 @@ Once your iMessage conversations are indexed, you can search with queries like:
### MCP Integration: RAG on Live Data from Any Platform
+**Web documents:** [Search and fetch with Parallel Search MCP](docs/parallel-mcp.md)
+without an API key: `python -m apps.parallel_rag` searches, indexes, and answers in one
+command, or export Markdown for `leann build`.
+
Connect to live data sources through the Model Context Protocol (MCP). LEANN now supports real-time RAG on platforms like Slack, Twitter, and more through standardized MCP servers.
**Key Benefits:**
diff --git a/apps/parallel_data/__init__.py b/apps/parallel_data/__init__.py
new file mode 100644
index 0000000..269d0c4
--- /dev/null
+++ b/apps/parallel_data/__init__.py
@@ -0,0 +1 @@
+"""Opt-in web document ingestion through Parallel Search MCP."""
diff --git a/apps/parallel_data/__main__.py b/apps/parallel_data/__main__.py
new file mode 100644
index 0000000..df979ff
--- /dev/null
+++ b/apps/parallel_data/__main__.py
@@ -0,0 +1,70 @@
+"""Export Parallel MCP excerpts to Markdown for LEANN's document workflow."""
+
+import argparse
+import asyncio
+import hashlib
+import logging
+import re
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from .parallel_mcp_reader import ParallelMCPReader
+
+
+def document_filename(url: str) -> str:
+ """Name a file after its URL: readable host/path slug plus a short hash for uniqueness."""
+ parts = urlsplit(url)
+ slug = re.sub(r"[^A-Za-z0-9]+", "-", f"{parts.netloc}{parts.path}").strip("-").lower()
+ digest = hashlib.sha256(url.encode()).hexdigest()[:10]
+ return f"{slug[:80].rstrip('-') or 'page'}-{digest}.md"
+
+
+def write_documents(documents: list[dict[str, Any]], output_dir: Path) -> list[Path]:
+ """Write one UTF-8 Markdown document per URL, refreshing matching files."""
+ output_dir.mkdir(parents=True, exist_ok=True)
+ paths = []
+ for document in documents:
+ metadata = document["metadata"]
+ path = output_dir / document_filename(metadata["url"])
+ lines = [f"# {metadata['title']}", "", f"Source: {metadata['url']}"]
+ if metadata.get("publish_date"):
+ lines.append(f"Published: {metadata['publish_date']}")
+ lines.extend(["", document["text"], ""])
+ path.write_text("\n".join(lines), encoding="utf-8")
+ paths.append(path)
+ return paths
+
+
+def main() -> None:
+ """Run the opt-in MCP export command."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--objective", required=True, help="What information to find on the web")
+ parser.add_argument(
+ "--search-query", action="append", required=True, help="Repeat for related queries"
+ )
+ parser.add_argument(
+ "--max-results", type=int, default=5, help="Select up to 1-20 results locally"
+ )
+ parser.add_argument(
+ "--fetch", action="store_true", help="Fetch excerpts from selected result URLs"
+ )
+ parser.add_argument("--timeout", type=float, default=90, help="Overall timeout in seconds")
+ parser.add_argument(
+ "--output-dir", type=Path, required=True, help="Directory for exported Markdown"
+ )
+ args = parser.parse_args()
+ logging.basicConfig(level=logging.WARNING)
+ documents = asyncio.run(
+ ParallelMCPReader(timeout=args.timeout).read_web_data(
+ args.objective, args.search_query, args.max_results, args.fetch
+ )
+ )
+ if not documents:
+ parser.exit(1, "No nonempty web documents returned; nothing was written.\n")
+ paths = write_documents(documents, args.output_dir)
+ print(f"Exported {len(paths)} web documents to {args.output_dir}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/apps/parallel_data/parallel_mcp_reader.py b/apps/parallel_data/parallel_mcp_reader.py
new file mode 100644
index 0000000..d7069c8
--- /dev/null
+++ b/apps/parallel_data/parallel_mcp_reader.py
@@ -0,0 +1,146 @@
+"""Read source-attributed web documents using anonymous Parallel Search MCP."""
+
+import asyncio
+import json
+import logging
+from typing import Any
+from uuid import uuid4
+
+import httpx
+from mcp import ClientSession
+from mcp.client.streamable_http import streamable_http_client
+
+logger = logging.getLogger(__name__)
+MCP_URL = "https://search.parallel.ai/mcp"
+USER_AGENT = "LEANN-Parallel-MCP/0.1 (https://github.com/StarTrail-org/LEANN)"
+
+
+def _tool_payload(result: Any) -> dict[str, Any]:
+ """Decode structured MCP output, rejecting tool errors and invalid responses."""
+ if result.isError:
+ detail = "\n".join(getattr(item, "text", "") for item in result.content)
+ raise RuntimeError(f"Parallel MCP tool failed: {detail}")
+ if result.structuredContent is not None:
+ payload = result.structuredContent
+ else:
+ text = "\n".join(getattr(item, "text", "") for item in result.content)
+ try:
+ payload = json.loads(text)
+ except ValueError as exc:
+ raise RuntimeError("Parallel MCP returned invalid JSON") from exc
+ if not isinstance(payload, dict) or not isinstance(payload.get("results"), list):
+ raise RuntimeError("Parallel MCP response is missing its results list")
+ for warning in payload.get("warnings") or []:
+ logger.warning("Parallel MCP: %s", warning.get("message", warning))
+ return payload
+
+
+class ParallelMCPReader:
+ """Search the web and optionally fetch excerpts for local LEANN ingestion.
+
+ This reader sends objectives and queries to a remote service. It does not
+ read saved credentials, load dotenv files, or send a Parallel API key.
+ """
+
+ def __init__(self, timeout: float = 90.0):
+ if timeout <= 0:
+ raise ValueError("timeout must be positive")
+ self.timeout = timeout
+ self.session_id = str(uuid4())
+
+ async def read_web_data(
+ self,
+ objective: str,
+ search_queries: list[str],
+ max_results: int = 5,
+ fetch: bool = False,
+ ) -> list[dict[str, Any]]:
+ """Return text/metadata documents, bounded to 1-20 selected search results.
+
+ With fetch enabled, fetch the selected URLs in one call. Per-URL fetch
+ failures retain their search excerpts and emit warnings. Tool/transport
+ failures propagate; the reader does not retry rate-limited requests.
+ The timeout bounds the entire operation, including connection setup.
+ """
+ if (
+ not objective.strip()
+ or not search_queries
+ or any(not query.strip() for query in search_queries)
+ ):
+ raise ValueError("An objective and nonempty search queries are required")
+ if not 1 <= max_results <= 20:
+ raise ValueError("max_results must be between 1 and 20")
+ return await asyncio.wait_for(
+ self._read(objective, search_queries, max_results, fetch), timeout=self.timeout
+ )
+
+ async def _read(
+ self, objective: str, search_queries: list[str], max_results: int, fetch: bool
+ ) -> list[dict[str, Any]]:
+ async with httpx.AsyncClient(
+ headers={"User-Agent": USER_AGENT}, timeout=self.timeout
+ ) as client:
+ async with streamable_http_client(MCP_URL, http_client=client) as (read, write, _):
+ async with ClientSession(read, write) as session:
+ await session.initialize()
+ tools = {tool.name for tool in (await session.list_tools()).tools}
+ required = {"web_search", "web_fetch"} if fetch else {"web_search"}
+ if not required.issubset(tools):
+ raise RuntimeError(
+ f"Parallel MCP is missing tools: {sorted(required - tools)}"
+ )
+ search = _tool_payload(
+ await session.call_tool(
+ "web_search",
+ {
+ "objective": objective,
+ "search_queries": search_queries,
+ "session_id": self.session_id,
+ },
+ )
+ )
+ # Deduplicate before selecting URLs for the optional fetch call.
+ selected = {}
+ for item in search["results"]:
+ selected.setdefault(item["url"], item)
+ if len(selected) >= max_results:
+ break
+ fetched = {}
+ if fetch and selected:
+ extraction = _tool_payload(
+ await session.call_tool(
+ "web_fetch",
+ {
+ "urls": list(selected),
+ "search_queries": search_queries,
+ "session_id": self.session_id,
+ },
+ )
+ )
+ fetched = {item["url"]: item for item in extraction["results"]}
+ for error in extraction.get("errors", []):
+ logger.warning(
+ "Fetch failed for %s (%s); retaining search excerpts",
+ error.get("url"),
+ error.get("error_type"),
+ )
+ documents = []
+ for url, search_item in selected.items():
+ item = fetched.get(url, search_item)
+ text = "\n\n".join(item.get("excerpts", [])).strip()
+ if not text:
+ text = "\n\n".join(search_item.get("excerpts", [])).strip()
+ if text:
+ documents.append(
+ {
+ "text": text,
+ "metadata": {
+ "url": url,
+ "title": item.get("title") or search_item.get("title") or url,
+ "publish_date": item.get("publish_date")
+ or search_item.get("publish_date"),
+ "source": "parallel-mcp",
+ },
+ }
+ )
+ return documents
diff --git a/apps/parallel_data/requirements.txt b/apps/parallel_data/requirements.txt
new file mode 100644
index 0000000..9847782
--- /dev/null
+++ b/apps/parallel_data/requirements.txt
@@ -0,0 +1,2 @@
+mcp>=1.30.0,<2
+httpx>=0.28.1,<1
diff --git a/apps/parallel_rag.py b/apps/parallel_rag.py
new file mode 100644
index 0000000..9a6a267
--- /dev/null
+++ b/apps/parallel_rag.py
@@ -0,0 +1,85 @@
+"""
+Web RAG example using Parallel Search MCP and the unified interface.
+
+Searches the web (optionally fetching the selected pages) through anonymous
+Parallel Search MCP, then chunks and indexes the excerpts in LEANN.
+
+Usage:
+ python -m apps.parallel_rag --objective "Find Yichuan Wang's MLSys research" \\
+ --search-query "Yichuan Wang MLsys" --fetch --query "What did he publish at MLSys?"
+"""
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+from typing import Any
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
+from llama_index.core import Document
+
+from .parallel_data.parallel_mcp_reader import ParallelMCPReader
+
+
+class ParallelWebRAG(BaseRAGExample):
+ """RAG example for web documents from Parallel Search MCP."""
+
+ def __init__(self):
+ super().__init__(
+ name="Parallel Web",
+ description="Search the web with Parallel Search MCP and run RAG over the results",
+ default_index_name="parallel_web_index",
+ )
+
+ def _add_specific_arguments(self, parser: argparse.ArgumentParser):
+ web_group = parser.add_argument_group("Parallel Web Parameters")
+ web_group.add_argument(
+ "--objective", required=True, help="What information to find on the web"
+ )
+ web_group.add_argument(
+ "--search-query", action="append", required=True, help="Repeat for related queries"
+ )
+ web_group.add_argument(
+ "--max-results", type=int, default=5, help="Select up to 1-20 results locally"
+ )
+ web_group.add_argument(
+ "--fetch", action="store_true", help="Fetch excerpts from selected result URLs"
+ )
+ web_group.add_argument(
+ "--timeout", type=float, default=90, help="Overall web timeout in seconds"
+ )
+ web_group.add_argument(
+ "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
+ )
+ web_group.add_argument(
+ "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
+ )
+
+ async def load_data(self, args) -> list[dict[str, Any]]:
+ print(f"Searching the web via Parallel Search MCP: {args.search_query}")
+ web_documents = await ParallelMCPReader(timeout=args.timeout).read_web_data(
+ args.objective, args.search_query, args.max_results, args.fetch
+ )
+ for document in web_documents:
+ print(f" - {document['metadata']['title']} ({document['metadata']['url']})")
+ documents = [
+ # Keep title and URL in the text so the LLM can cite its sources.
+ Document(
+ text=f"{doc['metadata']['title']}\nSource: {doc['metadata']['url']}\n\n{doc['text']}",
+ metadata=doc["metadata"],
+ )
+ for doc in web_documents
+ ]
+ texts = create_text_chunks(
+ documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
+ )
+ print(f"Created {len(texts)} text chunks from {len(documents)} web documents")
+ return texts
+
+
+if __name__ == "__main__":
+ asyncio.run(ParallelWebRAG().run())
diff --git a/docs/parallel-mcp.md b/docs/parallel-mcp.md
new file mode 100644
index 0000000..2c820d4
--- /dev/null
+++ b/docs/parallel-mcp.md
@@ -0,0 +1,77 @@
+# Web documents from Parallel Search MCP
+
+Use the opt-in reader to search the web and save source-attributed Markdown for
+LEANN's document ingestion. [Parallel Search MCP](https://docs.parallel.ai/integrations/mcp/search-mcp)
+provides free anonymous search in Fast mode and page fetching, with no account or
+API key. Free access has rate limits; this reader does not retry failed requests.
+
+Search objectives, queries, and selected URLs are sent to Parallel. This is a
+remote data source, unlike LEANN's local retrieval. Avoid including private data
+in your search. No existing LEANN application or provider default changes.
+
+## Install and export
+
+From the repository root, create a separate environment for the reader (Python
+3.10 or newer). It does not require the LEANN runtime or an LLM:
+
+```bash
+uv venv /tmp/leann-parallel
+uv pip install --python /tmp/leann-parallel/bin/python -r apps/parallel_data/requirements.txt
+/tmp/leann-parallel/bin/python -m apps.parallel_data \
+ --objective "Find how LEANN reduces vector index storage" \
+ --search-query "LEANN low storage vector index" \
+ --max-results 3 \
+ --output-dir ./web-data
+```
+
+Add `--fetch` to fetch excerpts from the selected URLs after searching. This
+uses MCP Streamable HTTP at `https://search.parallel.ai/mcp`. The reader retains
+search excerpts when a particular URL cannot be fetched and prints a warning.
+It exports excerpts, not guaranteed full pages. `--max-results` limits local
+selection and fetching, not the server's search result count. `--timeout` bounds
+the entire operation (90 seconds by default).
+
+Each file is named after its URL (for example
+`arxiv-org-abs-2506-08276-<hash>.md`) and contains its title, source URL,
+publication date when available, and excerpts. Repeating a search refreshes files for matching URLs; other files remain.
+Use a fresh directory for a separate collection.
+
+## Search, index, and ask in one command
+
+Inside your LEANN environment, install the reader requirements and run the RAG
+app. It searches, chunks the excerpts, builds a LEANN index, and answers with the
+configured LLM (`--llm`, see the other `apps/*_rag.py` examples):
+
+```bash
+uv pip install -r apps/parallel_data/requirements.txt
+python -m apps.parallel_rag \
+ --objective "Find how LEANN reduces vector index storage" \
+ --search-query "LEANN low storage vector index" \
+ --fetch --query "How does LEANN save storage?"
+```
+
+Each chunk keeps the page title and source URL so answers can cite them. The
+index is reused on later runs; pass `--force-rebuild` after changing the search.
+
+## Use the exported documents
+
+In your existing [LEANN installation](../README.md#installation), ingest the
+exported directory using the normal document workflow:
+
+```bash
+leann build web-notes --docs ./web-data
+leann search web-notes "How does LEANN save storage?"
+```
+
+Embedding configuration follows your existing LEANN setup. The export command
+only collects documents; it does not build an index or run an agent loop.
+
+For Python callers, `ParallelMCPReader.read_web_data()` returns dictionaries with
+`text` and `metadata` fields, including `url`, `title`, `publish_date`, and `source`.
+
+## Reader tests
+
+```bash
+uv pip install --python /tmp/leann-parallel/bin/python pytest
+/tmp/leann-parallel/bin/python -m pytest -o addopts= tests/test_parallel_mcp_reader.py
+```
diff --git a/tests/test_parallel_mcp_reader.py b/tests/test_parallel_mcp_reader.py
new file mode 100644
index 0000000..89b35ac
--- /dev/null
+++ b/tests/test_parallel_mcp_reader.py
@@ -0,0 +1,222 @@
+"""Exercise the real MCP reader/transport against an HTTP protocol fixture."""
+
+import asyncio
+import importlib
+import json
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+# The opt-in reader has its own installation; do not add SDK requirements to
+# collection of LEANN's ordinary test suite.
+httpx = pytest.importorskip("httpx")
+pytest.importorskip("mcp")
+reader = importlib.import_module("apps.parallel_data.parallel_mcp_reader")
+export = importlib.import_module("apps.parallel_data.__main__")
+
+
+def install_server(monkeypatch, search, extraction=None, tool_error=False, tools=None):
+ """Install an HTTP server fixture that observes every actual SDK request."""
+ requests = []
+ calls = []
+
+ def handle(request):
+ requests.append(request)
+ assert str(request.url) == reader.MCP_URL
+ assert request.headers["User-Agent"] == reader.USER_AGENT
+ assert "authorization" not in request.headers
+ assert "cookie" not in request.headers
+ payload = json.loads(request.content)
+ if "id" not in payload:
+ return httpx.Response(202)
+ method = payload["method"]
+ if method == "initialize":
+ result = {
+ "protocolVersion": payload["params"]["protocolVersion"],
+ "capabilities": {"tools": {}},
+ "serverInfo": {"name": "fixture", "version": "1"},
+ }
+ elif method == "tools/list":
+ result = {
+ "tools": [
+ {"name": name, "inputSchema": {"type": "object"}}
+ for name in (tools if tools is not None else ["web_search", "web_fetch"])
+ ]
+ }
+ else:
+ assert method == "tools/call"
+ calls.append(payload["params"])
+ data = search if payload["params"]["name"] == "web_search" else extraction
+ result = {
+ "content": [{"type": "text", "text": json.dumps(data)}],
+ "isError": tool_error,
+ }
+ return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": result})
+
+ client_class = httpx.AsyncClient
+ monkeypatch.setattr(
+ reader.httpx,
+ "AsyncClient",
+ lambda **kwargs: client_class(transport=httpx.MockTransport(handle), **kwargs),
+ )
+ return requests, calls
+
+
+def item(url, text="Search excerpt", title="A source"):
+ return {"url": url, "title": title, "excerpts": [text], "publish_date": "2026-10-01"}
+
+
+def test_search_fetch_real_transport(monkeypatch, caplog):
+ first = item("https://example.com/one")
+ second = item("https://example.com/two", "Fallback excerpt")
+ requests, calls = install_server(
+ monkeypatch,
+ {"results": [first, first, second, item("https://example.com/three")]},
+ {
+ "results": [item(first["url"], "Fetched excerpt", "Fetched title")],
+ "errors": [{"url": second["url"], "error_type": "HTTP_ERROR"}],
+ },
+ )
+ monkeypatch.setenv("PARALLEL_API_KEY", "must-not-be-sent")
+ instance = reader.ParallelMCPReader()
+ documents = asyncio.run(instance.read_web_data("Research", ["research web sources"], 2, True))
+ assert len(requests) >= 5 # initialization, discovery, search, and fetch
+ assert [call["name"] for call in calls] == ["web_search", "web_fetch"]
+ assert calls[1]["arguments"]["urls"] == [first["url"], second["url"]]
+ assert calls[0]["arguments"]["session_id"] == calls[1]["arguments"]["session_id"]
+ assert calls[0]["arguments"]["session_id"] == instance.session_id
+ assert calls[1]["arguments"]["search_queries"] == ["research web sources"]
+ assert documents[0]["text"] == "Fetched excerpt"
+ assert documents[0]["metadata"]["title"] == "Fetched title"
+ assert documents[1]["text"] == "Fallback excerpt"
+ assert "retaining search excerpts" in caplog.text
+
+
+def test_search_only_and_blank_excerpts(monkeypatch):
+ _, calls = install_server(
+ monkeypatch,
+ {"results": [item("https://example.com/empty", " "), item("https://example.com")]},
+ )
+ documents = asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["web sources"]))
+ assert len(documents) == 1
+ assert documents[0]["metadata"]["url"] == "https://example.com"
+ assert [call["name"] for call in calls] == ["web_search"]
+
+
+def test_empty_search_does_not_fetch(monkeypatch):
+ _, calls = install_server(monkeypatch, {"results": []})
+ assert (
+ asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["sources"], fetch=True))
+ == []
+ )
+ assert len(calls) == 1
+
+
+def test_tool_error_propagates(monkeypatch):
+ install_server(monkeypatch, {"error": "rate limited"}, tool_error=True)
+ with pytest.raises(Exception) as error:
+ asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["sources"]))
+ assert "Parallel MCP tool failed" in repr(error.value)
+
+
+def test_missing_tool_propagates(monkeypatch):
+ install_server(monkeypatch, {"results": []}, tools=["web_search"])
+ with pytest.raises(Exception) as error:
+ asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["sources"], fetch=True))
+ assert "missing tools" in repr(error.value)
+
+
+def test_structured_output_and_invalid_json():
+ result = SimpleNamespace(isError=False, structuredContent={"results": []}, content=[])
+ assert reader._tool_payload(result) == {"results": []}
+ result.structuredContent = None
+ result.content = [SimpleNamespace(text="invalid")]
+ with pytest.raises(RuntimeError, match="invalid JSON"):
+ reader._tool_payload(result)
+
+
+ "objective,queries,limit", [("", ["x"], 1), ("x", [], 1), ("x", [" "], 1), ("x", ["y"], 21)]
+)
+def test_invalid_arguments(objective, queries, limit):
+ with pytest.raises(ValueError):
+ asyncio.run(reader.ParallelMCPReader().read_web_data(objective, queries, limit))
+
+
+def test_overall_timeout(monkeypatch):
+ async def stalled(*args):
+ await asyncio.Event().wait()
+
+ instance = reader.ParallelMCPReader(timeout=0.01)
+ monkeypatch.setattr(instance, "_read", stalled)
+ with pytest.raises(asyncio.TimeoutError):
+ asyncio.run(instance.read_web_data("Research", ["sources"]))
+
+
+def test_export_command_preserves_sources(monkeypatch, tmp_path, capsys):
+ async def read_data(self, objective, queries, limit, fetch):
+ assert (objective, queries, limit, fetch) == ("Research", ["web sources"], 5, False)
+ return [
+ {"text": "Useful café excerpt", "metadata": item("https://example.com/../../source")}
+ ]
+
+ monkeypatch.setattr(reader.ParallelMCPReader, "read_web_data", read_data)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "parallel_data",
+ "--objective",
+ "Research",
+ "--search-query",
+ "web sources",
+ "--output-dir",
+ str(tmp_path),
+ ],
+ )
+ export.main()
+ paths = list(tmp_path.glob("*.md"))
+ assert len(paths) == 1
+ assert paths[0].parent == tmp_path
+ assert paths[0].name.startswith("example-com-source-")
+ text = paths[0].read_text()
+ assert "Source: https://example.com/../../source" in text
+ assert "Published: 2026-10-01" in text
+ assert "Useful café excerpt" in text
+ assert "Exported 1" in capsys.readouterr().out
+
+
+ "url,prefix",
+ [
+ ("https://arxiv.org/abs/2506.08276", "arxiv-org-abs-2506-08276-"),
+ ("https://yichuan-w.github.io/", "yichuan-w-github-io-"),
+ ("https://example.com/" + "a" * 200, "example-com-" + "a" * 68 + "-"),
+ ],
+)
+def test_document_filename_is_readable_and_unique(url, prefix):
+ name = export.document_filename(url)
+ assert name.startswith(prefix) and name.endswith(".md")
+ assert export.document_filename(url) == name
+ assert export.document_filename(url + "?v=2") != name
+
+
+def test_rag_app_chunks_web_documents_with_sources(monkeypatch):
+ pytest.importorskip("leann")
+ pytest.importorskip("llama_index.core")
+ rag = importlib.import_module("apps.parallel_rag")
+
+ async def read_data(self, objective, queries, limit, fetch):
+ assert (objective, queries, limit, fetch) == ("Research", ["web sources"], 5, True)
+ return [{"text": "LEANN recomputes embeddings. " * 40, "metadata": item("https://a.io/x")}]
+
+ monkeypatch.setattr(reader.ParallelMCPReader, "read_web_data", read_data)
+ app = rag.ParallelWebRAG()
+ args = app.parser.parse_args(
+ ["--objective", "Research", "--search-query", "web sources", "--fetch"]
+ )
+ texts = asyncio.run(app.load_data(args))
+ assert len(texts) > 1
+ assert texts[0]["text"].startswith("A source\nSource: https://a.io/x")
+ assert all(text["metadata"]["url"] == "https://a.io/x" for text in texts)