[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: add an opt-in Parallel MCP web document reader (#433)
George Pickett035a92c64a7c
diff --git a/README.md b/README.mdindex c79eaa9..3305bb6 100755--- a/README.md+++ b/README.md@@ -909,6 +909,10 @@ Once your iMessage conversations are indexed, you can search with queries like:### MCP Integration: RAG on Live Data from Any Platform+**Web documents:** [Search and fetch with Parallel Search MCP](docs/parallel-mcp.md)+without an API key: `python -m apps.parallel_rag` searches, indexes, and answers in one+command, or export Markdown for `leann build`.+Connect to live data sources through the Model Context Protocol (MCP). LEANN now supports real-time RAG on platforms like Slack, Twitter, and more through standardized MCP servers.**Key Benefits:**diff --git a/apps/parallel_data/__init__.py b/apps/parallel_data/__init__.pynew file mode 100644index 0000000..269d0c4--- /dev/null+++ b/apps/parallel_data/__init__.py@@ -0,0 +1 @@+"""Opt-in web document ingestion through Parallel Search MCP."""diff --git a/apps/parallel_data/__main__.py b/apps/parallel_data/__main__.pynew file mode 100644index 0000000..df979ff--- /dev/null+++ b/apps/parallel_data/__main__.py@@ -0,0 +1,70 @@+"""Export Parallel MCP excerpts to Markdown for LEANN's document workflow."""++import argparse+import asyncio+import hashlib+import logging+import re+from pathlib import Path+from typing import Any+from urllib.parse import urlsplit++from .parallel_mcp_reader import ParallelMCPReader+++def document_filename(url: str) -> str:+ """Name a file after its URL: readable host/path slug plus a short hash for uniqueness."""+ parts = urlsplit(url)+ slug = re.sub(r"[^A-Za-z0-9]+", "-", f"{parts.netloc}{parts.path}").strip("-").lower()+ digest = hashlib.sha256(url.encode()).hexdigest()[:10]+ return f"{slug[:80].rstrip('-') or 'page'}-{digest}.md"+++def write_documents(documents: list[dict[str, Any]], output_dir: Path) -> list[Path]:+ """Write one UTF-8 Markdown document per URL, refreshing matching files."""+ output_dir.mkdir(parents=True, exist_ok=True)+ paths = []+ for document in documents:+ metadata = document["metadata"]+ path = output_dir / document_filename(metadata["url"])+ lines = [f"# {metadata['title']}", "", f"Source: {metadata['url']}"]+ if metadata.get("publish_date"):+ lines.append(f"Published: {metadata['publish_date']}")+ lines.extend(["", document["text"], ""])+ path.write_text("\n".join(lines), encoding="utf-8")+ paths.append(path)+ return paths+++def main() -> None:+ """Run the opt-in MCP export command."""+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--objective", required=True, help="What information to find on the web")+ parser.add_argument(+ "--search-query", action="append", required=True, help="Repeat for related queries"+ )+ parser.add_argument(+ "--max-results", type=int, default=5, help="Select up to 1-20 results locally"+ )+ parser.add_argument(+ "--fetch", action="store_true", help="Fetch excerpts from selected result URLs"+ )+ parser.add_argument("--timeout", type=float, default=90, help="Overall timeout in seconds")+ parser.add_argument(+ "--output-dir", type=Path, required=True, help="Directory for exported Markdown"+ )+ args = parser.parse_args()+ logging.basicConfig(level=logging.WARNING)+ documents = asyncio.run(+ ParallelMCPReader(timeout=args.timeout).read_web_data(+ args.objective, args.search_query, args.max_results, args.fetch+ )+ )+ if not documents:+ parser.exit(1, "No nonempty web documents returned; nothing was written.\n")+ paths = write_documents(documents, args.output_dir)+ print(f"Exported {len(paths)} web documents to {args.output_dir}")+++if __name__ == "__main__":+ main()diff --git a/apps/parallel_data/parallel_mcp_reader.py b/apps/parallel_data/parallel_mcp_reader.pynew file mode 100644index 0000000..d7069c8--- /dev/null+++ b/apps/parallel_data/parallel_mcp_reader.py@@ -0,0 +1,146 @@+"""Read source-attributed web documents using anonymous Parallel Search MCP."""++import asyncio+import json+import logging+from typing import Any+from uuid import uuid4++import httpx+from mcp import ClientSession+from mcp.client.streamable_http import streamable_http_client++logger = logging.getLogger(__name__)+MCP_URL = "https://search.parallel.ai/mcp"+USER_AGENT = "LEANN-Parallel-MCP/0.1 (https://github.com/StarTrail-org/LEANN)"+++def _tool_payload(result: Any) -> dict[str, Any]:+ """Decode structured MCP output, rejecting tool errors and invalid responses."""+ if result.isError:+ detail = "\n".join(getattr(item, "text", "") for item in result.content)+ raise RuntimeError(f"Parallel MCP tool failed: {detail}")+ if result.structuredContent is not None:+ payload = result.structuredContent+ else:+ text = "\n".join(getattr(item, "text", "") for item in result.content)+ try:+ payload = json.loads(text)+ except ValueError as exc:+ raise RuntimeError("Parallel MCP returned invalid JSON") from exc+ if not isinstance(payload, dict) or not isinstance(payload.get("results"), list):+ raise RuntimeError("Parallel MCP response is missing its results list")+ for warning in payload.get("warnings") or []:+ logger.warning("Parallel MCP: %s", warning.get("message", warning))+ return payload+++class ParallelMCPReader:+ """Search the web and optionally fetch excerpts for local LEANN ingestion.++ This reader sends objectives and queries to a remote service. It does not+ read saved credentials, load dotenv files, or send a Parallel API key.+ """++ def __init__(self, timeout: float = 90.0):+ if timeout <= 0:+ raise ValueError("timeout must be positive")+ self.timeout = timeout+ self.session_id = str(uuid4())++ async def read_web_data(+ self,+ objective: str,+ search_queries: list[str],+ max_results: int = 5,+ fetch: bool = False,+ ) -> list[dict[str, Any]]:+ """Return text/metadata documents, bounded to 1-20 selected search results.++ With fetch enabled, fetch the selected URLs in one call. Per-URL fetch+ failures retain their search excerpts and emit warnings. Tool/transport+ failures propagate; the reader does not retry rate-limited requests.+ The timeout bounds the entire operation, including connection setup.+ """+ if (+ not objective.strip()+ or not search_queries+ or any(not query.strip() for query in search_queries)+ ):+ raise ValueError("An objective and nonempty search queries are required")+ if not 1 <= max_results <= 20:+ raise ValueError("max_results must be between 1 and 20")+ return await asyncio.wait_for(+ self._read(objective, search_queries, max_results, fetch), timeout=self.timeout+ )++ async def _read(+ self, objective: str, search_queries: list[str], max_results: int, fetch: bool+ ) -> list[dict[str, Any]]:+ async with httpx.AsyncClient(+ headers={"User-Agent": USER_AGENT}, timeout=self.timeout+ ) as client:+ async with streamable_http_client(MCP_URL, http_client=client) as (read, write, _):+ async with ClientSession(read, write) as session:+ await session.initialize()+ tools = {tool.name for tool in (await session.list_tools()).tools}+ required = {"web_search", "web_fetch"} if fetch else {"web_search"}+ if not required.issubset(tools):+ raise RuntimeError(+ f"Parallel MCP is missing tools: {sorted(required - tools)}"+ )+ search = _tool_payload(+ await session.call_tool(+ "web_search",+ {+ "objective": objective,+ "search_queries": search_queries,+ "session_id": self.session_id,+ },+ )+ )+ # Deduplicate before selecting URLs for the optional fetch call.+ selected = {}+ for item in search["results"]:+ selected.setdefault(item["url"], item)+ if len(selected) >= max_results:+ break+ fetched = {}+ if fetch and selected:+ extraction = _tool_payload(+ await session.call_tool(+ "web_fetch",+ {+ "urls": list(selected),+ "search_queries": search_queries,+ "session_id": self.session_id,+ },+ )+ )+ fetched = {item["url"]: item for item in extraction["results"]}+ for error in extraction.get("errors", []):+ logger.warning(+ "Fetch failed for %s (%s); retaining search excerpts",+ error.get("url"),+ error.get("error_type"),+ )+ documents = []+ for url, search_item in selected.items():+ item = fetched.get(url, search_item)+ text = "\n\n".join(item.get("excerpts", [])).strip()+ if not text:+ text = "\n\n".join(search_item.get("excerpts", [])).strip()+ if text:+ documents.append(+ {+ "text": text,+ "metadata": {+ "url": url,+ "title": item.get("title") or search_item.get("title") or url,+ "publish_date": item.get("publish_date")+ or search_item.get("publish_date"),+ "source": "parallel-mcp",+ },+ }+ )+ return documentsdiff --git a/apps/parallel_data/requirements.txt b/apps/parallel_data/requirements.txtnew file mode 100644index 0000000..9847782--- /dev/null+++ b/apps/parallel_data/requirements.txt@@ -0,0 +1,2 @@+mcp>=1.30.0,<2+httpx>=0.28.1,<1diff --git a/apps/parallel_rag.py b/apps/parallel_rag.pynew file mode 100644index 0000000..9a6a267--- /dev/null+++ b/apps/parallel_rag.py@@ -0,0 +1,85 @@+"""+Web RAG example using Parallel Search MCP and the unified interface.++Searches the web (optionally fetching the selected pages) through anonymous+Parallel Search MCP, then chunks and indexes the excerpts in LEANN.++Usage:+ python -m apps.parallel_rag --objective "Find Yichuan Wang's MLSys research" \\+ --search-query "Yichuan Wang MLsys" --fetch --query "What did he publish at MLSys?"+"""++import argparse+import asyncio+import sys+from pathlib import Path+from typing import Any++# Add parent directory to path for imports+sys.path.insert(0, str(Path(__file__).parent))++from base_rag_example import BaseRAGExample+from chunking import create_text_chunks+from llama_index.core import Document++from .parallel_data.parallel_mcp_reader import ParallelMCPReader+++class ParallelWebRAG(BaseRAGExample):+ """RAG example for web documents from Parallel Search MCP."""++ def __init__(self):+ super().__init__(+ name="Parallel Web",+ description="Search the web with Parallel Search MCP and run RAG over the results",+ default_index_name="parallel_web_index",+ )++ def _add_specific_arguments(self, parser: argparse.ArgumentParser):+ web_group = parser.add_argument_group("Parallel Web Parameters")+ web_group.add_argument(+ "--objective", required=True, help="What information to find on the web"+ )+ web_group.add_argument(+ "--search-query", action="append", required=True, help="Repeat for related queries"+ )+ web_group.add_argument(+ "--max-results", type=int, default=5, help="Select up to 1-20 results locally"+ )+ web_group.add_argument(+ "--fetch", action="store_true", help="Fetch excerpts from selected result URLs"+ )+ web_group.add_argument(+ "--timeout", type=float, default=90, help="Overall web timeout in seconds"+ )+ web_group.add_argument(+ "--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"+ )+ web_group.add_argument(+ "--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"+ )++ async def load_data(self, args) -> list[dict[str, Any]]:+ print(f"Searching the web via Parallel Search MCP: {args.search_query}")+ web_documents = await ParallelMCPReader(timeout=args.timeout).read_web_data(+ args.objective, args.search_query, args.max_results, args.fetch+ )+ for document in web_documents:+ print(f" - {document['metadata']['title']} ({document['metadata']['url']})")+ documents = [+ # Keep title and URL in the text so the LLM can cite its sources.+ Document(+ text=f"{doc['metadata']['title']}\nSource: {doc['metadata']['url']}\n\n{doc['text']}",+ metadata=doc["metadata"],+ )+ for doc in web_documents+ ]+ texts = create_text_chunks(+ documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap+ )+ print(f"Created {len(texts)} text chunks from {len(documents)} web documents")+ return texts+++if __name__ == "__main__":+ asyncio.run(ParallelWebRAG().run())diff --git a/docs/parallel-mcp.md b/docs/parallel-mcp.mdnew file mode 100644index 0000000..2c820d4--- /dev/null+++ b/docs/parallel-mcp.md@@ -0,0 +1,77 @@+# Web documents from Parallel Search MCP++Use the opt-in reader to search the web and save source-attributed Markdown for+LEANN's document ingestion. [Parallel Search MCP](https://docs.parallel.ai/integrations/mcp/search-mcp)+provides free anonymous search in Fast mode and page fetching, with no account or+API key. Free access has rate limits; this reader does not retry failed requests.++Search objectives, queries, and selected URLs are sent to Parallel. This is a+remote data source, unlike LEANN's local retrieval. Avoid including private data+in your search. No existing LEANN application or provider default changes.++## Install and export++From the repository root, create a separate environment for the reader (Python+3.10 or newer). It does not require the LEANN runtime or an LLM:++```bash+uv venv /tmp/leann-parallel+uv pip install --python /tmp/leann-parallel/bin/python -r apps/parallel_data/requirements.txt+/tmp/leann-parallel/bin/python -m apps.parallel_data \+ --objective "Find how LEANN reduces vector index storage" \+ --search-query "LEANN low storage vector index" \+ --max-results 3 \+ --output-dir ./web-data+```++Add `--fetch` to fetch excerpts from the selected URLs after searching. This+uses MCP Streamable HTTP at `https://search.parallel.ai/mcp`. The reader retains+search excerpts when a particular URL cannot be fetched and prints a warning.+It exports excerpts, not guaranteed full pages. `--max-results` limits local+selection and fetching, not the server's search result count. `--timeout` bounds+the entire operation (90 seconds by default).++Each file is named after its URL (for example+`arxiv-org-abs-2506-08276-<hash>.md`) and contains its title, source URL,+publication date when available, and excerpts. Repeating a search refreshes files for matching URLs; other files remain.+Use a fresh directory for a separate collection.++## Search, index, and ask in one command++Inside your LEANN environment, install the reader requirements and run the RAG+app. It searches, chunks the excerpts, builds a LEANN index, and answers with the+configured LLM (`--llm`, see the other `apps/*_rag.py` examples):++```bash+uv pip install -r apps/parallel_data/requirements.txt+python -m apps.parallel_rag \+ --objective "Find how LEANN reduces vector index storage" \+ --search-query "LEANN low storage vector index" \+ --fetch --query "How does LEANN save storage?"+```++Each chunk keeps the page title and source URL so answers can cite them. The+index is reused on later runs; pass `--force-rebuild` after changing the search.++## Use the exported documents++In your existing [LEANN installation](../README.md#installation), ingest the+exported directory using the normal document workflow:++```bash+leann build web-notes --docs ./web-data+leann search web-notes "How does LEANN save storage?"+```++Embedding configuration follows your existing LEANN setup. The export command+only collects documents; it does not build an index or run an agent loop.++For Python callers, `ParallelMCPReader.read_web_data()` returns dictionaries with+`text` and `metadata` fields, including `url`, `title`, `publish_date`, and `source`.++## Reader tests++```bash+uv pip install --python /tmp/leann-parallel/bin/python pytest+/tmp/leann-parallel/bin/python -m pytest -o addopts= tests/test_parallel_mcp_reader.py+```diff --git a/tests/test_parallel_mcp_reader.py b/tests/test_parallel_mcp_reader.pynew file mode 100644index 0000000..89b35ac--- /dev/null+++ b/tests/test_parallel_mcp_reader.py@@ -0,0 +1,222 @@+"""Exercise the real MCP reader/transport against an HTTP protocol fixture."""++import asyncio+import importlib+import json+import sys+from types import SimpleNamespace++import pytest++# The opt-in reader has its own installation; do not add SDK requirements to+# collection of LEANN's ordinary test suite.+httpx = pytest.importorskip("httpx")+pytest.importorskip("mcp")+reader = importlib.import_module("apps.parallel_data.parallel_mcp_reader")+export = importlib.import_module("apps.parallel_data.__main__")+++def install_server(monkeypatch, search, extraction=None, tool_error=False, tools=None):+ """Install an HTTP server fixture that observes every actual SDK request."""+ requests = []+ calls = []++ def handle(request):+ requests.append(request)+ assert str(request.url) == reader.MCP_URL+ assert request.headers["User-Agent"] == reader.USER_AGENT+ assert "authorization" not in request.headers+ assert "cookie" not in request.headers+ payload = json.loads(request.content)+ if "id" not in payload:+ return httpx.Response(202)+ method = payload["method"]+ if method == "initialize":+ result = {+ "protocolVersion": payload["params"]["protocolVersion"],+ "capabilities": {"tools": {}},+ "serverInfo": {"name": "fixture", "version": "1"},+ }+ elif method == "tools/list":+ result = {+ "tools": [+ {"name": name, "inputSchema": {"type": "object"}}+ for name in (tools if tools is not None else ["web_search", "web_fetch"])+ ]+ }+ else:+ assert method == "tools/call"+ calls.append(payload["params"])+ data = search if payload["params"]["name"] == "web_search" else extraction+ result = {+ "content": [{"type": "text", "text": json.dumps(data)}],+ "isError": tool_error,+ }+ return httpx.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": result})++ client_class = httpx.AsyncClient+ monkeypatch.setattr(+ reader.httpx,+ "AsyncClient",+ lambda **kwargs: client_class(transport=httpx.MockTransport(handle), **kwargs),+ )+ return requests, calls+++def item(url, text="Search excerpt", title="A source"):+ return {"url": url, "title": title, "excerpts": [text], "publish_date": "2026-10-01"}+++def test_search_fetch_real_transport(monkeypatch, caplog):+ first = item("https://example.com/one")+ second = item("https://example.com/two", "Fallback excerpt")+ requests, calls = install_server(+ monkeypatch,+ {"results": [first, first, second, item("https://example.com/three")]},+ {+ "results": [item(first["url"], "Fetched excerpt", "Fetched title")],+ "errors": [{"url": second["url"], "error_type": "HTTP_ERROR"}],+ },+ )+ monkeypatch.setenv("PARALLEL_API_KEY", "must-not-be-sent")+ instance = reader.ParallelMCPReader()+ documents = asyncio.run(instance.read_web_data("Research", ["research web sources"], 2, True))+ assert len(requests) >= 5 # initialization, discovery, search, and fetch+ assert [call["name"] for call in calls] == ["web_search", "web_fetch"]+ assert calls[1]["arguments"]["urls"] == [first["url"], second["url"]]+ assert calls[0]["arguments"]["session_id"] == calls[1]["arguments"]["session_id"]+ assert calls[0]["arguments"]["session_id"] == instance.session_id+ assert calls[1]["arguments"]["search_queries"] == ["research web sources"]+ assert documents[0]["text"] == "Fetched excerpt"+ assert documents[0]["metadata"]["title"] == "Fetched title"+ assert documents[1]["text"] == "Fallback excerpt"+ assert "retaining search excerpts" in caplog.text+++def test_search_only_and_blank_excerpts(monkeypatch):+ _, calls = install_server(+ monkeypatch,+ {"results": [item("https://example.com/empty", " "), item("https://example.com")]},+ )+ documents = asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["web sources"]))+ assert len(documents) == 1+ assert documents[0]["metadata"]["url"] == "https://example.com"+ assert [call["name"] for call in calls] == ["web_search"]+++def test_empty_search_does_not_fetch(monkeypatch):+ _, calls = install_server(monkeypatch, {"results": []})+ assert (+ asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["sources"], fetch=True))+ == []+ )+ assert len(calls) == 1+++def test_tool_error_propagates(monkeypatch):+ install_server(monkeypatch, {"error": "rate limited"}, tool_error=True)+ with pytest.raises(Exception) as error:+ asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["sources"]))+ assert "Parallel MCP tool failed" in repr(error.value)+++def test_missing_tool_propagates(monkeypatch):+ install_server(monkeypatch, {"results": []}, tools=["web_search"])+ with pytest.raises(Exception) as error:+ asyncio.run(reader.ParallelMCPReader().read_web_data("Research", ["sources"], fetch=True))+ assert "missing tools" in repr(error.value)+++def test_structured_output_and_invalid_json():+ result = SimpleNamespace(isError=False, structuredContent={"results": []}, content=[])+ assert reader._tool_payload(result) == {"results": []}+ result.structuredContent = None+ result.content = [SimpleNamespace(text="invalid")]+ with pytest.raises(RuntimeError, match="invalid JSON"):+ reader._tool_payload(result)+++ "objective,queries,limit", [("", ["x"], 1), ("x", [], 1), ("x", [" "], 1), ("x", ["y"], 21)]+)+def test_invalid_arguments(objective, queries, limit):+ with pytest.raises(ValueError):+ asyncio.run(reader.ParallelMCPReader().read_web_data(objective, queries, limit))+++def test_overall_timeout(monkeypatch):+ async def stalled(*args):+ await asyncio.Event().wait()++ instance = reader.ParallelMCPReader(timeout=0.01)+ monkeypatch.setattr(instance, "_read", stalled)+ with pytest.raises(asyncio.TimeoutError):+ asyncio.run(instance.read_web_data("Research", ["sources"]))+++def test_export_command_preserves_sources(monkeypatch, tmp_path, capsys):+ async def read_data(self, objective, queries, limit, fetch):+ assert (objective, queries, limit, fetch) == ("Research", ["web sources"], 5, False)+ return [+ {"text": "Useful café excerpt", "metadata": item("https://example.com/../../source")}+ ]++ monkeypatch.setattr(reader.ParallelMCPReader, "read_web_data", read_data)+ monkeypatch.setattr(+ sys,+ "argv",+ [+ "parallel_data",+ "--objective",+ "Research",+ "--search-query",+ "web sources",+ "--output-dir",+ str(tmp_path),+ ],+ )+ export.main()+ paths = list(tmp_path.glob("*.md"))+ assert len(paths) == 1+ assert paths[0].parent == tmp_path+ assert paths[0].name.startswith("example-com-source-")+ text = paths[0].read_text()+ assert "Source: https://example.com/../../source" in text+ assert "Published: 2026-10-01" in text+ assert "Useful café excerpt" in text+ assert "Exported 1" in capsys.readouterr().out+++ "url,prefix",+ [+ ("https://arxiv.org/abs/2506.08276", "arxiv-org-abs-2506-08276-"),+ ("https://yichuan-w.github.io/", "yichuan-w-github-io-"),+ ("https://example.com/" + "a" * 200, "example-com-" + "a" * 68 + "-"),+ ],+)+def test_document_filename_is_readable_and_unique(url, prefix):+ name = export.document_filename(url)+ assert name.startswith(prefix) and name.endswith(".md")+ assert export.document_filename(url) == name+ assert export.document_filename(url + "?v=2") != name+++def test_rag_app_chunks_web_documents_with_sources(monkeypatch):+ pytest.importorskip("leann")+ pytest.importorskip("llama_index.core")+ rag = importlib.import_module("apps.parallel_rag")++ async def read_data(self, objective, queries, limit, fetch):+ assert (objective, queries, limit, fetch) == ("Research", ["web sources"], 5, True)+ return [{"text": "LEANN recomputes embeddings. " * 40, "metadata": item("https://a.io/x")}]++ monkeypatch.setattr(reader.ParallelMCPReader, "read_web_data", read_data)+ app = rag.ParallelWebRAG()+ args = app.parser.parse_args(+ ["--objective", "Research", "--search-query", "web sources", "--fetch"]+ )+ texts = asyncio.run(app.load_data(args))+ assert len(texts) > 1+ assert texts[0]["text"].startswith("A source\nSource: https://a.io/x")+ assert all(text["metadata"]["url"] == "https://a.io/x" for text in texts)