An open source, non-profit web search engine
Super Search: declarative YAML site recipes (PoC)
Daoud Clarkebe9c0e873881
diff --git a/mwmbl/tinysearchengine/super_search_sources/__init__.py b/mwmbl/tinysearchengine/super_search_sources/__init__.pyindex e0e28df..763019d 100644--- a/mwmbl/tinysearchengine/super_search_sources/__init__.py+++ b/mwmbl/tinysearchengine/super_search_sources/__init__.py@@ -10,6 +10,7 @@ from mwmbl.tinysearchengine.super_search_sources.github import search as search_from mwmbl.tinysearchengine.super_search_sources.hn import search as search_hnfrom mwmbl.tinysearchengine.super_search_sources.mwmbl_index import search as search_mwmblfrom mwmbl.tinysearchengine.super_search_sources.pypi import search as search_pypi+from mwmbl.tinysearchengine.super_search_sources.recipe import load_recipes, make_recipe_sourcefrom mwmbl.tinysearchengine.super_search_sources.stackexchange import search as search_stackexchangeSOURCES = {@@ -20,3 +21,8 @@ SOURCES = {"arxiv": search_arxiv,"pypi": search_pypi,}++# Declarative YAML recipes (recipes/*.yaml) are loaded at import time and added+# as ordinary sources. A hand-written adapter wins if it shares a recipe's name.+for _name, _recipe in load_recipes().items():+ SOURCES.setdefault(_name, make_recipe_source(_recipe))diff --git a/mwmbl/tinysearchengine/super_search_sources/recipe.py b/mwmbl/tinysearchengine/super_search_sources/recipe.pynew file mode 100644index 0000000..5767284--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipe.py@@ -0,0 +1,292 @@+"""Declarative, YAML-driven search adapters for Super Search.++A *recipe* is a YAML file describing how to search one external site: the query+URL and parameters, the response format, and where to find the URL / title /+extract in the response. A single generic engine (:func:`search_with_recipe`)+executes any recipe and returns ``list[Document]`` with the same signature as+the hand-written adapters in this package, so onboarding a new site becomes+"drop in a YAML file" instead of "write a Python adapter".++Like the hand-written adapters, the engine never raises on HTTP/parse errors —+it logs and returns ``[]`` so one slow or broken source can't sink the+orchestrator.++See ``recipes/*.yaml`` for examples covering JSON APIs (Wiktionary,+archive.org) and HTML scraping (Project Gutenberg).+"""+from __future__ import annotations++import logging+from dataclasses import dataclass+from pathlib import Path+from typing import Any, Callable, Coroutine+from urllib.parse import quote, urljoin+from xml.etree.ElementTree import Element, ParseError++import httpx+import yaml+from bs4 import BeautifulSoup+from defusedxml.common import DefusedXmlException+from defusedxml.ElementTree import fromstring++from mwmbl.tinysearchengine.indexer import Document++logger = logging.getLogger(__name__)++RECIPES_DIR = Path(__file__).parent / "recipes"+++@dataclass+class Recipe:+ """A parsed search recipe. ``request`` and ``response`` are the raw YAML maps."""+ name: str+ request: dict+ response: dict+ domain: str = ""+ field: str = ""++ @property+ def response_format(self) -> str:+ return self.response.get("format", "json")+++def load_recipe(path: Path | str) -> Recipe:+ data = yaml.safe_load(Path(path).read_text())+ return Recipe(+ name=data["name"],+ domain=data.get("domain", ""),+ field=data.get("field", ""),+ request=data["request"],+ response=data["response"],+ )+++def load_recipes(directory: Path | str = RECIPES_DIR) -> dict[str, Recipe]:+ """Load every ``*.yaml`` recipe in ``directory``, keyed by recipe name.++ Invalid recipes are logged and skipped rather than failing the import.+ """+ recipes: dict[str, Recipe] = {}+ directory = Path(directory)+ if not directory.is_dir():+ return recipes+ for path in sorted(directory.glob("*.yaml")):+ try:+ recipe = load_recipe(path)+ except (yaml.YAMLError, KeyError, OSError) as e:+ logger.warning("Skipping invalid recipe %s: %s", path, e)+ continue+ recipes[recipe.name] = recipe+ return recipes+++def make_recipe_source(recipe: Recipe) -> Callable[..., Coroutine[Any, Any, list[Document]]]:+ """Wrap a recipe as a ``search(client, query, limit)`` adapter for ``SOURCES``."""+ async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Document]:+ return await search_with_recipe(client, recipe, query, limit)+ return search+++async def search_with_recipe(+ client: httpx.AsyncClient, recipe: Recipe, query: str, limit: int+) -> list[Document]:+ req = recipe.request+ method = req.get("method", "GET").upper()+ params = _build_params(req.get("params"), query, limit)+ try:+ response = await client.request(method, req["url"], params=params)+ response.raise_for_status()+ fmt = recipe.response_format+ if fmt == "json":+ return _parse_json(response.json(), recipe.response)+ if fmt == "html":+ return _parse_html(response.text, recipe.response)+ if fmt == "xml":+ return _parse_xml(response.text, recipe.response)+ logger.warning("Recipe %s: unknown response format %r", recipe.name, fmt)+ return []+ except (httpx.HTTPError, ValueError, ParseError, DefusedXmlException) as e:+ logger.info("Recipe source %s failed: %s", recipe.name, e)+ return []+++# ---------------------------------------------------------------------------+# Request building+# ---------------------------------------------------------------------------++def _substitute(value, query: str, limit: int):+ if isinstance(value, str):+ return value.format(query=query, limit=limit)+ if isinstance(value, list):+ return [_substitute(v, query, limit) for v in value]+ return value+++def _build_params(params: dict | None, query: str, limit: int) -> dict:+ return {k: _substitute(v, query, limit) for k, v in (params or {}).items()}+++# ---------------------------------------------------------------------------+# Shared helpers+# ---------------------------------------------------------------------------++def _coerce_str(value, strip_html: bool = False) -> str:+ if value is None:+ return ""+ text = str(value)+ if strip_html:+ text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)+ return text.strip()+++def _fill_template(template: str, context: dict) -> str:+ """Fill ``template`` from ``context``, URL-quoting each substituted value."""+ safe = {k: quote(str(v), safe="") for k, v in context.items() if v is not None}+ try:+ return template.format(**safe)+ except (KeyError, IndexError):+ return ""+++def _make_doc(values: dict, url: str) -> Document | None:+ if not url:+ return None+ return Document(title=values.get("title", ""), url=url, extract=values.get("extract", ""))+++# ---------------------------------------------------------------------------+# JSON+# ---------------------------------------------------------------------------++def _walk(data, path: str):+ """Walk a dotted path (``a.b.c``) through nested dicts; None if absent."""+ if not path:+ return data+ current = data+ for part in path.split("."):+ if isinstance(current, dict) and part in current:+ current = current[part]+ else:+ return None+ return current+++def _parse_json(payload, spec: dict) -> list[Document]:+ items = _walk(payload, spec.get("results", "")) or []+ fields = spec["fields"]+ strip = set(spec.get("strip_html", []))+ docs: list[Document] = []+ for item in items:+ if not isinstance(item, dict):+ continue+ values = {+ name: _coerce_str(_walk(item, field_spec), name in strip)+ for name, field_spec in fields.items()+ if name != "url" and isinstance(field_spec, str)+ }+ doc = _make_doc(values, _resolve_url_json(fields.get("url"), item, values))+ if doc is not None:+ docs.append(doc)+ return docs+++def _resolve_url_json(url_spec, item: dict, values: dict) -> str:+ if url_spec is None:+ return ""+ if isinstance(url_spec, str):+ return _coerce_str(_walk(item, url_spec))+ if isinstance(url_spec, dict) and "template" in url_spec:+ return _fill_template(url_spec["template"], {**item, **values})+ return ""+++# ---------------------------------------------------------------------------+# HTML+# ---------------------------------------------------------------------------++def _parse_html(html_text: str, spec: dict) -> list[Document]:+ soup = BeautifulSoup(html_text, "html.parser")+ fields = spec["fields"]+ strip = set(spec.get("strip_html", []))+ base_url = spec.get("base_url", "")+ docs: list[Document] = []+ for el in soup.select(spec["results"]):+ values = {+ name: _select_html(el, field_spec, name in strip)+ for name, field_spec in fields.items()+ if name != "url"+ }+ doc = _make_doc(values, _resolve_url_html(fields.get("url"), el, base_url))+ if doc is not None:+ docs.append(doc)+ return docs+++def _select_html(el, spec, strip_html: bool = False) -> str:+ if spec is None:+ return ""+ if isinstance(spec, str):+ spec = {"selector": spec}+ target = el.select_one(spec["selector"]) if spec.get("selector") else el+ if target is None:+ return ""+ attr = spec.get("attr")+ value = target.get(attr, "") if attr else target.get_text(" ", strip=True)+ if not isinstance(value, str):+ return ""+ if strip_html and value:+ value = BeautifulSoup(value, "html.parser").get_text(" ", strip=True)+ return value.strip()+++def _resolve_url_html(url_spec, el, base_url: str) -> str:+ if url_spec is None:+ return ""+ href = _select_html(el, url_spec)+ return urljoin(base_url, href) if (href and base_url) else href+++# ---------------------------------------------------------------------------+# XML+# ---------------------------------------------------------------------------++def _parse_xml(xml_text: str, spec: dict) -> list[Document]:+ root = fromstring(xml_text)+ results_path = spec.get("results", "")+ items = root.findall(results_path) if results_path else [root]+ fields = spec["fields"]+ strip = set(spec.get("strip_html", []))+ docs: list[Document] = []+ for item in items:+ values = {+ name: _select_xml(item, field_spec, name in strip)+ for name, field_spec in fields.items()+ if name != "url"+ }+ doc = _make_doc(values, _resolve_url_xml(fields.get("url"), item, values))+ if doc is not None:+ docs.append(doc)+ return docs+++def _select_xml(item: Element, spec, strip_html: bool = False) -> str:+ value = ""+ if isinstance(spec, str):+ child = item.find(spec)+ value = (child.text or "") if child is not None else ""+ elif isinstance(spec, dict):+ attr, selector = spec.get("attr"), spec.get("selector")+ target = item.find(selector) if selector else item+ if target is not None:+ value = target.get(attr, "") if attr else (target.text or "")+ if strip_html and value:+ value = BeautifulSoup(value, "html.parser").get_text(" ", strip=True)+ return value.strip()+++def _resolve_url_xml(url_spec, item: Element, values: dict) -> str:+ if url_spec is None:+ return ""+ if isinstance(url_spec, dict) and "template" in url_spec:+ return _fill_template(url_spec["template"], {**item.attrib, **values})+ return _select_xml(item, url_spec)diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/archive_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/archive_org.yamlnew file mode 100644index 0000000..22480b1--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/archive_org.yaml@@ -0,0 +1,23 @@+# Internet Archive via the advancedsearch JSON API. The result list is nested+# under response.docs; the item URL is templated from the identifier.+name: archive_org+domain: archive.org+field: tech+request:+ url: https://archive.org/advancedsearch.php+ params:+ q: "{query}"+ "fl[]":+ - identifier+ - title+ - description+ rows: "{limit}"+ output: json+response:+ format: json+ results: response.docs+ fields:+ title: title+ extract: description+ url:+ template: "https://archive.org/details/{identifier}"diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/gutenberg.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/gutenberg.yamlnew file mode 100644index 0000000..f80139c--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/gutenberg.yaml@@ -0,0 +1,22 @@+# Project Gutenberg has no JSON search API, so we scrape the HTML results page.+# Each result is an <li class="booklink"> with a relative href that base_url+# resolves to an absolute URL.+name: gutenberg+domain: www.gutenberg.org+field: books-literature+request:+ url: https://www.gutenberg.org/ebooks/search/+ params:+ query: "{query}"+response:+ format: html+ results: "li.booklink"+ base_url: https://www.gutenberg.org+ fields:+ title:+ selector: "span.title"+ extract:+ selector: "span.subtitle"+ url:+ selector: "a.link"+ attr: hrefdiff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/wiktionary.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/wiktionary.yamlnew file mode 100644index 0000000..9b3ba0c--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/wiktionary.yaml@@ -0,0 +1,25 @@+# English Wiktionary via the MediaWiki search API.+# Representative of the whole MediaWiki class (Wikipedia, Wikisource,+# minecraft.wiki, ...): same API shape, just a different host. The result has a+# page title and an HTML snippet; the page URL is templated from the title.+name: wiktionary+domain: en.wiktionary.org+field: other+request:+ url: https://en.wiktionary.org/w/api.php+ params:+ action: query+ list: search+ format: json+ srsearch: "{query}"+ srlimit: "{limit}"+response:+ format: json+ results: query.search+ fields:+ title: title+ extract: snippet+ url:+ template: "https://en.wiktionary.org/wiki/{title}"+ strip_html:+ - extractdiff --git a/test/test_super_search_recipes.py b/test/test_super_search_recipes.pynew file mode 100644index 0000000..290a590--- /dev/null+++ b/test/test_super_search_recipes.py@@ -0,0 +1,171 @@+"""Tests for the declarative YAML recipe engine (recipe.py).++Each recipe is exercised against a mocked httpx transport, asserting the+url/title/extract coercion (templated URLs, relative-href resolution, HTML+stripping) and that an HTTP error returns [] rather than propagating. We also+verify the shipped recipe files load and register into SOURCES.+"""+import re+from pathlib import Path++import httpx++from mwmbl.tinysearchengine.super_search_sources.recipe import (+ RECIPES_DIR,+ Recipe,+ load_recipes,+ search_with_recipe,+)++RECIPES = load_recipes()+++def _recipe(name: str) -> Recipe:+ return RECIPES[name]+++# ---------------------------------------------------------------------------+# Recipe loading / registration+# ---------------------------------------------------------------------------++def test_shipped_recipes_load():+ assert {"wiktionary", "archive_org", "gutenberg"} <= set(RECIPES)+ assert _recipe("gutenberg").response_format == "html"+++def test_recipes_registered_in_sources():+ from mwmbl.tinysearchengine.super_search_sources import SOURCES+ assert "wiktionary" in SOURCES and "gutenberg" in SOURCES+++def test_load_recipes_skips_invalid(tmp_path):+ (tmp_path / "broken.yaml").write_text("name: x\nrequest: {}\n") # no response+ assert load_recipes(tmp_path) == {}+++# ---------------------------------------------------------------------------+# JSON: Wiktionary (MediaWiki) — templated URL + HTML-stripped snippet+# ---------------------------------------------------------------------------++async def test_wiktionary_templates_url_and_strips_html(httpx_mock):+ httpx_mock.add_response(+ url=re.compile(r"https://en\.wiktionary\.org/w/api\.php.*"),+ json={"query": {"search": [+ {"title": "serendipity",+ "snippet": 'fortunate <span class="searchmatch">serendipity</span>'},+ ]}},+ )+ async with httpx.AsyncClient() as client:+ docs = await search_with_recipe(client, _recipe("wiktionary"), "serendipity", 5)+ assert len(docs) == 1+ assert docs[0].url == "https://en.wiktionary.org/wiki/serendipity"+ assert docs[0].title == "serendipity"+ assert "<span" not in docs[0].extract and "serendipity" in docs[0].extract+++async def test_wiktionary_quotes_titles_with_spaces(httpx_mock):+ httpx_mock.add_response(+ url=re.compile(r"https://en\.wiktionary\.org/.*"),+ json={"query": {"search": [{"title": "United States", "snippet": "x"}]}},+ )+ async with httpx.AsyncClient() as client:+ docs = await search_with_recipe(client, _recipe("wiktionary"), "united", 5)+ assert docs[0].url == "https://en.wiktionary.org/wiki/United%20States"+++# ---------------------------------------------------------------------------+# JSON: archive.org — nested results path + templated URL from identifier+# ---------------------------------------------------------------------------++async def test_archive_org_nested_results_and_template(httpx_mock):+ httpx_mock.add_response(+ url=re.compile(r"https://archive\.org/advancedsearch\.php.*"),+ json={"response": {"docs": [+ {"identifier": "apollo11", "title": "Apollo 11", "description": "Moon landing"},+ {"title": "no id"}, # no identifier -> no URL -> skipped+ ]}},+ )+ async with httpx.AsyncClient() as client:+ docs = await search_with_recipe(client, _recipe("archive_org"), "apollo 11", 5)+ assert len(docs) == 1+ assert docs[0].url == "https://archive.org/details/apollo11"+ assert docs[0].extract == "Moon landing"+++# ---------------------------------------------------------------------------+# HTML: Project Gutenberg — CSS selectors + relative href resolution+# ---------------------------------------------------------------------------++GUTENBERG_HTML = """+<html><body><ul>+ <li class="booklink">+ <a class="link" href="/ebooks/84">+ <span class="title">Frankenstein</span>+ <span class="subtitle">Mary Wollstonecraft Shelley</span>+ </a>+ </li>+ <li class="other"><a class="link" href="/ignored">nope</a></li>+</ul></body></html>+"""+++async def test_gutenberg_scrapes_and_resolves_relative_url(httpx_mock):+ httpx_mock.add_response(+ url=re.compile(r"https://www\.gutenberg\.org/ebooks/search/.*"),+ text=GUTENBERG_HTML,+ )+ async with httpx.AsyncClient() as client:+ docs = await search_with_recipe(client, _recipe("gutenberg"), "frankenstein", 5)+ assert len(docs) == 1+ assert docs[0].url == "https://www.gutenberg.org/ebooks/84"+ assert docs[0].title == "Frankenstein"+ assert "Shelley" in docs[0].extract+++# ---------------------------------------------------------------------------+# XML: schema completeness (no XML recipe ships, but the engine supports it)+# ---------------------------------------------------------------------------++BGG_XML = """<?xml version="1.0" encoding="utf-8"?>+<items>+ <item type="boardgame" id="13">+ <name type="primary" value="Catan"/>+ </item>+</items>+"""+++async def test_xml_attributes_and_template(httpx_mock):+ recipe = Recipe(+ name="bgg",+ request={"url": "https://boardgamegeek.com/xmlapi2/search"},+ response={+ "format": "xml",+ "results": ".//item",+ "fields": {+ "title": {"selector": "name", "attr": "value"},+ "url": {"template": "https://boardgamegeek.com/boardgame/{id}"},+ },+ },+ )+ httpx_mock.add_response(url=re.compile(r"https://boardgamegeek\.com/.*"), text=BGG_XML)+ async with httpx.AsyncClient() as client:+ docs = await search_with_recipe(client, recipe, "catan", 5)+ assert len(docs) == 1+ assert docs[0].title == "Catan"+ assert docs[0].url == "https://boardgamegeek.com/boardgame/13"+++# ---------------------------------------------------------------------------+# Robustness: errors swallowed+# ---------------------------------------------------------------------------++async def test_http_error_returns_empty(httpx_mock):+ httpx_mock.add_response(status_code=500)+ async with httpx.AsyncClient() as client:+ docs = await search_with_recipe(client, _recipe("wiktionary"), "x", 5)+ assert docs == []+++def test_recipes_dir_exists():+ assert Path(RECIPES_DIR).is_dir()