Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search: declarative YAML site recipes (PoC)

Daoud Clarkebe9c0e873881
diff --git a/mwmbl/tinysearchengine/super_search_sources/__init__.py b/mwmbl/tinysearchengine/super_search_sources/__init__.py
index e0e28df..763019d 100644
--- a/mwmbl/tinysearchengine/super_search_sources/__init__.py
+++ b/mwmbl/tinysearchengine/super_search_sources/__init__.py
@@ -10,6 +10,7 @@ from mwmbl.tinysearchengine.super_search_sources.github import search as search_
from mwmbl.tinysearchengine.super_search_sources.hn import search as search_hn
from mwmbl.tinysearchengine.super_search_sources.mwmbl_index import search as search_mwmbl
from mwmbl.tinysearchengine.super_search_sources.pypi import search as search_pypi
+from mwmbl.tinysearchengine.super_search_sources.recipe import load_recipes, make_recipe_source
from mwmbl.tinysearchengine.super_search_sources.stackexchange import search as search_stackexchange
SOURCES = {
@@ -20,3 +21,8 @@ SOURCES = {
"arxiv": search_arxiv,
"pypi": search_pypi,
}
+
+# Declarative YAML recipes (recipes/*.yaml) are loaded at import time and added
+# as ordinary sources. A hand-written adapter wins if it shares a recipe's name.
+for _name, _recipe in load_recipes().items():
+ SOURCES.setdefault(_name, make_recipe_source(_recipe))
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipe.py b/mwmbl/tinysearchengine/super_search_sources/recipe.py
new file mode 100644
index 0000000..5767284
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipe.py
@@ -0,0 +1,292 @@
+"""Declarative, YAML-driven search adapters for Super Search.
+
+A *recipe* is a YAML file describing how to search one external site: the query
+URL and parameters, the response format, and where to find the URL / title /
+extract in the response. A single generic engine (:func:`search_with_recipe`)
+executes any recipe and returns ``list[Document]`` with the same signature as
+the hand-written adapters in this package, so onboarding a new site becomes
+"drop in a YAML file" instead of "write a Python adapter".
+
+Like the hand-written adapters, the engine never raises on HTTP/parse errors —
+it logs and returns ``[]`` so one slow or broken source can't sink the
+orchestrator.
+
+See ``recipes/*.yaml`` for examples covering JSON APIs (Wiktionary,
+archive.org) and HTML scraping (Project Gutenberg).
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Coroutine
+from urllib.parse import quote, urljoin
+from xml.etree.ElementTree import Element, ParseError
+
+import httpx
+import yaml
+from bs4 import BeautifulSoup
+from defusedxml.common import DefusedXmlException
+from defusedxml.ElementTree import fromstring
+
+from mwmbl.tinysearchengine.indexer import Document
+
+logger = logging.getLogger(__name__)
+
+RECIPES_DIR = Path(__file__).parent / "recipes"
+
+
+@dataclass
+class Recipe:
+ """A parsed search recipe. ``request`` and ``response`` are the raw YAML maps."""
+ name: str
+ request: dict
+ response: dict
+ domain: str = ""
+ field: str = ""
+
+ @property
+ def response_format(self) -> str:
+ return self.response.get("format", "json")
+
+
+def load_recipe(path: Path | str) -> Recipe:
+ data = yaml.safe_load(Path(path).read_text())
+ return Recipe(
+ name=data["name"],
+ domain=data.get("domain", ""),
+ field=data.get("field", ""),
+ request=data["request"],
+ response=data["response"],
+ )
+
+
+def load_recipes(directory: Path | str = RECIPES_DIR) -> dict[str, Recipe]:
+ """Load every ``*.yaml`` recipe in ``directory``, keyed by recipe name.
+
+ Invalid recipes are logged and skipped rather than failing the import.
+ """
+ recipes: dict[str, Recipe] = {}
+ directory = Path(directory)
+ if not directory.is_dir():
+ return recipes
+ for path in sorted(directory.glob("*.yaml")):
+ try:
+ recipe = load_recipe(path)
+ except (yaml.YAMLError, KeyError, OSError) as e:
+ logger.warning("Skipping invalid recipe %s: %s", path, e)
+ continue
+ recipes[recipe.name] = recipe
+ return recipes
+
+
+def make_recipe_source(recipe: Recipe) -> Callable[..., Coroutine[Any, Any, list[Document]]]:
+ """Wrap a recipe as a ``search(client, query, limit)`` adapter for ``SOURCES``."""
+ async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Document]:
+ return await search_with_recipe(client, recipe, query, limit)
+ return search
+
+
+async def search_with_recipe(
+ client: httpx.AsyncClient, recipe: Recipe, query: str, limit: int
+) -> list[Document]:
+ req = recipe.request
+ method = req.get("method", "GET").upper()
+ params = _build_params(req.get("params"), query, limit)
+ try:
+ response = await client.request(method, req["url"], params=params)
+ response.raise_for_status()
+ fmt = recipe.response_format
+ if fmt == "json":
+ return _parse_json(response.json(), recipe.response)
+ if fmt == "html":
+ return _parse_html(response.text, recipe.response)
+ if fmt == "xml":
+ return _parse_xml(response.text, recipe.response)
+ logger.warning("Recipe %s: unknown response format %r", recipe.name, fmt)
+ return []
+ except (httpx.HTTPError, ValueError, ParseError, DefusedXmlException) as e:
+ logger.info("Recipe source %s failed: %s", recipe.name, e)
+ return []
+
+
+# ---------------------------------------------------------------------------
+# Request building
+# ---------------------------------------------------------------------------
+
+def _substitute(value, query: str, limit: int):
+ if isinstance(value, str):
+ return value.format(query=query, limit=limit)
+ if isinstance(value, list):
+ return [_substitute(v, query, limit) for v in value]
+ return value
+
+
+def _build_params(params: dict | None, query: str, limit: int) -> dict:
+ return {k: _substitute(v, query, limit) for k, v in (params or {}).items()}
+
+
+# ---------------------------------------------------------------------------
+# Shared helpers
+# ---------------------------------------------------------------------------
+
+def _coerce_str(value, strip_html: bool = False) -> str:
+ if value is None:
+ return ""
+ text = str(value)
+ if strip_html:
+ text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
+ return text.strip()
+
+
+def _fill_template(template: str, context: dict) -> str:
+ """Fill ``template`` from ``context``, URL-quoting each substituted value."""
+ safe = {k: quote(str(v), safe="") for k, v in context.items() if v is not None}
+ try:
+ return template.format(**safe)
+ except (KeyError, IndexError):
+ return ""
+
+
+def _make_doc(values: dict, url: str) -> Document | None:
+ if not url:
+ return None
+ return Document(title=values.get("title", ""), url=url, extract=values.get("extract", ""))
+
+
+# ---------------------------------------------------------------------------
+# JSON
+# ---------------------------------------------------------------------------
+
+def _walk(data, path: str):
+ """Walk a dotted path (``a.b.c``) through nested dicts; None if absent."""
+ if not path:
+ return data
+ current = data
+ for part in path.split("."):
+ if isinstance(current, dict) and part in current:
+ current = current[part]
+ else:
+ return None
+ return current
+
+
+def _parse_json(payload, spec: dict) -> list[Document]:
+ items = _walk(payload, spec.get("results", "")) or []
+ fields = spec["fields"]
+ strip = set(spec.get("strip_html", []))
+ docs: list[Document] = []
+ for item in items:
+ if not isinstance(item, dict):
+ continue
+ values = {
+ name: _coerce_str(_walk(item, field_spec), name in strip)
+ for name, field_spec in fields.items()
+ if name != "url" and isinstance(field_spec, str)
+ }
+ doc = _make_doc(values, _resolve_url_json(fields.get("url"), item, values))
+ if doc is not None:
+ docs.append(doc)
+ return docs
+
+
+def _resolve_url_json(url_spec, item: dict, values: dict) -> str:
+ if url_spec is None:
+ return ""
+ if isinstance(url_spec, str):
+ return _coerce_str(_walk(item, url_spec))
+ if isinstance(url_spec, dict) and "template" in url_spec:
+ return _fill_template(url_spec["template"], {**item, **values})
+ return ""
+
+
+# ---------------------------------------------------------------------------
+# HTML
+# ---------------------------------------------------------------------------
+
+def _parse_html(html_text: str, spec: dict) -> list[Document]:
+ soup = BeautifulSoup(html_text, "html.parser")
+ fields = spec["fields"]
+ strip = set(spec.get("strip_html", []))
+ base_url = spec.get("base_url", "")
+ docs: list[Document] = []
+ for el in soup.select(spec["results"]):
+ values = {
+ name: _select_html(el, field_spec, name in strip)
+ for name, field_spec in fields.items()
+ if name != "url"
+ }
+ doc = _make_doc(values, _resolve_url_html(fields.get("url"), el, base_url))
+ if doc is not None:
+ docs.append(doc)
+ return docs
+
+
+def _select_html(el, spec, strip_html: bool = False) -> str:
+ if spec is None:
+ return ""
+ if isinstance(spec, str):
+ spec = {"selector": spec}
+ target = el.select_one(spec["selector"]) if spec.get("selector") else el
+ if target is None:
+ return ""
+ attr = spec.get("attr")
+ value = target.get(attr, "") if attr else target.get_text(" ", strip=True)
+ if not isinstance(value, str):
+ return ""
+ if strip_html and value:
+ value = BeautifulSoup(value, "html.parser").get_text(" ", strip=True)
+ return value.strip()
+
+
+def _resolve_url_html(url_spec, el, base_url: str) -> str:
+ if url_spec is None:
+ return ""
+ href = _select_html(el, url_spec)
+ return urljoin(base_url, href) if (href and base_url) else href
+
+
+# ---------------------------------------------------------------------------
+# XML
+# ---------------------------------------------------------------------------
+
+def _parse_xml(xml_text: str, spec: dict) -> list[Document]:
+ root = fromstring(xml_text)
+ results_path = spec.get("results", "")
+ items = root.findall(results_path) if results_path else [root]
+ fields = spec["fields"]
+ strip = set(spec.get("strip_html", []))
+ docs: list[Document] = []
+ for item in items:
+ values = {
+ name: _select_xml(item, field_spec, name in strip)
+ for name, field_spec in fields.items()
+ if name != "url"
+ }
+ doc = _make_doc(values, _resolve_url_xml(fields.get("url"), item, values))
+ if doc is not None:
+ docs.append(doc)
+ return docs
+
+
+def _select_xml(item: Element, spec, strip_html: bool = False) -> str:
+ value = ""
+ if isinstance(spec, str):
+ child = item.find(spec)
+ value = (child.text or "") if child is not None else ""
+ elif isinstance(spec, dict):
+ attr, selector = spec.get("attr"), spec.get("selector")
+ target = item.find(selector) if selector else item
+ if target is not None:
+ value = target.get(attr, "") if attr else (target.text or "")
+ if strip_html and value:
+ value = BeautifulSoup(value, "html.parser").get_text(" ", strip=True)
+ return value.strip()
+
+
+def _resolve_url_xml(url_spec, item: Element, values: dict) -> str:
+ if url_spec is None:
+ return ""
+ if isinstance(url_spec, dict) and "template" in url_spec:
+ return _fill_template(url_spec["template"], {**item.attrib, **values})
+ return _select_xml(item, url_spec)
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/archive_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/archive_org.yaml
new file mode 100644
index 0000000..22480b1
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/archive_org.yaml
@@ -0,0 +1,23 @@
+# Internet Archive via the advancedsearch JSON API. The result list is nested
+# under response.docs; the item URL is templated from the identifier.
+name: archive_org
+domain: archive.org
+field: tech
+request:
+ url: https://archive.org/advancedsearch.php
+ params:
+ q: "{query}"
+ "fl[]":
+ - identifier
+ - title
+ - description
+ rows: "{limit}"
+ output: json
+response:
+ format: json
+ results: response.docs
+ fields:
+ title: title
+ extract: description
+ url:
+ template: "https://archive.org/details/{identifier}"
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/gutenberg.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/gutenberg.yaml
new file mode 100644
index 0000000..f80139c
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/gutenberg.yaml
@@ -0,0 +1,22 @@
+# Project Gutenberg has no JSON search API, so we scrape the HTML results page.
+# Each result is an <li class="booklink"> with a relative href that base_url
+# resolves to an absolute URL.
+name: gutenberg
+domain: www.gutenberg.org
+field: books-literature
+request:
+ url: https://www.gutenberg.org/ebooks/search/
+ params:
+ query: "{query}"
+response:
+ format: html
+ results: "li.booklink"
+ base_url: https://www.gutenberg.org
+ fields:
+ title:
+ selector: "span.title"
+ extract:
+ selector: "span.subtitle"
+ url:
+ selector: "a.link"
+ attr: href
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/wiktionary.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/wiktionary.yaml
new file mode 100644
index 0000000..9b3ba0c
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/wiktionary.yaml
@@ -0,0 +1,25 @@
+# English Wiktionary via the MediaWiki search API.
+# Representative of the whole MediaWiki class (Wikipedia, Wikisource,
+# minecraft.wiki, ...): same API shape, just a different host. The result has a
+# page title and an HTML snippet; the page URL is templated from the title.
+name: wiktionary
+domain: en.wiktionary.org
+field: other
+request:
+ url: https://en.wiktionary.org/w/api.php
+ params:
+ action: query
+ list: search
+ format: json
+ srsearch: "{query}"
+ srlimit: "{limit}"
+response:
+ format: json
+ results: query.search
+ fields:
+ title: title
+ extract: snippet
+ url:
+ template: "https://en.wiktionary.org/wiki/{title}"
+ strip_html:
+ - extract
diff --git a/test/test_super_search_recipes.py b/test/test_super_search_recipes.py
new file mode 100644
index 0000000..290a590
--- /dev/null
+++ b/test/test_super_search_recipes.py
@@ -0,0 +1,171 @@
+"""Tests for the declarative YAML recipe engine (recipe.py).
+
+Each recipe is exercised against a mocked httpx transport, asserting the
+url/title/extract coercion (templated URLs, relative-href resolution, HTML
+stripping) and that an HTTP error returns [] rather than propagating. We also
+verify the shipped recipe files load and register into SOURCES.
+"""
+import re
+from pathlib import Path
+
+import httpx
+
+from mwmbl.tinysearchengine.super_search_sources.recipe import (
+ RECIPES_DIR,
+ Recipe,
+ load_recipes,
+ search_with_recipe,
+)
+
+RECIPES = load_recipes()
+
+
+def _recipe(name: str) -> Recipe:
+ return RECIPES[name]
+
+
+# ---------------------------------------------------------------------------
+# Recipe loading / registration
+# ---------------------------------------------------------------------------
+
+def test_shipped_recipes_load():
+ assert {"wiktionary", "archive_org", "gutenberg"} <= set(RECIPES)
+ assert _recipe("gutenberg").response_format == "html"
+
+
+def test_recipes_registered_in_sources():
+ from mwmbl.tinysearchengine.super_search_sources import SOURCES
+ assert "wiktionary" in SOURCES and "gutenberg" in SOURCES
+
+
+def test_load_recipes_skips_invalid(tmp_path):
+ (tmp_path / "broken.yaml").write_text("name: x\nrequest: {}\n") # no response
+ assert load_recipes(tmp_path) == {}
+
+
+# ---------------------------------------------------------------------------
+# JSON: Wiktionary (MediaWiki) — templated URL + HTML-stripped snippet
+# ---------------------------------------------------------------------------
+
+async def test_wiktionary_templates_url_and_strips_html(httpx_mock):
+ httpx_mock.add_response(
+ url=re.compile(r"https://en\.wiktionary\.org/w/api\.php.*"),
+ json={"query": {"search": [
+ {"title": "serendipity",
+ "snippet": 'fortunate <span class="searchmatch">serendipity</span>'},
+ ]}},
+ )
+ async with httpx.AsyncClient() as client:
+ docs = await search_with_recipe(client, _recipe("wiktionary"), "serendipity", 5)
+ assert len(docs) == 1
+ assert docs[0].url == "https://en.wiktionary.org/wiki/serendipity"
+ assert docs[0].title == "serendipity"
+ assert "<span" not in docs[0].extract and "serendipity" in docs[0].extract
+
+
+async def test_wiktionary_quotes_titles_with_spaces(httpx_mock):
+ httpx_mock.add_response(
+ url=re.compile(r"https://en\.wiktionary\.org/.*"),
+ json={"query": {"search": [{"title": "United States", "snippet": "x"}]}},
+ )
+ async with httpx.AsyncClient() as client:
+ docs = await search_with_recipe(client, _recipe("wiktionary"), "united", 5)
+ assert docs[0].url == "https://en.wiktionary.org/wiki/United%20States"
+
+
+# ---------------------------------------------------------------------------
+# JSON: archive.org — nested results path + templated URL from identifier
+# ---------------------------------------------------------------------------
+
+async def test_archive_org_nested_results_and_template(httpx_mock):
+ httpx_mock.add_response(
+ url=re.compile(r"https://archive\.org/advancedsearch\.php.*"),
+ json={"response": {"docs": [
+ {"identifier": "apollo11", "title": "Apollo 11", "description": "Moon landing"},
+ {"title": "no id"}, # no identifier -> no URL -> skipped
+ ]}},
+ )
+ async with httpx.AsyncClient() as client:
+ docs = await search_with_recipe(client, _recipe("archive_org"), "apollo 11", 5)
+ assert len(docs) == 1
+ assert docs[0].url == "https://archive.org/details/apollo11"
+ assert docs[0].extract == "Moon landing"
+
+
+# ---------------------------------------------------------------------------
+# HTML: Project Gutenberg — CSS selectors + relative href resolution
+# ---------------------------------------------------------------------------
+
+GUTENBERG_HTML = """
+<html><body><ul>
+ <li class="booklink">
+ <a class="link" href="/ebooks/84">
+ <span class="title">Frankenstein</span>
+ <span class="subtitle">Mary Wollstonecraft Shelley</span>
+ </a>
+ </li>
+ <li class="other"><a class="link" href="/ignored">nope</a></li>
+</ul></body></html>
+"""
+
+
+async def test_gutenberg_scrapes_and_resolves_relative_url(httpx_mock):
+ httpx_mock.add_response(
+ url=re.compile(r"https://www\.gutenberg\.org/ebooks/search/.*"),
+ text=GUTENBERG_HTML,
+ )
+ async with httpx.AsyncClient() as client:
+ docs = await search_with_recipe(client, _recipe("gutenberg"), "frankenstein", 5)
+ assert len(docs) == 1
+ assert docs[0].url == "https://www.gutenberg.org/ebooks/84"
+ assert docs[0].title == "Frankenstein"
+ assert "Shelley" in docs[0].extract
+
+
+# ---------------------------------------------------------------------------
+# XML: schema completeness (no XML recipe ships, but the engine supports it)
+# ---------------------------------------------------------------------------
+
+BGG_XML = """<?xml version="1.0" encoding="utf-8"?>
+<items>
+ <item type="boardgame" id="13">
+ <name type="primary" value="Catan"/>
+ </item>
+</items>
+"""
+
+
+async def test_xml_attributes_and_template(httpx_mock):
+ recipe = Recipe(
+ name="bgg",
+ request={"url": "https://boardgamegeek.com/xmlapi2/search"},
+ response={
+ "format": "xml",
+ "results": ".//item",
+ "fields": {
+ "title": {"selector": "name", "attr": "value"},
+ "url": {"template": "https://boardgamegeek.com/boardgame/{id}"},
+ },
+ },
+ )
+ httpx_mock.add_response(url=re.compile(r"https://boardgamegeek\.com/.*"), text=BGG_XML)
+ async with httpx.AsyncClient() as client:
+ docs = await search_with_recipe(client, recipe, "catan", 5)
+ assert len(docs) == 1
+ assert docs[0].title == "Catan"
+ assert docs[0].url == "https://boardgamegeek.com/boardgame/13"
+
+
+# ---------------------------------------------------------------------------
+# Robustness: errors swallowed
+# ---------------------------------------------------------------------------
+
+async def test_http_error_returns_empty(httpx_mock):
+ httpx_mock.add_response(status_code=500)
+ async with httpx.AsyncClient() as client:
+ docs = await search_with_recipe(client, _recipe("wiktionary"), "x", 5)
+ assert docs == []
+
+
+def test_recipes_dir_exists():
+ assert Path(RECIPES_DIR).is_dir()