Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

8.9 KB
Raw Blame
#!/usr/bin/env python3
"""Deterministically auto-generate Super Search recipes for a list of domains.

Instead of dispatching an agent per site (which writes throwaway probe scripts
and needs manual approval, with a near-zero pass rate), this brute-forces the
handful of *formulaic* search endpoints that account for essentially every
recipe we have ever landed:

  1. WordPress REST search   GET /wp-json/wp/v2/search?search=&per_page=
  2. WordPress REST posts     GET /wp-json/wp/v2/posts?search=&per_page=
  3. MediaWiki API            GET /w/api.php?action=query&list=search&srsearch=
                              GET /api.php?...   (fallback path)
  4. Discourse               GET /search.json?q=

For each domain it tries each template with a few generic probe queries. A
template "passes" when it returns >= MIN_RESULTS results AND some probe word is
echoed in a result title -- which makes the written recipe's smoke block
(query = that word, expect_title_contains = that word) self-consistent, so the
output passes scripts/smoke_recipe.py and test_super_search_recipes.py unchanged.

The first passing template wins; its YAML is written to recipes/<name>.yaml.
Existing recipes are never overwritten.

Usage:
  DATABASE_URL="postgres://daoud@" uv run python scripts/auto_recipe.py \
      recipe_chunks/input_000.json recipe_chunks/input_001.json ...
  DATABASE_URL="postgres://daoud@" uv run python scripts/auto_recipe.py --all
"""

from __future__ import annotations

import argparse
import asyncio
import json
import os
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")

import logging  # noqa: E402

import django  # noqa: E402

django.setup()
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("mwmbl.tinysearchengine.super_search_sources.recipe").setLevel(logging.ERROR)

import httpx  # noqa: E402
import yaml  # noqa: E402

from mwmbl.tinysearchengine.super_search_sources.recipe import (  # noqa: E402
    Recipe,
    search_with_recipe,
)

RECIPES_DIR = REPO_ROOT / "mwmbl" / "tinysearchengine" / "super_search_sources" / "recipes"
CHUNKS_DIR = REPO_ROOT / "recipe_chunks"

MIN_RESULTS = 3
CONCURRENCY = 16
TIMEOUT = 12.0

# A primary probe word per field (more likely to be echoed in that field's
# result titles), then generic fallbacks. Capped per domain to bound requests.
FIELD_WORD = {
    "academia": "research",
    "art-design": "design",
    "books-literature": "book",
    "business-finance": "business",
    "education": "learning",
    "gaming": "game",
    "history": "history",
    "science": "science",
    "technology": "technology",
    "health": "health",
    "news": "news",
    "music": "music",
    "food": "food",
    "sports": "sport",
    "law-politics": "law",
    "environment": "climate",
    "travel": "travel",
}
GENERIC_WORDS = ["history", "science", "music", "design", "world", "data"]


def _name(domain: str) -> str:
    return "".join(c if c.isalnum() else "_" for c in domain)


def _probe_words(field: str) -> list[str]:
    words = []
    primary = FIELD_WORD.get(field)
    if primary:
        words.append(primary)
    for w in GENERIC_WORDS:
        if w not in words:
            words.append(w)
    return words[:5]


def _templates(domain: str) -> list[tuple[str, dict, dict]]:
    """Return (label, request, response) candidate recipe pieces for a domain."""
    base = f"https://{domain}"
    return [
        (
            "wp-search",
            {"url": f"{base}/wp-json/wp/v2/search", "params": {"search": "{query}", "per_page": "{limit}"}},
            {"format": "json", "results": "", "fields": {"title": "title", "url": "url"}},
        ),
        (
            "wp-posts",
            {"url": f"{base}/wp-json/wp/v2/posts", "params": {"search": "{query}", "per_page": "{limit}"}},
            {
                "format": "json",
                "results": "",
                "fields": {"title": "title.rendered", "extract": "excerpt.rendered", "url": "link"},
                "strip_html": ["title", "extract"],
            },
        ),
        (
            "mediawiki",
            {
                "url": f"{base}/w/api.php",
                "params": {
                    "action": "query",
                    "list": "search",
                    "format": "json",
                    "srsearch": "{query}",
                    "srlimit": "{limit}",
                },
            },
            {
                "format": "json",
                "results": "query.search",
                "fields": {"title": "title", "extract": "snippet", "url": {"template": f"{base}/wiki/{{title}}"}},
                "strip_html": ["extract"],
            },
        ),
        (
            "mediawiki-root",
            {
                "url": f"{base}/api.php",
                "params": {
                    "action": "query",
                    "list": "search",
                    "format": "json",
                    "srsearch": "{query}",
                    "srlimit": "{limit}",
                },
            },
            {
                "format": "json",
                "results": "query.search",
                "fields": {"title": "title", "extract": "snippet", "url": {"template": f"{base}/wiki/{{title}}"}},
                "strip_html": ["extract"],
            },
        ),
        (
            "discourse",
            {"url": f"{base}/search.json", "params": {"q": "{query}"}},
            {
                "format": "json",
                "results": "topics",
                "fields": {"title": "title", "url": {"template": f"{base}/t/{{slug}}/{{id}}"}},
            },
        ),
    ]


async def _probe_domain(client: httpx.AsyncClient, entry: dict) -> dict:
    domain = entry["name"]
    field = entry.get("field", "")
    name = _name(domain)
    if (RECIPES_DIR / f"{name}.yaml").exists():
        return {"domain": domain, "status": "exists"}

    words = _probe_words(field)
    for label, request, response in _templates(domain):
        recipe = Recipe(name=name, request=request, response=response, domain=domain, field=field)
        for word in words:
            try:
                docs = await search_with_recipe(client, recipe, word, 10)
            except Exception:  # noqa: BLE001
                docs = []
            if len(docs) < MIN_RESULTS:
                # Pattern's endpoint clearly absent/broken -> stop trying words.
                if not docs:
                    break
                continue
            if any(word in (d.title or "").lower() for d in docs):
                doc = {
                    "name": name,
                    "domain": domain,
                    "field": field,
                    "request": request,
                    "response": response,
                    "smoke": {"query": word, "expect_title_contains": word},
                }
                (RECIPES_DIR / f"{name}.yaml").write_text(yaml.safe_dump(doc, sort_keys=False, allow_unicode=True))
                return {"domain": domain, "status": "pass", "label": label, "word": word, "n": len(docs)}
    return {"domain": domain, "status": "fail"}


async def _run(entries: list[dict]) -> list[dict]:
    sem = asyncio.Semaphore(CONCURRENCY)
    async with httpx.AsyncClient(
        follow_redirects=True,
        timeout=TIMEOUT,
        headers={"User-Agent": "mwmbl-super-search-smoke/0.1 (+https://mwmbl.org)"},
    ) as client:

        async def guarded(entry):
            async with sem:
                return await _probe_domain(client, entry)

        return await asyncio.gather(*[guarded(e) for e in entries])


def _load_entries(paths: list[str]) -> list[dict]:
    entries, seen = [], set()
    for p in paths:
        for entry in json.loads(Path(p).read_text()):
            if entry["name"] not in seen:
                seen.add(entry["name"])
                entries.append(entry)
    return entries


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("batches", nargs="*", help="batch JSON files")
    ap.add_argument("--all", action="store_true", help="process every recipe_chunks/input_*.json")
    args = ap.parse_args()

    paths = sorted(str(p) for p in CHUNKS_DIR.glob("input_*.json")) if args.all else args.batches
    if not paths:
        print("no batch files given (use --all or list files)", file=sys.stderr)
        return 2

    entries = _load_entries(paths)
    print(f"Probing {len(entries)} domains from {len(paths)} batch file(s)...")
    results = asyncio.run(_run(entries))

    passes = [r for r in results if r["status"] == "pass"]
    exists = [r for r in results if r["status"] == "exists"]
    for r in passes:
        print(f"PASS {r['domain']}  ({r['label']}, q={r['word']!r}, {r['n']} results)")
    print(
        f"\n{len(passes)} passed, {len(exists)} already existed, "
        f"{len(results) - len(passes) - len(exists)} failed "
        f"(of {len(results)} domains)."
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())