An open source, non-profit web search engine
Super Search v2: 8 more HTML recipes (6 auto + 2 agent-authored selectors)
Daoud Clarkeaf26fa6372ec
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/docs_djangoproject_com.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/docs_djangoproject_com.yamlnew file mode 100644index 0000000..5582b7a--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/docs_djangoproject_com.yaml@@ -0,0 +1,19 @@+name: docs_djangoproject_com+domain: docs.djangoproject.com+field: programming+request:+ url: https://docs.djangoproject.com/en/6.0/search/+ params:+ q: '{query}'+response:+ format: html+ results: h2.result-title+ base_url: https://docs.djangoproject.com+ fields:+ title: a+ url:+ selector: a+ attr: href+smoke:+ query: design+ expect_title_contains: designdiff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/engineering_nyu_edu.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/engineering_nyu_edu.yamlnew file mode 100644index 0000000..11b7195--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/engineering_nyu_edu.yaml@@ -0,0 +1,19 @@+name: engineering_nyu_edu+domain: engineering.nyu.edu+field: academia+request:+ url: https://engineering.nyu.edu/search+ params:+ search: '{query}'+response:+ format: html+ results: h3.mb-2+ base_url: https://engineering.nyu.edu+ fields:+ title: a+ url:+ selector: a+ attr: href+smoke:+ query: research+ expect_title_contains: researchdiff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/indieweb_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/indieweb_org.yamlnew file mode 100644index 0000000..17a825e--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/indieweb_org.yaml@@ -0,0 +1,19 @@+name: indieweb_org+domain: indieweb.org+field: tech+request:+ url: https://indieweb.org/wiki/index.php+ params:+ search: '{query}'+response:+ format: html+ results: div.mw-search-result-heading+ base_url: https://indieweb.org+ fields:+ title: a+ url:+ selector: a+ attr: href+smoke:+ query: science+ expect_title_contains: sciencediff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/mathbabe_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/mathbabe_org.yamlnew file mode 100644index 0000000..3b4f99e--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/mathbabe_org.yaml@@ -0,0 +1,19 @@+name: mathbabe_org+domain: mathbabe.org+field: programming+request:+ url: https://mathbabe.org+ params:+ s: '{query}'+response:+ format: html+ results: h2+ base_url: https://mathbabe.org+ fields:+ title: a+ url:+ selector: a+ attr: href+smoke:+ query: science+ expect_title_contains: sciencediff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/sourceforge_net.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/sourceforge_net.yamlnew file mode 100644index 0000000..a902efc--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/sourceforge_net.yaml@@ -0,0 +1,19 @@+name: sourceforge_net+domain: sourceforge.net+field: technology+request:+ url: https://sourceforge.net/directory/+ params:+ q: '{query}'+response:+ format: html+ results: li.project-oss+ base_url: https://sourceforge.net+ fields:+ title: a.result-heading-title+ url:+ selector: a.result-heading-title+ attr: href+smoke:+ query: science+ expect_title_contains: sciencediff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/www_euronews_com.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/www_euronews_com.yamlnew file mode 100644index 0000000..0df9b52--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/www_euronews_com.yaml@@ -0,0 +1,19 @@+name: www_euronews_com+domain: www.euronews.com+field: news+request:+ url: https://www.euronews.com/search+ params:+ query: '{query}'+response:+ format: html+ results: '#c-search-articles article.the-media-object'+ base_url: https://www.euronews.com+ fields:+ title: h2.the-media-object__title+ url:+ selector: a.the-media-object__link+ attr: href+smoke:+ query: world+ expect_title_contains: worlddiff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/www_lrb_co_uk.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/www_lrb_co_uk.yamlnew file mode 100644index 0000000..3768df2--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/www_lrb_co_uk.yaml@@ -0,0 +1,19 @@+name: www_lrb_co_uk+domain: www.lrb.co.uk+field: books-literature+request:+ url: https://www.lrb.co.uk/search-results+ params:+ search: '{query}'+response:+ format: html+ results: h3+ base_url: https://www.lrb.co.uk+ fields:+ title: a+ url:+ selector: a+ attr: href+smoke:+ query: book+ expect_title_contains: bookdiff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/www_nist_gov.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/www_nist_gov.yamlnew file mode 100644index 0000000..77ffcaf--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/www_nist_gov.yaml@@ -0,0 +1,19 @@+name: www_nist_gov+domain: www.nist.gov+field: science+request:+ url: https://www.nist.gov/search+ params:+ s: '{query}'+response:+ format: html+ results: h3.nist-teaser__title+ base_url: https://www.nist.gov+ fields:+ title: a+ url:+ selector: a+ attr: href+smoke:+ query: science+ expect_title_contains: sciencediff --git a/scripts/fetch_url.py b/scripts/fetch_url.pyindex 594358f..261aea1 100644--- a/scripts/fetch_url.py+++ b/scripts/fetch_url.py@@ -14,6 +14,7 @@ JSON it pretty-prints so the response shape is obvious to a recipe author.Usage:uv run python scripts/fetch_url.py "https://example.com/search?q=test"uv run python scripts/fetch_url.py "https://example.com/api?q=test" --max-chars 12000+ uv run python scripts/fetch_url.py "https://example.com/search?q=test" --out /tmp/page.html"""from __future__ import annotations@@ -29,7 +30,7 @@ ENGINE_HEADERS = {"User-Agent": "mwmbl-super-search-smoke/0.1 (+https://mwmbl.orTIMEOUT = 15.0-async def _fetch(url: str, max_chars: int) -> int:+async def _fetch(url: str, max_chars: int, out: str | None) -> int:try:async with httpx.AsyncClient(follow_redirects=True, timeout=TIMEOUT, headers=ENGINE_HEADERS,@@ -44,6 +45,14 @@ async def _fetch(url: str, max_chars: int) -> int:print(f"content-type: {ctype}")print(f"length: {len(r.text)} chars\n---")+ if out:+ # Write the full, untruncated body so a recipe author can read/grep the+ # real markup when picking a CSS selector.+ with open(out, "w", encoding="utf-8") as fh:+ fh.write(r.text)+ print(f"wrote {len(r.text)} chars to {out}")+ return 0+body = r.textif "json" in ctype.lower():try:@@ -60,8 +69,9 @@ def main() -> int:ap = argparse.ArgumentParser()ap.add_argument("url")ap.add_argument("--max-chars", type=int, default=6000)+ ap.add_argument("--out", help="write full untruncated body to this file")args = ap.parse_args()- return asyncio.run(_fetch(args.url, args.max_chars))+ return asyncio.run(_fetch(args.url, args.max_chars, args.out))if __name__ == "__main__":