Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search v2: 8 more HTML recipes (6 auto + 2 agent-authored selectors)

Daoud Clarkeaf26fa6372ec
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/docs_djangoproject_com.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/docs_djangoproject_com.yaml
new file mode 100644
index 0000000..5582b7a
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/docs_djangoproject_com.yaml
@@ -0,0 +1,19 @@
+name: docs_djangoproject_com
+domain: docs.djangoproject.com
+field: programming
+request:
+ url: https://docs.djangoproject.com/en/6.0/search/
+ params:
+ q: '{query}'
+response:
+ format: html
+ results: h2.result-title
+ base_url: https://docs.djangoproject.com
+ fields:
+ title: a
+ url:
+ selector: a
+ attr: href
+smoke:
+ query: design
+ expect_title_contains: design
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/engineering_nyu_edu.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/engineering_nyu_edu.yaml
new file mode 100644
index 0000000..11b7195
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/engineering_nyu_edu.yaml
@@ -0,0 +1,19 @@
+name: engineering_nyu_edu
+domain: engineering.nyu.edu
+field: academia
+request:
+ url: https://engineering.nyu.edu/search
+ params:
+ search: '{query}'
+response:
+ format: html
+ results: h3.mb-2
+ base_url: https://engineering.nyu.edu
+ fields:
+ title: a
+ url:
+ selector: a
+ attr: href
+smoke:
+ query: research
+ expect_title_contains: research
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/indieweb_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/indieweb_org.yaml
new file mode 100644
index 0000000..17a825e
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/indieweb_org.yaml
@@ -0,0 +1,19 @@
+name: indieweb_org
+domain: indieweb.org
+field: tech
+request:
+ url: https://indieweb.org/wiki/index.php
+ params:
+ search: '{query}'
+response:
+ format: html
+ results: div.mw-search-result-heading
+ base_url: https://indieweb.org
+ fields:
+ title: a
+ url:
+ selector: a
+ attr: href
+smoke:
+ query: science
+ expect_title_contains: science
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/mathbabe_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/mathbabe_org.yaml
new file mode 100644
index 0000000..3b4f99e
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/mathbabe_org.yaml
@@ -0,0 +1,19 @@
+name: mathbabe_org
+domain: mathbabe.org
+field: programming
+request:
+ url: https://mathbabe.org
+ params:
+ s: '{query}'
+response:
+ format: html
+ results: h2
+ base_url: https://mathbabe.org
+ fields:
+ title: a
+ url:
+ selector: a
+ attr: href
+smoke:
+ query: science
+ expect_title_contains: science
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/sourceforge_net.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/sourceforge_net.yaml
new file mode 100644
index 0000000..a902efc
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/sourceforge_net.yaml
@@ -0,0 +1,19 @@
+name: sourceforge_net
+domain: sourceforge.net
+field: technology
+request:
+ url: https://sourceforge.net/directory/
+ params:
+ q: '{query}'
+response:
+ format: html
+ results: li.project-oss
+ base_url: https://sourceforge.net
+ fields:
+ title: a.result-heading-title
+ url:
+ selector: a.result-heading-title
+ attr: href
+smoke:
+ query: science
+ expect_title_contains: science
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/www_euronews_com.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/www_euronews_com.yaml
new file mode 100644
index 0000000..0df9b52
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/www_euronews_com.yaml
@@ -0,0 +1,19 @@
+name: www_euronews_com
+domain: www.euronews.com
+field: news
+request:
+ url: https://www.euronews.com/search
+ params:
+ query: '{query}'
+response:
+ format: html
+ results: '#c-search-articles article.the-media-object'
+ base_url: https://www.euronews.com
+ fields:
+ title: h2.the-media-object__title
+ url:
+ selector: a.the-media-object__link
+ attr: href
+smoke:
+ query: world
+ expect_title_contains: world
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/www_lrb_co_uk.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/www_lrb_co_uk.yaml
new file mode 100644
index 0000000..3768df2
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/www_lrb_co_uk.yaml
@@ -0,0 +1,19 @@
+name: www_lrb_co_uk
+domain: www.lrb.co.uk
+field: books-literature
+request:
+ url: https://www.lrb.co.uk/search-results
+ params:
+ search: '{query}'
+response:
+ format: html
+ results: h3
+ base_url: https://www.lrb.co.uk
+ fields:
+ title: a
+ url:
+ selector: a
+ attr: href
+smoke:
+ query: book
+ expect_title_contains: book
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/www_nist_gov.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/www_nist_gov.yaml
new file mode 100644
index 0000000..77ffcaf
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/www_nist_gov.yaml
@@ -0,0 +1,19 @@
+name: www_nist_gov
+domain: www.nist.gov
+field: science
+request:
+ url: https://www.nist.gov/search
+ params:
+ s: '{query}'
+response:
+ format: html
+ results: h3.nist-teaser__title
+ base_url: https://www.nist.gov
+ fields:
+ title: a
+ url:
+ selector: a
+ attr: href
+smoke:
+ query: science
+ expect_title_contains: science
diff --git a/scripts/fetch_url.py b/scripts/fetch_url.py
index 594358f..261aea1 100644
--- a/scripts/fetch_url.py
+++ b/scripts/fetch_url.py
@@ -14,6 +14,7 @@ JSON it pretty-prints so the response shape is obvious to a recipe author.
Usage:
uv run python scripts/fetch_url.py "https://example.com/search?q=test"
uv run python scripts/fetch_url.py "https://example.com/api?q=test" --max-chars 12000
+ uv run python scripts/fetch_url.py "https://example.com/search?q=test" --out /tmp/page.html
"""
from __future__ import annotations
@@ -29,7 +30,7 @@ ENGINE_HEADERS = {"User-Agent": "mwmbl-super-search-smoke/0.1 (+https://mwmbl.or
TIMEOUT = 15.0
-async def _fetch(url: str, max_chars: int) -> int:
+async def _fetch(url: str, max_chars: int, out: str | None) -> int:
try:
async with httpx.AsyncClient(
follow_redirects=True, timeout=TIMEOUT, headers=ENGINE_HEADERS,
@@ -44,6 +45,14 @@ async def _fetch(url: str, max_chars: int) -> int:
print(f"content-type: {ctype}")
print(f"length: {len(r.text)} chars\n---")
+ if out:
+ # Write the full, untruncated body so a recipe author can read/grep the
+ # real markup when picking a CSS selector.
+ with open(out, "w", encoding="utf-8") as fh:
+ fh.write(r.text)
+ print(f"wrote {len(r.text)} chars to {out}")
+ return 0
+
body = r.text
if "json" in ctype.lower():
try:
@@ -60,8 +69,9 @@ def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("url")
ap.add_argument("--max-chars", type=int, default=6000)
+ ap.add_argument("--out", help="write full untruncated body to this file")
args = ap.parse_args()
- return asyncio.run(_fetch(args.url, args.max_chars))
+ return asyncio.run(_fetch(args.url, args.max_chars, args.out))
if __name__ == "__main__":