An open source, non-profit web search engine
Super Search v2: fix prescreen search-param harvest (wrong-param false failures)
Daoud Clarke4f12b3abcf33
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/slashdot_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/slashdot_org.yamlnew file mode 100644index 0000000..da4b947--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/slashdot_org.yaml@@ -0,0 +1,19 @@+name: slashdot_org+domain: slashdot.org+field: technology+request:+ url: https://slashdot.org/index2.pl+ params:+ fhfilter: '{query}'+response:+ format: html+ results: article.fhitem+ base_url: https://slashdot.org+ fields:+ title: span.story-title > a+ url:+ selector: span.story-title > a+ attr: href+smoke:+ query: linux+ expect_title_contains: linuxdiff --git a/scripts/prescreen_sites.py b/scripts/prescreen_sites.pyindex eb16a37..a55de5a 100644--- a/scripts/prescreen_sites.py+++ b/scripts/prescreen_sites.py@@ -81,13 +81,25 @@ def _harvest(base: str, html: str) -> dict:forms = []for form in soup.find_all("form"):action = form.get("action") or ""- names = {(i.get("name") or "").lower() for i in form.find_all("input")}- hit = names & SEARCH_INPUT_NAMES- looks_search = "search" in action.lower() or "search" in (form.get("id", "") +- form.get("class", [""])[0]).lower()- if hit or (looks_search and names):- field = next(iter(hit)) if hit else "q"- forms.append({"action": urljoin(base, action or "/"), "param": field})+ # The real search field is a text/search input. Prefer one whose name is+ # a conventional search param, but fall back to the form's ACTUAL first+ # text-input name -- never a guessed "q" (e.g. Slashdot uses "fhfilter").+ text_names = [+ (i.get("name") or "").strip()+ for i in form.find_all("input")+ if (i.get("name") or "").strip()+ and (i.get("type") or "text").lower() in ("", "text", "search")+ ]+ hit = [n for n in text_names if n.lower() in SEARCH_INPUT_NAMES]+ cls = (form.get("class") or [""])[0]+ looks_search = "search" in action.lower() or "search" in (form.get("id", "") + cls).lower()+ if hit:+ field = hit[0]+ elif looks_search and text_names:+ field = text_names[0]+ else:+ continue+ forms.append({"action": urljoin(base, action or "/"), "param": field})# De-dup forms by (action, param).seen, uniq = set(), []for f in forms: