Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search v2: fix prescreen search-param harvest (wrong-param false failures)

Daoud Clarke4f12b3abcf33
diff --git a/mwmbl/tinysearchengine/super_search_sources/recipes/slashdot_org.yaml b/mwmbl/tinysearchengine/super_search_sources/recipes/slashdot_org.yaml
new file mode 100644
index 0000000..da4b947
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/recipes/slashdot_org.yaml
@@ -0,0 +1,19 @@
+name: slashdot_org
+domain: slashdot.org
+field: technology
+request:
+ url: https://slashdot.org/index2.pl
+ params:
+ fhfilter: '{query}'
+response:
+ format: html
+ results: article.fhitem
+ base_url: https://slashdot.org
+ fields:
+ title: span.story-title > a
+ url:
+ selector: span.story-title > a
+ attr: href
+smoke:
+ query: linux
+ expect_title_contains: linux
diff --git a/scripts/prescreen_sites.py b/scripts/prescreen_sites.py
index eb16a37..a55de5a 100644
--- a/scripts/prescreen_sites.py
+++ b/scripts/prescreen_sites.py
@@ -81,13 +81,25 @@ def _harvest(base: str, html: str) -> dict:
forms = []
for form in soup.find_all("form"):
action = form.get("action") or ""
- names = {(i.get("name") or "").lower() for i in form.find_all("input")}
- hit = names & SEARCH_INPUT_NAMES
- looks_search = "search" in action.lower() or "search" in (form.get("id", "") +
- form.get("class", [""])[0]).lower()
- if hit or (looks_search and names):
- field = next(iter(hit)) if hit else "q"
- forms.append({"action": urljoin(base, action or "/"), "param": field})
+ # The real search field is a text/search input. Prefer one whose name is
+ # a conventional search param, but fall back to the form's ACTUAL first
+ # text-input name -- never a guessed "q" (e.g. Slashdot uses "fhfilter").
+ text_names = [
+ (i.get("name") or "").strip()
+ for i in form.find_all("input")
+ if (i.get("name") or "").strip()
+ and (i.get("type") or "text").lower() in ("", "text", "search")
+ ]
+ hit = [n for n in text_names if n.lower() in SEARCH_INPUT_NAMES]
+ cls = (form.get("class") or [""])[0]
+ looks_search = "search" in action.lower() or "search" in (form.get("id", "") + cls).lower()
+ if hit:
+ field = hit[0]
+ elif looks_search and text_names:
+ field = text_names[0]
+ else:
+ continue
+ forms.append({"action": urljoin(base, action or "/"), "param": field})
# De-dup forms by (action, param).
seen, uniq = set(), []
for f in forms: