SearXNG is a free internet metasearch engine which aggregates results from various search services and databases. Users are neither tracked nor profiled.
Python bingbravedegoogleduckduckgogooglemetasearchprivacypythonqwantsearchsearch-enginesearxsearxngstartpageyahoo
[mod] Anna's Archive engine: revision of the engine (Paper result)
Markus Heiser22e73727c0ee
diff --git a/searx/engines/annas_archive.py b/searx/engines/annas_archive.pyindex d2dd7e8f5..d3eda6800 100644--- a/searx/engines/annas_archive.py+++ b/searx/engines/annas_archive.py@@ -23,10 +23,12 @@ for *newest* articles and journals (PDF) / by shortcut ``!aaa <search-term>``.- name: annas articlesengine: annas_archive+ categories = ["general", "articles"]shortcut: aaa- aa_content: 'magazine'- aa_ext: 'pdf'- aa_sort: 'newest'+ aa_content: "magazine"+ aa_ext: "pdf"+ aa_sort: "newest"+Implementations===============@@ -47,6 +49,7 @@ from searx.result_types import EngineResultsif t.TYPE_CHECKING:from searx.extended_types import SXNG_Response+ from searx.search.processors import OnlineParams# aboutabout: dict[str, t.Any] = {@@ -59,7 +62,7 @@ about: dict[str, t.Any] = {}# engine dependent config-categories: list[str] = ["files"]+categories = ["files", "books"]paging: bool = True# search-url@@ -72,14 +75,14 @@ aa_content: str = ""To not filter use an empty string (default)."""-aa_sort: str = ''+aa_sort: str = """""Sort Anna's results, possible values::newest, oldest, largest, smallestTo sort by *most relevant* use an empty string (default)."""-aa_ext: str = ''+aa_ext: str = """""Filter Anna's results by a file ending. Common filters for example are``pdf`` and ``epub``.@@ -91,29 +94,31 @@ aa_ext: str = ''"""-def init(engine_settings: dict[str, t.Any]) -> None: # pylint: disable=unused-argument+def setup(engine_settings: dict[str, t.Any]) -> bool: # pylint: disable=unused-argument"""Check of engine's settings."""- traits = EngineTraits(**ENGINE_TRAITS['annas archive'])+ traits = EngineTraits(**ENGINE_TRAITS["annas archive"])++ if aa_content and aa_content not in traits.custom["content"]:+ raise ValueError(f"invalid setting content: {aa_content}")- if aa_content and aa_content not in traits.custom['content']:- raise ValueError(f'invalid setting content: {aa_content}')+ if aa_sort and aa_sort not in traits.custom["sort"]:+ raise ValueError(f"invalid setting sort: {aa_sort}")- if aa_sort and aa_sort not in traits.custom['sort']:- raise ValueError(f'invalid setting sort: {aa_sort}')+ if aa_ext and aa_ext not in traits.custom["ext"]:+ raise ValueError(f"invalid setting ext: {aa_ext}")- if aa_ext and aa_ext not in traits.custom['ext']:- raise ValueError(f'invalid setting ext: {aa_ext}')+ return True-def request(query: str, params: dict[str, t.Any]) -> None:- lang = traits.get_language(params["language"], traits.all_locale)+def request(query: str, params: "OnlineParams") -> None:+ lang = traits.get_language(params["searxng_locale"], traits.all_locale)args = {- 'lang': lang,- 'content': aa_content,- 'ext': aa_ext,- 'sort': aa_sort,- 'q': query,- 'page': params['pageno'],+ "lang": lang,+ "content": aa_content,+ "ext": aa_ext,+ "sort": aa_sort,+ "q": query,+ "page": params["pageno"],}# filter out None and empty valuesfiltered_args = dict((k, v) for k, v in args.items() if v)@@ -126,29 +131,28 @@ def response(resp: "SXNG_Response") -> EngineResults:# The rendering of the WEB page is strange; positions of Anna's result page# are enclosed in SGML comments. These comments are *uncommented* by some- # JS code, see query of class '.js-scroll-hidden' in Anna's HTML template:+ # JS code, see query of class ".js-scroll-hidden" in Anna's HTML template:# https://annas-software.org/AnnaArchivist/annas-archive/-/blob/main/allthethings/templates/macros/md5_list.html- for item in eval_xpath_list(dom, '//main//div[contains(@class, "js-aarecord-list-outer")]/div'):+ for item in eval_xpath_list(dom, "//main//div[contains(@class, 'js-aarecord-list-outer')]/div"):try:kwargs: dict[str, t.Any] = _get_result(item)except SearxEngineXPathException:continue- res.add(res.types.LegacyResult(**kwargs))+ res.add(res.types.Paper(**kwargs))return resdef _get_result(item: ElementBase) -> dict[str, t.Any]:return {- 'template': 'paper.html',- 'url': base_url + eval_xpath_getindex(item, './a/@href', 0),- 'title': extract_text(eval_xpath(item, './div//a[starts-with(@href, "/md5")]')),- 'authors': [extract_text(eval_xpath_getindex(item, './/a[starts-with(@href, "/search")]', 0))],- 'publisher': extract_text(- eval_xpath_getindex(item, './/a[starts-with(@href, "/search")]', 1, default=None), allow_none=True+ "url": base_url + eval_xpath_getindex(item, "./a/@href", 0),+ "title": extract_text(eval_xpath(item, "./div//a[starts-with(@href, '/md5')]")),+ "authors": [extract_text(eval_xpath_getindex(item, ".//a[starts-with(@href, '/search')]", 0))],+ "publisher": extract_text(+ eval_xpath_getindex(item, ".//a[starts-with(@href, '/search')]", 1, default=None), allow_none=True),- 'content': extract_text(eval_xpath(item, './/div[contains(@class, "relative")]')),- 'thumbnail': extract_text(eval_xpath_getindex(item, './/img/@src', 0, default=None), allow_none=True),+ "content": extract_text(eval_xpath(item, ".//div[contains(@class, 'relative')]")),+ "thumbnail": extract_text(eval_xpath_getindex(item, ".//img/@src", 0, default=None), allow_none=True),}@@ -160,25 +164,25 @@ def fetch_traits(engine_traits: EngineTraits):from searx.network import get # see https://github.com/searxng/searxng/issues/762from searx.locales import language_tag- engine_traits.all_locale = ''- engine_traits.custom['content'] = []- engine_traits.custom['ext'] = []- engine_traits.custom['sort'] = []+ engine_traits.all_locale = ""+ engine_traits.custom["content"] = []+ engine_traits.custom["ext"] = []+ engine_traits.custom["sort"] = []- resp = get(base_url + '/search')+ resp = get(base_url + "/search")if not resp.ok:raise RuntimeError("Response from Anna's search page is not OK.")dom = html.fromstring(resp.text)# supported language codes- lang_map = {}+ lang_map: dict[str, str] = {}for x in eval_xpath_list(dom, "//form//input[@name='lang']"):eng_lang = x.get("value")- if eng_lang in ('', '_empty', 'nl-BE', 'und') or eng_lang.startswith('anti__'):+ if eng_lang in ("", "_empty", "nl-BE", "und") or eng_lang.startswith("anti__"):continuetry:- locale = babel.Locale.parse(lang_map.get(eng_lang, eng_lang), sep='-')+ locale = babel.Locale.parse(lang_map.get(eng_lang, eng_lang), sep="-")except babel.UnknownLocaleError:# silently ignore unknown languages# print("ERROR: %s -> %s is unknown by babel" % (x.get("data-name"), eng_lang))@@ -193,16 +197,16 @@ def fetch_traits(engine_traits: EngineTraits):for x in eval_xpath_list(dom, "//form//input[@name='content']"):if not x.get("value").startswith("anti__"):- engine_traits.custom['content'].append(x.get("value"))+ engine_traits.custom["content"].append(x.get("value"))for x in eval_xpath_list(dom, "//form//input[@name='ext']"):if not x.get("value").startswith("anti__"):- engine_traits.custom['ext'].append(x.get("value"))+ engine_traits.custom["ext"].append(x.get("value"))for x in eval_xpath_list(dom, "//form//select[@name='sort']//option"):- engine_traits.custom['sort'].append(x.get("value"))+ engine_traits.custom["sort"].append(x.get("value"))# for better diff; sort the persistence of these traits- engine_traits.custom['content'].sort()- engine_traits.custom['ext'].sort()- engine_traits.custom['sort'].sort()+ engine_traits.custom["content"].sort()+ engine_traits.custom["ext"].sort()+ engine_traits.custom["sort"].sort()diff --git a/searx/settings.yml b/searx/settings.ymlindex 3b77bfe09..7d4d98c95 100644--- a/searx/settings.yml+++ b/searx/settings.yml@@ -424,6 +424,7 @@ engines:engine: annas_archivedisabled: trueshortcut: aa+ timeout: 5- name: ansaengine: ansa