SearXNG is a free internet metasearch engine which aggregates results from various search services and databases. Users are neither tracked nor profiled.
Python bingbravedegoogleduckduckgogooglemetasearchprivacypythonqwantsearchsearch-enginesearxsearxngstartpageyahoo
[feat] magnific: solve captcha to bypass botblocking
Bnyrofe5bdd1bf2c4
diff --git a/searx/engines/magnific.py b/searx/engines/magnific.pyindex ddec8dfa4..3460725fe 100644--- a/searx/engines/magnific.py+++ b/searx/engines/magnific.py@@ -4,11 +4,16 @@.. _Magnific: https://www.magnific.com"""+import refrom urllib.parse import urlencodeimport typing as t+from searx.enginelib import EngineCache+from searx.exceptions import SearxEngineAPIException+from searx.network import get, postfrom searx.result_types import EngineResults+from searx.utils import eval_xpath_getindex, extract_textif t.TYPE_CHECKING:from searx.extended_types import SXNG_Response@@ -26,7 +31,7 @@ about = {base_url = "https://www.magnific.com"-categories = ["images"]+categories = ["stock_images"]paging = Truefree_images_only = True@@ -34,6 +39,47 @@ free_images_only = TrueWhether to only load images that may be used for free, without a Magnific account."""+# regular expressions for solving the captcha+_NUMBER_RE = re.compile(r"\d+")+_BM_VERIFY_RE = re.compile(r"\"bm-verify\":\s*\"(.*?)\"")+_CAPTCHA_COOKIE_NAME = "ak_bmsc"++CACHE: EngineCache+"""Cache for storing the cookie to bypass botblocking, obtained by solving a CAPTCHA."""+++def setup(engine_settings: dict[str, t.Any]):+ global CACHE # pylint: disable=global-statement+ CACHE = EngineCache(engine_settings["name"])+++def _solve_captcha(original_resp: "SXNG_Response") -> "SXNG_Response":+ """Solves the Magnific CAPTCHA by summing all numbers in the response.+ E.g., the relevant part of the script that contains the numbers could look like+ ``var i = 1789729970; var j = i + Number("1947" + "43160");``."""+ doc = original_resp.html()+ script = extract_text(eval_xpath_getindex(doc, "//script", 0)) or ""++ bm_verify = _BM_VERIFY_RE.search(original_resp.text)+ if not bm_verify:+ raise SearxEngineAPIException("failed to extract bm verify token")+ bm_verify = bm_verify.group(1)++ # challenge is just adding a bunch of number together (some are strings, some are ints)+ numbers = [int(m) for m in _NUMBER_RE.findall(script)]+ solution = sum(numbers)++ challenge_resp = post(+ f"{base_url}/_sec/verify?provider=interstitial",+ json={"bm-verify": bm_verify, "pow": solution},+ cookies=original_resp.cookies,+ )++ resp = get(original_resp.url, headers=original_resp.search_params["headers"], cookies=challenge_resp.cookies)+ # cookie is valid for 2 hours+ CACHE.set(_CAPTCHA_COOKIE_NAME, challenge_resp.cookies[_CAPTCHA_COOKIE_NAME], expire=7200)+ return resp+def request(query: str, params: "OnlineParams") -> None:args = {"term": query, "filters[ai-generated][excluded]": 1, "page": params["pageno"], "locale": "en"}@@ -41,10 +87,18 @@ def request(query: str, params: "OnlineParams") -> None:args["filters[license]"] = "free"params["headers"]["Referer"] = f"{base_url}/search"++ # cookie that is obtained after solving the CAPTCHA+ if cookie := CACHE.get(_CAPTCHA_COOKIE_NAME):+ params["cookies"][_CAPTCHA_COOKIE_NAME] = cookie+params["url"] = f"{base_url}/api/regular/search?{urlencode(args)}"def response(resp: "SXNG_Response"):+ if resp.headers["content-type"] == "text/html":+ resp = _solve_captcha(resp)+res = EngineResults()result: dict[str, t.Any] # TBH: dict[str, t.Any]