Skip to content

SearXNG is a free internet metasearch engine which aggregates results from various search services and databases. Users are neither tracked nor profiled.

[feat] brave: parse JavaScript IIFE to fix engine

WookieeOnCrackd47cb96bfb79
diff --git a/searx/engines/brave.py b/searx/engines/brave.py
index a8f2b3087..9031269ea 100644
--- a/searx/engines/brave.py
+++ b/searx/engines/brave.py
@@ -118,6 +118,7 @@ Implementations
"""
import json
+import re
import typing as t
from collections.abc import Callable
from urllib.parse import urlencode
@@ -129,7 +130,14 @@ from searx.enginelib.traits import EngineTraits
from searx.exceptions import SearxEngineResponseException
from searx.result_types import EngineResults, MainResult, Video
from searx.result_types.image import Image
-from searx.utils import html_to_text, js_obj_str_to_json_str, js_obj_str_to_python
+from searx.utils import (
+ html_to_text,
+ js_find_matching_bracket,
+ js_obj_str_to_json_str,
+ js_obj_str_to_python,
+ js_split_args,
+ js_find_variable_positions,
+)
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
@@ -188,6 +196,16 @@ time_range_map: dict[str, str] = {
"year": "py",
}
+IIFE_RE = re.compile(
+ r"""
+ \(function\s*\( # (function(
+ (?P<params>[^)]*) # formal parameters
+ \)\s*\{\s*return\s* # ){ return
+ (?P<body>\{) # the object literal starts
+ """,
+ re.VERBOSE | re.DOTALL,
+)
+
def request(query: str, params: dict[str, t.Any]) -> None:
@@ -207,12 +225,10 @@ def request(query: str, params: dict[str, t.Any]) -> None:
if brave_category == "goggles":
args["goggles_id"] = Goggles
- params["headers"]["Accept-Encoding"] = "gzip, deflate"
params["url"] = f"{base_url}{brave_category}?{urlencode(args)}"
logger.debug("url %s", params["url"])
# set properties in the cookies
-
params["cookies"]["safesearch"] = safesearch_map.get(params["safesearch"], "off")
# the useLocation is IP based, we use cookie "country" for the region
params["cookies"]["useLocation"] = "0"
@@ -235,25 +251,90 @@ def _extract_published_date(published_date_raw: str | None):
return None
+def extract_iife_object(data_src: str, match: re.Match[str]) -> tuple[str, int]:
+ """
+ Extract and substitute the object literal from a JavaScript IIFE.
+ Returns the transformed object literal and the position immediately after the IIFE call.
+ """
+ body_start = match.start("body")
+ body_end = js_find_matching_bracket(data_src, body_start, "{", "}")
+ obj_str = data_src[body_start : body_end + 1]
+
+ # 1. Locate the argument call execution signature block: ...)(arg1, arg2)
+ call_start = data_src.find("(", body_end)
+ if call_start == -1:
+ raise ValueError("IIFE missing call arguments")
+ call_end = js_find_matching_bracket(data_src, call_start, "(", ")")
+
+ # 2. map arguments
+ args = js_split_args(data_src[call_start + 1 : call_end])
+ params = [p.strip() for p in match["params"].split(",") if p.strip()]
+ mapping = dict(zip(params, args))
+
+ # 3. substitute idents
+ variables = js_find_variable_positions(obj_str)
+ for variable_name, pos in reversed(variables):
+ if variable_name in mapping:
+ value = mapping[variable_name]
+ obj_str = obj_str[:pos] + value + obj_str[pos + len(variable_name) :]
+
+ return obj_str, call_end + 1
+
+
def extract_json_data(text: str) -> dict[str, t.Any]:
- # Example script source containing the data:
- #
- # kit.start(app, element, {
- # node_ids: [0, 19],
- # data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
- # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- # form: null,
- # error: null
- # });
- start = text.index("data: [{")
- newline = text.index("\n", start)
- end = text.rindex("}}]", start, newline)
- js_obj_str = "{" + text[start:end] + "}}]}"
- # js_obj_str = js_obj_str.replace("\xa0", "") # remove ASCII for &nbsp;
- # js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<") # fix broken HTML tags in strings
- json_str = js_obj_str_to_json_str(js_obj_str)
- data: dict[str, t.Any] = json.loads(json_str)
- return data
+ """
+ Extract the SvelteKit data payload that Brave embeds in the page.
+
+ Handles:
+ - plain-object form
+ - pure IIFE form
+ - mixed form (IIFE + plain object) that appears on some news pages
+ """
+ marker = "data: ["
+ start = text.find(marker)
+ if start == -1:
+ raise ValueError("Brave data payload not found (no 'data: [')")
+
+ array_start = start + len(marker) - 1 # Points directly at '['
+ array_end = js_find_matching_bracket(text, array_start, "[", "]")
+ data_src = text[array_start + 1 : array_end]
+
+ objects: list[str] = []
+ pos = 0
+ n = len(data_src)
+
+ while pos < n:
+ # Move past whitespace and commas safely
+ while pos < n and data_src[pos] in " \t\n\r,":
+ pos += 1
+ if pos >= n:
+ break
+
+ # Case 1: IIFE Object Signature Matches
+ if m := IIFE_RE.match(data_src, pos):
+ obj_str, next_pos = extract_iife_object(data_src, m)
+ objects.append(obj_str)
+ pos = next_pos
+ # Brave sometimes emits an extra ')' after the IIFE call
+ while pos < n and data_src[pos] in " \t\n\r)":
+ pos += 1
+ continue
+
+ # Case 2: Standard raw plain JSON mapping block {type:"data", ...}
+ if data_src[pos] == "{":
+ obj_end = js_find_matching_bracket(data_src, pos, "{", "}")
+ objects.append(data_src[pos : obj_end + 1])
+ pos = obj_end + 1
+ continue
+
+ raise ValueError(f"unexpected token in data array at offset {pos}: {data_src[pos : pos + 40]!r}")
+
+ if not objects:
+ raise ValueError("no data objects found inside Brave payload")
+
+ combined = '{"data":[' + ",".join(objects) + "]}"
+ json_str = js_obj_str_to_json_str(combined)
+ return json.loads(json_str)
def response(resp: "SXNG_Response") -> EngineResults:
diff --git a/searx/utils.py b/searx/utils.py
index de795ab8d..0a2a766e0 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -7,6 +7,7 @@ import re
import importlib
import importlib.util
import json
+import string
import types
import typing as t
@@ -440,7 +441,7 @@ def to_string(obj: t.Any) -> str:
return repr(obj)
-def ecma_unescape(string: str) -> str:
+def ecma_unescape(text: str) -> str:
"""Python implementation of the unescape javascript function
https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string
@@ -455,20 +456,20 @@ def ecma_unescape(string: str) -> str:
'ó'
"""
# "%u5409" becomes "吉"
- string = _ECMA_UNESCAPE4_RE.sub(lambda e: chr(int(e.group(1), 16)), string)
+ text = _ECMA_UNESCAPE4_RE.sub(lambda e: chr(int(e.group(1), 16)), text)
# "%20" becomes " ", "%F3" becomes "ó"
- string = _ECMA_UNESCAPE2_RE.sub(lambda e: chr(int(e.group(1), 16)), string)
- return string
+ text = _ECMA_UNESCAPE2_RE.sub(lambda e: chr(int(e.group(1), 16)), text)
+ return text
-def remove_pua_from_str(string: str):
+def remove_pua_from_str(text: str):
"""Removes unicode's "PRIVATE USE CHARACTER"s (PUA_) from a string.
.. _PUA: https://en.wikipedia.org/wiki/Private_Use_Areas
"""
pua_ranges = ((0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD))
s: list[str] = []
- for c in string:
+ for c in text:
i = ord(c)
if any(a <= i <= b for (a, b) in pua_ranges):
continue
@@ -793,6 +794,116 @@ def js_obj_str_to_json_str(js_obj_str: str) -> str:
return s
+def js_find_matching_bracket(text: str, start: int, open_char: str = "{", close_char: str = "}") -> int:
+ """Return index of the matching closing bracket, safely ignoring JS string literals."""
+ depth = 0
+ i = start
+ in_string = None
+ escape_ = False
+
+ while i < len(text):
+ c = text[i]
+ if in_string:
+ if escape_:
+ escape_ = False
+ elif c == "\\":
+ escape_ = True
+ elif c == in_string:
+ in_string = None
+ elif c in ('"', "'", "`"):
+ in_string = c
+ elif c == open_char:
+ depth += 1
+ elif c == close_char:
+ depth -= 1
+ if depth == 0:
+ return i
+ i += 1
+ raise ValueError(f"unbalanced {open_char}{close_char} starting at {start}")
+
+
+def js_split_args(args_str: str) -> list[str]:
+ """Split a JS argument list, respecting strings and nested brackets."""
+ args: list[str] = []
+ last_pos = 0
+ in_string = None
+ escape_ = False
+ depth = 0
+
+ for idx, c in enumerate(f"{args_str},"):
+ if in_string:
+ if escape_:
+ escape_ = False
+ elif c == "\\":
+ escape_ = True
+ elif c == in_string:
+ in_string = None
+ elif c in ('"', "'", "`"):
+ in_string = c
+ elif c in ("(", "[", "{"):
+ depth += 1
+ elif c in (")", "]", "}"):
+ depth -= 1
+ elif c == "," and depth == 0:
+ if item := args_str[last_pos:idx].strip():
+ args.append(item)
+ last_pos = idx + 1
+ return args
+
+
+def js_find_variable_positions(js_obj: str) -> list[tuple[str, int]]:
+ """Find all variables referenced in the JavaScript object."""
+ # pylint: disable=too-many-branches
+ # pylint: disable=unsubscriptable-object
+
+ matches: list[tuple[str, int]] = []
+
+ in_string = False
+ in_escape = False
+
+ # flat array like [1,2,3], not e.g. [{"a": 3}]
+ in_flat_array = False
+ # if the current position is inside the place of
+ # a JS value, e.g. the range of ` bar` in {"foo": bar}
+ in_value = False
+ current_var: tuple[str, int] | None = None
+
+ for pos, c in enumerate(js_obj):
+ if in_string:
+ if in_escape:
+ in_escape = False
+ elif c == "\\":
+ in_escape = True
+ elif c == in_string:
+ in_string = None
+ elif c in ("'", "\"", "`"):
+ in_string = c
+ elif c == ":":
+ in_value = True
+ elif c in (",", "{", "}", "[", "]"):
+ # handle matching variables in arrays like [a,b,c]
+ # i.e. if we encounter a "," in such a flat array,
+ # we're still inside a value
+ if c == "[":
+ in_flat_array = True
+ elif c != ",":
+ in_flat_array = False
+ in_value = in_flat_array
+ if current_var:
+ # reached end of variable name
+ if current_var[0] not in ("void0", "true", "false", "null", "undefined"):
+ matches.append(current_var)
+ current_var = None
+ elif in_value:
+ if current_var and c in (string.ascii_letters + string.digits + "_$"):
+ current_var = (current_var[0] + c, current_var[1])
+ elif c in string.ascii_letters + "_$":
+ # start of new variable name
+ current_var = (c, pos)
+
+ return matches
+
+
def parse_duration_string(duration_str: str) -> timedelta | None:
"""Parse a time string in format MM:SS or HH:MM:SS and convert it to a
:obj:`datetime.timedelta` object.
diff --git a/tests/unit/test_engine_brave.py b/tests/unit/test_engine_brave.py
new file mode 100644
index 000000000..d73f81a39
--- /dev/null
+++ b/tests/unit/test_engine_brave.py
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=missing-module-docstring,line-too-long
+
+import json
+
+from parameterized import parameterized
+
+from searx.engines import brave
+from tests import SearxTestCase
+
+
+class BraveEngineTests(SearxTestCase):
+ '''Unit tests specific to the Brave engine'''
+
+ # extract_iife_object
+ @parameterized.expand(
+ [
+ (
+ """(function (a, b, c){ return {"a": a, "b": b, "c": c };})("first", 2, {"third": 3})""", # basic shape, numeric, nested object
+ {"a": "first", "b": 2, "c": {"third": 3}},
+ ),
+ (
+ '(function ($,aa,a,a1){return {"first": $, "second": true, "third": [aa,a,a1]};})("actual","x","y","z")', # non-alpha param, literal, and array
+ {"first": "actual", "second": True, "third": ["x", "y", "z"]},
+ ),
+ ]
+ )
+ def test_javascript_iife_parser(self, js_code, expected_output):
+ iife_match = brave.IIFE_RE.match(js_code)
+ self.assertIsNotNone(iife_match)
+
+ iife_object, _ = brave.extract_iife_object(js_code, iife_match) # pyright: ignore[reportArgumentType]
+ parsed = json.loads(iife_object)
+ self.assertEqual(parsed, expected_output)
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index beb794417..9fab2f2a6 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -58,7 +58,7 @@ class TestUtils(SearxTestCase):
def test_extract_text_error_empty(self):
with self.assertRaises(ValueError):
- utils.extract_text({})
+ utils.extract_text({}) # pyright: ignore[reportArgumentType]
def test_extract_url(self):
def f(html_str, search_url):
@@ -132,7 +132,7 @@ class TestXPathUtils(SearxTestCase): # pylint: disable=missing-class-docstring
utils.get_xpath(lxml.etree.XPath('//a'))
with self.assertRaises(TypeError):
- utils.get_xpath([])
+ utils.get_xpath([]) # pyright: ignore[reportArgumentType]
def test_get_xpath_invalid(self):
invalid_xpath = '//a[0].text'
@@ -214,3 +214,57 @@ class TestXPathUtils(SearxTestCase): # pylint: disable=missing-class-docstring
key, counter = solution # pyright: ignore[reportGeneralTypeIssues]
self.assertEqual(key, "71087e0b10d819181fc5463588a230b5744e6b9f19b86667c71ff73a9c7567aa")
self.assertEqual(counter, 260)
+
+
+class TestJsHelpers(SearxTestCase):
+ # js_find_matching_bracket
+ @parameterized.expand(
+ [
+ # simple object
+ ('{"a": 1, "b": {"c": 2}}', 0, "{", "}", 22),
+ # nested with strings that contain braces
+ ('{"a": "hello { world }", "b": 2}', 0, "{", "}", 31),
+ # array
+ ("[1, 2, [3, 4], 5]", 0, "[", "]", 16),
+ # string with escaped quote
+ (r'{"a": "he said \"hi\"", "b": 1}', 0, "{", "}", 30),
+ ]
+ )
+ def test_js_find_matching_bracket(self, text, start, open_c, close_c, expected_end):
+ end = utils.js_find_matching_bracket(text, start, open_c, close_c)
+ self.assertEqual(end, expected_end)
+
+ def test_js_find_matching_bracket_unbalanced(self):
+ with self.assertRaises(ValueError):
+ utils.js_find_matching_bracket("{a: 1", 0)
+
+ # js_split_args
+ @parameterized.expand(
+ [
+ ('"first", 2, {"third": 3}', ['"first"', "2", '{"third": 3}']),
+ ('"a", "b,c", [1,2]', ['"a"', '"b,c"', "[1,2]"]),
+ ("$", ["$"]),
+ ("", []),
+ ('"only"', ['"only"']),
+ ('1, "two", true, null', ["1", '"two"', "true", "null"]),
+ ]
+ )
+ def test_js_split_args(self, args_str, expected):
+ self.assertEqual(utils.js_split_args(args_str), expected)
+
+ # js_substitute_identifiers
+ @parameterized.expand(
+ [
+ # basic
+ ('{"a": a, "b": b}', [("a", 6), ("b", 14)]),
+ # $ and multi-char
+ ('{"first": $, "second": aa}', [("$", 10), ("aa", 23)]),
+ # identifier inside a string must NOT be matched
+ ('{"msg": "hello a world", "val": a}', [("a", 32)]),
+ # ints, booleans and null are no identifiers
+ ('{"msg": null, "val": true, "status": 1}', []),
+ ]
+ )
+ def test_js_find_variables(self, js_obj, expected):
+ result = utils.js_find_variable_positions(js_obj)
+ self.assertEqual(result, expected)