Skip to content

SearXNG is a free internet metasearch engine which aggregates results from various search services and databases. Users are neither tracked nor profiled.

3.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring, global-statement

import typing as t

import asyncio
import logging
import os
import threading

from curl_cffi import AsyncSession, CurlHttpVersion, CurlOpt
from curl_cffi.requests.exceptions import RequestException

from searx.extended_types import SXNG_Response

LOOP: asyncio.AbstractEventLoop = None  # pyright: ignore[reportAssignmentType]

# chrome is used by default
DEFAULT_IMPERSONATE = "chrome"
NO_IMPERSONATE = "none"


class AsyncClient(AsyncSession):
    """:class:`curl_cffi.AsyncSession` with ``aclose`` / ``is_closed``."""

    def __init__(self, **kwargs: t.Any):
        self._closed = False
        super().__init__(**kwargs)

    @property
    def is_closed(self) -> bool:
        return self._closed

    async def aclose(self) -> None:
        if self._closed:
            return
        self._closed = True
        try:
            await self.close()
        except RequestException:
            pass


def _proxy_kwargs(proxies: dict[str, str], enable_http: bool) -> dict[str, t.Any]:
    """Map settings.yml proxy keys (``all://``, ``https://``) to curl_cffi."""
    mapped: dict[str, str] = {}
    all_proxy: str | None = None
    for pattern, proxy_url in proxies.items():
        if not enable_http and pattern.startswith("http://"):
            continue
        if pattern.startswith("https"):
            mapped["https"] = proxy_url
        elif pattern.startswith("http"):
            mapped["http"] = proxy_url
        else:
            all_proxy = proxy_url
    if all_proxy:
        return {"proxy": all_proxy}
    if mapped:
        return {"proxies": mapped}
    return {}


def new_client(
    # pylint: disable=too-many-arguments
    enable_http: bool,
    verify: bool | str,
    enable_http2: bool,
    enable_http3: bool,
    max_connections: int,
    proxies: dict[str, str],
    local_address: str | None,
    max_redirects: int,
    impersonate: str = DEFAULT_IMPERSONATE,
    curl_options: dict[int, t.Any] | None = None,
) -> AsyncClient:
    extra_curl = dict(curl_options or {})
    if not enable_http:
        extra_curl.setdefault(CurlOpt.PROTOCOLS_STR, "https")
        extra_curl.setdefault(CurlOpt.REDIR_PROTOCOLS_STR, "https")
    cert_file = os.environ.get("SSL_CERT_FILE")
    if cert_file:
        extra_curl.setdefault(CurlOpt.CAINFO, cert_file)
    cert_dir = os.environ.get("SSL_CERT_DIR")
    if cert_dir:
        extra_curl.setdefault(CurlOpt.CAPATH, cert_dir)
    use_impersonate = impersonate not in ("", NO_IMPERSONATE)
    kwargs: dict[str, t.Any] = {
        "verify": verify,
        "max_redirects": max_redirects,
        "max_clients": max_connections or 10,
        "response_class": SXNG_Response,
        "discard_cookies": True,
        **_proxy_kwargs(proxies, enable_http),
    }
    if use_impersonate:
        kwargs["impersonate"] = impersonate
        kwargs["default_headers"] = True
    if local_address:
        kwargs["interface"] = local_address
    if not enable_http2:
        kwargs["http_version"] = CurlHttpVersion.V1_1
    elif enable_http3 and not proxies:
        kwargs["http_version"] = CurlHttpVersion.V3
    else:
        kwargs["http_version"] = CurlHttpVersion.V2_0
    if extra_curl:
        kwargs["curl_options"] = extra_curl
    return AsyncClient(**kwargs)


def get_loop() -> asyncio.AbstractEventLoop:
    return LOOP


def init():
    logging.getLogger("curl_cffi").setLevel(logging.WARNING)

    ready = threading.Event()

    def loop_thread():
        global LOOP
        LOOP = asyncio.new_event_loop()
        ready.set()
        LOOP.run_forever()

    threading.Thread(target=loop_thread, name="asyncio_loop", daemon=True).start()
    ready.wait()


init()