Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: update flaresolver (#1278)

NTFSvolumed63b76fecf1b
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
new file mode 100644
index 000000000..d06cfd3a6
--- /dev/null
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import itertools
+import time
+from functools import partial
+from http.cookies import SimpleCookie
+from typing import TYPE_CHECKING, Any
+
+import aiohttp
+from bs4 import BeautifulSoup
+from multidict import CIMultiDict, CIMultiDictProxy
+
+from cyberdrop_dl.compat import StrEnum
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.exceptions import DDOSGuardError
+from cyberdrop_dl.utils.logger import log
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from cyberdrop_dl.managers.manager import Manager
+
+
+class _Command(StrEnum):
+ CREATE_SESSION = "sessions.create"
+ DESTROY_SESSION = "sessions.destroy"
+ LIST_SESSIONS = "sessions.list"
+
+ GET_REQUEST = "request.get"
+ POST_REQUEST = "request.post"
+
+
[email protected](frozen=True, slots=True)
+class FlareSolverrSolution:
+ content: str
+ cookies: SimpleCookie
+ headers: CIMultiDictProxy[str]
+ url: AbsoluteHttpURL
+ user_agent: str
+ status: int
+
+ @staticmethod
+ def from_dict(solution: dict[str, Any]) -> FlareSolverrSolution:
+ return FlareSolverrSolution(
+ status=int(solution["status"]),
+ cookies=_parse_cookies(solution.get("cookies") or []),
+ user_agent=solution["userAgent"],
+ content=solution["response"],
+ url=AbsoluteHttpURL(solution["url"]),
+ headers=CIMultiDictProxy(CIMultiDict(solution["headers"])),
+ )
+
+
[email protected](frozen=True, slots=True)
+class _FlareSolverrResponse:
+ status: str
+ message: str
+ ok: bool
+ solution: FlareSolverrSolution | None
+
+ @staticmethod
+ def from_dict(resp: dict[str, Any]) -> _FlareSolverrResponse:
+ status, message = resp["status"], resp["message"]
+ solution = FlareSolverrSolution.from_dict(sol) if (sol := resp.get("solution")) else None
+ return _FlareSolverrResponse(status, message, status == "ok", solution)
+
+
+class FlareSolverr:
+ """Class that handles communication with flaresolverr."""
+
+ __slots__ = ("_next_request_id", "_request_lock", "_session_id", "_session_lock", "manager", "url")
+
+ def __init__(self, manager: Manager) -> None:
+ self.manager = manager
+ self._session_id: str = ""
+ self._session_lock, self._request_lock = asyncio.Lock(), asyncio.Lock()
+ self._next_request_id: Callable[[], int] = partial(next, itertools.count(1))
+ if manager.global_config.general.flaresolverr:
+ self.url = manager.global_config.general.flaresolverr / "v1"
+ else:
+ self.url = None
+
+ def __repr__(self):
+ return f"{type(self).__name__}(url={self.url!r})"
+
+ async def close(self):
+ await self._destroy_session()
+
+ async def request(self, url: AbsoluteHttpURL, data: Any = None) -> FlareSolverrSolution:
+ # TODO: make this method return an abstract response
+
+ invalid_response_error = DDOSGuardError("Invalid response from flaresolverr")
+ try:
+ if not self._session_id:
+ async with self._session_lock:
+ if not self._session_id:
+ await self._create_session()
+
+ resp = await self._request(
+ _Command.POST_REQUEST if data else _Command.GET_REQUEST,
+ url=str(url),
+ data=data,
+ session=self._session_id,
+ )
+
+ except (TypeError, KeyError) as e:
+ raise invalid_response_error from e
+
+ if not resp.ok:
+ raise DDOSGuardError(f"Failed to resolve URL with flaresolverr. {resp.message}")
+
+ if not resp.solution:
+ raise invalid_response_error
+
+ self.manager.client_manager.cookies.update_cookies(resp.solution.cookies)
+ self._check_user_agent(resp.solution)
+ return resp.solution
+
+ def _check_user_agent(self, solution: FlareSolverrSolution) -> None:
+ cdl_user_agent = self.manager.global_config.general.user_agent
+ mismatch_ua_msg = (
+ "Config user_agent and flaresolverr user_agent do not match:"
+ f"\n Cyberdrop-DL: '{cdl_user_agent}'"
+ f"\n Flaresolverr: '{solution.user_agent}'"
+ )
+
+ soup = BeautifulSoup(solution.content, "html.parser")
+ if self.manager.client_manager.check_ddos_guard(soup) or self.manager.client_manager.check_cloudflare(soup):
+ if solution.user_agent != cdl_user_agent:
+ raise DDOSGuardError(mismatch_ua_msg)
+
+ if solution.user_agent != cdl_user_agent:
+ msg = f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid"
+ log(msg, 30)
+
+ async def _request(self, command: _Command, /, data: Any = None, **kwargs: Any) -> _FlareSolverrResponse:
+ if not self.url:
+ raise DDOSGuardError("Found DDoS challenge, but FlareSolverr is not configured")
+
+ timeout = self.manager.global_config.rate_limiting_options._scrape_timeout
+ if command is _Command.CREATE_SESSION:
+ timeout = aiohttp.ClientTimeout(total=5 * 60, connect=60) # 5 minutes to create session
+
+ # timeout in miliseconds (60s)
+ playload = {"cmd": command, "maxTimeout": 60_000} | kwargs
+
+ if data:
+ assert command is _Command.POST_REQUEST
+ playload["postData"] = aiohttp.FormData(data)().decode()
+
+ async with (
+ self._request_lock,
+ self.manager.progress_manager.show_status_msg(
+ f"Waiting For Flaresolverr Response [{self._next_request_id()}]"
+ ),
+ ):
+ async with self.manager.client_manager._session.post(
+ self.url,
+ json=playload,
+ timeout=timeout,
+ ) as response:
+ return _FlareSolverrResponse.from_dict(await response.json())
+
+ async def _create_session(self) -> None:
+ session_id = "cyberdrop-dl"
+ kwargs = {}
+ if proxy := self.manager.global_config.general.proxy:
+ kwargs["proxy"] = {"url": str(proxy)}
+
+ resp = await self._request(_Command.CREATE_SESSION, session=session_id, **kwargs)
+ if not resp.ok:
+ raise DDOSGuardError(f"Failed to create flaresolverr session: {resp.message}")
+ self._session_id = session_id
+
+ async def _destroy_session(self) -> None:
+ if self._session_id:
+ await self._request(_Command.DESTROY_SESSION)
+ self._session_id = ""
+
+
+def _parse_cookies(cookies: list[dict[str, Any]]) -> SimpleCookie:
+ simple_cookie = SimpleCookie()
+ now = time.time()
+ for cookie in cookies:
+ name: str = cookie["name"]
+ simple_cookie[name] = cookie["value"]
+ morsel = simple_cookie[name]
+ morsel["domain"] = cookie["domain"]
+ morsel["path"] = cookie["path"]
+ morsel["secure"] = "TRUE" if cookie["secure"] else ""
+ if expires := cookie["expires"]:
+ morsel["max-age"] = str(max(0, int(expires) - int(now)))
+ return simple_cookie
diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.py
index 53170d2df..d4e0a97f5 100644
--- a/cyberdrop_dl/clients/response.py
+++ b/cyberdrop_dl/clients/response.py
@@ -1,10 +1,12 @@
from __future__ import annotations
import asyncio
+import dataclasses
from json import loads as json_loads
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Self
-from aiohttp_client_cache.response import AnyResponse
+from aiohttp import ClientResponse
+from aiohttp_client_cache.response import CachedResponse
from bs4 import BeautifulSoup
from multidict import CIMultiDict, CIMultiDictProxy
@@ -15,50 +17,106 @@ from cyberdrop_dl.utils.utilities import parse_url
if TYPE_CHECKING:
from curl_cffi.requests.models import Response as CurlResponse
+ from cyberdrop_dl.clients.flaresolverr import FlareSolverrSolution
-class AbstractResponse:
- """Class to represent common methods and attributes between aiohttp ClientResponse and a CurlResponse"""
-
- __slots__ = ("_read_lock", "_resp", "content_type", "headers", "location", "status", "url")
- def __init__(self, response: AnyResponse | CurlResponse) -> None:
- self._resp = response
- self.content_type = (self._resp.headers.get("Content-Type") or "").lower()
- if isinstance(response, AnyResponse):
- self.status = response.status
- self.headers = response.headers
[email protected](slots=True, weakref_slot=True)
+class AbstractResponse:
+ """Class to represent common methods and attributes between:
+ - `aiohttp.ClientResponse`
+ - `curl_cffi.Response`
+ - `FlareSolverrSolution`"""
+
+ content_type: str
+ status: int
+ headers: CIMultiDictProxy[str]
+ url: AbsoluteHttpURL
+ location: AbsoluteHttpURL | None
+
+ _resp: ClientResponse | CachedResponse | CurlResponse | None = None
+ _text: str = ""
+ _read_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)
+
+ @classmethod
+ def from_resp(cls, response: ClientResponse | CachedResponse | CurlResponse) -> Self:
+ if isinstance(response, ClientResponse | CachedResponse):
+ status = response.status
+ headers = response.headers
else:
- self.status = response.status_code
- self.headers = CIMultiDictProxy(CIMultiDict({k: v or "" for k, v in response.headers}))
-
- self.url = AbsoluteHttpURL(response.url)
- if location := response.headers.get("location"):
- self.location = parse_url(location, self.url.origin(), trim=False)
+ status = response.status_code
+ headers = CIMultiDictProxy(CIMultiDict({k: v or "" for k, v in response.headers}))
+
+ url = AbsoluteHttpURL(response.url)
+ content_type, location = cls.parse_headers(url, headers)
+ return cls(
+ content_type=content_type,
+ status=status,
+ headers=headers,
+ url=url,
+ location=location,
+ _resp=response,
+ )
+
+ @classmethod
+ def from_flaresolverr(cls, solution: FlareSolverrSolution) -> Self:
+ content_type, location = cls.parse_headers(solution.url, solution.headers)
+ return cls(
+ content_type=content_type,
+ status=solution.status,
+ headers=solution.headers,
+ url=solution.url,
+ location=location,
+ _text=solution.content,
+ )
+
+ @staticmethod
+ def parse_headers(url: AbsoluteHttpURL, headers: CIMultiDictProxy[str]):
+ if location := headers.get("location"):
+ location = parse_url(location, url.origin(), trim=False)
else:
- self.location = None
-
- self._read_lock = asyncio.Lock()
+ location = None
- async def text(self) -> str:
- async with self._read_lock:
- if isinstance(self._resp, AnyResponse):
- return await self._resp.text()
- return self._resp.text
-
- async def soup(self) -> BeautifulSoup:
- if "text" in self.content_type or "html" in self.content_type:
- return BeautifulSoup(await self.text(), "html.parser")
+ content_type = (headers.get("Content-Type") or "").lower()
+ return content_type, location
- raise InvalidContentTypeError(message=f"Received {self.content_type}, was expecting text")
+ async def text(self, encoding: str | None = None) -> str:
+ if self._text:
+ return self._text
- async def json(self) -> Any:
+ assert self._resp is not None
+ async with self._read_lock:
+ if self._text:
+ return self._text
+ if isinstance(self._resp, ClientResponse | CachedResponse):
+ self._text = await self._resp.text(encoding)
+ else:
+ if encoding:
+ self._resp.encoding = encoding
+ self._text = await self._resp.atext()
+ return self._text
+
+ async def soup(self, encoding: str | None = None) -> BeautifulSoup:
+ self._check_content_type("text", "html", expecting="HTML")
+ return BeautifulSoup(await self.text(encoding), "html.parser")
+
+ async def json(self, encoding: str | None = None, content_type: str | bool = True) -> Any:
if self.status == 204:
raise ScrapeError(204)
- if "text/plain" in self.content_type or "json" in self.content_type:
- return json_loads(await self.text())
+ if content_type:
+ if isinstance(content_type, str):
+ check = (content_type,)
+ else:
+ check = ("text/plain", "json")
+
+ self._check_content_type(*check, expecting="JSON")
+
+ return json_loads(await self.text(encoding))
- raise InvalidContentTypeError(message=f"Received {self.content_type}, was expecting JSON")
+ def _check_content_type(self, *content_types, expecting: str):
+ if not any(type_ in self.content_type for type_ in content_types):
+ msg = f"Received {self.content_type}, was expecting {expecting}"
+ raise InvalidContentTypeError(message=msg)
def __repr__(self) -> str:
- return f"<{self.__class__.__name__} [{self.status}] ({self.url})>"
+ return f"<{self.__class__.__name__} [{self.status}] ({self.url!r})>"
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index 280839840..e00721edf 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -5,19 +5,20 @@ import contextlib
from datetime import datetime
from json import dumps as json_dumps
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
import cyberdrop_dl.constants as constants
from cyberdrop_dl.clients.response import AbstractResponse
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature
-from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError
+from cyberdrop_dl.exceptions import DDOSGuardError
from cyberdrop_dl.utils.utilities import sanitize_filename
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
+ from collections.abc import AsyncGenerator
+
from curl_cffi.requests.impersonate import BrowserTypeLiteral
from curl_cffi.requests.session import HttpMethod
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.managers.client_manager import ClientManager
@@ -39,8 +40,9 @@ class ScraperClient:
await self.client_manager.manager.states.RUNNING.wait()
yield
+ @contextlib.asynccontextmanager
async def _request(
- self,
+ self: object,
url: AbsoluteHttpURL,
/,
method: HttpMethod = "GET",
@@ -50,68 +52,85 @@ class ScraperClient:
json: Any = None,
cache_disabled: bool = False,
**request_params: Any,
- ) -> AbstractResponse:
+ ) -> AsyncGenerator[AbstractResponse]:
+ """
+ Asynchronous context manager for HTTP requests.
+
+ - If 'impersonate' is specified, uses curl_cffi for the request and updates cookies.
+ - Otherwise, uses aiohttp with optional cache control.
+ - Yield an AbstractResponse that wraps the underlying response with common methods.
+ - On DDOSGuardError, retries the request using FlareSolverr.
+ - Saves the HTML content to disk if the config option is enabled.
+ - Closes underliying response on exit.
+ """
+ self = cast("ScraperClient", self)
request_params["headers"] = self.client_manager._default_headers | (headers or {})
request_params["data"] = data
request_params["json"] = json
+ async with self.__request_context(url, method, request_params, impersonate, cache_disabled) as resp:
+ exc = None
+ try:
+ yield await self._check_response(resp, url)
+ except Exception as e:
+ exc = e
+ raise
+ finally:
+ await self.write_soup_to_disk(url, resp, exc)
+
+ @contextlib.asynccontextmanager
+ async def __request_context(
+ self,
+ url: AbsoluteHttpURL,
+ method: HttpMethod,
+ request_params: dict[str, Any],
+ impersonate: BrowserTypeLiteral | bool | None,
+ cache_disabled: bool,
+ ) -> AsyncGenerator[AbstractResponse]:
if impersonate:
self.client_manager.check_curl_cffi_is_available()
if impersonate is True:
impersonate = "chrome"
request_params["impersonate"] = impersonate
- response = await self.client_manager._curl_session.request(method, str(url), **request_params)
+ curl_resp = await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)
+ try:
+ yield AbstractResponse.from_resp(curl_resp)
+ curl_cookies = self.client_manager._curl_session.cookies.get_dict(url.host)
+ self.client_manager.cookies.update_cookies(curl_cookies, url)
+ finally:
+ await curl_resp.aclose()
+ return
- else:
- async with self.client_manager.cache_control(self.client_manager._session, disabled=cache_disabled):
- response = await self.client_manager._session._request(method, url, **request_params)
+ async with (
+ self.client_manager.cache_control(self.client_manager._session, disabled=cache_disabled),
+ self.client_manager._session.request(method, url, **request_params) as aio_resp,
+ ):
+ yield AbstractResponse.from_resp(aio_resp)
- abs_resp = AbstractResponse(response)
- exc = None
+ async def _check_response(self, abs_resp: AbstractResponse, url: AbsoluteHttpURL, data: Any | None = None):
+ """Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.
+
+ Returns an AbstractResponse confirmed to not be a DDOS Guard page."""
try:
- await self.client_manager.check_http_status(response)
- except (DDOSGuardError, DownloadError) as e:
- exc = e
- raise
-
- else:
- if impersonate:
- self.client_manager.cookies.update_cookies(
- self.client_manager._curl_session.cookies.get_dict(url.host), url
- )
+ await self.client_manager.check_http_status(abs_resp)
return abs_resp
- finally:
- self.client_manager.manager.task_group.create_task(self.write_soup_to_disk(url, abs_resp, exc))
-
- @copy_signature(_request)
- async def _request_json(self, *args, **kwargs) -> Any:
- return await (await self._request(*args, **kwargs)).json()
+ except DDOSGuardError:
+ flare_solution = await self.client_manager.flaresolverr.request(url, data)
+ return AbstractResponse.from_flaresolverr(flare_solution)
- @copy_signature(_request)
- async def _request_soup(self, *args, **kwargs) -> BeautifulSoup:
- return await (await self._request(*args, **kwargs)).soup()
-
- async def write_soup_to_disk(
- self,
- url: AbsoluteHttpURL,
- response: AbstractResponse,
- exc: Exception | None = None,
- ):
+ async def write_soup_to_disk(self, url: AbsoluteHttpURL, response: AbstractResponse, exc: Exception | None = None):
"""Writes html to a file."""
+
if not self._save_pages_html:
return
- content: str = ""
+ content: str = await response.text()
try:
- content: str = (await response.soup()).prettify(formatter="html")
- except (UnicodeDecodeError, InvalidContentTypeError):
+ content = cast("str", (await response.soup()).prettify(formatter="html"))
+ except Exception:
pass
- content = content or await response.text()
now = datetime.now()
-
- # The date is not really relevant in the filename and makes them longer, potencially truncating the URL part
- # But it garanties the filename will be unique
log_date = now.strftime(constants.LOGS_DATETIME_FORMAT)
url_str = str(url)
response_url_str = str(response.url)
@@ -130,7 +149,11 @@ class ScraperClient:
json_data = json_dumps(info, indent=4, ensure_ascii=False)
text = f"<!-- cyberdrop-dl scraping result\n{json_data}\n-->\n{content}"
- try:
- await asyncio.to_thread(file_path.write_text, text, "utf8")
- except OSError:
- pass
+ self.client_manager.manager.task_group.create_task(try_write(file_path, text))
+
+
+async def try_write(file: Path, content: str) -> None:
+ try:
+ await asyncio.to_thread(file.write_text, content, "utf8")
+ except OSError:
+ pass
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 33a02c442..20d7c3257 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -224,9 +224,9 @@ class BunkrrCrawler(Crawler):
link: AbsoluteHttpURL | None = None
soup: BeautifulSoup | None = None
if is_stream_redirect(scrape_item.url):
- response, soup = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)
- scrape_item.url = AbsoluteHttpURL(response.url)
- del response
+ async with self.request(scrape_item.url) as resp:
+ soup = await resp.soup()
+ scrape_item.url = resp.url
database_url = scrape_item.url.with_host(self.DATABASE_PRIMARY_HOST)
if await self.check_complete_from_referer(database_url):
@@ -267,9 +267,7 @@ class BunkrrCrawler(Crawler):
@error_handling_wrapper
async def reinforced_file(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url)
title: str = css.select_one_get_text(soup, "h1")
link: AbsoluteHttpURL | None = await self.get_download_url_from_api(scrape_item.url)
if not link:
@@ -328,9 +326,7 @@ class BunkrrCrawler(Crawler):
api_url = STREAMING_API_ENTRYPOINT.with_host(self.known_good_host)
data = json.dumps(data_dict)
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=data, headers=headers)
-
+ json_resp: dict[str, Any] = await self.request_json(api_url, "POST", data=data, headers=headers)
api_response = ApiResponse(**json_resp)
link_str = decrypt_api_response(api_response)
link = self.parse_url(link_str)
@@ -362,14 +358,10 @@ class BunkrrCrawler(Crawler):
If we don't know a valid host but the response was successful, register the host as a valid host"""
- async def get_soup(url: AbsoluteHttpURL) -> BeautifulSoup:
- async with self.request_limiter:
- return await self.client.get_soup(self.DOMAIN, url)
-
async def get_soup_no_error(url: AbsoluteHttpURL) -> BeautifulSoup | None:
global known_bad_hosts
try:
- soup: BeautifulSoup = await get_soup(url)
+ soup = await self.request_soup(url)
except (ClientConnectorError, DDOSGuardError):
known_bad_hosts.add(url.host)
if not HOST_OPTIONS - known_bad_hosts:
@@ -380,10 +372,10 @@ class BunkrrCrawler(Crawler):
return soup
if not is_root_domain(url):
- return await get_soup(url)
+ return await self.request_soup(url)
elif self.known_good_host:
- return await get_soup(url.with_host(self.known_good_host))
+ return await self.request_soup(url.with_host(self.known_good_host))
async with self.switch_host_locks[url.host]:
if url.host not in known_bad_hosts:
@@ -399,7 +391,7 @@ class BunkrrCrawler(Crawler):
return soup
# everything failed, do the request with the original URL to throw an exception
- return await get_soup(url)
+ return await self.request_soup(url)
def get_part_next_to(url: AbsoluteHttpURL, part: str) -> str:
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 9f213baa8..86db0caf1 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -18,7 +18,8 @@ from aiolimiter import AsyncLimiter
from yarl import URL
from cyberdrop_dl import constants
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
+from cyberdrop_dl.clients.scraper_client import ScraperClient
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature
from cyberdrop_dl.downloader.downloader import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.scraper import filters
@@ -54,7 +55,7 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
from rich.progress import TaskID
- from cyberdrop_dl.clients.scraper_client import ScraperClient
+ from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.managers.manager import Manager
@@ -104,24 +105,26 @@ class Crawler(ABC):
_RATE_LIMIT: ClassVar[tuple[float, float]] = 25, 1
_DOWNLOAD_SLOTS: ClassVar[int | None] = None
- if TYPE_CHECKING:
- request = ScraperClient._request
- request_json = ScraperClient._request_json
- request_soup = ScraperClient._request_soup
-
- else:
+ @copy_signature(ScraperClient._request)
+ @contextlib.asynccontextmanager
+ async def request(self, *args, **kwargs) -> AsyncGenerator[AbstractResponse]:
+ async with self.client._limiter(self.DOMAIN), self.client._request(*args, **kwargs) as resp:
+ yield resp
- async def request(self, *args, **kwargs):
- async with self.client._limiter(self.DOMAIN):
- return await self.client._request(*args, **kwargs)
+ @copy_signature(ScraperClient._request)
+ async def request_json(self, *args, **kwargs) -> Any:
+ async with self.request(*args, **kwargs) as resp:
+ return await resp.json()
- async def request_json(self, *args, **kwargs):
- async with self.client._limiter(self.DOMAIN):
- return await self.client._request_json(*args, **kwargs)
+ @copy_signature(ScraperClient._request)
+ async def request_soup(self, *args, **kwargs) -> BeautifulSoup:
+ async with self.request(*args, **kwargs) as resp:
+ return await resp.soup()
- async def request_soup(self, *args, **kwargs):
- async with self.client._limiter(self.DOMAIN):
- return await self.client._request_soup(*args, **kwargs)
+ @copy_signature(ScraperClient._request)
+ async def request_text(self, *args, **kwargs) -> str:
+ async with self.request(*args, **kwargs) as resp:
+ return await resp.text()
@final
def __init__(self, manager: Manager) -> None:
@@ -654,8 +657,8 @@ class Crawler(ABC):
log(msg, bug=True)
async def _get_redirect_url(self, url: AbsoluteHttpURL):
- head = await self.request(url, method="HEAD")
- return head.location or url
+ async with self.request(url, method="HEAD") as head:
+ return head.location or url
@staticmethod
def register_cache_filter(
@@ -688,7 +691,7 @@ class Crawler(ABC):
return m3u8.RenditionGroup(await self._get_m3u8(url, headers))
async def _get_m3u8(self, url: AbsoluteHttpURL, /, headers: dict[str, str] | None = None) -> m3u8.M3U8:
- content = await (await self.request(url, headers=headers)).text()
+ content = await self.request_text(url, headers=headers)
return m3u8.M3U8(content, url.parent)
def create_custom_filename(
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 65f4f28f5..46d6e1f1d 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -5,7 +5,6 @@ import contextlib
import ssl
from base64 import b64encode
from collections import defaultdict
-from dataclasses import dataclass
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Literal, Self, overload
@@ -21,6 +20,7 @@ from videoprops import get_audio_properties, get_video_properties
from cyberdrop_dl import constants, env
from cyberdrop_dl.clients.download_client import DownloadClient
+from cyberdrop_dl.clients.flaresolverr import FlareSolverr
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.clients.scraper_client import ScraperClient
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem
@@ -164,7 +164,7 @@ class ClientManager:
self.scraper_client = ScraperClient(self)
self.speed_limiter = DownloadSpeedLimiter(self.rate_limiting_options.download_speed_limit)
self.download_client = DownloadClient(manager, self)
- self.flaresolverr = Flaresolverr(self)
+ self.flaresolverr = FlareSolverr(manager)
self.file_locks = FileLocksVault()
self._default_headers = {"user-agent": self.manager.global_config.general.user_agent}
self.reddit_session: CachedSession
@@ -378,7 +378,7 @@ class ClientManager:
If the response is successful and has valid html, returns soup
"""
if not isinstance(response, AbstractResponse):
- response = AbstractResponse(response)
+ response = AbstractResponse.from_resp(response)
message = None
@@ -502,139 +502,7 @@ class ClientManager:
return min_audio_duration <= media_item.duration <= max_audio_duration
async def close(self) -> None:
- await self.flaresolverr._destroy_session()
-
-
-@dataclass(frozen=True, slots=True)
-class FlaresolverrResponse:
- status: str
- cookies: dict
- user_agent: str
- soup: BeautifulSoup | None
- url: AbsoluteHttpURL
-
- @classmethod
- def from_dict(cls, flaresolverr_resp: dict) -> FlaresolverrResponse:
- status = flaresolverr_resp["status"]
- solution: dict = flaresolverr_resp["solution"]
- response = solution["response"]
- user_agent = solution["userAgent"].strip()
- url_str: str = solution["url"]
- cookies: dict = solution.get("cookies") or {}
- soup = BeautifulSoup(response, "html.parser") if response else None
- url = AbsoluteHttpURL(url_str)
- return cls(status, cookies, user_agent, soup, url)
-
-
-class Flaresolverr:
- """Class that handles communication with flaresolverr."""
-
- def __init__(self, client_manager: ClientManager) -> None:
- self.client_manager = client_manager
- self.enabled = bool(client_manager.manager.global_config.general.flaresolverr)
- self.session_id: str = ""
- self.session_lock = asyncio.Lock()
- self.request_lock = asyncio.Lock()
- self.request_count = 0
-
- async def _request(
- self,
- command: str,
- client_session: ClientSession,
- **kwargs,
- ) -> dict:
- """Base request function to call flaresolverr."""
- if not self.enabled:
- raise DDOSGuardError(message="FlareSolverr is not configured")
- async with self.session_lock:
- if not (self.session_id or kwargs.get("session")):
- await self._create_session()
- return await self._make_request(command, client_session, **kwargs)
-
- async def _make_request(self, command: str, client_session: ClientSession, **kwargs) -> dict[str, Any]:
- timeout = self.client_manager.manager.global_config.rate_limiting_options._scrape_timeout
- if command == "sessions.create":
- timeout = aiohttp.ClientTimeout(total=5 * 60, connect=60) # 5 minutes to create session
-
- for key, value in kwargs.items():
- if isinstance(value, AbsoluteHttpURL):
- kwargs[key] = str(value)
-
- data = {
- "cmd": command,
- "maxTimeout": 60_000, # This timeout is in miliseconds (60s)
- "session": self.session_id,
- } | kwargs
-
- self.request_count += 1
- msg = f"Waiting For Flaresolverr Response [{self.request_count}]"
- assert self.client_manager.manager.global_config.general.flaresolverr
- async with (
- self.request_lock,
- self.client_manager.manager.progress_manager.show_status_msg(msg),
- ):
- response = await client_session.post(
- self.client_manager.manager.global_config.general.flaresolverr / "v1",
- json=data,
- timeout=timeout,
- )
- json_obj: dict[str, Any] = await response.json()
-
- return json_obj
-
- async def _create_session(self) -> None:
- """Creates a permanet flaresolverr session."""
- session_id = "cyberdrop-dl"
- async with self.client_manager._new_session() as client_session:
- flaresolverr_resp = await self._make_request("sessions.create", client_session, session=session_id)
- status = flaresolverr_resp.get("status")
- if status != "ok":
- raise DDOSGuardError(message="Failed to create flaresolverr session")
- self.session_id = session_id
-
- async def _destroy_session(self) -> None:
- if self.session_id:
- async with self.client_manager._new_session() as client_session:
- await self._make_request("sessions.destroy", client_session, session=self.session_id)
- self.session_id = ""
-
- async def get(
- self,
- url: AbsoluteHttpURL,
- client_session: ClientSession,
- update_cookies: bool = True,
- ) -> tuple[BeautifulSoup | None, AbsoluteHttpURL]:
- """Returns the resolved URL from the given URL."""
- json_resp: dict = await self._request("request.get", client_session, url=url)
-
- try:
- fs_resp = FlaresolverrResponse.from_dict(json_resp)
- except (AttributeError, KeyError):
- raise DDOSGuardError(message="Invalid response from flaresolverr") from None
-
- if fs_resp.status != "ok":
- raise DDOSGuardError(message="Failed to resolve URL with flaresolverr")
-
- user_agent = client_session.headers["User-Agent"].strip()
- mismatch_msg = f"Config user_agent and flaresolverr user_agent do not match: \n Cyberdrop-DL: '{user_agent}'\n Flaresolverr: '{fs_resp.user_agent}'"
- if fs_resp.soup and (
- self.client_manager.check_ddos_guard(fs_resp.soup) or self.client_manager.check_cloudflare(fs_resp.soup)
- ):
- if not update_cookies:
- raise DDOSGuardError(message="Invalid response from flaresolverr")
- if fs_resp.user_agent != user_agent:
- raise DDOSGuardError(message=mismatch_msg)
-
- if update_cookies:
- if fs_resp.user_agent != user_agent:
- log(f"{mismatch_msg}\nResponse was successful but cookies will not be valid", 30)
-
- for cookie in fs_resp.cookies:
- self.client_manager.cookies.update_cookies(
- {cookie["name"]: cookie["value"]}, AbsoluteHttpURL(f"https://{cookie['domain']}")
- )
-
- return fs_resp.soup, fs_resp.url
+ await self.flaresolverr.close()
async def _set_dns_resolver(loop: asyncio.AbstractEventLoop | None = None) -> None:
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index fd76e5e82..8f6b46454 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -64,7 +64,7 @@ class ProgressManager:
self.status_message_task_id: TaskID = field(init=False)
@asynccontextmanager
- async def show_status_msg(self, msg: str | None) -> AsyncGenerator:
+ async def show_status_msg(self, msg: str | None) -> AsyncGenerator[None]:
try:
self.status_message.update(self.status_message_task_id, description=msg, visible=bool(msg))
yield
diff --git a/tests/test_flaresolverr.py b/tests/test_flaresolverr.py
new file mode 100644
index 000000000..12dd13938
--- /dev/null
+++ b/tests/test_flaresolverr.py
@@ -0,0 +1,60 @@
+import os
+
+import pytest
+
+from cyberdrop_dl.clients.flaresolverr import FlareSolverr, _Command
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.managers.manager import Manager
+from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
+
+ENV_NAME = "CDL_FLARESOLVERR"
+FLARESOLVER_URL = os.environ.get(ENV_NAME, "") # or "http://localhost:8191"
+
+pytestmark = pytest.mark.skipif(not FLARESOLVER_URL, reason=f"{ENV_NAME} environment variable is not set")
+
+
+async def flaresolverr(running_manager: Manager):
+ async with ScrapeMapper(running_manager) as scrape_mapper:
+ await scrape_mapper.run()
+ flare = running_manager.client_manager.flaresolverr
+ flare.url = AbsoluteHttpURL(FLARESOLVER_URL) / "v1"
+ yield flare
+
+
+def test_flaresolver(flaresolverr: FlareSolverr):
+ assert flaresolverr.url
+ assert flaresolverr._next_request_id() == 1
+ assert flaresolverr._next_request_id() == 2
+
+
+async def test_create_session(flaresolverr: FlareSolverr):
+ assert flaresolverr._session_id == ""
+ resp = await flaresolverr._request(_Command.CREATE_SESSION, session="cyberdrop-dl")
+ assert resp.ok
+ assert "Session created successfully" in resp.message or "Session already exists" in resp.message
+ assert resp.solution is None
+ resp = await flaresolverr._request(_Command.DESTROY_SESSION, session="cyberdrop-dl")
+ assert "The session has been removed" in resp.message
+
+
+async def test_create_session_methods(flaresolverr: FlareSolverr):
+ assert flaresolverr._session_id == ""
+ await flaresolverr._create_session()
+ assert flaresolverr._session_id == "cyberdrop-dl"
+ await flaresolverr._destroy_session()
+ assert flaresolverr._session_id == ""
+
+
+async def test_request_w_solution(flaresolverr: FlareSolverr):
+ url = AbsoluteHttpURL("https://google.com")
+ solution = await flaresolverr.request(url)
+ assert solution.status == 200
+ assert solution.url != url # should have www. as prefix
+ assert solution.user_agent
+ assert isinstance(solution.content, str)
+ assert isinstance(solution.user_agent, str)
+ assert "html" in solution.content
+ assert solution.cookies
+ for cookie in solution.cookies.values():
+ assert url.host in cookie["domain"]