Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: update flaresolver (#1278)
NTFSvolumed63b76fecf1b
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pynew file mode 100644index 000000000..d06cfd3a6--- /dev/null+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -0,0 +1,195 @@+from __future__ import annotations++import asyncio+import dataclasses+import itertools+import time+from functools import partial+from http.cookies import SimpleCookie+from typing import TYPE_CHECKING, Any++import aiohttp+from bs4 import BeautifulSoup+from multidict import CIMultiDict, CIMultiDictProxy++from cyberdrop_dl.compat import StrEnum+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.exceptions import DDOSGuardError+from cyberdrop_dl.utils.logger import log++if TYPE_CHECKING:+ from collections.abc import Callable++ from cyberdrop_dl.managers.manager import Manager+++class _Command(StrEnum):+ CREATE_SESSION = "sessions.create"+ DESTROY_SESSION = "sessions.destroy"+ LIST_SESSIONS = "sessions.list"++ GET_REQUEST = "request.get"+ POST_REQUEST = "request.post"++[email protected](frozen=True, slots=True)+class FlareSolverrSolution:+ content: str+ cookies: SimpleCookie+ headers: CIMultiDictProxy[str]+ url: AbsoluteHttpURL+ user_agent: str+ status: int++ @staticmethod+ def from_dict(solution: dict[str, Any]) -> FlareSolverrSolution:+ return FlareSolverrSolution(+ status=int(solution["status"]),+ cookies=_parse_cookies(solution.get("cookies") or []),+ user_agent=solution["userAgent"],+ content=solution["response"],+ url=AbsoluteHttpURL(solution["url"]),+ headers=CIMultiDictProxy(CIMultiDict(solution["headers"])),+ )++[email protected](frozen=True, slots=True)+class _FlareSolverrResponse:+ status: str+ message: str+ ok: bool+ solution: FlareSolverrSolution | None++ @staticmethod+ def from_dict(resp: dict[str, Any]) -> _FlareSolverrResponse:+ status, message = resp["status"], resp["message"]+ solution = FlareSolverrSolution.from_dict(sol) if (sol := resp.get("solution")) else None+ return _FlareSolverrResponse(status, message, status == "ok", solution)+++class FlareSolverr:+ """Class that handles communication with flaresolverr."""++ __slots__ = ("_next_request_id", "_request_lock", "_session_id", "_session_lock", "manager", "url")++ def __init__(self, manager: Manager) -> None:+ self.manager = manager+ self._session_id: str = ""+ self._session_lock, self._request_lock = asyncio.Lock(), asyncio.Lock()+ self._next_request_id: Callable[[], int] = partial(next, itertools.count(1))+ if manager.global_config.general.flaresolverr:+ self.url = manager.global_config.general.flaresolverr / "v1"+ else:+ self.url = None++ def __repr__(self):+ return f"{type(self).__name__}(url={self.url!r})"++ async def close(self):+ await self._destroy_session()++ async def request(self, url: AbsoluteHttpURL, data: Any = None) -> FlareSolverrSolution:+ # TODO: make this method return an abstract response++ invalid_response_error = DDOSGuardError("Invalid response from flaresolverr")+ try:+ if not self._session_id:+ async with self._session_lock:+ if not self._session_id:+ await self._create_session()++ resp = await self._request(+ _Command.POST_REQUEST if data else _Command.GET_REQUEST,+ url=str(url),+ data=data,+ session=self._session_id,+ )++ except (TypeError, KeyError) as e:+ raise invalid_response_error from e++ if not resp.ok:+ raise DDOSGuardError(f"Failed to resolve URL with flaresolverr. {resp.message}")++ if not resp.solution:+ raise invalid_response_error++ self.manager.client_manager.cookies.update_cookies(resp.solution.cookies)+ self._check_user_agent(resp.solution)+ return resp.solution++ def _check_user_agent(self, solution: FlareSolverrSolution) -> None:+ cdl_user_agent = self.manager.global_config.general.user_agent+ mismatch_ua_msg = (+ "Config user_agent and flaresolverr user_agent do not match:"+ f"\n Cyberdrop-DL: '{cdl_user_agent}'"+ f"\n Flaresolverr: '{solution.user_agent}'"+ )++ soup = BeautifulSoup(solution.content, "html.parser")+ if self.manager.client_manager.check_ddos_guard(soup) or self.manager.client_manager.check_cloudflare(soup):+ if solution.user_agent != cdl_user_agent:+ raise DDOSGuardError(mismatch_ua_msg)++ if solution.user_agent != cdl_user_agent:+ msg = f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid"+ log(msg, 30)++ async def _request(self, command: _Command, /, data: Any = None, **kwargs: Any) -> _FlareSolverrResponse:+ if not self.url:+ raise DDOSGuardError("Found DDoS challenge, but FlareSolverr is not configured")++ timeout = self.manager.global_config.rate_limiting_options._scrape_timeout+ if command is _Command.CREATE_SESSION:+ timeout = aiohttp.ClientTimeout(total=5 * 60, connect=60) # 5 minutes to create session++ # timeout in miliseconds (60s)+ playload = {"cmd": command, "maxTimeout": 60_000} | kwargs++ if data:+ assert command is _Command.POST_REQUEST+ playload["postData"] = aiohttp.FormData(data)().decode()++ async with (+ self._request_lock,+ self.manager.progress_manager.show_status_msg(+ f"Waiting For Flaresolverr Response [{self._next_request_id()}]"+ ),+ ):+ async with self.manager.client_manager._session.post(+ self.url,+ json=playload,+ timeout=timeout,+ ) as response:+ return _FlareSolverrResponse.from_dict(await response.json())++ async def _create_session(self) -> None:+ session_id = "cyberdrop-dl"+ kwargs = {}+ if proxy := self.manager.global_config.general.proxy:+ kwargs["proxy"] = {"url": str(proxy)}++ resp = await self._request(_Command.CREATE_SESSION, session=session_id, **kwargs)+ if not resp.ok:+ raise DDOSGuardError(f"Failed to create flaresolverr session: {resp.message}")+ self._session_id = session_id++ async def _destroy_session(self) -> None:+ if self._session_id:+ await self._request(_Command.DESTROY_SESSION)+ self._session_id = ""+++def _parse_cookies(cookies: list[dict[str, Any]]) -> SimpleCookie:+ simple_cookie = SimpleCookie()+ now = time.time()+ for cookie in cookies:+ name: str = cookie["name"]+ simple_cookie[name] = cookie["value"]+ morsel = simple_cookie[name]+ morsel["domain"] = cookie["domain"]+ morsel["path"] = cookie["path"]+ morsel["secure"] = "TRUE" if cookie["secure"] else ""+ if expires := cookie["expires"]:+ morsel["max-age"] = str(max(0, int(expires) - int(now)))+ return simple_cookiediff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.pyindex 53170d2df..d4e0a97f5 100644--- a/cyberdrop_dl/clients/response.py+++ b/cyberdrop_dl/clients/response.py@@ -1,10 +1,12 @@from __future__ import annotationsimport asyncio+import dataclassesfrom json import loads as json_loads-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Self-from aiohttp_client_cache.response import AnyResponse+from aiohttp import ClientResponse+from aiohttp_client_cache.response import CachedResponsefrom bs4 import BeautifulSoupfrom multidict import CIMultiDict, CIMultiDictProxy@@ -15,50 +17,106 @@ from cyberdrop_dl.utils.utilities import parse_urlif TYPE_CHECKING:from curl_cffi.requests.models import Response as CurlResponse+ from cyberdrop_dl.clients.flaresolverr import FlareSolverrSolution-class AbstractResponse:- """Class to represent common methods and attributes between aiohttp ClientResponse and a CurlResponse"""-- __slots__ = ("_read_lock", "_resp", "content_type", "headers", "location", "status", "url")- def __init__(self, response: AnyResponse | CurlResponse) -> None:- self._resp = response- self.content_type = (self._resp.headers.get("Content-Type") or "").lower()- if isinstance(response, AnyResponse):- self.status = response.status- self.headers = response.headers[email protected](slots=True, weakref_slot=True)+class AbstractResponse:+ """Class to represent common methods and attributes between:+ - `aiohttp.ClientResponse`+ - `curl_cffi.Response`+ - `FlareSolverrSolution`"""++ content_type: str+ status: int+ headers: CIMultiDictProxy[str]+ url: AbsoluteHttpURL+ location: AbsoluteHttpURL | None++ _resp: ClientResponse | CachedResponse | CurlResponse | None = None+ _text: str = ""+ _read_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)++ @classmethod+ def from_resp(cls, response: ClientResponse | CachedResponse | CurlResponse) -> Self:+ if isinstance(response, ClientResponse | CachedResponse):+ status = response.status+ headers = response.headerselse:- self.status = response.status_code- self.headers = CIMultiDictProxy(CIMultiDict({k: v or "" for k, v in response.headers}))-- self.url = AbsoluteHttpURL(response.url)- if location := response.headers.get("location"):- self.location = parse_url(location, self.url.origin(), trim=False)+ status = response.status_code+ headers = CIMultiDictProxy(CIMultiDict({k: v or "" for k, v in response.headers}))++ url = AbsoluteHttpURL(response.url)+ content_type, location = cls.parse_headers(url, headers)+ return cls(+ content_type=content_type,+ status=status,+ headers=headers,+ url=url,+ location=location,+ _resp=response,+ )++ @classmethod+ def from_flaresolverr(cls, solution: FlareSolverrSolution) -> Self:+ content_type, location = cls.parse_headers(solution.url, solution.headers)+ return cls(+ content_type=content_type,+ status=solution.status,+ headers=solution.headers,+ url=solution.url,+ location=location,+ _text=solution.content,+ )++ @staticmethod+ def parse_headers(url: AbsoluteHttpURL, headers: CIMultiDictProxy[str]):+ if location := headers.get("location"):+ location = parse_url(location, url.origin(), trim=False)else:- self.location = None-- self._read_lock = asyncio.Lock()+ location = None- async def text(self) -> str:- async with self._read_lock:- if isinstance(self._resp, AnyResponse):- return await self._resp.text()- return self._resp.text-- async def soup(self) -> BeautifulSoup:- if "text" in self.content_type or "html" in self.content_type:- return BeautifulSoup(await self.text(), "html.parser")+ content_type = (headers.get("Content-Type") or "").lower()+ return content_type, location- raise InvalidContentTypeError(message=f"Received {self.content_type}, was expecting text")+ async def text(self, encoding: str | None = None) -> str:+ if self._text:+ return self._text- async def json(self) -> Any:+ assert self._resp is not None+ async with self._read_lock:+ if self._text:+ return self._text+ if isinstance(self._resp, ClientResponse | CachedResponse):+ self._text = await self._resp.text(encoding)+ else:+ if encoding:+ self._resp.encoding = encoding+ self._text = await self._resp.atext()+ return self._text++ async def soup(self, encoding: str | None = None) -> BeautifulSoup:+ self._check_content_type("text", "html", expecting="HTML")+ return BeautifulSoup(await self.text(encoding), "html.parser")++ async def json(self, encoding: str | None = None, content_type: str | bool = True) -> Any:if self.status == 204:raise ScrapeError(204)- if "text/plain" in self.content_type or "json" in self.content_type:- return json_loads(await self.text())+ if content_type:+ if isinstance(content_type, str):+ check = (content_type,)+ else:+ check = ("text/plain", "json")++ self._check_content_type(*check, expecting="JSON")++ return json_loads(await self.text(encoding))- raise InvalidContentTypeError(message=f"Received {self.content_type}, was expecting JSON")+ def _check_content_type(self, *content_types, expecting: str):+ if not any(type_ in self.content_type for type_ in content_types):+ msg = f"Received {self.content_type}, was expecting {expecting}"+ raise InvalidContentTypeError(message=msg)def __repr__(self) -> str:- return f"<{self.__class__.__name__} [{self.status}] ({self.url})>"+ return f"<{self.__class__.__name__} [{self.status}] ({self.url!r})>"diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex 280839840..e00721edf 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -5,19 +5,20 @@ import contextlibfrom datetime import datetimefrom json import dumps as json_dumpsfrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, castimport cyberdrop_dl.constants as constantsfrom cyberdrop_dl.clients.response import AbstractResponse-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature-from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError+from cyberdrop_dl.exceptions import DDOSGuardErrorfrom cyberdrop_dl.utils.utilities import sanitize_filenameif TYPE_CHECKING:- from bs4 import BeautifulSoup+ from collections.abc import AsyncGenerator+from curl_cffi.requests.impersonate import BrowserTypeLiteralfrom curl_cffi.requests.session import HttpMethod+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.managers.client_manager import ClientManager@@ -39,8 +40,9 @@ class ScraperClient:await self.client_manager.manager.states.RUNNING.wait()yield+ @contextlib.asynccontextmanagerasync def _request(- self,+ self: object,url: AbsoluteHttpURL,/,method: HttpMethod = "GET",@@ -50,68 +52,85 @@ class ScraperClient:json: Any = None,cache_disabled: bool = False,**request_params: Any,- ) -> AbstractResponse:+ ) -> AsyncGenerator[AbstractResponse]:+ """+ Asynchronous context manager for HTTP requests.++ - If 'impersonate' is specified, uses curl_cffi for the request and updates cookies.+ - Otherwise, uses aiohttp with optional cache control.+ - Yield an AbstractResponse that wraps the underlying response with common methods.+ - On DDOSGuardError, retries the request using FlareSolverr.+ - Saves the HTML content to disk if the config option is enabled.+ - Closes underliying response on exit.+ """+ self = cast("ScraperClient", self)request_params["headers"] = self.client_manager._default_headers | (headers or {})request_params["data"] = datarequest_params["json"] = json+ async with self.__request_context(url, method, request_params, impersonate, cache_disabled) as resp:+ exc = None+ try:+ yield await self._check_response(resp, url)+ except Exception as e:+ exc = e+ raise+ finally:+ await self.write_soup_to_disk(url, resp, exc)++ @contextlib.asynccontextmanager+ async def __request_context(+ self,+ url: AbsoluteHttpURL,+ method: HttpMethod,+ request_params: dict[str, Any],+ impersonate: BrowserTypeLiteral | bool | None,+ cache_disabled: bool,+ ) -> AsyncGenerator[AbstractResponse]:if impersonate:self.client_manager.check_curl_cffi_is_available()if impersonate is True:impersonate = "chrome"request_params["impersonate"] = impersonate- response = await self.client_manager._curl_session.request(method, str(url), **request_params)+ curl_resp = await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)+ try:+ yield AbstractResponse.from_resp(curl_resp)+ curl_cookies = self.client_manager._curl_session.cookies.get_dict(url.host)+ self.client_manager.cookies.update_cookies(curl_cookies, url)+ finally:+ await curl_resp.aclose()+ return- else:- async with self.client_manager.cache_control(self.client_manager._session, disabled=cache_disabled):- response = await self.client_manager._session._request(method, url, **request_params)+ async with (+ self.client_manager.cache_control(self.client_manager._session, disabled=cache_disabled),+ self.client_manager._session.request(method, url, **request_params) as aio_resp,+ ):+ yield AbstractResponse.from_resp(aio_resp)- abs_resp = AbstractResponse(response)- exc = None+ async def _check_response(self, abs_resp: AbstractResponse, url: AbsoluteHttpURL, data: Any | None = None):+ """Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.++ Returns an AbstractResponse confirmed to not be a DDOS Guard page."""try:- await self.client_manager.check_http_status(response)- except (DDOSGuardError, DownloadError) as e:- exc = e- raise-- else:- if impersonate:- self.client_manager.cookies.update_cookies(- self.client_manager._curl_session.cookies.get_dict(url.host), url- )+ await self.client_manager.check_http_status(abs_resp)return abs_resp- finally:- self.client_manager.manager.task_group.create_task(self.write_soup_to_disk(url, abs_resp, exc))-- @copy_signature(_request)- async def _request_json(self, *args, **kwargs) -> Any:- return await (await self._request(*args, **kwargs)).json()+ except DDOSGuardError:+ flare_solution = await self.client_manager.flaresolverr.request(url, data)+ return AbstractResponse.from_flaresolverr(flare_solution)- @copy_signature(_request)- async def _request_soup(self, *args, **kwargs) -> BeautifulSoup:- return await (await self._request(*args, **kwargs)).soup()-- async def write_soup_to_disk(- self,- url: AbsoluteHttpURL,- response: AbstractResponse,- exc: Exception | None = None,- ):+ async def write_soup_to_disk(self, url: AbsoluteHttpURL, response: AbstractResponse, exc: Exception | None = None):"""Writes html to a file."""+if not self._save_pages_html:return- content: str = ""+ content: str = await response.text()try:- content: str = (await response.soup()).prettify(formatter="html")- except (UnicodeDecodeError, InvalidContentTypeError):+ content = cast("str", (await response.soup()).prettify(formatter="html"))+ except Exception:pass- content = content or await response.text()now = datetime.now()-- # The date is not really relevant in the filename and makes them longer, potencially truncating the URL part- # But it garanties the filename will be uniquelog_date = now.strftime(constants.LOGS_DATETIME_FORMAT)url_str = str(url)response_url_str = str(response.url)@@ -130,7 +149,11 @@ class ScraperClient:json_data = json_dumps(info, indent=4, ensure_ascii=False)text = f"<!-- cyberdrop-dl scraping result\n{json_data}\n-->\n{content}"- try:- await asyncio.to_thread(file_path.write_text, text, "utf8")- except OSError:- pass+ self.client_manager.manager.task_group.create_task(try_write(file_path, text))+++async def try_write(file: Path, content: str) -> None:+ try:+ await asyncio.to_thread(file.write_text, content, "utf8")+ except OSError:+ passdiff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 33a02c442..20d7c3257 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -224,9 +224,9 @@ class BunkrrCrawler(Crawler):link: AbsoluteHttpURL | None = Nonesoup: BeautifulSoup | None = Noneif is_stream_redirect(scrape_item.url):- response, soup = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)- scrape_item.url = AbsoluteHttpURL(response.url)- del response+ async with self.request(scrape_item.url) as resp:+ soup = await resp.soup()+ scrape_item.url = resp.urldatabase_url = scrape_item.url.with_host(self.DATABASE_PRIMARY_HOST)if await self.check_complete_from_referer(database_url):@@ -267,9 +267,7 @@ class BunkrrCrawler(Crawler):@error_handling_wrapperasync def reinforced_file(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-+ soup = await self.request_soup(scrape_item.url)title: str = css.select_one_get_text(soup, "h1")link: AbsoluteHttpURL | None = await self.get_download_url_from_api(scrape_item.url)if not link:@@ -328,9 +326,7 @@ class BunkrrCrawler(Crawler):api_url = STREAMING_API_ENTRYPOINT.with_host(self.known_good_host)data = json.dumps(data_dict)- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=data, headers=headers)-+ json_resp: dict[str, Any] = await self.request_json(api_url, "POST", data=data, headers=headers)api_response = ApiResponse(**json_resp)link_str = decrypt_api_response(api_response)link = self.parse_url(link_str)@@ -362,14 +358,10 @@ class BunkrrCrawler(Crawler):If we don't know a valid host but the response was successful, register the host as a valid host"""- async def get_soup(url: AbsoluteHttpURL) -> BeautifulSoup:- async with self.request_limiter:- return await self.client.get_soup(self.DOMAIN, url)-async def get_soup_no_error(url: AbsoluteHttpURL) -> BeautifulSoup | None:global known_bad_hoststry:- soup: BeautifulSoup = await get_soup(url)+ soup = await self.request_soup(url)except (ClientConnectorError, DDOSGuardError):known_bad_hosts.add(url.host)if not HOST_OPTIONS - known_bad_hosts:@@ -380,10 +372,10 @@ class BunkrrCrawler(Crawler):return soupif not is_root_domain(url):- return await get_soup(url)+ return await self.request_soup(url)elif self.known_good_host:- return await get_soup(url.with_host(self.known_good_host))+ return await self.request_soup(url.with_host(self.known_good_host))async with self.switch_host_locks[url.host]:if url.host not in known_bad_hosts:@@ -399,7 +391,7 @@ class BunkrrCrawler(Crawler):return soup# everything failed, do the request with the original URL to throw an exception- return await get_soup(url)+ return await self.request_soup(url)def get_part_next_to(url: AbsoluteHttpURL, part: str) -> str:diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 9f213baa8..86db0caf1 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -18,7 +18,8 @@ from aiolimiter import AsyncLimiterfrom yarl import URLfrom cyberdrop_dl import constants-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem+from cyberdrop_dl.clients.scraper_client import ScraperClient+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signaturefrom cyberdrop_dl.downloader.downloader import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.scraper import filters@@ -54,7 +55,7 @@ if TYPE_CHECKING:from bs4 import BeautifulSoup, Tagfrom rich.progress import TaskID- from cyberdrop_dl.clients.scraper_client import ScraperClient+ from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.managers.manager import Manager@@ -104,24 +105,26 @@ class Crawler(ABC):_RATE_LIMIT: ClassVar[tuple[float, float]] = 25, 1_DOWNLOAD_SLOTS: ClassVar[int | None] = None- if TYPE_CHECKING:- request = ScraperClient._request- request_json = ScraperClient._request_json- request_soup = ScraperClient._request_soup-- else:+ @copy_signature(ScraperClient._request)+ @contextlib.asynccontextmanager+ async def request(self, *args, **kwargs) -> AsyncGenerator[AbstractResponse]:+ async with self.client._limiter(self.DOMAIN), self.client._request(*args, **kwargs) as resp:+ yield resp- async def request(self, *args, **kwargs):- async with self.client._limiter(self.DOMAIN):- return await self.client._request(*args, **kwargs)+ @copy_signature(ScraperClient._request)+ async def request_json(self, *args, **kwargs) -> Any:+ async with self.request(*args, **kwargs) as resp:+ return await resp.json()- async def request_json(self, *args, **kwargs):- async with self.client._limiter(self.DOMAIN):- return await self.client._request_json(*args, **kwargs)+ @copy_signature(ScraperClient._request)+ async def request_soup(self, *args, **kwargs) -> BeautifulSoup:+ async with self.request(*args, **kwargs) as resp:+ return await resp.soup()- async def request_soup(self, *args, **kwargs):- async with self.client._limiter(self.DOMAIN):- return await self.client._request_soup(*args, **kwargs)+ @copy_signature(ScraperClient._request)+ async def request_text(self, *args, **kwargs) -> str:+ async with self.request(*args, **kwargs) as resp:+ return await resp.text()@finaldef __init__(self, manager: Manager) -> None:@@ -654,8 +657,8 @@ class Crawler(ABC):log(msg, bug=True)async def _get_redirect_url(self, url: AbsoluteHttpURL):- head = await self.request(url, method="HEAD")- return head.location or url+ async with self.request(url, method="HEAD") as head:+ return head.location or url@staticmethoddef register_cache_filter(@@ -688,7 +691,7 @@ class Crawler(ABC):return m3u8.RenditionGroup(await self._get_m3u8(url, headers))async def _get_m3u8(self, url: AbsoluteHttpURL, /, headers: dict[str, str] | None = None) -> m3u8.M3U8:- content = await (await self.request(url, headers=headers)).text()+ content = await self.request_text(url, headers=headers)return m3u8.M3U8(content, url.parent)def create_custom_filename(diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 65f4f28f5..46d6e1f1d 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -5,7 +5,6 @@ import contextlibimport sslfrom base64 import b64encodefrom collections import defaultdict-from dataclasses import dataclassfrom http import HTTPStatusfrom typing import TYPE_CHECKING, Any, Literal, Self, overload@@ -21,6 +20,7 @@ from videoprops import get_audio_properties, get_video_propertiesfrom cyberdrop_dl import constants, envfrom cyberdrop_dl.clients.download_client import DownloadClient+from cyberdrop_dl.clients.flaresolverr import FlareSolverrfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem@@ -164,7 +164,7 @@ class ClientManager:self.scraper_client = ScraperClient(self)self.speed_limiter = DownloadSpeedLimiter(self.rate_limiting_options.download_speed_limit)self.download_client = DownloadClient(manager, self)- self.flaresolverr = Flaresolverr(self)+ self.flaresolverr = FlareSolverr(manager)self.file_locks = FileLocksVault()self._default_headers = {"user-agent": self.manager.global_config.general.user_agent}self.reddit_session: CachedSession@@ -378,7 +378,7 @@ class ClientManager:If the response is successful and has valid html, returns soup"""if not isinstance(response, AbstractResponse):- response = AbstractResponse(response)+ response = AbstractResponse.from_resp(response)message = None@@ -502,139 +502,7 @@ class ClientManager:return min_audio_duration <= media_item.duration <= max_audio_durationasync def close(self) -> None:- await self.flaresolverr._destroy_session()---@dataclass(frozen=True, slots=True)-class FlaresolverrResponse:- status: str- cookies: dict- user_agent: str- soup: BeautifulSoup | None- url: AbsoluteHttpURL-- @classmethod- def from_dict(cls, flaresolverr_resp: dict) -> FlaresolverrResponse:- status = flaresolverr_resp["status"]- solution: dict = flaresolverr_resp["solution"]- response = solution["response"]- user_agent = solution["userAgent"].strip()- url_str: str = solution["url"]- cookies: dict = solution.get("cookies") or {}- soup = BeautifulSoup(response, "html.parser") if response else None- url = AbsoluteHttpURL(url_str)- return cls(status, cookies, user_agent, soup, url)---class Flaresolverr:- """Class that handles communication with flaresolverr."""-- def __init__(self, client_manager: ClientManager) -> None:- self.client_manager = client_manager- self.enabled = bool(client_manager.manager.global_config.general.flaresolverr)- self.session_id: str = ""- self.session_lock = asyncio.Lock()- self.request_lock = asyncio.Lock()- self.request_count = 0-- async def _request(- self,- command: str,- client_session: ClientSession,- **kwargs,- ) -> dict:- """Base request function to call flaresolverr."""- if not self.enabled:- raise DDOSGuardError(message="FlareSolverr is not configured")- async with self.session_lock:- if not (self.session_id or kwargs.get("session")):- await self._create_session()- return await self._make_request(command, client_session, **kwargs)-- async def _make_request(self, command: str, client_session: ClientSession, **kwargs) -> dict[str, Any]:- timeout = self.client_manager.manager.global_config.rate_limiting_options._scrape_timeout- if command == "sessions.create":- timeout = aiohttp.ClientTimeout(total=5 * 60, connect=60) # 5 minutes to create session-- for key, value in kwargs.items():- if isinstance(value, AbsoluteHttpURL):- kwargs[key] = str(value)-- data = {- "cmd": command,- "maxTimeout": 60_000, # This timeout is in miliseconds (60s)- "session": self.session_id,- } | kwargs-- self.request_count += 1- msg = f"Waiting For Flaresolverr Response [{self.request_count}]"- assert self.client_manager.manager.global_config.general.flaresolverr- async with (- self.request_lock,- self.client_manager.manager.progress_manager.show_status_msg(msg),- ):- response = await client_session.post(- self.client_manager.manager.global_config.general.flaresolverr / "v1",- json=data,- timeout=timeout,- )- json_obj: dict[str, Any] = await response.json()-- return json_obj-- async def _create_session(self) -> None:- """Creates a permanet flaresolverr session."""- session_id = "cyberdrop-dl"- async with self.client_manager._new_session() as client_session:- flaresolverr_resp = await self._make_request("sessions.create", client_session, session=session_id)- status = flaresolverr_resp.get("status")- if status != "ok":- raise DDOSGuardError(message="Failed to create flaresolverr session")- self.session_id = session_id-- async def _destroy_session(self) -> None:- if self.session_id:- async with self.client_manager._new_session() as client_session:- await self._make_request("sessions.destroy", client_session, session=self.session_id)- self.session_id = ""-- async def get(- self,- url: AbsoluteHttpURL,- client_session: ClientSession,- update_cookies: bool = True,- ) -> tuple[BeautifulSoup | None, AbsoluteHttpURL]:- """Returns the resolved URL from the given URL."""- json_resp: dict = await self._request("request.get", client_session, url=url)-- try:- fs_resp = FlaresolverrResponse.from_dict(json_resp)- except (AttributeError, KeyError):- raise DDOSGuardError(message="Invalid response from flaresolverr") from None-- if fs_resp.status != "ok":- raise DDOSGuardError(message="Failed to resolve URL with flaresolverr")-- user_agent = client_session.headers["User-Agent"].strip()- mismatch_msg = f"Config user_agent and flaresolverr user_agent do not match: \n Cyberdrop-DL: '{user_agent}'\n Flaresolverr: '{fs_resp.user_agent}'"- if fs_resp.soup and (- self.client_manager.check_ddos_guard(fs_resp.soup) or self.client_manager.check_cloudflare(fs_resp.soup)- ):- if not update_cookies:- raise DDOSGuardError(message="Invalid response from flaresolverr")- if fs_resp.user_agent != user_agent:- raise DDOSGuardError(message=mismatch_msg)-- if update_cookies:- if fs_resp.user_agent != user_agent:- log(f"{mismatch_msg}\nResponse was successful but cookies will not be valid", 30)-- for cookie in fs_resp.cookies:- self.client_manager.cookies.update_cookies(- {cookie["name"]: cookie["value"]}, AbsoluteHttpURL(f"https://{cookie['domain']}")- )-- return fs_resp.soup, fs_resp.url+ await self.flaresolverr.close()async def _set_dns_resolver(loop: asyncio.AbstractEventLoop | None = None) -> None:diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex fd76e5e82..8f6b46454 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -64,7 +64,7 @@ class ProgressManager:self.status_message_task_id: TaskID = field(init=False)@asynccontextmanager- async def show_status_msg(self, msg: str | None) -> AsyncGenerator:+ async def show_status_msg(self, msg: str | None) -> AsyncGenerator[None]:try:self.status_message.update(self.status_message_task_id, description=msg, visible=bool(msg))yielddiff --git a/tests/test_flaresolverr.py b/tests/test_flaresolverr.pynew file mode 100644index 000000000..12dd13938--- /dev/null+++ b/tests/test_flaresolverr.py@@ -0,0 +1,60 @@+import os++import pytest++from cyberdrop_dl.clients.flaresolverr import FlareSolverr, _Command+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.managers.manager import Manager+from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper++ENV_NAME = "CDL_FLARESOLVERR"+FLARESOLVER_URL = os.environ.get(ENV_NAME, "") # or "http://localhost:8191"++pytestmark = pytest.mark.skipif(not FLARESOLVER_URL, reason=f"{ENV_NAME} environment variable is not set")+++async def flaresolverr(running_manager: Manager):+ async with ScrapeMapper(running_manager) as scrape_mapper:+ await scrape_mapper.run()+ flare = running_manager.client_manager.flaresolverr+ flare.url = AbsoluteHttpURL(FLARESOLVER_URL) / "v1"+ yield flare+++def test_flaresolver(flaresolverr: FlareSolverr):+ assert flaresolverr.url+ assert flaresolverr._next_request_id() == 1+ assert flaresolverr._next_request_id() == 2+++async def test_create_session(flaresolverr: FlareSolverr):+ assert flaresolverr._session_id == ""+ resp = await flaresolverr._request(_Command.CREATE_SESSION, session="cyberdrop-dl")+ assert resp.ok+ assert "Session created successfully" in resp.message or "Session already exists" in resp.message+ assert resp.solution is None+ resp = await flaresolverr._request(_Command.DESTROY_SESSION, session="cyberdrop-dl")+ assert "The session has been removed" in resp.message+++async def test_create_session_methods(flaresolverr: FlareSolverr):+ assert flaresolverr._session_id == ""+ await flaresolverr._create_session()+ assert flaresolverr._session_id == "cyberdrop-dl"+ await flaresolverr._destroy_session()+ assert flaresolverr._session_id == ""+++async def test_request_w_solution(flaresolverr: FlareSolverr):+ url = AbsoluteHttpURL("https://google.com")+ solution = await flaresolverr.request(url)+ assert solution.status == 200+ assert solution.url != url # should have www. as prefix+ assert solution.user_agent+ assert isinstance(solution.content, str)+ assert isinstance(solution.user_agent, str)+ assert "html" in solution.content+ assert solution.cookies+ for cookie in solution.cookies.values():+ assert url.host in cookie["domain"]