Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: unify http clients (part 1) (#69)
NTFSvolumeb6f149407f1b
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex e69de29bb..2638e5141 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -0,0 +1,278 @@+from __future__ import annotations++import asyncio+import contextlib+import dataclasses+import logging+import time+import uuid+from pathlib import Path+from typing import TYPE_CHECKING, Any, ClassVar, Self, cast++from multidict import CIMultiDict++from cyberdrop_dl import constants, ddos_guard, signature+from cyberdrop_dl.clients.response import AbstractResponse+from cyberdrop_dl.exceptions import DDOSGuardError+from cyberdrop_dl.utils.cookie_management import make_simple_cookie+from cyberdrop_dl.utils.utilities import sanitize_filename++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Mapping++ from bs4 import BeautifulSoup+ from curl_cffi.requests.impersonate import BrowserTypeLiteral+ from curl_cffi.requests.session import HttpMethod++ from cyberdrop_dl.clients import flaresolverr+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+ from cyberdrop_dl.managers.client_manager import ClientManager+++logger = logging.getLogger(__name__)++[email protected](slots=True)+class HTTPClient:+ client_manager: ClientManager+ _save_responses_to_disk: bool+ _responses_folder: Path++ @classmethod+ def from_client(cls, client_manager: ClientManager) -> Self:+ return cls(+ client_manager,+ client_manager.manager.config_manager.settings_data.files.save_pages_html,+ client_manager.manager.path_manager.pages_folder,+ )++ @property+ def _default_headers(self) -> dict[Any, Any]:+ return {}++ @contextlib.asynccontextmanager+ async def _limiter(self, domain: str) -> AsyncGenerator[None]:+ with self.client_manager.request_context(domain):+ domain_limiter = self.client_manager.get_rate_limiter(domain)+ async with self.client_manager.global_rate_limiter, domain_limiter:+ await self.client_manager.manager.states.RUNNING.wait()+ yield++ def _prepare_headers(self, headers: Mapping[str, str] | None = None) -> CIMultiDict[str]:+ """Add default headers and transform it to CIMultiDict"""+ combined = CIMultiDict(self._default_headers)+ if headers:+ headers = CIMultiDict(headers)+ new: set[str] = set()+ for key, value in headers.items():+ if key in new:+ combined.add(key, value)+ else:+ combined[key] = value+ new.add(key)+ return combined++ @contextlib.asynccontextmanager+ async def request(+ self: object,+ url: AbsoluteHttpURL,+ /,+ method: HttpMethod = "GET",+ headers: Mapping[str, str] | None = None,+ impersonate: BrowserTypeLiteral | bool | None = None,+ data: Any = None,+ json: Any = None,+ cache_disabled: bool = False,+ **request_params: Any,+ ) -> AsyncGenerator[AbstractResponse[Any]]:+ self = cast("HTTPClient", self)+ request_params["headers"] = headers = self._prepare_headers(headers)+ request_params["data"] = data+ request_params["json"] = json++ if method == "GET" and (data or json):+ method = "POST"++ impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate+ if impersonate:+ self.client_manager.check_curl_cffi_is_available()+ if impersonate is True:+ impersonate = "chrome"+ request_params["impersonate"] = impersonate++ else:+ _ = headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)+ request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)++ request_id = str(uuid.uuid4())+ logger.debug("Starting {} request to {} [id={}]\n{}", method, url, request_id, request_params)++ async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:+ exc = None+ try:+ yield await self._check_response(resp, url)+ except Exception as e:+ exc = e+ raise+ finally:+ logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)+ if self._save_responses_to_disk:+ _ = self.client_manager.manager.task_group.create_task(+ asyncio.to_thread(+ _write_resp_to_disk,+ self._responses_folder,+ url,+ resp,+ exc,+ )+ )++ def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:+ """+ Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL++ This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session++ The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped+ """+ now = time.time()+ for cookie in self.client_manager._curl_session.cookies.jar:+ simple_cookie = make_simple_cookie(cookie, now)+ self.client_manager.cookies.update_cookies(simple_cookie, url)++ @contextlib.asynccontextmanager+ async def __request(+ self,+ url: AbsoluteHttpURL,+ method: HttpMethod,+ request_params: Mapping[str, Any],+ *,+ impersonate: bool,+ ) -> AsyncGenerator[AbstractResponse[Any]]:++ if impersonate:+ async with contextlib.aclosing(+ await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)+ ) as curl_resp:+ yield AbstractResponse.create(curl_resp)+ self.__sync_session_cookies(url)++ return++ async with (+ self.client_manager._session.request(method, url, **request_params) as aio_resp,+ ):+ yield AbstractResponse.create(aio_resp)++ async def _check_response(self, abs_resp: AbstractResponse[Any], url: AbsoluteHttpURL, data: Any | None = None):+ """Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.++ Returns an AbstractResponse confirmed to not be a DDOS Guard page."""+ try:+ await self.client_manager.check_http_status(abs_resp)+ return abs_resp+ except DDOSGuardError as e:+ if not (flare := self.client_manager.flaresolverr):+ raise++ try:+ solution = await flare.request(url, data)+ except RuntimeError:+ raise e from None++ self.client_manager.cookies.update_cookies(solution.cookies)+ await _check_flaresolverr_resp(self.client_manager.manager.global_config.general.user_agent, solution)+ return AbstractResponse.create(solution)+++async def _check_flaresolverr_resp(cdl_user_agent: str, solution: flaresolverr.Solution) -> None:+ mismatch_ua_msg = (+ "Config user_agent and flaresolverr user_agent do not match:"+ f"\n Cyberdrop-DL: '{cdl_user_agent}'"+ f"\n Flaresolverr: '{solution.user_agent}'"+ )++ try:+ await ddos_guard.check(solution.content)+ except DDOSGuardError:+ if solution.user_agent != cdl_user_agent:+ raise DDOSGuardError(mismatch_ua_msg) from None++ if solution.user_agent != cdl_user_agent:+ logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")+++def _write_resp_to_disk(+ folder: Path,+ url: AbsoluteHttpURL,+ response: AbstractResponse[Any],+ exc: Exception | None = None,+) -> None:++ max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10++ log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)+ path_safe_url = sanitize_filename(Path(str(url)).as_posix().replace("/", "-"))+ filename = f"{path_safe_url[:max_stem_len]}_{log_date}.html"+ file = folder / filename+ content = response.create_report(exc)+ try:+ _ = file.write_text(content, "utf8")+ except OSError:+ pass+++class HTTPClientProxy:+ DOMAIN: ClassVar[str]+ _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None+ client: HTTPClient # pyright: ignore[reportUninitializedInstanceVariable]++ @classmethod+ def __json_resp_check__(cls, json_resp: Any, resp: AbstractResponse[Any], /) -> None:+ """Custom check for JSON responses.++ This method is called automatically by the `HttpClient` when a JSON response is received from `cls.DOMAIN`+ and it was **NOT** successful (`4xx` or `5xx` HTTP code).++ Override this method in subclasses to raise a custom `ScrapeError` instead of the default HTTP error++ Example:+ ```python+ if isinstance(json, dict) and json.get("status") == "error":+ raise ScrapeError(422, f"API error: {json['message']}")+ ```++ IMPORTANT:+ Cases were the response **IS** successful (200, OK) but the JSON indicates an error+ should be handled by the crawler itself+ """++ @signature.copy(HTTPClient.request)+ @contextlib.asynccontextmanager+ async def request(+ self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs+ ) -> AsyncGenerator[AbstractResponse[Any]]:+ if impersonate is None:+ impersonate = self._IMPERSONATE++ with self.client.client_manager.set_json_checker(self.__json_resp_check__):+ async with (+ self.client._limiter(self.DOMAIN),+ self.client.request(*args, impersonate=impersonate, **kwargs) as resp,+ ):+ yield resp++ @signature.copy(request)+ async def request_json(self, *args, **kwargs) -> Any:+ async with self.request(*args, **kwargs) as resp:+ return await resp.json()++ @signature.copy(request)+ async def request_soup(self, *args, **kwargs) -> BeautifulSoup:+ async with self.request(*args, **kwargs) as resp:+ return await resp.soup()++ @signature.copy(request)+ async def request_text(self, *args, **kwargs) -> str:+ async with self.request(*args, **kwargs) as resp:+ return await resp.text()diff --git a/cyberdrop_dl/clients/jdownloader.py b/cyberdrop_dl/clients/jdownloader.pynew file mode 100644index 000000000..6b3033426--- /dev/null+++ b/cyberdrop_dl/clients/jdownloader.py@@ -0,0 +1,122 @@+from __future__ import annotations++import asyncio+import contextlib+import dataclasses+from typing import TYPE_CHECKING, Self++from myjdapi import myjdapi++from cyberdrop_dl.exceptions import JDownloaderError++if TYPE_CHECKING:+ from collections.abc import Generator+ from pathlib import Path++ from myjdapi.myjdapi import Jddevice++ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+ from cyberdrop_dl.managers.manager import Manager++[email protected](frozen=True, slots=True)+class Config:+ enabled: bool+ username: str+ password: str+ device: str+ download_dir: Path+ autostart: bool+ whitelist: tuple[str, ...]++ @staticmethod+ def from_manager(manager: Manager) -> Config:+ download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.path_manager.download_folder+ return Config(+ enabled=manager.config.runtime_options.send_unsupported_to_jdownloader,+ device=manager.auth_config.jdownloader.device,+ username=manager.auth_config.jdownloader.username,+ password=manager.auth_config.jdownloader.password,+ download_dir=download_dir.resolve(),+ autostart=manager.config.runtime_options.jdownloader_autostart,+ whitelist=tuple(manager.config.runtime_options.jdownloader_whitelist),+ )++[email protected](slots=True)+class JDownloader:+ """Class that handles connecting and sending links to JDownloader."""++ config: Config+ _enabled: bool = dataclasses.field(init=False)+ _device: Jddevice | None = dataclasses.field(default=None, init=False)++ @classmethod+ def from_manager(cls, manager: Manager, /) -> Self:+ return cls(Config.from_manager(manager))++ @property+ def enabled(self) -> bool:+ return self._enabled++ def __post_init__(self) -> None:+ self._enabled = self.config.enabled++ def is_enabled_for(self, url: AbsoluteHttpURL) -> bool:+ if not self.enabled:+ return False+ if not self.config.whitelist:+ return True++ return any(domain in url.host for domain in self.config.whitelist)++ @contextlib.contextmanager+ def _wrap_errors(self) -> Generator[None]:+ try:+ yield+ except myjdapi.MYJDDeviceNotFoundException:+ raise JDownloaderError(f"Device not found ({self.config.device})") from None+ except myjdapi.MYJDApiException as e:+ raise JDownloaderError(str(e)) from e++ async def _connect(self) -> None:+ if not self._enabled or self._device is not None:+ return++ if not all((self.config.username, self.config.password, self.config.device)):+ raise JDownloaderError("JDownloader credentials were not provided")++ with self._wrap_errors():+ api = myjdapi.Myjdapi()+ api.set_app_key("CYBERDROP-DL")+ _ = await asyncio.to_thread(api.connect, self.config.username, self.config.password)+ self._device = api.get_device(self.config.device)++ async def connect(self) -> None:+ try:+ return await self._connect()+ except Exception:+ self._enabled = False+ raise++ async def send(self, url: AbsoluteHttpURL, title: str, download_path: Path | None = None) -> None:+ """Sends links to JDownloader."""++ assert self._device is not None and self.enabled+ with self._wrap_errors():+ download_folder = self.config.download_dir+ if download_path:+ download_folder = download_folder / download_path++ await asyncio.to_thread(+ self._device.linkgrabber.add_links,+ [+ {+ "autostart": self.config.autostart,+ "links": str(url),+ "packageName": title if title else "Cyberdrop-DL",+ "destinationFolder": str(download_folder),+ "overwritePackagizerRules": True,+ },+ ],+ )diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pydeleted file mode 100644index 3f26afb73..000000000--- a/cyberdrop_dl/clients/scraper_client.py+++ /dev/null@@ -1,204 +0,0 @@-from __future__ import annotations--import asyncio-import contextlib-import logging-import time-import uuid-from pathlib import Path-from typing import TYPE_CHECKING, Any, cast--from cyberdrop_dl import constants, ddos_guard-from cyberdrop_dl.clients.response import AbstractResponse-from cyberdrop_dl.exceptions import DDOSGuardError-from cyberdrop_dl.utils.cookie_management import make_simple_cookie-from cyberdrop_dl.utils.utilities import sanitize_filename--if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Mapping-- from curl_cffi.requests.impersonate import BrowserTypeLiteral- from curl_cffi.requests.session import HttpMethod-- from cyberdrop_dl.clients import flaresolverr- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL- from cyberdrop_dl.managers.client_manager import ClientManager--logger = logging.getLogger(__name__)---class ScraperClient:- """AIOHTTP / CURL operations for scraping."""-- def __init__(self, client_manager: ClientManager) -> None:- self.client_manager = client_manager- self._save_pages_html = client_manager.manager.config_manager.settings_data.files.save_pages_html- self._resp_folder = self.client_manager.manager.path_manager.pages_folder-- @contextlib.asynccontextmanager- async def _limiter(self, domain: str) -> AsyncGenerator[None]:- with self.client_manager.request_context(domain):- domain_limiter = self.client_manager.get_rate_limiter(domain)- async with self.client_manager.global_rate_limiter, domain_limiter:- await self.client_manager.manager.states.RUNNING.wait()- yield-- @contextlib.asynccontextmanager- async def _request(- self: object,- url: AbsoluteHttpURL,- /,- method: HttpMethod = "GET",- headers: dict[str, str] | None = None,- impersonate: BrowserTypeLiteral | bool | None = None,- data: Any = None,- json: Any = None,- cache_disabled: bool = False,- **request_params: Any,- ) -> AsyncGenerator[AbstractResponse[Any]]:- """- Asynchronous context manager for HTTP requests.-- - If 'impersonate' is specified, uses curl_cffi for the request and updates cookies.- - Otherwise, uses aiohttp with optional cache control.- - Yield an AbstractResponse that wraps the underlying response with common methods.- - On DDOSGuardError, retries the request using FlareSolverr.- - Saves the HTML content to disk if the config option is enabled.- - Closes underliying response on exit.- """- self = cast("ScraperClient", self)- request_params["headers"] = headers = headers or {}- request_params["data"] = data- request_params["json"] = json-- if method == "GET" and (data or json):- method = "POST"-- impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate- if impersonate:- self.client_manager.check_curl_cffi_is_available()- if impersonate is True:- impersonate = "chrome"- request_params["impersonate"] = impersonate-- else:- _ = headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)- request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)-- request_id = str(uuid.uuid4())- logger.debug("Starting {} request to {} [id={}]\n{}", method, url, request_id, request_params)-- async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:- exc = None- try:- yield await self._check_response(resp, url)- except Exception as e:- exc = e- raise- finally:- logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)- if self._save_pages_html:- _ = self.client_manager.manager.task_group.create_task(- asyncio.to_thread(- _write_resp_to_disk,- self._resp_folder,- url,- resp,- exc,- )- )-- def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:- """- Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL-- This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session-- The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped- """- now = time.time()- for cookie in self.client_manager._curl_session.cookies.jar:- simple_cookie = make_simple_cookie(cookie, now)- self.client_manager.cookies.update_cookies(simple_cookie, url)-- @contextlib.asynccontextmanager- async def __request(- self,- url: AbsoluteHttpURL,- method: HttpMethod,- request_params: Mapping[str, Any],- *,- impersonate: bool,- ) -> AsyncGenerator[AbstractResponse[Any]]:-- if impersonate:- async with contextlib.aclosing(- await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)- ) as curl_resp:- yield AbstractResponse.create(curl_resp)- self.__sync_session_cookies(url)-- return-- async with (- self.client_manager._session.request(method, url, **request_params) as aio_resp,- ):- yield AbstractResponse.create(aio_resp)-- async def _check_response(self, abs_resp: AbstractResponse[Any], url: AbsoluteHttpURL, data: Any | None = None):- """Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.-- Returns an AbstractResponse confirmed to not be a DDOS Guard page."""- try:- await self.client_manager.check_http_status(abs_resp)- return abs_resp- except DDOSGuardError as e:- if not (flare := self.client_manager.flaresolverr):- raise-- try:- solution = await flare.request(url, data)- except RuntimeError:- raise e from None-- self.client_manager.cookies.update_cookies(solution.cookies)- await _check_flaresolverr_resp(self.client_manager.manager.global_config.general.user_agent, solution)- return AbstractResponse.create(solution)---async def _check_flaresolverr_resp(cdl_user_agent: str, solution: flaresolverr.Solution) -> None:- mismatch_ua_msg = (- "Config user_agent and flaresolverr user_agent do not match:"- f"\n Cyberdrop-DL: '{cdl_user_agent}'"- f"\n Flaresolverr: '{solution.user_agent}'"- )-- try:- await ddos_guard.check(solution.content)- except DDOSGuardError:- if solution.user_agent != cdl_user_agent:- raise DDOSGuardError(mismatch_ua_msg) from None-- if solution.user_agent != cdl_user_agent:- logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")---def _write_resp_to_disk(- folder: Path,- url: AbsoluteHttpURL,- response: AbstractResponse[Any],- exc: Exception | None = None,-) -> None:-- max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10-- log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)- url_str = str(url)- clean_url = sanitize_filename(Path(url_str).as_posix().replace("/", "-"))- filename = f"{clean_url[:max_stem_len]}_{log_date}.html"- file = folder / filename- content = response.create_report(exc)- try:- _ = file.write_text(content, "utf8")- except OSError:- passdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex a67e05c07..68f1e2bd9 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -32,8 +32,8 @@ import yarlfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl import constants, signature-from cyberdrop_dl.clients.scraper_client import ScraperClient+from cyberdrop_dl import constants+from cyberdrop_dl.clients import HTTPClient, HTTPClientProxyfrom cyberdrop_dl.data_structures.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloader@@ -60,10 +60,8 @@ if TYPE_CHECKING:from http.cookies import BaseCookiefrom bs4 import BeautifulSoup, Tag- from curl_cffi.requests.impersonate import BrowserTypeLiteralfrom rich.progress import TaskID- from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.managers.manager import Managerlogger = logging.getLogger(__name__)@@ -154,63 +152,7 @@ class Registry:cls._import(module_info.name)-class HTTPClientProxy:- DOMAIN: ClassVar[str]- _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None- client: ScraperClient # pyright: ignore[reportUninitializedInstanceVariable]-- @classmethod- def __json_resp_check__(cls, json_resp: Any, resp: AbstractResponse[Any], /) -> None:- """Custom check for JSON responses.-- This method is called automatically by the `HttpClient` when a JSON response is received from `cls.DOMAIN`- and it was **NOT** successful (`4xx` or `5xx` HTTP code).-- Override this method in subclasses to raise a custom `ScrapeError` instead of the default HTTP error-- Example:- ```python- if isinstance(json, dict) and json.get("status") == "error":- raise ScrapeError(422, f"API error: {json['message']}")- ```-- IMPORTANT:- Cases were the response **IS** successful (200, OK) but the JSON indicates an error- should be handled by the crawler itself- """-- @signature.copy(ScraperClient._request)- @contextlib.asynccontextmanager- async def request(- self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs- ) -> AsyncGenerator[AbstractResponse[Any]]:- if impersonate is None:- impersonate = self._IMPERSONATE-- with self.client.client_manager.set_json_checker(self.__json_resp_check__):- async with (- self.client._limiter(self.DOMAIN),- self.client._request(*args, impersonate=impersonate, **kwargs) as resp,- ):- yield resp-- @signature.copy(request)- async def request_json(self, *args, **kwargs) -> Any:- async with self.request(*args, **kwargs) as resp:- return await resp.json()-- @signature.copy(request)- async def request_soup(self, *args, **kwargs) -> BeautifulSoup:- async with self.request(*args, **kwargs) as resp:- return await resp.soup()-- @signature.copy(request)- async def request_text(self, *args, **kwargs) -> str:- async with self.request(*args, **kwargs) as resp:- return await resp.text()---class Crawler(ABC, HTTPClientProxy):+class Crawler(HTTPClientProxy, ABC):OLD_DOMAINS: ClassVar[tuple[str, ...]] = ()SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ()SUPPORTED_PATHS: ClassVar[SupportedPaths] = {}@@ -235,7 +177,7 @@ class Crawler(ABC, HTTPClientProxy):def __init__(self, manager: Manager) -> None:self.manager = managerself.downloader: Downloader = field(init=False)- self.client: ScraperClient = field(init=False)+ self.client: HTTPClient = field(init=False)self.startup_lock = asyncio.Lock()self.ready: bool = Falseself.disabled: bool = Falsediff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex b888899a6..897ebce0e 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -206,6 +206,10 @@ class LoginError(CDLBaseError):class JDownloaderError(CDLBaseError):"""This error will be thrown for any Jdownloader error."""+ def __init__(self, message: str | None = None) -> None:+ ui_failure = "JDownloader Error"+ super().__init__(ui_failure, message=message)+class InvalidYamlError(CDLBaseError):def __init__(self, file: Path, e: YAMLError) -> None:diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex ddb2678e7..a9b10c073 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -19,10 +19,10 @@ from aiolimiter import AsyncLimiterfrom cyberdrop_dl import constants, ddos_guard, envfrom cyberdrop_dl.aio import WeakAsyncLocks+from cyberdrop_dl.clients import HTTPClientfrom cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.clients.flaresolverr import FlareSolverrClientfrom cyberdrop_dl.clients.response import AbstractResponse-from cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrorsfrom cyberdrop_dl.ffmpeg import probe@@ -136,7 +136,7 @@ class ClientManager:self.download_slots: dict[str, int] = {}self.global_rate_limiter = AsyncLimiter(self.rate_limiting_options.rate_limit, 1)self.global_download_slots = asyncio.Semaphore(self.rate_limiting_options.max_simultaneous_downloads)- self.scraper_client = ScraperClient(self)+ self.scraper_client = HTTPClient.from_client(self)self.speed_limiter = DownloadSpeedLimiter(self.rate_limiting_options.download_speed_limit)self.download_client = DownloadClient(manager, self)self._flaresolverr: FlareSolverrClient | None = Nonediff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.pydeleted file mode 100644index c3ad2e509..000000000--- a/cyberdrop_dl/scraper/jdownloader.py+++ /dev/null@@ -1,100 +0,0 @@-from __future__ import annotations--import dataclasses-import logging-from typing import TYPE_CHECKING--from myjdapi import myjdapi--from cyberdrop_dl.exceptions import JDownloaderError--logger = logging.getLogger(__name__)--if TYPE_CHECKING:- from pathlib import Path-- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL- from cyberdrop_dl.managers.manager import Manager--[email protected](slots=True)-class JDownloaderConfig:- enabled: bool- username: str- password: str- device: str- download_dir: Path- autostart: bool-- @staticmethod- def from_manager(manager: Manager) -> JDownloaderConfig:- download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.path_manager.download_folder- return JDownloaderConfig(- enabled=manager.config.runtime_options.send_unsupported_to_jdownloader,- device=manager.auth_config.jdownloader.device,- username=manager.auth_config.jdownloader.username,- password=manager.auth_config.jdownloader.password,- download_dir=download_dir.resolve(),- autostart=manager.config.runtime_options.jdownloader_autostart,- )---class JDownloader:- """Class that handles connecting and passing links to JDownloader."""-- def __init__(self, options: Manager | JDownloaderConfig, /) -> None:- if isinstance(options, JDownloaderConfig):- self._config = options- else:- self._config = JDownloaderConfig.from_manager(options)- self.enabled = self._config.enabled- self._agent = None-- def _connect(self) -> None:- if not all((self._config.username, self._config.password, self._config.device)):- raise JDownloaderError("JDownloader credentials were not provided.")- jd = myjdapi.Myjdapi()- jd.set_app_key("CYBERDROP-DL")- jd.connect(self._config.username, self._config.password)- self._agent = jd.get_device(self._config.device)-- def connect(self) -> None:- if not self.enabled or self._agent is not None:- return- try:- return self._connect()- except JDownloaderError as e:- msg = e.message- except myjdapi.MYJDDeviceNotFoundException:- msg = f"Device not found ({self._config.device})"- except myjdapi.MYJDApiException as e:- msg = e-- logger.error(f"Failed to connect to jDownloader: {msg}")- self.enabled = False-- def direct_unsupported_to_jdownloader(- self,- url: AbsoluteHttpURL,- title: str,- relative_download_path: Path | None = None,- ) -> None:- """Sends links to JDownloader."""- try:- assert self._agent is not None- download_folder = self._config.download_dir- if relative_download_path:- download_folder = download_folder / relative_download_path- self._agent.linkgrabber.add_links(- [- {- "autostart": self._config.autostart,- "links": str(url),- "packageName": title if title else "Cyberdrop-DL",- "destinationFolder": str(download_folder),- "overwritePackagizerRules": True,- },- ],- )- except (AssertionError, myjdapi.MYJDException) as e:- raise JDownloaderError(str(e)) from ediff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex a495bd5bb..c7b4ced10 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Literal, Selfimport aiofilesfrom yarl import URL+from cyberdrop_dl.clients.jdownloader import JDownloaderfrom cyberdrop_dl.constants import REGEX_LINKS, BlockedDomainsfrom cyberdrop_dl.crawlers._chevereto import CheveretoCrawlerfrom cyberdrop_dl.crawlers.crawler import Crawler, create_crawlers@@ -21,7 +22,6 @@ from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediafrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionErrorfrom cyberdrop_dl.scraper.filters import is_in_domain_list, is_outside_date_range, is_valid_url-from cyberdrop_dl.scraper.jdownloader import JDownloaderfrom cyberdrop_dl.utils.logger import log_spacerfrom cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash@@ -47,8 +47,7 @@ class ScrapeMapper:self.manager = managerself.existing_crawlers: dict[str, Crawler] = {}self.direct_crawler = DirectHttpFile(self.manager)- self.jdownloader = JDownloader(self.manager)- self.jdownloader_whitelist = self.manager.config_manager.settings_data.runtime_options.jdownloader_whitelist+ self.jdownloader = JDownloader.from_manager(self.manager)self.using_input_file = Falseself.groups = set()self.count = 0@@ -64,10 +63,6 @@ class ScrapeMapper:def global_settings(self) -> GlobalSettings:return self.manager.config_manager.global_settings_data- @property- def enable_generic_crawler(self) -> bool:- return self.global_settings.general.enable_generic_crawler-def start_scrapers(self) -> None:"""Starts all scrapers."""from cyberdrop_dl import plugins@@ -101,7 +96,11 @@ class ScrapeMapper:"""Starts the orchestra."""self.start_scrapers()await self.manager.db_manager.history_table.update_previously_unsupported(self.existing_crawlers)- self.jdownloader.connect()+ try:+ await self.jdownloader.connect()+ except JDownloaderError:+ logger.exception("Failed to connect to jDownloader")+await self.start_real_debrid()self.direct_crawler._init_downloader()async for item in self.get_input_items():@@ -213,9 +212,6 @@ class ScrapeMapper:"""Maps URLs to their respective handlers."""scrape_item.url = remove_trailing_slash(scrape_item.url)crawler_match = match_url_to_crawler(self.existing_crawlers, scrape_item.url)- jdownloader_whitelisted = True- if self.jdownloader_whitelist:- jdownloader_whitelisted = any(domain in scrape_item.url.host for domain in self.jdownloader_whitelist)if crawler_match:if not crawler_match.ready:@@ -235,13 +231,13 @@ class ScrapeMapper:except (NoExtensionError, ValueError):pass- if self.jdownloader.enabled and jdownloader_whitelisted:+ if self.jdownloader.is_enabled_for(scrape_item.url):logger.info(f"Sending unsupported URL to JDownloader: {scrape_item.url}")success = Falsetry:download_folder = get_download_path(self.manager, scrape_item, "jdownloader")relative_download_dir = download_folder.relative_to(self.manager.path_manager.download_folder)- self.jdownloader.direct_unsupported_to_jdownloader(+ await self.jdownloader.send(scrape_item.url,scrape_item.parent_title,relative_download_dir,