Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: unify http clients (part 1) (#69)

NTFSvolumeb6f149407f1b
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index e69de29bb..2638e5141 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import dataclasses
+import logging
+import time
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
+
+from multidict import CIMultiDict
+
+from cyberdrop_dl import constants, ddos_guard, signature
+from cyberdrop_dl.clients.response import AbstractResponse
+from cyberdrop_dl.exceptions import DDOSGuardError
+from cyberdrop_dl.utils.cookie_management import make_simple_cookie
+from cyberdrop_dl.utils.utilities import sanitize_filename
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Mapping
+
+ from bs4 import BeautifulSoup
+ from curl_cffi.requests.impersonate import BrowserTypeLiteral
+ from curl_cffi.requests.session import HttpMethod
+
+ from cyberdrop_dl.clients import flaresolverr
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+ from cyberdrop_dl.managers.client_manager import ClientManager
+
+
+logger = logging.getLogger(__name__)
+
+
[email protected](slots=True)
+class HTTPClient:
+ client_manager: ClientManager
+ _save_responses_to_disk: bool
+ _responses_folder: Path
+
+ @classmethod
+ def from_client(cls, client_manager: ClientManager) -> Self:
+ return cls(
+ client_manager,
+ client_manager.manager.config_manager.settings_data.files.save_pages_html,
+ client_manager.manager.path_manager.pages_folder,
+ )
+
+ @property
+ def _default_headers(self) -> dict[Any, Any]:
+ return {}
+
+ @contextlib.asynccontextmanager
+ async def _limiter(self, domain: str) -> AsyncGenerator[None]:
+ with self.client_manager.request_context(domain):
+ domain_limiter = self.client_manager.get_rate_limiter(domain)
+ async with self.client_manager.global_rate_limiter, domain_limiter:
+ await self.client_manager.manager.states.RUNNING.wait()
+ yield
+
+ def _prepare_headers(self, headers: Mapping[str, str] | None = None) -> CIMultiDict[str]:
+ """Add default headers and transform it to CIMultiDict"""
+ combined = CIMultiDict(self._default_headers)
+ if headers:
+ headers = CIMultiDict(headers)
+ new: set[str] = set()
+ for key, value in headers.items():
+ if key in new:
+ combined.add(key, value)
+ else:
+ combined[key] = value
+ new.add(key)
+ return combined
+
+ @contextlib.asynccontextmanager
+ async def request(
+ self: object,
+ url: AbsoluteHttpURL,
+ /,
+ method: HttpMethod = "GET",
+ headers: Mapping[str, str] | None = None,
+ impersonate: BrowserTypeLiteral | bool | None = None,
+ data: Any = None,
+ json: Any = None,
+ cache_disabled: bool = False,
+ **request_params: Any,
+ ) -> AsyncGenerator[AbstractResponse[Any]]:
+ self = cast("HTTPClient", self)
+ request_params["headers"] = headers = self._prepare_headers(headers)
+ request_params["data"] = data
+ request_params["json"] = json
+
+ if method == "GET" and (data or json):
+ method = "POST"
+
+ impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate
+ if impersonate:
+ self.client_manager.check_curl_cffi_is_available()
+ if impersonate is True:
+ impersonate = "chrome"
+ request_params["impersonate"] = impersonate
+
+ else:
+ _ = headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)
+ request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)
+
+ request_id = str(uuid.uuid4())
+ logger.debug("Starting {} request to {} [id={}]\n{}", method, url, request_id, request_params)
+
+ async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:
+ exc = None
+ try:
+ yield await self._check_response(resp, url)
+ except Exception as e:
+ exc = e
+ raise
+ finally:
+ logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)
+ if self._save_responses_to_disk:
+ _ = self.client_manager.manager.task_group.create_task(
+ asyncio.to_thread(
+ _write_resp_to_disk,
+ self._responses_folder,
+ url,
+ resp,
+ exc,
+ )
+ )
+
+ def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
+ """
+ Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL
+
+ This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session
+
+ The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped
+ """
+ now = time.time()
+ for cookie in self.client_manager._curl_session.cookies.jar:
+ simple_cookie = make_simple_cookie(cookie, now)
+ self.client_manager.cookies.update_cookies(simple_cookie, url)
+
+ @contextlib.asynccontextmanager
+ async def __request(
+ self,
+ url: AbsoluteHttpURL,
+ method: HttpMethod,
+ request_params: Mapping[str, Any],
+ *,
+ impersonate: bool,
+ ) -> AsyncGenerator[AbstractResponse[Any]]:
+
+ if impersonate:
+ async with contextlib.aclosing(
+ await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)
+ ) as curl_resp:
+ yield AbstractResponse.create(curl_resp)
+ self.__sync_session_cookies(url)
+
+ return
+
+ async with (
+ self.client_manager._session.request(method, url, **request_params) as aio_resp,
+ ):
+ yield AbstractResponse.create(aio_resp)
+
+ async def _check_response(self, abs_resp: AbstractResponse[Any], url: AbsoluteHttpURL, data: Any | None = None):
+ """Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.
+
+ Returns an AbstractResponse confirmed to not be a DDOS Guard page."""
+ try:
+ await self.client_manager.check_http_status(abs_resp)
+ return abs_resp
+ except DDOSGuardError as e:
+ if not (flare := self.client_manager.flaresolverr):
+ raise
+
+ try:
+ solution = await flare.request(url, data)
+ except RuntimeError:
+ raise e from None
+
+ self.client_manager.cookies.update_cookies(solution.cookies)
+ await _check_flaresolverr_resp(self.client_manager.manager.global_config.general.user_agent, solution)
+ return AbstractResponse.create(solution)
+
+
+async def _check_flaresolverr_resp(cdl_user_agent: str, solution: flaresolverr.Solution) -> None:
+ mismatch_ua_msg = (
+ "Config user_agent and flaresolverr user_agent do not match:"
+ f"\n Cyberdrop-DL: '{cdl_user_agent}'"
+ f"\n Flaresolverr: '{solution.user_agent}'"
+ )
+
+ try:
+ await ddos_guard.check(solution.content)
+ except DDOSGuardError:
+ if solution.user_agent != cdl_user_agent:
+ raise DDOSGuardError(mismatch_ua_msg) from None
+
+ if solution.user_agent != cdl_user_agent:
+ logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")
+
+
+def _write_resp_to_disk(
+ folder: Path,
+ url: AbsoluteHttpURL,
+ response: AbstractResponse[Any],
+ exc: Exception | None = None,
+) -> None:
+
+ max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10
+
+ log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)
+ path_safe_url = sanitize_filename(Path(str(url)).as_posix().replace("/", "-"))
+ filename = f"{path_safe_url[:max_stem_len]}_{log_date}.html"
+ file = folder / filename
+ content = response.create_report(exc)
+ try:
+ _ = file.write_text(content, "utf8")
+ except OSError:
+ pass
+
+
+class HTTPClientProxy:
+ DOMAIN: ClassVar[str]
+ _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None
+ client: HTTPClient # pyright: ignore[reportUninitializedInstanceVariable]
+
+ @classmethod
+ def __json_resp_check__(cls, json_resp: Any, resp: AbstractResponse[Any], /) -> None:
+ """Custom check for JSON responses.
+
+ This method is called automatically by the `HttpClient` when a JSON response is received from `cls.DOMAIN`
+ and it was **NOT** successful (`4xx` or `5xx` HTTP code).
+
+ Override this method in subclasses to raise a custom `ScrapeError` instead of the default HTTP error
+
+ Example:
+ ```python
+ if isinstance(json, dict) and json.get("status") == "error":
+ raise ScrapeError(422, f"API error: {json['message']}")
+ ```
+
+ IMPORTANT:
+ Cases were the response **IS** successful (200, OK) but the JSON indicates an error
+ should be handled by the crawler itself
+ """
+
+ @signature.copy(HTTPClient.request)
+ @contextlib.asynccontextmanager
+ async def request(
+ self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs
+ ) -> AsyncGenerator[AbstractResponse[Any]]:
+ if impersonate is None:
+ impersonate = self._IMPERSONATE
+
+ with self.client.client_manager.set_json_checker(self.__json_resp_check__):
+ async with (
+ self.client._limiter(self.DOMAIN),
+ self.client.request(*args, impersonate=impersonate, **kwargs) as resp,
+ ):
+ yield resp
+
+ @signature.copy(request)
+ async def request_json(self, *args, **kwargs) -> Any:
+ async with self.request(*args, **kwargs) as resp:
+ return await resp.json()
+
+ @signature.copy(request)
+ async def request_soup(self, *args, **kwargs) -> BeautifulSoup:
+ async with self.request(*args, **kwargs) as resp:
+ return await resp.soup()
+
+ @signature.copy(request)
+ async def request_text(self, *args, **kwargs) -> str:
+ async with self.request(*args, **kwargs) as resp:
+ return await resp.text()
diff --git a/cyberdrop_dl/clients/jdownloader.py b/cyberdrop_dl/clients/jdownloader.py
new file mode 100644
index 000000000..6b3033426
--- /dev/null
+++ b/cyberdrop_dl/clients/jdownloader.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import dataclasses
+from typing import TYPE_CHECKING, Self
+
+from myjdapi import myjdapi
+
+from cyberdrop_dl.exceptions import JDownloaderError
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+ from pathlib import Path
+
+ from myjdapi.myjdapi import Jddevice
+
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+ from cyberdrop_dl.managers.manager import Manager
+
+
[email protected](frozen=True, slots=True)
+class Config:
+ enabled: bool
+ username: str
+ password: str
+ device: str
+ download_dir: Path
+ autostart: bool
+ whitelist: tuple[str, ...]
+
+ @staticmethod
+ def from_manager(manager: Manager) -> Config:
+ download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.path_manager.download_folder
+ return Config(
+ enabled=manager.config.runtime_options.send_unsupported_to_jdownloader,
+ device=manager.auth_config.jdownloader.device,
+ username=manager.auth_config.jdownloader.username,
+ password=manager.auth_config.jdownloader.password,
+ download_dir=download_dir.resolve(),
+ autostart=manager.config.runtime_options.jdownloader_autostart,
+ whitelist=tuple(manager.config.runtime_options.jdownloader_whitelist),
+ )
+
+
[email protected](slots=True)
+class JDownloader:
+ """Class that handles connecting and sending links to JDownloader."""
+
+ config: Config
+ _enabled: bool = dataclasses.field(init=False)
+ _device: Jddevice | None = dataclasses.field(default=None, init=False)
+
+ @classmethod
+ def from_manager(cls, manager: Manager, /) -> Self:
+ return cls(Config.from_manager(manager))
+
+ @property
+ def enabled(self) -> bool:
+ return self._enabled
+
+ def __post_init__(self) -> None:
+ self._enabled = self.config.enabled
+
+ def is_enabled_for(self, url: AbsoluteHttpURL) -> bool:
+ if not self.enabled:
+ return False
+ if not self.config.whitelist:
+ return True
+
+ return any(domain in url.host for domain in self.config.whitelist)
+
+ @contextlib.contextmanager
+ def _wrap_errors(self) -> Generator[None]:
+ try:
+ yield
+ except myjdapi.MYJDDeviceNotFoundException:
+ raise JDownloaderError(f"Device not found ({self.config.device})") from None
+ except myjdapi.MYJDApiException as e:
+ raise JDownloaderError(str(e)) from e
+
+ async def _connect(self) -> None:
+ if not self._enabled or self._device is not None:
+ return
+
+ if not all((self.config.username, self.config.password, self.config.device)):
+ raise JDownloaderError("JDownloader credentials were not provided")
+
+ with self._wrap_errors():
+ api = myjdapi.Myjdapi()
+ api.set_app_key("CYBERDROP-DL")
+ _ = await asyncio.to_thread(api.connect, self.config.username, self.config.password)
+ self._device = api.get_device(self.config.device)
+
+ async def connect(self) -> None:
+ try:
+ return await self._connect()
+ except Exception:
+ self._enabled = False
+ raise
+
+ async def send(self, url: AbsoluteHttpURL, title: str, download_path: Path | None = None) -> None:
+ """Sends links to JDownloader."""
+
+ assert self._device is not None and self.enabled
+ with self._wrap_errors():
+ download_folder = self.config.download_dir
+ if download_path:
+ download_folder = download_folder / download_path
+
+ await asyncio.to_thread(
+ self._device.linkgrabber.add_links,
+ [
+ {
+ "autostart": self.config.autostart,
+ "links": str(url),
+ "packageName": title if title else "Cyberdrop-DL",
+ "destinationFolder": str(download_folder),
+ "overwritePackagizerRules": True,
+ },
+ ],
+ )
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
deleted file mode 100644
index 3f26afb73..000000000
--- a/cyberdrop_dl/clients/scraper_client.py
+++ /dev/null
@@ -1,204 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import logging
-import time
-import uuid
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
-
-from cyberdrop_dl import constants, ddos_guard
-from cyberdrop_dl.clients.response import AbstractResponse
-from cyberdrop_dl.exceptions import DDOSGuardError
-from cyberdrop_dl.utils.cookie_management import make_simple_cookie
-from cyberdrop_dl.utils.utilities import sanitize_filename
-
-if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Mapping
-
- from curl_cffi.requests.impersonate import BrowserTypeLiteral
- from curl_cffi.requests.session import HttpMethod
-
- from cyberdrop_dl.clients import flaresolverr
- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
- from cyberdrop_dl.managers.client_manager import ClientManager
-
-logger = logging.getLogger(__name__)
-
-
-class ScraperClient:
- """AIOHTTP / CURL operations for scraping."""
-
- def __init__(self, client_manager: ClientManager) -> None:
- self.client_manager = client_manager
- self._save_pages_html = client_manager.manager.config_manager.settings_data.files.save_pages_html
- self._resp_folder = self.client_manager.manager.path_manager.pages_folder
-
- @contextlib.asynccontextmanager
- async def _limiter(self, domain: str) -> AsyncGenerator[None]:
- with self.client_manager.request_context(domain):
- domain_limiter = self.client_manager.get_rate_limiter(domain)
- async with self.client_manager.global_rate_limiter, domain_limiter:
- await self.client_manager.manager.states.RUNNING.wait()
- yield
-
- @contextlib.asynccontextmanager
- async def _request(
- self: object,
- url: AbsoluteHttpURL,
- /,
- method: HttpMethod = "GET",
- headers: dict[str, str] | None = None,
- impersonate: BrowserTypeLiteral | bool | None = None,
- data: Any = None,
- json: Any = None,
- cache_disabled: bool = False,
- **request_params: Any,
- ) -> AsyncGenerator[AbstractResponse[Any]]:
- """
- Asynchronous context manager for HTTP requests.
-
- - If 'impersonate' is specified, uses curl_cffi for the request and updates cookies.
- - Otherwise, uses aiohttp with optional cache control.
- - Yield an AbstractResponse that wraps the underlying response with common methods.
- - On DDOSGuardError, retries the request using FlareSolverr.
- - Saves the HTML content to disk if the config option is enabled.
- - Closes underliying response on exit.
- """
- self = cast("ScraperClient", self)
- request_params["headers"] = headers = headers or {}
- request_params["data"] = data
- request_params["json"] = json
-
- if method == "GET" and (data or json):
- method = "POST"
-
- impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate
- if impersonate:
- self.client_manager.check_curl_cffi_is_available()
- if impersonate is True:
- impersonate = "chrome"
- request_params["impersonate"] = impersonate
-
- else:
- _ = headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)
- request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)
-
- request_id = str(uuid.uuid4())
- logger.debug("Starting {} request to {} [id={}]\n{}", method, url, request_id, request_params)
-
- async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:
- exc = None
- try:
- yield await self._check_response(resp, url)
- except Exception as e:
- exc = e
- raise
- finally:
- logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)
- if self._save_pages_html:
- _ = self.client_manager.manager.task_group.create_task(
- asyncio.to_thread(
- _write_resp_to_disk,
- self._resp_folder,
- url,
- resp,
- exc,
- )
- )
-
- def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
- """
- Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL
-
- This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session
-
- The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped
- """
- now = time.time()
- for cookie in self.client_manager._curl_session.cookies.jar:
- simple_cookie = make_simple_cookie(cookie, now)
- self.client_manager.cookies.update_cookies(simple_cookie, url)
-
- @contextlib.asynccontextmanager
- async def __request(
- self,
- url: AbsoluteHttpURL,
- method: HttpMethod,
- request_params: Mapping[str, Any],
- *,
- impersonate: bool,
- ) -> AsyncGenerator[AbstractResponse[Any]]:
-
- if impersonate:
- async with contextlib.aclosing(
- await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)
- ) as curl_resp:
- yield AbstractResponse.create(curl_resp)
- self.__sync_session_cookies(url)
-
- return
-
- async with (
- self.client_manager._session.request(method, url, **request_params) as aio_resp,
- ):
- yield AbstractResponse.create(aio_resp)
-
- async def _check_response(self, abs_resp: AbstractResponse[Any], url: AbsoluteHttpURL, data: Any | None = None):
- """Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.
-
- Returns an AbstractResponse confirmed to not be a DDOS Guard page."""
- try:
- await self.client_manager.check_http_status(abs_resp)
- return abs_resp
- except DDOSGuardError as e:
- if not (flare := self.client_manager.flaresolverr):
- raise
-
- try:
- solution = await flare.request(url, data)
- except RuntimeError:
- raise e from None
-
- self.client_manager.cookies.update_cookies(solution.cookies)
- await _check_flaresolverr_resp(self.client_manager.manager.global_config.general.user_agent, solution)
- return AbstractResponse.create(solution)
-
-
-async def _check_flaresolverr_resp(cdl_user_agent: str, solution: flaresolverr.Solution) -> None:
- mismatch_ua_msg = (
- "Config user_agent and flaresolverr user_agent do not match:"
- f"\n Cyberdrop-DL: '{cdl_user_agent}'"
- f"\n Flaresolverr: '{solution.user_agent}'"
- )
-
- try:
- await ddos_guard.check(solution.content)
- except DDOSGuardError:
- if solution.user_agent != cdl_user_agent:
- raise DDOSGuardError(mismatch_ua_msg) from None
-
- if solution.user_agent != cdl_user_agent:
- logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")
-
-
-def _write_resp_to_disk(
- folder: Path,
- url: AbsoluteHttpURL,
- response: AbstractResponse[Any],
- exc: Exception | None = None,
-) -> None:
-
- max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10
-
- log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)
- url_str = str(url)
- clean_url = sanitize_filename(Path(url_str).as_posix().replace("/", "-"))
- filename = f"{clean_url[:max_stem_len]}_{log_date}.html"
- file = folder / filename
- content = response.create_report(exc)
- try:
- _ = file.write_text(content, "utf8")
- except OSError:
- pass
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index a67e05c07..68f1e2bd9 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -32,8 +32,8 @@ import yarl
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl import constants, signature
-from cyberdrop_dl.clients.scraper_client import ScraperClient
+from cyberdrop_dl import constants
+from cyberdrop_dl.clients import HTTPClient, HTTPClientProxy
from cyberdrop_dl.data_structures.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
@@ -60,10 +60,8 @@ if TYPE_CHECKING:
from http.cookies import BaseCookie
from bs4 import BeautifulSoup, Tag
- from curl_cffi.requests.impersonate import BrowserTypeLiteral
from rich.progress import TaskID
- from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.managers.manager import Manager
logger = logging.getLogger(__name__)
@@ -154,63 +152,7 @@ class Registry:
cls._import(module_info.name)
-class HTTPClientProxy:
- DOMAIN: ClassVar[str]
- _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None
- client: ScraperClient # pyright: ignore[reportUninitializedInstanceVariable]
-
- @classmethod
- def __json_resp_check__(cls, json_resp: Any, resp: AbstractResponse[Any], /) -> None:
- """Custom check for JSON responses.
-
- This method is called automatically by the `HttpClient` when a JSON response is received from `cls.DOMAIN`
- and it was **NOT** successful (`4xx` or `5xx` HTTP code).
-
- Override this method in subclasses to raise a custom `ScrapeError` instead of the default HTTP error
-
- Example:
- ```python
- if isinstance(json, dict) and json.get("status") == "error":
- raise ScrapeError(422, f"API error: {json['message']}")
- ```
-
- IMPORTANT:
- Cases were the response **IS** successful (200, OK) but the JSON indicates an error
- should be handled by the crawler itself
- """
-
- @signature.copy(ScraperClient._request)
- @contextlib.asynccontextmanager
- async def request(
- self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs
- ) -> AsyncGenerator[AbstractResponse[Any]]:
- if impersonate is None:
- impersonate = self._IMPERSONATE
-
- with self.client.client_manager.set_json_checker(self.__json_resp_check__):
- async with (
- self.client._limiter(self.DOMAIN),
- self.client._request(*args, impersonate=impersonate, **kwargs) as resp,
- ):
- yield resp
-
- @signature.copy(request)
- async def request_json(self, *args, **kwargs) -> Any:
- async with self.request(*args, **kwargs) as resp:
- return await resp.json()
-
- @signature.copy(request)
- async def request_soup(self, *args, **kwargs) -> BeautifulSoup:
- async with self.request(*args, **kwargs) as resp:
- return await resp.soup()
-
- @signature.copy(request)
- async def request_text(self, *args, **kwargs) -> str:
- async with self.request(*args, **kwargs) as resp:
- return await resp.text()
-
-
-class Crawler(ABC, HTTPClientProxy):
+class Crawler(HTTPClientProxy, ABC):
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ()
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ()
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {}
@@ -235,7 +177,7 @@ class Crawler(ABC, HTTPClientProxy):
def __init__(self, manager: Manager) -> None:
self.manager = manager
self.downloader: Downloader = field(init=False)
- self.client: ScraperClient = field(init=False)
+ self.client: HTTPClient = field(init=False)
self.startup_lock = asyncio.Lock()
self.ready: bool = False
self.disabled: bool = False
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index b888899a6..897ebce0e 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -206,6 +206,10 @@ class LoginError(CDLBaseError):
class JDownloaderError(CDLBaseError):
"""This error will be thrown for any Jdownloader error."""
+ def __init__(self, message: str | None = None) -> None:
+ ui_failure = "JDownloader Error"
+ super().__init__(ui_failure, message=message)
+
class InvalidYamlError(CDLBaseError):
def __init__(self, file: Path, e: YAMLError) -> None:
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index ddb2678e7..a9b10c073 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -19,10 +19,10 @@ from aiolimiter import AsyncLimiter
from cyberdrop_dl import constants, ddos_guard, env
from cyberdrop_dl.aio import WeakAsyncLocks
+from cyberdrop_dl.clients import HTTPClient
from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.clients.flaresolverr import FlareSolverrClient
from cyberdrop_dl.clients.response import AbstractResponse
-from cyberdrop_dl.clients.scraper_client import ScraperClient
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrors
from cyberdrop_dl.ffmpeg import probe
@@ -136,7 +136,7 @@ class ClientManager:
self.download_slots: dict[str, int] = {}
self.global_rate_limiter = AsyncLimiter(self.rate_limiting_options.rate_limit, 1)
self.global_download_slots = asyncio.Semaphore(self.rate_limiting_options.max_simultaneous_downloads)
- self.scraper_client = ScraperClient(self)
+ self.scraper_client = HTTPClient.from_client(self)
self.speed_limiter = DownloadSpeedLimiter(self.rate_limiting_options.download_speed_limit)
self.download_client = DownloadClient(manager, self)
self._flaresolverr: FlareSolverrClient | None = None
diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.py
deleted file mode 100644
index c3ad2e509..000000000
--- a/cyberdrop_dl/scraper/jdownloader.py
+++ /dev/null
@@ -1,100 +0,0 @@
-from __future__ import annotations
-
-import dataclasses
-import logging
-from typing import TYPE_CHECKING
-
-from myjdapi import myjdapi
-
-from cyberdrop_dl.exceptions import JDownloaderError
-
-logger = logging.getLogger(__name__)
-
-if TYPE_CHECKING:
- from pathlib import Path
-
- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
- from cyberdrop_dl.managers.manager import Manager
-
-
[email protected](slots=True)
-class JDownloaderConfig:
- enabled: bool
- username: str
- password: str
- device: str
- download_dir: Path
- autostart: bool
-
- @staticmethod
- def from_manager(manager: Manager) -> JDownloaderConfig:
- download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.path_manager.download_folder
- return JDownloaderConfig(
- enabled=manager.config.runtime_options.send_unsupported_to_jdownloader,
- device=manager.auth_config.jdownloader.device,
- username=manager.auth_config.jdownloader.username,
- password=manager.auth_config.jdownloader.password,
- download_dir=download_dir.resolve(),
- autostart=manager.config.runtime_options.jdownloader_autostart,
- )
-
-
-class JDownloader:
- """Class that handles connecting and passing links to JDownloader."""
-
- def __init__(self, options: Manager | JDownloaderConfig, /) -> None:
- if isinstance(options, JDownloaderConfig):
- self._config = options
- else:
- self._config = JDownloaderConfig.from_manager(options)
- self.enabled = self._config.enabled
- self._agent = None
-
- def _connect(self) -> None:
- if not all((self._config.username, self._config.password, self._config.device)):
- raise JDownloaderError("JDownloader credentials were not provided.")
- jd = myjdapi.Myjdapi()
- jd.set_app_key("CYBERDROP-DL")
- jd.connect(self._config.username, self._config.password)
- self._agent = jd.get_device(self._config.device)
-
- def connect(self) -> None:
- if not self.enabled or self._agent is not None:
- return
- try:
- return self._connect()
- except JDownloaderError as e:
- msg = e.message
- except myjdapi.MYJDDeviceNotFoundException:
- msg = f"Device not found ({self._config.device})"
- except myjdapi.MYJDApiException as e:
- msg = e
-
- logger.error(f"Failed to connect to jDownloader: {msg}")
- self.enabled = False
-
- def direct_unsupported_to_jdownloader(
- self,
- url: AbsoluteHttpURL,
- title: str,
- relative_download_path: Path | None = None,
- ) -> None:
- """Sends links to JDownloader."""
- try:
- assert self._agent is not None
- download_folder = self._config.download_dir
- if relative_download_path:
- download_folder = download_folder / relative_download_path
- self._agent.linkgrabber.add_links(
- [
- {
- "autostart": self._config.autostart,
- "links": str(url),
- "packageName": title if title else "Cyberdrop-DL",
- "destinationFolder": str(download_folder),
- "overwritePackagizerRules": True,
- },
- ],
- )
- except (AssertionError, myjdapi.MYJDException) as e:
- raise JDownloaderError(str(e)) from e
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index a495bd5bb..c7b4ced10 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Literal, Self
import aiofiles
from yarl import URL
+from cyberdrop_dl.clients.jdownloader import JDownloader
from cyberdrop_dl.constants import REGEX_LINKS, BlockedDomains
from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
from cyberdrop_dl.crawlers.crawler import Crawler, create_crawlers
@@ -21,7 +22,6 @@ from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMedia
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.scraper.filters import is_in_domain_list, is_outside_date_range, is_valid_url
-from cyberdrop_dl.scraper.jdownloader import JDownloader
from cyberdrop_dl.utils.logger import log_spacer
from cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash
@@ -47,8 +47,7 @@ class ScrapeMapper:
self.manager = manager
self.existing_crawlers: dict[str, Crawler] = {}
self.direct_crawler = DirectHttpFile(self.manager)
- self.jdownloader = JDownloader(self.manager)
- self.jdownloader_whitelist = self.manager.config_manager.settings_data.runtime_options.jdownloader_whitelist
+ self.jdownloader = JDownloader.from_manager(self.manager)
self.using_input_file = False
self.groups = set()
self.count = 0
@@ -64,10 +63,6 @@ class ScrapeMapper:
def global_settings(self) -> GlobalSettings:
return self.manager.config_manager.global_settings_data
- @property
- def enable_generic_crawler(self) -> bool:
- return self.global_settings.general.enable_generic_crawler
-
def start_scrapers(self) -> None:
"""Starts all scrapers."""
from cyberdrop_dl import plugins
@@ -101,7 +96,11 @@ class ScrapeMapper:
"""Starts the orchestra."""
self.start_scrapers()
await self.manager.db_manager.history_table.update_previously_unsupported(self.existing_crawlers)
- self.jdownloader.connect()
+ try:
+ await self.jdownloader.connect()
+ except JDownloaderError:
+ logger.exception("Failed to connect to jDownloader")
+
await self.start_real_debrid()
self.direct_crawler._init_downloader()
async for item in self.get_input_items():
@@ -213,9 +212,6 @@ class ScrapeMapper:
"""Maps URLs to their respective handlers."""
scrape_item.url = remove_trailing_slash(scrape_item.url)
crawler_match = match_url_to_crawler(self.existing_crawlers, scrape_item.url)
- jdownloader_whitelisted = True
- if self.jdownloader_whitelist:
- jdownloader_whitelisted = any(domain in scrape_item.url.host for domain in self.jdownloader_whitelist)
if crawler_match:
if not crawler_match.ready:
@@ -235,13 +231,13 @@ class ScrapeMapper:
except (NoExtensionError, ValueError):
pass
- if self.jdownloader.enabled and jdownloader_whitelisted:
+ if self.jdownloader.is_enabled_for(scrape_item.url):
logger.info(f"Sending unsupported URL to JDownloader: {scrape_item.url}")
success = False
try:
download_folder = get_download_path(self.manager, scrape_item, "jdownloader")
relative_download_dir = download_folder.relative_to(self.manager.path_manager.download_folder)
- self.jdownloader.direct_unsupported_to_jdownloader(
+ await self.jdownloader.send(
scrape_item.url,
scrape_item.parent_title,
relative_download_dir,