Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
style: fix signature of functions that use wrappers (#946)
NTFSvolume835cb993aff0
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex dec6f4f4b..cfee1ca37 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -7,7 +7,7 @@ import itertoolsimport timefrom functools import partial, wrapsfrom http import HTTPStatus-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, ParamSpec, TypeVarimport aiofilesimport aiohttp@@ -32,16 +32,21 @@ if TYPE_CHECKING:from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem+P = ParamSpec("P")+R = TypeVar("R")+CONTENT_TYPES_OVERRIDES = {"text/vnd.trolltech.linguist": "video/MP2T"}-def limiter(func: Callable) -> Any:+def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:"""Wrapper handles limits for download session."""@wraps(func)- async def wrapper(self: DownloadClient, *args, **kwargs) -> Any:- domain_limiter = await self.client_manager.get_rate_limiter(args[0])- await asyncio.sleep(await self.client_manager.get_downloader_spacer(args[0]))+ async def wrapper(*args, **kwargs) -> R:+ self: DownloadClient = args[0]+ domain: str = args[1]+ domain_limiter = await self.client_manager.get_rate_limiter(domain)+ await asyncio.sleep(await self.client_manager.get_downloader_spacer(domain))await self._global_limiter.acquire()await domain_limiter.acquire()@@ -53,7 +58,7 @@ def limiter(func: Callable) -> Any:trace_configs=self.trace_configs,) as client:kwargs["client_session"] = client- return await func(self, *args, **kwargs)+ return await func(*args, **kwargs)return wrapperdiff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex ba8ee461a..e4a5f12db 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -7,7 +7,7 @@ from dataclasses import fieldfrom datetime import datetimefrom functools import wrapsfrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVarimport aiohttpfrom aiohttp_client_cache import CachedSession@@ -27,7 +27,7 @@ except ImportError as e:curl_import_error = eif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Coroutinefrom curl_cffi.requests.impersonate import BrowserTypeLiteralfrom curl_cffi.requests.models import Response as CurlResponse@@ -38,12 +38,17 @@ if TYPE_CHECKING:from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem-def limiter(func: Callable) -> Any:+P = ParamSpec("P")+R = TypeVar("R")+++def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:"""Wrapper handles limits for scrape session."""@wraps(func)- async def wrapper(self: ScraperClient, *args, **kwargs) -> Any:- domain = args[0]+ async def wrapper(*args, **kwargs) -> R:+ self: ScraperClient = args[0]+ domain: str = args[1]domain_limiter = await self.client_manager.get_rate_limiter(domain)async with self.client_manager.session_limit:await self._global_limiter.acquire()@@ -57,10 +62,10 @@ def limiter(func: Callable) -> Any:msg += f"See: https://github.com/lexiforest/curl_cffi/issues/74#issuecomment-1849365636\n{curl_import_error!r}"raise ScrapeError("Missing Dependency", msg)kwargs.pop("client_session", None)- return await func(self, *args, **kwargs)+ return await func(*args, **kwargs)kwargs["client_session"] = self._session- return await func(self, *args, **kwargs)+ return await func(*args, **kwargs)return wrapperdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 084c8155c..fb02d4ce3 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -8,7 +8,7 @@ from dataclasses import fieldfrom datetime import datetimefrom functools import wrapsfrom pathlib import Path-from typing import TYPE_CHECKING, Any, ClassVar, NewType, Protocol+from typing import TYPE_CHECKING, Any, ClassVar, NewType, ParamSpec, Protocol, TypeVarfrom aiolimiter import AsyncLimiterfrom dateutil import parser@@ -30,9 +30,11 @@ from cyberdrop_dl.utils.utilities import (_NEW_ISSUE_URL = "https://github.com/jbsparrow/CyberDropDownloader/issues/new/choose"TimeStamp = NewType("TimeStamp", int)+P = ParamSpec("P")+R = TypeVar("R")if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Awaitable, Callable, Generator+ from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generatorfrom aiohttp_client_cache.response import AnyResponsefrom bs4 import BeautifulSoup, Tag@@ -408,18 +410,19 @@ class Crawler(ABC):filters.cache_filter_functions[url.host] = filter_fn-def create_task_id(func: Callable) -> Callable:+def create_task_id(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R | None]]:"""Wrapper that handles `task_id` creation and removal for scrape items"""@wraps(func)- async def wrapper(self: Crawler, *args, **kwargs):- scrape_item: ScrapeItem = args[0]+ async def wrapper(*args, **kwargs) -> R | None:+ self: Crawler = args[0]+ scrape_item: ScrapeItem = args[1]await self.manager.states.RUNNING.wait()task_id = self.scraping_progress.add_task(scrape_item.url)try:if not self.skip_pre_check:pre_check_scrape_item(scrape_item)- return await func(self, *args, **kwargs)+ return await func(*args, **kwargs)except ValueError:log(f"Scrape Failed: {UNKNOWN_URL_PATH_MSG}: {scrape_item.url}", 40)self.manager.progress_manager.scrape_stats_progress.add_failure(UNKNOWN_URL_PATH_MSG)diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.pyindex 27f09c165..06cb6d2aa 100644--- a/cyberdrop_dl/crawlers/generic.py+++ b/cyberdrop_dl/crawlers/generic.py@@ -4,7 +4,7 @@ import mimetypesfrom dataclasses import dataclassfrom functools import wrapsfrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, ParamSpec, TypeVarfrom cyberdrop_dl.clients.errors import InvalidContentTypeError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id@@ -13,7 +13,7 @@ from cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_and_extif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Coroutinefrom bs4 import BeautifulSoupfrom yarl import URL@@ -22,14 +22,21 @@ if TYPE_CHECKING:from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem+P = ParamSpec("P")+R = TypeVar("R")+VIDEO_SELECTOR = "video > source"-def log_unsupported_wrapper(func: Callable) -> Callable:+def log_unsupported_wrapper(+ func: Callable[P, Coroutine[None, None, R]],+) -> Callable[P, Coroutine[None, None, R | None]]:@wraps(func)- async def wrapper(self: GenericCrawler, item: ScrapeItem, *args, **kwargs):+ async def wrapper(*args, **kwargs):+ self: GenericCrawler = args[0]+ item: ScrapeItem = args[1]try:- return await func(self, item, *args, **kwargs)+ return await func(*args, **kwargs)except (InvalidContentTypeError, ScrapeError) as e:await self.log_unsupported(item, f"({e})")except Exception:diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 798eb48d0..ff9fdca5c 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -9,7 +9,7 @@ from enum import IntEnumfrom functools import wrapsfrom pathlib import Pathfrom time import perf_counter-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, ParamSpec, TypeVarimport browser_cookie3from pydantic import ValidationError@@ -31,8 +31,10 @@ from cyberdrop_dl.utils.utilities import check_partials_and_empty_folders, send_from cyberdrop_dl.utils.yaml import handle_validation_errorif TYPE_CHECKING:- from collections.abc import Callable, Generator+ from collections.abc import Callable, Coroutine, Generator+P = ParamSpec("P")+R = TypeVar("R")startup_logger = logging.getLogger("cyberdrop_dl_startup")STARTUP_LOGGER_FILE = Path.cwd().joinpath("startup.log")@@ -46,11 +48,13 @@ class ExitCode(IntEnum):_C = ExitCode-def _ui_error_handling_wrapper(func: Callable) -> Callable:+def _ui_error_handling_wrapper(+ func: Callable[P, Coroutine[None, None, R]],+) -> Callable[P, Coroutine[None, None, R | None]]:"""Wrapper handles errors from the main UI."""@wraps(func)- async def wrapper(*args, **kwargs):+ async def wrapper(*args, **kwargs) -> R | None:try:return await func(*args, **kwargs)except* Exception as e:diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex c8a81aa7c..545252b75 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -6,7 +6,7 @@ from dataclasses import fieldfrom functools import wrapsfrom http import HTTPStatusfrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, ParamSpec, TypeVarfrom aiohttp import ClientConnectorError, ClientError, ClientResponseErrorfrom filedate import File@@ -24,11 +24,13 @@ from cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import Callable, Generator+ from collections.abc import Callable, Coroutine, Generatorfrom cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.managers.manager import Manager+P = ParamSpec("P")+R = TypeVar("R")KNOWN_BAD_URLS = {"https://i.imgur.com/removed.png": 404,@@ -40,15 +42,16 @@ KNOWN_BAD_URLS = {}-def retry(func: Callable) -> Callable:+def retry(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:"""This function is a wrapper that handles retrying for failed downloads."""@wraps(func)- async def wrapper(self: Downloader, *args, **kwargs) -> Any:- media_item: MediaItem = args[0]+ async def wrapper(*args, **kwargs) -> R:+ self: Downloader = args[0]+ media_item: MediaItem = args[1]while True:try:- return await func(self, *args, **kwargs)+ return await func(*args, **kwargs)except DownloadError as e:if not e.retry:raisediff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.pyindex b3a76c6d9..48d21e8d6 100644--- a/cyberdrop_dl/scraper/jdownloader.py+++ b/cyberdrop_dl/scraper/jdownloader.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom dataclasses import fieldfrom functools import wraps-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, ParamSpec, TypeVarfrom myjdapi import myjdapi@@ -17,14 +17,18 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+P = ParamSpec("P")+R = TypeVar("R")-def error_wrapper(func: Callable) -> Callable:++def error_wrapper(func: Callable[P, R]) -> Callable[P, R | None]:"""Wrapper handles limits for scrape session."""@wraps(func)- def wrapper(self: JDownloader, *args, **kwargs) -> None:+ def wrapper(*args, **kwargs) -> R | None:+ self: JDownloader = args[0]try:- return func(self, *args, **kwargs)+ return func(*args, **kwargs)except JDownloaderError as e:msg = e.messagediff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.pyindex 2959f47ad..91077b58f 100644--- a/cyberdrop_dl/ui/program_ui.py+++ b/cyberdrop_dl/ui/program_ui.py@@ -5,7 +5,7 @@ import sqlite3import sysfrom functools import wrapsfrom textwrap import dedent-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVarimport browser_cookie3from requests import request@@ -23,20 +23,23 @@ from cyberdrop_dl.utils.updates import check_latest_pypifrom cyberdrop_dl.utils.utilities import clear_term, open_in_text_editorif TYPE_CHECKING:+ from collections.abc import Callablefrom pathlib import Pathfrom InquirerPy.base.control import Choicefrom cyberdrop_dl.managers.manager import Manager+P = ParamSpec("P")+R = TypeVar("R")console = Console()ERROR_PREFIX = Text("ERROR: ", style="bold red")-def repeat_until_done(func):+def repeat_until_done(func: Callable[P, R]) -> Callable[P, R]:@wraps(func)- def wrapper(*args, **kwargs):+ def wrapper(*args, **kwargs) -> R:done = Falsewhile not done:done = func(*args, **kwargs)diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pyindex 7cfbf0a4f..dd303d858 100644--- a/cyberdrop_dl/utils/cookie_management.py+++ b/cyberdrop_dl/utils/cookie_management.py@@ -4,7 +4,7 @@ import osfrom functools import wrapsfrom http.cookiejar import CookieJar, MozillaCookieJarfrom textwrap import dedent-from typing import TYPE_CHECKING, NamedTuple+from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVarimport browser_cookie3@@ -14,6 +14,9 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.constants import BROWSERS+P = ParamSpec("P")+R = TypeVar("R")+class CookieExtractor(NamedTuple):name: str@@ -28,11 +31,11 @@ COOKIE_ERROR_FOOTER = "\n\nNothing has been saved."CHROMIUM_BROWSERS = ["chrome", "chromium", "opera", "opera_gx", "brave", "edge", "vivaldi", "arc"]-def cookie_wrapper(func: Callable) -> Callable:+def cookie_wrapper(func: Callable[P, R]) -> Callable[P, R]:"""Wrapper handles errors for cookie extraction."""@wraps(func)- def wrapper(*args, **kwargs) -> None:+ def wrapper(*args, **kwargs) -> R:try:return func(*args, **kwargs)except PermissionError as e:diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 25fac0060..cf287c7de 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -10,7 +10,7 @@ import subprocessfrom dataclasses import dataclass, fieldsfrom functools import lru_cache, partial, wrapsfrom pathlib import Path-from typing import TYPE_CHECKING, Any, ClassVar, Protocol+from typing import TYPE_CHECKING, Any, ClassVar, ParamSpec, Protocol, TypeVarimport aiofilesimport rich@@ -31,7 +31,7 @@ from cyberdrop_dl.utils import constantsfrom cyberdrop_dl.utils.logger import log, log_debug, log_spacer, log_with_colorif TYPE_CHECKING:- from collections.abc import Callable, Mapping+ from collections.abc import Callable, Coroutine, Mappingfrom curl_cffi.requests.models import Response as CurlResponsefrom rich.text import Text@@ -41,6 +41,11 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem++P = ParamSpec("P")+R = TypeVar("R")++TEXT_EDITORS = "micro", "nano", "vim" # Ordered by preferenceFILENAME_REGEX = re.compile(r"filename\*=UTF-8''(.+)|.*filename=\"(.*?)\"", re.IGNORECASE)subprocess_get_text = partial(subprocess.run, capture_output=True, text=True, check=False)@@ -66,17 +71,18 @@ class OGProperties:video: str = ""-def error_handling_wrapper(func: Callable) -> Callable:+def error_handling_wrapper(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R | None]]:"""Wrapper handles errors for url scraping."""@wraps(func)- async def wrapper(self: Crawler | Downloader, *args, **kwargs):- item: ScrapeItem | MediaItem | URL = args[0]+ async def wrapper(*args, **kwargs) -> R | None:+ self: Crawler | Downloader = args[0]+ item: ScrapeItem | MediaItem | URL = args[1]link: URL = item if isinstance(item, URL) else item.urlorigin = exc_info = Nonelink_to_show: URL | str = ""try:- return await func(self, *args, **kwargs)+ return await func(*args, **kwargs)except CDLBaseError as e:error_log_msg = ErrorLogMessage(e.ui_failure, str(e))origin = e.origin