Skip to content

Bulk asynchronous downloader for multiple file hosts

style: fix signature of functions that use wrappers (#946)

NTFSvolume835cb993aff0
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index dec6f4f4b..cfee1ca37 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -7,7 +7,7 @@ import itertools
import time
from functools import partial, wraps
from http import HTTPStatus
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
import aiofiles
import aiohttp
@@ -32,16 +32,21 @@ if TYPE_CHECKING:
from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem
+P = ParamSpec("P")
+R = TypeVar("R")
+
CONTENT_TYPES_OVERRIDES = {"text/vnd.trolltech.linguist": "video/MP2T"}
-def limiter(func: Callable) -> Any:
+def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:
"""Wrapper handles limits for download session."""
@wraps(func)
- async def wrapper(self: DownloadClient, *args, **kwargs) -> Any:
- domain_limiter = await self.client_manager.get_rate_limiter(args[0])
- await asyncio.sleep(await self.client_manager.get_downloader_spacer(args[0]))
+ async def wrapper(*args, **kwargs) -> R:
+ self: DownloadClient = args[0]
+ domain: str = args[1]
+ domain_limiter = await self.client_manager.get_rate_limiter(domain)
+ await asyncio.sleep(await self.client_manager.get_downloader_spacer(domain))
await self._global_limiter.acquire()
await domain_limiter.acquire()
@@ -53,7 +58,7 @@ def limiter(func: Callable) -> Any:
trace_configs=self.trace_configs,
) as client:
kwargs["client_session"] = client
- return await func(self, *args, **kwargs)
+ return await func(*args, **kwargs)
return wrapper
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index ba8ee461a..e4a5f12db 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -7,7 +7,7 @@ from dataclasses import field
from datetime import datetime
from functools import wraps
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
import aiohttp
from aiohttp_client_cache import CachedSession
@@ -27,7 +27,7 @@ except ImportError as e:
curl_import_error = e
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Coroutine
from curl_cffi.requests.impersonate import BrowserTypeLiteral
from curl_cffi.requests.models import Response as CurlResponse
@@ -38,12 +38,17 @@ if TYPE_CHECKING:
from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
-def limiter(func: Callable) -> Any:
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:
"""Wrapper handles limits for scrape session."""
@wraps(func)
- async def wrapper(self: ScraperClient, *args, **kwargs) -> Any:
- domain = args[0]
+ async def wrapper(*args, **kwargs) -> R:
+ self: ScraperClient = args[0]
+ domain: str = args[1]
domain_limiter = await self.client_manager.get_rate_limiter(domain)
async with self.client_manager.session_limit:
await self._global_limiter.acquire()
@@ -57,10 +62,10 @@ def limiter(func: Callable) -> Any:
msg += f"See: https://github.com/lexiforest/curl_cffi/issues/74#issuecomment-1849365636\n{curl_import_error!r}"
raise ScrapeError("Missing Dependency", msg)
kwargs.pop("client_session", None)
- return await func(self, *args, **kwargs)
+ return await func(*args, **kwargs)
kwargs["client_session"] = self._session
- return await func(self, *args, **kwargs)
+ return await func(*args, **kwargs)
return wrapper
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 084c8155c..fb02d4ce3 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -8,7 +8,7 @@ from dataclasses import field
from datetime import datetime
from functools import wraps
from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, NewType, Protocol
+from typing import TYPE_CHECKING, Any, ClassVar, NewType, ParamSpec, Protocol, TypeVar
from aiolimiter import AsyncLimiter
from dateutil import parser
@@ -30,9 +30,11 @@ from cyberdrop_dl.utils.utilities import (
_NEW_ISSUE_URL = "https://github.com/jbsparrow/CyberDropDownloader/issues/new/choose"
TimeStamp = NewType("TimeStamp", int)
+P = ParamSpec("P")
+R = TypeVar("R")
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
+ from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generator
from aiohttp_client_cache.response import AnyResponse
from bs4 import BeautifulSoup, Tag
@@ -408,18 +410,19 @@ class Crawler(ABC):
filters.cache_filter_functions[url.host] = filter_fn
-def create_task_id(func: Callable) -> Callable:
+def create_task_id(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R | None]]:
"""Wrapper that handles `task_id` creation and removal for scrape items"""
@wraps(func)
- async def wrapper(self: Crawler, *args, **kwargs):
- scrape_item: ScrapeItem = args[0]
+ async def wrapper(*args, **kwargs) -> R | None:
+ self: Crawler = args[0]
+ scrape_item: ScrapeItem = args[1]
await self.manager.states.RUNNING.wait()
task_id = self.scraping_progress.add_task(scrape_item.url)
try:
if not self.skip_pre_check:
pre_check_scrape_item(scrape_item)
- return await func(self, *args, **kwargs)
+ return await func(*args, **kwargs)
except ValueError:
log(f"Scrape Failed: {UNKNOWN_URL_PATH_MSG}: {scrape_item.url}", 40)
self.manager.progress_manager.scrape_stats_progress.add_failure(UNKNOWN_URL_PATH_MSG)
diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.py
index 27f09c165..06cb6d2aa 100644
--- a/cyberdrop_dl/crawlers/generic.py
+++ b/cyberdrop_dl/crawlers/generic.py
@@ -4,7 +4,7 @@ import mimetypes
from dataclasses import dataclass
from functools import wraps
from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
from cyberdrop_dl.clients.errors import InvalidContentTypeError, NoExtensionError, ScrapeError
from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
@@ -13,7 +13,7 @@ from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_and_ext
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Coroutine
from bs4 import BeautifulSoup
from yarl import URL
@@ -22,14 +22,21 @@ if TYPE_CHECKING:
from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
+P = ParamSpec("P")
+R = TypeVar("R")
+
VIDEO_SELECTOR = "video > source"
-def log_unsupported_wrapper(func: Callable) -> Callable:
+def log_unsupported_wrapper(
+ func: Callable[P, Coroutine[None, None, R]],
+) -> Callable[P, Coroutine[None, None, R | None]]:
@wraps(func)
- async def wrapper(self: GenericCrawler, item: ScrapeItem, *args, **kwargs):
+ async def wrapper(*args, **kwargs):
+ self: GenericCrawler = args[0]
+ item: ScrapeItem = args[1]
try:
- return await func(self, item, *args, **kwargs)
+ return await func(*args, **kwargs)
except (InvalidContentTypeError, ScrapeError) as e:
await self.log_unsupported(item, f"({e})")
except Exception:
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 798eb48d0..ff9fdca5c 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -9,7 +9,7 @@ from enum import IntEnum
from functools import wraps
from pathlib import Path
from time import perf_counter
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
import browser_cookie3
from pydantic import ValidationError
@@ -31,8 +31,10 @@ from cyberdrop_dl.utils.utilities import check_partials_and_empty_folders, send_
from cyberdrop_dl.utils.yaml import handle_validation_error
if TYPE_CHECKING:
- from collections.abc import Callable, Generator
+ from collections.abc import Callable, Coroutine, Generator
+P = ParamSpec("P")
+R = TypeVar("R")
startup_logger = logging.getLogger("cyberdrop_dl_startup")
STARTUP_LOGGER_FILE = Path.cwd().joinpath("startup.log")
@@ -46,11 +48,13 @@ class ExitCode(IntEnum):
_C = ExitCode
-def _ui_error_handling_wrapper(func: Callable) -> Callable:
+def _ui_error_handling_wrapper(
+ func: Callable[P, Coroutine[None, None, R]],
+) -> Callable[P, Coroutine[None, None, R | None]]:
"""Wrapper handles errors from the main UI."""
@wraps(func)
- async def wrapper(*args, **kwargs):
+ async def wrapper(*args, **kwargs) -> R | None:
try:
return await func(*args, **kwargs)
except* Exception as e:
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index c8a81aa7c..545252b75 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -6,7 +6,7 @@ from dataclasses import field
from functools import wraps
from http import HTTPStatus
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
from aiohttp import ClientConnectorError, ClientError, ClientResponseError
from filedate import File
@@ -24,11 +24,13 @@ from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import Callable, Generator
+ from collections.abc import Callable, Coroutine, Generator
from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.managers.manager import Manager
+P = ParamSpec("P")
+R = TypeVar("R")
KNOWN_BAD_URLS = {
"https://i.imgur.com/removed.png": 404,
@@ -40,15 +42,16 @@ KNOWN_BAD_URLS = {
}
-def retry(func: Callable) -> Callable:
+def retry(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:
"""This function is a wrapper that handles retrying for failed downloads."""
@wraps(func)
- async def wrapper(self: Downloader, *args, **kwargs) -> Any:
- media_item: MediaItem = args[0]
+ async def wrapper(*args, **kwargs) -> R:
+ self: Downloader = args[0]
+ media_item: MediaItem = args[1]
while True:
try:
- return await func(self, *args, **kwargs)
+ return await func(*args, **kwargs)
except DownloadError as e:
if not e.retry:
raise
diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.py
index b3a76c6d9..48d21e8d6 100644
--- a/cyberdrop_dl/scraper/jdownloader.py
+++ b/cyberdrop_dl/scraper/jdownloader.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from dataclasses import field
from functools import wraps
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
from myjdapi import myjdapi
@@ -17,14 +17,18 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+P = ParamSpec("P")
+R = TypeVar("R")
-def error_wrapper(func: Callable) -> Callable:
+
+def error_wrapper(func: Callable[P, R]) -> Callable[P, R | None]:
"""Wrapper handles limits for scrape session."""
@wraps(func)
- def wrapper(self: JDownloader, *args, **kwargs) -> None:
+ def wrapper(*args, **kwargs) -> R | None:
+ self: JDownloader = args[0]
try:
- return func(self, *args, **kwargs)
+ return func(*args, **kwargs)
except JDownloaderError as e:
msg = e.message
diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.py
index 2959f47ad..91077b58f 100644
--- a/cyberdrop_dl/ui/program_ui.py
+++ b/cyberdrop_dl/ui/program_ui.py
@@ -5,7 +5,7 @@ import sqlite3
import sys
from functools import wraps
from textwrap import dedent
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
import browser_cookie3
from requests import request
@@ -23,20 +23,23 @@ from cyberdrop_dl.utils.updates import check_latest_pypi
from cyberdrop_dl.utils.utilities import clear_term, open_in_text_editor
if TYPE_CHECKING:
+ from collections.abc import Callable
from pathlib import Path
from InquirerPy.base.control import Choice
from cyberdrop_dl.managers.manager import Manager
+P = ParamSpec("P")
+R = TypeVar("R")
console = Console()
ERROR_PREFIX = Text("ERROR: ", style="bold red")
-def repeat_until_done(func):
+def repeat_until_done(func: Callable[P, R]) -> Callable[P, R]:
@wraps(func)
- def wrapper(*args, **kwargs):
+ def wrapper(*args, **kwargs) -> R:
done = False
while not done:
done = func(*args, **kwargs)
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
index 7cfbf0a4f..dd303d858 100644
--- a/cyberdrop_dl/utils/cookie_management.py
+++ b/cyberdrop_dl/utils/cookie_management.py
@@ -4,7 +4,7 @@ import os
from functools import wraps
from http.cookiejar import CookieJar, MozillaCookieJar
from textwrap import dedent
-from typing import TYPE_CHECKING, NamedTuple
+from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVar
import browser_cookie3
@@ -14,6 +14,9 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.utils.constants import BROWSERS
+P = ParamSpec("P")
+R = TypeVar("R")
+
class CookieExtractor(NamedTuple):
name: str
@@ -28,11 +31,11 @@ COOKIE_ERROR_FOOTER = "\n\nNothing has been saved."
CHROMIUM_BROWSERS = ["chrome", "chromium", "opera", "opera_gx", "brave", "edge", "vivaldi", "arc"]
-def cookie_wrapper(func: Callable) -> Callable:
+def cookie_wrapper(func: Callable[P, R]) -> Callable[P, R]:
"""Wrapper handles errors for cookie extraction."""
@wraps(func)
- def wrapper(*args, **kwargs) -> None:
+ def wrapper(*args, **kwargs) -> R:
try:
return func(*args, **kwargs)
except PermissionError as e:
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 25fac0060..cf287c7de 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -10,7 +10,7 @@ import subprocess
from dataclasses import dataclass, fields
from functools import lru_cache, partial, wraps
from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, Protocol
+from typing import TYPE_CHECKING, Any, ClassVar, ParamSpec, Protocol, TypeVar
import aiofiles
import rich
@@ -31,7 +31,7 @@ from cyberdrop_dl.utils import constants
from cyberdrop_dl.utils.logger import log, log_debug, log_spacer, log_with_color
if TYPE_CHECKING:
- from collections.abc import Callable, Mapping
+ from collections.abc import Callable, Coroutine, Mapping
from curl_cffi.requests.models import Response as CurlResponse
from rich.text import Text
@@ -41,6 +41,11 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
TEXT_EDITORS = "micro", "nano", "vim" # Ordered by preference
FILENAME_REGEX = re.compile(r"filename\*=UTF-8''(.+)|.*filename=\"(.*?)\"", re.IGNORECASE)
subprocess_get_text = partial(subprocess.run, capture_output=True, text=True, check=False)
@@ -66,17 +71,18 @@ class OGProperties:
video: str = ""
-def error_handling_wrapper(func: Callable) -> Callable:
+def error_handling_wrapper(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R | None]]:
"""Wrapper handles errors for url scraping."""
@wraps(func)
- async def wrapper(self: Crawler | Downloader, *args, **kwargs):
- item: ScrapeItem | MediaItem | URL = args[0]
+ async def wrapper(*args, **kwargs) -> R | None:
+ self: Crawler | Downloader = args[0]
+ item: ScrapeItem | MediaItem | URL = args[1]
link: URL = item if isinstance(item, URL) else item.url
origin = exc_info = None
link_to_show: URL | str = ""
try:
- return await func(self, *args, **kwargs)
+ return await func(*args, **kwargs)
except CDLBaseError as e:
error_log_msg = ErrorLogMessage(e.ui_failure, str(e))
origin = e.origin