Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add signature module (#12)
NTFSvolume831abcf62d64
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 307dd0d4f..122262e34 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -16,10 +16,10 @@ import yarlfrom aiolimiter import AsyncLimiterfrom yarl import URL-from cyberdrop_dl import constants+from cyberdrop_dl import constants, signaturefrom cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.mediaprops import ISO639Subtitle, Resolution-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.utils import css, dates, m3u8@@ -129,7 +129,7 @@ class Crawler(ABC):create_db_path = staticmethod(DBPathBuilder.path)- @copy_signature(ScraperClient._request)+ @signature.copy(ScraperClient._request)@contextlib.asynccontextmanagerasync def request(self, *args, impersonate: str | bool | None = None, **kwargs) -> AsyncGenerator[AbstractResponse]:if impersonate is None:@@ -141,17 +141,17 @@ class Crawler(ABC):):yield resp- @copy_signature(ScraperClient._request)+ @signature.copy(ScraperClient._request)async def request_json(self, *args, **kwargs) -> Any:async with self.request(*args, **kwargs) as resp:return await resp.json()- @copy_signature(ScraperClient._request)+ @signature.copy(ScraperClient._request)async def request_soup(self, *args, **kwargs) -> BeautifulSoup:async with self.request(*args, **kwargs) as resp:return await resp.soup()- @copy_signature(ScraperClient._request)+ @signature.copy(ScraperClient._request)async def request_text(self, *args, **kwargs) -> str:async with self.request(*args, **kwargs) as resp:return await resp.text()diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex cab1d5463..d4894ebcc 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -1,4 +1,4 @@-# type: ignore[reportIncompatibleVariableOverride]+# pyright: ignore[reportIncompatibleVariableOverride]from __future__ import annotationsimport copy@@ -6,45 +6,28 @@ import datetimefrom dataclasses import asdict, dataclass, fieldfrom enum import IntEnumfrom pathlib import Path-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, ParamSpec, Self, TypeVar, overload+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Self, overloadimport yarlif TYPE_CHECKING:- _P = ParamSpec("_P")- _R = TypeVar("_R")- _T = TypeVar("_T")- import functools- import inspectfrom collections.abc import Callableimport aiohttpfrom propcache.api import under_cached_property as cached_propertyfrom rich.progress import TaskID+ from cyberdrop_dl import signaturefrom cyberdrop_dl.managers.manager import Manager- def copy_signature(target: Callable[_P, _R]) -> Callable[[Callable[..., _T]], Callable[_P, _T]]:- def decorator(func: Callable[..., _T]) -> Callable[_P, _T]:- @functools.wraps(func)- def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:- return func(*args, **kwargs)-- wrapper.__signature__ = inspect.signature(target).replace( # pyright: ignore[reportAttributeAccessIssue]- return_annotation=inspect.signature(func).return_annotation- )- return wrapper-- return decorator-class AbsoluteHttpURL(yarl.URL):- @copy_signature(yarl.URL.__new__)+ @signature.copy(yarl.URL.__new__)def __new__(cls) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.__truediv__)+ @signature.copy(yarl.URL.__truediv__)def __truediv__(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.__mod__)+ @signature.copy(yarl.URL.__mod__)def __mod__(self) -> AbsoluteHttpURL: ...@cached_property@@ -59,13 +42,13 @@ if TYPE_CHECKING:@cached_propertydef parent(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.with_path)+ @signature.copy(yarl.URL.with_path)def with_path(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.with_host)+ @signature.copy(yarl.URL.with_host)def with_host(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.origin)+ @signature.copy(yarl.URL.origin)def origin(self) -> AbsoluteHttpURL: ...@overload@@ -74,7 +57,7 @@ if TYPE_CHECKING:@overloaddef with_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.with_query)+ @signature.copy(yarl.URL.with_query)def with_query(self) -> AbsoluteHttpURL: ...@overload@@ -83,7 +66,7 @@ if TYPE_CHECKING:@overloaddef extend_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.extend_query)+ @signature.copy(yarl.URL.extend_query)def extend_query(self) -> AbsoluteHttpURL: ...@overload@@ -92,36 +75,30 @@ if TYPE_CHECKING:@overloaddef update_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.update_query)+ @signature.copy(yarl.URL.update_query)def update_query(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.without_query_params)+ @signature.copy(yarl.URL.without_query_params)def without_query_params(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.with_fragment)+ @signature.copy(yarl.URL.with_fragment)def with_fragment(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.with_name)+ @signature.copy(yarl.URL.with_name)def with_name(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.with_suffix)+ @signature.copy(yarl.URL.with_suffix)def with_suffix(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.join)+ @signature.copy(yarl.URL.join)def join(self) -> AbsoluteHttpURL: ...- @copy_signature(yarl.URL.joinpath)+ @signature.copy(yarl.URL.joinpath)def joinpath(self) -> AbsoluteHttpURL: ...else:AbsoluteHttpURL = yarl.URL- def copy_signature(_):- def call(y):- return y-- return call-class ScrapeItemType(IntEnum):FORUM = 0diff --git a/cyberdrop_dl/signature.py b/cyberdrop_dl/signature.pynew file mode 100644index 000000000..5c4ecbab5--- /dev/null+++ b/cyberdrop_dl/signature.py@@ -0,0 +1,40 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ParamSpec, TypeVar++if TYPE_CHECKING:+ _P = ParamSpec("_P")+ _R = TypeVar("_R")+ _T = TypeVar("_T")+ import functools+ import inspect+ from collections.abc import Callable++ def copy(target: Callable[_P, _R]) -> Callable[[Callable[..., _T]], Callable[_P, _T]]:+ def decorator(func: Callable[..., _T]) -> Callable[_P, _T]:+ @functools.wraps(func)+ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:+ return func(*args, **kwargs)++ wrapper.__signature__ = inspect.signature(target).replace( # pyright: ignore[reportAttributeAccessIssue]+ return_annotation=inspect.signature(func).return_annotation+ )+ return wrapper++ return decorator++else:++ def copy(_):+ def call(y):+ return y++ return call+++def easy_repr(*fields: str) -> Callable[..., str]:+ def _repr_(self: object) -> str:+ params = ", ".join(f"{f}={getattr(self, f)!r}" for f in fields)+ return f"{type(self).__name__}({params})"++ return _repr_