Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add signature module (#12)

NTFSvolume831abcf62d64
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 307dd0d4f..122262e34 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -16,10 +16,10 @@ import yarl
from aiolimiter import AsyncLimiter
from yarl import URL
-from cyberdrop_dl import constants
+from cyberdrop_dl import constants, signature
from cyberdrop_dl.clients.scraper_client import ScraperClient
from cyberdrop_dl.data_structures.mediaprops import ISO639Subtitle, Resolution
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, copy_signature
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.utils import css, dates, m3u8
@@ -129,7 +129,7 @@ class Crawler(ABC):
create_db_path = staticmethod(DBPathBuilder.path)
- @copy_signature(ScraperClient._request)
+ @signature.copy(ScraperClient._request)
@contextlib.asynccontextmanager
async def request(self, *args, impersonate: str | bool | None = None, **kwargs) -> AsyncGenerator[AbstractResponse]:
if impersonate is None:
@@ -141,17 +141,17 @@ class Crawler(ABC):
):
yield resp
- @copy_signature(ScraperClient._request)
+ @signature.copy(ScraperClient._request)
async def request_json(self, *args, **kwargs) -> Any:
async with self.request(*args, **kwargs) as resp:
return await resp.json()
- @copy_signature(ScraperClient._request)
+ @signature.copy(ScraperClient._request)
async def request_soup(self, *args, **kwargs) -> BeautifulSoup:
async with self.request(*args, **kwargs) as resp:
return await resp.soup()
- @copy_signature(ScraperClient._request)
+ @signature.copy(ScraperClient._request)
async def request_text(self, *args, **kwargs) -> str:
async with self.request(*args, **kwargs) as resp:
return await resp.text()
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index cab1d5463..d4894ebcc 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -1,4 +1,4 @@
-# type: ignore[reportIncompatibleVariableOverride]
+# pyright: ignore[reportIncompatibleVariableOverride]
from __future__ import annotations
import copy
@@ -6,45 +6,28 @@ import datetime
from dataclasses import asdict, dataclass, field
from enum import IntEnum
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, ParamSpec, Self, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Self, overload
import yarl
if TYPE_CHECKING:
- _P = ParamSpec("_P")
- _R = TypeVar("_R")
- _T = TypeVar("_T")
- import functools
- import inspect
from collections.abc import Callable
import aiohttp
from propcache.api import under_cached_property as cached_property
from rich.progress import TaskID
+ from cyberdrop_dl import signature
from cyberdrop_dl.managers.manager import Manager
- def copy_signature(target: Callable[_P, _R]) -> Callable[[Callable[..., _T]], Callable[_P, _T]]:
- def decorator(func: Callable[..., _T]) -> Callable[_P, _T]:
- @functools.wraps(func)
- def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:
- return func(*args, **kwargs)
-
- wrapper.__signature__ = inspect.signature(target).replace( # pyright: ignore[reportAttributeAccessIssue]
- return_annotation=inspect.signature(func).return_annotation
- )
- return wrapper
-
- return decorator
-
class AbsoluteHttpURL(yarl.URL):
- @copy_signature(yarl.URL.__new__)
+ @signature.copy(yarl.URL.__new__)
def __new__(cls) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.__truediv__)
+ @signature.copy(yarl.URL.__truediv__)
def __truediv__(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.__mod__)
+ @signature.copy(yarl.URL.__mod__)
def __mod__(self) -> AbsoluteHttpURL: ...
@cached_property
@@ -59,13 +42,13 @@ if TYPE_CHECKING:
@cached_property
def parent(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.with_path)
+ @signature.copy(yarl.URL.with_path)
def with_path(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.with_host)
+ @signature.copy(yarl.URL.with_host)
def with_host(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.origin)
+ @signature.copy(yarl.URL.origin)
def origin(self) -> AbsoluteHttpURL: ...
@overload
@@ -74,7 +57,7 @@ if TYPE_CHECKING:
@overload
def with_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.with_query)
+ @signature.copy(yarl.URL.with_query)
def with_query(self) -> AbsoluteHttpURL: ...
@overload
@@ -83,7 +66,7 @@ if TYPE_CHECKING:
@overload
def extend_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.extend_query)
+ @signature.copy(yarl.URL.extend_query)
def extend_query(self) -> AbsoluteHttpURL: ...
@overload
@@ -92,36 +75,30 @@ if TYPE_CHECKING:
@overload
def update_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.update_query)
+ @signature.copy(yarl.URL.update_query)
def update_query(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.without_query_params)
+ @signature.copy(yarl.URL.without_query_params)
def without_query_params(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.with_fragment)
+ @signature.copy(yarl.URL.with_fragment)
def with_fragment(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.with_name)
+ @signature.copy(yarl.URL.with_name)
def with_name(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.with_suffix)
+ @signature.copy(yarl.URL.with_suffix)
def with_suffix(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.join)
+ @signature.copy(yarl.URL.join)
def join(self) -> AbsoluteHttpURL: ...
- @copy_signature(yarl.URL.joinpath)
+ @signature.copy(yarl.URL.joinpath)
def joinpath(self) -> AbsoluteHttpURL: ...
else:
AbsoluteHttpURL = yarl.URL
- def copy_signature(_):
- def call(y):
- return y
-
- return call
-
class ScrapeItemType(IntEnum):
FORUM = 0
diff --git a/cyberdrop_dl/signature.py b/cyberdrop_dl/signature.py
new file mode 100644
index 000000000..5c4ecbab5
--- /dev/null
+++ b/cyberdrop_dl/signature.py
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ParamSpec, TypeVar
+
+if TYPE_CHECKING:
+ _P = ParamSpec("_P")
+ _R = TypeVar("_R")
+ _T = TypeVar("_T")
+ import functools
+ import inspect
+ from collections.abc import Callable
+
+ def copy(target: Callable[_P, _R]) -> Callable[[Callable[..., _T]], Callable[_P, _T]]:
+ def decorator(func: Callable[..., _T]) -> Callable[_P, _T]:
+ @functools.wraps(func)
+ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+ return func(*args, **kwargs)
+
+ wrapper.__signature__ = inspect.signature(target).replace( # pyright: ignore[reportAttributeAccessIssue]
+ return_annotation=inspect.signature(func).return_annotation
+ )
+ return wrapper
+
+ return decorator
+
+else:
+
+ def copy(_):
+ def call(y):
+ return y
+
+ return call
+
+
+def easy_repr(*fields: str) -> Callable[..., str]:
+ def _repr_(self: object) -> str:
+ params = ", ".join(f"{f}={getattr(self, f)!r}" for f in fields)
+ return f"{type(self).__name__}({params})"
+
+ return _repr_