Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add crawler config class (#2042)
NTFSvolumea9fde2bbc5e3
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 642809ba6..23e9b433c 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -8,7 +8,7 @@ import timeimport warningsfrom contextvars import ContextVarfrom http import HTTPStatus-from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, Unpack, final+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, Self, Unpack, finalimport aiohttpfrom aiohttp import hdrs@@ -23,7 +23,7 @@ from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.cookies import make_simple_cookiefrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadErrorfrom cyberdrop_dl.utils import enter_context, truncated_preview-from cyberdrop_dl.utils.dataclass import DictDataclass+from cyberdrop_dl.utils.dataclass import ConfigDataclass, frozenif TYPE_CHECKING:import ssl@@ -431,15 +431,14 @@ type RateLimit = tuple[float, float]@final[email protected](slots=True, frozen=True)-class HTTPConfig:+@frozen+class HTTPConfig(ConfigDataclass):+ __attr_name__: ClassVar[str] = "__http_config__"headers: dict[str, str] | None = Noneimpersonate: str | bool | None = Nonerate_limit: RateLimit | None = Nonejson_check: JSONCheck | None = None- __iter__ = DictDataclass.__iter__-@classmethoddef default_headers(cls,@@ -459,12 +458,8 @@ class HTTPConfig:} | kwargsreturn HTTPConfig(headers={k: v for k, v in headers.items() if v is not None})- @classmethod- def get(cls, obj: object) -> HTTPConfig | None:- return getattr(obj, "__http_config__", None)-- def __or__(self, other: HTTPConfig) -> HTTPConfig:- changes = {k: v for k, v in other if v is not None}+ def __or__(self, other: Self) -> Self:+ changes = other._changes()if self.headers and other.headers:changes["headers"] = self.headers | other.headers@@ -474,13 +469,6 @@ class HTTPConfig:return dataclasses.replace(self, **changes)- def __call__[T](self, obj: type[T]) -> type[T]:- if config := self.get(obj):- obj.__http_config__ = config | self # pyright: ignore[reportAttributeAccessIssue]- else:- obj.__http_config__ = self # pyright: ignore[reportAttributeAccessIssue]- return obj-@final@dataclasses.dataclass(slots=True, frozen=True)diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex ebd2a9c76..ff815fd0a 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -4,7 +4,7 @@ import urllib.parsefrom typing import TYPE_CHECKING, Any, ClassVar, finalfrom cyberdrop_dl import aio-from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfigfrom cyberdrop_dl.exceptions import PasswordProtectedErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, json, open_graph@@ -18,6 +18,7 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem+@finalclass Selector:ITEM_DESCRIPTION = "p[class*=description-meta]"ITEM = "a.image-container"@@ -29,6 +30,7 @@ class Selector:MAIN_IMAGE = css.CssAttributeSelector("div#image-viewer img", "src")+@URLConfig(trim=False)class CheveretoCrawler(Crawler, is_generic=True):SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {"Album": (@@ -54,7 +56,6 @@ class CheveretoCrawler(Crawler, is_generic=True):"Direct links": "",}NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE- DEFAULT_TRIM_URLS: ClassVar[bool] = FalseCHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = Truedef __init_subclass__(cls, **kwargs: Any) -> None:diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 799d3a3fa..40a0ffa89 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -8,7 +8,7 @@ import refrom typing import TYPE_CHECKING, Any, ClassVar, finalfrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.exceptions import DownloadError, ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.utils import css, extr_text, open_graph, parse_url@@ -67,7 +67,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):super().__init_subclass__(**kwargs)if ensure_trailing_slash:cls.transform_url = cls.transform_kvs_url- cls.DEFAULT_TRIM_URLS = False+ _ = URLConfig(trim=False)(cls)@final@classmethoddiff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.pyindex 11133441f..198ce5e27 100644--- a/cyberdrop_dl/crawlers/_one_manager.py+++ b/cyberdrop_dl/crawlers/_one_manager.py@@ -7,9 +7,9 @@ Gitee: https://gitee.com/qkqpttgf/OneManager-phpfrom __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, final-from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfigfrom cyberdrop_dl.exceptions import InvalidContentTypeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -21,6 +21,7 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem+@finalclass Selector:TABLE = "table#list-table"FILE_LINK = "a.download"@@ -30,9 +31,9 @@ class Selector:DATE = "td.updated_at"+@URLConfig(allow_empty_path=True)class OneManagerCrawler(Crawler, is_abc=True):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Any path": "/..."}- ALLOW_EMPTY_PATH: ClassVar[bool] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:scrape_item.url = scrape_item.url.with_query(None)diff --git a/cyberdrop_dl/crawlers/apkmirror.py b/cyberdrop_dl/crawlers/apkmirror.pyindex 97167add6..81f4d4c6d 100644--- a/cyberdrop_dl/crawlers/apkmirror.py+++ b/cyberdrop_dl/crawlers/apkmirror.py@@ -5,7 +5,7 @@ import randomfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css@@ -23,12 +23,12 @@ class Selector:@HTTPConfig(rate_limit=(4, 1))+@URLConfig(trim=False)class APKMirrorCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"APK": "/apk/<developer>/<application>/<release>/<variant>-download"}DOMAIN: ClassVar[str] = "apkmirror.com"FOLDER_DOMAIN: ClassVar[str] = "APK Mirror"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.apkmirror.com")- DEFAULT_TRIM_URLS: ClassVar[bool] = Falseasync def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:diff --git a/cyberdrop_dl/crawlers/bunkr_albums.py b/cyberdrop_dl/crawlers/bunkr_albums.pyindex 095d1a839..84ae5d732 100644--- a/cyberdrop_dl/crawlers/bunkr_albums.py+++ b/cyberdrop_dl/crawlers/bunkr_albums.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -10,6 +10,7 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem+@URLConfig(allow_empty_path=True)class BunkrAlbumsCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Search": "/?search=<query>",@@ -19,7 +20,6 @@ class BunkrAlbumsCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "Bunkr-Albums"OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("bunkr-albums.io",)NEXT_PAGE_SELECTOR: ClassVar[str] = "a.btn-ghost:-soup-contains(Next)[href*='search=']"- ALLOW_EMPTY_PATH: ClassVar[bool] = Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex da640f470..3c282daf6 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -27,6 +27,7 @@ from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.models.validators import stringsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, is_absolute_http_urlfrom cyberdrop_dl.utils import css, dates, enter_context, is_blob_or_svg, m3u8, parse_url, unique+from cyberdrop_dl.utils.dataclass import ConfigDataclass, frozenfrom cyberdrop_dl.utils.errors import error_handling_contextif TYPE_CHECKING:@@ -138,19 +139,27 @@ class _CrawlerLogger(logging.LoggerAdapter[logging.Logger]):return f"[{self._crawler_name}] {msg}", kwargs+@final+@frozen+class URLConfig(ConfigDataclass):+ __attr_name__: ClassVar[str] = "__url_config__"+ trim: bool | None = None+ allow_empty_path: bool | None = None+ ignore_fragment: bool | None = None++@HTTPConfig(rate_limit=(25, 1))+@URLConfig(trim=True, allow_empty_path=False, ignore_fragment=True)class Crawler(HTTPMixin, HLSMixin, ABC):+ __url_config__: ClassVar[URLConfig]+DOMAIN: ClassVar[str]OLD_DOMAINS: ClassVar[tuple[str, ...]] = ()SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ()SUPPORTED_PATHS: ClassVar[SupportedPaths] = {}DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"-- UPDATE_UNSUPPORTED: ClassVar[bool] = False- ALLOW_EMPTY_PATH: ClassVar[bool] = FalseNEXT_PAGE_SELECTOR: ClassVar[str] = ""- DEFAULT_TRIM_URLS: ClassVar[bool] = TrueFOLDER_DOMAIN: ClassVar[str] = ""PRIMARY_URL: ClassVar[AbsoluteHttpURL]_FORUM: ClassVar[bool] = False@@ -158,7 +167,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):_DOWNLOAD_SLOTS: ClassVar[int | None] = None_SCRAPE_SLOTS: ClassVar[int] = 20_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False- _IGNORE_FRAGMENT: ClassVar[bool] = Truedisabled: bool = False@@ -175,10 +183,9 @@ class Crawler(HTTPMixin, HLSMixin, ABC):@staticmethoddef db_path_builder(key: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"]):- def apply[T: Crawler](c: type[T]) -> type[T]:- c.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])- c._IGNORE_FRAGMENT = "frag" not in key # pyright: ignore[reportConstantRedefinition]- return c+ def apply[T: Crawler](cls: type[T]) -> type[T]:+ cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])+ return URLConfig(ignore_fragment="frag" not in key)(cls)return apply@@ -360,11 +367,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):return 0return len(self._semaphore._waiters)- @final- @property- def deep_scrape(self) -> bool:- return self.config.deep_scrape-@final@propertydef origin(self) -> AbsoluteHttpURL:@@ -395,14 +397,14 @@ class Crawler(HTTPMixin, HLSMixin, ABC):with scrape_item.track_changes:scrape_item.url = url = self.transform_url(scrape_item.url)- lookup = url.path_qs if self._IGNORE_FRAGMENT else _path_qs_frag(url)+ lookup = url.path_qs if self.__url_config__.ignore_fragment else _path_qs_frag(url)if lookup in self._scraped_items:logger.info(f"Skipping {url} as it has already been scraped")returnself._scraped_items.add(lookup)- if not self.ALLOW_EMPTY_PATH and url.path == "/":+ if not self.__url_config__.allow_empty_path and url.path == "/":self.raise_exc(scrape_item, ScrapeError.unsupported())return@@ -442,14 +444,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):def is_subdomain(url: AbsoluteHttpURL) -> bool:return url.host.removeprefix("www.").count(".") > 1- @classmethod- def is_self_subdomain(cls, url: AbsoluteHttpURL) -> bool:- primary_domain = cls.PRIMARY_URL.host.removeprefix("www.")- other_domain = url.host.removeprefix("www.")- if primary_domain == other_domain:- return False- return primary_domain in other_domain and other_domain.count(".") > primary_domain.count(".")-@finalasync def write_metadata(self, scrape_item: ScrapeItem, name: str, metadata: object) -> None:"""Write general metadata (not specific to a single file) to json output"""@@ -703,17 +697,15 @@ class Crawler(HTTPMixin, HLSMixin, ABC):base = relative_to or cls.PRIMARY_URLassert is_absolute_http_url(base)if trim is None:- trim = cls.DEFAULT_TRIM_URLS+ trim = cls.__url_config__.trim+ if trim is None:+ trim = Truereturn parse_url(url, base, trim=trim)@final@propertydef cookies(self) -> SiteCookies:- return self.filter_cookies(self.PRIMARY_URL)-- @final- def filter_cookies(self, url: AbsoluteHttpURL) -> SiteCookies:- return SiteCookies(self.client.cookies.filter_cookies(url))+ return SiteCookies(self.client.cookies.filter_cookies(self.PRIMARY_URL))@finaldef update_cookies(self, cookies: dict[str, Any], url: yarl.URL | None = None) -> None:diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.pyindex 18a201c02..ecf51c97b 100644--- a/cyberdrop_dl/crawlers/doodstream.py+++ b/cyberdrop_dl/crawlers/doodstream.py@@ -49,7 +49,6 @@ class DoodStreamCrawler(Crawler):"dooodster.com",)PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://doodstream.com/")- UPDATE_UNSUPPORTED: ClassVar[bool] = TrueDOMAIN: ClassVar[str] = "doodstream"FOLDER_DOMAIN: ClassVar[str] = "DoodStream"diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.pyindex a3cff967f..ed3834e4c 100644--- a/cyberdrop_dl/crawlers/efukt.py+++ b/cyberdrop_dl/crawlers/efukt.py@@ -1,10 +1,10 @@from __future__ import annotationsimport dataclasses-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, finalfrom cyberdrop_dl import aio-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, dates, parse_urlfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -17,6 +17,7 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem+@finalclass Selector:DATE = "div.videobox span.stat:-soup-contains('Uploaded')"TITLE = "div.videobox > div.heading > h1"@@ -28,6 +29,7 @@ class Selector:MEDIA = f"{_IMAGE}, {_VIDEO}"+@URLConfig(allow_empty_path=True)class EfuktCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": "/...",@@ -40,7 +42,6 @@ class EfuktCrawler(Crawler):DOMAIN: ClassVar[str] = "efukt.com"FOLDER_DOMAIN: ClassVar[str] = "eFukt"NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE- ALLOW_EMPTY_PATH: ClassVar[bool] = TrueDEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} {title}"async def fetch(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.pyindex 7f1aeaf9c..6fbd5536c 100644--- a/cyberdrop_dl/crawlers/imagepond.py+++ b/cyberdrop_dl/crawlers/imagepond.py@@ -4,7 +4,7 @@ import dataclassesfrom typing import TYPE_CHECKING, ClassVar, Selffrom cyberdrop_dl import aio-from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -56,6 +56,7 @@ class File:)+@URLConfig(trim=False)class ImagePondCrawler(Crawler):SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {"Image / Video / Archive": (@@ -75,7 +76,6 @@ class ImagePondCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.imagepond.net")DOMAIN: ClassVar[str] = "imagepond.net"FOLDER_DOMAIN: ClassVar[str] = "ImagePond"- DEFAULT_TRIM_URLS: ClassVar[bool] = FalseNEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGEasync def fetch(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex 1bb043a33..39ea8fa5a 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar, Final, overridefrom cyberdrop_dl.crawlers._chevereto import CheveretoCrawler+from cyberdrop_dl.crawlers.crawler import URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLif TYPE_CHECKING:@@ -13,6 +14,7 @@ if TYPE_CHECKING:IMAGES_CDN: Final = "i.ibb.co"+@URLConfig(allow_empty_path=True)class ImgBBCrawler(CheveretoCrawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/album/<album_id>",@@ -23,7 +25,6 @@ class ImgBBCrawler(CheveretoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://ibb.co")DOMAIN: ClassVar[str] = "imgbb"FOLDER_DOMAIN: ClassVar[str] = "ImgBB"- ALLOW_EMPTY_PATH: ClassVar[bool] = True@overrideasync def _get_final_album_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.pyindex 380a95b86..b5cc56e35 100644--- a/cyberdrop_dl/crawlers/luxuretv.py+++ b/cyberdrop_dl/crawlers/luxuretv.py@@ -3,7 +3,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar, finalfrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -21,6 +21,7 @@ class Selector:@HTTPConfig(rate_limit=(3, 10))+@URLConfig(trim=False)class LuxureTVCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": "/videos/<name>-<id>.html",@@ -28,7 +29,6 @@ class LuxureTVCrawler(Crawler):}DOMAIN: ClassVar[str] = "luxuretv"FOLDER_DOMAIN: ClassVar[str] = "LuxureTV"- DEFAULT_TRIM_URLS: ClassVar[bool] = FalsePRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://luxuretv.com")NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGEdiff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 1d73f76b7..264306bc8 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -8,7 +8,7 @@ import itertoolsfrom http import HTTPStatusfrom typing import TYPE_CHECKING, Any, ClassVar, Literal-from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_id+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, URLConfig, auto_task_idfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, is_blob_or_svg@@ -51,6 +51,7 @@ _API_ERRORS_OVERRIDES: dict[int, int] = {@Crawler.db_path_builder("name")+@URLConfig(allow_empty_path=True)class MediaFireCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": (@@ -61,7 +62,6 @@ class MediaFireCrawler(Crawler):}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.mediafire.com/")DOMAIN: ClassVar[str] = "mediafire"- ALLOW_EMPTY_PATH: ClassVar[bool] = Truedef __post_init__(self) -> None:self.api: MediaFireAPI = MediaFireAPI.from_crawler(self)diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 9fe6bdf61..49ab6b317 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -14,7 +14,7 @@ from mega.data_structures import Cryptofrom cyberdrop_dl.clients.http import HTTPConfigfrom cyberdrop_dl.constants import CDL_USER_AGENT-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig, auto_task_idfrom cyberdrop_dl.downloader.mega_nz import MegaDownloaderfrom cyberdrop_dl.exceptions import LoginError, PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.progress.scraping import show_msg@@ -29,6 +29,7 @@ if TYPE_CHECKING:@HTTPConfig.default_headers(user_agent=CDL_USER_AGENT)+@URLConfig(allow_empty_path=True)@Crawler.db_path_builder("path_qs_frag")class MegaNzCrawler(Crawler):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "mega.io", "mega.nz"@@ -46,7 +47,6 @@ class MegaNzCrawler(Crawler):"**NOTE**": "Downloads can not be resumed. Partial downloads will always be deleted and new downloads will start over",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://mega.nz")- ALLOW_EMPTY_PATH: ClassVar[bool] = TrueDOMAIN: ClassVar[str] = "mega.nz"FOLDER_DOMAIN: ClassVar[str] = "MegaNz"OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("mega.co.nz",)diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex cc3b9a052..a51b65833 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -9,7 +9,7 @@ import functoolsfrom typing import TYPE_CHECKING, Any, ClassVar, Selffrom cyberdrop_dl.cache import disk_cached_method-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfigfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import dates@@ -85,6 +85,7 @@ class OneDriveFolder(OneDriveItem):return cls(**info)+@URLConfig(allow_empty_path=True)class OneDriveCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Access Link": "https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>",@@ -97,7 +98,6 @@ class OneDriveCrawler(Crawler):}SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = SHARE_LINK_HOST, "onedrive.live.com"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onedrive.com/")- ALLOW_EMPTY_PATH: ClassVar[bool] = TrueDOMAIN: ClassVar[str] = "onedrive"FOLDER_DOMAIN: ClassVar[str] = "OneDrive"diff --git a/cyberdrop_dl/crawlers/onefichier.py b/cyberdrop_dl/crawlers/onefichier.pyindex d3cc0d5c6..f02ddbe31 100644--- a/cyberdrop_dl/crawlers/onefichier.py+++ b/cyberdrop_dl/crawlers/onefichier.py@@ -3,7 +3,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar, final, overridefrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfigfrom cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css@@ -27,6 +27,7 @@ class Selector:@HTTPConfig(rate_limit=(1, 2))+@URLConfig(allow_empty_path=True)class OneFichierCrawler(Crawler):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ("1fichier.com",@@ -44,7 +45,7 @@ class OneFichierCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": "?<file_id>",}- ALLOW_EMPTY_PATH: ClassVar[bool] = True+DOMAIN: ClassVar[str] = "1fichier"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://1fichier.com")_DOWNLOAD_SLOTS: ClassVar[int | None] = 1diff --git a/cyberdrop_dl/crawlers/owncloud.py b/cyberdrop_dl/crawlers/owncloud.pyindex 7549e8d97..d36976455 100644--- a/cyberdrop_dl/crawlers/owncloud.py+++ b/cyberdrop_dl/crawlers/owncloud.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.utils import webdavfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -12,9 +12,9 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem+@URLConfig(trim=False)class OwnCloudCrawler(Crawler, is_generic=True):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Public Share": ("/s/<share_token>")}- DEFAULT_TRIM_URLS: ClassVar[bool] = Falseasync def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.pyindex 613f473c0..18f5185ef 100644--- a/cyberdrop_dl/crawlers/pimp_bunny.py+++ b/cyberdrop_dl/crawlers/pimp_bunny.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl import aiofrom cyberdrop_dl.crawlers._kvs import extract_kvs_video-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -43,6 +43,7 @@ def _pagination_query(url: AbsoluteHttpURL) -> dict[str, Any]:}+@URLConfig(trim=False)class PimpBunnyCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Videos": "/videos/...",@@ -56,7 +57,6 @@ class PimpBunnyCrawler(Crawler):DOMAIN: ClassVar[str] = "pimpbunny.com"FOLDER_DOMAIN: ClassVar[str] = "PimpBunny"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pimpbunny.com")- DEFAULT_TRIM_URLS: ClassVar[bool] = FalseNEXT_PAGE_SELECTOR: ClassVar[str] = ".pb-pagination-list .next a[href]"async def fetch(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.pyindex e50367b64..fe989fe2a 100644--- a/cyberdrop_dl/crawlers/pixhost.py+++ b/cyberdrop_dl/crawlers/pixhost.py@@ -24,7 +24,6 @@ class PixHostCrawler(Crawler):"Thumbnail": "/thumbs/..",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pixhost.to")- UPDATE_UNSUPPORTED: ClassVar[bool] = TrueDOMAIN: ClassVar[str] = "pixhost"FOLDER_DOMAIN: ClassVar[str] = "PixHost"OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("pixhost.org",)diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex e98cb385b..39fee92e9 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -4,7 +4,7 @@ import itertoolsfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers._kvs import extract_kvs_video-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfigfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css@@ -33,6 +33,7 @@ class Selector:TITLE_TRASH = "Free HD ", "Most Relevant ", "New ", "Videos", "Porn", "for:", "New Videos", "Tagged with"+@URLConfig(trim=False)class PorntrexCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": "/video/...",@@ -47,7 +48,6 @@ class PorntrexCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.porntrex.com")NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGEDOMAIN: ClassVar[str] = "porntrex"- DEFAULT_TRIM_URLS: ClassVar[bool] = Falseasync def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.name: # The ending slash is necessary or we get a 404 errordiff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.pyindex 950a2c3dd..f52149f06 100644--- a/cyberdrop_dl/crawlers/redgifs.py+++ b/cyberdrop_dl/crawlers/redgifs.py@@ -77,7 +77,7 @@ class RedGifsCrawler(Crawler):return await self.user(scrape_item, user_name.lower())case ["i" | "watch" | "ifr", gif_id]:return await self.gif(scrape_item, _id(gif_id))- case [_, _] if self.is_self_subdomain(scrape_item.url):+ case [_, _] if self.is_subdomain(scrape_item.url):scrape_item.url = _canonical_url(scrape_item.url.name)return await self.gif(scrape_item, scrape_item.url.name)case _:diff --git a/cyberdrop_dl/crawlers/thothub.py b/cyberdrop_dl/crawlers/thothub.pyindex ad50a242d..5b3e28ced 100644--- a/cyberdrop_dl/crawlers/thothub.py+++ b/cyberdrop_dl/crawlers/thothub.py@@ -1,8 +1,9 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, finalfrom cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler+from cyberdrop_dl.crawlers.crawler import URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -12,11 +13,13 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem+@finalclass Selector:TITLE = ".main-container .headline h1"VIDEOS = ".list-videos .item a"+@URLConfig(trim=False)class ThotHubCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/albums/<id>/<name>",@@ -24,7 +27,6 @@ class ThotHubCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):"Video": "/videos/<id>/<slug>",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://thothub.to")- DEFAULT_TRIM_URLS: ClassVar[bool] = FalseDOMAIN: ClassVar[str] = "thothub"FOLDER_DOMAIN: ClassVar[str] = "ThotHub"diff --git a/cyberdrop_dl/crawlers/xasiat.py b/cyberdrop_dl/crawlers/xasiat.pyindex e3ce21aa4..1ccdb913a 100644--- a/cyberdrop_dl/crawlers/xasiat.py+++ b/cyberdrop_dl/crawlers/xasiat.py@@ -3,12 +3,14 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler+from cyberdrop_dl.crawlers.crawler import URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItemif TYPE_CHECKING:from cyberdrop_dl.crawlers.crawler import SupportedPaths+@URLConfig(trim=False)class XasiatCrawler(KernelVideoSharingCrawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/albums/<id>/<name>",@@ -18,7 +20,6 @@ class XasiatCrawler(KernelVideoSharingCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.xasiat.com")DOMAIN: ClassVar[str] = "xasiat"- DEFAULT_TRIM_URLS: ClassVar[bool] = Falseasync def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:diff --git a/cyberdrop_dl/crawlers/yourlesbians.py b/cyberdrop_dl/crawlers/yourlesbians.pyindex ff6166043..f66e61697 100644--- a/cyberdrop_dl/crawlers/yourlesbians.py+++ b/cyberdrop_dl/crawlers/yourlesbians.py@@ -3,16 +3,17 @@ from __future__ import annotationsfrom typing import ClassVarfrom cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler+from cyberdrop_dl.crawlers.crawler import URLConfigfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.utils import open_graphfrom cyberdrop_dl.utils.errors import error_handling_wrapper+@URLConfig(trim=False)class YourLesbiansCrawler(KernelVideoSharingCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://yourlesbians.com")DOMAIN: ClassVar[str] = "yourlesbians.com"FOLDER_DOMAIN: ClassVar[str] = "YourLesbians"- DEFAULT_TRIM_URLS: ClassVar[bool] = False@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str | None = None) -> None:diff --git a/cyberdrop_dl/utils/dataclass.py b/cyberdrop_dl/utils/dataclass.pyindex 876f54198..453d9317c 100644--- a/cyberdrop_dl/utils/dataclass.py+++ b/cyberdrop_dl/utils/dataclass.py@@ -1,7 +1,8 @@from __future__ import annotationsimport dataclasses-from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self+import sys+from typing import TYPE_CHECKING, Any, ClassVar, Final, Protocol, Self, dataclass_transform, overloadfrom cyberdrop_dl.constants import MISSINGfrom cyberdrop_dl.utils import fast_cache@@ -78,3 +79,41 @@ class DictDataclass(_DataClass, Protocol):if overrides:data.update(overrides)return cls(**data)+++@overload+def frozen[T](cls: None = None, *, order: bool = False) -> Callable[[type[T]], type[T]]: ...+++@overload+def frozen[T](cls: type[T], *, order: bool = False) -> type[T]: ...+++@dataclass_transform(frozen_default=True, kw_only_default=True)+def frozen[T](cls: type[T] | None = None, *, order: bool = False) -> Callable[[type[T]], type[T]] | type[T]:+ fn = dataclasses.dataclass(frozen=True, kw_only=True, order=order, slots=True)+ return fn if cls is None else fn(cls)+++@frozen+class ConfigDataclass:+ __attr_name__: ClassVar[str]+ __iter__: ClassVar[Final] = DictDataclass.__iter__++ def _changes(self) -> dict[str, Any]:+ return {k: v for k, v in self if v is not None}++ @classmethod+ def get(cls, obj: object) -> Self | None:+ return getattr(obj, cls.__attr_name__, None)++ if sys.version_info < (3, 13, 0):+ __replace__ = dataclasses.replace++ def __or__(self, other: Self) -> Self:+ return self.__replace__(**other._changes())++ def __call__[T](self, obj: type[T]) -> type[T]:+ cfg = current_cfg | self if (current_cfg := self.get(obj)) is not None else self+ setattr(obj, self.__attr_name__, cfg)+ return objdiff --git a/tests/test_client.py b/tests/test_client.pyindex 3723d4ef9..4e649aa82 100644--- a/tests/test_client.py+++ b/tests/test_client.py@@ -107,3 +107,9 @@ def test_http_config_as_decorator() -> None:B = config_2(A)assert HTTPConfig.get(B) == config_1 | config_2+++def test_http_config_is_frozen() -> None:+ config_1 = HTTPConfig(headers={"a": "b", "h": "l"}, rate_limit=(3, 1))+ with pytest.raises(AttributeError):+ config_1.rate_limit = None # pyright: ignore[reportAttributeAccessIssue]