Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add crawler config class (#2042)

NTFSvolumea9fde2bbc5e3
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index 642809ba6..23e9b433c 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -8,7 +8,7 @@ import time
import warnings
from contextvars import ContextVar
from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, Unpack, final
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, Self, Unpack, final
import aiohttp
from aiohttp import hdrs
@@ -23,7 +23,7 @@ from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.cookies import make_simple_cookie
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError
from cyberdrop_dl.utils import enter_context, truncated_preview
-from cyberdrop_dl.utils.dataclass import DictDataclass
+from cyberdrop_dl.utils.dataclass import ConfigDataclass, frozen
if TYPE_CHECKING:
import ssl
@@ -431,15 +431,14 @@ type RateLimit = tuple[float, float]
@final
[email protected](slots=True, frozen=True)
-class HTTPConfig:
+@frozen
+class HTTPConfig(ConfigDataclass):
+ __attr_name__: ClassVar[str] = "__http_config__"
headers: dict[str, str] | None = None
impersonate: str | bool | None = None
rate_limit: RateLimit | None = None
json_check: JSONCheck | None = None
- __iter__ = DictDataclass.__iter__
-
@classmethod
def default_headers(
cls,
@@ -459,12 +458,8 @@ class HTTPConfig:
} | kwargs
return HTTPConfig(headers={k: v for k, v in headers.items() if v is not None})
- @classmethod
- def get(cls, obj: object) -> HTTPConfig | None:
- return getattr(obj, "__http_config__", None)
-
- def __or__(self, other: HTTPConfig) -> HTTPConfig:
- changes = {k: v for k, v in other if v is not None}
+ def __or__(self, other: Self) -> Self:
+ changes = other._changes()
if self.headers and other.headers:
changes["headers"] = self.headers | other.headers
@@ -474,13 +469,6 @@ class HTTPConfig:
return dataclasses.replace(self, **changes)
- def __call__[T](self, obj: type[T]) -> type[T]:
- if config := self.get(obj):
- obj.__http_config__ = config | self # pyright: ignore[reportAttributeAccessIssue]
- else:
- obj.__http_config__ = self # pyright: ignore[reportAttributeAccessIssue]
- return obj
-
@final
@dataclasses.dataclass(slots=True, frozen=True)
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index ebd2a9c76..ff815fd0a 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -4,7 +4,7 @@ import urllib.parse
from typing import TYPE_CHECKING, Any, ClassVar, final
from cyberdrop_dl import aio
-from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfig
from cyberdrop_dl.exceptions import PasswordProtectedError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, json, open_graph
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
+@final
class Selector:
ITEM_DESCRIPTION = "p[class*=description-meta]"
ITEM = "a.image-container"
@@ -29,6 +30,7 @@ class Selector:
MAIN_IMAGE = css.CssAttributeSelector("div#image-viewer img", "src")
+@URLConfig(trim=False)
class CheveretoCrawler(Crawler, is_generic=True):
SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {
"Album": (
@@ -54,7 +56,6 @@ class CheveretoCrawler(Crawler, is_generic=True):
"Direct links": "",
}
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = True
def __init_subclass__(cls, **kwargs: Any) -> None:
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 799d3a3fa..40a0ffa89 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -8,7 +8,7 @@ import re
from typing import TYPE_CHECKING, Any, ClassVar, final
from cyberdrop_dl.clients.http import HTTPConfig
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.exceptions import DownloadError, ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.utils import css, extr_text, open_graph, parse_url
@@ -67,7 +67,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
super().__init_subclass__(**kwargs)
if ensure_trailing_slash:
cls.transform_url = cls.transform_kvs_url
- cls.DEFAULT_TRIM_URLS = False
+ _ = URLConfig(trim=False)(cls)
@final
@classmethod
diff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.py
index 11133441f..198ce5e27 100644
--- a/cyberdrop_dl/crawlers/_one_manager.py
+++ b/cyberdrop_dl/crawlers/_one_manager.py
@@ -7,9 +7,9 @@ Gitee: https://gitee.com/qkqpttgf/OneManager-php
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, final
-from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfig
from cyberdrop_dl.exceptions import InvalidContentTypeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
+@final
class Selector:
TABLE = "table#list-table"
FILE_LINK = "a.download"
@@ -30,9 +31,9 @@ class Selector:
DATE = "td.updated_at"
+@URLConfig(allow_empty_path=True)
class OneManagerCrawler(Crawler, is_abc=True):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Any path": "/..."}
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
scrape_item.url = scrape_item.url.with_query(None)
diff --git a/cyberdrop_dl/crawlers/apkmirror.py b/cyberdrop_dl/crawlers/apkmirror.py
index 97167add6..81f4d4c6d 100644
--- a/cyberdrop_dl/crawlers/apkmirror.py
+++ b/cyberdrop_dl/crawlers/apkmirror.py
@@ -5,7 +5,7 @@ import random
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.clients.http import HTTPConfig
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
@@ -23,12 +23,12 @@ class Selector:
@HTTPConfig(rate_limit=(4, 1))
+@URLConfig(trim=False)
class APKMirrorCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"APK": "/apk/<developer>/<application>/<release>/<variant>-download"}
DOMAIN: ClassVar[str] = "apkmirror.com"
FOLDER_DOMAIN: ClassVar[str] = "APK Mirror"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.apkmirror.com")
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
diff --git a/cyberdrop_dl/crawlers/bunkr_albums.py b/cyberdrop_dl/crawlers/bunkr_albums.py
index 095d1a839..84ae5d732 100644
--- a/cyberdrop_dl/crawlers/bunkr_albums.py
+++ b/cyberdrop_dl/crawlers/bunkr_albums.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -10,6 +10,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
+@URLConfig(allow_empty_path=True)
class BunkrAlbumsCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Search": "/?search=<query>",
@@ -19,7 +20,6 @@ class BunkrAlbumsCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "Bunkr-Albums"
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("bunkr-albums.io",)
NEXT_PAGE_SELECTOR: ClassVar[str] = "a.btn-ghost:-soup-contains(Next)[href*='search=']"
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index da640f470..3c282daf6 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -27,6 +27,7 @@ from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.models.validators import strings
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, is_absolute_http_url
from cyberdrop_dl.utils import css, dates, enter_context, is_blob_or_svg, m3u8, parse_url, unique
+from cyberdrop_dl.utils.dataclass import ConfigDataclass, frozen
from cyberdrop_dl.utils.errors import error_handling_context
if TYPE_CHECKING:
@@ -138,19 +139,27 @@ class _CrawlerLogger(logging.LoggerAdapter[logging.Logger]):
return f"[{self._crawler_name}] {msg}", kwargs
+@final
+@frozen
+class URLConfig(ConfigDataclass):
+ __attr_name__: ClassVar[str] = "__url_config__"
+ trim: bool | None = None
+ allow_empty_path: bool | None = None
+ ignore_fragment: bool | None = None
+
+
@HTTPConfig(rate_limit=(25, 1))
+@URLConfig(trim=True, allow_empty_path=False, ignore_fragment=True)
class Crawler(HTTPMixin, HLSMixin, ABC):
+ __url_config__: ClassVar[URLConfig]
+
DOMAIN: ClassVar[str]
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ()
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ()
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {}
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"
-
- UPDATE_UNSUPPORTED: ClassVar[bool] = False
- ALLOW_EMPTY_PATH: ClassVar[bool] = False
NEXT_PAGE_SELECTOR: ClassVar[str] = ""
- DEFAULT_TRIM_URLS: ClassVar[bool] = True
FOLDER_DOMAIN: ClassVar[str] = ""
PRIMARY_URL: ClassVar[AbsoluteHttpURL]
_FORUM: ClassVar[bool] = False
@@ -158,7 +167,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
_DOWNLOAD_SLOTS: ClassVar[int | None] = None
_SCRAPE_SLOTS: ClassVar[int] = 20
_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False
- _IGNORE_FRAGMENT: ClassVar[bool] = True
disabled: bool = False
@@ -175,10 +183,9 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
@staticmethod
def db_path_builder(key: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"]):
- def apply[T: Crawler](c: type[T]) -> type[T]:
- c.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])
- c._IGNORE_FRAGMENT = "frag" not in key # pyright: ignore[reportConstantRedefinition]
- return c
+ def apply[T: Crawler](cls: type[T]) -> type[T]:
+ cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])
+ return URLConfig(ignore_fragment="frag" not in key)(cls)
return apply
@@ -360,11 +367,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
return 0
return len(self._semaphore._waiters)
- @final
- @property
- def deep_scrape(self) -> bool:
- return self.config.deep_scrape
-
@final
@property
def origin(self) -> AbsoluteHttpURL:
@@ -395,14 +397,14 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
with scrape_item.track_changes:
scrape_item.url = url = self.transform_url(scrape_item.url)
- lookup = url.path_qs if self._IGNORE_FRAGMENT else _path_qs_frag(url)
+ lookup = url.path_qs if self.__url_config__.ignore_fragment else _path_qs_frag(url)
if lookup in self._scraped_items:
logger.info(f"Skipping {url} as it has already been scraped")
return
self._scraped_items.add(lookup)
- if not self.ALLOW_EMPTY_PATH and url.path == "/":
+ if not self.__url_config__.allow_empty_path and url.path == "/":
self.raise_exc(scrape_item, ScrapeError.unsupported())
return
@@ -442,14 +444,6 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
def is_subdomain(url: AbsoluteHttpURL) -> bool:
return url.host.removeprefix("www.").count(".") > 1
- @classmethod
- def is_self_subdomain(cls, url: AbsoluteHttpURL) -> bool:
- primary_domain = cls.PRIMARY_URL.host.removeprefix("www.")
- other_domain = url.host.removeprefix("www.")
- if primary_domain == other_domain:
- return False
- return primary_domain in other_domain and other_domain.count(".") > primary_domain.count(".")
-
@final
async def write_metadata(self, scrape_item: ScrapeItem, name: str, metadata: object) -> None:
"""Write general metadata (not specific to a single file) to json output"""
@@ -703,17 +697,15 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
base = relative_to or cls.PRIMARY_URL
assert is_absolute_http_url(base)
if trim is None:
- trim = cls.DEFAULT_TRIM_URLS
+ trim = cls.__url_config__.trim
+ if trim is None:
+ trim = True
return parse_url(url, base, trim=trim)
@final
@property
def cookies(self) -> SiteCookies:
- return self.filter_cookies(self.PRIMARY_URL)
-
- @final
- def filter_cookies(self, url: AbsoluteHttpURL) -> SiteCookies:
- return SiteCookies(self.client.cookies.filter_cookies(url))
+ return SiteCookies(self.client.cookies.filter_cookies(self.PRIMARY_URL))
@final
def update_cookies(self, cookies: dict[str, Any], url: yarl.URL | None = None) -> None:
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.py
index 18a201c02..ecf51c97b 100644
--- a/cyberdrop_dl/crawlers/doodstream.py
+++ b/cyberdrop_dl/crawlers/doodstream.py
@@ -49,7 +49,6 @@ class DoodStreamCrawler(Crawler):
"dooodster.com",
)
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://doodstream.com/")
- UPDATE_UNSUPPORTED: ClassVar[bool] = True
DOMAIN: ClassVar[str] = "doodstream"
FOLDER_DOMAIN: ClassVar[str] = "DoodStream"
diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.py
index a3cff967f..ed3834e4c 100644
--- a/cyberdrop_dl/crawlers/efukt.py
+++ b/cyberdrop_dl/crawlers/efukt.py
@@ -1,10 +1,10 @@
from __future__ import annotations
import dataclasses
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, final
from cyberdrop_dl import aio
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, dates, parse_url
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
+@final
class Selector:
DATE = "div.videobox span.stat:-soup-contains('Uploaded')"
TITLE = "div.videobox > div.heading > h1"
@@ -28,6 +29,7 @@ class Selector:
MEDIA = f"{_IMAGE}, {_VIDEO}"
+@URLConfig(allow_empty_path=True)
class EfuktCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Video": "/...",
@@ -40,7 +42,6 @@ class EfuktCrawler(Crawler):
DOMAIN: ClassVar[str] = "efukt.com"
FOLDER_DOMAIN: ClassVar[str] = "eFukt"
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} {title}"
async def fetch(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.py
index 7f1aeaf9c..6fbd5536c 100644
--- a/cyberdrop_dl/crawlers/imagepond.py
+++ b/cyberdrop_dl/crawlers/imagepond.py
@@ -4,7 +4,7 @@ import dataclasses
from typing import TYPE_CHECKING, ClassVar, Self
from cyberdrop_dl import aio
-from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -56,6 +56,7 @@ class File:
)
+@URLConfig(trim=False)
class ImagePondCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {
"Image / Video / Archive": (
@@ -75,7 +76,6 @@ class ImagePondCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.imagepond.net")
DOMAIN: ClassVar[str] = "imagepond.net"
FOLDER_DOMAIN: ClassVar[str] = "ImagePond"
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
async def fetch(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index 1bb043a33..39ea8fa5a 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar, Final, override
from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
+from cyberdrop_dl.crawlers.crawler import URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
if TYPE_CHECKING:
@@ -13,6 +14,7 @@ if TYPE_CHECKING:
IMAGES_CDN: Final = "i.ibb.co"
+@URLConfig(allow_empty_path=True)
class ImgBBCrawler(CheveretoCrawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Album": "/album/<album_id>",
@@ -23,7 +25,6 @@ class ImgBBCrawler(CheveretoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://ibb.co")
DOMAIN: ClassVar[str] = "imgbb"
FOLDER_DOMAIN: ClassVar[str] = "ImgBB"
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
@override
async def _get_final_album_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.py
index 380a95b86..b5cc56e35 100644
--- a/cyberdrop_dl/crawlers/luxuretv.py
+++ b/cyberdrop_dl/crawlers/luxuretv.py
@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar, final
from cyberdrop_dl.clients.http import HTTPConfig
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -21,6 +21,7 @@ class Selector:
@HTTPConfig(rate_limit=(3, 10))
+@URLConfig(trim=False)
class LuxureTVCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Video": "/videos/<name>-<id>.html",
@@ -28,7 +29,6 @@ class LuxureTVCrawler(Crawler):
}
DOMAIN: ClassVar[str] = "luxuretv"
FOLDER_DOMAIN: ClassVar[str] = "LuxureTV"
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://luxuretv.com")
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 1d73f76b7..264306bc8 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -8,7 +8,7 @@ import itertools
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, ClassVar, Literal
-from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_id
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, URLConfig, auto_task_id
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, is_blob_or_svg
@@ -51,6 +51,7 @@ _API_ERRORS_OVERRIDES: dict[int, int] = {
@Crawler.db_path_builder("name")
+@URLConfig(allow_empty_path=True)
class MediaFireCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"File": (
@@ -61,7 +62,6 @@ class MediaFireCrawler(Crawler):
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.mediafire.com/")
DOMAIN: ClassVar[str] = "mediafire"
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
def __post_init__(self) -> None:
self.api: MediaFireAPI = MediaFireAPI.from_crawler(self)
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 9fe6bdf61..49ab6b317 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -14,7 +14,7 @@ from mega.data_structures import Crypto
from cyberdrop_dl.clients.http import HTTPConfig
from cyberdrop_dl.constants import CDL_USER_AGENT
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig, auto_task_id
from cyberdrop_dl.downloader.mega_nz import MegaDownloader
from cyberdrop_dl.exceptions import LoginError, PasswordProtectedError, ScrapeError
from cyberdrop_dl.progress.scraping import show_msg
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
@HTTPConfig.default_headers(user_agent=CDL_USER_AGENT)
+@URLConfig(allow_empty_path=True)
@Crawler.db_path_builder("path_qs_frag")
class MegaNzCrawler(Crawler):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "mega.io", "mega.nz"
@@ -46,7 +47,6 @@ class MegaNzCrawler(Crawler):
"**NOTE**": "Downloads can not be resumed. Partial downloads will always be deleted and new downloads will start over",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://mega.nz")
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
DOMAIN: ClassVar[str] = "mega.nz"
FOLDER_DOMAIN: ClassVar[str] = "MegaNz"
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("mega.co.nz",)
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index cc3b9a052..a51b65833 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -9,7 +9,7 @@ import functools
from typing import TYPE_CHECKING, Any, ClassVar, Self
from cyberdrop_dl.cache import disk_cached_method
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import dates
@@ -85,6 +85,7 @@ class OneDriveFolder(OneDriveItem):
return cls(**info)
+@URLConfig(allow_empty_path=True)
class OneDriveCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Access Link": "https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>",
@@ -97,7 +98,6 @@ class OneDriveCrawler(Crawler):
}
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = SHARE_LINK_HOST, "onedrive.live.com"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onedrive.com/")
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
DOMAIN: ClassVar[str] = "onedrive"
FOLDER_DOMAIN: ClassVar[str] = "OneDrive"
diff --git a/cyberdrop_dl/crawlers/onefichier.py b/cyberdrop_dl/crawlers/onefichier.py
index d3cc0d5c6..f02ddbe31 100644
--- a/cyberdrop_dl/crawlers/onefichier.py
+++ b/cyberdrop_dl/crawlers/onefichier.py
@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar, final, override
from cyberdrop_dl.clients.http import HTTPConfig
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig
from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
@@ -27,6 +27,7 @@ class Selector:
@HTTPConfig(rate_limit=(1, 2))
+@URLConfig(allow_empty_path=True)
class OneFichierCrawler(Crawler):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
"1fichier.com",
@@ -44,7 +45,7 @@ class OneFichierCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"File": "?<file_id>",
}
- ALLOW_EMPTY_PATH: ClassVar[bool] = True
+
DOMAIN: ClassVar[str] = "1fichier"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://1fichier.com")
_DOWNLOAD_SLOTS: ClassVar[int | None] = 1
diff --git a/cyberdrop_dl/crawlers/owncloud.py b/cyberdrop_dl/crawlers/owncloud.py
index 7549e8d97..d36976455 100644
--- a/cyberdrop_dl/crawlers/owncloud.py
+++ b/cyberdrop_dl/crawlers/owncloud.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.utils import webdav
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -12,9 +12,9 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
+@URLConfig(trim=False)
class OwnCloudCrawler(Crawler, is_generic=True):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Public Share": ("/s/<share_token>")}
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.py
index 613f473c0..18f5185ef 100644
--- a/cyberdrop_dl/crawlers/pimp_bunny.py
+++ b/cyberdrop_dl/crawlers/pimp_bunny.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl import aio
from cyberdrop_dl.crawlers._kvs import extract_kvs_video
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -43,6 +43,7 @@ def _pagination_query(url: AbsoluteHttpURL) -> dict[str, Any]:
}
+@URLConfig(trim=False)
class PimpBunnyCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Videos": "/videos/...",
@@ -56,7 +57,6 @@ class PimpBunnyCrawler(Crawler):
DOMAIN: ClassVar[str] = "pimpbunny.com"
FOLDER_DOMAIN: ClassVar[str] = "PimpBunny"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pimpbunny.com")
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
NEXT_PAGE_SELECTOR: ClassVar[str] = ".pb-pagination-list .next a[href]"
async def fetch(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.py
index e50367b64..fe989fe2a 100644
--- a/cyberdrop_dl/crawlers/pixhost.py
+++ b/cyberdrop_dl/crawlers/pixhost.py
@@ -24,7 +24,6 @@ class PixHostCrawler(Crawler):
"Thumbnail": "/thumbs/..",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pixhost.to")
- UPDATE_UNSUPPORTED: ClassVar[bool] = True
DOMAIN: ClassVar[str] = "pixhost"
FOLDER_DOMAIN: ClassVar[str] = "PixHost"
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("pixhost.org",)
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index e98cb385b..39fee92e9 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -4,7 +4,7 @@ import itertools
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers._kvs import extract_kvs_video
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, URLConfig
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
@@ -33,6 +33,7 @@ class Selector:
TITLE_TRASH = "Free HD ", "Most Relevant ", "New ", "Videos", "Porn", "for:", "New Videos", "Tagged with"
+@URLConfig(trim=False)
class PorntrexCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Video": "/video/...",
@@ -47,7 +48,6 @@ class PorntrexCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.porntrex.com")
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
DOMAIN: ClassVar[str] = "porntrex"
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.name: # The ending slash is necessary or we get a 404 error
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index 950a2c3dd..f52149f06 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -77,7 +77,7 @@ class RedGifsCrawler(Crawler):
return await self.user(scrape_item, user_name.lower())
case ["i" | "watch" | "ifr", gif_id]:
return await self.gif(scrape_item, _id(gif_id))
- case [_, _] if self.is_self_subdomain(scrape_item.url):
+ case [_, _] if self.is_subdomain(scrape_item.url):
scrape_item.url = _canonical_url(scrape_item.url.name)
return await self.gif(scrape_item, scrape_item.url.name)
case _:
diff --git a/cyberdrop_dl/crawlers/thothub.py b/cyberdrop_dl/crawlers/thothub.py
index ad50a242d..5b3e28ced 100644
--- a/cyberdrop_dl/crawlers/thothub.py
+++ b/cyberdrop_dl/crawlers/thothub.py
@@ -1,8 +1,9 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, final
from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler
+from cyberdrop_dl.crawlers.crawler import URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -12,11 +13,13 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
+@final
class Selector:
TITLE = ".main-container .headline h1"
VIDEOS = ".list-videos .item a"
+@URLConfig(trim=False)
class ThotHubCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Album": "/albums/<id>/<name>",
@@ -24,7 +27,6 @@ class ThotHubCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):
"Video": "/videos/<id>/<slug>",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://thothub.to")
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
DOMAIN: ClassVar[str] = "thothub"
FOLDER_DOMAIN: ClassVar[str] = "ThotHub"
diff --git a/cyberdrop_dl/crawlers/xasiat.py b/cyberdrop_dl/crawlers/xasiat.py
index e3ce21aa4..1ccdb913a 100644
--- a/cyberdrop_dl/crawlers/xasiat.py
+++ b/cyberdrop_dl/crawlers/xasiat.py
@@ -3,12 +3,14 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler
+from cyberdrop_dl.crawlers.crawler import URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
if TYPE_CHECKING:
from cyberdrop_dl.crawlers.crawler import SupportedPaths
+@URLConfig(trim=False)
class XasiatCrawler(KernelVideoSharingCrawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Album": "/albums/<id>/<name>",
@@ -18,7 +20,6 @@ class XasiatCrawler(KernelVideoSharingCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.xasiat.com")
DOMAIN: ClassVar[str] = "xasiat"
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
diff --git a/cyberdrop_dl/crawlers/yourlesbians.py b/cyberdrop_dl/crawlers/yourlesbians.py
index ff6166043..f66e61697 100644
--- a/cyberdrop_dl/crawlers/yourlesbians.py
+++ b/cyberdrop_dl/crawlers/yourlesbians.py
@@ -3,16 +3,17 @@ from __future__ import annotations
from typing import ClassVar
from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler
+from cyberdrop_dl.crawlers.crawler import URLConfig
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.utils import open_graph
from cyberdrop_dl.utils.errors import error_handling_wrapper
+@URLConfig(trim=False)
class YourLesbiansCrawler(KernelVideoSharingCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://yourlesbians.com")
DOMAIN: ClassVar[str] = "yourlesbians.com"
FOLDER_DOMAIN: ClassVar[str] = "YourLesbians"
- DEFAULT_TRIM_URLS: ClassVar[bool] = False
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem, album_id: str | None = None) -> None:
diff --git a/cyberdrop_dl/utils/dataclass.py b/cyberdrop_dl/utils/dataclass.py
index 876f54198..453d9317c 100644
--- a/cyberdrop_dl/utils/dataclass.py
+++ b/cyberdrop_dl/utils/dataclass.py
@@ -1,7 +1,8 @@
from __future__ import annotations
import dataclasses
-from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self
+import sys
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Protocol, Self, dataclass_transform, overload
from cyberdrop_dl.constants import MISSING
from cyberdrop_dl.utils import fast_cache
@@ -78,3 +79,41 @@ class DictDataclass(_DataClass, Protocol):
if overrides:
data.update(overrides)
return cls(**data)
+
+
+@overload
+def frozen[T](cls: None = None, *, order: bool = False) -> Callable[[type[T]], type[T]]: ...
+
+
+@overload
+def frozen[T](cls: type[T], *, order: bool = False) -> type[T]: ...
+
+
+@dataclass_transform(frozen_default=True, kw_only_default=True)
+def frozen[T](cls: type[T] | None = None, *, order: bool = False) -> Callable[[type[T]], type[T]] | type[T]:
+ fn = dataclasses.dataclass(frozen=True, kw_only=True, order=order, slots=True)
+ return fn if cls is None else fn(cls)
+
+
+@frozen
+class ConfigDataclass:
+ __attr_name__: ClassVar[str]
+ __iter__: ClassVar[Final] = DictDataclass.__iter__
+
+ def _changes(self) -> dict[str, Any]:
+ return {k: v for k, v in self if v is not None}
+
+ @classmethod
+ def get(cls, obj: object) -> Self | None:
+ return getattr(obj, cls.__attr_name__, None)
+
+ if sys.version_info < (3, 13, 0):
+ __replace__ = dataclasses.replace
+
+ def __or__(self, other: Self) -> Self:
+ return self.__replace__(**other._changes())
+
+ def __call__[T](self, obj: type[T]) -> type[T]:
+ cfg = current_cfg | self if (current_cfg := self.get(obj)) is not None else self
+ setattr(obj, self.__attr_name__, cfg)
+ return obj
diff --git a/tests/test_client.py b/tests/test_client.py
index 3723d4ef9..4e649aa82 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -107,3 +107,9 @@ def test_http_config_as_decorator() -> None:
B = config_2(A)
assert HTTPConfig.get(B) == config_1 | config_2
+
+
+def test_http_config_is_frozen() -> None:
+ config_1 = HTTPConfig(headers={"a": "b", "h": "l"}, rate_limit=(3, 1))
+ with pytest.raises(AttributeError):
+ config_1.rate_limit = None # pyright: ignore[reportAttributeAccessIssue]