Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add betterome support
NTFSvolume2353e7270187
diff --git a/cyberdrop_dl/crawlers/betterome.py b/cyberdrop_dl/crawlers/betterome.pynew file mode 100644index 000000000..2a19ac484--- /dev/null+++ b/cyberdrop_dl/crawlers/betterome.py@@ -0,0 +1,34 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.erome import EromeCrawler+from cyberdrop_dl.url_objects import AbsoluteHttpURL++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import ScrapeItem+++class BetteromeCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Album": "/album/<album_id>/...",+ "Profile": "/u/<name>",+ "Search": "/search?q=<query>",+ }+ DOMAIN: ClassVar[str] = "betterome"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://betterome.com")++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ with scrape_item.track_changes:+ match scrape_item.url.parts[1:]:+ case ["a", album_id, *_]:+ scrape_item.url = EromeCrawler.PRIMARY_URL / "a" / album_id+ case ["u", user]:+ scrape_item.url = EromeCrawler.PRIMARY_URL / user+ case ["search"] if scrape_item.url.query.get("q"):+ scrape_item.url = scrape_item.url.with_host(EromeCrawler.PRIMARY_URL.host)+ case _:+ raise ValueError++ self.handle_external_links(scrape_item)diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.pyindex 33a53c423..c4e49b413 100644--- a/cyberdrop_dl/utils/json.py+++ b/cyberdrop_dl/utils/json.py@@ -13,6 +13,8 @@ import reimport timefrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, Self, TypeGuard+from cyberdrop_dl.utils import b64_pad+if TYPE_CHECKING:from collections.abc import Callable, Generator, Iterablefrom pathlib import Path@@ -188,12 +190,12 @@ class JSONWebToken:if not looks_like_jwt(jwt):raise ValueError("Not a valid JSON Web Token", jwt)b64_headers, b64_payload, b64_signature = jwt.split(".")- headers = cls._decode(b64_headers)- return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)+ headers = cls.decode_payload(b64_headers)+ return cls(headers["alg"], headers, cls.decode_payload(b64_payload), b64_signature, jwt)@classmethod- def _decode(cls, payload: str, /) -> dict[str, Any]:- return loads(base64.urlsafe_b64decode(f"{payload}==="))+ def decode_payload(cls, payload: str, /) -> dict[str, Any]:+ return loads(base64.urlsafe_b64decode(b64_pad(payload)))def is_expired(self, threshold: int = 0) -> bool:"""Checks if the token has expired or is about to expire.