Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add betterome support

NTFSvolume2353e7270187
diff --git a/cyberdrop_dl/crawlers/betterome.py b/cyberdrop_dl/crawlers/betterome.py
new file mode 100644
index 000000000..2a19ac484
--- /dev/null
+++ b/cyberdrop_dl/crawlers/betterome.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.erome import EromeCrawler
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+
+class BetteromeCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Album": "/album/<album_id>/...",
+ "Profile": "/u/<name>",
+ "Search": "/search?q=<query>",
+ }
+ DOMAIN: ClassVar[str] = "betterome"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://betterome.com")
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ with scrape_item.track_changes:
+ match scrape_item.url.parts[1:]:
+ case ["a", album_id, *_]:
+ scrape_item.url = EromeCrawler.PRIMARY_URL / "a" / album_id
+ case ["u", user]:
+ scrape_item.url = EromeCrawler.PRIMARY_URL / user
+ case ["search"] if scrape_item.url.query.get("q"):
+ scrape_item.url = scrape_item.url.with_host(EromeCrawler.PRIMARY_URL.host)
+ case _:
+ raise ValueError
+
+ self.handle_external_links(scrape_item)
diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.py
index 33a53c423..c4e49b413 100644
--- a/cyberdrop_dl/utils/json.py
+++ b/cyberdrop_dl/utils/json.py
@@ -13,6 +13,8 @@ import re
import time
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, Self, TypeGuard
+from cyberdrop_dl.utils import b64_pad
+
if TYPE_CHECKING:
from collections.abc import Callable, Generator, Iterable
from pathlib import Path
@@ -188,12 +190,12 @@ class JSONWebToken:
if not looks_like_jwt(jwt):
raise ValueError("Not a valid JSON Web Token", jwt)
b64_headers, b64_payload, b64_signature = jwt.split(".")
- headers = cls._decode(b64_headers)
- return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)
+ headers = cls.decode_payload(b64_headers)
+ return cls(headers["alg"], headers, cls.decode_payload(b64_payload), b64_signature, jwt)
@classmethod
- def _decode(cls, payload: str, /) -> dict[str, Any]:
- return loads(base64.urlsafe_b64decode(f"{payload}==="))
+ def decode_payload(cls, payload: str, /) -> dict[str, Any]:
+ return loads(base64.urlsafe_b64decode(b64_pad(payload)))
def is_expired(self, threshold: int = 0) -> bool:
"""Checks if the token has expired or is about to expire.