Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add support to thothub (#1533)

Barbarella666666610a374ac1049
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 289fcd49e..0775f0675 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -107,6 +107,7 @@ from .spankbang import SpankBangCrawler
from .streamable import StreamableCrawler
from .streamtape import StreamtapeCrawler
from .thisvid import ThisVidCrawler
+from .thothub import ThotHubCrawler
from .tiktok import TikTokCrawler
from .tnaflix import TNAFlixCrawler
from .tokyomotion import TokioMotionCrawler
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 0c814ac8c..c88442786 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -30,7 +30,7 @@ class KVSVideo:
class Selectors:
UNAUTHORIZED = "div.video-holder:-soup-contains('This video is a private video')"
- FLASHVARS = "script:-soup-contains('video_title:')"
+ FLASHVARS = "script:-soup-contains('video_id:')"
USER_NAME = "div.headline > h2"
ALBUM_NAME = "div.headline > h1"
ALBUM_PICTURES = "div.album-list > a, .images a"
@@ -203,7 +203,7 @@ def _parse_video_vars(video_vars: str) -> KVSVideo:
yield resolution, url
resolution, url = max(get_formats())
- return KVSVideo(flashvars["video_id"], flashvars["video_title"], url, resolution)
+ return KVSVideo(flashvars["video_id"], flashvars.get("video_title", ""), url, resolution)
def _get_license_token(license_code: str) -> tuple[int, ...]:
diff --git a/cyberdrop_dl/crawlers/thothub.py b/cyberdrop_dl/crawlers/thothub.py
new file mode 100644
index 000000000..807d51974
--- /dev/null
+++ b/cyberdrop_dl/crawlers/thothub.py
@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
+class ThotHubCrawler(KernelVideoSharingCrawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Albums": "/albums/<album_id>/<album_name>",
+ "Videos": "/videos/...",
+ }
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://thothub.to")
+ DEFAULT_TRIM_URLS: ClassVar[bool] = False
+ DOMAIN: ClassVar[str] = "thothub"
+ FOLDER_DOMAIN: ClassVar[str] = "ThotHub"
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ scrape_item.url = scrape_item.url / ""
+ match scrape_item.url.parts[1:]:
+ case ["albums", album_id, _, *_]:
+ return await self.album(scrape_item, album_id)
+ case ["videos", _, _, *_]:
+ return await self.video(scrape_item)
+ case ["get_image", _, *_]:
+ return await self.direct_file(scrape_item)
+ case _:
+ raise ValueError
diff --git a/tests/crawlers/test_cases/thothub.py b/tests/crawlers/test_cases/thothub.py
new file mode 100644
index 000000000..194774cae
--- /dev/null
+++ b/tests/crawlers/test_cases/thothub.py
@@ -0,0 +1,28 @@
+DOMAIN = "thothub"
+TEST_CASES = [
+ (
+ "https://thothub.to/videos/1415181/kateharperxo/",
+ [
+ {
+ "url": "https://thothub.to/videos/1415181/kateharperxo/",
+ "filename": "Kateharperxo [1415181].mp4",
+ "referer": "https://thothub.to/videos/1415181/kateharperxo/",
+ "album_id": None,
+ }
+ ],
+ ),
+ (
+ "https://thothub.lol/albums/42921/ivy-king-of-selects/",
+ [
+ {
+ "url": "https://thothub.lol/get_image/45/0fd48eeb530b9232d216b579cd58ce9b/sources/42000/42921/2113169.jpg/",
+ "filename": "2113169.jpg",
+ "referer": "https://thothub.lol/get_image/45/0fd48eeb530b9232d216b579cd58ce9b/sources/42000/42921/2113169.jpg/",
+ "album_id": "42921",
+ "datetime": None,
+ "download_folder": r"re:Ivy King \- OF Selects \[album\] \(ThotHub\)",
+ }
+ ],
+ 112,
+ ),
+]