Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add support to thothub (#1533)
Barbarella666666610a374ac1049
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 289fcd49e..0775f0675 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -107,6 +107,7 @@ from .spankbang import SpankBangCrawlerfrom .streamable import StreamableCrawlerfrom .streamtape import StreamtapeCrawlerfrom .thisvid import ThisVidCrawler+from .thothub import ThotHubCrawlerfrom .tiktok import TikTokCrawlerfrom .tnaflix import TNAFlixCrawlerfrom .tokyomotion import TokioMotionCrawlerdiff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 0c814ac8c..c88442786 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -30,7 +30,7 @@ class KVSVideo:class Selectors:UNAUTHORIZED = "div.video-holder:-soup-contains('This video is a private video')"- FLASHVARS = "script:-soup-contains('video_title:')"+ FLASHVARS = "script:-soup-contains('video_id:')"USER_NAME = "div.headline > h2"ALBUM_NAME = "div.headline > h1"ALBUM_PICTURES = "div.album-list > a, .images a"@@ -203,7 +203,7 @@ def _parse_video_vars(video_vars: str) -> KVSVideo:yield resolution, urlresolution, url = max(get_formats())- return KVSVideo(flashvars["video_id"], flashvars["video_title"], url, resolution)+ return KVSVideo(flashvars["video_id"], flashvars.get("video_title", ""), url, resolution)def _get_license_token(license_code: str) -> tuple[int, ...]:diff --git a/cyberdrop_dl/crawlers/thothub.py b/cyberdrop_dl/crawlers/thothub.pynew file mode 100644index 000000000..807d51974--- /dev/null+++ b/cyberdrop_dl/crawlers/thothub.py@@ -0,0 +1,33 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL++if TYPE_CHECKING:+ from cyberdrop_dl.crawlers.crawler import SupportedPaths+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem+++class ThotHubCrawler(KernelVideoSharingCrawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Albums": "/albums/<album_id>/<album_name>",+ "Videos": "/videos/...",+ }+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://thothub.to")+ DEFAULT_TRIM_URLS: ClassVar[bool] = False+ DOMAIN: ClassVar[str] = "thothub"+ FOLDER_DOMAIN: ClassVar[str] = "ThotHub"++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ scrape_item.url = scrape_item.url / ""+ match scrape_item.url.parts[1:]:+ case ["albums", album_id, _, *_]:+ return await self.album(scrape_item, album_id)+ case ["videos", _, _, *_]:+ return await self.video(scrape_item)+ case ["get_image", _, *_]:+ return await self.direct_file(scrape_item)+ case _:+ raise ValueErrordiff --git a/tests/crawlers/test_cases/thothub.py b/tests/crawlers/test_cases/thothub.pynew file mode 100644index 000000000..194774cae--- /dev/null+++ b/tests/crawlers/test_cases/thothub.py@@ -0,0 +1,28 @@+DOMAIN = "thothub"+TEST_CASES = [+ (+ "https://thothub.to/videos/1415181/kateharperxo/",+ [+ {+ "url": "https://thothub.to/videos/1415181/kateharperxo/",+ "filename": "Kateharperxo [1415181].mp4",+ "referer": "https://thothub.to/videos/1415181/kateharperxo/",+ "album_id": None,+ }+ ],+ ),+ (+ "https://thothub.lol/albums/42921/ivy-king-of-selects/",+ [+ {+ "url": "https://thothub.lol/get_image/45/0fd48eeb530b9232d216b579cd58ce9b/sources/42000/42921/2113169.jpg/",+ "filename": "2113169.jpg",+ "referer": "https://thothub.lol/get_image/45/0fd48eeb530b9232d216b579cd58ce9b/sources/42000/42921/2113169.jpg/",+ "album_id": "42921",+ "datetime": None,+ "download_folder": r"re:Ivy King \- OF Selects \[album\] \(ThotHub\)",+ }+ ],+ 112,+ ),+]