Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: adding new crawlers (leakedzone and hotleak.vip) (#1131)

Barbarella6666666f51ecb498a89
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 95b632c28..147713d1d 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -37,6 +37,7 @@ from .generic import GenericCrawler
from .gofile import GoFileCrawler
from .google_drive import GoogleDriveCrawler
from .hitomi_la import HitomiLaCrawler
+from .hotleak_vip import HotLeakVipCrawler
from .hotpic import HotPicCrawler
from .iceyfile import IceyFileCrawler
from .imagebam import ImageBamCrawler
@@ -50,6 +51,7 @@ from .influencer_bitches import InfluencerBitchesCrawler
from .invision import INVISION_CRAWLERS
from .jpg5 import JPG5Crawler
from .kemono import KemonoCrawler
+from .leakedzone import LeakedZoneCrawler
from .luscious import LusciousCrawler
from .mediafire import MediaFireCrawler
from .mega_nz import MegaNzCrawler
diff --git a/cyberdrop_dl/crawlers/hotleak_vip.py b/cyberdrop_dl/crawlers/hotleak_vip.py
new file mode 100644
index 000000000..8720e25e2
--- /dev/null
+++ b/cyberdrop_dl/crawlers/hotleak_vip.py
@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from cyberdrop_dl.crawlers.leakedzone import LeakedZoneCrawler
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import css
+
+if TYPE_CHECKING:
+ from bs4 import BeautifulSoup
+
+
+LIGHT_GALLERY_ITEM_SELECTOR = "div.light-gallery-item"
+PRIMARY_URL = AbsoluteHttpURL("https://hotleak.vip")
+IMAGES_CDN = AbsoluteHttpURL("https://image-cdn.hotleak.vip")
+
+
+class HotLeakVipCrawler(LeakedZoneCrawler):
+ DOMAIN: ClassVar[str] = "hotleak.vip"
+ FOLDER_DOMAIN: ClassVar[str] = "HotLeakVip"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
+ IMAGES_CDN: ClassVar[AbsoluteHttpURL] = IMAGES_CDN
+
+ @classmethod
+ def get_encoded_video_url(cls, soup: BeautifulSoup) -> str:
+ video_info_text = css.select_one_get_attr(soup, LIGHT_GALLERY_ITEM_SELECTOR, "data-video")
+ video_data: dict[str, Any] = json.loads(video_info_text)
+ return video_data["source"][0]["src"]
diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.py
new file mode 100644
index 000000000..6d0dc7c3c
--- /dev/null
+++ b/cyberdrop_dl/crawlers/leakedzone.py
@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import binascii
+import dataclasses
+import itertools
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from aiolimiter import AsyncLimiter
+
+from cyberdrop_dl.compat import IntEnum
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.m3u8 import M3U8Media
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
+
+if TYPE_CHECKING:
+ from bs4 import BeautifulSoup
+
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
+class PostType(IntEnum):
+ IMAGE = 0
+ VIDEO = 1
+
+
+class Selectors:
+ JW_PLAYER = "script:contains('playerInstance.setup')"
+ MODEL_NAME_FROM_PROFILE = "div.actor-name > h1"
+ MODEL_NAME_FROM_VIDEO = "h2.actor-title-port"
+ MODEL_NAME = f"{MODEL_NAME_FROM_VIDEO}, {MODEL_NAME_FROM_PROFILE}"
+
+
+_SELECTORS = Selectors()
+PRIMARY_URL = AbsoluteHttpURL("https://leakedzone.com")
+IMAGES_CDN = AbsoluteHttpURL("https://image-cdn.leakedzone.com/storage/")
+
+
[email protected](frozen=True, slots=True)
+class Post:
+ id: str
+ type: PostType
+ created_at: str | None = None
+ image: str = ""
+ stream_url_play: str = ""
+
+ @staticmethod
+ def from_dict(post: dict[str, Any]) -> Post:
+ return Post(
+ str(post["id"]),
+ PostType(post["type"]),
+ post["created_at"],
+ post["image"].replace("_thumb", ""),
+ post.get("stream_url_play", ""),
+ )
+
+
+class LeakedZoneCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": "/<model_id>/video/<video_id>",
+ "Model": "/<model_id>",
+ }
+ DOMAIN: ClassVar[str] = "leakedzone"
+ FOLDER_DOMAIN: ClassVar[str] = "LeakedZone"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
+ IMAGES_CDN: ClassVar[AbsoluteHttpURL] = IMAGES_CDN
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case [_, "video", video_id]:
+ return await self.video(scrape_item, video_id)
+ case [_]:
+ return await self.model(scrape_item)
+ case _:
+ raise ValueError
+
+ def __post_init__(self) -> None:
+ self.request_limiter = AsyncLimiter(3, 10)
+
+ @classmethod
+ def get_encoded_video_url(cls, soup: BeautifulSoup) -> str:
+ js_text = css.select_one_get_text(soup, _SELECTORS.JW_PLAYER)
+ return get_text_between(js_text, 'file: f("', '"),')
+
+ @error_handling_wrapper
+ async def model(self, scrape_item: ScrapeItem) -> None:
+ async with self.request_limiter:
+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+
+ model_name: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME_FROM_PROFILE)
+ scrape_item.setup_as_profile(self.create_title(model_name))
+ headers = {"X-Requested-With": "XMLHttpRequest"}
+ for page in itertools.count(1):
+ async with self.request_limiter:
+ posts = await self.client.get_json(self.DOMAIN, scrape_item.url.with_query(page=page), headers=headers)
+ # We may be able to omit the last request by just checking the number of posts
+ # Seens to always return 48 posts
+ if not posts:
+ break
+ for post in (Post.from_dict(post) for post in posts):
+ if post.type is PostType.VIDEO:
+ post_url = self.PRIMARY_URL / model_name / "video" / post.id
+ await self._handle_video(scrape_item.create_child(post_url), post)
+ else:
+ post_url = self.PRIMARY_URL / model_name / "photo" / post.id
+ await self._handle_image(scrape_item.create_child(post_url), post)
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ async with self.request_limiter:
+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+
+ model_name = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME)
+ scrape_item.setup_as_album(self.create_title(model_name))
+ encoded_url = self.get_encoded_video_url(soup)
+ post = Post(video_id, PostType.VIDEO, stream_url_play=encoded_url)
+ await self._handle_video(scrape_item, post, check_referer=False)
+
+ async def _handle_video(self, scrape_item: ScrapeItem, post: Post, check_referer: bool = True) -> None:
+ if check_referer and await self.check_complete_from_referer(scrape_item):
+ return
+ url = self.parse_url(_decode_video_url(post.stream_url_play))
+ m3u8_media = M3U8Media(await self._get_m3u8(url))
+ filename, ext = self.get_filename_and_ext(f"{post.id}.mp4")
+ if post.created_at:
+ scrape_item.possible_datetime = self.parse_iso_date(post.created_at)
+ await self.handle_file(scrape_item.url, scrape_item, filename, ext, m3u8_media=m3u8_media)
+
+ async def _handle_image(self, scrape_item: ScrapeItem, post: Post) -> None:
+ image_url = self.IMAGES_CDN / post.image
+ filename, ext = self.get_filename_and_ext(image_url.name)
+ assert post.created_at
+ scrape_item.possible_datetime = self.parse_iso_date(post.created_at)
+ custom_filename = self.create_custom_filename(filename, ext, file_id=post.id)
+ await self.handle_file(image_url, scrape_item, filename, ext, custom_filename=custom_filename)
+
+
+def _decode_video_url(url: str) -> str:
+ # cut first and last 16 characters, reverse, base64 decode
+ # TODO: Research if this work on any JW Player
+ return binascii.a2b_base64(url[-17:15:-1]).decode()