Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: pmvhaven crawler (#1550)

Barbarella66666664be094eadc99
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index fba91091d..3e96b3269 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -1,35 +1,33 @@
from __future__ import annotations
-import itertools
-import json
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.utils.logger import log_debug
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator
+ from typing import Any
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-API_ENTRYPOINT = AbsoluteHttpURL("https://pmvhaven.com/api/v2/")
+class Selectors:
+ USER_NAME = "div.user-profile-card h1"
+
+
PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")
-CATEGORIES = "Hmv", "Pmv", "Hypno", "Tiktok", "KoreanBJ"
class PMVHavenCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Category": "/category/...",
- "Music": "/music/...",
- "Playlist": "/playlist/...",
+ "Playlist": "/playlists/...",
"Search results": "/search/...",
- "Star": "/star/...",
- "Tag": "/tags/...",
- "Users": "/profile/...",
+ "Users": (
+ "/profile/...",
+ "/users/...",
+ ),
"Video": "/video/...",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
@@ -38,197 +36,68 @@ class PMVHavenCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "video" in scrape_item.url.parts:
- return await self.video_from_api(scrape_item)
- if "star" in scrape_item.url.parts:
- return await self.model(scrape_item)
- if "tags" in scrape_item.url.parts:
- return await self.tag(scrape_item)
- if "music" in scrape_item.url.parts:
- return await self.music(scrape_item)
+ return await self.video(scrape_item)
if "search" in scrape_item.url.parts:
return await self.search(scrape_item)
- if "category" in scrape_item.url.parts:
- return await self.category(scrape_item)
- if "profile" in scrape_item.url.parts:
+ if any(u in scrape_item.url.parts for u in ("users", "profile")):
return await self.profile(scrape_item)
- if "playlist" in scrape_item.url.parts:
+ if "playlists" in scrape_item.url.parts:
return await self.playlist(scrape_item)
raise ValueError
@error_handling_wrapper
async def profile(self, scrape_item: ScrapeItem) -> None:
- username = scrape_item.url.name
- api_url = API_ENTRYPOINT / "profileInput"
+ soup = await self.request_soup(scrape_item.url)
+ username = css.select_text(soup, Selectors.USER_NAME)
title = f"{username} [user]"
title = self.create_title(title)
scrape_item.setup_as_profile(title)
- # Videos
- add_data = {"mode": "GetMoreProfileVideos", "user": username}
- async for json_resp in self.api_pager(api_url, add_data):
- await self.iter_video_info(scrape_item, json_resp["data"], "Videos")
-
- # Favorites
- add_data = {"mode": "GetMoreFavoritedVideos", "user": username, "search": None, "date": "Date", "sort": "Sort"}
- async for json_resp in self.api_pager(api_url, add_data):
- await self.iter_video_info(scrape_item, json_resp["data"], "Favorites")
-
- # Playlist
- # TODO: add pagination support for user playlists
-
- json_resp: dict[str, Any] = await self.request_json(
- api_url,
- method="POST",
- data=json.dumps({"profile": username, "mode": "GetUser"}),
- headers={"Content-Type": "text/plain;charset=UTF-8"},
- )
-
- user_info: dict[str, dict] = json_resp["data"]
- for playlist in user_info["playlists"]:
- playlist_id = playlist["_id"]
- link = PRIMARY_URL / "playlist" / playlist_id
- new_scrape_item = scrape_item.create_child(link, new_title_part="Playlists")
- self.create_task(self.playlist(new_scrape_item, add_suffix=False))
- scrape_item.add_children()
-
- @error_handling_wrapper
- async def playlist(self, scrape_item: ScrapeItem, add_suffix: bool = True) -> None:
- playlist_id = scrape_item.url.name
- add_data = {"mode": "loadPlaylistVideos", "playlist": playlist_id}
- api_url = API_ENTRYPOINT / "playlists"
- title: str = ""
- async for json_resp in self.api_pager(api_url, add_data, check_key="videos"):
- if not title:
- playlist_name: str = json_resp["playlist"]["name"]
- title = f"{playlist_name} [playlist]" if add_suffix else playlist_name
- title = self.create_title(title, playlist_id)
- scrape_item.setup_as_album(title, album_id=playlist_id)
- await self.iter_video_info(scrape_item, json_resp["videos"])
-
- @error_handling_wrapper
- async def model(self, scrape_item: ScrapeItem) -> None:
- model_name = scrape_item.url.name
- add_data = {"mode": "SearchStar", "star": model_name, "profile": None}
- await self._generic_search_pager(scrape_item, add_data, model_name, "model")
-
- @error_handling_wrapper
- async def creator(self, scrape_item: ScrapeItem) -> None:
- creator_name = scrape_item.url.name
- add_data = {"mode": "SearchCreator", "creator": creator_name, "profile": None}
- await self._generic_search_pager(scrape_item, add_data, creator_name, "creator")
+ nuxt_data = css.get_nuxt_data(soup)
+ await self._process_video_list(scrape_item, nuxt_data)
@error_handling_wrapper
- async def music(self, scrape_item: ScrapeItem) -> None:
- song_name = scrape_item.url.name
- add_data = {"mode": "SearchMusic", "music": song_name, "profile": None}
- await self._generic_search_pager(scrape_item, add_data, song_name, "music")
+ async def playlist(self, scrape_item: ScrapeItem) -> None:
+ soup = await self.request_soup(scrape_item.url)
+ nuxt_data = css.get_nuxt_data(soup)
+ playlist = css.parse_nuxt_obj(nuxt_data, "playlist")
+ name = playlist["name"]
+ title = self.create_title(f"{name} [playlist]")
+ scrape_item.setup_as_album(title)
+ await self._process_video_list(scrape_item, nuxt_data)
@error_handling_wrapper
async def search(self, scrape_item: ScrapeItem) -> None:
- search_value = scrape_item.url.name
- add_data = {"mode": "DefaultSearch", "data": search_value, "profile": None}
- await self._generic_search_pager(scrape_item, add_data, search_value, "search")
-
- @error_handling_wrapper
- async def category(self, scrape_item: ScrapeItem) -> None:
- category_name = ""
- for cat in CATEGORIES: # Search values are case sensitive but the URL for categories is always lowercase
- if cat.casefold() == scrape_item.url.name.casefold():
- category_name = cat
- break
- if not category_name:
- raise ScrapeError(422)
- add_data = {"mode": category_name, "profile": None}
- await self._generic_search_pager(scrape_item, add_data, category_name, "category")
+ soup = await self.request_soup(scrape_item.url)
+ tags = scrape_item.url.query.get("tags") or scrape_item.url.query.get("musicSong")
+ title = self.create_title(f"{tags} [search]")
+ scrape_item.setup_as_album(title)
+ nuxt_data = css.get_nuxt_data(soup)
+ await self._process_video_list(scrape_item, nuxt_data)
@error_handling_wrapper
- async def tag(self, scrape_item: ScrapeItem) -> None:
- tag_name = scrape_item.url.name
- add_data = {"mode": "SearchMoreTag", "tag": tag_name, "profile": None}
- await self._generic_search_pager(scrape_item, add_data, tag_name, "tag")
-
- @error_handling_wrapper
- async def video_from_api(self, scrape_item: ScrapeItem) -> None:
+ async def video(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- video_id: str = scrape_item.url.name.rsplit("_", 1)[-1]
- add_data = {"video": video_id, "mode": "InitVideo", "view": True}
- api_url = API_ENTRYPOINT / "videoInput"
- json_resp: dict[str, Any] = await self.request_json(api_url, method="POST", data=add_data)
- videos: list[dict[str, Any]] = json_resp.get("video", [])
- video_info = videos[0] if videos else {}
- await self.process_video_info(scrape_item, video_info)
-
- async def _generic_search_pager(self, scrape_item: ScrapeItem, add_data: dict, name: str, type: str = "") -> None:
- title: str = ""
- api_url = API_ENTRYPOINT / "search"
- async for json_resp in self.api_pager(api_url, add_data):
- if not title:
- title = f"{name} [{type}]" if type else name
- title = self.create_title(title)
- scrape_item.setup_as_album(title)
-
- await self.iter_video_info(scrape_item, json_resp["data"])
+ soup = await self.request_soup(scrape_item.url)
+ nuxt_data = css.get_nuxt_data(soup)
+ video = css.parse_nuxt_obj(nuxt_data, "video", "uploaderVideosCount")
+ await self._process_video_info(scrape_item, video)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@error_handling_wrapper
- async def process_video_info(self, scrape_item: ScrapeItem, video_info: dict) -> None:
- log_debug(json.dumps(video_info, indent=4))
- link_str: str = video_info.get("url") or ""
- if not link_str:
- raise ScrapeError(422, message="No video source found")
-
- video_id: str = video_info["_id"]
- resolution: str | None = video_info.get("height")
- title: str = video_info.get("title") or video_info["uploadTitle"]
- link_str: str = video_info["url"]
- date = self.parse_date(video_info["isoDate"])
-
- scrape_item.possible_datetime = date
- link = self.parse_url(link_str)
+ async def _process_video_list(self, scrape_item: ScrapeItem, nuxt_data: list[Any]) -> None:
+ for video_info in css.parse_nuxt_objs(nuxt_data, "videoUrl"):
+ await self._process_video_info(scrape_item, video_info)
+
+ @error_handling_wrapper
+ async def _process_video_info(self, scrape_item: ScrapeItem, video_info: dict[str, Any]) -> None:
+ scrape_item.possible_datetime = self.parse_date(video_info["uploadDate"])
+ link = self.parse_url(video_info["videoUrl"])
filename, ext = self.get_filename_and_ext(link.name, assume_ext=".mp4")
- custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=resolution)
- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
-
- async def iter_video_info(self, scrape_item: ScrapeItem, videos: list[dict], new_title_part: str = "") -> None:
- for video in videos:
- link = _canonical_url(video)
- new_scrape_item = scrape_item.create_child(link, new_title_part=new_title_part)
- await self.process_video_info(new_scrape_item, video)
- scrape_item.add_children()
-
- async def api_pager(
- self, api_url: AbsoluteHttpURL, add_data: dict, *, check_key: str = "data"
- ) -> AsyncGenerator[dict]:
- """Generator of API pages."""
- page: int = 1
- is_profile = api_url.name == "profileInput"
- add_headers = {"Content-Type": "text/plain;charset=UTF-8"} if is_profile else None
-
- for page in itertools.count(1):
- data = {"index": page} | add_data
- if is_profile:
- data = json.dumps(data)
-
- json_resp: dict[str, Any] = await self.request_json(
- api_url,
- method="POST",
- data=data,
- headers=add_headers,
- )
- has_videos = bool(json_resp[check_key])
- if not has_videos:
- break
- yield json_resp
-
-
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-
-def _canonical_url(video_info: dict[str, str]) -> AbsoluteHttpURL:
- title = video_info.get("title") or video_info["uploadTitle"]
- video_id = video_info["_id"]
- path = f"{title}_{video_id}"
- return PRIMARY_URL / "video" / path
+ custom_filename = self.create_custom_filename(
+ video_info["title"], ext, file_id=video_info["_id"], resolution=video_info["height"]
+ )
+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video_info)
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 130027d91..636acbfa8 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -190,7 +190,8 @@ def parse_nuxt_objs(nuxt_data: list[Any], *attributes: str) -> Generator[dict[st
try:
index: int = obj[first_key]
index_map: dict[str, int] = nuxt_data[index]
- except LookupError:
+ assert isinstance(index_map, dict)
+ except (LookupError, AssertionError):
index_map = obj
yield _parse_nuxt_obj(nuxt_data, index_map)
diff --git a/tests/crawlers/test_cases/pmvhaven.py b/tests/crawlers/test_cases/pmvhaven.py
new file mode 100644
index 000000000..1fcdd0309
--- /dev/null
+++ b/tests/crawlers/test_cases/pmvhaven.py
@@ -0,0 +1,56 @@
+DOMAIN = "pmvhaven"
+TEST_CASES = [
+ # Video
+ (
+ "https://pmvhaven.com/video/real-hookers-pmv_69301129194866d69ec96b5d",
+ [
+ {
+ "url": "https://video.pmvhaven.com/videos/1764757703719_yxthkzjbhbp_Hookers.mp4",
+ "filename": "Real Hookers - PMV [69301129194866d69ec96b5d][1080p].mp4",
+ "original_filename": "1764757703719_yxthkzjbhbp_Hookers.mp4",
+ "download_folder": "re:Loose Files (PMVHaven)",
+ "referer": "https://pmvhaven.com/video/real-hookers-pmv_69301129194866d69ec96b5d",
+ "datetime": 1764757801,
+ }
+ ],
+ ),
+ # Profile
+ (
+ "https://pmvhaven.com/profile/68feea724e81de92343a01ef",
+ [
+ {
+ "url": "ANY",
+ "download_folder": r"re:CheekyGooner [user] (PMVHaven)",
+ "referer": "ANY",
+ "album_id": None,
+ }
+ ],
+ range(170, 200),
+ ),
+ # Users
+ (
+ "https://pmvhaven.com/users/68feea8c4e81de92343a1fd7",
+ [
+ {
+ "url": "ANY",
+ "download_folder": r"re:arthurcclarke [user] (PMVHaven)",
+ "referer": "ANY",
+ "album_id": None,
+ }
+ ],
+ range(8, 20),
+ ),
+ # Playlist
+ (
+ "https://pmvhaven.com/playlists/6932d1e5e234628c68d50b71",
+ [
+ {
+ "url": "ANY",
+ "download_folder": r"re:10 GODDESS [playlist] (PMVHaven)",
+ "referer": "ANY",
+ "album_id": None,
+ }
+ ],
+ range(10, 20),
+ ),
+]