Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: pmvhaven crawler (#1550)
Barbarella66666664be094eadc99
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex fba91091d..3e96b3269 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -1,35 +1,33 @@from __future__ import annotations-import itertools-import json-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils.logger import log_debug+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import AsyncGenerator+ from typing import Anyfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem-API_ENTRYPOINT = AbsoluteHttpURL("https://pmvhaven.com/api/v2/")+class Selectors:+ USER_NAME = "div.user-profile-card h1"++PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")-CATEGORIES = "Hmv", "Pmv", "Hypno", "Tiktok", "KoreanBJ"class PMVHavenCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Category": "/category/...",- "Music": "/music/...",- "Playlist": "/playlist/...",+ "Playlist": "/playlists/...","Search results": "/search/...",- "Star": "/star/...",- "Tag": "/tags/...",- "Users": "/profile/...",+ "Users": (+ "/profile/...",+ "/users/...",+ ),"Video": "/video/...",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL@@ -38,197 +36,68 @@ class PMVHavenCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:if "video" in scrape_item.url.parts:- return await self.video_from_api(scrape_item)- if "star" in scrape_item.url.parts:- return await self.model(scrape_item)- if "tags" in scrape_item.url.parts:- return await self.tag(scrape_item)- if "music" in scrape_item.url.parts:- return await self.music(scrape_item)+ return await self.video(scrape_item)if "search" in scrape_item.url.parts:return await self.search(scrape_item)- if "category" in scrape_item.url.parts:- return await self.category(scrape_item)- if "profile" in scrape_item.url.parts:+ if any(u in scrape_item.url.parts for u in ("users", "profile")):return await self.profile(scrape_item)- if "playlist" in scrape_item.url.parts:+ if "playlists" in scrape_item.url.parts:return await self.playlist(scrape_item)raise ValueError@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem) -> None:- username = scrape_item.url.name- api_url = API_ENTRYPOINT / "profileInput"+ soup = await self.request_soup(scrape_item.url)+ username = css.select_text(soup, Selectors.USER_NAME)title = f"{username} [user]"title = self.create_title(title)scrape_item.setup_as_profile(title)- # Videos- add_data = {"mode": "GetMoreProfileVideos", "user": username}- async for json_resp in self.api_pager(api_url, add_data):- await self.iter_video_info(scrape_item, json_resp["data"], "Videos")-- # Favorites- add_data = {"mode": "GetMoreFavoritedVideos", "user": username, "search": None, "date": "Date", "sort": "Sort"}- async for json_resp in self.api_pager(api_url, add_data):- await self.iter_video_info(scrape_item, json_resp["data"], "Favorites")-- # Playlist- # TODO: add pagination support for user playlists-- json_resp: dict[str, Any] = await self.request_json(- api_url,- method="POST",- data=json.dumps({"profile": username, "mode": "GetUser"}),- headers={"Content-Type": "text/plain;charset=UTF-8"},- )-- user_info: dict[str, dict] = json_resp["data"]- for playlist in user_info["playlists"]:- playlist_id = playlist["_id"]- link = PRIMARY_URL / "playlist" / playlist_id- new_scrape_item = scrape_item.create_child(link, new_title_part="Playlists")- self.create_task(self.playlist(new_scrape_item, add_suffix=False))- scrape_item.add_children()-- @error_handling_wrapper- async def playlist(self, scrape_item: ScrapeItem, add_suffix: bool = True) -> None:- playlist_id = scrape_item.url.name- add_data = {"mode": "loadPlaylistVideos", "playlist": playlist_id}- api_url = API_ENTRYPOINT / "playlists"- title: str = ""- async for json_resp in self.api_pager(api_url, add_data, check_key="videos"):- if not title:- playlist_name: str = json_resp["playlist"]["name"]- title = f"{playlist_name} [playlist]" if add_suffix else playlist_name- title = self.create_title(title, playlist_id)- scrape_item.setup_as_album(title, album_id=playlist_id)- await self.iter_video_info(scrape_item, json_resp["videos"])-- @error_handling_wrapper- async def model(self, scrape_item: ScrapeItem) -> None:- model_name = scrape_item.url.name- add_data = {"mode": "SearchStar", "star": model_name, "profile": None}- await self._generic_search_pager(scrape_item, add_data, model_name, "model")-- @error_handling_wrapper- async def creator(self, scrape_item: ScrapeItem) -> None:- creator_name = scrape_item.url.name- add_data = {"mode": "SearchCreator", "creator": creator_name, "profile": None}- await self._generic_search_pager(scrape_item, add_data, creator_name, "creator")+ nuxt_data = css.get_nuxt_data(soup)+ await self._process_video_list(scrape_item, nuxt_data)@error_handling_wrapper- async def music(self, scrape_item: ScrapeItem) -> None:- song_name = scrape_item.url.name- add_data = {"mode": "SearchMusic", "music": song_name, "profile": None}- await self._generic_search_pager(scrape_item, add_data, song_name, "music")+ async def playlist(self, scrape_item: ScrapeItem) -> None:+ soup = await self.request_soup(scrape_item.url)+ nuxt_data = css.get_nuxt_data(soup)+ playlist = css.parse_nuxt_obj(nuxt_data, "playlist")+ name = playlist["name"]+ title = self.create_title(f"{name} [playlist]")+ scrape_item.setup_as_album(title)+ await self._process_video_list(scrape_item, nuxt_data)@error_handling_wrapperasync def search(self, scrape_item: ScrapeItem) -> None:- search_value = scrape_item.url.name- add_data = {"mode": "DefaultSearch", "data": search_value, "profile": None}- await self._generic_search_pager(scrape_item, add_data, search_value, "search")-- @error_handling_wrapper- async def category(self, scrape_item: ScrapeItem) -> None:- category_name = ""- for cat in CATEGORIES: # Search values are case sensitive but the URL for categories is always lowercase- if cat.casefold() == scrape_item.url.name.casefold():- category_name = cat- break- if not category_name:- raise ScrapeError(422)- add_data = {"mode": category_name, "profile": None}- await self._generic_search_pager(scrape_item, add_data, category_name, "category")+ soup = await self.request_soup(scrape_item.url)+ tags = scrape_item.url.query.get("tags") or scrape_item.url.query.get("musicSong")+ title = self.create_title(f"{tags} [search]")+ scrape_item.setup_as_album(title)+ nuxt_data = css.get_nuxt_data(soup)+ await self._process_video_list(scrape_item, nuxt_data)@error_handling_wrapper- async def tag(self, scrape_item: ScrapeItem) -> None:- tag_name = scrape_item.url.name- add_data = {"mode": "SearchMoreTag", "tag": tag_name, "profile": None}- await self._generic_search_pager(scrape_item, add_data, tag_name, "tag")-- @error_handling_wrapper- async def video_from_api(self, scrape_item: ScrapeItem) -> None:+ async def video(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- video_id: str = scrape_item.url.name.rsplit("_", 1)[-1]- add_data = {"video": video_id, "mode": "InitVideo", "view": True}- api_url = API_ENTRYPOINT / "videoInput"- json_resp: dict[str, Any] = await self.request_json(api_url, method="POST", data=add_data)- videos: list[dict[str, Any]] = json_resp.get("video", [])- video_info = videos[0] if videos else {}- await self.process_video_info(scrape_item, video_info)-- async def _generic_search_pager(self, scrape_item: ScrapeItem, add_data: dict, name: str, type: str = "") -> None:- title: str = ""- api_url = API_ENTRYPOINT / "search"- async for json_resp in self.api_pager(api_url, add_data):- if not title:- title = f"{name} [{type}]" if type else name- title = self.create_title(title)- scrape_item.setup_as_album(title)-- await self.iter_video_info(scrape_item, json_resp["data"])+ soup = await self.request_soup(scrape_item.url)+ nuxt_data = css.get_nuxt_data(soup)+ video = css.parse_nuxt_obj(nuxt_data, "video", "uploaderVideosCount")+ await self._process_video_info(scrape_item, video)# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@error_handling_wrapper- async def process_video_info(self, scrape_item: ScrapeItem, video_info: dict) -> None:- log_debug(json.dumps(video_info, indent=4))- link_str: str = video_info.get("url") or ""- if not link_str:- raise ScrapeError(422, message="No video source found")-- video_id: str = video_info["_id"]- resolution: str | None = video_info.get("height")- title: str = video_info.get("title") or video_info["uploadTitle"]- link_str: str = video_info["url"]- date = self.parse_date(video_info["isoDate"])-- scrape_item.possible_datetime = date- link = self.parse_url(link_str)+ async def _process_video_list(self, scrape_item: ScrapeItem, nuxt_data: list[Any]) -> None:+ for video_info in css.parse_nuxt_objs(nuxt_data, "videoUrl"):+ await self._process_video_info(scrape_item, video_info)++ @error_handling_wrapper+ async def _process_video_info(self, scrape_item: ScrapeItem, video_info: dict[str, Any]) -> None:+ scrape_item.possible_datetime = self.parse_date(video_info["uploadDate"])+ link = self.parse_url(video_info["videoUrl"])filename, ext = self.get_filename_and_ext(link.name, assume_ext=".mp4")- custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=resolution)- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)-- async def iter_video_info(self, scrape_item: ScrapeItem, videos: list[dict], new_title_part: str = "") -> None:- for video in videos:- link = _canonical_url(video)- new_scrape_item = scrape_item.create_child(link, new_title_part=new_title_part)- await self.process_video_info(new_scrape_item, video)- scrape_item.add_children()-- async def api_pager(- self, api_url: AbsoluteHttpURL, add_data: dict, *, check_key: str = "data"- ) -> AsyncGenerator[dict]:- """Generator of API pages."""- page: int = 1- is_profile = api_url.name == "profileInput"- add_headers = {"Content-Type": "text/plain;charset=UTF-8"} if is_profile else None-- for page in itertools.count(1):- data = {"index": page} | add_data- if is_profile:- data = json.dumps(data)-- json_resp: dict[str, Any] = await self.request_json(- api_url,- method="POST",- data=data,- headers=add_headers,- )- has_videos = bool(json_resp[check_key])- if not has_videos:- break- yield json_resp---# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~---def _canonical_url(video_info: dict[str, str]) -> AbsoluteHttpURL:- title = video_info.get("title") or video_info["uploadTitle"]- video_id = video_info["_id"]- path = f"{title}_{video_id}"- return PRIMARY_URL / "video" / path+ custom_filename = self.create_custom_filename(+ video_info["title"], ext, file_id=video_info["_id"], resolution=video_info["height"]+ )+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video_info)diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 130027d91..636acbfa8 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -190,7 +190,8 @@ def parse_nuxt_objs(nuxt_data: list[Any], *attributes: str) -> Generator[dict[sttry:index: int = obj[first_key]index_map: dict[str, int] = nuxt_data[index]- except LookupError:+ assert isinstance(index_map, dict)+ except (LookupError, AssertionError):index_map = objyield _parse_nuxt_obj(nuxt_data, index_map)diff --git a/tests/crawlers/test_cases/pmvhaven.py b/tests/crawlers/test_cases/pmvhaven.pynew file mode 100644index 000000000..1fcdd0309--- /dev/null+++ b/tests/crawlers/test_cases/pmvhaven.py@@ -0,0 +1,56 @@+DOMAIN = "pmvhaven"+TEST_CASES = [+ # Video+ (+ "https://pmvhaven.com/video/real-hookers-pmv_69301129194866d69ec96b5d",+ [+ {+ "url": "https://video.pmvhaven.com/videos/1764757703719_yxthkzjbhbp_Hookers.mp4",+ "filename": "Real Hookers - PMV [69301129194866d69ec96b5d][1080p].mp4",+ "original_filename": "1764757703719_yxthkzjbhbp_Hookers.mp4",+ "download_folder": "re:Loose Files (PMVHaven)",+ "referer": "https://pmvhaven.com/video/real-hookers-pmv_69301129194866d69ec96b5d",+ "datetime": 1764757801,+ }+ ],+ ),+ # Profile+ (+ "https://pmvhaven.com/profile/68feea724e81de92343a01ef",+ [+ {+ "url": "ANY",+ "download_folder": r"re:CheekyGooner [user] (PMVHaven)",+ "referer": "ANY",+ "album_id": None,+ }+ ],+ range(170, 200),+ ),+ # Users+ (+ "https://pmvhaven.com/users/68feea8c4e81de92343a1fd7",+ [+ {+ "url": "ANY",+ "download_folder": r"re:arthurcclarke [user] (PMVHaven)",+ "referer": "ANY",+ "album_id": None,+ }+ ],+ range(8, 20),+ ),+ # Playlist+ (+ "https://pmvhaven.com/playlists/6932d1e5e234628c68d50b71",+ [+ {+ "url": "ANY",+ "download_folder": r"re:10 GODDESS [playlist] (PMVHaven)",+ "referer": "ANY",+ "album_id": None,+ }+ ],+ range(10, 20),+ ),+]