Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: handle videos with no resolution info (PMVhaven) (#2033)
NTFSvolumeca9ce699f7f2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 26c1c2ea0..c680c1b6a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Solve proof of work challenge (Fileditch)- User downloads (Motherless)- Accept embeded URLs with no metadata (Pawchive)+- Handle videos with no resolution information (PMVHaven)## [10.2.0] - 2026-07-13diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex cfc953d6a..fd5f6c58a 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -3,147 +3,220 @@ from __future__ import annotationsimport dataclassesimport itertools+import operatorfrom typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl import aio-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import nuxt-from cyberdrop_dl.utils.dataclass import DictDataclass+from cyberdrop_dl.utils import nuxt, parse_url, unique+from cyberdrop_dl.utils.dataclass import Deserializerfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator, Iterable+ from collections.abc import AsyncGenerator, AsyncIterable, Iterablefrom typing import Anyfrom cyberdrop_dl.url_objects import ScrapeItem-PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")--[email protected](slots=True)-class Video(DictDataclass):- # TODO: parse and download previews and thumbnails- id: str- title: str- videoUrl: str- uploadDate: str- width: int- height: int-- hlsMasterPlaylistUrl: str | None = None-- @classmethod- def filter_dict(cls, data: dict[str, Any]) -> dict[str, Any]:- data["id"] = data["_id"]- return super(Video, cls).filter_dict(data)-- @property- def web_url(self) -> AbsoluteHttpURL:- # The title does not matter, the website parses the id from the url and redirects to the correct video- return PRIMARY_URL / f"video/{self.title.lower()}_{self.id}"--class PMVHavenCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Playlist": "/playlists/...",- "Search results": "/search/...",+ "Playlist": "/playlists/<playlist_id>",+ "Search results": "/search?q=<query>","Users": (- "/profile/...",- "/users/...",+ "/profile/<user_id>",+ "/users/<user_id>",),- "Video": "/video/...",+ "Video": "/video/<video_name>_<video_id>",}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pmvhaven.com")DOMAIN: ClassVar[str] = "pmvhaven"FOLDER_DOMAIN: ClassVar[str] = "PMVHaven"+ def __post_init__(self) -> None:+ self.api: PMVAPI = PMVAPI.from_crawler(self)+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["video", slug] if video_id := slug.rsplit("_", 1)[-1]:return await self.video(scrape_item, video_id)case ["search"] if query := scrape_item.url.query.get("q"):return await self.search(scrape_item, query)- case ["users" | "profile", _]:- return await self.profile(scrape_item)+ case ["users" | "profile", user_id]:+ return await self.profile(scrape_item, user_id)case ["playlists", playlist_id]:return await self.playlist(scrape_item, playlist_id)case _:raise ValueError@error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem) -> None:- soup = await self.request_soup(scrape_item.url)- data = nuxt.extract(soup)- username: str = nuxt.find(data, "username")["username"]- title = self.create_title(f"{username} [user]")- scrape_item.setup_as_profile(title)-- for video in _parse_videos(nuxt.ifind(data, "videoUrl", "title")):- await self._video(scrape_item.copy(), video)- scrape_item.add_children()+ async def profile(self, scrape_item: ScrapeItem, user_id: str) -> None:+ username, videos = await self.api.profile(user_id)+ scrape_item.setup_as_profile(self.create_title(f"{username} [user]"))+ self._iter_videos(scrape_item, videos)@error_handling_wrapperasync def playlist(self, scrape_item: ScrapeItem, playlist_id: str) -> None:# TODO: use playlist as album_id to skip downloads faster- api_url = self.PRIMARY_URL / "api/playlists" / playlist_id- data, pages = await aio.peek_first(self._api_pager(api_url))- title = self.create_title(f"{data['name']} [playlist]")+ playlist, video_pages = await self.api.playlist(playlist_id)+ title = self.create_title(f"{playlist.name} [playlist]")scrape_item.setup_as_album(title)-- async for data in pages:- for video in _parse_videos(data["videoDetails"]):- await self._video(scrape_item.copy(), video)- scrape_item.add_children()-- async def _api_pager(self, api_url: AbsoluteHttpURL, init_page: int = 1) -> AsyncGenerator[Any]:- for page in itertools.count(init_page):- resp: dict[str, Any] = await self.request_json(api_url.update_query(limit=100, page=page))- yield resp["data"]- if not resp["pagination"]["hasMore"]:- break+ await self._playlist(scrape_item, playlist)+ video_ids = await self._iter_video_pages(scrape_item, video_pages)+ missing_videos = set(playlist.videos) - video_ids+ if missing_videos:+ self.log.warning(+ "Playlist %s reports %s videos but only %s videos were found. The following videos have been removed: %s",+ scrape_item.url,+ len(playlist.videos),+ len(video_ids),+ sorted(missing_videos),+ )++ async def _playlist(self, scrape_item: ScrapeItem, playlist: Playlist) -> None:+ self.create_eager_task(self.write_metadata(scrape_item, playlist.id, playlist))+ with self.catch_errors(playlist.thumbnail):+ _, ext = self.get_filename_and_ext(playlist.thumbnail.name)+ filename = self.create_custom_filename(playlist.name, ext)+ await self.handle_file(+ playlist.thumbnail,+ scrape_item,+ playlist.thumbnail.name,+ ext,+ frag="thumbnail",+ custom_filename=filename,+ )++ async def _iter_video_pages(self, scrape_item: ScrapeItem, video_pages: AsyncIterable[Iterable[Video]]) -> set[str]:+ seen: set[str] = set()+ async for videos in video_pages:+ seen.update(self._iter_videos(scrape_item, videos))+ return seen++ def _iter_videos(self, scrape_item: ScrapeItem, videos: Iterable[Video]) -> set[str]:+ seen: set[str] = set()+ for video in videos:+ seen.add(video.id)+ new_item = scrape_item.copy()+ new_item.url = self.PRIMARY_URL / video.href+ self.create_eager_task(self._video(new_item, video))+ scrape_item.add_children()+ return seen@error_handling_wrapperasync def search(self, scrape_item: ScrapeItem, query: str) -> None:title = self.create_title(f"{query} [search]")scrape_item.setup_as_profile(title)- api_url = (self.PRIMARY_URL / "api/videos/search").with_query(q=query)- data: list[dict[str, Any]]- async for data in self._api_pager(api_url):- for video in _parse_videos(data):- await self._video(scrape_item.copy(), video)- scrape_item.add_children()+ await self._iter_video_pages(scrape_item, self.api.search(query))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:if await self.check_complete_from_referer(scrape_item.url):return- api_url = self.PRIMARY_URL / "api/videos" / video_id- data = (await self.request_json(api_url))["data"]- video = Video.from_dict(data)+ video = await self.api.video(video_id)await self._video(scrape_item, video)@error_handling_wrapperasync def _video(self, scrape_item: ScrapeItem, video: Video) -> None:- scrape_item.url = video.web_urlscrape_item.uploaded_at = self.parse_iso_date(video.uploadDate)- link = self.parse_url(video.videoUrl)- filename, ext = self.get_filename_and_ext(link.name, assume_ext=".mp4")+ _, ext = self.get_filename_and_ext(video.videoUrl.name, assume_ext=".mp4")custom_filename = self.create_custom_filename(video.title,ext,file_id=video.id,- resolution=Resolution(video.width, video.height),+ resolution=Resolution(video.width, video.height) if video.width and video.height else None,)- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video)+ await self.handle_file(+ video.videoUrl,+ scrape_item,+ video.title,+ ext,+ custom_filename=custom_filename,+ metadata=video,+ )+++_deserialize = Deserializer(+ aliases={"id": "_id"},+ converters={+ "thumbnail": parse_url,+ "videoUrl": parse_url,+ "hlsMasterPlaylistUrl": lambda url: url and parse_url(url),+ },+)++[email protected](slots=True)+class Video:+ # TODO: parse and download previews and thumbnails+ id: str+ title: str+ videoUrl: AbsoluteHttpURL+ uploadDate: str+ width: int | None = None+ height: int | None = None+ hlsMasterPlaylistUrl: AbsoluteHttpURL | None = None++ @property+ def href(self) -> str:+ # The title does not matter, the website parses the id from the url and redirects to the correct video+ return f"video/{self.title.lower()}_{self.id}"++ parse = classmethod(_deserialize)++[email protected](slots=True)+class Playlist:+ id: str+ name: str+ description: str+ thumbnail: AbsoluteHttpURL+ createdAt: str+ updatedAt: str+ videos: list[str]+++class PMVAPI(API):+ async def video(self, video_id: str) -> Video:+ api_url = self.PRIMARY_URL / "api/videos" / video_id+ data = (await self.request_json(api_url))["data"]+ return Video.parse(data)++ async def search(self, query: str) -> AsyncGenerator[map[Video]]:+ api_url = (self.PRIMARY_URL / "api/videos/search").with_query(q=query)+ data: list[dict[str, Any]]+ async for data in self.pager(api_url):+ yield _parse_videos(data)++ async def playlist(self, playlist_id: str) -> tuple[Playlist, AsyncGenerator[map[Video]]]:+ api_url = self.PRIMARY_URL / "api/playlists" / playlist_id+ data, pages = await aio.peek_first(self.pager(api_url))+ playlist = _deserialize(Playlist, data)++ async def video_pages():+ async for data in pages:+ yield _parse_videos(data["videoDetails"])++ return playlist, video_pages()++ async def pager(self, api_url: AbsoluteHttpURL, init_page: int = 1) -> AsyncGenerator[Any]:+ for page in itertools.count(init_page):+ resp: dict[str, Any] = await self.request_json(api_url.update_query(limit=100, page=page))+ yield resp["data"]+ if not resp["pagination"]["hasMore"]:+ break++ async def profile(self, user_id: str) -> tuple[str, map[Video]]:+ url = self.PRIMARY_URL / "profile" / user_id+ soup = await self.request_soup(url)+ data = nuxt.extract(soup)+ username: str = nuxt.find(data, "username")["username"]+ videos = _parse_videos(nuxt.ifind(data, "videoUrl", "title"))+ return username, videos-def _parse_videos(videos: Iterable[dict[str, Any]]) -> Generator[Video]:- ids: set[str] = set()- for data in videos:- if data["_id"] not in ids:- ids.add(data["_id"])- yield Video.from_dict(data)+def _parse_videos(videos: Iterable[dict[str, Any]]) -> map[Video]:+ return map(Video.parse, unique(videos, operator.itemgetter("_id")))diff --git a/tests/crawlers/test_cases/pmvhaven.py b/tests/crawlers/test_cases/pmvhaven.pyindex 80fb85475..ad8faadde 100644--- a/tests/crawlers/test_cases/pmvhaven.py+++ b/tests/crawlers/test_cases/pmvhaven.py@@ -4,21 +4,38 @@ TEST_CASES = ["url": "https://pmvhaven.com/video/real-hookers-pmv_69301129194866d69ec96b5d","results": [{- "url": "https://video.pmvhaven.com/videos/1764757703719_yxthkzjbhbp_Hookers.mp4",+ "url": "re:/videos/1764757703719_yxthkzjbhbp_Hookers.mp4","filename": "Real Hookers - PMV [69301129194866d69ec96b5d][1080p].mp4",- "original_filename": "1764757703719_yxthkzjbhbp_Hookers.mp4",+ "original_filename": "Real Hookers - PMV","download_folder": "re:Loose Files (PMVHaven)","referer": "https://pmvhaven.com/video/real-hookers-pmv_69301129194866d69ec96b5d","uploaded_at": 1764757801,}],},+ {+ "url": "https://pmvhaven.com/video/6944a3a0c7a6aeae188b1fa8",+ "description": "video with no resolution info",+ "results": [+ {+ "url": "https://pmvhavencloud.s3.eu-west-par.io.cloud.ovh.net/videos/LordOfShade_-_Big_Ass_Compilation__1766106016843_gs09jmf7.mp4",+ "filename": "Big Ass Compilation [6944a3a0c7a6aeae188b1fa8].mp4",+ "debrid_url": None,+ "original_filename": "Big Ass Compilation",+ "referer": "https://pmvhaven.com/video/6944a3a0c7a6aeae188b1fa8",+ "album_id": None,+ "uploaded_at": 1766106016,+ "download_folder": "re:Loose Files (PMVHaven)",+ }+ ],+ "count": 1,+ },{"url": "https://pmvhaven.com/profile/68feea724e81de92343a01ef","results": [{"url": "ANY", "download_folder": "re:CheekyGooner [user] (PMVHaven)", "referer": "ANY", "album_id": None}],- "count": range(240, 300),+ "count": range(280, 300),},{"url": "https://pmvhaven.com/users/68feea8c4e81de92343a1fd7",@@ -37,6 +54,6 @@ TEST_CASES = ["album_id": None,}],- "count": 10,+ "count": 11,},]