Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: switch to API (PMVHaven) (#52)
NTFSvolume333bb60c9ff0
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex 121961078..c5009f911 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -1,23 +1,47 @@+# ruff: noqa: N815from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+import dataclasses+import itertools+from typing import TYPE_CHECKING, Any, ClassVar, Selffrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.data_structures.mediaprops import Resolutionfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, nuxt-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.utils import nuxt+from cyberdrop_dl.utils.utilities import call_w_valid_kwargs, error_handling_wrapperif TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Generator, Iterablefrom typing import Anyfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem-class Selectors:- USER_NAME = "div.user-profile-card h1"+PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")-PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")[email protected](slots=True)+class Video:+ # TODO: parse and download previews and thumbnails+ id: str+ title: str+ videoUrl: str+ uploadDate: str+ width: int+ height: int++ hlsMasterPlaylistUrl: str | None = None++ @classmethod+ def from_dict(cls, data: dict[str, Any]) -> Self:+ data["id"] = data["_id"]+ return call_w_valid_kwargs(cls, data)++ @property+ def web_url(self) -> AbsoluteHttpURL:+ # The title does not matter, the website parses the id from the url and redirects to the correct video+ return PRIMARY_URL / f"video/{self.title.lower()}_{self.id}"class PMVHavenCrawler(Crawler):@@ -35,69 +59,90 @@ class PMVHavenCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "PMVHaven"async def fetch(self, scrape_item: ScrapeItem) -> None:- if "video" in scrape_item.url.parts:- return await self.video(scrape_item)- if "search" in scrape_item.url.parts:- return await self.search(scrape_item)- if any(u in scrape_item.url.parts for u in ("users", "profile")):- return await self.profile(scrape_item)- if "playlists" in scrape_item.url.parts:- return await self.playlist(scrape_item)- raise ValueError+ match scrape_item.url.parts[1:]:+ case ["video", slug] if video_id := slug.rsplit("_", 1)[-1]:+ return await self.video(scrape_item, video_id)+ case ["search"] if query := scrape_item.url.query.get("q"):+ return await self.search(scrape_item, query)+ case ["users" | "profile", _]:+ return await self.profile(scrape_item)+ case ["playlists", playlist_id]:+ return await self.playlist(scrape_item, playlist_id)+ case _:+ raise ValueError@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(scrape_item.url)- username = css.select_text(soup, Selectors.USER_NAME)- title = f"{username} [user]"- title = self.create_title(title)+ data = nuxt.extract(soup)+ username: str = nuxt.find(data, "username")["username"]+ title = self.create_title(f"{username} [user]")scrape_item.setup_as_profile(title)- nuxt_data = nuxt.extract(soup)- await self._process_video_list(scrape_item, nuxt_data)+ for video in _parse_videos(nuxt.ifind(data, "videoUrl", "title")):+ await self._video(scrape_item.copy(), video)+ scrape_item.add_children()@error_handling_wrapper- async def playlist(self, scrape_item: ScrapeItem) -> None:- soup = await self.request_soup(scrape_item.url)- nuxt_data = nuxt.extract(soup)- playlist = nuxt.parse_obj(nuxt_data, "playlist")- name = playlist["name"]- title = self.create_title(f"{name} [playlist]")- scrape_item.setup_as_album(title)- await self._process_video_list(scrape_item, nuxt_data)+ async def playlist(self, scrape_item: ScrapeItem, playlist_id: str) -> None:+ title: str = ""+ # TODO: use playlist as album_id to skip downloads faster+ api_url = self.PRIMARY_URL / "api/playlists" / playlist_id+ async for data in self._api_pager(api_url):+ if not title:+ title = self.create_title(f"{data['name']} [playlist]")+ scrape_item.setup_as_album(title)++ for video in _parse_videos(data["videoDetails"]):+ await self._video(scrape_item.copy(), video)+ scrape_item.add_children()++ async def _api_pager(self, api_url: AbsoluteHttpURL, init_page: int = 1) -> AsyncGenerator[Any]:+ for page in itertools.count(init_page):+ resp: dict[str, Any] = await self.request_json(api_url.update_query(limit=100, page=page))+ yield resp["data"]+ if not resp["pagination"]["hasMore"]:+ break@error_handling_wrapper- async def search(self, scrape_item: ScrapeItem) -> None:- soup = await self.request_soup(scrape_item.url)- tags = scrape_item.url.query.get("tags") or scrape_item.url.query.get("musicSong")- title = self.create_title(f"{tags} [search]")- scrape_item.setup_as_album(title)- nuxt_data = nuxt.extract(soup)- await self._process_video_list(scrape_item, nuxt_data)+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:+ title = self.create_title(f"{query} [search]")+ scrape_item.setup_as_profile(title)+ api_url = (self.PRIMARY_URL / "api/videos/search").with_query(q=query)+ data: list[dict[str, Any]]+ async for data in self._api_pager(api_url):+ for video in _parse_videos(data):+ await self._video(scrape_item.copy(), video)+ scrape_item.add_children()@error_handling_wrapper- async def video(self, scrape_item: ScrapeItem) -> None:+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:if await self.check_complete_from_referer(scrape_item):return- soup = await self.request_soup(scrape_item.url)- nuxt_data = nuxt.extract(soup)- video = nuxt.parse_obj(nuxt_data, "video", "uploaderVideosCount")- await self._process_video_info(scrape_item, video)-- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ api_url = self.PRIMARY_URL / "api/videos" / video_id+ data = (await self.request_json(api_url))["data"]+ video = Video.from_dict(data)+ await self._video(scrape_item, video)@error_handling_wrapper- async def _process_video_list(self, scrape_item: ScrapeItem, nuxt_data: list[Any]) -> None:- for video_info in nuxt.parse_objs(nuxt_data, "videoUrl"):- await self._process_video_info(scrape_item, video_info)-- @error_handling_wrapper- async def _process_video_info(self, scrape_item: ScrapeItem, video_info: dict[str, Any]) -> None:- scrape_item.possible_datetime = self.parse_date(video_info["uploadDate"])- link = self.parse_url(video_info["videoUrl"])+ async def _video(self, scrape_item: ScrapeItem, video: Video) -> None:+ scrape_item.url = video.web_url+ scrape_item.possible_datetime = self.parse_iso_date(video.uploadDate)+ link = self.parse_url(video.videoUrl)filename, ext = self.get_filename_and_ext(link.name, assume_ext=".mp4")custom_filename = self.create_custom_filename(- video_info["title"], ext, file_id=video_info["_id"], resolution=video_info["height"]+ video.title,+ ext,+ file_id=video.id,+ resolution=Resolution(video.width, video.height),)- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video_info)+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video)+++def _parse_videos(videos: Iterable[dict[str, Any]]) -> Generator[Video]:+ ids: set[str] = set()+ for data in videos:+ if data["_id"] not in ids:+ ids.add(data["_id"])+ yield Video.from_dict(data)diff --git a/cyberdrop_dl/utils/nuxt.py b/cyberdrop_dl/utils/nuxt.pyindex 4377e220f..e7c97d999 100644--- a/cyberdrop_dl/utils/nuxt.py+++ b/cyberdrop_dl/utils/nuxt.py@@ -19,16 +19,16 @@ def extract(soup: Tag) -> list[Any]:return json.loads(css.select_text(soup, "script#__NUXT_DATA__"))-def parse_obj(nuxt_data: list[Any], attribute: str, *attributes: str) -> dict[str, Any]:+def find(nuxt_data: list[Any], attribute: str, *attributes: str) -> dict[str, Any]:"""Parses a single object from a NUXT rich JSON payload response (__NUXT_DATA__)It iterates over each object until it finds an object with the desired attributes"""- if obj := next(parse_objs(nuxt_data, attribute, *attributes), None):+ if obj := next(ifind(nuxt_data, attribute, *attributes), None):return objraise css.SelectorError(f"Unable to find object with {attributes = } in NUXT_DATA")-def parse_objs(nuxt_data: list[Any], attribute: str, *attributes: str) -> Generator[dict[str, Any]]:+def ifind(nuxt_data: list[Any], attribute: str, *attributes: str) -> Generator[dict[str, Any]]:"""Iterates over each object from a NUXT rich JSON payload response (__NUXT_DATA__)diff --git a/tests/crawlers/test_cases/pmvhaven.py b/tests/crawlers/test_cases/pmvhaven.pyindex 1fcdd0309..ab768e954 100644--- a/tests/crawlers/test_cases/pmvhaven.py+++ b/tests/crawlers/test_cases/pmvhaven.py@@ -25,7 +25,7 @@ TEST_CASES = ["album_id": None,}],- range(170, 200),+ range(240, 300),),# Users(@@ -46,11 +46,11 @@ TEST_CASES = [[{"url": "ANY",- "download_folder": r"re:10 GODDESS [playlist] (PMVHaven)",+ "download_folder": r"re:10 FUCKING GODDESSES [playlist] (PMVHaven)","referer": "ANY","album_id": None,}],- range(10, 20),+ 10,),]