Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: switch to API (PMVHaven) (#52)

NTFSvolume333bb60c9ff0
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index 121961078..c5009f911 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -1,23 +1,47 @@
+# ruff: noqa: N815
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+import dataclasses
+import itertools
+from typing import TYPE_CHECKING, Any, ClassVar, Self
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures.mediaprops import Resolution
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, nuxt
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
+from cyberdrop_dl.utils import nuxt
+from cyberdrop_dl.utils.utilities import call_w_valid_kwargs, error_handling_wrapper
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Generator, Iterable
from typing import Any
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-class Selectors:
- USER_NAME = "div.user-profile-card h1"
+PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")
-PRIMARY_URL = AbsoluteHttpURL("https://pmvhaven.com")
[email protected](slots=True)
+class Video:
+ # TODO: parse and download previews and thumbnails
+ id: str
+ title: str
+ videoUrl: str
+ uploadDate: str
+ width: int
+ height: int
+
+ hlsMasterPlaylistUrl: str | None = None
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> Self:
+ data["id"] = data["_id"]
+ return call_w_valid_kwargs(cls, data)
+
+ @property
+ def web_url(self) -> AbsoluteHttpURL:
+ # The title does not matter, the website parses the id from the url and redirects to the correct video
+ return PRIMARY_URL / f"video/{self.title.lower()}_{self.id}"
class PMVHavenCrawler(Crawler):
@@ -35,69 +59,90 @@ class PMVHavenCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "PMVHaven"
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "video" in scrape_item.url.parts:
- return await self.video(scrape_item)
- if "search" in scrape_item.url.parts:
- return await self.search(scrape_item)
- if any(u in scrape_item.url.parts for u in ("users", "profile")):
- return await self.profile(scrape_item)
- if "playlists" in scrape_item.url.parts:
- return await self.playlist(scrape_item)
- raise ValueError
+ match scrape_item.url.parts[1:]:
+ case ["video", slug] if video_id := slug.rsplit("_", 1)[-1]:
+ return await self.video(scrape_item, video_id)
+ case ["search"] if query := scrape_item.url.query.get("q"):
+ return await self.search(scrape_item, query)
+ case ["users" | "profile", _]:
+ return await self.profile(scrape_item)
+ case ["playlists", playlist_id]:
+ return await self.playlist(scrape_item, playlist_id)
+ case _:
+ raise ValueError
@error_handling_wrapper
async def profile(self, scrape_item: ScrapeItem) -> None:
soup = await self.request_soup(scrape_item.url)
- username = css.select_text(soup, Selectors.USER_NAME)
- title = f"{username} [user]"
- title = self.create_title(title)
+ data = nuxt.extract(soup)
+ username: str = nuxt.find(data, "username")["username"]
+ title = self.create_title(f"{username} [user]")
scrape_item.setup_as_profile(title)
- nuxt_data = nuxt.extract(soup)
- await self._process_video_list(scrape_item, nuxt_data)
+ for video in _parse_videos(nuxt.ifind(data, "videoUrl", "title")):
+ await self._video(scrape_item.copy(), video)
+ scrape_item.add_children()
@error_handling_wrapper
- async def playlist(self, scrape_item: ScrapeItem) -> None:
- soup = await self.request_soup(scrape_item.url)
- nuxt_data = nuxt.extract(soup)
- playlist = nuxt.parse_obj(nuxt_data, "playlist")
- name = playlist["name"]
- title = self.create_title(f"{name} [playlist]")
- scrape_item.setup_as_album(title)
- await self._process_video_list(scrape_item, nuxt_data)
+ async def playlist(self, scrape_item: ScrapeItem, playlist_id: str) -> None:
+ title: str = ""
+ # TODO: use playlist as album_id to skip downloads faster
+ api_url = self.PRIMARY_URL / "api/playlists" / playlist_id
+ async for data in self._api_pager(api_url):
+ if not title:
+ title = self.create_title(f"{data['name']} [playlist]")
+ scrape_item.setup_as_album(title)
+
+ for video in _parse_videos(data["videoDetails"]):
+ await self._video(scrape_item.copy(), video)
+ scrape_item.add_children()
+
+ async def _api_pager(self, api_url: AbsoluteHttpURL, init_page: int = 1) -> AsyncGenerator[Any]:
+ for page in itertools.count(init_page):
+ resp: dict[str, Any] = await self.request_json(api_url.update_query(limit=100, page=page))
+ yield resp["data"]
+ if not resp["pagination"]["hasMore"]:
+ break
@error_handling_wrapper
- async def search(self, scrape_item: ScrapeItem) -> None:
- soup = await self.request_soup(scrape_item.url)
- tags = scrape_item.url.query.get("tags") or scrape_item.url.query.get("musicSong")
- title = self.create_title(f"{tags} [search]")
- scrape_item.setup_as_album(title)
- nuxt_data = nuxt.extract(soup)
- await self._process_video_list(scrape_item, nuxt_data)
+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:
+ title = self.create_title(f"{query} [search]")
+ scrape_item.setup_as_profile(title)
+ api_url = (self.PRIMARY_URL / "api/videos/search").with_query(q=query)
+ data: list[dict[str, Any]]
+ async for data in self._api_pager(api_url):
+ for video in _parse_videos(data):
+ await self._video(scrape_item.copy(), video)
+ scrape_item.add_children()
@error_handling_wrapper
- async def video(self, scrape_item: ScrapeItem) -> None:
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- soup = await self.request_soup(scrape_item.url)
- nuxt_data = nuxt.extract(soup)
- video = nuxt.parse_obj(nuxt_data, "video", "uploaderVideosCount")
- await self._process_video_info(scrape_item, video)
-
- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ api_url = self.PRIMARY_URL / "api/videos" / video_id
+ data = (await self.request_json(api_url))["data"]
+ video = Video.from_dict(data)
+ await self._video(scrape_item, video)
@error_handling_wrapper
- async def _process_video_list(self, scrape_item: ScrapeItem, nuxt_data: list[Any]) -> None:
- for video_info in nuxt.parse_objs(nuxt_data, "videoUrl"):
- await self._process_video_info(scrape_item, video_info)
-
- @error_handling_wrapper
- async def _process_video_info(self, scrape_item: ScrapeItem, video_info: dict[str, Any]) -> None:
- scrape_item.possible_datetime = self.parse_date(video_info["uploadDate"])
- link = self.parse_url(video_info["videoUrl"])
+ async def _video(self, scrape_item: ScrapeItem, video: Video) -> None:
+ scrape_item.url = video.web_url
+ scrape_item.possible_datetime = self.parse_iso_date(video.uploadDate)
+ link = self.parse_url(video.videoUrl)
filename, ext = self.get_filename_and_ext(link.name, assume_ext=".mp4")
custom_filename = self.create_custom_filename(
- video_info["title"], ext, file_id=video_info["_id"], resolution=video_info["height"]
+ video.title,
+ ext,
+ file_id=video.id,
+ resolution=Resolution(video.width, video.height),
)
- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video_info)
+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, metadata=video)
+
+
+def _parse_videos(videos: Iterable[dict[str, Any]]) -> Generator[Video]:
+ ids: set[str] = set()
+ for data in videos:
+ if data["_id"] not in ids:
+ ids.add(data["_id"])
+ yield Video.from_dict(data)
diff --git a/cyberdrop_dl/utils/nuxt.py b/cyberdrop_dl/utils/nuxt.py
index 4377e220f..e7c97d999 100644
--- a/cyberdrop_dl/utils/nuxt.py
+++ b/cyberdrop_dl/utils/nuxt.py
@@ -19,16 +19,16 @@ def extract(soup: Tag) -> list[Any]:
return json.loads(css.select_text(soup, "script#__NUXT_DATA__"))
-def parse_obj(nuxt_data: list[Any], attribute: str, *attributes: str) -> dict[str, Any]:
+def find(nuxt_data: list[Any], attribute: str, *attributes: str) -> dict[str, Any]:
"""Parses a single object from a NUXT rich JSON payload response (__NUXT_DATA__)
It iterates over each object until it finds an object with the desired attributes"""
- if obj := next(parse_objs(nuxt_data, attribute, *attributes), None):
+ if obj := next(ifind(nuxt_data, attribute, *attributes), None):
return obj
raise css.SelectorError(f"Unable to find object with {attributes = } in NUXT_DATA")
-def parse_objs(nuxt_data: list[Any], attribute: str, *attributes: str) -> Generator[dict[str, Any]]:
+def ifind(nuxt_data: list[Any], attribute: str, *attributes: str) -> Generator[dict[str, Any]]:
"""
Iterates over each object from a NUXT rich JSON payload response (__NUXT_DATA__)
diff --git a/tests/crawlers/test_cases/pmvhaven.py b/tests/crawlers/test_cases/pmvhaven.py
index 1fcdd0309..ab768e954 100644
--- a/tests/crawlers/test_cases/pmvhaven.py
+++ b/tests/crawlers/test_cases/pmvhaven.py
@@ -25,7 +25,7 @@ TEST_CASES = [
"album_id": None,
}
],
- range(170, 200),
+ range(240, 300),
),
# Users
(
@@ -46,11 +46,11 @@ TEST_CASES = [
[
{
"url": "ANY",
- "download_folder": r"re:10 GODDESS [playlist] (PMVHaven)",
+ "download_folder": r"re:10 FUCKING GODDESSES [playlist] (PMVHaven)",
"referer": "ANY",
"album_id": None,
}
],
- range(10, 20),
+ 10,
),
]