Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rework crawler (Pornhub) (#2113)

NTFSvolume880bbdc405c2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 86b3f4103..affdda4e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Corrupted/mixed up video segments when downloading multiple concurrent HLS streams and `--subfolders.create` is `False`
- `403 Forbidden` on all downloads (AdobeLightroom)
+- `400 Bad request` for albums (PornHub)
- URL matching (Anysex)
## [10.7.0] - 2026-08-29
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.py
index 5b7167682..61fd5950c 100644
--- a/cyberdrop_dl/config/crawlers.py
+++ b/cyberdrop_dl/config/crawlers.py
@@ -122,6 +122,16 @@ class ClonrConfig(ConfigModel):
raise ValueError("'clonr.zip' and 'clonr.use_source' are mutually exclusive")
+class PornHubConfig(ConfigModel):
+ profile_paths: tuple[NonEmptyStr, ...] = "photos/public", "gifs/public", "videos", "videos/upload"
+ "Subpaths to scrape when an input URL is a profile's homepage"
+
+ @override
+ def model_post_init(self, context: Any, /) -> None:
+ super().model_post_init(context)
+ self.profile_paths = tuple(p.lstrip("/") for p in self.profile_paths)
+
+
class GenericCrawlers(ConfigModel):
wordpress_media: tuple[HttpURL, ...] = ()
wordpress_html: tuple[HttpURL, ...] = ()
@@ -145,3 +155,4 @@ class Crawlers(ConfigGroup, name=None):
only_haven: KemonoConfig = Field(default_factory=KemonoConfig)
octave_music: OctaveMusicConfig = Field(default_factory=OctaveMusicConfig)
clonr: ClonrConfig = Field(default_factory=ClonrConfig)
+ pornhub: PornHubConfig = Field(default_factory=PornHubConfig)
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index 0e3e8cac0..bc79b85a6 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -54,6 +54,12 @@ crawlers:
content_urls: true
embed: true
file: true
+ pornhub:
+ profile_paths:
+ - photos/public
+ - gifs/public
+ - videos
+ - videos/upload
tiktok:
original: false
twitter:
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index ab0529bc9..f96ddab46 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -188,6 +188,9 @@
},
"clonr": {
"$ref": "#/$defs/ClonrConfig"
+ },
+ "pornhub": {
+ "$ref": "#/$defs/PornHubConfig"
}
},
"title": "Crawlers",
@@ -934,6 +937,26 @@
"title": "OnePaceConfig",
"type": "object"
},
+ "PornHubConfig": {
+ "additionalProperties": false,
+ "properties": {
+ "profile_paths": {
+ "default": [
+ "photos/public",
+ "gifs/public",
+ "videos",
+ "videos/upload"
+ ],
+ "items": {
+ "$ref": "#/$defs/NonEmptyStr"
+ },
+ "title": "Profile Paths",
+ "type": "array"
+ }
+ },
+ "title": "PornHubConfig",
+ "type": "object"
+ },
"Sort": {
"additionalProperties": false,
"properties": {
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index dbecf8942..e534cdff2 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -1070,6 +1070,10 @@ class API(HTTPMixin, ABC):
self.parse_url = crawler.parse_url
self.log = crawler.log
self.__http_config__ = config # pyright: ignore[reportAttributeAccessIssue]
+ try:
+ self.__http_ctx__.headers.setdefault("Referer", str(self.PRIMARY_URL))
+ except AttributeError:
+ pass
return self
def __post_init__(self) -> None: ...
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index 39ee2c979..8971b5217 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -1,318 +1,404 @@
from __future__ import annotations
import dataclasses
-import datetime
import json
from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, TypedDict
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict, final
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.cache import cached_method
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, extr_text
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Generator, Iterable
+
from bs4 import BeautifulSoup
from cyberdrop_dl.url_objects import ScrapeItem
-PRIMARY_URL = AbsoluteHttpURL("https://www.pornhub.com")
-MP4_NOT_AVAILABLE_SINCE = datetime.datetime(2025, 6, 25, tzinfo=datetime.UTC).timestamp()
-TOKEN_SELECTOR = css.CssAttributeSelector("input#xsrfToken", "value")
-
-
[email protected](frozen=True, slots=True)
-class Profile:
- type: str
- name: str
- download_videos: bool = dataclasses.field(compare=False)
- download_gifs: bool = dataclasses.field(compare=False)
- download_photos: bool = dataclasses.field(compare=False)
-
- @property
- def url(self) -> AbsoluteHttpURL:
- return PRIMARY_URL / self.type / self.name
-
- @staticmethod
- def new(type_: str, name: str, rest: list[str]) -> Profile:
- videos = gifs = photos = False
- match rest:
- case ["videos", *_]:
- videos = True
- case ["gifs", *_]:
- gifs = True
- case ["photos", *_]:
- photos = True
- case []:
- videos = True
- gifs = photos = "channel" not in type_
- case _:
- raise ValueError
- return Profile(type_, name, videos, gifs, photos)
-
+@final
class Selector:
- ALBUM_FROM_PHOTO = "div#thumbSlider > h2 > a"
- ALBUM_TITLE = "h1[class*=photoAlbumTitle]"
- DATE = "script:-soup-contains('uploadDate')"
+ FLASHVARS = "script:-soup-contains-own('var flashvars_')"
GIF = "div#js-gifToWebm"
- JS_VIDEO_INFO = "script:-soup-contains('var flashvars_')"
NEXT_PAGE = "li.page_next a"
PHOTO = "div#photoImageSection img"
- PLAYLIST_TITLE = "h1.playlistTitle"
- PLAYLIST_VIDEOS = "ul#videoPlaylist a.linkVideoThumb"
- TITLE = "div.title-container > h1.title"
- GEO_BLOCKED = ".geoBlocked > h1:-soup-contains('page is not available')"
- NO_VIDEO = "section.noVideo"
- REMOVED = "div.removed"
+ @final
+ class Playlist:
+ TITLE = "h1.playlistTitle"
+ VIDEOS = "ul#videoPlaylist a.linkVideoThumb"
- PROFILE_NAME = ".topProfileHeader h1[itemprop=name], div.title h1"
- PROFILE_VIDEOS = "div.container a.linkVideoThumb"
- PROFILE_GIFS = "#moreData li.gifLi a"
- PROFILE_ALBUMS = "#moreData.photosAlbumsListing a"
+ @final
+ class Album:
+ FROM_PHOTO = "div#thumbSlider > h2 > a"
+ TITLE = "h1[class*=photoAlbumTitle]"
-
-class Media(TypedDict):
- height: int
- width: int
- format: Literal["hls", "mp4"]
- videoUrl: str
- quality: str | list[str]
-
-
-class Format(NamedTuple):
- quality: int
- format: Literal["hls", "mp4"]
- url: str # "videoUrl"
-
- @staticmethod
- def new(media: Media) -> Format:
- quality = media["quality"]
- if isinstance(quality, str):
- try:
- quality = int(quality)
- except ValueError:
- pass
- if not isinstance(quality, int):
- quality = min(media["height"], media["width"])
- values: dict[str, Any] = {k: v for k, v in media.items() if k in Format._fields} | {"quality": quality}
- return Format(url=media["videoUrl"], **values)
+ @final
+ class Profile:
+ NAME = ".topProfileHeader h1[itemprop=name], div.title h1"
+ VIDEOS = "div.container a.linkVideoThumb"
+ GIFS = "#moreData li.gifLi a"
+ ALBUMS = "#moreData.photosAlbumsListing a"
class PornHubCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Album": "/album/...",
- "Channel": "/channel/...",
- "Gif": "/gif/...",
- "Photo": "/photo/...",
- "Playlist": "/playlist/...",
+ "Album": "/album/<album_id>",
+ "Channel": "/channel/<name>",
+ "Gif": "/gif/<gif_id>",
+ "Photo": "/photo/<photo_id>",
+ "Playlist": "/playlist/<playlist_id>",
"Profile": (
- "/user/...",
- "/model/...",
- "/pornstar/...",
+ "/users/<name>",
+ "/model/<name>",
+ "/pornstar/<name>",
+ ),
+ "Profile videos": (
+ "/users/<name>/videos",
+ "/model/<name>/videos",
+ "/pornstar/<name>/videos",
+ ),
+ "Profile uploaded videos": (
+ "/users/<name>/videos/upload",
+ "/model/<name>/videos/upload",
+ "/pornstar/<name>/videos/upload",
+ ),
+ "Profile clips": (
+ "/users/<name>/clips",
+ "/model/<name>/clips",
+ "/pornstar/<name>/clips",
+ ),
+ "Profile albums": (
+ "/users/<name>/photos",
+ "/model/<name>/photos",
+ "/pornstar/<name>/photos",
+ ),
+ "Profile gifs": (
+ "/users/<name>/gifs",
+ "/model/<name>/gifs",
+ "/pornstar/<name>/gifs",
),
"Video": (
"/embed/<video_id>",
"/view_video.php?viewkey=<video_id>",
),
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.pornhub.com")
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
DOMAIN: ClassVar[str] = "pornhub"
FOLDER_DOMAIN: ClassVar[str] = "PornHub"
def __post_init__(self) -> None:
- self.seen_profiles: set[Profile] = set()
-
- async def __async_post_init__(self) -> None:
- keys = ("age_verified", "accessPH", "accessAgeDisclaimerPH", "accessAgeDisclaimerUK", "expiredEnterModalShown")
- self.update_cookies(dict.fromkeys(keys, 1))
+ self.api: PornHubAPI = PornHubAPI.from_crawler(self)
+ self.update_cookies(
+ dict.fromkeys(
+ (
+ "age_verified",
+ "accessPH",
+ "accessAgeDisclaimerPH",
+ "accessAgeDisclaimerUK",
+ "expiredEnterModalShown",
+ ),
+ 2,
+ )
+ )
- async def fetch(self, scrape_item: ScrapeItem) -> None: # noqa: PLR0911
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
- case ["user" | "channel" | "channels" | "model" | "pornstar" as type_, name, *rest]:
- profile = Profile.new(type_, name, rest)
- if profile in self.seen_profiles:
- return None
- self.seen_profiles.add(profile)
- return await self.profile(scrape_item, profile)
+ case [
+ "user" | "users" | "channel" | "channels" | "model" | "models" | "pornstar" | "pornstars" as type_,
+ name,
+ *_,
+ ]:
+ await PornHubProfileCrawler(self, Profile(type_, name), scrape_item).fetch()
case ["album", album_id]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["playlist", playlist_id]:
- return await self.playlist(scrape_item, playlist_id)
- case ["photo", _]:
- return await self.photo(scrape_item)
- case ["gif", _]:
- return await self.gif(scrape_item)
+ await self.playlist(scrape_item, playlist_id)
+ case ["photo", photo_id]:
+ await self.photo(scrape_item, photo_id)
+ case ["gif", gif_id]:
+ await self.gif(scrape_item, gif_id)
case ["embed", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["view_video.php"] if video_id := scrape_item.url.query.get("viewkey"):
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
- @error_handling_wrapper
- async def profile(self, scrape_item: ScrapeItem, profile: Profile) -> None:
- title = await self._get_profile_title(profile.url)
- title = self.create_title(f"{title} [{profile.type.removesuffix('s')}]")
- scrape_item.setup_as_profile(title)
-
- if profile.download_videos:
- await self.iter_profile_pages(scrape_item, profile.url / "videos", Selector.PROFILE_VIDEOS)
- if profile.download_gifs:
- await self.iter_profile_pages(scrape_item, profile.url / "gifs/public", Selector.PROFILE_GIFS)
- if profile.download_photos:
- await self.iter_profile_pages(scrape_item, profile.url / "photos/public", Selector.PROFILE_ALBUMS)
-
- async def _get_profile_title(self, url: AbsoluteHttpURL) -> str:
- soup = await self.request_soup(url)
- return css.select_text(soup, Selector.PROFILE_NAME, decompose="span")
-
- @error_handling_wrapper
- async def iter_profile_pages(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str) -> None:
- async for soup in self.web_pager(url):
- for new_scrape_item in self.iter_children(scrape_item, soup, selector):
- self.create_task(self.run(new_scrape_item))
-
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
- soup = await self.request_soup(scrape_item.url)
- album_name = css.select_text(soup, Selector.ALBUM_TITLE)
- scrape_item.setup_as_album(self.create_title(album_name, album_id), album_id=album_id)
-
- api_url = self.PRIMARY_URL / "api/v1/album" / album_id / "show_album_json"
- json_resp: dict[str, Any] = await self.request_json(api_url.with_query(token=TOKEN_SELECTOR(soup)))
- photos: dict[str, dict[str, Any]] = json_resp["photos"]
- results = await self.get_album_results(album_id)
- for id_, photo in photos.items():
- web_url = PRIMARY_URL / "photo" / id_
- link = self.parse_url(photo["img_large"])
- new_scrape_item = scrape_item.create_new(web_url)
- self.create_task(self._process_photo(new_scrape_item, link, results))
+ album = await self.api.album(album_id)
+ scrape_item.setup_as_album(self.create_title(album.name, album.id), album_id=album.id)
+ downloaded = await self.get_album_results(album.id)
+
+ async for photo in self.api.album_photos(album.id):
+ if self.check_album_results(photo.canonical_url, downloaded):
+ continue
+ web_url = self.PRIMARY_URL / "photo" / photo.id
+ new_item = scrape_item.create_child(web_url)
+ self.create_eager_task(self._photo(new_item, photo))
scrape_item.add_children()
@error_handling_wrapper
- async def photo(self, scrape_item: ScrapeItem) -> None:
+ async def photo(self, scrape_item: ScrapeItem, photo_id: str) -> None:
if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
- link_str: str = css.select(soup, Selector.PHOTO, "src")
- link = self.parse_url(link_str)
- album_tag = css.select(soup, Selector.ALBUM_FROM_PHOTO)
- album_name = css.text(album_tag)
- album_link_str: str = css.attr(album_tag, "href")
- album_id: str = album_link_str.rsplit("/", maxsplit=1)[-1]
- title = self.create_title(album_name, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
- await self._process_photo(scrape_item, link)
+ src = css.select(soup, Selector.PHOTO, "src")
+ album = _extr_album(soup)
+ scrape_item.setup_as_album(self.create_title(album.name, album.id), album_id=album.id)
+ await self._photo(scrape_item, Photo(photo_id, self.parse_url(src)))
@error_handling_wrapper
- async def gif(self, scrape_item: ScrapeItem) -> None:
+ async def gif(self, scrape_item: ScrapeItem, gif_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
soup = await self.request_soup(scrape_item.url)
- attributes = "data-mp4", "data-fallback", "data-webm"
- gif_tag = css.select(soup, Selector.GIF)
- link_str = next(value for attr in attributes if (value := css.attr_or_none(gif_tag, attr)))
- link = self.parse_url(link_str)
- await self._process_photo(scrape_item, link)
+ gif = css.select(soup, Selector.GIF)
+ attrs = ("data-mp4", "data-fallback", "data-webm")
+ src = next(value for attr in attrs if (value := css.attr_or_none(gif, attr)))
+ scrape_item.uploaded_at = self.parse_iso_date(_extr_upload_date(soup))
+ await self._photo(scrape_item, Photo(gif_id, self.parse_url(src)))
@error_handling_wrapper
- async def _process_photo(
- self, scrape_item: ScrapeItem, link: AbsoluteHttpURL, results: dict[str, Any] | None = None
- ) -> None:
- results = results or {}
- original_name = link.name.rsplit(")")[-1]
- name = original_name.removeprefix("original_")
- canonical_url = link.with_name(name)
- if self.check_album_results(canonical_url, results):
- return
- custom_filename, ext = self.get_filename_and_ext(name, assume_ext=".jpg")
+ async def _photo(self, scrape_item: ScrapeItem, photo: Photo) -> None:
+ filename, ext = self.get_filename_and_ext(photo.name, assume_ext=".jpg")
await self.handle_file(
- canonical_url, scrape_item, original_name, ext, custom_filename=custom_filename, debrid_link=link
+ photo.canonical_url,
+ scrape_item,
+ photo.original_name,
+ ext,
+ custom_filename=filename,
+ debrid_link=photo.src,
)
@error_handling_wrapper
async def playlist(self, scrape_item: ScrapeItem, playlist_id: str) -> None:
soup = await self.request_soup(scrape_item.url)
- title: str = css.select_text(soup, Selector.PLAYLIST_TITLE)
+ title: str = css.select_text(soup, Selector.Playlist.TITLE)
title = self.create_title(title, playlist_id)
scrape_item.setup_as_album(f"{title} [playlist]", album_id=playlist_id)
- for new_scrape_item in self.iter_children(scrape_item, soup, Selector.PLAYLIST_VIDEOS):
+ for new_scrape_item in self.iter_children(scrape_item, soup, Selector.Playlist.VIDEOS):
self.create_task(self.run(new_scrape_item, check_referer=True))
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- embed_url = PRIMARY_URL / "embed" / video_id
- page_url = PRIMARY_URL.joinpath("view_video.php").with_query(viewkey=video_id)
+ embed_url = self.PRIMARY_URL / "embed" / video_id
- if await self.check_complete_from_referer(page_url):
+ if await self.check_complete(embed_url):
return
- soup = await self.request_soup(page_url)
- _check_video_is_available(soup)
- title = css.select_text(soup, Selector.TITLE)
- formats = [Format.new(media) for media in get_media_list(soup)]
- best_hls = max(f for f in formats if f.format == "hls")
- debrid_link = m3u8 = best_format = None
- scrape_item.uploaded_at = date = self.parse_iso_date(get_upload_date_str(soup))
- assert date
- use_hls = date >= MP4_NOT_AVAILABLE_SINCE
-
- if not use_hls:
- best_format = await self.get_best_mp4_format(formats)
- if best_format is None:
- self.log.warning(f"Video {video_id} has no mp4 formats available. Falling back to HLS")
-
- else:
- debrid_link = self.parse_url(best_format.url)
-
- if use_hls or best_format is None:
- m3u8, _ = await self.request_m3u8_playlist(self.parse_url(best_hls.url), headers={"Referer": str(page_url)})
- best_format = best_hls
-
- scrape_item.url = page_url
- filename, ext = self.get_filename_and_ext(f"{video_id}.mp4")
- custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.quality)
+ video = await self.api.video(video_id)
+ scrape_item.uploaded_at = self.parse_iso_date(video.uploaded)
+ src = max(f for f in video.formats if f.format == "hls")
+ m3u8, _ = await self.request_m3u8_playlist(self.parse_url(src.url), headers={"Referer": str(video.url)})
+
+ scrape_item.url = video.url
+ filename = self.create_custom_filename(video.title, ext := ".mp4", file_id=video_id, resolution=src.resolution)
await self.handle_file(
embed_url,
scrape_item,
- filename,
+ video.title,
ext,
- custom_filename=custom_filename,
- debrid_link=debrid_link,
+ custom_filename=filename,
m3u8=m3u8,
+ thumbnail=video.thumb,
+ )
+
+
[email protected](frozen=True, slots=True)
+class PornHubProfileCrawler:
+ crawler: PornHubCrawler
+ profile: Profile
+ item: ScrapeItem
+
+ async def fetch(self) -> None:
+ with self.crawler.catch_errors(self.item):
+ match self.item.url.parts[3:]:
+ case ["videos" | "clips", *_]:
+ await self.pages(Selector.Profile.VIDEOS)
+ case ["gifs", *_]:
+ await self.pages(Selector.Profile.GIFS)
+ case ["photos", *_]:
+ await self.pages(Selector.Profile.ALBUMS)
+ case []:
+ await self.dispatch()
+ case _:
+ raise ScrapeError.unsupported()
+
+ async def dispatch(self) -> None:
+ await self._init()
+ for path in self.crawler.config.crawlers.pornhub.profile_paths:
+ new_item = self.item.create_child(self.profile.url / path)
+ self.crawler.create_task(self.crawler.run(new_item))
+ self.item.add_children()
+
+ async def pages(self, selector: str) -> None:
+ await self._init()
+ self.item.append_folders(*filter(None, self.item.url.parts[3:]))
+ await self._iter_pages(selector)
+
+ async def _init(self) -> None:
+ if self.profile in self.item.markers:
+ return
+
+ soup = await self.crawler.request_soup(self.profile.url)
+ name = css.select_text(soup, Selector.Profile.NAME, decompose="span")
+ title = self.crawler.create_title(f"{name} [{self.profile.type.removesuffix('s')}]")
+ self.item.setup_as_profile(title)
+ self.item.markers.append(self.profile)
+
+ async def _iter_pages(self, selector: str) -> None:
+ async for soup in self.crawler.web_pager(self.item.url):
+ for new_item in self.crawler.iter_children(self.item, soup, selector):
+ self.crawler.create_task(self.crawler.run(new_item))
+
+
[email protected](frozen=True, slots=True)
+class Profile:
+ type: str
+ name: str
+
+ @property
+ def url(self) -> AbsoluteHttpURL:
+ return PornHubCrawler.PRIMARY_URL / self.type / self.name
+
+
[email protected](slots=True, order=True)
+class Photo:
+ id: str
+ src: AbsoluteHttpURL
+
+ name: str = dataclasses.field(init=False)
+ original_name: str = dataclasses.field(init=False)
+ canonical_url: AbsoluteHttpURL = dataclasses.field(init=False)
+
+ def __post_init__(self) -> None:
+ self.original_name = self.src.name.rpartition(")")[-1]
+ self.name = self.original_name.removeprefix("original_")
+ self.canonical_url = self.src.with_name(self.name)
+
+
[email protected](order=True, slots=True, frozen=True)
+class Album:
+ id: str
+ name: str
+
+
+class Media(TypedDict):
+ height: int
+ width: int
+ format: Literal["hls", "mp4"]
+ videoUrl: str
+ quality: str | int | list[str]
+
+
[email protected](slots=True, order=True, frozen=True)
+class Format:
+ resolution: Resolution
+ format: Literal["hls", "mp4"]
+ url: str
+
+
[email protected](slots=True, frozen=True, order=True)
+class Video:
+ id: str
+ title: str
+ thumb: str | None
+ uploaded: str
+ formats: tuple[Format, ...]
+ url: AbsoluteHttpURL
+
+
+class PornHubAPI(API):
+ @cached_method(ttl=600)
+ async def csrf_token(self) -> str:
+ soup = await self.request_soup(self.PRIMARY_URL)
+ return css.select(soup, "input[data-token]", "data-token")
+
+ async def album(self, album_id: str) -> Album:
+ url = self.PRIMARY_URL / "album" / album_id
+ soup = await self.request_soup(url, impersonate="firefox")
+ return Album(album_id, name=css.select_text(soup, Selector.Album.TITLE))
+
+ async def album_photos(self, album_id: str) -> AsyncGenerator[Photo]:
+ url = self.PRIMARY_URL / "api/v1/album" / album_id / "show_album_json"
+ resp: dict[str, Any] = await self.request_json(url.with_query(token=await self.csrf_token()))
+ photos: dict[str, dict[str, Any]] = resp["photos"]
+ for photo_id, photo in photos.items():
+ yield Photo(photo_id, self.parse_url(photo["img_large"]))
+
+ async def video(self, video_id: str) -> Video:
+ page_url = self.PRIMARY_URL.joinpath("view_video.php").with_query(viewkey=video_id)
+ soup = await self.request_soup(page_url)
+ _check_video_is_available(soup)
+ flashvars = _extr_flashvars(soup)
+ if flashvars.get("video_unavailable_country", "false") != "false":
+ raise ScrapeError(HTTPStatus.FORBIDDEN, "Video is geo restricted")
+
+ return Video(
+ id=video_id,
+ title=flashvars["video_title"],
+ thumb=flashvars.get("image_url"),
+ formats=tuple(_parse_formats(flashvars["mediaDefinitions"])),
+ uploaded=_extr_upload_date(soup),
+ url=page_url,
)
- async def get_best_mp4_format(self, formats: list[Format]) -> Format | None:
- mp4_format = next((f for f in formats if f.format == "mp4"), None)
- if not mp4_format:
- raise ScrapeError(422, message="Unable to get mp4 format")
- mp4_media_url = self.parse_url(mp4_format.url)
- # This returns an empty list when downloading multiple videos concurrently
- mp4_media: list[Media] = await self.request_json(mp4_media_url)
- return max((Format.new(media) for media in mp4_media), default=None)
+def _extr_album(soup: BeautifulSoup) -> Album:
+ album = css.select(soup, Selector.Album.FROM_PHOTO)
+ url: str = css.attr(album, "href")
+ return Album(id=url.rpartition("/")[-1], name=css.text(album))
-def get_upload_date_str(soup: BeautifulSoup) -> str:
- date_text = css.select_text(soup, Selector.DATE)
- return extr_text(date_text, 'uploadDate": "', '",')
+def _extr_upload_date(soup: BeautifulSoup) -> str:
+ return css.json_ld(soup, "uploadDate")["uploadDate"]
-def get_media_list(soup: BeautifulSoup) -> list[Media]:
- flashvars: str = css.select(soup, Selector.JS_VIDEO_INFO).text
- media_text = extr_text(flashvars, '"mediaDefinitions":', '"isVertical"').strip().removesuffix(",")
- return json.loads(media_text)
+def _extr_flashvars(soup: BeautifulSoup) -> dict[str, Any]:
+ flashvars: str = css.select_text(soup, Selector.FLASHVARS)
+ payload = extr_text(flashvars, "{", "};").strip()
+ return json.loads("{" + payload + "}")
+
+
+def _parse_formats(medias: Iterable[Media]) -> Generator[Format]:
+ for media in medias:
+ quality = media["quality"]
+ res = None
+ if not isinstance(quality, list):
+ try:
+ quality = int(quality)
+ except ValueError:
+ pass
+
+ try:
+ res = Resolution.parse(quality)
+ except ValueError:
+ pass
+
+ res = res or Resolution(media["height"], media["width"])
+
+ yield Format(url=media["videoUrl"], format=media["format"], resolution=res)
def _check_video_is_available(soup: BeautifulSoup) -> None:
- if soup.select_one(Selector.NO_VIDEO):
+ if soup.select_one("section.noVideo"):
raise ScrapeError(HTTPStatus.NOT_FOUND)
page_text = soup.text
- if soup.select_one(Selector.GEO_BLOCKED) or "This content is unavailable in your country" in page_text:
+ if (
+ soup.select_one(".geoBlocked > h1:-soup-contains('page is not available')")
+ or "This content is unavailable in your country" in page_text
+ ):
raise ScrapeError(HTTPStatus.FORBIDDEN, "Video is geo restricted")
if (
@@ -322,7 +408,7 @@ def _check_video_is_available(soup: BeautifulSoup) -> None:
raise ScrapeError(HTTPStatus.UNAVAILABLE_FOR_LEGAL_REASONS)
if (
- soup.select_one(Selector.REMOVED)
+ soup.select_one("div.removed")
or "This video has been removed" in page_text
or "This video is currently unavailable" in page_text
):
diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.py
index 118bd2e83..c48c1501a 100644
--- a/cyberdrop_dl/url_objects.py
+++ b/cyberdrop_dl/url_objects.py
@@ -233,6 +233,7 @@ class ScrapeItem:
_type: ScrapeItemType | None = None
_uploaded_at: int | None = None
retry_info: RetryInfo | None = None
+ markers: list[Any] = dataclasses.field(default_factory=list)
__repr__ = signature.simple_repr("url", "folders", "uploaded_at")
@@ -389,7 +390,9 @@ class ScrapeItem:
def copy(self) -> Self:
"""Returns a deep copy of this scrape_item"""
- return copy.deepcopy(self)
+ me = copy.deepcopy(self)
+ me.markers = self.markers.copy()
+ return me
def get_referer(self) -> AbsoluteHttpURL | None:
if self.referer:
diff --git a/docs/reference/config/crawlers.md b/docs/reference/config/crawlers.md
index 90c68b8f2..03db14010 100644
--- a/docs/reference/config/crawlers.md
+++ b/docs/reference/config/crawlers.md
@@ -171,6 +171,26 @@ crawlers:
file: true
```
+# PornHub
+
+```yaml
+crawlers:
+ pornhub:
+ profile_paths:
+ - "photos/public"
+ - "gifs/public"
+ - "videos"
+ - "videos/upload"
+```
+
+## `profile_paths`
+
+| Type | Default |
+| --------------------- | ----------------------------------------------------------- |
+| list of `NonEmptyStr` | [`photos/public`, `gifs/public`, `videos`, `videos/upload`] |
+
+Subpaths to scrape when an input URL is a profile's homepage. ex: `https://www.pornhub.com/model/<model_name>`
+
# Tiktok
```yaml
diff --git a/pyproject.toml b/pyproject.toml
index ea3770fbc..64d5bb020 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "cyberdrop-dl-patched"
-version = "10.8.0dev1"
+version = "10.8.0dev2"
authors = [{name = "Jacob B", email = "[email protected]"}]
classifiers = [
"Development Status :: 5 - Production/Stable",
diff --git a/tests/crawlers/test_cases/pornhub.py b/tests/crawlers/test_cases/pornhub.py
index d249b49c9..16bc135f5 100644
--- a/tests/crawlers/test_cases/pornhub.py
+++ b/tests/crawlers/test_cases/pornhub.py
@@ -1,45 +1,109 @@
DOMAIN = "pornhub"
TEST_CASES = [
+ {
+ "url": "https://www.pornhub.com/gif/54960161",
+ "results": [
+ {
+ "url": "re:/pics/gifs/054/960/161/54960161a.mp4",
+ "filename": "54960161a.mp4",
+ "debrid_url": "re:/pics/gifs/054/960/161/54960161a.mp4?validfrom=",
+ "original_filename": "54960161a.mp4",
+ "referer": "https://www.pornhub.com/gif/54960161",
+ "album_id": None,
+ "uploaded_at": 1786683600,
+ "download_folder": "re:Loose Files (PornHub)",
+ "thumbnail": None,
+ }
+ ],
+ },
+ {
+ "url": "https://www.pornhub.com/view_video.php?viewkey=6890ddc0b4b11",
+ "results": [
+ {
+ "url": "https://www.pornhub.com/embed/6890ddc0b4b11",
+ "filename": "MILF Celebrates Victory in Public Parking Ramp by Flashing Her Titti [6890ddc0b4b11][1080p].mp4",
+ "debrid_url": None,
+ "original_filename": "MILF Celebrates Victory in Public Parking Ramp by Flashing Her Titties! Onlyfans Snookybaby28",
+ "referer": "https://www.pornhub.com/view_video.php?viewkey=6890ddc0b4b11",
+ "album_id": None,
+ "uploaded_at": 1754324606,
+ "download_folder": "re:Loose Files (PornHub)",
+ "thumbnail": "re:/c6371/videos/202508/04/18163005/original_18163005.mp4",
+ }
+ ],
+ },
{
"url": "https://www.pornhub.com/album/36252941",
"results": [
{
- "url": "re:ei.phncdn.com/pics/albums/036/252/941/453277801",
+ "url": "re:/pics/albums/036/252/941/453277841/453277841.jpg",
+ "filename": "453277841.jpg",
+ "debrid_url": "re:/pics/albums/036/252/941/453277841/",
+ "original_filename": "original_453277841.jpg",
+ "referer": "https://www.pornhub.com/photo/453277841",
+ "album_id": "36252941",
+ "uploaded_at": None,
+ "download_folder": "re:White Top White Striped Calvins (PornHub)",
+ "thumbnail": None,
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277871/453277871.jpg",
+ "filename": "453277871.jpg",
+ "original_filename": "original_453277871.jpg",
+ "referer": "https://www.pornhub.com/photo/453277871",
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277801/453277801.jpg",
"filename": "453277801.jpg",
"original_filename": "original_453277801.jpg",
"referer": "https://www.pornhub.com/photo/453277801",
- "album_id": "36252941",
- "download_folder": "re:White Top White Striped Calvins \\(PornHub\\)",
- "uploaded_at": None,
- }
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277861/453277861.jpg",
+ "filename": "453277861.jpg",
+ "original_filename": "original_453277861.jpg",
+ "referer": "https://www.pornhub.com/photo/453277861",
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277821/453277821.jpg",
+ "filename": "453277821.jpg",
+ "original_filename": "original_453277821.jpg",
+ "referer": "https://www.pornhub.com/photo/453277821",
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277901/453277901.jpg",
+ "filename": "453277901.jpg",
+ "original_filename": "original_453277901.jpg",
+ "referer": "https://www.pornhub.com/photo/453277901",
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277831/453277831.jpg",
+ "filename": "453277831.jpg",
+ "original_filename": "original_453277831.jpg",
+ "referer": "https://www.pornhub.com/photo/453277831",
+ },
+ {
+ "url": "re:/pics/albums/036/252/941/453277851/453277851.jpg",
+ "filename": "453277851.jpg",
+ "original_filename": "original_453277851.jpg",
+ "referer": "https://www.pornhub.com/photo/453277851",
+ },
],
"count": 8,
},
{
"url": "https://www.pornhub.com/view_video.php?viewkey=ph5d530fd885a81",
- "description": "mp4 available",
"results": [
{
"url": "https://www.pornhub.com/embed/ph5d530fd885a81",
"filename": "masturbating with my tail butt plug in - Ally Blake [ph5d530fd885a81][1080p].mp4",
- "original_filename": "ph5d530fd885a81.mp4",
+ "debrid_url": None,
+ "original_filename": "masturbating with my tail butt plug in - Ally Blake",
"referer": "https://www.pornhub.com/view_video.php?viewkey=ph5d530fd885a81",
"album_id": None,
"uploaded_at": 1565724764,
- }
- ],
- },
- {
- "url": "https://www.pornhub.com/embed/6890ddc0b4b11",
- "description": "m3u8 download",
- "results": [
- {
- "url": "https://www.pornhub.com/embed/6890ddc0b4b11",
- "filename": "MILF Celebrates Victory in Public Parking Ramp by Flashing Her Titti [6890ddc0b4b11][1080p].mp4",
- "original_filename": "6890ddc0b4b11.mp4",
- "referer": "https://www.pornhub.com/view_video.php?viewkey=6890ddc0b4b11",
- "album_id": None,
- "uploaded_at": 1754324606,
+ "download_folder": "re:Loose Files (PornHub)",
+ "thumbnail": "NOT_NONE",
}
],
},
diff --git a/tests/test_config.py b/tests/test_config.py
index 073d4fe59..074ee4805 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -391,13 +391,14 @@ def test_config_union() -> None:
def test_additive_args_list() -> None:
- args = Config()._additive_args()
- assert args == (
+ args = set(Config()._additive_args())
+ assert args == {
("crawlers", "bandcamp", "formats"),
("crawlers", "disabled"),
+ ("crawlers", "pornhub", "profile_paths"),
("filters", "only_hosts"),
("filters", "skip_hosts"),
("hashing", "algorithms"),
("jdownloader", "whitelist"),
("restrict_path",),
- )
+ }
diff --git a/uv.lock b/uv.lock
index 16f8ecea1..b2e1a02fa 100644
--- a/uv.lock
+++ b/uv.lock
@@ -683,7 +683,7 @@ wheels = [
[[package]]
name = "cyberdrop-dl-patched"
-version = "10.8.0.dev1"
+version = "10.8.0.dev2"
source = { editable = "." }
dependencies = [
{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },