Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework crawler (Pornhub) (#2113)
NTFSvolume880bbdc405c2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 86b3f4103..affdda4e8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Corrupted/mixed up video segments when downloading multiple concurrent HLS streams and `--subfolders.create` is `False`- `403 Forbidden` on all downloads (AdobeLightroom)+- `400 Bad request` for albums (PornHub)- URL matching (Anysex)## [10.7.0] - 2026-08-29diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.pyindex 5b7167682..61fd5950c 100644--- a/cyberdrop_dl/config/crawlers.py+++ b/cyberdrop_dl/config/crawlers.py@@ -122,6 +122,16 @@ class ClonrConfig(ConfigModel):raise ValueError("'clonr.zip' and 'clonr.use_source' are mutually exclusive")+class PornHubConfig(ConfigModel):+ profile_paths: tuple[NonEmptyStr, ...] = "photos/public", "gifs/public", "videos", "videos/upload"+ "Subpaths to scrape when an input URL is a profile's homepage"++ @override+ def model_post_init(self, context: Any, /) -> None:+ super().model_post_init(context)+ self.profile_paths = tuple(p.lstrip("/") for p in self.profile_paths)++class GenericCrawlers(ConfigModel):wordpress_media: tuple[HttpURL, ...] = ()wordpress_html: tuple[HttpURL, ...] = ()@@ -145,3 +155,4 @@ class Crawlers(ConfigGroup, name=None):only_haven: KemonoConfig = Field(default_factory=KemonoConfig)octave_music: OctaveMusicConfig = Field(default_factory=OctaveMusicConfig)clonr: ClonrConfig = Field(default_factory=ClonrConfig)+ pornhub: PornHubConfig = Field(default_factory=PornHubConfig)diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex 0e3e8cac0..bc79b85a6 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -54,6 +54,12 @@ crawlers:content_urls: trueembed: truefile: true+ pornhub:+ profile_paths:+ - photos/public+ - gifs/public+ - videos+ - videos/uploadtiktok:original: falsetwitter:diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex ab0529bc9..f96ddab46 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -188,6 +188,9 @@},"clonr": {"$ref": "#/$defs/ClonrConfig"+ },+ "pornhub": {+ "$ref": "#/$defs/PornHubConfig"}},"title": "Crawlers",@@ -934,6 +937,26 @@"title": "OnePaceConfig","type": "object"},+ "PornHubConfig": {+ "additionalProperties": false,+ "properties": {+ "profile_paths": {+ "default": [+ "photos/public",+ "gifs/public",+ "videos",+ "videos/upload"+ ],+ "items": {+ "$ref": "#/$defs/NonEmptyStr"+ },+ "title": "Profile Paths",+ "type": "array"+ }+ },+ "title": "PornHubConfig",+ "type": "object"+ },"Sort": {"additionalProperties": false,"properties": {diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex dbecf8942..e534cdff2 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -1070,6 +1070,10 @@ class API(HTTPMixin, ABC):self.parse_url = crawler.parse_urlself.log = crawler.logself.__http_config__ = config # pyright: ignore[reportAttributeAccessIssue]+ try:+ self.__http_ctx__.headers.setdefault("Referer", str(self.PRIMARY_URL))+ except AttributeError:+ passreturn selfdef __post_init__(self) -> None: ...diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex 39ee2c979..8971b5217 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -1,318 +1,404 @@from __future__ import annotationsimport dataclasses-import datetimeimport jsonfrom http import HTTPStatus-from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, TypedDict+from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict, final-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.cache import cached_method+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, extr_textfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Generator, Iterable+from bs4 import BeautifulSoupfrom cyberdrop_dl.url_objects import ScrapeItem-PRIMARY_URL = AbsoluteHttpURL("https://www.pornhub.com")-MP4_NOT_AVAILABLE_SINCE = datetime.datetime(2025, 6, 25, tzinfo=datetime.UTC).timestamp()-TOKEN_SELECTOR = css.CssAttributeSelector("input#xsrfToken", "value")--[email protected](frozen=True, slots=True)-class Profile:- type: str- name: str- download_videos: bool = dataclasses.field(compare=False)- download_gifs: bool = dataclasses.field(compare=False)- download_photos: bool = dataclasses.field(compare=False)-- @property- def url(self) -> AbsoluteHttpURL:- return PRIMARY_URL / self.type / self.name-- @staticmethod- def new(type_: str, name: str, rest: list[str]) -> Profile:- videos = gifs = photos = False- match rest:- case ["videos", *_]:- videos = True- case ["gifs", *_]:- gifs = True- case ["photos", *_]:- photos = True- case []:- videos = True- gifs = photos = "channel" not in type_- case _:- raise ValueError- return Profile(type_, name, videos, gifs, photos)-+@finalclass Selector:- ALBUM_FROM_PHOTO = "div#thumbSlider > h2 > a"- ALBUM_TITLE = "h1[class*=photoAlbumTitle]"- DATE = "script:-soup-contains('uploadDate')"+ FLASHVARS = "script:-soup-contains-own('var flashvars_')"GIF = "div#js-gifToWebm"- JS_VIDEO_INFO = "script:-soup-contains('var flashvars_')"NEXT_PAGE = "li.page_next a"PHOTO = "div#photoImageSection img"- PLAYLIST_TITLE = "h1.playlistTitle"- PLAYLIST_VIDEOS = "ul#videoPlaylist a.linkVideoThumb"- TITLE = "div.title-container > h1.title"- GEO_BLOCKED = ".geoBlocked > h1:-soup-contains('page is not available')"- NO_VIDEO = "section.noVideo"- REMOVED = "div.removed"+ @final+ class Playlist:+ TITLE = "h1.playlistTitle"+ VIDEOS = "ul#videoPlaylist a.linkVideoThumb"- PROFILE_NAME = ".topProfileHeader h1[itemprop=name], div.title h1"- PROFILE_VIDEOS = "div.container a.linkVideoThumb"- PROFILE_GIFS = "#moreData li.gifLi a"- PROFILE_ALBUMS = "#moreData.photosAlbumsListing a"+ @final+ class Album:+ FROM_PHOTO = "div#thumbSlider > h2 > a"+ TITLE = "h1[class*=photoAlbumTitle]"--class Media(TypedDict):- height: int- width: int- format: Literal["hls", "mp4"]- videoUrl: str- quality: str | list[str]---class Format(NamedTuple):- quality: int- format: Literal["hls", "mp4"]- url: str # "videoUrl"-- @staticmethod- def new(media: Media) -> Format:- quality = media["quality"]- if isinstance(quality, str):- try:- quality = int(quality)- except ValueError:- pass- if not isinstance(quality, int):- quality = min(media["height"], media["width"])- values: dict[str, Any] = {k: v for k, v in media.items() if k in Format._fields} | {"quality": quality}- return Format(url=media["videoUrl"], **values)+ @final+ class Profile:+ NAME = ".topProfileHeader h1[itemprop=name], div.title h1"+ VIDEOS = "div.container a.linkVideoThumb"+ GIFS = "#moreData li.gifLi a"+ ALBUMS = "#moreData.photosAlbumsListing a"class PornHubCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Album": "/album/...",- "Channel": "/channel/...",- "Gif": "/gif/...",- "Photo": "/photo/...",- "Playlist": "/playlist/...",+ "Album": "/album/<album_id>",+ "Channel": "/channel/<name>",+ "Gif": "/gif/<gif_id>",+ "Photo": "/photo/<photo_id>",+ "Playlist": "/playlist/<playlist_id>","Profile": (- "/user/...",- "/model/...",- "/pornstar/...",+ "/users/<name>",+ "/model/<name>",+ "/pornstar/<name>",+ ),+ "Profile videos": (+ "/users/<name>/videos",+ "/model/<name>/videos",+ "/pornstar/<name>/videos",+ ),+ "Profile uploaded videos": (+ "/users/<name>/videos/upload",+ "/model/<name>/videos/upload",+ "/pornstar/<name>/videos/upload",+ ),+ "Profile clips": (+ "/users/<name>/clips",+ "/model/<name>/clips",+ "/pornstar/<name>/clips",+ ),+ "Profile albums": (+ "/users/<name>/photos",+ "/model/<name>/photos",+ "/pornstar/<name>/photos",+ ),+ "Profile gifs": (+ "/users/<name>/gifs",+ "/model/<name>/gifs",+ "/pornstar/<name>/gifs",),"Video": ("/embed/<video_id>","/view_video.php?viewkey=<video_id>",),}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.pornhub.com")NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGEDOMAIN: ClassVar[str] = "pornhub"FOLDER_DOMAIN: ClassVar[str] = "PornHub"def __post_init__(self) -> None:- self.seen_profiles: set[Profile] = set()-- async def __async_post_init__(self) -> None:- keys = ("age_verified", "accessPH", "accessAgeDisclaimerPH", "accessAgeDisclaimerUK", "expiredEnterModalShown")- self.update_cookies(dict.fromkeys(keys, 1))+ self.api: PornHubAPI = PornHubAPI.from_crawler(self)+ self.update_cookies(+ dict.fromkeys(+ (+ "age_verified",+ "accessPH",+ "accessAgeDisclaimerPH",+ "accessAgeDisclaimerUK",+ "expiredEnterModalShown",+ ),+ 2,+ )+ )- async def fetch(self, scrape_item: ScrapeItem) -> None: # noqa: PLR0911+ async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:- case ["user" | "channel" | "channels" | "model" | "pornstar" as type_, name, *rest]:- profile = Profile.new(type_, name, rest)- if profile in self.seen_profiles:- return None- self.seen_profiles.add(profile)- return await self.profile(scrape_item, profile)+ case [+ "user" | "users" | "channel" | "channels" | "model" | "models" | "pornstar" | "pornstars" as type_,+ name,+ *_,+ ]:+ await PornHubProfileCrawler(self, Profile(type_, name), scrape_item).fetch()case ["album", album_id]:- return await self.album(scrape_item, album_id)+ await self.album(scrape_item, album_id)case ["playlist", playlist_id]:- return await self.playlist(scrape_item, playlist_id)- case ["photo", _]:- return await self.photo(scrape_item)- case ["gif", _]:- return await self.gif(scrape_item)+ await self.playlist(scrape_item, playlist_id)+ case ["photo", photo_id]:+ await self.photo(scrape_item, photo_id)+ case ["gif", gif_id]:+ await self.gif(scrape_item, gif_id)case ["embed", video_id]:- return await self.video(scrape_item, video_id)+ await self.video(scrape_item, video_id)case ["view_video.php"] if video_id := scrape_item.url.query.get("viewkey"):- return await self.video(scrape_item, video_id)+ await self.video(scrape_item, video_id)case _:raise ValueError- @error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem, profile: Profile) -> None:- title = await self._get_profile_title(profile.url)- title = self.create_title(f"{title} [{profile.type.removesuffix('s')}]")- scrape_item.setup_as_profile(title)-- if profile.download_videos:- await self.iter_profile_pages(scrape_item, profile.url / "videos", Selector.PROFILE_VIDEOS)- if profile.download_gifs:- await self.iter_profile_pages(scrape_item, profile.url / "gifs/public", Selector.PROFILE_GIFS)- if profile.download_photos:- await self.iter_profile_pages(scrape_item, profile.url / "photos/public", Selector.PROFILE_ALBUMS)-- async def _get_profile_title(self, url: AbsoluteHttpURL) -> str:- soup = await self.request_soup(url)- return css.select_text(soup, Selector.PROFILE_NAME, decompose="span")-- @error_handling_wrapper- async def iter_profile_pages(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str) -> None:- async for soup in self.web_pager(url):- for new_scrape_item in self.iter_children(scrape_item, soup, selector):- self.create_task(self.run(new_scrape_item))-@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str) -> None:- soup = await self.request_soup(scrape_item.url)- album_name = css.select_text(soup, Selector.ALBUM_TITLE)- scrape_item.setup_as_album(self.create_title(album_name, album_id), album_id=album_id)-- api_url = self.PRIMARY_URL / "api/v1/album" / album_id / "show_album_json"- json_resp: dict[str, Any] = await self.request_json(api_url.with_query(token=TOKEN_SELECTOR(soup)))- photos: dict[str, dict[str, Any]] = json_resp["photos"]- results = await self.get_album_results(album_id)- for id_, photo in photos.items():- web_url = PRIMARY_URL / "photo" / id_- link = self.parse_url(photo["img_large"])- new_scrape_item = scrape_item.create_new(web_url)- self.create_task(self._process_photo(new_scrape_item, link, results))+ album = await self.api.album(album_id)+ scrape_item.setup_as_album(self.create_title(album.name, album.id), album_id=album.id)+ downloaded = await self.get_album_results(album.id)++ async for photo in self.api.album_photos(album.id):+ if self.check_album_results(photo.canonical_url, downloaded):+ continue+ web_url = self.PRIMARY_URL / "photo" / photo.id+ new_item = scrape_item.create_child(web_url)+ self.create_eager_task(self._photo(new_item, photo))scrape_item.add_children()@error_handling_wrapper- async def photo(self, scrape_item: ScrapeItem) -> None:+ async def photo(self, scrape_item: ScrapeItem, photo_id: str) -> None:if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)- link_str: str = css.select(soup, Selector.PHOTO, "src")- link = self.parse_url(link_str)- album_tag = css.select(soup, Selector.ALBUM_FROM_PHOTO)- album_name = css.text(album_tag)- album_link_str: str = css.attr(album_tag, "href")- album_id: str = album_link_str.rsplit("/", maxsplit=1)[-1]- title = self.create_title(album_name, album_id)- scrape_item.setup_as_album(title, album_id=album_id)- await self._process_photo(scrape_item, link)+ src = css.select(soup, Selector.PHOTO, "src")+ album = _extr_album(soup)+ scrape_item.setup_as_album(self.create_title(album.name, album.id), album_id=album.id)+ await self._photo(scrape_item, Photo(photo_id, self.parse_url(src)))@error_handling_wrapper- async def gif(self, scrape_item: ScrapeItem) -> None:+ async def gif(self, scrape_item: ScrapeItem, gif_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return+soup = await self.request_soup(scrape_item.url)- attributes = "data-mp4", "data-fallback", "data-webm"- gif_tag = css.select(soup, Selector.GIF)- link_str = next(value for attr in attributes if (value := css.attr_or_none(gif_tag, attr)))- link = self.parse_url(link_str)- await self._process_photo(scrape_item, link)+ gif = css.select(soup, Selector.GIF)+ attrs = ("data-mp4", "data-fallback", "data-webm")+ src = next(value for attr in attrs if (value := css.attr_or_none(gif, attr)))+ scrape_item.uploaded_at = self.parse_iso_date(_extr_upload_date(soup))+ await self._photo(scrape_item, Photo(gif_id, self.parse_url(src)))@error_handling_wrapper- async def _process_photo(- self, scrape_item: ScrapeItem, link: AbsoluteHttpURL, results: dict[str, Any] | None = None- ) -> None:- results = results or {}- original_name = link.name.rsplit(")")[-1]- name = original_name.removeprefix("original_")- canonical_url = link.with_name(name)- if self.check_album_results(canonical_url, results):- return- custom_filename, ext = self.get_filename_and_ext(name, assume_ext=".jpg")+ async def _photo(self, scrape_item: ScrapeItem, photo: Photo) -> None:+ filename, ext = self.get_filename_and_ext(photo.name, assume_ext=".jpg")await self.handle_file(- canonical_url, scrape_item, original_name, ext, custom_filename=custom_filename, debrid_link=link+ photo.canonical_url,+ scrape_item,+ photo.original_name,+ ext,+ custom_filename=filename,+ debrid_link=photo.src,)@error_handling_wrapperasync def playlist(self, scrape_item: ScrapeItem, playlist_id: str) -> None:soup = await self.request_soup(scrape_item.url)- title: str = css.select_text(soup, Selector.PLAYLIST_TITLE)+ title: str = css.select_text(soup, Selector.Playlist.TITLE)title = self.create_title(title, playlist_id)scrape_item.setup_as_album(f"{title} [playlist]", album_id=playlist_id)- for new_scrape_item in self.iter_children(scrape_item, soup, Selector.PLAYLIST_VIDEOS):+ for new_scrape_item in self.iter_children(scrape_item, soup, Selector.Playlist.VIDEOS):self.create_task(self.run(new_scrape_item, check_referer=True))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- embed_url = PRIMARY_URL / "embed" / video_id- page_url = PRIMARY_URL.joinpath("view_video.php").with_query(viewkey=video_id)+ embed_url = self.PRIMARY_URL / "embed" / video_id- if await self.check_complete_from_referer(page_url):+ if await self.check_complete(embed_url):return- soup = await self.request_soup(page_url)- _check_video_is_available(soup)- title = css.select_text(soup, Selector.TITLE)- formats = [Format.new(media) for media in get_media_list(soup)]- best_hls = max(f for f in formats if f.format == "hls")- debrid_link = m3u8 = best_format = None- scrape_item.uploaded_at = date = self.parse_iso_date(get_upload_date_str(soup))- assert date- use_hls = date >= MP4_NOT_AVAILABLE_SINCE-- if not use_hls:- best_format = await self.get_best_mp4_format(formats)- if best_format is None:- self.log.warning(f"Video {video_id} has no mp4 formats available. Falling back to HLS")-- else:- debrid_link = self.parse_url(best_format.url)-- if use_hls or best_format is None:- m3u8, _ = await self.request_m3u8_playlist(self.parse_url(best_hls.url), headers={"Referer": str(page_url)})- best_format = best_hls-- scrape_item.url = page_url- filename, ext = self.get_filename_and_ext(f"{video_id}.mp4")- custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.quality)+ video = await self.api.video(video_id)+ scrape_item.uploaded_at = self.parse_iso_date(video.uploaded)+ src = max(f for f in video.formats if f.format == "hls")+ m3u8, _ = await self.request_m3u8_playlist(self.parse_url(src.url), headers={"Referer": str(video.url)})++ scrape_item.url = video.url+ filename = self.create_custom_filename(video.title, ext := ".mp4", file_id=video_id, resolution=src.resolution)await self.handle_file(embed_url,scrape_item,- filename,+ video.title,ext,- custom_filename=custom_filename,- debrid_link=debrid_link,+ custom_filename=filename,m3u8=m3u8,+ thumbnail=video.thumb,+ )++[email protected](frozen=True, slots=True)+class PornHubProfileCrawler:+ crawler: PornHubCrawler+ profile: Profile+ item: ScrapeItem++ async def fetch(self) -> None:+ with self.crawler.catch_errors(self.item):+ match self.item.url.parts[3:]:+ case ["videos" | "clips", *_]:+ await self.pages(Selector.Profile.VIDEOS)+ case ["gifs", *_]:+ await self.pages(Selector.Profile.GIFS)+ case ["photos", *_]:+ await self.pages(Selector.Profile.ALBUMS)+ case []:+ await self.dispatch()+ case _:+ raise ScrapeError.unsupported()++ async def dispatch(self) -> None:+ await self._init()+ for path in self.crawler.config.crawlers.pornhub.profile_paths:+ new_item = self.item.create_child(self.profile.url / path)+ self.crawler.create_task(self.crawler.run(new_item))+ self.item.add_children()++ async def pages(self, selector: str) -> None:+ await self._init()+ self.item.append_folders(*filter(None, self.item.url.parts[3:]))+ await self._iter_pages(selector)++ async def _init(self) -> None:+ if self.profile in self.item.markers:+ return++ soup = await self.crawler.request_soup(self.profile.url)+ name = css.select_text(soup, Selector.Profile.NAME, decompose="span")+ title = self.crawler.create_title(f"{name} [{self.profile.type.removesuffix('s')}]")+ self.item.setup_as_profile(title)+ self.item.markers.append(self.profile)++ async def _iter_pages(self, selector: str) -> None:+ async for soup in self.crawler.web_pager(self.item.url):+ for new_item in self.crawler.iter_children(self.item, soup, selector):+ self.crawler.create_task(self.crawler.run(new_item))++[email protected](frozen=True, slots=True)+class Profile:+ type: str+ name: str++ @property+ def url(self) -> AbsoluteHttpURL:+ return PornHubCrawler.PRIMARY_URL / self.type / self.name++[email protected](slots=True, order=True)+class Photo:+ id: str+ src: AbsoluteHttpURL++ name: str = dataclasses.field(init=False)+ original_name: str = dataclasses.field(init=False)+ canonical_url: AbsoluteHttpURL = dataclasses.field(init=False)++ def __post_init__(self) -> None:+ self.original_name = self.src.name.rpartition(")")[-1]+ self.name = self.original_name.removeprefix("original_")+ self.canonical_url = self.src.with_name(self.name)++[email protected](order=True, slots=True, frozen=True)+class Album:+ id: str+ name: str+++class Media(TypedDict):+ height: int+ width: int+ format: Literal["hls", "mp4"]+ videoUrl: str+ quality: str | int | list[str]++[email protected](slots=True, order=True, frozen=True)+class Format:+ resolution: Resolution+ format: Literal["hls", "mp4"]+ url: str++[email protected](slots=True, frozen=True, order=True)+class Video:+ id: str+ title: str+ thumb: str | None+ uploaded: str+ formats: tuple[Format, ...]+ url: AbsoluteHttpURL+++class PornHubAPI(API):+ @cached_method(ttl=600)+ async def csrf_token(self) -> str:+ soup = await self.request_soup(self.PRIMARY_URL)+ return css.select(soup, "input[data-token]", "data-token")++ async def album(self, album_id: str) -> Album:+ url = self.PRIMARY_URL / "album" / album_id+ soup = await self.request_soup(url, impersonate="firefox")+ return Album(album_id, name=css.select_text(soup, Selector.Album.TITLE))++ async def album_photos(self, album_id: str) -> AsyncGenerator[Photo]:+ url = self.PRIMARY_URL / "api/v1/album" / album_id / "show_album_json"+ resp: dict[str, Any] = await self.request_json(url.with_query(token=await self.csrf_token()))+ photos: dict[str, dict[str, Any]] = resp["photos"]+ for photo_id, photo in photos.items():+ yield Photo(photo_id, self.parse_url(photo["img_large"]))++ async def video(self, video_id: str) -> Video:+ page_url = self.PRIMARY_URL.joinpath("view_video.php").with_query(viewkey=video_id)+ soup = await self.request_soup(page_url)+ _check_video_is_available(soup)+ flashvars = _extr_flashvars(soup)+ if flashvars.get("video_unavailable_country", "false") != "false":+ raise ScrapeError(HTTPStatus.FORBIDDEN, "Video is geo restricted")++ return Video(+ id=video_id,+ title=flashvars["video_title"],+ thumb=flashvars.get("image_url"),+ formats=tuple(_parse_formats(flashvars["mediaDefinitions"])),+ uploaded=_extr_upload_date(soup),+ url=page_url,)- async def get_best_mp4_format(self, formats: list[Format]) -> Format | None:- mp4_format = next((f for f in formats if f.format == "mp4"), None)- if not mp4_format:- raise ScrapeError(422, message="Unable to get mp4 format")- mp4_media_url = self.parse_url(mp4_format.url)- # This returns an empty list when downloading multiple videos concurrently- mp4_media: list[Media] = await self.request_json(mp4_media_url)- return max((Format.new(media) for media in mp4_media), default=None)+def _extr_album(soup: BeautifulSoup) -> Album:+ album = css.select(soup, Selector.Album.FROM_PHOTO)+ url: str = css.attr(album, "href")+ return Album(id=url.rpartition("/")[-1], name=css.text(album))-def get_upload_date_str(soup: BeautifulSoup) -> str:- date_text = css.select_text(soup, Selector.DATE)- return extr_text(date_text, 'uploadDate": "', '",')+def _extr_upload_date(soup: BeautifulSoup) -> str:+ return css.json_ld(soup, "uploadDate")["uploadDate"]-def get_media_list(soup: BeautifulSoup) -> list[Media]:- flashvars: str = css.select(soup, Selector.JS_VIDEO_INFO).text- media_text = extr_text(flashvars, '"mediaDefinitions":', '"isVertical"').strip().removesuffix(",")- return json.loads(media_text)+def _extr_flashvars(soup: BeautifulSoup) -> dict[str, Any]:+ flashvars: str = css.select_text(soup, Selector.FLASHVARS)+ payload = extr_text(flashvars, "{", "};").strip()+ return json.loads("{" + payload + "}")+++def _parse_formats(medias: Iterable[Media]) -> Generator[Format]:+ for media in medias:+ quality = media["quality"]+ res = None+ if not isinstance(quality, list):+ try:+ quality = int(quality)+ except ValueError:+ pass++ try:+ res = Resolution.parse(quality)+ except ValueError:+ pass++ res = res or Resolution(media["height"], media["width"])++ yield Format(url=media["videoUrl"], format=media["format"], resolution=res)def _check_video_is_available(soup: BeautifulSoup) -> None:- if soup.select_one(Selector.NO_VIDEO):+ if soup.select_one("section.noVideo"):raise ScrapeError(HTTPStatus.NOT_FOUND)page_text = soup.text- if soup.select_one(Selector.GEO_BLOCKED) or "This content is unavailable in your country" in page_text:+ if (+ soup.select_one(".geoBlocked > h1:-soup-contains('page is not available')")+ or "This content is unavailable in your country" in page_text+ ):raise ScrapeError(HTTPStatus.FORBIDDEN, "Video is geo restricted")if (@@ -322,7 +408,7 @@ def _check_video_is_available(soup: BeautifulSoup) -> None:raise ScrapeError(HTTPStatus.UNAVAILABLE_FOR_LEGAL_REASONS)if (- soup.select_one(Selector.REMOVED)+ soup.select_one("div.removed")or "This video has been removed" in page_textor "This video is currently unavailable" in page_text):diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.pyindex 118bd2e83..c48c1501a 100644--- a/cyberdrop_dl/url_objects.py+++ b/cyberdrop_dl/url_objects.py@@ -233,6 +233,7 @@ class ScrapeItem:_type: ScrapeItemType | None = None_uploaded_at: int | None = Noneretry_info: RetryInfo | None = None+ markers: list[Any] = dataclasses.field(default_factory=list)__repr__ = signature.simple_repr("url", "folders", "uploaded_at")@@ -389,7 +390,9 @@ class ScrapeItem:def copy(self) -> Self:"""Returns a deep copy of this scrape_item"""- return copy.deepcopy(self)+ me = copy.deepcopy(self)+ me.markers = self.markers.copy()+ return medef get_referer(self) -> AbsoluteHttpURL | None:if self.referer:diff --git a/docs/reference/config/crawlers.md b/docs/reference/config/crawlers.mdindex 90c68b8f2..03db14010 100644--- a/docs/reference/config/crawlers.md+++ b/docs/reference/config/crawlers.md@@ -171,6 +171,26 @@ crawlers:file: true```+# PornHub++```yaml+crawlers:+ pornhub:+ profile_paths:+ - "photos/public"+ - "gifs/public"+ - "videos"+ - "videos/upload"+```++## `profile_paths`++| Type | Default |+| --------------------- | ----------------------------------------------------------- |+| list of `NonEmptyStr` | [`photos/public`, `gifs/public`, `videos`, `videos/upload`] |++Subpaths to scrape when an input URL is a profile's homepage. ex: `https://www.pornhub.com/model/<model_name>`+# Tiktok```yamldiff --git a/pyproject.toml b/pyproject.tomlindex ea3770fbc..64d5bb020 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "cyberdrop-dl-patched"-version = "10.8.0dev1"+version = "10.8.0dev2"authors = [{name = "Jacob B", email = "[email protected]"}]classifiers = ["Development Status :: 5 - Production/Stable",diff --git a/tests/crawlers/test_cases/pornhub.py b/tests/crawlers/test_cases/pornhub.pyindex d249b49c9..16bc135f5 100644--- a/tests/crawlers/test_cases/pornhub.py+++ b/tests/crawlers/test_cases/pornhub.py@@ -1,45 +1,109 @@DOMAIN = "pornhub"TEST_CASES = [+ {+ "url": "https://www.pornhub.com/gif/54960161",+ "results": [+ {+ "url": "re:/pics/gifs/054/960/161/54960161a.mp4",+ "filename": "54960161a.mp4",+ "debrid_url": "re:/pics/gifs/054/960/161/54960161a.mp4?validfrom=",+ "original_filename": "54960161a.mp4",+ "referer": "https://www.pornhub.com/gif/54960161",+ "album_id": None,+ "uploaded_at": 1786683600,+ "download_folder": "re:Loose Files (PornHub)",+ "thumbnail": None,+ }+ ],+ },+ {+ "url": "https://www.pornhub.com/view_video.php?viewkey=6890ddc0b4b11",+ "results": [+ {+ "url": "https://www.pornhub.com/embed/6890ddc0b4b11",+ "filename": "MILF Celebrates Victory in Public Parking Ramp by Flashing Her Titti [6890ddc0b4b11][1080p].mp4",+ "debrid_url": None,+ "original_filename": "MILF Celebrates Victory in Public Parking Ramp by Flashing Her Titties! Onlyfans Snookybaby28",+ "referer": "https://www.pornhub.com/view_video.php?viewkey=6890ddc0b4b11",+ "album_id": None,+ "uploaded_at": 1754324606,+ "download_folder": "re:Loose Files (PornHub)",+ "thumbnail": "re:/c6371/videos/202508/04/18163005/original_18163005.mp4",+ }+ ],+ },{"url": "https://www.pornhub.com/album/36252941","results": [{- "url": "re:ei.phncdn.com/pics/albums/036/252/941/453277801",+ "url": "re:/pics/albums/036/252/941/453277841/453277841.jpg",+ "filename": "453277841.jpg",+ "debrid_url": "re:/pics/albums/036/252/941/453277841/",+ "original_filename": "original_453277841.jpg",+ "referer": "https://www.pornhub.com/photo/453277841",+ "album_id": "36252941",+ "uploaded_at": None,+ "download_folder": "re:White Top White Striped Calvins (PornHub)",+ "thumbnail": None,+ },+ {+ "url": "re:/pics/albums/036/252/941/453277871/453277871.jpg",+ "filename": "453277871.jpg",+ "original_filename": "original_453277871.jpg",+ "referer": "https://www.pornhub.com/photo/453277871",+ },+ {+ "url": "re:/pics/albums/036/252/941/453277801/453277801.jpg","filename": "453277801.jpg","original_filename": "original_453277801.jpg","referer": "https://www.pornhub.com/photo/453277801",- "album_id": "36252941",- "download_folder": "re:White Top White Striped Calvins \\(PornHub\\)",- "uploaded_at": None,- }+ },+ {+ "url": "re:/pics/albums/036/252/941/453277861/453277861.jpg",+ "filename": "453277861.jpg",+ "original_filename": "original_453277861.jpg",+ "referer": "https://www.pornhub.com/photo/453277861",+ },+ {+ "url": "re:/pics/albums/036/252/941/453277821/453277821.jpg",+ "filename": "453277821.jpg",+ "original_filename": "original_453277821.jpg",+ "referer": "https://www.pornhub.com/photo/453277821",+ },+ {+ "url": "re:/pics/albums/036/252/941/453277901/453277901.jpg",+ "filename": "453277901.jpg",+ "original_filename": "original_453277901.jpg",+ "referer": "https://www.pornhub.com/photo/453277901",+ },+ {+ "url": "re:/pics/albums/036/252/941/453277831/453277831.jpg",+ "filename": "453277831.jpg",+ "original_filename": "original_453277831.jpg",+ "referer": "https://www.pornhub.com/photo/453277831",+ },+ {+ "url": "re:/pics/albums/036/252/941/453277851/453277851.jpg",+ "filename": "453277851.jpg",+ "original_filename": "original_453277851.jpg",+ "referer": "https://www.pornhub.com/photo/453277851",+ },],"count": 8,},{"url": "https://www.pornhub.com/view_video.php?viewkey=ph5d530fd885a81",- "description": "mp4 available","results": [{"url": "https://www.pornhub.com/embed/ph5d530fd885a81","filename": "masturbating with my tail butt plug in - Ally Blake [ph5d530fd885a81][1080p].mp4",- "original_filename": "ph5d530fd885a81.mp4",+ "debrid_url": None,+ "original_filename": "masturbating with my tail butt plug in - Ally Blake","referer": "https://www.pornhub.com/view_video.php?viewkey=ph5d530fd885a81","album_id": None,"uploaded_at": 1565724764,- }- ],- },- {- "url": "https://www.pornhub.com/embed/6890ddc0b4b11",- "description": "m3u8 download",- "results": [- {- "url": "https://www.pornhub.com/embed/6890ddc0b4b11",- "filename": "MILF Celebrates Victory in Public Parking Ramp by Flashing Her Titti [6890ddc0b4b11][1080p].mp4",- "original_filename": "6890ddc0b4b11.mp4",- "referer": "https://www.pornhub.com/view_video.php?viewkey=6890ddc0b4b11",- "album_id": None,- "uploaded_at": 1754324606,+ "download_folder": "re:Loose Files (PornHub)",+ "thumbnail": "NOT_NONE",}],},diff --git a/tests/test_config.py b/tests/test_config.pyindex 073d4fe59..074ee4805 100644--- a/tests/test_config.py+++ b/tests/test_config.py@@ -391,13 +391,14 @@ def test_config_union() -> None:def test_additive_args_list() -> None:- args = Config()._additive_args()- assert args == (+ args = set(Config()._additive_args())+ assert args == {("crawlers", "bandcamp", "formats"),("crawlers", "disabled"),+ ("crawlers", "pornhub", "profile_paths"),("filters", "only_hosts"),("filters", "skip_hosts"),("hashing", "algorithms"),("jdownloader", "whitelist"),("restrict_path",),- )+ }diff --git a/uv.lock b/uv.lockindex 16f8ecea1..b2e1a02fa 100644--- a/uv.lock+++ b/uv.lock@@ -683,7 +683,7 @@ wheels = [[[package]]name = "cyberdrop-dl-patched"-version = "10.8.0.dev1"+version = "10.8.0.dev2"source = { editable = "." }dependencies = [{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },