Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: handle videos without http sources (xhamster) (#1225)
NTFSvolume35daff7dc2f3
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 133a79777..d38c48b94 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -1,183 +1,307 @@from __future__ import annotations+import dataclasses+import itertoolsimport jsonfrom functools import partial-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, NamedTuple+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple-from pydantic import AliasPath, Field, PlainValidator+from aiolimiter import AsyncLimiterfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.models import AliasModel-from cyberdrop_dl.utils.logger import log_debug+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_urlif TYPE_CHECKING:- from collections.abc import Generator-- from bs4 import BeautifulSoup+ from collections.abc import Iterablefrom cyberdrop_dl.data_structures.url_objects import ScrapeItem-PRIMARY_URL = AbsoluteHttpURL("https://xhamster.com/")+_PRIMARY_URL = AbsoluteHttpURL("https://xhamster.com/")+_ALLOW_AV1 = False+_ALLOW_HLS = False-class Selectors:- JS_VIDEO_INFO = "script#initials-script"+class Selector:+ JS_WINDOW_INITIALS = "script#initials-script"VIDEO = "a.video-thumb__image-container"GALLERY = "a.gallery-thumb__link"NEXT_PAGE = "a[data-page='next']"-_SELECTORS = Selectors()---HttpURL = Annotated[AbsoluteHttpURL, PlainValidator(partial(parse_url, relative_to=PRIMARY_URL))]+_parse_url = partial(parse_url, relative_to=_PRIMARY_URL)class XhamsterCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Video": "/videos/<video_title>",- "User": "/users/<user_name>",- "Creator": "/creatos/<creator_name>",- "Gallery": "photos/gallery/<gallery_name>",+ "Video": "/videos/<title>",+ "User": (+ "/users/<user_name>",+ "/users/profiles/<user_name>",+ ),+ "User Videos": "/users/<user_name>/videos",+ "User Galleries": "/users/<user_name>/photos",+ "Creator": "/creators/<creator_name>",+ "Creator Videos": "/creators/<creator_name>/exclusive",+ "Creator Galleries": "/creators/<creator_name>/photos",+ "Gallery": "/photos/gallery/<gallery_name_or_id>",}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE- DOMAIN: ClassVar[str] = "xhamster"- FOLDER_DOMAIN: ClassVar[str] = "xHamster"+ PRIMARY_URL = _PRIMARY_URL+ NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE+ DOMAIN = "xhamster"+ FOLDER_DOMAIN = "xHamster"++ def __post_init__(self) -> None:+ self.request_limiter = AsyncLimiter(4, 1)+ self._seen_hosts: set[str] = set()++ def _disable_title_auto_translations(self, url: AbsoluteHttpURL) -> None:+ if url.host not in self._seen_hosts:+ self.update_cookies({"lang": "en", "video_titles_translation": "0"}, url.origin())+ self._seen_hosts.add(url.host)async def fetch(self, scrape_item: ScrapeItem) -> None:- if "gallery" in scrape_item.url.parts:- return await self.gallery(scrape_item)- if any(p in scrape_item.url.parts for p in ("creators", "users")):- return await self.profile(scrape_item)- if any(p in scrape_item.url.parts for p in ("movies", "videos")):- return await self.video(scrape_item)- raise ValueError+ match scrape_item.url.parts[1:]:+ case ["photos", "gallery", _]:+ return await self.gallery(scrape_item)+ case ["videos", _]:+ return await self.video(scrape_item)+ case ["users" | "creators" as type_, _, *rest]:+ match rest:+ case []:+ return await self.profile(scrape_item)+ case ["photos"]:+ return await self.profile(scrape_item, only_photos=True)+ case ["videos"] if type_ == "users":+ return await self.profile(scrape_item, only_videos=True)+ case ["exclusive"] if type_ == "creators":+ return await self.profile(scrape_item, only_videos=True)+ case _:+ raise ValueError+ case _:+ raise ValueError++ @classmethod+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url)+ match url.parts[1:]:+ case ["photos", "gallery", name, *rest] if rest:+ return url.origin() / "photos/gallery" / name+ case ["users", "profiles", name]:+ return url.origin() / "users" / name+ case _:+ return url@error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem) -> None:- profile_type, username = scrape_item.url.parts[1:3]- title = self.create_title(f"{username} [{profile_type.removesuffix('s')}]")+ async def profile(self, scrape_item: ScrapeItem, only_photos: bool = False, only_videos: bool = False) -> None:+ url_type, username = scrape_item.url.parts[1:3]+ base_url = scrape_item.url.origin() / url_type / username+ initials = await self._get_window_initials(base_url)+ is_creator_url = url_type == "creators"+ info = initials["infoComponent"]["displayUserModel"] if is_creator_url else initials["displayUserModel"]++ # every creator is actually an user+ # 'pageTitle' has the creator's name (for a creator's URL)+ # 'name' and 'displayName' are the actual user's name+ # they are different but we will use the user name in both cases+ name: str = info.get("displayName") or info["name"]+ title = self.create_title(f"{name} [user]")scrape_item.setup_as_profile(title)- is_user = "users" in scrape_item.url.parts- last_part = "videos" if is_user else "exclusive"- base_url = PRIMARY_URL / profile_type / username- all_paths = ("videos", "photos")- paths_to_scrape = next(((p,) for p in all_paths if p in scrape_item.url.parts), all_paths)+ if is_creator_url:+ user_url = self.parse_url(info["pageURL"])+ has_videos = bool(initials["infoComponent"].get("pornstarTop", {}).get("videoCount", 0))+ has_galleries = bool(initials.get("galleriesComponent", {}).get("galleriesTotal", 0))++ else:+ user_url = base_url+ has_videos = bool(initials["counters"]["videos"])+ has_galleries = bool(initials["counters"]["galleries"])- if "videos" in paths_to_scrape:- videos_url = base_url / last_part- await self.process_children(scrape_item, videos_url, _SELECTORS.VIDEO, last_part)+ if has_videos and not only_photos:+ videos_url = user_url / "videos"+ await self._iter_profile_pages(scrape_item, videos_url, Selector.VIDEO, "videos")- if is_user and "photos" in paths_to_scrape:- gallerys_url = base_url / "photos"- await self.process_children(scrape_item, gallerys_url, _SELECTORS.GALLERY, "galleries")+ if has_galleries and not only_videos:+ gallerys_url = user_url / "photos"+ await self._iter_profile_pages(scrape_item, gallerys_url, Selector.GALLERY, "galleries")@error_handling_wrapper- async def process_children(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str, name: str) -> None:+ async def _iter_profile_pages(+ self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str, name: str+ ) -> None:async for soup in self.web_pager(url):for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem) -> None:- json_info = await self.get_model_details(scrape_item.url, "photosGalleryModel", "galleryModel")- gallery = Gallery(**json_info)- title = f"{gallery.title} [gallery]"- scrape_item.setup_as_album(title, album_id=gallery.id)- scrape_item.possible_datetime = gallery.created-- padding = max(3, len(str(gallery.quantity)))- for index, image in enumerate(gallery.photos, 1):- filename, ext = self.get_filename_and_ext(image.url.name)- # TODO: Adding an index prefix should be handled by `create_custom_filename`- custom_filename = f"{str(index).zfill(padding)} - {filename.removesuffix(ext)}"- custom_filename = self.create_custom_filename(custom_filename, ext, file_id=image.id)- new_scrape_item = scrape_item.create_child(image.page_url)- await self.handle_file(image.url, new_scrape_item, filename, ext, custom_filename=custom_filename)- scrape_item.add_children()+ initials = await self._get_window_initials(scrape_item.url)+ page_details: dict[str, Any] = initials["galleryPage"]+ gallery: dict[str, Any] = page_details["galleryModel"]+ gallery_id = str(gallery["id"])+ title = self.create_title(f"{gallery['title']} [gallery]", gallery_id)+ scrape_item.setup_as_album(title, album_id=gallery_id)+ scrape_item.possible_datetime = gallery["created"]++ results = await self.get_album_results(gallery_id)+ n_pages: int = page_details["paginationProps"]["lastPageNumber"]+ index = 0+ images: list[dict[str, Any]] = initials["photosGalleryModel"]["photos"]++ for next_page in itertools.count(2):+ for img in images:+ img["index"] = index = index + 1+ self._handle_img(scrape_item, img, results)++ if next_page > n_pages:+ break++ next_page_url = scrape_item.url / str(next_page)+ initials = await self._get_window_initials(next_page_url)+ images = initials["photosGalleryModel"]["photos"]++ def _handle_img(self, scrape_item: ScrapeItem, img: dict[str, Any], results: dict[str, int]):+ src, page_url = self.parse_url(img["imageURL"]), self.parse_url(img["pageURL"])+ if self.check_album_results(src, results):+ return++ _, ext = self.get_filename_and_ext(src.name)+ stem = f"{str(img['index']).zfill(3)} - {src.name.removesuffix(ext)}"+ filename = self.create_custom_filename(stem, ext, file_id=img["id"])+ new_scrape_item = scrape_item.create_child(page_url)+ self.create_task(self.handle_file(src, new_scrape_item, src.name, ext, custom_filename=filename))+ scrape_item.add_children()@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- json_info = await self.get_model_details(scrape_item.url, "videoModel")- video = Video(**json_info)-- scrape_item.url = video.page_url+ initials = await self._get_window_initials(scrape_item.url)+ video = _parse_video(initials)scrape_item.possible_datetime = video.created- _, resolution, download_url = max(video.get_formats())- link = PRIMARY_URL / "movies" / video.id / "download" / resolution- filename, ext = self.get_filename_and_ext(f"{video.id}.mp4")- custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=resolution)+ custom_filename = self.create_custom_filename(+ video.title,+ ".mp4",+ file_id=video.id,+ video_codec=video.best_mp4.codec,+ resolution=video.best_mp4.resolution,+ )await self.handle_file(- link, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=download_url+ video.url,+ scrape_item,+ filename=video.id + ".mp4",+ custom_filename=custom_filename,+ debrid_link=video.best_mp4.url,)- async def get_model_details(self, url: AbsoluteHttpURL, *model_name_choices: str) -> dict:- model_names = model_name_choices or []-+ async def _get_window_initials(self, url: AbsoluteHttpURL) -> dict[str, Any]:+ self._disable_title_auto_translations(url)async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, url)+ soup = await self.client.get_soup(self.DOMAIN, url)++ js_script = css.select_one(soup, Selector.JS_WINDOW_INITIALS)+ json_text = get_text_between(str(js_script), "window.initials=", ";</script>")+ return json.loads(json_text)- json_info = get_window_initials_json(soup)- del soup- def get_model_value() -> dict[str, Any] | None:- for model_name in model_names:- if (value := json_info.get(model_name)) is not None:- return value[email protected](frozen=True, slots=True)+class Video:+ id: str+ title: str+ created: int+ url: AbsoluteHttpURL+ best_hls: Format | None+ best_mp4: Format+- json_info = get_model_value() or json_info- log_debug(json_info)- return json_info+def _parse_video(initials: dict[str, Any]):+ video: dict[str, Any] = initials["videoModel"]+ hls_sources: list[Format] = []+ mp4_sources: list[Format] = []-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ sources = itertools.chain(_parse_http_sources(initials), _parse_xplayer_sources(initials))+ for src in sources:+ if src.codec == "av1" and not _ALLOW_AV1:+ continue+ if src.url.suffix == ".m3u8":+ if not _ALLOW_HLS:+ continue+ hls_sources.append(src)+ else:+ mp4_sources.append(src)-def get_window_initials_json(soup: BeautifulSoup) -> dict[str, dict]:- js_script = soup.select_one(_SELECTORS.JS_VIDEO_INFO)- js_code: str = str(js_script)- json_text: str = get_text_between(js_code, "window.initials=", "</script>").removesuffix(";").strip()- return json.loads(json_text)+ return Video(+ id=video["idHashSlug"],+ title=video["title"],+ created=video["created"],+ url=_parse_url(video["pageURL"]),+ best_hls=max(hls_sources, default=None),+ best_mp4=max(mp4_sources),+ )class Format(NamedTuple):- height: int- resolution: str+ resolution: int+ codec: str # h264 > av1url: AbsoluteHttpURL-class XHamsterItem(AliasModel):- id: str = Field(alias="idHashSlug")- page_url: HttpURL = Field(alias="pageURL")- created: int = 0+def _parse_http_sources(initials: dict[str, Any]) -> Iterable[Format]:+ seen_urls: set[AbsoluteHttpURL] = set()+ http_sources: dict[str, dict[str, str]] = initials["videoModel"].get("sources") or {}+ if not http_sources:+ return-class Image(XHamsterItem):- url: HttpURL = Field(alias="imageURL")+ for codec, formats_dict in http_sources.items():+ for quality, url in formats_dict.items():+ if codec == "download":+ continue+ url = _parse_url(url)+ if url in seen_urls:+ continue-class Gallery(XHamsterItem):- title: str- photos: list[Image]- quantity: int+ seen_urls.add(url)+ yield Format(int(quality.removesuffix("p")), codec, url)-class Video(XHamsterItem):- title: str- mp4_file: HttpURL = Field(alias="mp4File")- mp4_sources: dict[str, Any] = Field({}, validation_alias=AliasPath("sources", "mp4"))-- def get_formats(self) -> Generator[Format]:- yield Format(0, "", self.mp4_file)- for resolution, details in self.mp4_sources.items():- height = int(resolution.removesuffix("p"))- link: str = details["link"] if isinstance(details, dict) else details- url = parse_url(link, relative_to=PRIMARY_URL)- yield Format(height, f"{height}p", url)+def _parse_xplayer_sources(initials: dict[str, Any]) -> Iterable[Format]:+ xplayer_sources: dict[str, Any] = initials.get("xplayerSettings", {}).get("sources", {})+ if not xplayer_sources:+ return++ seen_urls: set[AbsoluteHttpURL] = set()++ def parse_format(format_dict: dict[str, str], codec: str):+ for key in ("url", "fallback"):+ url = format_dict.get(key)+ if not url:+ continue++ url = _parse_url(url)+ if url in seen_urls:+ continue++ seen_urls.add(url)+ if url.suffix == ".m3u8":+ resolution = 0+ else:+ quality: str = format_dict.get("quality") or format_dict["label"]+ resolution = int(quality.removesuffix("p"))++ yield Format(resolution, codec, url)++ hls_sources: dict[str, dict[str, str]] = xplayer_sources.get("hls", {})+ for codec, format_dict in hls_sources.items():+ yield from parse_format(format_dict, codec)++ standard_sources: dict[str, list[dict[str, Any]]] = xplayer_sources.get("standard", {})+ for codec, formats_list in standard_sources.items():+ for format_dict in formats_list:+ yield from parse_format(format_dict, codec)diff --git a/tests/crawlers/test_cases/xhamster.py b/tests/crawlers/test_cases/xhamster.pynew file mode 100644index 000000000..99f8f1f8b--- /dev/null+++ b/tests/crawlers/test_cases/xhamster.py@@ -0,0 +1,104 @@+DOMAIN = "xhamster"+TEST_CASES = [+ (+ "https://ge.xhamster.desi/videos/horny-asian-slut-fucking-with-big-toy-and-cumming-a-lot-xhW9s1z",+ [+ {+ "url": "https://ge.xhamster.desi/videos/horny-asian-slut-fucking-with-big-toy-and-cumming-a-lot-xhW9s1z",+ "filename": "Horny Asian slut fucking with Big Toy and CUMMING a lot [xhW9s1z][h264][720p].mp4",+ "original_filename": "xhW9s1z.mp4",+ "referer": "https://ge.xhamster.desi/videos/horny-asian-slut-fucking-with-big-toy-and-cumming-a-lot-xhW9s1z",+ "album_id": None,+ "datetime": 1609776316,+ }+ ],+ ),+ (+ "https://ge.xhamster.com/photos/gallery/15860946/511409507",+ [+ {+ "url": "https://ic-ph-nss.xhcdn.com/a/M2JlNDhlZTJkNjk0NjY1ZGMyMWMwY2Q2ZTU3NGNlNjQ/webp/000/511/409/501_1000.jpg",+ "filename": "009 - 501_1000 [511409501].jpg",+ "original_filename": "501_1000.jpg",+ "referer": "https://ge.xhamster.com/photos/gallery/15860946/511409501",+ "download_folder": r"re:Take me to the carwash at DigitalDesire \[gallery\] \(xHamster\)",+ "album_id": "15860946",+ "datetime": 1643783409,+ }+ ],+ 19,+ ),+ (+ "https://ge.xhamster.com/photos/gallery/15860946",+ [+ {+ "url": "https://ic-ph-nss.xhcdn.com/a/M2JlNDhlZTJkNjk0NjY1ZGMyMWMwY2Q2ZTU3NGNlNjQ/webp/000/511/409/501_1000.jpg",+ "filename": "009 - 501_1000 [511409501].jpg",+ "original_filename": "501_1000.jpg",+ "referer": "https://ge.xhamster.com/photos/gallery/15860946/511409501",+ "download_folder": r"re:Take me to the carwash at DigitalDesire \[gallery\] \(xHamster\)",+ "album_id": "15860946",+ "datetime": 1643783409,+ }+ ],+ 19,+ ),+ (+ "https://xhamster.com/users/ombreto/videos",+ [+ {+ "url": "https://xhamster.com/videos/18yo-alexandra-student-bdsm-real-casting-2023793",+ "filename": "18yo Alexandra student BDSM real casting [2023793][h264][240p].mp4",+ "original_filename": "2023793.mp4",+ "referer": "https://xhamster.com/videos/18yo-alexandra-student-bdsm-real-casting-2023793",+ "download_folder": r"re:ombreto \[user\] \(xHamster\)/videos",+ "album_id": None,+ "datetime": 1372456260,+ }+ ],+ 42,+ ),+ (+ "https://xhamster.com/creators/danielagomezz",+ [+ {+ "url": "https://ic-ph-nss.xhcdn.com/a/Mzc4ODY4M2VkNzM0ZTM3OGVjMGQ1ZTFjZGU1MzhhZDU/webp/000/517/308/699_1000.jpg",+ "filename": "002 - 699_1000 [517308699].jpg",+ "original_filename": "699_1000.jpg",+ "referer": "https://xhamster.com/photos/gallery/16252066/517308699",+ "download_folder": r"re:DanielaGomezx \[user\] \(xHamster\)/galleries/New image change",+ "album_id": "16252066",+ "datetime": 1738917721,+ },+ ],+ 31,+ ),+ (+ "https://xhamster.com/creators/danielagomezz/exclusive",+ [],+ 29,+ ),+ (+ "https://xhamster.com/creators/danielagomezz/photos",+ [+ {+ "url": "https://ic-ph-nss.xhcdn.com/a/NGNmMmIxMGJlM2E2YjgyZWI3ZGE4OTcwNmNhNzAxZTU/webp/000/517/308/698_1000.jpg",+ "filename": "001 - 698_1000 [517308698].jpg",+ "original_filename": "698_1000.jpg",+ "referer": "https://xhamster.com/photos/gallery/16252066/517308698",+ "download_folder": r"re:DanielaGomezx \[user\] \(xHamster\)/galleries/New image change",+ "album_id": "16252066",+ "datetime": 1738917721,+ },+ {+ "url": "https://ic-ph-nss.xhcdn.com/a/Mzc4ODY4M2VkNzM0ZTM3OGVjMGQ1ZTFjZGU1MzhhZDU/webp/000/517/308/699_1000.jpg",+ "filename": "002 - 699_1000 [517308699].jpg",+ "original_filename": "699_1000.jpg",+ "referer": "https://xhamster.com/photos/gallery/16252066/517308699",+ "download_folder": r"re:DanielaGomezx \[user\] \(xHamster\)/galleries/New image change",+ "album_id": "16252066",+ "datetime": 1738917721,+ },+ ],+ ),+]