Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: handle videos without http sources (xhamster) (#1225)

NTFSvolume35daff7dc2f3
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index 133a79777..d38c48b94 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -1,183 +1,307 @@
from __future__ import annotations
+import dataclasses
+import itertools
import json
from functools import partial
-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, NamedTuple
+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
-from pydantic import AliasPath, Field, PlainValidator
+from aiolimiter import AsyncLimiter
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.models import AliasModel
-from cyberdrop_dl.utils.logger import log_debug
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url
if TYPE_CHECKING:
- from collections.abc import Generator
-
- from bs4 import BeautifulSoup
+ from collections.abc import Iterable
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-PRIMARY_URL = AbsoluteHttpURL("https://xhamster.com/")
+_PRIMARY_URL = AbsoluteHttpURL("https://xhamster.com/")
+_ALLOW_AV1 = False
+_ALLOW_HLS = False
-class Selectors:
- JS_VIDEO_INFO = "script#initials-script"
+class Selector:
+ JS_WINDOW_INITIALS = "script#initials-script"
VIDEO = "a.video-thumb__image-container"
GALLERY = "a.gallery-thumb__link"
NEXT_PAGE = "a[data-page='next']"
-_SELECTORS = Selectors()
-
-
-HttpURL = Annotated[AbsoluteHttpURL, PlainValidator(partial(parse_url, relative_to=PRIMARY_URL))]
+_parse_url = partial(parse_url, relative_to=_PRIMARY_URL)
class XhamsterCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Video": "/videos/<video_title>",
- "User": "/users/<user_name>",
- "Creator": "/creatos/<creator_name>",
- "Gallery": "photos/gallery/<gallery_name>",
+ "Video": "/videos/<title>",
+ "User": (
+ "/users/<user_name>",
+ "/users/profiles/<user_name>",
+ ),
+ "User Videos": "/users/<user_name>/videos",
+ "User Galleries": "/users/<user_name>/photos",
+ "Creator": "/creators/<creator_name>",
+ "Creator Videos": "/creators/<creator_name>/exclusive",
+ "Creator Galleries": "/creators/<creator_name>/photos",
+ "Gallery": "/photos/gallery/<gallery_name_or_id>",
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
- NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE
- DOMAIN: ClassVar[str] = "xhamster"
- FOLDER_DOMAIN: ClassVar[str] = "xHamster"
+ PRIMARY_URL = _PRIMARY_URL
+ NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE
+ DOMAIN = "xhamster"
+ FOLDER_DOMAIN = "xHamster"
+
+ def __post_init__(self) -> None:
+ self.request_limiter = AsyncLimiter(4, 1)
+ self._seen_hosts: set[str] = set()
+
+ def _disable_title_auto_translations(self, url: AbsoluteHttpURL) -> None:
+ if url.host not in self._seen_hosts:
+ self.update_cookies({"lang": "en", "video_titles_translation": "0"}, url.origin())
+ self._seen_hosts.add(url.host)
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "gallery" in scrape_item.url.parts:
- return await self.gallery(scrape_item)
- if any(p in scrape_item.url.parts for p in ("creators", "users")):
- return await self.profile(scrape_item)
- if any(p in scrape_item.url.parts for p in ("movies", "videos")):
- return await self.video(scrape_item)
- raise ValueError
+ match scrape_item.url.parts[1:]:
+ case ["photos", "gallery", _]:
+ return await self.gallery(scrape_item)
+ case ["videos", _]:
+ return await self.video(scrape_item)
+ case ["users" | "creators" as type_, _, *rest]:
+ match rest:
+ case []:
+ return await self.profile(scrape_item)
+ case ["photos"]:
+ return await self.profile(scrape_item, only_photos=True)
+ case ["videos"] if type_ == "users":
+ return await self.profile(scrape_item, only_videos=True)
+ case ["exclusive"] if type_ == "creators":
+ return await self.profile(scrape_item, only_videos=True)
+ case _:
+ raise ValueError
+ case _:
+ raise ValueError
+
+ @classmethod
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = super().transform_url(url)
+ match url.parts[1:]:
+ case ["photos", "gallery", name, *rest] if rest:
+ return url.origin() / "photos/gallery" / name
+ case ["users", "profiles", name]:
+ return url.origin() / "users" / name
+ case _:
+ return url
@error_handling_wrapper
- async def profile(self, scrape_item: ScrapeItem) -> None:
- profile_type, username = scrape_item.url.parts[1:3]
- title = self.create_title(f"{username} [{profile_type.removesuffix('s')}]")
+ async def profile(self, scrape_item: ScrapeItem, only_photos: bool = False, only_videos: bool = False) -> None:
+ url_type, username = scrape_item.url.parts[1:3]
+ base_url = scrape_item.url.origin() / url_type / username
+ initials = await self._get_window_initials(base_url)
+ is_creator_url = url_type == "creators"
+ info = initials["infoComponent"]["displayUserModel"] if is_creator_url else initials["displayUserModel"]
+
+ # every creator is actually an user
+ # 'pageTitle' has the creator's name (for a creator's URL)
+ # 'name' and 'displayName' are the actual user's name
+ # they are different but we will use the user name in both cases
+ name: str = info.get("displayName") or info["name"]
+ title = self.create_title(f"{name} [user]")
scrape_item.setup_as_profile(title)
- is_user = "users" in scrape_item.url.parts
- last_part = "videos" if is_user else "exclusive"
- base_url = PRIMARY_URL / profile_type / username
- all_paths = ("videos", "photos")
- paths_to_scrape = next(((p,) for p in all_paths if p in scrape_item.url.parts), all_paths)
+ if is_creator_url:
+ user_url = self.parse_url(info["pageURL"])
+ has_videos = bool(initials["infoComponent"].get("pornstarTop", {}).get("videoCount", 0))
+ has_galleries = bool(initials.get("galleriesComponent", {}).get("galleriesTotal", 0))
+
+ else:
+ user_url = base_url
+ has_videos = bool(initials["counters"]["videos"])
+ has_galleries = bool(initials["counters"]["galleries"])
- if "videos" in paths_to_scrape:
- videos_url = base_url / last_part
- await self.process_children(scrape_item, videos_url, _SELECTORS.VIDEO, last_part)
+ if has_videos and not only_photos:
+ videos_url = user_url / "videos"
+ await self._iter_profile_pages(scrape_item, videos_url, Selector.VIDEO, "videos")
- if is_user and "photos" in paths_to_scrape:
- gallerys_url = base_url / "photos"
- await self.process_children(scrape_item, gallerys_url, _SELECTORS.GALLERY, "galleries")
+ if has_galleries and not only_videos:
+ gallerys_url = user_url / "photos"
+ await self._iter_profile_pages(scrape_item, gallerys_url, Selector.GALLERY, "galleries")
@error_handling_wrapper
- async def process_children(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str, name: str) -> None:
+ async def _iter_profile_pages(
+ self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str, name: str
+ ) -> None:
async for soup in self.web_pager(url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem) -> None:
- json_info = await self.get_model_details(scrape_item.url, "photosGalleryModel", "galleryModel")
- gallery = Gallery(**json_info)
- title = f"{gallery.title} [gallery]"
- scrape_item.setup_as_album(title, album_id=gallery.id)
- scrape_item.possible_datetime = gallery.created
-
- padding = max(3, len(str(gallery.quantity)))
- for index, image in enumerate(gallery.photos, 1):
- filename, ext = self.get_filename_and_ext(image.url.name)
- # TODO: Adding an index prefix should be handled by `create_custom_filename`
- custom_filename = f"{str(index).zfill(padding)} - {filename.removesuffix(ext)}"
- custom_filename = self.create_custom_filename(custom_filename, ext, file_id=image.id)
- new_scrape_item = scrape_item.create_child(image.page_url)
- await self.handle_file(image.url, new_scrape_item, filename, ext, custom_filename=custom_filename)
- scrape_item.add_children()
+ initials = await self._get_window_initials(scrape_item.url)
+ page_details: dict[str, Any] = initials["galleryPage"]
+ gallery: dict[str, Any] = page_details["galleryModel"]
+ gallery_id = str(gallery["id"])
+ title = self.create_title(f"{gallery['title']} [gallery]", gallery_id)
+ scrape_item.setup_as_album(title, album_id=gallery_id)
+ scrape_item.possible_datetime = gallery["created"]
+
+ results = await self.get_album_results(gallery_id)
+ n_pages: int = page_details["paginationProps"]["lastPageNumber"]
+ index = 0
+ images: list[dict[str, Any]] = initials["photosGalleryModel"]["photos"]
+
+ for next_page in itertools.count(2):
+ for img in images:
+ img["index"] = index = index + 1
+ self._handle_img(scrape_item, img, results)
+
+ if next_page > n_pages:
+ break
+
+ next_page_url = scrape_item.url / str(next_page)
+ initials = await self._get_window_initials(next_page_url)
+ images = initials["photosGalleryModel"]["photos"]
+
+ def _handle_img(self, scrape_item: ScrapeItem, img: dict[str, Any], results: dict[str, int]):
+ src, page_url = self.parse_url(img["imageURL"]), self.parse_url(img["pageURL"])
+ if self.check_album_results(src, results):
+ return
+
+ _, ext = self.get_filename_and_ext(src.name)
+ stem = f"{str(img['index']).zfill(3)} - {src.name.removesuffix(ext)}"
+ filename = self.create_custom_filename(stem, ext, file_id=img["id"])
+ new_scrape_item = scrape_item.create_child(page_url)
+ self.create_task(self.handle_file(src, new_scrape_item, src.name, ext, custom_filename=filename))
+ scrape_item.add_children()
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- json_info = await self.get_model_details(scrape_item.url, "videoModel")
- video = Video(**json_info)
-
- scrape_item.url = video.page_url
+ initials = await self._get_window_initials(scrape_item.url)
+ video = _parse_video(initials)
scrape_item.possible_datetime = video.created
- _, resolution, download_url = max(video.get_formats())
- link = PRIMARY_URL / "movies" / video.id / "download" / resolution
- filename, ext = self.get_filename_and_ext(f"{video.id}.mp4")
- custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=resolution)
+ custom_filename = self.create_custom_filename(
+ video.title,
+ ".mp4",
+ file_id=video.id,
+ video_codec=video.best_mp4.codec,
+ resolution=video.best_mp4.resolution,
+ )
await self.handle_file(
- link, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=download_url
+ video.url,
+ scrape_item,
+ filename=video.id + ".mp4",
+ custom_filename=custom_filename,
+ debrid_link=video.best_mp4.url,
)
- async def get_model_details(self, url: AbsoluteHttpURL, *model_name_choices: str) -> dict:
- model_names = model_name_choices or []
-
+ async def _get_window_initials(self, url: AbsoluteHttpURL) -> dict[str, Any]:
+ self._disable_title_auto_translations(url)
async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, url)
+ soup = await self.client.get_soup(self.DOMAIN, url)
+
+ js_script = css.select_one(soup, Selector.JS_WINDOW_INITIALS)
+ json_text = get_text_between(str(js_script), "window.initials=", ";</script>")
+ return json.loads(json_text)
- json_info = get_window_initials_json(soup)
- del soup
- def get_model_value() -> dict[str, Any] | None:
- for model_name in model_names:
- if (value := json_info.get(model_name)) is not None:
- return value
[email protected](frozen=True, slots=True)
+class Video:
+ id: str
+ title: str
+ created: int
+ url: AbsoluteHttpURL
+ best_hls: Format | None
+ best_mp4: Format
+
- json_info = get_model_value() or json_info
- log_debug(json_info)
- return json_info
+def _parse_video(initials: dict[str, Any]):
+ video: dict[str, Any] = initials["videoModel"]
+ hls_sources: list[Format] = []
+ mp4_sources: list[Format] = []
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ sources = itertools.chain(_parse_http_sources(initials), _parse_xplayer_sources(initials))
+ for src in sources:
+ if src.codec == "av1" and not _ALLOW_AV1:
+ continue
+ if src.url.suffix == ".m3u8":
+ if not _ALLOW_HLS:
+ continue
+ hls_sources.append(src)
+ else:
+ mp4_sources.append(src)
-def get_window_initials_json(soup: BeautifulSoup) -> dict[str, dict]:
- js_script = soup.select_one(_SELECTORS.JS_VIDEO_INFO)
- js_code: str = str(js_script)
- json_text: str = get_text_between(js_code, "window.initials=", "</script>").removesuffix(";").strip()
- return json.loads(json_text)
+ return Video(
+ id=video["idHashSlug"],
+ title=video["title"],
+ created=video["created"],
+ url=_parse_url(video["pageURL"]),
+ best_hls=max(hls_sources, default=None),
+ best_mp4=max(mp4_sources),
+ )
class Format(NamedTuple):
- height: int
- resolution: str
+ resolution: int
+ codec: str # h264 > av1
url: AbsoluteHttpURL
-class XHamsterItem(AliasModel):
- id: str = Field(alias="idHashSlug")
- page_url: HttpURL = Field(alias="pageURL")
- created: int = 0
+def _parse_http_sources(initials: dict[str, Any]) -> Iterable[Format]:
+ seen_urls: set[AbsoluteHttpURL] = set()
+ http_sources: dict[str, dict[str, str]] = initials["videoModel"].get("sources") or {}
+ if not http_sources:
+ return
-class Image(XHamsterItem):
- url: HttpURL = Field(alias="imageURL")
+ for codec, formats_dict in http_sources.items():
+ for quality, url in formats_dict.items():
+ if codec == "download":
+ continue
+ url = _parse_url(url)
+ if url in seen_urls:
+ continue
-class Gallery(XHamsterItem):
- title: str
- photos: list[Image]
- quantity: int
+ seen_urls.add(url)
+ yield Format(int(quality.removesuffix("p")), codec, url)
-class Video(XHamsterItem):
- title: str
- mp4_file: HttpURL = Field(alias="mp4File")
- mp4_sources: dict[str, Any] = Field({}, validation_alias=AliasPath("sources", "mp4"))
-
- def get_formats(self) -> Generator[Format]:
- yield Format(0, "", self.mp4_file)
- for resolution, details in self.mp4_sources.items():
- height = int(resolution.removesuffix("p"))
- link: str = details["link"] if isinstance(details, dict) else details
- url = parse_url(link, relative_to=PRIMARY_URL)
- yield Format(height, f"{height}p", url)
+def _parse_xplayer_sources(initials: dict[str, Any]) -> Iterable[Format]:
+ xplayer_sources: dict[str, Any] = initials.get("xplayerSettings", {}).get("sources", {})
+ if not xplayer_sources:
+ return
+
+ seen_urls: set[AbsoluteHttpURL] = set()
+
+ def parse_format(format_dict: dict[str, str], codec: str):
+ for key in ("url", "fallback"):
+ url = format_dict.get(key)
+ if not url:
+ continue
+
+ url = _parse_url(url)
+ if url in seen_urls:
+ continue
+
+ seen_urls.add(url)
+ if url.suffix == ".m3u8":
+ resolution = 0
+ else:
+ quality: str = format_dict.get("quality") or format_dict["label"]
+ resolution = int(quality.removesuffix("p"))
+
+ yield Format(resolution, codec, url)
+
+ hls_sources: dict[str, dict[str, str]] = xplayer_sources.get("hls", {})
+ for codec, format_dict in hls_sources.items():
+ yield from parse_format(format_dict, codec)
+
+ standard_sources: dict[str, list[dict[str, Any]]] = xplayer_sources.get("standard", {})
+ for codec, formats_list in standard_sources.items():
+ for format_dict in formats_list:
+ yield from parse_format(format_dict, codec)
diff --git a/tests/crawlers/test_cases/xhamster.py b/tests/crawlers/test_cases/xhamster.py
new file mode 100644
index 000000000..99f8f1f8b
--- /dev/null
+++ b/tests/crawlers/test_cases/xhamster.py
@@ -0,0 +1,104 @@
+DOMAIN = "xhamster"
+TEST_CASES = [
+ (
+ "https://ge.xhamster.desi/videos/horny-asian-slut-fucking-with-big-toy-and-cumming-a-lot-xhW9s1z",
+ [
+ {
+ "url": "https://ge.xhamster.desi/videos/horny-asian-slut-fucking-with-big-toy-and-cumming-a-lot-xhW9s1z",
+ "filename": "Horny Asian slut fucking with Big Toy and CUMMING a lot [xhW9s1z][h264][720p].mp4",
+ "original_filename": "xhW9s1z.mp4",
+ "referer": "https://ge.xhamster.desi/videos/horny-asian-slut-fucking-with-big-toy-and-cumming-a-lot-xhW9s1z",
+ "album_id": None,
+ "datetime": 1609776316,
+ }
+ ],
+ ),
+ (
+ "https://ge.xhamster.com/photos/gallery/15860946/511409507",
+ [
+ {
+ "url": "https://ic-ph-nss.xhcdn.com/a/M2JlNDhlZTJkNjk0NjY1ZGMyMWMwY2Q2ZTU3NGNlNjQ/webp/000/511/409/501_1000.jpg",
+ "filename": "009 - 501_1000 [511409501].jpg",
+ "original_filename": "501_1000.jpg",
+ "referer": "https://ge.xhamster.com/photos/gallery/15860946/511409501",
+ "download_folder": r"re:Take me to the carwash at DigitalDesire \[gallery\] \(xHamster\)",
+ "album_id": "15860946",
+ "datetime": 1643783409,
+ }
+ ],
+ 19,
+ ),
+ (
+ "https://ge.xhamster.com/photos/gallery/15860946",
+ [
+ {
+ "url": "https://ic-ph-nss.xhcdn.com/a/M2JlNDhlZTJkNjk0NjY1ZGMyMWMwY2Q2ZTU3NGNlNjQ/webp/000/511/409/501_1000.jpg",
+ "filename": "009 - 501_1000 [511409501].jpg",
+ "original_filename": "501_1000.jpg",
+ "referer": "https://ge.xhamster.com/photos/gallery/15860946/511409501",
+ "download_folder": r"re:Take me to the carwash at DigitalDesire \[gallery\] \(xHamster\)",
+ "album_id": "15860946",
+ "datetime": 1643783409,
+ }
+ ],
+ 19,
+ ),
+ (
+ "https://xhamster.com/users/ombreto/videos",
+ [
+ {
+ "url": "https://xhamster.com/videos/18yo-alexandra-student-bdsm-real-casting-2023793",
+ "filename": "18yo Alexandra student BDSM real casting [2023793][h264][240p].mp4",
+ "original_filename": "2023793.mp4",
+ "referer": "https://xhamster.com/videos/18yo-alexandra-student-bdsm-real-casting-2023793",
+ "download_folder": r"re:ombreto \[user\] \(xHamster\)/videos",
+ "album_id": None,
+ "datetime": 1372456260,
+ }
+ ],
+ 42,
+ ),
+ (
+ "https://xhamster.com/creators/danielagomezz",
+ [
+ {
+ "url": "https://ic-ph-nss.xhcdn.com/a/Mzc4ODY4M2VkNzM0ZTM3OGVjMGQ1ZTFjZGU1MzhhZDU/webp/000/517/308/699_1000.jpg",
+ "filename": "002 - 699_1000 [517308699].jpg",
+ "original_filename": "699_1000.jpg",
+ "referer": "https://xhamster.com/photos/gallery/16252066/517308699",
+ "download_folder": r"re:DanielaGomezx \[user\] \(xHamster\)/galleries/New image change",
+ "album_id": "16252066",
+ "datetime": 1738917721,
+ },
+ ],
+ 31,
+ ),
+ (
+ "https://xhamster.com/creators/danielagomezz/exclusive",
+ [],
+ 29,
+ ),
+ (
+ "https://xhamster.com/creators/danielagomezz/photos",
+ [
+ {
+ "url": "https://ic-ph-nss.xhcdn.com/a/NGNmMmIxMGJlM2E2YjgyZWI3ZGE4OTcwNmNhNzAxZTU/webp/000/517/308/698_1000.jpg",
+ "filename": "001 - 698_1000 [517308698].jpg",
+ "original_filename": "698_1000.jpg",
+ "referer": "https://xhamster.com/photos/gallery/16252066/517308698",
+ "download_folder": r"re:DanielaGomezx \[user\] \(xHamster\)/galleries/New image change",
+ "album_id": "16252066",
+ "datetime": 1738917721,
+ },
+ {
+ "url": "https://ic-ph-nss.xhcdn.com/a/Mzc4ODY4M2VkNzM0ZTM3OGVjMGQ1ZTFjZGU1MzhhZDU/webp/000/517/308/699_1000.jpg",
+ "filename": "002 - 699_1000 [517308699].jpg",
+ "original_filename": "699_1000.jpg",
+ "referer": "https://xhamster.com/photos/gallery/16252066/517308699",
+ "download_folder": r"re:DanielaGomezx \[user\] \(xHamster\)/galleries/New image change",
+ "album_id": "16252066",
+ "datetime": 1738917721,
+ },
+ ],
+ ),
+]