Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: bypass login check for 1080p+ videos (ePorner) (#1735)
NTFSvolume301c14472dec
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 366a8234f..41d488861 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -73,6 +73,7 @@ class _PlaceHolderConfigInclude:video_codec: bool = Trueaudio_codec: bool = Trueresolution: bool = True+ fps: bool = Truehash: bool = True@@ -843,6 +844,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):video_codec: str | None = None,audio_codec: str | None = None,resolution: Resolution | str | int | None = None,+ fps: float | None = None,hash_string: str | None = None,) -> str:@@ -856,7 +858,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):if _include.resolution and resolution and resolution not in (Resolution.highest(), Resolution.unknown()):res = resolution if type(resolution) is Resolution else Resolution.parse(resolution)- yield res.name+ if fps and _include.fps:+ yield res.name + "@" + (str(int(fps)) if fps.is_integer() else f"{fps:.1f}") + "fps"+ else:+ yield res.nameif _include.hash and hash_string:assert any(hash_string.startswith(x) for x in _HASH_PREFIXES), f"Invalid: {hash_string = }"diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex 0a3408d57..2a26e545b 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -1,35 +1,30 @@from __future__ import annotationsimport dataclasses-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVar-from cyberdrop_dl import env-from cyberdrop_dl.compat import IntEnum-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from bs4 import BeautifulSoup++from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, parse_urlif TYPE_CHECKING:- from bs4 import BeautifulSoup, Tag+ from collections.abc import Generatorfrom cyberdrop_dl.url_objects import ScrapeItemclass Selector:- _DOWNLOADS = "div#hd-porn-dload > div.dloaddivcol"- _H264 = "span.download-h264 > a"- _AV1 = "span.download-av1 > a"- FORMATS = f"{_DOWNLOADS} {_H264},{_DOWNLOADS} {_AV1}"-PHOTO = "div#gridphoto > a.photohref"VIDEO = "div[id^='vf'] div.mbcontent a"NEXT_PAGE = "div.numlist2 a.nmnext"PROFILE_GALLERY = "div[id^='pf'] a"PROFILE_PLAYLIST = "div.streameventsday.showAll > div#pl > a"- DATE_JS = "main script:-soup-contains('uploadDate')"+ DATE_JS = "script[type='application/ld+json']:-soup-contains('uploadDate')"GALLERY_TITLE = "div#galleryheader > h1"@@ -40,33 +35,21 @@ _PROFILE_URL_PARTS = {}[email protected](frozen=True, slots=True)[email protected](slots=True)class Video:title: strdate: strbest_src: VideoSource+ sources: tuple[VideoSource, ...]-class Codec(IntEnum):- H264 = 0- AV1 = -1 if env.EPORNER_PREFER_H264 else 1--[email protected](frozen=True, order=True, slots=True)[email protected](order=True, slots=True)class VideoSource:resolution: Resolution- codec: Codec- size: str- url: str-- @staticmethod- def parse(tag: Tag) -> VideoSource:- link_str: str = css.attr(tag, "href")- name = tag.get_text(strip=True).removeprefix("Download")- details = name.split("(", 1)[1].removesuffix(")").split(",")- res, codec, size = [d.strip() for d in details]- codec = Codec[codec.upper()]- return VideoSource(Resolution.parse(res), codec, size, link_str)+ fps: float+ url: AbsoluteHttpURL+ name: str+ format: strclass EpornerCrawler(Crawler):@@ -77,7 +60,11 @@ class EpornerCrawler(Crawler):"Profile": "/profile/...","Search": "/search/...","Search Photos": "/search-photos/...",- "Video": ("/<video_name>-<video-id>", "/hd-porn/<video_id>", "/embed/<video_id>"),+ "Video": (+ "/<video_name>-<video-id>",+ "/hd-porn/<video_id>",+ "/embed/<video_id>",+ ),"Photo": "/photo/...","Gallery": "/gallery/...",}@@ -85,7 +72,7 @@ class EpornerCrawler(Crawler):DOMAIN: ClassVar[str] = "eporner"FOLDER_DOMAIN: ClassVar[str] = "ePorner"NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE- _RATE_LIMIT: ClassVar[tuple[float, float]] = 2, 1+ _RATE_LIMIT: ClassVar[RateLimit] = 2, 1async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -189,47 +176,121 @@ class EpornerCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- soup = await self.request_soup(scrape_item.url)-- soup_str = soup.get_text()- if "File has been removed due to copyright owner request" in soup_str:- raise ScrapeError(451)- if "Video has been deleted" in soup_str:- raise ScrapeError(410)-+ video = await self._request_video(canonical_url, video_id)scrape_item.url = canonical_url- # TODO: Force utf8 for soup- video = _parse_video(soup)- link = self.parse_url(video.best_src.url)+ src = video.best_src.urlscrape_item.uploaded_at = self.parse_iso_date(video.date)- _, ext = self.get_filename_and_ext(link.name)filename = self.create_custom_filename(video.title,- ext,+ ext := src.suffix,file_id=video_id,+ video_codec="h264",resolution=video.best_src.resolution,- video_codec=video.best_src.codec.name.lower(),+ fps=video.best_src.fps,)- dl_link = await self._request_location_reencoded(link)- await self.handle_file(link, scrape_item, video.title, ext, custom_filename=filename, debrid_link=dl_link)- async def _request_location_reencoded(self, link: AbsoluteHttpURL):- # The location header is not encoded and the "requote_url_redirect" param of aiohttp is session scoped- # so we manually requote this redirect- async with self.request(link, allow_redirects=False) as resp:- assert resp.location- return AbsoluteHttpURL(f"{resp.location.origin()}/{resp.location.raw_path_qs}", encoded=True)+ await self.handle_file(+ scrape_item.url,+ scrape_item,+ video.title,+ ext,+ custom_filename=filename,+ debrid_link=src,+ )++ async def _request_video(self, url: AbsoluteHttpURL, video_id: str) -> Video:+ html = await self.request_text(url)+ player_hash = _extract_player_hash(html)+ xhr_url = (self.PRIMARY_URL / "xhr/video" / video_id).with_query(+ hash=_encode_hash(player_hash),+ domain=self.PRIMARY_URL.host,+ fallback="false",+ embed="false",+ supportedFormats="hls,h265,vp9,av1,mp4",+ )+ resp: dict[str, Any] = await self.request_json(xhr_url)+ if resp.get("available") is False:+ raise ScrapeError(404, resp.get("message"))+ return _parse_video(html, resp)-def _parse_video(soup: BeautifulSoup) -> Video:- ld_json = css.select_text(soup, Selector.DATE_JS).encode("raw_unicode_escape").decode("unicode-escape")++def _parse_video(html: str, video: dict[str, Any]) -> Video:++ ld_json = (+ css.select_text(BeautifulSoup(html, "html.parser"), Selector.DATE_JS)+ .encode("raw_unicode_escape")+ .decode("unicode-escape")+ )# This may have invalid json. They do not sanitize the description field# See: https://github.com/Cyberdrop-DL/cyberdrop-dl/issues/1211- formats = [VideoSource.parse(tag) for tag in soup.select(Selector.FORMATS)]-+ sources = tuple(_parse_sources(video["sources"]))return Video(title=extr_text(ld_json, 'name": "', '",'),date=extr_text(ld_json, 'uploadDate": "', '"'),- best_src=max(formats),+ sources=sources,+ best_src=max(src for src in sources if src.format == "mp4"),)+++def _parse_sources(sources: dict[str, dict[str, dict[str, Any]]]) -> Generator[VideoSource]:+ for format, formats in sources.items():+ for name, source in formats.items():+ url = parse_url(source["src"])+ if format == "hls":+ resolution = _parse_hls_res(url)+ fps = 0.0++ else:+ resolution = Resolution.parse(source["labelShort"])+ fps = _parse_fps(name)++ yield VideoSource(+ resolution=resolution,+ fps=fps,+ url=url,+ name=name,+ format=format,+ )+++def _extract_player_hash(html: str) -> str:+ if "File has been removed due to copyright owner request" in html:+ raise ScrapeError(451)+ if "Video has been deleted" in html:+ raise ScrapeError(410)++ return extr_text(html, "EP.video.player.hash = '", "';")+++def _parse_hls_res(url: AbsoluteHttpURL) -> Resolution:+ for part in reversed(url.parts[-2].split(",")):+ try:+ return Resolution.parse(part)+ except ValueError:+ continue+ return Resolution.unknown()+++def _parse_fps(name: str) -> float:+ try:+ return float(extr_text(name, "@", "fps"))+ except ValueError:+ return 0.0+++def _encode_hash(hex_hash: str) -> str:+ assert len(hex_hash) == 32+ return "".join(_encode_base_36(int(hex_hash[idx : idx + 8], base=16)) for idx in range(0, 32, 8))+++def _encode_base_36(number: int) -> str:+ alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"+ result = ""++ while number != 0:+ number, index = divmod(number, 36)+ result = alphabet[index] + result++ return result or "0"diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex 1a69509bd..70f0f0523 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -40,7 +40,6 @@ EDITOR = os.getenv("EDITOR")PIXELDRAIN_PROXY = _env("PIXELDRAIN_PROXY")BANDCAMP_FORMATS = _env("BANDCAMP_FORMATS")-EPORNER_PREFER_H264 = bool(_env("EPORNER_PREFER_H264"))ENABLE_TWITTER = bool(_env("ENABLE_TWITTER"))ONEPACE_PREFER_DUB = bool(_env("ONEPACE_PREFER_DUB"))diff --git a/tests/crawlers/test_cases/eporner.py b/tests/crawlers/test_cases/eporner.pyindex 280fcb7e2..080e2b838 100644--- a/tests/crawlers/test_cases/eporner.py+++ b/tests/crawlers/test_cases/eporner.py@@ -4,26 +4,45 @@ TEST_CASES = ["https://www.eporner.com/video-ia7DS1X427X/yoga-pmv/",[{- "url": "https://www.eporner.com/dload/ia7DS1X427X/2160/11435843-2160p-av1.mp4",- "filename": "Yoga PMV [ia7DS1X427X][av1][4K].mp4",+ "url": "https://www.eporner.com/video-ia7DS1X427X",+ "filename": "Yoga PMV [ia7DS1X427X][h264][4K].mp4",+ "debrid_link": "re:11435843-2160p.mp4","original_filename": '"Yoga" PMV',"referer": "https://www.eporner.com/video-ia7DS1X427X","album_id": None,- "datetime": 1722220285,+ "uploaded_at": 1722220285,+ "download_folder": "re:Loose Files (ePorner)",}],+ 1,),(- # Invalid json on the page- "https://www.eporner.com/video-6kuhFUubtC2/nightqueen-presents-jessa-fucked-peta-bailey-fucked-both-later/",+ "https://www.eporner.com/video-5XdtfLAMNOV",[{- "url": "https://www.eporner.com/dload/6kuhFUubtC2/1080/13445569-1080p.mp4",- "filename": "NightQueen Presents Jessa Fucked Peta Bailey Fucked Both Later 🤤 [6kuhFUubtC2][h264][1080p].mp4",- "original_filename": "NightQueen Presents Jessa Fucked Peta Bailey Fucked Both Later 🤤",- "referer": "https://www.eporner.com/video-6kuhFUubtC2",+ "url": "https://www.eporner.com/video-5XdtfLAMNOV",+ "filename": "I’m A Tubera And I Love It In The Tubo, Full Tubo. Horny Bl [5XdtfLAMNOV][h264][4K@60fps].mp4", # noqa: RUF001+ "debrid_link": "re:12538936-2160p.mp4",+ "original_filename": "I’m A Tubera And I Love It In The Tubo, Full Tubo. Horny Blonde Fucks With Her Boss", # noqa: RUF001+ "referer": "https://www.eporner.com/video-5XdtfLAMNOV","album_id": None,+ "uploaded_at": 1736796270,+ "download_folder": "re:Loose Files (ePorner)",}],+ 1,+ ),+ (+ "https://www.eporner.com/embed/5XdtfLAMNOV",+ [+ {+ "url": "https://www.eporner.com/video-5XdtfLAMNOV",+ "filename": "I’m A Tubera And I Love It In The Tubo, Full Tubo. Horny Bl [5XdtfLAMNOV][h264][4K@60fps].mp4", # noqa: RUF001+ "referer": "https://www.eporner.com/video-5XdtfLAMNOV",+ "uploaded_at": 1736796270,+ "download_folder": "re:Loose Files (ePorner)",+ }+ ],+ 1,),]diff --git a/tests/crawlers/test_eporner.py b/tests/crawlers/test_eporner.pynew file mode 100644index 000000000..04b67a470--- /dev/null+++ b/tests/crawlers/test_eporner.py@@ -0,0 +1,26 @@+import pytest++from cyberdrop_dl.crawlers import eporner+++ "raw, expected",+ [+ ("744fced59f9d5ac7824daaaeb24aa3d8", "w9t04518acgxz105kb721dgwkig"),+ ],+)+def test_encode_hash(raw: str, expected: str) -> None:+ assert eporner._encode_hash(raw) == expected+++ "raw, expected",+ [+ ("720p@60fps HD", 60.0),+ ("2160p(4K)@60fps HD", 60.0),+ ("1440p(2K)@60fps HD", 60.0),+ ("360p", 0.0),+ ],+)+def test_parse_fps(raw: str, expected: float) -> None:+ assert eporner._parse_fps(raw) == expected