Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: bypass login check for 1080p+ videos (ePorner) (#1735)

NTFSvolume301c14472dec
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 366a8234f..41d488861 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -73,6 +73,7 @@ class _PlaceHolderConfigInclude:
video_codec: bool = True
audio_codec: bool = True
resolution: bool = True
+ fps: bool = True
hash: bool = True
@@ -843,6 +844,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
video_codec: str | None = None,
audio_codec: str | None = None,
resolution: Resolution | str | int | None = None,
+ fps: float | None = None,
hash_string: str | None = None,
) -> str:
@@ -856,7 +858,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
if _include.resolution and resolution and resolution not in (Resolution.highest(), Resolution.unknown()):
res = resolution if type(resolution) is Resolution else Resolution.parse(resolution)
- yield res.name
+ if fps and _include.fps:
+ yield res.name + "@" + (str(int(fps)) if fps.is_integer() else f"{fps:.1f}") + "fps"
+ else:
+ yield res.name
if _include.hash and hash_string:
assert any(hash_string.startswith(x) for x in _HASH_PREFIXES), f"Invalid: {hash_string = }"
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index 0a3408d57..2a26e545b 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -1,35 +1,30 @@
from __future__ import annotations
import dataclasses
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
-from cyberdrop_dl import env
-from cyberdrop_dl.compat import IntEnum
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from bs4 import BeautifulSoup
+
+from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
-from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, parse_url
if TYPE_CHECKING:
- from bs4 import BeautifulSoup, Tag
+ from collections.abc import Generator
from cyberdrop_dl.url_objects import ScrapeItem
class Selector:
- _DOWNLOADS = "div#hd-porn-dload > div.dloaddivcol"
- _H264 = "span.download-h264 > a"
- _AV1 = "span.download-av1 > a"
- FORMATS = f"{_DOWNLOADS} {_H264},{_DOWNLOADS} {_AV1}"
-
PHOTO = "div#gridphoto > a.photohref"
VIDEO = "div[id^='vf'] div.mbcontent a"
NEXT_PAGE = "div.numlist2 a.nmnext"
PROFILE_GALLERY = "div[id^='pf'] a"
PROFILE_PLAYLIST = "div.streameventsday.showAll > div#pl > a"
- DATE_JS = "main script:-soup-contains('uploadDate')"
+ DATE_JS = "script[type='application/ld+json']:-soup-contains('uploadDate')"
GALLERY_TITLE = "div#galleryheader > h1"
@@ -40,33 +35,21 @@ _PROFILE_URL_PARTS = {
}
[email protected](frozen=True, slots=True)
[email protected](slots=True)
class Video:
title: str
date: str
best_src: VideoSource
+ sources: tuple[VideoSource, ...]
-class Codec(IntEnum):
- H264 = 0
- AV1 = -1 if env.EPORNER_PREFER_H264 else 1
-
-
[email protected](frozen=True, order=True, slots=True)
[email protected](order=True, slots=True)
class VideoSource:
resolution: Resolution
- codec: Codec
- size: str
- url: str
-
- @staticmethod
- def parse(tag: Tag) -> VideoSource:
- link_str: str = css.attr(tag, "href")
- name = tag.get_text(strip=True).removeprefix("Download")
- details = name.split("(", 1)[1].removesuffix(")").split(",")
- res, codec, size = [d.strip() for d in details]
- codec = Codec[codec.upper()]
- return VideoSource(Resolution.parse(res), codec, size, link_str)
+ fps: float
+ url: AbsoluteHttpURL
+ name: str
+ format: str
class EpornerCrawler(Crawler):
@@ -77,7 +60,11 @@ class EpornerCrawler(Crawler):
"Profile": "/profile/...",
"Search": "/search/...",
"Search Photos": "/search-photos/...",
- "Video": ("/<video_name>-<video-id>", "/hd-porn/<video_id>", "/embed/<video_id>"),
+ "Video": (
+ "/<video_name>-<video-id>",
+ "/hd-porn/<video_id>",
+ "/embed/<video_id>",
+ ),
"Photo": "/photo/...",
"Gallery": "/gallery/...",
}
@@ -85,7 +72,7 @@ class EpornerCrawler(Crawler):
DOMAIN: ClassVar[str] = "eporner"
FOLDER_DOMAIN: ClassVar[str] = "ePorner"
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
- _RATE_LIMIT: ClassVar[tuple[float, float]] = 2, 1
+ _RATE_LIMIT: ClassVar[RateLimit] = 2, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
@@ -189,47 +176,121 @@ class EpornerCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- soup = await self.request_soup(scrape_item.url)
-
- soup_str = soup.get_text()
- if "File has been removed due to copyright owner request" in soup_str:
- raise ScrapeError(451)
- if "Video has been deleted" in soup_str:
- raise ScrapeError(410)
-
+ video = await self._request_video(canonical_url, video_id)
scrape_item.url = canonical_url
- # TODO: Force utf8 for soup
- video = _parse_video(soup)
- link = self.parse_url(video.best_src.url)
+ src = video.best_src.url
scrape_item.uploaded_at = self.parse_iso_date(video.date)
- _, ext = self.get_filename_and_ext(link.name)
filename = self.create_custom_filename(
video.title,
- ext,
+ ext := src.suffix,
file_id=video_id,
+ video_codec="h264",
resolution=video.best_src.resolution,
- video_codec=video.best_src.codec.name.lower(),
+ fps=video.best_src.fps,
)
- dl_link = await self._request_location_reencoded(link)
- await self.handle_file(link, scrape_item, video.title, ext, custom_filename=filename, debrid_link=dl_link)
- async def _request_location_reencoded(self, link: AbsoluteHttpURL):
- # The location header is not encoded and the "requote_url_redirect" param of aiohttp is session scoped
- # so we manually requote this redirect
- async with self.request(link, allow_redirects=False) as resp:
- assert resp.location
- return AbsoluteHttpURL(f"{resp.location.origin()}/{resp.location.raw_path_qs}", encoded=True)
+ await self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ video.title,
+ ext,
+ custom_filename=filename,
+ debrid_link=src,
+ )
+
+ async def _request_video(self, url: AbsoluteHttpURL, video_id: str) -> Video:
+ html = await self.request_text(url)
+ player_hash = _extract_player_hash(html)
+ xhr_url = (self.PRIMARY_URL / "xhr/video" / video_id).with_query(
+ hash=_encode_hash(player_hash),
+ domain=self.PRIMARY_URL.host,
+ fallback="false",
+ embed="false",
+ supportedFormats="hls,h265,vp9,av1,mp4",
+ )
+ resp: dict[str, Any] = await self.request_json(xhr_url)
+ if resp.get("available") is False:
+ raise ScrapeError(404, resp.get("message"))
+ return _parse_video(html, resp)
-def _parse_video(soup: BeautifulSoup) -> Video:
- ld_json = css.select_text(soup, Selector.DATE_JS).encode("raw_unicode_escape").decode("unicode-escape")
+
+def _parse_video(html: str, video: dict[str, Any]) -> Video:
+
+ ld_json = (
+ css.select_text(BeautifulSoup(html, "html.parser"), Selector.DATE_JS)
+ .encode("raw_unicode_escape")
+ .decode("unicode-escape")
+ )
# This may have invalid json. They do not sanitize the description field
# See: https://github.com/Cyberdrop-DL/cyberdrop-dl/issues/1211
- formats = [VideoSource.parse(tag) for tag in soup.select(Selector.FORMATS)]
-
+ sources = tuple(_parse_sources(video["sources"]))
return Video(
title=extr_text(ld_json, 'name": "', '",'),
date=extr_text(ld_json, 'uploadDate": "', '"'),
- best_src=max(formats),
+ sources=sources,
+ best_src=max(src for src in sources if src.format == "mp4"),
)
+
+
+def _parse_sources(sources: dict[str, dict[str, dict[str, Any]]]) -> Generator[VideoSource]:
+ for format, formats in sources.items():
+ for name, source in formats.items():
+ url = parse_url(source["src"])
+ if format == "hls":
+ resolution = _parse_hls_res(url)
+ fps = 0.0
+
+ else:
+ resolution = Resolution.parse(source["labelShort"])
+ fps = _parse_fps(name)
+
+ yield VideoSource(
+ resolution=resolution,
+ fps=fps,
+ url=url,
+ name=name,
+ format=format,
+ )
+
+
+def _extract_player_hash(html: str) -> str:
+ if "File has been removed due to copyright owner request" in html:
+ raise ScrapeError(451)
+ if "Video has been deleted" in html:
+ raise ScrapeError(410)
+
+ return extr_text(html, "EP.video.player.hash = '", "';")
+
+
+def _parse_hls_res(url: AbsoluteHttpURL) -> Resolution:
+ for part in reversed(url.parts[-2].split(",")):
+ try:
+ return Resolution.parse(part)
+ except ValueError:
+ continue
+ return Resolution.unknown()
+
+
+def _parse_fps(name: str) -> float:
+ try:
+ return float(extr_text(name, "@", "fps"))
+ except ValueError:
+ return 0.0
+
+
+def _encode_hash(hex_hash: str) -> str:
+ assert len(hex_hash) == 32
+ return "".join(_encode_base_36(int(hex_hash[idx : idx + 8], base=16)) for idx in range(0, 32, 8))
+
+
+def _encode_base_36(number: int) -> str:
+ alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
+ result = ""
+
+ while number != 0:
+ number, index = divmod(number, 36)
+ result = alphabet[index] + result
+
+ return result or "0"
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index 1a69509bd..70f0f0523 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -40,7 +40,6 @@ EDITOR = os.getenv("EDITOR")
PIXELDRAIN_PROXY = _env("PIXELDRAIN_PROXY")
BANDCAMP_FORMATS = _env("BANDCAMP_FORMATS")
-EPORNER_PREFER_H264 = bool(_env("EPORNER_PREFER_H264"))
ENABLE_TWITTER = bool(_env("ENABLE_TWITTER"))
ONEPACE_PREFER_DUB = bool(_env("ONEPACE_PREFER_DUB"))
diff --git a/tests/crawlers/test_cases/eporner.py b/tests/crawlers/test_cases/eporner.py
index 280fcb7e2..080e2b838 100644
--- a/tests/crawlers/test_cases/eporner.py
+++ b/tests/crawlers/test_cases/eporner.py
@@ -4,26 +4,45 @@ TEST_CASES = [
"https://www.eporner.com/video-ia7DS1X427X/yoga-pmv/",
[
{
- "url": "https://www.eporner.com/dload/ia7DS1X427X/2160/11435843-2160p-av1.mp4",
- "filename": "Yoga PMV [ia7DS1X427X][av1][4K].mp4",
+ "url": "https://www.eporner.com/video-ia7DS1X427X",
+ "filename": "Yoga PMV [ia7DS1X427X][h264][4K].mp4",
+ "debrid_link": "re:11435843-2160p.mp4",
"original_filename": '"Yoga" PMV',
"referer": "https://www.eporner.com/video-ia7DS1X427X",
"album_id": None,
- "datetime": 1722220285,
+ "uploaded_at": 1722220285,
+ "download_folder": "re:Loose Files (ePorner)",
}
],
+ 1,
),
(
- # Invalid json on the page
- "https://www.eporner.com/video-6kuhFUubtC2/nightqueen-presents-jessa-fucked-peta-bailey-fucked-both-later/",
+ "https://www.eporner.com/video-5XdtfLAMNOV",
[
{
- "url": "https://www.eporner.com/dload/6kuhFUubtC2/1080/13445569-1080p.mp4",
- "filename": "NightQueen Presents Jessa Fucked Peta Bailey Fucked Both Later 🤤 [6kuhFUubtC2][h264][1080p].mp4",
- "original_filename": "NightQueen Presents Jessa Fucked Peta Bailey Fucked Both Later 🤤",
- "referer": "https://www.eporner.com/video-6kuhFUubtC2",
+ "url": "https://www.eporner.com/video-5XdtfLAMNOV",
+ "filename": "I’m A Tubera And I Love It In The Tubo, Full Tubo. Horny Bl [5XdtfLAMNOV][h264][4K@60fps].mp4", # noqa: RUF001
+ "debrid_link": "re:12538936-2160p.mp4",
+ "original_filename": "I’m A Tubera And I Love It In The Tubo, Full Tubo. Horny Blonde Fucks With Her Boss", # noqa: RUF001
+ "referer": "https://www.eporner.com/video-5XdtfLAMNOV",
"album_id": None,
+ "uploaded_at": 1736796270,
+ "download_folder": "re:Loose Files (ePorner)",
}
],
+ 1,
+ ),
+ (
+ "https://www.eporner.com/embed/5XdtfLAMNOV",
+ [
+ {
+ "url": "https://www.eporner.com/video-5XdtfLAMNOV",
+ "filename": "I’m A Tubera And I Love It In The Tubo, Full Tubo. Horny Bl [5XdtfLAMNOV][h264][4K@60fps].mp4", # noqa: RUF001
+ "referer": "https://www.eporner.com/video-5XdtfLAMNOV",
+ "uploaded_at": 1736796270,
+ "download_folder": "re:Loose Files (ePorner)",
+ }
+ ],
+ 1,
),
]
diff --git a/tests/crawlers/test_eporner.py b/tests/crawlers/test_eporner.py
new file mode 100644
index 000000000..04b67a470
--- /dev/null
+++ b/tests/crawlers/test_eporner.py
@@ -0,0 +1,26 @@
+import pytest
+
+from cyberdrop_dl.crawlers import eporner
+
+
+ "raw, expected",
+ [
+ ("744fced59f9d5ac7824daaaeb24aa3d8", "w9t04518acgxz105kb721dgwkig"),
+ ],
+)
+def test_encode_hash(raw: str, expected: str) -> None:
+ assert eporner._encode_hash(raw) == expected
+
+
+ "raw, expected",
+ [
+ ("720p@60fps HD", 60.0),
+ ("2160p(4K)@60fps HD", 60.0),
+ ("1440p(2K)@60fps HD", 60.0),
+ ("360p", 0.0),
+ ],
+)
+def test_parse_fps(raw: str, expected: float) -> None:
+ assert eporner._parse_fps(raw) == expected