Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove unused javascript parses

NTFSvolumea69cbdb4bb94
diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.py
index c0cf6eb5b..ab3a08cd5 100644
--- a/cyberdrop_dl/crawlers/youjizz.py
+++ b/cyberdrop_dl/crawlers/youjizz.py
@@ -1,57 +1,62 @@
from __future__ import annotations
+import dataclasses
import json
-import re
-from typing import TYPE_CHECKING, ClassVar, NamedTuple
+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.utils import css, javascript
-from cyberdrop_dl.utils.logger import log_debug
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
+from cyberdrop_dl.utils import css, open_graph
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
+ from collections.abc import Iterable
+
from bs4 import BeautifulSoup
- from yarl import URL
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://www.youjizz.com/")
-DEFAULT_QUALITY = "Auto"
-RESOLUTIONS = ["4k", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"] # best to worst
-DATE_PATTERN = re.compile(r"(\d+)\s*(weeks?|days?|hours?|minutes?|seconds?)", re.IGNORECASE)
-
-
JS_SELECTOR = "div#content > script:contains('var dataEncodings')"
DATE_SELECTOR = "span.pretty-date"
-class Format(NamedTuple):
- resolution: str | None
- link_str: str
+class VideoSource(NamedTuple):
+ resolution: int
+ url: str
-## TODO: convert to global dataclass with constructor from dict to use in multiple crawlers
-class VideoInfo(dict): ...
[email protected](frozen=True, slots=True)
+class Video:
+ date: str # Human date
+ title: str
+ best_src: VideoSource
class YouJizzCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": ("/video/embed/<video_id>", "/video/<video_id>/...")}
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": (
+ "/videos/embed/<video_id>",
+ "/videos/<video_name>",
+ )
+ }
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "youjizz"
FOLDER_DOMAIN: ClassVar[str] = "YouJizz"
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if any(p in scrape_item.url.parts for p in ("videos", "embed")):
- return await self.video(scrape_item)
- raise ValueError
+ match scrape_item.url.parts[1:]:
+ case ["videos", "embed", video_id]:
+ return await self.video(scrape_item, video_id)
+ case ["videos", video_name]:
+ video_id = video_name.rsplit("-", 1)[-1].removesuffix(".html")
+ return await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
@error_handling_wrapper
- async def video(self, scrape_item: ScrapeItem) -> None:
- video_id = get_video_id(scrape_item.url)
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
canonical_url = PRIMARY_URL / "videos" / "embed" / video_id
-
if await self.check_complete_from_referer(canonical_url):
return
@@ -59,51 +64,37 @@ class YouJizzCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
scrape_item.url = canonical_url
- info = get_info(soup)
- v_format = get_best_quality(info)
- if not v_format:
- raise ScrapeError(422)
-
- resolution, link_str = v_format
- link = self.parse_url(link_str)
- scrape_item.possible_datetime = self.parse_date(info["date"])
+ video = _parse_video(soup)
+ link = self.parse_url(video.best_src.url)
+ scrape_item.possible_datetime = self.parse_date(video.date)
filename, ext = self.get_filename_and_ext(link.name)
- if ext == ".m3u8":
- raise ScrapeError(422)
- custom_filename = self.create_custom_filename(info["title"], ext, file_id=video_id, resolution=resolution)
- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
-
-
-def get_video_id(url: URL) -> str:
- if "embed" in url.parts:
- embed_index = url.parts.index("embed")
- return url.parts[embed_index + 1]
-
- video_id = url.parts[2].rsplit("-", 1)[-1]
- return video_id.removesuffix(".html")
-
-
-def get_info(soup: BeautifulSoup) -> VideoInfo:
- info_js_script_text: str = css.select_one_get_text(soup, JS_SELECTOR)
- info: dict[str, str | None | dict] = javascript.parse_js_vars(info_js_script_text)
- info["title"] = css.get_attr(soup, "title").replace("\n", "").strip()
- date_tag = soup.select_one(DATE_SELECTOR)
- date_str: str | None = date_tag.text if date_tag else None
- info["date"] = date_str.replace("(s)", "s").strip() if date_str else None
- javascript.clean_dict(info, "stream_data")
- log_debug(json.dumps(info, indent=4))
- return VideoInfo(**info)
-
-
-def get_best_quality(info_dict: dict) -> Format | None:
- qualities: dict = info_dict["dataEncodings"]
- for res in RESOLUTIONS:
- avaliable_formats = [f for f in qualities if f["name"] == res]
- for format_info in avaliable_formats:
- link_str = format_info["filename"]
- if "/_hls/" not in link_str:
- return Format(res, link_str)
- default_quality: dict = next((f for f in qualities if f["name"] == DEFAULT_QUALITY), {})
- if default_quality:
- default_link_str = default_quality.get("filename") or ""
- return Format(None, default_link_str)
+ custom_filename = self.create_custom_filename(
+ video.title, ext, file_id=video_id, resolution=video.best_src.resolution
+ )
+ await self.handle_file(
+ scrape_item.url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link
+ )
+
+
+def _parse_video(soup: BeautifulSoup) -> Video:
+ js_text = css.select_one_get_text(soup, JS_SELECTOR)
+ encodings_text = get_text_between(js_text, "var dataEncodings =", "var encodings").strip().removesuffix(";")
+ data_encodings = json.loads(encodings_text)
+ return Video(
+ title=open_graph.title(soup),
+ date=css.select_one_get_text(soup, DATE_SELECTOR),
+ best_src=_get_best_src(data_encodings),
+ )
+
+
+def _get_best_src(data_encodings: list[dict[str, Any]]) -> VideoSource:
+ def parse() -> Iterable[VideoSource]:
+ for format_info in data_encodings:
+ try:
+ res = int(format_info["quality"])
+ except ValueError:
+ continue
+ if "/_hls/" not in (link_str := format_info["filename"]):
+ yield VideoSource(res, link_str)
+
+ return max(parse())
diff --git a/cyberdrop_dl/utils/javascript.py b/cyberdrop_dl/utils/javascript.py
index 5ef1bc01e..bd1df790f 100644
--- a/cyberdrop_dl/utils/javascript.py
+++ b/cyberdrop_dl/utils/javascript.py
@@ -4,18 +4,30 @@ import json
import re
from typing import Any
-HTTPS_PLACEHOLDER = "<<SAFE_HTTPS>>"
-HTTP_PLACEHOLDER = "<<SAFE_HTTP>>"
-QUOTE_KEYS_REGEX = r"(\w+)\s?:", r'"\1":' # wrap keys with double quotes
-QUOTE_VALUES_REGEX = r":\s?(?!(\d+|true|false))(\w+)", r':"\2"' # wrap values with double quotes, skip int or bool
+_HTTPS_PLACEHOLDER = "<<SAFE_HTTPS>>"
+_HTTP_PLACEHOLDER = "<<SAFE_HTTP>>"
+_QUOTE_KEYS_REGEX = r"(\w+)\s?:", r'"\1":' # wrap keys with double quotes
+_QUOTE_VALUES_REGEX = r":\s?(?!(\d+|true|false))(\w+)", r':"\2"' # wrap values with double quotes, skip int or bool
+_REPLACE_PAIRS = [
+ ("{'", '{"'),
+ ("'}", '"}'),
+ ("['", '["'),
+ ("']", '"]'),
+ (",'", ',"'),
+ ("':", '":'),
+ (", '", ', "'),
+ ("' :", '" :'),
+ ("',", '",'),
+ (": '", ': "'),
+]
-def scape_urls(js_text: str) -> str:
- return js_text.replace("https:", HTTPS_PLACEHOLDER).replace("http:", HTTP_PLACEHOLDER)
+def _escape_urls(js_text: str) -> str:
+ return js_text.replace("https:", _HTTPS_PLACEHOLDER).replace("http:", _HTTP_PLACEHOLDER)
-def recover_urls(js_text: str) -> str:
- return js_text.replace(HTTPS_PLACEHOLDER, "https:").replace(HTTP_PLACEHOLDER, "http:")
+def _recover_urls(js_text: str) -> str:
+ return js_text.replace(_HTTPS_PLACEHOLDER, "https:").replace(_HTTP_PLACEHOLDER, "http:")
def parse_js_vars(js_text: str, use_regex: bool = False) -> dict:
@@ -37,68 +49,51 @@ def parse_js_vars(js_text: str, use_regex: bool = False) -> dict:
def parse_obj(js_text: str, use_regex: bool = False) -> Any:
json_str = js_text.replace("\t", "").replace("\n", "").strip()
- json_str = replace_quotes(json_str)
+ json_str = _replace_quotes(json_str)
if use_regex:
- json_str = scape_urls(json_str)
- json_str = re.sub(*QUOTE_KEYS_REGEX, json_str)
- json_str = re.sub(*QUOTE_VALUES_REGEX, json_str)
- json_str = recover_urls(json_str)
+ json_str = _escape_urls(json_str)
+ json_str = re.sub(*_QUOTE_KEYS_REGEX, json_str)
+ json_str = re.sub(*_QUOTE_VALUES_REGEX, json_str)
+ json_str = _recover_urls(json_str)
result = json.loads(json_str)
- is_list = isinstance(result, list)
- if is_list:
+ is_dict = isinstance(result, dict)
+ if not is_dict:
result = {"data": result}
- clean_dict(result)
- if is_list:
+ _coerce_dict_values(result)
+ if not is_dict:
return result["data"]
return result
-def replace_quotes(js_text: str) -> str:
+def _replace_quotes(js_text: str) -> str:
# We can't just replace every single ' with " because it will brake if the json has english words like: it's
- replace_pairs = [
- ("{'", '{"'),
- ("'}", '"}'),
- ("['", '["'),
- ("']", '"]'),
- (",'", ',"'),
- ("':", '":'),
- (", '", ', "'),
- ("' :", '" :'),
- ("',", '",'),
- (": '", ': "'),
- ]
clean_js_text = js_text
- for old, new in replace_pairs:
+ for old, new in _REPLACE_PAIRS:
clean_js_text = clean_js_text.replace(old, new)
return clean_js_text
-def is_valid_key(key: str) -> bool:
- return not any(p in key for p in ("@", "m3u8"))
-
-
-def clean_dict(data: dict, *keys_to_clean) -> None:
- """Modifies dict in place"""
-
- for key in keys_to_clean:
- inner_dict = data.get(key)
- if inner_dict and isinstance(inner_dict, dict):
- data[key] = {k: v for k, v in inner_dict.items() if is_valid_key(k)}
-
+def _coerce_dict_values(data: dict[str, Any]) -> None:
for k, v in data.items():
if isinstance(v, dict):
continue
- data[k] = clean_value(v)
+ data[k] = _literal_value(v)
-def clean_value(value: list | str | int) -> list | str | int | None:
+def _literal_value(value: list | str | int | None) -> list[Any] | str | int | bool | None:
if isinstance(value, str):
value = value.removesuffix("'").removeprefix("'")
if value.isdigit():
return int(value)
+ if value == "undefined":
+ return None
+ if value == "true":
+ return True
+ if value == "false":
+ return False
return value
if isinstance(value, list):
- return [clean_value(v) for v in value]
+ return [_literal_value(v) for v in value]
return value