Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove unused javascript parses
NTFSvolumea69cbdb4bb94
diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.pyindex c0cf6eb5b..ab3a08cd5 100644--- a/cyberdrop_dl/crawlers/youjizz.py+++ b/cyberdrop_dl/crawlers/youjizz.py@@ -1,57 +1,62 @@from __future__ import annotations+import dataclassesimport json-import re-from typing import TYPE_CHECKING, ClassVar, NamedTuple+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuplefrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils import css, javascript-from cyberdrop_dl.utils.logger import log_debug-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.utils import css, open_graph+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:+ from collections.abc import Iterable+from bs4 import BeautifulSoup- from yarl import URLfrom cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://www.youjizz.com/")-DEFAULT_QUALITY = "Auto"-RESOLUTIONS = ["4k", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"] # best to worst-DATE_PATTERN = re.compile(r"(\d+)\s*(weeks?|days?|hours?|minutes?|seconds?)", re.IGNORECASE)--JS_SELECTOR = "div#content > script:contains('var dataEncodings')"DATE_SELECTOR = "span.pretty-date"-class Format(NamedTuple):- resolution: str | None- link_str: str+class VideoSource(NamedTuple):+ resolution: int+ url: str-## TODO: convert to global dataclass with constructor from dict to use in multiple crawlers-class VideoInfo(dict): ...[email protected](frozen=True, slots=True)+class Video:+ date: str # Human date+ title: str+ best_src: VideoSourceclass YouJizzCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": ("/video/embed/<video_id>", "/video/<video_id>/...")}+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": (+ "/videos/embed/<video_id>",+ "/videos/<video_name>",+ )+ }PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "youjizz"FOLDER_DOMAIN: ClassVar[str] = "YouJizz"async def fetch(self, scrape_item: ScrapeItem) -> None:- if any(p in scrape_item.url.parts for p in ("videos", "embed")):- return await self.video(scrape_item)- raise ValueError+ match scrape_item.url.parts[1:]:+ case ["videos", "embed", video_id]:+ return await self.video(scrape_item, video_id)+ case ["videos", video_name]:+ video_id = video_name.rsplit("-", 1)[-1].removesuffix(".html")+ return await self.video(scrape_item, video_id)+ case _:+ raise ValueError@error_handling_wrapper- async def video(self, scrape_item: ScrapeItem) -> None:- video_id = get_video_id(scrape_item.url)+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:canonical_url = PRIMARY_URL / "videos" / "embed" / video_id-if await self.check_complete_from_referer(canonical_url):return@@ -59,51 +64,37 @@ class YouJizzCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)scrape_item.url = canonical_url- info = get_info(soup)- v_format = get_best_quality(info)- if not v_format:- raise ScrapeError(422)-- resolution, link_str = v_format- link = self.parse_url(link_str)- scrape_item.possible_datetime = self.parse_date(info["date"])+ video = _parse_video(soup)+ link = self.parse_url(video.best_src.url)+ scrape_item.possible_datetime = self.parse_date(video.date)filename, ext = self.get_filename_and_ext(link.name)- if ext == ".m3u8":- raise ScrapeError(422)- custom_filename = self.create_custom_filename(info["title"], ext, file_id=video_id, resolution=resolution)- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)---def get_video_id(url: URL) -> str:- if "embed" in url.parts:- embed_index = url.parts.index("embed")- return url.parts[embed_index + 1]-- video_id = url.parts[2].rsplit("-", 1)[-1]- return video_id.removesuffix(".html")---def get_info(soup: BeautifulSoup) -> VideoInfo:- info_js_script_text: str = css.select_one_get_text(soup, JS_SELECTOR)- info: dict[str, str | None | dict] = javascript.parse_js_vars(info_js_script_text)- info["title"] = css.get_attr(soup, "title").replace("\n", "").strip()- date_tag = soup.select_one(DATE_SELECTOR)- date_str: str | None = date_tag.text if date_tag else None- info["date"] = date_str.replace("(s)", "s").strip() if date_str else None- javascript.clean_dict(info, "stream_data")- log_debug(json.dumps(info, indent=4))- return VideoInfo(**info)---def get_best_quality(info_dict: dict) -> Format | None:- qualities: dict = info_dict["dataEncodings"]- for res in RESOLUTIONS:- avaliable_formats = [f for f in qualities if f["name"] == res]- for format_info in avaliable_formats:- link_str = format_info["filename"]- if "/_hls/" not in link_str:- return Format(res, link_str)- default_quality: dict = next((f for f in qualities if f["name"] == DEFAULT_QUALITY), {})- if default_quality:- default_link_str = default_quality.get("filename") or ""- return Format(None, default_link_str)+ custom_filename = self.create_custom_filename(+ video.title, ext, file_id=video_id, resolution=video.best_src.resolution+ )+ await self.handle_file(+ scrape_item.url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link+ )+++def _parse_video(soup: BeautifulSoup) -> Video:+ js_text = css.select_one_get_text(soup, JS_SELECTOR)+ encodings_text = get_text_between(js_text, "var dataEncodings =", "var encodings").strip().removesuffix(";")+ data_encodings = json.loads(encodings_text)+ return Video(+ title=open_graph.title(soup),+ date=css.select_one_get_text(soup, DATE_SELECTOR),+ best_src=_get_best_src(data_encodings),+ )+++def _get_best_src(data_encodings: list[dict[str, Any]]) -> VideoSource:+ def parse() -> Iterable[VideoSource]:+ for format_info in data_encodings:+ try:+ res = int(format_info["quality"])+ except ValueError:+ continue+ if "/_hls/" not in (link_str := format_info["filename"]):+ yield VideoSource(res, link_str)++ return max(parse())diff --git a/cyberdrop_dl/utils/javascript.py b/cyberdrop_dl/utils/javascript.pyindex 5ef1bc01e..bd1df790f 100644--- a/cyberdrop_dl/utils/javascript.py+++ b/cyberdrop_dl/utils/javascript.py@@ -4,18 +4,30 @@ import jsonimport refrom typing import Any-HTTPS_PLACEHOLDER = "<<SAFE_HTTPS>>"-HTTP_PLACEHOLDER = "<<SAFE_HTTP>>"-QUOTE_KEYS_REGEX = r"(\w+)\s?:", r'"\1":' # wrap keys with double quotes-QUOTE_VALUES_REGEX = r":\s?(?!(\d+|true|false))(\w+)", r':"\2"' # wrap values with double quotes, skip int or bool+_HTTPS_PLACEHOLDER = "<<SAFE_HTTPS>>"+_HTTP_PLACEHOLDER = "<<SAFE_HTTP>>"+_QUOTE_KEYS_REGEX = r"(\w+)\s?:", r'"\1":' # wrap keys with double quotes+_QUOTE_VALUES_REGEX = r":\s?(?!(\d+|true|false))(\w+)", r':"\2"' # wrap values with double quotes, skip int or bool+_REPLACE_PAIRS = [+ ("{'", '{"'),+ ("'}", '"}'),+ ("['", '["'),+ ("']", '"]'),+ (",'", ',"'),+ ("':", '":'),+ (", '", ', "'),+ ("' :", '" :'),+ ("',", '",'),+ (": '", ': "'),+]-def scape_urls(js_text: str) -> str:- return js_text.replace("https:", HTTPS_PLACEHOLDER).replace("http:", HTTP_PLACEHOLDER)+def _escape_urls(js_text: str) -> str:+ return js_text.replace("https:", _HTTPS_PLACEHOLDER).replace("http:", _HTTP_PLACEHOLDER)-def recover_urls(js_text: str) -> str:- return js_text.replace(HTTPS_PLACEHOLDER, "https:").replace(HTTP_PLACEHOLDER, "http:")+def _recover_urls(js_text: str) -> str:+ return js_text.replace(_HTTPS_PLACEHOLDER, "https:").replace(_HTTP_PLACEHOLDER, "http:")def parse_js_vars(js_text: str, use_regex: bool = False) -> dict:@@ -37,68 +49,51 @@ def parse_js_vars(js_text: str, use_regex: bool = False) -> dict:def parse_obj(js_text: str, use_regex: bool = False) -> Any:json_str = js_text.replace("\t", "").replace("\n", "").strip()- json_str = replace_quotes(json_str)+ json_str = _replace_quotes(json_str)if use_regex:- json_str = scape_urls(json_str)- json_str = re.sub(*QUOTE_KEYS_REGEX, json_str)- json_str = re.sub(*QUOTE_VALUES_REGEX, json_str)- json_str = recover_urls(json_str)+ json_str = _escape_urls(json_str)+ json_str = re.sub(*_QUOTE_KEYS_REGEX, json_str)+ json_str = re.sub(*_QUOTE_VALUES_REGEX, json_str)+ json_str = _recover_urls(json_str)result = json.loads(json_str)- is_list = isinstance(result, list)- if is_list:+ is_dict = isinstance(result, dict)+ if not is_dict:result = {"data": result}- clean_dict(result)- if is_list:+ _coerce_dict_values(result)+ if not is_dict:return result["data"]return result-def replace_quotes(js_text: str) -> str:+def _replace_quotes(js_text: str) -> str:# We can't just replace every single ' with " because it will brake if the json has english words like: it's- replace_pairs = [- ("{'", '{"'),- ("'}", '"}'),- ("['", '["'),- ("']", '"]'),- (",'", ',"'),- ("':", '":'),- (", '", ', "'),- ("' :", '" :'),- ("',", '",'),- (": '", ': "'),- ]clean_js_text = js_text- for old, new in replace_pairs:+ for old, new in _REPLACE_PAIRS:clean_js_text = clean_js_text.replace(old, new)return clean_js_text-def is_valid_key(key: str) -> bool:- return not any(p in key for p in ("@", "m3u8"))---def clean_dict(data: dict, *keys_to_clean) -> None:- """Modifies dict in place"""-- for key in keys_to_clean:- inner_dict = data.get(key)- if inner_dict and isinstance(inner_dict, dict):- data[key] = {k: v for k, v in inner_dict.items() if is_valid_key(k)}-+def _coerce_dict_values(data: dict[str, Any]) -> None:for k, v in data.items():if isinstance(v, dict):continue- data[k] = clean_value(v)+ data[k] = _literal_value(v)-def clean_value(value: list | str | int) -> list | str | int | None:+def _literal_value(value: list | str | int | None) -> list[Any] | str | int | bool | None:if isinstance(value, str):value = value.removesuffix("'").removeprefix("'")if value.isdigit():return int(value)+ if value == "undefined":+ return None+ if value == "true":+ return True+ if value == "false":+ return Falsereturn valueif isinstance(value, list):- return [clean_value(v) for v in value]+ return [_literal_value(v) for v in value]return value