Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: parse 1440p and 4K videos (KVS) (#1314)
NTFSvolumebfbf62d3d345
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 08b0999cd..1d334b689 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -2,31 +2,35 @@from __future__ import annotations+import dataclassesimport re-from typing import TYPE_CHECKING, ClassVar, NamedTuple+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.data_structures import AbsoluteHttpURL, Resolutionfrom cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils import css-from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, open_graph+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_urlif TYPE_CHECKING:+ from collections.abc import Sequence+from bs4 import BeautifulSoup- from cyberdrop_dl.data_structures.url_objects import ScrapeItem+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem-class Video(NamedTuple):[email protected](slots=True)+class KVSVideo:id: str- res: str | None+ title: strurl: AbsoluteHttpURL- title: str = ""+ resolution: Resolutionclass Selectors:UNAUTHORIZED = "div.video-holder:contains('This video is a private video')"- FLASHVARS = "script:contains('var flashvars')"+ FLASHVARS = "script:contains('video_title:')"USER_NAME = "div.headline > h2"ALBUM_NAME = "div.headline > h1"ALBUM_PICTURES = "div.album-list > a"@@ -121,9 +125,9 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):returnsoup = await self.request_soup(scrape_item.url)- video = self.get_video_info(soup)+ video = extract_kvs_video(self, soup)filename, ext = self.get_filename_and_ext(video.url.name)- custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.res)+ custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.resolution)date_str = css.select_one_get_text(soup, _SELECTORS.DATE).split(":", 1)[-1].strip()scrape_item.possible_datetime = self.parse_date(date_str)await self.handle_file(@@ -152,41 +156,48 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):filename, ext = self.get_filename_and_ext(url.name)await self.handle_file(url, scrape_item, filename, ext)- def get_video_info(self, soup: BeautifulSoup) -> Video:- if soup.select_one(_SELECTORS.UNAUTHORIZED):- raise ScrapeError(401)- video = parse_flash_vars(script.text) if (script := soup.select_one(_SELECTORS.FLASHVARS)) else None- if not video:- raise ScrapeError(404)- return video++def extract_kvs_video(cls: Crawler, soup: BeautifulSoup) -> KVSVideo:+ if soup.select_one(_SELECTORS.UNAUTHORIZED):+ raise ScrapeError(401, "Private video")++ script = css.select_one_get_text(soup, _SELECTORS.FLASHVARS)+ video = _parse_video_vars(script)+ if not video.title:+ title = open_graph.get_title(soup) or css.page_title(soup)+ assert title+ video.title = css.sanitize_page_title(title, cls.DOMAIN)+ return video# URL de-obfuscation code for kvs, adapted from yt-dlp# https://github.com/yt-dlp/yt-dlp/blob/e1847535e28788414a25546a45bebcada2f34558/yt_dlp/extractor/generic.py-HASH_LENGTH = 32-VIDEO_RESOLUTION_PATTERN = re.compile(r"video_url_text:\s*'([^']+)'")-VIDEO_INFO_PATTTERN = re.compile(- r"video_id:\s*'(?P<video_id>[^']+)'[^}]*?"- r"video_title:\s*'(?P<video_title>[^']+)'[^}]*?"- r"license_code:\s*'(?P<license_code>[^']+)'[^}]*?"- r"video_url:\s*'(?P<video_url>[^']+)'[^}]*?"-)+_HASH_LENGTH = 32+_match_video_url_keys = re.compile(r"^video_(?:url|alt_url\d*)$").match+_find_flashvars = re.compile(r"(\w+):\s*'([^']*)'").findall-def parse_flash_vars(flashvars: str) -> Video | None:- if match_id := VIDEO_INFO_PATTTERN.search(flashvars):- video_id, title, license_code, url_str = match_id.groups()- real_url = get_real_url(url_str, license_code)- if match_res := VIDEO_RESOLUTION_PATTERN.search(flashvars):- resolution = match_res.group(1)- else:- resolution = None- return Video(video_id, resolution, real_url, title)+def _parse_video_vars(video_vars: str) -> KVSVideo:+ flashvars: dict[str, str] = dict(_find_flashvars(video_vars))+ url_keys = filter(_match_video_url_keys, flashvars.keys())+ license_token = _get_license_token(flashvars["license_code"])++ def get_formats():+ for key in url_keys:+ url_str = flashvars[key]+ if "/get_file/" not in url_str:+ continue+ resolution = Resolution.parse(flashvars[f"{key}_text"])+ url = _deobfuscate_url(url_str, license_token)+ yield resolution, url++ resolution, url = max(get_formats())+ return KVSVideo(flashvars["video_id"], flashvars["video_title"], url, resolution)-def get_license_token(license_code: str) -> list[int]:+def _get_license_token(license_code: str) -> tuple[int, ...]:license_code = license_code.removeprefix("$")license_values = [int(char) for char in license_code]modlicense = license_code.replace("0", "1")@@ -195,31 +206,30 @@ def get_license_token(license_code: str) -> list[int]:backhalf = int(modlicense[middle:])modlicense = str(4 * abs(fronthalf - backhalf))[: middle + 1]- return [+ return tuple((license_values[index + offset] + current) % 10for index, current in enumerate(map(int, modlicense))for offset in range(4)- ]+ )-def get_real_url(video_url_str: str, license_code: str) -> AbsoluteHttpURL:- if not video_url_str.startswith("function/0/"):- return AbsoluteHttpURL(video_url_str) # not obfuscated+def _deobfuscate_url(video_url_str: str, license_token: Sequence[int]) -> AbsoluteHttpURL:+ raw_url_str = video_url_str.removeprefix("function/0/")+ url = parse_url(raw_url_str)+ is_obfuscated = raw_url_str != video_url_str+ if not is_obfuscated:+ return url- parsed_url = AbsoluteHttpURL(video_url_str.removeprefix("function/0/"))- license_token = get_license_token(license_code)- hash, tail = parsed_url.parts[3][:HASH_LENGTH], parsed_url.parts[3][HASH_LENGTH:]- indices = list(range(HASH_LENGTH))+ hash, tail = url.parts[3][:_HASH_LENGTH], url.parts[3][_HASH_LENGTH:]+ indices = list(range(_HASH_LENGTH))# Swap indices of hash according to the destination calculated from the license tokenaccum = 0- for src in reversed(range(HASH_LENGTH)):+ for src in reversed(range(_HASH_LENGTH)):accum += license_token[src]- dest = (src + accum) % HASH_LENGTH+ dest = (src + accum) % _HASH_LENGTHindices[src], indices[dest] = indices[dest], indices[src]- new_parts = list(parsed_url.parts)- if not parsed_url.name:- _ = new_parts.pop()+ new_parts = list(url.parts)new_parts[3] = "".join(hash[index] for index in indices) + tail- return parsed_url.with_path("/".join(new_parts[1:]), keep_query=True, keep_fragment=True)+ return url.with_path("/".join(new_parts[1:]), keep_query=True, keep_fragment=True)diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.pyindex ef03d8387..135c33763 100644--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py@@ -2,18 +2,15 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler, Video+from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawlerfrom cyberdrop_dl.crawlers.porntrex import PorntrexCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.utils import css, open_graph+from cyberdrop_dl.utils import cssif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.crawlers.crawler import SupportedPaths-PRIMARY_URL = AbsoluteHttpURL("https://www.camwhores.tv")LAST_PAGE_SELECTOR = "div.pagination-holder li.page"@@ -26,7 +23,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):"Members": "/members/<member_id>",}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.camwhores.tv")DOMAIN: ClassVar[str] = "camwhores.tv"async def fetch(self, scrape_item: ScrapeItem) -> None:@@ -35,10 +32,6 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):scrape_item.url = scrape_item.url / ""await super().fetch(scrape_item)- def get_video_info(self, soup: BeautifulSoup) -> Video:- video = super().get_video_info(soup)- return video._replace(title=open_graph.title(soup))-def parse_url(self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None) -> AbsoluteHttpURL:diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.pyindex c3cce178a..49766f2c1 100644--- a/cyberdrop_dl/crawlers/pimp_bunny.py+++ b/cyberdrop_dl/crawlers/pimp_bunny.py@@ -1,20 +1,15 @@from __future__ import annotationsimport dataclasses-import refrom typing import TYPE_CHECKING, Any, ClassVar, Final+from cyberdrop_dl.crawlers._kvs import extract_kvs_videofrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures import AbsoluteHttpURL, Resolution-from cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils import css, open_graph-from cyberdrop_dl.utils.utilities import error_handling_wrapper, filter_query, parse_url+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.utilities import error_handling_wrapper, filter_queryif TYPE_CHECKING:- from collections.abc import Sequence-- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -124,7 +119,7 @@ class PimpBunnyCrawler(Crawler):returnsoup = await self.request_soup(scrape_item.url)- video = self.extract_kvs_video(soup)+ video = extract_kvs_video(self, soup)custom_filename = self.create_custom_filename(video.title, video.url.suffix, file_id=video.id, resolution=video.resolution)@@ -150,77 +145,3 @@ class PimpBunnyCrawler(Crawler):for _, image in self.iter_tags(soup, Selector.ITEM):self.create_task(self.direct_file(scrape_item, image))scrape_item.add_children()-- @classmethod- def extract_kvs_video(cls, soup: BeautifulSoup) -> KVSVideo:- if soup.select_one(Selector.UNAUTHORIZED):- raise ScrapeError(401, "Private video")-- script = css.select_one_get_text(soup, Selector.VIDEO_VARS)- video = _parse_video_vars(script)- if not video.title:- title = open_graph.get_title(soup) or css.page_title(soup)- assert title- video.title = css.sanitize_page_title(title, cls.DOMAIN)- return video---_HASH_LENGTH = 32-_match_video_url_keys = re.compile(r"^video_(?:url|alt_url\d*)$").match-_find_flashvars = re.compile(r"(\w+):\s*'([^']*)'").findall---def _parse_video_vars(video_vars: str) -> KVSVideo:- flashvars: dict[str, str] = dict(_find_flashvars(video_vars))- url_keys = filter(_match_video_url_keys, flashvars.keys())- license_token = _get_license_token(flashvars["license_code"])-- def get_formats():- for key in url_keys:- url_str = flashvars[key]- if "/get_file/" not in url_str:- continue- resolution = Resolution.parse(flashvars[f"{key}_text"])- url = _deobfuscate_url(url_str, license_token)- yield resolution, url-- resolution, url = max(get_formats())- return KVSVideo(flashvars["video_id"], flashvars["video_title"], url, resolution)---def _deobfuscate_url(video_url_str: str, license_token: Sequence[int]) -> AbsoluteHttpURL:- raw_url_str = video_url_str.removeprefix("function/0/")- url = parse_url(raw_url_str)- is_obfuscated = raw_url_str != video_url_str- if not is_obfuscated:- return url-- hash, tail = url.parts[3][:_HASH_LENGTH], url.parts[3][_HASH_LENGTH:]- indices = list(range(_HASH_LENGTH))-- # Swap indices of hash according to the destination calculated from the license token- accum = 0- for src in reversed(range(_HASH_LENGTH)):- accum += license_token[src]- dest = (src + accum) % _HASH_LENGTH- indices[src], indices[dest] = indices[dest], indices[src]-- new_parts = list(url.parts)- new_parts[3] = "".join(hash[index] for index in indices) + tail- return url.with_path("/".join(new_parts[1:]), keep_query=True, keep_fragment=True)---def _get_license_token(license_code: str) -> tuple[int, ...]:- license_code = license_code.removeprefix("$")- license_values = [int(char) for char in license_code]- modlicense = license_code.replace("0", "1")- middle = len(modlicense) // 2- fronthalf = int(modlicense[: middle + 1])- backhalf = int(modlicense[middle:])- modlicense = str(4 * abs(fronthalf - backhalf))[: middle + 1]-- return tuple(- (license_values[index + offset] + current) % 10- for index, current in enumerate(map(int, modlicense))- for offset in range(4)- )diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex 172a671cb..75c18a6cf 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -1,28 +1,19 @@from __future__ import annotationsimport itertools-import re-from typing import TYPE_CHECKING, ClassVar, NamedTuple+from typing import TYPE_CHECKING, ClassVar+from cyberdrop_dl.crawlers._kvs import extract_kvs_videofrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import css-from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between+from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem-class Video(NamedTuple):- id: str- title: str- url: str- res: str--class Selectors:VIDEO_JS = "div.video-holder script:contains('var flashvars')"NEXT_PAGE = "div#list_videos_videos_pagination li.next"@@ -39,7 +30,6 @@ class Selectors:VIDEOS_OR_ALBUMS = f"{VIDEOS}, {ALBUMS}"-VIDEO_INFO_FIELDS_PATTERN = re.compile(r"(\w+):\s*'([^']*)'")COLLECTION_PARTS = "tags", "categories", "models", "playlists", "search", "members"TITLE_TRASH = "Free HD ", "Most Relevant ", "New ", "Videos", "Porn", "for:", "New Videos", "Tagged with"_SELECTORS = Selectors()@@ -105,13 +95,14 @@ class PorntrexCrawler(Crawler):soup = await self.request_soup(scrape_item.url)- video = get_video_info(soup)- link = self.parse_url(video.url)- filename, ext = self.get_filename_and_ext(link.name)+ video = extract_kvs_video(self, soup)+ link = video.urlscrape_item.url = canonical_url- custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.res)+ custom_filename = self.create_custom_filename(+ video.title, link.suffix, file_id=video.id, resolution=video.resolution+ )await self.handle_file(- canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link+ canonical_url, scrape_item, link.name, link.suffix, custom_filename=custom_filename, debrid_link=link)@error_handling_wrapper@@ -196,39 +187,3 @@ class PorntrexCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):self.create_task(self.run(new_scrape_item))---def get_video_info(soup: BeautifulSoup) -> Video:- script = soup.select_one(_SELECTORS.VIDEO_JS)- if not script:- raise ScrapeError(404)-- flashvars = get_text_between(script.text, "var flashvars =", "var player_obj =")-- def extract_resolution(res_text):- match = re.search(r"(\d+)", res_text)- return int(match.group(1)) if match else 0-- video_info: dict[str, str] = dict(VIDEO_INFO_FIELDS_PATTERN.findall(flashvars))- if video_info:- resolutions: list[tuple[str, str]] = []- if "video_url" in video_info and "video_url_text" in video_info:- resolutions.append((video_info["video_url_text"], video_info["video_url"]))-- for key in video_info:- if (- key.startswith("video_alt_url")- and not key.endswith("_hd")- and not key.endswith("_4k")- and not key.endswith("_text")- ):- text_key = f"{key}_text"- if text_key in video_info:- resolutions.append((video_info[text_key], video_info[key]))-- if not resolutions:- resolutions.append((video_info["video_url"], ""))-- best = max(resolutions, key=lambda x: extract_resolution(x[0]))- return Video(video_info["video_id"], video_info["video_title"], best[1].strip("/"), best[0].split()[0])- raise ScrapeError(404)diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.pyindex b01f7b8ef..452efcd85 100644--- a/cyberdrop_dl/crawlers/thisvid.py+++ b/cyberdrop_dl/crawlers/thisvid.py@@ -1,24 +1,12 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import ClassVar-from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler, Video+from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css--if TYPE_CHECKING:- from bs4 import BeautifulSoup---PRIMARY_URL = AbsoluteHttpURL("https://thisvid.com")class ThisVidCrawler(KernelVideoSharingCrawler):- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://thisvid.com")DOMAIN: ClassVar[str] = "thisvid"FOLDER_DOMAIN: ClassVar[str] = "ThisVid"-- def get_video_info(self, soup: BeautifulSoup) -> Video:- video = super().get_video_info(soup)- title = css.select_one_get_text(soup, "title").split("- ThisVid.com")[0].strip()- return video._replace(title=title)