Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: support subtitles (VKvideo)
NTFSvolumefc682c6ef3eb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 922059f1b..5f9b96792 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- Support subfolders (Upfile)+- Support subtitles (VKVideo)### Fixeddiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex e95d1336f..1975f09da 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -645,11 +645,11 @@ class Crawler(HTTPMixin, HLSMixin, ABC):async def sniff_ext(self, thumbnail: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> str:try:- _, ext = self.get_filename_and_ext(thumbnail.name)+ return self.get_filename_and_ext(thumbnail.name).suffixexcept NoExtensionError:async with self.request(thumbnail, self._THUMB_HTTP_METHOD, headers=headers or {}) as resp:- _, ext = self.get_filename_and_ext(thumbnail.name, mime_type=resp.content_type)- return ext+ name = thumbnail.name or thumbnail.parent.name or "file"+ return self.get_filename_and_ext(name, mime_type=resp.content_type).suffixdef _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:return {diff --git a/cyberdrop_dl/crawlers/vk/types.py b/cyberdrop_dl/crawlers/vk/types.pyindex 96ea0dca4..2452fdfd4 100644--- a/cyberdrop_dl/crawlers/vk/types.py+++ b/cyberdrop_dl/crawlers/vk/types.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Self, overridefrom cyberdrop_dl.crawlers.vk import full_res_img, try_parse_urlfrom cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.mediaprops import ISO639Subtitlefrom cyberdrop_dl.utils import css, parse_urlfrom cyberdrop_dl.utils.dataclass import deserialize@@ -74,6 +75,7 @@ class Video(Owned):access_key: str | None = Nonedescription: str | None = Nonerestriction: str | None = None+ subs: tuple[ISO639Subtitle, ...] = ()@override@classmethod@@ -86,6 +88,15 @@ class Video(Owned):if restriction := video.get("content_restricted_message") or video.get("restriction", {}).get("text"):restriction = css.unescape(restriction)+ def subs():+ for sub in video.get("subtitles") or video.get("subs") or ():+ if url := sub.get("url"):+ yield ISO639Subtitle(+ url=url,+ name=sub.get("title") or sub.get("manifest_name"),+ lang_code=sub.get("lang") or "en",+ )+return deserialize(cls,video,@@ -93,6 +104,7 @@ class Video(Owned):formats=tuple(_extract_formats(video)),embed=video.get("files", {}).get("external"),restriction=restriction or None,+ subs=tuple(subs()),)@propertydiff --git a/cyberdrop_dl/crawlers/vk/vk.py b/cyberdrop_dl/crawlers/vk/vk.pyindex 3203a8085..8d0ea4cfe 100644--- a/cyberdrop_dl/crawlers/vk/vk.py+++ b/cyberdrop_dl/crawlers/vk/vk.py@@ -26,7 +26,7 @@ from cyberdrop_dl.crawlers.vk.types import (from cyberdrop_dl.exceptions import DownloadError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import b64_pad-from cyberdrop_dl.utils._url import remove_query_params+from cyberdrop_dl.utils._url import is_ip_address, remove_query_paramsfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:@@ -83,7 +83,20 @@ INNER_PAGES = {_PAGE_REGEX = rf"^(?P<type>{'|'.join(map(re.escape, _PAGES_TYPES))})(?P<owner_id>-?\d+)(?:_(?P<item_id>\d+))?"[email protected]_path_builder("path_qs")+def remove_tracking_params(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ return url.without_query_params("pl", "fromTrackCode")+++def remove_ip_params(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ return url.with_query(+ sorted(+ (k, v)+ for k, v in url.query.items()+ if k not in {"tkn", "urls"} and not is_ip_address(v) and not is_ip_address(k)+ )+ )++class VKBaseCrawler(Crawler, is_abc=True):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": (@@ -97,6 +110,23 @@ class VKBaseCrawler(Crawler, is_abc=True):FOLDER_DOMAIN: ClassVar[str] = "VK"DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} [{id}]"+ @staticmethod+ @override+ def __db_path__(url: AbsoluteHttpURL, /) -> str:+ url = remove_ip_params(remove_tracking_params(url))+ query = url.query+ match url.parts[1:]:+ case ["getVideoPreview" as thumb, *_]:+ return f"/{thumb}?id={url.query['id']}"+ case [] | [""] if (sub_id := query.get("subId")) and (vid := query.get("id")):+ return f"vksubtitle://{VKVideoCrawler.PRIMARY_URL.host}?vid={vid}&subId={sub_id}"+ case [slug] if re.match(_PAGE_REGEX, slug):+ return remove_query_params(url, keep=("list",)).path_qs+ case ["s", "v1", *__]:+ return url.path_qs+ case _:+ return str(url)+def __post_init__(self) -> None:for host in self.SUPPORTED_DOMAINS:self.update_cookies(@@ -108,9 +138,7 @@ class VKBaseCrawler(Crawler, is_abc=True):@classmethod@overridedef transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:- url = full_res_img(- super().transform_url(url).without_query_params("pl", "fromTrackCode")- ) # TODO: support playlists+ url = full_res_img(remove_tracking_params(super().transform_url(url))) # TODO: support playlistsmatch url.parts[1:]:case ["video_ext.php"] if (owner_id := url.query.get("oid")) and (vid := url.query.get("id")):@@ -146,14 +174,6 @@ class VKVideoCrawler(VKBaseCrawler, wiki_site="VKVideo"):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "vkvideo.ru", "vkvideo.com"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://vkvideo.com")- @staticmethod- @override- def __db_path__(url: AbsoluteHttpURL, /) -> str:- url = url.without_query_params("tkn")- if url.parts[1].startswith(("video", "clip", "photo")):- url = remove_query_params(url, keep=("list",))- return url.path_qs-async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case [owner, "clips" | "all" | "donut" as kind]:@@ -236,22 +256,24 @@ class VKVideoCrawler(VKBaseCrawler, wiki_site="VKVideo"):else:debrid_url = best_fmt.url+ filename = self.create_custom_filename(+ title,+ ext := ".mp4",+ file_id=video.slug,+ resolution=info.resolution if info else best_fmt.height,+ )await self.handle_file(scrape_item.url,scrape_item,title,- ext := ".mp4",+ ext,m3u8=m3u8,- custom_filename=self.create_custom_filename(- title,- ext,- file_id=video.slug,- resolution=info.resolution if info else best_fmt.height,- ),+ custom_filename=filename,thumbnail=video.thumb,debrid_link=debrid_url,metadata=video,)+ await self.handle_subs_v2(scrape_item, filename, video.subs)@URLConfig(allow_empty_path=True)diff --git a/cyberdrop_dl/utils/_url.py b/cyberdrop_dl/utils/_url.pyindex d5301a5e4..77b5a15cd 100644--- a/cyberdrop_dl/utils/_url.py+++ b/cyberdrop_dl/utils/_url.py@@ -100,3 +100,14 @@ def _is_absolute_http_url(url: yarl.URL) -> TypeIs[AbsoluteHttpURL]:def remove_query_params[T: yarl.URL](url: T, keep: tuple[str, ...]) -> T:return cast("T", url.with_query([(name, value) for name, value in url.query.items() if name in keep] or None))+++def is_ip_address(address: str) -> bool:+ import ipaddress++ try:+ ipaddress.ip_address(address)+ except ValueError:+ return False+ else:+ return True