Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: support subtitles (VKvideo)

NTFSvolumefc682c6ef3eb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 922059f1b..5f9b96792 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Support subfolders (Upfile)
+- Support subtitles (VKVideo)
### Fixed
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index e95d1336f..1975f09da 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -645,11 +645,11 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
async def sniff_ext(self, thumbnail: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> str:
try:
- _, ext = self.get_filename_and_ext(thumbnail.name)
+ return self.get_filename_and_ext(thumbnail.name).suffix
except NoExtensionError:
async with self.request(thumbnail, self._THUMB_HTTP_METHOD, headers=headers or {}) as resp:
- _, ext = self.get_filename_and_ext(thumbnail.name, mime_type=resp.content_type)
- return ext
+ name = thumbnail.name or thumbnail.parent.name or "file"
+ return self.get_filename_and_ext(name, mime_type=resp.content_type).suffix
def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
return {
diff --git a/cyberdrop_dl/crawlers/vk/types.py b/cyberdrop_dl/crawlers/vk/types.py
index 96ea0dca4..2452fdfd4 100644
--- a/cyberdrop_dl/crawlers/vk/types.py
+++ b/cyberdrop_dl/crawlers/vk/types.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Self, override
from cyberdrop_dl.crawlers.vk import full_res_img, try_parse_url
from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.mediaprops import ISO639Subtitle
from cyberdrop_dl.utils import css, parse_url
from cyberdrop_dl.utils.dataclass import deserialize
@@ -74,6 +75,7 @@ class Video(Owned):
access_key: str | None = None
description: str | None = None
restriction: str | None = None
+ subs: tuple[ISO639Subtitle, ...] = ()
@override
@classmethod
@@ -86,6 +88,15 @@ class Video(Owned):
if restriction := video.get("content_restricted_message") or video.get("restriction", {}).get("text"):
restriction = css.unescape(restriction)
+ def subs():
+ for sub in video.get("subtitles") or video.get("subs") or ():
+ if url := sub.get("url"):
+ yield ISO639Subtitle(
+ url=url,
+ name=sub.get("title") or sub.get("manifest_name"),
+ lang_code=sub.get("lang") or "en",
+ )
+
return deserialize(
cls,
video,
@@ -93,6 +104,7 @@ class Video(Owned):
formats=tuple(_extract_formats(video)),
embed=video.get("files", {}).get("external"),
restriction=restriction or None,
+ subs=tuple(subs()),
)
@property
diff --git a/cyberdrop_dl/crawlers/vk/vk.py b/cyberdrop_dl/crawlers/vk/vk.py
index 3203a8085..8d0ea4cfe 100644
--- a/cyberdrop_dl/crawlers/vk/vk.py
+++ b/cyberdrop_dl/crawlers/vk/vk.py
@@ -26,7 +26,7 @@ from cyberdrop_dl.crawlers.vk.types import (
from cyberdrop_dl.exceptions import DownloadError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import b64_pad
-from cyberdrop_dl.utils._url import remove_query_params
+from cyberdrop_dl.utils._url import is_ip_address, remove_query_params
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
@@ -83,7 +83,20 @@ INNER_PAGES = {
_PAGE_REGEX = rf"^(?P<type>{'|'.join(map(re.escape, _PAGES_TYPES))})(?P<owner_id>-?\d+)(?:_(?P<item_id>\d+))?"
[email protected]_path_builder("path_qs")
+def remove_tracking_params(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ return url.without_query_params("pl", "fromTrackCode")
+
+
+def remove_ip_params(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ return url.with_query(
+ sorted(
+ (k, v)
+ for k, v in url.query.items()
+ if k not in {"tkn", "urls"} and not is_ip_address(v) and not is_ip_address(k)
+ )
+ )
+
+
class VKBaseCrawler(Crawler, is_abc=True):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Video": (
@@ -97,6 +110,23 @@ class VKBaseCrawler(Crawler, is_abc=True):
FOLDER_DOMAIN: ClassVar[str] = "VK"
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} [{id}]"
+ @staticmethod
+ @override
+ def __db_path__(url: AbsoluteHttpURL, /) -> str:
+ url = remove_ip_params(remove_tracking_params(url))
+ query = url.query
+ match url.parts[1:]:
+ case ["getVideoPreview" as thumb, *_]:
+ return f"/{thumb}?id={url.query['id']}"
+ case [] | [""] if (sub_id := query.get("subId")) and (vid := query.get("id")):
+ return f"vksubtitle://{VKVideoCrawler.PRIMARY_URL.host}?vid={vid}&subId={sub_id}"
+ case [slug] if re.match(_PAGE_REGEX, slug):
+ return remove_query_params(url, keep=("list",)).path_qs
+ case ["s", "v1", *__]:
+ return url.path_qs
+ case _:
+ return str(url)
+
def __post_init__(self) -> None:
for host in self.SUPPORTED_DOMAINS:
self.update_cookies(
@@ -108,9 +138,7 @@ class VKBaseCrawler(Crawler, is_abc=True):
@classmethod
@override
def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- url = full_res_img(
- super().transform_url(url).without_query_params("pl", "fromTrackCode")
- ) # TODO: support playlists
+ url = full_res_img(remove_tracking_params(super().transform_url(url))) # TODO: support playlists
match url.parts[1:]:
case ["video_ext.php"] if (owner_id := url.query.get("oid")) and (vid := url.query.get("id")):
@@ -146,14 +174,6 @@ class VKVideoCrawler(VKBaseCrawler, wiki_site="VKVideo"):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "vkvideo.ru", "vkvideo.com"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://vkvideo.com")
- @staticmethod
- @override
- def __db_path__(url: AbsoluteHttpURL, /) -> str:
- url = url.without_query_params("tkn")
- if url.parts[1].startswith(("video", "clip", "photo")):
- url = remove_query_params(url, keep=("list",))
- return url.path_qs
-
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [owner, "clips" | "all" | "donut" as kind]:
@@ -236,22 +256,24 @@ class VKVideoCrawler(VKBaseCrawler, wiki_site="VKVideo"):
else:
debrid_url = best_fmt.url
+ filename = self.create_custom_filename(
+ title,
+ ext := ".mp4",
+ file_id=video.slug,
+ resolution=info.resolution if info else best_fmt.height,
+ )
await self.handle_file(
scrape_item.url,
scrape_item,
title,
- ext := ".mp4",
+ ext,
m3u8=m3u8,
- custom_filename=self.create_custom_filename(
- title,
- ext,
- file_id=video.slug,
- resolution=info.resolution if info else best_fmt.height,
- ),
+ custom_filename=filename,
thumbnail=video.thumb,
debrid_link=debrid_url,
metadata=video,
)
+ await self.handle_subs_v2(scrape_item, filename, video.subs)
@URLConfig(allow_empty_path=True)
diff --git a/cyberdrop_dl/utils/_url.py b/cyberdrop_dl/utils/_url.py
index d5301a5e4..77b5a15cd 100644
--- a/cyberdrop_dl/utils/_url.py
+++ b/cyberdrop_dl/utils/_url.py
@@ -100,3 +100,14 @@ def _is_absolute_http_url(url: yarl.URL) -> TypeIs[AbsoluteHttpURL]:
def remove_query_params[T: yarl.URL](url: T, keep: tuple[str, ...]) -> T:
return cast("T", url.with_query([(name, value) for name, value in url.query.items() if name in keep] or None))
+
+
+def is_ip_address(address: str) -> bool:
+ import ipaddress
+
+ try:
+ ipaddress.ip_address(address)
+ except ValueError:
+ return False
+ else:
+ return True