Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add odnoklassniki (ok.ru) support (#1213)

NTFSvolumefe227ef11279
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 47e8f6a98..bcef05823 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -128,6 +128,14 @@ class DownloadClient:
api_key = gofile_cookies.get("accountToken", "")
if api_key:
download_headers["Authorization"] = f"Bearer {api_key.value}" # type: ignore
+ elif domain == "odnoklassniki":
+ # TODO: Add "headers" attribute to MediaItem to use custom headers for downloads
+ download_headers |= {
+ "Accept-Language": "en-gb, en;q=0.8",
+ "User-Agent": "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36",
+ "Referer": "https://m.ok.ru/",
+ "Origin": "https://m.ok.ru",
+ }
return download_headers
@limiter
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 86263e438..c34e2ff85 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -62,6 +62,7 @@ from .nekohouse import NekohouseCrawler
from .nhentai import NHentaiCrawler
from .noodle_magazine import NoodleMagazineCrawler
from .nudostartv import NudoStarTVCrawler
+from .odnoklassniki import OdnoklassnikiCrawler
from .omegascans import OmegaScansCrawler
from .onedrive import OneDriveCrawler
from .pimpandhost import PimpAndHostCrawler
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index ac7e75fb4..97c90f007 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -682,8 +682,7 @@ class Crawler(ABC):
only_truncate_stem: bool = True,
) -> str:
calling_args = {name: value for name, value in locals().items() if value is not None and name not in ("self",)}
- clean_name = sanitize_filename(Path(name).as_posix().replace("/", "-")) # remove OS separators (if any)
- stem = Path(clean_name).stem.removesuffix(".") # remove extensions (if any)
+ stem = sanitize_filename(Path(name).as_posix().replace("/", "-")) # remove OS separators (if any)
extra_info: list[str] = []
if _placeholder_config.include_file_id and file_id:
diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.py
new file mode 100644
index 000000000..185954347
--- /dev/null
+++ b/cyberdrop_dl/crawlers/odnoklassniki.py
@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from aiolimiter import AsyncLimiter
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.utils import css, json
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
+
+if TYPE_CHECKING:
+ from bs4 import BeautifulSoup
+
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+_find_video_ids = re.compile("/video/(\\d+)").finditer
+
+_HEADERS = {
+ "Accept-Language": "en-gb, en;q=0.8",
+ "Referer": "https://ok.ru/",
+ "Origin": "https://ok.ru",
+}
+
+_MOBILE_HEADERS = _HEADERS | {
+ "User-Agent": "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36",
+ "Referer": "https://m.ok.ru/",
+ "Origin": "https://m.ok.ru",
+}
+
+
+class VideoProvider:
+ # This site also embeds videos from other sources as their own
+ OK_RU = "UPLOADED_ODKL"
+ YOUTUBE = "USER_YOUTUBE"
+ OG = "OPEN_GRAPH"
+ LIVESTREAM = "LIVE_TV_APP"
+
+
+class Selector:
+ VIDEO_DELETED = "a:contains('видео') div.empty"
+ UNAUTHORIZED = "div:contains('Access to this video is restricted')"
+ GEO_BLOCKED = "div:contains('This video is not available in your region')"
+ CHANNEL_NAME = ".album-info_name"
+ CHANNEL_HASH = "script:contains('gwtHash:')"
+
+ CHANNEL_LAST_ELEMENT = css.CssAttributeSelector("[data-last-element]", "data-last-element")
+ FLASHVARS = css.CssAttributeSelector("[data-options*='flashvars']", "data-options")
+
+
+class OdnoklassnikiCrawler(Crawler):
+ SUPPORTED_DOMAINS = "ok.ru", "odnoklassniki.ru"
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": "/video/<video_id>",
+ "Channel": (
+ "/video/c<channel_id>",
+ "/profile/<username>/c<channel_id>",
+ ),
+ }
+ PRIMARY_URL = AbsoluteHttpURL("https://ok.ru")
+ DOMAIN = "odnoklassniki"
+ FOLDER_DOMAIN = "ok.ru"
+
+ def __post_init__(self) -> None:
+ self.request_limiter = AsyncLimiter(3, 10)
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["video", id_]:
+ if id_.startswith("c"):
+ return await self.channel(scrape_item, id_)
+ return await self.video(scrape_item, id_)
+ case ["profile", _, channel] if channel.startswith("c"):
+ return await self.channel(scrape_item, channel)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def channel(self, scrape_item: ScrapeItem, channel_str: str):
+ async with self.request_limiter:
+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url, _HEADERS)
+
+ channel_id = channel_str.removeprefix("c")
+ gwt_hash = get_text_between(css.select_one_get_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')
+ last_element_id = css.select_one_get_attr_or_none(soup, *Selector.CHANNEL_LAST_ELEMENT)
+ name = css.select_one_get_text(soup, Selector.CHANNEL_NAME)
+ scrape_item.setup_as_album(self.create_title(name, channel_id), album_id=channel_id)
+
+ page_url = (self.PRIMARY_URL / "video" / channel_str).with_query(
+ {
+ "st.cmd": "anonymVideo",
+ "st.m": "ALBUM",
+ "st.ft": "album",
+ "st.aid": channel_str,
+ "cmd": "VideoAlbumBlock",
+ }
+ )
+ seen: set[str] = set()
+ content = str(soup)
+ page = 1
+ while True:
+ page_had_new_videos = False
+ for match in _find_video_ids(content):
+ if (video_path := match.group()) not in seen:
+ seen.add(video_path)
+ page_had_new_videos = True
+ video_url = self.PRIMARY_URL.with_path(video_path)
+ new_scrape_item = scrape_item.create_child(video_url)
+ self.create_task(self.run(new_scrape_item))
+ scrape_item.add_children()
+
+ if not page_had_new_videos or not last_element_id:
+ break
+
+ page += 1
+ async with self.request_limiter:
+ resp = await self.client._post_data(
+ self.DOMAIN,
+ page_url,
+ _HEADERS,
+ data={
+ "fetch": "false",
+ "st.page": page,
+ "st.lastelem": last_element_id,
+ "gwt.requested": gwt_hash,
+ },
+ )
+ last_element_id = resp.headers.get("lastelem")
+ content = await resp.text()
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem, video_id: str):
+ mobile_url = AbsoluteHttpURL(f"https://m.ok.ru/video/{video_id}")
+ async with self.request_limiter:
+ soup = await self.client.get_soup(self.DOMAIN, mobile_url, _MOBILE_HEADERS)
+
+ _check_video_is_available(soup)
+ metadata: dict[str, Any] = json.loads(Selector.FLASHVARS(soup))["flashvars"]["metadata"]
+
+ if (provider := metadata["provider"]) != VideoProvider.OK_RU:
+ raise ScrapeError(422, f"Unsupported provider: {provider}")
+
+ resolution, src = _get_best_src(metadata)
+ cdn_url = self.parse_url(src)
+ # downloads may fail if we have cdn cookies
+ self.client.client_manager.cookies.clear_domain(cdn_url.host)
+ json_ld = css.get_json_ld(soup)
+ title: str = metadata["movie"].get("title") or json_ld["name"]
+ scrape_item.possible_datetime = self.parse_iso_date(json_ld["uploadDate"])
+ filename = self.create_custom_filename(title, ".mp4", file_id=video_id, resolution=resolution)
+ await self.handle_file(
+ mobile_url, scrape_item, video_id + ".mp4", custom_filename=filename, debrid_link=cdn_url
+ )
+
+
+def _get_best_src(metadata: dict[str, Any]) -> tuple[int, str]:
+ def parse():
+ for video in metadata["videos"]:
+ resolution = {"full": 1080, "hd": 720, "sd": 480, "low": 360, "lowest": 240, "mobile": 144}.get(
+ video["name"], 0
+ )
+ yield resolution, video["url"]
+
+ return max(parse())
+
+
+def _check_video_is_available(soup: BeautifulSoup) -> None:
+ if error := soup.select_one(Selector.VIDEO_DELETED):
+ raise ScrapeError(404, css.get_text(error))
+
+ if soup.select_one(Selector.UNAUTHORIZED):
+ raise ScrapeError(503)
+
+ if soup.select_one(Selector.GEO_BLOCKED):
+ raise ScrapeError(403)
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 1d50ca5e3..66df8cb9c 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -153,7 +153,10 @@ def get_json_ld(soup: Tag, /, contains: str | None = None) -> dict[str, Any]:
if contains:
selector += f":contains('{contains}')"
- ld_json = json.loads(select_one_get_text(soup, selector))
+ ld_json = json.loads(select_one_get_text(soup, selector)) or {}
+ if isinstance(ld_json, list):
+ return ld_json[0]
+
return ld_json
diff --git a/tests/crawlers/test_cases/odnoklassniki.py b/tests/crawlers/test_cases/odnoklassniki.py
new file mode 100644
index 000000000..6d1756d87
--- /dev/null
+++ b/tests/crawlers/test_cases/odnoklassniki.py
@@ -0,0 +1,43 @@
+DOMAIN = "odnoklassniki"
+TEST_CASES = [
+ (
+ "https://ok.ru/video/1416667269705",
+ [
+ {
+ "url": "https://m.ok.ru/video/1416667269705",
+ "filename": "73 Réplika_x264 [1416667269705][480p].mp4",
+ "original_filename": "1416667269705.mp4",
+ "referer": "https://ok.ru/video/1416667269705",
+ "album_id": None,
+ "datetime": 1566236730,
+ }
+ ],
+ ),
+ (
+ "https://ok.ru/video/9469624191496",
+ [
+ {
+ "url": "https://m.ok.ru/video/9469624191496",
+ "filename": "Платформа 2 (2024) 1080p [9469624191496][1080p].mp4",
+ "original_filename": "9469624191496.mp4",
+ "referer": "https://ok.ru/video/9469624191496",
+ "album_id": None,
+ "datetime": 1753377221,
+ }
+ ],
+ ),
+ (
+ "https://ok.ru/video/c637817",
+ [
+ {
+ "url": "https://m.ok.ru/video/35013659257",
+ "filename": "3. Las Voces de Zim [35013659257][480p].mp4",
+ "original_filename": "35013659257.mp4",
+ "referer": "https://ok.ru/video/35013659257",
+ "download_folder": "re:Invader Zim",
+ "album_id": "637817",
+ }
+ ],
+ 34,
+ ),
+]