Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add odnoklassniki (ok.ru) support (#1213)
NTFSvolumefe227ef11279
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 47e8f6a98..bcef05823 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -128,6 +128,14 @@ class DownloadClient:api_key = gofile_cookies.get("accountToken", "")if api_key:download_headers["Authorization"] = f"Bearer {api_key.value}" # type: ignore+ elif domain == "odnoklassniki":+ # TODO: Add "headers" attribute to MediaItem to use custom headers for downloads+ download_headers |= {+ "Accept-Language": "en-gb, en;q=0.8",+ "User-Agent": "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36",+ "Referer": "https://m.ok.ru/",+ "Origin": "https://m.ok.ru",+ }return download_headers@limiterdiff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 86263e438..c34e2ff85 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -62,6 +62,7 @@ from .nekohouse import NekohouseCrawlerfrom .nhentai import NHentaiCrawlerfrom .noodle_magazine import NoodleMagazineCrawlerfrom .nudostartv import NudoStarTVCrawler+from .odnoklassniki import OdnoklassnikiCrawlerfrom .omegascans import OmegaScansCrawlerfrom .onedrive import OneDriveCrawlerfrom .pimpandhost import PimpAndHostCrawlerdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex ac7e75fb4..97c90f007 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -682,8 +682,7 @@ class Crawler(ABC):only_truncate_stem: bool = True,) -> str:calling_args = {name: value for name, value in locals().items() if value is not None and name not in ("self",)}- clean_name = sanitize_filename(Path(name).as_posix().replace("/", "-")) # remove OS separators (if any)- stem = Path(clean_name).stem.removesuffix(".") # remove extensions (if any)+ stem = sanitize_filename(Path(name).as_posix().replace("/", "-")) # remove OS separators (if any)extra_info: list[str] = []if _placeholder_config.include_file_id and file_id:diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.pynew file mode 100644index 000000000..185954347--- /dev/null+++ b/cyberdrop_dl/crawlers/odnoklassniki.py@@ -0,0 +1,177 @@+from __future__ import annotations++import re+from typing import TYPE_CHECKING, Any, ClassVar++from aiolimiter import AsyncLimiter++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.utils import css, json+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between++if TYPE_CHECKING:+ from bs4 import BeautifulSoup++ from cyberdrop_dl.data_structures.url_objects import ScrapeItem++_find_video_ids = re.compile("/video/(\\d+)").finditer++_HEADERS = {+ "Accept-Language": "en-gb, en;q=0.8",+ "Referer": "https://ok.ru/",+ "Origin": "https://ok.ru",+}++_MOBILE_HEADERS = _HEADERS | {+ "User-Agent": "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36",+ "Referer": "https://m.ok.ru/",+ "Origin": "https://m.ok.ru",+}+++class VideoProvider:+ # This site also embeds videos from other sources as their own+ OK_RU = "UPLOADED_ODKL"+ YOUTUBE = "USER_YOUTUBE"+ OG = "OPEN_GRAPH"+ LIVESTREAM = "LIVE_TV_APP"+++class Selector:+ VIDEO_DELETED = "a:contains('видео') div.empty"+ UNAUTHORIZED = "div:contains('Access to this video is restricted')"+ GEO_BLOCKED = "div:contains('This video is not available in your region')"+ CHANNEL_NAME = ".album-info_name"+ CHANNEL_HASH = "script:contains('gwtHash:')"++ CHANNEL_LAST_ELEMENT = css.CssAttributeSelector("[data-last-element]", "data-last-element")+ FLASHVARS = css.CssAttributeSelector("[data-options*='flashvars']", "data-options")+++class OdnoklassnikiCrawler(Crawler):+ SUPPORTED_DOMAINS = "ok.ru", "odnoklassniki.ru"+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": "/video/<video_id>",+ "Channel": (+ "/video/c<channel_id>",+ "/profile/<username>/c<channel_id>",+ ),+ }+ PRIMARY_URL = AbsoluteHttpURL("https://ok.ru")+ DOMAIN = "odnoklassniki"+ FOLDER_DOMAIN = "ok.ru"++ def __post_init__(self) -> None:+ self.request_limiter = AsyncLimiter(3, 10)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["video", id_]:+ if id_.startswith("c"):+ return await self.channel(scrape_item, id_)+ return await self.video(scrape_item, id_)+ case ["profile", _, channel] if channel.startswith("c"):+ return await self.channel(scrape_item, channel)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def channel(self, scrape_item: ScrapeItem, channel_str: str):+ async with self.request_limiter:+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url, _HEADERS)++ channel_id = channel_str.removeprefix("c")+ gwt_hash = get_text_between(css.select_one_get_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')+ last_element_id = css.select_one_get_attr_or_none(soup, *Selector.CHANNEL_LAST_ELEMENT)+ name = css.select_one_get_text(soup, Selector.CHANNEL_NAME)+ scrape_item.setup_as_album(self.create_title(name, channel_id), album_id=channel_id)++ page_url = (self.PRIMARY_URL / "video" / channel_str).with_query(+ {+ "st.cmd": "anonymVideo",+ "st.m": "ALBUM",+ "st.ft": "album",+ "st.aid": channel_str,+ "cmd": "VideoAlbumBlock",+ }+ )+ seen: set[str] = set()+ content = str(soup)+ page = 1+ while True:+ page_had_new_videos = False+ for match in _find_video_ids(content):+ if (video_path := match.group()) not in seen:+ seen.add(video_path)+ page_had_new_videos = True+ video_url = self.PRIMARY_URL.with_path(video_path)+ new_scrape_item = scrape_item.create_child(video_url)+ self.create_task(self.run(new_scrape_item))+ scrape_item.add_children()++ if not page_had_new_videos or not last_element_id:+ break++ page += 1+ async with self.request_limiter:+ resp = await self.client._post_data(+ self.DOMAIN,+ page_url,+ _HEADERS,+ data={+ "fetch": "false",+ "st.page": page,+ "st.lastelem": last_element_id,+ "gwt.requested": gwt_hash,+ },+ )+ last_element_id = resp.headers.get("lastelem")+ content = await resp.text()++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, video_id: str):+ mobile_url = AbsoluteHttpURL(f"https://m.ok.ru/video/{video_id}")+ async with self.request_limiter:+ soup = await self.client.get_soup(self.DOMAIN, mobile_url, _MOBILE_HEADERS)++ _check_video_is_available(soup)+ metadata: dict[str, Any] = json.loads(Selector.FLASHVARS(soup))["flashvars"]["metadata"]++ if (provider := metadata["provider"]) != VideoProvider.OK_RU:+ raise ScrapeError(422, f"Unsupported provider: {provider}")++ resolution, src = _get_best_src(metadata)+ cdn_url = self.parse_url(src)+ # downloads may fail if we have cdn cookies+ self.client.client_manager.cookies.clear_domain(cdn_url.host)+ json_ld = css.get_json_ld(soup)+ title: str = metadata["movie"].get("title") or json_ld["name"]+ scrape_item.possible_datetime = self.parse_iso_date(json_ld["uploadDate"])+ filename = self.create_custom_filename(title, ".mp4", file_id=video_id, resolution=resolution)+ await self.handle_file(+ mobile_url, scrape_item, video_id + ".mp4", custom_filename=filename, debrid_link=cdn_url+ )+++def _get_best_src(metadata: dict[str, Any]) -> tuple[int, str]:+ def parse():+ for video in metadata["videos"]:+ resolution = {"full": 1080, "hd": 720, "sd": 480, "low": 360, "lowest": 240, "mobile": 144}.get(+ video["name"], 0+ )+ yield resolution, video["url"]++ return max(parse())+++def _check_video_is_available(soup: BeautifulSoup) -> None:+ if error := soup.select_one(Selector.VIDEO_DELETED):+ raise ScrapeError(404, css.get_text(error))++ if soup.select_one(Selector.UNAUTHORIZED):+ raise ScrapeError(503)++ if soup.select_one(Selector.GEO_BLOCKED):+ raise ScrapeError(403)diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 1d50ca5e3..66df8cb9c 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -153,7 +153,10 @@ def get_json_ld(soup: Tag, /, contains: str | None = None) -> dict[str, Any]:if contains:selector += f":contains('{contains}')"- ld_json = json.loads(select_one_get_text(soup, selector))+ ld_json = json.loads(select_one_get_text(soup, selector)) or {}+ if isinstance(ld_json, list):+ return ld_json[0]+return ld_jsondiff --git a/tests/crawlers/test_cases/odnoklassniki.py b/tests/crawlers/test_cases/odnoklassniki.pynew file mode 100644index 000000000..6d1756d87--- /dev/null+++ b/tests/crawlers/test_cases/odnoklassniki.py@@ -0,0 +1,43 @@+DOMAIN = "odnoklassniki"+TEST_CASES = [+ (+ "https://ok.ru/video/1416667269705",+ [+ {+ "url": "https://m.ok.ru/video/1416667269705",+ "filename": "73 Réplika_x264 [1416667269705][480p].mp4",+ "original_filename": "1416667269705.mp4",+ "referer": "https://ok.ru/video/1416667269705",+ "album_id": None,+ "datetime": 1566236730,+ }+ ],+ ),+ (+ "https://ok.ru/video/9469624191496",+ [+ {+ "url": "https://m.ok.ru/video/9469624191496",+ "filename": "Платформа 2 (2024) 1080p [9469624191496][1080p].mp4",+ "original_filename": "9469624191496.mp4",+ "referer": "https://ok.ru/video/9469624191496",+ "album_id": None,+ "datetime": 1753377221,+ }+ ],+ ),+ (+ "https://ok.ru/video/c637817",+ [+ {+ "url": "https://m.ok.ru/video/35013659257",+ "filename": "3. Las Voces de Zim [35013659257][480p].mp4",+ "original_filename": "35013659257.mp4",+ "referer": "https://ok.ru/video/35013659257",+ "download_folder": "re:Invader Zim",+ "album_id": "637817",+ }+ ],+ 34,+ ),+]