Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework crawler (YandexDisk) (#2172)
NTFSvolumec6ce5e7ff905
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.pyindex 5e2191e5d..11cfb4aa9 100644--- a/cyberdrop_dl/crawlers/yandex_disk.py+++ b/cyberdrop_dl/crawlers/yandex_disk.py@@ -3,43 +3,29 @@ from __future__ import annotationsimport contextlibimport dataclassesimport json+from collections import deque+from contextvars import ContextVar+from pathlib import PurePosixPathfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, castfrom cyberdrop_dl import aiofrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css-from cyberdrop_dl.utils.dataclass import DictDataclass+from cyberdrop_dl.utils.dataclass import deserializefrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:from collections.abc import Generator- from bs4 import BeautifulSoup-from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.url_objects import ScrapeItem-_PRIMARY_URL = AbsoluteHttpURL("https://disk.yandex.com.tr/")-_DOWNLOAD_API_ENTRYPOINT = AbsoluteHttpURL("https://disk.yandex.com.tr/public/api/download-url")-_KEYS_TO_KEEP = "currentResourceId", "resources", "environment"-_DEFAULT_HEADERS = {- "Accept": "*/*",- "Accept-Language": "en-US,en;q=0.5",- "Accept-Encoding": "gzip, deflate, br, zstd",- "DNT": "1",- "Sec-GPC": "1",- "Connection": "keep-alive",- "Sec-Fetch-Dest": "empty",- "Sec-Fetch-Mode": "cors",- "Sec-Fetch-Site: same-originPriority": "u=0",- "Pragma": "no-cache",- "Cache-Control": "no-cache",-}-_MAX_SK_RETRIES = 2+POSIX_ROOT = PurePosixPath("/")+SHARE_ID = ContextVar[str]("SHARE_ID")class _WrongSkError(ScrapeError):@@ -48,42 +34,39 @@ class _WrongSkError(ScrapeError):self.new_sk: str = new_sk-def _raise_for_wrong_sk(json_resp: Any, _: AbstractResponse[Any] | None = None) -> None:+def _raise_for_wrong_sk(resp: Any, _: AbstractResponse[Any] | None = None) -> None:# Yandex answers the download API with a 400 when the sk does not match the session cookie- if not isinstance(json_resp, dict):+ if not isinstance(resp, dict):return- resp = cast("dict[str, Any]", json_resp)++ resp = cast("dict[str, Any]", resp)new_sk: str | None = resp.get("newSk")if resp.get("wrongSk") and new_sk:raise _WrongSkError(new_sk)-@HTTPConfig(json_check=_raise_for_wrong_sk)+@HTTPConfig(json_check=_raise_for_wrong_sk, headers={"Accept-Language": "en-US,en;q=0.5"})class YandexDiskCrawler(Crawler):- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "disk.yandex", "yadi.sk"+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "disk.yandex", "yadi.sk", "disk.360.yandex"SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Folder": "/d/<folder_id>",+ "Share": "/d/<share_id>","File": (- "/d/<folder_id>/<file_name>",+ "/d/<share_id>/<sub_path>","/i/<file_id>",),- "**NOTE**": "Does NOT support nested folders",}DOMAIN: ClassVar[str] = "disk.yandex"FOLDER_DOMAIN: ClassVar[str] = "YandexDisk"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://disk.yandex.com")def __post_init__(self) -> None:- self._session_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()- self._hosts_with_session: set[str] = set()+ self.api: YandexDiskAPI = YandexDiskAPI.from_crawler(self)async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:- case ["d", folder_id, file_name]:- await self.folder(scrape_item, folder_id, file_name)- case ["d", folder_id]:- await self.folder(scrape_item, folder_id)+ case ["d", share_id, *rest]:+ await self.share(scrape_item, share_id, subpath=POSIX_ROOT.joinpath(*rest))case ["i", _]:await self.file(scrape_item)case _:@@ -92,44 +75,83 @@ class YandexDiskCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item.url):- return None+ return- soup = await self._request_page(scrape_item.url)- item_info = _get_item_info(soup)- assert _is_single_item(item_info)- item_info["sk"] = item_info["environment"]["sk"]- item_info["file_url"] = scrape_item.url- file = YandexFile.from_json(item_info)- return await self._process_file(scrape_item, file)+ share = await self.api.share(scrape_item.url)+ file = YandexFile.from_share(share)+ await self._file(scrape_item, file)@error_handling_wrapper- async def folder(self, scrape_item: ScrapeItem, folder_id: str, single_file_name: str | None = None) -> None:- canonical_url = _PRIMARY_URL / "d" / folder_id- if single_file_name and await self.check_complete_from_referer(scrape_item.url):- return None-+ async def share(self, scrape_item: ScrapeItem, share_id: str, subpath: PurePosixPath | None = None) -> None:+ canonical_url = self.PRIMARY_URL / "d" / share_idscrape_item.url = canonical_url- soup = await self._request_page(scrape_item.url)- item_info = _get_item_info(soup)- del soup- if _is_single_item(item_info):- item_info["sk"] = item_info["environment"]["sk"]- item_info["file_url"] = scrape_item.url- file = YandexFile.from_json(item_info)- return await self._process_file(scrape_item, file)-- folder = YandexFolder.from_json(item_info)- title = self.create_title(folder.name, folder.id)- scrape_item.setup_as_album(title, album_id=folder.id)-- for file in folder.files:- if single_file_name and file.name != single_file_name:- continue- new_scrape_item = scrape_item.create_child(file.url)- await self._process_file(new_scrape_item, file)- scrape_item.add_children()- if single_file_name:- return None+ SHARE_ID.set(share_id)++ share = await self.api.share(scrape_item.url)+ if len(share.resources) == 1 and not share.currentResourceId:+ file = YandexFile.from_share(share)+ await self._file(scrape_item, file)+ return++ folder = YandexFolder.from_share(share)+ scrape_item.setup_as_album(self.create_title(folder.name, folder.id), album_id=folder.id)+ await self._walk_nodes(scrape_item, folder, subpath)++ async def _walk_nodes(+ self,+ scrape_item: ScrapeItem,+ folder: YandexFolder,+ subpath: PurePosixPath | None,+ ) -> None:+ subfolders: deque[tuple[PurePosixPath, str, str]] = deque()+ current_path = POSIX_ROOT+ subpath = subpath or POSIX_ROOT+ nodes = [folder.resources[child_id] for child_id in folder.children_ids]++ while True:+ for node in nodes:+ if node["type"] == "dir":+ inner_path = current_path / node["name"]+ if subpath.is_relative_to(inner_path) or inner_path.is_relative_to(subpath):+ subfolders.append((current_path / node["name"], node["id"], node["path"]))+ continue++ if node["type"] != "file":+ self.log.warning("Unknown node type: %s", node)+ continue++ file = YandexFile.from_json(node, sk=folder.sk)+ if not (current_path / file.name).is_relative_to(subpath):+ continue++ new_item = scrape_item.create_child(self.PRIMARY_URL / file.web_path)+ new_item.append_folders(*current_path.parts[1:])+ self.create_eager_task(self._file(new_item, file))+ scrape_item.add_children()++ if not subfolders:+ return++ current_path, node_id, node_path = subfolders.popleft()+ nodes = await self.api.list_folder(node_id, node_path, sk=folder.sk)++ @error_handling_wrapper+ async def _file(self, scrape_item: ScrapeItem, file: YandexFile) -> None:+ url = self.origin / file.web_path+ if await self.check_complete(url):+ return++ scrape_item.uploaded_at = file.modified+ src = await self.api.download(file)+ name = src.query.get("filename") or file.name+ filename, ext = self.get_filename_and_ext(name, mime_type=file.mimetype)+ await self.handle_file(url, scrape_item, name, ext, debrid_link=src, custom_filename=filename)+++class YandexDiskAPI(API):+ def __post_init__(self) -> None:+ self._host_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()+ self._hosts_with_session: set[str] = set()@contextlib.contextmanagerdef _request_context(self) -> Generator[None]:@@ -140,97 +162,92 @@ class YandexDiskCrawler(Crawler):raise DDOSGuardError from Noneraise- @error_handling_wrapper- async def _process_file(self, scrape_item: ScrapeItem, file: YandexFile) -> None:- if await self.check_complete_from_referer(scrape_item.url):- return-- json_resp = await self._request_download_url(scrape_item.url, file)- error = json_resp.get("error")- if error:- raise ScrapeError(422, message=json.dumps(json_resp)[:50])-- self.log.debug(json_resp)- scrape_item.uploaded_at = file.modified- link_str: str = json_resp["data"]["url"]- link = self.parse_url(link_str)-- filename = link.query.get("filename") or file.name- filename, ext = self.get_filename_and_ext(filename)- await self.handle_file(file.url, scrape_item, filename, ext, debrid_link=link)+ async def share(self, url: AbsoluteHttpURL) -> Share:+ html = await self._request_html(url)+ data = json.loads(css.select_text(await css.asoup(html, parse_only="script"), "script#store-prefetch"))+ return deserialize(Share, data, sk=data["environment"]["sk"], path=url.path.removeprefix("/"))- async def _request_page(self, url: AbsoluteHttpURL) -> BeautifulSoup:+ async def _request_html(self, url: AbsoluteHttpURL) -> str:# Yandex ties the sk embedded in a page to the session cookie it sets with that page.# If the first pages of a host are requested concurrently, each one starts a new session and# overwrites the previous cookie, so only the sk of the last page stays valid.# Let the first request of each host set the session cookie before any other page is requested- if url.host not in self._hosts_with_session:- async with self._session_locks[url.host]:- if url.host not in self._hosts_with_session:- soup = await self._get_soup(url)- self._hosts_with_session.add(url.host)- return soup-- return await self._get_soup(url)-- async def _get_soup(self, url: AbsoluteHttpURL) -> BeautifulSoup:with self._request_context():- return await self.request_soup(url, headers=_DEFAULT_HEADERS)-- async def _request_download_url(self, url: AbsoluteHttpURL, file: YandexFile) -> dict[str, Any]:- # The session can still change mid-run. Yandex then answers with a 400 and the sk that is valid now- for _ in range(_MAX_SK_RETRIES):- try:- return await self._post_download_url(url, file)- except _WrongSkError as e:- self.log.debug("Yandex rejected the sk for %s, retrying with the one it sent back", url)- file = file.with_sk(e.new_sk)-- return await self._post_download_url(url, file)+ if url.host not in self._hosts_with_session:+ async with self._host_locks[url.host]:+ if url.host not in self._hosts_with_session:+ html = await self.request_text(url)+ self._hosts_with_session.add(url.host)+ return html++ return await self.request_text(url)++ async def _download(self, file: YandexFile) -> dict[str, Any]:+ api_url = self.origin / "public/api/download-url"+ referer = self.origin / file.web_path+ return await self.post_json(+ api_url,+ headers={+ "Referer": str(referer),+ "X-Retpath-Y": str(referer),+ },+ params={"hash": file.path, "inline": False, "sk": file.sk},+ )- async def _post_download_url(self, url: AbsoluteHttpURL, file: YandexFile) -> dict[str, Any]:- api_url = _DOWNLOAD_API_ENTRYPOINT.with_host(url.host)+ async def post_json(+ self, api_url: AbsoluteHttpURL, params: dict[str, Any], headers: dict[str, Any] | None = None+ ) -> dict[str, Any]:with self._request_context():return await self.request_json(api_url,method="POST",- headers=_download_url_headers(url, file.url),- data=file.post_data,+ headers={+ "Content-Type": "text/plain",+ "X-Requested-With": "XMLHttpRequest",+ "Origin": str(self.origin),+ **(headers or {}),+ },+ data=json.dumps(params, separators=(",", ":")),)+ async def download(self, file: YandexFile) -> AbsoluteHttpURL:+ # The session can still change mid-run. Yandex then answers with a 400 and the sk that is valid now+ for retry in (True, False):+ try:+ resp = await self._download(file)+ except _WrongSkError as e:+ if not retry:+ raise+ self.log.debug("Yandex rejected the sk for %s, retrying with the one it sent back", file.path)+ file = file.with_sk(e.new_sk)+ else:+ if resp.get("error"):+ raise ScrapeError(422, message=str(resp)[:50])+ return self.parse_url(resp["data"]["url"])++ raise RuntimeError-def _download_url_headers(url: AbsoluteHttpURL, referer: AbsoluteHttpURL) -> dict[str, str]:- return _DEFAULT_HEADERS | {- "Content-Type": "text/plain",- "X-Requested-With": "XMLHttpRequest",- "Origin": str(url.origin()),- "Referer": str(referer),- "X-Retpath-Y": str(referer),- }+ async def list_folder(self, node_id: str, path: str, sk: str) -> list[dict[str, Any]]:+ url = self.origin / "public/api/fetch-list"+ resp = await self.post_json(url, params={"hash": path, "offset": 0, "withSizes": "true", "sk": sk})+ return [node for node in resp["resources"] if node.get("id") != node_id]-def _get_item_info(soup: BeautifulSoup) -> dict[str, Any]:- js_text: str = css.select_text(soup, "script#store-prefetch")- info_json: dict[str, dict[str, Any]] = json.loads(js_text)- info_json = {k: v for k, v in info_json.items() if k in _KEYS_TO_KEEP}- env: dict[str, str] = info_json["environment"]- info_json["environment"] = {"sk": env["sk"]} # We don't need any other info from env- return info_json[email protected](frozen=True, kw_only=True)+class Share:+ sk: str+ path: str+ resources: dict[str, dict[str, Any]]+ currentResourceId: str | None = None # noqa: N815@dataclasses.dataclass(frozen=True, kw_only=True)-class YandexItem(DictDataclass):- name: str- modified: int- type: Literal["file", "dir"]+class YandexItem:id: str- path: strsk: str- short_url: AbsoluteHttpURL # https://yadi.sk/d/<id>-- @property- def post_data(self) -> str:- return json.dumps({"hash": self.path, "sk": self.sk})+ path: str+ name: str+ modified: intdef with_sk(self, new_sk: str) -> Self:return dataclasses.replace(self, sk=new_sk)@@ -238,80 +255,42 @@ class YandexItem(DictDataclass):@dataclasses.dataclass(frozen=True, kw_only=True)class YandexFolder(YandexItem):- resources: dict[str, Any]+ type: Literal["dir"]+ resources: dict[str, dict[str, Any]]children_ids: list[str]- @property- def public_id(self) -> str:- return self.short_url.name-- @property- def files(self) -> Generator[YandexFile]:- for child_id in self.children_ids:- item_info: dict[str, Any] = self.resources[child_id]- if item_info["type"] != "file":- continue # TODO handle subfolders-- yield YandexFile.from_dict(- YandexFile.filter_dict(item_info),- parent_folder_public_id=self.public_id,- sk=self.sk,- )-- @property- def subfolders(self) -> Generator[YandexFolder]:- for child_id in self.children_ids:- item_info: dict[str, Any] = self.resources[child_id]- if item_info["type"] != "folder":- continue- raise NotImplementedError-- @property- def url(self) -> AbsoluteHttpURL:- return _PRIMARY_URL / "d" / self.id-@classmethod- def from_json(cls, json_resp: dict[str, Any]) -> Self:- resources: dict[str, dict[str, Any]] = json_resp["resources"]- folder_id: str = json_resp["currentResourceId"]- sk: str = json_resp["environment"]["sk"]-- folder_details = resources[folder_id]- short_url = AbsoluteHttpURL(folder_details["meta"]["short_url"])- children_ids: list[str] = folder_details["children"]- return cls.from_dict(- cls.filter_dict(folder_details),- resources=resources,- sk=sk,- short_url=short_url,- children_ids=children_ids,+ def from_share(cls, share: Share) -> Self:+ assert share.currentResourceId+ details = share.resources.pop(share.currentResourceId)+ return deserialize(+ cls,+ details,+ resources=share.resources,+ sk=share.sk,+ children_ids=details["children"],)@dataclasses.dataclass(frozen=True, kw_only=True)class YandexFile(YandexItem):- parent_folder_public_id: str = ""- file_url: AbsoluteHttpURL | None = None+ type: Literal["file"]+ mimetype: str+ web_path: str = ""- @property- def url(self) -> AbsoluteHttpURL:- if self.parent_folder_public_id:- return _PRIMARY_URL / "d" / self.parent_folder_public_id / self.name- if self.file_url:- return self.file_url- return self.short_url+ def __post_init__(self) -> None:+ if not self.web_path:+ _hash, _, path = self.path.partition(":/")+ assert path+ object.__setattr__(self, "web_path", f"d/{SHARE_ID.get()}/{path}")@classmethod- def from_json(cls, json_resp: dict[str, Any]) -> Self:- resources: dict[str, dict[str, Any]] = json_resp["resources"]- assert len(resources) == 1- file_details = next(iter(resources.items()))[1]- sk: str = json_resp["environment"]["sk"]+ def from_json(cls, details: dict[str, Any], sk: str) -> Self:+ return deserialize(cls, details, sk=sk, mimetype=details["meta"]["mimetype"])- short_url = AbsoluteHttpURL(file_details["meta"]["short_url"])- valid_dict: dict[str, Any] = cls.filter_dict(file_details)- return cls.from_dict(valid_dict, sk=sk, short_url=short_url)---def _is_single_item(json_resp: dict[str, Any]) -> bool:- return len(json_resp["resources"]) == 1 and not bool(json_resp["currentResourceId"])+ @classmethod+ def from_share(cls, share: Share) -> Self:+ assert len(share.resources) == 1+ details = next(iter(share.resources.values()))+ details["web_path"] = share.path+ return cls.from_json(details, sk=share.sk)diff --git a/tests/crawlers/test_cases/yandex_disk.py b/tests/crawlers/test_cases/yandex_disk.pynew file mode 100644index 000000000..4eb4ce187--- /dev/null+++ b/tests/crawlers/test_cases/yandex_disk.py@@ -0,0 +1,37 @@+DOMAIN = "disk.yandex"+TEST_CASES = [+ {+ "url": "https://disk.yandex.com/d/wSoTAc690rO3wg",+ "results": [+ {+ "url": "https://disk.yandex.com/d/wSoTAc690rO3wg/ace%20ventura/Ace%20Ventura%20-%20Pet%20Detective%20(1994).mp4",+ "filename": "Ace Ventura - Pet Detective (1994).mp4",+ "debrid_url": "re:https://downloader.disk.yandex.com/disk/",+ "original_filename": "Ace Ventura - Pet Detective (1994).mp4",+ "referer": "https://disk.yandex.com/d/wSoTAc690rO3wg/ace%20ventura/Ace%20Ventura%20-%20Pet%20Detective%20(1994).mp4",+ "album_id": "808eb04e24ecb99c8cca60edf6b4a24dbf26a575e48bcbef49ae42e4742f51a0",+ "uploaded_at": 1789752612,+ "download_folder": "re:Sample (YandexDisk)/ace ventura",+ "thumbnail": None,+ }+ ],+ "count": 1,+ },+ {+ "url": "https://disk.360.yandex.ru/i/TM2xsIVsgjY4uw",+ "results": [+ {+ "url": "https://disk.360.yandex.ru/i/TM2xsIVsgjY4uw",+ "filename": "Cars для суперсет 18.07.2025 14-19-50 - запись.mp4",+ "debrid_url": "re:https://downloader.disk.yandex.ru/disk/",+ "original_filename": "Cars для суперсет 18.07.2025 14-19-50 - запись.mp4",+ "referer": "https://disk.360.yandex.ru/i/TM2xsIVsgjY4uw",+ "album_id": None,+ "uploaded_at": 1753712643,+ "download_folder": "re:Loose Files (YandexDisk)",+ "thumbnail": None,+ }+ ],+ "count": 1,+ },+]diff --git a/tests/crawlers/test_yandex_disk.py b/tests/crawlers/test_yandex_disk.pyindex 5b7676086..7233b2426 100644--- a/tests/crawlers/test_yandex_disk.py+++ b/tests/crawlers/test_yandex_disk.py@@ -3,22 +3,6 @@ from typing import Anyimport pytestfrom cyberdrop_dl.crawlers import yandex_disk-from cyberdrop_dl.url_objects import AbsoluteHttpURL--- ("url", "expected"),- [- ("https://disk.yandex.com/i/AbCdEf123", "https://disk.yandex.com"),- ("https://disk.yandex.com.tr/d/AbCdEf123/video.mp4", "https://disk.yandex.com.tr"),- ("https://disk.yandex.ru/i/AbCdEf123", "https://disk.yandex.ru"),- ],-)-def test_download_url_headers_origin(url: str, expected: str) -> None:- referer = AbsoluteHttpURL("https://yadi.sk/i/AbCdEf123")- headers = yandex_disk._download_url_headers(AbsoluteHttpURL(url), referer)- assert headers["Origin"] == expected- assert headers["Referer"] == headers["X-Retpath-Y"] == str(referer)def test_raise_for_wrong_sk() -> None: