Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: concurrent session race causing DDoS-Guard errors (yandex disk) (#2147)

Michael Menanno77dd43b2873c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4f607a37b..b51d8f435 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,12 @@ All notable changes to this project will be documented here. For more details, v
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## UNRELEASED
+
+### Fixed
+
+- Downloads failing with "DDoS-Guard" errors when several links are scraped at once (Yandex Disk)
+
## [10.9.1] - 2026-09-13
### Changed
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.py
index 7e8ef5ee3..6026d6110 100644
--- a/cyberdrop_dl/crawlers/yandex_disk.py
+++ b/cyberdrop_dl/crawlers/yandex_disk.py
@@ -3,8 +3,10 @@ from __future__ import annotations
import contextlib
import dataclasses
import json
-from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, cast
+from cyberdrop_dl import aio
+from cyberdrop_dl.clients.http import HTTPConfig
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -17,6 +19,7 @@ if TYPE_CHECKING:
from bs4 import BeautifulSoup
+ from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.url_objects import ScrapeItem
@@ -36,8 +39,26 @@ _DEFAULT_HEADERS = {
"Pragma": "no-cache",
"Cache-Control": "no-cache",
}
+_MAX_SK_RETRIES = 2
+class _WrongSkError(ScrapeError):
+ def __init__(self, new_sk: str) -> None:
+ super().__init__(403, "Yandex rejected the session token (sk)")
+ self.new_sk: str = new_sk
+
+
+def _raise_for_wrong_sk(json_resp: Any, _: AbstractResponse[Any] | None = None) -> None:
+ # Yandex answers the download API with a 400 when the sk does not match the session cookie
+ if not isinstance(json_resp, dict):
+ return
+ resp = cast("dict[str, Any]", json_resp)
+ new_sk: str | None = resp.get("newSk")
+ if resp.get("wrongSk") and new_sk:
+ raise _WrongSkError(new_sk)
+
+
+@HTTPConfig(json_check=_raise_for_wrong_sk)
class YandexDiskCrawler(Crawler):
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "disk.yandex", "yadi.sk"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
@@ -53,6 +74,10 @@ class YandexDiskCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "YandexDisk"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
+ def __post_init__(self) -> None:
+ self._session_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()
+ self._hosts_with_session: set[str] = set()
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["d", folder_id, file_name]:
@@ -68,9 +93,7 @@ class YandexDiskCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item.url):
return None
- with self._request_context():
- soup = await self.request_soup(scrape_item.url, headers=_DEFAULT_HEADERS)
-
+ soup = await self._request_page(scrape_item.url)
item_info = _get_item_info(soup)
assert _is_single_item(item_info)
item_info["sk"] = item_info["environment"]["sk"]
@@ -85,9 +108,7 @@ class YandexDiskCrawler(Crawler):
return None
scrape_item.url = canonical_url
- with self._request_context():
- soup = await self.request_soup(scrape_item.url, headers=_DEFAULT_HEADERS)
-
+ soup = await self._request_page(scrape_item.url)
item_info = _get_item_info(soup)
del soup
if _is_single_item(item_info):
@@ -121,24 +142,9 @@ class YandexDiskCrawler(Crawler):
@error_handling_wrapper
async def _process_file(self, scrape_item: ScrapeItem, file: YandexFile) -> None:
if await self.check_complete_from_referer(scrape_item.url):
- return None
-
- headers = _download_url_headers(scrape_item.url, file.url)
-
- api_url = _DOWNLOAD_API_ENTRYPOINT.with_host(scrape_item.url.host)
- with self._request_context():
- json_resp: dict[str, Any] = await self.request_json(
- api_url,
- method="POST",
- headers=headers,
- data=file.post_data,
- )
-
- new_sk = json_resp.get("new_sk")
- if new_sk:
- new_file = file.with_sk(new_sk)
- return await self._process_file(scrape_item, new_file)
+ return
+ json_resp = await self._request_download_url(scrape_item.url, file)
error = json_resp.get("error")
if error:
raise ScrapeError(422, message=json.dumps(json_resp)[:50])
@@ -152,6 +158,45 @@ class YandexDiskCrawler(Crawler):
filename, ext = self.get_filename_and_ext(filename)
await self.handle_file(file.url, scrape_item, filename, ext, debrid_link=link)
+ async def _request_page(self, url: AbsoluteHttpURL) -> BeautifulSoup:
+ # Yandex ties the sk embedded in a page to the session cookie it sets with that page.
+ # If the first pages of a host are requested concurrently, each one starts a new session and
+ # overwrites the previous cookie, so only the sk of the last page stays valid.
+ # Let the first request of each host set the session cookie before any other page is requested
+ if url.host not in self._hosts_with_session:
+ async with self._session_locks[url.host]:
+ if url.host not in self._hosts_with_session:
+ soup = await self._get_soup(url)
+ self._hosts_with_session.add(url.host)
+ return soup
+
+ return await self._get_soup(url)
+
+ async def _get_soup(self, url: AbsoluteHttpURL) -> BeautifulSoup:
+ with self._request_context():
+ return await self.request_soup(url, headers=_DEFAULT_HEADERS)
+
+ async def _request_download_url(self, url: AbsoluteHttpURL, file: YandexFile) -> dict[str, Any]:
+ # The session can still change mid-run. Yandex then answers with a 400 and the sk that is valid now
+ for _ in range(_MAX_SK_RETRIES):
+ try:
+ return await self._post_download_url(url, file)
+ except _WrongSkError as e:
+ self.log.debug("Yandex rejected the sk for %s, retrying with the one it sent back", url)
+ file = file.with_sk(e.new_sk)
+
+ return await self._post_download_url(url, file)
+
+ async def _post_download_url(self, url: AbsoluteHttpURL, file: YandexFile) -> dict[str, Any]:
+ api_url = _DOWNLOAD_API_ENTRYPOINT.with_host(url.host)
+ with self._request_context():
+ return await self.request_json(
+ api_url,
+ method="POST",
+ headers=_download_url_headers(url, file.url),
+ data=file.post_data,
+ )
+
def _download_url_headers(url: AbsoluteHttpURL, referer: AbsoluteHttpURL) -> dict[str, str]:
return _DEFAULT_HEADERS | {
diff --git a/tests/crawlers/test_yandex_disk.py b/tests/crawlers/test_yandex_disk.py
index 3b851e4ae..5b7676086 100644
--- a/tests/crawlers/test_yandex_disk.py
+++ b/tests/crawlers/test_yandex_disk.py
@@ -1,3 +1,5 @@
+from typing import Any
+
import pytest
from cyberdrop_dl.crawlers import yandex_disk
@@ -17,3 +19,22 @@ def test_download_url_headers_origin(url: str, expected: str) -> None:
headers = yandex_disk._download_url_headers(AbsoluteHttpURL(url), referer)
assert headers["Origin"] == expected
assert headers["Referer"] == headers["X-Retpath-Y"] == str(referer)
+
+
+def test_raise_for_wrong_sk() -> None:
+ with pytest.raises(yandex_disk._WrongSkError) as exc_info:
+ yandex_disk._raise_for_wrong_sk({"error": True, "wrongSk": True, "newSk": "y2"})
+ assert exc_info.value.new_sk == "y2"
+
+
+ "json_resp",
+ [
+ {"error": True},
+ {"error": True, "wrongSk": True},
+ {"error": False, "statusCode": 200, "data": {"url": "https://downloader.disk.yandex.com/disk/x"}},
+ ["not", "a", "dict"],
+ ],
+)
+def test_raise_for_wrong_sk_ignores_other_responses(json_resp: Any) -> None:
+ yandex_disk._raise_for_wrong_sk(json_resp)