Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: use the full file size when resuming a download (#2139)
Michael Menanno3b851bc7d34d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0a6581610..a007bca00 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Downloads of embeded videos and direct URLs (Imagepond)- Thumbnails downloads always failing with 404 (Cyberdrop)- Downloads failing with "DDoS-Guard" errors (Yandex Disk)+- Corrupted file when resuming a partial download that was already 50% or more done## [10.8.0] - 2026-09-06diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.pyindex 5300251c5..fa29c00db 100644--- a/cyberdrop_dl/clients/downloads.py+++ b/cyberdrop_dl/clients/downloads.py@@ -116,13 +116,17 @@ class DownloadClient:) -> bool:await _check_response(media_item, resp, resume_point)media_item.size = _get_content_length(resp.headers)+ if resp.status == HTTPStatus.PARTIAL_CONTENT:+ # Content-Length of a ranged response only counts the bytes after resume_point.+ # Every check below (partial size, final size, filesize limits) needs the size of the whole file+ media_item.size += resume_point_set_upload_date(media_item, resp.headers)if not media_item.path:downloaded = await self._predownload_skip(media_item, domain)if downloaded is not None:return downloaded- hook = self._make_hook(media_item, resume_point)+ hook = self._make_hook(media_item)if resume_point:hook.advance(resume_point)@@ -130,11 +134,11 @@ class DownloadClient:await self._append_content(media_item, hook, resp)return True- def _make_hook(self, media_item: MediaItem, resume_point: int) -> ProgressHook:+ def _make_hook(self, media_item: MediaItem) -> ProgressHook:if media_item.is_segment and not media_item.extra_info.get("MUX_STREAM"):return self.manager.scrape_mapper.tui.downloads.download_hls_seg()- size = (media_item.size + resume_point) if media_item.size is not None else None+ size = media_item.sizereturn self.manager.scrape_mapper.tui.downloads.download_file(media_item.filename,media_item.domain,@@ -153,7 +157,7 @@ class DownloadClient:async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:check_free_space = storage.create_free_space_checker(media_item)check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)- await check_free_space(media_item.size)+ await check_free_space(_get_content_length(resp.headers))await self._pre_download_check(media_item)async with self._track_speed(hook), aio.open(media_item.partial_file, mode="ab") as f:diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 5560dc09e..2ee091fc0 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, final, overridefrom mega.chunker import MegaChunker, get_chunksfrom cyberdrop_dl import aio, storage-from cyberdrop_dl.clients.downloads import DownloadClient, make_speed_checker+from cyberdrop_dl.clients.downloads import DownloadClient, _get_content_length, make_speed_checkerfrom cyberdrop_dl.downloader.http import Downloaderif TYPE_CHECKING:@@ -24,7 +24,7 @@ class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIscheck_free_space = storage.create_free_space_checker(media_item)check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)- await check_free_space(media_item.size)+ await check_free_space(_get_content_length(resp.headers))await self._pre_download_check(media_item)crypto, file_size = media_item.extra_info[media_item.domain]["key"]diff --git a/tests/test_download_resume.py b/tests/test_download_resume.pynew file mode 100644index 000000000..f608e303e--- /dev/null+++ b/tests/test_download_resume.py@@ -0,0 +1,123 @@+from __future__ import annotations++import hashlib+from typing import TYPE_CHECKING, Self+from unittest import mock++import pytest+from multidict import CIMultiDict, CIMultiDictProxy++from cyberdrop_dl.clients.downloads import DownloadClient+from cyberdrop_dl.clients.response import AbstractResponse+from cyberdrop_dl.exceptions import DownloadError+from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem++if TYPE_CHECKING:+ from collections.abc import AsyncIterator+ from pathlib import Path++ from cyberdrop_dl.manager import Manager+++_PAYLOAD = bytes(range(256)) * 4_000 # 1,024,000 bytes+_DOMAIN = "example"+++class _FakeResponse(AbstractResponse[bytes]):+ __slots__ = ()++ async def _read(self) -> bytes:+ return self._resp++ async def _read_text(self, encoding: str | None = None) -> str:+ return self._resp.decode(encoding or "utf-8", errors="replace")++ async def iter_chunked(self, size: int) -> AsyncIterator[bytes]:+ for start in range(0, len(self._resp), size):+ yield self._resp[start : start + size]++ async def aclose(self) -> None:+ return None++ @classmethod+ def partial_content(cls, start: int, *, truncate_to: int | None = None) -> Self:+ body = _PAYLOAD[start:]+ headers = {+ "Content-Type": "video/mp4",+ "Content-Length": str(len(body)),+ "Content-Range": f"bytes {start}-{len(_PAYLOAD) - 1}/{len(_PAYLOAD)}",+ }+ return cls(+ content_type="video/mp4",+ status=206,+ headers=CIMultiDictProxy(CIMultiDict(headers)),+ url=AbsoluteHttpURL("https://cdn.example.com/video.mp4"),+ location=None,+ _resp=body[:truncate_to],+ )+++class _Hook:+ def __enter__(self) -> Self:+ return self++ def __exit__(self, *_: object) -> None:+ return None++ def advance(self, _: int) -> None:+ return None++ def get_speed(self) -> float:+ return 0+++async def _resumable_item(manager: Manager, tmp_path: Path, resume_point: int) -> MediaItem:+ folder = tmp_path / "downloads"+ folder.mkdir()+ item = MediaItem(+ url=AbsoluteHttpURL("https://cdn.example.com/video.mp4"),+ domain=_DOMAIN,+ download_folder=folder,+ filename="video.mp4",+ db_path="/video.mp4",+ referer=AbsoluteHttpURL("https://example.com/v/1"),+ album_id=None,+ ext=".mp4",+ original_filename="video.mp4",+ uploaded_at=None,+ )+ item.download_filename = "video.mp4"+ await manager.database.history.insert_incompleted(_DOMAIN, item)++ item.partial_file = folder / "video.mp4.part"+ item.partial_file.write_bytes(_PAYLOAD[:resume_point])+ return item++[email protected]("resume_pct", [0.2, 0.4, 0.6, 0.8])+async def test_resumed_download_keeps_partial_bytes(+ running_manager: Manager, tmp_path: Path, resume_pct: float+) -> None:+ resume_point = int(len(_PAYLOAD) * resume_pct)+ item = await _resumable_item(running_manager, tmp_path, resume_point)+ client = DownloadClient(running_manager)+ resp = _FakeResponse.partial_content(resume_point)+ with mock.patch.object(DownloadClient, "_make_hook", return_value=_Hook()):+ assert await client._process_response(item, _DOMAIN, resume_point, resp)++ data = item.partial_file.read_bytes()+ assert len(data) == len(_PAYLOAD)+ assert hashlib.sha256(data).digest() == hashlib.sha256(_PAYLOAD).digest()+++async def test_truncated_resume_is_not_marked_complete(running_manager: Manager, tmp_path: Path) -> None:+ resume_point = int(len(_PAYLOAD) * 0.3)+ truncate_to = int(len(_PAYLOAD) * 0.5)+ item = await _resumable_item(running_manager, tmp_path, resume_point)+ client = DownloadClient(running_manager)+ resp = _FakeResponse.partial_content(resume_point, truncate_to=truncate_to)+ with (+ mock.patch.object(DownloadClient, "_make_hook", return_value=_Hook()),+ pytest.raises(DownloadError, match="Corrupted File"),+ ):+ await client._process_response(item, _DOMAIN, resume_point, resp)