Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: use the full file size when resuming a download (#2139)

Michael Menanno3b851bc7d34d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a6581610..a007bca00 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Downloads of embeded videos and direct URLs (Imagepond)
- Thumbnails downloads always failing with 404 (Cyberdrop)
- Downloads failing with "DDoS-Guard" errors (Yandex Disk)
+- Corrupted file when resuming a partial download that was already 50% or more done
## [10.8.0] - 2026-09-06
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.py
index 5300251c5..fa29c00db 100644
--- a/cyberdrop_dl/clients/downloads.py
+++ b/cyberdrop_dl/clients/downloads.py
@@ -116,13 +116,17 @@ class DownloadClient:
) -> bool:
await _check_response(media_item, resp, resume_point)
media_item.size = _get_content_length(resp.headers)
+ if resp.status == HTTPStatus.PARTIAL_CONTENT:
+ # Content-Length of a ranged response only counts the bytes after resume_point.
+ # Every check below (partial size, final size, filesize limits) needs the size of the whole file
+ media_item.size += resume_point
_set_upload_date(media_item, resp.headers)
if not media_item.path:
downloaded = await self._predownload_skip(media_item, domain)
if downloaded is not None:
return downloaded
- hook = self._make_hook(media_item, resume_point)
+ hook = self._make_hook(media_item)
if resume_point:
hook.advance(resume_point)
@@ -130,11 +134,11 @@ class DownloadClient:
await self._append_content(media_item, hook, resp)
return True
- def _make_hook(self, media_item: MediaItem, resume_point: int) -> ProgressHook:
+ def _make_hook(self, media_item: MediaItem) -> ProgressHook:
if media_item.is_segment and not media_item.extra_info.get("MUX_STREAM"):
return self.manager.scrape_mapper.tui.downloads.download_hls_seg()
- size = (media_item.size + resume_point) if media_item.size is not None else None
+ size = media_item.size
return self.manager.scrape_mapper.tui.downloads.download_file(
media_item.filename,
media_item.domain,
@@ -153,7 +157,7 @@ class DownloadClient:
async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:
check_free_space = storage.create_free_space_checker(media_item)
check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)
- await check_free_space(media_item.size)
+ await check_free_space(_get_content_length(resp.headers))
await self._pre_download_check(media_item)
async with self._track_speed(hook), aio.open(media_item.partial_file, mode="ab") as f:
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 5560dc09e..2ee091fc0 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, final, override
from mega.chunker import MegaChunker, get_chunks
from cyberdrop_dl import aio, storage
-from cyberdrop_dl.clients.downloads import DownloadClient, make_speed_checker
+from cyberdrop_dl.clients.downloads import DownloadClient, _get_content_length, make_speed_checker
from cyberdrop_dl.downloader.http import Downloader
if TYPE_CHECKING:
@@ -24,7 +24,7 @@ class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIs
check_free_space = storage.create_free_space_checker(media_item)
check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)
- await check_free_space(media_item.size)
+ await check_free_space(_get_content_length(resp.headers))
await self._pre_download_check(media_item)
crypto, file_size = media_item.extra_info[media_item.domain]["key"]
diff --git a/tests/test_download_resume.py b/tests/test_download_resume.py
new file mode 100644
index 000000000..f608e303e
--- /dev/null
+++ b/tests/test_download_resume.py
@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING, Self
+from unittest import mock
+
+import pytest
+from multidict import CIMultiDict, CIMultiDictProxy
+
+from cyberdrop_dl.clients.downloads import DownloadClient
+from cyberdrop_dl.clients.response import AbstractResponse
+from cyberdrop_dl.exceptions import DownloadError
+from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+ from pathlib import Path
+
+ from cyberdrop_dl.manager import Manager
+
+
+_PAYLOAD = bytes(range(256)) * 4_000 # 1,024,000 bytes
+_DOMAIN = "example"
+
+
+class _FakeResponse(AbstractResponse[bytes]):
+ __slots__ = ()
+
+ async def _read(self) -> bytes:
+ return self._resp
+
+ async def _read_text(self, encoding: str | None = None) -> str:
+ return self._resp.decode(encoding or "utf-8", errors="replace")
+
+ async def iter_chunked(self, size: int) -> AsyncIterator[bytes]:
+ for start in range(0, len(self._resp), size):
+ yield self._resp[start : start + size]
+
+ async def aclose(self) -> None:
+ return None
+
+ @classmethod
+ def partial_content(cls, start: int, *, truncate_to: int | None = None) -> Self:
+ body = _PAYLOAD[start:]
+ headers = {
+ "Content-Type": "video/mp4",
+ "Content-Length": str(len(body)),
+ "Content-Range": f"bytes {start}-{len(_PAYLOAD) - 1}/{len(_PAYLOAD)}",
+ }
+ return cls(
+ content_type="video/mp4",
+ status=206,
+ headers=CIMultiDictProxy(CIMultiDict(headers)),
+ url=AbsoluteHttpURL("https://cdn.example.com/video.mp4"),
+ location=None,
+ _resp=body[:truncate_to],
+ )
+
+
+class _Hook:
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(self, *_: object) -> None:
+ return None
+
+ def advance(self, _: int) -> None:
+ return None
+
+ def get_speed(self) -> float:
+ return 0
+
+
+async def _resumable_item(manager: Manager, tmp_path: Path, resume_point: int) -> MediaItem:
+ folder = tmp_path / "downloads"
+ folder.mkdir()
+ item = MediaItem(
+ url=AbsoluteHttpURL("https://cdn.example.com/video.mp4"),
+ domain=_DOMAIN,
+ download_folder=folder,
+ filename="video.mp4",
+ db_path="/video.mp4",
+ referer=AbsoluteHttpURL("https://example.com/v/1"),
+ album_id=None,
+ ext=".mp4",
+ original_filename="video.mp4",
+ uploaded_at=None,
+ )
+ item.download_filename = "video.mp4"
+ await manager.database.history.insert_incompleted(_DOMAIN, item)
+
+ item.partial_file = folder / "video.mp4.part"
+ item.partial_file.write_bytes(_PAYLOAD[:resume_point])
+ return item
+
+
[email protected]("resume_pct", [0.2, 0.4, 0.6, 0.8])
+async def test_resumed_download_keeps_partial_bytes(
+ running_manager: Manager, tmp_path: Path, resume_pct: float
+) -> None:
+ resume_point = int(len(_PAYLOAD) * resume_pct)
+ item = await _resumable_item(running_manager, tmp_path, resume_point)
+ client = DownloadClient(running_manager)
+ resp = _FakeResponse.partial_content(resume_point)
+ with mock.patch.object(DownloadClient, "_make_hook", return_value=_Hook()):
+ assert await client._process_response(item, _DOMAIN, resume_point, resp)
+
+ data = item.partial_file.read_bytes()
+ assert len(data) == len(_PAYLOAD)
+ assert hashlib.sha256(data).digest() == hashlib.sha256(_PAYLOAD).digest()
+
+
+async def test_truncated_resume_is_not_marked_complete(running_manager: Manager, tmp_path: Path) -> None:
+ resume_point = int(len(_PAYLOAD) * 0.3)
+ truncate_to = int(len(_PAYLOAD) * 0.5)
+ item = await _resumable_item(running_manager, tmp_path, resume_point)
+ client = DownloadClient(running_manager)
+ resp = _FakeResponse.partial_content(resume_point, truncate_to=truncate_to)
+ with (
+ mock.patch.object(DownloadClient, "_make_hook", return_value=_Hook()),
+ pytest.raises(DownloadError, match="Corrupted File"),
+ ):
+ await client._process_response(item, _DOMAIN, resume_point, resp)