Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: media_item task removal

NTFSvolumea2abbc5f8386
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 854911076..3d092375e 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -216,14 +216,18 @@ class DownloadClient:
log(msg, 30)
media_item.datetime = last_modified
- media_item.task_id = self.manager.progress_manager.file_progress.add_task(
- domain=domain,
- filename=media_item.filename,
- expected_size=media_item.filesize + resume_point,
- )
+ task_id = media_item.task_id
+ if task_id is None:
+ task_id = self.manager.progress_manager.file_progress.add_task(
+ domain=domain,
+ filename=media_item.filename,
+ expected_size=media_item.filesize + resume_point,
+ )
+ media_item.set_task_id(task_id)
+
if media_item.partial_file.is_file():
resume_point = media_item.partial_file.stat().st_size
- self.manager.progress_manager.file_progress.advance_file(media_item.task_id, resume_point)
+ self.manager.progress_manager.file_progress.advance_file(task_id, resume_point)
await save_content(resp.content)
return True
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index ab3a3d161..5718e7ee3 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
-import contextlib
import re
from dataclasses import field
from functools import wraps
@@ -234,9 +233,12 @@ class Downloader:
def attempt_task_removal(self, media_item: MediaItem) -> None:
"""Attempts to remove the task from the progress bar."""
if media_item.task_id is not None:
- with contextlib.suppress(ValueError):
+ try:
self.manager.progress_manager.file_progress.remove_task(media_item.task_id)
- media_item.task_id = None
+ except ValueError:
+ pass
+
+ media_item.set_task_id(None)
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
diff --git a/cyberdrop_dl/utils/data_enums_classes/url_objects.py b/cyberdrop_dl/utils/data_enums_classes/url_objects.py
index ab84a9fb2..63b987689 100644
--- a/cyberdrop_dl/utils/data_enums_classes/url_objects.py
+++ b/cyberdrop_dl/utils/data_enums_classes/url_objects.py
@@ -51,15 +51,16 @@ class MediaItem:
is_segment: bool = False
# exclude from __init__
+ parent_media_item: MediaItem | None = field(init=False, default=None, hash=False, compare=False)
file_lock_reference_name: str | None = field(default=None, init=False)
download_filename: str | None = field(default=None, init=False)
filesize: int | None = field(default=None, init=False)
current_attempt: int = field(default=0, init=False, hash=False, compare=False)
partial_file: Path | None = field(default=None, init=False)
complete_file: Path = field(default=None, init=False) # type: ignore
- task_id: TaskID = field(init=False, hash=False, compare=False)
hash: str | None = field(default=None, init=False, hash=False, compare=False)
downloaded: bool = field(default=False, init=False, hash=False, compare=False)
+ _task_id: TaskID | None = field(default=None, init=False, hash=False, compare=False)
# slots for __post_init__
referer: URL = field(init=False)
@@ -75,8 +76,26 @@ class MediaItem:
self.original_filename = self.original_filename or self.filename
self.parents = origin.parents.copy()
self.datetime = origin.possible_datetime if isinstance(origin, ScrapeItem) else origin.datetime
+ self.parent_media_item = None if isinstance(origin, ScrapeItem) else origin
self.parent_threads = origin.parent_threads.copy()
+ @property
+ def task_id(self) -> TaskID | None:
+ if self.parent_media_item is not None:
+ return self.parent_media_item.task_id
+ return self._task_id
+
+ def set_task_id(self, task_id: TaskID | None) -> None:
+ if self.task_id is not None and task_id is not None:
+ # We already have a task_id; we can't replace it, only reset it.
+ # This should never happen. Calling code should always check the value before making a new task.
+ # We can't silently ignore it either because we will lose any reference to the created task.
+ raise ValueError("task_id is already set")
+ if self.parent_media_item is not None:
+ self.parent_media_item.set_task_id(task_id)
+ else:
+ self._task_id = task_id
+
@dataclass(kw_only=True, slots=True)
class ScrapeItem: