Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: media_item task removal
NTFSvolumea2abbc5f8386
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 854911076..3d092375e 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -216,14 +216,18 @@ class DownloadClient:log(msg, 30)media_item.datetime = last_modified- media_item.task_id = self.manager.progress_manager.file_progress.add_task(- domain=domain,- filename=media_item.filename,- expected_size=media_item.filesize + resume_point,- )+ task_id = media_item.task_id+ if task_id is None:+ task_id = self.manager.progress_manager.file_progress.add_task(+ domain=domain,+ filename=media_item.filename,+ expected_size=media_item.filesize + resume_point,+ )+ media_item.set_task_id(task_id)+if media_item.partial_file.is_file():resume_point = media_item.partial_file.stat().st_size- self.manager.progress_manager.file_progress.advance_file(media_item.task_id, resume_point)+ self.manager.progress_manager.file_progress.advance_file(task_id, resume_point)await save_content(resp.content)return Truediff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex ab3a3d161..5718e7ee3 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -1,7 +1,6 @@from __future__ import annotationsimport asyncio-import contextlibimport refrom dataclasses import fieldfrom functools import wraps@@ -234,9 +233,12 @@ class Downloader:def attempt_task_removal(self, media_item: MediaItem) -> None:"""Attempts to remove the task from the progress bar."""if media_item.task_id is not None:- with contextlib.suppress(ValueError):+ try:self.manager.progress_manager.file_progress.remove_task(media_item.task_id)- media_item.task_id = None+ except ValueError:+ pass++ media_item.set_task_id(None)"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""diff --git a/cyberdrop_dl/utils/data_enums_classes/url_objects.py b/cyberdrop_dl/utils/data_enums_classes/url_objects.pyindex ab84a9fb2..63b987689 100644--- a/cyberdrop_dl/utils/data_enums_classes/url_objects.py+++ b/cyberdrop_dl/utils/data_enums_classes/url_objects.py@@ -51,15 +51,16 @@ class MediaItem:is_segment: bool = False# exclude from __init__+ parent_media_item: MediaItem | None = field(init=False, default=None, hash=False, compare=False)file_lock_reference_name: str | None = field(default=None, init=False)download_filename: str | None = field(default=None, init=False)filesize: int | None = field(default=None, init=False)current_attempt: int = field(default=0, init=False, hash=False, compare=False)partial_file: Path | None = field(default=None, init=False)complete_file: Path = field(default=None, init=False) # type: ignore- task_id: TaskID = field(init=False, hash=False, compare=False)hash: str | None = field(default=None, init=False, hash=False, compare=False)downloaded: bool = field(default=False, init=False, hash=False, compare=False)+ _task_id: TaskID | None = field(default=None, init=False, hash=False, compare=False)# slots for __post_init__referer: URL = field(init=False)@@ -75,8 +76,26 @@ class MediaItem:self.original_filename = self.original_filename or self.filenameself.parents = origin.parents.copy()self.datetime = origin.possible_datetime if isinstance(origin, ScrapeItem) else origin.datetime+ self.parent_media_item = None if isinstance(origin, ScrapeItem) else originself.parent_threads = origin.parent_threads.copy()+ @property+ def task_id(self) -> TaskID | None:+ if self.parent_media_item is not None:+ return self.parent_media_item.task_id+ return self._task_id++ def set_task_id(self, task_id: TaskID | None) -> None:+ if self.task_id is not None and task_id is not None:+ # We already have a task_id; we can't replace it, only reset it.+ # This should never happen. Calling code should always check the value before making a new task.+ # We can't silently ignore it either because we will lose any reference to the created task.+ raise ValueError("task_id is already set")+ if self.parent_media_item is not None:+ self.parent_media_item.set_task_id(task_id)+ else:+ self._task_id = task_id+@dataclass(kw_only=True, slots=True)class ScrapeItem: