Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add alternative constructor for media_item (#1183)
NTFSvolumea1f6de52ff46
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 9c2f15e4c..47e8f6a98 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -162,7 +162,7 @@ class DownloadClient:if isinstance(gen, list):fallback_urls: list[AbsoluteHttpURL] | None = genelse:- fallback_call: Callable[..., AbsoluteHttpURL] | None = gen+ fallback_call: Callable[[aiohttp.ClientResponse, int], AbsoluteHttpURL] | None = gendef gen_fallback() -> Generator[AbsoluteHttpURL | None, aiohttp.ClientResponse, None]:response = yielddiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 6a7209941..4df693ed5 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -288,7 +288,9 @@ class Crawler(ABC):if isinstance(debrid_link, URL):assert is_absolute_http_url(debrid_link)download_folder = get_download_path(self.manager, scrape_item, self.FOLDER_DOMAIN)- media_item = MediaItem(url, scrape_item, download_folder, filename, original_filename, debrid_link, ext=ext)+ media_item = MediaItem.from_item(+ scrape_item, url, download_folder, filename, original_filename, debrid_link, ext=ext+ )await self.handle_media_item(media_item, m3u8)async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 5536652af..8773eb046 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -1,7 +1,7 @@from __future__ import annotationsimport copy-from dataclasses import InitVar, dataclass, field+from dataclasses import dataclass, fieldfrom enum import IntEnumfrom functools import partialmethodfrom pathlib import Path@@ -21,6 +21,7 @@ if TYPE_CHECKING:import inspectfrom collections.abc import Callable+ import aiohttpfrom propcache.api import under_cached_property as cached_propertyfrom rich.progress import TaskID@@ -140,49 +141,65 @@ class HlsSegment(NamedTuple):url: AbsoluteHttpURL-@dataclass(unsafe_hash=True, slots=True)+@dataclass(unsafe_hash=True, slots=True, kw_only=True)class MediaItem:url: AbsoluteHttpURL- origin: InitVar[ScrapeItem | MediaItem]+ referer: AbsoluteHttpURLdownload_folder: Pathfilename: str- original_filename: str | None = None- debrid_link: AbsoluteHttpURL | None = field(default=None, hash=False, compare=False)- duration: float | None = field(default=None, hash=False, compare=False)- ext: str = ""+ original_filename: str+ ext: str+ debrid_link: AbsoluteHttpURL | None = field(default=None, compare=False)+ duration: float | None = field(default=None, compare=False)is_segment: bool = False- fallbacks: Callable[..., AbsoluteHttpURL] | list[AbsoluteHttpURL] | None = field(- default=None, hash=False, compare=False+ fallbacks: Callable[[aiohttp.ClientResponse, int], AbsoluteHttpURL] | list[AbsoluteHttpURL] | None = field(+ default=None, compare=False)+ album_id: str | None = None+ datetime: int | None = field(default=None, compare=False)+ parents: list[AbsoluteHttpURL] = field(default_factory=list, compare=False)+ parent_threads: set[AbsoluteHttpURL] = field(default_factory=set, compare=False)+ parent_media_item: MediaItem | None = field(default=None, compare=False)+ download_filename: str | None = field(default=None)+ filesize: int | None = field(default=None, compare=False)+ current_attempt: int = field(default=0, compare=False)+ partial_file: Path = None # type: ignore+ complete_file: Path = None # type: ignore+ hash: str | None = field(default=None, compare=False)+ downloaded: bool = field(default=False, compare=False)+ _task_id: TaskID | None = field(default=None, compare=False)- # exclude from __init__- parent_media_item: MediaItem | None = field(init=False, default=None, hash=False, compare=False)- file_lock_reference_name: str | None = field(default=None, init=False)- download_filename: str | None = field(default=None, init=False)- filesize: int | None = field(default=None, init=False)- current_attempt: int = field(default=0, init=False, hash=False, compare=False)- partial_file: Path = field(default=None, init=False) # type: ignore- complete_file: Path = field(default=None, init=False) # type: ignore- hash: str | None = field(default=None, init=False, hash=False, compare=False)- downloaded: bool = field(default=False, init=False, hash=False, compare=False)- _task_id: TaskID | None = field(default=None, init=False, hash=False, compare=False)-- # slots for __post_init__- referer: AbsoluteHttpURL = field(init=False)- album_id: str | None = field(init=False)- datetime: int | None = field(init=False, hash=False, compare=False)- parents: list[AbsoluteHttpURL] = field(init=False, hash=False, compare=False)- parent_threads: set[AbsoluteHttpURL] = field(init=False, hash=False, compare=False)-- def __post_init__(self, origin: ScrapeItem | MediaItem) -> None:- self.referer = origin.url- self.album_id = origin.album_id- self.ext = self.ext or Path(self.filename).suffix- self.original_filename = self.original_filename or self.filename- self.parents = origin.parents.copy()- self.datetime = origin.possible_datetime if isinstance(origin, ScrapeItem) else origin.datetime- self.parent_media_item = None if isinstance(origin, ScrapeItem) else origin- self.parent_threads = origin.parent_threads.copy()+ @staticmethod+ def from_item(+ origin: ScrapeItem | MediaItem,+ url: AbsoluteHttpURL,+ /,+ download_folder: Path,+ filename: str,+ original_filename: str | None = None,+ debrid_link: AbsoluteHttpURL | None = None,+ duration: float | None = None,+ ext: str = "",+ is_segment: bool = False,+ fallbacks: Callable[[aiohttp.ClientResponse, int], AbsoluteHttpURL] | list[AbsoluteHttpURL] | None = None,+ ) -> MediaItem:+ return MediaItem(+ url=url,+ download_folder=download_folder,+ filename=filename,+ debrid_link=debrid_link,+ duration=duration,+ referer=origin.url,+ album_id=origin.album_id,+ ext=ext or Path(filename).suffix,+ original_filename=original_filename or filename,+ is_segment=is_segment,+ fallbacks=fallbacks,+ parents=origin.parents.copy(),+ datetime=origin.possible_datetime if isinstance(origin, ScrapeItem) else origin.datetime,+ parent_media_item=None if isinstance(origin, ScrapeItem) else origin,+ parent_threads=origin.parent_threads.copy(),+ )@propertydef task_id(self) -> TaskID | None:diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex f6065b31b..5d6842fee 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -219,9 +219,9 @@ class Downloader:yield HlsSegment(segment.title, name, parse_url(segment.absolute_uri))def make_download_task(segment: HlsSegment) -> Coroutine:- seg_media_item = MediaItem(- url=segment.url,- origin=media_item,+ seg_media_item = MediaItem.from_item(+ media_item,+ segment.url,download_folder=download_folder,filename=segment.name,ext=media_item.ext,@@ -343,9 +343,7 @@ class Downloader:async def start_download(self, media_item: MediaItem) -> bool:if not media_item.is_segment:log(f"{self.log_prefix} starting: {media_item.url}", 20)- if not media_item.file_lock_reference_name:- media_item.file_lock_reference_name = media_item.filename- lock = self._file_lock_vault.get_lock(media_item.file_lock_reference_name)+ lock = self._file_lock_vault.get_lock(media_item.filename)async with lock:return bool(await self.download(media_item))diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex bf7a550ed..0c32b97d9 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -244,7 +244,7 @@ class ScrapeMapper:filename, _ = get_filename_and_ext(scrape_item.url.name)except NoExtensionError:filename, _ = get_filename_and_ext(scrape_item.url.name, forum=True)- media_item = MediaItem(scrape_item.url, scrape_item, download_folder, filename)+ media_item = MediaItem.from_item(scrape_item, scrape_item.url, download_folder, filename)self.manager.task_group.create_task(self.no_crawler_downloader.run(media_item))return