Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add alternative constructor for media_item (#1183)

NTFSvolumea1f6de52ff46
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 9c2f15e4c..47e8f6a98 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -162,7 +162,7 @@ class DownloadClient:
if isinstance(gen, list):
fallback_urls: list[AbsoluteHttpURL] | None = gen
else:
- fallback_call: Callable[..., AbsoluteHttpURL] | None = gen
+ fallback_call: Callable[[aiohttp.ClientResponse, int], AbsoluteHttpURL] | None = gen
def gen_fallback() -> Generator[AbsoluteHttpURL | None, aiohttp.ClientResponse, None]:
response = yield
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 6a7209941..4df693ed5 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -288,7 +288,9 @@ class Crawler(ABC):
if isinstance(debrid_link, URL):
assert is_absolute_http_url(debrid_link)
download_folder = get_download_path(self.manager, scrape_item, self.FOLDER_DOMAIN)
- media_item = MediaItem(url, scrape_item, download_folder, filename, original_filename, debrid_link, ext=ext)
+ media_item = MediaItem.from_item(
+ scrape_item, url, download_folder, filename, original_filename, debrid_link, ext=ext
+ )
await self.handle_media_item(media_item, m3u8)
async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 5536652af..8773eb046 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import copy
-from dataclasses import InitVar, dataclass, field
+from dataclasses import dataclass, field
from enum import IntEnum
from functools import partialmethod
from pathlib import Path
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
import inspect
from collections.abc import Callable
+ import aiohttp
from propcache.api import under_cached_property as cached_property
from rich.progress import TaskID
@@ -140,49 +141,65 @@ class HlsSegment(NamedTuple):
url: AbsoluteHttpURL
-@dataclass(unsafe_hash=True, slots=True)
+@dataclass(unsafe_hash=True, slots=True, kw_only=True)
class MediaItem:
url: AbsoluteHttpURL
- origin: InitVar[ScrapeItem | MediaItem]
+ referer: AbsoluteHttpURL
download_folder: Path
filename: str
- original_filename: str | None = None
- debrid_link: AbsoluteHttpURL | None = field(default=None, hash=False, compare=False)
- duration: float | None = field(default=None, hash=False, compare=False)
- ext: str = ""
+ original_filename: str
+ ext: str
+ debrid_link: AbsoluteHttpURL | None = field(default=None, compare=False)
+ duration: float | None = field(default=None, compare=False)
is_segment: bool = False
- fallbacks: Callable[..., AbsoluteHttpURL] | list[AbsoluteHttpURL] | None = field(
- default=None, hash=False, compare=False
+ fallbacks: Callable[[aiohttp.ClientResponse, int], AbsoluteHttpURL] | list[AbsoluteHttpURL] | None = field(
+ default=None, compare=False
)
+ album_id: str | None = None
+ datetime: int | None = field(default=None, compare=False)
+ parents: list[AbsoluteHttpURL] = field(default_factory=list, compare=False)
+ parent_threads: set[AbsoluteHttpURL] = field(default_factory=set, compare=False)
+ parent_media_item: MediaItem | None = field(default=None, compare=False)
+ download_filename: str | None = field(default=None)
+ filesize: int | None = field(default=None, compare=False)
+ current_attempt: int = field(default=0, compare=False)
+ partial_file: Path = None # type: ignore
+ complete_file: Path = None # type: ignore
+ hash: str | None = field(default=None, compare=False)
+ downloaded: bool = field(default=False, compare=False)
+ _task_id: TaskID | None = field(default=None, compare=False)
- # exclude from __init__
- parent_media_item: MediaItem | None = field(init=False, default=None, hash=False, compare=False)
- file_lock_reference_name: str | None = field(default=None, init=False)
- download_filename: str | None = field(default=None, init=False)
- filesize: int | None = field(default=None, init=False)
- current_attempt: int = field(default=0, init=False, hash=False, compare=False)
- partial_file: Path = field(default=None, init=False) # type: ignore
- complete_file: Path = field(default=None, init=False) # type: ignore
- hash: str | None = field(default=None, init=False, hash=False, compare=False)
- downloaded: bool = field(default=False, init=False, hash=False, compare=False)
- _task_id: TaskID | None = field(default=None, init=False, hash=False, compare=False)
-
- # slots for __post_init__
- referer: AbsoluteHttpURL = field(init=False)
- album_id: str | None = field(init=False)
- datetime: int | None = field(init=False, hash=False, compare=False)
- parents: list[AbsoluteHttpURL] = field(init=False, hash=False, compare=False)
- parent_threads: set[AbsoluteHttpURL] = field(init=False, hash=False, compare=False)
-
- def __post_init__(self, origin: ScrapeItem | MediaItem) -> None:
- self.referer = origin.url
- self.album_id = origin.album_id
- self.ext = self.ext or Path(self.filename).suffix
- self.original_filename = self.original_filename or self.filename
- self.parents = origin.parents.copy()
- self.datetime = origin.possible_datetime if isinstance(origin, ScrapeItem) else origin.datetime
- self.parent_media_item = None if isinstance(origin, ScrapeItem) else origin
- self.parent_threads = origin.parent_threads.copy()
+ @staticmethod
+ def from_item(
+ origin: ScrapeItem | MediaItem,
+ url: AbsoluteHttpURL,
+ /,
+ download_folder: Path,
+ filename: str,
+ original_filename: str | None = None,
+ debrid_link: AbsoluteHttpURL | None = None,
+ duration: float | None = None,
+ ext: str = "",
+ is_segment: bool = False,
+ fallbacks: Callable[[aiohttp.ClientResponse, int], AbsoluteHttpURL] | list[AbsoluteHttpURL] | None = None,
+ ) -> MediaItem:
+ return MediaItem(
+ url=url,
+ download_folder=download_folder,
+ filename=filename,
+ debrid_link=debrid_link,
+ duration=duration,
+ referer=origin.url,
+ album_id=origin.album_id,
+ ext=ext or Path(filename).suffix,
+ original_filename=original_filename or filename,
+ is_segment=is_segment,
+ fallbacks=fallbacks,
+ parents=origin.parents.copy(),
+ datetime=origin.possible_datetime if isinstance(origin, ScrapeItem) else origin.datetime,
+ parent_media_item=None if isinstance(origin, ScrapeItem) else origin,
+ parent_threads=origin.parent_threads.copy(),
+ )
@property
def task_id(self) -> TaskID | None:
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index f6065b31b..5d6842fee 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -219,9 +219,9 @@ class Downloader:
yield HlsSegment(segment.title, name, parse_url(segment.absolute_uri))
def make_download_task(segment: HlsSegment) -> Coroutine:
- seg_media_item = MediaItem(
- url=segment.url,
- origin=media_item,
+ seg_media_item = MediaItem.from_item(
+ media_item,
+ segment.url,
download_folder=download_folder,
filename=segment.name,
ext=media_item.ext,
@@ -343,9 +343,7 @@ class Downloader:
async def start_download(self, media_item: MediaItem) -> bool:
if not media_item.is_segment:
log(f"{self.log_prefix} starting: {media_item.url}", 20)
- if not media_item.file_lock_reference_name:
- media_item.file_lock_reference_name = media_item.filename
- lock = self._file_lock_vault.get_lock(media_item.file_lock_reference_name)
+ lock = self._file_lock_vault.get_lock(media_item.filename)
async with lock:
return bool(await self.download(media_item))
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index bf7a550ed..0c32b97d9 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -244,7 +244,7 @@ class ScrapeMapper:
filename, _ = get_filename_and_ext(scrape_item.url.name)
except NoExtensionError:
filename, _ = get_filename_and_ext(scrape_item.url.name, forum=True)
- media_item = MediaItem(scrape_item.url, scrape_item, download_folder, filename)
+ media_item = MediaItem.from_item(scrape_item, scrape_item.url, download_folder, filename)
self.manager.task_group.create_task(self.no_crawler_downloader.run(media_item))
return