Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rename some URL objs properties
NTFSvolumee901a417732c
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.pyindex e58de99d2..f5057463c 100644--- a/cyberdrop_dl/clients/downloads.py+++ b/cyberdrop_dl/clients/downloads.py@@ -115,7 +115,7 @@ class DownloadClient:_check_content_type(content_type, media_item.ext)try:- media_item.filesize = int(resp.headers[hdrs.CONTENT_LENGTH])+ media_item.size = int(resp.headers[hdrs.CONTENT_LENGTH])except KeyError:msg = f"Download response has no `{hdrs.CONTENT_LENGTH}` header. Refusing to download"raise DownloadError(HTTPStatus.LENGTH_REQUIRED, msg, retry=False) from None@@ -152,7 +152,7 @@ class DownloadClient:if media_item.is_segment:return self.manager.scrape_mapper.tui.downloads.download_hls_seg()- size = (media_item.filesize + resume_point) if media_item.filesize is not None else None+ size = (media_item.size + resume_point) if media_item.size is not None else Nonereturn self.manager.scrape_mapper.tui.downloads.download_file(media_item.filename,media_item.domain,@@ -189,11 +189,11 @@ class DownloadClient:await aio.unlink(media_item.partial_file, missing_ok=True)raise DownloadError(HTTPStatus.INTERNAL_SERVER_ERROR, "File is empty")- assert media_item.filesize- if size < media_item.filesize:+ assert media_item.size+ if size < media_item.size:await aio.unlink(media_item.partial_file, missing_ok=True)raise DownloadError(- "Corrupted File", f"Expected at least {media_item.filesize:,} bytes but only got {size:,} bytes"+ "Corrupted File", f"Expected at least {media_item.size:,} bytes but only got {size:,} bytes")async def download_file(self, domain: str, media_item: MediaItem) -> bool:@@ -277,7 +277,7 @@ class DownloadClient:part_suffix = media_item.path.suffix + constants.TempExt.PARTmedia_item.partial_file = media_item.path.with_suffix(part_suffix)- expected_size = media_item.filesize+ expected_size = media_item.sizeproceed = Trueskip = False@@ -293,7 +293,7 @@ class DownloadClient:if not media_item.path.exists() and not media_item.partial_file.exists():break- if media_item.path.exists() and media_item.path.stat().st_size == media_item.filesize:+ if media_item.path.exists() and media_item.path.stat().st_size == media_item.size:logger.info(f"Found {media_item.path.name} locally, skipping download")proceed = Falsebreak@@ -312,13 +312,13 @@ class DownloadClient:if media_item.filename == downloaded_filename:if media_item.partial_file.exists():logger.info(f"Found {downloaded_filename} locally, trying to resume")- assert media_item.filesize+ assert media_item.sizesize = media_item.partial_file.stat().st_size- if size >= media_item.filesize:+ if size >= media_item.size:logger.info(f"Deleting partial file {media_item.partial_file}. Size is out of bound")media_item.partial_file.unlink()- elif size == media_item.filesize:+ elif size == media_item.size:if media_item.path.exists():logger.warning(f"Found conflicting complete file '{media_item.path}' locally, iterating filename"@@ -339,7 +339,7 @@ class DownloadClient:f"Renaming found partial file '{media_item.partial_file}' to complete file {media_item.path}")elif media_item.path.exists():- if media_item.path.stat().st_size == media_item.filesize:+ if media_item.path.stat().st_size == media_item.size:logger.info(f"Found complete file '{media_item.path}' locally, skipping download")proceed = Falseelse:@@ -377,13 +377,13 @@ class DownloadClient:"""Checks if the file size is within the limits."""limits = self.manager.config.file_size_limits.ranges- assert media.filesize is not None+ assert media.size is not Noneif media.ext in FileExt.IMAGE:- return media.filesize in limits.image+ return media.size in limits.imageif media.ext in FileExt.VIDEO:- return media.filesize in limits.video+ return media.size in limits.video- return media.filesize in limits.other+ return media.size in limits.otherdef _check_content_type(content_type: str, ext: str) -> str | None:diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.pyindex 2711ecab7..65112b8eb 100644--- a/cyberdrop_dl/hasher.py+++ b/cyberdrop_dl/hasher.py@@ -197,7 +197,7 @@ class Hasher:assert sizeself.hashed_media_items.append(media_item)if hash_value:- media_item.hash = hash_value+ media_item.xxhash = hash_valueself.hashes_dict[hash_value][size].add(absolute_path)self._hashed_items.add(media_item.id)diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.pyindex 857546cc3..7740898c0 100644--- a/cyberdrop_dl/url_objects.py+++ b/cyberdrop_dl/url_objects.py@@ -122,25 +122,25 @@ class MediaItem:domain: strreferer: AbsoluteHttpURLdownload_folder: Path+ db_path: strfilename: str+original_filename: str = ""download_filename: str | None = None- filesize: int | None = None- ext: str- db_path: str-- debrid_link: AbsoluteHttpURL | None = None+ ext: str = ""+ size: int | None = None+ debrid_url: AbsoluteHttpURL | None = Noneduration: float | None = Noneis_segment: bool = Falsealbum_id: str | None = Noneuploaded_at: int | None = None+ xxhash: str | None = Noneparents: tuple[AbsoluteHttpURL, ...] = dataclasses.field(default_factory=tuple)attempts: int = dataclasses.field(init=False, default=0)partial_file: Path = dataclasses.field(init=False)path: Path = dataclasses.field(init=False)- hash: str | None = None- downloaded: bool = dataclasses.field(default=False)+ downloaded: bool = dataclasses.field(init=False, default=False)metadata: object = dataclasses.field(init=False, default_factory=dict)@@ -149,9 +149,10 @@ class MediaItem:id: tuple[str, ...] = dataclasses.field(init=False)base64_id: str = dataclasses.field(init=False)- headers: dict[str, str] = dataclasses.field(default_factory=dict)+ headers: dict[str, str] = dataclasses.field(init=False, default_factory=dict)def __post_init__(self) -> None:+ self.ext = self.ext or Path(self.filename).suffixself.id = self.domain, self.db_pathself.original_filename = self.original_filename or self.filenameif self.url.scheme == "metadata":@@ -169,7 +170,7 @@ class MediaItem:@propertydef real_url(self) -> AbsoluteHttpURL:- return self.debrid_link or self.url+ return self.debrid_url or self.url@staticmethoddef from_item( # noqa: PLR0913@@ -192,7 +193,7 @@ class MediaItem:db_path=db_path,referer=origin.url,album_id=origin.album_id,- ext=ext or Path(filename).suffix,+ ext=ext,original_filename=original_filename or filename,parents=tuple(origin.parents),uploaded_at=origin.uploaded_at,@@ -203,8 +204,8 @@ class MediaItem:if not hasattr(self, attr):setattr(self, attr, None)me = dataclasses.asdict(self)- if self.hash:- me["hash"] = f"xxh128:{self.hash}"+ if self.xxhash:+ me["xxhash"] = f"xxh128:{self.xxhash}"for name in ("is_segment",):del me[name]return me@@ -219,14 +220,14 @@ class ScrapeItem:download_folder: Path = Path("downloads")folders: list[str] = dataclasses.field(init=False, default_factory=list)- parents: list[AbsoluteHttpURL] = dataclasses.field(default_factory=list, init=False)- parent_threads: set[AbsoluteHttpURL] = dataclasses.field(default_factory=set, init=False)- children_limits: tuple[int, ...] = dataclasses.field(default_factory=tuple, init=False)+ parents: list[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=list)+ parent_threads: set[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=set)+ children_limits: tuple[int, ...] = dataclasses.field(init=False, default_factory=tuple)password: str | None = dataclasses.field(init=False)- _children_count: int = dataclasses.field(default=0, init=False)- _children_limit: int = dataclasses.field(default=0, init=False)- _type: ScrapeItemType | None = dataclasses.field(default=None, init=False)+ _children_count: int = dataclasses.field(init=False, default=0)+ _children_limit: int = dataclasses.field(init=False, default=0)+ _type: ScrapeItemType | None = dataclasses.field(init=False, default=None)_uploaded_at: int | None = dataclasses.field(init=False, default=None)def __post_init__(self) -> None: