Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rename some URL objs properties

NTFSvolumee901a417732c
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.py
index e58de99d2..f5057463c 100644
--- a/cyberdrop_dl/clients/downloads.py
+++ b/cyberdrop_dl/clients/downloads.py
@@ -115,7 +115,7 @@ class DownloadClient:
_check_content_type(content_type, media_item.ext)
try:
- media_item.filesize = int(resp.headers[hdrs.CONTENT_LENGTH])
+ media_item.size = int(resp.headers[hdrs.CONTENT_LENGTH])
except KeyError:
msg = f"Download response has no `{hdrs.CONTENT_LENGTH}` header. Refusing to download"
raise DownloadError(HTTPStatus.LENGTH_REQUIRED, msg, retry=False) from None
@@ -152,7 +152,7 @@ class DownloadClient:
if media_item.is_segment:
return self.manager.scrape_mapper.tui.downloads.download_hls_seg()
- size = (media_item.filesize + resume_point) if media_item.filesize is not None else None
+ size = (media_item.size + resume_point) if media_item.size is not None else None
return self.manager.scrape_mapper.tui.downloads.download_file(
media_item.filename,
media_item.domain,
@@ -189,11 +189,11 @@ class DownloadClient:
await aio.unlink(media_item.partial_file, missing_ok=True)
raise DownloadError(HTTPStatus.INTERNAL_SERVER_ERROR, "File is empty")
- assert media_item.filesize
- if size < media_item.filesize:
+ assert media_item.size
+ if size < media_item.size:
await aio.unlink(media_item.partial_file, missing_ok=True)
raise DownloadError(
- "Corrupted File", f"Expected at least {media_item.filesize:,} bytes but only got {size:,} bytes"
+ "Corrupted File", f"Expected at least {media_item.size:,} bytes but only got {size:,} bytes"
)
async def download_file(self, domain: str, media_item: MediaItem) -> bool:
@@ -277,7 +277,7 @@ class DownloadClient:
part_suffix = media_item.path.suffix + constants.TempExt.PART
media_item.partial_file = media_item.path.with_suffix(part_suffix)
- expected_size = media_item.filesize
+ expected_size = media_item.size
proceed = True
skip = False
@@ -293,7 +293,7 @@ class DownloadClient:
if not media_item.path.exists() and not media_item.partial_file.exists():
break
- if media_item.path.exists() and media_item.path.stat().st_size == media_item.filesize:
+ if media_item.path.exists() and media_item.path.stat().st_size == media_item.size:
logger.info(f"Found {media_item.path.name} locally, skipping download")
proceed = False
break
@@ -312,13 +312,13 @@ class DownloadClient:
if media_item.filename == downloaded_filename:
if media_item.partial_file.exists():
logger.info(f"Found {downloaded_filename} locally, trying to resume")
- assert media_item.filesize
+ assert media_item.size
size = media_item.partial_file.stat().st_size
- if size >= media_item.filesize:
+ if size >= media_item.size:
logger.info(f"Deleting partial file {media_item.partial_file}. Size is out of bound")
media_item.partial_file.unlink()
- elif size == media_item.filesize:
+ elif size == media_item.size:
if media_item.path.exists():
logger.warning(
f"Found conflicting complete file '{media_item.path}' locally, iterating filename"
@@ -339,7 +339,7 @@ class DownloadClient:
f"Renaming found partial file '{media_item.partial_file}' to complete file {media_item.path}"
)
elif media_item.path.exists():
- if media_item.path.stat().st_size == media_item.filesize:
+ if media_item.path.stat().st_size == media_item.size:
logger.info(f"Found complete file '{media_item.path}' locally, skipping download")
proceed = False
else:
@@ -377,13 +377,13 @@ class DownloadClient:
"""Checks if the file size is within the limits."""
limits = self.manager.config.file_size_limits.ranges
- assert media.filesize is not None
+ assert media.size is not None
if media.ext in FileExt.IMAGE:
- return media.filesize in limits.image
+ return media.size in limits.image
if media.ext in FileExt.VIDEO:
- return media.filesize in limits.video
+ return media.size in limits.video
- return media.filesize in limits.other
+ return media.size in limits.other
def _check_content_type(content_type: str, ext: str) -> str | None:
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.py
index 2711ecab7..65112b8eb 100644
--- a/cyberdrop_dl/hasher.py
+++ b/cyberdrop_dl/hasher.py
@@ -197,7 +197,7 @@ class Hasher:
assert size
self.hashed_media_items.append(media_item)
if hash_value:
- media_item.hash = hash_value
+ media_item.xxhash = hash_value
self.hashes_dict[hash_value][size].add(absolute_path)
self._hashed_items.add(media_item.id)
diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.py
index 857546cc3..7740898c0 100644
--- a/cyberdrop_dl/url_objects.py
+++ b/cyberdrop_dl/url_objects.py
@@ -122,25 +122,25 @@ class MediaItem:
domain: str
referer: AbsoluteHttpURL
download_folder: Path
+ db_path: str
filename: str
+
original_filename: str = ""
download_filename: str | None = None
- filesize: int | None = None
- ext: str
- db_path: str
-
- debrid_link: AbsoluteHttpURL | None = None
+ ext: str = ""
+ size: int | None = None
+ debrid_url: AbsoluteHttpURL | None = None
duration: float | None = None
is_segment: bool = False
album_id: str | None = None
uploaded_at: int | None = None
+ xxhash: str | None = None
parents: tuple[AbsoluteHttpURL, ...] = dataclasses.field(default_factory=tuple)
attempts: int = dataclasses.field(init=False, default=0)
partial_file: Path = dataclasses.field(init=False)
path: Path = dataclasses.field(init=False)
- hash: str | None = None
- downloaded: bool = dataclasses.field(default=False)
+ downloaded: bool = dataclasses.field(init=False, default=False)
metadata: object = dataclasses.field(init=False, default_factory=dict)
@@ -149,9 +149,10 @@ class MediaItem:
id: tuple[str, ...] = dataclasses.field(init=False)
base64_id: str = dataclasses.field(init=False)
- headers: dict[str, str] = dataclasses.field(default_factory=dict)
+ headers: dict[str, str] = dataclasses.field(init=False, default_factory=dict)
def __post_init__(self) -> None:
+ self.ext = self.ext or Path(self.filename).suffix
self.id = self.domain, self.db_path
self.original_filename = self.original_filename or self.filename
if self.url.scheme == "metadata":
@@ -169,7 +170,7 @@ class MediaItem:
@property
def real_url(self) -> AbsoluteHttpURL:
- return self.debrid_link or self.url
+ return self.debrid_url or self.url
@staticmethod
def from_item( # noqa: PLR0913
@@ -192,7 +193,7 @@ class MediaItem:
db_path=db_path,
referer=origin.url,
album_id=origin.album_id,
- ext=ext or Path(filename).suffix,
+ ext=ext,
original_filename=original_filename or filename,
parents=tuple(origin.parents),
uploaded_at=origin.uploaded_at,
@@ -203,8 +204,8 @@ class MediaItem:
if not hasattr(self, attr):
setattr(self, attr, None)
me = dataclasses.asdict(self)
- if self.hash:
- me["hash"] = f"xxh128:{self.hash}"
+ if self.xxhash:
+ me["xxhash"] = f"xxh128:{self.xxhash}"
for name in ("is_segment",):
del me[name]
return me
@@ -219,14 +220,14 @@ class ScrapeItem:
download_folder: Path = Path("downloads")
folders: list[str] = dataclasses.field(init=False, default_factory=list)
- parents: list[AbsoluteHttpURL] = dataclasses.field(default_factory=list, init=False)
- parent_threads: set[AbsoluteHttpURL] = dataclasses.field(default_factory=set, init=False)
- children_limits: tuple[int, ...] = dataclasses.field(default_factory=tuple, init=False)
+ parents: list[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=list)
+ parent_threads: set[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=set)
+ children_limits: tuple[int, ...] = dataclasses.field(init=False, default_factory=tuple)
password: str | None = dataclasses.field(init=False)
- _children_count: int = dataclasses.field(default=0, init=False)
- _children_limit: int = dataclasses.field(default=0, init=False)
- _type: ScrapeItemType | None = dataclasses.field(default=None, init=False)
+ _children_count: int = dataclasses.field(init=False, default=0)
+ _children_limit: int = dataclasses.field(init=False, default=0)
+ _type: ScrapeItemType | None = dataclasses.field(init=False, default=None)
_uploaded_at: int | None = dataclasses.field(init=False, default=None)
def __post_init__(self) -> None: