Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: make all file system calls async (#951)

NTFSvolume7ea85d42f86e
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index a354aeca3..ec1d8dee0 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -230,8 +230,8 @@ class DownloadClient:
return False
- if resp.status != HTTPStatus.PARTIAL_CONTENT and media_item.partial_file.is_file():
- media_item.partial_file.unlink()
+ if resp.status != HTTPStatus.PARTIAL_CONTENT:
+ await asyncio.to_thread(media_item.partial_file.unlink, missing_ok=True)
if not media_item.datetime and (last_modified := get_last_modified(resp.headers)):
msg = f"Unable to parse upload date for {media_item.url}, using `Last-Modified` header as file datetime"
@@ -298,9 +298,12 @@ class DownloadClient:
check_free_space = partial(self.manager.storage_manager.check_free_space, media_item)
await check_free_space()
- media_item.partial_file.parent.mkdir(parents=True, exist_ok=True)
- if not media_item.partial_file.is_file():
- media_item.partial_file.touch()
+ def prepare():
+ media_item.partial_file.parent.mkdir(parents=True, exist_ok=True)
+ if not media_item.partial_file.is_file():
+ media_item.partial_file.touch()
+
+ await asyncio.to_thread(prepare)
last_slow_speed_read = None
@@ -381,7 +384,7 @@ class DownloadClient:
async def add_file_size(self, domain: str, media_item: MediaItem) -> None:
if not media_item.complete_file:
media_item.complete_file = self.get_file_location(media_item)
- if media_item.complete_file.is_file():
+ if await asyncio.to_thread(media_item.complete_file.is_file):
await self.manager.db_manager.history_table.add_filesize(domain, media_item)
async def handle_media_item_completion(self, media_item: MediaItem, downloaded: bool = False) -> None:
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index dc24d9052..af6ebd408 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -12,6 +12,7 @@ from send2trash import send2trash
from cyberdrop_dl.ui.prompts.basic_prompts import enter_to_continue
from cyberdrop_dl.utils.data_enums_classes.hash import Hashing
from cyberdrop_dl.utils.logger import log
+from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
from yarl import URL
@@ -50,7 +51,7 @@ class HashClient:
async def hash_directory(self, path: Path) -> None:
path = Path(path)
with self.manager.live_manager.get_hash_live(stop=True):
- if not path.is_dir():
+ if not await asyncio.to_thread(path.is_dir):
raise NotADirectoryError
for file in path.rglob("*"):
await self.update_db_and_retrive_hash(file)
@@ -61,7 +62,7 @@ class HashClient:
hash = await self.update_db_and_retrive_hash(
media_item.complete_file, media_item.original_filename, media_item.referer
)
- self.save_hash_data(media_item, hash)
+ await self.save_hash_data(media_item, hash)
async def hash_item_during_download(self, media_item: MediaItem) -> None:
if media_item.is_segment:
@@ -73,7 +74,7 @@ class HashClient:
hash = await self.update_db_and_retrive_hash(
media_item.complete_file, media_item.original_filename, media_item.referer
)
- self.save_hash_data(media_item, hash)
+ await self.save_hash_data(media_item, hash)
except Exception as e:
log(f"After hash processing failed: {media_item.complete_file} with error {e}", 40, exc_info=True)
@@ -81,11 +82,9 @@ class HashClient:
self, file: Path | str, original_filename: str | None = None, referer: URL | None = None
):
file = Path(file)
- if not file.is_file():
+ if file.suffix == ".part":
return
- elif file.stat().st_size == 0:
- return
- elif file.suffix == ".part":
+ if not await asyncio.to_thread(get_size_or_none, file):
return
hash = await self._update_db_and_retrive_hash_helper(file, original_filename, referer, hash_type=self.xxhash)
if self.manager.config_manager.settings_data.dupe_cleanup_options.add_md5_hash:
@@ -128,9 +127,10 @@ class HashClient:
log(f"Error hashing {file} : {e}", 40, exc_info=True)
return hash
- def save_hash_data(self, media_item: MediaItem, hash: str):
- absolute_path = media_item.complete_file.resolve()
- size = media_item.complete_file.stat().st_size
+ async def save_hash_data(self, media_item: MediaItem, hash: str):
+ absolute_path = await asyncio.to_thread(media_item.complete_file.resolve)
+ size = await asyncio.to_thread(get_size_or_none, media_item.complete_file)
+ assert size
self.hashed_media_items.add(media_item)
if hash:
media_item.hash = hash
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 545252b75..e3ed22def 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -21,7 +21,7 @@ from cyberdrop_dl.clients.errors import (
from cyberdrop_dl.utils.constants import CustomHTTPStatus
from cyberdrop_dl.utils.data_enums_classes.url_objects import HlsSegment, MediaItem
from cyberdrop_dl.utils.logger import log
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine, Generator
@@ -290,8 +290,7 @@ class Downloader:
ClientError,
) as e:
ui_message = getattr(e, "status", type(e).__name__)
- if media_item.partial_file and media_item.partial_file.is_file():
- size = media_item.partial_file.stat().st_size
+ if size := await asyncio.to_thread(get_size_or_none, media_item.partial_file):
if (
media_item.filename in self._current_attempt_filesize
and self._current_attempt_filesize[media_item.filename] >= size
diff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.py
index b74aeeeef..167c0a510 100644
--- a/cyberdrop_dl/managers/log_manager.py
+++ b/cyberdrop_dl/managers/log_manager.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import csv
from asyncio import Lock
from pathlib import Path
@@ -38,7 +39,7 @@ class LogManager:
"""Write to the specified csv file. kwargs are columns for the CSV."""
self._csv_locks[file] = self._csv_locks.get(file, Lock())
async with self._csv_locks[file]:
- write_headers = not file.is_file()
+ write_headers = not await asyncio.to_thread(file.is_file)
async with aiofiles.open(file, "a", encoding="utf8", newline="") as csv_file:
writer = csv.DictWriter(
csv_file, fieldnames=kwargs.keys(), delimiter=CSV_DELIMITER, quoting=csv.QUOTE_ALL
@@ -71,7 +72,11 @@ class LogManager:
async def update_last_forum_post(self) -> None:
"""Updates the last forum post."""
input_file = self.manager.path_manager.input_file
- if not input_file.is_file() or not self.last_post_log.is_file():
+
+ def proceed():
+ return input_file.is_file() or self.last_post_log.is_file()
+
+ if not await asyncio.to_thread(proceed):
return
log_spacer(20)
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 403a6570c..b06983b4a 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import re
from dataclasses import Field
from datetime import date, datetime
@@ -114,6 +115,10 @@ class ScrapeMapper:
async def parse_input_file_groups(self) -> AsyncGenerator[tuple[str, list[URL]]]:
"""Split URLs from input file by their groups."""
input_file = self.manager.path_manager.input_file
+ if not await asyncio.to_thread(input_file.is_file):
+ yield ("", [])
+ return
+
block_quote = False
current_group_name = ""
async with aiofiles.open(input_file, encoding="utf8") as f:
@@ -136,10 +141,6 @@ class ScrapeMapper:
async def load_links(self) -> AsyncGenerator[ScrapeItem]:
"""Loads links from args / input file."""
- input_file = self.manager.path_manager.input_file
- # we need to touch the file just in case, purge_tree deletes it
- if not input_file.is_file():
- input_file.touch(exist_ok=True)
if not self.manager.parsed_args.cli_only_args.links:
self.using_input_file = True
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index e5191e667..d5b3ce1dd 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import contextlib
import itertools
from fractions import Fraction
@@ -17,6 +18,7 @@ from cyberdrop_dl.utils.logger import log_with_color
from cyberdrop_dl.utils.utilities import purge_dir_tree
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
from datetime import datetime
from cyberdrop_dl.managers.manager import Manager
@@ -40,9 +42,16 @@ class Sorter:
self.video_format: str | None = settings.sorted_video
self.other_format: str | None = settings.sorted_other
- def _get_files(self, directory: Path) -> list[Path]:
+ async def _get_files(self, directory: Path) -> AsyncGenerator[Path]:
"""Finds all files in a directory and returns them in a list."""
- return [file.resolve() for file in directory.rglob("*") if file.is_file()]
+
+ def resolve_if_file(path: Path) -> Path | None:
+ if path.is_file():
+ return path.resolve()
+
+ for file in directory.rglob("*"):
+ if file := await asyncio.to_thread(resolve_if_file, file):
+ yield file
def _move_file(self, old_path: Path, new_path: Path) -> bool:
"""Moves a file to a destination folder."""
@@ -74,19 +83,20 @@ class Sorter:
async def run(self) -> None:
"""Sorts the files in the download directory into their respective folders."""
- if not self.download_folder.is_dir():
+ if not asyncio.to_thread(self.download_folder.is_dir):
log_with_color(f"Download directory ({self.download_folder}) does not exist", "red", 40)
return
log_with_color("\nSorting downloads, please wait", "cyan", 20)
- self.sorted_folder.mkdir(parents=True, exist_ok=True)
+ await asyncio.to_thread(self.sorted_folder.mkdir, parents=True, exist_ok=True)
files_to_sort: dict[str, list[Path]] = {}
with self.manager.live_manager.get_sort_live(stop=True):
- subfolders = [f for f in self.download_folder.iterdir() if f.is_dir()]
- for folder in subfolders:
- files_to_sort[folder.name] = self._get_files(folder)
+ for subfolder in self.download_folder.iterdir():
+ if not await asyncio.to_thread(subfolder.is_dir):
+ continue
+ files_to_sort[subfolder.name] = [file async for file in self._get_files(subfolder)]
await self._sort_files(files_to_sort)
log_with_color("DONE!", "green", 20)
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 6f7493d97..bb77b021e 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import contextlib
import json
import os
@@ -10,6 +11,7 @@ import subprocess
from dataclasses import dataclass, fields
from functools import lru_cache, partial, wraps
from pathlib import Path
+from stat import S_ISREG
from typing import TYPE_CHECKING, Any, ClassVar, ParamSpec, Protocol, TypeVar
import aiofiles
@@ -311,8 +313,8 @@ async def send_webhook_message(manager: Manager) -> None:
form = FormData()
- if "attach_logs" in webhook.tags and main_log.is_file():
- if main_log.stat().st_size <= 25 * 1024 * 1024:
+ if "attach_logs" in webhook.tags and (size := await asyncio.to_thread(get_size_or_none, main_log)):
+ if size <= 25 * 1024 * 1024: # 25MB
async with aiofiles.open(main_log, "rb") as f:
form.add_field("file", await f.read(), filename=main_log.name)
@@ -498,6 +500,19 @@ def get_filename_from_headers(headers: Mapping[str, Any]) -> str | None:
return matches[0] or matches[1]
+def get_size_or_none(path: Path) -> int | None:
+ """Checks if this is a file and returns its size with a single system call.
+
+ Returns `None` otherwise"""
+
+ try:
+ stat = path.stat()
+ if S_ISREG(stat.st_mode):
+ return stat.st_size
+ except (OSError, ValueError):
+ return None
+
+
log_cyan = partial(log_with_color, style="cyan", level=20)
log_yellow = partial(log_with_color, style="yellow", level=20)
log_green = partial(log_with_color, style="green", level=20)