Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: make setting file dates async (#953)
NTFSvolume932532ea31fe
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex e3ed22def..b0425bf9b 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -1,15 +1,19 @@from __future__ import annotationsimport asyncio+import osimport re+import shutil+import subprocess+import sysfrom dataclasses import field+from datetime import datetimefrom functools import wrapsfrom http import HTTPStatusfrom pathlib import Pathfrom typing import TYPE_CHECKING, ParamSpec, TypeVarfrom aiohttp import ClientConnectorError, ClientError, ClientResponseError-from filedate import Filefrom cyberdrop_dl.clients.errors import (DownloadError,@@ -23,6 +27,20 @@ from cyberdrop_dl.utils.data_enums_classes.url_objects import HlsSegment, MediaIfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none+# Windows epoch is January 1, 1601. Unix epoch is January 1, 1970+WIN_EPOCH_OFFSET = 116444736e9+MAC_OS_SET_FILE = None++if sys.platform == "win32":+ from ctypes import byref, windll, wintypes++ import win32con++elif sys.platform == "darwin":+ # SetFile is non standard in macOS. Only users that have xcode installed will have SetFile+ MAC_OS_SET_FILE = shutil.which("SetFile")++if TYPE_CHECKING:from collections.abc import Callable, Coroutine, Generator@@ -197,12 +215,12 @@ class Downloader:await self.client.process_completed(media_item, self.domain)await self.client.handle_media_item_completion(media_item, downloaded=ffmpeg_result.success)- self.finalize_download(media_item, ffmpeg_result.success)+ await self.finalize_download(media_item, ffmpeg_result.success)- def finalize_download(self, media_item: MediaItem, downloaded: bool) -> None:+ async def finalize_download(self, media_item: MediaItem, downloaded: bool) -> None:if downloaded:- Path.chmod(media_item.complete_file, 0o666)- self.set_file_datetime(media_item, media_item.complete_file)+ await asyncio.to_thread(Path.chmod, media_item.complete_file, 0o666)+ await self.set_file_datetime(media_item, media_item.complete_file)self.attempt_task_removal(media_item)self.manager.progress_manager.download_progress.add_completed()log(f"Download finished: {media_item.url}", 20)@@ -217,7 +235,7 @@ class Downloader:if not self.manager.download_manager.pre_check_duration(media_item):raise DurationError(origin=media_item)- def set_file_datetime(self, media_item: MediaItem, complete_file: Path) -> None:+ async def set_file_datetime(self, media_item: MediaItem, complete_file: Path) -> None:"""Sets the file's datetime."""if self.manager.config_manager.settings_data.download_options.disable_file_timestamps:return@@ -225,12 +243,64 @@ class Downloader:log(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime", 30)return- file = File(str(complete_file))- file.set(- created=media_item.datetime,- modified=media_item.datetime,- accessed=media_item.datetime,- )+ # TODO: Make this entire method async (run in another thread)++ # 1. try setting creation date+ try:+ if sys.platform == "win32":++ def set_win_time():+ nano_ts: float = media_item.datetime * 1e7 # Windows uses nano seconds for dates+ timestamp = int(nano_ts + WIN_EPOCH_OFFSET)++ # Windows dates are 64bits, split into 2 32bits unsigned ints (dwHighDateTime , dwLowDateTime)+ # XOR to get the date as bytes, then shift to get the first 32 bits (dwHighDateTime)+ ctime = wintypes.FILETIME(timestamp & 0xFFFFFFFF, timestamp >> 32)+ access_mode = 256 # FILE_WRITE_ATTRIBUTES+ sharing_mode = 0 # Exclusive access+ security_mode = None # Use default security attributes+ creation_disposition = win32con.OPEN_EXISTING++ # FILE_FLAG_BACKUP_SEMANTICS allows access to directories+ flags = win32con.FILE_ATTRIBUTE_NORMAL | win32con.FILE_FLAG_BACKUP_SEMANTICS+ template_file = None++ params = (+ access_mode,+ sharing_mode,+ security_mode,+ creation_disposition,+ flags,+ template_file,+ )++ handle = windll.kernel32.CreateFileW(complete_file, *params)+ windll.kernel32.SetFileTime(+ handle,+ byref(ctime), # Creation time+ None, # Access time+ None, # Modification time+ )+ windll.kernel32.CloseHandle(handle)++ await asyncio.to_thread(set_win_time)++ elif sys.platform == "darwin" and MAC_OS_SET_FILE:+ date_string = datetime.fromtimestamp(media_item.datetime).strftime("%m/%d/%Y %H:%M:%S")+ cmd = ["-d", date_string, complete_file]+ process = await asyncio.subprocess.create_subprocess_exec(+ MAC_OS_SET_FILE, *cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL+ )+ _ = await process.wait()++ except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError, ValueError):+ pass++ # 2. try setting modification and access date+ try:+ await asyncio.to_thread(os.utime, complete_file, (media_item.datetime, media_item.datetime))+ except OSError:+ passdef attempt_task_removal(self, media_item: MediaItem) -> None:"""Attempts to remove the task from the progress bar."""@@ -267,8 +337,8 @@ class Downloader:await self.check_file_can_download(media_item)downloaded = await self.client.download_file(self.manager, self.domain, media_item)if downloaded:- Path.chmod(media_item.complete_file, 0o666)- self.set_file_datetime(media_item, media_item.complete_file)+ await asyncio.to_thread(Path.chmod, media_item.complete_file, 0o666)+ await self.set_file_datetime(media_item, media_item.complete_file)self.attempt_task_removal(media_item)self.manager.progress_manager.download_progress.add_completed()log(f"Download finished: {media_item.url}", 20)diff --git a/poetry.lock b/poetry.lockindex 2eb128ca9..ba7a53ad8 100644--- a/poetry.lock+++ b/poetry.lock@@ -731,20 +731,6 @@ files = [{file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"},]-[[package]]-name = "filedate"-version = "3.0"-description = ""-optional = false-python-versions = ">=3.4"-groups = ["main"]-files = [- {file = "filedate-3.0-py3-none-any.whl", hash = "sha256:10af3be124bc17c041a4fb0381f7ab6761384be8f455e785936c9a8cf614dedb"},-]--[package.dependencies]-python-dateutil = "*"-[[package]]name = "filelock"version = "3.18.0"@@ -2675,4 +2661,4 @@ propcache = ">=0.2.0"[metadata]lock-version = "2.1"python-versions = ">=3.11,<4"-content-hash = "9c656f1b51a3b5d438450245fc66a62151c013a48a936eb35485f776e16b9cfd"+content-hash = "a47a76bece39df8b7fbf7c9fc53421daee7155ba89b2aec16a6ce3fe6495b339"diff --git a/pyproject.toml b/pyproject.tomlindex 35dc996a4..01cda1f08 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -14,7 +14,6 @@ dependencies = ["browser-cookie3 (>=0.20.1,<0.21.0)","certifi (>=2025.1.31,<2026.0.0)","curl-cffi (>=0.9.0,<0.10.0)",- "filedate (>=3.0,<4.0)","get-video-properties (>=0.1.1,<0.2.0)","inquirerpy (>=0.3.4,<0.4.0)","mediafire (>=0.6.1,<0.7.0)",