Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: make setting file dates async (#953)

NTFSvolume932532ea31fe
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index e3ed22def..b0425bf9b 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -1,15 +1,19 @@
from __future__ import annotations
import asyncio
+import os
import re
+import shutil
+import subprocess
+import sys
from dataclasses import field
+from datetime import datetime
from functools import wraps
from http import HTTPStatus
from pathlib import Path
from typing import TYPE_CHECKING, ParamSpec, TypeVar
from aiohttp import ClientConnectorError, ClientError, ClientResponseError
-from filedate import File
from cyberdrop_dl.clients.errors import (
DownloadError,
@@ -23,6 +27,20 @@ from cyberdrop_dl.utils.data_enums_classes.url_objects import HlsSegment, MediaI
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none
+# Windows epoch is January 1, 1601. Unix epoch is January 1, 1970
+WIN_EPOCH_OFFSET = 116444736e9
+MAC_OS_SET_FILE = None
+
+if sys.platform == "win32":
+ from ctypes import byref, windll, wintypes
+
+ import win32con
+
+elif sys.platform == "darwin":
+ # SetFile is non standard in macOS. Only users that have xcode installed will have SetFile
+ MAC_OS_SET_FILE = shutil.which("SetFile")
+
+
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine, Generator
@@ -197,12 +215,12 @@ class Downloader:
await self.client.process_completed(media_item, self.domain)
await self.client.handle_media_item_completion(media_item, downloaded=ffmpeg_result.success)
- self.finalize_download(media_item, ffmpeg_result.success)
+ await self.finalize_download(media_item, ffmpeg_result.success)
- def finalize_download(self, media_item: MediaItem, downloaded: bool) -> None:
+ async def finalize_download(self, media_item: MediaItem, downloaded: bool) -> None:
if downloaded:
- Path.chmod(media_item.complete_file, 0o666)
- self.set_file_datetime(media_item, media_item.complete_file)
+ await asyncio.to_thread(Path.chmod, media_item.complete_file, 0o666)
+ await self.set_file_datetime(media_item, media_item.complete_file)
self.attempt_task_removal(media_item)
self.manager.progress_manager.download_progress.add_completed()
log(f"Download finished: {media_item.url}", 20)
@@ -217,7 +235,7 @@ class Downloader:
if not self.manager.download_manager.pre_check_duration(media_item):
raise DurationError(origin=media_item)
- def set_file_datetime(self, media_item: MediaItem, complete_file: Path) -> None:
+ async def set_file_datetime(self, media_item: MediaItem, complete_file: Path) -> None:
"""Sets the file's datetime."""
if self.manager.config_manager.settings_data.download_options.disable_file_timestamps:
return
@@ -225,12 +243,64 @@ class Downloader:
log(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime", 30)
return
- file = File(str(complete_file))
- file.set(
- created=media_item.datetime,
- modified=media_item.datetime,
- accessed=media_item.datetime,
- )
+ # TODO: Make this entire method async (run in another thread)
+
+ # 1. try setting creation date
+ try:
+ if sys.platform == "win32":
+
+ def set_win_time():
+ nano_ts: float = media_item.datetime * 1e7 # Windows uses nano seconds for dates
+ timestamp = int(nano_ts + WIN_EPOCH_OFFSET)
+
+ # Windows dates are 64bits, split into 2 32bits unsigned ints (dwHighDateTime , dwLowDateTime)
+ # XOR to get the date as bytes, then shift to get the first 32 bits (dwHighDateTime)
+ ctime = wintypes.FILETIME(timestamp & 0xFFFFFFFF, timestamp >> 32)
+ access_mode = 256 # FILE_WRITE_ATTRIBUTES
+ sharing_mode = 0 # Exclusive access
+ security_mode = None # Use default security attributes
+ creation_disposition = win32con.OPEN_EXISTING
+
+ # FILE_FLAG_BACKUP_SEMANTICS allows access to directories
+ flags = win32con.FILE_ATTRIBUTE_NORMAL | win32con.FILE_FLAG_BACKUP_SEMANTICS
+ template_file = None
+
+ params = (
+ access_mode,
+ sharing_mode,
+ security_mode,
+ creation_disposition,
+ flags,
+ template_file,
+ )
+
+ handle = windll.kernel32.CreateFileW(complete_file, *params)
+ windll.kernel32.SetFileTime(
+ handle,
+ byref(ctime), # Creation time
+ None, # Access time
+ None, # Modification time
+ )
+ windll.kernel32.CloseHandle(handle)
+
+ await asyncio.to_thread(set_win_time)
+
+ elif sys.platform == "darwin" and MAC_OS_SET_FILE:
+ date_string = datetime.fromtimestamp(media_item.datetime).strftime("%m/%d/%Y %H:%M:%S")
+ cmd = ["-d", date_string, complete_file]
+ process = await asyncio.subprocess.create_subprocess_exec(
+ MAC_OS_SET_FILE, *cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+ )
+ _ = await process.wait()
+
+ except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError, ValueError):
+ pass
+
+ # 2. try setting modification and access date
+ try:
+ await asyncio.to_thread(os.utime, complete_file, (media_item.datetime, media_item.datetime))
+ except OSError:
+ pass
def attempt_task_removal(self, media_item: MediaItem) -> None:
"""Attempts to remove the task from the progress bar."""
@@ -267,8 +337,8 @@ class Downloader:
await self.check_file_can_download(media_item)
downloaded = await self.client.download_file(self.manager, self.domain, media_item)
if downloaded:
- Path.chmod(media_item.complete_file, 0o666)
- self.set_file_datetime(media_item, media_item.complete_file)
+ await asyncio.to_thread(Path.chmod, media_item.complete_file, 0o666)
+ await self.set_file_datetime(media_item, media_item.complete_file)
self.attempt_task_removal(media_item)
self.manager.progress_manager.download_progress.add_completed()
log(f"Download finished: {media_item.url}", 20)
diff --git a/poetry.lock b/poetry.lock
index 2eb128ca9..ba7a53ad8 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -731,20 +731,6 @@ files = [
{file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"},
]
-[[package]]
-name = "filedate"
-version = "3.0"
-description = ""
-optional = false
-python-versions = ">=3.4"
-groups = ["main"]
-files = [
- {file = "filedate-3.0-py3-none-any.whl", hash = "sha256:10af3be124bc17c041a4fb0381f7ab6761384be8f455e785936c9a8cf614dedb"},
-]
-
-[package.dependencies]
-python-dateutil = "*"
-
[[package]]
name = "filelock"
version = "3.18.0"
@@ -2675,4 +2661,4 @@ propcache = ">=0.2.0"
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4"
-content-hash = "9c656f1b51a3b5d438450245fc66a62151c013a48a936eb35485f776e16b9cfd"
+content-hash = "a47a76bece39df8b7fbf7c9fc53421daee7155ba89b2aec16a6ce3fe6495b339"
diff --git a/pyproject.toml b/pyproject.toml
index 35dc996a4..01cda1f08 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,7 +14,6 @@ dependencies = [
"browser-cookie3 (>=0.20.1,<0.21.0)",
"certifi (>=2025.1.31,<2026.0.0)",
"curl-cffi (>=0.9.0,<0.10.0)",
- "filedate (>=3.0,<4.0)",
"get-video-properties (>=0.1.1,<0.2.0)",
"inquirerpy (>=0.3.4,<0.4.0)",
"mediafire (>=0.6.1,<0.7.0)",