Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from __future__ import annotations
import asyncio
import dataclasses
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Self
import send2trash
from cyberdrop_dl import aio
from cyberdrop_dl.csv_logs import CSVFiles, CSVLogsManager
from cyberdrop_dl.progress.dedupe import DedupeStats, DedupeUI
if TYPE_CHECKING:
import sqlite3
from collections.abc import Generator, Iterable
from cyberdrop_dl.database import Database
from cyberdrop_dl.hasher import FileHashes
from cyberdrop_dl.manager import Manager
logger = logging.getLogger(__name__)
@dataclasses.dataclass(slots=True)
class Czkawka:
base_dir: Path
database: Database
use_trash_bin: bool
csv_files: CSVFiles
_sem: asyncio.BoundedSemaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.BoundedSemaphore(20))
_tui: DedupeUI = dataclasses.field(init=False, repr=False)
_csv_logs: CSVLogsManager = dataclasses.field(init=False, repr=False)
@classmethod
def from_manager(cls, manager: Manager) -> Self:
return cls(
base_dir=manager.config.download_folder.expanduser().resolve().absolute(),
database=manager.database,
use_trash_bin=manager.config.hashing.dedupe.use_trash_bin,
csv_files=CSVFiles.from_config(manager.config),
)
def __post_init__(self) -> None:
self._tui = DedupeUI(self.base_dir)
@property
def stats(self) -> DedupeStats:
return self._tui.stats
async def run(self, file_hashes: FileHashes) -> None:
logger.info(f"Starting dedupe of new downloads in '{self.base_dir}'")
with self._tui():
# The CSV writer schedules its writes into this task group, so it lives exactly as long as the dedupe
async with asyncio.TaskGroup() as tg:
self._csv_logs = CSVLogsManager(self.csv_files, tg)
await self._dedupe(file_hashes, tg)
if self.stats.deleted:
logger.info(f"Saved the details of every deleted duplicate to '{self.csv_files.dedupe}'")
logger.info(f"Dedupe finished. Deleted {self.stats.deleted:,} of {self.stats.total:,} duplicates found")
async def _dedupe(self, file_hashes: FileHashes, tg: asyncio.TaskGroup) -> None:
async def delete_dupes(hash_value: str, size: int, paths: set[Path]) -> None:
db_matches = await self.database.hash.get_files_with_hash_matches(hash_value, size, "xxh128")
for original, file in _filter_db_matches(db_matches, self.base_dir):
if file not in paths:
continue
await self._sem.acquire()
tg.create_task(self._delete_and_log(file, original, hash_value))
for hash_value, sizes in file_hashes.items():
for size, paths in sizes.items():
tg.create_task(delete_dupes(hash_value, size, paths))
async def _delete_and_log(self, file: Path, original: Path, xxh128_value: str) -> None:
hash_string = f"xxh128:{xxh128_value}"
suffix = "Sent to trash" if self.use_trash_bin else "Permanently deleted"
with self._tui.new_file(file):
try:
deleted = await _delete_file(file, to_trash=self.use_trash_bin)
except OSError as e:
logger.error(f"Unable to remove '{file}' ({hash_string}): {e!r}")
else:
if deleted:
logger.info(
f"Removed new download '{file}' [{suffix}]. It's a duplicate of '{original}' ({hash_string})"
)
self._tui.stats.deleted += 1
self._csv_logs.write_dedupe(file, original, hash_string)
finally:
self._sem.release()
async def _delete_file(path: Path, *, to_trash: bool) -> bool:
"""Deletes a file and return `True` on success, `False` is the file was not found.
Any other exception is propagated"""
coro = asyncio.to_thread(send2trash.send2trash, path) if to_trash else aio.unlink(path)
try:
await coro
except FileNotFoundError:
return False
except OSError as e:
# send2trash raises everything as a bare OSError. We should only ignore FileNotFound and raise everything else
if "file not found" in str(e).casefold():
return False
raise
else:
return True
def _filter_db_matches(db_matches: Iterable[sqlite3.Row], base_dir: Path) -> Generator[tuple[Path, Path]]:
"""Yields `(original, duplicate)` pairs.
The original is the first row, AKA the first file ever downloaded with this hash. It's always kept,
even if it's outside of `base_dir`."""
rows = iter(db_matches)
if (first := next(rows, None)) is None:
return
original = _row_path(first)
for row in rows:
file = _row_path(row)
if file.is_relative_to(base_dir):
yield original, file
def _row_path(row: sqlite3.Row) -> Path:
return Path(row["folder"], row["download_filename"])