Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: report which file each deleted duplicate matched (#2150)
Michael Menannofb2c2f25a2aa
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex b6c84a827..22301df56 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -153,6 +153,7 @@ logs:console_level: nullexpire_after: nullfiles:+ dedupe: dedupe.csvdownload_errors: download_errors.csvlast_forum_post: last_forum_post.csvmain: downloader.logdiff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex c0d03b539..492a7c9a9 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -785,6 +785,10 @@"last_forum_post": {"$ref": "#/$defs/CSVPath","default": "last_forum_post.csv"+ },+ "dedupe": {+ "$ref": "#/$defs/CSVPath",+ "default": "dedupe.csv"}},"title": "LogFiles",diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.pyindex a6b4f1f59..5fb4b2a07 100644--- a/cyberdrop_dl/config/settings.py+++ b/cyberdrop_dl/config/settings.py@@ -78,6 +78,9 @@ class LogFiles(ConfigModel):last_forum_post: CSVPath = Path("last_forum_post.csv")"Save the URL of the last scraped post from each thread to this file (MUST BE .csv)"+ dedupe: CSVPath = Path("dedupe.csv")+ "Save every duplicate deleted by the deduper, and the file it matched, to this file (MUST BE .csv)"+@propertydef jsonl_file(self) -> Path:return self.main.with_suffix(".results.jsonl")diff --git a/cyberdrop_dl/csv_logs.py b/cyberdrop_dl/csv_logs.pyindex 41e1c389f..a8cc39f6a 100644--- a/cyberdrop_dl/csv_logs.py+++ b/cyberdrop_dl/csv_logs.py@@ -35,6 +35,7 @@ class CSVFiles:download_errors: Pathscrape_errors: Pathlast_forum_post: Path+ dedupe: Pathjsonl_file: Path__iter__ = DictDataclass.__iter__@@ -48,6 +49,7 @@ class CSVFiles:scrape_errors=files.scrape_errors,jsonl_file=files.jsonl_file,last_forum_post=files.last_forum_post,+ dedupe=files.dedupe,)@@ -101,6 +103,11 @@ class CSVLogsManager:def write_last_forum_post(self, url: AbsoluteHttpURL) -> None:_ = self.task_group.create_task(self._write_to_csv(self.files.last_forum_post, url=url))+ def write_dedupe(self, duplicate: Path, original: Path, file_hash: str) -> None:+ _ = self.task_group.create_task(+ self._write_to_csv(self.files.dedupe, duplicate=duplicate, original=original, hash=file_hash)+ )+def write_download_error(self,url: AbsoluteHttpURL,diff --git a/cyberdrop_dl/dedupe.py b/cyberdrop_dl/dedupe.pyindex 868d89cd6..3e3bf5b68 100644--- a/cyberdrop_dl/dedupe.py+++ b/cyberdrop_dl/dedupe.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport asyncioimport dataclasses-import itertoolsimport loggingfrom pathlib import Pathfrom typing import TYPE_CHECKING, Self@@ -10,6 +9,7 @@ from typing import TYPE_CHECKING, Selfimport send2trashfrom cyberdrop_dl import aio+from cyberdrop_dl.csv_logs import CSVFiles, CSVLogsManagerfrom cyberdrop_dl.progress.dedupe import DedupeStats, DedupeUIif TYPE_CHECKING:@@ -28,8 +28,10 @@ class Czkawka:base_dir: Pathdatabase: Databaseuse_trash_bin: bool+ csv_files: CSVFiles_sem: asyncio.BoundedSemaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.BoundedSemaphore(20))_tui: DedupeUI = dataclasses.field(init=False, repr=False)+ _csv_logs: CSVLogsManager = dataclasses.field(init=False, repr=False)@classmethoddef from_manager(cls, manager: Manager) -> Self:@@ -37,6 +39,7 @@ class Czkawka:base_dir=manager.config.download_folder.expanduser().resolve().absolute(),database=manager.database,use_trash_bin=manager.config.hashing.dedupe.use_trash_bin,+ csv_files=CSVFiles.from_config(manager.config),)def __post_init__(self) -> None:@@ -47,25 +50,32 @@ class Czkawka:return self._tui.statsasync def run(self, file_hashes: FileHashes) -> None:+ logger.info(f"Starting dedupe of new downloads in '{self.base_dir}'")with self._tui():- await self._dedupe(file_hashes)-- async def _dedupe(self, file_hashes: FileHashes) -> None:- async with asyncio.TaskGroup() as tg:-- async def delete_dupes(hash_value: str, size: int, paths: set[Path]) -> None:- db_matches = await self.database.hash.get_files_with_hash_matches(hash_value, size, "xxh128")- for file in _filter_db_matches(db_matches, self.base_dir):- if file not in paths:- continue- await self._sem.acquire()- tg.create_task(self._delete_and_log(file, hash_value))-- for hash_value, sizes in file_hashes.items():- for size, paths in sizes.items():- tg.create_task(delete_dupes(hash_value, size, paths))-- async def _delete_and_log(self, file: Path, xxh128_value: str) -> None:+ # The CSV writer schedules its writes into this task group, so it lives exactly as long as the dedupe+ async with asyncio.TaskGroup() as tg:+ self._csv_logs = CSVLogsManager(self.csv_files, tg)+ await self._dedupe(file_hashes, tg)++ if self.stats.deleted:+ logger.info(f"Saved the details of every deleted duplicate to '{self.csv_files.dedupe}'")++ logger.info(f"Dedupe finished. Deleted {self.stats.deleted:,} of {self.stats.total:,} duplicates found")++ async def _dedupe(self, file_hashes: FileHashes, tg: asyncio.TaskGroup) -> None:+ async def delete_dupes(hash_value: str, size: int, paths: set[Path]) -> None:+ db_matches = await self.database.hash.get_files_with_hash_matches(hash_value, size, "xxh128")+ for original, file in _filter_db_matches(db_matches, self.base_dir):+ if file not in paths:+ continue+ await self._sem.acquire()+ tg.create_task(self._delete_and_log(file, original, hash_value))++ for hash_value, sizes in file_hashes.items():+ for size, paths in sizes.items():+ tg.create_task(delete_dupes(hash_value, size, paths))++ async def _delete_and_log(self, file: Path, original: Path, xxh128_value: str) -> None:hash_string = f"xxh128:{xxh128_value}"suffix = "Sent to trash" if self.use_trash_bin else "Permanently deleted"@@ -78,9 +88,10 @@ class Czkawka:else:if deleted:logger.info(- f"Removed new download '{file}' [{suffix}]. File hash matches with a previous download ({hash_string})"+ f"Removed new download '{file}' [{suffix}]. It's a duplicate of '{original}' ({hash_string})")self._tui.stats.deleted += 1+ self._csv_logs.write_dedupe(file, original, hash_string)finally:self._sem.release()@@ -106,9 +117,22 @@ async def _delete_file(path: Path, *, to_trash: bool) -> bool:return True-def _filter_db_matches(db_matches: Iterable[sqlite3.Row], base_dir: Path) -> Generator[Path]:- # always keep the first row, AKA the first file ever downloaded with this hash- for row in itertools.islice(db_matches, 1, None):- file = Path(row["folder"], row["download_filename"])+def _filter_db_matches(db_matches: Iterable[sqlite3.Row], base_dir: Path) -> Generator[tuple[Path, Path]]:+ """Yields `(original, duplicate)` pairs.++ The original is the first row, AKA the first file ever downloaded with this hash. It's always kept,+ even if it's outside of `base_dir`."""++ rows = iter(db_matches)+ if (first := next(rows, None)) is None:+ return++ original = _row_path(first)+ for row in rows:+ file = _row_path(row)if file.is_relative_to(base_dir):- yield file+ yield original, file+++def _row_path(row: sqlite3.Row) -> Path:+ return Path(row["folder"], row["download_filename"])diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.mdindex 47203a873..1a65adfbb 100644--- a/docs/reference/cli-arguments.md+++ b/docs/reference/cli-arguments.md@@ -74,7 +74,7 @@ If provided, this file _must_ exists already, but it can be empty<!-- START_CLI_OVERVIEW -->```shell-cyberdrop-dl v10.9.0+cyberdrop-dl v10.9.1Bulk asynchronous downloader for multiple file hostsUsage: cyberdrop-dl COMMAND [OPTIONS]@@ -106,7 +106,7 @@ Wiki (docs): https://script-ware.gitbook.io/cyberdrop-dl────────────────────────────────────────────────────────────────────────────────────────────────────-cyberdrop-dl v10.9.0+cyberdrop-dl v10.9.1Bulk asynchronous downloader for multiple file hostsUsage: cyberdrop-dl download [OPTIONS] [ARGS...]@@ -376,6 +376,9 @@ Download URLs│ --logs.files.last-forum-post Save the URL of the last scraped post from each thread to ││ this file (MUST BE .csv) ││ [default: last_forum_post.csv] │+│ --logs.files.dedupe Save every duplicate deleted by the deduper, and the file │+│ it matched, to this file (MUST BE .csv) │+│ [default: dedupe.csv] ││ --logs.folder Base folder to prepend to log files paths (if they are not ││ absolute) ││ --logs.expire-after Delete all log files inside `--logs.folder` if they are │diff --git a/docs/reference/config/logs.md b/docs/reference/config/logs.mdindex cda4de473..99ee5b995 100644--- a/docs/reference/config/logs.md+++ b/docs/reference/config/logs.md@@ -190,3 +190,19 @@ logs:files:unsupported: last_forum_post.csv```++## `dedupe`++| Type | Default | Restrictions |+| ------ | ------------ | --------------------------------------------- |+| `Path` | `dedupe.csv` | extension will always be overridden to `.csv` |++Path of the dedupe log file. For relative paths, the final path will be `logs.folder` / `logs.files.dedupe`++Every duplicate that the deduper deletes is saved here, together with the file it matched and their shared hash. The file is only created if at least one duplicate was deleted.++```yaml+logs:+ files:+ dedupe: dedupe.csv+```diff --git a/tests/test_csv_logs.py b/tests/test_csv_logs.pyindex 8903a75f0..0fecdda7a 100644--- a/tests/test_csv_logs.py+++ b/tests/test_csv_logs.py@@ -1,9 +1,11 @@+import asyncioimport datetimefrom pathlib import Pathimport pytest-from cyberdrop_dl.csv_logs import _prepare_resp_file, _write_to_csv+from cyberdrop_dl.config import Config+from cyberdrop_dl.csv_logs import CSVFiles, CSVLogsManager, _prepare_resp_file, _write_to_csvfrom cyberdrop_dl.url_objects import AbsoluteHttpURLnow = datetime.datetime(2026, 5, 8, tzinfo=datetime.UTC)@@ -32,6 +34,19 @@ def test_prepare_resp_filename(url: str, expected: str) -> None:assert result.as_posix() == expected+async def test_delete_old_logs_removes_a_stale_dedupe_log(tmp_cwd: Path) -> None:+ config = Config()+ config.logs.resolve_filenames(tmp_cwd)+ stale_log = config.logs.files.dedupe+ stale_log.parent.mkdir(parents=True, exist_ok=True)+ _ = stale_log.write_text("from a previous run")++ async with asyncio.TaskGroup() as task_group:+ CSVLogsManager(CSVFiles.from_config(config), task_group).delete_old_logs()++ assert not stale_log.exists()++class TestWriteToCsv:@pytest.fixturedef row(self) -> dict[str, object]:diff --git a/tests/test_dedupe.py b/tests/test_dedupe.pyindex 5c055590d..12ff71bb7 100644--- a/tests/test_dedupe.py+++ b/tests/test_dedupe.py@@ -1,4 +1,5 @@# ruff: noqa: ASYNC240+import dataclassesimport sqlite3from pathlib import Pathfrom typing import cast@@ -6,7 +7,13 @@ from unittest.mock import patchimport pytest-from cyberdrop_dl.dedupe import _delete_file, _filter_db_matches+from cyberdrop_dl.config import Config+from cyberdrop_dl.csv_logs import CSVFiles+from cyberdrop_dl.dedupe import Czkawka, _delete_file, _filter_db_matches+from cyberdrop_dl.hasher import FileHashes+from cyberdrop_dl.manager import Manager++_HASH = "a5aec63d0fef8660b00570f23be727e9"@pytest.fixture@@ -60,16 +67,17 @@ class TestFilterDbMatches:def _row(folder: Path | str, download_filename: str) -> sqlite3.Row:return cast("sqlite3.Row", {"folder": folder, "download_filename": download_filename}) # pyright: ignore[reportInvalidCast]- def test_filter_db_matches_should_skip_first_row(self, tmp_path: Path) -> None:+ def test_filter_db_matches_should_pair_every_duplicate_with_the_first_row(self, tmp_path: Path) -> None:rows = [self._row(tmp_path, "skip_me.txt"),self._row(tmp_path, "do not skip me.txt"),self._row(tmp_path, "do not skip me 2.txt"),]+ original = tmp_path / "skip_me.txt"found = list(_filter_db_matches(rows, tmp_path))assert found == [- tmp_path / "do not skip me.txt",- tmp_path / "do not skip me 2.txt",+ (original, tmp_path / "do not skip me.txt"),+ (original, tmp_path / "do not skip me 2.txt"),]def test_filter_db_matches_should_skip_non_relative_paths(self, tmp_path: Path) -> None:@@ -79,11 +87,116 @@ class TestFilterDbMatches:self._row("/another/dir", "skip me 2.txt"),self._row(tmp_path, "do not skip me 2.txt"),]+ original = tmp_path / "skip_me.txt"found = list(_filter_db_matches(rows, tmp_path))assert found == [- tmp_path / "do not skip me.txt",- tmp_path / "do not skip me 2.txt",+ (original, tmp_path / "do not skip me.txt"),+ (original, tmp_path / "do not skip me 2.txt"),]+ def test_filter_db_matches_should_report_originals_outside_base_dir(self, tmp_path: Path) -> None:+ rows = [+ self._row("/another/dir", "original.txt"),+ self._row(tmp_path, "dupe.txt"),+ ]+ found = list(_filter_db_matches(rows, tmp_path))+ assert found == [(Path("/another/dir/original.txt"), tmp_path / "dupe.txt")]++ def test_filtering_a_single_row(self, tmp_path: Path) -> None:+ assert list(_filter_db_matches([self._row(tmp_path, "alone.txt")], tmp_path)) == []+def test_filtering_no_rows(self) -> None:assert list(_filter_db_matches([], Path("/a/dir"))) == []++[email protected](frozen=True, slots=True)+class _DedupeCase:+ deduper: Czkawka+ file_hashes: FileHashes+ original: Path+ duplicate: Path++ @property+ def csv_file(self) -> Path:+ return self.deduper.csv_files.dedupe++ async def run(self) -> None:+ await self.deduper.run(self.file_hashes)+++async def _seed_duplicates(manager: Manager, base_dir: Path, *names: str) -> tuple[list[Path], FileHashes]:+ """Registers files that all share the same hash in the database. The first one is the original"""++ files: list[Path] = []+ for name in names:+ file = base_dir / name+ file.parent.mkdir(parents=True, exist_ok=True)+ _ = file.write_text("same content")+ await manager.database.hash.insert_or_update_hash_db(_HASH, "xxh128", file, None, None)+ files.append(file)++ return files, {_HASH: {files[0].stat().st_size: set(files)}}+++def _deduper(manager: Manager, base_dir: Path, log_folder: Path) -> Czkawka:+ config = Config()+ config.logs.resolve_filenames(log_folder)+ return Czkawka(+ base_dir=base_dir,+ database=manager.database,+ use_trash_bin=False,+ csv_files=CSVFiles.from_config(config),+ )+++class TestDedupeRun:+ @pytest.fixture+ async def case(self, running_manager: Manager, tmp_path: Path) -> _DedupeCase:+ base_dir = tmp_path / "downloads"+ files, file_hashes = await _seed_duplicates(running_manager, base_dir, "original.jpg", "sub/duplicate.jpg")+ deduper = _deduper(running_manager, base_dir, tmp_path / "logs")+ return _DedupeCase(deduper, file_hashes, files[0], files[1])++ async def test_deletes_the_duplicate_and_keeps_the_original(self, case: _DedupeCase) -> None:+ await case.run()+ assert case.original.exists()+ assert not case.duplicate.exists()++ async def test_logs_the_path_of_the_original(self, case: _DedupeCase, logs: pytest.LogCaptureFixture) -> None:+ await case.run()+ assert f"Removed new download '{case.duplicate}'" in logs.text+ assert f"duplicate of '{case.original}'" in logs.text++ async def test_logs_the_start_and_the_end_of_the_dedupe(+ self, case: _DedupeCase, logs: pytest.LogCaptureFixture+ ) -> None:+ await case.run()+ assert "Starting dedupe" in logs.text+ assert "Dedupe finished. Deleted 1 of 1 duplicates found" in logs.text++ async def test_writes_the_csv_log(self, case: _DedupeCase) -> None:+ await case.run()+ assert case.csv_file.read_text("utf8").splitlines() == [+ '"duplicate","original","hash"',+ f'"{case.duplicate}","{case.original}","xxh128:{_HASH}"',+ ]++ async def test_writes_one_csv_row_per_duplicate_under_a_single_header(+ self, running_manager: Manager, tmp_path: Path+ ) -> None:+ base_dir = tmp_path / "downloads"+ files, file_hashes = await _seed_duplicates(running_manager, base_dir, "original.jpg", "one.jpg", "two.jpg")+ deduper = _deduper(running_manager, base_dir, tmp_path / "logs")+ await deduper.run(file_hashes)++ original, *duplicates = files+ header, *rows = deduper.csv_files.dedupe.read_text("utf8").splitlines()+ assert header == '"duplicate","original","hash"'+ assert sorted(rows) == sorted(f'"{dupe}","{original}","xxh128:{_HASH}"' for dupe in duplicates)++ async def test_no_csv_log_when_nothing_was_deleted(self, case: _DedupeCase) -> None:+ await case.deduper.run({})+ assert not case.csv_file.exists()++ def test_from_manager_uses_the_csv_file_from_the_config(self, manager: Manager) -> None:+ assert Czkawka.from_manager(manager).csv_files.dedupe == manager.config.logs.files.dedupe