Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: report which file each deleted duplicate matched (#2150)

Michael Menannofb2c2f25a2aa
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index b6c84a827..22301df56 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -153,6 +153,7 @@ logs:
console_level: null
expire_after: null
files:
+ dedupe: dedupe.csv
download_errors: download_errors.csv
last_forum_post: last_forum_post.csv
main: downloader.log
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index c0d03b539..492a7c9a9 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -785,6 +785,10 @@
"last_forum_post": {
"$ref": "#/$defs/CSVPath",
"default": "last_forum_post.csv"
+ },
+ "dedupe": {
+ "$ref": "#/$defs/CSVPath",
+ "default": "dedupe.csv"
}
},
"title": "LogFiles",
diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.py
index a6b4f1f59..5fb4b2a07 100644
--- a/cyberdrop_dl/config/settings.py
+++ b/cyberdrop_dl/config/settings.py
@@ -78,6 +78,9 @@ class LogFiles(ConfigModel):
last_forum_post: CSVPath = Path("last_forum_post.csv")
"Save the URL of the last scraped post from each thread to this file (MUST BE .csv)"
+ dedupe: CSVPath = Path("dedupe.csv")
+ "Save every duplicate deleted by the deduper, and the file it matched, to this file (MUST BE .csv)"
+
@property
def jsonl_file(self) -> Path:
return self.main.with_suffix(".results.jsonl")
diff --git a/cyberdrop_dl/csv_logs.py b/cyberdrop_dl/csv_logs.py
index 41e1c389f..a8cc39f6a 100644
--- a/cyberdrop_dl/csv_logs.py
+++ b/cyberdrop_dl/csv_logs.py
@@ -35,6 +35,7 @@ class CSVFiles:
download_errors: Path
scrape_errors: Path
last_forum_post: Path
+ dedupe: Path
jsonl_file: Path
__iter__ = DictDataclass.__iter__
@@ -48,6 +49,7 @@ class CSVFiles:
scrape_errors=files.scrape_errors,
jsonl_file=files.jsonl_file,
last_forum_post=files.last_forum_post,
+ dedupe=files.dedupe,
)
@@ -101,6 +103,11 @@ class CSVLogsManager:
def write_last_forum_post(self, url: AbsoluteHttpURL) -> None:
_ = self.task_group.create_task(self._write_to_csv(self.files.last_forum_post, url=url))
+ def write_dedupe(self, duplicate: Path, original: Path, file_hash: str) -> None:
+ _ = self.task_group.create_task(
+ self._write_to_csv(self.files.dedupe, duplicate=duplicate, original=original, hash=file_hash)
+ )
+
def write_download_error(
self,
url: AbsoluteHttpURL,
diff --git a/cyberdrop_dl/dedupe.py b/cyberdrop_dl/dedupe.py
index 868d89cd6..3e3bf5b68 100644
--- a/cyberdrop_dl/dedupe.py
+++ b/cyberdrop_dl/dedupe.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import dataclasses
-import itertools
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Self
@@ -10,6 +9,7 @@ from typing import TYPE_CHECKING, Self
import send2trash
from cyberdrop_dl import aio
+from cyberdrop_dl.csv_logs import CSVFiles, CSVLogsManager
from cyberdrop_dl.progress.dedupe import DedupeStats, DedupeUI
if TYPE_CHECKING:
@@ -28,8 +28,10 @@ class Czkawka:
base_dir: Path
database: Database
use_trash_bin: bool
+ csv_files: CSVFiles
_sem: asyncio.BoundedSemaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.BoundedSemaphore(20))
_tui: DedupeUI = dataclasses.field(init=False, repr=False)
+ _csv_logs: CSVLogsManager = dataclasses.field(init=False, repr=False)
@classmethod
def from_manager(cls, manager: Manager) -> Self:
@@ -37,6 +39,7 @@ class Czkawka:
base_dir=manager.config.download_folder.expanduser().resolve().absolute(),
database=manager.database,
use_trash_bin=manager.config.hashing.dedupe.use_trash_bin,
+ csv_files=CSVFiles.from_config(manager.config),
)
def __post_init__(self) -> None:
@@ -47,25 +50,32 @@ class Czkawka:
return self._tui.stats
async def run(self, file_hashes: FileHashes) -> None:
+ logger.info(f"Starting dedupe of new downloads in '{self.base_dir}'")
with self._tui():
- await self._dedupe(file_hashes)
-
- async def _dedupe(self, file_hashes: FileHashes) -> None:
- async with asyncio.TaskGroup() as tg:
-
- async def delete_dupes(hash_value: str, size: int, paths: set[Path]) -> None:
- db_matches = await self.database.hash.get_files_with_hash_matches(hash_value, size, "xxh128")
- for file in _filter_db_matches(db_matches, self.base_dir):
- if file not in paths:
- continue
- await self._sem.acquire()
- tg.create_task(self._delete_and_log(file, hash_value))
-
- for hash_value, sizes in file_hashes.items():
- for size, paths in sizes.items():
- tg.create_task(delete_dupes(hash_value, size, paths))
-
- async def _delete_and_log(self, file: Path, xxh128_value: str) -> None:
+ # The CSV writer schedules its writes into this task group, so it lives exactly as long as the dedupe
+ async with asyncio.TaskGroup() as tg:
+ self._csv_logs = CSVLogsManager(self.csv_files, tg)
+ await self._dedupe(file_hashes, tg)
+
+ if self.stats.deleted:
+ logger.info(f"Saved the details of every deleted duplicate to '{self.csv_files.dedupe}'")
+
+ logger.info(f"Dedupe finished. Deleted {self.stats.deleted:,} of {self.stats.total:,} duplicates found")
+
+ async def _dedupe(self, file_hashes: FileHashes, tg: asyncio.TaskGroup) -> None:
+ async def delete_dupes(hash_value: str, size: int, paths: set[Path]) -> None:
+ db_matches = await self.database.hash.get_files_with_hash_matches(hash_value, size, "xxh128")
+ for original, file in _filter_db_matches(db_matches, self.base_dir):
+ if file not in paths:
+ continue
+ await self._sem.acquire()
+ tg.create_task(self._delete_and_log(file, original, hash_value))
+
+ for hash_value, sizes in file_hashes.items():
+ for size, paths in sizes.items():
+ tg.create_task(delete_dupes(hash_value, size, paths))
+
+ async def _delete_and_log(self, file: Path, original: Path, xxh128_value: str) -> None:
hash_string = f"xxh128:{xxh128_value}"
suffix = "Sent to trash" if self.use_trash_bin else "Permanently deleted"
@@ -78,9 +88,10 @@ class Czkawka:
else:
if deleted:
logger.info(
- f"Removed new download '{file}' [{suffix}]. File hash matches with a previous download ({hash_string})"
+ f"Removed new download '{file}' [{suffix}]. It's a duplicate of '{original}' ({hash_string})"
)
self._tui.stats.deleted += 1
+ self._csv_logs.write_dedupe(file, original, hash_string)
finally:
self._sem.release()
@@ -106,9 +117,22 @@ async def _delete_file(path: Path, *, to_trash: bool) -> bool:
return True
-def _filter_db_matches(db_matches: Iterable[sqlite3.Row], base_dir: Path) -> Generator[Path]:
- # always keep the first row, AKA the first file ever downloaded with this hash
- for row in itertools.islice(db_matches, 1, None):
- file = Path(row["folder"], row["download_filename"])
+def _filter_db_matches(db_matches: Iterable[sqlite3.Row], base_dir: Path) -> Generator[tuple[Path, Path]]:
+ """Yields `(original, duplicate)` pairs.
+
+ The original is the first row, AKA the first file ever downloaded with this hash. It's always kept,
+ even if it's outside of `base_dir`."""
+
+ rows = iter(db_matches)
+ if (first := next(rows, None)) is None:
+ return
+
+ original = _row_path(first)
+ for row in rows:
+ file = _row_path(row)
if file.is_relative_to(base_dir):
- yield file
+ yield original, file
+
+
+def _row_path(row: sqlite3.Row) -> Path:
+ return Path(row["folder"], row["download_filename"])
diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.md
index 47203a873..1a65adfbb 100644
--- a/docs/reference/cli-arguments.md
+++ b/docs/reference/cli-arguments.md
@@ -74,7 +74,7 @@ If provided, this file _must_ exists already, but it can be empty
<!-- START_CLI_OVERVIEW -->
```shell
-cyberdrop-dl v10.9.0
+cyberdrop-dl v10.9.1
Bulk asynchronous downloader for multiple file hosts
Usage: cyberdrop-dl COMMAND [OPTIONS]
@@ -106,7 +106,7 @@ Wiki (docs): https://script-ware.gitbook.io/cyberdrop-dl
────────────────────────────────────────────────────────────────────────────────────────────────────
-cyberdrop-dl v10.9.0
+cyberdrop-dl v10.9.1
Bulk asynchronous downloader for multiple file hosts
Usage: cyberdrop-dl download [OPTIONS] [ARGS...]
@@ -376,6 +376,9 @@ Download URLs
│ --logs.files.last-forum-post Save the URL of the last scraped post from each thread to │
│ this file (MUST BE .csv) │
│ [default: last_forum_post.csv] │
+│ --logs.files.dedupe Save every duplicate deleted by the deduper, and the file │
+│ it matched, to this file (MUST BE .csv) │
+│ [default: dedupe.csv] │
│ --logs.folder Base folder to prepend to log files paths (if they are not │
│ absolute) │
│ --logs.expire-after Delete all log files inside `--logs.folder` if they are │
diff --git a/docs/reference/config/logs.md b/docs/reference/config/logs.md
index cda4de473..99ee5b995 100644
--- a/docs/reference/config/logs.md
+++ b/docs/reference/config/logs.md
@@ -190,3 +190,19 @@ logs:
files:
unsupported: last_forum_post.csv
```
+
+## `dedupe`
+
+| Type | Default | Restrictions |
+| ------ | ------------ | --------------------------------------------- |
+| `Path` | `dedupe.csv` | extension will always be overridden to `.csv` |
+
+Path of the dedupe log file. For relative paths, the final path will be `logs.folder` / `logs.files.dedupe`
+
+Every duplicate that the deduper deletes is saved here, together with the file it matched and their shared hash. The file is only created if at least one duplicate was deleted.
+
+```yaml
+logs:
+ files:
+ dedupe: dedupe.csv
+```
diff --git a/tests/test_csv_logs.py b/tests/test_csv_logs.py
index 8903a75f0..0fecdda7a 100644
--- a/tests/test_csv_logs.py
+++ b/tests/test_csv_logs.py
@@ -1,9 +1,11 @@
+import asyncio
import datetime
from pathlib import Path
import pytest
-from cyberdrop_dl.csv_logs import _prepare_resp_file, _write_to_csv
+from cyberdrop_dl.config import Config
+from cyberdrop_dl.csv_logs import CSVFiles, CSVLogsManager, _prepare_resp_file, _write_to_csv
from cyberdrop_dl.url_objects import AbsoluteHttpURL
now = datetime.datetime(2026, 5, 8, tzinfo=datetime.UTC)
@@ -32,6 +34,19 @@ def test_prepare_resp_filename(url: str, expected: str) -> None:
assert result.as_posix() == expected
+async def test_delete_old_logs_removes_a_stale_dedupe_log(tmp_cwd: Path) -> None:
+ config = Config()
+ config.logs.resolve_filenames(tmp_cwd)
+ stale_log = config.logs.files.dedupe
+ stale_log.parent.mkdir(parents=True, exist_ok=True)
+ _ = stale_log.write_text("from a previous run")
+
+ async with asyncio.TaskGroup() as task_group:
+ CSVLogsManager(CSVFiles.from_config(config), task_group).delete_old_logs()
+
+ assert not stale_log.exists()
+
+
class TestWriteToCsv:
@pytest.fixture
def row(self) -> dict[str, object]:
diff --git a/tests/test_dedupe.py b/tests/test_dedupe.py
index 5c055590d..12ff71bb7 100644
--- a/tests/test_dedupe.py
+++ b/tests/test_dedupe.py
@@ -1,4 +1,5 @@
# ruff: noqa: ASYNC240
+import dataclasses
import sqlite3
from pathlib import Path
from typing import cast
@@ -6,7 +7,13 @@ from unittest.mock import patch
import pytest
-from cyberdrop_dl.dedupe import _delete_file, _filter_db_matches
+from cyberdrop_dl.config import Config
+from cyberdrop_dl.csv_logs import CSVFiles
+from cyberdrop_dl.dedupe import Czkawka, _delete_file, _filter_db_matches
+from cyberdrop_dl.hasher import FileHashes
+from cyberdrop_dl.manager import Manager
+
+_HASH = "a5aec63d0fef8660b00570f23be727e9"
@pytest.fixture
@@ -60,16 +67,17 @@ class TestFilterDbMatches:
def _row(folder: Path | str, download_filename: str) -> sqlite3.Row:
return cast("sqlite3.Row", {"folder": folder, "download_filename": download_filename}) # pyright: ignore[reportInvalidCast]
- def test_filter_db_matches_should_skip_first_row(self, tmp_path: Path) -> None:
+ def test_filter_db_matches_should_pair_every_duplicate_with_the_first_row(self, tmp_path: Path) -> None:
rows = [
self._row(tmp_path, "skip_me.txt"),
self._row(tmp_path, "do not skip me.txt"),
self._row(tmp_path, "do not skip me 2.txt"),
]
+ original = tmp_path / "skip_me.txt"
found = list(_filter_db_matches(rows, tmp_path))
assert found == [
- tmp_path / "do not skip me.txt",
- tmp_path / "do not skip me 2.txt",
+ (original, tmp_path / "do not skip me.txt"),
+ (original, tmp_path / "do not skip me 2.txt"),
]
def test_filter_db_matches_should_skip_non_relative_paths(self, tmp_path: Path) -> None:
@@ -79,11 +87,116 @@ class TestFilterDbMatches:
self._row("/another/dir", "skip me 2.txt"),
self._row(tmp_path, "do not skip me 2.txt"),
]
+ original = tmp_path / "skip_me.txt"
found = list(_filter_db_matches(rows, tmp_path))
assert found == [
- tmp_path / "do not skip me.txt",
- tmp_path / "do not skip me 2.txt",
+ (original, tmp_path / "do not skip me.txt"),
+ (original, tmp_path / "do not skip me 2.txt"),
]
+ def test_filter_db_matches_should_report_originals_outside_base_dir(self, tmp_path: Path) -> None:
+ rows = [
+ self._row("/another/dir", "original.txt"),
+ self._row(tmp_path, "dupe.txt"),
+ ]
+ found = list(_filter_db_matches(rows, tmp_path))
+ assert found == [(Path("/another/dir/original.txt"), tmp_path / "dupe.txt")]
+
+ def test_filtering_a_single_row(self, tmp_path: Path) -> None:
+ assert list(_filter_db_matches([self._row(tmp_path, "alone.txt")], tmp_path)) == []
+
def test_filtering_no_rows(self) -> None:
assert list(_filter_db_matches([], Path("/a/dir"))) == []
+
+
[email protected](frozen=True, slots=True)
+class _DedupeCase:
+ deduper: Czkawka
+ file_hashes: FileHashes
+ original: Path
+ duplicate: Path
+
+ @property
+ def csv_file(self) -> Path:
+ return self.deduper.csv_files.dedupe
+
+ async def run(self) -> None:
+ await self.deduper.run(self.file_hashes)
+
+
+async def _seed_duplicates(manager: Manager, base_dir: Path, *names: str) -> tuple[list[Path], FileHashes]:
+ """Registers files that all share the same hash in the database. The first one is the original"""
+
+ files: list[Path] = []
+ for name in names:
+ file = base_dir / name
+ file.parent.mkdir(parents=True, exist_ok=True)
+ _ = file.write_text("same content")
+ await manager.database.hash.insert_or_update_hash_db(_HASH, "xxh128", file, None, None)
+ files.append(file)
+
+ return files, {_HASH: {files[0].stat().st_size: set(files)}}
+
+
+def _deduper(manager: Manager, base_dir: Path, log_folder: Path) -> Czkawka:
+ config = Config()
+ config.logs.resolve_filenames(log_folder)
+ return Czkawka(
+ base_dir=base_dir,
+ database=manager.database,
+ use_trash_bin=False,
+ csv_files=CSVFiles.from_config(config),
+ )
+
+
+class TestDedupeRun:
+ @pytest.fixture
+ async def case(self, running_manager: Manager, tmp_path: Path) -> _DedupeCase:
+ base_dir = tmp_path / "downloads"
+ files, file_hashes = await _seed_duplicates(running_manager, base_dir, "original.jpg", "sub/duplicate.jpg")
+ deduper = _deduper(running_manager, base_dir, tmp_path / "logs")
+ return _DedupeCase(deduper, file_hashes, files[0], files[1])
+
+ async def test_deletes_the_duplicate_and_keeps_the_original(self, case: _DedupeCase) -> None:
+ await case.run()
+ assert case.original.exists()
+ assert not case.duplicate.exists()
+
+ async def test_logs_the_path_of_the_original(self, case: _DedupeCase, logs: pytest.LogCaptureFixture) -> None:
+ await case.run()
+ assert f"Removed new download '{case.duplicate}'" in logs.text
+ assert f"duplicate of '{case.original}'" in logs.text
+
+ async def test_logs_the_start_and_the_end_of_the_dedupe(
+ self, case: _DedupeCase, logs: pytest.LogCaptureFixture
+ ) -> None:
+ await case.run()
+ assert "Starting dedupe" in logs.text
+ assert "Dedupe finished. Deleted 1 of 1 duplicates found" in logs.text
+
+ async def test_writes_the_csv_log(self, case: _DedupeCase) -> None:
+ await case.run()
+ assert case.csv_file.read_text("utf8").splitlines() == [
+ '"duplicate","original","hash"',
+ f'"{case.duplicate}","{case.original}","xxh128:{_HASH}"',
+ ]
+
+ async def test_writes_one_csv_row_per_duplicate_under_a_single_header(
+ self, running_manager: Manager, tmp_path: Path
+ ) -> None:
+ base_dir = tmp_path / "downloads"
+ files, file_hashes = await _seed_duplicates(running_manager, base_dir, "original.jpg", "one.jpg", "two.jpg")
+ deduper = _deduper(running_manager, base_dir, tmp_path / "logs")
+ await deduper.run(file_hashes)
+
+ original, *duplicates = files
+ header, *rows = deduper.csv_files.dedupe.read_text("utf8").splitlines()
+ assert header == '"duplicate","original","hash"'
+ assert sorted(rows) == sorted(f'"{dupe}","{original}","xxh128:{_HASH}"' for dupe in duplicates)
+
+ async def test_no_csv_log_when_nothing_was_deleted(self, case: _DedupeCase) -> None:
+ await case.deduper.run({})
+ assert not case.csv_file.exists()
+
+ def test_from_manager_uses_the_csv_file_from_the_config(self, manager: Manager) -> None:
+ assert Czkawka.from_manager(manager).csv_files.dedupe == manager.config.logs.files.dedupe