Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add a command to delete hashes of files that no longer exist (#2156)
Michael Menanno0b094d5bc4f0
diff --git a/cyberdrop_dl/commands/database.py b/cyberdrop_dl/commands/database.pyindex eb8cb3801..ea74a609d 100644--- a/cyberdrop_dl/commands/database.py+++ b/cyberdrop_dl/commands/database.py@@ -1,11 +1,19 @@-from typing import Annotated+from __future__ import annotations++from typing import TYPE_CHECKING, Annotatedfrom cyclopts import App, Parameterfrom cyberdrop_dl.commands import CLIarguments, SQLiteFile, open_folderfrom cyberdrop_dl.config.appdata import AppData+if TYPE_CHECKING:+ from cyberdrop_dl.database import Database+ from cyberdrop_dl.database.hash import PruneStats+app = App(name="database", help="Commands for managing the database")+prune_app = App(name="prune", help="Delete database entries that are no longer useful")+app.command(prune_app)@app.command()@@ -28,6 +36,31 @@ def transfer(transfer_db(database_file, force=force)+@prune_app.command()+def hashes(+ *,+ dry_run: Annotated[+ bool,+ Parameter(help="Report what would be deleted without modifying the database"),+ ] = False,+ cli: CLIarguments | None = None,+) -> None:+ """Delete the hashes of files that no longer exist on disk.++ Only affects the `hash` and `files` tables.+ """+ from cyberdrop_dl import aio, stats+ from cyberdrop_dl.database import Database++ db_file = (cli and cli.database_file) or AppData.default().db_file+ stats.print(aio.run(_prune_hashes(Database(db_file), dry_run=dry_run)))+++async def _prune_hashes(database: Database, *, dry_run: bool) -> PruneStats:+ async with database:+ return await database.hash.prune_missing_files(dry_run=dry_run)++@app.command()def file(*, cli: CLIarguments | None = None) -> None:"Show file path to the database"diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.pyindex 08a41cdbc..43fed84be 100644--- a/cyberdrop_dl/database/hash.py+++ b/cyberdrop_dl/database/hash.py@@ -1,7 +1,9 @@from __future__ import annotationsimport dataclasses+import functoolsimport logging+import osfrom pathlib import Pathfrom typing import TYPE_CHECKING, cast@@ -18,6 +20,18 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+_PATH_EXISTS = "path_exists"+"""Name of the custom SQLite function registered by `HashTable.prune_missing_files`"""++_WHERE_MISSING = f"WHERE {_PATH_EXISTS}(folder, download_filename) = 0"++[email protected](slots=True)+class PruneStats:+ hash_rows: int = 0+ file_rows: int = 0+ dry_run: bool = False+@dataclasses.dataclass(slots=True)class HashTable(Table, name="hash"):@@ -92,6 +106,32 @@ class HashTable(Table, name="hash"):cursor = await db_conn.execute(query, (hash_type, hash_value))return await cursor.fetchone() is not None+ async def prune_missing_files(self, *, dry_run: bool = False) -> PruneStats:+ """Delete every `hash` and `files` row whose file is no longer on disk."""+ stats = PruneStats(dry_run=dry_run)+ try:+ if dry_run:+ async with self.db.reader() as db_conn:+ await _register_path_exists(db_conn)+ stats.hash_rows = await _count_missing(db_conn, "hash")+ stats.file_rows = await _count_missing(db_conn, "files")++ return stats++ # Foreign keys are never enabled at runtime, so deletes do not cascade. `hash` goes first+ # so a failure can never leave a hash row pointing at a files row that is already gone+ async with self.db.writer() as db_conn:+ await _register_path_exists(db_conn)+ stats.hash_rows = await _delete_missing(db_conn, "hash")+ stats.file_rows = await _delete_missing(db_conn, "files")+ await db_conn.commit()++ return stats+ finally:+ # The answers are only good for as long as this one prune. Do not hold on to them, or to+ # the memory, once it is over+ _path_exists_inner.cache_clear()+async def insert_or_update_hash_db(self,hash_value: str,@@ -168,3 +208,43 @@ class HashTable(Table, name="hash"):),)await db_conn.commit()+++def _path_exists(folder: str, download_filename: str) -> bool:+ # `os.path` instead of `pathlib` because a large library means hundreds of thousands of these,+ # and building a Path for each one is not free+ return _path_exists_inner(os.path.join(folder, download_filename)) # noqa: PTH118++[email protected]_cache(maxsize=100_000)+def _path_exists_inner(path: str) -> bool:+ # Every hash of a file shares its path, so the same path is looked up once per `hash` row plus+ # once for its `files` row. Bounded because a large library can hold millions of rows; the worst+ # an eviction costs us is one repeated system call+ try:+ _ = os.stat(path) # noqa: PTH116+ except FileNotFoundError:+ return False+ except (OSError, ValueError):+ # `os.path.exists` reports all of these as "missing". Here they mean we could not find out+ # (no permission, an unreachable network drive), which is not a reason to delete the row+ return True+ return True+++async def _register_path_exists(db_conn: aiosqlite.Connection) -> None:+ # Deterministic does not memoize calls with different arguments, that is what the cache above is+ # for. It lets SQLite treat the call as a pure expression, which is what we want here+ await db_conn.create_function(_PATH_EXISTS, 2, _path_exists, deterministic=True)+++async def _delete_missing(db_conn: aiosqlite.Connection, table: str) -> int:+ cursor = await db_conn.execute(f"DELETE FROM {table} {_WHERE_MISSING};") # noqa: S608+ return cursor.rowcount+++async def _count_missing(db_conn: aiosqlite.Connection, table: str) -> int:+ cursor = await db_conn.execute(f"SELECT COUNT(*) AS count FROM {table} {_WHERE_MISSING};") # noqa: S608+ row = await cursor.fetchone()+ assert row is not None+ return row["count"]diff --git a/cyberdrop_dl/program_ui.py b/cyberdrop_dl/program_ui.pyindex bc31463b8..d4cad3c84 100644--- a/cyberdrop_dl/program_ui.py+++ b/cyberdrop_dl/program_ui.py@@ -28,6 +28,8 @@ if TYPE_CHECKING:from collections.abc import Callablefrom pathlib import Path+ from cyberdrop_dl.database import Database+ from cyberdrop_dl.database.hash import PruneStatsfrom cyberdrop_dl.manager import Manager_INPUT_FILE: ContextVar[Path] = ContextVar("_INPUT_FILE")@@ -49,6 +51,7 @@ def run(manager: Manager, input_file: Path) -> RetrySource | Path:"Download": lambda _: input_file,"Retry failed downloads": _retry_failed,"Create file hashes": _scan_and_create_hashes,+ "Delete hashes of missing files": _prune_hashes,"Sort files in download folder": _sort_files,"Edit URLs.txt": lambda _: _edit_urls(),"Edit config": _edit_config,@@ -77,6 +80,20 @@ def _scan_and_create_hashes(manager: Manager) -> None:enter_to_continue()+def _prune_hashes(manager: Manager) -> None:+ console.warning(+ "You are about to delete the hashes of every file in the database that no longer exists on disk",+ )+ if ask_confirmation(explicit=True):+ stats.print(aio.run(_prune(manager.database)))+ enter_to_continue()+++async def _prune(database: Database) -> PruneStats:+ async with database:+ return await database.hash.prune_missing_files()++def _sort_files(manager: Manager) -> None:sorter = Sorter.from_config(manager.config)console.warning(diff --git a/cyberdrop_dl/stats.py b/cyberdrop_dl/stats.pyindex 0dbe0fdfb..2b248c1c7 100644--- a/cyberdrop_dl/stats.py+++ b/cyberdrop_dl/stats.py@@ -4,6 +4,7 @@ import functoolsimport loggingfrom typing import TYPE_CHECKING, final+from cyberdrop_dl.database.hash import PruneStatsfrom cyberdrop_dl.logs import log_spacerfrom cyberdrop_dl.progress.dedupe import DedupeStatsfrom cyberdrop_dl.progress.hashing import HashingStats@@ -64,6 +65,15 @@ def _(stats: DedupeStats) -> None:logger.info(f" Errors: {errors:,} files", extra=Color.RED if errors else None)+def _(stats: PruneStats) -> None:+ log_spacer()+ logger.info("Prune Stats:", extra=Color.CYAN)+ verb = "Would delete" if stats.dry_run else "Deleted"+ logger.info(f" {verb}: {stats.hash_rows:,} hash entries", extra=Color.GREEN if stats.hash_rows else None)+ logger.info(f" {verb}: {stats.file_rows:,} file entries", extra=Color.GREEN if stats.file_rows else None)++@print.registerdef _(stats: FileStats) -> None:log_spacer()diff --git a/docs/reference/retry-and-history.md b/docs/reference/retry-and-history.mdindex 29218c074..e6fb6fdaf 100644--- a/docs/reference/retry-and-history.md+++ b/docs/reference/retry-and-history.md@@ -95,6 +95,30 @@ cyberdrop-dl retry all --from 2026-08-01 --ignore-history[`downloads.skip_and_mark_completed`](config/downloads.md) does the reverse of a retry. It marks every scraped file as completed without downloading anything. Use it to make CDL permanently skip a set of URLs.+## Pruning stale hashes++Auto dedupe deletes a new download when its hash matches a file CDL downloaded before, without checking that the older file is still there. If you moved, renamed or lost those files (a dead drive, a manual cleanup), the hash table keeps pointing at paths that no longer exist and new downloads keep getting deleted to preserve originals you do not have.++`prune hashes` clears that out. It walks every path recorded in the hash table, checks whether the file is still on disk, and deletes the entries of the ones that are gone:++```shell+cyberdrop-dl database prune hashes+```++Use `--dry-run` to see what it would delete without touching the database:++```shell+cyberdrop-dl database prune hashes --dry-run+```++The same action is available in the main menu as **Delete hashes of missing files**.++{% hint style="info" %}+This only touches the hash table (check 3). Download history (check 2) is left alone, so pruning never causes a URL you already downloaded to be downloaded again.+{% endhint %}++Pruning is a one time cleanup, not a setting. If you move files around again, run it again.+## Common scenarios| What happened | What to run | What CDL does |@@ -105,6 +129,7 @@ cyberdrop-dl retry all --from 2026-08-01 --ignore-history| You deleted some downloaded files and want them back | `cyberdrop-dl download <url> --ignore-history` | Ignores the database and downloads them again || You want to replace files you still have on disk | Move or rename them, then run with `--ignore-history` | Without moving them, the on disk check skips the download || You want CDL to never download a set of URLs | `cyberdrop-dl download <url> --skip-and-mark-completed` | Marks them as completed without downloading |+| Dedupe deleted new downloads whose "original" is gone | `cyberdrop-dl database prune hashes` | Deletes the hashes of files that are no longer on disk || You want to start over from scratch | Delete the database file | `cyberdrop-dl database file` prints its path |## Options that no longer existdiff --git a/tests/test_prune.py b/tests/test_prune.pynew file mode 100644index 000000000..6cc4ee674--- /dev/null+++ b/tests/test_prune.py@@ -0,0 +1,268 @@+# ruff: noqa: ASYNC240+from __future__ import annotations++import contextlib+import os+from typing import TYPE_CHECKING+from unittest.mock import patch++import pytest++from cyberdrop_dl import aio, program_ui+from cyberdrop_dl.__main__ import run_cdl+from cyberdrop_dl.database import Database+from cyberdrop_dl.database.hash import _path_exists, _path_exists_inner+from cyberdrop_dl.url_objects import AbsoluteHttpURL++if TYPE_CHECKING:+ from collections.abc import Generator+ from pathlib import Path++ from cyberdrop_dl.manager import Manager++_HASH = "a5aec63d0fef8660b00570f23be727e9"+_REFERER = AbsoluteHttpURL("https://example.com/file.txt")+++def _stat_raises(target: Path, error: OSError) -> Generator[None]:+ """Make `os.stat` fail for a single path, leaving every other path alone."""+ real_stat = os.stat++ def fake_stat(path, *args, **kwargs):+ if str(path) == str(target):+ raise error+ return real_stat(path, *args, **kwargs)++ with patch("os.stat", fake_stat):+ yield+++async def _add_file(database: Database, file: Path) -> None:+ file.parent.mkdir(parents=True, exist_ok=True)+ file.write_text(file.name)+ await database.hash.insert_or_update_hash_db(_HASH, "md5", file, file.name, _REFERER)+++async def _count(database: Database, table: str) -> int:+ async with database.reader() as db_conn:+ cursor = await db_conn.execute(f"SELECT COUNT(*) AS count FROM {table};") # noqa: S608+ row = await cursor.fetchone()+ assert row is not None+ return row["count"]+++async def _filenames(database: Database, table: str) -> set[str]:+ async with database.reader() as db_conn:+ rows = await db_conn.execute_fetchall(f"SELECT download_filename FROM {table};") # noqa: S608+ return {row["download_filename"] for row in rows}+++async def database(running_manager: Manager) -> Database:+ return running_manager.database+++def txt_file(tmp_cwd: Path) -> Path:+ file = tmp_cwd / "text_file.txt"+ file.write_text("12345")+ return file++[email protected](autouse=True)+def _clear_path_cache() -> Generator[None]:+ _path_exists_inner.cache_clear()+ yield+ _path_exists_inner.cache_clear()+++class TestPruneMissingFiles:+ async def test_removes_rows_of_deleted_files(self, database: Database, tmp_cwd: Path) -> None:+ kept, gone = tmp_cwd / "kept.txt", tmp_cwd / "gone.txt"+ await _add_file(database, kept)+ await _add_file(database, gone)+ gone.unlink()++ stats = await database.hash.prune_missing_files()++ assert stats.hash_rows == 1+ assert stats.file_rows == 1+ assert await _filenames(database, "files") == {"kept.txt"}+ assert await _filenames(database, "hash") == {"kept.txt"}++ async def test_keeps_everything_when_no_file_is_missing(self, database: Database, tmp_cwd: Path) -> None:+ await _add_file(database, tmp_cwd / "kept.txt")++ stats = await database.hash.prune_missing_files()++ assert stats.hash_rows == 0+ assert stats.file_rows == 0+ assert await _count(database, "files") == 1+ assert await _count(database, "hash") == 1++ async def test_removes_every_hash_algo_of_a_missing_file(self, database: Database, tmp_cwd: Path) -> None:+ gone = tmp_cwd / "gone.txt"+ await _add_file(database, gone)+ await database.hash.insert_or_update_hashes(_HASH, "sha256", gone)+ assert await _count(database, "hash") == 2+ gone.unlink()++ stats = await database.hash.prune_missing_files()++ assert stats.hash_rows == 2+ assert stats.file_rows == 1+ assert await _count(database, "hash") == 0++ async def test_removes_orphan_hash_row_with_no_files_row(self, database: Database, tmp_cwd: Path) -> None:+ orphan = tmp_cwd / "orphan.txt"+ await database.hash.insert_or_update_hashes(_HASH, "md5", orphan)+ assert await _count(database, "files") == 0++ stats = await database.hash.prune_missing_files()++ assert stats.hash_rows == 1+ assert stats.file_rows == 0+ assert await _count(database, "hash") == 0++ async def test_dry_run_reports_counts_without_deleting(self, database: Database, tmp_cwd: Path) -> None:+ gone = tmp_cwd / "gone.txt"+ await _add_file(database, tmp_cwd / "kept.txt")+ await _add_file(database, gone)+ gone.unlink()++ stats = await database.hash.prune_missing_files(dry_run=True)++ assert stats.dry_run is True+ assert stats.hash_rows == 1+ assert stats.file_rows == 1+ assert await _count(database, "files") == 2+ assert await _count(database, "hash") == 2++ async def test_keeps_rows_whose_path_cannot_be_checked(self, database: Database, tmp_cwd: Path) -> None:+ unreadable = tmp_cwd / "unreadable.txt"+ await _add_file(database, unreadable)+ unreadable.unlink()++ with _stat_raises(unreadable, PermissionError(13, "Permission denied")):+ stats = await database.hash.prune_missing_files()++ assert stats.hash_rows == 0+ assert stats.file_rows == 0+ assert await _count(database, "files") == 1+ assert await _count(database, "hash") == 1++ async def test_clears_the_path_cache_when_done(self, database: Database, tmp_cwd: Path) -> None:+ await _add_file(database, tmp_cwd / "kept.txt")+ _path_exists(str(tmp_cwd), "primed.txt")+ assert _path_exists_inner.cache_info().currsize == 1++ await database.hash.prune_missing_files()++ assert _path_exists_inner.cache_info().currsize == 0+++class TestPathExists:+ def test_reports_an_existing_file(self, txt_file: Path) -> None:+ assert _path_exists(str(txt_file.parent), txt_file.name) is True++ def test_reports_a_deleted_file(self, txt_file: Path) -> None:+ txt_file.unlink()+ assert _path_exists(str(txt_file.parent), txt_file.name) is False++ def test_reports_a_path_it_cannot_read_as_still_there(self, txt_file: Path) -> None:+ with _stat_raises(txt_file, PermissionError(13, "Permission denied")):+ assert _path_exists(str(txt_file.parent), txt_file.name) is True++ def test_looks_a_repeated_path_up_only_once(self, txt_file: Path) -> None:+ _path_exists(str(txt_file.parent), txt_file.name)+ _path_exists(str(txt_file.parent), txt_file.name)++ assert _path_exists_inner.cache_info().hits == 1+++class TestPruneCommand:+ @staticmethod+ def _seed(db_file: Path, *files: Path) -> None:+ async def seed() -> None:+ async with Database(db_file) as database:+ for file in files:+ await _add_file(database, file)++ aio.run(seed())++ @staticmethod+ def _read(db_file: Path, table: str) -> set[str]:+ async def read() -> set[str]:+ async with Database(db_file) as database:+ return await _filenames(database, table)++ return aio.run(read())++ def test_removes_rows_of_deleted_files(self, capsys: pytest.CaptureFixture[str], tmp_cwd: Path) -> None:+ db_file, kept, gone = tmp_cwd / "test.db", tmp_cwd / "kept.txt", tmp_cwd / "gone.txt"+ self._seed(db_file, kept, gone)+ gone.unlink()++ exit_code = run_cdl(["database", "prune", "hashes", "--database-file", str(db_file)])++ assert exit_code == 0+ assert "Deleted: 1 hash entries" in capsys.readouterr().out+ assert self._read(db_file, "files") == {"kept.txt"}+ assert self._read(db_file, "hash") == {"kept.txt"}++ def test_dry_run_keeps_every_row(self, tmp_cwd: Path) -> None:+ db_file, kept, gone = tmp_cwd / "test.db", tmp_cwd / "kept.txt", tmp_cwd / "gone.txt"+ self._seed(db_file, kept, gone)+ gone.unlink()++ exit_code = run_cdl(["database", "prune", "hashes", "--database-file", str(db_file), "--dry-run"])++ assert exit_code == 0+ assert self._read(db_file, "files") == {"kept.txt", "gone.txt"}+ assert self._read(db_file, "hash") == {"kept.txt", "gone.txt"}+++class TestPruneMenuEntry:+ @staticmethod+ def _seed(manager: Manager, *files: Path) -> None:+ async def seed() -> None:+ async with manager.database:+ for file in files:+ await _add_file(manager.database, file)++ aio.run(seed())++ @staticmethod+ def _read(manager: Manager, table: str) -> set[str]:+ async def read() -> set[str]:+ async with manager.database:+ return await _filenames(manager.database, table)++ return aio.run(read())++ @pytest.fixture(autouse=True)+ def _no_prompts(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(program_ui, "enter_to_continue", lambda: None)++ def test_prunes_after_explicit_confirmation(self, manager: Manager, tmp_cwd: Path) -> None:+ gone = tmp_cwd / "gone.txt"+ self._seed(manager, tmp_cwd / "kept.txt", gone)+ gone.unlink()++ with patch.object(program_ui, "ask_confirmation", return_value=True):+ program_ui._prune_hashes(manager)++ assert self._read(manager, "files") == {"kept.txt"}+ assert self._read(manager, "hash") == {"kept.txt"}++ def test_keeps_every_row_when_confirmation_is_declined(self, manager: Manager, tmp_cwd: Path) -> None:+ gone = tmp_cwd / "gone.txt"+ self._seed(manager, tmp_cwd / "kept.txt", gone)+ gone.unlink()++ with patch.object(program_ui, "ask_confirmation", return_value=False):+ program_ui._prune_hashes(manager)++ assert self._read(manager, "files") == {"kept.txt", "gone.txt"}+ assert self._read(manager, "hash") == {"kept.txt", "gone.txt"}