Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add a command to delete hashes of files that no longer exist (#2156)

Michael Menanno0b094d5bc4f0
diff --git a/cyberdrop_dl/commands/database.py b/cyberdrop_dl/commands/database.py
index eb8cb3801..ea74a609d 100644
--- a/cyberdrop_dl/commands/database.py
+++ b/cyberdrop_dl/commands/database.py
@@ -1,11 +1,19 @@
-from typing import Annotated
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Annotated
from cyclopts import App, Parameter
from cyberdrop_dl.commands import CLIarguments, SQLiteFile, open_folder
from cyberdrop_dl.config.appdata import AppData
+if TYPE_CHECKING:
+ from cyberdrop_dl.database import Database
+ from cyberdrop_dl.database.hash import PruneStats
+
app = App(name="database", help="Commands for managing the database")
+prune_app = App(name="prune", help="Delete database entries that are no longer useful")
+app.command(prune_app)
@app.command()
@@ -28,6 +36,31 @@ def transfer(
transfer_db(database_file, force=force)
+@prune_app.command()
+def hashes(
+ *,
+ dry_run: Annotated[
+ bool,
+ Parameter(help="Report what would be deleted without modifying the database"),
+ ] = False,
+ cli: CLIarguments | None = None,
+) -> None:
+ """Delete the hashes of files that no longer exist on disk.
+
+ Only affects the `hash` and `files` tables.
+ """
+ from cyberdrop_dl import aio, stats
+ from cyberdrop_dl.database import Database
+
+ db_file = (cli and cli.database_file) or AppData.default().db_file
+ stats.print(aio.run(_prune_hashes(Database(db_file), dry_run=dry_run)))
+
+
+async def _prune_hashes(database: Database, *, dry_run: bool) -> PruneStats:
+ async with database:
+ return await database.hash.prune_missing_files(dry_run=dry_run)
+
+
@app.command()
def file(*, cli: CLIarguments | None = None) -> None:
"Show file path to the database"
diff --git a/cyberdrop_dl/database/hash.py b/cyberdrop_dl/database/hash.py
index 08a41cdbc..43fed84be 100644
--- a/cyberdrop_dl/database/hash.py
+++ b/cyberdrop_dl/database/hash.py
@@ -1,7 +1,9 @@
from __future__ import annotations
import dataclasses
+import functools
import logging
+import os
from pathlib import Path
from typing import TYPE_CHECKING, cast
@@ -18,6 +20,18 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+_PATH_EXISTS = "path_exists"
+"""Name of the custom SQLite function registered by `HashTable.prune_missing_files`"""
+
+_WHERE_MISSING = f"WHERE {_PATH_EXISTS}(folder, download_filename) = 0"
+
+
[email protected](slots=True)
+class PruneStats:
+ hash_rows: int = 0
+ file_rows: int = 0
+ dry_run: bool = False
+
@dataclasses.dataclass(slots=True)
class HashTable(Table, name="hash"):
@@ -92,6 +106,32 @@ class HashTable(Table, name="hash"):
cursor = await db_conn.execute(query, (hash_type, hash_value))
return await cursor.fetchone() is not None
+ async def prune_missing_files(self, *, dry_run: bool = False) -> PruneStats:
+ """Delete every `hash` and `files` row whose file is no longer on disk."""
+ stats = PruneStats(dry_run=dry_run)
+ try:
+ if dry_run:
+ async with self.db.reader() as db_conn:
+ await _register_path_exists(db_conn)
+ stats.hash_rows = await _count_missing(db_conn, "hash")
+ stats.file_rows = await _count_missing(db_conn, "files")
+
+ return stats
+
+ # Foreign keys are never enabled at runtime, so deletes do not cascade. `hash` goes first
+ # so a failure can never leave a hash row pointing at a files row that is already gone
+ async with self.db.writer() as db_conn:
+ await _register_path_exists(db_conn)
+ stats.hash_rows = await _delete_missing(db_conn, "hash")
+ stats.file_rows = await _delete_missing(db_conn, "files")
+ await db_conn.commit()
+
+ return stats
+ finally:
+ # The answers are only good for as long as this one prune. Do not hold on to them, or to
+ # the memory, once it is over
+ _path_exists_inner.cache_clear()
+
async def insert_or_update_hash_db(
self,
hash_value: str,
@@ -168,3 +208,43 @@ class HashTable(Table, name="hash"):
),
)
await db_conn.commit()
+
+
+def _path_exists(folder: str, download_filename: str) -> bool:
+ # `os.path` instead of `pathlib` because a large library means hundreds of thousands of these,
+ # and building a Path for each one is not free
+ return _path_exists_inner(os.path.join(folder, download_filename)) # noqa: PTH118
+
+
[email protected]_cache(maxsize=100_000)
+def _path_exists_inner(path: str) -> bool:
+ # Every hash of a file shares its path, so the same path is looked up once per `hash` row plus
+ # once for its `files` row. Bounded because a large library can hold millions of rows; the worst
+ # an eviction costs us is one repeated system call
+ try:
+ _ = os.stat(path) # noqa: PTH116
+ except FileNotFoundError:
+ return False
+ except (OSError, ValueError):
+ # `os.path.exists` reports all of these as "missing". Here they mean we could not find out
+ # (no permission, an unreachable network drive), which is not a reason to delete the row
+ return True
+ return True
+
+
+async def _register_path_exists(db_conn: aiosqlite.Connection) -> None:
+ # Deterministic does not memoize calls with different arguments, that is what the cache above is
+ # for. It lets SQLite treat the call as a pure expression, which is what we want here
+ await db_conn.create_function(_PATH_EXISTS, 2, _path_exists, deterministic=True)
+
+
+async def _delete_missing(db_conn: aiosqlite.Connection, table: str) -> int:
+ cursor = await db_conn.execute(f"DELETE FROM {table} {_WHERE_MISSING};") # noqa: S608
+ return cursor.rowcount
+
+
+async def _count_missing(db_conn: aiosqlite.Connection, table: str) -> int:
+ cursor = await db_conn.execute(f"SELECT COUNT(*) AS count FROM {table} {_WHERE_MISSING};") # noqa: S608
+ row = await cursor.fetchone()
+ assert row is not None
+ return row["count"]
diff --git a/cyberdrop_dl/program_ui.py b/cyberdrop_dl/program_ui.py
index bc31463b8..d4cad3c84 100644
--- a/cyberdrop_dl/program_ui.py
+++ b/cyberdrop_dl/program_ui.py
@@ -28,6 +28,8 @@ if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
+ from cyberdrop_dl.database import Database
+ from cyberdrop_dl.database.hash import PruneStats
from cyberdrop_dl.manager import Manager
_INPUT_FILE: ContextVar[Path] = ContextVar("_INPUT_FILE")
@@ -49,6 +51,7 @@ def run(manager: Manager, input_file: Path) -> RetrySource | Path:
"Download": lambda _: input_file,
"Retry failed downloads": _retry_failed,
"Create file hashes": _scan_and_create_hashes,
+ "Delete hashes of missing files": _prune_hashes,
"Sort files in download folder": _sort_files,
"Edit URLs.txt": lambda _: _edit_urls(),
"Edit config": _edit_config,
@@ -77,6 +80,20 @@ def _scan_and_create_hashes(manager: Manager) -> None:
enter_to_continue()
+def _prune_hashes(manager: Manager) -> None:
+ console.warning(
+ "You are about to delete the hashes of every file in the database that no longer exists on disk",
+ )
+ if ask_confirmation(explicit=True):
+ stats.print(aio.run(_prune(manager.database)))
+ enter_to_continue()
+
+
+async def _prune(database: Database) -> PruneStats:
+ async with database:
+ return await database.hash.prune_missing_files()
+
+
def _sort_files(manager: Manager) -> None:
sorter = Sorter.from_config(manager.config)
console.warning(
diff --git a/cyberdrop_dl/stats.py b/cyberdrop_dl/stats.py
index 0dbe0fdfb..2b248c1c7 100644
--- a/cyberdrop_dl/stats.py
+++ b/cyberdrop_dl/stats.py
@@ -4,6 +4,7 @@ import functools
import logging
from typing import TYPE_CHECKING, final
+from cyberdrop_dl.database.hash import PruneStats
from cyberdrop_dl.logs import log_spacer
from cyberdrop_dl.progress.dedupe import DedupeStats
from cyberdrop_dl.progress.hashing import HashingStats
@@ -64,6 +65,15 @@ def _(stats: DedupeStats) -> None:
logger.info(f" Errors: {errors:,} files", extra=Color.RED if errors else None)
+def _(stats: PruneStats) -> None:
+ log_spacer()
+ logger.info("Prune Stats:", extra=Color.CYAN)
+ verb = "Would delete" if stats.dry_run else "Deleted"
+ logger.info(f" {verb}: {stats.hash_rows:,} hash entries", extra=Color.GREEN if stats.hash_rows else None)
+ logger.info(f" {verb}: {stats.file_rows:,} file entries", extra=Color.GREEN if stats.file_rows else None)
+
+
@print.register
def _(stats: FileStats) -> None:
log_spacer()
diff --git a/docs/reference/retry-and-history.md b/docs/reference/retry-and-history.md
index 29218c074..e6fb6fdaf 100644
--- a/docs/reference/retry-and-history.md
+++ b/docs/reference/retry-and-history.md
@@ -95,6 +95,30 @@ cyberdrop-dl retry all --from 2026-08-01 --ignore-history
[`downloads.skip_and_mark_completed`](config/downloads.md) does the reverse of a retry. It marks every scraped file as completed without downloading anything. Use it to make CDL permanently skip a set of URLs.
+## Pruning stale hashes
+
+Auto dedupe deletes a new download when its hash matches a file CDL downloaded before, without checking that the older file is still there. If you moved, renamed or lost those files (a dead drive, a manual cleanup), the hash table keeps pointing at paths that no longer exist and new downloads keep getting deleted to preserve originals you do not have.
+
+`prune hashes` clears that out. It walks every path recorded in the hash table, checks whether the file is still on disk, and deletes the entries of the ones that are gone:
+
+```shell
+cyberdrop-dl database prune hashes
+```
+
+Use `--dry-run` to see what it would delete without touching the database:
+
+```shell
+cyberdrop-dl database prune hashes --dry-run
+```
+
+The same action is available in the main menu as **Delete hashes of missing files**.
+
+{% hint style="info" %}
+This only touches the hash table (check 3). Download history (check 2) is left alone, so pruning never causes a URL you already downloaded to be downloaded again.
+{% endhint %}
+
+Pruning is a one time cleanup, not a setting. If you move files around again, run it again.
+
## Common scenarios
| What happened | What to run | What CDL does |
@@ -105,6 +129,7 @@ cyberdrop-dl retry all --from 2026-08-01 --ignore-history
| You deleted some downloaded files and want them back | `cyberdrop-dl download <url> --ignore-history` | Ignores the database and downloads them again |
| You want to replace files you still have on disk | Move or rename them, then run with `--ignore-history` | Without moving them, the on disk check skips the download |
| You want CDL to never download a set of URLs | `cyberdrop-dl download <url> --skip-and-mark-completed` | Marks them as completed without downloading |
+| Dedupe deleted new downloads whose "original" is gone | `cyberdrop-dl database prune hashes` | Deletes the hashes of files that are no longer on disk |
| You want to start over from scratch | Delete the database file | `cyberdrop-dl database file` prints its path |
## Options that no longer exist
diff --git a/tests/test_prune.py b/tests/test_prune.py
new file mode 100644
index 000000000..6cc4ee674
--- /dev/null
+++ b/tests/test_prune.py
@@ -0,0 +1,268 @@
+# ruff: noqa: ASYNC240
+from __future__ import annotations
+
+import contextlib
+import os
+from typing import TYPE_CHECKING
+from unittest.mock import patch
+
+import pytest
+
+from cyberdrop_dl import aio, program_ui
+from cyberdrop_dl.__main__ import run_cdl
+from cyberdrop_dl.database import Database
+from cyberdrop_dl.database.hash import _path_exists, _path_exists_inner
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+ from pathlib import Path
+
+ from cyberdrop_dl.manager import Manager
+
+_HASH = "a5aec63d0fef8660b00570f23be727e9"
+_REFERER = AbsoluteHttpURL("https://example.com/file.txt")
+
+
+def _stat_raises(target: Path, error: OSError) -> Generator[None]:
+ """Make `os.stat` fail for a single path, leaving every other path alone."""
+ real_stat = os.stat
+
+ def fake_stat(path, *args, **kwargs):
+ if str(path) == str(target):
+ raise error
+ return real_stat(path, *args, **kwargs)
+
+ with patch("os.stat", fake_stat):
+ yield
+
+
+async def _add_file(database: Database, file: Path) -> None:
+ file.parent.mkdir(parents=True, exist_ok=True)
+ file.write_text(file.name)
+ await database.hash.insert_or_update_hash_db(_HASH, "md5", file, file.name, _REFERER)
+
+
+async def _count(database: Database, table: str) -> int:
+ async with database.reader() as db_conn:
+ cursor = await db_conn.execute(f"SELECT COUNT(*) AS count FROM {table};") # noqa: S608
+ row = await cursor.fetchone()
+ assert row is not None
+ return row["count"]
+
+
+async def _filenames(database: Database, table: str) -> set[str]:
+ async with database.reader() as db_conn:
+ rows = await db_conn.execute_fetchall(f"SELECT download_filename FROM {table};") # noqa: S608
+ return {row["download_filename"] for row in rows}
+
+
+async def database(running_manager: Manager) -> Database:
+ return running_manager.database
+
+
+def txt_file(tmp_cwd: Path) -> Path:
+ file = tmp_cwd / "text_file.txt"
+ file.write_text("12345")
+ return file
+
+
[email protected](autouse=True)
+def _clear_path_cache() -> Generator[None]:
+ _path_exists_inner.cache_clear()
+ yield
+ _path_exists_inner.cache_clear()
+
+
+class TestPruneMissingFiles:
+ async def test_removes_rows_of_deleted_files(self, database: Database, tmp_cwd: Path) -> None:
+ kept, gone = tmp_cwd / "kept.txt", tmp_cwd / "gone.txt"
+ await _add_file(database, kept)
+ await _add_file(database, gone)
+ gone.unlink()
+
+ stats = await database.hash.prune_missing_files()
+
+ assert stats.hash_rows == 1
+ assert stats.file_rows == 1
+ assert await _filenames(database, "files") == {"kept.txt"}
+ assert await _filenames(database, "hash") == {"kept.txt"}
+
+ async def test_keeps_everything_when_no_file_is_missing(self, database: Database, tmp_cwd: Path) -> None:
+ await _add_file(database, tmp_cwd / "kept.txt")
+
+ stats = await database.hash.prune_missing_files()
+
+ assert stats.hash_rows == 0
+ assert stats.file_rows == 0
+ assert await _count(database, "files") == 1
+ assert await _count(database, "hash") == 1
+
+ async def test_removes_every_hash_algo_of_a_missing_file(self, database: Database, tmp_cwd: Path) -> None:
+ gone = tmp_cwd / "gone.txt"
+ await _add_file(database, gone)
+ await database.hash.insert_or_update_hashes(_HASH, "sha256", gone)
+ assert await _count(database, "hash") == 2
+ gone.unlink()
+
+ stats = await database.hash.prune_missing_files()
+
+ assert stats.hash_rows == 2
+ assert stats.file_rows == 1
+ assert await _count(database, "hash") == 0
+
+ async def test_removes_orphan_hash_row_with_no_files_row(self, database: Database, tmp_cwd: Path) -> None:
+ orphan = tmp_cwd / "orphan.txt"
+ await database.hash.insert_or_update_hashes(_HASH, "md5", orphan)
+ assert await _count(database, "files") == 0
+
+ stats = await database.hash.prune_missing_files()
+
+ assert stats.hash_rows == 1
+ assert stats.file_rows == 0
+ assert await _count(database, "hash") == 0
+
+ async def test_dry_run_reports_counts_without_deleting(self, database: Database, tmp_cwd: Path) -> None:
+ gone = tmp_cwd / "gone.txt"
+ await _add_file(database, tmp_cwd / "kept.txt")
+ await _add_file(database, gone)
+ gone.unlink()
+
+ stats = await database.hash.prune_missing_files(dry_run=True)
+
+ assert stats.dry_run is True
+ assert stats.hash_rows == 1
+ assert stats.file_rows == 1
+ assert await _count(database, "files") == 2
+ assert await _count(database, "hash") == 2
+
+ async def test_keeps_rows_whose_path_cannot_be_checked(self, database: Database, tmp_cwd: Path) -> None:
+ unreadable = tmp_cwd / "unreadable.txt"
+ await _add_file(database, unreadable)
+ unreadable.unlink()
+
+ with _stat_raises(unreadable, PermissionError(13, "Permission denied")):
+ stats = await database.hash.prune_missing_files()
+
+ assert stats.hash_rows == 0
+ assert stats.file_rows == 0
+ assert await _count(database, "files") == 1
+ assert await _count(database, "hash") == 1
+
+ async def test_clears_the_path_cache_when_done(self, database: Database, tmp_cwd: Path) -> None:
+ await _add_file(database, tmp_cwd / "kept.txt")
+ _path_exists(str(tmp_cwd), "primed.txt")
+ assert _path_exists_inner.cache_info().currsize == 1
+
+ await database.hash.prune_missing_files()
+
+ assert _path_exists_inner.cache_info().currsize == 0
+
+
+class TestPathExists:
+ def test_reports_an_existing_file(self, txt_file: Path) -> None:
+ assert _path_exists(str(txt_file.parent), txt_file.name) is True
+
+ def test_reports_a_deleted_file(self, txt_file: Path) -> None:
+ txt_file.unlink()
+ assert _path_exists(str(txt_file.parent), txt_file.name) is False
+
+ def test_reports_a_path_it_cannot_read_as_still_there(self, txt_file: Path) -> None:
+ with _stat_raises(txt_file, PermissionError(13, "Permission denied")):
+ assert _path_exists(str(txt_file.parent), txt_file.name) is True
+
+ def test_looks_a_repeated_path_up_only_once(self, txt_file: Path) -> None:
+ _path_exists(str(txt_file.parent), txt_file.name)
+ _path_exists(str(txt_file.parent), txt_file.name)
+
+ assert _path_exists_inner.cache_info().hits == 1
+
+
+class TestPruneCommand:
+ @staticmethod
+ def _seed(db_file: Path, *files: Path) -> None:
+ async def seed() -> None:
+ async with Database(db_file) as database:
+ for file in files:
+ await _add_file(database, file)
+
+ aio.run(seed())
+
+ @staticmethod
+ def _read(db_file: Path, table: str) -> set[str]:
+ async def read() -> set[str]:
+ async with Database(db_file) as database:
+ return await _filenames(database, table)
+
+ return aio.run(read())
+
+ def test_removes_rows_of_deleted_files(self, capsys: pytest.CaptureFixture[str], tmp_cwd: Path) -> None:
+ db_file, kept, gone = tmp_cwd / "test.db", tmp_cwd / "kept.txt", tmp_cwd / "gone.txt"
+ self._seed(db_file, kept, gone)
+ gone.unlink()
+
+ exit_code = run_cdl(["database", "prune", "hashes", "--database-file", str(db_file)])
+
+ assert exit_code == 0
+ assert "Deleted: 1 hash entries" in capsys.readouterr().out
+ assert self._read(db_file, "files") == {"kept.txt"}
+ assert self._read(db_file, "hash") == {"kept.txt"}
+
+ def test_dry_run_keeps_every_row(self, tmp_cwd: Path) -> None:
+ db_file, kept, gone = tmp_cwd / "test.db", tmp_cwd / "kept.txt", tmp_cwd / "gone.txt"
+ self._seed(db_file, kept, gone)
+ gone.unlink()
+
+ exit_code = run_cdl(["database", "prune", "hashes", "--database-file", str(db_file), "--dry-run"])
+
+ assert exit_code == 0
+ assert self._read(db_file, "files") == {"kept.txt", "gone.txt"}
+ assert self._read(db_file, "hash") == {"kept.txt", "gone.txt"}
+
+
+class TestPruneMenuEntry:
+ @staticmethod
+ def _seed(manager: Manager, *files: Path) -> None:
+ async def seed() -> None:
+ async with manager.database:
+ for file in files:
+ await _add_file(manager.database, file)
+
+ aio.run(seed())
+
+ @staticmethod
+ def _read(manager: Manager, table: str) -> set[str]:
+ async def read() -> set[str]:
+ async with manager.database:
+ return await _filenames(manager.database, table)
+
+ return aio.run(read())
+
+ @pytest.fixture(autouse=True)
+ def _no_prompts(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(program_ui, "enter_to_continue", lambda: None)
+
+ def test_prunes_after_explicit_confirmation(self, manager: Manager, tmp_cwd: Path) -> None:
+ gone = tmp_cwd / "gone.txt"
+ self._seed(manager, tmp_cwd / "kept.txt", gone)
+ gone.unlink()
+
+ with patch.object(program_ui, "ask_confirmation", return_value=True):
+ program_ui._prune_hashes(manager)
+
+ assert self._read(manager, "files") == {"kept.txt"}
+ assert self._read(manager, "hash") == {"kept.txt"}
+
+ def test_keeps_every_row_when_confirmation_is_declined(self, manager: Manager, tmp_cwd: Path) -> None:
+ gone = tmp_cwd / "gone.txt"
+ self._seed(manager, tmp_cwd / "kept.txt", gone)
+ gone.unlink()
+
+ with patch.object(program_ui, "ask_confirmation", return_value=False):
+ program_ui._prune_hashes(manager)
+
+ assert self._read(manager, "files") == {"kept.txt", "gone.txt"}
+ assert self._read(manager, "hash") == {"kept.txt", "gone.txt"}