Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: enforce that CLI paths must exists

NTFSvolumea6686b839c75
diff --git a/cyberdrop_dl/cli/__init__.py b/cyberdrop_dl/cli.py
similarity index 73%
rename from cyberdrop_dl/cli/__init__.py
rename to cyberdrop_dl/cli.py
index 34f9ef440..ca7375efe 100644
--- a/cyberdrop_dl/cli/__init__.py
+++ b/cyberdrop_dl/cli.py
@@ -3,11 +3,11 @@ from cyclopts.core import App
from cyclopts.help import DefaultFormatter
from cyberdrop_dl import __version__
-from cyberdrop_dl.cli.clean_up import app as cleanup
-from cyberdrop_dl.cli.database import app as database
-from cyberdrop_dl.cli.hash import compute_hashes
-from cyberdrop_dl.cli.main import download, interactive
-from cyberdrop_dl.cli.report import report
+from cyberdrop_dl.commands.clean_up import app as cleanup
+from cyberdrop_dl.commands.database import app as database
+from cyberdrop_dl.commands.hash import compute_hashes
+from cyberdrop_dl.commands.report import report
+from cyberdrop_dl.commands.scrape import download, interactive
app = App(
name="cyberdrop-dl",
diff --git a/cyberdrop_dl/cli/hash.py b/cyberdrop_dl/cli/hash.py
deleted file mode 100644
index 3998e6f63..000000000
--- a/cyberdrop_dl/cli/hash.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from typing import Literal
-
-from cyclopts.types import ExistingDirectory, ExistingFile
-
-from cyberdrop_dl.cli.compat import check_for_v9_files
-
-
-async def compute_hashes(
- folder: ExistingDirectory,
- /,
- *,
- database_file: ExistingFile | None = None,
- hashes: tuple[Literal["xxh128", "md5", "sha256"], ...] = ("xxh128", "md5", "sha256"),
-) -> None:
- """Compute and save hashes of every file inside `folder` (recursively)"""
- check_for_v9_files()
- from cyberdrop_dl import stats
- from cyberdrop_dl.config.appdata import AppData
- from cyberdrop_dl.database import Database
- from cyberdrop_dl.hasher import Hasher, hash_directory
-
- folder = folder.expanduser().resolve().absolute()
- database = Database(database_file or AppData.default().db_file)
- extra_hashes = set(hashes)
- extra_hashes.discard("xxh128")
- hasher = Hasher(tuple(extra_hashes), database, folder) # pyright: ignore[reportArgumentType]
- hash_stats = await hash_directory(hasher)
- stats.print(hash_stats)
diff --git a/cyberdrop_dl/commands/__init__.py b/cyberdrop_dl/commands/__init__.py
new file mode 100644
index 000000000..a23153288
--- /dev/null
+++ b/cyberdrop_dl/commands/__init__.py
@@ -0,0 +1,21 @@
+from typing import Annotated
+
+from cyclopts import Parameter
+from cyclopts.types import ResolvedExistingFile
+
+from cyberdrop_dl.models import ConfigModel
+from cyberdrop_dl.models.types import HttpURL
+
+
+@Parameter(name="*")
+class CLIarguments(ConfigModel):
+ urls: Annotated[tuple[HttpURL, ...], Parameter(show=False)] = ()
+
+ config_file: ResolvedExistingFile | None = None
+ "YAML file to use as config"
+
+ cache_file: ResolvedExistingFile | None = None
+ "JSON file to use as cache"
+
+ database_file: ResolvedExistingFile | None = None
+ "SQLite file to use as database"
diff --git a/cyberdrop_dl/cli/clean_up.py b/cyberdrop_dl/commands/clean_up.py
similarity index 60%
rename from cyberdrop_dl/cli/clean_up.py
rename to cyberdrop_dl/commands/clean_up.py
index 431375550..09a91f45b 100644
--- a/cyberdrop_dl/cli/clean_up.py
+++ b/cyberdrop_dl/commands/clean_up.py
@@ -1,8 +1,8 @@
import logging
-from pathlib import Path
from typing import Annotated
-from cyclopts import App, Parameter, validators
+from cyclopts import App, Parameter
+from cyclopts.types import ResolvedExistingDirectory
from cyberdrop_dl.utils import cleanup
@@ -13,17 +13,13 @@ logger = logging.getLogger(__name__)
@app.command()
def files(
path: Annotated[
- Path,
- Parameter(
- help="Path of the folder to clean up",
- validator=validators.Path(exists=True, file_okay=False, dir_okay=True),
- ),
+ ResolvedExistingDirectory,
+ Parameter(help="Path of the folder to clean up"),
],
/,
) -> None:
- """Delete partial (`.cdl_hls` and `.part`) files, empty folders and empty files inside `dir` (recursive)"""
+ """Delete partial (`.cdl_hls` and `.part`) files, empty folders and empty files inside `path` (recursive)"""
- path = path.expanduser().resolve().absolute()
logger.info("Deleting partial downloads...")
cleanup.rm_partial_files(path)
logger.info("Deleting empty files and folders...")
diff --git a/cyberdrop_dl/cli/compat.py b/cyberdrop_dl/commands/compat.py
similarity index 98%
rename from cyberdrop_dl/cli/compat.py
rename to cyberdrop_dl/commands/compat.py
index 4347f5c3e..df0977941 100644
--- a/cyberdrop_dl/cli/compat.py
+++ b/cyberdrop_dl/commands/compat.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
import logging
import time
from pathlib import Path
diff --git a/cyberdrop_dl/cli/database.py b/cyberdrop_dl/commands/database.py
similarity index 60%
rename from cyberdrop_dl/cli/database.py
rename to cyberdrop_dl/commands/database.py
index 732f77a9f..116251386 100644
--- a/cyberdrop_dl/cli/database.py
+++ b/cyberdrop_dl/commands/database.py
@@ -1,7 +1,9 @@
-from pathlib import Path
from typing import Annotated
-from cyclopts import App, Parameter, validators
+from cyclopts import App, Parameter
+from cyclopts.types import ResolvedExistingFile
+
+from cyberdrop_dl.config.appdata import AppData
app = App(name="database", help="Commands for managing the database")
@@ -9,11 +11,8 @@ app = App(name="database", help="Commands for managing the database")
@app.command()
def transfer(
db_path: Annotated[
- Path,
- Parameter(
- help="Path to the SQLite database file to migrate",
- validator=validators.Path(exists=True, file_okay=True, dir_okay=False, ext=".db"),
- ),
+ ResolvedExistingFile,
+ Parameter(help="Path to the SQLite database file to migrate"),
],
*,
force: Annotated[
@@ -27,3 +26,9 @@ def transfer(
from cyberdrop_dl.database.transfer import run as transfer_db
transfer_db(db_path, force=force)
+
+
+def file() -> None:
+ "Show path of default database"
+ app.console.print(AppData.default().db_file)
diff --git a/cyberdrop_dl/commands/hash.py b/cyberdrop_dl/commands/hash.py
new file mode 100644
index 000000000..b1527e140
--- /dev/null
+++ b/cyberdrop_dl/commands/hash.py
@@ -0,0 +1,40 @@
+from typing import Literal
+
+from cyclopts.types import ResolvedExistingDirectory
+
+from cyberdrop_dl.commands import CLIarguments
+from cyberdrop_dl.commands.compat import check_for_v9_files
+from cyberdrop_dl.config import Config
+
+
+async def compute_hashes(
+ folder: ResolvedExistingDirectory,
+ /,
+ *,
+ hashes: tuple[Literal["xxh128", "md5", "sha256"], ...] | None = None,
+ cli: CLIarguments | None = None,
+) -> None:
+ """Compute and save hashes of every file inside `folder` (recursively)"""
+ check_for_v9_files()
+ from cyberdrop_dl import stats
+ from cyberdrop_dl.config.appdata import AppData
+ from cyberdrop_dl.database import Database
+ from cyberdrop_dl.hasher import Hasher, hash_directory
+
+ db_file = cli.database_file if cli else None
+ extra_hashes = _choose_hash_algos(hashes, cli)
+ database = Database(db_file or AppData.default().db_file)
+ hasher = Hasher(extra_hashes, database, folder) # pyright: ignore[reportArgumentType]
+ hash_stats = await hash_directory(hasher)
+ stats.print(hash_stats)
+
+
+def _choose_hash_algos(
+ hashes: tuple[Literal["xxh128", "md5", "sha256"], ...] | None,
+ cli: CLIarguments | None = None,
+) -> tuple[str, ...]:
+ if hashes:
+ return tuple(set(hashes) - {"xxh128"})
+ if cli and cli.config_file:
+ return Config.from_file(cli.config_file).hashing.extra_hashes
+ return "md5", "sha256"
diff --git a/cyberdrop_dl/cli/report.py b/cyberdrop_dl/commands/report.py
similarity index 100%
rename from cyberdrop_dl/cli/report.py
rename to cyberdrop_dl/commands/report.py
diff --git a/cyberdrop_dl/cli/main.py b/cyberdrop_dl/commands/scrape.py
similarity index 88%
rename from cyberdrop_dl/cli/main.py
rename to cyberdrop_dl/commands/scrape.py
index ca30b9d36..d048d27ba 100644
--- a/cyberdrop_dl/cli/main.py
+++ b/cyberdrop_dl/commands/scrape.py
@@ -9,16 +9,17 @@ import cyclopts.validators
from cyclopts import Parameter
from cyclopts.group import Group
-from cyberdrop_dl.cli.compat import check_for_v9_files
+from cyberdrop_dl.commands import CLIarguments
+from cyberdrop_dl.commands.compat import check_for_v9_files
from cyberdrop_dl.config import Config
from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup
from cyberdrop_dl.logs import log_spacer, set_console_level, setup_file_logging
-from cyberdrop_dl.models import ConfigModel, merge_models
+from cyberdrop_dl.models import merge_models
from cyberdrop_dl.models.types import HttpURL # noqa: TC001
from cyberdrop_dl.utils import cleanup
-_INTERACTIVE: ContextVar[bool] = ContextVar("_INTERACTIVE", default=False)
+INTERACTIVE: ContextVar[bool] = ContextVar("_INTERACTIVE", default=False)
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
@@ -27,20 +28,6 @@ if TYPE_CHECKING:
from cyberdrop_dl.manager import Manager
-@Parameter(name="*")
-class CLIargs(ConfigModel):
- urls: Annotated[tuple[HttpURL, ...], Parameter(show=False)] = ()
-
- config_file: Path | None = None
- "YAML file to use as config"
-
- cache_file: Path | None = None
- "JSON file to use as cache"
-
- database_file: Path | None = None
- "SQLite file to use as database"
-
-
async def _scrape(manager: Manager) -> None:
from cyberdrop_dl import ffmpeg
from cyberdrop_dl.scrape_mapper import ScrapeMapper
@@ -105,7 +92,7 @@ async def _notify(config: Config, stats_summary: str) -> None:
def _show_interactive_ui(manager: Manager) -> None:
- if not _INTERACTIVE.get():
+ if not INTERACTIVE.get():
return
from cyberdrop_dl import program_ui
@@ -150,11 +137,11 @@ def interactive(
validator=cyclopts.validators.Path(dir_okay=False),
),
] = Path("URLs.txt"), # pyright: ignore[reportCallInDefaultInitializer]
- cli: CLIargs | None = None,
+ cli: CLIarguments | None = None,
) -> None:
"Show a TUI menu equivalent to the CLI commands"
- _INTERACTIVE.set(True)
- download(input_file=input_file, cli=cli or CLIargs())
+ INTERACTIVE.set(True)
+ download(input_file=input_file, cli=cli or CLIarguments())
def download(
@@ -177,7 +164,7 @@ def download(
validator=cyclopts.validators.Path(exists=True, dir_okay=False),
),
] = None,
- cli: CLIargs | None = None,
+ cli: CLIarguments | None = None,
cli_overrides: Config | None = None,
) -> None:
"Download URLs"
@@ -187,13 +174,13 @@ def download(
from cyberdrop_dl.manager import Manager
- cli = cli or CLIargs()
+ cli = cli or CLIarguments()
cli.urls = urls
appdata, config = _prepare_appdata_and_config(cli, cli_overrides)
_main(Manager(cli, appdata, config, input_file))
-def _prepare_appdata_and_config(cli: CLIargs, cli_overrides: Config | None = None) -> tuple[AppData, Config]:
+def _prepare_appdata_and_config(cli: CLIarguments, cli_overrides: Config | None = None) -> tuple[AppData, Config]:
appdata = AppData.create(
config_file=cli.config_file,
cache_file=cli.cache_file,
diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.py
index a5f556476..e3601beff 100644
--- a/cyberdrop_dl/manager.py
+++ b/cyberdrop_dl/manager.py
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
from collections.abc import Generator
from pathlib import Path
- from cyberdrop_dl.cli.main import CLIargs
+ from cyberdrop_dl.commands import CLIarguments
from cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStats
from cyberdrop_dl.url_objects import MediaItem
@@ -41,16 +41,16 @@ logger = logging.getLogger(__name__)
class Manager:
def __init__(
self,
- cli_args: CLIargs | None = None,
+ cli_args: CLIarguments | None = None,
appdata: AppData | None = None,
config: Config | None = None,
input_file: Path | None = None,
) -> None:
- from cyberdrop_dl.cli.main import CLIargs
+ from cyberdrop_dl.commands import CLIarguments
self.cache: dict[str, Any] = {}
self._appdata: AppData | None = appdata
- self.cli_args: CLIargs = cli_args or CLIargs()
+ self.cli_args: CLIarguments = cli_args or CLIarguments()
self._config: Config | None = config
self.input_file: Path | None = input_file
@@ -228,7 +228,7 @@ def _log_enviroment() -> None:
)
-def _log_cli_args(cli_args: CLIargs) -> None:
+def _log_cli_args(cli_args: CLIarguments) -> None:
logger.debug({"CLI options": cli_args.model_dump(mode="json")})
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 129a29a41..9e50d7386 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -2,7 +2,7 @@ import pytest
from cyberdrop_dl import __version__
from cyberdrop_dl.__main__ import run_cdl
-from cyberdrop_dl.cli.report import generate_report
+from cyberdrop_dl.commands.report import generate_report
@pytest.mark.parametrize(
diff --git a/tests/test_config.py b/tests/test_config.py
index a33fa955c..b605ef631 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -10,7 +10,7 @@ import yaml
from cyclopts.exceptions import UnknownOptionError
from pydantic import BaseModel
-import cyberdrop_dl.cli.main
+import cyberdrop_dl.commands.scrape
from cyberdrop_dl.config import Config, Files, _resolve_paths, merge_additive_args, settings
from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.config.auth import Authentication, Notifications
@@ -154,7 +154,7 @@ class TestRuntimeLogsConfig:
def test_default_config_does_not_need_ffmpeg() -> None:
- cyberdrop_dl.cli.main._check_ffmpeg(Config())
+ cyberdrop_dl.commands.scrape._check_ffmpeg(Config())
def test_media_durations_need_ffmpeg() -> None:
@@ -164,7 +164,7 @@ def test_media_durations_need_ffmpeg() -> None:
assert duration.total_seconds() == 20
assert config.filters.duration.needs_ffmpeg
with pytest.raises(CDLConfigRuntimeErrorsGroup) as exc:
- cyberdrop_dl.cli.main._check_ffmpeg(config)
+ cyberdrop_dl.commands.scrape._check_ffmpeg(config)
assert len(exc.value.exceptions) == 1
assert type(exc.value.exceptions[0]) is RuntimeError