Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: enforce that CLI paths must exists
NTFSvolumea6686b839c75
diff --git a/cyberdrop_dl/cli/__init__.py b/cyberdrop_dl/cli.pysimilarity index 73%rename from cyberdrop_dl/cli/__init__.pyrename to cyberdrop_dl/cli.pyindex 34f9ef440..ca7375efe 100644--- a/cyberdrop_dl/cli/__init__.py+++ b/cyberdrop_dl/cli.py@@ -3,11 +3,11 @@ from cyclopts.core import Appfrom cyclopts.help import DefaultFormatterfrom cyberdrop_dl import __version__-from cyberdrop_dl.cli.clean_up import app as cleanup-from cyberdrop_dl.cli.database import app as database-from cyberdrop_dl.cli.hash import compute_hashes-from cyberdrop_dl.cli.main import download, interactive-from cyberdrop_dl.cli.report import report+from cyberdrop_dl.commands.clean_up import app as cleanup+from cyberdrop_dl.commands.database import app as database+from cyberdrop_dl.commands.hash import compute_hashes+from cyberdrop_dl.commands.report import report+from cyberdrop_dl.commands.scrape import download, interactiveapp = App(name="cyberdrop-dl",diff --git a/cyberdrop_dl/cli/hash.py b/cyberdrop_dl/cli/hash.pydeleted file mode 100644index 3998e6f63..000000000--- a/cyberdrop_dl/cli/hash.py+++ /dev/null@@ -1,28 +0,0 @@-from typing import Literal--from cyclopts.types import ExistingDirectory, ExistingFile--from cyberdrop_dl.cli.compat import check_for_v9_files---async def compute_hashes(- folder: ExistingDirectory,- /,- *,- database_file: ExistingFile | None = None,- hashes: tuple[Literal["xxh128", "md5", "sha256"], ...] = ("xxh128", "md5", "sha256"),-) -> None:- """Compute and save hashes of every file inside `folder` (recursively)"""- check_for_v9_files()- from cyberdrop_dl import stats- from cyberdrop_dl.config.appdata import AppData- from cyberdrop_dl.database import Database- from cyberdrop_dl.hasher import Hasher, hash_directory-- folder = folder.expanduser().resolve().absolute()- database = Database(database_file or AppData.default().db_file)- extra_hashes = set(hashes)- extra_hashes.discard("xxh128")- hasher = Hasher(tuple(extra_hashes), database, folder) # pyright: ignore[reportArgumentType]- hash_stats = await hash_directory(hasher)- stats.print(hash_stats)diff --git a/cyberdrop_dl/commands/__init__.py b/cyberdrop_dl/commands/__init__.pynew file mode 100644index 000000000..a23153288--- /dev/null+++ b/cyberdrop_dl/commands/__init__.py@@ -0,0 +1,21 @@+from typing import Annotated++from cyclopts import Parameter+from cyclopts.types import ResolvedExistingFile++from cyberdrop_dl.models import ConfigModel+from cyberdrop_dl.models.types import HttpURL+++@Parameter(name="*")+class CLIarguments(ConfigModel):+ urls: Annotated[tuple[HttpURL, ...], Parameter(show=False)] = ()++ config_file: ResolvedExistingFile | None = None+ "YAML file to use as config"++ cache_file: ResolvedExistingFile | None = None+ "JSON file to use as cache"++ database_file: ResolvedExistingFile | None = None+ "SQLite file to use as database"diff --git a/cyberdrop_dl/cli/clean_up.py b/cyberdrop_dl/commands/clean_up.pysimilarity index 60%rename from cyberdrop_dl/cli/clean_up.pyrename to cyberdrop_dl/commands/clean_up.pyindex 431375550..09a91f45b 100644--- a/cyberdrop_dl/cli/clean_up.py+++ b/cyberdrop_dl/commands/clean_up.py@@ -1,8 +1,8 @@import logging-from pathlib import Pathfrom typing import Annotated-from cyclopts import App, Parameter, validators+from cyclopts import App, Parameter+from cyclopts.types import ResolvedExistingDirectoryfrom cyberdrop_dl.utils import cleanup@@ -13,17 +13,13 @@ logger = logging.getLogger(__name__)@app.command()def files(path: Annotated[- Path,- Parameter(- help="Path of the folder to clean up",- validator=validators.Path(exists=True, file_okay=False, dir_okay=True),- ),+ ResolvedExistingDirectory,+ Parameter(help="Path of the folder to clean up"),],/,) -> None:- """Delete partial (`.cdl_hls` and `.part`) files, empty folders and empty files inside `dir` (recursive)"""+ """Delete partial (`.cdl_hls` and `.part`) files, empty folders and empty files inside `path` (recursive)"""- path = path.expanduser().resolve().absolute()logger.info("Deleting partial downloads...")cleanup.rm_partial_files(path)logger.info("Deleting empty files and folders...")diff --git a/cyberdrop_dl/cli/compat.py b/cyberdrop_dl/commands/compat.pysimilarity index 98%rename from cyberdrop_dl/cli/compat.pyrename to cyberdrop_dl/commands/compat.pyindex 4347f5c3e..df0977941 100644--- a/cyberdrop_dl/cli/compat.py+++ b/cyberdrop_dl/commands/compat.py@@ -1,5 +1,3 @@-from __future__ import annotations-import loggingimport timefrom pathlib import Pathdiff --git a/cyberdrop_dl/cli/database.py b/cyberdrop_dl/commands/database.pysimilarity index 60%rename from cyberdrop_dl/cli/database.pyrename to cyberdrop_dl/commands/database.pyindex 732f77a9f..116251386 100644--- a/cyberdrop_dl/cli/database.py+++ b/cyberdrop_dl/commands/database.py@@ -1,7 +1,9 @@-from pathlib import Pathfrom typing import Annotated-from cyclopts import App, Parameter, validators+from cyclopts import App, Parameter+from cyclopts.types import ResolvedExistingFile++from cyberdrop_dl.config.appdata import AppDataapp = App(name="database", help="Commands for managing the database")@@ -9,11 +11,8 @@ app = App(name="database", help="Commands for managing the database")@app.command()def transfer(db_path: Annotated[- Path,- Parameter(- help="Path to the SQLite database file to migrate",- validator=validators.Path(exists=True, file_okay=True, dir_okay=False, ext=".db"),- ),+ ResolvedExistingFile,+ Parameter(help="Path to the SQLite database file to migrate"),],*,force: Annotated[@@ -27,3 +26,9 @@ def transfer(from cyberdrop_dl.database.transfer import run as transfer_dbtransfer_db(db_path, force=force)+++def file() -> None:+ "Show path of default database"+ app.console.print(AppData.default().db_file)diff --git a/cyberdrop_dl/commands/hash.py b/cyberdrop_dl/commands/hash.pynew file mode 100644index 000000000..b1527e140--- /dev/null+++ b/cyberdrop_dl/commands/hash.py@@ -0,0 +1,40 @@+from typing import Literal++from cyclopts.types import ResolvedExistingDirectory++from cyberdrop_dl.commands import CLIarguments+from cyberdrop_dl.commands.compat import check_for_v9_files+from cyberdrop_dl.config import Config+++async def compute_hashes(+ folder: ResolvedExistingDirectory,+ /,+ *,+ hashes: tuple[Literal["xxh128", "md5", "sha256"], ...] | None = None,+ cli: CLIarguments | None = None,+) -> None:+ """Compute and save hashes of every file inside `folder` (recursively)"""+ check_for_v9_files()+ from cyberdrop_dl import stats+ from cyberdrop_dl.config.appdata import AppData+ from cyberdrop_dl.database import Database+ from cyberdrop_dl.hasher import Hasher, hash_directory++ db_file = cli.database_file if cli else None+ extra_hashes = _choose_hash_algos(hashes, cli)+ database = Database(db_file or AppData.default().db_file)+ hasher = Hasher(extra_hashes, database, folder) # pyright: ignore[reportArgumentType]+ hash_stats = await hash_directory(hasher)+ stats.print(hash_stats)+++def _choose_hash_algos(+ hashes: tuple[Literal["xxh128", "md5", "sha256"], ...] | None,+ cli: CLIarguments | None = None,+) -> tuple[str, ...]:+ if hashes:+ return tuple(set(hashes) - {"xxh128"})+ if cli and cli.config_file:+ return Config.from_file(cli.config_file).hashing.extra_hashes+ return "md5", "sha256"diff --git a/cyberdrop_dl/cli/report.py b/cyberdrop_dl/commands/report.pysimilarity index 100%rename from cyberdrop_dl/cli/report.pyrename to cyberdrop_dl/commands/report.pydiff --git a/cyberdrop_dl/cli/main.py b/cyberdrop_dl/commands/scrape.pysimilarity index 88%rename from cyberdrop_dl/cli/main.pyrename to cyberdrop_dl/commands/scrape.pyindex ca30b9d36..d048d27ba 100644--- a/cyberdrop_dl/cli/main.py+++ b/cyberdrop_dl/commands/scrape.py@@ -9,16 +9,17 @@ import cyclopts.validatorsfrom cyclopts import Parameterfrom cyclopts.group import Group-from cyberdrop_dl.cli.compat import check_for_v9_files+from cyberdrop_dl.commands import CLIarguments+from cyberdrop_dl.commands.compat import check_for_v9_filesfrom cyberdrop_dl.config import Configfrom cyberdrop_dl.config.appdata import AppDatafrom cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroupfrom cyberdrop_dl.logs import log_spacer, set_console_level, setup_file_logging-from cyberdrop_dl.models import ConfigModel, merge_models+from cyberdrop_dl.models import merge_modelsfrom cyberdrop_dl.models.types import HttpURL # noqa: TC001from cyberdrop_dl.utils import cleanup-_INTERACTIVE: ContextVar[bool] = ContextVar("_INTERACTIVE", default=False)+INTERACTIVE: ContextVar[bool] = ContextVar("_INTERACTIVE", default=False)logger = logging.getLogger(__name__)if TYPE_CHECKING:@@ -27,20 +28,6 @@ if TYPE_CHECKING:from cyberdrop_dl.manager import Manager-@Parameter(name="*")-class CLIargs(ConfigModel):- urls: Annotated[tuple[HttpURL, ...], Parameter(show=False)] = ()-- config_file: Path | None = None- "YAML file to use as config"-- cache_file: Path | None = None- "JSON file to use as cache"-- database_file: Path | None = None- "SQLite file to use as database"--async def _scrape(manager: Manager) -> None:from cyberdrop_dl import ffmpegfrom cyberdrop_dl.scrape_mapper import ScrapeMapper@@ -105,7 +92,7 @@ async def _notify(config: Config, stats_summary: str) -> None:def _show_interactive_ui(manager: Manager) -> None:- if not _INTERACTIVE.get():+ if not INTERACTIVE.get():returnfrom cyberdrop_dl import program_ui@@ -150,11 +137,11 @@ def interactive(validator=cyclopts.validators.Path(dir_okay=False),),] = Path("URLs.txt"), # pyright: ignore[reportCallInDefaultInitializer]- cli: CLIargs | None = None,+ cli: CLIarguments | None = None,) -> None:"Show a TUI menu equivalent to the CLI commands"- _INTERACTIVE.set(True)- download(input_file=input_file, cli=cli or CLIargs())+ INTERACTIVE.set(True)+ download(input_file=input_file, cli=cli or CLIarguments())def download(@@ -177,7 +164,7 @@ def download(validator=cyclopts.validators.Path(exists=True, dir_okay=False),),] = None,- cli: CLIargs | None = None,+ cli: CLIarguments | None = None,cli_overrides: Config | None = None,) -> None:"Download URLs"@@ -187,13 +174,13 @@ def download(from cyberdrop_dl.manager import Manager- cli = cli or CLIargs()+ cli = cli or CLIarguments()cli.urls = urlsappdata, config = _prepare_appdata_and_config(cli, cli_overrides)_main(Manager(cli, appdata, config, input_file))-def _prepare_appdata_and_config(cli: CLIargs, cli_overrides: Config | None = None) -> tuple[AppData, Config]:+def _prepare_appdata_and_config(cli: CLIarguments, cli_overrides: Config | None = None) -> tuple[AppData, Config]:appdata = AppData.create(config_file=cli.config_file,cache_file=cli.cache_file,diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.pyindex a5f556476..e3601beff 100644--- a/cyberdrop_dl/manager.py+++ b/cyberdrop_dl/manager.py@@ -29,7 +29,7 @@ if TYPE_CHECKING:from collections.abc import Generatorfrom pathlib import Path- from cyberdrop_dl.cli.main import CLIargs+ from cyberdrop_dl.commands import CLIargumentsfrom cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStatsfrom cyberdrop_dl.url_objects import MediaItem@@ -41,16 +41,16 @@ logger = logging.getLogger(__name__)class Manager:def __init__(self,- cli_args: CLIargs | None = None,+ cli_args: CLIarguments | None = None,appdata: AppData | None = None,config: Config | None = None,input_file: Path | None = None,) -> None:- from cyberdrop_dl.cli.main import CLIargs+ from cyberdrop_dl.commands import CLIargumentsself.cache: dict[str, Any] = {}self._appdata: AppData | None = appdata- self.cli_args: CLIargs = cli_args or CLIargs()+ self.cli_args: CLIarguments = cli_args or CLIarguments()self._config: Config | None = configself.input_file: Path | None = input_file@@ -228,7 +228,7 @@ def _log_enviroment() -> None:)-def _log_cli_args(cli_args: CLIargs) -> None:+def _log_cli_args(cli_args: CLIarguments) -> None:logger.debug({"CLI options": cli_args.model_dump(mode="json")})diff --git a/tests/test_cli.py b/tests/test_cli.pyindex 129a29a41..9e50d7386 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -2,7 +2,7 @@ import pytestfrom cyberdrop_dl import __version__from cyberdrop_dl.__main__ import run_cdl-from cyberdrop_dl.cli.report import generate_report+from cyberdrop_dl.commands.report import generate_report@pytest.mark.parametrize(diff --git a/tests/test_config.py b/tests/test_config.pyindex a33fa955c..b605ef631 100644--- a/tests/test_config.py+++ b/tests/test_config.py@@ -10,7 +10,7 @@ import yamlfrom cyclopts.exceptions import UnknownOptionErrorfrom pydantic import BaseModel-import cyberdrop_dl.cli.main+import cyberdrop_dl.commands.scrapefrom cyberdrop_dl.config import Config, Files, _resolve_paths, merge_additive_args, settingsfrom cyberdrop_dl.config.appdata import AppDatafrom cyberdrop_dl.config.auth import Authentication, Notifications@@ -154,7 +154,7 @@ class TestRuntimeLogsConfig:def test_default_config_does_not_need_ffmpeg() -> None:- cyberdrop_dl.cli.main._check_ffmpeg(Config())+ cyberdrop_dl.commands.scrape._check_ffmpeg(Config())def test_media_durations_need_ffmpeg() -> None:@@ -164,7 +164,7 @@ def test_media_durations_need_ffmpeg() -> None:assert duration.total_seconds() == 20assert config.filters.duration.needs_ffmpegwith pytest.raises(CDLConfigRuntimeErrorsGroup) as exc:- cyberdrop_dl.cli.main._check_ffmpeg(config)+ cyberdrop_dl.commands.scrape._check_ffmpeg(config)assert len(exc.value.exceptions) == 1assert type(exc.value.exceptions[0]) is RuntimeError