Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove path manager (part 1) (#80)

NTFSvolume4f208b2ebfdf
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index 2638e5141..79e09872c 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -43,7 +43,7 @@ class HTTPClient:
return cls(
client_manager,
client_manager.manager.config_manager.settings_data.files.save_pages_html,
- client_manager.manager.path_manager.pages_folder,
+ client_manager.manager.config.logs.main_log.parent / "cdl_responses",
)
@property
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 71ccb4371..277bf880a 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -346,7 +346,7 @@ class DownloadClient:
try:
media_item.downloaded = downloaded
await self.manager.hash_manager.hash_client.hash_item_during_download(media_item)
- self.manager.path_manager.add_completed(media_item)
+ self.manager.add_completed(media_item)
except Exception:
logger.exception(f"Error handling media item completion of: {media_item.path}")
@@ -359,7 +359,7 @@ class DownloadClient:
return download_folder
if self.manager.config_manager.settings_data.download_options.block_download_sub_folders:
- while download_folder.parent != self.manager.path_manager.download_folder:
+ while download_folder.parent != self.manager.config.files.download_folder:
download_folder = download_folder.parent
media_item.download_folder = download_folder
return download_folder
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index 5a8004c10..712006c97 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -206,7 +206,7 @@ class HashClient:
async def get_file_hashes_dict(self) -> dict:
"""Get a dictionary of files based on matching file hashes and file size."""
- downloads = self.manager.path_manager.completed_downloads
+ downloads = self.manager.completed_downloads
async def exists(item: MediaItem) -> MediaItem | None:
if await asyncio.to_thread(item.path.is_file):
diff --git a/cyberdrop_dl/clients/jdownloader.py b/cyberdrop_dl/clients/jdownloader.py
index 6b3033426..fb83e6aa1 100644
--- a/cyberdrop_dl/clients/jdownloader.py
+++ b/cyberdrop_dl/clients/jdownloader.py
@@ -31,7 +31,7 @@ class Config:
@staticmethod
def from_manager(manager: Manager) -> Config:
- download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.path_manager.download_folder
+ download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.config.files.download_folder
return Config(
enabled=manager.config.runtime_options.send_unsupported_to_jdownloader,
device=manager.auth_config.jdownloader.device,
diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.py
index c1d711864..5f62b6836 100755
--- a/cyberdrop_dl/config/__init__.py
+++ b/cyberdrop_dl/config/__init__.py
@@ -123,9 +123,7 @@ class Config:
self.apprise_urls = get_apprise_urls(file=self.apprise_file)
def _resolve_all_paths(self) -> None:
- self.auth.resolve_paths()
self.settings.resolve_paths()
- self.global_settings.resolve_paths()
def _all_settings(self) -> tuple[ConfigSettings, AuthSettings, GlobalSettings]:
return self.settings, self.auth, self.global_settings
diff --git a/cyberdrop_dl/config/_common.py b/cyberdrop_dl/config/_common.py
index debbbe48c..aa533aa35 100755
--- a/cyberdrop_dl/config/_common.py
+++ b/cyberdrop_dl/config/_common.py
@@ -1,8 +1,6 @@
from pathlib import Path
from typing import Self
-from pydantic import BaseModel
-
from cyberdrop_dl import yaml
from cyberdrop_dl.exceptions import InvalidYamlError
from cyberdrop_dl.models import AliasModel, get_model_fields
@@ -30,18 +28,6 @@ class ConfigModel(AliasModel):
def save_to_file(self, file: Path) -> None:
yaml.save(file, self)
- def resolve_paths(self) -> None:
- self._resolve_paths(self)
-
- @classmethod
- def _resolve_paths(cls, model: BaseModel) -> None:
- for name, value in vars(model).items():
- if isinstance(value, Path):
- setattr(model, name, value.resolve())
-
- elif isinstance(value, BaseModel):
- cls._resolve_paths(value)
-
def _is_in_file(search_value: str, file: Path) -> bool:
try:
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py
index 0b33aa67e..f46f17bc4 100755
--- a/cyberdrop_dl/config/config_model.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -1,5 +1,5 @@
-import itertools
import re
+from collections.abc import Generator
from datetime import date, datetime, timedelta
from logging import DEBUG
from pathlib import Path
@@ -66,14 +66,14 @@ class DownloadOptions(BaseModel):
class Files(AliasModel):
download_folder: Path = Field(default=DEFAULT_DOWNLOAD_STORAGE, validation_alias="d")
dump_json: bool = Field(default=False, validation_alias="j")
- input_file: Path = Field(default=DEFAULT_APP_STORAGE / "Configs/{config}/URLs.txt", validation_alias="i")
+ input_file: Path = Field(default=Path("URLs.txt"), validation_alias="i")
save_pages_html: bool = False
class Logs(AliasModel):
download_error_urls: LogPath = Path("Download_Error_URLs.csv")
last_forum_post: LogPath = Path("Last_Scraped_Forum_Posts.csv")
- log_folder: Path = DEFAULT_APP_STORAGE / "Configs/{config}/Logs"
+ log_folder: Path = DEFAULT_APP_STORAGE / "Logs"
logs_expire_after: timedelta | None = None
main_log: MainLogPath = Path("downloader.log")
rotate_logs: bool = False
@@ -92,28 +92,38 @@ class Logs(AliasModel):
if value := falsy_as(input_date, None):
return to_timedelta(value)
+ def _files(self) -> Generator[tuple[str, Path]]:
+ for name, log_file in vars(self).items():
+ if name == "log_folder" or not isinstance(log_file, Path) or log_file.suffix not in (".csv", ".log"):
+ continue
+ yield name, log_file
+
def _set_output_filenames(self, now: datetime) -> None:
+ self.log_folder = self.log_folder.resolve()
self.log_folder.mkdir(exist_ok=True, parents=True)
current_time_file_iso: str = now.strftime(constants.LOGS_DATETIME_FORMAT)
current_time_folder_iso: str = now.strftime(constants.LOGS_DATE_FORMAT)
- for attr, log_file in vars(self).items():
- if not isinstance(log_file, Path) or log_file.suffix not in (".csv", ".log"):
- continue
-
+ for name, log_file in self._files():
+ log_file = self.log_folder / log_file
if self.rotate_logs:
new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}"
log_file = log_file.parent / current_time_folder_iso / new_name
- setattr(self, attr, self.log_folder / log_file)
+ setattr(self, name, log_file)
+
+ def mkdirs(self):
+ for _, log_file in self._files():
log_file.parent.mkdir(exist_ok=True, parents=True)
- def _delete_old_logs_and_folders(self, now: datetime | None = None) -> None:
- if now and self.logs_expire_after:
- for file in itertools.chain(self.log_folder.rglob("*.log"), self.log_folder.rglob("*.csv")):
- file_date = file.stat().st_ctime
- t_delta = now - datetime.fromtimestamp(file_date)
- if t_delta > self.logs_expire_after:
- file.unlink(missing_ok=True)
+ def _delete_old_logs_and_folders(self, now: datetime) -> None:
+ if self.logs_expire_after:
+ for file in self.log_folder.rglob("*"):
+ if file.suffix.lower() not in (".log", ".csv"):
+ continue
+
+ if (now - datetime.fromtimestamp(file.stat().st_ctime)) > self.logs_expire_after:
+ file.unlink()
+
purge_dir_tree(self.log_folder)
@@ -292,3 +302,27 @@ class ConfigSettings(ConfigModel):
logs: Logs = Logs()
runtime_options: RuntimeOptions = RuntimeOptions()
sorting: Sorting = Sorting()
+ _resolved: bool = False
+
+ def resolve_paths(self) -> None:
+ if self._resolved:
+ return
+
+ now = datetime.now()
+ self.logs._set_output_filenames(now)
+ self._resolve_paths(self)
+ self.logs._delete_old_logs_and_folders(now)
+ self.logs.mkdirs()
+ self._resolved = True
+
+ @classmethod
+ def _resolve_paths(cls, model: BaseModel) -> None:
+
+ for name, value in vars(model).items():
+ if isinstance(value, Path):
+ if "{config}" in str(value):
+ raise RuntimeError(f"Using '{{config}}' as reference on a path is no longer support: {value}")
+ setattr(model, name, value.expanduser().resolve().absolute())
+
+ elif isinstance(value, BaseModel):
+ cls._resolve_paths(value)
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 2907b77db..dd21cf945 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -62,7 +62,7 @@ def _ui_error_handling_wrapper(
@_ui_error_handling_wrapper
async def _run_manager(manager: Manager) -> None:
"""Runs the program and handles the UI."""
- manager.path_manager.startup()
+ manager.config.resolve_paths()
manager.logs.delete_old_logs()
debug_log_file_path = _setup_debug_logger(manager)
start_time = manager.start_time
@@ -126,7 +126,7 @@ async def _post_runtime(manager: Manager) -> None:
if manager.config_manager.settings_data.runtime_options.update_last_forum_post:
log_spacer(20)
- await manager.logs.update_last_forum_post(manager.path_manager.input_file)
+ await manager.logs.update_last_forum_post(manager.config.files.input_file)
def _setup_debug_logger(manager: Manager) -> Path | None:
@@ -162,7 +162,7 @@ def _setup_debug_logger(manager: Manager) -> Path | None:
def _setup_main_logger(manager: Manager) -> None:
logger = logging.getLogger("cyberdrop_dl")
- file_io = manager.path_manager.main_log.open("w", encoding="utf8")
+ file_io = manager.config.logs.main_log.open("w", encoding="utf8")
settings_data = manager.config_manager.settings_data
log_level = settings_data.runtime_options.log_level
logger.setLevel(log_level)
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 7bcc11703..06638b226 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -144,9 +144,9 @@ class Downloader:
self.client = self.manager.client_manager.download_client
self._semaphore = asyncio.Semaphore(self.manager.client_manager.get_download_slots(self.domain))
- self.manager.path_manager.download_folder.mkdir(parents=True, exist_ok=True)
+ self.manager.config.files.download_folder.mkdir(parents=True, exist_ok=True)
if self.manager.config_manager.settings_data.sorting.sort_downloads:
- self.manager.path_manager.sorted_folder.mkdir(parents=True, exist_ok=True)
+ self.manager.config.sorting.sort_folder.mkdir(parents=True, exist_ok=True)
def update_queued_files(self, increase_total: bool = True):
queued_files = self.manager.progress_manager.file_progress.get_queue_length()
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index f9a34b40f..bf949a549 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -326,7 +326,7 @@ class ClientManager:
get_cookies_from_browsers(
self.manager, browser=self.manager.config_manager.settings_data.browser_cookies.browser
)
- cookie_files = sorted(self.manager.path_manager.cookies_dir.glob("*.txt"))
+ cookie_files = sorted(self.manager.appdata.cookies.glob("*.txt"))
if not cookie_files:
return
async for domain, cookie in read_netscape_files(cookie_files):
diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py
index ec35eb917..b4d6592f6 100644
--- a/cyberdrop_dl/managers/config_manager.py
+++ b/cyberdrop_dl/managers/config_manager.py
@@ -1,6 +1,7 @@
from __future__ import annotations
from dataclasses import field
+from pathlib import Path
from time import sleep
from typing import TYPE_CHECKING
@@ -11,8 +12,6 @@ from cyberdrop_dl.managers.logs import LogManager
from cyberdrop_dl.utils.apprise import get_apprise_urls
if TYPE_CHECKING:
- from pathlib import Path
-
from pydantic import BaseModel
from cyberdrop_dl.managers.manager import Manager
@@ -38,10 +37,10 @@ class ConfigManager:
def startup(self) -> None:
"""Startup process for the config manager."""
self.loaded_config = self.manager.cache.get("default_config", "Default")
- self.settings = self.manager.path_manager.config_folder / self.loaded_config / "settings.yaml"
- self.global_settings = self.manager.path_manager.config_folder / "global_settings.yaml"
- self.authentication_settings = self.manager.path_manager.config_folder / "authentication.yaml"
- auth_override = self.manager.path_manager.config_folder / self.loaded_config / "authentication.yaml"
+ self.settings = self.manager.appdata.configs / self.loaded_config / "settings.yaml"
+ self.global_settings = self.manager.appdata.configs / "global_settings.yaml"
+ self.authentication_settings = self.manager.appdata.configs / "authentication.yaml"
+ auth_override = self.manager.appdata.configs / self.loaded_config / "authentication.yaml"
if auth_override.is_file():
self.authentication_settings = auth_override
@@ -54,7 +53,7 @@ class ConfigManager:
self._load_authentication_config()
self._load_global_settings_config()
self._load_settings_config()
- self.apprise_file = self.manager.path_manager.config_folder / self.loaded_config / "apprise.txt"
+ self.apprise_file = self.manager.appdata.configs / self.loaded_config / "apprise.txt"
self.apprise_urls = get_apprise_urls(file=self.apprise_file)
@staticmethod
@@ -98,15 +97,11 @@ class ConfigManager:
return
else:
self.settings_data = ConfigSettings()
- self.settings_data.files.input_file = (
- self.manager.path_manager.appdata / "Configs" / self.loaded_config / "URLs.txt"
- )
- downloads = self.manager.path_manager.cwd / "Downloads"
+ self.settings_data.files.input_file = self.manager.appdata.configs / self.loaded_config / "URLs.txt"
+ downloads = Path("Downloads").resolve()
self.settings_data.sorting.sort_folder = downloads / "Cyberdrop-DL Sorted Downloads"
self.settings_data.files.download_folder = downloads / "Cyberdrop-DL Downloads"
- self.settings_data.logs.log_folder = (
- self.manager.path_manager.appdata / "Configs" / self.loaded_config / "Logs"
- )
+ self.settings_data.logs.log_folder = self.manager.appdata.configs / self.loaded_config / "Logs"
yaml.save(self.settings, self.settings_data)
@@ -126,7 +121,7 @@ class ConfigManager:
def reload_config(self) -> None:
self.startup()
- self.manager.path_manager.startup()
+ self.manager.config.resolve_paths()
sleep(1)
self.manager.logs = LogManager.from_manager(self.manager)
sleep(1)
diff --git a/cyberdrop_dl/managers/logs.py b/cyberdrop_dl/managers/logs.py
index 2f5138391..668e8e034 100644
--- a/cyberdrop_dl/managers/logs.py
+++ b/cyberdrop_dl/managers/logs.py
@@ -54,11 +54,11 @@ class LogManager:
@classmethod
def from_manager(cls, manager: Manager) -> Self:
files = LogFiles(
- main_log=manager.path_manager.main_log,
- last_post_log=manager.path_manager.last_forum_post_log,
- unsupported_urls_log=manager.path_manager.unsupported_urls_log,
- download_error_log=manager.path_manager.download_error_urls_log,
- scrape_error_log=manager.path_manager.scrape_error_urls_log,
+ main_log=manager.config_manager.settings_data.logs.main_log,
+ last_post_log=manager.config_manager.settings_data.logs.last_forum_post,
+ unsupported_urls_log=manager.config_manager.settings_data.logs.unsupported_urls,
+ download_error_log=manager.config_manager.settings_data.logs.download_error_urls,
+ scrape_error_log=manager.config_manager.settings_data.logs.scrape_error_urls,
)
return cls(files, manager.task_group)
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 93034035d..7e5e8d944 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -1,9 +1,11 @@
from __future__ import annotations
import asyncio
+import dataclasses
import json
import logging
from dataclasses import Field, field
+from pathlib import Path
from time import perf_counter
from typing import TYPE_CHECKING, Any, NamedTuple, Self, TypeVar
@@ -18,7 +20,6 @@ from cyberdrop_dl.managers.config_manager import ConfigManager
from cyberdrop_dl.managers.hash_manager import HashManager
from cyberdrop_dl.managers.live_manager import LiveManager
from cyberdrop_dl.managers.logs import LogManager
-from cyberdrop_dl.managers.path_manager import PathManager
from cyberdrop_dl.managers.progress_manager import ProgressManager
from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger
from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information
@@ -26,7 +27,9 @@ from cyberdrop_dl.utils.utilities import close_if_defined, get_system_informatio
if TYPE_CHECKING:
from asyncio import TaskGroup
from collections.abc import Sequence
+ from os import PathLike
+ from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
@@ -45,7 +48,6 @@ class Manager:
self.parsed_args: ParsedArgs = field(init=False)
self.cache: dict[str, Any] = {}
- self.path_manager: PathManager = field(init=False)
self.config_manager: ConfigManager = field(init=False)
self.hash_manager: HashManager = field(init=False)
@@ -67,6 +69,8 @@ class Manager:
self.loggers: dict[str, QueuedLogger] = {}
self.args = args
self.states: AsyncioEvents
+ self._appdata: AppData | None = None
+ self._completed_downloads: list[MediaItem] = []
constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)
@@ -83,9 +87,9 @@ class Manager:
return self.config_manager.global_settings_data
async def __aenter__(self) -> Self:
- cache_file = self.path_manager.cache_folder / "cache.yaml"
+ cache_file = self.appdata.cache_file
try:
- self.cache.update(yaml.load(self.path_manager.cache_folder / "cache.yaml"))
+ self.cache.update(yaml.load(cache_file))
except FileNotFoundError:
cache_file.parent.mkdir(exist_ok=True, parents=True)
cache_file.touch()
@@ -93,7 +97,7 @@ class Manager:
async def __aexit__(self, *_) -> None:
self.cache["version"] = __version__
- yaml.save(self.path_manager.cache_folder / "cache.yaml", self.cache)
+ yaml.save(self.appdata.cache_file, self.cache)
def startup(self) -> None:
"""Startup process for the manager."""
@@ -101,17 +105,24 @@ class Manager:
if isinstance(self.parsed_args, Field):
self.parsed_args = parse_args(self.args)
- self.path_manager = PathManager(self)
- self.path_manager.pre_startup()
+ self.appdata.mkdirs()
self.config_manager = ConfigManager(self)
self.config_manager.startup()
self.args_consolidation()
-
- self.path_manager.startup()
+ self.config.resolve_paths()
self.logs = LogManager.from_manager(self)
+ def add_completed(self, media_item: MediaItem) -> None:
+ if media_item.is_segment:
+ return
+ self._completed_downloads.append(media_item)
+
+ @property
+ def completed_downloads(self) -> list[MediaItem]:
+ return self._completed_downloads
+
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
async def async_startup(self) -> None:
@@ -131,11 +142,11 @@ class Manager:
async def async_db_hash_startup(self) -> None:
if not isinstance(self.db_manager, Database):
self.db_manager = Database(
- self.path_manager.history_db,
+ self.appdata.db_file,
self.config.runtime_options.ignore_history,
)
await self.db_manager.startup()
- transfer_v5_db_to_v6(self.path_manager.history_db)
+ transfer_v5_db_to_v6(self.appdata.db_file)
if not isinstance(self.hash_manager, HashManager):
self.hash_manager = HashManager(self)
if not isinstance(self.live_manager, LiveManager):
@@ -186,9 +197,9 @@ class Manager:
f"System Info: {system_info}",
f"Using Config: {self.config_manager.loaded_config}",
f"Using Config File: {self.config_manager.settings}",
- f"Using Input File: {self.path_manager.input_file}",
- f"Using Download Folder: {self.path_manager.download_folder}",
- f"Using Database File: {self.path_manager.history_db}",
+ f"Using Input File: {self.config.files.input_file}",
+ f"Using Download Folder: {self.config.files.download_folder}",
+ f"Using Database File: {self.appdata.db_file}",
f"Using CLI only options: {cli_only_args}",
f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}",
f"Using Settings: \n{config_settings}",
@@ -216,6 +227,18 @@ class Manager:
_, queued_logger = self.loggers.popitem()
queued_logger.stop()
+ @property
+ def appdata(self) -> AppData:
+ if self._appdata is None:
+ if folder := self.parsed_args.cli_only_args.appdata_folder:
+ path = folder / "AppData"
+ else:
+ path = Path("AppData")
+
+ self._appdata = AppData.from_path(path.resolve())
+
+ return self._appdata
+
def add_or_remove_lists(cli_values: list[str], config_values: list[str]) -> None:
exclude = {"+", "-"}
@@ -255,3 +278,45 @@ def merge_models(default: M, new: M) -> M:
updated_dict = merge_dicts(default_dict, new_dict)
return default.model_validate(updated_dict)
+
+
[email protected](slots=True, frozen=True, kw_only=True)
+class AppData:
+ path: Path
+ cache_file: Path
+ db_file: Path
+ config_file: Path
+
+ cache: Path
+ cookies: Path
+ configs: Path
+
+ @classmethod
+ def from_path(cls, path: Path) -> Self:
+ assert path.is_absolute()
+ cache = path / "Cache"
+ return cls(
+ path=path,
+ cache=cache,
+ configs=path / "Configs",
+ cookies=path / "Cookies",
+ config_file=path / "config.yaml",
+ cache_file=cache / "cache.yaml",
+ db_file=cache / "cyberdrop.db",
+ )
+
+ def __truediv__(self, other: PathLike[str]):
+ try:
+ return self.path / other
+ except TypeError:
+ return NotImplemented
+
+ def __fspath__(self) -> str:
+ return str(self)
+
+ def __str__(self) -> str:
+ return str(self.path)
+
+ def mkdirs(self) -> None:
+ for dir in (self.cache, self.configs, self.cookies):
+ dir.mkdir(parents=True, exist_ok=True)
diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.py
deleted file mode 100644
index 4073d57b9..000000000
--- a/cyberdrop_dl/managers/path_manager.py
+++ /dev/null
@@ -1,160 +0,0 @@
-from __future__ import annotations
-
-import os
-from dataclasses import Field, field
-from datetime import datetime
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-from cyberdrop_dl import env
-from cyberdrop_dl.utils.utilities import purge_dir_tree
-
-if TYPE_CHECKING:
- from cyberdrop_dl.data_structures.url_objects import MediaItem
- from cyberdrop_dl.managers.manager import Manager
-
-
-class PathManager:
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
-
- self.download_folder: Path = field(init=False)
- self.sorted_folder: Path = field(init=False)
- self.scan_folder: Path | None = field(init=False)
-
- self.log_folder: Path = field(init=False)
-
- self.cache_folder: Path = field(init=False)
- self.config_folder: Path = field(init=False)
-
- self.input_file: Path = field(init=False)
- self.history_db: Path = field(init=False)
- self.cache_db: Path = field(init=False)
-
- self._completed_downloads: list[MediaItem] = []
-
- self.main_log: Path = field(init=False)
- self.last_forum_post_log: Path = field(init=False)
- self.unsupported_urls_log: Path = field(init=False)
- self.download_error_urls_log: Path = field(init=False)
- self.scrape_error_urls_log: Path = field(init=False)
- self.pages_folder: Path = field(init=False)
-
- self._logs_model_names = [
- "main_log",
- "last_forum_post",
- "unsupported_urls",
- "download_error_urls",
- "scrape_error_urls",
- ]
- self._appdata: Path = field(init=False)
-
- @property
- def cwd(self) -> Path:
- if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":
- # This is for testing purposes only"""
- return Path("..").resolve()
- return Path().resolve()
-
- @property
- def appdata(self) -> Path:
- if isinstance(self._appdata, Field):
- if self.manager.parsed_args.cli_only_args.appdata_folder:
- path = self.manager.parsed_args.cli_only_args.appdata_folder / "AppData"
- self._appdata = self.cwd / path
- else:
- self._appdata = self.cwd / "AppData"
-
- return self._appdata
-
- def pre_startup(self) -> None:
- self.cache_folder = self.appdata / "Cache"
- self.config_folder = self.appdata / "Configs"
- self.cookies_dir = self.appdata / "Cookies"
- self.cache_db = self.cache_folder / "request_cache.db"
-
- self.cache_folder.mkdir(parents=True, exist_ok=True)
- self.config_folder.mkdir(parents=True, exist_ok=True)
- self.cookies_dir.mkdir(parents=True, exist_ok=True)
- self.cache_db.touch(exist_ok=True)
-
- def startup(self) -> None:
- """Startup process for the Directory Manager."""
- settings_data = self.manager.config_manager.settings_data
- current_config = self.manager.config_manager.loaded_config
-
- def replace(path: Path) -> Path:
- path_w_config = str(path).replace("{config}", current_config)
- if os.name == "nt":
- return self.cwd.joinpath(Path(path_w_config)).resolve()
- normalized_path_str = path_w_config.replace("\\", "/")
- return self.cwd.joinpath(Path(normalized_path_str)).resolve()
-
- self.download_folder = replace(settings_data.files.download_folder)
- self.sorted_folder = replace(settings_data.sorting.sort_folder)
- self.log_folder = replace(settings_data.logs.log_folder)
- self.input_file = replace(settings_data.files.input_file)
- self.history_db = self.cache_folder / "cyberdrop.db"
- self.scan_folder = settings_data.sorting.scan_folder
- if self.scan_folder:
- self.scan_folder = replace(self.scan_folder)
-
- self.log_folder.mkdir(parents=True, exist_ok=True)
-
- now = datetime.now()
- self._set_output_filenames(now)
- self._delete_logs_and_folders(now)
- self._create_output_folders()
-
- if not self.input_file.is_file():
- self.input_file.touch(exist_ok=True)
- self.history_db.touch(exist_ok=True)
-
- def _set_output_filenames(self, now: datetime) -> None:
- current_time_file_iso: str = now.strftime("%Y%m%d_%H%M%S")
- current_time_folder_iso: str = now.strftime("%Y_%m_%d")
- log_settings_config = self.manager.config_manager.settings_data.logs
- log_files: dict[str, Path] = log_settings_config.model_dump()
-
- for model_name, log_file in log_files.items():
- if model_name not in self._logs_model_names:
- continue
- if log_settings_config.rotate_logs:
- new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}"
- log_file: Path = log_file.parent / current_time_folder_iso / new_name
- log_files[model_name] = log_file
-
- log_settings_config = log_settings_config.model_copy(update=log_files)
-
- for model_name in self._logs_model_names:
- internal_name = f"{model_name.replace('_log', '')}_log"
- setattr(self, internal_name, self.log_folder / getattr(log_settings_config, model_name))
-
- self.pages_folder = self.main_log.parent / "cdl_responses"
-
- def _delete_logs_and_folders(self, now: datetime):
- if self.manager.config_manager.settings_data.logs.logs_expire_after:
- for file in set(self.log_folder.rglob("*.log")) | set(self.log_folder.rglob("*.csv")):
- file_date = Path(file).stat().st_ctime
- t_delta = now - datetime.fromtimestamp(file_date)
- if t_delta > self.manager.config_manager.settings_data.logs.logs_expire_after:
- file.unlink(missing_ok=True)
- purge_dir_tree(self.log_folder)
-
- def _create_output_folders(self):
- for model_name in self._logs_model_names:
- internal_name = f"{model_name.replace('_log', '')}_log"
- path: Path = getattr(self, internal_name)
- path.parent.mkdir(parents=True, exist_ok=True)
-
- if self.manager.config_manager.settings_data.files.save_pages_html:
- self.pages_folder.mkdir(parents=True, exist_ok=True)
-
- def add_completed(self, media_item: MediaItem) -> None:
- if media_item.is_segment:
- return
- self._completed_downloads.append(media_item)
-
- @property
- def completed_downloads(self) -> list[MediaItem]:
- return self._completed_downloads
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index 1ca7bd486..6cb867ddf 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -149,10 +149,10 @@ class ProgressManager:
log_spacer(20)
logger.info("Printing Stats...\n")
- config_path = self.manager.path_manager.config_folder / self.manager.config_manager.loaded_config
+ config_path = self.manager.appdata.configs / self.manager.config_manager.loaded_config
config_path_text = get_console_hyperlink(config_path, text=self.manager.config_manager.loaded_config)
input_file_text = get_input(self.manager)
- log_folder_text = get_console_hyperlink(self.manager.path_manager.log_folder)
+ log_folder_text = get_console_hyperlink(self.manager.config.logs.log_folder)
log_concat("Run Stats (config: ", config_path_text, ")", style="cyan")
log_concat(" Input File: ", input_file_text, style="yellow")
@@ -218,7 +218,7 @@ def get_input(manager: Manager) -> Text | str:
if manager.parsed_args.cli_only_args.retry_maintenance:
return "--retry-maintenance"
if manager.scrape_mapper.using_input_file:
- return get_console_hyperlink(manager.path_manager.input_file)
+ return get_console_hyperlink(manager.config.files.input_file)
return "--links (CLI args)"
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 5274f7677..b5cff10d8 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -135,7 +135,7 @@ class ScrapeMapper:
async def parse_input_file_groups(self) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:
"""Split URLs from input file by their groups."""
- input_file = self.manager.path_manager.input_file
+ input_file = self.manager.config.files.input_file
if not await asyncio.to_thread(input_file.is_file):
yield ("", [])
return
@@ -235,7 +235,7 @@ class ScrapeMapper:
success = False
try:
download_folder = get_download_path(self.manager, scrape_item, "jdownloader")
- relative_download_dir = download_folder.relative_to(self.manager.path_manager.download_folder)
+ relative_download_dir = download_folder.relative_to(self.manager.config.files.download_folder)
await self.jdownloader.send(
scrape_item.url,
scrape_item.parent_title,
diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.py
index 72575a0d6..59783cce7 100644
--- a/cyberdrop_dl/ui/program_ui.py
+++ b/cyberdrop_dl/ui/program_ui.py
@@ -81,7 +81,7 @@ class ProgramUI:
def _scan_and_create_hashes(self) -> None:
"""Scans a folder and creates hashes for all of its files."""
- path = ask_dir_path("Select the directory to scan", default=str(self.manager.path_manager.download_folder))
+ path = ask_dir_path("Select the directory to scan", default=str(self.manager.config.files.download_folder))
hash_directory_scanner(self.manager, path)
def _sort_files(self) -> None:
@@ -101,7 +101,7 @@ class ProgramUI:
console.print(Markdown(changelog_content, justify="left"))
def _edit_urls(self) -> None:
- self._open_in_text_editor(self.manager.path_manager.input_file, reload_config=False)
+ self._open_in_text_editor(self.manager.config.files.input_file, reload_config=False)
def _open_in_text_editor(self, file_path: Path, *, reload_config: bool = True):
try:
diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.py
index 5f77257e9..41ed775ec 100644
--- a/cyberdrop_dl/utils/apprise.py
+++ b/cyberdrop_dl/utils/apprise.py
@@ -189,7 +189,7 @@ async def send_apprise_notifications(manager: Manager) -> tuple[constants.Notifi
for apprise_url in apprise_urls:
apprise_obj.add(apprise_url.url, tag=list(apprise_url.tags))
- main_log = manager.path_manager.main_log
+ main_log = manager.config.logs.main_log
all_urls = [x.raw_url for x in apprise_urls]
log_lines = []
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
index c3556dc0e..297c7cbf9 100644
--- a/cyberdrop_dl/utils/cookie_management.py
+++ b/cyberdrop_dl/utils/cookie_management.py
@@ -93,10 +93,10 @@ def get_cookies_from_browsers(manager: Manager, *, browser: BROWSERS, domains: l
msg = "None of the provided browsers is supported for extraction"
raise ValueError(msg)
- manager.path_manager.cookies_dir.mkdir(parents=True, exist_ok=True)
+ manager.appdata.cookies.mkdir(parents=True, exist_ok=True)
domains_with_cookies: set[str] = set()
for domain in domains_to_extract:
- cookie_file_path = manager.path_manager.cookies_dir / f"{domain}.txt"
+ cookie_file_path = manager.appdata.cookies / f"{domain}.txt"
cdl_cookie_jar = MozillaCookieJar(cookie_file_path)
for cookie in extracted_cookies:
if domain in cookie.domain:
@@ -113,9 +113,9 @@ def clear_cookies(manager: Manager, domains: list[str]) -> None:
if not domains:
raise ValueError("No domains selected")
- manager.path_manager.cookies_dir.mkdir(parents=True, exist_ok=True)
+ manager.appdata.cookies.mkdir(parents=True, exist_ok=True)
for domain in domains:
- cookie_file_path = manager.path_manager.cookies_dir / f"{domain}.txt"
+ cookie_file_path = manager.appdata.cookies / f"{domain}.txt"
cookie_jar = MozillaCookieJar(cookie_file_path)
cookie_jar.save(ignore_discard=True, ignore_expires=True)
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index 142f062f0..2c0cae833 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -32,8 +32,8 @@ async def get_modified_date(file: Path) -> datetime:
class Sorter:
def __init__(self, manager: Manager) -> None:
self.manager = manager
- self.download_folder = manager.path_manager.scan_folder or manager.path_manager.download_folder
- self.sorted_folder = manager.path_manager.sorted_folder
+ self.download_folder = manager.config.sorting.scan_folder or manager.config.files.download_folder
+ self.sorted_folder = manager.config.sorting.sort_folder
self.incrementer_format: str = manager.config_manager.settings_data.sorting.sort_incrementer_format
self.db_manager = manager.db_manager
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index ba54f310e..5502efbb8 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -262,7 +262,7 @@ def get_filename_and_ext(filename: str, forum: bool = False, mime_type: str | No
def get_download_path(manager: Manager, scrape_item: ScrapeItem, domain: str) -> Path:
"""Returns the path to the download folder."""
- download_dir = manager.path_manager.download_folder
+ download_dir = manager.config.files.download_folder
return download_dir / scrape_item.create_download_path(domain)
@@ -365,14 +365,14 @@ def _partial_files(dir: Path | str) -> Generator[Path]:
def delete_partial_files(manager: Manager) -> None:
"""Deletes partial download files recursively."""
log_red("Deleting partial downloads...")
- for file in _partial_files(manager.path_manager.download_folder):
+ for file in _partial_files(manager.config.files.download_folder):
file.unlink(missing_ok=True)
def check_for_partial_files(manager: Manager) -> None:
"""Checks if there are partial downloads in any subdirectory and logs if found."""
log_yellow("Checking for partial downloads...")
- has_partial_files = next(_partial_files(manager.path_manager.download_folder), None)
+ has_partial_files = next(_partial_files(manager.config.files.download_folder), None)
if has_partial_files:
log_yellow("There are partial downloads in the downloads folder")
@@ -380,9 +380,9 @@ def check_for_partial_files(manager: Manager) -> None:
def delete_empty_folders(manager: Manager):
"""Deletes empty folders efficiently."""
log_yellow("Checking for empty folders...")
- purge_dir_tree(manager.path_manager.download_folder)
+ purge_dir_tree(manager.config.files.download_folder)
- sorted_folder = manager.path_manager.sorted_folder
+ sorted_folder = manager.config.sorting.sort_folder
if sorted_folder and manager.config_manager.settings_data.sorting.sort_downloads:
purge_dir_tree(sorted_folder)
diff --git a/cyberdrop_dl/utils/webhook.py b/cyberdrop_dl/utils/webhook.py
index f29846cc9..3f99b7017 100644
--- a/cyberdrop_dl/utils/webhook.py
+++ b/cyberdrop_dl/utils/webhook.py
@@ -71,7 +71,7 @@ async def send_webhook_message(manager: Manager) -> None:
rich.print("\nSending Webhook Notifications.. ")
url = cast("AbsoluteHttpURL", webhook.url.get_secret_value())
- form = await _prepare_form(webhook, manager.path_manager.main_log)
+ form = await _prepare_form(webhook, manager.config.logs.main_log)
result = constants.NotificationResult.FAILED.value
diff --git a/tests/conftest.py b/tests/conftest.py
index 06ac8fec1..599b256dc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -70,7 +70,7 @@ def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Man
monkeypatch.chdir(tmp_path)
bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads, "--download-tiktok-audios"))
bare_manager.startup()
- bare_manager.path_manager.startup()
+ bare_manager.config.resolve_paths()
bare_manager.logs.delete_old_logs()
return bare_manager
diff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.py
index 82eb6ddfd..6feb7cbe9 100644
--- a/tests/crawlers/test_xenforo.py
+++ b/tests/crawlers/test_xenforo.py
@@ -75,7 +75,7 @@ async def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
monkeypatch.chdir(tmp_path)
manager = Manager(("--appdata-folder", appdata, "-d", downloads))
manager.startup()
- manager.path_manager.startup()
+ manager.config.resolve_paths()
manager.logs.delete_old_logs()
await manager.async_startup()
yield manager
diff --git a/tests/test_apprise.py b/tests/test_apprise.py
index ed62be1fa..2a010d243 100644
--- a/tests/test_apprise.py
+++ b/tests/test_apprise.py
@@ -10,7 +10,6 @@ from cyberdrop_dl import constants
from cyberdrop_dl.constants import NotificationResult
from cyberdrop_dl.managers.config_manager import ConfigManager
from cyberdrop_dl.managers.manager import Manager
-from cyberdrop_dl.managers.path_manager import PathManager
from cyberdrop_dl.utils import apprise
TEST_FILES_PATH = Path(__file__).parent / "test_files/apprise"
@@ -82,8 +81,7 @@ async def send_notification(test_case: AppriseTestCase) -> None:
FAKE_MANAGER.config_manager.apprise_urls = []
if test_case.urls and any(test_case.urls):
FAKE_MANAGER.config_manager.apprise_urls = apprise.get_apprise_urls(urls=test_case.urls)
- FAKE_MANAGER.path_manager = PathManager(FAKE_MANAGER)
- FAKE_MANAGER.path_manager.main_log = test_case.file or TEST_FILES_PATH / "valid_single_url.txt"
+ FAKE_MANAGER.config.logs.main_log = test_case.file or TEST_FILES_PATH / "valid_single_url.txt"
constants.LOG_OUTPUT_TEXT = Text(test_case.name)
result, logs = await apprise.send_apprise_notifications(FAKE_MANAGER)
assert result.value == test_case.result.value, f"Result for this case should be {test_case.result.value}"
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 58a87cb86..0d0456920 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -3,7 +3,7 @@ from cyberdrop_dl.managers.manager import Manager
def test_cache_file_is_not_saved_outside_ctx(manager: Manager) -> None:
- cache_file = manager.path_manager.cache_folder / "cache.yaml"
+ cache_file = manager.appdata.cache_file
manager.cache["test"] = 1
assert manager.cache == {"test": 1}
@@ -11,7 +11,7 @@ def test_cache_file_is_not_saved_outside_ctx(manager: Manager) -> None:
async def test_cache_file_is_saved_in_ctx(manager: Manager) -> None:
- cache_file = manager.path_manager.cache_folder / "cache.yaml"
+ cache_file = manager.appdata.cache_file
async with manager:
manager.cache["test"] = 1
assert manager.cache == {"test": 1}
diff --git a/tests/test_hashing.py b/tests/test_hashing.py
index 0ae9498de..810adfc3f 100644
--- a/tests/test_hashing.py
+++ b/tests/test_hashing.py
@@ -57,12 +57,12 @@ def test_hash_directory_scanner(manager: Manager, expected_results: set[tuple[st
manager.config.dupe_cleanup_options.add_md5_hash = "md5" in algos
manager.config.dupe_cleanup_options.add_sha256_hash = "sha256" in algos
- manager.path_manager.download_folder.mkdir(parents=True)
- db_path = manager.path_manager.history_db
- hash_directory_scanner(manager, manager.path_manager.download_folder)
+ manager.config.files.download_folder.mkdir(parents=True)
+ db_path = manager.appdata.db_file
+ hash_directory_scanner(manager, manager.config.files.download_folder)
assert not get_hashes(db_path)
- create_files(manager.path_manager.download_folder, n_files)
- hash_directory_scanner(manager, manager.path_manager.download_folder)
+ create_files(manager.config.files.download_folder, n_files)
+ hash_directory_scanner(manager, manager.config.files.download_folder)
results = get_hashes(db_path)
assert len(results) == len(expected_results)
assert results == expected_results