Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove path manager (part 1) (#80)
NTFSvolume4f208b2ebfdf
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex 2638e5141..79e09872c 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -43,7 +43,7 @@ class HTTPClient:return cls(client_manager,client_manager.manager.config_manager.settings_data.files.save_pages_html,- client_manager.manager.path_manager.pages_folder,+ client_manager.manager.config.logs.main_log.parent / "cdl_responses",)@propertydiff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 71ccb4371..277bf880a 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -346,7 +346,7 @@ class DownloadClient:try:media_item.downloaded = downloadedawait self.manager.hash_manager.hash_client.hash_item_during_download(media_item)- self.manager.path_manager.add_completed(media_item)+ self.manager.add_completed(media_item)except Exception:logger.exception(f"Error handling media item completion of: {media_item.path}")@@ -359,7 +359,7 @@ class DownloadClient:return download_folderif self.manager.config_manager.settings_data.download_options.block_download_sub_folders:- while download_folder.parent != self.manager.path_manager.download_folder:+ while download_folder.parent != self.manager.config.files.download_folder:download_folder = download_folder.parentmedia_item.download_folder = download_folderreturn download_folderdiff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex 5a8004c10..712006c97 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -206,7 +206,7 @@ class HashClient:async def get_file_hashes_dict(self) -> dict:"""Get a dictionary of files based on matching file hashes and file size."""- downloads = self.manager.path_manager.completed_downloads+ downloads = self.manager.completed_downloadsasync def exists(item: MediaItem) -> MediaItem | None:if await asyncio.to_thread(item.path.is_file):diff --git a/cyberdrop_dl/clients/jdownloader.py b/cyberdrop_dl/clients/jdownloader.pyindex 6b3033426..fb83e6aa1 100644--- a/cyberdrop_dl/clients/jdownloader.py+++ b/cyberdrop_dl/clients/jdownloader.py@@ -31,7 +31,7 @@ class Config:@staticmethoddef from_manager(manager: Manager) -> Config:- download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.path_manager.download_folder+ download_dir = manager.config.runtime_options.jdownloader_download_dir or manager.config.files.download_folderreturn Config(enabled=manager.config.runtime_options.send_unsupported_to_jdownloader,device=manager.auth_config.jdownloader.device,diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.pyindex c1d711864..5f62b6836 100755--- a/cyberdrop_dl/config/__init__.py+++ b/cyberdrop_dl/config/__init__.py@@ -123,9 +123,7 @@ class Config:self.apprise_urls = get_apprise_urls(file=self.apprise_file)def _resolve_all_paths(self) -> None:- self.auth.resolve_paths()self.settings.resolve_paths()- self.global_settings.resolve_paths()def _all_settings(self) -> tuple[ConfigSettings, AuthSettings, GlobalSettings]:return self.settings, self.auth, self.global_settingsdiff --git a/cyberdrop_dl/config/_common.py b/cyberdrop_dl/config/_common.pyindex debbbe48c..aa533aa35 100755--- a/cyberdrop_dl/config/_common.py+++ b/cyberdrop_dl/config/_common.py@@ -1,8 +1,6 @@from pathlib import Pathfrom typing import Self-from pydantic import BaseModel-from cyberdrop_dl import yamlfrom cyberdrop_dl.exceptions import InvalidYamlErrorfrom cyberdrop_dl.models import AliasModel, get_model_fields@@ -30,18 +28,6 @@ class ConfigModel(AliasModel):def save_to_file(self, file: Path) -> None:yaml.save(file, self)- def resolve_paths(self) -> None:- self._resolve_paths(self)-- @classmethod- def _resolve_paths(cls, model: BaseModel) -> None:- for name, value in vars(model).items():- if isinstance(value, Path):- setattr(model, name, value.resolve())-- elif isinstance(value, BaseModel):- cls._resolve_paths(value)-def _is_in_file(search_value: str, file: Path) -> bool:try:diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.pyindex 0b33aa67e..f46f17bc4 100755--- a/cyberdrop_dl/config/config_model.py+++ b/cyberdrop_dl/config/config_model.py@@ -1,5 +1,5 @@-import itertoolsimport re+from collections.abc import Generatorfrom datetime import date, datetime, timedeltafrom logging import DEBUGfrom pathlib import Path@@ -66,14 +66,14 @@ class DownloadOptions(BaseModel):class Files(AliasModel):download_folder: Path = Field(default=DEFAULT_DOWNLOAD_STORAGE, validation_alias="d")dump_json: bool = Field(default=False, validation_alias="j")- input_file: Path = Field(default=DEFAULT_APP_STORAGE / "Configs/{config}/URLs.txt", validation_alias="i")+ input_file: Path = Field(default=Path("URLs.txt"), validation_alias="i")save_pages_html: bool = Falseclass Logs(AliasModel):download_error_urls: LogPath = Path("Download_Error_URLs.csv")last_forum_post: LogPath = Path("Last_Scraped_Forum_Posts.csv")- log_folder: Path = DEFAULT_APP_STORAGE / "Configs/{config}/Logs"+ log_folder: Path = DEFAULT_APP_STORAGE / "Logs"logs_expire_after: timedelta | None = Nonemain_log: MainLogPath = Path("downloader.log")rotate_logs: bool = False@@ -92,28 +92,38 @@ class Logs(AliasModel):if value := falsy_as(input_date, None):return to_timedelta(value)+ def _files(self) -> Generator[tuple[str, Path]]:+ for name, log_file in vars(self).items():+ if name == "log_folder" or not isinstance(log_file, Path) or log_file.suffix not in (".csv", ".log"):+ continue+ yield name, log_file+def _set_output_filenames(self, now: datetime) -> None:+ self.log_folder = self.log_folder.resolve()self.log_folder.mkdir(exist_ok=True, parents=True)current_time_file_iso: str = now.strftime(constants.LOGS_DATETIME_FORMAT)current_time_folder_iso: str = now.strftime(constants.LOGS_DATE_FORMAT)- for attr, log_file in vars(self).items():- if not isinstance(log_file, Path) or log_file.suffix not in (".csv", ".log"):- continue-+ for name, log_file in self._files():+ log_file = self.log_folder / log_fileif self.rotate_logs:new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}"log_file = log_file.parent / current_time_folder_iso / new_name- setattr(self, attr, self.log_folder / log_file)+ setattr(self, name, log_file)++ def mkdirs(self):+ for _, log_file in self._files():log_file.parent.mkdir(exist_ok=True, parents=True)- def _delete_old_logs_and_folders(self, now: datetime | None = None) -> None:- if now and self.logs_expire_after:- for file in itertools.chain(self.log_folder.rglob("*.log"), self.log_folder.rglob("*.csv")):- file_date = file.stat().st_ctime- t_delta = now - datetime.fromtimestamp(file_date)- if t_delta > self.logs_expire_after:- file.unlink(missing_ok=True)+ def _delete_old_logs_and_folders(self, now: datetime) -> None:+ if self.logs_expire_after:+ for file in self.log_folder.rglob("*"):+ if file.suffix.lower() not in (".log", ".csv"):+ continue++ if (now - datetime.fromtimestamp(file.stat().st_ctime)) > self.logs_expire_after:+ file.unlink()+purge_dir_tree(self.log_folder)@@ -292,3 +302,27 @@ class ConfigSettings(ConfigModel):logs: Logs = Logs()runtime_options: RuntimeOptions = RuntimeOptions()sorting: Sorting = Sorting()+ _resolved: bool = False++ def resolve_paths(self) -> None:+ if self._resolved:+ return++ now = datetime.now()+ self.logs._set_output_filenames(now)+ self._resolve_paths(self)+ self.logs._delete_old_logs_and_folders(now)+ self.logs.mkdirs()+ self._resolved = True++ @classmethod+ def _resolve_paths(cls, model: BaseModel) -> None:++ for name, value in vars(model).items():+ if isinstance(value, Path):+ if "{config}" in str(value):+ raise RuntimeError(f"Using '{{config}}' as reference on a path is no longer support: {value}")+ setattr(model, name, value.expanduser().resolve().absolute())++ elif isinstance(value, BaseModel):+ cls._resolve_paths(value)diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 2907b77db..dd21cf945 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -62,7 +62,7 @@ def _ui_error_handling_wrapper(@_ui_error_handling_wrapperasync def _run_manager(manager: Manager) -> None:"""Runs the program and handles the UI."""- manager.path_manager.startup()+ manager.config.resolve_paths()manager.logs.delete_old_logs()debug_log_file_path = _setup_debug_logger(manager)start_time = manager.start_time@@ -126,7 +126,7 @@ async def _post_runtime(manager: Manager) -> None:if manager.config_manager.settings_data.runtime_options.update_last_forum_post:log_spacer(20)- await manager.logs.update_last_forum_post(manager.path_manager.input_file)+ await manager.logs.update_last_forum_post(manager.config.files.input_file)def _setup_debug_logger(manager: Manager) -> Path | None:@@ -162,7 +162,7 @@ def _setup_debug_logger(manager: Manager) -> Path | None:def _setup_main_logger(manager: Manager) -> None:logger = logging.getLogger("cyberdrop_dl")- file_io = manager.path_manager.main_log.open("w", encoding="utf8")+ file_io = manager.config.logs.main_log.open("w", encoding="utf8")settings_data = manager.config_manager.settings_datalog_level = settings_data.runtime_options.log_levellogger.setLevel(log_level)diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 7bcc11703..06638b226 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -144,9 +144,9 @@ class Downloader:self.client = self.manager.client_manager.download_clientself._semaphore = asyncio.Semaphore(self.manager.client_manager.get_download_slots(self.domain))- self.manager.path_manager.download_folder.mkdir(parents=True, exist_ok=True)+ self.manager.config.files.download_folder.mkdir(parents=True, exist_ok=True)if self.manager.config_manager.settings_data.sorting.sort_downloads:- self.manager.path_manager.sorted_folder.mkdir(parents=True, exist_ok=True)+ self.manager.config.sorting.sort_folder.mkdir(parents=True, exist_ok=True)def update_queued_files(self, increase_total: bool = True):queued_files = self.manager.progress_manager.file_progress.get_queue_length()diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex f9a34b40f..bf949a549 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -326,7 +326,7 @@ class ClientManager:get_cookies_from_browsers(self.manager, browser=self.manager.config_manager.settings_data.browser_cookies.browser)- cookie_files = sorted(self.manager.path_manager.cookies_dir.glob("*.txt"))+ cookie_files = sorted(self.manager.appdata.cookies.glob("*.txt"))if not cookie_files:returnasync for domain, cookie in read_netscape_files(cookie_files):diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.pyindex ec35eb917..b4d6592f6 100644--- a/cyberdrop_dl/managers/config_manager.py+++ b/cyberdrop_dl/managers/config_manager.py@@ -1,6 +1,7 @@from __future__ import annotationsfrom dataclasses import field+from pathlib import Pathfrom time import sleepfrom typing import TYPE_CHECKING@@ -11,8 +12,6 @@ from cyberdrop_dl.managers.logs import LogManagerfrom cyberdrop_dl.utils.apprise import get_apprise_urlsif TYPE_CHECKING:- from pathlib import Path-from pydantic import BaseModelfrom cyberdrop_dl.managers.manager import Manager@@ -38,10 +37,10 @@ class ConfigManager:def startup(self) -> None:"""Startup process for the config manager."""self.loaded_config = self.manager.cache.get("default_config", "Default")- self.settings = self.manager.path_manager.config_folder / self.loaded_config / "settings.yaml"- self.global_settings = self.manager.path_manager.config_folder / "global_settings.yaml"- self.authentication_settings = self.manager.path_manager.config_folder / "authentication.yaml"- auth_override = self.manager.path_manager.config_folder / self.loaded_config / "authentication.yaml"+ self.settings = self.manager.appdata.configs / self.loaded_config / "settings.yaml"+ self.global_settings = self.manager.appdata.configs / "global_settings.yaml"+ self.authentication_settings = self.manager.appdata.configs / "authentication.yaml"+ auth_override = self.manager.appdata.configs / self.loaded_config / "authentication.yaml"if auth_override.is_file():self.authentication_settings = auth_override@@ -54,7 +53,7 @@ class ConfigManager:self._load_authentication_config()self._load_global_settings_config()self._load_settings_config()- self.apprise_file = self.manager.path_manager.config_folder / self.loaded_config / "apprise.txt"+ self.apprise_file = self.manager.appdata.configs / self.loaded_config / "apprise.txt"self.apprise_urls = get_apprise_urls(file=self.apprise_file)@staticmethod@@ -98,15 +97,11 @@ class ConfigManager:returnelse:self.settings_data = ConfigSettings()- self.settings_data.files.input_file = (- self.manager.path_manager.appdata / "Configs" / self.loaded_config / "URLs.txt"- )- downloads = self.manager.path_manager.cwd / "Downloads"+ self.settings_data.files.input_file = self.manager.appdata.configs / self.loaded_config / "URLs.txt"+ downloads = Path("Downloads").resolve()self.settings_data.sorting.sort_folder = downloads / "Cyberdrop-DL Sorted Downloads"self.settings_data.files.download_folder = downloads / "Cyberdrop-DL Downloads"- self.settings_data.logs.log_folder = (- self.manager.path_manager.appdata / "Configs" / self.loaded_config / "Logs"- )+ self.settings_data.logs.log_folder = self.manager.appdata.configs / self.loaded_config / "Logs"yaml.save(self.settings, self.settings_data)@@ -126,7 +121,7 @@ class ConfigManager:def reload_config(self) -> None:self.startup()- self.manager.path_manager.startup()+ self.manager.config.resolve_paths()sleep(1)self.manager.logs = LogManager.from_manager(self.manager)sleep(1)diff --git a/cyberdrop_dl/managers/logs.py b/cyberdrop_dl/managers/logs.pyindex 2f5138391..668e8e034 100644--- a/cyberdrop_dl/managers/logs.py+++ b/cyberdrop_dl/managers/logs.py@@ -54,11 +54,11 @@ class LogManager:@classmethoddef from_manager(cls, manager: Manager) -> Self:files = LogFiles(- main_log=manager.path_manager.main_log,- last_post_log=manager.path_manager.last_forum_post_log,- unsupported_urls_log=manager.path_manager.unsupported_urls_log,- download_error_log=manager.path_manager.download_error_urls_log,- scrape_error_log=manager.path_manager.scrape_error_urls_log,+ main_log=manager.config_manager.settings_data.logs.main_log,+ last_post_log=manager.config_manager.settings_data.logs.last_forum_post,+ unsupported_urls_log=manager.config_manager.settings_data.logs.unsupported_urls,+ download_error_log=manager.config_manager.settings_data.logs.download_error_urls,+ scrape_error_log=manager.config_manager.settings_data.logs.scrape_error_urls,)return cls(files, manager.task_group)diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 93034035d..7e5e8d944 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -1,9 +1,11 @@from __future__ import annotationsimport asyncio+import dataclassesimport jsonimport loggingfrom dataclasses import Field, field+from pathlib import Pathfrom time import perf_counterfrom typing import TYPE_CHECKING, Any, NamedTuple, Self, TypeVar@@ -18,7 +20,6 @@ from cyberdrop_dl.managers.config_manager import ConfigManagerfrom cyberdrop_dl.managers.hash_manager import HashManagerfrom cyberdrop_dl.managers.live_manager import LiveManagerfrom cyberdrop_dl.managers.logs import LogManager-from cyberdrop_dl.managers.path_manager import PathManagerfrom cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.utils.logger import LogHandler, QueuedLoggerfrom cyberdrop_dl.utils.utilities import close_if_defined, get_system_information@@ -26,7 +27,9 @@ from cyberdrop_dl.utils.utilities import close_if_defined, get_system_informatioif TYPE_CHECKING:from asyncio import TaskGroupfrom collections.abc import Sequence+ from os import PathLike+ from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper@@ -45,7 +48,6 @@ class Manager:self.parsed_args: ParsedArgs = field(init=False)self.cache: dict[str, Any] = {}- self.path_manager: PathManager = field(init=False)self.config_manager: ConfigManager = field(init=False)self.hash_manager: HashManager = field(init=False)@@ -67,6 +69,8 @@ class Manager:self.loggers: dict[str, QueuedLogger] = {}self.args = argsself.states: AsyncioEvents+ self._appdata: AppData | None = None+ self._completed_downloads: list[MediaItem] = []constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)@@ -83,9 +87,9 @@ class Manager:return self.config_manager.global_settings_dataasync def __aenter__(self) -> Self:- cache_file = self.path_manager.cache_folder / "cache.yaml"+ cache_file = self.appdata.cache_filetry:- self.cache.update(yaml.load(self.path_manager.cache_folder / "cache.yaml"))+ self.cache.update(yaml.load(cache_file))except FileNotFoundError:cache_file.parent.mkdir(exist_ok=True, parents=True)cache_file.touch()@@ -93,7 +97,7 @@ class Manager:async def __aexit__(self, *_) -> None:self.cache["version"] = __version__- yaml.save(self.path_manager.cache_folder / "cache.yaml", self.cache)+ yaml.save(self.appdata.cache_file, self.cache)def startup(self) -> None:"""Startup process for the manager."""@@ -101,17 +105,24 @@ class Manager:if isinstance(self.parsed_args, Field):self.parsed_args = parse_args(self.args)- self.path_manager = PathManager(self)- self.path_manager.pre_startup()+ self.appdata.mkdirs()self.config_manager = ConfigManager(self)self.config_manager.startup()self.args_consolidation()-- self.path_manager.startup()+ self.config.resolve_paths()self.logs = LogManager.from_manager(self)+ def add_completed(self, media_item: MediaItem) -> None:+ if media_item.is_segment:+ return+ self._completed_downloads.append(media_item)++ @property+ def completed_downloads(self) -> list[MediaItem]:+ return self._completed_downloads+"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""async def async_startup(self) -> None:@@ -131,11 +142,11 @@ class Manager:async def async_db_hash_startup(self) -> None:if not isinstance(self.db_manager, Database):self.db_manager = Database(- self.path_manager.history_db,+ self.appdata.db_file,self.config.runtime_options.ignore_history,)await self.db_manager.startup()- transfer_v5_db_to_v6(self.path_manager.history_db)+ transfer_v5_db_to_v6(self.appdata.db_file)if not isinstance(self.hash_manager, HashManager):self.hash_manager = HashManager(self)if not isinstance(self.live_manager, LiveManager):@@ -186,9 +197,9 @@ class Manager:f"System Info: {system_info}",f"Using Config: {self.config_manager.loaded_config}",f"Using Config File: {self.config_manager.settings}",- f"Using Input File: {self.path_manager.input_file}",- f"Using Download Folder: {self.path_manager.download_folder}",- f"Using Database File: {self.path_manager.history_db}",+ f"Using Input File: {self.config.files.input_file}",+ f"Using Download Folder: {self.config.files.download_folder}",+ f"Using Database File: {self.appdata.db_file}",f"Using CLI only options: {cli_only_args}",f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}",f"Using Settings: \n{config_settings}",@@ -216,6 +227,18 @@ class Manager:_, queued_logger = self.loggers.popitem()queued_logger.stop()+ @property+ def appdata(self) -> AppData:+ if self._appdata is None:+ if folder := self.parsed_args.cli_only_args.appdata_folder:+ path = folder / "AppData"+ else:+ path = Path("AppData")++ self._appdata = AppData.from_path(path.resolve())++ return self._appdata+def add_or_remove_lists(cli_values: list[str], config_values: list[str]) -> None:exclude = {"+", "-"}@@ -255,3 +278,45 @@ def merge_models(default: M, new: M) -> M:updated_dict = merge_dicts(default_dict, new_dict)return default.model_validate(updated_dict)++[email protected](slots=True, frozen=True, kw_only=True)+class AppData:+ path: Path+ cache_file: Path+ db_file: Path+ config_file: Path++ cache: Path+ cookies: Path+ configs: Path++ @classmethod+ def from_path(cls, path: Path) -> Self:+ assert path.is_absolute()+ cache = path / "Cache"+ return cls(+ path=path,+ cache=cache,+ configs=path / "Configs",+ cookies=path / "Cookies",+ config_file=path / "config.yaml",+ cache_file=cache / "cache.yaml",+ db_file=cache / "cyberdrop.db",+ )++ def __truediv__(self, other: PathLike[str]):+ try:+ return self.path / other+ except TypeError:+ return NotImplemented++ def __fspath__(self) -> str:+ return str(self)++ def __str__(self) -> str:+ return str(self.path)++ def mkdirs(self) -> None:+ for dir in (self.cache, self.configs, self.cookies):+ dir.mkdir(parents=True, exist_ok=True)diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.pydeleted file mode 100644index 4073d57b9..000000000--- a/cyberdrop_dl/managers/path_manager.py+++ /dev/null@@ -1,160 +0,0 @@-from __future__ import annotations--import os-from dataclasses import Field, field-from datetime import datetime-from pathlib import Path-from typing import TYPE_CHECKING--from cyberdrop_dl import env-from cyberdrop_dl.utils.utilities import purge_dir_tree--if TYPE_CHECKING:- from cyberdrop_dl.data_structures.url_objects import MediaItem- from cyberdrop_dl.managers.manager import Manager---class PathManager:- def __init__(self, manager: Manager) -> None:- self.manager = manager-- self.download_folder: Path = field(init=False)- self.sorted_folder: Path = field(init=False)- self.scan_folder: Path | None = field(init=False)-- self.log_folder: Path = field(init=False)-- self.cache_folder: Path = field(init=False)- self.config_folder: Path = field(init=False)-- self.input_file: Path = field(init=False)- self.history_db: Path = field(init=False)- self.cache_db: Path = field(init=False)-- self._completed_downloads: list[MediaItem] = []-- self.main_log: Path = field(init=False)- self.last_forum_post_log: Path = field(init=False)- self.unsupported_urls_log: Path = field(init=False)- self.download_error_urls_log: Path = field(init=False)- self.scrape_error_urls_log: Path = field(init=False)- self.pages_folder: Path = field(init=False)-- self._logs_model_names = [- "main_log",- "last_forum_post",- "unsupported_urls",- "download_error_urls",- "scrape_error_urls",- ]- self._appdata: Path = field(init=False)-- @property- def cwd(self) -> Path:- if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":- # This is for testing purposes only"""- return Path("..").resolve()- return Path().resolve()-- @property- def appdata(self) -> Path:- if isinstance(self._appdata, Field):- if self.manager.parsed_args.cli_only_args.appdata_folder:- path = self.manager.parsed_args.cli_only_args.appdata_folder / "AppData"- self._appdata = self.cwd / path- else:- self._appdata = self.cwd / "AppData"-- return self._appdata-- def pre_startup(self) -> None:- self.cache_folder = self.appdata / "Cache"- self.config_folder = self.appdata / "Configs"- self.cookies_dir = self.appdata / "Cookies"- self.cache_db = self.cache_folder / "request_cache.db"-- self.cache_folder.mkdir(parents=True, exist_ok=True)- self.config_folder.mkdir(parents=True, exist_ok=True)- self.cookies_dir.mkdir(parents=True, exist_ok=True)- self.cache_db.touch(exist_ok=True)-- def startup(self) -> None:- """Startup process for the Directory Manager."""- settings_data = self.manager.config_manager.settings_data- current_config = self.manager.config_manager.loaded_config-- def replace(path: Path) -> Path:- path_w_config = str(path).replace("{config}", current_config)- if os.name == "nt":- return self.cwd.joinpath(Path(path_w_config)).resolve()- normalized_path_str = path_w_config.replace("\\", "/")- return self.cwd.joinpath(Path(normalized_path_str)).resolve()-- self.download_folder = replace(settings_data.files.download_folder)- self.sorted_folder = replace(settings_data.sorting.sort_folder)- self.log_folder = replace(settings_data.logs.log_folder)- self.input_file = replace(settings_data.files.input_file)- self.history_db = self.cache_folder / "cyberdrop.db"- self.scan_folder = settings_data.sorting.scan_folder- if self.scan_folder:- self.scan_folder = replace(self.scan_folder)-- self.log_folder.mkdir(parents=True, exist_ok=True)-- now = datetime.now()- self._set_output_filenames(now)- self._delete_logs_and_folders(now)- self._create_output_folders()-- if not self.input_file.is_file():- self.input_file.touch(exist_ok=True)- self.history_db.touch(exist_ok=True)-- def _set_output_filenames(self, now: datetime) -> None:- current_time_file_iso: str = now.strftime("%Y%m%d_%H%M%S")- current_time_folder_iso: str = now.strftime("%Y_%m_%d")- log_settings_config = self.manager.config_manager.settings_data.logs- log_files: dict[str, Path] = log_settings_config.model_dump()-- for model_name, log_file in log_files.items():- if model_name not in self._logs_model_names:- continue- if log_settings_config.rotate_logs:- new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}"- log_file: Path = log_file.parent / current_time_folder_iso / new_name- log_files[model_name] = log_file-- log_settings_config = log_settings_config.model_copy(update=log_files)-- for model_name in self._logs_model_names:- internal_name = f"{model_name.replace('_log', '')}_log"- setattr(self, internal_name, self.log_folder / getattr(log_settings_config, model_name))-- self.pages_folder = self.main_log.parent / "cdl_responses"-- def _delete_logs_and_folders(self, now: datetime):- if self.manager.config_manager.settings_data.logs.logs_expire_after:- for file in set(self.log_folder.rglob("*.log")) | set(self.log_folder.rglob("*.csv")):- file_date = Path(file).stat().st_ctime- t_delta = now - datetime.fromtimestamp(file_date)- if t_delta > self.manager.config_manager.settings_data.logs.logs_expire_after:- file.unlink(missing_ok=True)- purge_dir_tree(self.log_folder)-- def _create_output_folders(self):- for model_name in self._logs_model_names:- internal_name = f"{model_name.replace('_log', '')}_log"- path: Path = getattr(self, internal_name)- path.parent.mkdir(parents=True, exist_ok=True)-- if self.manager.config_manager.settings_data.files.save_pages_html:- self.pages_folder.mkdir(parents=True, exist_ok=True)-- def add_completed(self, media_item: MediaItem) -> None:- if media_item.is_segment:- return- self._completed_downloads.append(media_item)-- @property- def completed_downloads(self) -> list[MediaItem]:- return self._completed_downloadsdiff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex 1ca7bd486..6cb867ddf 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -149,10 +149,10 @@ class ProgressManager:log_spacer(20)logger.info("Printing Stats...\n")- config_path = self.manager.path_manager.config_folder / self.manager.config_manager.loaded_config+ config_path = self.manager.appdata.configs / self.manager.config_manager.loaded_configconfig_path_text = get_console_hyperlink(config_path, text=self.manager.config_manager.loaded_config)input_file_text = get_input(self.manager)- log_folder_text = get_console_hyperlink(self.manager.path_manager.log_folder)+ log_folder_text = get_console_hyperlink(self.manager.config.logs.log_folder)log_concat("Run Stats (config: ", config_path_text, ")", style="cyan")log_concat(" Input File: ", input_file_text, style="yellow")@@ -218,7 +218,7 @@ def get_input(manager: Manager) -> Text | str:if manager.parsed_args.cli_only_args.retry_maintenance:return "--retry-maintenance"if manager.scrape_mapper.using_input_file:- return get_console_hyperlink(manager.path_manager.input_file)+ return get_console_hyperlink(manager.config.files.input_file)return "--links (CLI args)"diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 5274f7677..b5cff10d8 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -135,7 +135,7 @@ class ScrapeMapper:async def parse_input_file_groups(self) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:"""Split URLs from input file by their groups."""- input_file = self.manager.path_manager.input_file+ input_file = self.manager.config.files.input_fileif not await asyncio.to_thread(input_file.is_file):yield ("", [])return@@ -235,7 +235,7 @@ class ScrapeMapper:success = Falsetry:download_folder = get_download_path(self.manager, scrape_item, "jdownloader")- relative_download_dir = download_folder.relative_to(self.manager.path_manager.download_folder)+ relative_download_dir = download_folder.relative_to(self.manager.config.files.download_folder)await self.jdownloader.send(scrape_item.url,scrape_item.parent_title,diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.pyindex 72575a0d6..59783cce7 100644--- a/cyberdrop_dl/ui/program_ui.py+++ b/cyberdrop_dl/ui/program_ui.py@@ -81,7 +81,7 @@ class ProgramUI:def _scan_and_create_hashes(self) -> None:"""Scans a folder and creates hashes for all of its files."""- path = ask_dir_path("Select the directory to scan", default=str(self.manager.path_manager.download_folder))+ path = ask_dir_path("Select the directory to scan", default=str(self.manager.config.files.download_folder))hash_directory_scanner(self.manager, path)def _sort_files(self) -> None:@@ -101,7 +101,7 @@ class ProgramUI:console.print(Markdown(changelog_content, justify="left"))def _edit_urls(self) -> None:- self._open_in_text_editor(self.manager.path_manager.input_file, reload_config=False)+ self._open_in_text_editor(self.manager.config.files.input_file, reload_config=False)def _open_in_text_editor(self, file_path: Path, *, reload_config: bool = True):try:diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.pyindex 5f77257e9..41ed775ec 100644--- a/cyberdrop_dl/utils/apprise.py+++ b/cyberdrop_dl/utils/apprise.py@@ -189,7 +189,7 @@ async def send_apprise_notifications(manager: Manager) -> tuple[constants.Notififor apprise_url in apprise_urls:apprise_obj.add(apprise_url.url, tag=list(apprise_url.tags))- main_log = manager.path_manager.main_log+ main_log = manager.config.logs.main_logall_urls = [x.raw_url for x in apprise_urls]log_lines = []diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pyindex c3556dc0e..297c7cbf9 100644--- a/cyberdrop_dl/utils/cookie_management.py+++ b/cyberdrop_dl/utils/cookie_management.py@@ -93,10 +93,10 @@ def get_cookies_from_browsers(manager: Manager, *, browser: BROWSERS, domains: lmsg = "None of the provided browsers is supported for extraction"raise ValueError(msg)- manager.path_manager.cookies_dir.mkdir(parents=True, exist_ok=True)+ manager.appdata.cookies.mkdir(parents=True, exist_ok=True)domains_with_cookies: set[str] = set()for domain in domains_to_extract:- cookie_file_path = manager.path_manager.cookies_dir / f"{domain}.txt"+ cookie_file_path = manager.appdata.cookies / f"{domain}.txt"cdl_cookie_jar = MozillaCookieJar(cookie_file_path)for cookie in extracted_cookies:if domain in cookie.domain:@@ -113,9 +113,9 @@ def clear_cookies(manager: Manager, domains: list[str]) -> None:if not domains:raise ValueError("No domains selected")- manager.path_manager.cookies_dir.mkdir(parents=True, exist_ok=True)+ manager.appdata.cookies.mkdir(parents=True, exist_ok=True)for domain in domains:- cookie_file_path = manager.path_manager.cookies_dir / f"{domain}.txt"+ cookie_file_path = manager.appdata.cookies / f"{domain}.txt"cookie_jar = MozillaCookieJar(cookie_file_path)cookie_jar.save(ignore_discard=True, ignore_expires=True)diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex 142f062f0..2c0cae833 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -32,8 +32,8 @@ async def get_modified_date(file: Path) -> datetime:class Sorter:def __init__(self, manager: Manager) -> None:self.manager = manager- self.download_folder = manager.path_manager.scan_folder or manager.path_manager.download_folder- self.sorted_folder = manager.path_manager.sorted_folder+ self.download_folder = manager.config.sorting.scan_folder or manager.config.files.download_folder+ self.sorted_folder = manager.config.sorting.sort_folderself.incrementer_format: str = manager.config_manager.settings_data.sorting.sort_incrementer_formatself.db_manager = manager.db_managerdiff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex ba54f310e..5502efbb8 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -262,7 +262,7 @@ def get_filename_and_ext(filename: str, forum: bool = False, mime_type: str | Nodef get_download_path(manager: Manager, scrape_item: ScrapeItem, domain: str) -> Path:"""Returns the path to the download folder."""- download_dir = manager.path_manager.download_folder+ download_dir = manager.config.files.download_folderreturn download_dir / scrape_item.create_download_path(domain)@@ -365,14 +365,14 @@ def _partial_files(dir: Path | str) -> Generator[Path]:def delete_partial_files(manager: Manager) -> None:"""Deletes partial download files recursively."""log_red("Deleting partial downloads...")- for file in _partial_files(manager.path_manager.download_folder):+ for file in _partial_files(manager.config.files.download_folder):file.unlink(missing_ok=True)def check_for_partial_files(manager: Manager) -> None:"""Checks if there are partial downloads in any subdirectory and logs if found."""log_yellow("Checking for partial downloads...")- has_partial_files = next(_partial_files(manager.path_manager.download_folder), None)+ has_partial_files = next(_partial_files(manager.config.files.download_folder), None)if has_partial_files:log_yellow("There are partial downloads in the downloads folder")@@ -380,9 +380,9 @@ def check_for_partial_files(manager: Manager) -> None:def delete_empty_folders(manager: Manager):"""Deletes empty folders efficiently."""log_yellow("Checking for empty folders...")- purge_dir_tree(manager.path_manager.download_folder)+ purge_dir_tree(manager.config.files.download_folder)- sorted_folder = manager.path_manager.sorted_folder+ sorted_folder = manager.config.sorting.sort_folderif sorted_folder and manager.config_manager.settings_data.sorting.sort_downloads:purge_dir_tree(sorted_folder)diff --git a/cyberdrop_dl/utils/webhook.py b/cyberdrop_dl/utils/webhook.pyindex f29846cc9..3f99b7017 100644--- a/cyberdrop_dl/utils/webhook.py+++ b/cyberdrop_dl/utils/webhook.py@@ -71,7 +71,7 @@ async def send_webhook_message(manager: Manager) -> None:rich.print("\nSending Webhook Notifications.. ")url = cast("AbsoluteHttpURL", webhook.url.get_secret_value())- form = await _prepare_form(webhook, manager.path_manager.main_log)+ form = await _prepare_form(webhook, manager.config.logs.main_log)result = constants.NotificationResult.FAILED.valuediff --git a/tests/conftest.py b/tests/conftest.pyindex 06ac8fec1..599b256dc 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -70,7 +70,7 @@ def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manmonkeypatch.chdir(tmp_path)bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads, "--download-tiktok-audios"))bare_manager.startup()- bare_manager.path_manager.startup()+ bare_manager.config.resolve_paths()bare_manager.logs.delete_old_logs()return bare_managerdiff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.pyindex 82eb6ddfd..6feb7cbe9 100644--- a/tests/crawlers/test_xenforo.py+++ b/tests/crawlers/test_xenforo.py@@ -75,7 +75,7 @@ async def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)monkeypatch.chdir(tmp_path)manager = Manager(("--appdata-folder", appdata, "-d", downloads))manager.startup()- manager.path_manager.startup()+ manager.config.resolve_paths()manager.logs.delete_old_logs()await manager.async_startup()yield managerdiff --git a/tests/test_apprise.py b/tests/test_apprise.pyindex ed62be1fa..2a010d243 100644--- a/tests/test_apprise.py+++ b/tests/test_apprise.py@@ -10,7 +10,6 @@ from cyberdrop_dl import constantsfrom cyberdrop_dl.constants import NotificationResultfrom cyberdrop_dl.managers.config_manager import ConfigManagerfrom cyberdrop_dl.managers.manager import Manager-from cyberdrop_dl.managers.path_manager import PathManagerfrom cyberdrop_dl.utils import appriseTEST_FILES_PATH = Path(__file__).parent / "test_files/apprise"@@ -82,8 +81,7 @@ async def send_notification(test_case: AppriseTestCase) -> None:FAKE_MANAGER.config_manager.apprise_urls = []if test_case.urls and any(test_case.urls):FAKE_MANAGER.config_manager.apprise_urls = apprise.get_apprise_urls(urls=test_case.urls)- FAKE_MANAGER.path_manager = PathManager(FAKE_MANAGER)- FAKE_MANAGER.path_manager.main_log = test_case.file or TEST_FILES_PATH / "valid_single_url.txt"+ FAKE_MANAGER.config.logs.main_log = test_case.file or TEST_FILES_PATH / "valid_single_url.txt"constants.LOG_OUTPUT_TEXT = Text(test_case.name)result, logs = await apprise.send_apprise_notifications(FAKE_MANAGER)assert result.value == test_case.result.value, f"Result for this case should be {test_case.result.value}"diff --git a/tests/test_cache.py b/tests/test_cache.pyindex 58a87cb86..0d0456920 100644--- a/tests/test_cache.py+++ b/tests/test_cache.py@@ -3,7 +3,7 @@ from cyberdrop_dl.managers.manager import Managerdef test_cache_file_is_not_saved_outside_ctx(manager: Manager) -> None:- cache_file = manager.path_manager.cache_folder / "cache.yaml"+ cache_file = manager.appdata.cache_filemanager.cache["test"] = 1assert manager.cache == {"test": 1}@@ -11,7 +11,7 @@ def test_cache_file_is_not_saved_outside_ctx(manager: Manager) -> None:async def test_cache_file_is_saved_in_ctx(manager: Manager) -> None:- cache_file = manager.path_manager.cache_folder / "cache.yaml"+ cache_file = manager.appdata.cache_fileasync with manager:manager.cache["test"] = 1assert manager.cache == {"test": 1}diff --git a/tests/test_hashing.py b/tests/test_hashing.pyindex 0ae9498de..810adfc3f 100644--- a/tests/test_hashing.py+++ b/tests/test_hashing.py@@ -57,12 +57,12 @@ def test_hash_directory_scanner(manager: Manager, expected_results: set[tuple[stmanager.config.dupe_cleanup_options.add_md5_hash = "md5" in algosmanager.config.dupe_cleanup_options.add_sha256_hash = "sha256" in algos- manager.path_manager.download_folder.mkdir(parents=True)- db_path = manager.path_manager.history_db- hash_directory_scanner(manager, manager.path_manager.download_folder)+ manager.config.files.download_folder.mkdir(parents=True)+ db_path = manager.appdata.db_file+ hash_directory_scanner(manager, manager.config.files.download_folder)assert not get_hashes(db_path)- create_files(manager.path_manager.download_folder, n_files)- hash_directory_scanner(manager, manager.path_manager.download_folder)+ create_files(manager.config.files.download_folder, n_files)+ hash_directory_scanner(manager, manager.config.files.download_folder)results = get_hashes(db_path)assert len(results) == len(expected_results)assert results == expected_results