Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: move to std logging (part 2) (#108)
NTFSvolume3cd16e728dfe
diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.pyindex e3014a485..6dd457037 100644--- a/cyberdrop_dl/crawlers/luscious.py+++ b/cyberdrop_dl/crawlers/luscious.py@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -62,7 +61,7 @@ class LusciousCrawler(Crawler):filters = [{"name": i, "value": v} for i, v in query.items() if i not in ("page", "display", "q")]data["variables"] = {"input": {"display": sorting, "filters": filters, "page": page}}- log_debug(data)+ self.log.debug(data)return json.dumps(data)@error_handling_wrapper@@ -118,5 +117,5 @@ class LusciousCrawler(Crawler):data=query,headers={"Content-Type": "application/json"},)- log_debug(json_resp)+ self.log.debug(json_resp)return json_respdiff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.pyindex 8fe5b59be..d1aefb2be 100644--- a/cyberdrop_dl/crawlers/realdebrid.py+++ b/cyberdrop_dl/crawlers/realdebrid.py@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimitfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -118,7 +117,7 @@ class RealDebridCrawler(Crawler):title = self.create_title(f"files [{url.host}]")scrape_item.setup_as_album(title)debrid_link = await self.api.unrestrict(url, scrape_item.password)- log(f"[{self.FOLDER_DOMAIN}]:\n Original URL: {url}\n Debrid URL: {debrid_link}", 10)+ self.log.info(f"\n Original URL: {url}\n Debrid URL: {debrid_link}")filename, ext = self.get_filename_and_ext(debrid_link.name)await self.handle_file(url,diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 062b816ed..d15232a19 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -2,31 +2,31 @@ from __future__ import annotationsimport contextlibimport logging-from datetime import datetimefrom enum import IntEnum-from functools import wraps-from pathlib import Pathfrom typing import TYPE_CHECKING, ParamSpec, TypeVar-from cyberdrop_dl import aio, constants, env, storage-from cyberdrop_dl.dependencies import browser_cookie3+from rich.traceback import install as install_rich_tracebacks++from cyberdrop_dl import aio, storagefrom cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.scraper.scrape_mapper import ScrapeMapperfrom cyberdrop_dl.ui import program_uifrom cyberdrop_dl.updates import check_latest_pypifrom cyberdrop_dl.utils.apprise import send_apprise_notifications-from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log_spacer, log_with_color+from cyberdrop_dl.utils.logger import log_spacer, setup_loggingfrom cyberdrop_dl.utils.sorting import Sorterfrom cyberdrop_dl.utils.utilities import check_partials_and_empty_foldersfrom cyberdrop_dl.utils.webhook import send_webhook_messageif TYPE_CHECKING:- from collections.abc import Callable, Coroutine, Sequence+ from collections.abc import Sequencelogger = logging.getLogger(__name__)P = ParamSpec("P")R = TypeVar("R")+_ = install_rich_tracebacks()+class ExitCode(IntEnum):OK = 0@@ -36,62 +36,32 @@ class ExitCode(IntEnum):_C = ExitCode-def _ui_error_handling_wrapper(- func: Callable[P, Coroutine[None, None, R]],-) -> Callable[P, Coroutine[None, None, R | None]]:- """Wrapper handles errors from the main UI."""-- @wraps(func)- async def wrapper(*args, **kwargs) -> R | None:- try:- return await func(*args, **kwargs)- except* Exception as e:- exceptions = [e]- if isinstance(e, ExceptionGroup):- exceptions = e.exceptions- if not isinstance(exceptions[0], browser_cookie3.BrowserCookieError):- msg = "An error occurred, please report this to the developer with your logs file:"- log_with_color(msg, "bold red", 50, show_in_stats=False)- for exc in exceptions:- log_with_color(f" {exc}", "bold red", 50, show_in_stats=False, exc_info=exc)-- return wrapper---@_ui_error_handling_wrapperasync def _run_manager(manager: Manager) -> None:"""Runs the program and handles the UI."""manager.config.resolve_paths()manager.logs.delete_old_logs()- debug_log_file_path = _setup_debug_logger(manager)start_time = manager.start_time- _setup_main_logger(manager)- logger.info(f"Using Debug Log: {debug_log_file_path}")- logger.info("Starting Async Processes...")- await manager.async_startup()-- log_spacer(10)- async with manager.database:- logger.info("Starting CDL...\n")- await _scheduler(manager)+ with setup_logging(manager.config.logs.main_log):+ await manager.async_startup()- manager.progress_manager.print_stats(start_time)+ log_spacer()+ async with manager.database:+ logger.info("Starting CDL...\n")- log_spacer(20)- logger.info("Checking for Updates...")- check_latest_pypi()- log_spacer(20)- logger.info("Closing Program...")- log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)+ await _runtime(manager)+ await _post_runtime(manager)- await send_webhook_message(manager)- await send_apprise_notifications(manager)+ manager.progress_manager.print_stats(start_time)+ log_spacer()+ check_latest_pypi()+ log_spacer()+ logger.info("Closing program...")+ logger.info("Finished downloading. Enjoy :)", extra={"color": "green"})-async def _scheduler(manager: Manager) -> None:- for func in (_runtime, _post_runtime):- await func(manager)+ await send_webhook_message(manager)+ await send_apprise_notifications(manager)async def _runtime(manager: Manager) -> None:@@ -104,9 +74,8 @@ async def _runtime(manager: Manager) -> None:async def _post_runtime(manager: Manager) -> None:"""Actions to complete after main runtime, and before ui shutdown."""- log_spacer(20, log_to_console=False)- msg = f"Running Post-Download Processes For Config: {manager.config_manager.loaded_config}"- log_with_color(msg, "green", 20)+ log_spacer()+ logger.info("Running Post-Download Processes", extra={"color": "green"})await manager.hash_manager.hash_client.cleanup_dupes_after_download()@@ -117,58 +86,9 @@ async def _post_runtime(manager: Manager) -> None:check_partials_and_empty_folders(manager)if manager.config_manager.settings_data.runtime_options.update_last_forum_post:- log_spacer(20)await manager.logs.update_last_forum_post(manager.config.files.input_file)-def _setup_debug_logger(manager: Manager) -> Path | None:- if not env.DEBUG_VAR:- return-- log_level = logging.DEBUG- settings_data = manager.config_manager.settings_data- settings_data.runtime_options.log_level = log_level- debug_log_file_path = Path(__file__).parents[1] / "cyberdrop_dl_debug.log"- if env.DEBUG_LOG_FOLDER:- debug_log_folder = Path(env.DEBUG_LOG_FOLDER)- if not debug_log_folder.is_dir():- msg = "Value of env var 'CDL_DEBUG_LOG_FOLDER' is invalid."- msg += f" Folder '{debug_log_folder}' does not exists"- raise FileNotFoundError(None, msg, env.DEBUG_LOG_FOLDER)- date = datetime.now().strftime("%Y%m%d_%H%M%S")- debug_log_file_path = debug_log_folder / f"cyberdrop_dl_debug_{date}.log"-- file_io = debug_log_file_path.open("w", encoding="utf8")-- file_handler = LogHandler(level=log_level, file=file_io, width=500, debug=True)- queued_logger = QueuedLogger(manager, file_handler, "debug")-- logging.getLogger("cyberdrop_dl").addHandler(queued_logger.handler)-- # aiosqlite_log = logging.getLogger("aiosqlite")- # aiosqlite_log.setLevel(log_level)- # aiosqlite_log.addHandler(file_handler_debug)-- return debug_log_file_path.resolve()---def _setup_main_logger(manager: Manager) -> None:- logger = logging.getLogger("cyberdrop_dl")- file_io = manager.config.logs.main_log.open("w", encoding="utf8")- settings_data = manager.config_manager.settings_data- log_level = settings_data.runtime_options.log_level- logger.setLevel(log_level)-- if not manager.parsed_args.cli_only_args.fullscreen_ui:- constants.CONSOLE_LEVEL = settings_data.runtime_options.console_log_level- constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)-- logger.addHandler(constants.console_handler)- file_handler = LogHandler(level=log_level, file=file_io, width=500)- queued_logger = QueuedLogger(manager, file_handler)- logger.addHandler(queued_logger.handler)--class Director:"""Creates a manager and runs it"""diff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.pyindex d448196a5..8884500cb 100644--- a/cyberdrop_dl/main.py+++ b/cyberdrop_dl/main.py@@ -14,9 +14,7 @@ def main(args: Sequence[str] | None = None) -> None:def run(args: Sequence[str] | None = None) -> str | int | None:- from cyberdrop_dl.utils.logger import catch_exceptions- @catch_exceptionsdef run_() -> int:return _create_director(args).run()diff --git a/cyberdrop_dl/managers/logs.py b/cyberdrop_dl/managers/logs.pyindex 668e8e034..711ba5e55 100644--- a/cyberdrop_dl/managers/logs.py+++ b/cyberdrop_dl/managers/logs.py@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Selffrom cyberdrop_dl.exceptions import get_originfrom cyberdrop_dl.utils import json+from cyberdrop_dl.utils.logger import log_spacerif TYPE_CHECKING:from collections.abc import Iterable, Iterator@@ -133,6 +134,7 @@ class LogManager:def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:+ log_spacer()logger.info("Updating Last Forum Posts...\n")current_urls, current_base_urls, new_urls, new_base_urls = [], [], [], []diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 81a310a1e..13ba14053 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport asyncioimport dataclasses-import jsonimport loggingfrom dataclasses import Field, fieldfrom pathlib import Path@@ -167,35 +166,32 @@ class Manager:def args_logging(self) -> None:"""Logs the runtime arguments."""- auth_provided = {}- for site, auth_entries in self.config_manager.authentication_data.model_dump().items():- auth_provided[site] = all(auth_entries.values())+ auth = {+ site: all(credentials.values())+ for site, credentials in self.config_manager.authentication_data.model_dump().items()+ }config_settings = self.config_manager.settings_data.model_copy()config_settings.runtime_options.deep_scrape = self.config_manager.deep_scrape- config_settings = config_settings.model_dump_json(indent=4)- global_settings = self.config_manager.global_settings_data.model_dump_json(indent=4)- cli_only_args = self.parsed_args.cli_only_args.model_dump_json(indent=4)- system_info = get_system_information()-- args_info = (- "Starting Cyberdrop-DL Process",- f"Running Version: {__version__}",- f"System Info: {system_info}",- f"Using Config: {self.config_manager.loaded_config}",- f"Using Config File: {self.config_manager.settings}",- f"Using Input File: {self.config.files.input_file}",- f"Using Download Folder: {self.config.files.download_folder}",- f"Using Database File: {self.appdata.db_file}",- f"Using CLI only options: {cli_only_args}",- f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}",- f"Using Settings: \n{config_settings}",- f"Using Global Settings: \n{global_settings}",- f"Using ffmpeg version: {ffmpeg.get_ffmpeg_version()}",- f"Using ffprobe version: {ffmpeg.get_ffprobe_version()}",- )- logger.info("\n".join(args_info))++ logger.info(f"Running cyberdrop-dl v{__version__}")++ args_info = {+ "System": get_system_information(),+ "Config": self.config_manager.loaded_config,+ "Config File": self.config_manager.settings,+ "Input File": self.config.files.input_file,+ "Download Folder": self.config.files.download_folder,+ "Database File": self.appdata.db_file,+ "CLI only options": self.parsed_args.cli_only_args.model_dump(mode="json"),+ "Auth": auth,+ "Settings": config_settings.model_dump(mode="json"),+ "Global Settings": self.config_manager.global_settings_data.model_dump(mode="json"),+ "ffmpeg version": ffmpeg.get_ffmpeg_version(),+ "ffprobe version": ffmpeg.get_ffprobe_version(),+ }+ logger.info(args_info)async def close(self) -> None:diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex bfb399743..3d2f7c798 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -4,7 +4,6 @@ import loggingimport timefrom contextlib import asynccontextmanagerfrom datetime import timedelta-from functools import partialfrom typing import TYPE_CHECKINGfrom pydantic import ByteSize@@ -22,10 +21,10 @@ from cyberdrop_dl.ui.progress.hash_progress import HashProgressfrom cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgressfrom cyberdrop_dl.ui.progress.sort_progress import SortProgressfrom cyberdrop_dl.ui.progress.statistic_progress import DownloadStatsProgress, ScrapeStatsProgress-from cyberdrop_dl.utils.logger import log_spacer, log_with_color+from cyberdrop_dl.utils.logger import log_spacerif TYPE_CHECKING:- from collections.abc import AsyncGenerator+ from collections.abc import AsyncGenerator, Sequencefrom pathlib import Pathfrom rich.panel import Panel@@ -36,10 +35,6 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-log_cyan = partial(log_with_color, style="cyan", level=20)-log_yellow = partial(log_with_color, style="yellow", level=20)-log_green = partial(log_with_color, style="green", level=20)-log_red = partial(log_with_color, style="red", level=20)spinner = SpinnerColumn(style="green", spinner_name="dots")@@ -126,67 +121,73 @@ class ProgressManager:runtime = timedelta(seconds=int(end_time - start_time))total_data_written = ByteSize(self.file_progress.total_data_written).human_readable(decimal=True)- log_spacer(20)+ log_spacer()logger.info("Printing Stats...\n")config_path = self.manager.appdata.configs / self.manager.config_manager.loaded_configconfig_path_text = get_console_hyperlink(config_path, text=self.manager.config_manager.loaded_config)input_file_text = get_input(self.manager)log_folder_text = get_console_hyperlink(self.manager.config.logs.log_folder)- log_concat("Run Stats (config: ", config_path_text, ")", style="cyan")- log_concat(" Input File: ", input_file_text, style="yellow")- log_yellow(f" Input URLs: {self.manager.scrape_mapper.count:,}")- log_yellow(f" Input URL Groups: {self.manager.scrape_mapper.group_count:,}")- log_concat(" Log Folder: ", log_folder_text, style="yellow")- log_yellow(f" Total Runtime: {runtime}")- log_yellow(f" Total Downloaded Data: {total_data_written}")-- log_spacer(20, "")- log_cyan("Download Stats:")- log_green(f" Downloaded: {self.download_progress.completed_files:,} files")- log_yellow(f" Skipped (By Config): {self.download_progress.skipped_files:,} files")- log_yellow(f" Skipped (Previously Downloaded): {self.download_progress.previously_completed_files:,} files")- log_red(f" Failed: {self.download_stats_progress.failed_files:,} files")-- log_spacer(20, "")- log_cyan("Unsupported URLs Stats:")- log_yellow(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")- log_yellow(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")+ logger.info("Run Stats: ", config_path_text)+ logger.info(" Input File: ", input_file_text)+ logger.info(f" Input URLs: {self.manager.scrape_mapper.count:,}")+ logger.info(f" Input URL Groups: {self.manager.scrape_mapper.group_count:,}")+ logger.info(" Log Folder: ", log_folder_text)+ logger.info(f" Total Runtime: {runtime}")+ logger.info(f" Total Downloaded Data: {total_data_written}")++ log_spacer()+ logger.info("Download Stats:")+ logger.info(f" Downloaded: {self.download_progress.completed_files:,} files")+ logger.info(f" Skipped (By Config): {self.download_progress.skipped_files:,} files")+ logger.info(f" Skipped (Previously Downloaded): {self.download_progress.previously_completed_files:,} files")+ logger.info(f" Failed: {self.download_stats_progress.failed_files:,} files")++ log_spacer()+ logger.info("Unsupported URLs Stats:")+ logger.info(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")+ logger.info(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")self.print_dedupe_stats()- log_spacer(20, "")- log_cyan("Sort Stats:")- log_green(f" Audios: {self.sort_progress.audio_count:,}")- log_green(f" Images: {self.sort_progress.image_count:,}")- log_green(f" Videos: {self.sort_progress.video_count:,}")- log_green(f" Other Files: {self.sort_progress.other_count:,}")+ log_spacer()+ logger.info("Sort Stats:")+ logger.info(f" Audios: {self.sort_progress.audio_count:,}")+ logger.info(f" Images: {self.sort_progress.image_count:,}")+ logger.info(f" Videos: {self.sort_progress.video_count:,}")+ logger.info(f" Other Files: {self.sort_progress.other_count:,}")- last_padding = log_failures(self.scrape_stats_progress.return_totals(), "Scrape Failures:")- log_failures(self.download_stats_progress.return_totals(), "Download Failures:", last_padding)+ _log_errors(self.scrape_stats_progress.return_totals(), self.download_stats_progress.return_totals())def print_dedupe_stats(self) -> None:- log_spacer(20, "")- log_cyan("Dupe Stats:")- log_yellow(f" Newly Hashed: {self.hash_progress.hashed_files:,} files")- log_yellow(f" Previously Hashed: {self.hash_progress.prev_hashed_files:,} files")- log_yellow(f" Removed (Downloads): {self.hash_progress.removed_files:,} files")---def log_failures(failures: list[UiFailureTotal], title: str = "Failures:", last_padding: int = 0) -> int:- log_spacer(20, "")- log_cyan(title)- if not failures:- log_green(" None")- return 0- error_padding = last_padding- error_codes = [f.error_code for f in failures if f.error_code is not None]- if error_codes:- error_padding = max(len(str(max(error_codes))), error_padding)- for f in failures:- error = f.error_code if f.error_code is not None else ""- log_red(f" {error:>{error_padding}}{' ' if error_padding else ''}{f.msg}: {f.total:,}")- return error_padding+ log_spacer()+ logger.info("Dupe Stats:")+ logger.info(f" Newly Hashed: {self.hash_progress.hashed_files:,} files")+ logger.info(f" Previously Hashed: {self.hash_progress.prev_hashed_files:,} files")+ logger.info(f" Removed (Downloads): {self.hash_progress.removed_files:,} files")+++def _log_errors(scrape_errors: Sequence[UiFailureTotal], download_errors: Sequence[UiFailureTotal]) -> None:+ error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)++ try:+ padding = len(str(max(error_codes)))+ except ValueError:+ padding = 0++ for title, errors in (+ ("Scrape Failures:", scrape_errors),+ ("Download Failures:", download_errors),+ ):+ log_spacer()+ logger.info(title, extra={"color": "red"})+ if not errors:+ logger.info(f" {'None':>{padding}}")+ return++ for error in scrape_errors:+ error_code = error.code if error.code is not None else ""+ logger.info(f" {error_code:>{padding}}{' ' if padding else ''}{error.msg}: {error.count:,}")def get_input(manager: Manager) -> Text | str:@@ -206,22 +207,3 @@ def get_console_hyperlink(file_path: Path, text: str = "") -> Text:show_text = text or full_pathfile_url = URL(full_path.as_posix()).with_scheme("file")return Text(str(show_text), style=f"link {file_url}")---def concat_as_text(*text_or_str, style: str = "") -> Text:- result = Text()- for elem in text_or_str:- if isinstance(elem, Text):- text = elem- if style and text.style != style:- text.stylize(f"{style} {text.style}")- else:- text = Text(elem, style=style)-- result.append(text)- return result---def log_concat(*text_or_str, style: str = "", **kwargs) -> None:- text = concat_as_text(*text_or_str, style=style)- log_with_color(text, style, **kwargs)diff --git a/cyberdrop_dl/progress/overflow.py b/cyberdrop_dl/progress/overflow.pyindex 5be484910..fea91faa5 100644--- a/cyberdrop_dl/progress/overflow.py+++ b/cyberdrop_dl/progress/overflow.py@@ -51,7 +51,7 @@ class OverFlow:return ""-class OverflowPanel:+class OverFlowPanel:unit: ClassVar[str]def __init__(self, *columns: ProgressColumn | str, max_rows: int, expand: bool = True) -> None:diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.pyindex 58ff4a5a5..b9659c934 100644--- a/cyberdrop_dl/progress/scraping/downloads.py+++ b/cyberdrop_dl/progress/scraping/downloads.py@@ -27,7 +27,7 @@ from rich.text import Textfrom typing_extensions import overridefrom cyberdrop_dl.progress import DictProgress, ProgressHook, create_test_live-from cyberdrop_dl.progress.overflow import OverflowPanel+from cyberdrop_dl.progress.overflow import OverFlowPanelif TYPE_CHECKING:from collections.abc import Generator, Iterable@@ -112,7 +112,7 @@ class AutoDownloadColumn(DownloadColumn):@final-class DownloadsPanel(OverflowPanel):+class DownloadsPanel(OverFlowPanel):unit: ClassVar[str] = "file"@propertydiff --git a/cyberdrop_dl/progress/scraping/panel.py b/cyberdrop_dl/progress/scraping/panel.pyindex 9a31a3318..22c0ffffe 100644--- a/cyberdrop_dl/progress/scraping/panel.py+++ b/cyberdrop_dl/progress/scraping/panel.py@@ -15,7 +15,7 @@ from rich.text import Textfrom cyberdrop_dl import __version__from cyberdrop_dl.progress import create_test_live-from cyberdrop_dl.progress.overflow import OverflowPanel+from cyberdrop_dl.progress.overflow import OverFlowPanelif TYPE_CHECKING:from collections.abc import Generator@@ -61,7 +61,7 @@ class StatusMessage:@final-class ScrapingPanel(OverflowPanel):+class ScrapingPanel(OverFlowPanel):unit: ClassVar[str] = "URL"def __init__(self) -> None:diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 0cfb24caf..4e880f7a2 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -463,7 +463,7 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, type[Crawler]], craw)logger.info(f"Crawlers disabled by config: \n{crawlers_info}")- log_spacer(10)+ log_spacer()def match_url_to_crawler(existing_crawlers: dict[str, _T], url: AbsoluteHttpURL) -> _T | None:diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.pyindex f31965aef..b45e3c4ec 100644--- a/cyberdrop_dl/ui/progress/statistic_progress.py+++ b/cyberdrop_dl/ui/progress/statistic_progress.py@@ -42,7 +42,7 @@ class TaskInfo(NamedTuple):class UiFailureTotal(NamedTuple):full_msg: strtotal: int- error_code: int | None+ code: int | Nonemsg: str@classmethoddiff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.pyindex 09bde9413..fe6127b9d 100644--- a/cyberdrop_dl/utils/apprise.py+++ b/cyberdrop_dl/utils/apprise.py@@ -95,13 +95,13 @@ def _process_results(else:final_result = constants.NotificationResult.FAILED- log_spacer(10, log_to_console=False, log_to_file=not all(result))+ log_spacer()rich.print("Apprise notifications results:", final_result.value)log = logger.debug if all(result) else logger.infolog(f"Apprise notifications results: {final_result.value}")log(f"PARSED_APPRISE_URLs: \n{json.dumps(all_urls, indent=4)}\n")log(f"RESULTS_BY_TAGS: \n{json.dumps(result_dict, indent=4)}")- log_spacer(10, log_to_console=False, log_to_file=not all(result))+ log_spacer()parsed_log_lines = _parse_apprise_logs(apprise_logs)for line in parsed_log_lines:log(line.msg)diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.pyindex 7886fa0c9..709b0eef4 100644--- a/cyberdrop_dl/utils/logger.py+++ b/cyberdrop_dl/utils/logger.py@@ -1,39 +1,44 @@from __future__ import annotationsimport contextlib+import itertoolsimport jsonimport logging+import osimport queue-import sys-from functools import wraps+from contextvars import ContextVar+from datetime import datetimefrom logging.handlers import QueueHandler, QueueListenerfrom pathlib import Path-from typing import IO, TYPE_CHECKING, ParamSpec+from typing import TYPE_CHECKING, ParamSpec, castfrom rich._log_render import LogRenderfrom rich.console import Console, Group-from rich.containers import Lines, Renderablesfrom rich.logging import RichHandler-from rich.measure import Measurementfrom rich.padding import Paddingfrom rich.text import Text, TextType+from typing_extensions import override-from cyberdrop_dl import constants, env-from cyberdrop_dl.dependencies import browser_cookie3-from cyberdrop_dl.exceptions import InvalidYamlError+from cyberdrop_dl import env++if TYPE_CHECKING:+ import threading+ from collections.abc import Generatorlogger = logging.getLogger("cyberdrop_dl")-logger_debug = logging.getLogger("cyberdrop_dl_debug")-startup_logger = logging.getLogger("cyberdrop_dl_startup")_DEFAULT_CONSOLE = Console()-_USER_NAME = Path.home().resolve().name+_USER_NAME = Path.home().name_NEW_ISSUE_URL = "https://github.com/NTFSvolume/cdl/issues/new/choose"+_DEFAULT_CONSOLE_WIDTH = 240+_LOCK: threading.RLock = cast("threading.RLock", logging._lock) # pyright: ignore[ reportAttributeAccessIssue]+_MAIN_LOGGER: ContextVar[LogHandler] = ContextVar("_MAIN_LOGGER")++_capture_logs: ContextVar[bool] = ContextVar("_capture_logs", default=False)if TYPE_CHECKING:- from collections.abc import Callable, Generator, Iterable- from datetime import datetime+ from collections.abc import Callable, Iterablefrom rich.console import ConsoleRenderable@@ -46,18 +51,28 @@ if TYPE_CHECKING:class RedactedConsole(Console):"""Custom console to remove username from logs"""+ @overridedef _render_buffer(self, buffer) -> str:- output: str = super()._render_buffer(buffer)- return _redact_message(output)+ return self._redact_message(super()._render_buffer(buffer))++ @classmethod+ def _redact_message(cls, message: object) -> str:+ redacted = str(message)+ for sep in ("\\", "\\\\", "/"):+ as_tail = sep + _USER_NAME+ as_part = _USER_NAME + sep+ redacted = redacted.replace(as_tail, f"{sep}[REDACTED]").replace(as_part, f"[REDACTED]{sep}")+ return redactedclass JsonLogRecord(logging.LogRecord):- def getMessage(self) -> str: # noqa: N802- """`dicts` will be logged as json, lazily"""+ """`dicts` will be logged as json, lazily"""+ @override+ def getMessage(self) -> str:msg = str(self._proccess_msg(self.msg))if self.args:- args = map(self._proccess_msg, self.args)+ args = tuple(map(self._proccess_msg, self.args))try:return msg.format(*args)except Exception:@@ -67,7 +82,8 @@ class JsonLogRecord(logging.LogRecord):@staticmethoddef _proccess_msg(msg: object) -> object:- # TODO: Use our custom decoder to support more types+ if callable(dump := getattr(msg, "model_dump_json", None)):+ return dump()if isinstance(msg, dict):return json.dumps(msg, indent=2, ensure_ascii=False, default=str)return msg@@ -77,36 +93,64 @@ logging.setLogRecordFactory(JsonLogRecord)class LogHandler(RichHandler):- """Rich Handler with default settings, automatic console creation and custom log render to remove padding in files."""-- def __init__(- self, level: int = 10, file: IO[str] | None = None, width: int | None = None, debug: bool = False- ) -> None:- is_file: bool = file is not None- redacted: bool = is_file and not debug- console_cls = RedactedConsole if redacted else Console- if file is None and width is None:- console = _DEFAULT_CONSOLE- else:- console = console_cls(file=file, width=width)+ """Rich Handler with default settings, custom log render to remove padding in files and `color` extra"""+ def __init__(self, level: int = logging.DEBUG, console: Console | None = None) -> None:+ self.is_file: bool = bool(console)+ self._buffer: list[Text] = []super().__init__(level,console,- show_time=False,- show_path=False,- tracebacks_show_locals=debug,- locals_max_string=constants.DEFAULT_CONSOLE_WIDTH,+ show_time=self.is_file,+ rich_tracebacks=True,+ tracebacks_show_locals=True,+ locals_max_string=_DEFAULT_CONSOLE_WIDTH,tracebacks_extra_lines=2,locals_max_length=20,+ show_path=False,)- if is_file:+ if self.is_file:self._log_render = NoPaddingLogRender(- show_path=False,show_level=True,+ show_path=False,+ level_width=10,time_format=lambda dt: Text(f"[{dt.isoformat(sep=' ', timespec='milliseconds')}]", style="log.time"),)+ @override+ def render_message(self, record: logging.LogRecord, message: str) -> ConsoleRenderable:+ """This is the same as the base class, just added the `color` parsing from the extras"""+ use_markup = bool(getattr(record, "markup", self.markup))+ color = getattr(record, "color", "")+ message_text = Text.from_markup(message, style=color) if use_markup else Text(message, style=color)++ highlighter = getattr(record, "highlighter", self.highlighter)+ if highlighter:+ message_text = highlighter(message_text)++ if self.keywords is None:+ self.keywords = self.KEYWORDS++ if self.keywords:+ _ = message_text.highlight_words(self.keywords, "logging.keyword")++ if self.is_file and _capture_logs.get():+ self._buffer.append(message_text)++ return message_text++ def export_text(self) -> Text:+ assert self.lock is not None+ with self.lock:+ lines = self._buffer[:]+ self._buffer.clear()++ eof = Text("\n")+ text = Text()+ for line in itertools.chain.from_iterable((line, eof) for line in lines):+ _ = text.append_text(line)+ return text+class BareQueueHandler(QueueHandler):"""Sends the log record to the queue as is.@@ -117,7 +161,7 @@ class BareQueueHandler(QueueHandler):This made tracebacks render improperly because when the rich handler picks the log record from the queue, it has no traceback.The original traceback was being formatted as normal text and included as part of the message.- Having not pickleable objects is only an issue in multi-processing operations (multiprocessing.Queue)+ We never log from other processes so we do not need that safety check"""def prepare(self, record: logging.LogRecord) -> logging.LogRecord:@@ -144,11 +188,31 @@ class QueuedLogger:self.log_handler.close()+def _threaded_logger(log_handler: LogHandler) -> Generator[BareQueueHandler]:+ """Context-manager to process logs from this handler in another thread.++ It starts a QueueListener and yields the QueueHandler."""+ q: queue.Queue[logging.LogRecord] = queue.Queue()+ q_handler: BareQueueHandler = BareQueueHandler(q)+ listener: QueueListener = QueueListener(q, log_handler, respect_handler_level=True)+ listener.start()+ try:+ yield q_handler+ finally:+ try:+ q_handler.close()+ finally:+ listener.stop()+ for handler in listener.handlers[:]:+ handler.close()++class NoPaddingLogRender(LogRender):- cdl_padding: int = 0+ _cdl_padding: int = 0EXCLUDE_PATH_LOGGING_FROM: tuple[str, ...] = "logger.py", "base.py", "session.py", "cache_control.py"- def __call__( # type: ignore[reportIncompatibleMethodOverride]+ def __call__( # type: ignore[reportIncompatibleMethodOverride] # pyright: ignore[reportIncompatibleMethodOverride]self,console: Console,renderables: Iterable[ConsoleRenderable],@@ -169,170 +233,112 @@ class NoPaddingLogRender(LogRender):else Text(log_time.strftime(time_format), style="log.time"))if log_time_display == self._last_time and self.omit_repeated_times:- output.append(" " * len(log_time_display), style="log.time")+ _ = output.append(" " * len(log_time_display), style="log.time")output.pad_right(1)else:- output.append(log_time_display)+ _ = output.append_text(log_time_display)output.pad_right(1)self._last_time = log_time_display+if self.show_level:- output.append(level)+ _ = output.append(level)output.pad_right(1)- if not self.cdl_padding:- self.cdl_padding = _get_renderable_length(output)+ if not self._cdl_padding:+ self._cdl_padding = console.measure(output).maximumif self.show_path and path and not any(path.startswith(p) for p in self.EXCLUDE_PATH_LOGGING_FROM):path_text = Text(style="log.path")- path_text.append(path, style=f"link file://{link_path}" if link_path else "")+ _ = path_text.append(path, style=f"link file://{link_path}" if link_path else "")if line_no:- path_text.append(":")- path_text.append(+ _ = path_text.append(":")+ _ = path_text.append(f"{line_no}",style=f"link file://{link_path}#{line_no}" if link_path else "",)- output.append(path_text)+ _ = output.append_text(path_text)output.pad_right(1)padded_lines: list[ConsoleRenderable] = []- for renderable in Renderables(renderables): # type: ignore+ for renderable in renderables:if isinstance(renderable, Text):- renderable = _indent_text(renderable, console, self.cdl_padding)+ renderable = _indent_text(renderable, console, self._cdl_padding)renderable.stylize("log.message")- output.append(renderable)+ _ = output.append_text(renderable)continue- padded_lines.append(Padding(renderable, (0, 0, 0, self.cdl_padding), expand=False))-- return Group(output, *padded_lines)+ padded_lines.append(Padding(renderable, (0, 0, 0, self._cdl_padding), expand=False))-def _get_renderable_length(renderable) -> int:- measurement = Measurement.get(_DEFAULT_CONSOLE, _DEFAULT_CONSOLE.options, renderable)- return measurement.maximum+ return Group(output, *padded_lines)-def _indent_text(text: Text, console: Console, indent: int = 30) -> Text:+def _indent_text(text: Text, console: Console, indent: int) -> Text:"""Indents each line of a Text object except the first one."""- indent_str = Text("\n" + (" " * indent))+ padding = Text("\n" + (" " * indent))new_text = Text()- new_width = console.width - indent- lines: Lines = text.wrap(console, width=new_width)- first_line = lines[0]- other_lines = lines[1:]- for line in other_lines:+ first_line, *rest = text.wrap(console, width=console.width - indent)+ for line in rest:line.rstrip()- new_text.append(indent_str + line)+ _ = new_text.append_text(padding + line)first_line.rstrip()- return first_line.append(new_text)---def log(message: object, level: int = 10, bug: bool = False, **kwargs) -> None:- """Simple logging function."""- log_debug(message, level, **kwargs)- if bug:- args = message, f"Please open a bug report at {_NEW_ISSUE_URL}"- message = "{}. {}"- level = 30- else:- args = ()- logger.log(level, message, *args, **kwargs)---def log_debug(message: object, level: int = 10, **kwargs) -> None:- """Simple logging function."""- if env.DEBUG_VAR:- logger_debug.log(level, message, **kwargs)-+ return first_line.append_text(new_text)-def log_with_color(message: Text | str, style: str, level: int = 20, show_in_stats: bool = True, **kwargs) -> None:- """Simple logging function with color."""- text = message if isinstance(message, Text) else Text(message, style=style)- log(text.plain, level, **kwargs)- if constants.CONSOLE_LEVEL >= 50 and "pytest" not in sys.modules:- _DEFAULT_CONSOLE.print(text)- if show_in_stats:- constants.LOG_OUTPUT_TEXT.append_text(text.append("\n"))--def log_spacer(level: int, char: str = "-", *, log_to_console: bool = True, log_to_file: bool = True) -> None:- spacer = char * min(int(constants.DEFAULT_CONSOLE_WIDTH / 2), 50)- if log_to_file:- log(spacer, level)- if log_to_console and constants.CONSOLE_LEVEL >= 50:- _DEFAULT_CONSOLE.print("")- constants.LOG_OUTPUT_TEXT.append("\n", style="black")---def _redact_message(message: Exception | Text | str) -> str:- redacted = str(message)- separators = ["\\", "\\\\", "/"]- for sep in separators:- as_tail = sep + _USER_NAME- as_part = _USER_NAME + sep- redacted = redacted.replace(as_tail, f"{sep}[REDACTED]").replace(as_part, f"[REDACTED]{sep}")- return redacted+def log_spacer(char: str = "-") -> None:+ logger.info(char * (_DEFAULT_CONSOLE_WIDTH // 2), stacklevel=2)@contextlib.contextmanager-def _setup_startup_logger() -> Generator[None]:- """Context manager to add a file handler to the startup logger-- It will only add it if we have an exception, to prevent creating an empty file"""- startup_logger.setLevel(10)- if "pytest" not in sys.modules:- console_handler = LogHandler(level=10)- startup_logger.addHandler(console_handler)- try:- yield-- except Exception:- try:- file = Path.cwd() / "startup.log"- file_handler = LogHandler(- level=10,- file=file.open("w", encoding="utf8"),- width=constants.DEFAULT_CONSOLE_WIDTH,+def setup_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[None]:+ logger.setLevel(level)+ file.parent.mkdir(parents=True, exist_ok=True)+ with (+ file.open("w+" if os.name == "nt" else "w", encoding="utf8") as fp,+ _threaded_logger(LogHandler(level=level)) as console_out,+ _threaded_logger(+ main_logger := LogHandler(+ level=level,+ console=RedactedConsole(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),)- startup_logger.addHandler(file_handler)- except OSError:- # We could not create the file for some reason- # Ignore this and just log to the console- pass- raise---def catch_exceptions(func: Callable[_P, _ExitCode]) -> Callable[_P, _ExitCode]:- """Decorator to automatically log uncaught exceptions.-- Exceptions will be logged to a file in the current working directory- because the manager setup itself may have failed, therefore we don't know- what the proper log file path is.- """-- @wraps(func)- def catch(*args, **kwargs) -> _ExitCode | None:+ ) as file_out,+ _setup_debug_logger() as debug_log_file,+ ):+ token = _MAIN_LOGGER.set(main_logger)+ logger.addHandler(console_out)+ logger.addHandler(file_out)try:- with _setup_startup_logger():- return func(*args, **kwargs)+ logger.info(f"Debug log file: '{debug_log_file}'")+ yield+ finally:+ _MAIN_LOGGER.reset(token)- except InvalidYamlError as e:- startup_logger.error(e.message)- except browser_cookie3.BrowserCookieError:- startup_logger.exception("")+def _setup_debug_logger() -> Generator[Path | None]:+ if not env.DEBUG_VAR:+ yield+ return- except OSError as e:- startup_logger.exception(str(e))+ debug_log_file = Path(__file__).parent.parent.parent / "cyberdrop_dl_debug.log"- except KeyboardInterrupt:- startup_logger.info("Exiting...")- return+ if env.DEBUG_LOG_FOLDER:+ debug_log_folder = Path(env.DEBUG_LOG_FOLDER)- except Exception:- msg = "An error occurred, please report this to the developer with your logs file:"- startup_logger.exception(msg)+ if not debug_log_folder.is_dir():+ msg = f"Value of env var 'CDL_DEBUG_LOG_FOLDER' is invalid. Folder '{debug_log_folder}' does not exists"+ raise FileNotFoundError(None, msg, env.DEBUG_LOG_FOLDER)- return 1+ now = datetime.now().strftime("%Y%m%d_%H%M%S")+ debug_log_file = debug_log_folder / f"cyberdrop_dl_debug_{now}.log"- return catch+ debug_log_file = debug_log_file.expanduser().resolve().absolute()++ with (+ debug_log_file.open("w", encoding="utf8") as fp,+ _threaded_logger(+ LogHandler(level=logging.DEBUG, console=Console(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2))+ ) as debug_handler,+ ):+ logger.addHandler(debug_handler)+ yield debug_log_filediff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex ee29704f6..77f4c7e4e 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -11,7 +11,7 @@ import platformimport reimport sysfrom collections.abc import Generator-from functools import partial, wraps+from functools import wrapsfrom http import HTTPStatusfrom pathlib import Pathfrom stat import S_ISREG@@ -45,7 +45,6 @@ from cyberdrop_dl.exceptions import (get_origin,)from cyberdrop_dl.utils import json-from cyberdrop_dl.utils.logger import log_with_colorif TYPE_CHECKING:from collections.abc import Callable, Coroutine, Generator, Iterable@@ -282,22 +281,22 @@ def _partial_files(dir: Path | str) -> Generator[Path]:def delete_partial_files(manager: Manager) -> None:"""Deletes partial download files recursively."""- log_red("Deleting partial downloads...")+ logger.info("Deleting partial downloads...")for file in _partial_files(manager.config.files.download_folder):file.unlink(missing_ok=True)def check_for_partial_files(manager: Manager) -> None:"""Checks if there are partial downloads in any subdirectory and logs if found."""- log_yellow("Checking for partial downloads...")+ logger.info("Checking for partial downloads...")has_partial_files = next(_partial_files(manager.config.files.download_folder), None)if has_partial_files:- log_yellow("There are partial downloads in the downloads folder")+ logger.warning("There are partial downloads in the downloads folder")def delete_empty_folders(manager: Manager) -> None:"""Deletes empty folders efficiently."""- log_yellow("Checking for empty folders...")+ logger.info("Checking for empty folders...")purge_dir_tree(manager.config.files.download_folder)sorted_folder = manager.config.sorting.sort_folder@@ -389,7 +388,7 @@ def get_size_or_none(path: Path) -> int | None:@functools.cache-def get_system_information() -> str:+def get_system_information() -> dict[str, Any]:def get_common_name() -> str:system = platform.system()@@ -418,7 +417,7 @@ def get_system_information() -> str:"common_name": get_common_name(),}_ = system_info.pop("node", None)- return json.dumps(system_info, indent=4)+ return system_infodef is_blob_or_svg(link: str) -> bool:@@ -443,7 +442,3 @@ def unique(iterable: Iterable[_T], *, hashable: bool = True) -> Iterable[_T]:def xor_decrypt(encrypted_data: bytes, key: bytes) -> str:data = bytearray(b_input ^ b_key for b_input, b_key in zip(encrypted_data, itertools.cycle(key)))return data.decode("utf-8", errors="ignore")---log_yellow = partial(log_with_color, style="yellow", level=20)-log_red = partial(log_with_color, style="red", level=20)diff --git a/cyberdrop_dl/utils/webhook.py b/cyberdrop_dl/utils/webhook.pyindex 22d66569e..22d0daecd 100644--- a/cyberdrop_dl/utils/webhook.py+++ b/cyberdrop_dl/utils/webhook.py@@ -90,7 +90,7 @@ async def send_webhook_message(manager: Manager) -> None:logger.exception("Unable to send webhook notification")result_to_log = result, str(e)- log_spacer(10, log_to_console=False)+ log_spacer()rich.print("Webhook Notifications Results:", result)result_to_log = "\n".join(map(str, result_to_log))logger.info(f"Webhook Notifications Results: {result_to_log}")diff --git a/tests/test_manager.py b/tests/test_manager.pyindex c2127d16a..9beb6fa9d 100644--- a/tests/test_manager.py+++ b/tests/test_manager.py@@ -108,7 +108,7 @@ def test_args_logging_should_censor_webhook(running_manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)running_manager.args_logging()assert logs.messages- assert "Starting Cyberdrop-DL Process" in logs.text+ assert "Running cyberdrop-dl " in logs.textassert webhook not in logs.textwebhook_line = next(msg for msg in logs.text.splitlines() if '"webhook"' in msg)_, _, webhook_text = webhook_line.partition(":")