Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: switch to std logging (#68)
NTFSvolume4ce75c50accd
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 64abe409c..2ceb82aa4 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport asyncioimport contextlibimport itertools+import loggingimport timefrom collections.abc import Generatorfrom http import HTTPStatus@@ -16,7 +17,6 @@ from cyberdrop_dl.constants import FILE_FORMATSfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadErrorfrom cyberdrop_dl.utils import dates-from cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:@@ -31,6 +31,9 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)++_CONTENT_TYPES_OVERRIDES: dict[str, str] = {"text/vnd.trolltech.linguist": "video/MP2T"}_SLOW_DOWNLOAD_PERIOD: int = 10 # seconds_CHROME_ANDROID_USER_AGENT: str = (@@ -142,7 +145,7 @@ class DownloadClient:if not proceed:if media_item.is_segment:return True- log(f"Skipping {media_item.url} as it has already been downloaded", 10)+ logger.info(f"Skipping {media_item.url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed(False)await self.process_completed(media_item, domain)await self.handle_media_item_completion(media_item, downloaded=False)@@ -157,8 +160,9 @@ class DownloadClient:and not media_item.uploaded_atand (last_modified := get_last_modified(resp.headers))):- msg = f"Unable to parse upload date for {media_item.url}, using `Last-Modified` header as file datetime"- log(msg, 30)+ logger.warning(+ f"Unable to parse upload date for {media_item.url}, using `Last-Modified` header as file datetime"+ )media_item.uploaded_at = last_modifiedtask_id = media_item.task_id@@ -229,12 +233,14 @@ class DownloadClient:else:fallback_count += 1msg = f" with fallback URL #{fallback_count} {download_url} failed, retrying with new fallback URL: "- log(f"Download of {media_item.url}{msg}{next_download_url}", 40)+ logger.error(f"Download of {media_item.url}{msg}{next_download_url}")download_url = next_download_urlcontinueraise- async def _append_content(self, media_item: MediaItem, content: aiohttp.StreamReader | AbstractResponse) -> None:+ async def _append_content(+ self, media_item: MediaItem, content: aiohttp.StreamReader | AbstractResponse[Any]+ ) -> None:"""Appends content to a file."""assert media_item.task_id is not None@@ -289,7 +295,7 @@ class DownloadClient:async def download_file(self, domain: str, media_item: MediaItem) -> bool:"""Starts a file."""if self.manager.config.download_options.skip_download_mark_completed and not media_item.is_segment:- log(f"Download Removed {media_item.url} due to mark completed option", 10)+ logger.info(f"Download removed {media_item.url} due to mark completed option")self.manager.progress_manager.download_progress.add_skipped()# set completed pathawait self.process_completed(media_item, domain)@@ -304,7 +310,7 @@ class DownloadClient:proceed = await self.client_manager.check_file_duration(media_item)await self.manager.db_manager.history_table.add_duration(domain, media_item)if not proceed:- log(f"Download Skip {media_item.url} due to runtime restrictions", 10)+ logger.info(f"Download skipped {media_item.url} due to runtime restrictions")await asyncio.to_thread(media_item.path.unlink)await self.mark_incomplete(media_item, domain)self.manager.progress_manager.download_progress.add_skipped()@@ -342,7 +348,7 @@ class DownloadClient:await self.manager.hash_manager.hash_client.hash_item_during_download(media_item)self.manager.path_manager.add_completed(media_item)except Exception:- log(f"Error handling media item completion of: {media_item.path}", 10, exc_info=True)+ logger.exception(f"Error handling media item completion of: {media_item.path}")"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""@@ -372,14 +378,11 @@ class DownloadClient:proceed = Trueskip = False- if not TYPE_CHECKING:- log = log_debug if media_item.is_segment else globals()["log"]-while True:if expected_size and not media_item.is_segment:file_size_check = self.check_filesize_limits(media_item)if not file_size_check:- log(f"Download Skip {media_item.url} due to filesize restrictions", 10)+ logger.info(f"Download skipped {media_item.url} due to filesize restrictions")proceed = Falseskip = Truereturn proceed, skip@@ -388,7 +391,7 @@ class DownloadClient:breakif media_item.path.exists() and media_item.path.stat().st_size == media_item.filesize:- log(f"Found {media_item.path.name} locally, skipping download")+ logger.info(f"Found {media_item.path.name} locally, skipping download")proceed = Falsebreak@@ -405,18 +408,17 @@ class DownloadClient:if media_item.filename == downloaded_filename:if media_item.partial_file.exists():- log(f"Found {downloaded_filename} locally, trying to resume")+ logger.info(f"Found {downloaded_filename} locally, trying to resume")assert media_item.filesizesize = media_item.partial_file.stat().st_size- if size >= media_item.filesize != 0:- log(f"Deleting partial file {media_item.partial_file}")+ if size >= media_item.filesize:+ logger.info(f"Deleting partial file {media_item.partial_file}. Size is out of bound")media_item.partial_file.unlink()elif size == media_item.filesize:if media_item.path.exists():- log(- f"Found conflicting complete file '{media_item.path}' locally, iterating filename",- 30,+ logger.warning(+ f"Found conflicting complete file '{media_item.path}' locally, iterating filename")new_complete_filename, new_partial_file = await self.iterate_filename(media_item.path,@@ -430,17 +432,16 @@ class DownloadClient:else:proceed = Falsemedia_item.partial_file.rename(media_item.path)- log(+ logger.info(f"Renaming found partial file '{media_item.partial_file}' to complete file {media_item.path}")elif media_item.path.exists():if media_item.path.stat().st_size == media_item.filesize:- log(f"Found complete file '{media_item.path}' locally, skipping download")+ logger.info(f"Found complete file '{media_item.path}' locally, skipping download")proceed = Falseelse:- log(- f"Found conflicting complete file '{media_item.path}' locally, iterating filename",- 30,+ logger.warning(+ f"Found conflicting complete file '{media_item.path}' locally, iterating filename")media_item.path, media_item.partial_file = await self.iterate_filename(media_item.path,diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex 59bf912e2..5a8004c10 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import loggingfrom collections import defaultdictfrom pathlib import Pathfrom typing import TYPE_CHECKING, Literal@@ -10,7 +11,6 @@ from send2trash import send2trashfrom cyberdrop_dl import constantsfrom cyberdrop_dl.constants import Hashingfrom cyberdrop_dl.ui.prompts.basic_prompts import enter_to_continue-from cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:@@ -20,6 +20,8 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)+def hash_directory_scanner(manager: Manager, path: Path) -> None:asyncio.run(_hash_directory_scanner_helper(manager, path))@@ -87,7 +89,7 @@ class HashClient:)await self.save_hash_data(media_item, hash)except Exception as e:- log(f"After hash processing failed: '{media_item.path}' with error {e}", 40, exc_info=True)+ logger.exception(f"After hash processing failed: '{media_item.path}' with error {e}")async def update_db_and_retrive_hash(self, file: Path | str, original_filename: str | None = None, referer: URL | None = None@@ -138,7 +140,7 @@ class HashClient:referer,)except Exception as e:- log(f"Error hashing '{file}' : {e}", 40, exc_info=True)+ logger.exception(f"Error hashing '{file}' : {e}")else:return hash@@ -187,7 +189,7 @@ class HashClient:try:deleted = await _delete_file(file, self._to_trash)except OSError as e:- log(f"Unable to remove '{file}' ({hash_string}): {e}", 40)+ logger.exception(f"Unable to remove '{file}' ({hash_string}): {e}")else:if not deleted:return@@ -196,7 +198,7 @@ class HashClient:f"Removed new download '{file}' [{self._deleted_file_suffix}]. "f"File hash matches with a previous download ({hash_string})")- log(msg, 10)+ logger.info(msg)self.manager.progress_manager.hash_progress.add_removed_file()finally:@@ -216,9 +218,8 @@ class HashClient:continuetry:await self.hash_item(media_item)- except Exception as e:- msg = f"Unable to hash file = {media_item.path}: {e}"- log(msg, 40)+ except Exception:+ logger.exception(msg=f"Unable to hash '{media_item.path}'")return self.hashes_dictdiff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.pyindex fcab3091e..1770756c9 100644--- a/cyberdrop_dl/clients/response.py+++ b/cyberdrop_dl/clients/response.py@@ -69,7 +69,10 @@ class AbstractResponse(ABC, Generic[_ResponseT]):return f"<{type(self).__name__} [{self.status}] ({self.url})>"def __str__(self) -> str:- return self.create_report()+ me = self.__json__()+ for key in ("content", "created_at"):+ del me[key]+ return json.dumps(me, indent=2, ensure_ascii=False)def __json__(self) -> dict[str, Any]:if content := self._text:@@ -81,8 +84,9 @@ class AbstractResponse(ABC, Generic[_ResponseT]):return {"url": str(self.url),+ "location": str(self.location) if self.location else None,"status_code": self.status,- "created_at": self.created_at.isoformat(),+ "created_at": str(self.created_at),"response_headers": dict(self.headers),"content": content,}diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex 8941d368d..3f26afb73 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -4,8 +4,7 @@ import asyncioimport contextlibimport loggingimport time-from datetime import datetime-from json import dumps as json_dumps+import uuidfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, cast@@ -16,7 +15,7 @@ from cyberdrop_dl.utils.cookie_management import make_simple_cookiefrom cyberdrop_dl.utils.utilities import sanitize_filenameif TYPE_CHECKING:- from collections.abc import AsyncGenerator+ from collections.abc import AsyncGenerator, Mappingfrom curl_cffi.requests.impersonate import BrowserTypeLiteralfrom curl_cffi.requests.session import HttpMethod@@ -34,9 +33,7 @@ class ScraperClient:def __init__(self, client_manager: ClientManager) -> None:self.client_manager = client_managerself._save_pages_html = client_manager.manager.config_manager.settings_data.files.save_pages_html- self._pages_folder = self.client_manager.manager.path_manager.pages_folder- min_html_file_path_len = len(str(self._pages_folder)) + len(constants.STARTUP_TIME_STR) + 10- self._max_html_stem_len = 245 - min_html_file_path_len+ self._resp_folder = self.client_manager.manager.path_manager.pages_folder@contextlib.asynccontextmanagerasync def _limiter(self, domain: str) -> AsyncGenerator[None]:@@ -77,10 +74,21 @@ class ScraperClient:if method == "GET" and (data or json):method = "POST"- if not impersonate:- headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)+ impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate+ if impersonate:+ self.client_manager.check_curl_cffi_is_available()+ if impersonate is True:+ impersonate = "chrome"+ request_params["impersonate"] = impersonate- async with self.__request_context(url, method, request_params, impersonate) as resp:+ else:+ _ = headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)+ request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)++ request_id = str(uuid.uuid4())+ logger.debug("Starting {} request to {} [id={}]\n{}", method, url, request_id, request_params)++ async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:exc = Nonetry:yield await self._check_response(resp, url)@@ -88,7 +96,17 @@ class ScraperClient:exc = eraisefinally:- await self.write_soup_to_disk(url, resp, exc)+ logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)+ if self._save_pages_html:+ _ = self.client_manager.manager.task_group.create_task(+ asyncio.to_thread(+ _write_resp_to_disk,+ self._resp_folder,+ url,+ resp,+ exc,+ )+ )def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:"""@@ -104,28 +122,24 @@ class ScraperClient:self.client_manager.cookies.update_cookies(simple_cookie, url)@contextlib.asynccontextmanager- async def __request_context(+ async def __request(self,url: AbsoluteHttpURL,method: HttpMethod,- request_params: dict[str, Any],- impersonate: BrowserTypeLiteral | bool | None,+ request_params: Mapping[str, Any],+ *,+ impersonate: bool,) -> AsyncGenerator[AbstractResponse[Any]]:- impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate+if impersonate:- self.client_manager.check_curl_cffi_is_available()- if impersonate is True:- impersonate = "chrome"- request_params["impersonate"] = impersonate- curl_resp = await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)- try:+ async with contextlib.aclosing(+ await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)+ ) as curl_resp:yield AbstractResponse.create(curl_resp)self.__sync_session_cookies(url)- finally:- await curl_resp.aclose()+return- request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)async with (self.client_manager._session.request(method, url, **request_params) as aio_resp,):@@ -148,63 +162,43 @@ class ScraperClient:raise e from Noneself.client_manager.cookies.update_cookies(solution.cookies)- await self._check_flaresolverr_resp(solution)+ await _check_flaresolverr_resp(self.client_manager.manager.global_config.general.user_agent, solution)return AbstractResponse.create(solution)- async def _check_flaresolverr_resp(self, solution: flaresolverr.Solution) -> None:- cdl_user_agent = self.client_manager.manager.global_config.general.user_agent- mismatch_ua_msg = (- "Config user_agent and flaresolverr user_agent do not match:"- f"\n Cyberdrop-DL: '{cdl_user_agent}'"- f"\n Flaresolverr: '{solution.user_agent}'"- )- try:- await ddos_guard.check(solution.content)- except DDOSGuardError:- if solution.user_agent != cdl_user_agent:- raise DDOSGuardError(mismatch_ua_msg) from None+async def _check_flaresolverr_resp(cdl_user_agent: str, solution: flaresolverr.Solution) -> None:+ mismatch_ua_msg = (+ "Config user_agent and flaresolverr user_agent do not match:"+ f"\n Cyberdrop-DL: '{cdl_user_agent}'"+ f"\n Flaresolverr: '{solution.user_agent}'"+ )+ try:+ await ddos_guard.check(solution.content)+ except DDOSGuardError:if solution.user_agent != cdl_user_agent:- logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")+ raise DDOSGuardError(mismatch_ua_msg) from None- async def write_soup_to_disk(- self, url: AbsoluteHttpURL, response: AbstractResponse[Any], exc: Exception | None = None- ):- """Writes html to a file."""+ if solution.user_agent != cdl_user_agent:+ logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")- if not (- self._save_pages_html- and "html" in response.content_type- and response.consumed # Do not consume the response if the crawler didn't- ):- return- content = cast("str", (await response.soup()).prettify(formatter="html"))- now = datetime.now()- log_date = now.strftime(constants.LOGS_DATETIME_FORMAT)- url_str = str(url)- response_url_str = str(response.url)- clean_url = sanitize_filename(Path(url_str).as_posix().replace("/", "-"))- filename = f"{clean_url[: self._max_html_stem_len]}_{log_date}.html"- file_path = self._pages_folder / filename- info = {- "url": url_str,- "response_url": response_url_str,- "status_code": response.status,- "datetime": now.isoformat(),- "response_headers": dict(response.headers),- }- if exc:- info |= {"error": str(exc), "exception": repr(exc)}-- json_data = json_dumps(info, indent=4, ensure_ascii=False)- text = f"<!-- cyberdrop-dl scraping result\n{json_data}\n-->\n{content}"- self.client_manager.manager.task_group.create_task(try_write(file_path, text))---async def try_write(file: Path, content: str) -> None:+def _write_resp_to_disk(+ folder: Path,+ url: AbsoluteHttpURL,+ response: AbstractResponse[Any],+ exc: Exception | None = None,+) -> None:++ max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10++ log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)+ url_str = str(url)+ clean_url = sanitize_filename(Path(url_str).as_posix().replace("/", "-"))+ filename = f"{clean_url[:max_stem_len]}_{log_date}.html"+ file = folder / filename+ content = response.create_report(exc)try:- await asyncio.to_thread(file.write_text, content, "utf8")+ _ = file.write_text(content, "utf8")except OSError:passdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 15c458002..a67e05c07 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -5,6 +5,7 @@ import contextlibimport datetimeimport importlibimport inspect+import loggingimport pkgutilimport reimport weakref@@ -54,24 +55,24 @@ from cyberdrop_dl.utils.utilities import (truncate_str,)-P = ParamSpec("P")-R = TypeVar("R")-T = TypeVar("T")-_T_co = TypeVar("_T_co", covariant=True)--if TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterablefrom http.cookies import BaseCookiefrom bs4 import BeautifulSoup, Tag+ from curl_cffi.requests.impersonate import BrowserTypeLiteralfrom rich.progress import TaskIDfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)-OneOrTuple: TypeAlias = T | tuple[T, ...]+_P = ParamSpec("_P")+_R = TypeVar("_R")+_T = TypeVar("_T")+_T_co = TypeVar("_T_co", covariant=True)+OneOrTuple: TypeAlias = _T | tuple[_T, ...]SupportedPaths: TypeAlias = dict[str, OneOrTuple[str]]SupportedDomains: TypeAlias = OneOrTuple[str]RateLimit = tuple[float, float]@@ -155,7 +156,7 @@ class Registry:class HTTPClientProxy:DOMAIN: ClassVar[str]- _IMPERSONATE: ClassVar[str | bool | None] = None+ _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = Noneclient: ScraperClient # pyright: ignore[reportUninitializedInstanceVariable]@classmethod@@ -181,7 +182,7 @@ class HTTPClientProxy:@signature.copy(ScraperClient._request)@contextlib.asynccontextmanagerasync def request(- self, *args, impersonate: str | bool | None = None, **kwargs+ self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs) -> AsyncGenerator[AbstractResponse[Any]]:if impersonate is None:impersonate = self._IMPERSONATE@@ -365,7 +366,7 @@ class Crawler(ABC, HTTPClientProxy):try:yieldexcept Exception:- self.log(f"[{self.FOLDER_DOMAIN}] {msg}. Crawler has been disabled", 40)+ logger.error(f"[{self.FOLDER_DOMAIN}] {msg}. Crawler has been disabled")self.disabled = Trueraise@@ -387,7 +388,7 @@ class Crawler(ABC, HTTPClientProxy):scrape_item.url = url = self.transform_url(scrape_item.url)if url.path_qs in self.scraped_items:- return log(f"Skipping {url} as it has already been scraped", 10)+ return logger.info(f"Skipping {url} as it has already been scraped")self.scraped_items.add(url.path_qs)async with self._fetch_context(scrape_item):@@ -431,7 +432,7 @@ class Crawler(ABC, HTTPClientProxy):@contextlib.contextmanagerdef new_task_id(self, url: AbsoluteHttpURL) -> Generator[TaskID]:"""Creates a new task_id (shows the URL in the UI and logs)"""- log(f"Scraping [{self.FOLDER_DOMAIN}]: {url}", 20)+ logger.info(f"Scraping [{self.FOLDER_DOMAIN}]: {url}")task_id = self.manager.progress_manager.scraping_progress.add_task(url)try:yield task_id@@ -456,7 +457,7 @@ class Crawler(ABC, HTTPClientProxy):filename = f"{name}.metadata" # we won't write to fs, so we skip name sanitizationdownload_folder = get_download_path(self.manager, scrape_item, self.FOLDER_DOMAIN)- url = scrape_item.url.with_scheme("metadata")+ url = AbsoluteHttpURL(scrape_item.url.with_scheme("metadata"))media_item = MediaItem.from_item(scrape_item,url,@@ -532,18 +533,12 @@ class Crawler(ABC, HTTPClientProxy):db_path = self.create_db_path(url)check_complete = await self.manager.db_manager.history_table.check_complete(self.DOMAIN, url, referer, db_path)if check_complete:- log(f"Skipping {url} as it has already been downloaded", 10)+ logger.info(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()return check_completeasync def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:await self.manager.states.RUNNING.wait()- if media_item.uploaded_at and not isinstance(media_item.uploaded_at, int):- msg = (- f"Invalid datetime from '{self.FOLDER_DOMAIN}' crawler . Got {media_item.uploaded_at!r}, expected int."- )- log(msg, bug=True)-check_complete = await self.check_complete(media_item.url, media_item.referer)if check_complete:if media_item.album_id:@@ -561,17 +556,17 @@ class Crawler(ABC, HTTPClientProxy):media_host = media_item.url.hostif (hosts := self.manager.config.ignore_options.skip_hosts) and any(host in media_host for host in hosts):- log(f"Download skip {media_item.url} due to skip_hosts config", 10)+ logger.info(f"Download skipped {media_item.url} due to skip_hosts config")return Trueif (hosts := self.manager.config.ignore_options.only_hosts) and not any(host in media_host for host in hosts):- log(f"Download skip {media_item.url} due to only_hosts config", 10)+ logger.info(f"Download skipped {media_item.url} due to only_hosts config")return Trueif (regex := self.manager.config.ignore_options.filename_regex_filter) and re.search(regex, media_item.filename):- log(f"Download skip {media_item.url} due to filename regex filter config", 10)+ logger.info(f"Download skipped {media_item.url} due to filename regex filter config")return Truereturn False@@ -588,7 +583,7 @@ class Crawler(ABC, HTTPClientProxy):domain = None if any_crawler else self.DOMAINdownloaded = await self.manager.db_manager.history_table.check_complete_by_referer(domain, url)if downloaded:- log(f"Skipping {url} as it has already been downloaded", 10)+ logger.info(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()return Truereturn False@@ -601,7 +596,7 @@ class Crawler(ABC, HTTPClientProxy):downloaded = await self.manager.db_manager.hash_table.check_hash_exists(hash_type, hash_value)if downloaded:url = scrape_item if isinstance(scrape_item, URL) else scrape_item.url- log(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded", 10)+ logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()return downloaded@@ -637,7 +632,7 @@ class Crawler(ABC, HTTPClientProxy):return Falseurl_path = self.create_db_path(url)if url_path in album_results and album_results[url_path] != 0:- log(f"Skipping {url} as it has already been downloaded")+ logger.info(f"Skipping {url} as it has already been downloaded")self.manager.progress_manager.download_progress.add_previously_completed()return Truereturn False@@ -829,7 +824,9 @@ class Crawler(ABC, HTTPClientProxy):page_url = self.parse_url(page_url_str, **kwargs)@error_handling_wrapper- async def direct_file(self, scrape_item: ScrapeItem, url: URL | None = None, assume_ext: str | None = None) -> None:+ async def direct_file(+ self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None, assume_ext: str | None = None+ ) -> None:"""Download a direct link file. Filename will be the url slug"""link = url or scrape_item.urlfilename, ext = self.get_filename_and_ext(link.name or link.parent.name, assume_ext=assume_ext)@@ -946,7 +943,7 @@ class Crawler(ABC, HTTPClientProxy):f"Important information was removed while creating a filename. "f"\n{calling_args}")- log(msg, bug=True)+ logger.warning(msg)return filename@final@@ -1042,7 +1039,7 @@ def _create_subclass(url: yarl.URL | str, base_class: type[_CrawlerT]) -> type[_domain = primary_url.host.removeprefix("www.")class_name = _make_crawler_name(domain)class_attributes = Site(primary_url, domain)._asdict()- return type(class_name, (base_class,), class_attributes) # type: ignore+ return type(class_name, (base_class,), class_attributes) # type: ignore # pyright: ignore[reportReturnType]def _make_crawler_name(input_string: str) -> str:@@ -1091,12 +1088,12 @@ def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, OneOrTupdef auto_task_id(- func: Callable[Concatenate[_CrawlerT, ScrapeItem, P], R | Coroutine[None, None, R]],-) -> Callable[Concatenate[_CrawlerT, ScrapeItem, P], Coroutine[None, None, R]]:+ func: Callable[Concatenate[_CrawlerT, ScrapeItem, _P], _R | Coroutine[None, None, _R]],+) -> Callable[Concatenate[_CrawlerT, ScrapeItem, _P], Coroutine[None, None, _R]]:"""Autocreate a new `task_id` from the scrape_item of the method"""@wraps(func)- async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: P.args, **kwargs: P.kwargs) -> R:+ async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: _P.args, **kwargs: _P.kwargs) -> _R:await self.manager.states.RUNNING.wait()with self.new_task_id(scrape_item.url):result = func(self, scrape_item, *args, **kwargs)diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 18b0c963b..514d2ac9e 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -1,4 +1,3 @@-# pyright: ignore[reportIncompatibleVariableOverride]from __future__ import annotationsimport base64@@ -18,7 +17,6 @@ if TYPE_CHECKING:from collections.abc import Callable, Generatorimport aiohttp- from propcache.api import under_cached_property as cached_propertyfrom rich.progress import TaskIDfrom cyberdrop_dl import signature@@ -34,17 +32,17 @@ if TYPE_CHECKING:@signature.copy(yarl.URL.__mod__)def __mod__(self) -> AbsoluteHttpURL: ...- @cached_property- def host(self) -> str: ...+ @property+ def host(self) -> str: ... # pyright: ignore[reportIncompatibleVariableOverride]- @cached_property- def scheme(self) -> Literal["http", "https"]: ...+ @property+ def scheme(self) -> Literal["http", "https"]: ... # pyright: ignore[reportIncompatibleVariableOverride]- @cached_property- def absolute(self) -> Literal[True]: ...+ @property+ def absolute(self) -> Literal[True]: ... # pyright: ignore[reportIncompatibleVariableOverride]- @cached_property- def parent(self) -> AbsoluteHttpURL: ...+ @property+ def parent(self) -> AbsoluteHttpURL: ... # pyright: ignore[reportIncompatibleVariableOverride]@signature.copy(yarl.URL.with_path)def with_path(self) -> AbsoluteHttpURL: ...diff --git a/cyberdrop_dl/database/tables/hash.py b/cyberdrop_dl/database/tables/hash.pyindex 2d318c8f2..d809dc45c 100644--- a/cyberdrop_dl/database/tables/hash.py+++ b/cyberdrop_dl/database/tables/hash.py@@ -1,10 +1,9 @@from __future__ import annotations+import loggingfrom pathlib import Pathfrom typing import TYPE_CHECKING, cast-from cyberdrop_dl.utils.logger import log-from .definitions import create_files, create_hash, create_hash_indexif TYPE_CHECKING:@@ -14,6 +13,9 @@ if TYPE_CHECKING:from cyberdrop_dl.database import Database+logger = logging.getLogger(__name__)++class HashTable:def __init__(self, database: Database) -> None:self._database = database@@ -51,7 +53,7 @@ class HashTable:return row[0]except Exception as e:- log(f"Error checking file: {e}", 40, exc_info=e)+ logger.exception(f"Error checking file: {e}")async def get_files_with_hash_matches(self, hash_value: str, size: int, hash_type: str | None = None@@ -83,8 +85,8 @@ class HashTable:cursor = await self.db_conn.execute(query, (hash_value, size, hash_type))return cast("list[aiosqlite.Row]", await cursor.fetchall())- except Exception as e:- log(f"Error retrieving folder and filename: {e}", 40, exc_info=e)+ except Exception:+ logger.exception("Error retrieving folder and filename")return []async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:@@ -129,7 +131,7 @@ class HashTable:await self.db_conn.execute(query, (hash_value, hash_type, folder, download_filename, hash_value))await self.db_conn.commit()except Exception as e:- log(f"Error inserting/updating record: {e}", 40, exc_info=e)+ logger.exception(f"Error inserting/updating record: {e}")return Falsereturn True@@ -166,7 +168,7 @@ class HashTable:)await self.db_conn.commit()except Exception as e:- log(f"Error inserting/updating record: {e}", 40, exc_info=e)+ logger.exception(f"Error inserting/updating record: {e}")return Falsereturn True@@ -190,5 +192,5 @@ class HashTable:rows = await cursor.fetchall()return [row[0] for row in rows]except Exception as e:- log(f"Error retrieving folder and filename: {e}", 40, exc_info=e)+ logger.exception(f"Error retrieving folder and filename: {e}")return []diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.pyindex 481c32d04..4cc381bda 100644--- a/cyberdrop_dl/database/tables/history.py+++ b/cyberdrop_dl/database/tables/history.py@@ -4,8 +4,6 @@ import loggingfrom sqlite3 import IntegrityError, Rowfrom typing import TYPE_CHECKING, cast-from cyberdrop_dl.utils.utilities import log-from .definitions import create_fixed_history, create_historyif TYPE_CHECKING:@@ -116,7 +114,7 @@ class HistoryTable:current_referer, completed = await select_referer_and_completed()if completed and url != referer and str(referer) != current_referer:# Update the referer if it has changed so that check_complete_by_referer can work- log(f"Updating referer of {url} from {current_referer} to {referer}")+ logger.info(f"Updating referer of {url} from {current_referer} to {referer}")await update_referer()return completed@@ -299,7 +297,7 @@ class HistoryTable:yield cast("list[Row]", rows)except Exception as e:- log(f"Error getting bunkr failed via size: {e}", 40, exc_info=e)+ logger.exception(f"Error getting bunkr failed via size: {e}")async def get_all_bunkr_failed_via_hash(self) -> AsyncGenerator[list[Row]]:query = """@@ -314,7 +312,7 @@ class HistoryTable:yield cast("list[Row]", rows)except Exception as e:- log(f"Error getting bunkr failed via hash: {e}", 40, exc_info=e)+ logger.exception(f"Error getting bunkr failed via hash: {e}")async def fix_primary_keys(self) -> None:domain_column, *_ = await self._get_media_table_columns()diff --git a/cyberdrop_dl/database/tables/schema.py b/cyberdrop_dl/database/tables/schema.pyindex 704270c2f..63a21f05f 100644--- a/cyberdrop_dl/database/tables/schema.py+++ b/cyberdrop_dl/database/tables/schema.py@@ -1,12 +1,11 @@from __future__ import annotations+import loggingfrom typing import TYPE_CHECKINGimport aiosqlitefrom packaging.version import Version-from cyberdrop_dl.utils.logger import log, log_spacer-from .definitions import create_schema_versionif TYPE_CHECKING:@@ -17,6 +16,8 @@ if TYPE_CHECKING:CURRENT_APP_SCHEMA_VERSION = "8.10.0"+logger = logging.getLogger(__name__)+class SchemaVersionTable:def __init__(self, database: Database) -> None:@@ -52,13 +53,12 @@ class SchemaVersionTable:await self.db_conn.commit()async def startup(self) -> None:- log_spacer(10)- log(f"Expected database schema version: {CURRENT_APP_SCHEMA_VERSION}")+ logger.info(f"Expected database schema version: {CURRENT_APP_SCHEMA_VERSION}")version = await self.get_version()- log(f"Database reports installed version: {version}")+ logger.info(f"Database reports installed version: {version}")if version is not None and version >= Version(CURRENT_APP_SCHEMA_VERSION):return# TODO: on v9, raise SystemExit if db version is None or older than 8.0.0- log(f"Updating database version to {CURRENT_APP_SCHEMA_VERSION}")+ logger.info(f"Updating database version to {CURRENT_APP_SCHEMA_VERSION}")await self.__update_schema_version()diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 54ef0c60f..384c24008 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -16,13 +16,7 @@ from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapperfrom cyberdrop_dl.ui.program_ui import ProgramUIfrom cyberdrop_dl.updates import check_latest_pypifrom cyberdrop_dl.utils.apprise import send_apprise_notifications-from cyberdrop_dl.utils.logger import (- LogHandler,- QueuedLogger,- log,- log_spacer,- log_with_color,-)+from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log_spacer, log_with_colorfrom cyberdrop_dl.utils.sorting import Sorterfrom cyberdrop_dl.utils.utilities import check_partials_and_empty_foldersfrom cyberdrop_dl.utils.webhook import send_webhook_message@@ -30,6 +24,7 @@ from cyberdrop_dl.utils.webhook import send_webhook_messageif TYPE_CHECKING:from collections.abc import Callable, Coroutine, Sequence+logger = logging.getLogger(__name__)P = ParamSpec("P")R = TypeVar("R")@@ -72,22 +67,22 @@ async def _run_manager(manager: Manager) -> None:debug_log_file_path = _setup_debug_logger(manager)start_time = manager.start_time_setup_main_logger(manager)- log(f"Using Debug Log: {debug_log_file_path}", 10)- log("Starting Async Processes...", 10)+ logger.info(f"Using Debug Log: {debug_log_file_path}")+ logger.info("Starting Async Processes...")await manager.async_startup()log_spacer(10)- log("Starting CDL...\n", 20)+ logger.info("Starting CDL...\n")await _scheduler(manager)manager.progress_manager.print_stats(start_time)log_spacer(20)- log("Checking for Updates...", 20)+ logger.info("Checking for Updates...")check_latest_pypi()log_spacer(20)- log("Closing Program...", 20)+ logger.info("Closing Program...")log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)await send_webhook_message(manager)@@ -137,11 +132,9 @@ def _setup_debug_logger(manager: Manager) -> Path | None:if not env.DEBUG_VAR:return- debug_logger = logging.getLogger("cyberdrop_dl_debug")- log_level = 10+ log_level = logging.DEBUGsettings_data = manager.config_manager.settings_datasettings_data.runtime_options.log_level = log_level- debug_logger.setLevel(log_level)debug_log_file_path = Path(__file__).parents[1] / "cyberdrop_dl_debug.log"if env.DEBUG_LOG_FOLDER:debug_log_folder = Path(env.DEBUG_LOG_FOLDER)@@ -156,7 +149,8 @@ def _setup_debug_logger(manager: Manager) -> Path | None:file_handler = LogHandler(level=log_level, file=file_io, width=500, debug=True)queued_logger = QueuedLogger(manager, file_handler, "debug")- debug_logger.addHandler(queued_logger.handler)++ logging.getLogger("cyberdrop_dl").addHandler(queued_logger.handler)# aiosqlite_log = logging.getLogger("aiosqlite")# aiosqlite_log.setLevel(log_level)diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 7fc0e3f33..a17200aee 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import loggingimport osimport shutilimport subprocess@@ -26,9 +27,10 @@ from cyberdrop_dl.exceptions import (SkipDownloadError,TooManyCrawlerErrors,)-from cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, parse_url+logger = logging.getLogger(__name__)+# Windows epoch is January 1, 1601. Unix epoch is January 1, 1970WIN_EPOCH_OFFSET = 116444736e9MAC_OS_SET_FILE = None@@ -101,14 +103,13 @@ def retry(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[if e.status != 999:media_item.attempts += 1- log(f"{self.log_prefix} failed: {media_item.url} with error: {e!s}", 40)+ logger.error(f"{self.log_prefix} failed: {media_item.url} with error: {e!s}")if media_item.attempts >= self.max_attempts:raise- retry_msg = (+ logger.info(f"Retrying {self.log_prefix.lower()}: {media_item.url} , retry attempt: {media_item.attempts + 1}")- log(retry_msg, 20)return wrapper@@ -271,11 +272,10 @@ class Downloader:try:subtitles = await download(m3u8_group.subtitle)except Exception as e:- log(f"Unable to download subtitles for {media_item.url}, Skipping. {e!r}", 40)+ logger.exception(f"Unable to download subtitles for {media_item.url}, Skipping. {e!r}")else:- log(- f"Found subtitles for {media_item.url}, but CDL is currently unable to merge them. Subtitle were saved at {subtitles} ",- 30,+ logger.warning(+ f"Found subtitles for {media_item.url}, but CDL is currently unable to merge them. Subtitle were saved at {subtitles} ")if m3u8_group.audio:@@ -326,7 +326,7 @@ class Downloader:await self.set_file_datetime(media_item, media_item.path)self.attempt_task_removal(media_item)self.manager.progress_manager.download_progress.add_completed()- log(f"Download finished: {media_item.url}", 20)+ logger.info(f"Download finished: {media_item.url}")"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""@@ -348,7 +348,7 @@ class Downloader:if self.manager.config_manager.settings_data.download_options.disable_file_timestamps:returnif not media_item.uploaded_at:- log(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime", 30)+ logger.warning(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime")return# TODO: Make this entire method async (run in another thread)@@ -431,14 +431,14 @@ class Downloader:return Falseif not media_item.is_segment:- log(f"{self.log_prefix} starting: {media_item.url}", 20)+ logger.info(f"{self.log_prefix} starting: {media_item.url}")async with self._file_lock_vault[media_item.filename]:- log_debug(f"Lock for {media_item.filename} acquired", 20)+ logger.debug(f"Lock for '{media_item.filename}' acquired")try:return bool(await self.download(media_item))finally:- log_debug(f"Lock for {media_item.filename} released", 20)+ logger.debug(f"Lock for '{media_item.filename}' released")@error_handling_wrapper@retry@@ -460,13 +460,13 @@ class Downloader:if not media_item.is_segment:await self.set_file_datetime(media_item, media_item.path)self.manager.progress_manager.download_progress.add_completed()- log(f"Download finished: {media_item.url}", 20)+ logger.info(f"Download finished: {media_item.url}")self.attempt_task_removal(media_item)return downloadedexcept SkipDownloadError as e:if not media_item.is_segment:- log(f"Download skip {media_item.url}: {e}", 10)+ logger.info(f"Download skipped {media_item.url}: {e}")self.manager.progress_manager.download_progress.add_skipped()self.attempt_task_removal(media_item)@@ -499,8 +499,10 @@ class Downloader:exc_info: Exception | None = None,) -> None:self.attempt_task_removal(media_item)- full_message = f"{self.log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}"- log(full_message, 40, exc_info=exc_info)+ logger.error(+ f"{self.log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}",+ exc_info=exc_info,+ )self.manager.log_manager.write_download_error_log(media_item, error_log_msg.csv_log_msg)self.manager.progress_manager.download_stats_progress.add_failure(error_log_msg.ui_failure)self.manager.progress_manager.download_progress.add_failed()diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex 2eeed1476..ee54532bb 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -1,5 +1,4 @@import os-from hashlib import sha256os.environ["PYDANTIC_ERRORS_INCLUDE_URL"] = "0"RUNNING_IN_IDE = bool(os.getenv("PYCHARM_HOSTED") or os.getenv("TERM_PROGRAM") == "vscode")@@ -7,9 +6,7 @@ RUNNING_IN_TERMUX = bool(os.getenv("TERMUX_VERSION") or os.getenv("TERMUX_MAIN_PACKAGE_FORMAT") or "com.termux" in os.getenv("$PREFIX", ""))PORTRAIT_MODE = bool(RUNNING_IN_TERMUX or os.getenv("CDL_PORTRAIT_MODE"))-ENABLE_DEBUG_CRAWLERS = os.getenv("CDL_ENABLE_DEBUG_CRAWLERS")-if ENABLE_DEBUG_CRAWLERS:- ENABLE_DEBUG_CRAWLERS = sha256(ENABLE_DEBUG_CRAWLERS.encode("utf-8")).hexdigest()+DEBUG_LOG_FOLDER = os.getenv("CDL_DEBUG_LOG_FOLDER")PROFILING = os.getenv("CDL_PROFILING")diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 8cc0937e0..ddb2678e7 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -2,13 +2,14 @@ from __future__ import annotationsimport asyncioimport contextlib+import loggingimport sslfrom base64 import b64encodefrom collections import defaultdictfrom collections.abc import Generatorfrom contextvars import ContextVarfrom http import HTTPStatus-from typing import TYPE_CHECKING, Any, Literal, Self+from typing import TYPE_CHECKING, Any, Selfimport aiohttpimport certifi@@ -27,11 +28,6 @@ from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError,from cyberdrop_dl.ffmpeg import probefrom cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsersfrom cyberdrop_dl.utils.cookie_management import read_netscape_files-from cyberdrop_dl.utils.logger import log, log_debug, log_spacer--_VALID_EXTENSIONS = (- constants.FILE_FORMATS["Images"] | constants.FILE_FORMATS["Videos"] | constants.FILE_FORMATS["Audio"]-)if TYPE_CHECKING:from collections.abc import Callable, Generator, Iterable, Mapping@@ -48,6 +44,11 @@ try:except ImportError as e:_curl_import_error = e+logger = logging.getLogger(__name__)+_VALID_EXTENSIONS = (+ constants.FILE_FORMATS["Images"] | constants.FILE_FORMATS["Videos"] | constants.FILE_FORMATS["Audio"]+)+_DOWNLOAD_ERROR_ETAGS = {"d835884373f4d6c8f24742ceabe74946": "Imgur image has been removed","65b7753c-528a": "SC Scrape Image",@@ -159,8 +160,8 @@ class ClientManager:return self._flaresolverrdef _startup(self) -> None:- self._session = self.new_scrape_session()- self._download_session = self.new_download_session()+ self._session = self.create_aiohttp_session()+ self._download_session = self.create_aiohttp_session()if _curl_import_error is not None:return@@ -275,24 +276,16 @@ class ClientManager:cookies={cookie.key: cookie.value for cookie in self.cookies},)- def new_scrape_session(self) -> ClientSession:- trace_configs = _create_request_log_hooks("scrape")- return self._new_session(cached=True, trace_configs=trace_configs)-- def new_download_session(self) -> ClientSession:- trace_configs = _create_request_log_hooks("download")- return self._new_session(cached=False, trace_configs=trace_configs)-- def _new_session(- self, cached: bool = False, trace_configs: list[aiohttp.TraceConfig] | None = None+ def create_aiohttp_session(+ self,) -> ClientSession:- timeout = self.rate_limiting_options._aiohttp_timeoutreturn ClientSession(- headers={"user-agent": self.manager.global_config.general.user_agent},+ headers={+ "user-agent": self.manager.global_config.general.user_agent,+ },raise_for_status=False,cookie_jar=self.cookies,- timeout=timeout,- trace_configs=trace_configs,+ timeout=self.rate_limiting_options._aiohttp_timeout,proxy=self.manager.global_config.general.proxy,connector=self._new_tcp_connector(),requote_redirect_url=False,@@ -311,7 +304,7 @@ class ClientManager:f"{crawler.__class__.__name__} has been disabled after too many errors. "f"URLs from the following domains will be ignored: {crawler.SCRAPE_MAPPER_KEYS}")- log(msg, 40)+ logger.error(msg)raise TooManyCrawlerErrors@contextlib.contextmanager@@ -341,8 +334,6 @@ class ClientManager:async for domain, cookie in read_netscape_files(cookie_files):self.cookies.update_cookies(cookie, response_url=AbsoluteHttpURL(f"https://{domain}"))- log_spacer(20, log_to_console=False)-def get_rate_limiter(self, domain: str) -> AsyncLimiter:"""Get a rate limiter for a domain."""if domain in self.rate_limits:@@ -466,7 +457,7 @@ async def _set_dns_resolver(loop: asyncio.AbstractEventLoop | None = None) -> Noconstants.DNS_RESOLVER = aiohttp.AsyncResolverexcept Exception as e:constants.DNS_RESOLVER = aiohttp.ThreadedResolver- log(f"Unable to setup asynchronous DNS resolver. Falling back to thread based resolver: {e}", 30)+ logger.warning(f"Unable to setup asynchronous DNS resolver. Falling back to thread based resolver: {e}")async def _test_async_resolver(loop: asyncio.AbstractEventLoop | None = None) -> None:@@ -478,20 +469,3 @@ async def _test_async_resolver(loop: asyncio.AbstractEventLoop | None = None) ->async with aiodns.DNSResolver(loop=loop, timeout=5.0) as resolver:_ = await resolver.query_dns("github.com", "A")---def _create_request_log_hooks(client_type: Literal["scrape", "download"]) -> list[aiohttp.TraceConfig]:- async def on_request_start(*args) -> None:- params: aiohttp.TraceRequestStartParams = args[2]- log_debug(f"Starting {client_type} {params.method} request to {params.url}", 10)-- async def on_request_end(*args) -> None:- params: aiohttp.TraceRequestEndParams = args[2]- msg = f"Finishing {client_type} {params.method} request to {params.url}"- msg += f" -> response status: {params.response.status}"- log_debug(msg, 10)-- trace_config = aiohttp.TraceConfig()- trace_config.on_request_start.append(on_request_start)- trace_config.on_request_end.append(on_request_end)- return [trace_config]diff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.pyindex 80e06f5c7..248607564 100644--- a/cyberdrop_dl/managers/log_manager.py+++ b/cyberdrop_dl/managers/log_manager.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport csv+import loggingfrom collections import defaultdictfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any@@ -9,7 +10,7 @@ from typing import TYPE_CHECKING, Anyfrom cyberdrop_dl.constants import CSV_DELIMITERfrom cyberdrop_dl.exceptions import get_originfrom cyberdrop_dl.utils import json-from cyberdrop_dl.utils.logger import log, log_spacer+from cyberdrop_dl.utils.logger import log_spacerif TYPE_CHECKING:from collections.abc import Iterable@@ -20,6 +21,9 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)++class LogManager:def __init__(self, manager: Manager) -> None:self.manager = manager@@ -99,7 +103,7 @@ class LogManager:def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:log_spacer(20)- log("Updating Last Forum Posts...\n", 20)+ logger.info("Updating Last Forum Posts...\n")current_urls, current_base_urls, new_urls, new_base_urls = [], [], [], []try:@@ -115,13 +119,13 @@ def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:current_urls.append(url)current_base_urls.append(base_url)except UnicodeDecodeError:- log("Unable to read input file, skipping update_last_forum_post", 40)+ logger.exception("Unable to read input file, skipping update_last_forum_post")returnwith last_post_log.open(encoding="utf8") as f:reader = csv.DictReader(f.readlines())for row in reader:- new_url = base_url = row.get("url").strip().removesuffix("/") # type: ignore+ new_url = base_url = row["url"].strip().removesuffix("/") # type: ignoreif "https" in new_url and "/post-" in new_url:base_url = new_url.rsplit("/post", 1)[0]@@ -138,11 +142,11 @@ def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:old_url = current_urls[index]if old_url == new_url:continue- log(f"Updating {base}\n {old_url = }\n {new_url = }", 20)+ logger.info(f"Updating {base}\n {old_url = }\n {new_url = }")updated_urls[index] = new_urlif updated_urls == current_urls:- log("No URLs updated", 20)+ logger.info("No URLs updated")returnwith input_file.open("w", encoding="utf8") as f:diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex c82ff0b8c..ffb61cd8e 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport json+import loggingfrom dataclasses import Field, fieldfrom time import perf_counterfrom typing import TYPE_CHECKING, Any, NamedTuple, TypeVar@@ -20,7 +21,7 @@ from cyberdrop_dl.managers.live_manager import LiveManagerfrom cyberdrop_dl.managers.log_manager import LogManagerfrom cyberdrop_dl.managers.path_manager import PathManagerfrom cyberdrop_dl.managers.progress_manager import ProgressManager-from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log+from cyberdrop_dl.utils.logger import LogHandler, QueuedLoggerfrom cyberdrop_dl.utils.utilities import close_if_defined, get_system_informationif TYPE_CHECKING:@@ -35,6 +36,9 @@ class AsyncioEvents(NamedTuple):RUNNING: asyncio.Event+logger = logging.getLogger(__name__)++class Manager:def __init__(self, args: Sequence[str] | None = None) -> None:if isinstance(args, str):@@ -180,7 +184,7 @@ class Manager:f"Using ffmpeg version: {ffmpeg.get_ffmpeg_version()}",f"Using ffprobe version: {ffmpeg.get_ffprobe_version()}",)- log("\n".join(args_info))+ logger.info("\n".join(args_info))async def async_db_close(self) -> None:"Partial shutdown for managers used for hash directory scanner"diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex df99171a6..1ca7bd486 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -1,5 +1,6 @@from __future__ import annotations+import loggingimport timefrom contextlib import asynccontextmanagerfrom dataclasses import field@@ -22,7 +23,7 @@ from cyberdrop_dl.ui.progress.hash_progress import HashProgressfrom cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgressfrom cyberdrop_dl.ui.progress.sort_progress import SortProgressfrom cyberdrop_dl.ui.progress.statistic_progress import DownloadStatsProgress, ScrapeStatsProgress-from cyberdrop_dl.utils.logger import log, log_spacer, log_with_color+from cyberdrop_dl.utils.logger import log_spacer, log_with_colorif TYPE_CHECKING:from collections.abc import AsyncGenerator@@ -33,6 +34,9 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.ui.progress.statistic_progress import UiFailureTotal++logger = logging.getLogger(__name__)+log_cyan = partial(log_with_color, style="cyan", level=20)log_yellow = partial(log_with_color, style="yellow", level=20)log_green = partial(log_with_color, style="green", level=20)@@ -144,7 +148,7 @@ class ProgressManager:total_data_written = ByteSize(self.file_progress.total_data_written).human_readable(decimal=True)log_spacer(20)- log("Printing Stats...\n", 20)+ logger.info("Printing Stats...\n")config_path = self.manager.path_manager.config_folder / self.manager.config_manager.loaded_configconfig_path_text = get_console_hyperlink(config_path, text=self.manager.config_manager.loaded_config)input_file_text = get_input(self.manager)diff --git a/cyberdrop_dl/plugins.py b/cyberdrop_dl/plugins.pyindex 0284045e8..96f7d47a5 100644--- a/cyberdrop_dl/plugins.py+++ b/cyberdrop_dl/plugins.py@@ -15,11 +15,11 @@ See: https://packaging.python.org/en/latest/guides/creating-and-discovering-plugfrom __future__ import annotationsimport dataclasses+import loggingfrom importlib.metadata import entry_pointsfrom typing import TYPE_CHECKING, Finalfrom cyberdrop_dl import env-from cyberdrop_dl.utils.logger import logif TYPE_CHECKING:from collections.abc import Iterable@@ -27,6 +27,7 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)_GROUP_NAME: Final = "cyberdrop_dl_plugins"@@ -51,8 +52,8 @@ def _get_plugins() -> Iterable[Plugin]:def load(manager: Manager) -> None:for plugin in _get_plugins():if env.NO_PLUGINS:- log(f"Found plugins installed but plugins are disabled. Ignored: {tuple(_get_plugins())}", 40)+ logger.error(f"Found plugins installed but plugins are disabled. Ignored: {tuple(_get_plugins())}")returnplugin.entrypoint.load()(manager)- log(f"Loaded {plugin}", 20)+ logger.info(f"Loaded {plugin}")diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.pyindex d3c9fd94d..c3ad2e509 100644--- a/cyberdrop_dl/scraper/jdownloader.py+++ b/cyberdrop_dl/scraper/jdownloader.py@@ -1,12 +1,14 @@from __future__ import annotationsimport dataclasses+import loggingfrom typing import TYPE_CHECKINGfrom myjdapi import myjdapifrom cyberdrop_dl.exceptions import JDownloaderError-from cyberdrop_dl.utils.logger import log++logger = logging.getLogger(__name__)if TYPE_CHECKING:from pathlib import Path@@ -68,7 +70,7 @@ class JDownloader:except myjdapi.MYJDApiException as e:msg = e- log(f"Failed to connect to jDownloader: {msg}", 40)+ logger.error(f"Failed to connect to jDownloader: {msg}")self.enabled = Falsedef direct_unsupported_to_jdownloader(diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex a2165ceae..a495bd5bb 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import loggingimport refrom datetime import date, datetimefrom pathlib import Path@@ -21,7 +22,7 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionErrorfrom cyberdrop_dl.scraper.filters import is_in_domain_list, is_outside_date_range, is_valid_urlfrom cyberdrop_dl.scraper.jdownloader import JDownloader-from cyberdrop_dl.utils.logger import log, log_spacer+from cyberdrop_dl.utils.logger import log_spacerfrom cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slashif TYPE_CHECKING:@@ -33,6 +34,7 @@ if TYPE_CHECKING:from cyberdrop_dl.crawlers import Crawlerfrom cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)existing_crawlers: dict[str, Crawler] = {}_seen_urls: set[AbsoluteHttpURL] = set()_crawlers_disabled_at_runtime: set[str] = set()@@ -129,7 +131,7 @@ class ScrapeMapper:self.count += 1if not self.count:- log("No valid links found.", 30)+ logger.warning("No valid links found")"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""@@ -222,7 +224,7 @@ class ScrapeMapper:returnif not self.real_debrid.disabled and self.real_debrid.api.is_supported(scrape_item.url):- log(f"Using RealDebrid for unsupported URL: {scrape_item.url}", 10)+ logger.info(f"Using RealDebrid for unsupported URL: {scrape_item.url}")self.manager.task_group.create_task(self.real_debrid.run(scrape_item))return@@ -234,7 +236,7 @@ class ScrapeMapper:passif self.jdownloader.enabled and jdownloader_whitelisted:- log(f"Sending unsupported URL to JDownloader: {scrape_item.url}", 20)+ logger.info(f"Sending unsupported URL to JDownloader: {scrape_item.url}")success = Falsetry:download_folder = get_download_path(self.manager, scrape_item, "jdownloader")@@ -246,7 +248,7 @@ class ScrapeMapper:)success = Trueexcept JDownloaderError as e:- log(f"Failed to send {scrape_item.url} to JDownloader\n{e.message}", 40)+ logger.error(f"Failed to send {scrape_item.url} to JDownloader\n{e.message}")self.manager.log_manager.write_unsupported_urls_log(scrape_item.url,scrape_item.parents[0] if scrape_item.parents else None,@@ -254,7 +256,7 @@ class ScrapeMapper:self.manager.progress_manager.scrape_stats_progress.add_unsupported(sent_to_jdownloader=success)return- log(f"Unsupported URL: {scrape_item.url}", 30)+ logger.warning(f"Unsupported URL: {scrape_item.url}")self.manager.log_manager.write_unsupported_urls_log(scrape_item.url,scrape_item.parents[0] if scrape_item.parents else None,@@ -274,23 +276,23 @@ class ScrapeMapper:is_in_domain_list(scrape_item, BlockedDomains.partial_match)or scrape_item.url.host in BlockedDomains.exact_match):- log(f"Skipping {scrape_item.url} as it is a blocked domain", 10)+ logger.info(f"Skipping {scrape_item.url} as it is a blocked domain")return Falsebefore = self.manager.parsed_args.cli_only_args.completed_beforeafter = self.manager.parsed_args.cli_only_args.completed_afterif is_outside_date_range(scrape_item, before, after):- log(f"Skipping {scrape_item.url} as it is outside of the desired date range", 10)+ logger.info(f"Skipping {scrape_item.url} as it is outside of the desired date range")return Falseskip_hosts = self.manager.config_manager.settings_data.ignore_options.skip_hostsif skip_hosts and is_in_domain_list(scrape_item, skip_hosts):- log(f"Skipping URL by skip_hosts config: {scrape_item.url}", 10)+ logger.info(f"Skipping URL by skip_hosts config: {scrape_item.url}")return Falseonly_hosts = self.manager.config_manager.settings_data.ignore_options.only_hostsif only_hosts and not is_in_domain_list(scrape_item, only_hosts):- log(f"Skipping URL by only_hosts config: {scrape_item.url}", 10)+ logger.info(f"Skipping URL by only_hosts config: {scrape_item.url}")return Falsereturn True@@ -334,7 +336,7 @@ def regex_links(line: str) -> Generator[AbsoluteHttpURL]:encoded = "%" in linkyield AbsoluteHttpURL(link, encoded=encoded)except Exception as e:- log(f"Unable to parse URL from input file: {link} {e:!r}", 40)+ logger.error(f"Unable to parse URL from input file: {link} {e:!r}")def _create_item_from_row(row: aiosqlite.Row) -> ScrapeItem:@@ -391,10 +393,10 @@ def register_crawler()if from_user == "raise":raise ValueError(msg)- log(msg, 40)+ logger.error(msg)continueelse:- log(f"Successfully mapped {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}")+ logger.info(f"Successfully mapped {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}")elif other:msg = f"{domain} from {crawler.NAME} already registered by {other}"@@ -436,7 +438,7 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_tf"{len(crawlers_to_disable)} Crawler names where provided to disable"f", but only {len(disabled_crawlers)} {'is' if len(disabled_crawlers) == 1 else 'are'} a valid crawler's name.")- log(msg, 30)+ logger.warning(msg)if disabled_crawlers:existing_crawlers.clear()@@ -444,7 +446,7 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_tcrawlers_info = "\n".join(str({info.site: info.supported_domains}) for info in sorted(crawlers.INFO for crawlers in disabled_crawlers))- log(f"Crawlers disabled by config: \n{crawlers_info}")+ logger.info(f"Crawlers disabled by config: \n{crawlers_info}")log_spacer(10)diff --git a/cyberdrop_dl/signature.py b/cyberdrop_dl/signature.pyindex 5c4ecbab5..47c643707 100644--- a/cyberdrop_dl/signature.py+++ b/cyberdrop_dl/signature.py@@ -30,11 +30,3 @@ else:return yreturn call---def easy_repr(*fields: str) -> Callable[..., str]:- def _repr_(self: object) -> str:- params = ", ".join(f"{f}={getattr(self, f)!r}" for f in fields)- return f"{type(self).__name__}({params})"-- return _repr_diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.pyindex 3418c505e..5f77257e9 100644--- a/cyberdrop_dl/utils/apprise.py+++ b/cyberdrop_dl/utils/apprise.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport json+import loggingimport shutilimport tempfilefrom dataclasses import dataclass@@ -16,11 +17,13 @@ from rich.text import Textfrom cyberdrop_dl import constantsfrom cyberdrop_dl.dependencies import apprisefrom cyberdrop_dl.models import AppriseURLModel-from cyberdrop_dl.utils.logger import log, log_debug, log_spacer+from cyberdrop_dl.utils.logger import log_spacerif TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)+@dataclassclass AppriseURL:@@ -80,7 +83,7 @@ def get_apprise_urls(*, file: Path | None = None, urls: list[str] | None = None)if not urls:return []if apprise is None:- log("Found apprise URLs for notifications but apprise is not installed. Ignoring", 30)+ logger.warning("Found apprise URLs for notifications but apprise is not installed. Ignoring")return []return _simplify_urls([AppriseURLModel.model_validate({"url": url}) for url in set(urls)])@@ -128,14 +131,14 @@ def _process_results(log_spacer(10, log_to_console=False, log_to_file=not all(result))rich.print("Apprise notifications results:", final_result.value)- logger = log_debug if all(result) else log- logger(f"Apprise notifications results: {final_result.value}")- logger(f"PARSED_APPRISE_URLs: \n{json.dumps(all_urls, indent=4)}\n")- logger(f"RESULTS_BY_TAGS: \n{json.dumps(result_dict, indent=4)}")+ log = logger.debug if all(result) else logger.info+ log(f"Apprise notifications results: {final_result.value}")+ log(f"PARSED_APPRISE_URLs: \n{json.dumps(all_urls, indent=4)}\n")+ log(f"RESULTS_BY_TAGS: \n{json.dumps(result_dict, indent=4)}")log_spacer(10, log_to_console=False, log_to_file=not all(result))parsed_log_lines = _parse_apprise_logs(apprise_logs)for line in parsed_log_lines:- logger(level=line.level.value, message=line.msg)+ log(line.msg)return final_result, parsed_log_lines@@ -208,7 +211,7 @@ async def send_apprise_notifications(manager: Manager) -> tuple[constants.Notifishutil.copy(main_log, temp_main_log)notifications_to_send["attach_logs"]["attach"] = str(temp_main_log.resolve())except OSError:- log(attach_file_failed_msg, 40)+ logger.exception(attach_file_failed_msg)async def notify(tag: str, msg: dict[str, Any]) -> tuple[str, bool | None]:result = await apprise_obj.async_notify(**msg, tag=tag)diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pyindex e1c936fd2..c3556dc0e 100644--- a/cyberdrop_dl/utils/cookie_management.py+++ b/cyberdrop_dl/utils/cookie_management.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import loggingimport osimport timefrom functools import wraps@@ -10,7 +11,6 @@ from textwrap import dedentfrom typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVarfrom cyberdrop_dl.dependencies import browser_cookie3-from cyberdrop_dl.utils.logger import logif TYPE_CHECKING:from collections.abc import AsyncIterable, Callable@@ -19,7 +19,7 @@ if TYPE_CHECKING:from cyberdrop_dl.constants import BROWSERSfrom cyberdrop_dl.managers.manager import Manager-+logger = logging.getLogger(__name__)P = ParamSpec("P")R = TypeVar("R")@@ -136,7 +136,7 @@ def extract_cookies(extractor_name: str) -> CookieJar:async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[str, SimpleCookie]]:- now = time.time()+ now = int(time.time())domains_seen = set()cookie_jars = await asyncio.gather(*(_read_netscape_file(file) for file in cookie_files))for file, cookie_jar in zip(cookie_files, cookie_jars, strict=True):@@ -149,14 +149,16 @@ async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[scontinuesimplified_domain = cookie.domain.removeprefix(".")if simplified_domain not in current_cookie_file_domains:- log(f"Found cookies for {simplified_domain} in file '{file.name}'", 20)+ logger.info(f"Found cookies for {simplified_domain} in file '{file.name}'")current_cookie_file_domains.add(simplified_domain)if simplified_domain in domains_seen:- log(f"Previous cookies for domain {simplified_domain} detected. They will be overwritten", 30)+ logger.warning(+ f"Previous cookies for domain {simplified_domain} detected. They will be overwritten"+ )if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignoreexpired_cookies_domains.add(simplified_domain)- log(f"Cookies for {simplified_domain} are expired", 30)+ logger.warning(f"Cookies for {simplified_domain} are expired")domains_seen.add(simplified_domain)simple_cookie = make_simple_cookie(cookie, now)@@ -170,7 +172,7 @@ async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:cookie_jar.load(ignore_discard=True)return cookie_jarexcept OSError as e:- log(f"Unable to load cookies from '{file.name}':\n {e!s}", 40)+ logger.error(f"Unable to load cookies from '{file.name}':\n {e!s}")return await asyncio.to_thread(read)diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.pyindex 662f2f213..7886fa0c9 100644--- a/cyberdrop_dl/utils/logger.py+++ b/cyberdrop_dl/utils/logger.py@@ -80,7 +80,7 @@ class LogHandler(RichHandler):"""Rich Handler with default settings, automatic console creation and custom log render to remove padding in files."""def __init__(- self, level: int = 10, file: IO[str] | None = None, width: int | None = None, debug: bool = False, **kwargs+ self, level: int = 10, file: IO[str] | None = None, width: int | None = None, debug: bool = False) -> None:is_file: bool = file is not Noneredacted: bool = is_file and not debug@@ -89,11 +89,23 @@ class LogHandler(RichHandler):console = _DEFAULT_CONSOLEelse:console = console_cls(file=file, width=width)- options = constants.RICH_HANDLER_DEBUG_CONFIG if debug else constants.RICH_HANDLER_CONFIG- options = options | kwargs- super().__init__(level, console, show_time=is_file, **options)++ super().__init__(+ level,+ console,+ show_time=False,+ show_path=False,+ tracebacks_show_locals=debug,+ locals_max_string=constants.DEFAULT_CONSOLE_WIDTH,+ tracebacks_extra_lines=2,+ locals_max_length=20,+ )if is_file:- self._log_render = NoPaddingLogRender(show_level=True)+ self._log_render = NoPaddingLogRender(+ show_path=False,+ show_level=True,+ time_format=lambda dt: Text(f"[{dt.isoformat(sep=' ', timespec='milliseconds')}]", style="log.time"),+ )class BareQueueHandler(QueueHandler):diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex 60b350b9f..79e6342a5 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport itertools+import loggingfrom datetime import datetimefrom pathlib import Pathfrom subprocess import CalledProcessError@@ -13,13 +14,16 @@ from cyberdrop_dl import constantsfrom cyberdrop_dl.constants import FILE_FORMATSfrom cyberdrop_dl.ffmpeg import probefrom cyberdrop_dl.utils import strings-from cyberdrop_dl.utils.logger import log, log_with_color+from cyberdrop_dl.utils.logger import log_with_colorfrom cyberdrop_dl.utils.utilities import purge_dir_treeif TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+logger = logging.getLogger(__name__)++async def get_modified_date(file: Path) -> datetime:stat = await asyncio.to_thread(file.stat)return datetime.fromtimestamp(stat.st_mtime).replace(microsecond=0)@@ -128,8 +132,8 @@ class Sorter:bitrate = audio.bitratesample_rate = audio.sample_rate- except (RuntimeError, CalledProcessError, OSError) as e:- log(f"Unable to get audio properties of '{file}'", 40, exc_info=e)+ except (RuntimeError, CalledProcessError, OSError):+ logger.exception(f"Unable to get audio properties of '{file}'")if await self._process_file_move(file,@@ -156,7 +160,7 @@ class Sorter:width = height = resolution = Noneexcept (OSError, ValueError):- log(f"Unable to get some image properties of '{file}'")+ logger.exception(f"Unable to get some image properties of '{file}'")if await self._process_file_move(file,@@ -185,8 +189,8 @@ class Sorter:duration = video.duration or probe_output.format.durationframerate = video.fps- except (RuntimeError, CalledProcessError, OSError) as e:- log(f"Unable to get some video properties of '{file}'", 40, exc_info=e)+ except (RuntimeError, CalledProcessError, OSError):+ logger.exception(f"Unable to get some video properties of '{file}'")if await self._process_file_move(file,diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex d0ec91bc3..c766a0341 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -5,6 +5,7 @@ import dataclassesimport functoolsimport inspectimport itertools+import loggingimport mimetypesimport osimport platform@@ -48,7 +49,7 @@ from cyberdrop_dl.exceptions import (get_origin,)from cyberdrop_dl.utils import json-from cyberdrop_dl.utils.logger import log, log_with_color+from cyberdrop_dl.utils.logger import log_with_colorif TYPE_CHECKING:from collections.abc import Callable, Coroutine, Generator, Iterable, Mapping@@ -69,6 +70,7 @@ if TYPE_CHECKING:__dataclass_fields__: ClassVar[dict]+logger = logging.getLogger(__name__)_ALLOWED_FILEPATH_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"_BLOB_OR_SVG = ("data:", "blob:", "javascript:")@@ -137,7 +139,7 @@ def error_handling_context(self: Crawler | Downloader, item: ScrapeItem | MediaIself.write_download_error(item, error_log_msg, exc_info)return- log(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", 40, exc_info=exc_info)+ logger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)self.manager.log_manager.write_scrape_error_log(link_to_show, error_log_msg.csv_log_msg, origin)self.manager.progress_manager.scrape_stats_progress.add_failure(error_log_msg.ui_failure)@@ -288,7 +290,7 @@ def clear_term():os.system("cls" if os.name == "nt" else "clear")-def get_size(path: os.DirEntry) -> int | None:+def get_size(path: os.DirEntry[str]) -> int | None:try:return path.stat(follow_symlinks=False).st_sizeexcept (OSError, ValueError):@@ -573,9 +575,7 @@ def xor_decrypt(encrypted_data: bytes, key: bytes) -> str:return data.decode("utf-8", errors="ignore")-log_cyan = partial(log_with_color, style="cyan", level=20)log_yellow = partial(log_with_color, style="yellow", level=20)-log_green = partial(log_with_color, style="green", level=20)log_red = partial(log_with_color, style="red", level=20)diff --git a/cyberdrop_dl/utils/webhook.py b/cyberdrop_dl/utils/webhook.pyindex ed3b001a7..f29846cc9 100644--- a/cyberdrop_dl/utils/webhook.py+++ b/cyberdrop_dl/utils/webhook.py@@ -1,6 +1,7 @@from __future__ import annotationsimport json+import loggingfrom typing import TYPE_CHECKING, Any, castimport aiofiles@@ -8,7 +9,7 @@ import richfrom aiohttp import FormDatafrom cyberdrop_dl import aio, constants-from cyberdrop_dl.utils.logger import log, log_debug, log_spacer+from cyberdrop_dl.utils.logger import log_spacerif TYPE_CHECKING:from pathlib import Path@@ -17,7 +18,7 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.models.base_models import HttpAppriseURL-+logger = logging.getLogger(__name__)_DEFAULT_DIFF_LINE_FORMAT: str = "{}"_STYLE_TO_DIFF_MAP = {"green": "+ {}",@@ -71,15 +72,14 @@ async def send_webhook_message(manager: Manager) -> None:rich.print("\nSending Webhook Notifications.. ")url = cast("AbsoluteHttpURL", webhook.url.get_secret_value())form = await _prepare_form(webhook, manager.path_manager.main_log)- logger = log+result = constants.NotificationResult.FAILED.valuetry:- async with manager.client_manager._new_session() as session, session.post(url, data=form) as response:+ async with manager.client_manager.create_aiohttp_session() as session, session.post(url, data=form) as response:if response.ok:result = constants.NotificationResult.SUCCESS.valueresult_to_log = [result]- logger = log_debugelse:json_resp: dict[str, Any] = await response.json()@@ -88,10 +88,10 @@ async def send_webhook_message(manager: Manager) -> None:result_to_log = constants.NotificationResult.FAILED.value, resp_textexcept Exception as e:- logger("Unable to send webhook notification", 40, exc_info=e)+ logger.exception("Unable to send webhook notification")result_to_log = result, str(e)log_spacer(10, log_to_console=False)rich.print("Webhook Notifications Results:", result)result_to_log = "\n".join(map(str, result_to_log))- logger(f"Webhook Notifications Results: {result_to_log}")+ logger.info(f"Webhook Notifications Results: {result_to_log}")