Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: switch to std logging (#68)

NTFSvolume4ce75c50accd
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 64abe409c..2ceb82aa4 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import contextlib
import itertools
+import logging
import time
from collections.abc import Generator
from http import HTTPStatus
@@ -16,7 +17,6 @@ from cyberdrop_dl.constants import FILE_FORMATS
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadError
from cyberdrop_dl.utils import dates
-from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
@@ -31,6 +31,9 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
+
+
_CONTENT_TYPES_OVERRIDES: dict[str, str] = {"text/vnd.trolltech.linguist": "video/MP2T"}
_SLOW_DOWNLOAD_PERIOD: int = 10 # seconds
_CHROME_ANDROID_USER_AGENT: str = (
@@ -142,7 +145,7 @@ class DownloadClient:
if not proceed:
if media_item.is_segment:
return True
- log(f"Skipping {media_item.url} as it has already been downloaded", 10)
+ logger.info(f"Skipping {media_item.url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed(False)
await self.process_completed(media_item, domain)
await self.handle_media_item_completion(media_item, downloaded=False)
@@ -157,8 +160,9 @@ class DownloadClient:
and not media_item.uploaded_at
and (last_modified := get_last_modified(resp.headers))
):
- msg = f"Unable to parse upload date for {media_item.url}, using `Last-Modified` header as file datetime"
- log(msg, 30)
+ logger.warning(
+ f"Unable to parse upload date for {media_item.url}, using `Last-Modified` header as file datetime"
+ )
media_item.uploaded_at = last_modified
task_id = media_item.task_id
@@ -229,12 +233,14 @@ class DownloadClient:
else:
fallback_count += 1
msg = f" with fallback URL #{fallback_count} {download_url} failed, retrying with new fallback URL: "
- log(f"Download of {media_item.url}{msg}{next_download_url}", 40)
+ logger.error(f"Download of {media_item.url}{msg}{next_download_url}")
download_url = next_download_url
continue
raise
- async def _append_content(self, media_item: MediaItem, content: aiohttp.StreamReader | AbstractResponse) -> None:
+ async def _append_content(
+ self, media_item: MediaItem, content: aiohttp.StreamReader | AbstractResponse[Any]
+ ) -> None:
"""Appends content to a file."""
assert media_item.task_id is not None
@@ -289,7 +295,7 @@ class DownloadClient:
async def download_file(self, domain: str, media_item: MediaItem) -> bool:
"""Starts a file."""
if self.manager.config.download_options.skip_download_mark_completed and not media_item.is_segment:
- log(f"Download Removed {media_item.url} due to mark completed option", 10)
+ logger.info(f"Download removed {media_item.url} due to mark completed option")
self.manager.progress_manager.download_progress.add_skipped()
# set completed path
await self.process_completed(media_item, domain)
@@ -304,7 +310,7 @@ class DownloadClient:
proceed = await self.client_manager.check_file_duration(media_item)
await self.manager.db_manager.history_table.add_duration(domain, media_item)
if not proceed:
- log(f"Download Skip {media_item.url} due to runtime restrictions", 10)
+ logger.info(f"Download skipped {media_item.url} due to runtime restrictions")
await asyncio.to_thread(media_item.path.unlink)
await self.mark_incomplete(media_item, domain)
self.manager.progress_manager.download_progress.add_skipped()
@@ -342,7 +348,7 @@ class DownloadClient:
await self.manager.hash_manager.hash_client.hash_item_during_download(media_item)
self.manager.path_manager.add_completed(media_item)
except Exception:
- log(f"Error handling media item completion of: {media_item.path}", 10, exc_info=True)
+ logger.exception(f"Error handling media item completion of: {media_item.path}")
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
@@ -372,14 +378,11 @@ class DownloadClient:
proceed = True
skip = False
- if not TYPE_CHECKING:
- log = log_debug if media_item.is_segment else globals()["log"]
-
while True:
if expected_size and not media_item.is_segment:
file_size_check = self.check_filesize_limits(media_item)
if not file_size_check:
- log(f"Download Skip {media_item.url} due to filesize restrictions", 10)
+ logger.info(f"Download skipped {media_item.url} due to filesize restrictions")
proceed = False
skip = True
return proceed, skip
@@ -388,7 +391,7 @@ class DownloadClient:
break
if media_item.path.exists() and media_item.path.stat().st_size == media_item.filesize:
- log(f"Found {media_item.path.name} locally, skipping download")
+ logger.info(f"Found {media_item.path.name} locally, skipping download")
proceed = False
break
@@ -405,18 +408,17 @@ class DownloadClient:
if media_item.filename == downloaded_filename:
if media_item.partial_file.exists():
- log(f"Found {downloaded_filename} locally, trying to resume")
+ logger.info(f"Found {downloaded_filename} locally, trying to resume")
assert media_item.filesize
size = media_item.partial_file.stat().st_size
- if size >= media_item.filesize != 0:
- log(f"Deleting partial file {media_item.partial_file}")
+ if size >= media_item.filesize:
+ logger.info(f"Deleting partial file {media_item.partial_file}. Size is out of bound")
media_item.partial_file.unlink()
elif size == media_item.filesize:
if media_item.path.exists():
- log(
- f"Found conflicting complete file '{media_item.path}' locally, iterating filename",
- 30,
+ logger.warning(
+ f"Found conflicting complete file '{media_item.path}' locally, iterating filename"
)
new_complete_filename, new_partial_file = await self.iterate_filename(
media_item.path,
@@ -430,17 +432,16 @@ class DownloadClient:
else:
proceed = False
media_item.partial_file.rename(media_item.path)
- log(
+ logger.info(
f"Renaming found partial file '{media_item.partial_file}' to complete file {media_item.path}"
)
elif media_item.path.exists():
if media_item.path.stat().st_size == media_item.filesize:
- log(f"Found complete file '{media_item.path}' locally, skipping download")
+ logger.info(f"Found complete file '{media_item.path}' locally, skipping download")
proceed = False
else:
- log(
- f"Found conflicting complete file '{media_item.path}' locally, iterating filename",
- 30,
+ logger.warning(
+ f"Found conflicting complete file '{media_item.path}' locally, iterating filename"
)
media_item.path, media_item.partial_file = await self.iterate_filename(
media_item.path,
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index 59bf912e2..5a8004c10 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import logging
from collections import defaultdict
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -10,7 +11,6 @@ from send2trash import send2trash
from cyberdrop_dl import constants
from cyberdrop_dl.constants import Hashing
from cyberdrop_dl.ui.prompts.basic_prompts import enter_to_continue
-from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
+
def hash_directory_scanner(manager: Manager, path: Path) -> None:
asyncio.run(_hash_directory_scanner_helper(manager, path))
@@ -87,7 +89,7 @@ class HashClient:
)
await self.save_hash_data(media_item, hash)
except Exception as e:
- log(f"After hash processing failed: '{media_item.path}' with error {e}", 40, exc_info=True)
+ logger.exception(f"After hash processing failed: '{media_item.path}' with error {e}")
async def update_db_and_retrive_hash(
self, file: Path | str, original_filename: str | None = None, referer: URL | None = None
@@ -138,7 +140,7 @@ class HashClient:
referer,
)
except Exception as e:
- log(f"Error hashing '{file}' : {e}", 40, exc_info=True)
+ logger.exception(f"Error hashing '{file}' : {e}")
else:
return hash
@@ -187,7 +189,7 @@ class HashClient:
try:
deleted = await _delete_file(file, self._to_trash)
except OSError as e:
- log(f"Unable to remove '{file}' ({hash_string}): {e}", 40)
+ logger.exception(f"Unable to remove '{file}' ({hash_string}): {e}")
else:
if not deleted:
return
@@ -196,7 +198,7 @@ class HashClient:
f"Removed new download '{file}' [{self._deleted_file_suffix}]. "
f"File hash matches with a previous download ({hash_string})"
)
- log(msg, 10)
+ logger.info(msg)
self.manager.progress_manager.hash_progress.add_removed_file()
finally:
@@ -216,9 +218,8 @@ class HashClient:
continue
try:
await self.hash_item(media_item)
- except Exception as e:
- msg = f"Unable to hash file = {media_item.path}: {e}"
- log(msg, 40)
+ except Exception:
+ logger.exception(msg=f"Unable to hash '{media_item.path}'")
return self.hashes_dict
diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.py
index fcab3091e..1770756c9 100644
--- a/cyberdrop_dl/clients/response.py
+++ b/cyberdrop_dl/clients/response.py
@@ -69,7 +69,10 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
return f"<{type(self).__name__} [{self.status}] ({self.url})>"
def __str__(self) -> str:
- return self.create_report()
+ me = self.__json__()
+ for key in ("content", "created_at"):
+ del me[key]
+ return json.dumps(me, indent=2, ensure_ascii=False)
def __json__(self) -> dict[str, Any]:
if content := self._text:
@@ -81,8 +84,9 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
return {
"url": str(self.url),
+ "location": str(self.location) if self.location else None,
"status_code": self.status,
- "created_at": self.created_at.isoformat(),
+ "created_at": str(self.created_at),
"response_headers": dict(self.headers),
"content": content,
}
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index 8941d368d..3f26afb73 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -4,8 +4,7 @@ import asyncio
import contextlib
import logging
import time
-from datetime import datetime
-from json import dumps as json_dumps
+import uuid
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
@@ -16,7 +15,7 @@ from cyberdrop_dl.utils.cookie_management import make_simple_cookie
from cyberdrop_dl.utils.utilities import sanitize_filename
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator
+ from collections.abc import AsyncGenerator, Mapping
from curl_cffi.requests.impersonate import BrowserTypeLiteral
from curl_cffi.requests.session import HttpMethod
@@ -34,9 +33,7 @@ class ScraperClient:
def __init__(self, client_manager: ClientManager) -> None:
self.client_manager = client_manager
self._save_pages_html = client_manager.manager.config_manager.settings_data.files.save_pages_html
- self._pages_folder = self.client_manager.manager.path_manager.pages_folder
- min_html_file_path_len = len(str(self._pages_folder)) + len(constants.STARTUP_TIME_STR) + 10
- self._max_html_stem_len = 245 - min_html_file_path_len
+ self._resp_folder = self.client_manager.manager.path_manager.pages_folder
@contextlib.asynccontextmanager
async def _limiter(self, domain: str) -> AsyncGenerator[None]:
@@ -77,10 +74,21 @@ class ScraperClient:
if method == "GET" and (data or json):
method = "POST"
- if not impersonate:
- headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)
+ impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate
+ if impersonate:
+ self.client_manager.check_curl_cffi_is_available()
+ if impersonate is True:
+ impersonate = "chrome"
+ request_params["impersonate"] = impersonate
- async with self.__request_context(url, method, request_params, impersonate) as resp:
+ else:
+ _ = headers.setdefault("user-agent", self.client_manager.manager.global_config.general.user_agent)
+ request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)
+
+ request_id = str(uuid.uuid4())
+ logger.debug("Starting {} request to {} [id={}]\n{}", method, url, request_id, request_params)
+
+ async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:
exc = None
try:
yield await self._check_response(resp, url)
@@ -88,7 +96,17 @@ class ScraperClient:
exc = e
raise
finally:
- await self.write_soup_to_disk(url, resp, exc)
+ logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)
+ if self._save_pages_html:
+ _ = self.client_manager.manager.task_group.create_task(
+ asyncio.to_thread(
+ _write_resp_to_disk,
+ self._resp_folder,
+ url,
+ resp,
+ exc,
+ )
+ )
def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
"""
@@ -104,28 +122,24 @@ class ScraperClient:
self.client_manager.cookies.update_cookies(simple_cookie, url)
@contextlib.asynccontextmanager
- async def __request_context(
+ async def __request(
self,
url: AbsoluteHttpURL,
method: HttpMethod,
- request_params: dict[str, Any],
- impersonate: BrowserTypeLiteral | bool | None,
+ request_params: Mapping[str, Any],
+ *,
+ impersonate: bool,
) -> AsyncGenerator[AbstractResponse[Any]]:
- impersonate = self.client_manager.manager.parsed_args.cli_only_args.impersonate or impersonate
+
if impersonate:
- self.client_manager.check_curl_cffi_is_available()
- if impersonate is True:
- impersonate = "chrome"
- request_params["impersonate"] = impersonate
- curl_resp = await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)
- try:
+ async with contextlib.aclosing(
+ await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)
+ ) as curl_resp:
yield AbstractResponse.create(curl_resp)
self.__sync_session_cookies(url)
- finally:
- await curl_resp.aclose()
+
return
- request_params.setdefault("max_redirects", constants.MAX_REDIRECTS)
async with (
self.client_manager._session.request(method, url, **request_params) as aio_resp,
):
@@ -148,63 +162,43 @@ class ScraperClient:
raise e from None
self.client_manager.cookies.update_cookies(solution.cookies)
- await self._check_flaresolverr_resp(solution)
+ await _check_flaresolverr_resp(self.client_manager.manager.global_config.general.user_agent, solution)
return AbstractResponse.create(solution)
- async def _check_flaresolverr_resp(self, solution: flaresolverr.Solution) -> None:
- cdl_user_agent = self.client_manager.manager.global_config.general.user_agent
- mismatch_ua_msg = (
- "Config user_agent and flaresolverr user_agent do not match:"
- f"\n Cyberdrop-DL: '{cdl_user_agent}'"
- f"\n Flaresolverr: '{solution.user_agent}'"
- )
- try:
- await ddos_guard.check(solution.content)
- except DDOSGuardError:
- if solution.user_agent != cdl_user_agent:
- raise DDOSGuardError(mismatch_ua_msg) from None
+async def _check_flaresolverr_resp(cdl_user_agent: str, solution: flaresolverr.Solution) -> None:
+ mismatch_ua_msg = (
+ "Config user_agent and flaresolverr user_agent do not match:"
+ f"\n Cyberdrop-DL: '{cdl_user_agent}'"
+ f"\n Flaresolverr: '{solution.user_agent}'"
+ )
+ try:
+ await ddos_guard.check(solution.content)
+ except DDOSGuardError:
if solution.user_agent != cdl_user_agent:
- logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")
+ raise DDOSGuardError(mismatch_ua_msg) from None
- async def write_soup_to_disk(
- self, url: AbsoluteHttpURL, response: AbstractResponse[Any], exc: Exception | None = None
- ):
- """Writes html to a file."""
+ if solution.user_agent != cdl_user_agent:
+ logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")
- if not (
- self._save_pages_html
- and "html" in response.content_type
- and response.consumed # Do not consume the response if the crawler didn't
- ):
- return
- content = cast("str", (await response.soup()).prettify(formatter="html"))
- now = datetime.now()
- log_date = now.strftime(constants.LOGS_DATETIME_FORMAT)
- url_str = str(url)
- response_url_str = str(response.url)
- clean_url = sanitize_filename(Path(url_str).as_posix().replace("/", "-"))
- filename = f"{clean_url[: self._max_html_stem_len]}_{log_date}.html"
- file_path = self._pages_folder / filename
- info = {
- "url": url_str,
- "response_url": response_url_str,
- "status_code": response.status,
- "datetime": now.isoformat(),
- "response_headers": dict(response.headers),
- }
- if exc:
- info |= {"error": str(exc), "exception": repr(exc)}
-
- json_data = json_dumps(info, indent=4, ensure_ascii=False)
- text = f"<!-- cyberdrop-dl scraping result\n{json_data}\n-->\n{content}"
- self.client_manager.manager.task_group.create_task(try_write(file_path, text))
-
-
-async def try_write(file: Path, content: str) -> None:
+def _write_resp_to_disk(
+ folder: Path,
+ url: AbsoluteHttpURL,
+ response: AbstractResponse[Any],
+ exc: Exception | None = None,
+) -> None:
+
+ max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10
+
+ log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)
+ url_str = str(url)
+ clean_url = sanitize_filename(Path(url_str).as_posix().replace("/", "-"))
+ filename = f"{clean_url[:max_stem_len]}_{log_date}.html"
+ file = folder / filename
+ content = response.create_report(exc)
try:
- await asyncio.to_thread(file.write_text, content, "utf8")
+ _ = file.write_text(content, "utf8")
except OSError:
pass
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 15c458002..a67e05c07 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -5,6 +5,7 @@ import contextlib
import datetime
import importlib
import inspect
+import logging
import pkgutil
import re
import weakref
@@ -54,24 +55,24 @@ from cyberdrop_dl.utils.utilities import (
truncate_str,
)
-P = ParamSpec("P")
-R = TypeVar("R")
-T = TypeVar("T")
-_T_co = TypeVar("_T_co", covariant=True)
-
-
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable
from http.cookies import BaseCookie
from bs4 import BeautifulSoup, Tag
+ from curl_cffi.requests.impersonate import BrowserTypeLiteral
from rich.progress import TaskID
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
-OneOrTuple: TypeAlias = T | tuple[T, ...]
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+_T = TypeVar("_T")
+_T_co = TypeVar("_T_co", covariant=True)
+OneOrTuple: TypeAlias = _T | tuple[_T, ...]
SupportedPaths: TypeAlias = dict[str, OneOrTuple[str]]
SupportedDomains: TypeAlias = OneOrTuple[str]
RateLimit = tuple[float, float]
@@ -155,7 +156,7 @@ class Registry:
class HTTPClientProxy:
DOMAIN: ClassVar[str]
- _IMPERSONATE: ClassVar[str | bool | None] = None
+ _IMPERSONATE: ClassVar[BrowserTypeLiteral | bool | None] = None
client: ScraperClient # pyright: ignore[reportUninitializedInstanceVariable]
@classmethod
@@ -181,7 +182,7 @@ class HTTPClientProxy:
@signature.copy(ScraperClient._request)
@contextlib.asynccontextmanager
async def request(
- self, *args, impersonate: str | bool | None = None, **kwargs
+ self, *args, impersonate: BrowserTypeLiteral | bool | None = None, **kwargs
) -> AsyncGenerator[AbstractResponse[Any]]:
if impersonate is None:
impersonate = self._IMPERSONATE
@@ -365,7 +366,7 @@ class Crawler(ABC, HTTPClientProxy):
try:
yield
except Exception:
- self.log(f"[{self.FOLDER_DOMAIN}] {msg}. Crawler has been disabled", 40)
+ logger.error(f"[{self.FOLDER_DOMAIN}] {msg}. Crawler has been disabled")
self.disabled = True
raise
@@ -387,7 +388,7 @@ class Crawler(ABC, HTTPClientProxy):
scrape_item.url = url = self.transform_url(scrape_item.url)
if url.path_qs in self.scraped_items:
- return log(f"Skipping {url} as it has already been scraped", 10)
+ return logger.info(f"Skipping {url} as it has already been scraped")
self.scraped_items.add(url.path_qs)
async with self._fetch_context(scrape_item):
@@ -431,7 +432,7 @@ class Crawler(ABC, HTTPClientProxy):
@contextlib.contextmanager
def new_task_id(self, url: AbsoluteHttpURL) -> Generator[TaskID]:
"""Creates a new task_id (shows the URL in the UI and logs)"""
- log(f"Scraping [{self.FOLDER_DOMAIN}]: {url}", 20)
+ logger.info(f"Scraping [{self.FOLDER_DOMAIN}]: {url}")
task_id = self.manager.progress_manager.scraping_progress.add_task(url)
try:
yield task_id
@@ -456,7 +457,7 @@ class Crawler(ABC, HTTPClientProxy):
filename = f"{name}.metadata" # we won't write to fs, so we skip name sanitization
download_folder = get_download_path(self.manager, scrape_item, self.FOLDER_DOMAIN)
- url = scrape_item.url.with_scheme("metadata")
+ url = AbsoluteHttpURL(scrape_item.url.with_scheme("metadata"))
media_item = MediaItem.from_item(
scrape_item,
url,
@@ -532,18 +533,12 @@ class Crawler(ABC, HTTPClientProxy):
db_path = self.create_db_path(url)
check_complete = await self.manager.db_manager.history_table.check_complete(self.DOMAIN, url, referer, db_path)
if check_complete:
- log(f"Skipping {url} as it has already been downloaded", 10)
+ logger.info(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
return check_complete
async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:
await self.manager.states.RUNNING.wait()
- if media_item.uploaded_at and not isinstance(media_item.uploaded_at, int):
- msg = (
- f"Invalid datetime from '{self.FOLDER_DOMAIN}' crawler . Got {media_item.uploaded_at!r}, expected int."
- )
- log(msg, bug=True)
-
check_complete = await self.check_complete(media_item.url, media_item.referer)
if check_complete:
if media_item.album_id:
@@ -561,17 +556,17 @@ class Crawler(ABC, HTTPClientProxy):
media_host = media_item.url.host
if (hosts := self.manager.config.ignore_options.skip_hosts) and any(host in media_host for host in hosts):
- log(f"Download skip {media_item.url} due to skip_hosts config", 10)
+ logger.info(f"Download skipped {media_item.url} due to skip_hosts config")
return True
if (hosts := self.manager.config.ignore_options.only_hosts) and not any(host in media_host for host in hosts):
- log(f"Download skip {media_item.url} due to only_hosts config", 10)
+ logger.info(f"Download skipped {media_item.url} due to only_hosts config")
return True
if (regex := self.manager.config.ignore_options.filename_regex_filter) and re.search(
regex, media_item.filename
):
- log(f"Download skip {media_item.url} due to filename regex filter config", 10)
+ logger.info(f"Download skipped {media_item.url} due to filename regex filter config")
return True
return False
@@ -588,7 +583,7 @@ class Crawler(ABC, HTTPClientProxy):
domain = None if any_crawler else self.DOMAIN
downloaded = await self.manager.db_manager.history_table.check_complete_by_referer(domain, url)
if downloaded:
- log(f"Skipping {url} as it has already been downloaded", 10)
+ logger.info(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
return True
return False
@@ -601,7 +596,7 @@ class Crawler(ABC, HTTPClientProxy):
downloaded = await self.manager.db_manager.hash_table.check_hash_exists(hash_type, hash_value)
if downloaded:
url = scrape_item if isinstance(scrape_item, URL) else scrape_item.url
- log(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded", 10)
+ logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
return downloaded
@@ -637,7 +632,7 @@ class Crawler(ABC, HTTPClientProxy):
return False
url_path = self.create_db_path(url)
if url_path in album_results and album_results[url_path] != 0:
- log(f"Skipping {url} as it has already been downloaded")
+ logger.info(f"Skipping {url} as it has already been downloaded")
self.manager.progress_manager.download_progress.add_previously_completed()
return True
return False
@@ -829,7 +824,9 @@ class Crawler(ABC, HTTPClientProxy):
page_url = self.parse_url(page_url_str, **kwargs)
@error_handling_wrapper
- async def direct_file(self, scrape_item: ScrapeItem, url: URL | None = None, assume_ext: str | None = None) -> None:
+ async def direct_file(
+ self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None, assume_ext: str | None = None
+ ) -> None:
"""Download a direct link file. Filename will be the url slug"""
link = url or scrape_item.url
filename, ext = self.get_filename_and_ext(link.name or link.parent.name, assume_ext=assume_ext)
@@ -946,7 +943,7 @@ class Crawler(ABC, HTTPClientProxy):
f"Important information was removed while creating a filename. "
f"\n{calling_args}"
)
- log(msg, bug=True)
+ logger.warning(msg)
return filename
@final
@@ -1042,7 +1039,7 @@ def _create_subclass(url: yarl.URL | str, base_class: type[_CrawlerT]) -> type[_
domain = primary_url.host.removeprefix("www.")
class_name = _make_crawler_name(domain)
class_attributes = Site(primary_url, domain)._asdict()
- return type(class_name, (base_class,), class_attributes) # type: ignore
+ return type(class_name, (base_class,), class_attributes) # type: ignore # pyright: ignore[reportReturnType]
def _make_crawler_name(input_string: str) -> str:
@@ -1091,12 +1088,12 @@ def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, OneOrTup
def auto_task_id(
- func: Callable[Concatenate[_CrawlerT, ScrapeItem, P], R | Coroutine[None, None, R]],
-) -> Callable[Concatenate[_CrawlerT, ScrapeItem, P], Coroutine[None, None, R]]:
+ func: Callable[Concatenate[_CrawlerT, ScrapeItem, _P], _R | Coroutine[None, None, _R]],
+) -> Callable[Concatenate[_CrawlerT, ScrapeItem, _P], Coroutine[None, None, _R]]:
"""Autocreate a new `task_id` from the scrape_item of the method"""
@wraps(func)
- async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: P.args, **kwargs: P.kwargs) -> R:
+ async def wrapper(self: _CrawlerT, scrape_item: ScrapeItem, *args: _P.args, **kwargs: _P.kwargs) -> _R:
await self.manager.states.RUNNING.wait()
with self.new_task_id(scrape_item.url):
result = func(self, scrape_item, *args, **kwargs)
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 18b0c963b..514d2ac9e 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -1,4 +1,3 @@
-# pyright: ignore[reportIncompatibleVariableOverride]
from __future__ import annotations
import base64
@@ -18,7 +17,6 @@ if TYPE_CHECKING:
from collections.abc import Callable, Generator
import aiohttp
- from propcache.api import under_cached_property as cached_property
from rich.progress import TaskID
from cyberdrop_dl import signature
@@ -34,17 +32,17 @@ if TYPE_CHECKING:
@signature.copy(yarl.URL.__mod__)
def __mod__(self) -> AbsoluteHttpURL: ...
- @cached_property
- def host(self) -> str: ...
+ @property
+ def host(self) -> str: ... # pyright: ignore[reportIncompatibleVariableOverride]
- @cached_property
- def scheme(self) -> Literal["http", "https"]: ...
+ @property
+ def scheme(self) -> Literal["http", "https"]: ... # pyright: ignore[reportIncompatibleVariableOverride]
- @cached_property
- def absolute(self) -> Literal[True]: ...
+ @property
+ def absolute(self) -> Literal[True]: ... # pyright: ignore[reportIncompatibleVariableOverride]
- @cached_property
- def parent(self) -> AbsoluteHttpURL: ...
+ @property
+ def parent(self) -> AbsoluteHttpURL: ... # pyright: ignore[reportIncompatibleVariableOverride]
@signature.copy(yarl.URL.with_path)
def with_path(self) -> AbsoluteHttpURL: ...
diff --git a/cyberdrop_dl/database/tables/hash.py b/cyberdrop_dl/database/tables/hash.py
index 2d318c8f2..d809dc45c 100644
--- a/cyberdrop_dl/database/tables/hash.py
+++ b/cyberdrop_dl/database/tables/hash.py
@@ -1,10 +1,9 @@
from __future__ import annotations
+import logging
from pathlib import Path
from typing import TYPE_CHECKING, cast
-from cyberdrop_dl.utils.logger import log
-
from .definitions import create_files, create_hash, create_hash_index
if TYPE_CHECKING:
@@ -14,6 +13,9 @@ if TYPE_CHECKING:
from cyberdrop_dl.database import Database
+logger = logging.getLogger(__name__)
+
+
class HashTable:
def __init__(self, database: Database) -> None:
self._database = database
@@ -51,7 +53,7 @@ class HashTable:
return row[0]
except Exception as e:
- log(f"Error checking file: {e}", 40, exc_info=e)
+ logger.exception(f"Error checking file: {e}")
async def get_files_with_hash_matches(
self, hash_value: str, size: int, hash_type: str | None = None
@@ -83,8 +85,8 @@ class HashTable:
cursor = await self.db_conn.execute(query, (hash_value, size, hash_type))
return cast("list[aiosqlite.Row]", await cursor.fetchall())
- except Exception as e:
- log(f"Error retrieving folder and filename: {e}", 40, exc_info=e)
+ except Exception:
+ logger.exception("Error retrieving folder and filename")
return []
async def check_hash_exists(self, hash_type: str, hash_value: str) -> bool:
@@ -129,7 +131,7 @@ class HashTable:
await self.db_conn.execute(query, (hash_value, hash_type, folder, download_filename, hash_value))
await self.db_conn.commit()
except Exception as e:
- log(f"Error inserting/updating record: {e}", 40, exc_info=e)
+ logger.exception(f"Error inserting/updating record: {e}")
return False
return True
@@ -166,7 +168,7 @@ class HashTable:
)
await self.db_conn.commit()
except Exception as e:
- log(f"Error inserting/updating record: {e}", 40, exc_info=e)
+ logger.exception(f"Error inserting/updating record: {e}")
return False
return True
@@ -190,5 +192,5 @@ class HashTable:
rows = await cursor.fetchall()
return [row[0] for row in rows]
except Exception as e:
- log(f"Error retrieving folder and filename: {e}", 40, exc_info=e)
+ logger.exception(f"Error retrieving folder and filename: {e}")
return []
diff --git a/cyberdrop_dl/database/tables/history.py b/cyberdrop_dl/database/tables/history.py
index 481c32d04..4cc381bda 100644
--- a/cyberdrop_dl/database/tables/history.py
+++ b/cyberdrop_dl/database/tables/history.py
@@ -4,8 +4,6 @@ import logging
from sqlite3 import IntegrityError, Row
from typing import TYPE_CHECKING, cast
-from cyberdrop_dl.utils.utilities import log
-
from .definitions import create_fixed_history, create_history
if TYPE_CHECKING:
@@ -116,7 +114,7 @@ class HistoryTable:
current_referer, completed = await select_referer_and_completed()
if completed and url != referer and str(referer) != current_referer:
# Update the referer if it has changed so that check_complete_by_referer can work
- log(f"Updating referer of {url} from {current_referer} to {referer}")
+ logger.info(f"Updating referer of {url} from {current_referer} to {referer}")
await update_referer()
return completed
@@ -299,7 +297,7 @@ class HistoryTable:
yield cast("list[Row]", rows)
except Exception as e:
- log(f"Error getting bunkr failed via size: {e}", 40, exc_info=e)
+ logger.exception(f"Error getting bunkr failed via size: {e}")
async def get_all_bunkr_failed_via_hash(self) -> AsyncGenerator[list[Row]]:
query = """
@@ -314,7 +312,7 @@ class HistoryTable:
yield cast("list[Row]", rows)
except Exception as e:
- log(f"Error getting bunkr failed via hash: {e}", 40, exc_info=e)
+ logger.exception(f"Error getting bunkr failed via hash: {e}")
async def fix_primary_keys(self) -> None:
domain_column, *_ = await self._get_media_table_columns()
diff --git a/cyberdrop_dl/database/tables/schema.py b/cyberdrop_dl/database/tables/schema.py
index 704270c2f..63a21f05f 100644
--- a/cyberdrop_dl/database/tables/schema.py
+++ b/cyberdrop_dl/database/tables/schema.py
@@ -1,12 +1,11 @@
from __future__ import annotations
+import logging
from typing import TYPE_CHECKING
import aiosqlite
from packaging.version import Version
-from cyberdrop_dl.utils.logger import log, log_spacer
-
from .definitions import create_schema_version
if TYPE_CHECKING:
@@ -17,6 +16,8 @@ if TYPE_CHECKING:
CURRENT_APP_SCHEMA_VERSION = "8.10.0"
+logger = logging.getLogger(__name__)
+
class SchemaVersionTable:
def __init__(self, database: Database) -> None:
@@ -52,13 +53,12 @@ class SchemaVersionTable:
await self.db_conn.commit()
async def startup(self) -> None:
- log_spacer(10)
- log(f"Expected database schema version: {CURRENT_APP_SCHEMA_VERSION}")
+ logger.info(f"Expected database schema version: {CURRENT_APP_SCHEMA_VERSION}")
version = await self.get_version()
- log(f"Database reports installed version: {version}")
+ logger.info(f"Database reports installed version: {version}")
if version is not None and version >= Version(CURRENT_APP_SCHEMA_VERSION):
return
# TODO: on v9, raise SystemExit if db version is None or older than 8.0.0
- log(f"Updating database version to {CURRENT_APP_SCHEMA_VERSION}")
+ logger.info(f"Updating database version to {CURRENT_APP_SCHEMA_VERSION}")
await self.__update_schema_version()
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 54ef0c60f..384c24008 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -16,13 +16,7 @@ from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
from cyberdrop_dl.ui.program_ui import ProgramUI
from cyberdrop_dl.updates import check_latest_pypi
from cyberdrop_dl.utils.apprise import send_apprise_notifications
-from cyberdrop_dl.utils.logger import (
- LogHandler,
- QueuedLogger,
- log,
- log_spacer,
- log_with_color,
-)
+from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log_spacer, log_with_color
from cyberdrop_dl.utils.sorting import Sorter
from cyberdrop_dl.utils.utilities import check_partials_and_empty_folders
from cyberdrop_dl.utils.webhook import send_webhook_message
@@ -30,6 +24,7 @@ from cyberdrop_dl.utils.webhook import send_webhook_message
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine, Sequence
+logger = logging.getLogger(__name__)
P = ParamSpec("P")
R = TypeVar("R")
@@ -72,22 +67,22 @@ async def _run_manager(manager: Manager) -> None:
debug_log_file_path = _setup_debug_logger(manager)
start_time = manager.start_time
_setup_main_logger(manager)
- log(f"Using Debug Log: {debug_log_file_path}", 10)
- log("Starting Async Processes...", 10)
+ logger.info(f"Using Debug Log: {debug_log_file_path}")
+ logger.info("Starting Async Processes...")
await manager.async_startup()
log_spacer(10)
- log("Starting CDL...\n", 20)
+ logger.info("Starting CDL...\n")
await _scheduler(manager)
manager.progress_manager.print_stats(start_time)
log_spacer(20)
- log("Checking for Updates...", 20)
+ logger.info("Checking for Updates...")
check_latest_pypi()
log_spacer(20)
- log("Closing Program...", 20)
+ logger.info("Closing Program...")
log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)
await send_webhook_message(manager)
@@ -137,11 +132,9 @@ def _setup_debug_logger(manager: Manager) -> Path | None:
if not env.DEBUG_VAR:
return
- debug_logger = logging.getLogger("cyberdrop_dl_debug")
- log_level = 10
+ log_level = logging.DEBUG
settings_data = manager.config_manager.settings_data
settings_data.runtime_options.log_level = log_level
- debug_logger.setLevel(log_level)
debug_log_file_path = Path(__file__).parents[1] / "cyberdrop_dl_debug.log"
if env.DEBUG_LOG_FOLDER:
debug_log_folder = Path(env.DEBUG_LOG_FOLDER)
@@ -156,7 +149,8 @@ def _setup_debug_logger(manager: Manager) -> Path | None:
file_handler = LogHandler(level=log_level, file=file_io, width=500, debug=True)
queued_logger = QueuedLogger(manager, file_handler, "debug")
- debug_logger.addHandler(queued_logger.handler)
+
+ logging.getLogger("cyberdrop_dl").addHandler(queued_logger.handler)
# aiosqlite_log = logging.getLogger("aiosqlite")
# aiosqlite_log.setLevel(log_level)
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 7fc0e3f33..a17200aee 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import logging
import os
import shutil
import subprocess
@@ -26,9 +27,10 @@ from cyberdrop_dl.exceptions import (
SkipDownloadError,
TooManyCrawlerErrors,
)
-from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper, parse_url
+logger = logging.getLogger(__name__)
+
# Windows epoch is January 1, 1601. Unix epoch is January 1, 1970
WIN_EPOCH_OFFSET = 116444736e9
MAC_OS_SET_FILE = None
@@ -101,14 +103,13 @@ def retry(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[
if e.status != 999:
media_item.attempts += 1
- log(f"{self.log_prefix} failed: {media_item.url} with error: {e!s}", 40)
+ logger.error(f"{self.log_prefix} failed: {media_item.url} with error: {e!s}")
if media_item.attempts >= self.max_attempts:
raise
- retry_msg = (
+ logger.info(
f"Retrying {self.log_prefix.lower()}: {media_item.url} , retry attempt: {media_item.attempts + 1}"
)
- log(retry_msg, 20)
return wrapper
@@ -271,11 +272,10 @@ class Downloader:
try:
subtitles = await download(m3u8_group.subtitle)
except Exception as e:
- log(f"Unable to download subtitles for {media_item.url}, Skipping. {e!r}", 40)
+ logger.exception(f"Unable to download subtitles for {media_item.url}, Skipping. {e!r}")
else:
- log(
- f"Found subtitles for {media_item.url}, but CDL is currently unable to merge them. Subtitle were saved at {subtitles} ",
- 30,
+ logger.warning(
+ f"Found subtitles for {media_item.url}, but CDL is currently unable to merge them. Subtitle were saved at {subtitles} "
)
if m3u8_group.audio:
@@ -326,7 +326,7 @@ class Downloader:
await self.set_file_datetime(media_item, media_item.path)
self.attempt_task_removal(media_item)
self.manager.progress_manager.download_progress.add_completed()
- log(f"Download finished: {media_item.url}", 20)
+ logger.info(f"Download finished: {media_item.url}")
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
@@ -348,7 +348,7 @@ class Downloader:
if self.manager.config_manager.settings_data.download_options.disable_file_timestamps:
return
if not media_item.uploaded_at:
- log(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime", 30)
+ logger.warning(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime")
return
# TODO: Make this entire method async (run in another thread)
@@ -431,14 +431,14 @@ class Downloader:
return False
if not media_item.is_segment:
- log(f"{self.log_prefix} starting: {media_item.url}", 20)
+ logger.info(f"{self.log_prefix} starting: {media_item.url}")
async with self._file_lock_vault[media_item.filename]:
- log_debug(f"Lock for {media_item.filename} acquired", 20)
+ logger.debug(f"Lock for '{media_item.filename}' acquired")
try:
return bool(await self.download(media_item))
finally:
- log_debug(f"Lock for {media_item.filename} released", 20)
+ logger.debug(f"Lock for '{media_item.filename}' released")
@error_handling_wrapper
@retry
@@ -460,13 +460,13 @@ class Downloader:
if not media_item.is_segment:
await self.set_file_datetime(media_item, media_item.path)
self.manager.progress_manager.download_progress.add_completed()
- log(f"Download finished: {media_item.url}", 20)
+ logger.info(f"Download finished: {media_item.url}")
self.attempt_task_removal(media_item)
return downloaded
except SkipDownloadError as e:
if not media_item.is_segment:
- log(f"Download skip {media_item.url}: {e}", 10)
+ logger.info(f"Download skipped {media_item.url}: {e}")
self.manager.progress_manager.download_progress.add_skipped()
self.attempt_task_removal(media_item)
@@ -499,8 +499,10 @@ class Downloader:
exc_info: Exception | None = None,
) -> None:
self.attempt_task_removal(media_item)
- full_message = f"{self.log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}"
- log(full_message, 40, exc_info=exc_info)
+ logger.error(
+ f"{self.log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}",
+ exc_info=exc_info,
+ )
self.manager.log_manager.write_download_error_log(media_item, error_log_msg.csv_log_msg)
self.manager.progress_manager.download_stats_progress.add_failure(error_log_msg.ui_failure)
self.manager.progress_manager.download_progress.add_failed()
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index 2eeed1476..ee54532bb 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -1,5 +1,4 @@
import os
-from hashlib import sha256
os.environ["PYDANTIC_ERRORS_INCLUDE_URL"] = "0"
RUNNING_IN_IDE = bool(os.getenv("PYCHARM_HOSTED") or os.getenv("TERM_PROGRAM") == "vscode")
@@ -7,9 +6,7 @@ RUNNING_IN_TERMUX = bool(
os.getenv("TERMUX_VERSION") or os.getenv("TERMUX_MAIN_PACKAGE_FORMAT") or "com.termux" in os.getenv("$PREFIX", "")
)
PORTRAIT_MODE = bool(RUNNING_IN_TERMUX or os.getenv("CDL_PORTRAIT_MODE"))
-ENABLE_DEBUG_CRAWLERS = os.getenv("CDL_ENABLE_DEBUG_CRAWLERS")
-if ENABLE_DEBUG_CRAWLERS:
- ENABLE_DEBUG_CRAWLERS = sha256(ENABLE_DEBUG_CRAWLERS.encode("utf-8")).hexdigest()
+
DEBUG_LOG_FOLDER = os.getenv("CDL_DEBUG_LOG_FOLDER")
PROFILING = os.getenv("CDL_PROFILING")
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 8cc0937e0..ddb2678e7 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -2,13 +2,14 @@ from __future__ import annotations
import asyncio
import contextlib
+import logging
import ssl
from base64 import b64encode
from collections import defaultdict
from collections.abc import Generator
from contextvars import ContextVar
from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, Literal, Self
+from typing import TYPE_CHECKING, Any, Self
import aiohttp
import certifi
@@ -27,11 +28,6 @@ from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError,
from cyberdrop_dl.ffmpeg import probe
from cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsers
from cyberdrop_dl.utils.cookie_management import read_netscape_files
-from cyberdrop_dl.utils.logger import log, log_debug, log_spacer
-
-_VALID_EXTENSIONS = (
- constants.FILE_FORMATS["Images"] | constants.FILE_FORMATS["Videos"] | constants.FILE_FORMATS["Audio"]
-)
if TYPE_CHECKING:
from collections.abc import Callable, Generator, Iterable, Mapping
@@ -48,6 +44,11 @@ try:
except ImportError as e:
_curl_import_error = e
+logger = logging.getLogger(__name__)
+_VALID_EXTENSIONS = (
+ constants.FILE_FORMATS["Images"] | constants.FILE_FORMATS["Videos"] | constants.FILE_FORMATS["Audio"]
+)
+
_DOWNLOAD_ERROR_ETAGS = {
"d835884373f4d6c8f24742ceabe74946": "Imgur image has been removed",
"65b7753c-528a": "SC Scrape Image",
@@ -159,8 +160,8 @@ class ClientManager:
return self._flaresolverr
def _startup(self) -> None:
- self._session = self.new_scrape_session()
- self._download_session = self.new_download_session()
+ self._session = self.create_aiohttp_session()
+ self._download_session = self.create_aiohttp_session()
if _curl_import_error is not None:
return
@@ -275,24 +276,16 @@ class ClientManager:
cookies={cookie.key: cookie.value for cookie in self.cookies},
)
- def new_scrape_session(self) -> ClientSession:
- trace_configs = _create_request_log_hooks("scrape")
- return self._new_session(cached=True, trace_configs=trace_configs)
-
- def new_download_session(self) -> ClientSession:
- trace_configs = _create_request_log_hooks("download")
- return self._new_session(cached=False, trace_configs=trace_configs)
-
- def _new_session(
- self, cached: bool = False, trace_configs: list[aiohttp.TraceConfig] | None = None
+ def create_aiohttp_session(
+ self,
) -> ClientSession:
- timeout = self.rate_limiting_options._aiohttp_timeout
return ClientSession(
- headers={"user-agent": self.manager.global_config.general.user_agent},
+ headers={
+ "user-agent": self.manager.global_config.general.user_agent,
+ },
raise_for_status=False,
cookie_jar=self.cookies,
- timeout=timeout,
- trace_configs=trace_configs,
+ timeout=self.rate_limiting_options._aiohttp_timeout,
proxy=self.manager.global_config.general.proxy,
connector=self._new_tcp_connector(),
requote_redirect_url=False,
@@ -311,7 +304,7 @@ class ClientManager:
f"{crawler.__class__.__name__} has been disabled after too many errors. "
f"URLs from the following domains will be ignored: {crawler.SCRAPE_MAPPER_KEYS}"
)
- log(msg, 40)
+ logger.error(msg)
raise TooManyCrawlerErrors
@contextlib.contextmanager
@@ -341,8 +334,6 @@ class ClientManager:
async for domain, cookie in read_netscape_files(cookie_files):
self.cookies.update_cookies(cookie, response_url=AbsoluteHttpURL(f"https://{domain}"))
- log_spacer(20, log_to_console=False)
-
def get_rate_limiter(self, domain: str) -> AsyncLimiter:
"""Get a rate limiter for a domain."""
if domain in self.rate_limits:
@@ -466,7 +457,7 @@ async def _set_dns_resolver(loop: asyncio.AbstractEventLoop | None = None) -> No
constants.DNS_RESOLVER = aiohttp.AsyncResolver
except Exception as e:
constants.DNS_RESOLVER = aiohttp.ThreadedResolver
- log(f"Unable to setup asynchronous DNS resolver. Falling back to thread based resolver: {e}", 30)
+ logger.warning(f"Unable to setup asynchronous DNS resolver. Falling back to thread based resolver: {e}")
async def _test_async_resolver(loop: asyncio.AbstractEventLoop | None = None) -> None:
@@ -478,20 +469,3 @@ async def _test_async_resolver(loop: asyncio.AbstractEventLoop | None = None) ->
async with aiodns.DNSResolver(loop=loop, timeout=5.0) as resolver:
_ = await resolver.query_dns("github.com", "A")
-
-
-def _create_request_log_hooks(client_type: Literal["scrape", "download"]) -> list[aiohttp.TraceConfig]:
- async def on_request_start(*args) -> None:
- params: aiohttp.TraceRequestStartParams = args[2]
- log_debug(f"Starting {client_type} {params.method} request to {params.url}", 10)
-
- async def on_request_end(*args) -> None:
- params: aiohttp.TraceRequestEndParams = args[2]
- msg = f"Finishing {client_type} {params.method} request to {params.url}"
- msg += f" -> response status: {params.response.status}"
- log_debug(msg, 10)
-
- trace_config = aiohttp.TraceConfig()
- trace_config.on_request_start.append(on_request_start)
- trace_config.on_request_end.append(on_request_end)
- return [trace_config]
diff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.py
index 80e06f5c7..248607564 100644
--- a/cyberdrop_dl/managers/log_manager.py
+++ b/cyberdrop_dl/managers/log_manager.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import csv
+import logging
from collections import defaultdict
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -9,7 +10,7 @@ from typing import TYPE_CHECKING, Any
from cyberdrop_dl.constants import CSV_DELIMITER
from cyberdrop_dl.exceptions import get_origin
from cyberdrop_dl.utils import json
-from cyberdrop_dl.utils.logger import log, log_spacer
+from cyberdrop_dl.utils.logger import log_spacer
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -20,6 +21,9 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
+
+
class LogManager:
def __init__(self, manager: Manager) -> None:
self.manager = manager
@@ -99,7 +103,7 @@ class LogManager:
def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:
log_spacer(20)
- log("Updating Last Forum Posts...\n", 20)
+ logger.info("Updating Last Forum Posts...\n")
current_urls, current_base_urls, new_urls, new_base_urls = [], [], [], []
try:
@@ -115,13 +119,13 @@ def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:
current_urls.append(url)
current_base_urls.append(base_url)
except UnicodeDecodeError:
- log("Unable to read input file, skipping update_last_forum_post", 40)
+ logger.exception("Unable to read input file, skipping update_last_forum_post")
return
with last_post_log.open(encoding="utf8") as f:
reader = csv.DictReader(f.readlines())
for row in reader:
- new_url = base_url = row.get("url").strip().removesuffix("/") # type: ignore
+ new_url = base_url = row["url"].strip().removesuffix("/") # type: ignore
if "https" in new_url and "/post-" in new_url:
base_url = new_url.rsplit("/post", 1)[0]
@@ -138,11 +142,11 @@ def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:
old_url = current_urls[index]
if old_url == new_url:
continue
- log(f"Updating {base}\n {old_url = }\n {new_url = }", 20)
+ logger.info(f"Updating {base}\n {old_url = }\n {new_url = }")
updated_urls[index] = new_url
if updated_urls == current_urls:
- log("No URLs updated", 20)
+ logger.info("No URLs updated")
return
with input_file.open("w", encoding="utf8") as f:
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index c82ff0b8c..ffb61cd8e 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import json
+import logging
from dataclasses import Field, field
from time import perf_counter
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
@@ -20,7 +21,7 @@ from cyberdrop_dl.managers.live_manager import LiveManager
from cyberdrop_dl.managers.log_manager import LogManager
from cyberdrop_dl.managers.path_manager import PathManager
from cyberdrop_dl.managers.progress_manager import ProgressManager
-from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log
+from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger
from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information
if TYPE_CHECKING:
@@ -35,6 +36,9 @@ class AsyncioEvents(NamedTuple):
RUNNING: asyncio.Event
+logger = logging.getLogger(__name__)
+
+
class Manager:
def __init__(self, args: Sequence[str] | None = None) -> None:
if isinstance(args, str):
@@ -180,7 +184,7 @@ class Manager:
f"Using ffmpeg version: {ffmpeg.get_ffmpeg_version()}",
f"Using ffprobe version: {ffmpeg.get_ffprobe_version()}",
)
- log("\n".join(args_info))
+ logger.info("\n".join(args_info))
async def async_db_close(self) -> None:
"Partial shutdown for managers used for hash directory scanner"
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index df99171a6..1ca7bd486 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import logging
import time
from contextlib import asynccontextmanager
from dataclasses import field
@@ -22,7 +23,7 @@ from cyberdrop_dl.ui.progress.hash_progress import HashProgress
from cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgress
from cyberdrop_dl.ui.progress.sort_progress import SortProgress
from cyberdrop_dl.ui.progress.statistic_progress import DownloadStatsProgress, ScrapeStatsProgress
-from cyberdrop_dl.utils.logger import log, log_spacer, log_with_color
+from cyberdrop_dl.utils.logger import log_spacer, log_with_color
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
@@ -33,6 +34,9 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.ui.progress.statistic_progress import UiFailureTotal
+
+logger = logging.getLogger(__name__)
+
log_cyan = partial(log_with_color, style="cyan", level=20)
log_yellow = partial(log_with_color, style="yellow", level=20)
log_green = partial(log_with_color, style="green", level=20)
@@ -144,7 +148,7 @@ class ProgressManager:
total_data_written = ByteSize(self.file_progress.total_data_written).human_readable(decimal=True)
log_spacer(20)
- log("Printing Stats...\n", 20)
+ logger.info("Printing Stats...\n")
config_path = self.manager.path_manager.config_folder / self.manager.config_manager.loaded_config
config_path_text = get_console_hyperlink(config_path, text=self.manager.config_manager.loaded_config)
input_file_text = get_input(self.manager)
diff --git a/cyberdrop_dl/plugins.py b/cyberdrop_dl/plugins.py
index 0284045e8..96f7d47a5 100644
--- a/cyberdrop_dl/plugins.py
+++ b/cyberdrop_dl/plugins.py
@@ -15,11 +15,11 @@ See: https://packaging.python.org/en/latest/guides/creating-and-discovering-plug
from __future__ import annotations
import dataclasses
+import logging
from importlib.metadata import entry_points
from typing import TYPE_CHECKING, Final
from cyberdrop_dl import env
-from cyberdrop_dl.utils.logger import log
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
_GROUP_NAME: Final = "cyberdrop_dl_plugins"
@@ -51,8 +52,8 @@ def _get_plugins() -> Iterable[Plugin]:
def load(manager: Manager) -> None:
for plugin in _get_plugins():
if env.NO_PLUGINS:
- log(f"Found plugins installed but plugins are disabled. Ignored: {tuple(_get_plugins())}", 40)
+ logger.error(f"Found plugins installed but plugins are disabled. Ignored: {tuple(_get_plugins())}")
return
plugin.entrypoint.load()(manager)
- log(f"Loaded {plugin}", 20)
+ logger.info(f"Loaded {plugin}")
diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.py
index d3c9fd94d..c3ad2e509 100644
--- a/cyberdrop_dl/scraper/jdownloader.py
+++ b/cyberdrop_dl/scraper/jdownloader.py
@@ -1,12 +1,14 @@
from __future__ import annotations
import dataclasses
+import logging
from typing import TYPE_CHECKING
from myjdapi import myjdapi
from cyberdrop_dl.exceptions import JDownloaderError
-from cyberdrop_dl.utils.logger import log
+
+logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from pathlib import Path
@@ -68,7 +70,7 @@ class JDownloader:
except myjdapi.MYJDApiException as e:
msg = e
- log(f"Failed to connect to jDownloader: {msg}", 40)
+ logger.error(f"Failed to connect to jDownloader: {msg}")
self.enabled = False
def direct_unsupported_to_jdownloader(
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index a2165ceae..a495bd5bb 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import logging
import re
from datetime import date, datetime
from pathlib import Path
@@ -21,7 +22,7 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.scraper.filters import is_in_domain_list, is_outside_date_range, is_valid_url
from cyberdrop_dl.scraper.jdownloader import JDownloader
-from cyberdrop_dl.utils.logger import log, log_spacer
+from cyberdrop_dl.utils.logger import log_spacer
from cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash
if TYPE_CHECKING:
@@ -33,6 +34,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.crawlers import Crawler
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
existing_crawlers: dict[str, Crawler] = {}
_seen_urls: set[AbsoluteHttpURL] = set()
_crawlers_disabled_at_runtime: set[str] = set()
@@ -129,7 +131,7 @@ class ScrapeMapper:
self.count += 1
if not self.count:
- log("No valid links found.", 30)
+ logger.warning("No valid links found")
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
@@ -222,7 +224,7 @@ class ScrapeMapper:
return
if not self.real_debrid.disabled and self.real_debrid.api.is_supported(scrape_item.url):
- log(f"Using RealDebrid for unsupported URL: {scrape_item.url}", 10)
+ logger.info(f"Using RealDebrid for unsupported URL: {scrape_item.url}")
self.manager.task_group.create_task(self.real_debrid.run(scrape_item))
return
@@ -234,7 +236,7 @@ class ScrapeMapper:
pass
if self.jdownloader.enabled and jdownloader_whitelisted:
- log(f"Sending unsupported URL to JDownloader: {scrape_item.url}", 20)
+ logger.info(f"Sending unsupported URL to JDownloader: {scrape_item.url}")
success = False
try:
download_folder = get_download_path(self.manager, scrape_item, "jdownloader")
@@ -246,7 +248,7 @@ class ScrapeMapper:
)
success = True
except JDownloaderError as e:
- log(f"Failed to send {scrape_item.url} to JDownloader\n{e.message}", 40)
+ logger.error(f"Failed to send {scrape_item.url} to JDownloader\n{e.message}")
self.manager.log_manager.write_unsupported_urls_log(
scrape_item.url,
scrape_item.parents[0] if scrape_item.parents else None,
@@ -254,7 +256,7 @@ class ScrapeMapper:
self.manager.progress_manager.scrape_stats_progress.add_unsupported(sent_to_jdownloader=success)
return
- log(f"Unsupported URL: {scrape_item.url}", 30)
+ logger.warning(f"Unsupported URL: {scrape_item.url}")
self.manager.log_manager.write_unsupported_urls_log(
scrape_item.url,
scrape_item.parents[0] if scrape_item.parents else None,
@@ -274,23 +276,23 @@ class ScrapeMapper:
is_in_domain_list(scrape_item, BlockedDomains.partial_match)
or scrape_item.url.host in BlockedDomains.exact_match
):
- log(f"Skipping {scrape_item.url} as it is a blocked domain", 10)
+ logger.info(f"Skipping {scrape_item.url} as it is a blocked domain")
return False
before = self.manager.parsed_args.cli_only_args.completed_before
after = self.manager.parsed_args.cli_only_args.completed_after
if is_outside_date_range(scrape_item, before, after):
- log(f"Skipping {scrape_item.url} as it is outside of the desired date range", 10)
+ logger.info(f"Skipping {scrape_item.url} as it is outside of the desired date range")
return False
skip_hosts = self.manager.config_manager.settings_data.ignore_options.skip_hosts
if skip_hosts and is_in_domain_list(scrape_item, skip_hosts):
- log(f"Skipping URL by skip_hosts config: {scrape_item.url}", 10)
+ logger.info(f"Skipping URL by skip_hosts config: {scrape_item.url}")
return False
only_hosts = self.manager.config_manager.settings_data.ignore_options.only_hosts
if only_hosts and not is_in_domain_list(scrape_item, only_hosts):
- log(f"Skipping URL by only_hosts config: {scrape_item.url}", 10)
+ logger.info(f"Skipping URL by only_hosts config: {scrape_item.url}")
return False
return True
@@ -334,7 +336,7 @@ def regex_links(line: str) -> Generator[AbsoluteHttpURL]:
encoded = "%" in link
yield AbsoluteHttpURL(link, encoded=encoded)
except Exception as e:
- log(f"Unable to parse URL from input file: {link} {e:!r}", 40)
+ logger.error(f"Unable to parse URL from input file: {link} {e:!r}")
def _create_item_from_row(row: aiosqlite.Row) -> ScrapeItem:
@@ -391,10 +393,10 @@ def register_crawler(
)
if from_user == "raise":
raise ValueError(msg)
- log(msg, 40)
+ logger.error(msg)
continue
else:
- log(f"Successfully mapped {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}")
+ logger.info(f"Successfully mapped {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}")
elif other:
msg = f"{domain} from {crawler.NAME} already registered by {other}"
@@ -436,7 +438,7 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_t
f"{len(crawlers_to_disable)} Crawler names where provided to disable"
f", but only {len(disabled_crawlers)} {'is' if len(disabled_crawlers) == 1 else 'are'} a valid crawler's name."
)
- log(msg, 30)
+ logger.warning(msg)
if disabled_crawlers:
existing_crawlers.clear()
@@ -444,7 +446,7 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_t
crawlers_info = "\n".join(
str({info.site: info.supported_domains}) for info in sorted(crawlers.INFO for crawlers in disabled_crawlers)
)
- log(f"Crawlers disabled by config: \n{crawlers_info}")
+ logger.info(f"Crawlers disabled by config: \n{crawlers_info}")
log_spacer(10)
diff --git a/cyberdrop_dl/signature.py b/cyberdrop_dl/signature.py
index 5c4ecbab5..47c643707 100644
--- a/cyberdrop_dl/signature.py
+++ b/cyberdrop_dl/signature.py
@@ -30,11 +30,3 @@ else:
return y
return call
-
-
-def easy_repr(*fields: str) -> Callable[..., str]:
- def _repr_(self: object) -> str:
- params = ", ".join(f"{f}={getattr(self, f)!r}" for f in fields)
- return f"{type(self).__name__}({params})"
-
- return _repr_
diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.py
index 3418c505e..5f77257e9 100644
--- a/cyberdrop_dl/utils/apprise.py
+++ b/cyberdrop_dl/utils/apprise.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import json
+import logging
import shutil
import tempfile
from dataclasses import dataclass
@@ -16,11 +17,13 @@ from rich.text import Text
from cyberdrop_dl import constants
from cyberdrop_dl.dependencies import apprise
from cyberdrop_dl.models import AppriseURLModel
-from cyberdrop_dl.utils.logger import log, log_debug, log_spacer
+from cyberdrop_dl.utils.logger import log_spacer
if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
+
@dataclass
class AppriseURL:
@@ -80,7 +83,7 @@ def get_apprise_urls(*, file: Path | None = None, urls: list[str] | None = None)
if not urls:
return []
if apprise is None:
- log("Found apprise URLs for notifications but apprise is not installed. Ignoring", 30)
+ logger.warning("Found apprise URLs for notifications but apprise is not installed. Ignoring")
return []
return _simplify_urls([AppriseURLModel.model_validate({"url": url}) for url in set(urls)])
@@ -128,14 +131,14 @@ def _process_results(
log_spacer(10, log_to_console=False, log_to_file=not all(result))
rich.print("Apprise notifications results:", final_result.value)
- logger = log_debug if all(result) else log
- logger(f"Apprise notifications results: {final_result.value}")
- logger(f"PARSED_APPRISE_URLs: \n{json.dumps(all_urls, indent=4)}\n")
- logger(f"RESULTS_BY_TAGS: \n{json.dumps(result_dict, indent=4)}")
+ log = logger.debug if all(result) else logger.info
+ log(f"Apprise notifications results: {final_result.value}")
+ log(f"PARSED_APPRISE_URLs: \n{json.dumps(all_urls, indent=4)}\n")
+ log(f"RESULTS_BY_TAGS: \n{json.dumps(result_dict, indent=4)}")
log_spacer(10, log_to_console=False, log_to_file=not all(result))
parsed_log_lines = _parse_apprise_logs(apprise_logs)
for line in parsed_log_lines:
- logger(level=line.level.value, message=line.msg)
+ log(line.msg)
return final_result, parsed_log_lines
@@ -208,7 +211,7 @@ async def send_apprise_notifications(manager: Manager) -> tuple[constants.Notifi
shutil.copy(main_log, temp_main_log)
notifications_to_send["attach_logs"]["attach"] = str(temp_main_log.resolve())
except OSError:
- log(attach_file_failed_msg, 40)
+ logger.exception(attach_file_failed_msg)
async def notify(tag: str, msg: dict[str, Any]) -> tuple[str, bool | None]:
result = await apprise_obj.async_notify(**msg, tag=tag)
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
index e1c936fd2..c3556dc0e 100644
--- a/cyberdrop_dl/utils/cookie_management.py
+++ b/cyberdrop_dl/utils/cookie_management.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import logging
import os
import time
from functools import wraps
@@ -10,7 +11,6 @@ from textwrap import dedent
from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVar
from cyberdrop_dl.dependencies import browser_cookie3
-from cyberdrop_dl.utils.logger import log
if TYPE_CHECKING:
from collections.abc import AsyncIterable, Callable
@@ -19,7 +19,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.constants import BROWSERS
from cyberdrop_dl.managers.manager import Manager
-
+logger = logging.getLogger(__name__)
P = ParamSpec("P")
R = TypeVar("R")
@@ -136,7 +136,7 @@ def extract_cookies(extractor_name: str) -> CookieJar:
async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[str, SimpleCookie]]:
- now = time.time()
+ now = int(time.time())
domains_seen = set()
cookie_jars = await asyncio.gather(*(_read_netscape_file(file) for file in cookie_files))
for file, cookie_jar in zip(cookie_files, cookie_jars, strict=True):
@@ -149,14 +149,16 @@ async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[s
continue
simplified_domain = cookie.domain.removeprefix(".")
if simplified_domain not in current_cookie_file_domains:
- log(f"Found cookies for {simplified_domain} in file '{file.name}'", 20)
+ logger.info(f"Found cookies for {simplified_domain} in file '{file.name}'")
current_cookie_file_domains.add(simplified_domain)
if simplified_domain in domains_seen:
- log(f"Previous cookies for domain {simplified_domain} detected. They will be overwritten", 30)
+ logger.warning(
+ f"Previous cookies for domain {simplified_domain} detected. They will be overwritten"
+ )
if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore
expired_cookies_domains.add(simplified_domain)
- log(f"Cookies for {simplified_domain} are expired", 30)
+ logger.warning(f"Cookies for {simplified_domain} are expired")
domains_seen.add(simplified_domain)
simple_cookie = make_simple_cookie(cookie, now)
@@ -170,7 +172,7 @@ async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:
cookie_jar.load(ignore_discard=True)
return cookie_jar
except OSError as e:
- log(f"Unable to load cookies from '{file.name}':\n {e!s}", 40)
+ logger.error(f"Unable to load cookies from '{file.name}':\n {e!s}")
return await asyncio.to_thread(read)
diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
index 662f2f213..7886fa0c9 100644
--- a/cyberdrop_dl/utils/logger.py
+++ b/cyberdrop_dl/utils/logger.py
@@ -80,7 +80,7 @@ class LogHandler(RichHandler):
"""Rich Handler with default settings, automatic console creation and custom log render to remove padding in files."""
def __init__(
- self, level: int = 10, file: IO[str] | None = None, width: int | None = None, debug: bool = False, **kwargs
+ self, level: int = 10, file: IO[str] | None = None, width: int | None = None, debug: bool = False
) -> None:
is_file: bool = file is not None
redacted: bool = is_file and not debug
@@ -89,11 +89,23 @@ class LogHandler(RichHandler):
console = _DEFAULT_CONSOLE
else:
console = console_cls(file=file, width=width)
- options = constants.RICH_HANDLER_DEBUG_CONFIG if debug else constants.RICH_HANDLER_CONFIG
- options = options | kwargs
- super().__init__(level, console, show_time=is_file, **options)
+
+ super().__init__(
+ level,
+ console,
+ show_time=False,
+ show_path=False,
+ tracebacks_show_locals=debug,
+ locals_max_string=constants.DEFAULT_CONSOLE_WIDTH,
+ tracebacks_extra_lines=2,
+ locals_max_length=20,
+ )
if is_file:
- self._log_render = NoPaddingLogRender(show_level=True)
+ self._log_render = NoPaddingLogRender(
+ show_path=False,
+ show_level=True,
+ time_format=lambda dt: Text(f"[{dt.isoformat(sep=' ', timespec='milliseconds')}]", style="log.time"),
+ )
class BareQueueHandler(QueueHandler):
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index 60b350b9f..79e6342a5 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import itertools
+import logging
from datetime import datetime
from pathlib import Path
from subprocess import CalledProcessError
@@ -13,13 +14,16 @@ from cyberdrop_dl import constants
from cyberdrop_dl.constants import FILE_FORMATS
from cyberdrop_dl.ffmpeg import probe
from cyberdrop_dl.utils import strings
-from cyberdrop_dl.utils.logger import log, log_with_color
+from cyberdrop_dl.utils.logger import log_with_color
from cyberdrop_dl.utils.utilities import purge_dir_tree
if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+logger = logging.getLogger(__name__)
+
+
async def get_modified_date(file: Path) -> datetime:
stat = await asyncio.to_thread(file.stat)
return datetime.fromtimestamp(stat.st_mtime).replace(microsecond=0)
@@ -128,8 +132,8 @@ class Sorter:
bitrate = audio.bitrate
sample_rate = audio.sample_rate
- except (RuntimeError, CalledProcessError, OSError) as e:
- log(f"Unable to get audio properties of '{file}'", 40, exc_info=e)
+ except (RuntimeError, CalledProcessError, OSError):
+ logger.exception(f"Unable to get audio properties of '{file}'")
if await self._process_file_move(
file,
@@ -156,7 +160,7 @@ class Sorter:
width = height = resolution = None
except (OSError, ValueError):
- log(f"Unable to get some image properties of '{file}'")
+ logger.exception(f"Unable to get some image properties of '{file}'")
if await self._process_file_move(
file,
@@ -185,8 +189,8 @@ class Sorter:
duration = video.duration or probe_output.format.duration
framerate = video.fps
- except (RuntimeError, CalledProcessError, OSError) as e:
- log(f"Unable to get some video properties of '{file}'", 40, exc_info=e)
+ except (RuntimeError, CalledProcessError, OSError):
+ logger.exception(f"Unable to get some video properties of '{file}'")
if await self._process_file_move(
file,
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index d0ec91bc3..c766a0341 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -5,6 +5,7 @@ import dataclasses
import functools
import inspect
import itertools
+import logging
import mimetypes
import os
import platform
@@ -48,7 +49,7 @@ from cyberdrop_dl.exceptions import (
get_origin,
)
from cyberdrop_dl.utils import json
-from cyberdrop_dl.utils.logger import log, log_with_color
+from cyberdrop_dl.utils.logger import log_with_color
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine, Generator, Iterable, Mapping
@@ -69,6 +70,7 @@ if TYPE_CHECKING:
__dataclass_fields__: ClassVar[dict]
+logger = logging.getLogger(__name__)
_ALLOWED_FILEPATH_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"
_BLOB_OR_SVG = ("data:", "blob:", "javascript:")
@@ -137,7 +139,7 @@ def error_handling_context(self: Crawler | Downloader, item: ScrapeItem | MediaI
self.write_download_error(item, error_log_msg, exc_info)
return
- log(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", 40, exc_info=exc_info)
+ logger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)
self.manager.log_manager.write_scrape_error_log(link_to_show, error_log_msg.csv_log_msg, origin)
self.manager.progress_manager.scrape_stats_progress.add_failure(error_log_msg.ui_failure)
@@ -288,7 +290,7 @@ def clear_term():
os.system("cls" if os.name == "nt" else "clear")
-def get_size(path: os.DirEntry) -> int | None:
+def get_size(path: os.DirEntry[str]) -> int | None:
try:
return path.stat(follow_symlinks=False).st_size
except (OSError, ValueError):
@@ -573,9 +575,7 @@ def xor_decrypt(encrypted_data: bytes, key: bytes) -> str:
return data.decode("utf-8", errors="ignore")
-log_cyan = partial(log_with_color, style="cyan", level=20)
log_yellow = partial(log_with_color, style="yellow", level=20)
-log_green = partial(log_with_color, style="green", level=20)
log_red = partial(log_with_color, style="red", level=20)
diff --git a/cyberdrop_dl/utils/webhook.py b/cyberdrop_dl/utils/webhook.py
index ed3b001a7..f29846cc9 100644
--- a/cyberdrop_dl/utils/webhook.py
+++ b/cyberdrop_dl/utils/webhook.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import logging
from typing import TYPE_CHECKING, Any, cast
import aiofiles
@@ -8,7 +9,7 @@ import rich
from aiohttp import FormData
from cyberdrop_dl import aio, constants
-from cyberdrop_dl.utils.logger import log, log_debug, log_spacer
+from cyberdrop_dl.utils.logger import log_spacer
if TYPE_CHECKING:
from pathlib import Path
@@ -17,7 +18,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.models.base_models import HttpAppriseURL
-
+logger = logging.getLogger(__name__)
_DEFAULT_DIFF_LINE_FORMAT: str = "{}"
_STYLE_TO_DIFF_MAP = {
"green": "+ {}",
@@ -71,15 +72,14 @@ async def send_webhook_message(manager: Manager) -> None:
rich.print("\nSending Webhook Notifications.. ")
url = cast("AbsoluteHttpURL", webhook.url.get_secret_value())
form = await _prepare_form(webhook, manager.path_manager.main_log)
- logger = log
+
result = constants.NotificationResult.FAILED.value
try:
- async with manager.client_manager._new_session() as session, session.post(url, data=form) as response:
+ async with manager.client_manager.create_aiohttp_session() as session, session.post(url, data=form) as response:
if response.ok:
result = constants.NotificationResult.SUCCESS.value
result_to_log = [result]
- logger = log_debug
else:
json_resp: dict[str, Any] = await response.json()
@@ -88,10 +88,10 @@ async def send_webhook_message(manager: Manager) -> None:
result_to_log = constants.NotificationResult.FAILED.value, resp_text
except Exception as e:
- logger("Unable to send webhook notification", 40, exc_info=e)
+ logger.exception("Unable to send webhook notification")
result_to_log = result, str(e)
log_spacer(10, log_to_console=False)
rich.print("Webhook Notifications Results:", result)
result_to_log = "\n".join(map(str, result_to_log))
- logger(f"Webhook Notifications Results: {result_to_log}")
+ logger.info(f"Webhook Notifications Results: {result_to_log}")