Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move enter_context to utils

NTFSvolume8d2a306c49d1
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index 23494c4fb..6f0a40785 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -36,7 +36,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.manager import Manager
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-_JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("_JSON_CHECK", default=None)
+JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("JSON_CHECK", default=None)
RequestContext = contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]] # pyright: ignore[reportPrivateUsage]
logger = logging.getLogger(__name__)
@@ -318,20 +318,12 @@ class HTTPClient:
flaresolverr.verify_solution(self.config.user_agent, solution)
return AbstractResponse.create(solution)
- @contextlib.contextmanager
- def json_context(self, check: Callable[[Any, AbstractResponse[Any]], None], /):
- token = _JSON_CHECK.set(check)
- try:
- yield
- finally:
- _JSON_CHECK.reset(token)
-
async def _check_json(response: AbstractResponse[Any]) -> None:
if "json" not in response.content_type:
return
- if check := _JSON_CHECK.get():
+ if check := JSON_CHECK.get():
check(await response.json(), response)
return
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 0476de815..64bcfb735 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -17,20 +17,24 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, Self, final
from cyberdrop_dl import aio, env, signature
-from cyberdrop_dl.clients.http import HTTPClient, HTTPMixin, RequestContext
+from cyberdrop_dl.clients.http import JSON_CHECK, HTTPClient, HTTPMixin, RequestContext
from cyberdrop_dl.constants import CDL_USER_AGENT
from cyberdrop_dl.crawlers._hls import HLSMixin
from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
-from cyberdrop_dl.filepath import (
- check_dangerous_filename,
- check_path_traversal,
- compose_filename,
- get_filename_and_ext,
-)
+from cyberdrop_dl.filepath import check_dangerous_filename, check_path_traversal, compose_filename, get_filename_and_ext
from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, is_absolute_http_url
-from cyberdrop_dl.utils import css, dates, error_handling_context, is_blob_or_svg, m3u8, parse_url, unique
+from cyberdrop_dl.utils import (
+ css,
+ dates,
+ enter_context,
+ error_handling_context,
+ is_blob_or_svg,
+ m3u8,
+ parse_url,
+ unique,
+)
from cyberdrop_dl.utils.strings import safe_format
if TYPE_CHECKING:
@@ -321,7 +325,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
if impersonate is None:
impersonate = self._IMPERSONATE
- with self.client.json_context(self.__json_resp_check__):
+ with enter_context(JSON_CHECK, self.__json_resp_check__):
async with (
self.client.rate_limits[self.DOMAIN],
self.client.global_rate_limiter,
diff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.py
index eff54d06a..4a36a8417 100644
--- a/cyberdrop_dl/logs.py
+++ b/cyberdrop_dl/logs.py
@@ -21,7 +21,7 @@ from rich.text import Text, TextType
from cyberdrop_dl import env
from cyberdrop_dl.constants import MAIN_LOG_FILE
from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup
-from cyberdrop_dl.utils import dates
+from cyberdrop_dl.utils import dates, enter_context
if TYPE_CHECKING:
import datetime
@@ -195,7 +195,7 @@ def _threaded_logger(
q_listener: QueueListener = QueueListener(q, log_handler, respect_handler_level=True)
q_listener.start()
- with _enter_context(context_var, q_listener) if context_var else contextlib.nullcontext():
+ with enter_context(context_var, q_listener) if context_var else contextlib.nullcontext():
logging.getLogger().addHandler(q_handler)
try:
yield q_handler
@@ -209,15 +209,6 @@ def _threaded_logger(
handler.close()
-def _enter_context[T](context_var: ContextVar[T], value: T, /) -> Generator[None]:
- token = context_var.set(value)
- try:
- yield
- finally:
- context_var.reset(token)
-
-
@final
class NoPaddingLogRender(LogRender):
_cdl_padding: int = 0
@@ -333,9 +324,9 @@ def setup_file_logging(file: Path, /, *, level: int = logging.DEBUG) -> Generato
with (
_setup_debug_logger() as debug_log_file,
file.open("w", encoding="utf8") as fp,
- _enter_context(MAIN_LOG_FILE, file),
- _enter_context(mega.LOG_HTTP_TRAFFIC, True),
- _enter_context(mega.LOG_FILE_PROGRESS, False),
+ enter_context(MAIN_LOG_FILE, file),
+ enter_context(mega.LOG_HTTP_TRAFFIC, True),
+ enter_context(mega.LOG_FILE_PROGRESS, False),
_threaded_logger(
log_handler=LogHandler(
level=logging.DEBUG,
@@ -351,11 +342,11 @@ def setup_file_logging(file: Path, /, *, level: int = logging.DEBUG) -> Generato
try:
yield
except CDLConfigRuntimeErrorsGroup as e:
- with _enter_context(_LOG_TO_CONSOLE, False):
+ with enter_context(_LOG_TO_CONSOLE, False):
logger.critical("Unrecoverable error", exc_info=e.with_traceback(None))
raise
except Exception:
- with _enter_context(_LOG_TO_CONSOLE, False):
+ with enter_context(_LOG_TO_CONSOLE, False):
logger.critical("Unrecoverable error", exc_info=True)
raise
@@ -433,7 +424,7 @@ def disable_console_logging():
else:
listener.stop()
listener.start()
- return _enter_context(_LOG_TO_CONSOLE, False)
+ return enter_context(_LOG_TO_CONSOLE, False)
@contextlib.contextmanager
diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.py
index d09994595..d00373561 100644
--- a/cyberdrop_dl/manager.py
+++ b/cyberdrop_dl/manager.py
@@ -21,10 +21,10 @@ from cyberdrop_dl.database import Database
from cyberdrop_dl.dedupe import Czkawka
from cyberdrop_dl.dependencies import ALL_DEPENDENCIES
from cyberdrop_dl.hasher import Hasher
-from cyberdrop_dl.logs import _enter_context, capture_logs, log_spacer
+from cyberdrop_dl.logs import capture_logs, log_spacer
from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLED
from cyberdrop_dl.sorter import Sorter
-from cyberdrop_dl.utils import get_system_information
+from cyberdrop_dl.utils import enter_context, get_system_information
if TYPE_CHECKING:
from collections.abc import Generator
@@ -93,8 +93,8 @@ class Manager:
self.sorter = Sorter.from_manager(self)
with (
_cache_context(self.appdata.cache_file, self.cache),
- _enter_context(REFRESH_RATE, self.config.ui_options.refresh_rate),
- _enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),
+ enter_context(REFRESH_RATE, self.config.ui_options.refresh_rate),
+ enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),
):
try:
yield self
diff --git a/cyberdrop_dl/progress/scraping/__init__.py b/cyberdrop_dl/progress/scraping/__init__.py
index c0d8482f0..a90f609b6 100644
--- a/cyberdrop_dl/progress/scraping/__init__.py
+++ b/cyberdrop_dl/progress/scraping/__init__.py
@@ -19,6 +19,7 @@ from cyberdrop_dl.progress.scraping.downloads import DownloadsPanel
from cyberdrop_dl.progress.scraping.errors import DownloadErrorsPanel, ScrapeErrorsPanel
from cyberdrop_dl.progress.scraping.files import FileStatsPanel
from cyberdrop_dl.progress.scraping.panel import ScrapingPanel, StatusMessage
+from cyberdrop_dl.utils import enter_context
if TYPE_CHECKING:
from collections.abc import Generator, Iterator
@@ -82,14 +83,13 @@ class ScrapingUI(LiveUI):
@contextlib.contextmanager
def __call__(self, *, transient: bool = True, force: bool = False) -> Generator[None]:
- token = _STATUS.set(self.status)
- if self.mode is not UIOptions.FULLSCREEN:
- transient = False
- try:
- with super(ScrapingUI, self).__call__(transient=transient, force=force):
- yield
- finally:
- _STATUS.reset(token)
+ with (
+ enter_context(_STATUS, self.status),
+ super(ScrapingUI, self).__call__(
+ transient=transient if self.mode is UIOptions.FULLSCREEN else False, force=force
+ ),
+ ):
+ yield
def _create_screen(self) -> Screen:
horizontal, vertical = Layout(), Layout()
diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.py
index 1276a675a..c318270bb 100644
--- a/cyberdrop_dl/progress/scraping/downloads.py
+++ b/cyberdrop_dl/progress/scraping/downloads.py
@@ -35,6 +35,7 @@ from cyberdrop_dl.progress import (
truncate_float,
)
from cyberdrop_dl.progress.overflow import OverFlowPanel
+from cyberdrop_dl.utils import enter_context
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
@@ -176,13 +177,12 @@ class DownloadsPanel(OverFlowPanel):
},
)
- token = _current_hls_task.set(segments_task.id)
- try:
- yield
- finally:
- self._remove_task(segments_task)
- self._hls_progress.remove_task(task_id)
- _current_hls_task.reset(token)
+ with enter_context(_current_hls_task, segments_task.id):
+ try:
+ yield
+ finally:
+ self._remove_task(segments_task)
+ self._hls_progress.remove_task(task_id)
def download_file(
self, description: object, /, domain: str, total: float | None, *, url: AbsoluteHttpURL | None = None
diff --git a/cyberdrop_dl/storage/__init__.py b/cyberdrop_dl/storage/__init__.py
index fe6ea7f76..79d11b6b4 100644
--- a/cyberdrop_dl/storage/__init__.py
+++ b/cyberdrop_dl/storage/__init__.py
@@ -8,6 +8,7 @@ from contextvars import ContextVar
from typing import TYPE_CHECKING
from cyberdrop_dl.exceptions import InsufficientFreeSpaceError
+from cyberdrop_dl.utils import enter_context
logger = logging.getLogger(__name__)
@@ -17,7 +18,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import MediaItem
-_required_free_space: ContextVar[int] = ContextVar("_required_free_space")
+_REQUIRED_FREE_SPACE: ContextVar[int] = ContextVar("_REQUIRED_FREE_SPACE")
def _disk_usage(folder: Path) -> int:
@@ -43,7 +44,7 @@ except ImportError:
async def has_sufficient_space(folder: Path, /, required_free_space: int | None = None) -> bool:
free_space = await _get_free_space(folder)
- return free_space == -1 or free_space > (required_free_space or _required_free_space.get())
+ return free_space == -1 or free_space > (required_free_space or _REQUIRED_FREE_SPACE.get())
def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> Callable[[], Awaitable[None]]:
@@ -61,7 +62,6 @@ def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> C
@contextlib.asynccontextmanager
async def monitor(required_free_space: int) -> AsyncGenerator[None]:
- token = _required_free_space.set(required_free_space)
if _psutil_loop is None:
logger.warning("psutil is not available on this system. Falling back to eager checks for free space")
loop = None
@@ -69,9 +69,9 @@ async def monitor(required_free_space: int) -> AsyncGenerator[None]:
loop = asyncio.create_task(_psutil_loop(), name="storage monitor")
await asyncio.sleep(0)
try:
- yield
+ with enter_context(_REQUIRED_FREE_SPACE, required_free_space):
+ yield
finally:
- _required_free_space.reset(token)
if loop is not None:
try:
_ = loop.cancel("On monitor exit")
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index 5a306a769..7bf3ec04c 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import base64
+import contextlib
import itertools
import logging
import platform
@@ -16,6 +17,7 @@ from cyberdrop_dl.utils._url import remove_trailing_slash # noqa: F401
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
+ from contextvars import ContextVar
from pathlib import Path
from cyberdrop_dl.config import Config
@@ -24,6 +26,15 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+def enter_context[T](context_var: ContextVar[T], value: T, /) -> Generator[None]:
+ token = context_var.set(value)
+ try:
+ yield
+ finally:
+ context_var.reset(token)
+
+
def delete_empty_files_and_folders(path: Path) -> None:
"""walks and removes in place"""
if not path.is_dir():
diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.py
index 6e0a5c291..c2dde6385 100644
--- a/cyberdrop_dl/utils/json.py
+++ b/cyberdrop_dl/utils/json.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import base64
+import contextlib
import dataclasses
import datetime
import enum
@@ -10,11 +11,10 @@ import json.decoder
import json.scanner
import re
import time
-from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, Self, TypeGuard
if TYPE_CHECKING:
- from collections.abc import Callable, Iterable
+ from collections.abc import Callable, Generator, Iterable
from pathlib import Path
from pydantic import BaseModel
@@ -79,15 +79,21 @@ class JSDecoder(json.JSONDecoder):
JSONDecodeError = json.JSONDecodeError
+def _verbose_context() -> Generator[None]:
+ try:
+ yield
+ except json.JSONDecodeError as e:
+ sub_string = e.doc[e.pos - 10 : e.pos + 10]
+ msg = f"{e.msg} at around '{sub_string}', char: '{e.doc[e.pos]}'"
+ raise json.JSONDecodeError(msg, e.doc, e.pos) from None
+
+
def _verbose[**P, R](func: Callable[P, R]) -> Callable[P, R]:
@functools.wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
- try:
+ with _verbose_context():
return func(*args, **kwargs)
- except json.JSONDecodeError as e:
- sub_string = e.doc[e.pos - 10 : e.pos + 10]
- msg = f"{e.msg} at around '{sub_string}', char: '{e.doc[e.pos]}'"
- raise json.JSONDecodeError(msg, e.doc, e.pos) from None
return wrapper
@@ -144,9 +150,9 @@ def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path) -> None:
f.write("\n")
-loads = _verbose(json.loads)
_JS_DECODER = JSDecoder()
_DEFAULT_ENCODER = _get_encoder()
+loads = _verbose(json.loads)
@_verbose
@@ -177,7 +183,7 @@ class JSONWebToken:
@classmethod
def decode(cls, jwt: str, /) -> Self:
- if not is_jwt(jwt):
+ if not looks_like_jwt(jwt):
raise ValueError("Not a valid JSON Web Token", jwt)
b64_headers, b64_payload, b64_signature = jwt.split(".")
headers = cls._decode(b64_headers)
@@ -201,7 +207,7 @@ class JSONWebToken:
return self.encoded
-def is_jwt(string: str) -> bool:
+def looks_like_jwt(string: str) -> bool:
return string.startswith("eyJ") and string.count(".") == 2
diff --git a/pyproject.toml b/pyproject.toml
index a9c762903..f7e04e264 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -167,7 +167,7 @@ ignore = [
]
[tool.ruff.lint.flake8-boolean-trap]
-extend-allowed-calls = ["cyberdrop_dl.logs._enter_context"]
+extend-allowed-calls = ["cyberdrop_dl.utils.enter_context"]
[tool.ruff.lint.flake8-import-conventions]
banned-from = [