Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: move enter_context to utils
NTFSvolume8d2a306c49d1
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 23494c4fb..6f0a40785 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -36,7 +36,7 @@ if TYPE_CHECKING:from cyberdrop_dl.manager import Managerfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-_JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("_JSON_CHECK", default=None)+JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("JSON_CHECK", default=None)RequestContext = contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]] # pyright: ignore[reportPrivateUsage]logger = logging.getLogger(__name__)@@ -318,20 +318,12 @@ class HTTPClient:flaresolverr.verify_solution(self.config.user_agent, solution)return AbstractResponse.create(solution)- @contextlib.contextmanager- def json_context(self, check: Callable[[Any, AbstractResponse[Any]], None], /):- token = _JSON_CHECK.set(check)- try:- yield- finally:- _JSON_CHECK.reset(token)-async def _check_json(response: AbstractResponse[Any]) -> None:if "json" not in response.content_type:return- if check := _JSON_CHECK.get():+ if check := JSON_CHECK.get():check(await response.json(), response)returndiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 0476de815..64bcfb735 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -17,20 +17,24 @@ from types import MappingProxyTypefrom typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, Self, finalfrom cyberdrop_dl import aio, env, signature-from cyberdrop_dl.clients.http import HTTPClient, HTTPMixin, RequestContext+from cyberdrop_dl.clients.http import JSON_CHECK, HTTPClient, HTTPMixin, RequestContextfrom cyberdrop_dl.constants import CDL_USER_AGENTfrom cyberdrop_dl.crawlers._hls import HLSMixinfrom cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError-from cyberdrop_dl.filepath import (- check_dangerous_filename,- check_path_traversal,- compose_filename,- get_filename_and_ext,-)+from cyberdrop_dl.filepath import check_dangerous_filename, check_path_traversal, compose_filename, get_filename_and_extfrom cyberdrop_dl.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, is_absolute_http_url-from cyberdrop_dl.utils import css, dates, error_handling_context, is_blob_or_svg, m3u8, parse_url, unique+from cyberdrop_dl.utils import (+ css,+ dates,+ enter_context,+ error_handling_context,+ is_blob_or_svg,+ m3u8,+ parse_url,+ unique,+)from cyberdrop_dl.utils.strings import safe_formatif TYPE_CHECKING:@@ -321,7 +325,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):if impersonate is None:impersonate = self._IMPERSONATE- with self.client.json_context(self.__json_resp_check__):+ with enter_context(JSON_CHECK, self.__json_resp_check__):async with (self.client.rate_limits[self.DOMAIN],self.client.global_rate_limiter,diff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.pyindex eff54d06a..4a36a8417 100644--- a/cyberdrop_dl/logs.py+++ b/cyberdrop_dl/logs.py@@ -21,7 +21,7 @@ from rich.text import Text, TextTypefrom cyberdrop_dl import envfrom cyberdrop_dl.constants import MAIN_LOG_FILEfrom cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup-from cyberdrop_dl.utils import dates+from cyberdrop_dl.utils import dates, enter_contextif TYPE_CHECKING:import datetime@@ -195,7 +195,7 @@ def _threaded_logger(q_listener: QueueListener = QueueListener(q, log_handler, respect_handler_level=True)q_listener.start()- with _enter_context(context_var, q_listener) if context_var else contextlib.nullcontext():+ with enter_context(context_var, q_listener) if context_var else contextlib.nullcontext():logging.getLogger().addHandler(q_handler)try:yield q_handler@@ -209,15 +209,6 @@ def _threaded_logger(handler.close()-def _enter_context[T](context_var: ContextVar[T], value: T, /) -> Generator[None]:- token = context_var.set(value)- try:- yield- finally:- context_var.reset(token)--@finalclass NoPaddingLogRender(LogRender):_cdl_padding: int = 0@@ -333,9 +324,9 @@ def setup_file_logging(file: Path, /, *, level: int = logging.DEBUG) -> Generatowith (_setup_debug_logger() as debug_log_file,file.open("w", encoding="utf8") as fp,- _enter_context(MAIN_LOG_FILE, file),- _enter_context(mega.LOG_HTTP_TRAFFIC, True),- _enter_context(mega.LOG_FILE_PROGRESS, False),+ enter_context(MAIN_LOG_FILE, file),+ enter_context(mega.LOG_HTTP_TRAFFIC, True),+ enter_context(mega.LOG_FILE_PROGRESS, False),_threaded_logger(log_handler=LogHandler(level=logging.DEBUG,@@ -351,11 +342,11 @@ def setup_file_logging(file: Path, /, *, level: int = logging.DEBUG) -> Generatotry:yieldexcept CDLConfigRuntimeErrorsGroup as e:- with _enter_context(_LOG_TO_CONSOLE, False):+ with enter_context(_LOG_TO_CONSOLE, False):logger.critical("Unrecoverable error", exc_info=e.with_traceback(None))raiseexcept Exception:- with _enter_context(_LOG_TO_CONSOLE, False):+ with enter_context(_LOG_TO_CONSOLE, False):logger.critical("Unrecoverable error", exc_info=True)raise@@ -433,7 +424,7 @@ def disable_console_logging():else:listener.stop()listener.start()- return _enter_context(_LOG_TO_CONSOLE, False)+ return enter_context(_LOG_TO_CONSOLE, False)@contextlib.contextmanagerdiff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.pyindex d09994595..d00373561 100644--- a/cyberdrop_dl/manager.py+++ b/cyberdrop_dl/manager.py@@ -21,10 +21,10 @@ from cyberdrop_dl.database import Databasefrom cyberdrop_dl.dedupe import Czkawkafrom cyberdrop_dl.dependencies import ALL_DEPENDENCIESfrom cyberdrop_dl.hasher import Hasher-from cyberdrop_dl.logs import _enter_context, capture_logs, log_spacer+from cyberdrop_dl.logs import capture_logs, log_spacerfrom cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLEDfrom cyberdrop_dl.sorter import Sorter-from cyberdrop_dl.utils import get_system_information+from cyberdrop_dl.utils import enter_context, get_system_informationif TYPE_CHECKING:from collections.abc import Generator@@ -93,8 +93,8 @@ class Manager:self.sorter = Sorter.from_manager(self)with (_cache_context(self.appdata.cache_file, self.cache),- _enter_context(REFRESH_RATE, self.config.ui_options.refresh_rate),- _enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),+ enter_context(REFRESH_RATE, self.config.ui_options.refresh_rate),+ enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),):try:yield selfdiff --git a/cyberdrop_dl/progress/scraping/__init__.py b/cyberdrop_dl/progress/scraping/__init__.pyindex c0d8482f0..a90f609b6 100644--- a/cyberdrop_dl/progress/scraping/__init__.py+++ b/cyberdrop_dl/progress/scraping/__init__.py@@ -19,6 +19,7 @@ from cyberdrop_dl.progress.scraping.downloads import DownloadsPanelfrom cyberdrop_dl.progress.scraping.errors import DownloadErrorsPanel, ScrapeErrorsPanelfrom cyberdrop_dl.progress.scraping.files import FileStatsPanelfrom cyberdrop_dl.progress.scraping.panel import ScrapingPanel, StatusMessage+from cyberdrop_dl.utils import enter_contextif TYPE_CHECKING:from collections.abc import Generator, Iterator@@ -82,14 +83,13 @@ class ScrapingUI(LiveUI):@contextlib.contextmanagerdef __call__(self, *, transient: bool = True, force: bool = False) -> Generator[None]:- token = _STATUS.set(self.status)- if self.mode is not UIOptions.FULLSCREEN:- transient = False- try:- with super(ScrapingUI, self).__call__(transient=transient, force=force):- yield- finally:- _STATUS.reset(token)+ with (+ enter_context(_STATUS, self.status),+ super(ScrapingUI, self).__call__(+ transient=transient if self.mode is UIOptions.FULLSCREEN else False, force=force+ ),+ ):+ yielddef _create_screen(self) -> Screen:horizontal, vertical = Layout(), Layout()diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.pyindex 1276a675a..c318270bb 100644--- a/cyberdrop_dl/progress/scraping/downloads.py+++ b/cyberdrop_dl/progress/scraping/downloads.py@@ -35,6 +35,7 @@ from cyberdrop_dl.progress import (truncate_float,)from cyberdrop_dl.progress.overflow import OverFlowPanel+from cyberdrop_dl.utils import enter_contextif TYPE_CHECKING:from collections.abc import Generator, Iterable@@ -176,13 +177,12 @@ class DownloadsPanel(OverFlowPanel):},)- token = _current_hls_task.set(segments_task.id)- try:- yield- finally:- self._remove_task(segments_task)- self._hls_progress.remove_task(task_id)- _current_hls_task.reset(token)+ with enter_context(_current_hls_task, segments_task.id):+ try:+ yield+ finally:+ self._remove_task(segments_task)+ self._hls_progress.remove_task(task_id)def download_file(self, description: object, /, domain: str, total: float | None, *, url: AbsoluteHttpURL | None = Nonediff --git a/cyberdrop_dl/storage/__init__.py b/cyberdrop_dl/storage/__init__.pyindex fe6ea7f76..79d11b6b4 100644--- a/cyberdrop_dl/storage/__init__.py+++ b/cyberdrop_dl/storage/__init__.py@@ -8,6 +8,7 @@ from contextvars import ContextVarfrom typing import TYPE_CHECKINGfrom cyberdrop_dl.exceptions import InsufficientFreeSpaceError+from cyberdrop_dl.utils import enter_contextlogger = logging.getLogger(__name__)@@ -17,7 +18,7 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import MediaItem-_required_free_space: ContextVar[int] = ContextVar("_required_free_space")+_REQUIRED_FREE_SPACE: ContextVar[int] = ContextVar("_REQUIRED_FREE_SPACE")def _disk_usage(folder: Path) -> int:@@ -43,7 +44,7 @@ except ImportError:async def has_sufficient_space(folder: Path, /, required_free_space: int | None = None) -> bool:free_space = await _get_free_space(folder)- return free_space == -1 or free_space > (required_free_space or _required_free_space.get())+ return free_space == -1 or free_space > (required_free_space or _REQUIRED_FREE_SPACE.get())def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> Callable[[], Awaitable[None]]:@@ -61,7 +62,6 @@ def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> C@contextlib.asynccontextmanagerasync def monitor(required_free_space: int) -> AsyncGenerator[None]:- token = _required_free_space.set(required_free_space)if _psutil_loop is None:logger.warning("psutil is not available on this system. Falling back to eager checks for free space")loop = None@@ -69,9 +69,9 @@ async def monitor(required_free_space: int) -> AsyncGenerator[None]:loop = asyncio.create_task(_psutil_loop(), name="storage monitor")await asyncio.sleep(0)try:- yield+ with enter_context(_REQUIRED_FREE_SPACE, required_free_space):+ yieldfinally:- _required_free_space.reset(token)if loop is not None:try:_ = loop.cancel("On monitor exit")diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex 5a306a769..7bf3ec04c 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -1,6 +1,7 @@from __future__ import annotationsimport base64+import contextlibimport itertoolsimport loggingimport platform@@ -16,6 +17,7 @@ from cyberdrop_dl.utils._url import remove_trailing_slash # noqa: F401if TYPE_CHECKING:from collections.abc import Generator, Iterable+ from contextvars import ContextVarfrom pathlib import Pathfrom cyberdrop_dl.config import Config@@ -24,6 +26,15 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+def enter_context[T](context_var: ContextVar[T], value: T, /) -> Generator[None]:+ token = context_var.set(value)+ try:+ yield+ finally:+ context_var.reset(token)++def delete_empty_files_and_folders(path: Path) -> None:"""walks and removes in place"""if not path.is_dir():diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.pyindex 6e0a5c291..c2dde6385 100644--- a/cyberdrop_dl/utils/json.py+++ b/cyberdrop_dl/utils/json.py@@ -1,6 +1,7 @@from __future__ import annotationsimport base64+import contextlibimport dataclassesimport datetimeimport enum@@ -10,11 +11,10 @@ import json.decoderimport json.scannerimport reimport time-from collections.abc import Iterablefrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, Self, TypeGuardif TYPE_CHECKING:- from collections.abc import Callable, Iterable+ from collections.abc import Callable, Generator, Iterablefrom pathlib import Pathfrom pydantic import BaseModel@@ -79,15 +79,21 @@ class JSDecoder(json.JSONDecoder):JSONDecodeError = json.JSONDecodeError+def _verbose_context() -> Generator[None]:+ try:+ yield+ except json.JSONDecodeError as e:+ sub_string = e.doc[e.pos - 10 : e.pos + 10]+ msg = f"{e.msg} at around '{sub_string}', char: '{e.doc[e.pos]}'"+ raise json.JSONDecodeError(msg, e.doc, e.pos) from None++def _verbose[**P, R](func: Callable[P, R]) -> Callable[P, R]:@functools.wraps(func)def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:- try:+ with _verbose_context():return func(*args, **kwargs)- except json.JSONDecodeError as e:- sub_string = e.doc[e.pos - 10 : e.pos + 10]- msg = f"{e.msg} at around '{sub_string}', char: '{e.doc[e.pos]}'"- raise json.JSONDecodeError(msg, e.doc, e.pos) from Nonereturn wrapper@@ -144,9 +150,9 @@ def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path) -> None:f.write("\n")-loads = _verbose(json.loads)_JS_DECODER = JSDecoder()_DEFAULT_ENCODER = _get_encoder()+loads = _verbose(json.loads)@_verbose@@ -177,7 +183,7 @@ class JSONWebToken:@classmethoddef decode(cls, jwt: str, /) -> Self:- if not is_jwt(jwt):+ if not looks_like_jwt(jwt):raise ValueError("Not a valid JSON Web Token", jwt)b64_headers, b64_payload, b64_signature = jwt.split(".")headers = cls._decode(b64_headers)@@ -201,7 +207,7 @@ class JSONWebToken:return self.encoded-def is_jwt(string: str) -> bool:+def looks_like_jwt(string: str) -> bool:return string.startswith("eyJ") and string.count(".") == 2diff --git a/pyproject.toml b/pyproject.tomlindex a9c762903..f7e04e264 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -167,7 +167,7 @@ ignore = [][tool.ruff.lint.flake8-boolean-trap]-extend-allowed-calls = ["cyberdrop_dl.logs._enter_context"]+extend-allowed-calls = ["cyberdrop_dl.utils.enter_context"][tool.ruff.lint.flake8-import-conventions]banned-from = [