Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: unify aio cache with TTL cache
NTFSvolumebe15468d1a32
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.pyindex f3591cfcb..f32ed0d33 100644--- a/cyberdrop_dl/aio.py+++ b/cyberdrop_dl/aio.py@@ -6,7 +6,6 @@ from __future__ import annotationsimport asyncioimport contextlibimport dataclasses-import functoolsimport shutilimport timefrom pathlib import Path@@ -15,7 +14,6 @@ from typing import IO, TYPE_CHECKING, Any, Self, cast, overloadfrom weakref import WeakValueDictionaryfrom aiolimiter.leakybucket import AsyncLimiter-from typing_extensions import Sentinelfrom cyberdrop_dl.constants import MISSING@@ -31,7 +29,6 @@ if TYPE_CHECKING:Iterator,Sequence,)- from types import CoroutineTypefrom _typeshed import OpenBinaryMode, OpenTextMode@@ -120,55 +117,6 @@ class _CachedValue[T]:return time.monotonic() - self.created_at >= self.ttl-def cache_wrapper[T](- *, ttl: float | None = None-) -> Callable[[Callable[[], Awaitable[T]]], Callable[[], CoroutineType[Any, Any, T]]]:-- return lambda x: cached(x, ttl=ttl)---def cached[T](fn: Callable[[], Awaitable[T]], *, ttl: float | None = None) -> Callable[[], CoroutineType[Any, Any, T]]:- if ttl is None:- return _perpetual_cache(fn)-- lock = asyncio.Lock()- cached_value: _CachedValue[T] | None = None-- @functools.wraps(fn)- async def wrapper() -> T:- nonlocal cached_value- async with lock:- if cached_value is not None and not cached_value.has_expired:- return cached_value.value-- cached_value = _CachedValue(await fn(), ttl=ttl)-- return cached_value.value-- return wrapper---def _perpetual_cache[T](fn: Callable[[], Awaitable[T]]) -> Callable[[], CoroutineType[Any, Any, T]]:- lock = asyncio.Lock()- cached_value: T | Sentinel = MISSING-- @functools.wraps(fn)- async def wrapper() -> T:- nonlocal cached_value- if cached_value is not MISSING:- return cast("T", cached_value)-- async with lock:- if cached_value is not MISSING:- return cast("T", cached_value)-- cached_value = await fn()-- return cached_value-- return wrapper--@dataclasses.dataclass(slots=True, eq=False)class AsyncIOWrapper[AnyStr: (bytes, str)]:"""An asynchronous context manager wrapper for a file object."""diff --git a/cyberdrop_dl/cache.py b/cyberdrop_dl/cache.pyindex 92b632459..d6b791211 100644--- a/cyberdrop_dl/cache.py+++ b/cyberdrop_dl/cache.py@@ -1,19 +1,24 @@from __future__ import annotations+import asyncioimport contextlibimport dataclasses+import functoolsimport jsonimport loggingimport time-from typing import TYPE_CHECKING, Any, TypedDict+from typing import TYPE_CHECKING, Any, Protocol, TypedDict, castfrom cyberdrop_dl import __version__+from cyberdrop_dl.constants import MISSINGif TYPE_CHECKING:- from collections.abc import Generator+ from collections.abc import Awaitable, Callable, Generatorfrom pathlib import Path+ from types import CoroutineTypelogger = logging.getLogger(__name__)+_IN_MEMORY_CACHE: dict[str, Any] = {}class _CachedValue[T](TypedDict):@@ -32,7 +37,7 @@ def _has_expired(self: _CachedValue[Any]) -> bool:class TTLCacheAdapter[T]:_cache: dict[str, Any]_keys: tuple[str, ...] = ()- _root: dict[str, _CachedValue[Any]] = dataclasses.field(init=False)+ _root: dict[str, _CachedValue[Any]] = dataclasses.field(init=False, repr=False)def create_lookup_path(self, *keys: str) -> str:return ".".join([*self._keys, *keys])@@ -56,15 +61,15 @@ class TTLCacheAdapter[T]:def __getitem__(self, key: str, /) -> T:cache_hit = self._get(key)- if cache_hit is None:+ if cache_hit is MISSING:raise KeyError(key)return cache_hit["value"]- def _get(self, key: str) -> _CachedValue[T] | None:+ def _get(self, key: str) -> _CachedValue[T] | MISSING: # pyright: ignore[reportInvalidTypeForm]try:cache_hit = self.root[key]except KeyError:- return None+ return MISSINGtry:expired = _has_expired(cache_hit)@@ -74,7 +79,7 @@ class TTLCacheAdapter[T]:if expired:del self[key]- return None+ return MISSINGreturn cache_hitdef __delitem__(self, key: str) -> None:@@ -82,7 +87,7 @@ class TTLCacheAdapter[T]:def get(self, key: str) -> T | None:cache_hit = self._get(key)- return None if cache_hit is None else cache_hit["value"]+ return None if cache_hit is MISSING else cache_hit["value"]def save(self, key: str, value: T, *, ttl: float | None = None) -> None:"""NOTE: cached values MUST be JSON serializable"""@@ -96,6 +101,12 @@ class TTLCacheAdapter[T]:"""NOTE: cached values MUST be JSON serializable"""self.save(name, value, ttl=None)+ def discard(self, key: str) -> None:+ try:+ del self[key]+ except KeyError:+ return+@contextlib.contextmanagerdef cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:@@ -114,3 +125,44 @@ def cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:cache["version"] = __version__cache_file.parent.mkdir(parents=True, exist_ok=True)cache_file.write_text(json.dumps(cache, indent=2, ensure_ascii=False, sort_keys=True))+++class CachedAsyncFunc[T](Protocol):+ def __call__(self) -> CoroutineType[Any, Any, T]: ...++ def clear(self) -> None: ...+++def cached_fn[T](+ fn: Callable[[], Awaitable[T]],+ cache: TTLCacheAdapter[T] | None = None,+ *,+ key: str | None = None,+ ttl: float | None = None,+) -> CachedAsyncFunc[T]:++ lock = asyncio.Lock()++ *parts, fn_name = ".".join([fn.__module__, fn.__qualname__]).split(".")+ key = key or fn_name+ cache = cache or TTLCacheAdapter(_IN_MEMORY_CACHE, tuple(parts))++ @functools.wraps(fn)+ async def wrapper() -> T:+ try:+ return cache[key]+ except KeyError:+ pass++ async with lock:+ try:+ return cache[key]+ except KeyError:+ pass++ value = await fn()+ cache.save(key, value, ttl=ttl)+ return value++ wrapper.clear = lambda: cache.discard(key) # pyright: ignore[reportAttributeAccessIssue]+ return cast("CachedAsyncFunc[T]", cast("object", wrapper))diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.pyindex c5b7dd278..da662532d 100644--- a/cyberdrop_dl/crawlers/hitomi_la.py+++ b/cyberdrop_dl/crawlers/hitomi_la.py@@ -11,6 +11,7 @@ from collections import defaultdictfrom typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl import aio+from cyberdrop_dl.cache import cached_fnfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.filepath import get_filename_and_ext@@ -219,7 +220,7 @@ class HitomiAPI(API):}def __post_init__(self) -> None:- self.servers = aio.cached(self.servers, ttl=40 * 60)+ self.servers = cached_fn(self.servers, ttl=40 * 60)async def servers(self) -> Servers:# https://ltn.gold-usergeneratedcontent.net/gg.jsdiff --git a/tests/test_aio.py b/tests/test_aio.pydeleted file mode 100644index 75e4c0764..000000000--- a/tests/test_aio.py+++ /dev/null@@ -1,42 +0,0 @@-from __future__ import annotations--import asyncio--from cyberdrop_dl import aio---class Spy:- def __init__(self, value: object) -> None:- self.value: object = value- self.calls: int = 0-- async def __call__(self) -> object:- self.calls += 1- return self.value---async def test_cache_returns_same_value_while_not_expired() -> None:- ttl = 2.0- spy = Spy("test 1")- factory = aio.cache_wrapper(ttl=ttl)(spy)- first = await factory()- second = await factory()-- assert first == second- assert spy.calls == 1- await asyncio.sleep(ttl + 0.1)- _ = await factory()- assert spy.calls == 2---async def test_concurrent_calls_execute_only_once() -> None:- spy = Spy("test 2")-- @aio.cache_wrapper(ttl=60)- async def factory() -> object:- await asyncio.sleep(0.02)- return await spy()-- results = await asyncio.gather(*(factory() for _ in range(10)))- assert len(set(results)) == 1- assert spy.calls == 1diff --git a/tests/test_cache.py b/tests/test_cache.pyindex db22e6724..fbdfd38ab 100644--- a/tests/test_cache.py+++ b/tests/test_cache.py@@ -1,11 +1,13 @@+import asyncioimport timefrom pathlib import Pathfrom typing import Any+from unittest import mockimport pytestfrom cyberdrop_dl import __version__-from cyberdrop_dl.cache import TTLCacheAdapter, cache_context+from cyberdrop_dl.cache import _IN_MEMORY_CACHE, TTLCacheAdapter, cache_context, cached_fnfrom cyberdrop_dl.config.appdata import AppDatafrom cyberdrop_dl.manager import Managerfrom cyberdrop_dl.utils import json@@ -101,3 +103,83 @@ def test_invalid_cache(logs: pytest.LogCaptureFixture) -> None:},}assert logs.messages == ["Invalid cache entry a.b, ignoring"]+++async def test_cache_fn() -> None:++ call = 0++ async def update() -> int:+ nonlocal call+ call += 1+ return call++ fn = cached_fn(update)+ assert await fn() == 1+ assert await fn() == 1++ fn = cached_fn(update, ttl=0.2)+ assert await fn() == 1+ assert await fn() == 1++ fn2 = cached_fn(update, key="update2", ttl=0.2)+ assert await fn2() == 2+ assert await fn2() == 2+ await asyncio.sleep(0.2 + 0.1)+ assert await fn2() == 3+ assert _IN_MEMORY_CACHE["test_cache"]["test_cache_fn"]["<locals>"] == {+ "update": {+ "value": 1,+ "ttl": None,+ "created_at": mock.ANY,+ },+ "update2": {+ "value": 3,+ "ttl": 0.2,+ "created_at": mock.ANY,+ },+ }+ fn.clear()+ assert "update" not in _IN_MEMORY_CACHE["test_cache"]["test_cache_fn"]["<locals>"]+ assert await fn() == 4+++class Spy:+ def __init__(self, value: object) -> None:+ self.value: object = value+ self.calls: int = 0++ async def __call__(self) -> object:+ self.calls += 1+ return self.value+++async def test_cached_fn_concurrent_calls_execute_only_once() -> None:+ spy = Spy("test 2")++ async def factory() -> object:+ await asyncio.sleep(0.02)+ return await spy()++ factory = cached_fn(factory, ttl=60)+ results = await asyncio.gather(*(factory() for _ in range(10)))+ assert len(set(results)) == 1+ assert spy.calls == 1+++async def test_cache_returns_same_value_while_not_expired() -> None:+ ttl = 0.5+ spy_ = Spy("test 1")++ def spy():+ return spy_()++ factory = cached_fn(spy, ttl=ttl)+ first = await factory()+ second = await factory()++ assert first == second+ assert spy_.calls == 1+ await asyncio.sleep(ttl + 0.1)+ _ = await factory()+ assert spy_.calls == 2