Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: unify aio cache with TTL cache

NTFSvolumebe15468d1a32
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.py
index f3591cfcb..f32ed0d33 100644
--- a/cyberdrop_dl/aio.py
+++ b/cyberdrop_dl/aio.py
@@ -6,7 +6,6 @@ from __future__ import annotations
import asyncio
import contextlib
import dataclasses
-import functools
import shutil
import time
from pathlib import Path
@@ -15,7 +14,6 @@ from typing import IO, TYPE_CHECKING, Any, Self, cast, overload
from weakref import WeakValueDictionary
from aiolimiter.leakybucket import AsyncLimiter
-from typing_extensions import Sentinel
from cyberdrop_dl.constants import MISSING
@@ -31,7 +29,6 @@ if TYPE_CHECKING:
Iterator,
Sequence,
)
- from types import CoroutineType
from _typeshed import OpenBinaryMode, OpenTextMode
@@ -120,55 +117,6 @@ class _CachedValue[T]:
return time.monotonic() - self.created_at >= self.ttl
-def cache_wrapper[T](
- *, ttl: float | None = None
-) -> Callable[[Callable[[], Awaitable[T]]], Callable[[], CoroutineType[Any, Any, T]]]:
-
- return lambda x: cached(x, ttl=ttl)
-
-
-def cached[T](fn: Callable[[], Awaitable[T]], *, ttl: float | None = None) -> Callable[[], CoroutineType[Any, Any, T]]:
- if ttl is None:
- return _perpetual_cache(fn)
-
- lock = asyncio.Lock()
- cached_value: _CachedValue[T] | None = None
-
- @functools.wraps(fn)
- async def wrapper() -> T:
- nonlocal cached_value
- async with lock:
- if cached_value is not None and not cached_value.has_expired:
- return cached_value.value
-
- cached_value = _CachedValue(await fn(), ttl=ttl)
-
- return cached_value.value
-
- return wrapper
-
-
-def _perpetual_cache[T](fn: Callable[[], Awaitable[T]]) -> Callable[[], CoroutineType[Any, Any, T]]:
- lock = asyncio.Lock()
- cached_value: T | Sentinel = MISSING
-
- @functools.wraps(fn)
- async def wrapper() -> T:
- nonlocal cached_value
- if cached_value is not MISSING:
- return cast("T", cached_value)
-
- async with lock:
- if cached_value is not MISSING:
- return cast("T", cached_value)
-
- cached_value = await fn()
-
- return cached_value
-
- return wrapper
-
-
@dataclasses.dataclass(slots=True, eq=False)
class AsyncIOWrapper[AnyStr: (bytes, str)]:
"""An asynchronous context manager wrapper for a file object."""
diff --git a/cyberdrop_dl/cache.py b/cyberdrop_dl/cache.py
index 92b632459..d6b791211 100644
--- a/cyberdrop_dl/cache.py
+++ b/cyberdrop_dl/cache.py
@@ -1,19 +1,24 @@
from __future__ import annotations
+import asyncio
import contextlib
import dataclasses
+import functools
import json
import logging
import time
-from typing import TYPE_CHECKING, Any, TypedDict
+from typing import TYPE_CHECKING, Any, Protocol, TypedDict, cast
from cyberdrop_dl import __version__
+from cyberdrop_dl.constants import MISSING
if TYPE_CHECKING:
- from collections.abc import Generator
+ from collections.abc import Awaitable, Callable, Generator
from pathlib import Path
+ from types import CoroutineType
logger = logging.getLogger(__name__)
+_IN_MEMORY_CACHE: dict[str, Any] = {}
class _CachedValue[T](TypedDict):
@@ -32,7 +37,7 @@ def _has_expired(self: _CachedValue[Any]) -> bool:
class TTLCacheAdapter[T]:
_cache: dict[str, Any]
_keys: tuple[str, ...] = ()
- _root: dict[str, _CachedValue[Any]] = dataclasses.field(init=False)
+ _root: dict[str, _CachedValue[Any]] = dataclasses.field(init=False, repr=False)
def create_lookup_path(self, *keys: str) -> str:
return ".".join([*self._keys, *keys])
@@ -56,15 +61,15 @@ class TTLCacheAdapter[T]:
def __getitem__(self, key: str, /) -> T:
cache_hit = self._get(key)
- if cache_hit is None:
+ if cache_hit is MISSING:
raise KeyError(key)
return cache_hit["value"]
- def _get(self, key: str) -> _CachedValue[T] | None:
+ def _get(self, key: str) -> _CachedValue[T] | MISSING: # pyright: ignore[reportInvalidTypeForm]
try:
cache_hit = self.root[key]
except KeyError:
- return None
+ return MISSING
try:
expired = _has_expired(cache_hit)
@@ -74,7 +79,7 @@ class TTLCacheAdapter[T]:
if expired:
del self[key]
- return None
+ return MISSING
return cache_hit
def __delitem__(self, key: str) -> None:
@@ -82,7 +87,7 @@ class TTLCacheAdapter[T]:
def get(self, key: str) -> T | None:
cache_hit = self._get(key)
- return None if cache_hit is None else cache_hit["value"]
+ return None if cache_hit is MISSING else cache_hit["value"]
def save(self, key: str, value: T, *, ttl: float | None = None) -> None:
"""NOTE: cached values MUST be JSON serializable"""
@@ -96,6 +101,12 @@ class TTLCacheAdapter[T]:
"""NOTE: cached values MUST be JSON serializable"""
self.save(name, value, ttl=None)
+ def discard(self, key: str) -> None:
+ try:
+ del self[key]
+ except KeyError:
+ return
+
@contextlib.contextmanager
def cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:
@@ -114,3 +125,44 @@ def cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:
cache["version"] = __version__
cache_file.parent.mkdir(parents=True, exist_ok=True)
cache_file.write_text(json.dumps(cache, indent=2, ensure_ascii=False, sort_keys=True))
+
+
+class CachedAsyncFunc[T](Protocol):
+ def __call__(self) -> CoroutineType[Any, Any, T]: ...
+
+ def clear(self) -> None: ...
+
+
+def cached_fn[T](
+ fn: Callable[[], Awaitable[T]],
+ cache: TTLCacheAdapter[T] | None = None,
+ *,
+ key: str | None = None,
+ ttl: float | None = None,
+) -> CachedAsyncFunc[T]:
+
+ lock = asyncio.Lock()
+
+ *parts, fn_name = ".".join([fn.__module__, fn.__qualname__]).split(".")
+ key = key or fn_name
+ cache = cache or TTLCacheAdapter(_IN_MEMORY_CACHE, tuple(parts))
+
+ @functools.wraps(fn)
+ async def wrapper() -> T:
+ try:
+ return cache[key]
+ except KeyError:
+ pass
+
+ async with lock:
+ try:
+ return cache[key]
+ except KeyError:
+ pass
+
+ value = await fn()
+ cache.save(key, value, ttl=ttl)
+ return value
+
+ wrapper.clear = lambda: cache.discard(key) # pyright: ignore[reportAttributeAccessIssue]
+ return cast("CachedAsyncFunc[T]", cast("object", wrapper))
diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.py
index c5b7dd278..da662532d 100644
--- a/cyberdrop_dl/crawlers/hitomi_la.py
+++ b/cyberdrop_dl/crawlers/hitomi_la.py
@@ -11,6 +11,7 @@ from collections import defaultdict
from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl import aio
+from cyberdrop_dl.cache import cached_fn
from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.filepath import get_filename_and_ext
@@ -219,7 +220,7 @@ class HitomiAPI(API):
}
def __post_init__(self) -> None:
- self.servers = aio.cached(self.servers, ttl=40 * 60)
+ self.servers = cached_fn(self.servers, ttl=40 * 60)
async def servers(self) -> Servers:
# https://ltn.gold-usergeneratedcontent.net/gg.js
diff --git a/tests/test_aio.py b/tests/test_aio.py
deleted file mode 100644
index 75e4c0764..000000000
--- a/tests/test_aio.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-
-from cyberdrop_dl import aio
-
-
-class Spy:
- def __init__(self, value: object) -> None:
- self.value: object = value
- self.calls: int = 0
-
- async def __call__(self) -> object:
- self.calls += 1
- return self.value
-
-
-async def test_cache_returns_same_value_while_not_expired() -> None:
- ttl = 2.0
- spy = Spy("test 1")
- factory = aio.cache_wrapper(ttl=ttl)(spy)
- first = await factory()
- second = await factory()
-
- assert first == second
- assert spy.calls == 1
- await asyncio.sleep(ttl + 0.1)
- _ = await factory()
- assert spy.calls == 2
-
-
-async def test_concurrent_calls_execute_only_once() -> None:
- spy = Spy("test 2")
-
- @aio.cache_wrapper(ttl=60)
- async def factory() -> object:
- await asyncio.sleep(0.02)
- return await spy()
-
- results = await asyncio.gather(*(factory() for _ in range(10)))
- assert len(set(results)) == 1
- assert spy.calls == 1
diff --git a/tests/test_cache.py b/tests/test_cache.py
index db22e6724..fbdfd38ab 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -1,11 +1,13 @@
+import asyncio
import time
from pathlib import Path
from typing import Any
+from unittest import mock
import pytest
from cyberdrop_dl import __version__
-from cyberdrop_dl.cache import TTLCacheAdapter, cache_context
+from cyberdrop_dl.cache import _IN_MEMORY_CACHE, TTLCacheAdapter, cache_context, cached_fn
from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.manager import Manager
from cyberdrop_dl.utils import json
@@ -101,3 +103,83 @@ def test_invalid_cache(logs: pytest.LogCaptureFixture) -> None:
},
}
assert logs.messages == ["Invalid cache entry a.b, ignoring"]
+
+
+async def test_cache_fn() -> None:
+
+ call = 0
+
+ async def update() -> int:
+ nonlocal call
+ call += 1
+ return call
+
+ fn = cached_fn(update)
+ assert await fn() == 1
+ assert await fn() == 1
+
+ fn = cached_fn(update, ttl=0.2)
+ assert await fn() == 1
+ assert await fn() == 1
+
+ fn2 = cached_fn(update, key="update2", ttl=0.2)
+ assert await fn2() == 2
+ assert await fn2() == 2
+ await asyncio.sleep(0.2 + 0.1)
+ assert await fn2() == 3
+ assert _IN_MEMORY_CACHE["test_cache"]["test_cache_fn"]["<locals>"] == {
+ "update": {
+ "value": 1,
+ "ttl": None,
+ "created_at": mock.ANY,
+ },
+ "update2": {
+ "value": 3,
+ "ttl": 0.2,
+ "created_at": mock.ANY,
+ },
+ }
+ fn.clear()
+ assert "update" not in _IN_MEMORY_CACHE["test_cache"]["test_cache_fn"]["<locals>"]
+ assert await fn() == 4
+
+
+class Spy:
+ def __init__(self, value: object) -> None:
+ self.value: object = value
+ self.calls: int = 0
+
+ async def __call__(self) -> object:
+ self.calls += 1
+ return self.value
+
+
+async def test_cached_fn_concurrent_calls_execute_only_once() -> None:
+ spy = Spy("test 2")
+
+ async def factory() -> object:
+ await asyncio.sleep(0.02)
+ return await spy()
+
+ factory = cached_fn(factory, ttl=60)
+ results = await asyncio.gather(*(factory() for _ in range(10)))
+ assert len(set(results)) == 1
+ assert spy.calls == 1
+
+
+async def test_cache_returns_same_value_while_not_expired() -> None:
+ ttl = 0.5
+ spy_ = Spy("test 1")
+
+ def spy():
+ return spy_()
+
+ factory = cached_fn(spy, ttl=ttl)
+ first = await factory()
+ second = await factory()
+
+ assert first == second
+ assert spy_.calls == 1
+ await asyncio.sleep(ttl + 0.1)
+ _ = await factory()
+ assert spy_.calls == 2