Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: cache imgur client id

NTFSvolume22d3e3e79ea3
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 6806ff09b..9c5a2af44 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -891,12 +891,14 @@ class API(HTTPMixin, ABC):
PRIMARY_URL: AbsoluteHttpURL, # noqa: N803
config: Config,
request: Callable[..., RequestContext],
+ cache: TTLCacheAdapter[Any],
parse_url: Callable[[str | yarl.URL], AbsoluteHttpURL] = parse_url,
) -> None:
self.PRIMARY_URL: Final = PRIMARY_URL
self.parse_url: Final = parse_url
self.request: Final = request
self.config: Final = config
+ self.cache: Final = cache
self.__post_init__()
@classmethod
@@ -905,6 +907,7 @@ class API(HTTPMixin, ABC):
PRIMARY_URL=crawler.PRIMARY_URL,
parse_url=crawler.parse_url,
request=crawler.request,
+ cache=crawler.cache,
config=crawler.manager.config,
)
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index b73144d6e..915115a7b 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, ClassVar
+from cyberdrop_dl.cache import disk_cached_method
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -38,11 +39,14 @@ class ImgurCrawler(Crawler):
async def __async_post_init__(self) -> None:
with self.catch_errors(self.PRIMARY_URL), self.disable_on_error("Unable to get client id"):
- # TODO: cache this
- soup = await self.request_soup(self.PRIMARY_URL)
- js_src = css.select(soup, "script[src*='/desktop-assets/js/main']", "src")
- js_text = await self.request_text(self.parse_url(js_src))
- self.client_id = extr_text(js_text, 'apiClientId:"', '"')
+ self.client_id = await self._get_client_id()
+
+ @disk_cached_method("client_id", ttl=86400 * 90)
+ async def _get_client_id(self) -> str:
+ soup = await self.request_soup(self.PRIMARY_URL)
+ js_src = css.select(soup, "script[src*='/desktop-assets/js/main']", "src")
+ js_text = await self.request_text(self.parse_url(js_src))
+ return extr_text(js_text, 'apiClientId:"', '"')
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == _IMAGE_CDN.host:
diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.py
index be96f9b99..dad9d37a4 100644
--- a/cyberdrop_dl/crawlers/koofr.py
+++ b/cyberdrop_dl/crawlers/koofr.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
from typing import Any, ClassVar, Literal
from pydantic import dataclasses
@@ -15,7 +14,7 @@ _PRIMARY_URL = AbsoluteHttpURL("https://koofr.eu")
_SHORT_LINK_CDN = AbsoluteHttpURL("https://k00.fr")
[email protected](slots=True, frozen=True)
[email protected](slots=True)
class Node:
name: str
type: Literal["file", "dir"]
@@ -69,7 +68,7 @@ class KooFrCrawler(Crawler, db_path="path_qs_frag"):
@error_handling_wrapper
async def _walk_folder(self, scrape_item: ScrapeItem, content_id: str, path: str) -> None:
children = await self.api.get_children(content_id, path, scrape_item.password)
- async with asyncio.TaskGroup() as tg:
+ async with self.new_task_group(scrape_item) as tg:
for node in children:
if node.type == "file":
tg.create_task(self._file(scrape_item, node))