Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: cache imgur client id
NTFSvolume22d3e3e79ea3
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 6806ff09b..9c5a2af44 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -891,12 +891,14 @@ class API(HTTPMixin, ABC):PRIMARY_URL: AbsoluteHttpURL, # noqa: N803config: Config,request: Callable[..., RequestContext],+ cache: TTLCacheAdapter[Any],parse_url: Callable[[str | yarl.URL], AbsoluteHttpURL] = parse_url,) -> None:self.PRIMARY_URL: Final = PRIMARY_URLself.parse_url: Final = parse_urlself.request: Final = requestself.config: Final = config+ self.cache: Final = cacheself.__post_init__()@classmethod@@ -905,6 +907,7 @@ class API(HTTPMixin, ABC):PRIMARY_URL=crawler.PRIMARY_URL,parse_url=crawler.parse_url,request=crawler.request,+ cache=crawler.cache,config=crawler.manager.config,)diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex b73144d6e..915115a7b 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any, ClassVar+from cyberdrop_dl.cache import disk_cached_methodfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -38,11 +39,14 @@ class ImgurCrawler(Crawler):async def __async_post_init__(self) -> None:with self.catch_errors(self.PRIMARY_URL), self.disable_on_error("Unable to get client id"):- # TODO: cache this- soup = await self.request_soup(self.PRIMARY_URL)- js_src = css.select(soup, "script[src*='/desktop-assets/js/main']", "src")- js_text = await self.request_text(self.parse_url(js_src))- self.client_id = extr_text(js_text, 'apiClientId:"', '"')+ self.client_id = await self._get_client_id()++ @disk_cached_method("client_id", ttl=86400 * 90)+ async def _get_client_id(self) -> str:+ soup = await self.request_soup(self.PRIMARY_URL)+ js_src = css.select(soup, "script[src*='/desktop-assets/js/main']", "src")+ js_text = await self.request_text(self.parse_url(js_src))+ return extr_text(js_text, 'apiClientId:"', '"')async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.host == _IMAGE_CDN.host:diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.pyindex be96f9b99..dad9d37a4 100644--- a/cyberdrop_dl/crawlers/koofr.py+++ b/cyberdrop_dl/crawlers/koofr.py@@ -1,6 +1,5 @@from __future__ import annotations-import asynciofrom typing import Any, ClassVar, Literalfrom pydantic import dataclasses@@ -15,7 +14,7 @@ _PRIMARY_URL = AbsoluteHttpURL("https://koofr.eu")_SHORT_LINK_CDN = AbsoluteHttpURL("https://k00.fr")[email protected](slots=True, frozen=True)[email protected](slots=True)class Node:name: strtype: Literal["file", "dir"]@@ -69,7 +68,7 @@ class KooFrCrawler(Crawler, db_path="path_qs_frag"):@error_handling_wrapperasync def _walk_folder(self, scrape_item: ScrapeItem, content_id: str, path: str) -> None:children = await self.api.get_children(content_id, path, scrape_item.password)- async with asyncio.TaskGroup() as tg:+ async with self.new_task_group(scrape_item) as tg:for node in children:if node.type == "file":tg.create_task(self._file(scrape_item, node))