Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: use a single, global session for the entire run (#948)

NTFSvolumed86b84259fcf
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index de5ce016d..ba8ee461a 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
from contextlib import asynccontextmanager
+from dataclasses import field
from datetime import datetime
from functools import wraps
from pathlib import Path
@@ -58,16 +59,8 @@ def limiter(func: Callable) -> Any:
kwargs.pop("client_session", None)
return await func(self, *args, **kwargs)
- async with CachedSession(
- headers=self._headers,
- raise_for_status=False,
- cookie_jar=self.client_manager.cookies,
- timeout=self._timeouts,
- trace_configs=self.trace_configs,
- cache=self.client_manager.manager.cache_manager.request_cache,
- ) as client:
- kwargs["client_session"] = client
- return await func(self, *args, **kwargs)
+ kwargs["client_session"] = self._session
+ return await func(self, *args, **kwargs)
return wrapper
@@ -94,7 +87,21 @@ class ScraperClient:
# folder len + date_prefix len + 10 [suffix (.html) + 1 OS separator + 4 (padding)]
min_html_file_path_len = len(str(self.pages_folder)) + len(constants.STARTUP_TIME_STR) + 10
self.max_html_stem_len = 245 - min_html_file_path_len
+ self._session: CachedSession = field(init=False)
+
+ def startup(self):
self.add_request_log_hooks()
+ self._session = CachedSession(
+ headers=self._headers,
+ raise_for_status=False,
+ cookie_jar=self.client_manager.cookies,
+ timeout=self._timeouts,
+ trace_configs=self.trace_configs,
+ cache=self.client_manager.manager.cache_manager.request_cache,
+ )
+
+ async def close(self):
+ await self._session.close()
@asynccontextmanager
async def write_soup_on_error(self, domain: str, url, response: CurlResponse | aiohttp.ClientResponse):
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 96b072437..326185fce 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -295,6 +295,8 @@ class Manager:
"""Closes the manager."""
self.states.RUNNING.clear()
if not isinstance(self.client_manager, Field):
+ if not isinstance(self.client_manager.scraper_session, Field):
+ await self.client_manager.scraper_session.close()
await self.client_manager.close()
await self.async_db_close()
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index dc9fef448..579e8fc05 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -72,6 +72,7 @@ class ScrapeMapper:
"""Starts the orchestra."""
self.manager.scrape_mapper = self
self.manager.client_manager.load_cookie_files()
+ self.manager.client_manager.scraper_session.startup()
self.start_scrapers()
await self.manager.db_manager.history_table.update_previously_unsupported(self.existing_crawlers)
self.start_jdownloader()