Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: use a single, global session for the entire run (#948)
NTFSvolumed86b84259fcf
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex de5ce016d..ba8ee461a 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport asyncioimport jsonfrom contextlib import asynccontextmanager+from dataclasses import fieldfrom datetime import datetimefrom functools import wrapsfrom pathlib import Path@@ -58,16 +59,8 @@ def limiter(func: Callable) -> Any:kwargs.pop("client_session", None)return await func(self, *args, **kwargs)- async with CachedSession(- headers=self._headers,- raise_for_status=False,- cookie_jar=self.client_manager.cookies,- timeout=self._timeouts,- trace_configs=self.trace_configs,- cache=self.client_manager.manager.cache_manager.request_cache,- ) as client:- kwargs["client_session"] = client- return await func(self, *args, **kwargs)+ kwargs["client_session"] = self._session+ return await func(self, *args, **kwargs)return wrapper@@ -94,7 +87,21 @@ class ScraperClient:# folder len + date_prefix len + 10 [suffix (.html) + 1 OS separator + 4 (padding)]min_html_file_path_len = len(str(self.pages_folder)) + len(constants.STARTUP_TIME_STR) + 10self.max_html_stem_len = 245 - min_html_file_path_len+ self._session: CachedSession = field(init=False)++ def startup(self):self.add_request_log_hooks()+ self._session = CachedSession(+ headers=self._headers,+ raise_for_status=False,+ cookie_jar=self.client_manager.cookies,+ timeout=self._timeouts,+ trace_configs=self.trace_configs,+ cache=self.client_manager.manager.cache_manager.request_cache,+ )++ async def close(self):+ await self._session.close()@asynccontextmanagerasync def write_soup_on_error(self, domain: str, url, response: CurlResponse | aiohttp.ClientResponse):diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 96b072437..326185fce 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -295,6 +295,8 @@ class Manager:"""Closes the manager."""self.states.RUNNING.clear()if not isinstance(self.client_manager, Field):+ if not isinstance(self.client_manager.scraper_session, Field):+ await self.client_manager.scraper_session.close()await self.client_manager.close()await self.async_db_close()diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex dc9fef448..579e8fc05 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -72,6 +72,7 @@ class ScrapeMapper:"""Starts the orchestra."""self.manager.scrape_mapper = selfself.manager.client_manager.load_cookie_files()+ self.manager.client_manager.scraper_session.startup()self.start_scrapers()await self.manager.db_manager.history_table.update_previously_unsupported(self.existing_crawlers)self.start_jdownloader()