Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework cookie management (#1196)
NTFSvolume2d81d5e42f29
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 15800ef2a..6d2cca9d9 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -336,9 +336,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):@final@error_handling_wrapperasync def _login(self, login_url: AbsoluteHttpURL) -> None:- host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)- session_cookie = host_cookies.get(self.LOGIN_USER_COOKIE_NAME)- session_cookie = session_cookie.value if session_cookie else None+ session_cookie = self.get_cookie_value(self.LOGIN_USER_COOKIE_NAME)msg = f"No cookies found for {self.FOLDER_DOMAIN}"if not session_cookie and self.login_required:raise LoginError(message=msg)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 97c90f007..9ff60ebf5 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -48,6 +48,7 @@ _T_co = TypeVar("_T_co", covariant=True)if TYPE_CHECKING:from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generator+ from http.cookies import BaseCookiefrom aiohttp_client_cache.response import AnyResponsefrom bs4 import BeautifulSoup, Tag@@ -721,6 +722,23 @@ class Crawler(ABC):log(msg, bug=True)return filename+ @final+ def get_cookies(self, partial_match_domain: bool = False) -> Iterable[tuple[str, BaseCookie[str]]]:+ if partial_match_domain:+ yield from self.client.client_manager.filter_cookies_by_word_in_domain(self.DOMAIN)+ else:+ yield str(self.PRIMARY_URL.host), self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)++ @final+ def get_cookie_value(self, cookie_name: str, partial_match_domain: bool = False) -> str | None:+ def get_morsels_by_name():+ for _, cookie in self.get_cookies(partial_match_domain):+ if morsel := cookie.get(cookie_name):+ yield morsel++ if newest := max(get_morsels_by_name(), key=lambda x: int(x["max-age"] or 0), default=None):+ return newest.value+def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:if cls.SUPPORTED_DOMAINS:diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 5d3497e18..ed97bb778 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -3,11 +3,9 @@ from __future__ import annotationsimport asyncioimport contextlibimport ssl-import timefrom collections import defaultdictfrom dataclasses import dataclassfrom http import HTTPStatus-from http.cookiejar import MozillaCookieJarfrom typing import TYPE_CHECKING, Any, Literal, overloadimport aiohttp@@ -25,11 +23,13 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrorsfrom cyberdrop_dl.managers.download_speed_manager import DownloadSpeedLimiterfrom cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsers+from cyberdrop_dl.utils.cookie_management import read_netscape_filesfrom cyberdrop_dl.utils.logger import log, log_debug, log_spacerfrom cyberdrop_dl.utils.utilities import get_soup_no_errorif TYPE_CHECKING:- from collections.abc import Generator, Mapping+ from collections.abc import Generator, Iterable, Mapping+ from http.cookies import BaseCookiefrom aiohttp_client_cache.response import CachedResponsefrom curl_cffi.requests import AsyncSession@@ -126,6 +126,15 @@ class ClientManager:self.flaresolverr = Flaresolverr(self)self._headers = {"user-agent": self.manager.global_config.general.user_agent}+ def filter_cookies_by_word_in_domain(self, word: str) -> Iterable[tuple[str, BaseCookie[str]]]:+ """Yields pairs of `[domain, BaseCookie]` for every cookie with a domain that has `word` in it"""+ if not self.cookies:+ return+ self.cookies._do_expiration()+ for domain, _ in self.cookies._cookies:+ if word in domain:+ yield domain, self.cookies.filter_cookies(AbsoluteHttpURL(f"https://{domain}"))+async def startup(self) -> None:await _set_dns_resolver()@@ -215,7 +224,7 @@ class ClientManager:finally:pass- def load_cookie_files(self) -> None:+ async def load_cookie_files(self) -> None:if self.manager.config_manager.settings_data.browser_cookies.auto_import:assert self.manager.config_manager.settings_data.browser_cookies.browserget_cookies_from_browsers(@@ -224,37 +233,8 @@ class ClientManager:cookie_files = sorted(self.manager.path_manager.cookies_dir.glob("*.txt"))if not cookie_files:return-- now = time.time()- domains_seen = set()- for file in cookie_files:- cookie_jar = MozillaCookieJar(file)- try:- cookie_jar.load(ignore_discard=True)- except OSError as e:- log(f"Unable to load cookies from '{file.name}':\n {e!s}", 40)- continue- current_cookie_file_domains: set[str] = set()- expired_cookies_domains: set[str] = set()- for cookie in cookie_jar:- simplified_domain = cookie.domain.removeprefix(".")- if simplified_domain not in current_cookie_file_domains:- log(f"Found cookies for {simplified_domain} in file '{file.name}'", 20)- current_cookie_file_domains.add(simplified_domain)- if simplified_domain in domains_seen:- log(f"Previous cookies for domain {simplified_domain} detected. They will be overwritten", 30)-- if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore- expired_cookies_domains.add(simplified_domain)-- domains_seen.add(simplified_domain)- self.cookies.update_cookies(- {cookie.name: cookie.value}, # type: ignore- response_url=AbsoluteHttpURL(f"https://{cookie.domain}"),- )-- for simplified_domain in expired_cookies_domains:- log(f"Cookies for {simplified_domain} are expired", 30)+ async for domain, cookie in read_netscape_files(cookie_files):+ self.cookies.update_cookies(cookie, response_url=AbsoluteHttpURL(f"https://{domain}"))log_spacer(20, log_to_console=False)diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex b937f9994..8e4c785eb 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -84,7 +84,7 @@ class ScrapeMapper:async def __aenter__(self) -> Self:self.manager.scrape_mapper = self- self.manager.client_manager.load_cookie_files()+ await self.manager.client_manager.load_cookie_files()await self.manager.client_manager.scraper_session.__aenter__()self.manager.task_group = asyncio.TaskGroup()await self.manager.task_group.__aenter__()diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pyindex f3daf35f3..5e6d5afc3 100644--- a/cyberdrop_dl/utils/cookie_management.py+++ b/cyberdrop_dl/utils/cookie_management.py@@ -1,19 +1,26 @@from __future__ import annotations+import asyncioimport os+import timefrom functools import wraps-from http.cookiejar import CookieJar, MozillaCookieJar+from http.cookiejar import Cookie, CookieJar, MozillaCookieJar+from http.cookies import SimpleCookiefrom textwrap import dedentfrom typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVarimport browser_cookie3+from cyberdrop_dl.utils.logger import log+if TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import AsyncIterable, Callable+ from pathlib import Pathfrom cyberdrop_dl.constants import BROWSERSfrom cyberdrop_dl.managers.manager import Manager+from cyberdrop_dl.data_structures.supported_domains import (SUPPORTED_FORUMS,SUPPORTED_SITES_DOMAINS,@@ -131,3 +138,58 @@ def extract_cookies(extractor_name: str) -> CookieJar:msg = f"Cookie extraction from {extractor.name.capitalize()} is not supported on Windows - {msg}"raise UnsupportedBrowserError(msg) from Noneraise+++async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[str, SimpleCookie]]:+ now = time.time()+ domains_seen = set()+ cookie_jars = await asyncio.gather(*(_read_netscape_file(file) for file in cookie_files))+ for file, cookie_jar in zip(cookie_files, cookie_jars, strict=True):+ if not cookie_jar:+ continue+ current_cookie_file_domains: set[str] = set()+ expired_cookies_domains: set[str] = set()+ for cookie in cookie_jar:+ if not cookie.value:+ continue+ simplified_domain = cookie.domain.removeprefix(".")+ if simplified_domain not in current_cookie_file_domains:+ log(f"Found cookies for {simplified_domain} in file '{file.name}'", 20)+ current_cookie_file_domains.add(simplified_domain)+ if simplified_domain in domains_seen:+ log(f"Previous cookies for domain {simplified_domain} detected. They will be overwritten", 30)++ if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore+ expired_cookies_domains.add(simplified_domain)+ log(f"Cookies for {simplified_domain} are expired", 30)++ domains_seen.add(simplified_domain)+ simple_cookie = _make_simple_cookie(cookie, now)+ yield cookie.domain, simple_cookie+++async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:+ def read():+ cookie_jar = MozillaCookieJar(file)+ try:+ cookie_jar.load(ignore_discard=True)+ return cookie_jar+ except OSError as e:+ log(f"Unable to load cookies from '{file.name}':\n {e!s}", 40)++ return await asyncio.to_thread(read)+++def _make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:+ simple_cookie = SimpleCookie()+ assert cookie.value is not None+ simple_cookie[cookie.name] = cookie.value+ morsel = simple_cookie[cookie.name]+ morsel["domain"] = cookie.domain+ morsel["path"] = cookie.path+ morsel["secure"] = cookie.secure+ if cookie.expires:+ morsel["max-age"] = str(max(0, cookie.expires - int(now)))+ else:+ morsel["max-age"] = ""+ return simple_cookie