Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rework cookie management (#1196)

NTFSvolume2d81d5e42f29
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 15800ef2a..6d2cca9d9 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -336,9 +336,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
@final
@error_handling_wrapper
async def _login(self, login_url: AbsoluteHttpURL) -> None:
- host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)
- session_cookie = host_cookies.get(self.LOGIN_USER_COOKIE_NAME)
- session_cookie = session_cookie.value if session_cookie else None
+ session_cookie = self.get_cookie_value(self.LOGIN_USER_COOKIE_NAME)
msg = f"No cookies found for {self.FOLDER_DOMAIN}"
if not session_cookie and self.login_required:
raise LoginError(message=msg)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 97c90f007..9ff60ebf5 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -48,6 +48,7 @@ _T_co = TypeVar("_T_co", covariant=True)
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generator
+ from http.cookies import BaseCookie
from aiohttp_client_cache.response import AnyResponse
from bs4 import BeautifulSoup, Tag
@@ -721,6 +722,23 @@ class Crawler(ABC):
log(msg, bug=True)
return filename
+ @final
+ def get_cookies(self, partial_match_domain: bool = False) -> Iterable[tuple[str, BaseCookie[str]]]:
+ if partial_match_domain:
+ yield from self.client.client_manager.filter_cookies_by_word_in_domain(self.DOMAIN)
+ else:
+ yield str(self.PRIMARY_URL.host), self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)
+
+ @final
+ def get_cookie_value(self, cookie_name: str, partial_match_domain: bool = False) -> str | None:
+ def get_morsels_by_name():
+ for _, cookie in self.get_cookies(partial_match_domain):
+ if morsel := cookie.get(cookie_name):
+ yield morsel
+
+ if newest := max(get_morsels_by_name(), key=lambda x: int(x["max-age"] or 0), default=None):
+ return newest.value
+
def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:
if cls.SUPPORTED_DOMAINS:
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 5d3497e18..ed97bb778 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -3,11 +3,9 @@ from __future__ import annotations
import asyncio
import contextlib
import ssl
-import time
from collections import defaultdict
from dataclasses import dataclass
from http import HTTPStatus
-from http.cookiejar import MozillaCookieJar
from typing import TYPE_CHECKING, Any, Literal, overload
import aiohttp
@@ -25,11 +23,13 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrors
from cyberdrop_dl.managers.download_speed_manager import DownloadSpeedLimiter
from cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsers
+from cyberdrop_dl.utils.cookie_management import read_netscape_files
from cyberdrop_dl.utils.logger import log, log_debug, log_spacer
from cyberdrop_dl.utils.utilities import get_soup_no_error
if TYPE_CHECKING:
- from collections.abc import Generator, Mapping
+ from collections.abc import Generator, Iterable, Mapping
+ from http.cookies import BaseCookie
from aiohttp_client_cache.response import CachedResponse
from curl_cffi.requests import AsyncSession
@@ -126,6 +126,15 @@ class ClientManager:
self.flaresolverr = Flaresolverr(self)
self._headers = {"user-agent": self.manager.global_config.general.user_agent}
+ def filter_cookies_by_word_in_domain(self, word: str) -> Iterable[tuple[str, BaseCookie[str]]]:
+ """Yields pairs of `[domain, BaseCookie]` for every cookie with a domain that has `word` in it"""
+ if not self.cookies:
+ return
+ self.cookies._do_expiration()
+ for domain, _ in self.cookies._cookies:
+ if word in domain:
+ yield domain, self.cookies.filter_cookies(AbsoluteHttpURL(f"https://{domain}"))
+
async def startup(self) -> None:
await _set_dns_resolver()
@@ -215,7 +224,7 @@ class ClientManager:
finally:
pass
- def load_cookie_files(self) -> None:
+ async def load_cookie_files(self) -> None:
if self.manager.config_manager.settings_data.browser_cookies.auto_import:
assert self.manager.config_manager.settings_data.browser_cookies.browser
get_cookies_from_browsers(
@@ -224,37 +233,8 @@ class ClientManager:
cookie_files = sorted(self.manager.path_manager.cookies_dir.glob("*.txt"))
if not cookie_files:
return
-
- now = time.time()
- domains_seen = set()
- for file in cookie_files:
- cookie_jar = MozillaCookieJar(file)
- try:
- cookie_jar.load(ignore_discard=True)
- except OSError as e:
- log(f"Unable to load cookies from '{file.name}':\n {e!s}", 40)
- continue
- current_cookie_file_domains: set[str] = set()
- expired_cookies_domains: set[str] = set()
- for cookie in cookie_jar:
- simplified_domain = cookie.domain.removeprefix(".")
- if simplified_domain not in current_cookie_file_domains:
- log(f"Found cookies for {simplified_domain} in file '{file.name}'", 20)
- current_cookie_file_domains.add(simplified_domain)
- if simplified_domain in domains_seen:
- log(f"Previous cookies for domain {simplified_domain} detected. They will be overwritten", 30)
-
- if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore
- expired_cookies_domains.add(simplified_domain)
-
- domains_seen.add(simplified_domain)
- self.cookies.update_cookies(
- {cookie.name: cookie.value}, # type: ignore
- response_url=AbsoluteHttpURL(f"https://{cookie.domain}"),
- )
-
- for simplified_domain in expired_cookies_domains:
- log(f"Cookies for {simplified_domain} are expired", 30)
+ async for domain, cookie in read_netscape_files(cookie_files):
+ self.cookies.update_cookies(cookie, response_url=AbsoluteHttpURL(f"https://{domain}"))
log_spacer(20, log_to_console=False)
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index b937f9994..8e4c785eb 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -84,7 +84,7 @@ class ScrapeMapper:
async def __aenter__(self) -> Self:
self.manager.scrape_mapper = self
- self.manager.client_manager.load_cookie_files()
+ await self.manager.client_manager.load_cookie_files()
await self.manager.client_manager.scraper_session.__aenter__()
self.manager.task_group = asyncio.TaskGroup()
await self.manager.task_group.__aenter__()
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
index f3daf35f3..5e6d5afc3 100644
--- a/cyberdrop_dl/utils/cookie_management.py
+++ b/cyberdrop_dl/utils/cookie_management.py
@@ -1,19 +1,26 @@
from __future__ import annotations
+import asyncio
import os
+import time
from functools import wraps
-from http.cookiejar import CookieJar, MozillaCookieJar
+from http.cookiejar import Cookie, CookieJar, MozillaCookieJar
+from http.cookies import SimpleCookie
from textwrap import dedent
from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVar
import browser_cookie3
+from cyberdrop_dl.utils.logger import log
+
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import AsyncIterable, Callable
+ from pathlib import Path
from cyberdrop_dl.constants import BROWSERS
from cyberdrop_dl.managers.manager import Manager
+
from cyberdrop_dl.data_structures.supported_domains import (
SUPPORTED_FORUMS,
SUPPORTED_SITES_DOMAINS,
@@ -131,3 +138,58 @@ def extract_cookies(extractor_name: str) -> CookieJar:
msg = f"Cookie extraction from {extractor.name.capitalize()} is not supported on Windows - {msg}"
raise UnsupportedBrowserError(msg) from None
raise
+
+
+async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[str, SimpleCookie]]:
+ now = time.time()
+ domains_seen = set()
+ cookie_jars = await asyncio.gather(*(_read_netscape_file(file) for file in cookie_files))
+ for file, cookie_jar in zip(cookie_files, cookie_jars, strict=True):
+ if not cookie_jar:
+ continue
+ current_cookie_file_domains: set[str] = set()
+ expired_cookies_domains: set[str] = set()
+ for cookie in cookie_jar:
+ if not cookie.value:
+ continue
+ simplified_domain = cookie.domain.removeprefix(".")
+ if simplified_domain not in current_cookie_file_domains:
+ log(f"Found cookies for {simplified_domain} in file '{file.name}'", 20)
+ current_cookie_file_domains.add(simplified_domain)
+ if simplified_domain in domains_seen:
+ log(f"Previous cookies for domain {simplified_domain} detected. They will be overwritten", 30)
+
+ if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore
+ expired_cookies_domains.add(simplified_domain)
+ log(f"Cookies for {simplified_domain} are expired", 30)
+
+ domains_seen.add(simplified_domain)
+ simple_cookie = _make_simple_cookie(cookie, now)
+ yield cookie.domain, simple_cookie
+
+
+async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:
+ def read():
+ cookie_jar = MozillaCookieJar(file)
+ try:
+ cookie_jar.load(ignore_discard=True)
+ return cookie_jar
+ except OSError as e:
+ log(f"Unable to load cookies from '{file.name}':\n {e!s}", 40)
+
+ return await asyncio.to_thread(read)
+
+
+def _make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:
+ simple_cookie = SimpleCookie()
+ assert cookie.value is not None
+ simple_cookie[cookie.name] = cookie.value
+ morsel = simple_cookie[cookie.name]
+ morsel["domain"] = cookie.domain
+ morsel["path"] = cookie.path
+ morsel["secure"] = cookie.secure
+ if cookie.expires:
+ morsel["max-age"] = str(max(0, cookie.expires - int(now)))
+ else:
+ morsel["max-age"] = ""
+ return simple_cookie