Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: simplify cookies extraction (#84)
NTFSvolumefbf9f96f9182
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex 79e09872c..44dc5a6db 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -13,8 +13,8 @@ from multidict import CIMultiDictfrom cyberdrop_dl import constants, ddos_guard, signaturefrom cyberdrop_dl.clients.response import AbstractResponse+from cyberdrop_dl.cookies import make_simple_cookiefrom cyberdrop_dl.exceptions import DDOSGuardError-from cyberdrop_dl.utils.cookie_management import make_simple_cookiefrom cyberdrop_dl.utils.utilities import sanitize_filenameif TYPE_CHECKING:diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.pyindex a074fbebc..3d960767d 100755--- a/cyberdrop_dl/config/config_model.py+++ b/cyberdrop_dl/config/config_model.py@@ -7,7 +7,7 @@ from pathlib import Pathfrom pydantic import BaseModel, ByteSize, Field, NonNegativeInt, field_serializer, field_validatorfrom cyberdrop_dl import constants-from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE, Hashing+from cyberdrop_dl.constants import DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE, Browser, Hashingfrom cyberdrop_dl.models import AliasModel, AppriseURLfrom cyberdrop_dl.models.types import (ByteSizeSerilized,@@ -259,7 +259,7 @@ class Sorting(BaseModel):class BrowserCookies(BaseModel):auto_import: bool = False- browser: BROWSERS | None = BROWSERS.firefox+ browser: Browser | None = Browser.firefoxsites: list[NonEmptyStr] = SUPPORTED_SITES_DOMAINSdef model_post_init(self, *_) -> None:diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.pyindex 03f08d608..d09b4927d 100644--- a/cyberdrop_dl/constants.py+++ b/cyberdrop_dl/constants.py@@ -98,7 +98,7 @@ class Hashing(CIStrEnum):POST_DOWNLOAD = auto()-class BROWSERS(StrEnum):+class Browser(StrEnum):chrome = auto()firefox = auto()safari = auto()diff --git a/cyberdrop_dl/cookies.py b/cyberdrop_dl/cookies.pynew file mode 100644index 000000000..40defd2a4--- /dev/null+++ b/cyberdrop_dl/cookies.py@@ -0,0 +1,166 @@+from __future__ import annotations++import asyncio+import logging+import os+import sys+import time+from http.cookiejar import Cookie, CookieJar, MozillaCookieJar+from http.cookies import SimpleCookie+from typing import TYPE_CHECKING, Final++from cyberdrop_dl.dependencies import browser_cookie3++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Generator, Sequence+ from pathlib import Path++ from cyberdrop_dl.constants import Browser+++if sys.version_info < (3, 14):+ from http import cookies++ # https://github.com/python/cpython/issues/112713+ cookies.Morsel._reserved["partitioned"] = "partitioned" # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]+ cookies.Morsel._flags.add("partitioned") # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]+++logger = logging.getLogger(__name__)+++_COOKIE_EXTRACTORS: Final = {func.__name__: func for func in browser_cookie3.all_browsers}+_CHROMIUM_BROWSERS = frozenset(+ (+ "chrome",+ "chromium",+ "opera",+ "opera_gx",+ "brave",+ "edge",+ "vivaldi",+ "arc",+ )+)+++async def extract_cookies(browser: Browser) -> CookieJar:+ extract = _COOKIE_EXTRACTORS[browser]+ try:+ return await asyncio.to_thread(extract)+ except PermissionError as e:+ msg = (+ "We've encountered a Permissions Error. Please close all browsers and try again\n"+ "If you are still having issues, make sure all browsers processes are closed in Task Manager\n"+ f"ERROR: {e!s}"+ )++ except ValueError as e:+ msg = f"ERROR: {e!s}"++ except browser_cookie3.BrowserCookieError as e:+ if (+ "Unable to get key for cookie decryption" in (msg := str(e))+ and browser in _CHROMIUM_BROWSERS+ and os.name == "nt"+ ):+ msg = f"ERROR: Cookie extraction from {browser.capitalize()} is not supported on Windows - {msg}"++ else:+ msg = (+ "Browser extraction ran into an error, the selected browser may not be available on your system\n"+ "If you are still having issues, make sure all browsers processes are closed in Task Manager.\n"+ f"ERROR: {e!s}"+ )++ raise browser_cookie3.BrowserCookieError(f"{msg}\n\nNothing has been saved.")+++def split_cookies(extracted_cookies: CookieJar) -> dict[str, MozillaCookieJar]:+ cookie_jars: dict[str, MozillaCookieJar] = {}++ for domain, cookie in ((cookie.domain.lstrip(".").removeprefix("www."), cookie) for cookie in extracted_cookies):+ cookie_jar = cookie_jars.get(domain)+ if cookie_jar is None:+ cookie_jar = MozillaCookieJar()+ cookie_jar.set_cookie(cookie)++ return cookie_jars+++async def export_cookies(cookies: CookieJar, output_path: Path) -> None:+ cookie_jars = split_cookies(cookies)+ await asyncio.to_thread(output_path.mkdir, parents=True, exist_ok=True)+ _ = await asyncio.gather(+ *(+ asyncio.to_thread(+ cj.save,+ str(output_path / f"{domain}.txt"),+ ignore_discard=True,+ ignore_expires=True,+ )+ for domain, cj in cookie_jars.items()+ )+ )+++async def read_netscape_files(cookie_files: Sequence[Path]) -> AsyncGenerator[SimpleCookie]:+ now = int(time.time())+ domains: set[str] = set()+ for fut in asyncio.as_completed([asyncio.create_task(_read_netscape_file(file)) for file in cookie_files]):+ cookie_jar = await fut+ if not cookie_jar:+ continue++ for domain, cookie in _parse_cookie_jar(cookie_jar, now):+ if domain in domains:+ logger.warning(f"Multiple files have cookies for {domain}. They will be overwritten")+ domains.add(domain)+ yield cookie+++def _parse_cookie_jar(cookie_jar: MozillaCookieJar, now: int) -> Generator[tuple[str, SimpleCookie]]:++ domains: set[str] = set()+ has_expired_cookies: set[str] = set()+ for cookie in cookie_jar:+ if not cookie.value:+ continue++ domain = cookie.domain.lstrip(".").removeprefix("www.")+ if domain not in domains:+ logger.info(f"Found cookies for {domain} in {cookie_jar.filename}")+ domains.add(domain)++ if (domain not in has_expired_cookies) and cookie.is_expired(now):+ has_expired_cookies.add(domain)+ logger.warning(f"Cookies for {domain} are expired")++ yield domain, make_simple_cookie(cookie, now)+++async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:+ def read():+ cookie_jar = MozillaCookieJar(file)+ try:+ cookie_jar.load(ignore_discard=True)+ return cookie_jar+ except OSError as e:+ logger.error(f"Unable to load cookies from '{file.name}':\n {e!s}")++ return await asyncio.to_thread(read)+++def make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:+ simple_cookie = SimpleCookie()+ assert cookie.value is not None+ simple_cookie[cookie.name] = cookie.value+ morsel = simple_cookie[cookie.name]+ morsel["domain"] = cookie.domain+ morsel["path"] = cookie.path+ morsel["secure"] = cookie.secure+ if cookie.expires:+ morsel["max-age"] = str(max(0, cookie.expires - int(now)))+ else:+ morsel["max-age"] = ""+ return simple_cookiediff --git a/cyberdrop_dl/dependencies/__init__.py b/cyberdrop_dl/dependencies/__init__.pyindex 08ea8e5ee..3b2dd32cf 100644--- a/cyberdrop_dl/dependencies/__init__.py+++ b/cyberdrop_dl/dependencies/__init__.py@@ -1,7 +1,5 @@from typing import TYPE_CHECKING-from . import _browser_cookie3 as browser_cookie3-if TYPE_CHECKING:import apprise@@ -11,4 +9,4 @@ else:except ImportError:apprise = None-__all__ = ["apprise", "browser_cookie3"]+__all__ = ["apprise"]diff --git a/cyberdrop_dl/dependencies/_browser_cookie3.py b/cyberdrop_dl/dependencies/browser_cookie3.pysimilarity index 100%rename from cyberdrop_dl/dependencies/_browser_cookie3.pyrename to cyberdrop_dl/dependencies/browser_cookie3.pydiff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex bf949a549..da5ccb294 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -24,10 +24,10 @@ from cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.clients.flaresolverr import FlareSolverrClientfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.constants import FileExt+from cyberdrop_dl.cookies import export_cookies, extract_cookies, read_netscape_filesfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrorsfrom cyberdrop_dl.ffmpeg import probe-from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers, read_netscape_filesif TYPE_CHECKING:from collections.abc import Callable, Generator, Iterable, Mapping@@ -247,7 +247,7 @@ class ClientManager:async def startup(self) -> None:await _set_dns_resolver()- def new_curl_cffi_session(self) -> AsyncSession:+ def new_curl_cffi_session(self) -> AsyncSession[CurlResponse]:# Calling code should have validated if curl is actually availableimport warnings@@ -322,15 +322,16 @@ class ClientManager:async def load_cookie_files(self) -> None:if self.manager.config_manager.settings_data.browser_cookies.auto_import:- assert self.manager.config_manager.settings_data.browser_cookies.browser- get_cookies_from_browsers(- self.manager, browser=self.manager.config_manager.settings_data.browser_cookies.browser- )- cookie_files = sorted(self.manager.appdata.cookies.glob("*.txt"))+ assert self.manager.config.browser_cookies.browser+ cookies = await extract_cookies(self.manager.config.browser_cookies.browser)+ await export_cookies(cookies, output_path=self.manager.appdata.cookies)++ cookie_files = await asyncio.to_thread(lambda: sorted(self.manager.appdata.cookies.glob("*.txt")))if not cookie_files:return- async for domain, cookie in read_netscape_files(cookie_files):- self.cookies.update_cookies(cookie, response_url=AbsoluteHttpURL(f"https://{domain}"))++ async for cookie in read_netscape_files(cookie_files):+ self.cookies.update_cookies(cookie)def get_rate_limiter(self, domain: str) -> AsyncLimiter:"""Get a rate limiter for a domain."""diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pydeleted file mode 100644index 297c7cbf9..000000000--- a/cyberdrop_dl/utils/cookie_management.py+++ /dev/null@@ -1,192 +0,0 @@-from __future__ import annotations--import asyncio-import logging-import os-import time-from functools import wraps-from http.cookiejar import Cookie, CookieJar, MozillaCookieJar-from http.cookies import SimpleCookie-from textwrap import dedent-from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVar--from cyberdrop_dl.dependencies import browser_cookie3--if TYPE_CHECKING:- from collections.abc import AsyncIterable, Callable- from pathlib import Path-- from cyberdrop_dl.constants import BROWSERS- from cyberdrop_dl.managers.manager import Manager--logger = logging.getLogger(__name__)-P = ParamSpec("P")-R = TypeVar("R")---class CookieExtractor(NamedTuple):- name: str- extract: Callable[..., CookieJar]---class UnsupportedBrowserError(browser_cookie3.BrowserCookieError): ...---COOKIE_EXTRACTORS = [CookieExtractor(func.__name__, func) for func in browser_cookie3.all_browsers]-COOKIE_ERROR_FOOTER = "\n\nNothing has been saved."-CHROMIUM_BROWSERS = ["chrome", "chromium", "opera", "opera_gx", "brave", "edge", "vivaldi", "arc"]---def cookie_wrapper(func: Callable[P, R]) -> Callable[P, R]:- """Wrapper handles errors for cookie extraction."""-- @wraps(func)- def wrapper(*args, **kwargs) -> R:- try:- return func(*args, **kwargs)- except PermissionError as e:- msg = """We've encountered a Permissions Error. Please close all browsers and try again- If you are still having issues, make sure all browsers processes are closed in Task Manager"""- msg = f"{dedent(msg)}\nERROR: {e!s}"-- except (ValueError, UnsupportedBrowserError) as e:- msg = f"ERROR: {e!s}"-- except browser_cookie3.BrowserCookieError as e:- msg = """Browser extraction ran into an error, the selected browser(s) may not be available on your system- If you are still having issues, make sure all browsers processes are closed in Task Manager."""- msg = f"{dedent(msg)}\nERROR: {e!s}"-- raise browser_cookie3.BrowserCookieError(f"{msg}{COOKIE_ERROR_FOOTER}")-- return wrapper---@cookie_wrapper-def get_cookies_from_browsers(manager: Manager, *, browser: BROWSERS, domains: list[str] | None = None) -> set[str]:- """Extract cookies from browsers.-- :param browsers: list of browsers to extract from. If `None`, config `browser_cookies.browsers` will be used- :param domains: list of domains to filter cookies. If `None`, config `browser_cookies.sites` will be used- :return: A set with all the domains that actually had cookies- :raises BrowserCookieError: If there's any error while extracting cookies"""- from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES-- if domains == []:- msg = "No domains selected"- raise ValueError(msg)-- extractor_name = browser.lower()- domains_to_extract: list[str] = domains or manager.config_manager.settings_data.browser_cookies.sites- if "all" in domains_to_extract:- domains_to_extract.remove("all")- domains_to_extract.extend(SUPPORTED_SITES_DOMAINS)- elif "all_forums" in domains_to_extract:- domains_to_extract.remove("all_forums")- domains_to_extract.extend(SUPPORTED_FORUMS.values())- elif "all_file_hosts" in domains_to_extract:- domains_to_extract.remove("all_file_hosts")- domains_to_extract.extend(SUPPORTED_WEBSITES.values())-- extracted_cookies = extract_cookies(extractor_name)- if not extracted_cookies:- msg = "None of the provided browsers is supported for extraction"- raise ValueError(msg)-- manager.appdata.cookies.mkdir(parents=True, exist_ok=True)- domains_with_cookies: set[str] = set()- for domain in domains_to_extract:- cookie_file_path = manager.appdata.cookies / f"{domain}.txt"- cdl_cookie_jar = MozillaCookieJar(cookie_file_path)- for cookie in extracted_cookies:- if domain in cookie.domain:- domains_with_cookies.add(domain)- cdl_cookie_jar.set_cookie(cookie)-- if domain in domains_with_cookies:- cdl_cookie_jar.save(ignore_discard=True, ignore_expires=True)-- return domains_with_cookies---def clear_cookies(manager: Manager, domains: list[str]) -> None:- if not domains:- raise ValueError("No domains selected")-- manager.appdata.cookies.mkdir(parents=True, exist_ok=True)- for domain in domains:- cookie_file_path = manager.appdata.cookies / f"{domain}.txt"- cookie_jar = MozillaCookieJar(cookie_file_path)- cookie_jar.save(ignore_discard=True, ignore_expires=True)---def extract_cookies(extractor_name: str) -> CookieJar:- def is_decrypt_error(msg: str) -> bool:- return "Unable to get key for cookie decryption" in msg-- extractor = next(extractor for extractor in COOKIE_EXTRACTORS if extractor.name == extractor_name)- try:- return extractor.extract()- except browser_cookie3.BrowserCookieError as e:- msg = str(e)- if is_decrypt_error(msg) and extractor.name in CHROMIUM_BROWSERS and os.name == "nt":- msg = f"Cookie extraction from {extractor.name.capitalize()} is not supported on Windows - {msg}"- raise UnsupportedBrowserError(msg) from None- raise---async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[str, SimpleCookie]]:- now = int(time.time())- domains_seen = set()- cookie_jars = await asyncio.gather(*(_read_netscape_file(file) for file in cookie_files))- for file, cookie_jar in zip(cookie_files, cookie_jars, strict=True):- if not cookie_jar:- continue- current_cookie_file_domains: set[str] = set()- expired_cookies_domains: set[str] = set()- for cookie in cookie_jar:- if not cookie.value:- continue- simplified_domain = cookie.domain.removeprefix(".")- if simplified_domain not in current_cookie_file_domains:- logger.info(f"Found cookies for {simplified_domain} in file '{file.name}'")- current_cookie_file_domains.add(simplified_domain)- if simplified_domain in domains_seen:- logger.warning(- f"Previous cookies for domain {simplified_domain} detected. They will be overwritten"- )-- if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore- expired_cookies_domains.add(simplified_domain)- logger.warning(f"Cookies for {simplified_domain} are expired")-- domains_seen.add(simplified_domain)- simple_cookie = make_simple_cookie(cookie, now)- yield cookie.domain, simple_cookie---async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:- def read():- cookie_jar = MozillaCookieJar(file)- try:- cookie_jar.load(ignore_discard=True)- return cookie_jar- except OSError as e:- logger.error(f"Unable to load cookies from '{file.name}':\n {e!s}")-- return await asyncio.to_thread(read)---def make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:- simple_cookie = SimpleCookie()- assert cookie.value is not None- simple_cookie[cookie.name] = cookie.value- morsel = simple_cookie[cookie.name]- morsel["domain"] = cookie.domain- morsel["path"] = cookie.path- morsel["secure"] = cookie.secure- if cookie.expires:- morsel["max-age"] = str(max(0, cookie.expires - int(now)))- else:- morsel["max-age"] = ""- return simple_cookiediff --git a/pyproject.toml b/pyproject.tomlindex e682ead63..bc05cae5d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -98,7 +98,7 @@ markers = [minversion = "8.4.1"[tool.ruff]-exclude = ["cyberdrop_dl/dependencies/_browser_cookie3.py"]+exclude = ["cyberdrop_dl/dependencies/browser_cookie3.py"]line-length = 120target-version = "py311"