Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: simplify cookies extraction (#84)

NTFSvolumefbf9f96f9182
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index 79e09872c..44dc5a6db 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -13,8 +13,8 @@ from multidict import CIMultiDict
from cyberdrop_dl import constants, ddos_guard, signature
from cyberdrop_dl.clients.response import AbstractResponse
+from cyberdrop_dl.cookies import make_simple_cookie
from cyberdrop_dl.exceptions import DDOSGuardError
-from cyberdrop_dl.utils.cookie_management import make_simple_cookie
from cyberdrop_dl.utils.utilities import sanitize_filename
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py
index a074fbebc..3d960767d 100755
--- a/cyberdrop_dl/config/config_model.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -7,7 +7,7 @@ from pathlib import Path
from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, field_serializer, field_validator
from cyberdrop_dl import constants
-from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE, Hashing
+from cyberdrop_dl.constants import DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE, Browser, Hashing
from cyberdrop_dl.models import AliasModel, AppriseURL
from cyberdrop_dl.models.types import (
ByteSizeSerilized,
@@ -259,7 +259,7 @@ class Sorting(BaseModel):
class BrowserCookies(BaseModel):
auto_import: bool = False
- browser: BROWSERS | None = BROWSERS.firefox
+ browser: Browser | None = Browser.firefox
sites: list[NonEmptyStr] = SUPPORTED_SITES_DOMAINS
def model_post_init(self, *_) -> None:
diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.py
index 03f08d608..d09b4927d 100644
--- a/cyberdrop_dl/constants.py
+++ b/cyberdrop_dl/constants.py
@@ -98,7 +98,7 @@ class Hashing(CIStrEnum):
POST_DOWNLOAD = auto()
-class BROWSERS(StrEnum):
+class Browser(StrEnum):
chrome = auto()
firefox = auto()
safari = auto()
diff --git a/cyberdrop_dl/cookies.py b/cyberdrop_dl/cookies.py
new file mode 100644
index 000000000..40defd2a4
--- /dev/null
+++ b/cyberdrop_dl/cookies.py
@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import sys
+import time
+from http.cookiejar import Cookie, CookieJar, MozillaCookieJar
+from http.cookies import SimpleCookie
+from typing import TYPE_CHECKING, Final
+
+from cyberdrop_dl.dependencies import browser_cookie3
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Generator, Sequence
+ from pathlib import Path
+
+ from cyberdrop_dl.constants import Browser
+
+
+if sys.version_info < (3, 14):
+ from http import cookies
+
+ # https://github.com/python/cpython/issues/112713
+ cookies.Morsel._reserved["partitioned"] = "partitioned" # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
+ cookies.Morsel._flags.add("partitioned") # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
+
+
+logger = logging.getLogger(__name__)
+
+
+_COOKIE_EXTRACTORS: Final = {func.__name__: func for func in browser_cookie3.all_browsers}
+_CHROMIUM_BROWSERS = frozenset(
+ (
+ "chrome",
+ "chromium",
+ "opera",
+ "opera_gx",
+ "brave",
+ "edge",
+ "vivaldi",
+ "arc",
+ )
+)
+
+
+async def extract_cookies(browser: Browser) -> CookieJar:
+ extract = _COOKIE_EXTRACTORS[browser]
+ try:
+ return await asyncio.to_thread(extract)
+ except PermissionError as e:
+ msg = (
+ "We've encountered a Permissions Error. Please close all browsers and try again\n"
+ "If you are still having issues, make sure all browsers processes are closed in Task Manager\n"
+ f"ERROR: {e!s}"
+ )
+
+ except ValueError as e:
+ msg = f"ERROR: {e!s}"
+
+ except browser_cookie3.BrowserCookieError as e:
+ if (
+ "Unable to get key for cookie decryption" in (msg := str(e))
+ and browser in _CHROMIUM_BROWSERS
+ and os.name == "nt"
+ ):
+ msg = f"ERROR: Cookie extraction from {browser.capitalize()} is not supported on Windows - {msg}"
+
+ else:
+ msg = (
+ "Browser extraction ran into an error, the selected browser may not be available on your system\n"
+ "If you are still having issues, make sure all browsers processes are closed in Task Manager.\n"
+ f"ERROR: {e!s}"
+ )
+
+ raise browser_cookie3.BrowserCookieError(f"{msg}\n\nNothing has been saved.")
+
+
+def split_cookies(extracted_cookies: CookieJar) -> dict[str, MozillaCookieJar]:
+ cookie_jars: dict[str, MozillaCookieJar] = {}
+
+ for domain, cookie in ((cookie.domain.lstrip(".").removeprefix("www."), cookie) for cookie in extracted_cookies):
+ cookie_jar = cookie_jars.get(domain)
+ if cookie_jar is None:
+ cookie_jar = MozillaCookieJar()
+ cookie_jar.set_cookie(cookie)
+
+ return cookie_jars
+
+
+async def export_cookies(cookies: CookieJar, output_path: Path) -> None:
+ cookie_jars = split_cookies(cookies)
+ await asyncio.to_thread(output_path.mkdir, parents=True, exist_ok=True)
+ _ = await asyncio.gather(
+ *(
+ asyncio.to_thread(
+ cj.save,
+ str(output_path / f"{domain}.txt"),
+ ignore_discard=True,
+ ignore_expires=True,
+ )
+ for domain, cj in cookie_jars.items()
+ )
+ )
+
+
+async def read_netscape_files(cookie_files: Sequence[Path]) -> AsyncGenerator[SimpleCookie]:
+ now = int(time.time())
+ domains: set[str] = set()
+ for fut in asyncio.as_completed([asyncio.create_task(_read_netscape_file(file)) for file in cookie_files]):
+ cookie_jar = await fut
+ if not cookie_jar:
+ continue
+
+ for domain, cookie in _parse_cookie_jar(cookie_jar, now):
+ if domain in domains:
+ logger.warning(f"Multiple files have cookies for {domain}. They will be overwritten")
+ domains.add(domain)
+ yield cookie
+
+
+def _parse_cookie_jar(cookie_jar: MozillaCookieJar, now: int) -> Generator[tuple[str, SimpleCookie]]:
+
+ domains: set[str] = set()
+ has_expired_cookies: set[str] = set()
+ for cookie in cookie_jar:
+ if not cookie.value:
+ continue
+
+ domain = cookie.domain.lstrip(".").removeprefix("www.")
+ if domain not in domains:
+ logger.info(f"Found cookies for {domain} in {cookie_jar.filename}")
+ domains.add(domain)
+
+ if (domain not in has_expired_cookies) and cookie.is_expired(now):
+ has_expired_cookies.add(domain)
+ logger.warning(f"Cookies for {domain} are expired")
+
+ yield domain, make_simple_cookie(cookie, now)
+
+
+async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:
+ def read():
+ cookie_jar = MozillaCookieJar(file)
+ try:
+ cookie_jar.load(ignore_discard=True)
+ return cookie_jar
+ except OSError as e:
+ logger.error(f"Unable to load cookies from '{file.name}':\n {e!s}")
+
+ return await asyncio.to_thread(read)
+
+
+def make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:
+ simple_cookie = SimpleCookie()
+ assert cookie.value is not None
+ simple_cookie[cookie.name] = cookie.value
+ morsel = simple_cookie[cookie.name]
+ morsel["domain"] = cookie.domain
+ morsel["path"] = cookie.path
+ morsel["secure"] = cookie.secure
+ if cookie.expires:
+ morsel["max-age"] = str(max(0, cookie.expires - int(now)))
+ else:
+ morsel["max-age"] = ""
+ return simple_cookie
diff --git a/cyberdrop_dl/dependencies/__init__.py b/cyberdrop_dl/dependencies/__init__.py
index 08ea8e5ee..3b2dd32cf 100644
--- a/cyberdrop_dl/dependencies/__init__.py
+++ b/cyberdrop_dl/dependencies/__init__.py
@@ -1,7 +1,5 @@
from typing import TYPE_CHECKING
-from . import _browser_cookie3 as browser_cookie3
-
if TYPE_CHECKING:
import apprise
@@ -11,4 +9,4 @@ else:
except ImportError:
apprise = None
-__all__ = ["apprise", "browser_cookie3"]
+__all__ = ["apprise"]
diff --git a/cyberdrop_dl/dependencies/_browser_cookie3.py b/cyberdrop_dl/dependencies/browser_cookie3.py
similarity index 100%
rename from cyberdrop_dl/dependencies/_browser_cookie3.py
rename to cyberdrop_dl/dependencies/browser_cookie3.py
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index bf949a549..da5ccb294 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -24,10 +24,10 @@ from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.clients.flaresolverr import FlareSolverrClient
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.constants import FileExt
+from cyberdrop_dl.cookies import export_cookies, extract_cookies, read_netscape_files
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrors
from cyberdrop_dl.ffmpeg import probe
-from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers, read_netscape_files
if TYPE_CHECKING:
from collections.abc import Callable, Generator, Iterable, Mapping
@@ -247,7 +247,7 @@ class ClientManager:
async def startup(self) -> None:
await _set_dns_resolver()
- def new_curl_cffi_session(self) -> AsyncSession:
+ def new_curl_cffi_session(self) -> AsyncSession[CurlResponse]:
# Calling code should have validated if curl is actually available
import warnings
@@ -322,15 +322,16 @@ class ClientManager:
async def load_cookie_files(self) -> None:
if self.manager.config_manager.settings_data.browser_cookies.auto_import:
- assert self.manager.config_manager.settings_data.browser_cookies.browser
- get_cookies_from_browsers(
- self.manager, browser=self.manager.config_manager.settings_data.browser_cookies.browser
- )
- cookie_files = sorted(self.manager.appdata.cookies.glob("*.txt"))
+ assert self.manager.config.browser_cookies.browser
+ cookies = await extract_cookies(self.manager.config.browser_cookies.browser)
+ await export_cookies(cookies, output_path=self.manager.appdata.cookies)
+
+ cookie_files = await asyncio.to_thread(lambda: sorted(self.manager.appdata.cookies.glob("*.txt")))
if not cookie_files:
return
- async for domain, cookie in read_netscape_files(cookie_files):
- self.cookies.update_cookies(cookie, response_url=AbsoluteHttpURL(f"https://{domain}"))
+
+ async for cookie in read_netscape_files(cookie_files):
+ self.cookies.update_cookies(cookie)
def get_rate_limiter(self, domain: str) -> AsyncLimiter:
"""Get a rate limiter for a domain."""
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
deleted file mode 100644
index 297c7cbf9..000000000
--- a/cyberdrop_dl/utils/cookie_management.py
+++ /dev/null
@@ -1,192 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import os
-import time
-from functools import wraps
-from http.cookiejar import Cookie, CookieJar, MozillaCookieJar
-from http.cookies import SimpleCookie
-from textwrap import dedent
-from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVar
-
-from cyberdrop_dl.dependencies import browser_cookie3
-
-if TYPE_CHECKING:
- from collections.abc import AsyncIterable, Callable
- from pathlib import Path
-
- from cyberdrop_dl.constants import BROWSERS
- from cyberdrop_dl.managers.manager import Manager
-
-logger = logging.getLogger(__name__)
-P = ParamSpec("P")
-R = TypeVar("R")
-
-
-class CookieExtractor(NamedTuple):
- name: str
- extract: Callable[..., CookieJar]
-
-
-class UnsupportedBrowserError(browser_cookie3.BrowserCookieError): ...
-
-
-COOKIE_EXTRACTORS = [CookieExtractor(func.__name__, func) for func in browser_cookie3.all_browsers]
-COOKIE_ERROR_FOOTER = "\n\nNothing has been saved."
-CHROMIUM_BROWSERS = ["chrome", "chromium", "opera", "opera_gx", "brave", "edge", "vivaldi", "arc"]
-
-
-def cookie_wrapper(func: Callable[P, R]) -> Callable[P, R]:
- """Wrapper handles errors for cookie extraction."""
-
- @wraps(func)
- def wrapper(*args, **kwargs) -> R:
- try:
- return func(*args, **kwargs)
- except PermissionError as e:
- msg = """We've encountered a Permissions Error. Please close all browsers and try again
- If you are still having issues, make sure all browsers processes are closed in Task Manager"""
- msg = f"{dedent(msg)}\nERROR: {e!s}"
-
- except (ValueError, UnsupportedBrowserError) as e:
- msg = f"ERROR: {e!s}"
-
- except browser_cookie3.BrowserCookieError as e:
- msg = """Browser extraction ran into an error, the selected browser(s) may not be available on your system
- If you are still having issues, make sure all browsers processes are closed in Task Manager."""
- msg = f"{dedent(msg)}\nERROR: {e!s}"
-
- raise browser_cookie3.BrowserCookieError(f"{msg}{COOKIE_ERROR_FOOTER}")
-
- return wrapper
-
-
-@cookie_wrapper
-def get_cookies_from_browsers(manager: Manager, *, browser: BROWSERS, domains: list[str] | None = None) -> set[str]:
- """Extract cookies from browsers.
-
- :param browsers: list of browsers to extract from. If `None`, config `browser_cookies.browsers` will be used
- :param domains: list of domains to filter cookies. If `None`, config `browser_cookies.sites` will be used
- :return: A set with all the domains that actually had cookies
- :raises BrowserCookieError: If there's any error while extracting cookies"""
- from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES
-
- if domains == []:
- msg = "No domains selected"
- raise ValueError(msg)
-
- extractor_name = browser.lower()
- domains_to_extract: list[str] = domains or manager.config_manager.settings_data.browser_cookies.sites
- if "all" in domains_to_extract:
- domains_to_extract.remove("all")
- domains_to_extract.extend(SUPPORTED_SITES_DOMAINS)
- elif "all_forums" in domains_to_extract:
- domains_to_extract.remove("all_forums")
- domains_to_extract.extend(SUPPORTED_FORUMS.values())
- elif "all_file_hosts" in domains_to_extract:
- domains_to_extract.remove("all_file_hosts")
- domains_to_extract.extend(SUPPORTED_WEBSITES.values())
-
- extracted_cookies = extract_cookies(extractor_name)
- if not extracted_cookies:
- msg = "None of the provided browsers is supported for extraction"
- raise ValueError(msg)
-
- manager.appdata.cookies.mkdir(parents=True, exist_ok=True)
- domains_with_cookies: set[str] = set()
- for domain in domains_to_extract:
- cookie_file_path = manager.appdata.cookies / f"{domain}.txt"
- cdl_cookie_jar = MozillaCookieJar(cookie_file_path)
- for cookie in extracted_cookies:
- if domain in cookie.domain:
- domains_with_cookies.add(domain)
- cdl_cookie_jar.set_cookie(cookie)
-
- if domain in domains_with_cookies:
- cdl_cookie_jar.save(ignore_discard=True, ignore_expires=True)
-
- return domains_with_cookies
-
-
-def clear_cookies(manager: Manager, domains: list[str]) -> None:
- if not domains:
- raise ValueError("No domains selected")
-
- manager.appdata.cookies.mkdir(parents=True, exist_ok=True)
- for domain in domains:
- cookie_file_path = manager.appdata.cookies / f"{domain}.txt"
- cookie_jar = MozillaCookieJar(cookie_file_path)
- cookie_jar.save(ignore_discard=True, ignore_expires=True)
-
-
-def extract_cookies(extractor_name: str) -> CookieJar:
- def is_decrypt_error(msg: str) -> bool:
- return "Unable to get key for cookie decryption" in msg
-
- extractor = next(extractor for extractor in COOKIE_EXTRACTORS if extractor.name == extractor_name)
- try:
- return extractor.extract()
- except browser_cookie3.BrowserCookieError as e:
- msg = str(e)
- if is_decrypt_error(msg) and extractor.name in CHROMIUM_BROWSERS and os.name == "nt":
- msg = f"Cookie extraction from {extractor.name.capitalize()} is not supported on Windows - {msg}"
- raise UnsupportedBrowserError(msg) from None
- raise
-
-
-async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[str, SimpleCookie]]:
- now = int(time.time())
- domains_seen = set()
- cookie_jars = await asyncio.gather(*(_read_netscape_file(file) for file in cookie_files))
- for file, cookie_jar in zip(cookie_files, cookie_jars, strict=True):
- if not cookie_jar:
- continue
- current_cookie_file_domains: set[str] = set()
- expired_cookies_domains: set[str] = set()
- for cookie in cookie_jar:
- if not cookie.value:
- continue
- simplified_domain = cookie.domain.removeprefix(".")
- if simplified_domain not in current_cookie_file_domains:
- logger.info(f"Found cookies for {simplified_domain} in file '{file.name}'")
- current_cookie_file_domains.add(simplified_domain)
- if simplified_domain in domains_seen:
- logger.warning(
- f"Previous cookies for domain {simplified_domain} detected. They will be overwritten"
- )
-
- if (simplified_domain not in expired_cookies_domains) and cookie.is_expired(now): # type: ignore
- expired_cookies_domains.add(simplified_domain)
- logger.warning(f"Cookies for {simplified_domain} are expired")
-
- domains_seen.add(simplified_domain)
- simple_cookie = make_simple_cookie(cookie, now)
- yield cookie.domain, simple_cookie
-
-
-async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:
- def read():
- cookie_jar = MozillaCookieJar(file)
- try:
- cookie_jar.load(ignore_discard=True)
- return cookie_jar
- except OSError as e:
- logger.error(f"Unable to load cookies from '{file.name}':\n {e!s}")
-
- return await asyncio.to_thread(read)
-
-
-def make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:
- simple_cookie = SimpleCookie()
- assert cookie.value is not None
- simple_cookie[cookie.name] = cookie.value
- morsel = simple_cookie[cookie.name]
- morsel["domain"] = cookie.domain
- morsel["path"] = cookie.path
- morsel["secure"] = cookie.secure
- if cookie.expires:
- morsel["max-age"] = str(max(0, cookie.expires - int(now)))
- else:
- morsel["max-age"] = ""
- return simple_cookie
diff --git a/pyproject.toml b/pyproject.toml
index e682ead63..bc05cae5d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -98,7 +98,7 @@ markers = [
minversion = "8.4.1"
[tool.ruff]
-exclude = ["cyberdrop_dl/dependencies/_browser_cookie3.py"]
+exclude = ["cyberdrop_dl/dependencies/browser_cookie3.py"]
line-length = 120
target-version = "py311"