Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: create http client with manager (#1791)

NTFSvolume6a2d3600fbed
diff --git a/cyberdrop_dl/clients/client.py b/cyberdrop_dl/clients/client.py
index b7c3f6116..56774cc55 100644
--- a/cyberdrop_dl/clients/client.py
+++ b/cyberdrop_dl/clients/client.py
@@ -11,13 +11,11 @@ from http import HTTPStatus
from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self, cast, final
import aiohttp
-from aiohttp import ClientResponse, ClientSession
from aiolimiter import AsyncLimiter
from multidict import CIMultiDict
from cyberdrop_dl import cookies, ddos_guard, signature
from cyberdrop_dl.clients import flaresolverr, tcp
-from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.cookies import make_simple_cookie
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError
@@ -35,6 +33,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.manager import Manager
from cyberdrop_dl.url_objects import AbsoluteHttpURL
+_JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("_JSON_CHECK", default=None)
logger = logging.getLogger(__name__)
@@ -61,50 +60,33 @@ class _LazyResponseLog:
def __json__(self) -> dict[str, Any]:
resp = self.response.__json__()
del resp["created_at"]
- if type(content := resp["content"]) is str:
- resp["content"] = truncated_preview(content)
+ if type(resp["content"]) is str:
+ resp["content"] = truncated_preview(resp["content"])
return resp
def __str__(self) -> str:
return str(self.__json__())
-_JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("_JSON_CHECK", default=None)
-
-
-class DownloadSpeedLimiter(AsyncLimiter):
- __slots__ = ()
-
- def __init__(self, speed_limit: int) -> None:
- super().__init__(speed_limit, 1)
-
- async def acquire(self, amount: float = 1) -> None:
- if self.max_rate <= 0:
- return
- await super().acquire(amount)
-
-
@final
class HTTPClient:
def __init__(self, manager: Manager) -> None:
self.manager = manager
self.ssl_context = tcp.create_ssl_context(self.manager.config.global_settings.general.ssl_context)
- self._cookies: aiohttp.CookieJar | None = None
self.rate_limits: dict[str, AsyncLimiter] = {}
- self.global_rate_limiter = AsyncLimiter(self.manager.config.global_settings.rate_limiting_options.rate_limit, 1)
+ self.global_rate_limiter = AsyncLimiter(
+ self.manager.config.global_settings.rate_limiting_options.rate_limit, time_period=1
+ )
self.global_download_limiter = asyncio.Semaphore(
self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads
)
- speed_limit = self.manager.config.global_settings.rate_limiting_options.download_speed_limit
- self.speed_limiter = DownloadSpeedLimiter(speed_limit)
- self.chunk_size: int = 1024 * 1024 * 10 # 10MB
- if speed_limit:
- self.chunk_size = min(self.chunk_size, speed_limit)
-
- self.download_client = DownloadClient(manager, self)
- self._save_responses_to_disk = manager.config.settings.files.save_pages_html
- self._responses_folder = manager.config.settings.logs.main_log.parent / "cdl_responses"
+ self._cookies: aiohttp.CookieJar | None = None
+ self._responses_folder: Path | None = (
+ manager.config.settings.logs.main_log.parent / "cdl_responses"
+ if manager.config.settings.files.save_pages_html
+ else None
+ )
self._flaresolverr: flaresolverr.Client | None = None
self._curl_session: AsyncSession[CurlResponse] | None = None
@@ -181,8 +163,8 @@ class HTTPClient:
cookies={cookie.key: cookie.value for cookie in self.cookies},
)
- def create_aiohttp_session(self) -> ClientSession:
- return ClientSession(
+ def create_aiohttp_session(self) -> aiohttp.ClientSession:
+ return aiohttp.ClientSession(
headers={"User-Agent": self.manager.config.global_settings.general.user_agent},
raise_for_status=False,
cookie_jar=self.cookies,
@@ -199,7 +181,7 @@ class HTTPClient:
async for cookie in cookies.read_netscape_files(cookie_files):
self.cookies.update_cookies(cookie)
- async def check_http_status(self, response: ClientResponse | CurlResponse | AbstractResponse[Any]) -> None:
+ async def check_http_status(self, response: aiohttp.ClientResponse | CurlResponse | AbstractResponse[Any]) -> None:
"""Checks the HTTP status code and raises an exception if it's not acceptable."""
if not isinstance(response, AbstractResponse):
response = AbstractResponse.create(response)
@@ -223,6 +205,7 @@ class HTTPClient:
impersonate: str | bool | None = None,
data: Any = None,
json: Any = None,
+ check: bool = True,
**request_params: Any,
) -> AsyncGenerator[AbstractResponse[Any]]:
self = cast("HTTPClient", self)
@@ -245,13 +228,17 @@ class HTTPClient:
async with self._request(url, method, request_params, impersonate=bool(impersonate)) as resp:
exc = None
try:
- yield await self._check_response(resp, url)
+ if check:
+ yield await self._check_response(resp, url)
+ else:
+ yield resp
except Exception as e:
exc = e
raise
finally:
- if self._save_responses_to_disk:
+ if self._responses_folder:
self.manager.logs.write_response(self._responses_folder, url, resp, exc)
+ exc = None
def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
"""
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 7aa78a9a8..cc0e9b621 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -1,32 +1,31 @@
from __future__ import annotations
import asyncio
-import contextlib
import itertools
import logging
import time
from http import HTTPStatus
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, final
+
+from aiolimiter import AsyncLimiter
from cyberdrop_dl import aio, constants, ffmpeg, storage
from cyberdrop_dl.clients import etag
-from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.exceptions import DownloadError, InvalidContentTypeError, SlowDownloadError
from cyberdrop_dl.utils import dates
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
+ from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Any
- import aiohttp
-
from cyberdrop_dl.clients.client import HTTPClient
+ from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.config import Config
from cyberdrop_dl.manager import Manager
from cyberdrop_dl.progress import ProgressHook
- from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
+ from cyberdrop_dl.url_objects import MediaItem
logger = logging.getLogger(__name__)
@@ -38,14 +37,32 @@ _FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks
_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}
+class DownloadSpeedLimiter(AsyncLimiter):
+ __slots__ = ()
+
+ async def acquire(self, amount: float = 1) -> None:
+ if self.max_rate <= 0:
+ return
+ await super().acquire(amount)
+
+
+@final
class DownloadClient:
"""Low level class that performs the actual HTTP download operations"""
- def __init__(self, manager: Manager, client: HTTPClient) -> None:
+ def __init__(self, manager: Manager) -> None:
self.manager = manager
- self.client = client
self.download_speed_threshold = self.manager.config.settings.runtime_options.slow_download_speed
self._supports_ranges: bool = True
+ speed_limit = self.manager.config.global_settings.rate_limiting_options.download_speed_limit
+ self.speed_limiter = DownloadSpeedLimiter(speed_limit, time_period=1)
+ self.chunk_size: int = 1024 * 1024 * 10 # 10MB
+ if speed_limit:
+ self.chunk_size = min(self.chunk_size, speed_limit)
+
+ @property
+ def http_client(self) -> HTTPClient:
+ return self.manager.http_client
async def _download(self, domain: str, media_item: MediaItem) -> bool:
"""Downloads a file."""
@@ -63,26 +80,29 @@ class DownloadClient:
await asyncio.sleep(self.manager.config.global_settings.rate_limiting_options.total_delay)
- def process_response(resp: aiohttp.ClientResponse | AbstractResponse[Any]):
- return self._process_response(media_item, domain, resume_point, resp)
-
- return await self._request_download(media_item, process_response)
+ async with self.http_client.request(
+ media_item.real_url,
+ headers=media_item.headers,
+ impersonate=media_item.domain in _USE_IMPERSONATION,
+ check=False,
+ ) as resp:
+ return await self._process_response(media_item, domain, resume_point, resp)
async def _process_response(
self,
media_item: MediaItem,
domain: str,
resume_point: int,
- resp: aiohttp.ClientResponse | AbstractResponse[Any],
+ resp: AbstractResponse[Any],
) -> bool:
if resp.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
await aio.unlink(media_item.partial_file)
etag.check(resp.headers)
- await self.client.check_http_status(resp)
+ await self.http_client.check_http_status(resp)
if not media_item.is_segment:
- _ = get_content_type(media_item.ext, resp.headers)
+ _check_content_type(_get_content_type(resp.headers), media_item.ext)
media_item.filesize = int(resp.headers.get("Content-Length", "0")) or None
if not media_item.path:
@@ -129,57 +149,20 @@ class DownloadClient:
hook.advance(resume_point)
with hook:
- await self._append_content(media_item, hook, self._get_resp_reader(resp))
+ await self._append_content(media_item, hook, resp)
return True
- def _get_resp_reader(
- self, resp: aiohttp.ClientResponse | AbstractResponse[Any]
- ) -> AbstractResponse[Any] | aiohttp.StreamReader:
- if isinstance(resp, AbstractResponse):
- return resp
- return resp.content
-
- @contextlib.asynccontextmanager
- async def __request_context(
- self, url: AbsoluteHttpURL, domain: str, headers: dict[str, str]
- ) -> AsyncGenerator[AbstractResponse[Any] | aiohttp.ClientResponse]:
- if domain in _USE_IMPERSONATION:
- resp = await self.client.curl_session.get(str(url), stream=True, headers=headers)
- try:
- yield AbstractResponse.create(resp)
- finally:
- await resp.aclose()
- return
-
- async with self.client._download_session.get(url, headers=headers) as resp:
- yield resp
-
- async def _request_download(
- self,
- media_item: MediaItem,
- process_response: Callable[[aiohttp.ClientResponse | AbstractResponse[Any]], Coroutine[None, None, bool]],
- ) -> bool:
- async with self.__request_context(media_item.real_url, media_item.domain, media_item.headers) as resp:
- return await process_response(resp)
-
- async def _append_content(
- self,
- media_item: MediaItem,
- hook: ProgressHook,
- content: aiohttp.StreamReader | AbstractResponse[Any],
- ) -> None:
- """Appends content to a file."""
-
+ async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:
check_free_space = storage.create_free_space_checker(media_item)
check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)
await check_free_space()
await self._pre_download_check(media_item)
async with aio.open(media_item.partial_file, mode="ab") as f:
- async for chunk in content.iter_chunked(self.client.chunk_size):
+ async for chunk in resp.iter_chunked(self.chunk_size):
await check_free_space()
chunk_size = len(chunk)
- await self.client.speed_limiter.acquire(chunk_size)
+ await self.speed_limiter.acquire(chunk_size)
await f.write(chunk)
hook.advance(chunk_size)
check_download_speed()
@@ -387,26 +370,20 @@ class DownloadClient:
return media.filesize in limits.other
-def get_content_type(ext: str, headers: Mapping[str, str]) -> str | None:
- content_type: str = headers.get("Content-Type", "")
- content_length = headers.get("Content-Length")
- if not content_type and not content_length:
- msg = "No content type in response headers"
+def _check_content_type(content_type: str, ext: str) -> str | None:
+ if _is_html_or_text(content_type) and ext.lower() not in FileExt.TEXT:
+ msg = f"Received '{content_type}', was expecting binary payload"
raise InvalidContentTypeError(message=msg)
- if not content_type:
- return None
-
- override_key = next((name for name in _CONTENT_TYPES_OVERRIDES if name in content_type), "<NO_OVERRIDE>")
- override: str | None = _CONTENT_TYPES_OVERRIDES.get(override_key)
- content_type = override or content_type
- content_type = content_type.lower()
- if is_html_or_text(content_type) and ext.lower() not in FileExt.TEXT:
- msg = f"Received '{content_type}', was expecting other"
+def _get_content_type(headers: Mapping[str, str]) -> str:
+ content_type = headers.get("Content-Type")
+ if not content_type:
+ msg = "No content type in response headers"
raise InvalidContentTypeError(message=msg)
- return content_type
+ override_key = next((name for name in _CONTENT_TYPES_OVERRIDES if name in content_type), "<NO_OVERRIDE>")
+ return _CONTENT_TYPES_OVERRIDES.get(override_key) or content_type
def get_last_modified(headers: Mapping[str, str]) -> int | None:
@@ -414,7 +391,7 @@ def get_last_modified(headers: Mapping[str, str]) -> int | None:
return dates.parse_http(date_str)
-def is_html_or_text(content_type: str) -> bool:
+def _is_html_or_text(content_type: str) -> bool:
return any(s in content_type for s in ("html", "text"))
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index bc5741ca1..744c84473 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -179,9 +179,10 @@ class Client:
with show_msg(msg):
logger.debug("Making FlareSolverr request [id=%s]\n%s", request_id, params)
async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:
- resp = await response.json()
- logger.debug("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp))
- return Response.from_dict(resp)
+ resp_json = await response.json()
+ resp = Response.from_dict(resp_json)
+ logger.debug("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp_json))
+ return resp
async def _create_session(self) -> None:
session_id = "cyberdrop-dl"
diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.py
index 984c8103f..721e52c49 100644
--- a/cyberdrop_dl/clients/response.py
+++ b/cyberdrop_dl/clients/response.py
@@ -31,8 +31,6 @@ else:
class CurlResponse: ...
-__all__ = ["AbstractResponse"]
-
_ResponseT = TypeVar(
"_ResponseT",
bound=ClientResponse | CurlResponse | FlaresolverrSolution,
@@ -92,6 +90,9 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
elif "html" in self.content_type:
content = BeautifulSoup(content, "html.parser").prettify(formatter="html")
+ elif not ("json" in self.content_type or "html" in self.content_type):
+ content = f"<{self.content_type} payload>"
+
return {
"url": str(self.url),
"status_code": self.status,
@@ -136,9 +137,12 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
filename = aiohttp.multipart.content_disposition_filename(params)
return ContentDisposition(disposition_type, params, filename)
+ @final
@property
- def consumed(self) -> bool:
- return bool(self._text)
+ def aiohttp_resp(self) -> ClientResponse:
+ if type(self._resp) is ClientResponse:
+ return self._resp
+ raise RuntimeError(f"Unexpected response type: {type(self._resp)!r}")
@property
def ok(self) -> bool:
@@ -190,7 +194,6 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
@final
def create_report(self, exc: Exception | None = None, **extras: Any) -> str:
- assert self.consumed
me = self.__json__()
if exc:
diff --git a/cyberdrop_dl/clients/tcp.py b/cyberdrop_dl/clients/tcp.py
index 546890f18..40a24aaa8 100644
--- a/cyberdrop_dl/clients/tcp.py
+++ b/cyberdrop_dl/clients/tcp.py
@@ -54,7 +54,8 @@ async def choose_dns_resolver() -> type[aiohttp.AsyncResolver] | type[aiohttp.Th
def create_connector(ssl_context: ssl.SSLContext | bool) -> aiohttp.TCPConnector:
- assert _DNS_RESOLVER is not None
+ if _DNS_RESOLVER is None:
+ raise RuntimeError("DNS resolver is unknown")
tcp_conn = aiohttp.TCPConnector(ssl=ssl_context, resolver=_DNS_RESOLVER())
tcp_conn._resolver_owner = True
return tcp_conn
diff --git a/cyberdrop_dl/csv_logs.py b/cyberdrop_dl/csv_logs.py
index 61fe5c10c..b2ad2dda8 100644
--- a/cyberdrop_dl/csv_logs.py
+++ b/cyberdrop_dl/csv_logs.py
@@ -15,6 +15,7 @@ from cyberdrop_dl.utils import json
from cyberdrop_dl.utils.filepath import sanitize_filename
if TYPE_CHECKING:
+ import datetime
from collections.abc import Iterable, Iterator
from yarl import URL
@@ -218,15 +219,16 @@ def _write_resp_to_disk(
response: AbstractResponse[Any],
exc: Exception | None = None,
) -> None:
+ file = _prepare_resp_file(folder, url, response.created_at)
+ try:
+ _ = file.write_text(response.create_report(exc), "utf8")
+ except OSError:
+ pass
- max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10
- log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)
+def _prepare_resp_file(folder: Path, url: AbsoluteHttpURL, created_at: datetime.datetime) -> Path:
+ max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10
+ log_date = created_at.strftime(constants.LOGS_DATETIME_FORMAT)
path_safe_url = sanitize_filename(Path(str(url)).as_posix().replace("/", "-"))
filename = f"{path_safe_url[:max_stem_len]}_{log_date}.html"
- file = folder / filename
- content = response.create_report(exc)
- try:
- _ = file.write_text(content, "utf8")
- except OSError:
- pass
+ return folder / filename
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index 7d1f2d996..820049531 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -89,7 +89,7 @@ class Downloader:
@property
def client(self) -> DownloadClient:
- return self.manager.http_client.download_client
+ return self.manager.download_client
@property
def config(self) -> Config:
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index ead57c55b..058d7c7ca 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import dataclasses
-from typing import TYPE_CHECKING, final
+from typing import TYPE_CHECKING, Any, final
from mega.chunker import MegaChunker, get_chunks
from typing_extensions import override
@@ -11,23 +11,24 @@ from cyberdrop_dl.clients.download_client import DownloadClient, make_speed_chec
from cyberdrop_dl.downloader.http import Downloader
if TYPE_CHECKING:
- import aiohttp
from mega.data_structures import Crypto
from yarl import URL
+ from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.manager import Manager
from cyberdrop_dl.progress import ProgressHook
from cyberdrop_dl.url_objects import MediaItem
@final
-class MegaDownloadClient(DownloadClient):
+class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIssues]
def __init__(self, manager: Manager) -> None:
- super().__init__(manager, manager.http_client)
+ super().__init__(manager)
self._decrypt_mapping: dict[URL, tuple[Crypto, int]] = {}
self._supports_ranges = False
- async def _append_content(self, media_item: MediaItem, hook: ProgressHook, content: aiohttp.StreamReader) -> None:
+ @override
+ async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:
"""Appends content to a file."""
check_free_space = storage.create_free_space_checker(media_item)
@@ -38,14 +39,15 @@ class MegaDownloadClient(DownloadClient):
crypto, file_size = self._decrypt_mapping.pop(media_item.url)
chunk_decryptor = MegaChunker(crypto.key, crypto.iv, crypto.meta_mac)
+ aiohttp_resp = resp.aiohttp_resp
async with aio.open(media_item.partial_file, mode="ab") as f:
for _, chunk_size in get_chunks(file_size):
- raw_chunk = await content.readexactly(chunk_size)
+ raw_chunk = await aiohttp_resp.content.readexactly(chunk_size)
chunk = chunk_decryptor.read(raw_chunk)
await check_free_space()
chunk_size = len(chunk)
- await self.client.speed_limiter.acquire(chunk_size)
+ await self.speed_limiter.acquire(chunk_size)
await f.write(chunk)
hook.advance(chunk_size)
check_download_speed()
@@ -68,8 +70,8 @@ class MegaDownloader(Downloader):
super(MegaDownloader, self).__post_init__()
self._client = MegaDownloadClient(self.manager)
- @override
@property
+ @override
def client(self) -> MegaDownloadClient:
return self._client
diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.py
index c14b9a887..5c58b8e38 100644
--- a/cyberdrop_dl/manager.py
+++ b/cyberdrop_dl/manager.py
@@ -16,6 +16,7 @@ from pydantic.types import ByteSize
from cyberdrop_dl import __version__, cookies, env, ffmpeg, stats, yaml
from cyberdrop_dl.cli import CLIargs
from cyberdrop_dl.clients.client import HTTPClient
+from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.config import Config
from cyberdrop_dl.csv_logs import CSVLogsManager
from cyberdrop_dl.database import Database
@@ -52,9 +53,11 @@ class Manager:
self._completed_downloads: list[MediaItem] = []
self.hasher: Hasher = Hasher(self)
self.logs: CSVLogsManager = CSVLogsManager.from_manager(self)
+ self.http_client: HTTPClient = HTTPClient(self)
+ self.download_client: DownloadClient = DownloadClient(self)
+
self.scrape_mapper: ScrapeMapper
self.database: Database
- self.http_client: HTTPClient
self.deduper: Czkawka
self.sorter: Sorter
@@ -85,13 +88,16 @@ class Manager:
)
self.deduper = Czkawka.from_manager(self)
self.sorter = Sorter.from_manager(self)
- self.http_client = HTTPClient(self)
with (
_cache_context(self.appdata.cache_file, self.cache),
_enter_context(REFRESH_RATE, self.config.global_settings.ui_options.refresh_rate),
_enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),
):
- yield self
+ try:
+ yield self
+ finally:
+ del self.deduper
+ del self.sorter
def add_completed(self, media_item: MediaItem) -> None:
if media_item.is_segment:
diff --git a/tests/test_client.py b/tests/test_client.py
index 9f58d1b2e..a2b167f47 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -1,3 +1,4 @@
+import ssl
import sys
import aiohttp
@@ -13,6 +14,13 @@ def client(manager: Manager) -> HTTPClient:
return HTTPClient(manager)
+def test_initial_state(client: HTTPClient) -> None:
+ assert client._cookies is None
+ assert client._flaresolverr is None
+ assert isinstance(client.ssl_context, ssl.SSLContext)
+ assert client.rate_limits == {}
+
+
async def test_context_manager(client: HTTPClient) -> None:
with pytest.raises(AttributeError):
_ = client._session
@@ -35,3 +43,18 @@ async def test_context_manager(client: HTTPClient) -> None:
with pytest.raises(ScrapeError):
_ = client.curl_session
# test_ssl_context.py
+
+
+def test_create_aiohttp_session_outside_loop(client: HTTPClient) -> None:
+ with pytest.raises(RuntimeError, match="no running event loop"):
+ _ = client.create_aiohttp_session()
+
+
+async def test_create_aiohttp_session(client: HTTPClient) -> None:
+ async with client, client.create_aiohttp_session() as session:
+ assert len(session.headers) == 1
+ assert session.headers.get("User-agent")
+ assert session.cookie_jar is client.cookies
+ assert session._raise_for_status is False
+ assert session.timeout == client.manager.config.global_settings.rate_limiting_options._aiohttp_timeout
+ assert session.requote_redirect_url is False
diff --git a/tests/test_csv_logs.py b/tests/test_csv_logs.py
new file mode 100644
index 000000000..3ac2402e4
--- /dev/null
+++ b/tests/test_csv_logs.py
@@ -0,0 +1,32 @@
+import datetime
+from pathlib import Path
+
+import pytest
+
+from cyberdrop_dl import csv_logs
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+
+now = datetime.datetime(2026, 5, 8)
+
+
+ "url, expected",
+ [
+ (
+ "https://static.scdn.st/e2f8a4c6-3d7b-4e19-9a5c-8b1d6f0e3a7c/thumbs/f7e8625e-b50b-42c3-93da-d74bce716c41-md8f1c79417cd16004-zU4LIum1.png",
+ "/https-static.scdn.st-e2f8a4c6-3d7b-4e19-9a5c-8b1d6f0e3a7c-thumbs-f7e8625e-b50b-42c3-93da-d74bce716c41-md8f1c79417cd16004-zU4LIum1.png_20260508_000000.html",
+ ),
+ (
+ "https://cloud.mail.ru/public/d1LR/hMBXG2mFo",
+ "/https-cloud.mail.ru-public-d1LR-hMBXG2mFo_20260508_000000.html",
+ ),
+ (
+ "https://www.tokyomotion.net/video/5884366/%E3%83%A1%E3%82%B9%E7%89%9B%E3%81%8A%E3%81%BB%E3%82%A4%E3%82%AD%E3%83%81%E3%82%AF%E3%83%8B%E3%83%BC",
+ "/https-www.tokyomotion.net-video-5884366-%E3%83%A1%E3%82%B9%E7%89%9B%E3%81%8A%E3%81%BB%E3%82%A4%E3%82%AD%E3%83%81%E3%82%AF%E3%83%8B%E3%83%BC_20260508_000000.html",
+ ),
+ ],
+)
+def test_prepare_resp_filename(url: str, expected: str) -> None:
+ result = csv_logs._prepare_resp_file(Path("/"), AbsoluteHttpURL(url), now)
+ assert result.as_posix().count("/") == 1
+ assert result.as_posix() == expected
diff --git a/tests/test_download_client.py b/tests/test_download_client.py
new file mode 100644
index 000000000..26262d171
--- /dev/null
+++ b/tests/test_download_client.py
@@ -0,0 +1,45 @@
+import pytest
+from multidict import CIMultiDict
+
+from cyberdrop_dl.clients.download_client import DownloadClient, _check_content_type, _get_content_type
+from cyberdrop_dl.exceptions import InvalidContentTypeError
+from cyberdrop_dl.manager import Manager
+
+
+ "limit",
+ [
+ (5_000_000,),
+ (500_000_000,),
+ ],
+)
+def test_chunk_size_is_never_greater_that_speed_limit(manager: Manager, limit: int) -> None:
+ max_expected = 1024 * 1024 * 10
+ limit = manager.config.global_settings.rate_limiting_options.download_speed_limit
+ assert limit == 0
+ client = DownloadClient(manager)
+ assert client.chunk_size != limit
+ assert client.chunk_size == max_expected
+
+ manager.config.global_settings.rate_limiting_options.download_speed_limit = limit
+ client = DownloadClient(manager)
+ assert client.chunk_size == min(limit or max_expected, max_expected)
+
+
+def test_get_content_type() -> None:
+ def get(value: str) -> str:
+ return _get_content_type(CIMultiDict({"content-type": value}))
+
+ assert get("text/vnd.trolltech.linguist") == "video/MP2T"
+ assert get("text/HTML") == "text/HTML"
+ assert get("application/json") == "application/json"
+
+ with pytest.raises(InvalidContentTypeError):
+ _ = get("")
+
+
+def test_check_content_type() -> None:
+ _check_content_type("text/html", ".txt")
+
+ with pytest.raises(InvalidContentTypeError, match="Received 'text/html', was expecting binary payload"):
+ _check_content_type("text/html", ".mp4")
diff --git a/tests/test_manager.py b/tests/test_manager.py
index f3690ddd3..28f14723f 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -48,7 +48,7 @@ def test_manager_context() -> None:
config = Config.parse_args(["--refresh-rate", "40"])
manager = Manager(config=config)
- for attr in ("database", "deduper", "sorter", "http_client"):
+ for attr in ("database", "deduper", "sorter"):
with pytest.raises(AttributeError):
getattr(manager, attr)
diff --git a/tests/test_response.py b/tests/test_response.py
new file mode 100644
index 000000000..a8c5ad570
--- /dev/null
+++ b/tests/test_response.py
@@ -0,0 +1,35 @@
+import datetime
+
+from multidict import CIMultiDict, CIMultiDictProxy
+
+from cyberdrop_dl.clients.response import _AIOHTTPResponse
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+
+
+def make_resp(content_type: str, url: str = "https://example.com") -> _AIOHTTPResponse:
+ resp = _AIOHTTPResponse(
+ content_type,
+ status=200,
+ headers=CIMultiDictProxy(CIMultiDict({"content-type": content_type})),
+ url=AbsoluteHttpURL(url),
+ location=None,
+ _resp=None, # pyright: ignore[reportArgumentType]
+ )
+ resp.created_at = datetime.datetime.min
+ return resp
+
+
+def test_json_dump_binary_resp() -> None:
+ content_type = "application/octet-stream"
+ resp = make_resp(content_type)
+ json = resp.__json__()
+ assert content_type in json["content"]
+ assert json == {
+ "url": "https://example.com",
+ "status_code": 200,
+ "created_at": "0001-01-01 00:00:00",
+ "response_headers": {
+ "content-type": "application/octet-stream",
+ },
+ "content": "<application/octet-stream payload>",
+ }