Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: create http client with manager (#1791)
NTFSvolume6a2d3600fbed
diff --git a/cyberdrop_dl/clients/client.py b/cyberdrop_dl/clients/client.pyindex b7c3f6116..56774cc55 100644--- a/cyberdrop_dl/clients/client.py+++ b/cyberdrop_dl/clients/client.py@@ -11,13 +11,11 @@ from http import HTTPStatusfrom typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self, cast, finalimport aiohttp-from aiohttp import ClientResponse, ClientSessionfrom aiolimiter import AsyncLimiterfrom multidict import CIMultiDictfrom cyberdrop_dl import cookies, ddos_guard, signaturefrom cyberdrop_dl.clients import flaresolverr, tcp-from cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.cookies import make_simple_cookiefrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError@@ -35,6 +33,7 @@ if TYPE_CHECKING:from cyberdrop_dl.manager import Managerfrom cyberdrop_dl.url_objects import AbsoluteHttpURL+_JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("_JSON_CHECK", default=None)logger = logging.getLogger(__name__)@@ -61,50 +60,33 @@ class _LazyResponseLog:def __json__(self) -> dict[str, Any]:resp = self.response.__json__()del resp["created_at"]- if type(content := resp["content"]) is str:- resp["content"] = truncated_preview(content)+ if type(resp["content"]) is str:+ resp["content"] = truncated_preview(resp["content"])return respdef __str__(self) -> str:return str(self.__json__())-_JSON_CHECK: ContextVar[Callable[[Any, AbstractResponse[Any]], None] | None] = ContextVar("_JSON_CHECK", default=None)---class DownloadSpeedLimiter(AsyncLimiter):- __slots__ = ()-- def __init__(self, speed_limit: int) -> None:- super().__init__(speed_limit, 1)-- async def acquire(self, amount: float = 1) -> None:- if self.max_rate <= 0:- return- await super().acquire(amount)--@finalclass HTTPClient:def __init__(self, manager: Manager) -> None:self.manager = managerself.ssl_context = tcp.create_ssl_context(self.manager.config.global_settings.general.ssl_context)- self._cookies: aiohttp.CookieJar | None = Noneself.rate_limits: dict[str, AsyncLimiter] = {}- self.global_rate_limiter = AsyncLimiter(self.manager.config.global_settings.rate_limiting_options.rate_limit, 1)+ self.global_rate_limiter = AsyncLimiter(+ self.manager.config.global_settings.rate_limiting_options.rate_limit, time_period=1+ )self.global_download_limiter = asyncio.Semaphore(self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads)- speed_limit = self.manager.config.global_settings.rate_limiting_options.download_speed_limit- self.speed_limiter = DownloadSpeedLimiter(speed_limit)- self.chunk_size: int = 1024 * 1024 * 10 # 10MB- if speed_limit:- self.chunk_size = min(self.chunk_size, speed_limit)-- self.download_client = DownloadClient(manager, self)- self._save_responses_to_disk = manager.config.settings.files.save_pages_html- self._responses_folder = manager.config.settings.logs.main_log.parent / "cdl_responses"+ self._cookies: aiohttp.CookieJar | None = None+ self._responses_folder: Path | None = (+ manager.config.settings.logs.main_log.parent / "cdl_responses"+ if manager.config.settings.files.save_pages_html+ else None+ )self._flaresolverr: flaresolverr.Client | None = Noneself._curl_session: AsyncSession[CurlResponse] | None = None@@ -181,8 +163,8 @@ class HTTPClient:cookies={cookie.key: cookie.value for cookie in self.cookies},)- def create_aiohttp_session(self) -> ClientSession:- return ClientSession(+ def create_aiohttp_session(self) -> aiohttp.ClientSession:+ return aiohttp.ClientSession(headers={"User-Agent": self.manager.config.global_settings.general.user_agent},raise_for_status=False,cookie_jar=self.cookies,@@ -199,7 +181,7 @@ class HTTPClient:async for cookie in cookies.read_netscape_files(cookie_files):self.cookies.update_cookies(cookie)- async def check_http_status(self, response: ClientResponse | CurlResponse | AbstractResponse[Any]) -> None:+ async def check_http_status(self, response: aiohttp.ClientResponse | CurlResponse | AbstractResponse[Any]) -> None:"""Checks the HTTP status code and raises an exception if it's not acceptable."""if not isinstance(response, AbstractResponse):response = AbstractResponse.create(response)@@ -223,6 +205,7 @@ class HTTPClient:impersonate: str | bool | None = None,data: Any = None,json: Any = None,+ check: bool = True,**request_params: Any,) -> AsyncGenerator[AbstractResponse[Any]]:self = cast("HTTPClient", self)@@ -245,13 +228,17 @@ class HTTPClient:async with self._request(url, method, request_params, impersonate=bool(impersonate)) as resp:exc = Nonetry:- yield await self._check_response(resp, url)+ if check:+ yield await self._check_response(resp, url)+ else:+ yield respexcept Exception as e:exc = eraisefinally:- if self._save_responses_to_disk:+ if self._responses_folder:self.manager.logs.write_response(self._responses_folder, url, resp, exc)+ exc = Nonedef __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:"""diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 7aa78a9a8..cc0e9b621 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -1,32 +1,31 @@from __future__ import annotationsimport asyncio-import contextlibimport itertoolsimport loggingimport timefrom http import HTTPStatus-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, final++from aiolimiter import AsyncLimiterfrom cyberdrop_dl import aio, constants, ffmpeg, storagefrom cyberdrop_dl.clients import etag-from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.constants import FileExtfrom cyberdrop_dl.exceptions import DownloadError, InvalidContentTypeError, SlowDownloadErrorfrom cyberdrop_dl.utils import datesif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping+ from collections.abc import Callable, Mappingfrom pathlib import Pathfrom typing import Any- import aiohttp-from cyberdrop_dl.clients.client import HTTPClient+ from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.config import Configfrom cyberdrop_dl.manager import Managerfrom cyberdrop_dl.progress import ProgressHook- from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem+ from cyberdrop_dl.url_objects import MediaItemlogger = logging.getLogger(__name__)@@ -38,14 +37,32 @@ _FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}+class DownloadSpeedLimiter(AsyncLimiter):+ __slots__ = ()++ async def acquire(self, amount: float = 1) -> None:+ if self.max_rate <= 0:+ return+ await super().acquire(amount)+++@finalclass DownloadClient:"""Low level class that performs the actual HTTP download operations"""- def __init__(self, manager: Manager, client: HTTPClient) -> None:+ def __init__(self, manager: Manager) -> None:self.manager = manager- self.client = clientself.download_speed_threshold = self.manager.config.settings.runtime_options.slow_download_speedself._supports_ranges: bool = True+ speed_limit = self.manager.config.global_settings.rate_limiting_options.download_speed_limit+ self.speed_limiter = DownloadSpeedLimiter(speed_limit, time_period=1)+ self.chunk_size: int = 1024 * 1024 * 10 # 10MB+ if speed_limit:+ self.chunk_size = min(self.chunk_size, speed_limit)++ @property+ def http_client(self) -> HTTPClient:+ return self.manager.http_clientasync def _download(self, domain: str, media_item: MediaItem) -> bool:"""Downloads a file."""@@ -63,26 +80,29 @@ class DownloadClient:await asyncio.sleep(self.manager.config.global_settings.rate_limiting_options.total_delay)- def process_response(resp: aiohttp.ClientResponse | AbstractResponse[Any]):- return self._process_response(media_item, domain, resume_point, resp)-- return await self._request_download(media_item, process_response)+ async with self.http_client.request(+ media_item.real_url,+ headers=media_item.headers,+ impersonate=media_item.domain in _USE_IMPERSONATION,+ check=False,+ ) as resp:+ return await self._process_response(media_item, domain, resume_point, resp)async def _process_response(self,media_item: MediaItem,domain: str,resume_point: int,- resp: aiohttp.ClientResponse | AbstractResponse[Any],+ resp: AbstractResponse[Any],) -> bool:if resp.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:await aio.unlink(media_item.partial_file)etag.check(resp.headers)- await self.client.check_http_status(resp)+ await self.http_client.check_http_status(resp)if not media_item.is_segment:- _ = get_content_type(media_item.ext, resp.headers)+ _check_content_type(_get_content_type(resp.headers), media_item.ext)media_item.filesize = int(resp.headers.get("Content-Length", "0")) or Noneif not media_item.path:@@ -129,57 +149,20 @@ class DownloadClient:hook.advance(resume_point)with hook:- await self._append_content(media_item, hook, self._get_resp_reader(resp))+ await self._append_content(media_item, hook, resp)return True- def _get_resp_reader(- self, resp: aiohttp.ClientResponse | AbstractResponse[Any]- ) -> AbstractResponse[Any] | aiohttp.StreamReader:- if isinstance(resp, AbstractResponse):- return resp- return resp.content-- @contextlib.asynccontextmanager- async def __request_context(- self, url: AbsoluteHttpURL, domain: str, headers: dict[str, str]- ) -> AsyncGenerator[AbstractResponse[Any] | aiohttp.ClientResponse]:- if domain in _USE_IMPERSONATION:- resp = await self.client.curl_session.get(str(url), stream=True, headers=headers)- try:- yield AbstractResponse.create(resp)- finally:- await resp.aclose()- return-- async with self.client._download_session.get(url, headers=headers) as resp:- yield resp-- async def _request_download(- self,- media_item: MediaItem,- process_response: Callable[[aiohttp.ClientResponse | AbstractResponse[Any]], Coroutine[None, None, bool]],- ) -> bool:- async with self.__request_context(media_item.real_url, media_item.domain, media_item.headers) as resp:- return await process_response(resp)-- async def _append_content(- self,- media_item: MediaItem,- hook: ProgressHook,- content: aiohttp.StreamReader | AbstractResponse[Any],- ) -> None:- """Appends content to a file."""-+ async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:check_free_space = storage.create_free_space_checker(media_item)check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)await check_free_space()await self._pre_download_check(media_item)async with aio.open(media_item.partial_file, mode="ab") as f:- async for chunk in content.iter_chunked(self.client.chunk_size):+ async for chunk in resp.iter_chunked(self.chunk_size):await check_free_space()chunk_size = len(chunk)- await self.client.speed_limiter.acquire(chunk_size)+ await self.speed_limiter.acquire(chunk_size)await f.write(chunk)hook.advance(chunk_size)check_download_speed()@@ -387,26 +370,20 @@ class DownloadClient:return media.filesize in limits.other-def get_content_type(ext: str, headers: Mapping[str, str]) -> str | None:- content_type: str = headers.get("Content-Type", "")- content_length = headers.get("Content-Length")- if not content_type and not content_length:- msg = "No content type in response headers"+def _check_content_type(content_type: str, ext: str) -> str | None:+ if _is_html_or_text(content_type) and ext.lower() not in FileExt.TEXT:+ msg = f"Received '{content_type}', was expecting binary payload"raise InvalidContentTypeError(message=msg)- if not content_type:- return None-- override_key = next((name for name in _CONTENT_TYPES_OVERRIDES if name in content_type), "<NO_OVERRIDE>")- override: str | None = _CONTENT_TYPES_OVERRIDES.get(override_key)- content_type = override or content_type- content_type = content_type.lower()- if is_html_or_text(content_type) and ext.lower() not in FileExt.TEXT:- msg = f"Received '{content_type}', was expecting other"+def _get_content_type(headers: Mapping[str, str]) -> str:+ content_type = headers.get("Content-Type")+ if not content_type:+ msg = "No content type in response headers"raise InvalidContentTypeError(message=msg)- return content_type+ override_key = next((name for name in _CONTENT_TYPES_OVERRIDES if name in content_type), "<NO_OVERRIDE>")+ return _CONTENT_TYPES_OVERRIDES.get(override_key) or content_typedef get_last_modified(headers: Mapping[str, str]) -> int | None:@@ -414,7 +391,7 @@ def get_last_modified(headers: Mapping[str, str]) -> int | None:return dates.parse_http(date_str)-def is_html_or_text(content_type: str) -> bool:+def _is_html_or_text(content_type: str) -> bool:return any(s in content_type for s in ("html", "text"))diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex bc5741ca1..744c84473 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -179,9 +179,10 @@ class Client:with show_msg(msg):logger.debug("Making FlareSolverr request [id=%s]\n%s", request_id, params)async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:- resp = await response.json()- logger.debug("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp))- return Response.from_dict(resp)+ resp_json = await response.json()+ resp = Response.from_dict(resp_json)+ logger.debug("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp_json))+ return respasync def _create_session(self) -> None:session_id = "cyberdrop-dl"diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.pyindex 984c8103f..721e52c49 100644--- a/cyberdrop_dl/clients/response.py+++ b/cyberdrop_dl/clients/response.py@@ -31,8 +31,6 @@ else:class CurlResponse: ...-__all__ = ["AbstractResponse"]-_ResponseT = TypeVar("_ResponseT",bound=ClientResponse | CurlResponse | FlaresolverrSolution,@@ -92,6 +90,9 @@ class AbstractResponse(ABC, Generic[_ResponseT]):elif "html" in self.content_type:content = BeautifulSoup(content, "html.parser").prettify(formatter="html")+ elif not ("json" in self.content_type or "html" in self.content_type):+ content = f"<{self.content_type} payload>"+return {"url": str(self.url),"status_code": self.status,@@ -136,9 +137,12 @@ class AbstractResponse(ABC, Generic[_ResponseT]):filename = aiohttp.multipart.content_disposition_filename(params)return ContentDisposition(disposition_type, params, filename)+ @final@property- def consumed(self) -> bool:- return bool(self._text)+ def aiohttp_resp(self) -> ClientResponse:+ if type(self._resp) is ClientResponse:+ return self._resp+ raise RuntimeError(f"Unexpected response type: {type(self._resp)!r}")@propertydef ok(self) -> bool:@@ -190,7 +194,6 @@ class AbstractResponse(ABC, Generic[_ResponseT]):@finaldef create_report(self, exc: Exception | None = None, **extras: Any) -> str:- assert self.consumedme = self.__json__()if exc:diff --git a/cyberdrop_dl/clients/tcp.py b/cyberdrop_dl/clients/tcp.pyindex 546890f18..40a24aaa8 100644--- a/cyberdrop_dl/clients/tcp.py+++ b/cyberdrop_dl/clients/tcp.py@@ -54,7 +54,8 @@ async def choose_dns_resolver() -> type[aiohttp.AsyncResolver] | type[aiohttp.Thdef create_connector(ssl_context: ssl.SSLContext | bool) -> aiohttp.TCPConnector:- assert _DNS_RESOLVER is not None+ if _DNS_RESOLVER is None:+ raise RuntimeError("DNS resolver is unknown")tcp_conn = aiohttp.TCPConnector(ssl=ssl_context, resolver=_DNS_RESOLVER())tcp_conn._resolver_owner = Truereturn tcp_conndiff --git a/cyberdrop_dl/csv_logs.py b/cyberdrop_dl/csv_logs.pyindex 61fe5c10c..b2ad2dda8 100644--- a/cyberdrop_dl/csv_logs.py+++ b/cyberdrop_dl/csv_logs.py@@ -15,6 +15,7 @@ from cyberdrop_dl.utils import jsonfrom cyberdrop_dl.utils.filepath import sanitize_filenameif TYPE_CHECKING:+ import datetimefrom collections.abc import Iterable, Iteratorfrom yarl import URL@@ -218,15 +219,16 @@ def _write_resp_to_disk(response: AbstractResponse[Any],exc: Exception | None = None,) -> None:+ file = _prepare_resp_file(folder, url, response.created_at)+ try:+ _ = file.write_text(response.create_report(exc), "utf8")+ except OSError:+ pass- max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10- log_date = response.created_at.strftime(constants.LOGS_DATETIME_FORMAT)+def _prepare_resp_file(folder: Path, url: AbsoluteHttpURL, created_at: datetime.datetime) -> Path:+ max_stem_len = 245 - len(str(folder)) + len(constants.STARTUP_TIME_STR) + 10+ log_date = created_at.strftime(constants.LOGS_DATETIME_FORMAT)path_safe_url = sanitize_filename(Path(str(url)).as_posix().replace("/", "-"))filename = f"{path_safe_url[:max_stem_len]}_{log_date}.html"- file = folder / filename- content = response.create_report(exc)- try:- _ = file.write_text(content, "utf8")- except OSError:- pass+ return folder / filenamediff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex 7d1f2d996..820049531 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -89,7 +89,7 @@ class Downloader:@propertydef client(self) -> DownloadClient:- return self.manager.http_client.download_client+ return self.manager.download_client@propertydef config(self) -> Config:diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex ead57c55b..058d7c7ca 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -1,7 +1,7 @@from __future__ import annotationsimport dataclasses-from typing import TYPE_CHECKING, final+from typing import TYPE_CHECKING, Any, finalfrom mega.chunker import MegaChunker, get_chunksfrom typing_extensions import override@@ -11,23 +11,24 @@ from cyberdrop_dl.clients.download_client import DownloadClient, make_speed_checfrom cyberdrop_dl.downloader.http import Downloaderif TYPE_CHECKING:- import aiohttpfrom mega.data_structures import Cryptofrom yarl import URL+ from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.manager import Managerfrom cyberdrop_dl.progress import ProgressHookfrom cyberdrop_dl.url_objects import MediaItem@final-class MegaDownloadClient(DownloadClient):+class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIssues]def __init__(self, manager: Manager) -> None:- super().__init__(manager, manager.http_client)+ super().__init__(manager)self._decrypt_mapping: dict[URL, tuple[Crypto, int]] = {}self._supports_ranges = False- async def _append_content(self, media_item: MediaItem, hook: ProgressHook, content: aiohttp.StreamReader) -> None:+ @override+ async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:"""Appends content to a file."""check_free_space = storage.create_free_space_checker(media_item)@@ -38,14 +39,15 @@ class MegaDownloadClient(DownloadClient):crypto, file_size = self._decrypt_mapping.pop(media_item.url)chunk_decryptor = MegaChunker(crypto.key, crypto.iv, crypto.meta_mac)+ aiohttp_resp = resp.aiohttp_respasync with aio.open(media_item.partial_file, mode="ab") as f:for _, chunk_size in get_chunks(file_size):- raw_chunk = await content.readexactly(chunk_size)+ raw_chunk = await aiohttp_resp.content.readexactly(chunk_size)chunk = chunk_decryptor.read(raw_chunk)await check_free_space()chunk_size = len(chunk)- await self.client.speed_limiter.acquire(chunk_size)+ await self.speed_limiter.acquire(chunk_size)await f.write(chunk)hook.advance(chunk_size)check_download_speed()@@ -68,8 +70,8 @@ class MegaDownloader(Downloader):super(MegaDownloader, self).__post_init__()self._client = MegaDownloadClient(self.manager)- @override@property+ @overridedef client(self) -> MegaDownloadClient:return self._clientdiff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.pyindex c14b9a887..5c58b8e38 100644--- a/cyberdrop_dl/manager.py+++ b/cyberdrop_dl/manager.py@@ -16,6 +16,7 @@ from pydantic.types import ByteSizefrom cyberdrop_dl import __version__, cookies, env, ffmpeg, stats, yamlfrom cyberdrop_dl.cli import CLIargsfrom cyberdrop_dl.clients.client import HTTPClient+from cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.config import Configfrom cyberdrop_dl.csv_logs import CSVLogsManagerfrom cyberdrop_dl.database import Database@@ -52,9 +53,11 @@ class Manager:self._completed_downloads: list[MediaItem] = []self.hasher: Hasher = Hasher(self)self.logs: CSVLogsManager = CSVLogsManager.from_manager(self)+ self.http_client: HTTPClient = HTTPClient(self)+ self.download_client: DownloadClient = DownloadClient(self)+self.scrape_mapper: ScrapeMapperself.database: Database- self.http_client: HTTPClientself.deduper: Czkawkaself.sorter: Sorter@@ -85,13 +88,16 @@ class Manager:)self.deduper = Czkawka.from_manager(self)self.sorter = Sorter.from_manager(self)- self.http_client = HTTPClient(self)with (_cache_context(self.appdata.cache_file, self.cache),_enter_context(REFRESH_RATE, self.config.global_settings.ui_options.refresh_rate),_enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),):- yield self+ try:+ yield self+ finally:+ del self.deduper+ del self.sorterdef add_completed(self, media_item: MediaItem) -> None:if media_item.is_segment:diff --git a/tests/test_client.py b/tests/test_client.pyindex 9f58d1b2e..a2b167f47 100644--- a/tests/test_client.py+++ b/tests/test_client.py@@ -1,3 +1,4 @@+import sslimport sysimport aiohttp@@ -13,6 +14,13 @@ def client(manager: Manager) -> HTTPClient:return HTTPClient(manager)+def test_initial_state(client: HTTPClient) -> None:+ assert client._cookies is None+ assert client._flaresolverr is None+ assert isinstance(client.ssl_context, ssl.SSLContext)+ assert client.rate_limits == {}++async def test_context_manager(client: HTTPClient) -> None:with pytest.raises(AttributeError):_ = client._session@@ -35,3 +43,18 @@ async def test_context_manager(client: HTTPClient) -> None:with pytest.raises(ScrapeError):_ = client.curl_session# test_ssl_context.py+++def test_create_aiohttp_session_outside_loop(client: HTTPClient) -> None:+ with pytest.raises(RuntimeError, match="no running event loop"):+ _ = client.create_aiohttp_session()+++async def test_create_aiohttp_session(client: HTTPClient) -> None:+ async with client, client.create_aiohttp_session() as session:+ assert len(session.headers) == 1+ assert session.headers.get("User-agent")+ assert session.cookie_jar is client.cookies+ assert session._raise_for_status is False+ assert session.timeout == client.manager.config.global_settings.rate_limiting_options._aiohttp_timeout+ assert session.requote_redirect_url is Falsediff --git a/tests/test_csv_logs.py b/tests/test_csv_logs.pynew file mode 100644index 000000000..3ac2402e4--- /dev/null+++ b/tests/test_csv_logs.py@@ -0,0 +1,32 @@+import datetime+from pathlib import Path++import pytest++from cyberdrop_dl import csv_logs+from cyberdrop_dl.url_objects import AbsoluteHttpURL++now = datetime.datetime(2026, 5, 8)+++ "url, expected",+ [+ (+ "https://static.scdn.st/e2f8a4c6-3d7b-4e19-9a5c-8b1d6f0e3a7c/thumbs/f7e8625e-b50b-42c3-93da-d74bce716c41-md8f1c79417cd16004-zU4LIum1.png",+ "/https-static.scdn.st-e2f8a4c6-3d7b-4e19-9a5c-8b1d6f0e3a7c-thumbs-f7e8625e-b50b-42c3-93da-d74bce716c41-md8f1c79417cd16004-zU4LIum1.png_20260508_000000.html",+ ),+ (+ "https://cloud.mail.ru/public/d1LR/hMBXG2mFo",+ "/https-cloud.mail.ru-public-d1LR-hMBXG2mFo_20260508_000000.html",+ ),+ (+ "https://www.tokyomotion.net/video/5884366/%E3%83%A1%E3%82%B9%E7%89%9B%E3%81%8A%E3%81%BB%E3%82%A4%E3%82%AD%E3%83%81%E3%82%AF%E3%83%8B%E3%83%BC",+ "/https-www.tokyomotion.net-video-5884366-%E3%83%A1%E3%82%B9%E7%89%9B%E3%81%8A%E3%81%BB%E3%82%A4%E3%82%AD%E3%83%81%E3%82%AF%E3%83%8B%E3%83%BC_20260508_000000.html",+ ),+ ],+)+def test_prepare_resp_filename(url: str, expected: str) -> None:+ result = csv_logs._prepare_resp_file(Path("/"), AbsoluteHttpURL(url), now)+ assert result.as_posix().count("/") == 1+ assert result.as_posix() == expecteddiff --git a/tests/test_download_client.py b/tests/test_download_client.pynew file mode 100644index 000000000..26262d171--- /dev/null+++ b/tests/test_download_client.py@@ -0,0 +1,45 @@+import pytest+from multidict import CIMultiDict++from cyberdrop_dl.clients.download_client import DownloadClient, _check_content_type, _get_content_type+from cyberdrop_dl.exceptions import InvalidContentTypeError+from cyberdrop_dl.manager import Manager+++ "limit",+ [+ (5_000_000,),+ (500_000_000,),+ ],+)+def test_chunk_size_is_never_greater_that_speed_limit(manager: Manager, limit: int) -> None:+ max_expected = 1024 * 1024 * 10+ limit = manager.config.global_settings.rate_limiting_options.download_speed_limit+ assert limit == 0+ client = DownloadClient(manager)+ assert client.chunk_size != limit+ assert client.chunk_size == max_expected++ manager.config.global_settings.rate_limiting_options.download_speed_limit = limit+ client = DownloadClient(manager)+ assert client.chunk_size == min(limit or max_expected, max_expected)+++def test_get_content_type() -> None:+ def get(value: str) -> str:+ return _get_content_type(CIMultiDict({"content-type": value}))++ assert get("text/vnd.trolltech.linguist") == "video/MP2T"+ assert get("text/HTML") == "text/HTML"+ assert get("application/json") == "application/json"++ with pytest.raises(InvalidContentTypeError):+ _ = get("")+++def test_check_content_type() -> None:+ _check_content_type("text/html", ".txt")++ with pytest.raises(InvalidContentTypeError, match="Received 'text/html', was expecting binary payload"):+ _check_content_type("text/html", ".mp4")diff --git a/tests/test_manager.py b/tests/test_manager.pyindex f3690ddd3..28f14723f 100644--- a/tests/test_manager.py+++ b/tests/test_manager.py@@ -48,7 +48,7 @@ def test_manager_context() -> None:config = Config.parse_args(["--refresh-rate", "40"])manager = Manager(config=config)- for attr in ("database", "deduper", "sorter", "http_client"):+ for attr in ("database", "deduper", "sorter"):with pytest.raises(AttributeError):getattr(manager, attr)diff --git a/tests/test_response.py b/tests/test_response.pynew file mode 100644index 000000000..a8c5ad570--- /dev/null+++ b/tests/test_response.py@@ -0,0 +1,35 @@+import datetime++from multidict import CIMultiDict, CIMultiDictProxy++from cyberdrop_dl.clients.response import _AIOHTTPResponse+from cyberdrop_dl.url_objects import AbsoluteHttpURL+++def make_resp(content_type: str, url: str = "https://example.com") -> _AIOHTTPResponse:+ resp = _AIOHTTPResponse(+ content_type,+ status=200,+ headers=CIMultiDictProxy(CIMultiDict({"content-type": content_type})),+ url=AbsoluteHttpURL(url),+ location=None,+ _resp=None, # pyright: ignore[reportArgumentType]+ )+ resp.created_at = datetime.datetime.min+ return resp+++def test_json_dump_binary_resp() -> None:+ content_type = "application/octet-stream"+ resp = make_resp(content_type)+ json = resp.__json__()+ assert content_type in json["content"]+ assert json == {+ "url": "https://example.com",+ "status_code": 200,+ "created_at": "0001-01-01 00:00:00",+ "response_headers": {+ "content-type": "application/octet-stream",+ },+ "content": "<application/octet-stream payload>",+ }