Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: allow a null socket read timeout (#1313)
NTFSvolume1900d6bb71ce
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex 5f059ab76..15f4bc643 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -136,7 +136,7 @@ class FlareSolverr:if not self.url:raise DDOSGuardError("Found DDoS challenge, but FlareSolverr is not configured")- timeout = self.manager.global_config.rate_limiting_options._scrape_timeout+ timeout = self.manager.global_config.rate_limiting_options._aiohttp_timeoutif command is _Command.CREATE_SESSION:timeout = aiohttp.ClientTimeout(total=5 * 60, connect=60) # 5 minutes to create sessiondiff --git a/cyberdrop_dl/config/global_model.py b/cyberdrop_dl/config/global_model.pyindex 6daa058f5..5b5e980a4 100755--- a/cyberdrop_dl/config/global_model.py+++ b/cyberdrop_dl/config/global_model.py@@ -1,28 +1,27 @@import randomfrom datetime import timedelta-from typing import Literal, NamedTuple+from typing import Literalimport aiohttp-from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt, field_serializer, field_validator+from pydantic import (+ BaseModel,+ ByteSize,+ NonNegativeFloat,+ PositiveFloat,+ PositiveInt,+ field_serializer,+ field_validator,+)from yarl import URLfrom cyberdrop_dl.config._common import ConfigModel, Fieldfrom cyberdrop_dl.models.types import ByteSizeSerilized, HttpURL, ListNonEmptyStr, ListPydanticURL, NonEmptyStr-from cyberdrop_dl.models.validators import falsy_as, to_bytesize, to_timedelta+from cyberdrop_dl.models.validators import falsy_as, falsy_as_none, to_bytesize, to_timedeltaMIN_REQUIRED_FREE_SPACE = to_bytesize("512MB")DEFAULT_REQUIRED_FREE_SPACE = to_bytesize("5GB")-class Timeout(NamedTuple):- connect: int- read: int-- @property- def total(self) -> int:- return self.read + self.connect--class General(BaseModel):# TODO: Move `ssl_context` to an advance config sectionssl_context: Literal["truststore", "certifi", "truststore+certifi"] | None = "truststore+certifi"@@ -63,7 +62,6 @@ class General(BaseModel):class RateLimiting(BaseModel):- connection_timeout: PositiveInt = 15download_attempts: PositiveInt = 2download_delay: NonNegativeFloat = 0.0download_speed_limit: ByteSizeSerilized = ByteSize(0)@@ -73,12 +71,24 @@ class RateLimiting(BaseModel):max_simultaneous_downloads_per_domain: PositiveInt = 5max_simultaneous_downloads: PositiveInt = 15rate_limit: PositiveInt = 50- read_timeout: PositiveInt = 300++ connection_timeout: PositiveFloat = 15+ read_timeout: PositiveFloat | None = 300++ @field_validator("read_timeout", mode="before")+ @classmethod+ def parse_timeouts(cls, value: object) -> object | None:+ return falsy_as_none(value)def model_post_init(self, *_) -> None:- self._timeout = Timeout(self.connection_timeout, self.read_timeout)- self._scrape_timeout = aiohttp.ClientTimeout(total=self._timeout.total, connect=self._timeout.connect)- self._download_timeout = aiohttp.ClientTimeout(total=None, connect=self._timeout.connect)+ self._curl_timeout = self.connection_timeout+ if self.read_timeout is not None:+ self._curl_timeout = self.connection_timeout, self.read_timeout+ self._aiohttp_timeout = aiohttp.ClientTimeout(+ total=None,+ sock_connect=self.connection_timeout,+ sock_read=self.read_timeout,+ )@field_validator("file_host_cache_expire_after", "forum_cache_expire_after", mode="before")@staticmethoddiff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 8bd576c8b..af0f6ac45 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -274,7 +274,7 @@ class ClientManager:impersonate="chrome",verify=bool(self.ssl_context),proxy=proxy_or_none,- timeout=self.rate_limiting_options._timeout,+ timeout=self.rate_limiting_options._curl_timeout,cookies={cookie.key: cookie.value for cookie in self.cookies},)@@ -300,11 +300,11 @@ class ClientManager:self, cached: bool = False, trace_configs: list[aiohttp.TraceConfig] | None = None) -> CachedSession | ClientSession:if cached:- timeout = self.rate_limiting_options._scrape_timeout+ timeout = self.rate_limiting_options._aiohttp_timeoutsession_cls = CachedSessionkwargs: dict[str, Any] = {"cache": self.manager.cache_manager.request_cache}else:- timeout = self.rate_limiting_options._download_timeout+ timeout = self.rate_limiting_options._aiohttp_timeoutsession_cls = ClientSessionkwargs = {}return session_cls(