Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: allow a null socket read timeout (#1313)

NTFSvolume1900d6bb71ce
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index 5f059ab76..15f4bc643 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -136,7 +136,7 @@ class FlareSolverr:
if not self.url:
raise DDOSGuardError("Found DDoS challenge, but FlareSolverr is not configured")
- timeout = self.manager.global_config.rate_limiting_options._scrape_timeout
+ timeout = self.manager.global_config.rate_limiting_options._aiohttp_timeout
if command is _Command.CREATE_SESSION:
timeout = aiohttp.ClientTimeout(total=5 * 60, connect=60) # 5 minutes to create session
diff --git a/cyberdrop_dl/config/global_model.py b/cyberdrop_dl/config/global_model.py
index 6daa058f5..5b5e980a4 100755
--- a/cyberdrop_dl/config/global_model.py
+++ b/cyberdrop_dl/config/global_model.py
@@ -1,28 +1,27 @@
import random
from datetime import timedelta
-from typing import Literal, NamedTuple
+from typing import Literal
import aiohttp
-from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt, field_serializer, field_validator
+from pydantic import (
+ BaseModel,
+ ByteSize,
+ NonNegativeFloat,
+ PositiveFloat,
+ PositiveInt,
+ field_serializer,
+ field_validator,
+)
from yarl import URL
from cyberdrop_dl.config._common import ConfigModel, Field
from cyberdrop_dl.models.types import ByteSizeSerilized, HttpURL, ListNonEmptyStr, ListPydanticURL, NonEmptyStr
-from cyberdrop_dl.models.validators import falsy_as, to_bytesize, to_timedelta
+from cyberdrop_dl.models.validators import falsy_as, falsy_as_none, to_bytesize, to_timedelta
MIN_REQUIRED_FREE_SPACE = to_bytesize("512MB")
DEFAULT_REQUIRED_FREE_SPACE = to_bytesize("5GB")
-class Timeout(NamedTuple):
- connect: int
- read: int
-
- @property
- def total(self) -> int:
- return self.read + self.connect
-
-
class General(BaseModel):
# TODO: Move `ssl_context` to an advance config section
ssl_context: Literal["truststore", "certifi", "truststore+certifi"] | None = "truststore+certifi"
@@ -63,7 +62,6 @@ class General(BaseModel):
class RateLimiting(BaseModel):
- connection_timeout: PositiveInt = 15
download_attempts: PositiveInt = 2
download_delay: NonNegativeFloat = 0.0
download_speed_limit: ByteSizeSerilized = ByteSize(0)
@@ -73,12 +71,24 @@ class RateLimiting(BaseModel):
max_simultaneous_downloads_per_domain: PositiveInt = 5
max_simultaneous_downloads: PositiveInt = 15
rate_limit: PositiveInt = 50
- read_timeout: PositiveInt = 300
+
+ connection_timeout: PositiveFloat = 15
+ read_timeout: PositiveFloat | None = 300
+
+ @field_validator("read_timeout", mode="before")
+ @classmethod
+ def parse_timeouts(cls, value: object) -> object | None:
+ return falsy_as_none(value)
def model_post_init(self, *_) -> None:
- self._timeout = Timeout(self.connection_timeout, self.read_timeout)
- self._scrape_timeout = aiohttp.ClientTimeout(total=self._timeout.total, connect=self._timeout.connect)
- self._download_timeout = aiohttp.ClientTimeout(total=None, connect=self._timeout.connect)
+ self._curl_timeout = self.connection_timeout
+ if self.read_timeout is not None:
+ self._curl_timeout = self.connection_timeout, self.read_timeout
+ self._aiohttp_timeout = aiohttp.ClientTimeout(
+ total=None,
+ sock_connect=self.connection_timeout,
+ sock_read=self.read_timeout,
+ )
@field_validator("file_host_cache_expire_after", "forum_cache_expire_after", mode="before")
@staticmethod
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 8bd576c8b..af0f6ac45 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -274,7 +274,7 @@ class ClientManager:
impersonate="chrome",
verify=bool(self.ssl_context),
proxy=proxy_or_none,
- timeout=self.rate_limiting_options._timeout,
+ timeout=self.rate_limiting_options._curl_timeout,
cookies={cookie.key: cookie.value for cookie in self.cookies},
)
@@ -300,11 +300,11 @@ class ClientManager:
self, cached: bool = False, trace_configs: list[aiohttp.TraceConfig] | None = None
) -> CachedSession | ClientSession:
if cached:
- timeout = self.rate_limiting_options._scrape_timeout
+ timeout = self.rate_limiting_options._aiohttp_timeout
session_cls = CachedSession
kwargs: dict[str, Any] = {"cache": self.manager.cache_manager.request_cache}
else:
- timeout = self.rate_limiting_options._download_timeout
+ timeout = self.rate_limiting_options._aiohttp_timeout
session_cls = ClientSession
kwargs = {}
return session_cls(