Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: allow multiple Flaresolverr requests

NTFSvolume405c9bb3d69b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 80fcb31df..5543047b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Support multiple files as input
- Support mixing URLs and input files on the same run (all files must exists and all URLs/files must be positional arguments)
+### Changed
+
+- Allow up to 10 concurrent requests with Flaresolverr
+
### Fixed
- URL matching (Anysex)
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index 24e5e5449..299d76016 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -108,7 +108,7 @@ class Client:
_session_id: str = dataclasses.field(init=False, default="")
_session_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)
- _request_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)
+ _sem: asyncio.Semaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.Semaphore(10))
_request_id: Callable[[], int] = dataclasses.field(
init=False, repr=False, default_factory=lambda: itertools.count(1).__next__
)
@@ -196,7 +196,7 @@ class Client:
) -> Response:
req_params, json_data = self._prepare_req(command, json_data, data=data, wait=wait)
- async with self._request_lock:
+ async with self._sem:
request_id = self._request_id()
msg = (
"Destroying Flaresolverr session"
@@ -227,7 +227,7 @@ class Client:
if proxy := self._aiohttp_session._default_proxy:
params.update(proxy={"url": str(proxy)})
- resp = await self._request(Command.CREATE_SESSION, session=session_id, **params)
+ resp = await self._request(Command.CREATE_SESSION, session=session_id, wait=None, **params)
if not resp.ok:
raise FlaresolverrError(f"FlareSolverr said: {resp.message}")
self._session_id = session_id
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index 03733105a..c2303cde1 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -384,6 +384,17 @@ class HTTPController(Protocol):
class HTTPMixin(HTTPController, Protocol):
+ @contextlib.asynccontextmanager
+ async def rate_limit_ctx(
+ self,
+ ) -> AsyncGenerator[None]:
+ ctx = self.__http_ctx__
+ if ctx.throttle is not None:
+ await ctx.throttle()
+
+ async with self.client.rate_limit_ctx(ctx.domain, ctx.json_check):
+ yield
+
@contextlib.asynccontextmanager
async def request(
self,
@@ -394,18 +405,21 @@ class HTTPMixin(HTTPController, Protocol):
) -> AsyncGenerator[AbstractResponse[Any]]:
ctx = self.__http_ctx__
- if ctx.throttle is not None:
- await ctx.throttle()
-
kwargs.setdefault("impersonate", ctx.impersonate)
kwargs["headers"] = ctx.headers | kwargs.setdefault("headers", {})
async with (
- self.client.rate_limit_ctx(ctx.domain, ctx.json_check),
+ self.rate_limit_ctx(),
self.client.request(url, method, **kwargs) as resp,
):
yield resp
+ async def flaresolverr_request(
+ self, url: AbsoluteHttpURL, data: Any | None = None, wait: int | None = None
+ ) -> AbstractResponse[Any]:
+ async with self.rate_limit_ctx():
+ return await self.client.flaresolverr_request(url, data, wait)
+
async def request_json(
self,
url: AbsoluteHttpURL,
diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.py
index b9a8eb290..1dfb425be 100644
--- a/cyberdrop_dl/crawlers/fileditch.py
+++ b/cyberdrop_dl/crawlers/fileditch.py
@@ -73,7 +73,7 @@ class FileditchCrawler(Crawler):
await self.handle_file(src, scrape_item, filename, ext, thumbnail=thumb)
async def request_download(self, url: AbsoluteHttpURL) -> tuple[AbsoluteHttpURL, str | None]:
- resp = await self.client.flaresolverr_request(url, wait=20)
+ resp = await self.flaresolverr_request(url, wait=20)
soup = await resp.soup()
if soup.select_one(".gone-path"):
raise ScrapeError(410)