Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: allow multiple Flaresolverr requests
NTFSvolume405c9bb3d69b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 80fcb31df..5543047b8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Support multiple files as input- Support mixing URLs and input files on the same run (all files must exists and all URLs/files must be positional arguments)+### Changed++- Allow up to 10 concurrent requests with Flaresolverr+### Fixed- URL matching (Anysex)diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex 24e5e5449..299d76016 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -108,7 +108,7 @@ class Client:_session_id: str = dataclasses.field(init=False, default="")_session_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)- _request_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)+ _sem: asyncio.Semaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.Semaphore(10))_request_id: Callable[[], int] = dataclasses.field(init=False, repr=False, default_factory=lambda: itertools.count(1).__next__)@@ -196,7 +196,7 @@ class Client:) -> Response:req_params, json_data = self._prepare_req(command, json_data, data=data, wait=wait)- async with self._request_lock:+ async with self._sem:request_id = self._request_id()msg = ("Destroying Flaresolverr session"@@ -227,7 +227,7 @@ class Client:if proxy := self._aiohttp_session._default_proxy:params.update(proxy={"url": str(proxy)})- resp = await self._request(Command.CREATE_SESSION, session=session_id, **params)+ resp = await self._request(Command.CREATE_SESSION, session=session_id, wait=None, **params)if not resp.ok:raise FlaresolverrError(f"FlareSolverr said: {resp.message}")self._session_id = session_iddiff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 03733105a..c2303cde1 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -384,6 +384,17 @@ class HTTPController(Protocol):class HTTPMixin(HTTPController, Protocol):+ @contextlib.asynccontextmanager+ async def rate_limit_ctx(+ self,+ ) -> AsyncGenerator[None]:+ ctx = self.__http_ctx__+ if ctx.throttle is not None:+ await ctx.throttle()++ async with self.client.rate_limit_ctx(ctx.domain, ctx.json_check):+ yield+@contextlib.asynccontextmanagerasync def request(self,@@ -394,18 +405,21 @@ class HTTPMixin(HTTPController, Protocol):) -> AsyncGenerator[AbstractResponse[Any]]:ctx = self.__http_ctx__- if ctx.throttle is not None:- await ctx.throttle()-kwargs.setdefault("impersonate", ctx.impersonate)kwargs["headers"] = ctx.headers | kwargs.setdefault("headers", {})async with (- self.client.rate_limit_ctx(ctx.domain, ctx.json_check),+ self.rate_limit_ctx(),self.client.request(url, method, **kwargs) as resp,):yield resp+ async def flaresolverr_request(+ self, url: AbsoluteHttpURL, data: Any | None = None, wait: int | None = None+ ) -> AbstractResponse[Any]:+ async with self.rate_limit_ctx():+ return await self.client.flaresolverr_request(url, data, wait)+async def request_json(self,url: AbsoluteHttpURL,diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.pyindex b9a8eb290..1dfb425be 100644--- a/cyberdrop_dl/crawlers/fileditch.py+++ b/cyberdrop_dl/crawlers/fileditch.py@@ -73,7 +73,7 @@ class FileditchCrawler(Crawler):await self.handle_file(src, scrape_item, filename, ext, thumbnail=thumb)async def request_download(self, url: AbsoluteHttpURL) -> tuple[AbsoluteHttpURL, str | None]:- resp = await self.client.flaresolverr_request(url, wait=20)+ resp = await self.flaresolverr_request(url, wait=20)soup = await resp.soup()if soup.select_one(".gone-path"):raise ScrapeError(410)