Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
import os
from collections.abc import AsyncGenerator
import aiohttp
import pytest
from cyberdrop_dl.clients import HttpMethod
from cyberdrop_dl.clients.flaresolverr import (
MAX_TIMEOUT,
Client,
Command,
Config,
Request,
RequestCommand,
Response,
_cmd_from_http_method,
_default_session_name,
)
from cyberdrop_dl.exceptions import CloudflareBlockedError, DDOSGuardError, FlaresolverrError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
ENV_NAME = "CDL_FLARESOLVERR"
FLARESOLVER_URL = os.environ.get(ENV_NAME, "") # or "http://localhost:8191"
def needs_flaresolverr() -> pytest.MarkDecorator:
return pytest.mark.skipif(not FLARESOLVER_URL, reason=f"{ENV_NAME} environment variable is not set")
@pytest.fixture
async def flaresolverr() -> AsyncGenerator[Client]:
async with aiohttp.ClientSession() as session:
yield Client(
session,
Config(
url=AbsoluteHttpURL(FLARESOLVER_URL) / "v1",
use_session=True,
),
)
@needs_flaresolverr()
def test_flaresolver(flaresolverr: Client) -> None:
assert flaresolverr.config.url
assert flaresolverr.request_id() == 1
assert flaresolverr.request_id() == 2
@needs_flaresolverr()
async def test_create_session(flaresolverr: Client) -> None:
assert flaresolverr.session is None
resp = await flaresolverr._request(Command.CREATE_SESSION, {"session": "cyberdrop-dl"})
assert resp.ok
assert "Session created successfully" in resp.message or "Session already exists" in resp.message
assert resp.solution is None
resp = await flaresolverr._request(Command.DESTROY_SESSION, {"session": "cyberdrop-dl"})
assert "The session has been removed" in resp.message
@needs_flaresolverr()
async def test_create_session_methods(flaresolverr: Client) -> None:
assert flaresolverr.session is None
name = "cyberdrop-dl test"
await flaresolverr.create_session(name)
assert flaresolverr.session is None
await flaresolverr.destroy_session(name)
assert flaresolverr.session is None
await flaresolverr._ensure_session()
assert flaresolverr.session
assert flaresolverr.session == _default_session_name()
@needs_flaresolverr()
async def test_request_w_solution(flaresolverr: Client) -> None:
url = AbsoluteHttpURL("https://google.com")
solution = await flaresolverr.request(url)
assert solution.status == 200
assert solution.url != url # should have www. as prefix
assert solution.user_agent
assert isinstance(solution.content, str)
assert isinstance(solution.user_agent, str)
assert "html" in solution.content
assert solution.cookies
for cookie in solution.cookies.values():
assert url.host in cookie["domain"]
@pytest.mark.parametrize(
("method", "expected"),
[
("GET", Command.GET_REQUEST),
("HEAD", Command.GET_REQUEST),
("POST", Command.POST_REQUEST),
("PUT", Command.POST_REQUEST),
("DELETE", Command.POST_REQUEST),
],
)
def test_cmd_from_http_method(method: HttpMethod, expected: Command) -> None:
assert _cmd_from_http_method(method) is expected
@pytest.mark.parametrize("method", [("PATCH", "QUERY", "OPTIONS")])
def test_cmd_from_unsupported_http_method(method: HttpMethod) -> None:
with pytest.raises(ValueError):
_cmd_from_http_method(method)
def _config(*, proxy: bool, use_session: bool) -> Config:
return Config(
url=AbsoluteHttpURL("https://example.com"),
use_session=use_session,
proxy=AbsoluteHttpURL("https://example.com/proxy") if proxy else None,
)
@pytest.mark.parametrize(
("command", "config", "wait", "session", "expected"),
[
(
Command.GET_REQUEST,
_config(proxy=True, use_session=True),
None,
"sessionA",
Request(
Command.GET_REQUEST,
{
"cmd": "request.get",
"maxTimeout": 60_000,
"url": "https://example.com",
"session": "sessionA",
},
),
),
(
Command.GET_REQUEST,
_config(proxy=True, use_session=True),
20,
"sessionA",
Request(
Command.GET_REQUEST,
{
"cmd": "request.get",
"maxTimeout": 60_000,
"url": "https://example.com",
"waitInSeconds": 20,
"session": "sessionA",
},
aiohttp.ClientTimeout(sock_read=80, sock_connect=60),
),
),
(
Command.POST_REQUEST,
_config(proxy=True, use_session=False),
20,
"sessionA",
Request(
Command.POST_REQUEST,
{
"cmd": "request.post",
"maxTimeout": 60_000,
"url": "https://example.com",
"waitInSeconds": 20,
"proxy": {
"url": "https://example.com/proxy",
},
},
aiohttp.ClientTimeout(sock_read=80, sock_connect=60),
),
),
],
)
def test_request_build(
command: RequestCommand, config: Config, *, wait: int | None, session: str | None, expected: Request
) -> None:
req = Request.build(command, AbsoluteHttpURL("https://example.com"), config, wait=wait, session=session)
assert req.command is expected.command
assert req.payload == expected.payload
assert req.timeout == expected.timeout
def test_request_always_overrides_min_timeout() -> None:
for req in (
Request(Command.GET_REQUEST, {}),
Request(Command.GET_REQUEST, {}, aiohttp.ClientTimeout(sock_read=1, sock_connect=1)),
):
assert req.timeout.sock_read
assert req.timeout.sock_read >= MAX_TIMEOUT
assert req.timeout.sock_connect
assert req.timeout.sock_connect >= MAX_TIMEOUT
def test_wait_in_seconds_overrides_read() -> None:
wait = 120
req = Request(Command.GET_REQUEST, {"waitInSeconds": wait})
assert req.timeout.sock_read
assert req.timeout.sock_read >= MAX_TIMEOUT + wait
assert req.timeout.sock_connect
assert req.timeout.sock_connect >= MAX_TIMEOUT
_CF_BLOCKED = (
"Error: Error solving the challenge. Cloudflare has blocked this request. "
"Probably your IP is banned for this site, check in your web browser."
)
@pytest.mark.parametrize(
("message", "expected_error", "disabled"),
[
(_CF_BLOCKED, CloudflareBlockedError, False),
("Error: Error solving the challenge. Timeout after 60.0 seconds.", FlaresolverrError, True),
],
)
async def test_request_error_response(
monkeypatch: pytest.MonkeyPatch, message: str, expected_error: type[Exception], *, disabled: bool
) -> None:
async def fake_request(*_, **__) -> Response:
return Response.parse(1, {"status": "error", "message": message})
async with aiohttp.ClientSession() as session:
client = Client(session, _config(proxy=False, use_session=False))
monkeypatch.setattr(client, "_request", fake_request)
for _ in range(client.MAX_INVALID_RESPONSES + 1):
assert client.is_down is False
with pytest.raises(expected_error):
await client.request(AbsoluteHttpURL("https://example.com"))
assert client.is_down is disabled
def test_cloudflare_blocked_is_ddos_guard_error() -> None:
assert issubclass(CloudflareBlockedError, DDOSGuardError)
assert CloudflareBlockedError().ui_failure == "Cloudflare Blocked"