Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

15.1 KB
Raw Blame
from __future__ import annotations

import asyncio
import logging
from contextlib import asynccontextmanager
from ipaddress import ip_address
from typing import TYPE_CHECKING, Any

import pytest
from w3lib.http import basic_auth_header

from scrapy.core.downloader.handlers._aioftp import _HAS_SOCKS, AioftpDownloadHandler
from scrapy.exceptions import (
    CannotResolveHostError,
    DownloadCancelledError,
    DownloadConnectionRefusedError,
    DownloadFailedError,
    DownloadTimeoutError,
    NotConfigured,
    ResponseDataLossError,
)
from scrapy.http import BodyFileResponse, HtmlResponse, Request, TextResponse
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import coroutine_test

try:
    from siosocks.io.asyncio import socks_server_handler
except ImportError:
    socks_server_handler = None

if TYPE_CHECKING:
    from collections.abc import AsyncGenerator, Callable, Generator
    from pathlib import Path

    from scrapy.http import Response

pytestmark = pytest.mark.only_asyncio

FILES = {
    "file.txt": b"I have the power!",
    "file with spaces.txt": b"Moooooooooo power!",
    "html-file-without-extension": b"<!DOCTYPE html>\n<title>.</title>",
}


def _write_files(server: MockFTPServer) -> MockFTPServer:
    for name, content in FILES.items():
        (server.path / name).write_bytes(content)
    return server


@pytest.fixture(scope="module")
def ftp_server() -> Generator[MockFTPServer]:
    with MockFTPServer() as server:
        yield _write_files(server)


@pytest.fixture(scope="module")
def ftps_server() -> Generator[MockFTPServer]:
    with MockFTPServer(tls=True) as server:
        yield _write_files(server)


async def _download(
    request: Request, settings: dict[str, Any] | None = None
) -> Response:
    crawler = get_crawler(settings_dict=settings)
    dh = build_from_crawler(AioftpDownloadHandler, crawler)
    try:
        return await dh.download_request(request)
    finally:
        await dh.close()


@pytest.mark.parametrize(
    ("path", "response_class"),
    [
        ("file.txt", TextResponse),
        ("file with spaces.txt", TextResponse),
        ("html-file-without-extension", HtmlResponse),
    ],
)
@coroutine_test
async def test_download(
    ftp_server: MockFTPServer, path: str, response_class: type[Response]
) -> None:
    response = await _download(Request(ftp_server.url(path)))
    assert type(response) is response_class
    assert response.status == 200
    assert response.body == FILES[path]
    assert response.headers == {
        b"Local Filename": [b""],
        b"Size": [str(len(FILES[path])).encode()],
    }
    assert response.protocol == "FTP"
    assert response.ip_address == ip_address("127.0.0.1")
    assert response.certificate is None


@coroutine_test
async def test_download_ftps(
    ftps_server: MockFTPServer, caplog: pytest.LogCaptureFixture
) -> None:
    settings = {"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}
    with caplog.at_level(logging.DEBUG):
        response = await _download(Request(ftps_server.url("file.txt")), settings)
    assert response.body == FILES["file.txt"]
    assert isinstance(response.certificate, bytes)
    assert "SSL connection to 127.0.0.1 using protocol" in caplog.text


@coroutine_test
async def test_tls_verbose_logging_without_tls(ftp_server: MockFTPServer) -> None:
    settings = {"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}
    response = await _download(Request(ftp_server.url("file.txt")), settings)
    assert response.body == FILES["file.txt"]


@coroutine_test
async def test_nonexistent(ftp_server: MockFTPServer) -> None:
    response = await _download(Request(ftp_server.url("nonexistent.txt")))
    assert response.status == 404
    assert response.body == b""


@pytest.mark.parametrize("path", ["", "/"])
@coroutine_test
async def test_list(ftp_server: MockFTPServer, path: str) -> None:
    url = f"ftp://{ftp_server.host}:{ftp_server.port}{path}"
    response = await _download(Request(url))
    assert isinstance(response, TextResponse)
    assert response.status == 200
    assert set(response.text.splitlines()) == set(FILES)


@coroutine_test
async def test_list_nonexistent(ftp_server: MockFTPServer) -> None:
    response = await _download(Request(ftp_server.url("nonexistent/")))
    assert response.status == 404
    assert response.body == b""


@coroutine_test
async def test_invalid_credentials(ftp_server: MockFTPServer) -> None:
    request = Request(ftp_server.url("file.txt"), meta={"ftp_user": "invalid"})
    with pytest.raises(DownloadFailedError):
        await _download(request)


@pytest.mark.parametrize("local", [False, True])
@coroutine_test
async def test_maxsize(ftp_server: MockFTPServer, tmp_path: Path, local: bool) -> None:
    meta: dict[str, Any] = {"download_maxsize": 10}
    if local:
        meta["ftp_local_filename"] = str(tmp_path / "file.txt").encode()
    request = Request(ftp_server.url("file.txt"), meta=meta)
    with pytest.raises(DownloadCancelledError):
        await _download(request)


@coroutine_test
async def test_maxsize_disabled(ftp_server: MockFTPServer) -> None:
    request = Request(ftp_server.url("file.txt"), meta={"download_maxsize": 0})
    response = await _download(request, {"DOWNLOAD_MAXSIZE": 10})
    assert response.body == FILES["file.txt"]


@coroutine_test
async def test_warnsize(
    ftp_server: MockFTPServer, caplog: pytest.LogCaptureFixture
) -> None:
    request = Request(ftp_server.url("file.txt"), meta={"download_warnsize": 10})
    response = await _download(request)
    assert response.body == FILES["file.txt"]
    assert "which is larger than download warn size (10)" in caplog.text


@coroutine_test
async def test_active_mode_meta(
    ftp_server: MockFTPServer, caplog: pytest.LogCaptureFixture
) -> None:
    request = Request(ftp_server.url("file.txt"), meta={"ftp_passive": False})
    response = await _download(request)
    assert response.body == FILES["file.txt"]
    assert "does not support active mode" in caplog.text


@coroutine_test
async def test_active_mode() -> None:
    crawler = get_crawler(settings_dict={"FTP_PASSIVE_MODE": False})
    with pytest.raises(NotConfigured):
        build_from_crawler(AioftpDownloadHandler, crawler)


@coroutine_test
async def test_connection_refused() -> None:
    server = await asyncio.start_server(lambda r, w: None, "127.0.0.1", 0)
    port = server.sockets[0].getsockname()[1]
    server.close()
    await server.wait_closed()
    with pytest.raises(DownloadConnectionRefusedError):
        await _download(Request(f"ftp://127.0.0.1:{port}/file.txt"))


@coroutine_test
async def test_cannot_resolve_host() -> None:
    with pytest.raises(CannotResolveHostError):
        await _download(Request("ftp://nonexistent.invalid/file.txt"))


@coroutine_test
async def test_timeout() -> None:
    async def never_greet(
        reader: asyncio.StreamReader, writer: asyncio.StreamWriter
    ) -> None:
        await reader.read()
        writer.close()

    async with _serve(never_greet) as url:
        request = Request(url, meta={"download_timeout": 0.1})
        with pytest.raises(DownloadTimeoutError):
            await _download(request)


@coroutine_test
async def test_connection_lost() -> None:
    async with _serve(lambda r, w: w.close()) as url:
        with pytest.raises(DownloadFailedError):
            await _download(Request(url))


async def _write_slowly(
    writer: asyncio.StreamWriter, data: bytes, interval: float
) -> None:
    """Write *data* one byte every *interval* seconds, until *writer* is
    closed."""
    for i in range(len(data)):
        if writer.is_closing():
            return
        writer.write(data[i : i + 1])
        await asyncio.sleep(interval)


def _fake_ftp(
    retr_reply: bytes | None, data: bytes = b"", interval: float = 0
) -> Callable[[asyncio.StreamReader, asyncio.StreamWriter], Any]:
    """Return a minimal FTP server connection handler that sends *data*, if
    any, and then answers RETR with *retr_reply*, or hangs if *retr_reply* is
    ``None``.

    If *interval* is set, *data* is sent one byte every *interval* seconds,
    until the client closes the data connection.
    """

    async def handle(
        reader: asyncio.StreamReader, writer: asyncio.StreamWriter
    ) -> None:
        data_writer: asyncio.Future[asyncio.StreamWriter] = (
            asyncio.get_running_loop().create_future()
        )

        def on_data_connection(
            _: asyncio.StreamReader, w: asyncio.StreamWriter
        ) -> None:
            data_writer.set_result(w)

        writer.write(b"220 Ready\r\n")
        while line := await reader.readline():
            command = line.split()[0].upper()
            if command == b"USER":
                writer.write(b"230 Logged in\r\n")
            elif command == b"TYPE":
                writer.write(b"200 OK\r\n")
            elif command == b"EPSV":
                data_server = await asyncio.start_server(
                    on_data_connection, "127.0.0.1", 0
                )
                port = data_server.sockets[0].getsockname()[1]
                writer.write(f"229 Entering passive mode (|||{port}|)\r\n".encode())
            elif command == b"RETR":
                w = await data_writer
                if data:
                    writer.write(b"150 Sending\r\n")
                    if interval:
                        await _write_slowly(w, data, interval)
                    else:
                        w.write(data)
                if retr_reply is None:
                    await reader.read()
                w.close()
                data_server.close()
                if retr_reply is None:
                    break
                writer.write(retr_reply)
            await writer.drain()
        writer.close()

    return handle


@asynccontextmanager
async def _serve(
    handler: Callable[[asyncio.StreamReader, asyncio.StreamWriter], Any],
) -> AsyncGenerator[str]:
    server = await asyncio.start_server(handler, "127.0.0.1", 0)
    port = server.sockets[0].getsockname()[1]
    try:
        yield f"ftp://127.0.0.1:{port}/file.txt"
    finally:
        server.close()
        await server.wait_closed()


@coroutine_test
async def test_retr_failure() -> None:
    async with _serve(_fake_ftp(b"450 Busy\r\n")) as url:
        response = await _download(Request(url))
    assert response.status == 503


_TRUNCATED = _fake_ftp(b"426 Transfer aborted\r\n", data=b"partial")


@coroutine_test
async def test_dataloss() -> None:
    async with _serve(_TRUNCATED) as url:
        with pytest.raises(ResponseDataLossError):
            await _download(Request(url))


@coroutine_test
async def test_dataloss_allowed() -> None:
    async with _serve(_TRUNCATED) as url:
        response = await _download(
            Request(url, meta={"download_fail_on_dataloss": False})
        )
    assert response.body == b"partial"
    assert response.flags == ["dataloss"]


@coroutine_test
async def test_timeout_partial_response() -> None:
    async with _serve(_fake_ftp(None, data=b"partial")) as url:
        response = await _download(
            Request(
                url,
                meta={"download_timeout": 0.5, "download_fail_on_dataloss": False},
            )
        )
    assert response.body == b"partial"
    assert response.flags == ["dataloss"]


@coroutine_test
async def test_timeout_slow_transfer() -> None:
    # the server sends data bytes well within the timeout of each other, but
    # the whole transfer takes longer than the timeout
    async with _serve(_fake_ftp(b"226 Done\r\n", b"x" * 20, interval=0.1)) as url:
        with pytest.raises(DownloadTimeoutError):
            await _download(Request(url, meta={"download_timeout": 0.5}))


@coroutine_test
async def test_local_filename(ftp_server: MockFTPServer, tmp_path: Path) -> None:
    local_filename = str(tmp_path / "file.txt").encode()
    request = Request(
        ftp_server.url("file.txt"), meta={"ftp_local_filename": local_filename}
    )
    response = await _download(request)
    assert response.body == local_filename
    assert response.headers == {
        b"Local Filename": [local_filename],
        b"Size": [b"17"],
    }
    assert (tmp_path / "file.txt").read_bytes() == FILES["file.txt"]


@coroutine_test
async def test_body_file(ftp_server: MockFTPServer, tmp_path: Path) -> None:
    body_file = tmp_path / "file.txt"
    request = Request(
        ftp_server.url("file.txt"),
        meta={"body_file": body_file, "ftp_local_filename": b"ignored"},
    )
    response = await _download(request)
    assert isinstance(response, BodyFileResponse)
    assert response.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
    assert body_file.read_bytes() == FILES["file.txt"]


@asynccontextmanager
async def _socks_proxy(**kwargs: Any) -> AsyncGenerator[str]:
    server = await asyncio.start_server(
        lambda r, w: socks_server_handler(r, w, **kwargs), "127.0.0.1", 0
    )
    port = server.sockets[0].getsockname()[1]
    try:
        yield f"127.0.0.1:{port}"
    finally:
        server.close()


requires_socks = pytest.mark.skipif(
    not _HAS_SOCKS or socks_server_handler is None,
    reason="Requires python-socks and siosocks",
)


@requires_socks
@pytest.mark.parametrize("scheme", ["socks4", "socks5"])
@coroutine_test
async def test_socks_proxy(ftp_server: MockFTPServer, scheme: str) -> None:
    async with _socks_proxy() as proxy:
        request = Request(
            ftp_server.url("file.txt"), meta={"proxy": f"{scheme}://{proxy}"}
        )
        response = await _download(request)
    assert response.body == FILES["file.txt"]


@requires_socks
@coroutine_test
async def test_socks_proxy_credentials(ftp_server: MockFTPServer) -> None:
    async with _socks_proxy(
        allowed_versions={5}, username="user", password="pass"
    ) as proxy:
        request = Request(
            ftp_server.url("file.txt"),
            meta={"proxy": f"socks5://{proxy}"},
            headers={"Proxy-Authorization": basic_auth_header("user", "pass")},
        )
        response = await _download(request)
    assert response.body == FILES["file.txt"]


@requires_socks
@coroutine_test
async def test_socks_proxy_wrong_credentials(ftp_server: MockFTPServer) -> None:
    async with _socks_proxy(
        allowed_versions={5}, username="user", password="pass"
    ) as proxy:
        request = Request(
            ftp_server.url("file.txt"),
            meta={"proxy": f"socks5://{proxy}"},
            headers={"Proxy-Authorization": basic_auth_header("user", "wrong")},
        )
        with pytest.raises(DownloadFailedError):
            await _download(request)


@requires_socks
@coroutine_test
async def test_socks_proxy_ftps(ftps_server: MockFTPServer) -> None:
    async with _socks_proxy() as proxy:
        request = Request(
            ftps_server.url("file.txt"), meta={"proxy": f"socks5://{proxy}"}
        )
        response = await _download(request)
    assert response.body == FILES["file.txt"]
    assert isinstance(response.certificate, bytes)


@coroutine_test
async def test_http_proxy(ftp_server: MockFTPServer) -> None:
    request = Request(ftp_server.url("file.txt"), meta={"proxy": "http://127.0.0.1:1"})
    with pytest.raises(NotImplementedError):
        await _download(request)