Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import os
from abc import ABC, abstractmethod
from pathlib import Path
from tempfile import mkstemp
from typing import TYPE_CHECKING, Any
import pytest
from pytest_twisted import async_yield_fixture
from twisted.cred import checkers, credentials, portal
from twisted.internet.protocol import Factory, Protocol
from scrapy import Spider
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import DownloadCancelledError, DownloadFailedError, NotConfigured
from scrapy.http import BodyFileResponse, HtmlResponse, Request, Response
from scrapy.http.response.text import TextResponse
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator
from twisted.protocols.ftp import FTPFactory
pytestmark = pytest.mark.requires_reactor # FTPDownloadHandler requires a reactor
class TestFTPBase(ABC):
username = "scrapy"
password = "passwd"
req_meta: dict[str, Any] = {"ftp_user": username, "ftp_password": password}
test_files = (
("file.txt", b"I have the power!"),
("file with spaces.txt", b"Moooooooooo power!"),
("html-file-without-extension", b"<!DOCTYPE html>\n<title>.</title>"),
("big.bin", b"x" * 2**20),
)
@abstractmethod
def _create_files(self, root: Path) -> None:
raise NotImplementedError
@abstractmethod
def _get_factory(self, tmp_path: Path) -> FTPFactory:
raise NotImplementedError
@async_yield_fixture # type: ignore[untyped-decorator]
async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:
from twisted.internet import reactor
self._create_files(tmp_path)
factory = self._get_factory(tmp_path)
port = reactor.listenTCP(0, factory, interface="127.0.0.1")
portno = port.getHost().port
yield f"ftp://127.0.0.1:{portno}/"
await port.stopListening()
@pytest.fixture
@staticmethod
def dh() -> Generator[FTPDownloadHandler]:
crawler = get_crawler()
dh = build_from_crawler(FTPDownloadHandler, crawler)
yield dh
# if the test was skipped, there will be no client attribute
if hasattr(dh, "client"):
assert dh.client.transport
dh.client.transport.loseConnection()
@deferred_f_from_coro_f
async def test_ftp_download_success(
self, server_url: str, dh: FTPDownloadHandler
) -> None:
request = Request(url=server_url + "file.txt", meta=self.req_meta)
r = await dh.download_request(request)
assert r.status == 200
assert r.body == b"I have the power!"
assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
assert r.protocol is None
@deferred_f_from_coro_f
async def test_ftp_download_path_with_spaces(
self, server_url: str, dh: FTPDownloadHandler
) -> None:
request = Request(
url=server_url + "file with spaces.txt",
meta=self.req_meta,
)
r = await dh.download_request(request)
assert r.status == 200
assert r.body == b"Moooooooooo power!"
assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]}
@deferred_f_from_coro_f
async def test_ftp_download_nonexistent(
self, server_url: str, dh: FTPDownloadHandler
) -> None:
request = Request(url=server_url + "nonexistent.txt", meta=self.req_meta)
r = await dh.download_request(request)
assert r.status == 404
assert r.body == b"['550 nonexistent.txt: No such file or directory.']"
@deferred_f_from_coro_f
async def test_ftp_list(self, server_url: str, dh: FTPDownloadHandler) -> None:
request = Request(url=server_url, meta=self.req_meta)
r = await dh.download_request(request)
assert isinstance(r, TextResponse)
assert r.status == 200
assert set(r.text.splitlines()) == {filename for filename, _ in self.test_files}
@deferred_f_from_coro_f
async def test_ftp_list_nonexistent(
self, server_url: str, dh: FTPDownloadHandler
) -> None:
request = Request(url=server_url + "nonexistent/", meta=self.req_meta)
r = await dh.download_request(request)
assert isinstance(r, TextResponse)
assert r.status == 200
assert r.text == ""
@deferred_f_from_coro_f
async def test_ftp_local_filename(
self, server_url: str, dh: FTPDownloadHandler
) -> None:
f, local_fname = mkstemp()
fname_bytes = local_fname.encode()
local_path = Path(local_fname)
os.close(f)
meta = {"ftp_local_filename": fname_bytes}
meta.update(self.req_meta)
request = Request(url=server_url + "file.txt", meta=meta)
r = await dh.download_request(request)
assert r.body == fname_bytes
assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]}
assert local_path.exists()
assert local_path.read_bytes() == b"I have the power!"
local_path.unlink()
@deferred_f_from_coro_f
async def test_body_file(
self, server_url: str, dh: FTPDownloadHandler, tmp_path: Path
) -> None:
body_file = tmp_path / "body"
meta = {**self.req_meta, "body_file": body_file}
request = Request(url=server_url + "file.txt", meta=meta)
r = await dh.download_request(request)
assert isinstance(r, BodyFileResponse)
assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]}
assert body_file.read_bytes() == b"I have the power!"
@deferred_f_from_coro_f
async def test_body_file_nonexistent(
self, server_url: str, dh: FTPDownloadHandler, tmp_path: Path
) -> None:
body_file = tmp_path / "body"
meta = {**self.req_meta, "body_file": body_file}
request = Request(url=server_url + "nonexistent.txt", meta=meta)
r = await dh.download_request(request)
assert isinstance(r, BodyFileResponse)
assert r.status == 404
assert (
body_file.read_bytes()
== b"['550 nonexistent.txt: No such file or directory.']"
)
@pytest.mark.parametrize("filename", ["file.txt", "big.bin"])
@pytest.mark.parametrize("local", [False, True])
@deferred_f_from_coro_f
async def test_maxsize(
self,
server_url: str,
dh: FTPDownloadHandler,
tmp_path: Path,
filename: str,
local: bool,
caplog: pytest.LogCaptureFixture,
) -> None:
meta = {**self.req_meta, "download_maxsize": 10}
if local:
meta["ftp_local_filename"] = str(tmp_path / "local.txt").encode()
request = Request(url=server_url + filename, meta=meta)
with pytest.raises(DownloadCancelledError):
await dh.download_request(request)
assert "which is larger than download max size (10)" in caplog.text
@deferred_f_from_coro_f
async def test_maxsize_disabled(self, server_url: str, tmp_path: Path) -> None:
crawler = get_crawler(settings_dict={"DOWNLOAD_MAXSIZE": 10})
dh = build_from_crawler(FTPDownloadHandler, crawler)
meta = {**self.req_meta, "download_maxsize": 0}
request = Request(url=server_url + "file.txt", meta=meta)
r = await dh.download_request(request)
assert r.body == b"I have the power!"
@pytest.mark.parametrize("filename", ["file.txt", "big.bin"])
@deferred_f_from_coro_f
async def test_warnsize(
self,
server_url: str,
dh: FTPDownloadHandler,
filename: str,
caplog: pytest.LogCaptureFixture,
) -> None:
meta = {**self.req_meta, "download_warnsize": 10}
request = Request(url=server_url + filename, meta=meta)
r = await dh.download_request(request)
assert r.status == 200
assert caplog.text.count("which is larger than download warn size (10)") == 1
@pytest.mark.parametrize(
("filename", "response_class"),
[
("file.txt", TextResponse),
("html-file-without-extension", HtmlResponse),
],
)
@deferred_f_from_coro_f
async def test_response_class(
self,
filename: str,
response_class: type[Response],
server_url: str,
dh: FTPDownloadHandler,
) -> None:
meta = {}
meta.update(self.req_meta)
request = Request(url=server_url + filename, meta=meta)
r = await dh.download_request(request)
assert type(r) is response_class
class TestFTP(TestFTPBase):
def _create_files(self, root: Path) -> None:
userdir = root / self.username
userdir.mkdir()
for filename, content in self.test_files:
(userdir / filename).write_bytes(content)
def _get_factory(self, root: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
# the FTP protocol authenticates with str credentials
users_checker.addUser(self.username, self.password) # type: ignore[arg-type]
p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type]
return FTPFactory(portal=p)
@deferred_f_from_coro_f
async def test_invalid_credentials(
self, server_url: str, dh: FTPDownloadHandler
) -> None:
meta = dict(self.req_meta)
meta.update({"ftp_password": "invalid"})
request = Request(url=server_url + "file.txt", meta=meta)
with pytest.raises(DownloadFailedError):
await dh.download_request(request)
class TestAnonymousFTP(TestFTPBase):
username = "anonymous"
req_meta = {}
def _create_files(self, root: Path) -> None:
for filename, content in self.test_files:
(root / filename).write_bytes(content)
def _get_factory(self, tmp_path: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(tmp_path))
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
p.registerChecker(
checkers.AllowAnonymousAccess(), # type: ignore[arg-type]
credentials.IAnonymous,
)
return FTPFactory(portal=p, userAnonymous=self.username)
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
build_from_crawler(FTPDownloadHandler, crawler)
@deferred_f_from_coro_f
async def test_connection_lost() -> None:
from twisted.internet import reactor
class DropConnection(Protocol):
def connectionMade(self) -> None:
assert self.transport
self.transport.loseConnection()
port = reactor.listenTCP(
0, Factory.forProtocol(DropConnection), interface="127.0.0.1"
)
portno = port.getHost().port
crawler = get_crawler()
dh = build_from_crawler(FTPDownloadHandler, crawler)
try:
request = Request(url=f"ftp://127.0.0.1:{portno}/file.txt")
with pytest.raises(DownloadFailedError):
await dh.download_request(request)
finally:
await maybe_deferred_to_future(port.stopListening())