Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Type tests.mockserver and other resource-defining files (#7865)

Adriana7385d6e5103
diff --git a/pyproject.toml b/pyproject.toml
index 609c70708..13267e427 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -118,13 +118,10 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
- "tests.mockserver.*",
"tests.spiders",
"tests.test_closespider",
"tests.test_cmdline",
"tests.test_contracts",
- "tests.test_core_downloader",
- "tests.test_downloader_handler_twisted_ftp",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
@@ -135,7 +132,6 @@ module = [
"tests.test_feedexport_postprocess",
"tests.test_feedexport_storages",
"tests.test_feedexport_uri_params",
- "tests.test_http2_client_protocol",
"tests.test_item",
"tests.test_linkextractors",
"tests.test_loader",
diff --git a/tests/mockserver/dns.py b/tests/mockserver/dns.py
index e19a2e61a..2af018c66 100644
--- a/tests/mockserver/dns.py
+++ b/tests/mockserver/dns.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import sys
from subprocess import PIPE, Popen
+from typing import TYPE_CHECKING
from twisted.internet import defer
from twisted.names import dns, error
@@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory
from tests.utils import get_script_run_env
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+ from types import TracebackType
+
+ from twisted.internet.defer import Deferred
+
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+
+_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]]
+
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
- def _resolve(self, name):
+ def _resolve(self, name: bytes) -> _Answers:
record = dns.Record_A(address=b"127.0.0.1")
- answer = dns.RRHeader(name=name, payload=record)
+ # zope.interface has no type hints, so mypy cannot tell that Record_A
+ # provides the IEncodableRecord interface.
+ answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type]
return [answer], [], []
- def query(self, query, timeout=None):
+ def query(
+ self, query: dns.Query, timeout: Sequence[int] | None = None
+ ) -> Deferred[_Answers]:
if query.type == dns.A:
return defer.succeed(self._resolve(query.name.name))
return defer.fail(error.DomainError())
- def lookupAllRecords(self, name, timeout=None):
+ def lookupAllRecords(
+ self, name: bytes, timeout: Sequence[int] | None = None
+ ) -> Deferred[_Answers]:
return defer.succeed(self._resolve(name))
class MockDNSServer:
- def __enter__(self):
+ def __enter__(self) -> Self:
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.dns"],
stdout=PIPE,
env=get_script_run_env(),
text=True,
)
+ assert self.proc.stdout is not None
self.host = "127.0.0.1"
self.port = int(self.proc.stdout.readline().strip().split(":")[1])
return self
- def __exit__(self, exc_type, exc_value, traceback):
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
self.proc.kill()
self.proc.communicate()
@@ -54,7 +79,7 @@ def main() -> None:
protocol = dns.DNSDatagramProtocol(controller=factory)
listener = reactor.listenUDP(0, protocol)
- def print_listening():
+ def print_listening() -> None:
host = listener.getHost()
print(f"{host.host}:{host.port}")
diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py
index 22efc966b..1edd64dda 100644
--- a/tests/mockserver/ftp.py
+++ b/tests/mockserver/ftp.py
@@ -7,6 +7,7 @@ from pathlib import Path
from shutil import rmtree
from subprocess import PIPE, Popen
from tempfile import mkdtemp
+from typing import TYPE_CHECKING
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import FTPHandler
@@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer
from tests.utils import get_script_run_env
+if TYPE_CHECKING:
+ from types import TracebackType
+
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
class MockFTPServer:
"""Creates an FTP server on a random port with a default passwordless user
@@ -26,7 +33,7 @@ class MockFTPServer:
self.port: int | None = None
self.path: Path | None = None
- def __enter__(self):
+ def __enter__(self) -> Self:
self.path = Path(mkdtemp())
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],
@@ -34,6 +41,7 @@ class MockFTPServer:
env=get_script_run_env(),
text=True,
)
+ assert self.proc.stderr is not None
for line in self.proc.stderr:
if "starting FTP server" in line and (
m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)
@@ -48,12 +56,18 @@ class MockFTPServer:
)
return self
- def __exit__(self, exc_type, exc_value, traceback):
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
rmtree(str(self.path))
+ assert self.proc is not None
self.proc.kill()
self.proc.communicate()
- def url(self, path):
+ def url(self, path: str) -> str:
return f"ftp://{self.host}:{self.port}/{path}"
diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py
index 7ad873c02..c4fd4464e 100644
--- a/tests/mockserver/http.py
+++ b/tests/mockserver/http.py
@@ -1,8 +1,8 @@
from __future__ import annotations
from pathlib import Path
+from typing import TYPE_CHECKING
-from twisted.web import resource
from twisted.web.static import Data, File
from twisted.web.util import Redirect
@@ -11,6 +11,7 @@ from tests import tests_datadir
from .http_base import BaseMockServer, main_factory
from .http_resources import (
ArbitraryLengthPayloadResource,
+ BaseResource,
BrokenChunkedResource,
BrokenDownloadResource,
ChunkedResource,
@@ -35,62 +36,68 @@ from .http_resources import (
SetCookie,
Status,
UriResource,
+ put_child,
)
+if TYPE_CHECKING:
+ from twisted.web.server import Request
-class Root(resource.Resource):
- def __init__(self):
+
+class Root(BaseResource):
+ def __init__(self) -> None:
super().__init__()
- self.putChild(b"status", Status())
- self.putChild(b"follow", Follow())
- self.putChild(b"delay", Delay())
- self.putChild(b"partial", Partial())
- self.putChild(b"drop", Drop())
- self.putChild(b"raw", Raw())
- self.putChild(b"echo", Echo())
- self.putChild(b"payload", PayloadResource())
- self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
- self.putChild(b"static", File(str(Path(tests_datadir, "test_site/"))))
- self.putChild(b"redirect-to", RedirectTo())
- self.putChild(b"text", Data(b"Works", "text/plain"))
- self.putChild(
+ put_child(self, b"status", Status())
+ put_child(self, b"follow", Follow())
+ put_child(self, b"delay", Delay())
+ put_child(self, b"partial", Partial())
+ put_child(self, b"drop", Drop())
+ put_child(self, b"raw", Raw())
+ put_child(self, b"echo", Echo())
+ put_child(self, b"payload", PayloadResource())
+ put_child(self, b"alpayload", ArbitraryLengthPayloadResource())
+ put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))
+ put_child(self, b"redirect-to", RedirectTo())
+ put_child(self, b"text", Data(b"Works", "text/plain"))
+ put_child(
+ self,
b"html",
Data(
b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
"text/html",
),
)
- self.putChild(
+ put_child(
+ self,
b"enc-gb18030",
Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
)
- self.putChild(b"redirect", Redirect(b"/redirected"))
- self.putChild(
- b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
+ put_child(self, b"redirect", Redirect(b"/redirected"))
+ put_child(
+ self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
)
- self.putChild(b"redirected", Data(b"Redirected here", "text/plain"))
+ put_child(self, b"redirected", Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)]
- self.putChild(b"numbers", Data(b"".join(numbers), "text/plain"))
- self.putChild(b"wait", ForeverTakingResource())
- self.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
- self.putChild(b"host", HostHeaderResource())
- self.putChild(b"client-ip", ClientIPResource())
- self.putChild(b"broken", BrokenDownloadResource())
- self.putChild(b"chunked", ChunkedResource())
- self.putChild(b"broken-chunked", BrokenChunkedResource())
- self.putChild(b"contentlength", ContentLengthHeaderResource())
- self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
- self.putChild(b"largechunkedfile", LargeChunkedFileResource())
- self.putChild(b"compress", Compress())
- self.putChild(b"duplicate-header", DuplicateHeaderResource())
- self.putChild(b"response-headers", ResponseHeadersResource())
- self.putChild(b"set-cookie", SetCookie())
- self.putChild(b"uri", UriResource())
+ put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))
+ put_child(self, b"wait", ForeverTakingResource())
+ put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))
+ put_child(self, b"host", HostHeaderResource())
+ put_child(self, b"client-ip", ClientIPResource())
+ put_child(self, b"broken", BrokenDownloadResource())
+ put_child(self, b"chunked", ChunkedResource())
+ put_child(self, b"broken-chunked", BrokenChunkedResource())
+ put_child(self, b"contentlength", ContentLengthHeaderResource())
+ put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource())
+ put_child(self, b"largechunkedfile", LargeChunkedFileResource())
+ put_child(self, b"compress", Compress())
+ put_child(self, b"duplicate-header", DuplicateHeaderResource())
+ put_child(self, b"response-headers", ResponseHeadersResource())
+ put_child(self, b"set-cookie", SetCookie())
+ put_child(self, b"uri", UriResource())
- def getChild(self, path, request):
+ def getChild(self, path: bytes, request: Request) -> Root:
return self
- def render(self, request):
+ def render(self, request: Request) -> bytes:
return b"Scrapy mock HTTP server\n"
diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py
index 343c79781..5bc1252d6 100644
--- a/tests/mockserver/http_base.py
+++ b/tests/mockserver/http_base.py
@@ -17,6 +17,7 @@ from .utils import ssl_context_factory
if TYPE_CHECKING:
from collections.abc import Callable
+ from types import TracebackType
from twisted.web import resource
@@ -60,7 +61,12 @@ class BaseMockServer(ABC):
self.https_port = https_parsed.port
return self
- def __exit__(self, exc_type, exc_value, traceback) -> None:
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
if self.proc:
self.proc.kill()
self.proc.communicate()
@@ -135,7 +141,7 @@ def main_factory(
context_factory = ssl_context_factory(**context_factory_kw)
https_port = reactor.listenSSL(0, factory, context_factory)
- def print_listening():
+ def print_listening() -> None:
if listen_http:
http_host = http_port.getHost()
http_address = f"http://{http_host.host}:{http_host.port}"
diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py
index 98ac6cf6a..cb028bc10 100644
--- a/tests/mockserver/http_resources.py
+++ b/tests/mockserver/http_resources.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import gzip
import json
import random
-from typing import TYPE_CHECKING, ParamSpec, TypeVar
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from urllib.parse import urlencode
from twisted.internet.task import deferLater
@@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
from twisted.internet.defer import Deferred
- from twisted.web.http import Request
+ from twisted.python.failure import Failure
+ from twisted.web.http import Request as HTTPRequest
+ from twisted.web.server import Request
_T = TypeVar("_T")
_P = ParamSpec("_P")
-def getarg(request, name, default=None, type_=None):
+def getarg(
+ request: Request,
+ name: bytes,
+ default: Any = None,
+ type_: Callable[[bytes], Any] | None = None,
+) -> Any:
if name in request.args:
value = request.args[name][0]
if type_ is not None:
@@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None):
return default
-def close_connection(request):
+def close_connection(request: Request) -> None:
# We have to force a disconnection for HTTP/1.1 clients. Otherwise
# client keeps the connection open waiting for more data.
request.channel.loseConnection()
request.finish()
+def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None:
+ # zope.interface has no type hints, so mypy cannot tell that Resource
+ # instances provide the IResource interface that putChild() expects.
+ parent.putChild(path, child) # type: ignore[arg-type]
+
+
+class BaseResource(resource.Resource):
+ """Base class for mockserver resources, with type hints."""
+
+ # Only needed to give subclasses a typed __init__ to call.
+ def __init__(self) -> None: # pylint: disable=useless-parent-delegation
+ super().__init__() # type: ignore[no-untyped-call]
+
+
# most of the following resources are copied from twisted.web.test.test_webclient
-class ForeverTakingResource(resource.Resource):
+class ForeverTakingResource(BaseResource):
"""
L{ForeverTakingResource} is a resource which never finishes responding
to requests.
"""
- def __init__(self, write=False):
- resource.Resource.__init__(self)
+ def __init__(self, write: bool = False):
+ super().__init__()
self._write = write
- def render(self, request):
+ def render(self, request: Request) -> int:
if self._write:
request.write(b"some bytes")
return server.NOT_DONE_YET
-class HostHeaderResource(resource.Resource):
+class HostHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the host header
from the request.
"""
- def render(self, request):
- return request.requestHeaders.getRawHeaders(b"host")[0]
+ def render(self, request: Request) -> bytes:
+ headers = request.requestHeaders.getRawHeaders(b"host")
+ assert headers
+ return headers[0]
-class ClientIPResource(resource.Resource):
+class ClientIPResource(BaseResource):
"""
A testing resource which renders itself as the request client IP address.
"""
- def render(self, request):
+ def render(self, request: Request) -> bytes:
client_address = request.getClientAddress()
if client_address is None or client_address.host is None:
return b""
return to_bytes(client_address.host)
-class PayloadResource(resource.Resource):
+class PayloadResource(BaseResource):
"""
A testing resource which renders itself as the contents of the request body
as long as the request body is 100 bytes long, otherwise which renders
itself as C{"ERROR"}.
"""
- def render(self, request):
- data = request.content.read()
- contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
- if len(data) != 100 or int(contentLength) != 100:
+ def render(self, request: Request) -> bytes:
+ assert request.content
+ data: bytes = request.content.read()
+ content_length = request.requestHeaders.getRawHeaders(b"content-length")
+ assert content_length
+ if len(data) != 100 or int(content_length[0]) != 100:
return b"ERROR"
return data
-class LeafResource(resource.Resource):
+class LeafResource(BaseResource):
isLeaf = True
def deferRequest(
self,
- request: Request,
+ request: HTTPRequest,
delay: float,
f: Callable[_P, _T],
*a: _P.args,
@@ -107,7 +132,7 @@ class LeafResource(resource.Resource):
) -> Deferred[_T]:
from twisted.internet import reactor
- def _cancelrequest(_):
+ def _cancelrequest(_: Failure) -> None:
# silence CancelledError
d.addErrback(lambda _: None)
d.cancel()
@@ -118,12 +143,13 @@ class LeafResource(resource.Resource):
class Follow(LeafResource):
- def render(self, request):
+ def render(self, request: Request) -> int:
total = getarg(request, b"total", 100, type_=int)
show = getarg(request, b"show", 1, type_=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type_=float)
n = getarg(request, b"n", total, type_=int)
+ nlist: Sequence[int]
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
@@ -133,7 +159,7 @@ class Follow(LeafResource):
self.deferRequest(request, lag, self.renderRequest, request, nlist)
return NOT_DONE_YET
- def renderRequest(self, request, nlist):
+ def renderRequest(self, request: Request, nlist: Sequence[int]) -> None:
s = """<html> <head></head> <body>"""
args = request.args.copy()
for nl in nlist:
@@ -146,45 +172,47 @@ class Follow(LeafResource):
class Delay(LeafResource):
- def render_GET(self, request):
+ def render_GET(self, request: Request) -> int:
n = getarg(request, b"n", 1, type_=float)
b = getarg(request, b"b", 1, type_=int)
if b:
# send headers now and delay body
- request.write("")
+ request.write(b"")
self.deferRequest(request, n, self._delayedRender, request, n)
return NOT_DONE_YET
- def _delayedRender(self, request, n):
+ def _delayedRender(self, request: Request, n: float) -> None:
request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))
request.finish()
class Status(LeafResource):
- def render_GET(self, request):
+ def render_GET(self, request: Request) -> bytes:
n = getarg(request, b"n", 200, type_=int)
request.setResponseCode(n)
return b""
class Raw(LeafResource):
- def render_GET(self, request):
+ def render_GET(self, request: Request) -> int:
request.startedWriting = 1
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
render_POST = render_GET
- def _delayedRender(self, request):
+ def _delayedRender(self, request: Request) -> None:
raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")
request.startedWriting = 1
request.write(raw)
+ assert request.channel.transport is not None
request.channel.transport.loseConnection()
request.finish()
class Echo(LeafResource):
- def render_GET(self, request):
+ def render_GET(self, request: Request) -> bytes:
+ assert request.content
output = {
"headers": {
to_unicode(k): [to_unicode(v) for v in vs]
@@ -198,27 +226,29 @@ class Echo(LeafResource):
class RedirectTo(LeafResource):
- def render(self, request):
+ def render(self, request: Request) -> bytes:
goto = getarg(request, b"goto", b"/")
# we force the body content, otherwise Twisted redirectTo()
# returns HTML with <meta http-equiv="refresh"
- redirectTo(goto, request)
+ # zope.interface has no type hints, so mypy cannot tell that Request
+ # provides the IRequest interface.
+ redirectTo(goto, request) # type: ignore[arg-type]
return b"redirecting..."
class Partial(LeafResource):
- def render_GET(self, request):
+ def render_GET(self, request: Request) -> int:
request.setHeader(b"Content-Length", b"1024")
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
- def _delayedRender(self, request):
+ def _delayedRender(self, request: Request) -> None:
request.write(b"partial content\n")
request.finish()
class Drop(Partial):
- def _delayedRender(self, request):
+ def _delayedRender(self, request: Request) -> None:
abort = getarg(request, b"abort", 0, type_=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
@@ -233,8 +263,10 @@ class Drop(Partial):
class ArbitraryLengthPayloadResource(LeafResource):
- def render(self, request):
- return request.content.read()
+ def render(self, request: Request) -> bytes:
+ assert request.content
+ data: bytes = request.content.read()
+ return data
class NoMetaRefreshRedirect(Redirect):
@@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect):
)
-class ContentLengthHeaderResource(resource.Resource):
+class ContentLengthHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the Content-Length
header from the request.
"""
- def render(self, request):
- return request.requestHeaders.getRawHeaders(b"content-length")[0]
+ def render(self, request: Request) -> bytes:
+ headers = request.requestHeaders.getRawHeaders(b"content-length")
+ assert headers
+ return headers[0]
-class ChunkedResource(resource.Resource):
- def render(self, request):
+class ChunkedResource(BaseResource):
+ def render(self, request: Request) -> int:
from twisted.internet import reactor
- def response():
+ def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
request.finish()
@@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource):
return server.NOT_DONE_YET
-class BrokenChunkedResource(resource.Resource):
- def render(self, request):
+class BrokenChunkedResource(BaseResource):
+ def render(self, request: Request) -> int:
from twisted.internet import reactor
- def response():
+ def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
# Disable terminating chunk on finish.
@@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource):
return server.NOT_DONE_YET
-class BrokenDownloadResource(resource.Resource):
- def render(self, request):
+class BrokenDownloadResource(BaseResource):
+ def render(self, request: Request) -> int:
from twisted.internet import reactor
- def response():
+ def response() -> None:
request.setHeader(b"Content-Length", b"20")
request.write(b"partial")
close_connection(request)
@@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource):
return server.NOT_DONE_YET
-class EmptyContentTypeHeaderResource(resource.Resource):
+class EmptyContentTypeHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of request body
without content-type header in response.
"""
- def render(self, request):
+ def render(self, request: Request) -> bytes:
+ assert request.content
request.setHeader("content-type", "")
- return request.content.read()
+ data: bytes = request.content.read()
+ return data
-class LargeChunkedFileResource(resource.Resource):
- def render(self, request):
+class LargeChunkedFileResource(BaseResource):
+ def render(self, request: Request) -> int:
from twisted.internet import reactor
- def response():
+ def response() -> None:
for _ in range(1024):
request.write(b"x" * 1024)
request.finish()
@@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource):
return server.NOT_DONE_YET
-class DuplicateHeaderResource(resource.Resource):
- def render(self, request):
+class DuplicateHeaderResource(BaseResource):
+ def render(self, request: Request) -> bytes:
request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
return b""
-class UriResource(resource.Resource):
+class UriResource(BaseResource):
"""Return the full uri that was requested"""
- def getChild(self, path, request):
+ def getChild(self, path: bytes, request: Request) -> resource.Resource:
return self
- def render(self, request):
+ def render(self, request: Request) -> bytes | int:
# Note: this is an ugly hack for CONNECT request timeout test.
# Returning some data here fail SSL/TLS handshake
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b"CONNECT":
return request.uri
+ assert request.transport is not None
request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
return NOT_DONE_YET
-class ResponseHeadersResource(resource.Resource):
+class ResponseHeadersResource(BaseResource):
"""Return a response with headers set from the JSON request body"""
- def render(self, request):
+ def render(self, request: Request) -> bytes:
+ assert request.content
body = json.loads(request.content.read().decode())
for header_name, header_value in body.items():
request.responseHeaders.setRawHeaders(header_name, [header_value])
return json.dumps(body).encode("utf-8")
-class Compress(resource.Resource):
+class Compress(BaseResource):
"""Compress the data sent in the request url params and set Content-Encoding header"""
- def render(self, request):
- data = request.args.get(b"data")[0]
+ def render(self, request: Request) -> bytes:
+ data = request.args[b"data"][0]
accept_encoding_header = request.getHeader(b"accept-encoding")
@@ -370,10 +408,10 @@ class Compress(resource.Resource):
return b"Did not receive a valid accept-encoding header"
-class SetCookie(resource.Resource):
+class SetCookie(BaseResource):
"""Return a response with a Set-Cookie header for each request url parameter"""
- def render(self, request):
+ def render(self, request: Request) -> bytes:
for cookie_name, cookie_values in request.args.items():
for cookie_value in cookie_values:
cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode()
diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py
index fdea666e1..2a6cb6dd8 100644
--- a/tests/mockserver/simple_https.py
+++ b/tests/mockserver/simple_https.py
@@ -2,18 +2,23 @@
from __future__ import annotations
-from twisted.web import resource
+from typing import TYPE_CHECKING
+
from twisted.web.static import Data
from .http_base import BaseMockServer, main_factory
+from .http_resources import BaseResource, put_child
+
+if TYPE_CHECKING:
+ from twisted.web.server import Request
-class Root(resource.Resource):
- def __init__(self):
- resource.Resource.__init__(self)
- self.putChild(b"file", Data(b"0123456789", "text/plain"))
+class Root(BaseResource):
+ def __init__(self) -> None:
+ super().__init__()
+ put_child(self, b"file", Data(b"0123456789", "text/plain"))
- def getChild(self, path, request):
+ def getChild(self, path: bytes, request: Request) -> Root:
return self
@@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer):
cipher_string: str | None = None,
tls_min_version: str | None = None,
tls_max_version: str | None = None,
- ):
+ ) -> None:
super().__init__()
self.keyfile = keyfile
self.certfile = certfile
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 3e4139b3e..912c0450b 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -14,7 +14,7 @@ from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
-from scrapy import Request
+from scrapy import Request, Spider
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
@@ -31,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
-from tests.mockserver.http_resources import PayloadResource
+from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
+ from twisted.internet.interfaces import IListeningPort
from twisted.web.iweb import IBodyProducer
+ from scrapy.http import Response
+
class TestSlot:
def test_repr(self):
@@ -52,7 +55,7 @@ class TestContextFactoryBase:
async def server_url(self, tmp_path):
(tmp_path / "file").write_bytes(b"0123456789")
r = static.File(str(tmp_path))
- r.putChild(b"payload", PayloadResource())
+ put_child(r, b"payload", PayloadResource())
site = server.Site(r, timeout=None)
port = self._listen(site)
portno = port.getHost().port
@@ -61,7 +64,7 @@ class TestContextFactoryBase:
await port.stopListening()
- def _listen(self, site):
+ def _listen(self, site: server.Site) -> IListeningPort:
from twisted.internet import reactor
return reactor.listenSSL(
@@ -317,7 +320,10 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
- def fetch(self, request, spider): # pylint: disable=signature-differs
+ # requiring the spider argument is what triggers the deprecation
+ def fetch( # type: ignore[override] # pylint: disable=signature-differs
+ self, request: Request, spider: Spider
+ ) -> Deferred[Response | Request]:
return super().fetch(request, spider)
crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})
diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py
index 489b70e74..14de97b21 100644
--- a/tests/test_downloader_handler_twisted_ftp.py
+++ b/tests/test_downloader_handler_twisted_ftp.py
@@ -156,14 +156,17 @@ class TestFTP(TestFTPBase):
for filename, content in self.test_files:
(userdir / filename).write_bytes(content)
- def _get_factory(self, root):
+ def _get_factory(self, root: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
- p = portal.Portal(realm)
+ # zope.interface has no type hints, so mypy cannot tell that these
+ # objects provide the interfaces that Portal expects.
+ p = portal.Portal(realm) # type: ignore[arg-type]
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
- users_checker.addUser(self.username, self.password)
- p.registerChecker(users_checker, credentials.IUsernamePassword)
+ # the FTP protocol authenticates with str credentials
+ users_checker.addUser(self.username, self.password) # type: ignore[arg-type]
+ p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type]
return FTPFactory(portal=p)
@deferred_f_from_coro_f
@@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase):
for filename, content in self.test_files:
(root / filename).write_bytes(content)
- def _get_factory(self, tmp_path):
+ def _get_factory(self, tmp_path: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(tmp_path))
- p = portal.Portal(realm)
- p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
+ # zope.interface has no type hints, so mypy cannot tell that these
+ # objects provide the interfaces that Portal expects.
+ p = portal.Portal(realm) # type: ignore[arg-type]
+ p.registerChecker(
+ checkers.AllowAnonymousAccess(), # type: ignore[arg-type]
+ credentials.IAnonymous,
+ )
return FTPFactory(portal=p, userAnonymous=self.username)
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 3c1347fd3..b8586d1ca 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -30,7 +30,7 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
-from tests.mockserver.http_resources import LeafResource, Status
+from tests.mockserver.http_resources import LeafResource, Status, put_child
from tests.mockserver.utils import ssl_context_factory
if TYPE_CHECKING:
@@ -199,18 +199,18 @@ class TestHttps2ClientProtocol:
@pytest.fixture
def site(self, tmp_path):
r = File(str(tmp_path))
- r.putChild(b"get-data-html-small", GetDataHtmlSmall())
- r.putChild(b"get-data-html-large", GetDataHtmlLarge())
-
- r.putChild(b"post-data-json-small", PostDataJsonSmall())
- r.putChild(b"post-data-json-large", PostDataJsonLarge())
-
- r.putChild(b"dataloss", Dataloss())
- r.putChild(b"no-content-length-header", NoContentLengthHeader())
- r.putChild(b"status", Status())
- r.putChild(b"query-params", QueryParams())
- r.putChild(b"timeout", TimeoutResponse())
- r.putChild(b"request-headers", RequestHeaders())
+ put_child(r, b"get-data-html-small", GetDataHtmlSmall())
+ put_child(r, b"get-data-html-large", GetDataHtmlLarge())
+
+ put_child(r, b"post-data-json-small", PostDataJsonSmall())
+ put_child(r, b"post-data-json-large", PostDataJsonLarge())
+
+ put_child(r, b"dataloss", Dataloss())
+ put_child(r, b"no-content-length-header", NoContentLengthHeader())
+ put_child(r, b"status", Status())
+ put_child(r, b"query-params", QueryParams())
+ put_child(r, b"timeout", TimeoutResponse())
+ put_child(r, b"request-headers", RequestHeaders())
return Site(r, timeout=None)
@async_yield_fixture # type: ignore[untyped-decorator]
diff --git a/tox.ini b/tox.ini
index edde83356..ac32064a6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -111,7 +111,8 @@ commands =
pre-commit run {posargs:--all-files}
[testenv:pylint]
-basepython = python3
+# Some checks are Python-version-dependent, so pin the version used in CI.
+basepython = python3.14
deps =
{[testenv:extra-deps]deps}
pylint==4.0.6