Scrapy, a fast high-level web crawling & scraping framework for Python.
Type tests.mockserver and other resource-defining files (#7865)
Adriana7385d6e5103
diff --git a/pyproject.toml b/pyproject.tomlindex 609c70708..13267e427 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -118,13 +118,10 @@ allow_incomplete_defs = true # 59 errors# TODO[[tool.mypy.overrides]]module = [- "tests.mockserver.*","tests.spiders","tests.test_closespider","tests.test_cmdline","tests.test_contracts",- "tests.test_core_downloader",- "tests.test_downloader_handler_twisted_ftp","tests.test_downloaderslotssettings","tests.test_dupefilters","tests.test_engine_loop",@@ -135,7 +132,6 @@ module = ["tests.test_feedexport_postprocess","tests.test_feedexport_storages","tests.test_feedexport_uri_params",- "tests.test_http2_client_protocol","tests.test_item","tests.test_linkextractors","tests.test_loader",diff --git a/tests/mockserver/dns.py b/tests/mockserver/dns.pyindex e19a2e61a..2af018c66 100644--- a/tests/mockserver/dns.py+++ b/tests/mockserver/dns.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport sysfrom subprocess import PIPE, Popen+from typing import TYPE_CHECKINGfrom twisted.internet import deferfrom twisted.names import dns, error@@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactoryfrom tests.utils import get_script_run_env+if TYPE_CHECKING:+ from collections.abc import Sequence+ from types import TracebackType++ from twisted.internet.defer import Deferred++ # typing.Self requires Python 3.11+ from typing_extensions import Self+++_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]]+class MockDNSResolver:"""Implements twisted.internet.interfaces.IResolver partially"""- def _resolve(self, name):+ def _resolve(self, name: bytes) -> _Answers:record = dns.Record_A(address=b"127.0.0.1")- answer = dns.RRHeader(name=name, payload=record)+ # zope.interface has no type hints, so mypy cannot tell that Record_A+ # provides the IEncodableRecord interface.+ answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type]return [answer], [], []- def query(self, query, timeout=None):+ def query(+ self, query: dns.Query, timeout: Sequence[int] | None = None+ ) -> Deferred[_Answers]:if query.type == dns.A:return defer.succeed(self._resolve(query.name.name))return defer.fail(error.DomainError())- def lookupAllRecords(self, name, timeout=None):+ def lookupAllRecords(+ self, name: bytes, timeout: Sequence[int] | None = None+ ) -> Deferred[_Answers]:return defer.succeed(self._resolve(name))class MockDNSServer:- def __enter__(self):+ def __enter__(self) -> Self:self.proc = Popen([sys.executable, "-u", "-m", "tests.mockserver.dns"],stdout=PIPE,env=get_script_run_env(),text=True,)+ assert self.proc.stdout is not Noneself.host = "127.0.0.1"self.port = int(self.proc.stdout.readline().strip().split(":")[1])return self- def __exit__(self, exc_type, exc_value, traceback):+ def __exit__(+ self,+ exc_type: type[BaseException] | None,+ exc_value: BaseException | None,+ traceback: TracebackType | None,+ ) -> None:self.proc.kill()self.proc.communicate()@@ -54,7 +79,7 @@ def main() -> None:protocol = dns.DNSDatagramProtocol(controller=factory)listener = reactor.listenUDP(0, protocol)- def print_listening():+ def print_listening() -> None:host = listener.getHost()print(f"{host.host}:{host.port}")diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.pyindex 22efc966b..1edd64dda 100644--- a/tests/mockserver/ftp.py+++ b/tests/mockserver/ftp.py@@ -7,6 +7,7 @@ from pathlib import Pathfrom shutil import rmtreefrom subprocess import PIPE, Popenfrom tempfile import mkdtemp+from typing import TYPE_CHECKINGfrom pyftpdlib.authorizers import DummyAuthorizerfrom pyftpdlib.handlers import FTPHandler@@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServerfrom tests.utils import get_script_run_env+if TYPE_CHECKING:+ from types import TracebackType++ # typing.Self requires Python 3.11+ from typing_extensions import Self+class MockFTPServer:"""Creates an FTP server on a random port with a default passwordless user@@ -26,7 +33,7 @@ class MockFTPServer:self.port: int | None = Noneself.path: Path | None = None- def __enter__(self):+ def __enter__(self) -> Self:self.path = Path(mkdtemp())self.proc = Popen([sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],@@ -34,6 +41,7 @@ class MockFTPServer:env=get_script_run_env(),text=True,)+ assert self.proc.stderr is not Nonefor line in self.proc.stderr:if "starting FTP server" in line and (m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)@@ -48,12 +56,18 @@ class MockFTPServer:)return self- def __exit__(self, exc_type, exc_value, traceback):+ def __exit__(+ self,+ exc_type: type[BaseException] | None,+ exc_value: BaseException | None,+ traceback: TracebackType | None,+ ) -> None:rmtree(str(self.path))+ assert self.proc is not Noneself.proc.kill()self.proc.communicate()- def url(self, path):+ def url(self, path: str) -> str:return f"ftp://{self.host}:{self.port}/{path}"diff --git a/tests/mockserver/http.py b/tests/mockserver/http.pyindex 7ad873c02..c4fd4464e 100644--- a/tests/mockserver/http.py+++ b/tests/mockserver/http.py@@ -1,8 +1,8 @@from __future__ import annotationsfrom pathlib import Path+from typing import TYPE_CHECKING-from twisted.web import resourcefrom twisted.web.static import Data, Filefrom twisted.web.util import Redirect@@ -11,6 +11,7 @@ from tests import tests_datadirfrom .http_base import BaseMockServer, main_factoryfrom .http_resources import (ArbitraryLengthPayloadResource,+ BaseResource,BrokenChunkedResource,BrokenDownloadResource,ChunkedResource,@@ -35,62 +36,68 @@ from .http_resources import (SetCookie,Status,UriResource,+ put_child,)+if TYPE_CHECKING:+ from twisted.web.server import Request-class Root(resource.Resource):- def __init__(self):++class Root(BaseResource):+ def __init__(self) -> None:super().__init__()- self.putChild(b"status", Status())- self.putChild(b"follow", Follow())- self.putChild(b"delay", Delay())- self.putChild(b"partial", Partial())- self.putChild(b"drop", Drop())- self.putChild(b"raw", Raw())- self.putChild(b"echo", Echo())- self.putChild(b"payload", PayloadResource())- self.putChild(b"alpayload", ArbitraryLengthPayloadResource())- self.putChild(b"static", File(str(Path(tests_datadir, "test_site/"))))- self.putChild(b"redirect-to", RedirectTo())- self.putChild(b"text", Data(b"Works", "text/plain"))- self.putChild(+ put_child(self, b"status", Status())+ put_child(self, b"follow", Follow())+ put_child(self, b"delay", Delay())+ put_child(self, b"partial", Partial())+ put_child(self, b"drop", Drop())+ put_child(self, b"raw", Raw())+ put_child(self, b"echo", Echo())+ put_child(self, b"payload", PayloadResource())+ put_child(self, b"alpayload", ArbitraryLengthPayloadResource())+ put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))+ put_child(self, b"redirect-to", RedirectTo())+ put_child(self, b"text", Data(b"Works", "text/plain"))+ put_child(+ self,b"html",Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>","text/html",),)- self.putChild(+ put_child(+ self,b"enc-gb18030",Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),)- self.putChild(b"redirect", Redirect(b"/redirected"))- self.putChild(- b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")+ put_child(self, b"redirect", Redirect(b"/redirected"))+ put_child(+ self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))- self.putChild(b"redirected", Data(b"Redirected here", "text/plain"))+ put_child(self, b"redirected", Data(b"Redirected here", "text/plain"))numbers = [str(x).encode("utf8") for x in range(2**18)]- self.putChild(b"numbers", Data(b"".join(numbers), "text/plain"))- self.putChild(b"wait", ForeverTakingResource())- self.putChild(b"hang-after-headers", ForeverTakingResource(write=True))- self.putChild(b"host", HostHeaderResource())- self.putChild(b"client-ip", ClientIPResource())- self.putChild(b"broken", BrokenDownloadResource())- self.putChild(b"chunked", ChunkedResource())- self.putChild(b"broken-chunked", BrokenChunkedResource())- self.putChild(b"contentlength", ContentLengthHeaderResource())- self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())- self.putChild(b"largechunkedfile", LargeChunkedFileResource())- self.putChild(b"compress", Compress())- self.putChild(b"duplicate-header", DuplicateHeaderResource())- self.putChild(b"response-headers", ResponseHeadersResource())- self.putChild(b"set-cookie", SetCookie())- self.putChild(b"uri", UriResource())+ put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))+ put_child(self, b"wait", ForeverTakingResource())+ put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))+ put_child(self, b"host", HostHeaderResource())+ put_child(self, b"client-ip", ClientIPResource())+ put_child(self, b"broken", BrokenDownloadResource())+ put_child(self, b"chunked", ChunkedResource())+ put_child(self, b"broken-chunked", BrokenChunkedResource())+ put_child(self, b"contentlength", ContentLengthHeaderResource())+ put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource())+ put_child(self, b"largechunkedfile", LargeChunkedFileResource())+ put_child(self, b"compress", Compress())+ put_child(self, b"duplicate-header", DuplicateHeaderResource())+ put_child(self, b"response-headers", ResponseHeadersResource())+ put_child(self, b"set-cookie", SetCookie())+ put_child(self, b"uri", UriResource())- def getChild(self, path, request):+ def getChild(self, path: bytes, request: Request) -> Root:return self- def render(self, request):+ def render(self, request: Request) -> bytes:return b"Scrapy mock HTTP server\n"diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.pyindex 343c79781..5bc1252d6 100644--- a/tests/mockserver/http_base.py+++ b/tests/mockserver/http_base.py@@ -17,6 +17,7 @@ from .utils import ssl_context_factoryif TYPE_CHECKING:from collections.abc import Callable+ from types import TracebackTypefrom twisted.web import resource@@ -60,7 +61,12 @@ class BaseMockServer(ABC):self.https_port = https_parsed.portreturn self- def __exit__(self, exc_type, exc_value, traceback) -> None:+ def __exit__(+ self,+ exc_type: type[BaseException] | None,+ exc_value: BaseException | None,+ traceback: TracebackType | None,+ ) -> None:if self.proc:self.proc.kill()self.proc.communicate()@@ -135,7 +141,7 @@ def main_factory(context_factory = ssl_context_factory(**context_factory_kw)https_port = reactor.listenSSL(0, factory, context_factory)- def print_listening():+ def print_listening() -> None:if listen_http:http_host = http_port.getHost()http_address = f"http://{http_host.host}:{http_host.port}"diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.pyindex 98ac6cf6a..cb028bc10 100644--- a/tests/mockserver/http_resources.py+++ b/tests/mockserver/http_resources.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport gzipimport jsonimport random-from typing import TYPE_CHECKING, ParamSpec, TypeVar+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVarfrom urllib.parse import urlencodefrom twisted.internet.task import deferLater@@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTofrom scrapy.utils.python import to_bytes, to_unicodeif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Sequencefrom twisted.internet.defer import Deferred- from twisted.web.http import Request+ from twisted.python.failure import Failure+ from twisted.web.http import Request as HTTPRequest+ from twisted.web.server import Request_T = TypeVar("_T")_P = ParamSpec("_P")-def getarg(request, name, default=None, type_=None):+def getarg(+ request: Request,+ name: bytes,+ default: Any = None,+ type_: Callable[[bytes], Any] | None = None,+) -> Any:if name in request.args:value = request.args[name][0]if type_ is not None:@@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None):return default-def close_connection(request):+def close_connection(request: Request) -> None:# We have to force a disconnection for HTTP/1.1 clients. Otherwise# client keeps the connection open waiting for more data.request.channel.loseConnection()request.finish()+def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None:+ # zope.interface has no type hints, so mypy cannot tell that Resource+ # instances provide the IResource interface that putChild() expects.+ parent.putChild(path, child) # type: ignore[arg-type]+++class BaseResource(resource.Resource):+ """Base class for mockserver resources, with type hints."""++ # Only needed to give subclasses a typed __init__ to call.+ def __init__(self) -> None: # pylint: disable=useless-parent-delegation+ super().__init__() # type: ignore[no-untyped-call]++# most of the following resources are copied from twisted.web.test.test_webclient-class ForeverTakingResource(resource.Resource):+class ForeverTakingResource(BaseResource):"""L{ForeverTakingResource} is a resource which never finishes respondingto requests."""- def __init__(self, write=False):- resource.Resource.__init__(self)+ def __init__(self, write: bool = False):+ super().__init__()self._write = write- def render(self, request):+ def render(self, request: Request) -> int:if self._write:request.write(b"some bytes")return server.NOT_DONE_YET-class HostHeaderResource(resource.Resource):+class HostHeaderResource(BaseResource):"""A testing resource which renders itself as the value of the host headerfrom the request."""- def render(self, request):- return request.requestHeaders.getRawHeaders(b"host")[0]+ def render(self, request: Request) -> bytes:+ headers = request.requestHeaders.getRawHeaders(b"host")+ assert headers+ return headers[0]-class ClientIPResource(resource.Resource):+class ClientIPResource(BaseResource):"""A testing resource which renders itself as the request client IP address."""- def render(self, request):+ def render(self, request: Request) -> bytes:client_address = request.getClientAddress()if client_address is None or client_address.host is None:return b""return to_bytes(client_address.host)-class PayloadResource(resource.Resource):+class PayloadResource(BaseResource):"""A testing resource which renders itself as the contents of the request bodyas long as the request body is 100 bytes long, otherwise which rendersitself as C{"ERROR"}."""- def render(self, request):- data = request.content.read()- contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]- if len(data) != 100 or int(contentLength) != 100:+ def render(self, request: Request) -> bytes:+ assert request.content+ data: bytes = request.content.read()+ content_length = request.requestHeaders.getRawHeaders(b"content-length")+ assert content_length+ if len(data) != 100 or int(content_length[0]) != 100:return b"ERROR"return data-class LeafResource(resource.Resource):+class LeafResource(BaseResource):isLeaf = Truedef deferRequest(self,- request: Request,+ request: HTTPRequest,delay: float,f: Callable[_P, _T],*a: _P.args,@@ -107,7 +132,7 @@ class LeafResource(resource.Resource):) -> Deferred[_T]:from twisted.internet import reactor- def _cancelrequest(_):+ def _cancelrequest(_: Failure) -> None:# silence CancelledErrord.addErrback(lambda _: None)d.cancel()@@ -118,12 +143,13 @@ class LeafResource(resource.Resource):class Follow(LeafResource):- def render(self, request):+ def render(self, request: Request) -> int:total = getarg(request, b"total", 100, type_=int)show = getarg(request, b"show", 1, type_=int)order = getarg(request, b"order", b"desc")maxlatency = getarg(request, b"maxlatency", 0, type_=float)n = getarg(request, b"n", total, type_=int)+ nlist: Sequence[int]if order == b"rand":nlist = [random.randint(1, total) for _ in range(show)]else: # order == "desc"@@ -133,7 +159,7 @@ class Follow(LeafResource):self.deferRequest(request, lag, self.renderRequest, request, nlist)return NOT_DONE_YET- def renderRequest(self, request, nlist):+ def renderRequest(self, request: Request, nlist: Sequence[int]) -> None:s = """<html> <head></head> <body>"""args = request.args.copy()for nl in nlist:@@ -146,45 +172,47 @@ class Follow(LeafResource):class Delay(LeafResource):- def render_GET(self, request):+ def render_GET(self, request: Request) -> int:n = getarg(request, b"n", 1, type_=float)b = getarg(request, b"b", 1, type_=int)if b:# send headers now and delay body- request.write("")+ request.write(b"")self.deferRequest(request, n, self._delayedRender, request, n)return NOT_DONE_YET- def _delayedRender(self, request, n):+ def _delayedRender(self, request: Request, n: float) -> None:request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))request.finish()class Status(LeafResource):- def render_GET(self, request):+ def render_GET(self, request: Request) -> bytes:n = getarg(request, b"n", 200, type_=int)request.setResponseCode(n)return b""class Raw(LeafResource):- def render_GET(self, request):+ def render_GET(self, request: Request) -> int:request.startedWriting = 1self.deferRequest(request, 0, self._delayedRender, request)return NOT_DONE_YETrender_POST = render_GET- def _delayedRender(self, request):+ def _delayedRender(self, request: Request) -> None:raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")request.startedWriting = 1request.write(raw)+ assert request.channel.transport is not Nonerequest.channel.transport.loseConnection()request.finish()class Echo(LeafResource):- def render_GET(self, request):+ def render_GET(self, request: Request) -> bytes:+ assert request.contentoutput = {"headers": {to_unicode(k): [to_unicode(v) for v in vs]@@ -198,27 +226,29 @@ class Echo(LeafResource):class RedirectTo(LeafResource):- def render(self, request):+ def render(self, request: Request) -> bytes:goto = getarg(request, b"goto", b"/")# we force the body content, otherwise Twisted redirectTo()# returns HTML with <meta http-equiv="refresh"- redirectTo(goto, request)+ # zope.interface has no type hints, so mypy cannot tell that Request+ # provides the IRequest interface.+ redirectTo(goto, request) # type: ignore[arg-type]return b"redirecting..."class Partial(LeafResource):- def render_GET(self, request):+ def render_GET(self, request: Request) -> int:request.setHeader(b"Content-Length", b"1024")self.deferRequest(request, 0, self._delayedRender, request)return NOT_DONE_YET- def _delayedRender(self, request):+ def _delayedRender(self, request: Request) -> None:request.write(b"partial content\n")request.finish()class Drop(Partial):- def _delayedRender(self, request):+ def _delayedRender(self, request: Request) -> None:abort = getarg(request, b"abort", 0, type_=int)request.write(b"this connection will be dropped\n")tr = request.channel.transport@@ -233,8 +263,10 @@ class Drop(Partial):class ArbitraryLengthPayloadResource(LeafResource):- def render(self, request):- return request.content.read()+ def render(self, request: Request) -> bytes:+ assert request.content+ data: bytes = request.content.read()+ return dataclass NoMetaRefreshRedirect(Redirect):@@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect):)-class ContentLengthHeaderResource(resource.Resource):+class ContentLengthHeaderResource(BaseResource):"""A testing resource which renders itself as the value of the Content-Lengthheader from the request."""- def render(self, request):- return request.requestHeaders.getRawHeaders(b"content-length")[0]+ def render(self, request: Request) -> bytes:+ headers = request.requestHeaders.getRawHeaders(b"content-length")+ assert headers+ return headers[0]-class ChunkedResource(resource.Resource):- def render(self, request):+class ChunkedResource(BaseResource):+ def render(self, request: Request) -> int:from twisted.internet import reactor- def response():+ def response() -> None:request.write(b"chunked ")request.write(b"content\n")request.finish()@@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource):return server.NOT_DONE_YET-class BrokenChunkedResource(resource.Resource):- def render(self, request):+class BrokenChunkedResource(BaseResource):+ def render(self, request: Request) -> int:from twisted.internet import reactor- def response():+ def response() -> None:request.write(b"chunked ")request.write(b"content\n")# Disable terminating chunk on finish.@@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource):return server.NOT_DONE_YET-class BrokenDownloadResource(resource.Resource):- def render(self, request):+class BrokenDownloadResource(BaseResource):+ def render(self, request: Request) -> int:from twisted.internet import reactor- def response():+ def response() -> None:request.setHeader(b"Content-Length", b"20")request.write(b"partial")close_connection(request)@@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource):return server.NOT_DONE_YET-class EmptyContentTypeHeaderResource(resource.Resource):+class EmptyContentTypeHeaderResource(BaseResource):"""A testing resource which renders itself as the value of request bodywithout content-type header in response."""- def render(self, request):+ def render(self, request: Request) -> bytes:+ assert request.contentrequest.setHeader("content-type", "")- return request.content.read()+ data: bytes = request.content.read()+ return data-class LargeChunkedFileResource(resource.Resource):- def render(self, request):+class LargeChunkedFileResource(BaseResource):+ def render(self, request: Request) -> int:from twisted.internet import reactor- def response():+ def response() -> None:for _ in range(1024):request.write(b"x" * 1024)request.finish()@@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource):return server.NOT_DONE_YET-class DuplicateHeaderResource(resource.Resource):- def render(self, request):+class DuplicateHeaderResource(BaseResource):+ def render(self, request: Request) -> bytes:request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])return b""-class UriResource(resource.Resource):+class UriResource(BaseResource):"""Return the full uri that was requested"""- def getChild(self, path, request):+ def getChild(self, path: bytes, request: Request) -> resource.Resource:return self- def render(self, request):+ def render(self, request: Request) -> bytes | int:# Note: this is an ugly hack for CONNECT request timeout test.# Returning some data here fail SSL/TLS handshake# ToDo: implement proper HTTPS proxy tests, not faking them.if request.method != b"CONNECT":return request.uri+ assert request.transport is not Nonerequest.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n")return NOT_DONE_YET-class ResponseHeadersResource(resource.Resource):+class ResponseHeadersResource(BaseResource):"""Return a response with headers set from the JSON request body"""- def render(self, request):+ def render(self, request: Request) -> bytes:+ assert request.contentbody = json.loads(request.content.read().decode())for header_name, header_value in body.items():request.responseHeaders.setRawHeaders(header_name, [header_value])return json.dumps(body).encode("utf-8")-class Compress(resource.Resource):+class Compress(BaseResource):"""Compress the data sent in the request url params and set Content-Encoding header"""- def render(self, request):- data = request.args.get(b"data")[0]+ def render(self, request: Request) -> bytes:+ data = request.args[b"data"][0]accept_encoding_header = request.getHeader(b"accept-encoding")@@ -370,10 +408,10 @@ class Compress(resource.Resource):return b"Did not receive a valid accept-encoding header"-class SetCookie(resource.Resource):+class SetCookie(BaseResource):"""Return a response with a Set-Cookie header for each request url parameter"""- def render(self, request):+ def render(self, request: Request) -> bytes:for cookie_name, cookie_values in request.args.items():for cookie_value in cookie_values:cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode()diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.pyindex fdea666e1..2a6cb6dd8 100644--- a/tests/mockserver/simple_https.py+++ b/tests/mockserver/simple_https.py@@ -2,18 +2,23 @@from __future__ import annotations-from twisted.web import resource+from typing import TYPE_CHECKING+from twisted.web.static import Datafrom .http_base import BaseMockServer, main_factory+from .http_resources import BaseResource, put_child++if TYPE_CHECKING:+ from twisted.web.server import Request-class Root(resource.Resource):- def __init__(self):- resource.Resource.__init__(self)- self.putChild(b"file", Data(b"0123456789", "text/plain"))+class Root(BaseResource):+ def __init__(self) -> None:+ super().__init__()+ put_child(self, b"file", Data(b"0123456789", "text/plain"))- def getChild(self, path, request):+ def getChild(self, path: bytes, request: Request) -> Root:return self@@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer):cipher_string: str | None = None,tls_min_version: str | None = None,tls_max_version: str | None = None,- ):+ ) -> None:super().__init__()self.keyfile = keyfileself.certfile = certfilediff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 3e4139b3e..912c0450b 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -14,7 +14,7 @@ from twisted.web import server, staticfrom twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBodyfrom twisted.web.client import Response as TxResponse-from scrapy import Request+from scrapy import Request, Spiderfrom scrapy.core.downloader import Downloader, Slot, tlsfrom scrapy.core.downloader.contextfactory import (_load_context_factory_from_settings,@@ -31,14 +31,17 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler-from tests.mockserver.http_resources import PayloadResource+from tests.mockserver.http_resources import PayloadResource, put_childfrom tests.mockserver.utils import ssl_context_factoryfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:from twisted.internet.defer import Deferred+ from twisted.internet.interfaces import IListeningPortfrom twisted.web.iweb import IBodyProducer+ from scrapy.http import Response+class TestSlot:def test_repr(self):@@ -52,7 +55,7 @@ class TestContextFactoryBase:async def server_url(self, tmp_path):(tmp_path / "file").write_bytes(b"0123456789")r = static.File(str(tmp_path))- r.putChild(b"payload", PayloadResource())+ put_child(r, b"payload", PayloadResource())site = server.Site(r, timeout=None)port = self._listen(site)portno = port.getHost().port@@ -61,7 +64,7 @@ class TestContextFactoryBase:await port.stopListening()- def _listen(self, site):+ def _listen(self, site: server.Site) -> IListeningPort:from twisted.internet import reactorreturn reactor.listenSSL(@@ -317,7 +320,10 @@ def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:@coroutine_testasync def test_fetch_deprecated_spider_arg():class CustomDownloader(Downloader):- def fetch(self, request, spider): # pylint: disable=signature-differs+ # requiring the spider argument is what triggers the deprecation+ def fetch( # type: ignore[override] # pylint: disable=signature-differs+ self, request: Request, spider: Spider+ ) -> Deferred[Response | Request]:return super().fetch(request, spider)crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.pyindex 489b70e74..14de97b21 100644--- a/tests/test_downloader_handler_twisted_ftp.py+++ b/tests/test_downloader_handler_twisted_ftp.py@@ -156,14 +156,17 @@ class TestFTP(TestFTPBase):for filename, content in self.test_files:(userdir / filename).write_bytes(content)- def _get_factory(self, root):+ def _get_factory(self, root: Path) -> FTPFactory:from twisted.protocols.ftp import FTPFactory, FTPRealmrealm = FTPRealm(anonymousRoot=str(root), userHome=str(root))- p = portal.Portal(realm)+ # zope.interface has no type hints, so mypy cannot tell that these+ # objects provide the interfaces that Portal expects.+ p = portal.Portal(realm) # type: ignore[arg-type]users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()- users_checker.addUser(self.username, self.password)- p.registerChecker(users_checker, credentials.IUsernamePassword)+ # the FTP protocol authenticates with str credentials+ users_checker.addUser(self.username, self.password) # type: ignore[arg-type]+ p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type]return FTPFactory(portal=p)@deferred_f_from_coro_f@@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase):for filename, content in self.test_files:(root / filename).write_bytes(content)- def _get_factory(self, tmp_path):+ def _get_factory(self, tmp_path: Path) -> FTPFactory:from twisted.protocols.ftp import FTPFactory, FTPRealmrealm = FTPRealm(anonymousRoot=str(tmp_path))- p = portal.Portal(realm)- p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)+ # zope.interface has no type hints, so mypy cannot tell that these+ # objects provide the interfaces that Portal expects.+ p = portal.Portal(realm) # type: ignore[arg-type]+ p.registerChecker(+ checkers.AllowAnonymousAccess(), # type: ignore[arg-type]+ credentials.IAnonymous,+ )return FTPFactory(portal=p, userAnonymous=self.username)diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 3c1347fd3..b8586d1ca 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -30,7 +30,7 @@ from scrapy.utils.defer import (deferred_from_coro,maybe_deferred_to_future,)-from tests.mockserver.http_resources import LeafResource, Status+from tests.mockserver.http_resources import LeafResource, Status, put_childfrom tests.mockserver.utils import ssl_context_factoryif TYPE_CHECKING:@@ -199,18 +199,18 @@ class TestHttps2ClientProtocol:@pytest.fixturedef site(self, tmp_path):r = File(str(tmp_path))- r.putChild(b"get-data-html-small", GetDataHtmlSmall())- r.putChild(b"get-data-html-large", GetDataHtmlLarge())-- r.putChild(b"post-data-json-small", PostDataJsonSmall())- r.putChild(b"post-data-json-large", PostDataJsonLarge())-- r.putChild(b"dataloss", Dataloss())- r.putChild(b"no-content-length-header", NoContentLengthHeader())- r.putChild(b"status", Status())- r.putChild(b"query-params", QueryParams())- r.putChild(b"timeout", TimeoutResponse())- r.putChild(b"request-headers", RequestHeaders())+ put_child(r, b"get-data-html-small", GetDataHtmlSmall())+ put_child(r, b"get-data-html-large", GetDataHtmlLarge())++ put_child(r, b"post-data-json-small", PostDataJsonSmall())+ put_child(r, b"post-data-json-large", PostDataJsonLarge())++ put_child(r, b"dataloss", Dataloss())+ put_child(r, b"no-content-length-header", NoContentLengthHeader())+ put_child(r, b"status", Status())+ put_child(r, b"query-params", QueryParams())+ put_child(r, b"timeout", TimeoutResponse())+ put_child(r, b"request-headers", RequestHeaders())return Site(r, timeout=None)@async_yield_fixture # type: ignore[untyped-decorator]diff --git a/tox.ini b/tox.iniindex edde83356..ac32064a6 100644--- a/tox.ini+++ b/tox.ini@@ -111,7 +111,8 @@ commands =pre-commit run {posargs:--all-files}[testenv:pylint]-basepython = python3+# Some checks are Python-version-dependent, so pin the version used in CI.+basepython = python3.14deps ={[testenv:extra-deps]deps}pylint==4.0.6