Scrapy, a fast high-level web crawling & scraping framework for Python.
Pin older Twisted in tests, update type hints. (#6882)
Andrey Rakhmatullin24a827c72e9f
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 74a6e54ee..54fef48b6 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -8,7 +8,7 @@ import refrom contextlib import suppressfrom io import BytesIOfrom time import time-from typing import TYPE_CHECKING, Any, TypedDict, TypeVar+from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, castfrom urllib.parse import urldefrag, urlparsefrom twisted.internet import ssl@@ -27,7 +27,7 @@ from twisted.web.client import (from twisted.web.client import Response as TxResponsefrom twisted.web.http import PotentialDataLoss, _DataLossfrom twisted.web.http_headers import Headers as TxHeaders-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponsefrom zope.interface import implementerfrom scrapy import Request, Spider, signals@@ -286,11 +286,11 @@ class TunnelingAgent(Agent):key: Any,endpoint: TCP4ClientEndpoint,method: bytes,- parsedURI: bytes,+ parsedURI: URI,headers: TxHeaders | None,bodyProducer: IBodyProducer | None,requestPath: bytes,- ) -> Deferred[TxResponse]:+ ) -> Deferred[IResponse]:# proxy host and port are required for HTTP pool `key`# otherwise, same remote host connection request could reuse# a cached tunneled connection to a different proxy@@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent):uri: bytes,headers: TxHeaders | None = None,bodyProducer: IBodyProducer | None = None,- ) -> Deferred[TxResponse]:+ ) -> Deferred[IResponse]:"""Issue a new request via the configured proxy."""# Cache *all* connections under the same key, since we are only# connecting to a single destination, the proxy:return self._requestWithEndpoint(- key=("http-proxy", self._proxyURI.host, self._proxyURI.port),+ key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port),endpoint=self._getEndpoint(self._proxyURI),method=method,parsedURI=URI.fromBytes(uri),@@ -426,8 +426,11 @@ class ScrapyAgent:headers.removeHeader(b"Proxy-Authorization")bodyproducer = _RequestBodyProducer(request.body) if request.body else Nonestart_time = time()- d: Deferred[TxResponse] = agent.request(- method, to_bytes(url, encoding="ascii"), headers, bodyproducer+ d: Deferred[IResponse] = agent.request(+ method,+ to_bytes(url, encoding="ascii"),+ headers,+ cast(IBodyProducer, bodyproducer),)# set download latencyd.addCallback(self._cb_latency, request, start_time)diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 4a732bd72..3780ad23e 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -10,7 +10,7 @@ import warningsfrom importlib import import_modulefrom pathlib import Pathfrom posixpath import split-from typing import TYPE_CHECKING, Any, TypeVar+from typing import TYPE_CHECKING, Any, TypeVar, castfrom unittest import TestCase, mockfrom twisted.trial.unittest import SkipTest@@ -216,4 +216,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:from twisted.web.client import Agent # imports twisted.internet.reactoragent = Agent(reactor)- return agent.request(b"GET", url.encode("utf-8"))+ return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 1bffd69ed..668a2cd1b 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -4,7 +4,7 @@ import shutilimport warningsfrom pathlib import Pathfrom tempfile import mkdtemp-from typing import Any+from typing import Any, castimport OpenSSL.SSLimport pytest@@ -15,6 +15,7 @@ from twisted.trial import unittestfrom twisted.web import server, staticfrom twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBodyfrom twisted.web.client import Response as TxResponse+from twisted.web.iweb import IBodyProducerfrom scrapy.core.downloader import Slotfrom scrapy.core.downloader.contextfactory import (@@ -73,8 +74,15 @@ class TestContextFactoryBase(unittest.TestCase):) -> bytes:agent = Agent(reactor, contextFactory=client_context_factory)body_producer = _RequestBodyProducer(body.encode()) if body else None- response: TxResponse = await maybe_deferred_to_future(- agent.request(b"GET", url.encode(), bodyProducer=body_producer)+ response: TxResponse = cast(+ TxResponse,+ await maybe_deferred_to_future(+ agent.request(+ b"GET",+ url.encode(),+ bodyProducer=cast(IBodyProducer, body_producer),+ )+ ),)with warnings.catch_warnings():# https://github.com/twisted/twisted/issues/8227diff --git a/tox.ini b/tox.iniindex 92cfc3794..5680d98d1 100644--- a/tox.ini+++ b/tox.ini@@ -20,6 +20,7 @@ deps =sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422testfixturespywin32; sys_platform == "win32"+ Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467[testenv]deps =