Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Pin older Twisted in tests, update type hints. (#6882)

Andrey Rakhmatullinba10dcfd1a44
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 74a6e54ee..54fef48b6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,7 +8,7 @@ import re
from contextlib import suppress
from io import BytesIO
from time import time
-from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
+from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
from urllib.parse import urldefrag, urlparse
from twisted.internet import ssl
@@ -27,7 +27,7 @@ from twisted.web.client import (
from twisted.web.client import Response as TxResponse
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse
from zope.interface import implementer
from scrapy import Request, Spider, signals
@@ -286,11 +286,11 @@ class TunnelingAgent(Agent):
key: Any,
endpoint: TCP4ClientEndpoint,
method: bytes,
- parsedURI: bytes,
+ parsedURI: URI,
headers: TxHeaders | None,
bodyProducer: IBodyProducer | None,
requestPath: bytes,
- ) -> Deferred[TxResponse]:
+ ) -> Deferred[IResponse]:
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
@@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent):
uri: bytes,
headers: TxHeaders | None = None,
bodyProducer: IBodyProducer | None = None,
- ) -> Deferred[TxResponse]:
+ ) -> Deferred[IResponse]:
"""
Issue a new request via the configured proxy.
"""
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
return self._requestWithEndpoint(
- key=("http-proxy", self._proxyURI.host, self._proxyURI.port),
+ key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port),
endpoint=self._getEndpoint(self._proxyURI),
method=method,
parsedURI=URI.fromBytes(uri),
@@ -426,8 +426,11 @@ class ScrapyAgent:
headers.removeHeader(b"Proxy-Authorization")
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
start_time = time()
- d: Deferred[TxResponse] = agent.request(
- method, to_bytes(url, encoding="ascii"), headers, bodyproducer
+ d: Deferred[IResponse] = agent.request(
+ method,
+ to_bytes(url, encoding="ascii"),
+ headers,
+ cast(IBodyProducer, bodyproducer),
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 2da526cd8..94b1a1fc7 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -10,7 +10,7 @@ import warnings
from importlib import import_module
from pathlib import Path
from posixpath import split
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar, cast
from unittest import TestCase, mock
from twisted.trial.unittest import SkipTest
@@ -211,4 +211,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
from twisted.web.client import Agent # imports twisted.internet.reactor
agent = Agent(reactor)
- return agent.request(b"GET", url.encode("utf-8"))
+ return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index ef77f7843..464320602 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -4,7 +4,7 @@ import shutil
import warnings
from pathlib import Path
from tempfile import mkdtemp
-from typing import Any
+from typing import Any, cast
import OpenSSL.SSL
import pytest
@@ -14,6 +14,7 @@ from twisted.trial import unittest
from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
+from twisted.web.iweb import IBodyProducer
from scrapy.core.downloader import Slot
from scrapy.core.downloader.contextfactory import (
@@ -76,8 +77,15 @@ class TestContextFactoryBase(unittest.TestCase):
agent = Agent(reactor, contextFactory=client_context_factory)
body_producer = _RequestBodyProducer(body.encode()) if body else None
- response: TxResponse = await maybe_deferred_to_future(
- agent.request(b"GET", url.encode(), bodyProducer=body_producer)
+ response: TxResponse = cast(
+ TxResponse,
+ await maybe_deferred_to_future(
+ agent.request(
+ b"GET",
+ url.encode(),
+ bodyProducer=cast(IBodyProducer, body_producer),
+ )
+ ),
)
with warnings.catch_warnings():
# https://github.com/twisted/twisted/issues/8227
diff --git a/tox.ini b/tox.ini
index 92cfc3794..5680d98d1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,6 +20,7 @@ deps =
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
testfixtures
pywin32; sys_platform == "win32"
+ Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467
[testenv]
deps =