Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix and ban the top-level twisted.internet.reactor imports. (#6835)
Andrey Rakhmatullin916fe509744c
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.pyindex 70c9003e5..734614aa5 100755--- a/extras/qps-bench-server.py+++ b/extras/qps-bench-server.py@@ -2,7 +2,7 @@from collections import dequefrom time import time-from twisted.internet import reactor+from twisted.internet import reactor # noqa: TID253from twisted.web.resource import Resourcefrom twisted.web.server import NOT_DONE_YET, Sitediff --git a/pyproject.toml b/pyproject.tomlindex 68c1e07bb..871da8020 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -299,6 +299,8 @@ extend-select = ["T10",# flake8-type-checking"TC",+ # flake8-tidy-imports+ "TID",# pyupgrade"UP",# pycodestyle warnings@@ -398,6 +400,11 @@ ignore = ["SIM112",]+[tool.ruff.lint.flake8-tidy-imports]+banned-module-level-imports = [+ "twisted.internet.reactor",+]+[tool.ruff.lint.per-file-ignores]# Circular import workarounds"scrapy/linkextractors/__init__.py" = ["E402"]diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.pyindex 8f59c035c..cbe6427ea 100644--- a/tests/CrawlerProcess/reactor_default.py+++ b/tests/CrawlerProcess/reactor_default.py@@ -1,4 +1,4 @@-from twisted.internet import reactor # noqa: F401+from twisted.internet import reactor # noqa: F401,TID253from twisted.python import logimport scrapydiff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.pyindex 9901dd634..f7802fbc6 100644--- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py@@ -1,4 +1,4 @@-from twisted.internet import reactor # noqa: F401+from twisted.internet import reactor # noqa: F401,TID253from twisted.python import logimport scrapydiff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.pyindex 6c0102241..c275e0583 100644--- a/tests/CrawlerRunner/change_reactor.py+++ b/tests/CrawlerRunner/change_reactor.py@@ -26,7 +26,7 @@ runner = CrawlerRunner()d = runner.crawl(NoRequestsSpider)-from twisted.internet import reactor # noqa: E402+from twisted.internet import reactor # noqa: E402,TID253d.addBoth(callback=lambda _: reactor.stop())reactor.run()diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pyindex 5e2184afb..207fc86ad 100644--- a/tests/CrawlerRunner/ip_address.py+++ b/tests/CrawlerRunner/ip_address.py@@ -6,7 +6,6 @@ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")from urllib.parse import urlparse-from twisted.internet import reactorfrom twisted.names import cache, resolvefrom twisted.names import hosts as hostsModulefrom twisted.names.client import Resolver@@ -44,6 +43,8 @@ class LocalhostSpider(Spider):if __name__ == "__main__":+ from twisted.internet import reactor+with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:port = urlparse(mock_http_server.http_address).porturl = f"http://not.a.real.domain:{port}/echo"diff --git a/tests/mockserver.py b/tests/mockserver.pyindex f5c12787a..e0ac127f2 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -13,7 +13,7 @@ from typing import TYPE_CHECKINGfrom urllib.parse import urlencodefrom OpenSSL import SSL-from twisted.internet import defer, reactor, ssl+from twisted.internet import defer, sslfrom twisted.internet.task import deferLaterfrom twisted.names import dns, errorfrom twisted.names.server import DNSServerFactory@@ -114,6 +114,8 @@ class LeafResource(resource.Resource):isLeaf = Truedef deferRequest(self, request, delay, f, *a, **kw):+ from twisted.internet import reactor+def _cancelrequest(_):# silence CancelledErrord.addErrback(lambda _: None)@@ -378,6 +380,8 @@ def ssl_context_factory(if __name__ == "__main__":+ from twisted.internet import reactor+parser = argparse.ArgumentParser()parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http"diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.pyindex 5006e3689..c6fdb13ea 100644--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py@@ -2,7 +2,7 @@ import sysfrom pathlib import Pathfrom subprocess import PIPE, Popen-from .. import TWISTED_KEEPS_TRACEBACKS+from tests import TWISTED_KEEPS_TRACEBACKSclass TestCmdlineCrawlPipeline:diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 1bffd69ed..ef77f7843 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -8,7 +8,6 @@ from typing import Anyimport OpenSSL.SSLimport pytest-from twisted.internet import reactorfrom twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.protocols.policies import WrappingFactoryfrom twisted.trial import unittest@@ -40,6 +39,8 @@ class TestContextFactoryBase(unittest.TestCase):context_factory = Nonedef _listen(self, site):+ from twisted.internet import reactor+return reactor.listenSSL(0,site,@@ -71,6 +72,8 @@ class TestContextFactoryBase(unittest.TestCase):client_context_factory: BrowserLikePolicyForHTTPS,body: str | None = None,) -> bytes:+ from twisted.internet import reactor+agent = Agent(reactor, contextFactory=client_context_factory)body_producer = _RequestBodyProducer(body.encode()) if body else Noneresponse: TxResponse = await maybe_deferred_to_future(diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex 46322a747..159f403d0 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -8,7 +8,7 @@ from unittest import mockimport pytestfrom testfixtures import LogCapture-from twisted.internet import defer, error, reactor+from twisted.internet import defer, errorfrom twisted.web import serverfrom twisted.web.error import SchemeNotSupportedfrom twisted.web.http import H2_ENABLED@@ -59,6 +59,8 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):@defer.inlineCallbacksdef test_download_with_maxsize_very_large_file(self):+ from twisted.internet import reactor+with mock.patch("scrapy.core.http2.stream.logger") as logger:request = Request(self.getURL("largechunkedfile"))@@ -207,6 +209,8 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase):expected_http_proxy_request_body = b"/"def setUp(self):+ from twisted.internet import reactor+site = server.Site(UriResource(), timeout=None)self.port = reactor.listenSSL(0,diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex fc6ac5aee..dacadb075 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -12,7 +12,6 @@ from unittest import mockimport pytestfrom twisted.cred import checkers, credentials, portal-from twisted.internet import reactorfrom twisted.protocols.ftp import FTPFactory, FTPRealmfrom twisted.trial import unittestfrom w3lib.url import path_to_file_uri@@ -310,6 +309,8 @@ class TestFTPBase(unittest.TestCase):)def setUp(self):+ from twisted.internet import reactor+# setup dirs and test fileself.directory = Path(mkdtemp())userdir = self.directory / self.username@@ -451,6 +452,8 @@ class TestAnonymousFTP(TestFTPBase):req_meta = {}def setUp(self):+ from twisted.internet import reactor+# setup dir and test fileself.directory = Path(mkdtemp())for filename, content in self.test_files:diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 46e5972f7..5eaf66966 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -12,7 +12,7 @@ from unittest import mockimport pytestfrom testfixtures import LogCapture-from twisted.internet import defer, error, reactor+from twisted.internet import defer, errorfrom twisted.protocols.policies import WrappingFactoryfrom twisted.trial import unittestfrom twisted.web import resource, server, static, util@@ -52,6 +52,8 @@ class ContentLengthHeaderResource(resource.Resource):class ChunkedResource(resource.Resource):def render(self, request):+ from twisted.internet import reactor+def response():request.write(b"chunked ")request.write(b"content\n")@@ -63,6 +65,8 @@ class ChunkedResource(resource.Resource):class BrokenChunkedResource(resource.Resource):def render(self, request):+ from twisted.internet import reactor+def response():request.write(b"chunked ")request.write(b"content\n")@@ -76,6 +80,8 @@ class BrokenChunkedResource(resource.Resource):class BrokenDownloadResource(resource.Resource):def render(self, request):+ from twisted.internet import reactor+def response():request.setHeader(b"Content-Length", b"20")request.write(b"partial")@@ -105,6 +111,8 @@ class EmptyContentTypeHeaderResource(resource.Resource):class LargeChunkedFileResource(resource.Resource):def render(self, request):+ from twisted.internet import reactor+def response():for i in range(1024):request.write(b"x" * 1024)@@ -133,6 +141,8 @@ class TestHttpBase(unittest.TestCase, ABC):raise NotImplementedErrordef setUp(self):+ from twisted.internet import reactor+self.tmpname = Path(mkdtemp())(self.tmpname / "file").write_bytes(b"0123456789")r = static.File(str(self.tmpname))@@ -365,6 +375,9 @@ class TestHttp11Base(TestHttpBase):@defer.inlineCallbacksdef test_download_with_maxsize_very_large_file(self):+ from twisted.internet import reactor++ # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger:request = Request(self.getURL("largechunkedfile"))@@ -501,6 +514,8 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):raise NotImplementedErrordef setUp(self):+ from twisted.internet import reactor+self.tmpname = Path(mkdtemp())(self.tmpname / "file").write_bytes(b"0123456789")r = static.File(str(self.tmpname))@@ -639,6 +654,8 @@ class TestHttpProxyBase(unittest.TestCase, ABC):raise NotImplementedErrordef setUp(self):+ from twisted.internet import reactor+site = server.Site(UriResource(), timeout=None)wrapper = WrappingFactory(site)self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1")diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 9518f1835..04800896c 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKINGfrom unittest import mockimport pytest-from twisted.internet import error, reactor+from twisted.internet import errorfrom twisted.internet.defer import Deferred, maybeDeferredfrom twisted.python import failurefrom twisted.trial import unittest@@ -53,6 +53,8 @@ Disallow: /some/randome/page.htmlresponse = TextResponse("http://site.local/robots.txt", body=ROBOTS)def return_response(request):+ from twisted.internet import reactor+deferred = Deferred()reactor.callFromThread(deferred.callback, response)return deferred@@ -102,6 +104,8 @@ Disallow: /some/randome/page.html)def return_response(request):+ from twisted.internet import reactor+deferred = Deferred()reactor.callFromThread(deferred.callback, response)return deferred@@ -124,6 +128,8 @@ Disallow: /some/randome/page.htmlresponse = Response("http://site.local/robots.txt")def return_response(request):+ from twisted.internet import reactor+deferred = Deferred()reactor.callFromThread(deferred.callback, response)return deferred@@ -145,6 +151,8 @@ Disallow: /some/randome/page.htmlerr = error.DNSLookupError("Robotstxt address not found")def return_failure(request):+ from twisted.internet import reactor+deferred = Deferred()reactor.callFromThread(deferred.errback, failure.Failure(err))return deferred@@ -178,6 +186,8 @@ Disallow: /some/randome/page.htmlself.crawler.settings.set("ROBOTSTXT_OBEY", True)def ignore_request(request):+ from twisted.internet import reactor+deferred = Deferred()reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))return deferreddiff --git a/tests/test_engine.py b/tests/test_engine.pyindex b60b510b2..1f79a081d 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -24,7 +24,7 @@ from urllib.parse import urlparseimport attrfrom itemadapter import ItemAdapterfrom pydispatch import dispatcher-from twisted.internet import defer, reactor+from twisted.internet import deferfrom twisted.trial import unittestfrom twisted.web import server, static, util@@ -130,6 +130,8 @@ class ChangeCloseReasonSpider(MySpider):def start_test_site(debug=False):+ from twisted.internet import reactor+root_dir = Path(tests_datadir, "test_site")r = static.File(str(root_dir))r.putChild(b"redirect", util.Redirect(b"/redirected"))@@ -514,6 +516,8 @@ def test_request_scheduled_signal(caplog):if __name__ == "__main__":+ from twisted.internet import reactor # pylint: disable=ungrouped-imports+if len(sys.argv) > 1 and sys.argv[1] == "runserver":start_test_site(debug=True)reactor.run()diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 7c1b38877..0605c2438 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -13,7 +13,6 @@ from unittest import mock, skipIffrom urllib.parse import urlencodeimport pytest-from twisted.internet import reactorfrom twisted.internet.defer import (CancelledError,Deferred,@@ -209,6 +208,8 @@ class TestHttps2ClientProtocol(TestCase):@inlineCallbacksdef setUp(self):+ from twisted.internet import reactor+# Initialize resource treeroot = self._init_resource()self.site = Site(root, timeout=None)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex d915fc2a3..2d0db6e25 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -4,7 +4,6 @@ import warningsimport pytestfrom testfixtures import LogCapture-from twisted.internet import reactorfrom twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.python.failure import Failurefrom twisted.trial import unittest@@ -329,6 +328,8 @@ class TestMediaPipeline(TestBaseMediaPipeline):rsp1 = Response("http://url")def rsp1_func():+ from twisted.internet import reactor+dfd = Deferred().addCallback(_check_downloading)reactor.callLater(0.1, dfd.callback, rsp1)return dfddiff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 6dff321da..9b3ce6b0b 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -3,7 +3,7 @@ import asyncioimport pytestfrom pydispatch import dispatcherfrom testfixtures import LogCapture-from twisted.internet import defer, reactor+from twisted.internet import deferfrom twisted.python.failure import Failurefrom twisted.trial import unittest@@ -65,6 +65,8 @@ class SendCatchLogDeferredTest(TestSendCatchLog):class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):def ok_handler(self, arg, handlers_called):+ from twisted.internet import reactor+handlers_called.add(self.ok_handler)assert arg == "test"d = defer.Deferred()@@ -97,6 +99,8 @@ class SendCatchLogAsyncTest(TestSendCatchLog):class SendCatchLogAsyncTest2(SendCatchLogAsyncTest):def ok_handler(self, arg, handlers_called):+ from twisted.internet import reactor+handlers_called.add(self.ok_handler)assert arg == "test"d = defer.Deferred()diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex c3c03d6c3..e580d51ca 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -12,7 +12,7 @@ from urllib.parse import urlparseimport OpenSSL.SSLimport pytest-from twisted.internet import defer, reactor+from twisted.internet import deferfrom twisted.internet.defer import inlineCallbacksfrom twisted.internet.testing import StringTransportfrom twisted.protocols.policies import WrappingFactory@@ -205,6 +205,8 @@ class EncodingResource(resource.Resource):@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")class TestWebClient(unittest.TestCase):def _listen(self, site):+ from twisted.internet import reactor+return reactor.listenTCP(0, site, interface="127.0.0.1")def setUp(self):@@ -318,6 +320,8 @@ class TestWebClient(unittest.TestCase):assert b"404 - No Such Resource" in pageDatadef testFactoryInfo(self):+ from twisted.internet import reactor+url = self.getURL("file")parsed = urlparse(url)factory = client.ScrapyHTTPClientFactory(Request(url))