Scrapy, a fast high-level web crawling & scraping framework for Python.
Address 2/3 of warnings from tests (#5561)
Mikhail Korobovb103664bf45b
diff --git a/pytest.ini b/pytest.iniindex ae2ed2029..af0f2fb6e 100644--- a/pytest.ini+++ b/pytest.ini@@ -21,3 +21,6 @@ addopts =markers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passedonly_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed+filterwarnings =+ ignore:scrapy.downloadermiddlewares.decompression is deprecated+ ignore:Module scrapy.utils.reqser is deprecateddiff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex d0f4a68c2..136056f50 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -5,11 +5,11 @@ from twisted.python import failurefrom twisted.trial import unittestfrom scrapy import FormRequest-from scrapy.crawler import CrawlerRunnerfrom scrapy.spidermiddlewares.httperror import HttpErrorfrom scrapy.spiders import Spiderfrom scrapy.http import Requestfrom scrapy.item import Item, Field+from scrapy.utils.test import get_crawlerfrom scrapy.contracts import ContractsManager, Contractfrom scrapy.contracts.default import (UrlContract,@@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase):TestSameUrlSpider.parse_first.__doc__ = contract_docTestSameUrlSpider.parse_second.__doc__ = contract_doc- crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)+ crawler = get_crawler(TestSameUrlSpider)yield crawler.crawl()self.assertEqual(crawler.spider.visited, 2)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex f9ffcd6bb..59c271868 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -18,6 +18,7 @@ from scrapy.exceptions import StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.python import to_unicode+from scrapy.utils.test import get_crawlerfrom tests import NON_EXISTING_RESOLVABLEfrom tests.mockserver import MockServerfrom tests.spiders import (@@ -49,14 +50,13 @@ class CrawlTestCase(TestCase):def setUp(self):self.mockserver = MockServer()self.mockserver.__enter__()- self.runner = CrawlerRunner()def tearDown(self):self.mockserver.__exit__(None, None, None)@defer.inlineCallbacksdef test_follow_all(self):- crawler = self.runner.create_crawler(FollowAllSpider)+ crawler = get_crawler(FollowAllSpider)yield crawler.crawl(mockserver=self.mockserver)self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url@@ -79,7 +79,7 @@ class CrawlTestCase(TestCase):settings = {"DOWNLOAD_DELAY": delay,'RANDOMIZE_DOWNLOAD_DELAY': randomize}- crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)+ crawler = get_crawler(FollowAllSpider, settings)yield crawler.crawl(**crawl_kwargs)times = crawler.spider.timestotal_time = times[-1] - times[0]@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):# of ``total`` and ``delay`` values that are too small for the test# code above to have any meaning.settings["DOWNLOAD_DELAY"] = 0- crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)+ crawler = get_crawler(FollowAllSpider, settings)yield crawler.crawl(**crawl_kwargs)times = crawler.spider.timestotal_time = times[-1] - times[0]@@ -102,7 +102,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_timeout_success(self):- crawler = self.runner.create_crawler(DelaySpider)+ crawler = get_crawler(DelaySpider)yield crawler.crawl(n=0.5, mockserver=self.mockserver)self.assertTrue(crawler.spider.t1 > 0)self.assertTrue(crawler.spider.t2 > 0)@@ -110,7 +110,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_timeout_failure(self):- crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)+ crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})yield crawler.crawl(n=0.5, mockserver=self.mockserver)self.assertTrue(crawler.spider.t1 > 0)self.assertTrue(crawler.spider.t2 == 0)@@ -125,14 +125,14 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_retry_503(self):- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)self._assert_retried(log)@defer.inlineCallbacksdef test_retry_conn_failed(self):- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)with LogCapture() as log:yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)self._assert_retried(log)@@ -141,7 +141,7 @@ class CrawlTestCase(TestCase):def test_retry_dns_error(self):if NON_EXISTING_RESOLVABLE:raise unittest.SkipTest("Non-existing hosts are resolvable")- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)with LogCapture() as log:# try to fetch the homepage of a non-existent domainyield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)@@ -150,7 +150,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_start_requests_bug_before_yield(self):with LogCapture('scrapy', level=logging.ERROR) as log:- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)+ crawler = get_crawler(BrokenStartRequestsSpider)yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)self.assertEqual(len(log.records), 1)@@ -161,7 +161,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_start_requests_bug_yielding(self):with LogCapture('scrapy', level=logging.ERROR) as log:- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)+ crawler = get_crawler(BrokenStartRequestsSpider)yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)self.assertEqual(len(log.records), 1)@@ -172,7 +172,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_start_requests_lazyness(self):settings = {"CONCURRENT_REQUESTS": 1}- crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)+ crawler = get_crawler(BrokenStartRequestsSpider, settings)yield crawler.crawl(mockserver=self.mockserver)self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),@@ -181,7 +181,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_start_requests_dupes(self):settings = {"CONCURRENT_REQUESTS": 1}- crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)+ crawler = get_crawler(DuplicateStartRequestsSpider, settings)yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)self.assertEqual(crawler.spider.visited, 6)@@ -210,7 +210,7 @@ Connection: closefoo bodywith multiples lines'''})- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)with LogCapture() as log:yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)self.assertEqual(str(log).count("Got response 200"), 1)@@ -218,7 +218,7 @@ with multiples lines@defer.inlineCallbacksdef test_retry_conn_lost(self):# connection lost after receiving data- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)self._assert_retried(log)@@ -226,7 +226,7 @@ with multiples lines@defer.inlineCallbacksdef test_retry_conn_aborted(self):# connection lost before receiving data- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)self._assert_retried(log)@@ -245,7 +245,7 @@ with multiples linesreq0.meta['next'] = req1req1.meta['next'] = req2req2.meta['next'] = req3- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)yield crawler.crawl(seed=req0, mockserver=self.mockserver)# basic asserts in case of weird communication errorsself.assertIn('responses', crawler.spider.meta)@@ -271,7 +271,7 @@ with multiples linesdef cb(response):est.append(get_engine_status(crawler.engine))- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)self.assertEqual(len(est), 1, est)s = dict(est[0])@@ -286,7 +286,7 @@ with multiples linesdef cb(response):est.append(format_engine_status(crawler.engine))- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)self.assertEqual(len(est), 1, est)est = est[0].split("\n")[2:-2] # remove header & footer@@ -317,7 +317,7 @@ with multiples linesdef start_requests(self):raise TestError- crawler = self.runner.create_crawler(FaultySpider)+ crawler = get_crawler(FaultySpider)yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)self.assertFalse(crawler.crawling)@@ -328,26 +328,28 @@ with multiples lines"tests.pipelines.ZeroDivisionErrorPipeline": 300,}}- crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider, settings)yield self.assertFailure(- self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),+ crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),ZeroDivisionError)self.assertFalse(crawler.crawling)@defer.inlineCallbacksdef test_crawlerrunner_accepts_crawler(self):- crawler = self.runner.create_crawler(SimpleSpider)+ crawler = get_crawler(SimpleSpider)+ runner = CrawlerRunner()with LogCapture() as log:- yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)self.assertIn("Got response 200", str(log))@defer.inlineCallbacksdef test_crawl_multiple(self):- self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)- self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)+ runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})+ runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)with LogCapture() as log:- yield self.runner.join()+ yield runner.join()self._assert_retried(log)self.assertIn("Got response 200", str(log))@@ -358,7 +360,6 @@ class CrawlSpiderTestCase(TestCase):def setUp(self):self.mockserver = MockServer()self.mockserver.__enter__()- self.runner = CrawlerRunner()def tearDown(self):self.mockserver.__exit__(None, None, None)@@ -370,7 +371,7 @@ class CrawlSpiderTestCase(TestCase):def _on_item_scraped(item):items.append(item)- crawler = self.runner.create_crawler(spider_cls)+ crawler = get_crawler(spider_cls)crawler.signals.connect(_on_item_scraped, signals.item_scraped)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)@@ -378,10 +379,9 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_crawlspider_with_parse(self):- self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)-+ crawler = get_crawler(CrawlSpiderWithParseMethod)with LogCapture() as log:- yield self.runner.join()+ yield crawler.crawl(mockserver=self.mockserver)self.assertIn("[parse] status 200 (foo: None)", str(log))self.assertIn("[parse] status 201 (foo: None)", str(log))@@ -389,10 +389,9 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_crawlspider_with_errback(self):- self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)-+ crawler = get_crawler(CrawlSpiderWithErrback)with LogCapture() as log:- yield self.runner.join()+ yield crawler.crawl(mockserver=self.mockserver)self.assertIn("[parse] status 200 (foo: None)", str(log))self.assertIn("[parse] status 201 (foo: None)", str(log))@@ -403,18 +402,19 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_async_def_parse(self):- self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ crawler = get_crawler(AsyncDefSpider)with LogCapture() as log:- yield self.runner.join()+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)self.assertIn("Got response 200", str(log))@mark.only_asyncio()@defer.inlineCallbacksdef test_async_def_asyncio_parse(self):- runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})- runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ crawler = get_crawler(AsyncDefAsyncioSpider, {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"+ })with LogCapture() as log:- yield runner.join()+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)self.assertIn("Got response 200", str(log))@mark.only_asyncio()@@ -433,7 +433,7 @@ class CrawlSpiderTestCase(TestCase):def _on_item_scraped(item):items.append(item)- crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)+ crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)crawler.signals.connect(_on_item_scraped, signals.item_scraped)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)@@ -479,14 +479,14 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_response_ssl_certificate_none(self):- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)url = self.mockserver.url("/echo?body=test", is_secure=False)yield crawler.crawl(seed=url, mockserver=self.mockserver)self.assertIsNone(crawler.spider.meta['responses'][0].certificate)@defer.inlineCallbacksdef test_response_ssl_certificate(self):- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)url = self.mockserver.url("/echo?body=test", is_secure=True)yield crawler.crawl(seed=url, mockserver=self.mockserver)cert = crawler.spider.meta['responses'][0].certificate@@ -497,7 +497,7 @@ class CrawlSpiderTestCase(TestCase):@mark.xfail(reason="Responses with no body return early and contain no certificate")@defer.inlineCallbacksdef test_response_ssl_certificate_empty_response(self):- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)url = self.mockserver.url("/status?n=200", is_secure=True)yield crawler.crawl(seed=url, mockserver=self.mockserver)cert = crawler.spider.meta['responses'][0].certificate@@ -507,7 +507,7 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_dns_server_ip_address_none(self):- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)url = self.mockserver.url('/status?n=200')yield crawler.crawl(seed=url, mockserver=self.mockserver)ip_address = crawler.spider.meta['responses'][0].ip_address@@ -515,7 +515,7 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_dns_server_ip_address(self):- crawler = self.runner.create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)url = self.mockserver.url('/echo?body=test')expected_netloc, _ = urlparse(url).netloc.split(':')yield crawler.crawl(seed=url, mockserver=self.mockserver)@@ -525,7 +525,7 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_bytes_received_stop_download_callback(self):- crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)+ crawler = get_crawler(BytesReceivedCallbackSpider)yield crawler.crawl(mockserver=self.mockserver)self.assertIsNone(crawler.spider.meta.get("failure"))self.assertIsInstance(crawler.spider.meta["response"], Response)@@ -534,7 +534,7 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_bytes_received_stop_download_errback(self):- crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)+ crawler = get_crawler(BytesReceivedErrbackSpider)yield crawler.crawl(mockserver=self.mockserver)self.assertIsNone(crawler.spider.meta.get("response"))self.assertIsInstance(crawler.spider.meta["failure"], Failure)@@ -549,7 +549,7 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_headers_received_stop_download_callback(self):- crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)+ crawler = get_crawler(HeadersReceivedCallbackSpider)yield crawler.crawl(mockserver=self.mockserver)self.assertIsNone(crawler.spider.meta.get("failure"))self.assertIsInstance(crawler.spider.meta["response"], Response)@@ -557,7 +557,7 @@ class CrawlSpiderTestCase(TestCase):@defer.inlineCallbacksdef test_headers_received_stop_download_errback(self):- crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)+ crawler = get_crawler(HeadersReceivedErrbackSpider)yield crawler.crawl(mockserver=self.mockserver)self.assertIsNone(crawler.spider.meta.get("response"))self.assertIsInstance(crawler.spider.meta["failure"], Failure)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex f7aa769e4..d67abed7c 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -13,11 +13,13 @@ from twisted.trial import unittestimport scrapyfrom scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoaderfrom scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.misc import load_object+from scrapy.utils.test import get_crawlerfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.extensions import telnetfrom scrapy.utils.test import get_testenv@@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase):class CrawlerTestCase(BaseCrawlerTest):- def setUp(self):- self.crawler = Crawler(DefaultSpider, Settings())-def test_populate_spidercls_settings(self):spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}project_settings = {'TEST1': 'project', 'TEST3': 'project'}@@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest):settings = Settings()settings.setdict(project_settings, priority='project')- crawler = Crawler(CustomSettingsSpider, settings)+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)+ crawler = Crawler(CustomSettingsSpider, settings)self.assertEqual(crawler.settings.get('TEST1'), 'spider')self.assertEqual(crawler.settings.get('TEST2'), 'spider')@@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest):self.assertTrue(crawler.settings.frozen)def test_crawler_accepts_dict(self):- crawler = Crawler(DefaultSpider, {'foo': 'bar'})+ crawler = get_crawler(DefaultSpider, {'foo': 'bar'})self.assertEqual(crawler.settings['foo'], 'bar')self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')def test_crawler_accepts_None(self):- crawler = Crawler(DefaultSpider)+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)+ crawler = Crawler(DefaultSpider)self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')def test_crawler_rejects_spider_objects(self):@@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase):'AUTOTHROTTLE_ENABLED': True}- crawler = Crawler(MySpider, {})+ crawler = get_crawler(MySpider)enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]self.assertIn(AutoThrottle, enabled_exts)@@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):class MySpider(scrapy.Spider):name = 'spider'- Crawler(MySpider, {})+ get_crawler(MySpider)assert get_scrapy_root_handler() is Nonedef test_spider_custom_settings_log_level(self):@@ -111,7 +114,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):configure_logging()self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)- crawler = Crawler(MySpider, {})+ crawler = get_crawler(MySpider)self.assertEqual(get_scrapy_root_handler().level, logging.INFO)info_count = crawler.stats.get_value('log_count/INFO')logging.debug('debug message')@@ -148,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):}configure_logging()- Crawler(MySpider, {})+ get_crawler(MySpider)logging.debug('debug message')with open(log_file, 'rb') as fo:@@ -229,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider):@mark.usefixtures('reactor_pytest')class CrawlerRunnerHasSpider(unittest.TestCase):+ def _runner(self):+ return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})+@defer.inlineCallbacksdef test_crawler_runner_bootstrap_successful(self):- runner = CrawlerRunner()+ runner = self._runner()yield runner.crawl(NoRequestsSpider)self.assertEqual(runner.bootstrap_failed, False)@defer.inlineCallbacksdef test_crawler_runner_bootstrap_successful_for_several(self):- runner = CrawlerRunner()+ runner = self._runner()yield runner.crawl(NoRequestsSpider)yield runner.crawl(NoRequestsSpider)self.assertEqual(runner.bootstrap_failed, False)@defer.inlineCallbacksdef test_crawler_runner_bootstrap_failed(self):- runner = CrawlerRunner()+ runner = self._runner()try:yield runner.crawl(ExceptionSpider)@@ -257,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):@defer.inlineCallbacksdef test_crawler_runner_bootstrap_failed_for_several(self):- runner = CrawlerRunner()+ runner = self._runner()try:yield runner.crawl(ExceptionSpider)@@ -275,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):if self.reactor_pytest == 'asyncio':CrawlerRunner(settings={"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",})else:msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"with self.assertRaisesRegex(Exception, msg):runner = CrawlerRunner(settings={"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",})yield runner.crawl(NoRequestsSpider)diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex 0362e2018..b9f3e24a4 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -1,7 +1,9 @@import unittest+import pytestfrom w3lib.http import basic_auth_header+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.downloadermiddlewares.httpauth import HttpAuthMiddlewarefrom scrapy.spiders import Spider@@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):self.spider = TestSpiderLegacy('foo')def test_auth(self):- mw = HttpAuthMiddleware()- mw.spider_opened(self.spider)+ with pytest.warns(ScrapyDeprecationWarning,+ match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):+ mw = HttpAuthMiddleware()+ mw.spider_opened(self.spider)# initial request, sets the domain and sends the headerreq = Request('http://example.com/')@@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):self.assertNotIn('Authorization', req.headers)def test_auth_already_set(self):- mw = HttpAuthMiddleware()- mw.spider_opened(self.spider)+ with pytest.warns(ScrapyDeprecationWarning,+ match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):+ mw = HttpAuthMiddleware()+ mw.spider_opened(self.spider)req = Request('http://example.com/',headers=dict(Authorization='Digest 123'))assert mw.process_request(req, self.spider) is Nonediff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 7c97bf32a..4ac85c1ec 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -1,13 +1,13 @@import os-from functools import partial++import pytestfrom twisted.trial.unittest import TestCasefrom scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddlewarefrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Requestfrom scrapy.spiders import Spider-from scrapy.crawler import Crawler-from scrapy.settings import Settings+from scrapy.utils.test import get_crawlerspider = Spider('foo')@@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase):os.environ = self._oldenvdef test_not_enabled(self):- settings = Settings({'HTTPPROXY_ENABLED': False})- crawler = Crawler(Spider, settings)- self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))+ crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})+ with pytest.raises(NotConfigured):+ HttpProxyMiddleware.from_crawler(crawler)def test_no_environment_proxies(self):os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 9e75f0a50..7d88ba4d2 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -1,7 +1,9 @@+import warningsfrom itertools import productfrom unittest import TestCasefrom scrapy.downloadermiddlewares.stats import DownloaderStats+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.response import response_httprepr@@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase):for test_response in test_responses:self.crawler.stats.set_value('downloader/response_bytes', 0)self.mw.process_response(self.req, test_response, self.spider)- self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)+ resp_size = len(response_httprepr(test_response))+ self.assertStatsEqual('downloader/response_bytes', resp_size)def test_process_exception(self):self.mw.process_exception(self.req, MyException(), self.spider)diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex b7df2554a..8a37a8ebe 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -10,7 +10,6 @@ from scrapy.dupefilters import RFPDupeFilterfrom scrapy.http import Requestfrom scrapy.core.scheduler import Schedulerfrom scrapy.utils.python import to_bytes-from scrapy.utils.job import job_dirfrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider@@ -29,8 +28,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):@classmethoddef from_crawler(cls, crawler):- debug = crawler.settings.getbool('DUPEFILTER_DEBUG')- df = cls(job_dir(crawler.settings), debug)+ df = super().from_crawler(crawler)df.method = 'from_crawler'return df@@ -38,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):class FromSettingsRFPDupeFilter(RFPDupeFilter):@classmethod- def from_settings(cls, settings):- debug = settings.getbool('DUPEFILTER_DEBUG')- df = cls(job_dir(settings), debug)+ def from_settings(cls, settings, *, fingerprinter=None):+ df = super().from_settings(settings, fingerprinter=fingerprinter)df.method = 'from_settings'return df@@ -53,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase):def test_df_from_crawler_scheduler(self):settings = {'DUPEFILTER_DEBUG': True,- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertTrue(scheduler.df.debug)@@ -61,14 +59,16 @@ class RFPDupeFilterTest(unittest.TestCase):def test_df_from_settings_scheduler(self):settings = {'DUPEFILTER_DEBUG': True,- 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}+ 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertTrue(scheduler.df.debug)self.assertEqual(scheduler.df.method, 'from_settings')def test_df_direct_scheduler(self):- settings = {'DUPEFILTER_CLASS': DirectDupeFilter}+ settings = {'DUPEFILTER_CLASS': DirectDupeFilter,+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertEqual(scheduler.df.method, 'n/a')@@ -171,7 +171,8 @@ class RFPDupeFilterTest(unittest.TestCase):def test_log(self):with LogCapture() as log:settings = {'DUPEFILTER_DEBUG': False,- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}crawler = get_crawler(SimpleSpider, settings_dict=settings)spider = SimpleSpider.from_crawler(crawler)dupefilter = _get_dupefilter(crawler=crawler)@@ -197,7 +198,8 @@ class RFPDupeFilterTest(unittest.TestCase):def test_log_debug(self):with LogCapture() as log:settings = {'DUPEFILTER_DEBUG': True,- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}crawler = get_crawler(SimpleSpider, settings_dict=settings)spider = SimpleSpider.from_crawler(crawler)dupefilter = _get_dupefilter(crawler=crawler)@@ -230,7 +232,8 @@ class RFPDupeFilterTest(unittest.TestCase):def test_log_debug_default_dupefilter(self):with LogCapture() as log:- settings = {'DUPEFILTER_DEBUG': True}+ settings = {'DUPEFILTER_DEBUG': True,+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}crawler = get_crawler(SimpleSpider, settings_dict=settings)spider = SimpleSpider.from_crawler(crawler)dupefilter = _get_dupefilter(crawler=crawler)diff --git a/tests/test_engine.py b/tests/test_engine.pyindex fa7d0c8d4..1bd802bcf 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -13,10 +13,11 @@ module with the ``runserver`` argument::import osimport reimport sys-import warningsfrom collections import defaultdictfrom urllib.parse import urlparse+from dataclasses import dataclass+import pytestimport attrfrom itemadapter import ItemAdapterfrom pydispatch import dispatcher@@ -50,6 +51,13 @@ class AttrsItem:price = attr.ib(default=0)+@dataclass+class DataClassItem:+ name: str = ""+ url: str = ""+ price: int = 0++class TestSpider(Spider):name = "scrapytest.org"allowed_domains = ["scrapytest.org", "localhost"]@@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider):item_cls = AttrsItem-try:- from dataclasses import make_dataclass-except ImportError:- DataClassItemsSpider = None-else:- TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])-- class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef]- def parse_item(self, response):- item = super().parse_item(response)- return TestDataClass(**item)+class DataClassItemsSpider(TestSpider):+ item_cls = DataClassItemclass ItemZeroDivisionErrorSpider(TestSpider):@@ -188,7 +187,7 @@ class CrawlerRun:return self.deferreddef stop(self):- self.port.stopListening()+ self.port.stopListening() # FIXME: wait for this Deferredfor name, signal in vars(signals).items():if not name.startswith('_'):disconnect_all(signal)@@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase):def test_crawler(self):for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):- if spider is None:- continue- self.run = CrawlerRun(spider)- yield self.run.run()- self._assert_visited_urls()- self._assert_scheduled_requests(count=9)- self._assert_downloaded_responses(count=9)- self._assert_scraped_items()- self._assert_signals_caught()- self._assert_bytes_received()+ run = CrawlerRun(spider)+ yield run.run()+ self._assert_visited_urls(run)+ self._assert_scheduled_requests(run, count=9)+ self._assert_downloaded_responses(run, count=9)+ self._assert_scraped_items(run)+ self._assert_signals_caught(run)+ self._assert_bytes_received(run)@defer.inlineCallbacksdef test_crawler_dupefilter(self):- self.run = CrawlerRun(TestDupeFilterSpider)- yield self.run.run()- self._assert_scheduled_requests(count=8)- self._assert_dropped_requests()+ run = CrawlerRun(TestDupeFilterSpider)+ yield run.run()+ self._assert_scheduled_requests(run, count=8)+ self._assert_dropped_requests(run)@defer.inlineCallbacksdef test_crawler_itemerror(self):- self.run = CrawlerRun(ItemZeroDivisionErrorSpider)- yield self.run.run()- self._assert_items_error()+ run = CrawlerRun(ItemZeroDivisionErrorSpider)+ yield run.run()+ self._assert_items_error(run)@defer.inlineCallbacksdef test_crawler_change_close_reason_on_idle(self):- self.run = CrawlerRun(ChangeCloseReasonSpider)- yield self.run.run()- self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},- self.run.signals_caught[signals.spider_closed])+ run = CrawlerRun(ChangeCloseReasonSpider)+ yield run.run()+ self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},+ run.signals_caught[signals.spider_closed])- def _assert_visited_urls(self):+ def _assert_visited_urls(self, run: CrawlerRun):must_be_visited = ["/", "/redirect", "/redirected","/item1.html", "/item2.html", "/item999.html"]- urls_visited = {rp[0].url for rp in self.run.respplug}- urls_expected = {self.run.geturl(p) for p in must_be_visited}+ urls_visited = {rp[0].url for rp in run.respplug}+ urls_expected = {run.geturl(p) for p in must_be_visited}assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"- def _assert_scheduled_requests(self, count=None):- self.assertEqual(count, len(self.run.reqplug))+ def _assert_scheduled_requests(self, run: CrawlerRun, count=None):+ self.assertEqual(count, len(run.reqplug))paths_expected = ['/item999.html', '/item2.html', '/item1.html']- urls_requested = {rq[0].url for rq in self.run.reqplug}- urls_expected = {self.run.geturl(p) for p in paths_expected}+ urls_requested = {rq[0].url for rq in run.reqplug}+ urls_expected = {run.geturl(p) for p in paths_expected}assert urls_expected <= urls_requested- scheduled_requests_count = len(self.run.reqplug)- dropped_requests_count = len(self.run.reqdropped)- responses_count = len(self.run.respplug)+ scheduled_requests_count = len(run.reqplug)+ dropped_requests_count = len(run.reqdropped)+ responses_count = len(run.respplug)self.assertEqual(scheduled_requests_count,dropped_requests_count + responses_count)- self.assertEqual(len(self.run.reqreached),+ self.assertEqual(len(run.reqreached),responses_count)- def _assert_dropped_requests(self):- self.assertEqual(len(self.run.reqdropped), 1)+ def _assert_dropped_requests(self, run: CrawlerRun):+ self.assertEqual(len(run.reqdropped), 1)- def _assert_downloaded_responses(self, count):+ def _assert_downloaded_responses(self, run: CrawlerRun, count):# response tests- self.assertEqual(count, len(self.run.respplug))- self.assertEqual(count, len(self.run.reqreached))+ self.assertEqual(count, len(run.respplug))+ self.assertEqual(count, len(run.reqreached))- for response, _ in self.run.respplug:- if self.run.getpath(response.url) == '/item999.html':+ for response, _ in run.respplug:+ if run.getpath(response.url) == '/item999.html':self.assertEqual(404, response.status)- if self.run.getpath(response.url) == '/redirect':+ if run.getpath(response.url) == '/redirect':self.assertEqual(302, response.status)- def _assert_items_error(self):- self.assertEqual(2, len(self.run.itemerror))- for item, response, spider, failure in self.run.itemerror:+ def _assert_items_error(self, run: CrawlerRun):+ self.assertEqual(2, len(run.itemerror))+ for item, response, spider, failure in run.itemerror:self.assertEqual(failure.value.__class__, ZeroDivisionError)- self.assertEqual(spider, self.run.spider)+ self.assertEqual(spider, run.spider)self.assertEqual(item['url'], response.url)if 'item1.html' in item['url']:@@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase):self.assertEqual('Item 2 name', item['name'])self.assertEqual('200', item['price'])- def _assert_scraped_items(self):- self.assertEqual(2, len(self.run.itemresp))- for item, response in self.run.itemresp:+ def _assert_scraped_items(self, run: CrawlerRun):+ self.assertEqual(2, len(run.itemresp))+ for item, response in run.itemresp:item = ItemAdapter(item)self.assertEqual(item['url'], response.url)if 'item1.html' in item['url']:@@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase):self.assertEqual('Item 2 name', item['name'])self.assertEqual('200', item['price'])- def _assert_headers_received(self):- for headers in self.run.headers.values():+ def _assert_headers_received(self, run: CrawlerRun):+ for headers in run.headers.values():self.assertIn(b"Server", headers)self.assertIn(b"TwistedWeb", headers[b"Server"])self.assertIn(b"Date", headers)self.assertIn(b"Content-Type", headers)- def _assert_bytes_received(self):- self.assertEqual(9, len(self.run.bytes))- for request, data in self.run.bytes.items():+ def _assert_bytes_received(self, run: CrawlerRun):+ self.assertEqual(9, len(run.bytes))+ for request, data in run.bytes.items():joined_data = b"".join(data)- if self.run.getpath(request.url) == "/":+ if run.getpath(request.url) == "/":self.assertEqual(joined_data, get_testdata("test_site", "index.html"))- elif self.run.getpath(request.url) == "/item1.html":+ elif run.getpath(request.url) == "/item1.html":self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))- elif self.run.getpath(request.url) == "/item2.html":+ elif run.getpath(request.url) == "/item2.html":self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))- elif self.run.getpath(request.url) == "/redirected":+ elif run.getpath(request.url) == "/redirected":self.assertEqual(joined_data, b"Redirected here")- elif self.run.getpath(request.url) == '/redirect':+ elif run.getpath(request.url) == '/redirect':self.assertEqual(joined_data,b"\n<html>\n"@@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase):b" </body>\n"b"</html>\n")- elif self.run.getpath(request.url) == "/tem999.html":+ elif run.getpath(request.url) == "/tem999.html":self.assertEqual(joined_data,b"\n<html>\n"@@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase):b" </body>\n"b"</html>\n")- elif self.run.getpath(request.url) == "/numbers":+ elif run.getpath(request.url) == "/numbers":# signal was fired multiple timesself.assertTrue(len(data) > 1)# bytes were received in ordernumbers = [str(x).encode("utf8") for x in range(2**18)]self.assertEqual(joined_data, b"".join(numbers))- def _assert_signals_caught(self):- assert signals.engine_started in self.run.signals_caught- assert signals.engine_stopped in self.run.signals_caught- assert signals.spider_opened in self.run.signals_caught- assert signals.spider_idle in self.run.signals_caught- assert signals.spider_closed in self.run.signals_caught- assert signals.headers_received in self.run.signals_caught-- self.assertEqual({'spider': self.run.spider},- self.run.signals_caught[signals.spider_opened])- self.assertEqual({'spider': self.run.spider},- self.run.signals_caught[signals.spider_idle])- self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},- self.run.signals_caught[signals.spider_closed])+ def _assert_signals_caught(self, run: CrawlerRun):+ assert signals.engine_started in run.signals_caught+ assert signals.engine_stopped in run.signals_caught+ assert signals.spider_opened in run.signals_caught+ assert signals.spider_idle in run.signals_caught+ assert signals.spider_closed in run.signals_caught+ assert signals.headers_received in run.signals_caught++ self.assertEqual({'spider': run.spider},+ run.signals_caught[signals.spider_opened])+ self.assertEqual({'spider': run.spider},+ run.signals_caught[signals.spider_idle])+ self.assertEqual({'spider': run.spider, 'reason': 'finished'},+ run.signals_caught[signals.spider_closed])@defer.inlineCallbacksdef test_close_downloader(self):@@ -407,28 +404,29 @@ class EngineTest(unittest.TestCase):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)yield e.open_spider(TestSpider(), [])e.start()- yield self.assertFailure(e.start(), RuntimeError).addBoth(- lambda exc: self.assertEqual(str(exc), "Engine already running")- )- yield e.stop()+ try:+ yield self.assertFailure(e.start(), RuntimeError).addBoth(+ lambda exc: self.assertEqual(str(exc), "Engine already running")+ )+ finally:+ yield e.stop()@defer.inlineCallbacksdef test_close_spiders_downloader(self):- with warnings.catch_warnings(record=True) as warning_list:+ with pytest.warns(ScrapyDeprecationWarning,+ match="ExecutionEngine.open_spiders is deprecated, "+ "please use ExecutionEngine.spider instead"):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)yield e.open_spider(TestSpider(), [])self.assertEqual(len(e.open_spiders), 1)yield e.close()self.assertEqual(len(e.open_spiders), 0)- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)- self.assertEqual(- str(warning_list[0].message),- "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",- )@defer.inlineCallbacksdef test_close_engine_spiders_downloader(self):- with warnings.catch_warnings(record=True) as warning_list:+ with pytest.warns(ScrapyDeprecationWarning,+ match="ExecutionEngine.open_spiders is deprecated, "+ "please use ExecutionEngine.spider instead"):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)yield e.open_spider(TestSpider(), [])e.start()@@ -436,61 +434,47 @@ class EngineTest(unittest.TestCase):yield e.close()self.assertFalse(e.running)self.assertEqual(len(e.open_spiders), 0)- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)- self.assertEqual(- str(warning_list[0].message),- "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",- )@defer.inlineCallbacksdef test_crawl_deprecated_spider_arg(self):- with warnings.catch_warnings(record=True) as warning_list:+ with pytest.warns(ScrapyDeprecationWarning,+ match="Passing a 'spider' argument to "+ "ExecutionEngine.crawl is deprecated"):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)spider = TestSpider()yield e.open_spider(spider, [])e.start()e.crawl(Request("data:,"), spider)yield e.close()- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)- self.assertEqual(- str(warning_list[0].message),- "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",- )@defer.inlineCallbacksdef test_download_deprecated_spider_arg(self):- with warnings.catch_warnings(record=True) as warning_list:+ with pytest.warns(ScrapyDeprecationWarning,+ match="Passing a 'spider' argument to "+ "ExecutionEngine.download is deprecated"):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)spider = TestSpider()yield e.open_spider(spider, [])e.start()e.download(Request("data:,"), spider)yield e.close()- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)- self.assertEqual(- str(warning_list[0].message),- "Passing a 'spider' argument to ExecutionEngine.download is deprecated",- )@defer.inlineCallbacksdef test_deprecated_schedule(self):- with warnings.catch_warnings(record=True) as warning_list:+ with pytest.warns(ScrapyDeprecationWarning,+ match="ExecutionEngine.schedule is deprecated, please use "+ "ExecutionEngine.crawl or ExecutionEngine.download instead"):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)spider = TestSpider()yield e.open_spider(spider, [])e.start()e.schedule(Request("data:,"), spider)yield e.close()- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)- self.assertEqual(- str(warning_list[0].message),- "ExecutionEngine.schedule is deprecated, please use "- "ExecutionEngine.crawl or ExecutionEngine.download instead",- )@defer.inlineCallbacksdef test_deprecated_has_capacity(self):- with warnings.catch_warnings(record=True) as warning_list:+ with pytest.warns(ScrapyDeprecationWarning,+ match="ExecutionEngine.has_capacity is deprecated"):e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)self.assertTrue(e.has_capacity())spider = TestSpider()@@ -499,8 +483,6 @@ class EngineTest(unittest.TestCase):e.start()yield e.close()self.assertTrue(e.has_capacity())- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)- self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")if __name__ == "__main__":diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.pyindex 0ba69e096..933e4067d 100644--- a/tests/test_engine_stop_download_bytes.py+++ b/tests/test_engine_stop_download_bytes.py@@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest):@defer.inlineCallbacksdef test_crawler(self):for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):- if spider is None:- continue- self.run = BytesReceivedCrawlerRun(spider)+ run = BytesReceivedCrawlerRun(spider)with LogCapture() as log:- yield self.run.run()+ yield run.run()log.check_present(("scrapy.core.downloader.handlers.http11","DEBUG",- f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "+ f"Download stopped for <GET http://localhost:{run.portno}/redirected> ""from signal handler BytesReceivedCrawlerRun.bytes_received"))log.check_present(("scrapy.core.downloader.handlers.http11","DEBUG",- f"Download stopped for <GET http://localhost:{self.run.portno}/> "+ f"Download stopped for <GET http://localhost:{run.portno}/> ""from signal handler BytesReceivedCrawlerRun.bytes_received"))log.check_present(("scrapy.core.downloader.handlers.http11","DEBUG",- f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "+ f"Download stopped for <GET http://localhost:{run.portno}/numbers> ""from signal handler BytesReceivedCrawlerRun.bytes_received"))- self._assert_visited_urls()- self._assert_scheduled_requests(count=9)- self._assert_downloaded_responses(count=9)- self._assert_signals_caught()- self._assert_headers_received()- self._assert_bytes_received()+ self._assert_visited_urls(run)+ self._assert_scheduled_requests(run, count=9)+ self._assert_downloaded_responses(run, count=9)+ self._assert_signals_caught(run)+ self._assert_headers_received(run)+ self._assert_bytes_received(run)- def _assert_bytes_received(self):- self.assertEqual(9, len(self.run.bytes))- for request, data in self.run.bytes.items():+ def _assert_bytes_received(self, run: CrawlerRun):+ self.assertEqual(9, len(run.bytes))+ for request, data in run.bytes.items():joined_data = b"".join(data)self.assertTrue(len(data) == 1) # signal was fired only once- if self.run.getpath(request.url) == "/numbers":+ if run.getpath(request.url) == "/numbers":# Received bytes are not the complete response. The exact amount depends# on the buffer size, which can vary, so we only check that the amount# of received bytes is strictly less than the full response.diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.pyindex fad6643ad..8975d0e3f 100644--- a/tests/test_engine_stop_download_headers.py+++ b/tests/test_engine_stop_download_headers.py@@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest):@defer.inlineCallbacksdef test_crawler(self):for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):- if spider is None:- continue- self.run = HeadersReceivedCrawlerRun(spider)+ run = HeadersReceivedCrawlerRun(spider)with LogCapture() as log:- yield self.run.run()+ yield run.run()log.check_present(("scrapy.core.downloader.handlers.http11","DEBUG",- f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"+ f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"" signal handler HeadersReceivedCrawlerRun.headers_received"))log.check_present(("scrapy.core.downloader.handlers.http11","DEBUG",- f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"+ f"Download stopped for <GET http://localhost:{run.portno}/> from signal"" handler HeadersReceivedCrawlerRun.headers_received"))log.check_present(("scrapy.core.downloader.handlers.http11","DEBUG",- f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"+ f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"" signal handler HeadersReceivedCrawlerRun.headers_received"))- self._assert_visited_urls()- self._assert_downloaded_responses(count=6)- self._assert_signals_caught()- self._assert_bytes_received()- self._assert_headers_received()+ self._assert_visited_urls(run)+ self._assert_downloaded_responses(run, count=6)+ self._assert_signals_caught(run)+ self._assert_bytes_received(run)+ self._assert_headers_received(run)- def _assert_bytes_received(self):- self.assertEqual(0, len(self.run.bytes))+ def _assert_bytes_received(self, run: CrawlerRun):+ self.assertEqual(0, len(run.bytes))- def _assert_visited_urls(self):+ def _assert_visited_urls(self, run: CrawlerRun):must_be_visited = ["/", "/redirect", "/redirected"]- urls_visited = {rp[0].url for rp in self.run.respplug}- urls_expected = {self.run.geturl(p) for p in must_be_visited}+ urls_visited = {rp[0].url for rp in run.respplug}+ urls_expected = {run.geturl(p) for p in must_be_visited}assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex ec48f8d4a..ecd1b59d3 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -22,6 +22,7 @@ from urllib.parse import urljoin, quotefrom urllib.request import pathname2urlimport lxml.etree+import pytestfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial import unittest@@ -30,7 +31,6 @@ from zope.interface import implementerfrom zope.interface.verify import verifyObjectimport scrapy-from scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.exporters import CsvItemExporterfrom scrapy.extensions.feedexport import (@@ -697,9 +697,9 @@ class FeedExportTest(FeedExportTestBase):content = {}try:with MockServer() as s:- runner = CrawlerRunner(Settings(settings))spider_cls.start_urls = [s.url('/')]- yield runner.crawl(spider_cls)+ crawler = get_crawler(spider_cls, settings)+ yield crawler.crawl()for file_path, feed_options in FEEDS.items():if not os.path.exists(str(file_path)):@@ -1554,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):content = {}try:with MockServer() as s:- runner = CrawlerRunner(Settings(settings))spider_cls.start_urls = [s.url('/')]- yield runner.crawl(spider_cls)+ crawler = get_crawler(spider_cls, settings)+ yield crawler.crawl()for file_path, feed_options in FEEDS.items():if not os.path.exists(str(file_path)):@@ -2026,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):content = defaultdict(list)try:with MockServer() as s:- runner = CrawlerRunner(Settings(settings))spider_cls.start_urls = [s.url('/')]- yield runner.crawl(spider_cls)+ crawler = get_crawler(spider_cls, settings)+ yield crawler.crawl()for path, feed in FEEDS.items():dir_name = os.path.dirname(path)@@ -2048,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},},})- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)for batch in data['jl']:@@ -2064,7 +2064,7 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},},})- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')data = yield self.exported_data(items, settings)for batch in data['csv']:got_batch = csv.DictReader(to_unicode(batch).splitlines())@@ -2080,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},},})- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)for batch in data['xml']:@@ -2098,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},},})- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)# XML@@ -2123,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},},})- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)import pickle@@ -2140,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},},})- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)import marshal@@ -2166,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase):'FEED_EXPORT_BATCH_ITEM_COUNT': 2}header = self.MyItem.fields.keys()- yield self.assertExported(items, header, rows, settings=Settings(settings))+ yield self.assertExported(items, header, rows, settings=settings)def test_wrong_path(self):""" If path is without %(batch_time)s and %(batch_id) an exception must be raised """@@ -2382,9 +2382,9 @@ class BatchDeliveriesTest(FeedExportTestBase):yield itemwith MockServer() as server:- runner = CrawlerRunner(Settings(settings))TestSpider.start_urls = [server.url('/')]- yield runner.crawl(TestSpider)+ crawler = get_crawler(TestSpider, settings)+ yield crawler.crawl()self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)for stub in CustomS3FeedStorage.stubs[:-1]:@@ -2434,25 +2434,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):'file': StdoutFeedStorageWithoutFeedOptions},}- crawler = get_crawler(settings_dict=settings_dict)- feed_exporter = FeedExporter.from_crawler(crawler)+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+spider = scrapy.Spider("default")- with warnings.catch_warnings(record=True) as w:+ with pytest.warns(ScrapyDeprecationWarning,+ match="StdoutFeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument."):feed_exporter.open_spider(spider)- messages = tuple(str(item.message) for item in w- if item.category is ScrapyDeprecationWarning)- self.assertEqual(- messages,- (- (- "StdoutFeedStorageWithoutFeedOptions does not support "- "the 'feed_options' keyword argument. Add a "- "'feed_options' parameter to its signature to remove "- "this warning. This parameter will become mandatory "- "in a future version of Scrapy."- ),- )- )class FileFeedStorageWithoutFeedOptions(FileFeedStorage):@@ -2476,25 +2467,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):'file': FileFeedStorageWithoutFeedOptions},}- crawler = get_crawler(settings_dict=settings_dict)- feed_exporter = FeedExporter.from_crawler(crawler)+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)spider = scrapy.Spider("default")- with warnings.catch_warnings(record=True) as w:++ with pytest.warns(ScrapyDeprecationWarning,+ match="FileFeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument."):feed_exporter.open_spider(spider)- messages = tuple(str(item.message) for item in w- if item.category is ScrapyDeprecationWarning)- self.assertEqual(- messages,- (- (- "FileFeedStorageWithoutFeedOptions does not support "- "the 'feed_options' keyword argument. Add a "- "'feed_options' parameter to its signature to remove "- "this warning. This parameter will become mandatory "- "in a future version of Scrapy."- ),- )- )class S3FeedStorageWithoutFeedOptions(S3FeedStorage):@@ -2524,26 +2506,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):'file': S3FeedStorageWithoutFeedOptions},}- crawler = get_crawler(settings_dict=settings_dict)- feed_exporter = FeedExporter.from_crawler(crawler)+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+spider = scrapy.Spider("default")spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with pytest.warns(ScrapyDeprecationWarning,+ match="S3FeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument."):feed_exporter.open_spider(spider)- messages = tuple(str(item.message) for item in w- if item.category is ScrapyDeprecationWarning)- self.assertEqual(- messages,- (- (- "S3FeedStorageWithoutFeedOptions does not support "- "the 'feed_options' keyword argument. Add a "- "'feed_options' parameter to its signature to remove "- "this warning. This parameter will become mandatory "- "in a future version of Scrapy."- ),- )- )def test_from_crawler(self):settings_dict = {@@ -2552,26 +2526,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler},}- crawler = get_crawler(settings_dict=settings_dict)- feed_exporter = FeedExporter.from_crawler(crawler)+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+spider = scrapy.Spider("default")spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with pytest.warns(ScrapyDeprecationWarning,+ match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "+ "the 'feed_options' keyword argument."):feed_exporter.open_spider(spider)- messages = tuple(str(item.message) for item in w- if item.category is ScrapyDeprecationWarning)- self.assertEqual(- messages,- (- (- "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "- "does not support the 'feed_options' keyword argument. Add a "- "'feed_options' parameter to its signature to remove "- "this warning. This parameter will become mandatory "- "in a future version of Scrapy."- ),- )- )class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):@@ -2601,26 +2567,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):'file': FTPFeedStorageWithoutFeedOptions},}- crawler = get_crawler(settings_dict=settings_dict)- feed_exporter = FeedExporter.from_crawler(crawler)+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+spider = scrapy.Spider("default")spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with pytest.warns(ScrapyDeprecationWarning,+ match="FTPFeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument."):feed_exporter.open_spider(spider)- messages = tuple(str(item.message) for item in w- if item.category is ScrapyDeprecationWarning)- self.assertEqual(- messages,- (- (- "FTPFeedStorageWithoutFeedOptions does not support "- "the 'feed_options' keyword argument. Add a "- "'feed_options' parameter to its signature to remove "- "this warning. This parameter will become mandatory "- "in a future version of Scrapy."- ),- )- )def test_from_crawler(self):settings_dict = {@@ -2629,50 +2587,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler},}- crawler = get_crawler(settings_dict=settings_dict)- feed_exporter = FeedExporter.from_crawler(crawler)+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+spider = scrapy.Spider("default")spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with pytest.warns(ScrapyDeprecationWarning,+ match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "+ "the 'feed_options' keyword argument."):feed_exporter.open_spider(spider)- messages = tuple(str(item.message) for item in w- if item.category is ScrapyDeprecationWarning)- self.assertEqual(- messages,- (- (- "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "- "does not support the 'feed_options' keyword argument. Add a "- "'feed_options' parameter to its signature to remove "- "this warning. This parameter will become mandatory "- "in a future version of Scrapy."- ),- )- )class URIParamsTest:spider_name = "uri_params_spider"+ deprecated_options = Falsedef build_settings(self, uri='file:///tmp/foobar', uri_params=None):raise NotImplementedError+ def _crawler_feed_exporter(self, settings):+ if self.deprecated_options:+ with pytest.warns(ScrapyDeprecationWarning,+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):+ crawler = get_crawler(settings_dict=settings)+ feed_exporter = FeedExporter.from_crawler(crawler)+ else:+ crawler = get_crawler(settings_dict=settings)+ feed_exporter = FeedExporter.from_crawler(crawler)+ return crawler, feed_exporter+def test_default(self):settings = self.build_settings(uri='file:///tmp/%(name)s',)- crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)+ crawler, feed_exporter = self._crawler_feed_exporter(settings)spider = scrapy.Spider(self.spider_name)spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with warnings.catch_warnings():+ warnings.simplefilter("error", ScrapyDeprecationWarning)feed_exporter.open_spider(spider)- messages = tuple(- str(item.message) for item in w- if item.category is ScrapyDeprecationWarning- )- self.assertEqual(messages, tuple())self.assertEqual(feed_exporter.slots[0].uri,@@ -2687,28 +2645,13 @@ class URIParamsTest:uri='file:///tmp/%(name)s',uri_params=uri_params,)- crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)+ crawler, feed_exporter = self._crawler_feed_exporter(settings)spider = scrapy.Spider(self.spider_name)spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with pytest.warns(ScrapyDeprecationWarning,+ match="Modifying the params dictionary in-place"):feed_exporter.open_spider(spider)- messages = tuple(- str(item.message) for item in w- if item.category is ScrapyDeprecationWarning- )- self.assertEqual(- messages,- (- (- 'Modifying the params dictionary in-place in the '- 'function defined in the FEED_URI_PARAMS setting or '- 'in the uri_params key of the FEEDS setting is '- 'deprecated. The function must return a new '- 'dictionary instead.'- ),- )- )self.assertEqual(feed_exporter.slots[0].uri,@@ -2723,18 +2666,14 @@ class URIParamsTest:uri='file:///tmp/%(name)s',uri_params=uri_params,)- crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)+ crawler, feed_exporter = self._crawler_feed_exporter(settings)spider = scrapy.Spider(self.spider_name)spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:++ with warnings.catch_warnings():+ warnings.simplefilter("error", ScrapyDeprecationWarning)with self.assertRaises(KeyError):feed_exporter.open_spider(spider)- messages = tuple(- str(item.message) for item in w- if item.category is ScrapyDeprecationWarning- )- self.assertEqual(messages, tuple())def test_params_as_is(self):def uri_params(params, spider):@@ -2744,17 +2683,12 @@ class URIParamsTest:uri='file:///tmp/%(name)s',uri_params=uri_params,)- crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)+ crawler, feed_exporter = self._crawler_feed_exporter(settings)spider = scrapy.Spider(self.spider_name)spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:+ with warnings.catch_warnings():+ warnings.simplefilter("error", ScrapyDeprecationWarning)feed_exporter.open_spider(spider)- messages = tuple(- str(item.message) for item in w- if item.category is ScrapyDeprecationWarning- )- self.assertEqual(messages, tuple())self.assertEqual(feed_exporter.slots[0].uri,@@ -2769,17 +2703,12 @@ class URIParamsTest:uri='file:///tmp/%(foo)s',uri_params=uri_params,)- crawler = get_crawler(settings_dict=settings)- feed_exporter = FeedExporter.from_crawler(crawler)+ crawler, feed_exporter = self._crawler_feed_exporter(settings)spider = scrapy.Spider(self.spider_name)spider.crawler = crawler- with warnings.catch_warnings(record=True) as w:+ with warnings.catch_warnings():+ warnings.simplefilter("error", ScrapyDeprecationWarning)feed_exporter.open_spider(spider)- messages = tuple(- str(item.message) for item in w- if item.category is ScrapyDeprecationWarning- )- self.assertEqual(messages, tuple())self.assertEqual(feed_exporter.slots[0].uri,@@ -2788,6 +2717,7 @@ class URIParamsTest:class URIParamsSettingTest(URIParamsTest, unittest.TestCase):+ deprecated_options = Truedef build_settings(self, uri='file:///tmp/foobar', uri_params=None):extra_settings = {}@@ -2800,6 +2730,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase):class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):+ deprecated_options = Falsedef build_settings(self, uri='file:///tmp/foobar', uri_params=None):options = {diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 6381f895b..f3bb23bda 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -5,8 +5,8 @@ from twisted.internet import deferfrom twisted.python.failure import Failurefrom twisted.trial.unittest import TestCase as TwistedTestCase-from scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import DropItem+from scrapy.utils.test import get_crawlerfrom scrapy.http import Request, Responsefrom scrapy.item import Item, Fieldfrom scrapy.logformatter import LogFormatter@@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):@defer.inlineCallbacksdef test_show_messages(self):- crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)+ crawler = get_crawler(ItemSpider, self.base_settings)with LogCapture() as lc:yield crawler.crawl(mockserver=self.mockserver)self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))@@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):def test_skip_messages(self):settings = self.base_settings.copy()settings['LOG_FORMATTER'] = SkipMessagesLogFormatter- crawler = CrawlerRunner(settings).create_crawler(ItemSpider)+ crawler = get_crawler(ItemSpider, settings)with LogCapture() as lc:yield crawler.crawl(mockserver=self.mockserver)self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex f49fda701..e46532a1c 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):self.tmpmediastore = self.mktemp()os.mkdir(self.tmpmediastore)self.settings = {+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION','ITEM_PIPELINES': {self.pipeline_class: 1},self.store_setting_key: self.tmpmediastore,}@@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase):def _on_item_scraped(self, item):self.items.append(item)- def _create_crawler(self, spider_class, **kwargs):- crawler = self.runner.create_crawler(spider_class, **kwargs)+ def _create_crawler(self, spider_class, runner=None, **kwargs):+ if runner is None:+ runner = self.runner+ crawler = runner.create_crawler(spider_class, **kwargs)crawler.signals.connect(self._on_item_scraped, signals.item_scraped)return crawler@@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase):def test_download_media_redirected_allowed(self):settings = dict(self.settings)settings.update({'MEDIA_ALLOW_REDIRECTS': True})- self.runner = CrawlerRunner(settings)-- crawler = self._create_crawler(RedirectedMediaDownloadSpider)+ runner = CrawlerRunner(settings)+ crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/files/images/"),diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex 25d9657d5..0406d906f 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -2,8 +2,8 @@ from twisted.internet import deferfrom twisted.trial.unittest import TestCasefrom scrapy import Request, signals-from scrapy.crawler import CrawlerRunnerfrom scrapy.http.response import Response+from scrapy.utils.test import get_crawlerfrom testfixtures import LogCapture@@ -71,7 +71,7 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_response_200(self):url = self.mockserver.url("/status?n=200")- crawler = CrawlerRunner().create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)yield crawler.crawl(seed=url, mockserver=self.mockserver)response = crawler.spider.meta["responses"][0]self.assertEqual(response.request.url, url)@@ -80,7 +80,7 @@ class CrawlTestCase(TestCase):def test_response_error(self):for status in ("404", "500"):url = self.mockserver.url(f"/status?n={status}")- crawler = CrawlerRunner().create_crawler(SingleRequestSpider)+ crawler = get_crawler(SingleRequestSpider)yield crawler.crawl(seed=url, mockserver=self.mockserver)failure = crawler.spider.meta["failure"]response = failure.value.response@@ -90,12 +90,11 @@ class CrawlTestCase(TestCase):@defer.inlineCallbacksdef test_downloader_middleware_raise_exception(self):url = self.mockserver.url("/status?n=200")- runner = CrawlerRunner(settings={+ crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {RaiseExceptionRequestMiddleware: 590,},})- crawler = runner.create_crawler(SingleRequestSpider)yield crawler.crawl(seed=url, mockserver=self.mockserver)failure = crawler.spider.meta["failure"]self.assertEqual(failure.request.url, url)@@ -117,12 +116,11 @@ class CrawlTestCase(TestCase):signal_params["request"] = requesturl = self.mockserver.url("/status?n=200")- runner = CrawlerRunner(settings={+ crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {ProcessResponseMiddleware: 595,}})- crawler = runner.create_crawler(SingleRequestSpider)crawler.signals.connect(signal_handler, signal=signals.response_received)with LogCapture() as log:@@ -147,13 +145,12 @@ class CrawlTestCase(TestCase):The spider callback should receive the overridden response.request"""url = self.mockserver.url("/status?n=200")- runner = CrawlerRunner(settings={+ crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {RaiseExceptionRequestMiddleware: 590,CatchExceptionOverrideRequestMiddleware: 595,},})- crawler = runner.create_crawler(SingleRequestSpider)yield crawler.crawl(seed=url, mockserver=self.mockserver)response = crawler.spider.meta["responses"][0]self.assertEqual(response.body, b"Caught ZeroDivisionError")@@ -168,13 +165,12 @@ class CrawlTestCase(TestCase):The spider callback should receive the original response.request"""url = self.mockserver.url("/status?n=200")- runner = CrawlerRunner(settings={+ crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER_MIDDLEWARES": {RaiseExceptionRequestMiddleware: 590,CatchExceptionDoNotOverrideRequestMiddleware: 595,},})- crawler = runner.create_crawler(SingleRequestSpider)yield crawler.crawl(seed=url, mockserver=self.mockserver)response = crawler.spider.meta["responses"][0]self.assertEqual(response.body, b"Caught ZeroDivisionError")@@ -186,12 +182,11 @@ class CrawlTestCase(TestCase):Downloader middleware which returns a response with aspecific 'request' attribute, with an alternative callback"""- runner = CrawlerRunner(settings={+ crawler = get_crawler(AlternativeCallbacksSpider, {"DOWNLOADER_MIDDLEWARES": {AlternativeCallbacksMiddleware: 595,}})- crawler = runner.create_crawler(AlternativeCallbacksSpider)with LogCapture() as log:url = self.mockserver.url("/status?n=200")diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 473a93e69..002a04358 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -3,7 +3,7 @@ from twisted.internet import deferfrom twisted.trial.unittest import TestCasefrom scrapy.http import Request-from scrapy.crawler import CrawlerRunner+from scrapy.utils.test import get_crawlerfrom tests.spiders import MockServerSpiderfrom tests.mockserver import MockServer@@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):def setUp(self):self.mockserver = MockServer()self.mockserver.__enter__()- self.runner = CrawlerRunner()def tearDown(self):self.mockserver.__exit__(None, None, None)@defer.inlineCallbacksdef test_callback_kwargs(self):- crawler = self.runner.create_crawler(KeywordArgumentsSpider)+ crawler = get_crawler(KeywordArgumentsSpider)with LogCapture() as log:yield crawler.crawl(mockserver=self.mockserver)self.assertTrue(all(crawler.spider.checks))diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 2d4bfa165..ac66056ba 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -52,6 +52,7 @@ class MockCrawler(Crawler):SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,JOBDIR=jobdir,DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',+ REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION',)super().__init__(Spider, settings)self.engine = MockEngine(downloader=MockDownloader())@@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase):SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',CONCURRENT_REQUESTS_PER_IP=1,)- crawler = Crawler(Spider, settings)+ crawler = get_crawler(Spider, settings)scheduler = Scheduler.from_crawler(crawler)spider = Spider(name='spider')scheduler.open(spider)diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex bf90b4320..fc234a83d 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -7,10 +7,10 @@ from twisted.internet import deferfrom twisted.trial.unittest import TestCase as TwistedTestCasefrom scrapy.core.scheduler import BaseScheduler-from scrapy.crawler import CrawlerRunnerfrom scrapy.http import Requestfrom scrapy.spiders import Spider-from scrapy.utils.request import request_fingerprint+from scrapy.utils.request import fingerprint+from scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer@@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS]class MinimalScheduler:def __init__(self) -> None:- self.requests: Dict[str, Request] = {}+ self.requests: Dict[bytes, Request] = {}def has_pending_requests(self) -> bool:return bool(self.requests)def enqueue_request(self, request: Request) -> bool:- fp = request_fingerprint(request)+ fp = fingerprint(request)if fp not in self.requests:self.requests[fp] = requestreturn True@@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):@defer.inlineCallbacksdef test_crawl(self):with MockServer() as mockserver:- settings = {"SCHEDULER": self.scheduler_cls}+ settings = {+ "SCHEDULER": self.scheduler_cls,+ }with LogCapture() as log:- yield CrawlerRunner(settings).crawl(TestSpider, mockserver)+ crawler = get_crawler(TestSpider, settings)+ yield crawler.crawl(mockserver)for path in PATHS:self.assertIn(f"{{'path': '{path}'}}", str(log))self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log))diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 8a35e9fd7..3719c7c9f 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase):def test_crawler_runner_loading(self):module = 'tests.test_spiderloader.test_spiders.spider1'- runner = CrawlerRunner({'SPIDER_MODULES': [module]})+ runner = CrawlerRunner({+ 'SPIDER_MODULES': [module],+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',+ })self.assertRaisesRegex(KeyError, 'Spider not found',runner.create_crawler, 'spider2')diff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 1ef4eeb14..46452415a 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -3,6 +3,7 @@ import osimport tempfileimport shutilimport contextlib+import warningsfrom pytest import warns@@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase):envvars = {'SCRAPY_SETTINGS_MODULE': value,}- with set_env(**envvars), warns(None) as warnings:- settings = get_project_settings()- assert not warnings+ with warnings.catch_warnings():+ warnings.simplefilter("error")+ with set_env(**envvars):+ settings = get_project_settings()+assert settings.get('SETTINGS_MODULE') == valuedef test_invalid_envvar(self):envvars = {'SCRAPY_FOO': 'bar',}- with set_env(**envvars), warns(None) as warnings:- get_project_settings()- assert len(warnings) == 1- assert warnings[0].category == ScrapyDeprecationWarning- assert str(warnings[0].message).endswith(': FOO')+ with warns(ScrapyDeprecationWarning, match=': FOO') as record:+ with set_env(**envvars):+ get_project_settings()+ assert len(record) == 1def test_valid_and_invalid_envvars(self):value = 'tests.test_cmdline.settings'@@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase):'SCRAPY_FOO': 'bar','SCRAPY_SETTINGS_MODULE': value,}- with set_env(**envvars), warns(None) as warnings:- settings = get_project_settings()- assert len(warnings) == 1- assert warnings[0].category == ScrapyDeprecationWarning- assert str(warnings[0].message).endswith(': FOO')+ with warns(ScrapyDeprecationWarning, match=': FOO') as record:+ with set_env(**envvars):+ settings = get_project_settings()+ assert len(record) == 1assert settings.get('SETTINGS_MODULE') == valuediff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex e9edfee98..5ee772c0b 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -308,13 +308,22 @@ class RequestFingerprintTest(FingerprintTest):),)+ def setUp(self) -> None:+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)++ def tearDown(self) -> None:+ warnings.simplefilter("default", ScrapyDeprecationWarning)+@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)def test_part_separation(self):super().test_part_separation()++class RequestFingerprintDeprecationTest(unittest.TestCase):+def test_deprecation_default_parameters(self):with pytest.warns(ScrapyDeprecationWarning) as warnings:- self.function(Request("http://www.example.com"))+ request_fingerprint(Request("http://www.example.com"))messages = [str(warning.message) for warning in warnings]self.assertTrue(any(@@ -326,7 +335,7 @@ class RequestFingerprintTest(FingerprintTest):def test_deprecation_non_default_parameters(self):with pytest.warns(ScrapyDeprecationWarning) as warnings:- self.function(Request("http://www.example.com"), keep_fragments=True)+ request_fingerprint(Request("http://www.example.com"), keep_fragments=True)messages = [str(warning.message) for warning in warnings]self.assertTrue(any(diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 0a09f6109..d20852e62 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -1,7 +1,9 @@import osimport unittest+import warningsfrom urllib.parse import urlparse+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Response, TextResponse, HtmlResponsefrom scrapy.utils.python import to_bytesfrom scrapy.utils.response import (response_httprepr, open_in_browser,@@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase):dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')def test_response_httprepr(self):- r1 = Response("http://www.example.com")- self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)- r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")- self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')+ r1 = Response("http://www.example.com")+ self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')- r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")- self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')+ r1 = Response("http://www.example.com", status=404,+ headers={"Content-type": "text/html"}, body=b"Some body")+ self.assertEqual(response_httprepr(r1),+ b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')++ r1 = Response("http://www.example.com", status=6666,+ headers={"Content-type": "text/html"}, body=b"Some body")+ self.assertEqual(response_httprepr(r1),+ b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')def test_open_in_browser(self):url = "http:///www.example.com/some/page.html"