Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Address 2/3 of warnings from tests (#5561)

Mikhail Korobovb103664bf45b
diff --git a/pytest.ini b/pytest.ini
index ae2ed2029..af0f2fb6e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,3 +21,6 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+filterwarnings =
+ ignore:scrapy.downloadermiddlewares.decompression is deprecated
+ ignore:Module scrapy.utils.reqser is deprecated
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d0f4a68c2..136056f50 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -5,11 +5,11 @@ from twisted.python import failure
from twisted.trial import unittest
from scrapy import FormRequest
-from scrapy.crawler import CrawlerRunner
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.item import Item, Field
+from scrapy.utils.test import get_crawler
from scrapy.contracts import ContractsManager, Contract
from scrapy.contracts.default import (
UrlContract,
@@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase):
TestSameUrlSpider.parse_first.__doc__ = contract_doc
TestSameUrlSpider.parse_second.__doc__ = contract_doc
- crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+ crawler = get_crawler(TestSameUrlSpider)
yield crawler.crawl()
self.assertEqual(crawler.spider.visited, 2)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f9ffcd6bb..59c271868 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -18,6 +18,7 @@ from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
+from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import MockServer
from tests.spiders import (
@@ -49,14 +50,13 @@ class CrawlTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
- self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_follow_all(self):
- crawler = self.runner.create_crawler(FollowAllSpider)
+ crawler = get_crawler(FollowAllSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url
@@ -79,7 +79,7 @@ class CrawlTestCase(TestCase):
settings = {"DOWNLOAD_DELAY": delay,
'RANDOMIZE_DOWNLOAD_DELAY': randomize}
- crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+ crawler = get_crawler(FollowAllSpider, settings)
yield crawler.crawl(**crawl_kwargs)
times = crawler.spider.times
total_time = times[-1] - times[0]
@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
# of ``total`` and ``delay`` values that are too small for the test
# code above to have any meaning.
settings["DOWNLOAD_DELAY"] = 0
- crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+ crawler = get_crawler(FollowAllSpider, settings)
yield crawler.crawl(**crawl_kwargs)
times = crawler.spider.times
total_time = times[-1] - times[0]
@@ -102,7 +102,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_success(self):
- crawler = self.runner.create_crawler(DelaySpider)
+ crawler = get_crawler(DelaySpider)
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 > 0)
@@ -110,7 +110,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_failure(self):
- crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
+ crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 == 0)
@@ -125,14 +125,14 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_retry_503(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_failed(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
self._assert_retried(log)
@@ -141,7 +141,7 @@ class CrawlTestCase(TestCase):
def test_retry_dns_error(self):
if NON_EXISTING_RESOLVABLE:
raise unittest.SkipTest("Non-existing hosts are resolvable")
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
# try to fetch the homepage of a non-existent domain
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
@@ -150,7 +150,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_bug_before_yield(self):
with LogCapture('scrapy', level=logging.ERROR) as log:
- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+ crawler = get_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
self.assertEqual(len(log.records), 1)
@@ -161,7 +161,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_bug_yielding(self):
with LogCapture('scrapy', level=logging.ERROR) as log:
- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+ crawler = get_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
self.assertEqual(len(log.records), 1)
@@ -172,7 +172,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_lazyness(self):
settings = {"CONCURRENT_REQUESTS": 1}
- crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
+ crawler = get_crawler(BrokenStartRequestsSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(
crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@@ -181,7 +181,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
- crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
+ crawler = get_crawler(DuplicateStartRequestsSpider, settings)
yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
self.assertEqual(crawler.spider.visited, 6)
@@ -210,7 +210,7 @@ Connection: close
foo body
with multiples lines
'''})
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)
self.assertEqual(str(log).count("Got response 200"), 1)
@@ -218,7 +218,7 @@ with multiples lines
@defer.inlineCallbacks
def test_retry_conn_lost(self):
# connection lost after receiving data
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
self._assert_retried(log)
@@ -226,7 +226,7 @@ with multiples lines
@defer.inlineCallbacks
def test_retry_conn_aborted(self):
# connection lost before receiving data
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
self._assert_retried(log)
@@ -245,7 +245,7 @@ with multiples lines
req0.meta['next'] = req1
req1.meta['next'] = req2
req2.meta['next'] = req3
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=req0, mockserver=self.mockserver)
# basic asserts in case of weird communication errors
self.assertIn('responses', crawler.spider.meta)
@@ -271,7 +271,7 @@ with multiples lines
def cb(response):
est.append(get_engine_status(crawler.engine))
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
self.assertEqual(len(est), 1, est)
s = dict(est[0])
@@ -286,7 +286,7 @@ with multiples lines
def cb(response):
est.append(format_engine_status(crawler.engine))
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
self.assertEqual(len(est), 1, est)
est = est[0].split("\n")[2:-2] # remove header & footer
@@ -317,7 +317,7 @@ with multiples lines
def start_requests(self):
raise TestError
- crawler = self.runner.create_crawler(FaultySpider)
+ crawler = get_crawler(FaultySpider)
yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
self.assertFalse(crawler.crawling)
@@ -328,26 +328,28 @@ with multiples lines
"tests.pipelines.ZeroDivisionErrorPipeline": 300,
}
}
- crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider, settings)
yield self.assertFailure(
- self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
+ crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
ZeroDivisionError)
self.assertFalse(crawler.crawling)
@defer.inlineCallbacks
def test_crawlerrunner_accepts_crawler(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
+ runner = CrawlerRunner()
with LogCapture() as log:
- yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@defer.inlineCallbacks
def test_crawl_multiple(self):
- self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
- self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
+ runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+ runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
with LogCapture() as log:
- yield self.runner.join()
+ yield runner.join()
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
@@ -358,7 +360,6 @@ class CrawlSpiderTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
- self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@@ -370,7 +371,7 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
- crawler = self.runner.create_crawler(spider_cls)
+ crawler = get_crawler(spider_cls)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -378,10 +379,9 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_crawlspider_with_parse(self):
- self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
-
+ crawler = get_crawler(CrawlSpiderWithParseMethod)
with LogCapture() as log:
- yield self.runner.join()
+ yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("[parse] status 200 (foo: None)", str(log))
self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -389,10 +389,9 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_crawlspider_with_errback(self):
- self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
-
+ crawler = get_crawler(CrawlSpiderWithErrback)
with LogCapture() as log:
- yield self.runner.join()
+ yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("[parse] status 200 (foo: None)", str(log))
self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -403,18 +402,19 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_async_def_parse(self):
- self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ crawler = get_crawler(AsyncDefSpider)
with LogCapture() as log:
- yield self.runner.join()
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse(self):
- runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})
- runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ crawler = get_crawler(AsyncDefAsyncioSpider, {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+ })
with LogCapture() as log:
- yield runner.join()
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@@ -433,7 +433,7 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
- crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+ crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -479,14 +479,14 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_response_ssl_certificate_none(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=False)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
@defer.inlineCallbacks
def test_response_ssl_certificate(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=True)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
cert = crawler.spider.meta['responses'][0].certificate
@@ -497,7 +497,7 @@ class CrawlSpiderTestCase(TestCase):
@mark.xfail(reason="Responses with no body return early and contain no certificate")
@defer.inlineCallbacks
def test_response_ssl_certificate_empty_response(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/status?n=200", is_secure=True)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
cert = crawler.spider.meta['responses'][0].certificate
@@ -507,7 +507,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_dns_server_ip_address_none(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
@@ -515,7 +515,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_dns_server_ip_address(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url('/echo?body=test')
expected_netloc, _ = urlparse(url).netloc.split(':')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
@@ -525,7 +525,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_bytes_received_stop_download_callback(self):
- crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
+ crawler = get_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -534,7 +534,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_bytes_received_stop_download_errback(self):
- crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
+ crawler = get_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
@@ -549,7 +549,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_headers_received_stop_download_callback(self):
- crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
+ crawler = get_crawler(HeadersReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -557,7 +557,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_headers_received_stop_download_errback(self):
- crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
+ crawler = get_crawler(HeadersReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f7aa769e4..d67abed7c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -13,11 +13,13 @@ from twisted.trial import unittest
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
+from scrapy.utils.test import get_crawler
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
from scrapy.utils.test import get_testenv
@@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase):
class CrawlerTestCase(BaseCrawlerTest):
- def setUp(self):
- self.crawler = Crawler(DefaultSpider, Settings())
-
def test_populate_spidercls_settings(self):
spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest):
settings = Settings()
settings.setdict(project_settings, priority='project')
- crawler = Crawler(CustomSettingsSpider, settings)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ crawler = Crawler(CustomSettingsSpider, settings)
self.assertEqual(crawler.settings.get('TEST1'), 'spider')
self.assertEqual(crawler.settings.get('TEST2'), 'spider')
@@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest):
self.assertTrue(crawler.settings.frozen)
def test_crawler_accepts_dict(self):
- crawler = Crawler(DefaultSpider, {'foo': 'bar'})
+ crawler = get_crawler(DefaultSpider, {'foo': 'bar'})
self.assertEqual(crawler.settings['foo'], 'bar')
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
def test_crawler_accepts_None(self):
- crawler = Crawler(DefaultSpider)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ crawler = Crawler(DefaultSpider)
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
def test_crawler_rejects_spider_objects(self):
@@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase):
'AUTOTHROTTLE_ENABLED': True
}
- crawler = Crawler(MySpider, {})
+ crawler = get_crawler(MySpider)
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
self.assertIn(AutoThrottle, enabled_exts)
@@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
class MySpider(scrapy.Spider):
name = 'spider'
- Crawler(MySpider, {})
+ get_crawler(MySpider)
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
@@ -111,7 +114,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
- crawler = Crawler(MySpider, {})
+ crawler = get_crawler(MySpider)
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
@@ -148,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
}
configure_logging()
- Crawler(MySpider, {})
+ get_crawler(MySpider)
logging.debug('debug message')
with open(log_file, 'rb') as fo:
@@ -229,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider):
@mark.usefixtures('reactor_pytest')
class CrawlerRunnerHasSpider(unittest.TestCase):
+ def _runner(self):
+ return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
- runner = CrawlerRunner()
+ runner = self._runner()
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, False)
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful_for_several(self):
- runner = CrawlerRunner()
+ runner = self._runner()
yield runner.crawl(NoRequestsSpider)
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, False)
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_failed(self):
- runner = CrawlerRunner()
+ runner = self._runner()
try:
yield runner.crawl(ExceptionSpider)
@@ -257,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_failed_for_several(self):
- runner = CrawlerRunner()
+ runner = self._runner()
try:
yield runner.crawl(ExceptionSpider)
@@ -275,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
if self.reactor_pytest == 'asyncio':
CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
})
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
})
yield runner.crawl(NoRequestsSpider)
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 0362e2018..b9f3e24a4 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,7 +1,9 @@
import unittest
+import pytest
from w3lib.http import basic_auth_header
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.spiders import Spider
@@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.spider = TestSpiderLegacy('foo')
def test_auth(self):
- mw = HttpAuthMiddleware()
- mw.spider_opened(self.spider)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+ mw = HttpAuthMiddleware()
+ mw.spider_opened(self.spider)
# initial request, sets the domain and sends the header
req = Request('http://example.com/')
@@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.assertNotIn('Authorization', req.headers)
def test_auth_already_set(self):
- mw = HttpAuthMiddleware()
- mw.spider_opened(self.spider)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+ mw = HttpAuthMiddleware()
+ mw.spider_opened(self.spider)
req = Request('http://example.com/',
headers=dict(Authorization='Digest 123'))
assert mw.process_request(req, self.spider) is None
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7c97bf32a..4ac85c1ec 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,13 +1,13 @@
import os
-from functools import partial
+
+import pytest
from twisted.trial.unittest import TestCase
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request
from scrapy.spiders import Spider
-from scrapy.crawler import Crawler
-from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler
spider = Spider('foo')
@@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase):
os.environ = self._oldenv
def test_not_enabled(self):
- settings = Settings({'HTTPPROXY_ENABLED': False})
- crawler = Crawler(Spider, settings)
- self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
+ crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})
+ with pytest.raises(NotConfigured):
+ HttpProxyMiddleware.from_crawler(crawler)
def test_no_environment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 9e75f0a50..7d88ba4d2 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,7 +1,9 @@
+import warnings
from itertools import product
from unittest import TestCase
from scrapy.downloadermiddlewares.stats import DownloaderStats
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.response import response_httprepr
@@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase):
for test_response in test_responses:
self.crawler.stats.set_value('downloader/response_bytes', 0)
self.mw.process_response(self.req, test_response, self.spider)
- self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ resp_size = len(response_httprepr(test_response))
+ self.assertStatsEqual('downloader/response_bytes', resp_size)
def test_process_exception(self):
self.mw.process_exception(self.req, MyException(), self.spider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index b7df2554a..8a37a8ebe 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -10,7 +10,6 @@ from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
from scrapy.core.scheduler import Scheduler
from scrapy.utils.python import to_bytes
-from scrapy.utils.job import job_dir
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@@ -29,8 +28,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@classmethod
def from_crawler(cls, crawler):
- debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
- df = cls(job_dir(crawler.settings), debug)
+ df = super().from_crawler(crawler)
df.method = 'from_crawler'
return df
@@ -38,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
class FromSettingsRFPDupeFilter(RFPDupeFilter):
@classmethod
- def from_settings(cls, settings):
- debug = settings.getbool('DUPEFILTER_DEBUG')
- df = cls(job_dir(settings), debug)
+ def from_settings(cls, settings, *, fingerprinter=None):
+ df = super().from_settings(settings, fingerprinter=fingerprinter)
df.method = 'from_settings'
return df
@@ -53,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_crawler_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
@@ -61,14 +59,16 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_settings_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
- 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
self.assertEqual(scheduler.df.method, 'from_settings')
def test_df_direct_scheduler(self):
- settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
+ settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertEqual(scheduler.df.method, 'n/a')
@@ -171,7 +171,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': False,
- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = _get_dupefilter(crawler=crawler)
@@ -197,7 +198,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log_debug(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True,
- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = _get_dupefilter(crawler=crawler)
@@ -230,7 +232,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log_debug_default_dupefilter(self):
with LogCapture() as log:
- settings = {'DUPEFILTER_DEBUG': True}
+ settings = {'DUPEFILTER_DEBUG': True,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = _get_dupefilter(crawler=crawler)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index fa7d0c8d4..1bd802bcf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -13,10 +13,11 @@ module with the ``runserver`` argument::
import os
import re
import sys
-import warnings
from collections import defaultdict
from urllib.parse import urlparse
+from dataclasses import dataclass
+import pytest
import attr
from itemadapter import ItemAdapter
from pydispatch import dispatcher
@@ -50,6 +51,13 @@ class AttrsItem:
price = attr.ib(default=0)
+@dataclass
+class DataClassItem:
+ name: str = ""
+ url: str = ""
+ price: int = 0
+
+
class TestSpider(Spider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
@@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider):
item_cls = AttrsItem
-try:
- from dataclasses import make_dataclass
-except ImportError:
- DataClassItemsSpider = None
-else:
- TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
-
- class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef]
- def parse_item(self, response):
- item = super().parse_item(response)
- return TestDataClass(**item)
+class DataClassItemsSpider(TestSpider):
+ item_cls = DataClassItem
class ItemZeroDivisionErrorSpider(TestSpider):
@@ -188,7 +187,7 @@ class CrawlerRun:
return self.deferred
def stop(self):
- self.port.stopListening()
+ self.port.stopListening() # FIXME: wait for this Deferred
for name, signal in vars(signals).items():
if not name.startswith('_'):
disconnect_all(signal)
@@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase):
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
- if spider is None:
- continue
- self.run = CrawlerRun(spider)
- yield self.run.run()
- self._assert_visited_urls()
- self._assert_scheduled_requests(count=9)
- self._assert_downloaded_responses(count=9)
- self._assert_scraped_items()
- self._assert_signals_caught()
- self._assert_bytes_received()
+ run = CrawlerRun(spider)
+ yield run.run()
+ self._assert_visited_urls(run)
+ self._assert_scheduled_requests(run, count=9)
+ self._assert_downloaded_responses(run, count=9)
+ self._assert_scraped_items(run)
+ self._assert_signals_caught(run)
+ self._assert_bytes_received(run)
@defer.inlineCallbacks
def test_crawler_dupefilter(self):
- self.run = CrawlerRun(TestDupeFilterSpider)
- yield self.run.run()
- self._assert_scheduled_requests(count=8)
- self._assert_dropped_requests()
+ run = CrawlerRun(TestDupeFilterSpider)
+ yield run.run()
+ self._assert_scheduled_requests(run, count=8)
+ self._assert_dropped_requests(run)
@defer.inlineCallbacks
def test_crawler_itemerror(self):
- self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
- yield self.run.run()
- self._assert_items_error()
+ run = CrawlerRun(ItemZeroDivisionErrorSpider)
+ yield run.run()
+ self._assert_items_error(run)
@defer.inlineCallbacks
def test_crawler_change_close_reason_on_idle(self):
- self.run = CrawlerRun(ChangeCloseReasonSpider)
- yield self.run.run()
- self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
- self.run.signals_caught[signals.spider_closed])
+ run = CrawlerRun(ChangeCloseReasonSpider)
+ yield run.run()
+ self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},
+ run.signals_caught[signals.spider_closed])
- def _assert_visited_urls(self):
+ def _assert_visited_urls(self, run: CrawlerRun):
must_be_visited = ["/", "/redirect", "/redirected",
"/item1.html", "/item2.html", "/item999.html"]
- urls_visited = {rp[0].url for rp in self.run.respplug}
- urls_expected = {self.run.geturl(p) for p in must_be_visited}
+ urls_visited = {rp[0].url for rp in run.respplug}
+ urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
- def _assert_scheduled_requests(self, count=None):
- self.assertEqual(count, len(self.run.reqplug))
+ def _assert_scheduled_requests(self, run: CrawlerRun, count=None):
+ self.assertEqual(count, len(run.reqplug))
paths_expected = ['/item999.html', '/item2.html', '/item1.html']
- urls_requested = {rq[0].url for rq in self.run.reqplug}
- urls_expected = {self.run.geturl(p) for p in paths_expected}
+ urls_requested = {rq[0].url for rq in run.reqplug}
+ urls_expected = {run.geturl(p) for p in paths_expected}
assert urls_expected <= urls_requested
- scheduled_requests_count = len(self.run.reqplug)
- dropped_requests_count = len(self.run.reqdropped)
- responses_count = len(self.run.respplug)
+ scheduled_requests_count = len(run.reqplug)
+ dropped_requests_count = len(run.reqdropped)
+ responses_count = len(run.respplug)
self.assertEqual(scheduled_requests_count,
dropped_requests_count + responses_count)
- self.assertEqual(len(self.run.reqreached),
+ self.assertEqual(len(run.reqreached),
responses_count)
- def _assert_dropped_requests(self):
- self.assertEqual(len(self.run.reqdropped), 1)
+ def _assert_dropped_requests(self, run: CrawlerRun):
+ self.assertEqual(len(run.reqdropped), 1)
- def _assert_downloaded_responses(self, count):
+ def _assert_downloaded_responses(self, run: CrawlerRun, count):
# response tests
- self.assertEqual(count, len(self.run.respplug))
- self.assertEqual(count, len(self.run.reqreached))
+ self.assertEqual(count, len(run.respplug))
+ self.assertEqual(count, len(run.reqreached))
- for response, _ in self.run.respplug:
- if self.run.getpath(response.url) == '/item999.html':
+ for response, _ in run.respplug:
+ if run.getpath(response.url) == '/item999.html':
self.assertEqual(404, response.status)
- if self.run.getpath(response.url) == '/redirect':
+ if run.getpath(response.url) == '/redirect':
self.assertEqual(302, response.status)
- def _assert_items_error(self):
- self.assertEqual(2, len(self.run.itemerror))
- for item, response, spider, failure in self.run.itemerror:
+ def _assert_items_error(self, run: CrawlerRun):
+ self.assertEqual(2, len(run.itemerror))
+ for item, response, spider, failure in run.itemerror:
self.assertEqual(failure.value.__class__, ZeroDivisionError)
- self.assertEqual(spider, self.run.spider)
+ self.assertEqual(spider, run.spider)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
@@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
- def _assert_scraped_items(self):
- self.assertEqual(2, len(self.run.itemresp))
- for item, response in self.run.itemresp:
+ def _assert_scraped_items(self, run: CrawlerRun):
+ self.assertEqual(2, len(run.itemresp))
+ for item, response in run.itemresp:
item = ItemAdapter(item)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
@@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
- def _assert_headers_received(self):
- for headers in self.run.headers.values():
+ def _assert_headers_received(self, run: CrawlerRun):
+ for headers in run.headers.values():
self.assertIn(b"Server", headers)
self.assertIn(b"TwistedWeb", headers[b"Server"])
self.assertIn(b"Date", headers)
self.assertIn(b"Content-Type", headers)
- def _assert_bytes_received(self):
- self.assertEqual(9, len(self.run.bytes))
- for request, data in self.run.bytes.items():
+ def _assert_bytes_received(self, run: CrawlerRun):
+ self.assertEqual(9, len(run.bytes))
+ for request, data in run.bytes.items():
joined_data = b"".join(data)
- if self.run.getpath(request.url) == "/":
+ if run.getpath(request.url) == "/":
self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
- elif self.run.getpath(request.url) == "/item1.html":
+ elif run.getpath(request.url) == "/item1.html":
self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
- elif self.run.getpath(request.url) == "/item2.html":
+ elif run.getpath(request.url) == "/item2.html":
self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
- elif self.run.getpath(request.url) == "/redirected":
+ elif run.getpath(request.url) == "/redirected":
self.assertEqual(joined_data, b"Redirected here")
- elif self.run.getpath(request.url) == '/redirect':
+ elif run.getpath(request.url) == '/redirect':
self.assertEqual(
joined_data,
b"\n<html>\n"
@@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase):
b" </body>\n"
b"</html>\n"
)
- elif self.run.getpath(request.url) == "/tem999.html":
+ elif run.getpath(request.url) == "/tem999.html":
self.assertEqual(
joined_data,
b"\n<html>\n"
@@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase):
b" </body>\n"
b"</html>\n"
)
- elif self.run.getpath(request.url) == "/numbers":
+ elif run.getpath(request.url) == "/numbers":
# signal was fired multiple times
self.assertTrue(len(data) > 1)
# bytes were received in order
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.assertEqual(joined_data, b"".join(numbers))
- def _assert_signals_caught(self):
- assert signals.engine_started in self.run.signals_caught
- assert signals.engine_stopped in self.run.signals_caught
- assert signals.spider_opened in self.run.signals_caught
- assert signals.spider_idle in self.run.signals_caught
- assert signals.spider_closed in self.run.signals_caught
- assert signals.headers_received in self.run.signals_caught
-
- self.assertEqual({'spider': self.run.spider},
- self.run.signals_caught[signals.spider_opened])
- self.assertEqual({'spider': self.run.spider},
- self.run.signals_caught[signals.spider_idle])
- self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
- self.run.signals_caught[signals.spider_closed])
+ def _assert_signals_caught(self, run: CrawlerRun):
+ assert signals.engine_started in run.signals_caught
+ assert signals.engine_stopped in run.signals_caught
+ assert signals.spider_opened in run.signals_caught
+ assert signals.spider_idle in run.signals_caught
+ assert signals.spider_closed in run.signals_caught
+ assert signals.headers_received in run.signals_caught
+
+ self.assertEqual({'spider': run.spider},
+ run.signals_caught[signals.spider_opened])
+ self.assertEqual({'spider': run.spider},
+ run.signals_caught[signals.spider_idle])
+ self.assertEqual({'spider': run.spider, 'reason': 'finished'},
+ run.signals_caught[signals.spider_closed])
@defer.inlineCallbacks
def test_close_downloader(self):
@@ -407,28 +404,29 @@ class EngineTest(unittest.TestCase):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
- yield self.assertFailure(e.start(), RuntimeError).addBoth(
- lambda exc: self.assertEqual(str(exc), "Engine already running")
- )
- yield e.stop()
+ try:
+ yield self.assertFailure(e.start(), RuntimeError).addBoth(
+ lambda exc: self.assertEqual(str(exc), "Engine already running")
+ )
+ finally:
+ yield e.stop()
@defer.inlineCallbacks
def test_close_spiders_downloader(self):
- with warnings.catch_warnings(record=True) as warning_list:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="ExecutionEngine.open_spiders is deprecated, "
+ "please use ExecutionEngine.spider instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
self.assertEqual(len(e.open_spiders), 1)
yield e.close()
self.assertEqual(len(e.open_spiders), 0)
- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
- self.assertEqual(
- str(warning_list[0].message),
- "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
- )
@defer.inlineCallbacks
def test_close_engine_spiders_downloader(self):
- with warnings.catch_warnings(record=True) as warning_list:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="ExecutionEngine.open_spiders is deprecated, "
+ "please use ExecutionEngine.spider instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
@@ -436,61 +434,47 @@ class EngineTest(unittest.TestCase):
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
- self.assertEqual(
- str(warning_list[0].message),
- "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
- )
@defer.inlineCallbacks
def test_crawl_deprecated_spider_arg(self):
- with warnings.catch_warnings(record=True) as warning_list:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Passing a 'spider' argument to "
+ "ExecutionEngine.crawl is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.crawl(Request("data:,"), spider)
yield e.close()
- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
- self.assertEqual(
- str(warning_list[0].message),
- "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
- )
@defer.inlineCallbacks
def test_download_deprecated_spider_arg(self):
- with warnings.catch_warnings(record=True) as warning_list:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Passing a 'spider' argument to "
+ "ExecutionEngine.download is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.download(Request("data:,"), spider)
yield e.close()
- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
- self.assertEqual(
- str(warning_list[0].message),
- "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
- )
@defer.inlineCallbacks
def test_deprecated_schedule(self):
- with warnings.catch_warnings(record=True) as warning_list:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="ExecutionEngine.schedule is deprecated, please use "
+ "ExecutionEngine.crawl or ExecutionEngine.download instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.schedule(Request("data:,"), spider)
yield e.close()
- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
- self.assertEqual(
- str(warning_list[0].message),
- "ExecutionEngine.schedule is deprecated, please use "
- "ExecutionEngine.crawl or ExecutionEngine.download instead",
- )
@defer.inlineCallbacks
def test_deprecated_has_capacity(self):
- with warnings.catch_warnings(record=True) as warning_list:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="ExecutionEngine.has_capacity is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
self.assertTrue(e.has_capacity())
spider = TestSpider()
@@ -499,8 +483,6 @@ class EngineTest(unittest.TestCase):
e.start()
yield e.close()
self.assertTrue(e.has_capacity())
- self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
- self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
if __name__ == "__main__":
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 0ba69e096..933e4067d 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
- if spider is None:
- continue
- self.run = BytesReceivedCrawlerRun(spider)
+ run = BytesReceivedCrawlerRun(spider)
with LogCapture() as log:
- yield self.run.run()
+ yield run.run()
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
- f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
+ f"Download stopped for <GET http://localhost:{run.portno}/redirected> "
"from signal handler BytesReceivedCrawlerRun.bytes_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
- f"Download stopped for <GET http://localhost:{self.run.portno}/> "
+ f"Download stopped for <GET http://localhost:{run.portno}/> "
"from signal handler BytesReceivedCrawlerRun.bytes_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
- f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
+ f"Download stopped for <GET http://localhost:{run.portno}/numbers> "
"from signal handler BytesReceivedCrawlerRun.bytes_received"))
- self._assert_visited_urls()
- self._assert_scheduled_requests(count=9)
- self._assert_downloaded_responses(count=9)
- self._assert_signals_caught()
- self._assert_headers_received()
- self._assert_bytes_received()
+ self._assert_visited_urls(run)
+ self._assert_scheduled_requests(run, count=9)
+ self._assert_downloaded_responses(run, count=9)
+ self._assert_signals_caught(run)
+ self._assert_headers_received(run)
+ self._assert_bytes_received(run)
- def _assert_bytes_received(self):
- self.assertEqual(9, len(self.run.bytes))
- for request, data in self.run.bytes.items():
+ def _assert_bytes_received(self, run: CrawlerRun):
+ self.assertEqual(9, len(run.bytes))
+ for request, data in run.bytes.items():
joined_data = b"".join(data)
self.assertTrue(len(data) == 1) # signal was fired only once
- if self.run.getpath(request.url) == "/numbers":
+ if run.getpath(request.url) == "/numbers":
# Received bytes are not the complete response. The exact amount depends
# on the buffer size, which can vary, so we only check that the amount
# of received bytes is strictly less than the full response.
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index fad6643ad..8975d0e3f 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
- if spider is None:
- continue
- self.run = HeadersReceivedCrawlerRun(spider)
+ run = HeadersReceivedCrawlerRun(spider)
with LogCapture() as log:
- yield self.run.run()
+ yield run.run()
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
- f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"
+ f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"
" signal handler HeadersReceivedCrawlerRun.headers_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
- f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"
+ f"Download stopped for <GET http://localhost:{run.portno}/> from signal"
" handler HeadersReceivedCrawlerRun.headers_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
- f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"
+ f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"
" signal handler HeadersReceivedCrawlerRun.headers_received"))
- self._assert_visited_urls()
- self._assert_downloaded_responses(count=6)
- self._assert_signals_caught()
- self._assert_bytes_received()
- self._assert_headers_received()
+ self._assert_visited_urls(run)
+ self._assert_downloaded_responses(run, count=6)
+ self._assert_signals_caught(run)
+ self._assert_bytes_received(run)
+ self._assert_headers_received(run)
- def _assert_bytes_received(self):
- self.assertEqual(0, len(self.run.bytes))
+ def _assert_bytes_received(self, run: CrawlerRun):
+ self.assertEqual(0, len(run.bytes))
- def _assert_visited_urls(self):
+ def _assert_visited_urls(self, run: CrawlerRun):
must_be_visited = ["/", "/redirect", "/redirected"]
- urls_visited = {rp[0].url for rp in self.run.respplug}
- urls_expected = {self.run.geturl(p) for p in must_be_visited}
+ urls_visited = {rp[0].url for rp in run.respplug}
+ urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ec48f8d4a..ecd1b59d3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -22,6 +22,7 @@ from urllib.parse import urljoin, quote
from urllib.request import pathname2url
import lxml.etree
+import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial import unittest
@@ -30,7 +31,6 @@ from zope.interface import implementer
from zope.interface.verify import verifyObject
import scrapy
-from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter
from scrapy.extensions.feedexport import (
@@ -697,9 +697,9 @@ class FeedExportTest(FeedExportTestBase):
content = {}
try:
with MockServer() as s:
- runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
- yield runner.crawl(spider_cls)
+ crawler = get_crawler(spider_cls, settings)
+ yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
@@ -1554,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
content = {}
try:
with MockServer() as s:
- runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
- yield runner.crawl(spider_cls)
+ crawler = get_crawler(spider_cls, settings)
+ yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
@@ -2026,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
content = defaultdict(list)
try:
with MockServer() as s:
- runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
- yield runner.crawl(spider_cls)
+ crawler = get_crawler(spider_cls, settings)
+ yield crawler.crawl()
for path, feed in FEEDS.items():
dir_name = os.path.dirname(path)
@@ -2048,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
},
})
- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['jl']:
@@ -2064,7 +2064,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
},
})
- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
data = yield self.exported_data(items, settings)
for batch in data['csv']:
got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -2080,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
},
})
- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['xml']:
@@ -2098,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
},
})
- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
# XML
@@ -2123,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
},
})
- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import pickle
@@ -2140,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
},
})
- batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+ batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import marshal
@@ -2166,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'FEED_EXPORT_BATCH_ITEM_COUNT': 2
}
header = self.MyItem.fields.keys()
- yield self.assertExported(items, header, rows, settings=Settings(settings))
+ yield self.assertExported(items, header, rows, settings=settings)
def test_wrong_path(self):
""" If path is without %(batch_time)s and %(batch_id) an exception must be raised """
@@ -2382,9 +2382,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
yield item
with MockServer() as server:
- runner = CrawlerRunner(Settings(settings))
TestSpider.start_urls = [server.url('/')]
- yield runner.crawl(TestSpider)
+ crawler = get_crawler(TestSpider, settings)
+ yield crawler.crawl()
self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
for stub in CustomS3FeedStorage.stubs[:-1]:
@@ -2434,25 +2434,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': StdoutFeedStorageWithoutFeedOptions
},
}
- crawler = get_crawler(settings_dict=settings_dict)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+
spider = scrapy.Spider("default")
- with warnings.catch_warnings(record=True) as w:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="StdoutFeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
- messages = tuple(str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning)
- self.assertEqual(
- messages,
- (
- (
- "StdoutFeedStorageWithoutFeedOptions does not support "
- "the 'feed_options' keyword argument. Add a "
- "'feed_options' parameter to its signature to remove "
- "this warning. This parameter will become mandatory "
- "in a future version of Scrapy."
- ),
- )
- )
class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
@@ -2476,25 +2467,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': FileFeedStorageWithoutFeedOptions
},
}
- crawler = get_crawler(settings_dict=settings_dict)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
- with warnings.catch_warnings(record=True) as w:
+
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="FileFeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
- messages = tuple(str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning)
- self.assertEqual(
- messages,
- (
- (
- "FileFeedStorageWithoutFeedOptions does not support "
- "the 'feed_options' keyword argument. Add a "
- "'feed_options' parameter to its signature to remove "
- "this warning. This parameter will become mandatory "
- "in a future version of Scrapy."
- ),
- )
- )
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
@@ -2524,26 +2506,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': S3FeedStorageWithoutFeedOptions
},
}
- crawler = get_crawler(settings_dict=settings_dict)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+
spider = scrapy.Spider("default")
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="S3FeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
- messages = tuple(str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning)
- self.assertEqual(
- messages,
- (
- (
- "S3FeedStorageWithoutFeedOptions does not support "
- "the 'feed_options' keyword argument. Add a "
- "'feed_options' parameter to its signature to remove "
- "this warning. This parameter will become mandatory "
- "in a future version of Scrapy."
- ),
- )
- )
def test_from_crawler(self):
settings_dict = {
@@ -2552,26 +2526,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
},
}
- crawler = get_crawler(settings_dict=settings_dict)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+
spider = scrapy.Spider("default")
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+ "the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
- messages = tuple(str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning)
- self.assertEqual(
- messages,
- (
- (
- "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
- "does not support the 'feed_options' keyword argument. Add a "
- "'feed_options' parameter to its signature to remove "
- "this warning. This parameter will become mandatory "
- "in a future version of Scrapy."
- ),
- )
- )
class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
@@ -2601,26 +2567,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': FTPFeedStorageWithoutFeedOptions
},
}
- crawler = get_crawler(settings_dict=settings_dict)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+
spider = scrapy.Spider("default")
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="FTPFeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
- messages = tuple(str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning)
- self.assertEqual(
- messages,
- (
- (
- "FTPFeedStorageWithoutFeedOptions does not support "
- "the 'feed_options' keyword argument. Add a "
- "'feed_options' parameter to its signature to remove "
- "this warning. This parameter will become mandatory "
- "in a future version of Scrapy."
- ),
- )
- )
def test_from_crawler(self):
settings_dict = {
@@ -2629,50 +2587,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
},
}
- crawler = get_crawler(settings_dict=settings_dict)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+
spider = scrapy.Spider("default")
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+ "the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
- messages = tuple(str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning)
- self.assertEqual(
- messages,
- (
- (
- "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
- "does not support the 'feed_options' keyword argument. Add a "
- "'feed_options' parameter to its signature to remove "
- "this warning. This parameter will become mandatory "
- "in a future version of Scrapy."
- ),
- )
- )
class URIParamsTest:
spider_name = "uri_params_spider"
+ deprecated_options = False
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
raise NotImplementedError
+ def _crawler_feed_exporter(self, settings):
+ if self.deprecated_options:
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+ crawler = get_crawler(settings_dict=settings)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ else:
+ crawler = get_crawler(settings_dict=settings)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ return crawler, feed_exporter
+
def test_default(self):
settings = self.build_settings(
uri='file:///tmp/%(name)s',
)
- crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
- messages = tuple(
- str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning
- )
- self.assertEqual(messages, tuple())
self.assertEqual(
feed_exporter.slots[0].uri,
@@ -2687,28 +2645,13 @@ class URIParamsTest:
uri='file:///tmp/%(name)s',
uri_params=uri_params,
)
- crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Modifying the params dictionary in-place"):
feed_exporter.open_spider(spider)
- messages = tuple(
- str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning
- )
- self.assertEqual(
- messages,
- (
- (
- 'Modifying the params dictionary in-place in the '
- 'function defined in the FEED_URI_PARAMS setting or '
- 'in the uri_params key of the FEEDS setting is '
- 'deprecated. The function must return a new '
- 'dictionary instead.'
- ),
- )
- )
self.assertEqual(
feed_exporter.slots[0].uri,
@@ -2723,18 +2666,14 @@ class URIParamsTest:
uri='file:///tmp/%(name)s',
uri_params=uri_params,
)
- crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", ScrapyDeprecationWarning)
with self.assertRaises(KeyError):
feed_exporter.open_spider(spider)
- messages = tuple(
- str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning
- )
- self.assertEqual(messages, tuple())
def test_params_as_is(self):
def uri_params(params, spider):
@@ -2744,17 +2683,12 @@ class URIParamsTest:
uri='file:///tmp/%(name)s',
uri_params=uri_params,
)
- crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
- messages = tuple(
- str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning
- )
- self.assertEqual(messages, tuple())
self.assertEqual(
feed_exporter.slots[0].uri,
@@ -2769,17 +2703,12 @@ class URIParamsTest:
uri='file:///tmp/%(foo)s',
uri_params=uri_params,
)
- crawler = get_crawler(settings_dict=settings)
- feed_exporter = FeedExporter.from_crawler(crawler)
+ crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
- with warnings.catch_warnings(record=True) as w:
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
- messages = tuple(
- str(item.message) for item in w
- if item.category is ScrapyDeprecationWarning
- )
- self.assertEqual(messages, tuple())
self.assertEqual(
feed_exporter.slots[0].uri,
@@ -2788,6 +2717,7 @@ class URIParamsTest:
class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
+ deprecated_options = True
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
extra_settings = {}
@@ -2800,6 +2730,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
+ deprecated_options = False
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
options = {
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 6381f895b..f3bb23bda 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -5,8 +5,8 @@ from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase as TwistedTestCase
-from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import DropItem
+from scrapy.utils.test import get_crawler
from scrapy.http import Request, Response
from scrapy.item import Item, Field
from scrapy.logformatter import LogFormatter
@@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
@defer.inlineCallbacks
def test_show_messages(self):
- crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+ crawler = get_crawler(ItemSpider, self.base_settings)
with LogCapture() as lc:
yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
@@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
def test_skip_messages(self):
settings = self.base_settings.copy()
settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
- crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+ crawler = get_crawler(ItemSpider, settings)
with LogCapture() as lc:
yield crawler.crawl(mockserver=self.mockserver)
self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index f49fda701..e46532a1c 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):
self.tmpmediastore = self.mktemp()
os.mkdir(self.tmpmediastore)
self.settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'ITEM_PIPELINES': {self.pipeline_class: 1},
self.store_setting_key: self.tmpmediastore,
}
@@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase):
def _on_item_scraped(self, item):
self.items.append(item)
- def _create_crawler(self, spider_class, **kwargs):
- crawler = self.runner.create_crawler(spider_class, **kwargs)
+ def _create_crawler(self, spider_class, runner=None, **kwargs):
+ if runner is None:
+ runner = self.runner
+ crawler = runner.create_crawler(spider_class, **kwargs)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
return crawler
@@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_redirected_allowed(self):
settings = dict(self.settings)
settings.update({'MEDIA_ALLOW_REDIRECTS': True})
- self.runner = CrawlerRunner(settings)
-
- crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+ runner = CrawlerRunner(settings)
+ crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
with LogCapture() as log:
yield crawler.crawl(
self.mockserver.url("/files/images/"),
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 25d9657d5..0406d906f 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -2,8 +2,8 @@ from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy import Request, signals
-from scrapy.crawler import CrawlerRunner
from scrapy.http.response import Response
+from scrapy.utils.test import get_crawler
from testfixtures import LogCapture
@@ -71,7 +71,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_response_200(self):
url = self.mockserver.url("/status?n=200")
- crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.request.url, url)
@@ -80,7 +80,7 @@ class CrawlTestCase(TestCase):
def test_response_error(self):
for status in ("404", "500"):
url = self.mockserver.url(f"/status?n={status}")
- crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
failure = crawler.spider.meta["failure"]
response = failure.value.response
@@ -90,12 +90,11 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_downloader_middleware_raise_exception(self):
url = self.mockserver.url("/status?n=200")
- runner = CrawlerRunner(settings={
+ crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
RaiseExceptionRequestMiddleware: 590,
},
})
- crawler = runner.create_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
failure = crawler.spider.meta["failure"]
self.assertEqual(failure.request.url, url)
@@ -117,12 +116,11 @@ class CrawlTestCase(TestCase):
signal_params["request"] = request
url = self.mockserver.url("/status?n=200")
- runner = CrawlerRunner(settings={
+ crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
ProcessResponseMiddleware: 595,
}
})
- crawler = runner.create_crawler(SingleRequestSpider)
crawler.signals.connect(signal_handler, signal=signals.response_received)
with LogCapture() as log:
@@ -147,13 +145,12 @@ class CrawlTestCase(TestCase):
The spider callback should receive the overridden response.request
"""
url = self.mockserver.url("/status?n=200")
- runner = CrawlerRunner(settings={
+ crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
RaiseExceptionRequestMiddleware: 590,
CatchExceptionOverrideRequestMiddleware: 595,
},
})
- crawler = runner.create_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -168,13 +165,12 @@ class CrawlTestCase(TestCase):
The spider callback should receive the original response.request
"""
url = self.mockserver.url("/status?n=200")
- runner = CrawlerRunner(settings={
+ crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
RaiseExceptionRequestMiddleware: 590,
CatchExceptionDoNotOverrideRequestMiddleware: 595,
},
})
- crawler = runner.create_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -186,12 +182,11 @@ class CrawlTestCase(TestCase):
Downloader middleware which returns a response with a
specific 'request' attribute, with an alternative callback
"""
- runner = CrawlerRunner(settings={
+ crawler = get_crawler(AlternativeCallbacksSpider, {
"DOWNLOADER_MIDDLEWARES": {
AlternativeCallbacksMiddleware: 595,
}
})
- crawler = runner.create_crawler(AlternativeCallbacksSpider)
with LogCapture() as log:
url = self.mockserver.url("/status?n=200")
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 473a93e69..002a04358 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -3,7 +3,7 @@ from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy.http import Request
-from scrapy.crawler import CrawlerRunner
+from scrapy.utils.test import get_crawler
from tests.spiders import MockServerSpider
from tests.mockserver import MockServer
@@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
- self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_callback_kwargs(self):
- crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+ crawler = get_crawler(KeywordArgumentsSpider)
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(all(crawler.spider.checks))
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 2d4bfa165..ac66056ba 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -52,6 +52,7 @@ class MockCrawler(Crawler):
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
JOBDIR=jobdir,
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
+ REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION',
)
super().__init__(Spider, settings)
self.engine = MockEngine(downloader=MockDownloader())
@@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase):
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
CONCURRENT_REQUESTS_PER_IP=1,
)
- crawler = Crawler(Spider, settings)
+ crawler = get_crawler(Spider, settings)
scheduler = Scheduler.from_crawler(crawler)
spider = Spider(name='spider')
scheduler.open(spider)
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index bf90b4320..fc234a83d 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -7,10 +7,10 @@ from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
from scrapy.core.scheduler import BaseScheduler
-from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.spiders import Spider
-from scrapy.utils.request import request_fingerprint
+from scrapy.utils.request import fingerprint
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
@@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS]
class MinimalScheduler:
def __init__(self) -> None:
- self.requests: Dict[str, Request] = {}
+ self.requests: Dict[bytes, Request] = {}
def has_pending_requests(self) -> bool:
return bool(self.requests)
def enqueue_request(self, request: Request) -> bool:
- fp = request_fingerprint(request)
+ fp = fingerprint(request)
if fp not in self.requests:
self.requests[fp] = request
return True
@@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):
@defer.inlineCallbacks
def test_crawl(self):
with MockServer() as mockserver:
- settings = {"SCHEDULER": self.scheduler_cls}
+ settings = {
+ "SCHEDULER": self.scheduler_cls,
+ }
with LogCapture() as log:
- yield CrawlerRunner(settings).crawl(TestSpider, mockserver)
+ crawler = get_crawler(TestSpider, settings)
+ yield crawler.crawl(mockserver)
for path in PATHS:
self.assertIn(f"{{'path': '{path}'}}", str(log))
self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log))
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 8a35e9fd7..3719c7c9f 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase):
def test_crawler_runner_loading(self):
module = 'tests.test_spiderloader.test_spiders.spider1'
- runner = CrawlerRunner({'SPIDER_MODULES': [module]})
+ runner = CrawlerRunner({
+ 'SPIDER_MODULES': [module],
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+ })
self.assertRaisesRegex(KeyError, 'Spider not found',
runner.create_crawler, 'spider2')
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 1ef4eeb14..46452415a 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -3,6 +3,7 @@ import os
import tempfile
import shutil
import contextlib
+import warnings
from pytest import warns
@@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase):
envvars = {
'SCRAPY_SETTINGS_MODULE': value,
}
- with set_env(**envvars), warns(None) as warnings:
- settings = get_project_settings()
- assert not warnings
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ with set_env(**envvars):
+ settings = get_project_settings()
+
assert settings.get('SETTINGS_MODULE') == value
def test_invalid_envvar(self):
envvars = {
'SCRAPY_FOO': 'bar',
}
- with set_env(**envvars), warns(None) as warnings:
- get_project_settings()
- assert len(warnings) == 1
- assert warnings[0].category == ScrapyDeprecationWarning
- assert str(warnings[0].message).endswith(': FOO')
+ with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+ with set_env(**envvars):
+ get_project_settings()
+ assert len(record) == 1
def test_valid_and_invalid_envvars(self):
value = 'tests.test_cmdline.settings'
@@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase):
'SCRAPY_FOO': 'bar',
'SCRAPY_SETTINGS_MODULE': value,
}
- with set_env(**envvars), warns(None) as warnings:
- settings = get_project_settings()
- assert len(warnings) == 1
- assert warnings[0].category == ScrapyDeprecationWarning
- assert str(warnings[0].message).endswith(': FOO')
+ with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+ with set_env(**envvars):
+ settings = get_project_settings()
+ assert len(record) == 1
assert settings.get('SETTINGS_MODULE') == value
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index e9edfee98..5ee772c0b 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -308,13 +308,22 @@ class RequestFingerprintTest(FingerprintTest):
),
)
+ def setUp(self) -> None:
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+
+ def tearDown(self) -> None:
+ warnings.simplefilter("default", ScrapyDeprecationWarning)
+
@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
def test_part_separation(self):
super().test_part_separation()
+
+class RequestFingerprintDeprecationTest(unittest.TestCase):
+
def test_deprecation_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
- self.function(Request("http://www.example.com"))
+ request_fingerprint(Request("http://www.example.com"))
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any(
@@ -326,7 +335,7 @@ class RequestFingerprintTest(FingerprintTest):
def test_deprecation_non_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
- self.function(Request("http://www.example.com"), keep_fragments=True)
+ request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any(
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 0a09f6109..d20852e62 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,7 +1,9 @@
import os
import unittest
+import warnings
from urllib.parse import urlparse
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
@@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase):
dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
def test_response_httprepr(self):
- r1 = Response("http://www.example.com")
- self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
- self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+ r1 = Response("http://www.example.com")
+ self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
- r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
- self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
+ r1 = Response("http://www.example.com", status=404,
+ headers={"Content-type": "text/html"}, body=b"Some body")
+ self.assertEqual(response_httprepr(r1),
+ b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+
+ r1 = Response("http://www.example.com", status=6666,
+ headers={"Content-type": "text/html"}, body=b"Some body")
+ self.assertEqual(response_httprepr(r1),
+ b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
def test_open_in_browser(self):
url = "http:///www.example.com/some/page.html"