Scrapy, a fast high-level web crawling & scraping framework for Python.
Switch to pytest.raises(). (#6680)
Andrey Rakhmatullin8d92c28a16c7
diff --git a/pyproject.toml b/pyproject.tomlindex ad62ea212..82d8056f6 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -378,8 +378,6 @@ ignore = [# Temporarily silenced PT rules# Use a regular `assert` instead of unittest-style `assertEqual`"PT009",- # Use `pytest.raises` instead of unittest-style `assertRaises`- "PT027",][tool.ruff.lint.per-file-ignores]diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex a04e0107b..324a9b955 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -214,7 +214,7 @@ class DownloaderAwarePriorityQueue:"DownloaderAwarePriorityQueue accepts ""``slot_startprios`` as a dict; "f"{slot_startprios.__class__!r} instance "- "is passed. Most likely, it means the state is"+ "is passed. Most likely, it means the state is ""created by an incompatible priority queue. ""Only a crawl started with the same priority ""queue class can be resumed."diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 425188d32..df5ebfa7b 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -185,9 +185,8 @@ class CrawlerTestCase(BaseCrawlerTest):def test_get_downloader_middleware_not_crawling(self):crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)- self.assertRaises(- RuntimeError, crawler.get_downloader_middleware, DefaultSpider- )+ with pytest.raises(RuntimeError):+ crawler.get_downloader_middleware(DefaultSpider)@inlineCallbacksdef test_get_downloader_middleware_no_engine(self):@@ -266,7 +265,8 @@ class CrawlerTestCase(BaseCrawlerTest):def test_get_extension_not_crawling(self):crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)- self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)+ with pytest.raises(RuntimeError):+ crawler.get_extension(DefaultSpider)@inlineCallbacksdef test_get_extension_no_engine(self):@@ -345,7 +345,8 @@ class CrawlerTestCase(BaseCrawlerTest):def test_get_item_pipeline_not_crawling(self):crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)- self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)+ with pytest.raises(RuntimeError):+ crawler.get_item_pipeline(DefaultSpider)@inlineCallbacksdef test_get_item_pipeline_no_engine(self):@@ -424,7 +425,8 @@ class CrawlerTestCase(BaseCrawlerTest):def test_get_spider_middleware_not_crawling(self):crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)- self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)+ with pytest.raises(RuntimeError):+ crawler.get_spider_middleware(DefaultSpider)@inlineCallbacksdef test_get_spider_middleware_no_engine(self):@@ -537,7 +539,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,})- self.assertRaises(MultipleInvalid, CrawlerRunner, settings)+ with pytest.raises(MultipleInvalid):+ CrawlerRunner(settings)def test_crawler_runner_accepts_dict(self):runner = CrawlerRunner({"foo": "bar"})@@ -630,13 +633,15 @@ class CrawlerRunnerHasSpider(unittest.TestCase):})else:- msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"- with self.assertRaisesRegex(Exception, msg):- runner = CrawlerRunner(- settings={- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",- }- )+ runner = CrawlerRunner(+ settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+ )+ with pytest.raises(+ Exception,+ match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",+ ):yield runner.crawl(NoRequestsSpider)diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.pyindex 174bf841e..17d5c2d0a 100644--- a/tests/test_downloader_handlers_http2.py+++ b/tests/test_downloader_handlers_http2.py@@ -248,5 +248,5 @@ class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase):@defer.inlineCallbacksdef test_download_with_proxy_https_timeout(self):- with self.assertRaises(NotImplementedError):+ with pytest.raises(NotImplementedError):yield super().test_download_with_proxy_https_timeout()diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 42051042c..49498375c 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -1,4 +1,5 @@import asyncio+from gzip import BadGzipFilefrom unittest import mockimport pytest@@ -106,7 +107,8 @@ class DefaultsTest(ManagerTestCase):"Location": "http://example.com/login",},)- self.assertRaises(OSError, self._download, request=req, response=resp)+ with pytest.raises(BadGzipFile):+ self._download(request=req, response=resp)class ResponseFromProcessRequestTest(ManagerTestCase):diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 772769690..694a669d4 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -83,11 +83,10 @@ class CookiesMiddlewareTest(TestCase):self.assertEqual(req2.headers.get("Cookie"), b"C1=value1")def test_setting_false_cookies_enabled(self):- self.assertRaises(- NotConfigured,- CookiesMiddleware.from_crawler,- get_crawler(settings_dict={"COOKIES_ENABLED": False}),- )+ with pytest.raises(NotConfigured):+ CookiesMiddleware.from_crawler(+ get_crawler(settings_dict={"COOKIES_ENABLED": False})+ )def test_setting_default_cookies_enabled(self):self.assertIsInstance(diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex 581fc1974..0f1489344 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -1,5 +1,6 @@import unittest+import pytestfrom w3lib.http import basic_auth_headerfrom scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware@@ -29,8 +30,8 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):self.spider = LegacySpider("foo")def test_auth(self):- with self.assertRaises(AttributeError):- mw = HttpAuthMiddleware()+ mw = HttpAuthMiddleware()+ with pytest.raises(AttributeError):mw.spider_opened(self.spider)diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 74db93f8a..de3a9689b 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -5,6 +5,8 @@ import timeimport unittestfrom contextlib import contextmanager+import pytest+from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddlewarefrom scrapy.exceptions import IgnoreRequestfrom scrapy.http import HtmlResponse, Request, Response@@ -192,9 +194,8 @@ class DummyPolicyTest(_BaseTest):def test_middleware_ignore_missing(self):with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:- self.assertRaises(- IgnoreRequest, mw.process_request, self.request, self.spider- )+ with pytest.raises(IgnoreRequest):+ mw.process_request(self.request, self.spider)mw.process_response(self.request, self.response, self.spider)response = mw.process_request(self.request, self.spider)assert isinstance(response, HtmlResponse)diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex a1c5883ec..b3e3b98d7 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -4,6 +4,7 @@ from logging import WARNINGfrom pathlib import Pathfrom unittest import SkipTest, TestCase+import pytestfrom testfixtures import LogCapturefrom w3lib.encoding import resolve_encoding@@ -87,11 +88,10 @@ class HttpCompressionTest(TestCase):)def test_setting_false_compression_enabled(self):- self.assertRaises(- NotConfigured,- HttpCompressionMiddleware.from_crawler,- get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),- )+ with pytest.raises(NotConfigured):+ HttpCompressionMiddleware.from_crawler(+ get_crawler(settings_dict={"COMPRESSION_ENABLED": False})+ )def test_setting_default_compression_enabled(self):self.assertIsInstance(@@ -520,13 +520,8 @@ class HttpCompressionTest(TestCase):mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")- self.assertRaises(- IgnoreRequest,- mw.process_response,- response.request,- response,- spider,- )+ with pytest.raises(IgnoreRequest):+ mw.process_response(response.request, response, spider)def test_compression_bomb_setting_br(self):try:@@ -561,13 +556,8 @@ class HttpCompressionTest(TestCase):mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")- self.assertRaises(- IgnoreRequest,- mw.process_response,- response.request,- response,- spider,- )+ with pytest.raises(IgnoreRequest):+ mw.process_response(response.request, response, spider)def test_compression_bomb_spider_attr_br(self):try:@@ -600,13 +590,8 @@ class HttpCompressionTest(TestCase):response = self._getresponse(f"bomb-{compression_id}")response.meta["download_maxsize"] = 10_000_000- self.assertRaises(- IgnoreRequest,- mw.process_response,- response.request,- response,- spider,- )+ with pytest.raises(IgnoreRequest):+ mw.process_response(response.request, response, spider)def test_compression_bomb_request_meta_br(self):try:diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex f950906e9..47abeee7a 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -72,9 +72,8 @@ class Base:assert isinstance(req, Request)assert "redirect_times" in req.metaself.assertEqual(req.meta["redirect_times"], 1)- self.assertRaises(- IgnoreRequest, self.mw.process_response, req, rsp, self.spider- )+ with pytest.raises(IgnoreRequest):+ self.mw.process_response(req, rsp, self.spider)def test_ttl(self):self.mw.max_redirect_times = 100@@ -83,9 +82,8 @@ class Base:req = self.mw.process_response(req, rsp, self.spider)assert isinstance(req, Request)- self.assertRaises(- IgnoreRequest, self.mw.process_response, req, rsp, self.spider- )+ with pytest.raises(IgnoreRequest):+ self.mw.process_response(req, rsp, self.spider)def test_redirect_urls(self):req1 = Request("http://scrapytest.org/first")diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 6b9b39413..36f48db69 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -1,6 +1,7 @@import loggingimport unittest+import pytestfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.internet.error import (@@ -407,7 +408,7 @@ class GetRetryRequestTest(unittest.TestCase):def test_no_spider(self):request = Request("https://example.com")- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):get_retry_request(request) # pylint: disable=missing-kwoadef test_max_retry_times_setting(self):diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 535e07c1f..9b95400fd 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -1,5 +1,6 @@from unittest import mock+import pytestfrom twisted.internet import error, reactorfrom twisted.internet.defer import Deferred, DeferredList, maybeDeferredfrom twisted.python import failure@@ -26,7 +27,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):def test_robotstxt_settings(self):self.crawler.settings = Settings()self.crawler.settings.set("USER_AGENT", "CustomAgent")- self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler)+ with pytest.raises(NotConfigured):+ RobotsTxtMiddleware(self.crawler)def _get_successful_crawler(self):crawler = self.crawlerdiff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex eb8d309b6..48728e078 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -10,6 +10,7 @@ from io import BytesIOfrom typing import Anyimport lxml.etree+import pytestfrom itemadapter import ItemAdapterfrom scrapy.exporters import (@@ -147,7 +148,7 @@ class PythonItemExporterTest(BaseItemExporterTest):return PythonItemExporter(**kwargs)def test_invalid_option(self):- with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):+ with pytest.raises(TypeError, match="Unexpected options: invalid_option"):PythonItemExporter(invalid_option="something")def test_nested_item(self):@@ -388,7 +389,7 @@ class CsvItemExporterTest(BaseItemExporterTest):)def test_errors_default(self):- with self.assertRaises(UnicodeEncodeError):+ with pytest.raises(UnicodeEncodeError):self.assertExportResult(item={"text": "W\u0275\u200brd"},expected=None,@@ -549,7 +550,8 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):self.ie = self._get_exporter(sort_keys=True)self.test_export_item()self._check_output()- self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True)+ with pytest.raises(TypeError):+ self._get_exporter(foo_unknown_keyword_bar=True)def test_nonstring_types_item(self):item = self._get_nonstring_types_item()@@ -602,7 +604,8 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):i3 = MyItem(name="Jesus", age="44")self.ie.start_exporting()self.ie.export_item(i1)- self.assertRaises(TypeError, self.ie.export_item, i2)+ with pytest.raises(TypeError):+ self.ie.export_item(i2)self.ie.export_item(i3)self.ie.finish_exporting()exported = json.loads(to_unicode(self.output.getvalue()))@@ -657,7 +660,8 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):i3 = MyItem(name="Jesus", age="44")self.ie.start_exporting()self.ie.export_item(i1)- self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)+ with pytest.raises(UnicodeEncodeError):+ self.ie.export_item(i2)self.ie.export_item(i3)self.ie.finish_exporting()exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin"))diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 1620d2d41..b4c1b9631 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -233,7 +233,8 @@ class BlockingFeedStorageTest(unittest.TestCase):invalid_path = tests_path / "invalid_path"spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})- self.assertRaises(OSError, b.open, spider=spider)+ with pytest.raises(OSError, match="Not a Directory:"):+ b.open(spider=spider)@pytest.mark.requires_boto3@@ -2437,7 +2438,8 @@ class BatchDeliveriesTest(FeedExportTestBase):"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}crawler = get_crawler(settings_dict=settings)- self.assertRaises(NotConfigured, FeedExporter, crawler)+ with pytest.raises(NotConfigured):+ FeedExporter(crawler)@defer.inlineCallbacksdef test_export_no_items_not_store_empty(self):@@ -2758,7 +2760,7 @@ class FeedExportInitTest(unittest.TestCase):},}crawler = get_crawler(settings_dict=settings)- with self.assertRaises(NotConfigured):+ with pytest.raises(NotConfigured):FeedExporter.from_crawler(crawler)def test_unsupported_format(self):@@ -2770,7 +2772,7 @@ class FeedExportInitTest(unittest.TestCase):},}crawler = get_crawler(settings_dict=settings)- with self.assertRaises(NotConfigured):+ with pytest.raises(NotConfigured):FeedExporter.from_crawler(crawler)def test_absolute_pathlib_as_uri(self):@@ -2863,7 +2865,7 @@ class URIParamsTest:with warnings.catch_warnings():warnings.simplefilter("error", ScrapyDeprecationWarning)- with self.assertRaises(KeyError):+ with pytest.raises(KeyError):feed_exporter.open_spider(spider)def test_params_as_is(self):diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex ddc772236..0881bbeca 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -12,6 +12,7 @@ from typing import TYPE_CHECKINGfrom unittest import mock, skipIffrom urllib.parse import urlencode+import pytestfrom twisted.internet import reactorfrom twisted.internet.defer import (CancelledError,@@ -406,7 +407,7 @@ class Https2ClientProtocolTestCase(TestCase):"scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"):request = Request(url=self.get_url("/status?n=200"))- with self.assertRaises(ResponseFailed):+ with pytest.raises(ResponseFailed):yield self.make_request(request)def test_cancel_request(self):@@ -560,7 +561,7 @@ class Https2ClientProtocolTestCase(TestCase):return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)def test_invalid_request_type(self):- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):self.make_request("https://InvalidDataTypePassed.com")def test_query_parameters(self):diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 7db1eb8c5..0bbbcda46 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -1,6 +1,8 @@import copyimport unittest+import pytest+from scrapy.http import Headers@@ -13,7 +15,8 @@ class HeadersTest(unittest.TestCase):assert h["Content-Type"]assert h["Content-Length"]- self.assertRaises(KeyError, h.__getitem__, "Accept")+ with pytest.raises(KeyError):+ h["Accept"]self.assertEqual(h.get("Accept"), None)self.assertEqual(h.getlist("Accept"), [])@@ -152,15 +155,11 @@ class HeadersTest(unittest.TestCase):self.assertEqual(h1.getlist("hey"), [b"5"])def test_invalid_value(self):- self.assertRaisesRegex(- TypeError, "Unsupported value type", Headers, {"foo": object()}- )- self.assertRaisesRegex(- TypeError, "Unsupported value type", Headers().__setitem__, "foo", object()- )- self.assertRaisesRegex(- TypeError, "Unsupported value type", Headers().setdefault, "foo", object()- )- self.assertRaisesRegex(- TypeError, "Unsupported value type", Headers().setlist, "foo", [object()]- )+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers({"foo": object()})+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers()["foo"] = object()+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers().setdefault("foo", object())+ with pytest.raises(TypeError, match="Unsupported value type"):+ Headers().setlist("foo", [object()])diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex a8ab8240f..e5291157d 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -7,6 +7,8 @@ from typing import Anyfrom unittest import mockfrom urllib.parse import parse_qs, unquote_to_bytes+import pytest+from scrapy.http import (FormRequest,Headers,@@ -28,10 +30,12 @@ class RequestTest(unittest.TestCase):def test_init(self):# Request requires url in the __init__ method- self.assertRaises(Exception, self.request_class)+ with pytest.raises(TypeError):+ self.request_class()# url argument must be basestring- self.assertRaises(TypeError, self.request_class, 123)+ with pytest.raises(TypeError):+ self.request_class(123)r = self.request_class("http://www.example.com")r = self.request_class("http://www.example.com")@@ -64,9 +68,13 @@ class RequestTest(unittest.TestCase):self.request_class("data:,Hello%2C%20World!")def test_url_no_scheme(self):- self.assertRaises(ValueError, self.request_class, "foo")- self.assertRaises(ValueError, self.request_class, "/foo/")- self.assertRaises(ValueError, self.request_class, "/foo:bar")+ msg = "Missing scheme in request url:"+ with pytest.raises(ValueError, match=msg):+ self.request_class("foo")+ with pytest.raises(ValueError, match=msg):+ self.request_class("/foo/")+ with pytest.raises(ValueError, match=msg):+ self.request_class("/foo:bar")def test_headers(self):# Different ways of setting headers attribute@@ -273,8 +281,10 @@ class RequestTest(unittest.TestCase):def test_immutable_attributes(self):r = self.request_class("http://example.com")- self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")- self.assertRaises(AttributeError, setattr, r, "body", "xxx")+ with pytest.raises(AttributeError):+ r.url = "http://example2.com"+ with pytest.raises(AttributeError):+ r.body = "xxx"def test_callback_and_errback(self):def a_function():@@ -309,11 +319,11 @@ class RequestTest(unittest.TestCase):self.assertIs(r5.errback, NO_CALLBACK)def test_callback_and_errback_type(self):- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):self.request_class("http://example.com", callback="a_function")- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):self.request_class("http://example.com", errback="a_function")- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):self.request_class(url="http://example.com",callback="a_function",@@ -321,7 +331,7 @@ class RequestTest(unittest.TestCase):)def test_no_callback(self):- with self.assertRaises(RuntimeError):+ with pytest.raises(RuntimeError):NO_CALLBACK()def test_from_curl(self):@@ -403,13 +413,11 @@ class RequestTest(unittest.TestCase):# If `ignore_unknown_options` is set to `False` it raises an error with# the unknown options: --foo and -z- self.assertRaises(- ValueError,- lambda: self.request_class.from_curl(+ with pytest.raises(ValueError, match="Unrecognized options:"):+ self.request_class.from_curl('curl -X PATCH "http://example.org" --foo -z',ignore_unknown_options=False,- ),- )+ )class FormRequestTest(RequestTest):@@ -428,7 +436,7 @@ class FormRequestTest(RequestTest):data = (("a", "one"), ("a", "two"), ("b", "2"))url = self.request_class("http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data- ).url.split("#")[0]+ ).url.split("#", maxsplit=1)[0]fs = _qs(self.request_class(url, method="GET", formdata=data))self.assertEqual(set(fs[b"a"]), {b"one", b"two"})self.assertEqual(fs[b"b"], [b"2"])@@ -897,12 +905,11 @@ class FormRequestTest(RequestTest):<input type="submit" name="clickable2" value="clicked2"></form>""")- self.assertRaises(+ with pytest.raises(ValueError,- self.request_class.from_response,- response,- clickdata={"type": "submit"},- )+ match="Multiple elements found .* matching the criteria in clickdata",+ ):+ self.request_class.from_response(response, clickdata={"type": "submit"})def test_from_response_non_matching_clickdata(self):response = _buildresponse(@@ -910,12 +917,12 @@ class FormRequestTest(RequestTest):<input type="submit" name="clickable" value="clicked"></form>""")- self.assertRaises(- ValueError,- self.request_class.from_response,- response,- clickdata={"nonexistent": "notme"},- )+ with pytest.raises(+ ValueError, match="No clickable element matching clickdata:"+ ):+ self.request_class.from_response(+ response, clickdata={"nonexistent": "notme"}+ )def test_from_response_nr_index_clickdata(self):response = _buildresponse(@@ -937,13 +944,15 @@ class FormRequestTest(RequestTest):</form>""")- self.assertRaises(- ValueError, self.request_class.from_response, response, clickdata={"nr": 1}- )+ with pytest.raises(+ ValueError, match="No clickable element matching clickdata:"+ ):+ self.request_class.from_response(response, clickdata={"nr": 1})def test_from_response_errors_noform(self):response = _buildresponse("""<html></html>""")- self.assertRaises(ValueError, self.request_class.from_response, response)+ with pytest.raises(ValueError, match="No <form> element found in"):+ self.request_class.from_response(response)def test_from_response_invalid_html5(self):response = _buildresponse(@@ -963,9 +972,8 @@ class FormRequestTest(RequestTest):<input type="hidden" name="test2" value="xxx"></form>""")- self.assertRaises(- IndexError, self.request_class.from_response, response, formnumber=1- )+ with pytest.raises(IndexError):+ self.request_class.from_response(response, formnumber=1)def test_from_response_noformname(self):response = _buildresponse(@@ -1021,13 +1029,8 @@ class FormRequestTest(RequestTest):<input type="hidden" name="two" value="2"></form>""")- self.assertRaises(- IndexError,- self.request_class.from_response,- response,- formname="form3",- formnumber=2,- )+ with pytest.raises(IndexError):+ self.request_class.from_response(response, formname="form3", formnumber=2)def test_from_response_formid_exists(self):response = _buildresponse(@@ -1086,13 +1089,8 @@ class FormRequestTest(RequestTest):<input type="hidden" name="two" value="2"></form>""")- self.assertRaises(- IndexError,- self.request_class.from_response,- response,- formid="form3",- formnumber=2,- )+ with pytest.raises(IndexError):+ self.request_class.from_response(response, formid="form3", formnumber=2)def test_from_response_select(self):res = _buildresponse(@@ -1245,12 +1243,10 @@ class FormRequestTest(RequestTest):fs = _qs(r1)self.assertEqual(fs[b"three"], [b"3"])- self.assertRaises(- ValueError,- self.request_class.from_response,- response,- formxpath="//form/input[@name='abc']",- )+ with pytest.raises(ValueError, match="No <form> element found with"):+ self.request_class.from_response(+ response, formxpath="//form/input[@name='abc']"+ )def test_from_response_unicode_xpath(self):response = _buildresponse(b'<form name="\xd1\x8a"></form>')@@ -1261,13 +1257,8 @@ class FormRequestTest(RequestTest):self.assertEqual(fs, {})xpath = "//form[@name='\u03b1']"- self.assertRaisesRegex(- ValueError,- re.escape(xpath),- self.request_class.from_response,- response,- formxpath=xpath,- )+ with pytest.raises(ValueError, match=re.escape(xpath)):+ self.request_class.from_response(response, formxpath=xpath)def test_from_response_button_submit(self):response = _buildresponse(@@ -1393,12 +1384,8 @@ class FormRequestTest(RequestTest):fs = _qs(r1)self.assertEqual(fs[b"three"], [b"3"])- self.assertRaises(- ValueError,- self.request_class.from_response,- response,- formcss="input[name='abc']",- )+ with pytest.raises(ValueError, match="No <form> element found with"):+ self.request_class.from_response(response, formcss="input[name='abc']")def test_from_response_valid_form_methods(self):form_methods = [@@ -1424,13 +1411,11 @@ class FormRequestTest(RequestTest):</form></body></html>""")- with self.assertRaises(ValueError) as context:+ with pytest.raises(+ ValueError, match="formdata should be a dict or iterable of tuples"+ ):FormRequest.from_response(response, formdata=123)- self.assertIn(- "formdata should be a dict or iterable of tuples", str(context.exception)- )-def test_form_response_with_custom_invalid_formdata_value_error(self):"""Test that a ValueError is raised for fault-inducing iterable formdata input"""response = _buildresponse(@@ -1441,13 +1426,11 @@ class FormRequestTest(RequestTest):</body></html>""")- with self.assertRaises(ValueError) as context:+ with pytest.raises(+ ValueError, match="formdata should be a dict or iterable of tuples"+ ):FormRequest.from_response(response, formdata=("a",))- self.assertIn(- "formdata should be a dict or iterable of tuples", str(context.exception)- )-def test_get_form_with_xpath_no_form_parent(self):"""Test that _get_from raised a ValueError when an XPath selects an elementnot nested within a <form> and no <form> parent is found"""@@ -1462,11 +1445,9 @@ class FormRequestTest(RequestTest):</body></html>""")- with self.assertRaises(ValueError) as context:+ with pytest.raises(ValueError, match="No <form> element found with"):FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p')- self.assertIn("No <form> element found with", str(context.exception))-def _buildresponse(body, **kwargs):kwargs.setdefault("body", body)@@ -1507,8 +1488,10 @@ class XmlRpcRequestTest(RequestTest):self._test_request(params=("response",), methodresponse="login")self._test_request(params=("pas£",), encoding="utf-8")self._test_request(params=(None,), allow_none=1)- self.assertRaises(TypeError, self._test_request)- self.assertRaises(TypeError, self._test_request, params=(None,))+ with pytest.raises(TypeError):+ self._test_request()+ with pytest.raises(TypeError):+ self._test_request(params=(None,))def test_latin1(self):self._test_request(params=("pas£",), encoding="latin1")diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex dde883451..5a943f084 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -27,14 +27,15 @@ class BaseResponseTest(unittest.TestCase):def test_init(self):# Response requires url in the constructor- self.assertRaises(Exception, self.response_class)+ with pytest.raises(TypeError):+ self.response_class()self.assertTrue(isinstance(self.response_class("http://example.com/"), self.response_class))- self.assertRaises(TypeError, self.response_class, b"http://example.com")- self.assertRaises(- TypeError, self.response_class, url="http://example.com", body={}- )+ with pytest.raises(TypeError):+ self.response_class(b"http://example.com")+ with pytest.raises(TypeError):+ self.response_class(url="http://example.com", body={})# body can be str or Noneself.assertTrue(isinstance(@@ -77,12 +78,8 @@ class BaseResponseTest(unittest.TestCase):self.assertEqual(r.status, 301)r = self.response_class("http://www.example.com", status="301")self.assertEqual(r.status, 301)- self.assertRaises(- ValueError,- self.response_class,- "http://example.com",- status="lala200",- )+ with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):+ self.response_class("http://example.com", status="lala200")def test_copy(self):"""Test Response copy"""@@ -122,14 +119,12 @@ class BaseResponseTest(unittest.TestCase):def test_unavailable_meta(self):r1 = self.response_class("http://www.example.com", body=b"Some body")- with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"):+ with pytest.raises(AttributeError, match=r"Response\.meta not available"):r1.metadef test_unavailable_cb_kwargs(self):r1 = self.response_class("http://www.example.com", body=b"Some body")- with self.assertRaisesRegex(- AttributeError, r"Response\.cb_kwargs not available"- ):+ with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):r1.cb_kwargsdef test_copy_inherited_classes(self):@@ -179,8 +174,10 @@ class BaseResponseTest(unittest.TestCase):def test_immutable_attributes(self):r = self.response_class("http://example.com")- self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")- self.assertRaises(AttributeError, setattr, r, "body", "xxx")+ with pytest.raises(AttributeError):+ r.url = "http://example2.com"+ with pytest.raises(AttributeError):+ r.body = "xxx"def test_urljoin(self):"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""@@ -192,10 +189,14 @@ class BaseResponseTest(unittest.TestCase):r = self.response_class("http://example.com", body=b"hello")if self.response_class == Response:msg = "Response content isn't text"- self.assertRaisesRegex(AttributeError, msg, getattr, r, "text")- self.assertRaisesRegex(NotSupported, msg, r.css, "body")- self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body")- self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body")+ with pytest.raises(AttributeError, match=msg):+ r.text+ with pytest.raises(NotSupported, match=msg):+ r.css("body")+ with pytest.raises(NotSupported, match=msg):+ r.xpath("//body")+ with pytest.raises(NotSupported, match=msg):+ r.jmespath("body")else:r.textr.css("body")@@ -216,7 +217,8 @@ class BaseResponseTest(unittest.TestCase):def test_follow_None_url(self):r = self.response_class("http://example.com")- self.assertRaises(ValueError, r.follow, None)+ with pytest.raises(ValueError, match="url can't be None"):+ r.follow(None)@pytest.mark.xfail(parse_version(w3lib_version) < parse_version("2.1.1"),@@ -279,18 +281,20 @@ class BaseResponseTest(unittest.TestCase):def test_follow_all_invalid(self):r = self.response_class("http://example.com")if self.response_class == Response:- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):list(r.follow_all(urls=None))- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):list(r.follow_all(urls=12345))- with self.assertRaises(ValueError):+ with pytest.raises(ValueError, match="url can't be None"):list(r.follow_all(urls=[None]))else:- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError, match="Please supply exactly one of the following arguments"+ ):list(r.follow_all(urls=None))- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):list(r.follow_all(urls=12345))- with self.assertRaises(ValueError):+ with pytest.raises(ValueError, match="url can't be None"):list(r.follow_all(urls=[None]))def test_follow_all_whitespace(self):@@ -399,12 +403,8 @@ class TextResponseTest(BaseResponseTest):"\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ""\u0442\u0435\u043a\u0441\u0442")- self.assertRaises(- TypeError,- self.response_class,- "http://www.example.com",- body="unicode body",- )+ with pytest.raises(TypeError):+ self.response_class("http://www.example.com", body="unicode body")original_string = unicode_string.encode("cp1251")r1 = self.response_class(@@ -483,12 +483,8 @@ class TextResponseTest(BaseResponseTest):self._assert_response_values(r9, "cp1252", "€")# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies- self.assertRaises(- TypeError,- self.response_class,- "http://www.example.com",- body="\xa3",- )+ with pytest.raises(TypeError):+ self.response_class("http://www.example.com", body="\xa3")def test_declared_encoding_invalid(self):"""Check that unknown declared encodings are ignored"""@@ -679,20 +675,20 @@ class TextResponseTest(BaseResponseTest):self._assert_followed_url(sel, url, response=resp)# non-a elements are not supported- self.assertRaises(ValueError, resp.follow, resp.css("div")[0])+ with pytest.raises(+ ValueError, match="Only <a> and <link> elements are supported"+ ):+ resp.follow(resp.css("div")[0])def test_follow_selector_list(self):resp = self._links_response()- self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a"))+ with pytest.raises(ValueError, match="SelectorList"):+ resp.follow(resp.css("a"))def test_follow_selector_invalid(self):resp = self._links_response()- self.assertRaisesRegex(- ValueError,- "Unsupported",- resp.follow,- resp.xpath("count(//div)")[0],- )+ with pytest.raises(ValueError, match="Unsupported"):+ resp.follow(resp.xpath("count(//div)")[0])def test_follow_selector_attribute(self):resp = self._links_response()@@ -704,7 +700,8 @@ class TextResponseTest(BaseResponseTest):url="http://example.com",body=b"<html><body><a name=123>click me</a></body></html>",)- self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0])+ with pytest.raises(ValueError, match="no href"):+ resp.follow(resp.css("a")[0])def test_follow_whitespace_selector(self):resp = self.response_class(@@ -812,7 +809,9 @@ class TextResponseTest(BaseResponseTest):def test_follow_all_too_many_arguments(self):response = self._links_response()- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError, match="Please supply exactly one of the following arguments"+ ):response.follow_all(css='a[href*="example.com"]',xpath='//a[contains(@href, "example.com")]',@@ -825,7 +824,9 @@ class TextResponseTest(BaseResponseTest):text_body = b"""<html><body>text</body></html>"""text_response = self.response_class("http://www.example.com", body=text_body)- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError, match="(Expecting value|Unexpected '<'): line 1"+ ):text_response.json()def test_cache_json_response(self):@@ -1023,10 +1024,8 @@ class CustomResponseTest(TextResponseTest):self.assertEqual(r4.bar, "bar")self.assertIsNone(r4.lost)- with self.assertRaises(TypeError) as ctx:+ with pytest.raises(+ TypeError,+ match=r"__init__\(\) got an unexpected keyword argument 'unknown'",+ ):r1.replace(unknown="unknown")- self.assertTrue(- str(ctx.exception).endswith(- "__init__() got an unexpected keyword argument 'unknown'"- )- )diff --git a/tests/test_item.py b/tests/test_item.pyindex 0399c8f8d..47c5c3db6 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -2,6 +2,8 @@ import unittestfrom abc import ABCMetafrom unittest import mock+import pytest+from scrapy.item import Field, Item, ItemMeta@@ -22,7 +24,8 @@ class ItemTest(unittest.TestCase):name = Field()i = TestItem()- self.assertRaises(KeyError, i.__getitem__, "name")+ with pytest.raises(KeyError):+ i["name"]i2 = TestItem(name="john doe")self.assertEqual(i2["name"], "john doe")@@ -33,15 +36,18 @@ class ItemTest(unittest.TestCase):i4 = TestItem(i3)self.assertEqual(i4["name"], "john doe")- self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"})+ with pytest.raises(KeyError):+ TestItem({"name": "john doe", "other": "foo"})def test_invalid_field(self):class TestItem(Item):passi = TestItem()- self.assertRaises(KeyError, i.__setitem__, "field", "text")- self.assertRaises(KeyError, i.__getitem__, "field")+ with pytest.raises(KeyError):+ i["field"] = "text"+ with pytest.raises(KeyError):+ i["field"]def test_repr(self):class TestItem(Item):@@ -72,14 +78,16 @@ class ItemTest(unittest.TestCase):name = Field()i = TestItem()- self.assertRaises(AttributeError, getattr, i, "name")+ with pytest.raises(AttributeError):+ i.namedef test_raise_setattr(self):class TestItem(Item):name = Field()i = TestItem()- self.assertRaises(AttributeError, setattr, i, "name", "john")+ with pytest.raises(AttributeError):+ i.name = "john"def test_custom_methods(self):class TestItem(Item):@@ -92,7 +100,8 @@ class ItemTest(unittest.TestCase):self["name"] = namei = TestItem()- self.assertRaises(KeyError, i.get_name)+ with pytest.raises(KeyError):+ i.get_name()i["name"] = "lala"self.assertEqual(i.get_name(), "lala")i.change_name("other")@@ -223,7 +232,8 @@ class ItemTest(unittest.TestCase):class D(B, C):pass- self.assertRaises(KeyError, D, not_allowed="value")+ with pytest.raises(KeyError):+ D(not_allowed="value")self.assertEqual(D(save="X")["save"], "X")self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}})@@ -231,7 +241,8 @@ class ItemTest(unittest.TestCase):class E(C, B):pass- self.assertRaises(KeyError, E, not_allowed="value")+ with pytest.raises(KeyError):+ E(not_allowed="value")self.assertEqual(E(save="X")["save"], "X")self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}})diff --git a/tests/test_link.py b/tests/test_link.pyindex 35723bbd6..ed9d27a37 100644--- a/tests/test_link.py+++ b/tests/test_link.py@@ -1,5 +1,7 @@import unittest+import pytest+from scrapy.link import Link@@ -53,5 +55,5 @@ class LinkTest(unittest.TestCase):self._assert_same_links(l1, l2)def test_bytes_url(self):- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):Link(b"http://www.example.com/\xc2\xa3")diff --git a/tests/test_loader.py b/tests/test_loader.pyindex b52d5ea2e..1a933bb8d 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -4,6 +4,7 @@ import dataclassesimport unittestimport attr+import pytestfrom itemadapter import ItemAdapterfrom itemloaders.processors import Compose, Identity, MapCompose, TakeFirst@@ -69,7 +70,8 @@ def processor_with_args(value, other=None, loader_context=None):class BasicItemLoaderTest(unittest.TestCase):def test_add_value_on_unknown_field(self):il = ProcessorItemLoader()- self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])+ with pytest.raises(KeyError):+ il.add_value("wrong_field", ["lala", "lolo"])def test_load_item_using_default_loader(self):i = SummaryItem()@@ -294,12 +296,18 @@ class SelectortemLoaderTest(unittest.TestCase):def test_init_method_errors(self):l = ProcessorItemLoader()- self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")- self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")- self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")- self.assertRaises(RuntimeError, l.add_css, "name", "#name::text")- self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text")- self.assertRaises(RuntimeError, l.get_css, "#name::text")+ with pytest.raises(RuntimeError):+ l.add_xpath("url", "//a/@href")+ with pytest.raises(RuntimeError):+ l.replace_xpath("url", "//a/@href")+ with pytest.raises(RuntimeError):+ l.get_xpath("//a/@href")+ with pytest.raises(RuntimeError):+ l.add_css("name", "#name::text")+ with pytest.raises(RuntimeError):+ l.replace_css("name", "#name::text")+ with pytest.raises(RuntimeError):+ l.get_css("#name::text")def test_init_method_with_selector(self):sel = Selector(text="<html><body><div>marta</div></body></html>")diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.pyindex 1e504f539..0d7921b1d 100644--- a/tests/test_loader_deprecated.py+++ b/tests/test_loader_deprecated.py@@ -6,6 +6,7 @@ Once we remove the references from scrapy, we can remove these tests.import unittestfrom functools import partial+import pytestfrom itemloaders.processors import (Compose,Identity,@@ -435,7 +436,13 @@ class BasicItemLoaderTest(unittest.TestCase):name_in = MapCompose(float)il = TestItemLoader()- self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"])+ with pytest.raises(+ ValueError,+ match="Error with input processor MapCompose: .* "+ "error='ValueError: Error in MapCompose .* "+ "error='ValueError: could not convert",+ ):+ il.add_value("name", ["marta", "other"])def test_error_output_processor(self):class TestItem(Item):@@ -447,7 +454,12 @@ class BasicItemLoaderTest(unittest.TestCase):il = TestItemLoader()il.add_value("name", "marta")- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError,+ match="Error with output processor: .* "+ "error='ValueError: Error in Compose .* "+ "error='ValueError: could not convert",+ ):il.load_item()def test_error_processor_as_argument(self):@@ -458,9 +470,13 @@ class BasicItemLoaderTest(unittest.TestCase):default_item_class = TestItemil = TestItemLoader()- self.assertRaises(- ValueError, il.add_value, "name", ["marta", "other"], Compose(float)- )+ with pytest.raises(+ ValueError,+ match=r"Error with processor Compose .* "+ r"error='ValueError: Error in Compose .* "+ r"error='TypeError: float\(\) argument",+ ):+ il.add_value("name", ["marta", "other"], Compose(float))class InitializationFromDictTest(unittest.TestCase):@@ -630,7 +646,8 @@ class ProcessorsTest(unittest.TestCase):def test_join(self):proc = Join()- self.assertRaises(TypeError, proc, [None, "", "hello", "world"])+ with pytest.raises(TypeError):+ proc([None, "", "hello", "world"])self.assertEqual(proc(["", "hello", "world"]), " hello world")self.assertEqual(proc(["hello", "world"]), "hello world")self.assertIsInstance(proc(["hello", "world"]), str)@@ -641,9 +658,17 @@ class ProcessorsTest(unittest.TestCase):proc = Compose(str.upper)self.assertEqual(proc(None), None)proc = Compose(str.upper, stop_on_none=False)- self.assertRaises(ValueError, proc, None)+ with pytest.raises(+ ValueError,+ match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",+ ):+ proc(None)proc = Compose(str.upper, lambda x: x + 1)- self.assertRaises(ValueError, proc, "hello")+ with pytest.raises(+ ValueError,+ match="Error in Compose with .* error='TypeError: (can only|unsupported operand)",+ ):+ proc("hello")def test_mapcompose(self):def filter_world(x):@@ -657,9 +682,17 @@ class ProcessorsTest(unittest.TestCase):proc = MapCompose(filter_world, str.upper)self.assertEqual(proc(None), [])proc = MapCompose(filter_world, str.upper)- self.assertRaises(ValueError, proc, [1])+ with pytest.raises(+ ValueError,+ match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",+ ):+ proc([1])proc = MapCompose(filter_world, lambda x: x + 1)- self.assertRaises(ValueError, proc, "hello")+ with pytest.raises(+ ValueError,+ match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)",+ ):+ proc("hello")class SelectJmesTestCase(unittest.TestCase):diff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex a4b002e34..6bc5b6f1f 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -1,6 +1,8 @@import unittestfrom datetime import datetime+import pytest+from scrapy.extensions.logstats import LogStatsfrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider@@ -18,8 +20,9 @@ class TestLogStats(unittest.TestCase):def test_stats_calculations(self):logstats = LogStats.from_crawler(self.crawler)- with self.assertRaises(AttributeError):+ with pytest.raises(AttributeError):logstats.pagesprev+ with pytest.raises(AttributeError):logstats.itemsprevlogstats.spider_opened(self.spider)diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 3d049843a..1d89e44ce 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -7,6 +7,7 @@ from shutil import rmtreefrom tempfile import mkdtempimport attr+import pytestfrom itemadapter import ItemAdapterfrom twisted.trial import unittest@@ -146,11 +147,11 @@ class ImagesPipelineTestCase(unittest.TestCase):resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue())req = Request(url="https://dev.mydeco.com/mydeco.gif")- with self.assertRaises(ImageException):+ with pytest.raises(ImageException):next(self.pipeline.get_images(response=resp1, request=req, info=object()))- with self.assertRaises(ImageException):+ with pytest.raises(ImageException):next(self.pipeline.get_images(response=resp2, request=req, info=object()))- with self.assertRaises(ImageException):+ with pytest.raises(ImageException):next(self.pipeline.get_images(response=resp3, request=req, info=object()))def test_get_images(self):diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex 1584014b8..c223c4562 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -1,6 +1,7 @@import tempfileimport unittest+import pytestimport queuelibfrom scrapy.http.request import Request@@ -40,9 +41,9 @@ class PriorityQueueTest(unittest.TestCase):self.crawler, FifoMemoryQueue, temp_dir)queue.push(Request("https://example.org"))- with self.assertRaises(+ with pytest.raises(NotImplementedError,- msg="The underlying queue class does not implement 'peek'",+ match="The underlying queue class does not implement 'peek'",):queue.peek()queue.close()@@ -129,9 +130,9 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase):if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")self.queue.push(Request("https://example.org"))- with self.assertRaises(+ with pytest.raises(NotImplementedError,- msg="The underlying queue class does not implement 'peek'",+ match="The underlying queue class does not implement 'peek'",):self.queue.peek()diff --git a/tests/test_request_dict.py b/tests/test_request_dict.pyindex 85133038a..2c605a015 100644--- a/tests/test_request_dict.py+++ b/tests/test_request_dict.py@@ -1,5 +1,7 @@import unittest+import pytest+from scrapy import Request, Spiderfrom scrapy.http import FormRequest, JsonRequestfrom scrapy.utils.request import request_from_dict@@ -134,11 +136,15 @@ class RequestSerializationTest(unittest.TestCase):def test_unserializable_callback1(self):r = Request("http://www.example.com", callback=lambda x: x)- self.assertRaises(ValueError, r.to_dict, spider=self.spider)+ with pytest.raises(+ ValueError, match="is not an instance method in: <MethodsSpider"+ ):+ r.to_dict(spider=self.spider)def test_unserializable_callback2(self):r = Request("http://www.example.com", callback=self.spider.parse_item)- self.assertRaises(ValueError, r.to_dict, spider=None)+ with pytest.raises(ValueError, match="is not an instance method in: None"):+ r.to_dict(spider=None)def test_unserializable_callback3(self):"""Parser method is removed or replaced dynamically."""@@ -152,14 +158,18 @@ class RequestSerializationTest(unittest.TestCase):spider = MySpider()r = Request("http://www.example.com", callback=spider.parse)spider.parse = None- self.assertRaises(ValueError, r.to_dict, spider=spider)+ with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):+ r.to_dict(spider=spider)def test_callback_not_available(self):"""Callback method is not available in the spider passed to from_dict"""spider = SpiderDelegation()r = Request("http://www.example.com", callback=spider.delegated_callback)d = r.to_dict(spider=spider)- self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))+ with pytest.raises(+ ValueError, match="Method 'delegated_callback' not found in: <Spider"+ ):+ request_from_dict(d, spider=Spider("foo"))class SpiderMixin:diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 3ac330ae2..f2f8b96cd 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -5,6 +5,7 @@ import tempfileimport unittestfrom typing import Any, NamedTuple+import pytestfrom twisted.internet import deferfrom twisted.trial.unittest import TestCase@@ -229,7 +230,10 @@ class TestMigration(unittest.TestCase):next_scheduler_handler.create_scheduler()def test_migration(self):- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError,+ match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict",+ ):self._migration(self.tmpdir)@@ -351,5 +355,7 @@ class TestIncompatibility(unittest.TestCase):scheduler.open(spider)def test_incompatibility(self):- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP"+ ):self._incompatible()diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 7c72805e2..c2bb8cec5 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom unittest import TestCasefrom urllib.parse import urljoin+import pytestfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial.unittest import TestCase as TwistedTestCase@@ -75,13 +76,12 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin):def test_methods(self):self.assertIsNone(self.scheduler.open(Spider("foo")))self.assertIsNone(self.scheduler.close("finished"))- self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests)- self.assertRaises(- NotImplementedError,- self.scheduler.enqueue_request,- Request("https://example.org"),- )- self.assertRaises(NotImplementedError, self.scheduler.next_request)+ with pytest.raises(NotImplementedError):+ self.scheduler.has_pending_requests()+ with pytest.raises(NotImplementedError):+ self.scheduler.enqueue_request(Request("https://example.org"))+ with pytest.raises(NotImplementedError):+ self.scheduler.next_request()class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):@@ -89,15 +89,15 @@ class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):self.scheduler = MinimalScheduler()def test_open_close(self):- with self.assertRaises(AttributeError):+ with pytest.raises(AttributeError):self.scheduler.open(Spider("foo"))- with self.assertRaises(AttributeError):+ with pytest.raises(AttributeError):self.scheduler.close("finished")def test_len(self):- with self.assertRaises(AttributeError):+ with pytest.raises(AttributeError):self.scheduler.__len__()- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):len(self.scheduler)def test_enqueue_dequeue(self):diff --git a/tests/test_selector.py b/tests/test_selector.pyindex 4eda0460f..2d7a1442e 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -112,7 +112,7 @@ class SelectorTestCase(unittest.TestCase):)def test_selector_bad_args(self):- with self.assertRaisesRegex(ValueError, "received both response and text"):+ with pytest.raises(ValueError, match="received both response and text"):Selector(TextResponse(url="http://example.com", body=b""), text="")diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 5c8a19d9b..b7a316eee 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -235,9 +235,9 @@ class BaseSettingsTest(unittest.TestCase):self.assertIn("key_highprio", settings)del settings["key_highprio"]self.assertNotIn("key_highprio", settings)- with self.assertRaises(KeyError):+ with pytest.raises(KeyError):settings.delete("notkey")- with self.assertRaises(KeyError):+ with pytest.raises(KeyError):del settings["notkey"]def test_get(self):@@ -303,9 +303,19 @@ class BaseSettingsTest(unittest.TestCase):self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3})self.assertEqual(settings.getdict("TEST_DICT3"), {})self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5})- self.assertRaises(ValueError, settings.getdict, "TEST_LIST1")- self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG")- self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG")+ with pytest.raises(+ ValueError,+ match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",+ ):+ settings.getdict("TEST_LIST1")+ with pytest.raises(+ ValueError, match="Supported values for boolean settings are"+ ):+ settings.getbool("TEST_ENABLED_WRONG")+ with pytest.raises(+ ValueError, match="Supported values for boolean settings are"+ ):+ settings.getbool("TEST_DISABLED_WRONG")def test_getpriority(self):settings = BaseSettings({"key": "value"}, priority=99)@@ -381,11 +391,10 @@ class BaseSettingsTest(unittest.TestCase):def test_freeze(self):self.settings.freeze()- with self.assertRaises(TypeError) as cm:+ with pytest.raises(+ TypeError, match="Trying to modify an immutable Settings object"+ ):self.settings.set("TEST_BOOL", False)- self.assertEqual(- str(cm.exception), "Trying to modify an immutable Settings object"- )def test_frozencopy(self):frozencopy = self.settings.frozencopy()@@ -476,7 +485,7 @@ class SettingsTest(unittest.TestCase):def test_pop_item_with_default_value(self):settings = Settings()- with self.assertRaises(KeyError):+ with pytest.raises(KeyError):settings.pop("DUMMY_CONFIG")dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value")@@ -491,9 +500,7 @@ class SettingsTest(unittest.TestCase):settings.freeze()- with self.assertRaises(TypeError) as error:+ with pytest.raises(+ TypeError, match="Trying to modify an immutable Settings object"+ ):settings.pop("OTHER_DUMMY_CONFIG")-- self.assertEqual(- str(error.exception), "Trying to modify an immutable Settings object"- )diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 18a863350..af29872a8 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -7,6 +7,7 @@ from pathlib import Pathfrom typing import Anyfrom unittest import mock+import pytestfrom testfixtures import LogCapturefrom twisted.internet.defer import inlineCallbacksfrom twisted.trial import unittest@@ -57,8 +58,11 @@ class SpiderTest(unittest.TestCase):def test_spider_without_name(self):"""``__init__`` method arguments are assigned to spider attributes"""- self.assertRaises(ValueError, self.spider_class)- self.assertRaises(ValueError, self.spider_class, somearg="foo")+ msg = "must have a name"+ with pytest.raises(ValueError, match=msg):+ self.spider_class()+ with pytest.raises(ValueError, match=msg):+ self.spider_class(somearg="foo")def test_from_crawler_crawler_and_settings_population(self):crawler = get_crawler()@@ -475,7 +479,7 @@ class CrawlSpiderTest(SpiderTest):spider = self.spider_class("example.com")spider.start_url = "https://www.example.com"- with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"):+ with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"):list(spider.start_requests())@@ -825,5 +829,5 @@ class NoParseMethodSpiderTest(unittest.TestCase):resp = TextResponse(url="http://www.example.com/random_url", body=text)exc_msg = "Spider.parse callback is not defined"- with self.assertRaisesRegex(NotImplementedError, exc_msg):+ with pytest.raises(NotImplementedError, match=exc_msg):spider.parse(resp)diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 705f722b3..b103e9ed0 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -7,6 +7,7 @@ from pathlib import Pathfrom tempfile import mkdtempfrom unittest import mock+import pytestfrom twisted.trial import unittestfrom zope.interface.verify import verifyObject@@ -124,9 +125,8 @@ class SpiderLoaderTest(unittest.TestCase):})- self.assertRaisesRegex(- KeyError, "Spider not found", runner.create_crawler, "spider2"- )+ with pytest.raises(KeyError, match="Spider not found"):+ runner.create_crawler("spider2")crawler = runner.create_crawler("spider1")self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))@@ -135,7 +135,8 @@ class SpiderLoaderTest(unittest.TestCase):def test_bad_spider_modules_exception(self):module = "tests.test_spiderloader.test_spiders.doesnotexist"settings = Settings({"SPIDER_MODULES": [module]})- self.assertRaises(ImportError, SpiderLoader.from_settings, settings)+ with pytest.raises(ImportError):+ SpiderLoader.from_settings(settings)def test_bad_spider_modules_warning(self):with warnings.catch_warnings(record=True) as w:@@ -159,7 +160,8 @@ class SpiderLoaderTest(unittest.TestCase):with mock.patch.object(SpiderLoader, "_load_spiders") as m:m.side_effect = SyntaxErrorsettings = Settings({"SPIDER_MODULES": [module]})- self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)+ with pytest.raises(SyntaxError):+ SpiderLoader.from_settings(settings)def test_syntax_error_warning(self):with (diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex a8507c789..a9f3876bb 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom collections.abc import AsyncIterator, Iterablefrom unittest import mock+import pytestfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.python.failure import Failure@@ -299,12 +300,9 @@ class ProcessSpiderOutputCoroutineMiddleware:class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):@defer.inlineCallbacksdef test_non_iterable(self):- with self.assertRaisesRegex(+ with pytest.raises(_InvalidOutput,- (- r"\.process_spider_output must return an iterable, got <class "- r"'NoneType'>"- ),+ match=r"\.process_spider_output must return an iterable, got <class 'NoneType'>",):yield self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware,@@ -312,9 +310,9 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):@defer.inlineCallbacksdef test_coroutine(self):- with self.assertRaisesRegex(+ with pytest.raises(_InvalidOutput,- r"\.process_spider_output must be an asynchronous generator",+ match=r"\.process_spider_output must be an asynchronous generator",):yield self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware,@@ -518,8 +516,8 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):@defer.inlineCallbacksdef _test_asyncgen_nodowngrade(self, *mw_classes):- with self.assertRaisesRegex(- _InvalidOutput, "Async iterable returned from .+ cannot be downgraded"+ with pytest.raises(+ _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"):yield self._get_middleware_result(*mw_classes)diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 307054de7..f9eb93d6b 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -1,6 +1,7 @@import loggingfrom unittest import TestCase+import pytestfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial.unittest import TestCase as TrialTestCase@@ -68,9 +69,8 @@ class TestHttpErrorMiddleware(TestCase):def test_process_spider_input(self):self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))- self.assertRaises(- HttpError, self.mw.process_spider_input, self.res404, self.spider- )+ with pytest.raises(HttpError):+ self.mw.process_spider_input(self.res404, self.spider)def test_process_spider_exception(self):self.assertEqual(@@ -105,9 +105,8 @@ class TestHttpErrorMiddlewareSettings(TestCase):def test_process_spider_input(self):self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))- self.assertRaises(- HttpError, self.mw.process_spider_input, self.res404, self.spider- )+ with pytest.raises(HttpError):+ self.mw.process_spider_input(self.res404, self.spider)self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))def test_meta_overrides_settings(self):@@ -120,14 +119,14 @@ class TestHttpErrorMiddlewareSettings(TestCase):res402.request = requestself.assertIsNone(self.mw.process_spider_input(res404, self.spider))- self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)+ with pytest.raises(HttpError):+ self.mw.process_spider_input(res402, self.spider)def test_spider_override_settings(self):self.spider.handle_httpstatus_list = [404]self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))- self.assertRaises(- HttpError, self.mw.process_spider_input, self.res402, self.spider- )+ with pytest.raises(HttpError):+ self.mw.process_spider_input(self.res402, self.spider)class TestHttpErrorMiddlewareHandleAll(TestCase):@@ -151,7 +150,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):res402.request = requestself.assertIsNone(self.mw.process_spider_input(res404, self.spider))- self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)+ with pytest.raises(HttpError):+ self.mw.process_spider_input(res402, self.spider)def test_httperror_allow_all_false(self):crawler = get_crawler(_HttpErrorSpider)@@ -167,7 +167,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):res402 = self.res402.copy()res402.request = request_httpstatus_true- self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)+ with pytest.raises(HttpError):+ mw.process_spider_input(res404, self.spider)self.assertIsNone(mw.process_spider_input(res402, self.spider))diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 4945ac25d..01a87c645 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -5,6 +5,8 @@ from typing import Anyfrom unittest import TestCasefrom urllib.parse import urlparse+import pytest+from scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.http import Request, Responsefrom scrapy.settings import Settings@@ -884,7 +886,7 @@ class TestSettingsPolicyByName(TestCase):def test_invalid_name(self):settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})- with self.assertRaises(RuntimeError):+ with pytest.raises(RuntimeError):RefererMiddleware(settings)def test_multiple_policy_tokens(self):@@ -925,7 +927,7 @@ class TestSettingsPolicyByName(TestCase):)})- with self.assertRaises(RuntimeError):+ with pytest.raises(RuntimeError):RefererMiddleware(settings)diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.pyindex 59d18d92e..72692afab 100644--- a/tests/test_spiderstate.py+++ b/tests/test_spiderstate.py@@ -2,6 +2,7 @@ import shutilfrom datetime import datetime, timezonefrom tempfile import mkdtemp+import pytestfrom twisted.trial import unittestfrom scrapy.exceptions import NotConfigured@@ -42,4 +43,5 @@ class SpiderStateTest(unittest.TestCase):def test_not_configured(self):crawler = get_crawler(Spider)- self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler)+ with pytest.raises(NotConfigured):+ SpiderState.from_crawler(crawler)diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex a2e7ae65d..8556b75dd 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -1,6 +1,7 @@import pickleimport sys+import pytestfrom queuelib.tests import test_queue as tfrom scrapy.http import Request@@ -30,10 +31,17 @@ class MyLoader(ItemLoader):def nonserializable_object_test(self):q = self.queue()- self.assertRaises(ValueError, q.push, lambda x: x)+ with pytest.raises(+ ValueError,+ match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",+ ):+ q.push(lambda x: x)# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)sel = Selector(text="<html><body><p>some text</p></body></html>")- self.assertRaises(ValueError, q.push, sel)+ with pytest.raises(+ ValueError, match="unmarshallable object|can't pickle Selector objects"+ ):+ q.push(sel)class FifoDiskQueueTestMixin:diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex 04eeae4dc..88f6657d8 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -6,6 +6,7 @@ import shutilimport tempfileimport unittest+import pytestimport queuelibfrom scrapy.http import Request@@ -69,9 +70,9 @@ class RequestQueueTestMixin:req = Request("http://www.example.com")q.push(req)self.assertEqual(len(q), 1)- with self.assertRaises(+ with pytest.raises(NotImplementedError,- msg="The underlying queue class does not implement 'peek'",+ match="The underlying queue class does not implement 'peek'",):q.peek()self.assertEqual(q.pop().url, req.url)@@ -120,9 +121,9 @@ class FifoQueueMixin(RequestQueueTestMixin):q.push(req1)q.push(req2)q.push(req3)- with self.assertRaises(+ with pytest.raises(NotImplementedError,- msg="The underlying queue class does not implement 'peek'",+ match="The underlying queue class does not implement 'peek'",):q.peek()self.assertEqual(len(q), 3)@@ -176,9 +177,9 @@ class LifoQueueMixin(RequestQueueTestMixin):q.push(req1)q.push(req2)q.push(req3)- with self.assertRaises(+ with pytest.raises(NotImplementedError,- msg="The underlying queue class does not implement 'peek'",+ match="The underlying queue class does not implement 'peek'",):q.peek()self.assertEqual(len(q), 3)diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex cbea41129..e27bb7b74 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -1,5 +1,7 @@import unittest+import pytest+from scrapy.exceptions import UsageErrorfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.conf import (@@ -32,7 +34,9 @@ class BuildComponentListTest(unittest.TestCase):)# Same priority raises ValueErrorduplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError, match="Some paths in .* convert to the same object"+ ):build_component_list(duplicate_bs, convert=lambda x: x.lower())def test_valid_numbers(self):@@ -58,21 +62,13 @@ class UtilsConfTestCase(unittest.TestCase):class FeedExportConfigTestCase(unittest.TestCase):def test_feed_export_config_invalid_format(self):settings = Settings()- self.assertRaises(- UsageError,- feed_process_params_from_cli,- settings,- ["items.dat"],- )+ with pytest.raises(UsageError):+ feed_process_params_from_cli(settings, ["items.dat"])def test_feed_export_config_mismatch(self):settings = Settings()- self.assertRaises(- UsageError,- feed_process_params_from_cli,- settings,- ["items1.dat", "items2.dat"],- )+ with pytest.raises(UsageError):+ feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"])def test_feed_export_config_explicit_formats(self):settings = Settings()@@ -117,11 +113,9 @@ class FeedExportConfigTestCase(unittest.TestCase):)def test_output_and_overwrite_output(self):- with self.assertRaises(UsageError):+ with pytest.raises(UsageError):feed_process_params_from_cli(- Settings(),- ["output1.json"],- overwrite_output=["output2.json"],+ Settings(), ["output1.json"], overwrite_output=["output2.json"])def test_feed_complete_default_values_from_settings_empty(self):diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 5d99161bf..a5b438645 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -1,6 +1,7 @@import unittestimport warnings+import pytestfrom w3lib.http import basic_auth_headerfrom scrapy import Request@@ -205,11 +206,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)def test_too_few_arguments_error(self):- self.assertRaisesRegex(+ with pytest.raises(ValueError,- r"too few arguments|the following arguments are required:\s*url",- lambda: curl_to_request_kwargs("curl"),- )+ match=r"too few arguments|the following arguments are required:\s*url",+ ):+ curl_to_request_kwargs("curl")def test_ignore_unknown_options(self):# case 1: ignore_unknown_options=True:@@ -220,16 +221,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)# case 2: ignore_unknown_options=False (raise exception):- self.assertRaisesRegex(- ValueError,- "Unrecognized options:.*--bar.*--baz",- lambda: curl_to_request_kwargs(+ with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"):+ curl_to_request_kwargs("curl --bar --baz http://www.example.com", ignore_unknown_options=False- ),- )+ )def test_must_start_with_curl_error(self):- self.assertRaises(- ValueError,- lambda: curl_to_request_kwargs("carl -X POST http://example.org"),- )+ with pytest.raises(ValueError, match="A curl command must start"):+ curl_to_request_kwargs("carl -X POST http://example.org")diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex fadbc6daa..2e35d339a 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -87,8 +87,10 @@ class CaseInsensitiveDictMixin:def test_delete(self):d = self.dict_class({"key_lower": 1})del d["key_LOWER"]- self.assertRaises(KeyError, d.__getitem__, "key_LOWER")- self.assertRaises(KeyError, d.__getitem__, "key_lower")+ with pytest.raises(KeyError):+ d["key_LOWER"]+ with pytest.raises(KeyError):+ d["key_lower"]@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_getdefault(self):@@ -138,7 +140,8 @@ class CaseInsensitiveDictMixin:d = self.dict_class()d["a"] = 1self.assertEqual(d.pop("A"), 1)- self.assertRaises(KeyError, d.pop, "A")+ with pytest.raises(KeyError):+ d.pop("A")def test_normkey(self):class MyDict(self.dict_class):@@ -279,8 +282,8 @@ class SequenceExcludeTest(unittest.TestCase):self.assertIn(set("bar"), d)# supplied sequence is a set, so checking for list (non)inclusion fails- self.assertRaises(TypeError, (0, 1, 2) in d)- self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"])+ with pytest.raises(TypeError):+ ["a", "b", "c"] in d # noqa: B015for v in [-3, "test", 1.1]:self.assertNotIn(v, d)diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex dc5fbd3c3..e917b6947 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -3,6 +3,8 @@ import unittestimport warningsfrom unittest import mock+import pytest+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.deprecate import create_deprecated_class, update_classpath@@ -181,7 +183,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)- self.assertRaises(TypeError, issubclass, object(), DeprecatedName)+ with pytest.raises(TypeError):+ issubclass(object(), DeprecatedName)def test_isinstance(self):with warnings.catch_warnings():diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.pyindex 7b7a25db8..d40cae9c7 100644--- a/tests/test_utils_gz.py+++ b/tests/test_utils_gz.py@@ -1,6 +1,8 @@import unittest+from gzip import BadGzipFilefrom pathlib import Path+import pytestfrom w3lib.encoding import html_to_unicodefrom scrapy.http import Response@@ -27,9 +29,8 @@ class GunzipTest(unittest.TestCase):assert text.endswith(b"</html")def test_gunzip_no_gzip_file_raises(self):- self.assertRaises(- OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()- )+ with pytest.raises(BadGzipFile):+ gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())def test_gunzip_truncated_short(self):r1 = Response(diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex 12507c6a3..9ad30617a 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -215,7 +215,7 @@ class XmliterBaseTestCase:"""response = XmlResponse(url="http://mydummycompany.com", body=body)my_iter = self.xmliter(response, "g:link_image")- with self.assertRaises(StopIteration):+ with pytest.raises(StopIteration):next(my_iter)@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")@@ -228,13 +228,14 @@ class XmliterBaseTestCase:iter = self.xmliter(body, "product")next(iter)next(iter)-- self.assertRaises(StopIteration, next, iter)+ with pytest.raises(StopIteration):+ next(iter)@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_objtype_exception(self):i = self.xmliter(42, "product")- self.assertRaises(TypeError, next, i)+ with pytest.raises(TypeError):+ next(i)@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_xmliter_encoding(self):@@ -344,7 +345,8 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase):def test_xmliter_objtype_exception(self):i = self.xmliter(42, "product")- self.assertRaises(TypeError, next, i)+ with pytest.raises(TypeError):+ next(i)class UtilsCsvTestCase(unittest.TestCase):@@ -491,8 +493,8 @@ class UtilsCsvTestCase(unittest.TestCase):next(iter)next(iter)next(iter)-- self.assertRaises(StopIteration, next, iter)+ with pytest.raises(StopIteration):+ next(iter)def test_csviter_encoding(self):body1 = get_testdata("feeds", "feed-sample4.csv")diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex e25bdfe3f..a67e16962 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -32,9 +32,12 @@ class UtilsMiscTestCase(unittest.TestCase):self.assertIs(obj, load_object)def test_load_object_exceptions(self):- self.assertRaises(ImportError, load_object, "nomodule999.mod.function")- self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999")- self.assertRaises(TypeError, load_object, {})+ with pytest.raises(ImportError):+ load_object("nomodule999.mod.function")+ with pytest.raises(NameError):+ load_object("scrapy.utils.misc.load_object999")+ with pytest.raises(TypeError):+ load_object({})def test_walk_modules(self):mods = walk_modules("tests.test_utils_misc.test_walk_modules")@@ -59,7 +62,8 @@ class UtilsMiscTestCase(unittest.TestCase):]self.assertEqual({m.__name__ for m in mods}, set(expected))- self.assertRaises(ImportError, walk_modules, "nomodule999")+ with pytest.raises(ImportError):+ walk_modules("nomodule999")def test_walk_modules_egg(self):egg = str(Path(__file__).parent / "test.egg")@@ -148,11 +152,13 @@ class UtilsMiscTestCase(unittest.TestCase):create_instance(m, None, crawler, *args, **kwargs)m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)- with self.assertRaises(ValueError):+ with pytest.raises(+ ValueError, match="Specify at least one of settings and crawler"+ ):create_instance(m, None, None)m.from_settings.return_value = None- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):create_instance(m, settings, None)def test_build_from_crawler(self):@@ -191,7 +197,7 @@ class UtilsMiscTestCase(unittest.TestCase):# Check adoption of crawlerm = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])m.from_crawler.return_value = None- with self.assertRaises(TypeError):+ with pytest.raises(TypeError):build_from_crawler(m, crawler, *args, **kwargs)def test_set_environ(self):diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex a693d6b53..3b0739276 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -87,7 +87,8 @@ class ToUnicodeTest(unittest.TestCase):self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e")def test_converting_a_strange_object_should_raise_TypeError(self):- self.assertRaises(TypeError, to_unicode, 423)+ with pytest.raises(TypeError):+ to_unicode(423)def test_errors_argument(self):self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb")@@ -104,7 +105,8 @@ class ToBytesTest(unittest.TestCase):self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e")def test_converting_a_strange_object_should_raise_TypeError(self):- self.assertRaises(TypeError, to_bytes, unittest)+ with pytest.raises(TypeError):+ to_bytes(pytest)def test_errors_argument(self):self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b")diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex c6ba8cbbb..af7906781 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -35,7 +35,8 @@ class ResponseUtilsTest(unittest.TestCase):assert open_in_browser(response, _openfunc=browser_open), "Browser not called"resp = Response(url, body=body)- self.assertRaises(TypeError, open_in_browser, resp, debug=True)+ with pytest.raises(TypeError):+ open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-argdef test_get_meta_refresh(self):r1 = HtmlResponse(