Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Switch to pytest.raises(). (#6680)

Andrey Rakhmatullin8d92c28a16c7
diff --git a/pyproject.toml b/pyproject.toml
index ad62ea212..82d8056f6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -378,8 +378,6 @@ ignore = [
# Temporarily silenced PT rules
# Use a regular `assert` instead of unittest-style `assertEqual`
"PT009",
- # Use `pytest.raises` instead of unittest-style `assertRaises`
- "PT027",
]
[tool.ruff.lint.per-file-ignores]
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index a04e0107b..324a9b955 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -214,7 +214,7 @@ class DownloaderAwarePriorityQueue:
"DownloaderAwarePriorityQueue accepts "
"``slot_startprios`` as a dict; "
f"{slot_startprios.__class__!r} instance "
- "is passed. Most likely, it means the state is"
+ "is passed. Most likely, it means the state is "
"created by an incompatible priority queue. "
"Only a crawl started with the same priority "
"queue class can be resumed."
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 425188d32..df5ebfa7b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -185,9 +185,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_downloader_middleware_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
- self.assertRaises(
- RuntimeError, crawler.get_downloader_middleware, DefaultSpider
- )
+ with pytest.raises(RuntimeError):
+ crawler.get_downloader_middleware(DefaultSpider)
@inlineCallbacks
def test_get_downloader_middleware_no_engine(self):
@@ -266,7 +265,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_extension_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
- self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)
+ with pytest.raises(RuntimeError):
+ crawler.get_extension(DefaultSpider)
@inlineCallbacks
def test_get_extension_no_engine(self):
@@ -345,7 +345,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_item_pipeline_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
- self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)
+ with pytest.raises(RuntimeError):
+ crawler.get_item_pipeline(DefaultSpider)
@inlineCallbacks
def test_get_item_pipeline_no_engine(self):
@@ -424,7 +425,8 @@ class CrawlerTestCase(BaseCrawlerTest):
def test_get_spider_middleware_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
- self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)
+ with pytest.raises(RuntimeError):
+ crawler.get_spider_middleware(DefaultSpider)
@inlineCallbacks
def test_get_spider_middleware_no_engine(self):
@@ -537,7 +539,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
- self.assertRaises(MultipleInvalid, CrawlerRunner, settings)
+ with pytest.raises(MultipleInvalid):
+ CrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self):
runner = CrawlerRunner({"foo": "bar"})
@@ -630,13 +633,15 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
}
)
else:
- msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
- with self.assertRaisesRegex(Exception, msg):
- runner = CrawlerRunner(
- settings={
- "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
- }
- )
+ runner = CrawlerRunner(
+ settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+ )
+ with pytest.raises(
+ Exception,
+ match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
+ ):
yield runner.crawl(NoRequestsSpider)
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 174bf841e..17d5c2d0a 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -248,5 +248,5 @@ class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase):
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
- with self.assertRaises(NotImplementedError):
+ with pytest.raises(NotImplementedError):
yield super().test_download_with_proxy_https_timeout()
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 42051042c..49498375c 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,4 +1,5 @@
import asyncio
+from gzip import BadGzipFile
from unittest import mock
import pytest
@@ -106,7 +107,8 @@ class DefaultsTest(ManagerTestCase):
"Location": "http://example.com/login",
},
)
- self.assertRaises(OSError, self._download, request=req, response=resp)
+ with pytest.raises(BadGzipFile):
+ self._download(request=req, response=resp)
class ResponseFromProcessRequestTest(ManagerTestCase):
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 772769690..694a669d4 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -83,11 +83,10 @@ class CookiesMiddlewareTest(TestCase):
self.assertEqual(req2.headers.get("Cookie"), b"C1=value1")
def test_setting_false_cookies_enabled(self):
- self.assertRaises(
- NotConfigured,
- CookiesMiddleware.from_crawler,
- get_crawler(settings_dict={"COOKIES_ENABLED": False}),
- )
+ with pytest.raises(NotConfigured):
+ CookiesMiddleware.from_crawler(
+ get_crawler(settings_dict={"COOKIES_ENABLED": False})
+ )
def test_setting_default_cookies_enabled(self):
self.assertIsInstance(
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 581fc1974..0f1489344 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,5 +1,6 @@
import unittest
+import pytest
from w3lib.http import basic_auth_header
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
@@ -29,8 +30,8 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.spider = LegacySpider("foo")
def test_auth(self):
- with self.assertRaises(AttributeError):
- mw = HttpAuthMiddleware()
+ mw = HttpAuthMiddleware()
+ with pytest.raises(AttributeError):
mw.spider_opened(self.spider)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 74db93f8a..de3a9689b 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -5,6 +5,8 @@ import time
import unittest
from contextlib import contextmanager
+import pytest
+
from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import HtmlResponse, Request, Response
@@ -192,9 +194,8 @@ class DummyPolicyTest(_BaseTest):
def test_middleware_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
- self.assertRaises(
- IgnoreRequest, mw.process_request, self.request, self.spider
- )
+ with pytest.raises(IgnoreRequest):
+ mw.process_request(self.request, self.spider)
mw.process_response(self.request, self.response, self.spider)
response = mw.process_request(self.request, self.spider)
assert isinstance(response, HtmlResponse)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a1c5883ec..b3e3b98d7 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -4,6 +4,7 @@ from logging import WARNING
from pathlib import Path
from unittest import SkipTest, TestCase
+import pytest
from testfixtures import LogCapture
from w3lib.encoding import resolve_encoding
@@ -87,11 +88,10 @@ class HttpCompressionTest(TestCase):
)
def test_setting_false_compression_enabled(self):
- self.assertRaises(
- NotConfigured,
- HttpCompressionMiddleware.from_crawler,
- get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),
- )
+ with pytest.raises(NotConfigured):
+ HttpCompressionMiddleware.from_crawler(
+ get_crawler(settings_dict={"COMPRESSION_ENABLED": False})
+ )
def test_setting_default_compression_enabled(self):
self.assertIsInstance(
@@ -520,13 +520,8 @@ class HttpCompressionTest(TestCase):
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
- self.assertRaises(
- IgnoreRequest,
- mw.process_response,
- response.request,
- response,
- spider,
- )
+ with pytest.raises(IgnoreRequest):
+ mw.process_response(response.request, response, spider)
def test_compression_bomb_setting_br(self):
try:
@@ -561,13 +556,8 @@ class HttpCompressionTest(TestCase):
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
- self.assertRaises(
- IgnoreRequest,
- mw.process_response,
- response.request,
- response,
- spider,
- )
+ with pytest.raises(IgnoreRequest):
+ mw.process_response(response.request, response, spider)
def test_compression_bomb_spider_attr_br(self):
try:
@@ -600,13 +590,8 @@ class HttpCompressionTest(TestCase):
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 10_000_000
- self.assertRaises(
- IgnoreRequest,
- mw.process_response,
- response.request,
- response,
- spider,
- )
+ with pytest.raises(IgnoreRequest):
+ mw.process_response(response.request, response, spider)
def test_compression_bomb_request_meta_br(self):
try:
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index f950906e9..47abeee7a 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -72,9 +72,8 @@ class Base:
assert isinstance(req, Request)
assert "redirect_times" in req.meta
self.assertEqual(req.meta["redirect_times"], 1)
- self.assertRaises(
- IgnoreRequest, self.mw.process_response, req, rsp, self.spider
- )
+ with pytest.raises(IgnoreRequest):
+ self.mw.process_response(req, rsp, self.spider)
def test_ttl(self):
self.mw.max_redirect_times = 100
@@ -83,9 +82,8 @@ class Base:
req = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req, Request)
- self.assertRaises(
- IgnoreRequest, self.mw.process_response, req, rsp, self.spider
- )
+ with pytest.raises(IgnoreRequest):
+ self.mw.process_response(req, rsp, self.spider)
def test_redirect_urls(self):
req1 = Request("http://scrapytest.org/first")
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 6b9b39413..36f48db69 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,6 +1,7 @@
import logging
import unittest
+import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.error import (
@@ -407,7 +408,7 @@ class GetRetryRequestTest(unittest.TestCase):
def test_no_spider(self):
request = Request("https://example.com")
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
def test_max_retry_times_setting(self):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 535e07c1f..9b95400fd 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,5 +1,6 @@
from unittest import mock
+import pytest
from twisted.internet import error, reactor
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python import failure
@@ -26,7 +27,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def test_robotstxt_settings(self):
self.crawler.settings = Settings()
self.crawler.settings.set("USER_AGENT", "CustomAgent")
- self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler)
+ with pytest.raises(NotConfigured):
+ RobotsTxtMiddleware(self.crawler)
def _get_successful_crawler(self):
crawler = self.crawler
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index eb8d309b6..48728e078 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -10,6 +10,7 @@ from io import BytesIO
from typing import Any
import lxml.etree
+import pytest
from itemadapter import ItemAdapter
from scrapy.exporters import (
@@ -147,7 +148,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
return PythonItemExporter(**kwargs)
def test_invalid_option(self):
- with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
+ with pytest.raises(TypeError, match="Unexpected options: invalid_option"):
PythonItemExporter(invalid_option="something")
def test_nested_item(self):
@@ -388,7 +389,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
)
def test_errors_default(self):
- with self.assertRaises(UnicodeEncodeError):
+ with pytest.raises(UnicodeEncodeError):
self.assertExportResult(
item={"text": "W\u0275\u200brd"},
expected=None,
@@ -549,7 +550,8 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.ie = self._get_exporter(sort_keys=True)
self.test_export_item()
self._check_output()
- self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True)
+ with pytest.raises(TypeError):
+ self._get_exporter(foo_unknown_keyword_bar=True)
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
@@ -602,7 +604,8 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
i3 = MyItem(name="Jesus", age="44")
self.ie.start_exporting()
self.ie.export_item(i1)
- self.assertRaises(TypeError, self.ie.export_item, i2)
+ with pytest.raises(TypeError):
+ self.ie.export_item(i2)
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
@@ -657,7 +660,8 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest):
i3 = MyItem(name="Jesus", age="44")
self.ie.start_exporting()
self.ie.export_item(i1)
- self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)
+ with pytest.raises(UnicodeEncodeError):
+ self.ie.export_item(i2)
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin"))
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1620d2d41..b4c1b9631 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -233,7 +233,8 @@ class BlockingFeedStorageTest(unittest.TestCase):
invalid_path = tests_path / "invalid_path"
spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
- self.assertRaises(OSError, b.open, spider=spider)
+ with pytest.raises(OSError, match="Not a Directory:"):
+ b.open(spider=spider)
@pytest.mark.requires_boto3
@@ -2437,7 +2438,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
crawler = get_crawler(settings_dict=settings)
- self.assertRaises(NotConfigured, FeedExporter, crawler)
+ with pytest.raises(NotConfigured):
+ FeedExporter(crawler)
@defer.inlineCallbacks
def test_export_no_items_not_store_empty(self):
@@ -2758,7 +2760,7 @@ class FeedExportInitTest(unittest.TestCase):
},
}
crawler = get_crawler(settings_dict=settings)
- with self.assertRaises(NotConfigured):
+ with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
def test_unsupported_format(self):
@@ -2770,7 +2772,7 @@ class FeedExportInitTest(unittest.TestCase):
},
}
crawler = get_crawler(settings_dict=settings)
- with self.assertRaises(NotConfigured):
+ with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
def test_absolute_pathlib_as_uri(self):
@@ -2863,7 +2865,7 @@ class URIParamsTest:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
- with self.assertRaises(KeyError):
+ with pytest.raises(KeyError):
feed_exporter.open_spider(spider)
def test_params_as_is(self):
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index ddc772236..0881bbeca 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
from unittest import mock, skipIf
from urllib.parse import urlencode
+import pytest
from twisted.internet import reactor
from twisted.internet.defer import (
CancelledError,
@@ -406,7 +407,7 @@ class Https2ClientProtocolTestCase(TestCase):
"scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
):
request = Request(url=self.get_url("/status?n=200"))
- with self.assertRaises(ResponseFailed):
+ with pytest.raises(ResponseFailed):
yield self.make_request(request)
def test_cancel_request(self):
@@ -560,7 +561,7 @@ class Https2ClientProtocolTestCase(TestCase):
return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
def test_invalid_request_type(self):
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
self.make_request("https://InvalidDataTypePassed.com")
def test_query_parameters(self):
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 7db1eb8c5..0bbbcda46 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -1,6 +1,8 @@
import copy
import unittest
+import pytest
+
from scrapy.http import Headers
@@ -13,7 +15,8 @@ class HeadersTest(unittest.TestCase):
assert h["Content-Type"]
assert h["Content-Length"]
- self.assertRaises(KeyError, h.__getitem__, "Accept")
+ with pytest.raises(KeyError):
+ h["Accept"]
self.assertEqual(h.get("Accept"), None)
self.assertEqual(h.getlist("Accept"), [])
@@ -152,15 +155,11 @@ class HeadersTest(unittest.TestCase):
self.assertEqual(h1.getlist("hey"), [b"5"])
def test_invalid_value(self):
- self.assertRaisesRegex(
- TypeError, "Unsupported value type", Headers, {"foo": object()}
- )
- self.assertRaisesRegex(
- TypeError, "Unsupported value type", Headers().__setitem__, "foo", object()
- )
- self.assertRaisesRegex(
- TypeError, "Unsupported value type", Headers().setdefault, "foo", object()
- )
- self.assertRaisesRegex(
- TypeError, "Unsupported value type", Headers().setlist, "foo", [object()]
- )
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers({"foo": object()})
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers()["foo"] = object()
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers().setdefault("foo", object())
+ with pytest.raises(TypeError, match="Unsupported value type"):
+ Headers().setlist("foo", [object()])
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a8ab8240f..e5291157d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -7,6 +7,8 @@ from typing import Any
from unittest import mock
from urllib.parse import parse_qs, unquote_to_bytes
+import pytest
+
from scrapy.http import (
FormRequest,
Headers,
@@ -28,10 +30,12 @@ class RequestTest(unittest.TestCase):
def test_init(self):
# Request requires url in the __init__ method
- self.assertRaises(Exception, self.request_class)
+ with pytest.raises(TypeError):
+ self.request_class()
# url argument must be basestring
- self.assertRaises(TypeError, self.request_class, 123)
+ with pytest.raises(TypeError):
+ self.request_class(123)
r = self.request_class("http://www.example.com")
r = self.request_class("http://www.example.com")
@@ -64,9 +68,13 @@ class RequestTest(unittest.TestCase):
self.request_class("data:,Hello%2C%20World!")
def test_url_no_scheme(self):
- self.assertRaises(ValueError, self.request_class, "foo")
- self.assertRaises(ValueError, self.request_class, "/foo/")
- self.assertRaises(ValueError, self.request_class, "/foo:bar")
+ msg = "Missing scheme in request url:"
+ with pytest.raises(ValueError, match=msg):
+ self.request_class("foo")
+ with pytest.raises(ValueError, match=msg):
+ self.request_class("/foo/")
+ with pytest.raises(ValueError, match=msg):
+ self.request_class("/foo:bar")
def test_headers(self):
# Different ways of setting headers attribute
@@ -273,8 +281,10 @@ class RequestTest(unittest.TestCase):
def test_immutable_attributes(self):
r = self.request_class("http://example.com")
- self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
- self.assertRaises(AttributeError, setattr, r, "body", "xxx")
+ with pytest.raises(AttributeError):
+ r.url = "http://example2.com"
+ with pytest.raises(AttributeError):
+ r.body = "xxx"
def test_callback_and_errback(self):
def a_function():
@@ -309,11 +319,11 @@ class RequestTest(unittest.TestCase):
self.assertIs(r5.errback, NO_CALLBACK)
def test_callback_and_errback_type(self):
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
self.request_class("http://example.com", callback="a_function")
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
self.request_class("http://example.com", errback="a_function")
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
self.request_class(
url="http://example.com",
callback="a_function",
@@ -321,7 +331,7 @@ class RequestTest(unittest.TestCase):
)
def test_no_callback(self):
- with self.assertRaises(RuntimeError):
+ with pytest.raises(RuntimeError):
NO_CALLBACK()
def test_from_curl(self):
@@ -403,13 +413,11 @@ class RequestTest(unittest.TestCase):
# If `ignore_unknown_options` is set to `False` it raises an error with
# the unknown options: --foo and -z
- self.assertRaises(
- ValueError,
- lambda: self.request_class.from_curl(
+ with pytest.raises(ValueError, match="Unrecognized options:"):
+ self.request_class.from_curl(
'curl -X PATCH "http://example.org" --foo -z',
ignore_unknown_options=False,
- ),
- )
+ )
class FormRequestTest(RequestTest):
@@ -428,7 +436,7 @@ class FormRequestTest(RequestTest):
data = (("a", "one"), ("a", "two"), ("b", "2"))
url = self.request_class(
"http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data
- ).url.split("#")[0]
+ ).url.split("#", maxsplit=1)[0]
fs = _qs(self.request_class(url, method="GET", formdata=data))
self.assertEqual(set(fs[b"a"]), {b"one", b"two"})
self.assertEqual(fs[b"b"], [b"2"])
@@ -897,12 +905,11 @@ class FormRequestTest(RequestTest):
<input type="submit" name="clickable2" value="clicked2">
</form>"""
)
- self.assertRaises(
+ with pytest.raises(
ValueError,
- self.request_class.from_response,
- response,
- clickdata={"type": "submit"},
- )
+ match="Multiple elements found .* matching the criteria in clickdata",
+ ):
+ self.request_class.from_response(response, clickdata={"type": "submit"})
def test_from_response_non_matching_clickdata(self):
response = _buildresponse(
@@ -910,12 +917,12 @@ class FormRequestTest(RequestTest):
<input type="submit" name="clickable" value="clicked">
</form>"""
)
- self.assertRaises(
- ValueError,
- self.request_class.from_response,
- response,
- clickdata={"nonexistent": "notme"},
- )
+ with pytest.raises(
+ ValueError, match="No clickable element matching clickdata:"
+ ):
+ self.request_class.from_response(
+ response, clickdata={"nonexistent": "notme"}
+ )
def test_from_response_nr_index_clickdata(self):
response = _buildresponse(
@@ -937,13 +944,15 @@ class FormRequestTest(RequestTest):
</form>
"""
)
- self.assertRaises(
- ValueError, self.request_class.from_response, response, clickdata={"nr": 1}
- )
+ with pytest.raises(
+ ValueError, match="No clickable element matching clickdata:"
+ ):
+ self.request_class.from_response(response, clickdata={"nr": 1})
def test_from_response_errors_noform(self):
response = _buildresponse("""<html></html>""")
- self.assertRaises(ValueError, self.request_class.from_response, response)
+ with pytest.raises(ValueError, match="No <form> element found in"):
+ self.request_class.from_response(response)
def test_from_response_invalid_html5(self):
response = _buildresponse(
@@ -963,9 +972,8 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="test2" value="xxx">
</form>"""
)
- self.assertRaises(
- IndexError, self.request_class.from_response, response, formnumber=1
- )
+ with pytest.raises(IndexError):
+ self.request_class.from_response(response, formnumber=1)
def test_from_response_noformname(self):
response = _buildresponse(
@@ -1021,13 +1029,8 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="two" value="2">
</form>"""
)
- self.assertRaises(
- IndexError,
- self.request_class.from_response,
- response,
- formname="form3",
- formnumber=2,
- )
+ with pytest.raises(IndexError):
+ self.request_class.from_response(response, formname="form3", formnumber=2)
def test_from_response_formid_exists(self):
response = _buildresponse(
@@ -1086,13 +1089,8 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="two" value="2">
</form>"""
)
- self.assertRaises(
- IndexError,
- self.request_class.from_response,
- response,
- formid="form3",
- formnumber=2,
- )
+ with pytest.raises(IndexError):
+ self.request_class.from_response(response, formid="form3", formnumber=2)
def test_from_response_select(self):
res = _buildresponse(
@@ -1245,12 +1243,10 @@ class FormRequestTest(RequestTest):
fs = _qs(r1)
self.assertEqual(fs[b"three"], [b"3"])
- self.assertRaises(
- ValueError,
- self.request_class.from_response,
- response,
- formxpath="//form/input[@name='abc']",
- )
+ with pytest.raises(ValueError, match="No <form> element found with"):
+ self.request_class.from_response(
+ response, formxpath="//form/input[@name='abc']"
+ )
def test_from_response_unicode_xpath(self):
response = _buildresponse(b'<form name="\xd1\x8a"></form>')
@@ -1261,13 +1257,8 @@ class FormRequestTest(RequestTest):
self.assertEqual(fs, {})
xpath = "//form[@name='\u03b1']"
- self.assertRaisesRegex(
- ValueError,
- re.escape(xpath),
- self.request_class.from_response,
- response,
- formxpath=xpath,
- )
+ with pytest.raises(ValueError, match=re.escape(xpath)):
+ self.request_class.from_response(response, formxpath=xpath)
def test_from_response_button_submit(self):
response = _buildresponse(
@@ -1393,12 +1384,8 @@ class FormRequestTest(RequestTest):
fs = _qs(r1)
self.assertEqual(fs[b"three"], [b"3"])
- self.assertRaises(
- ValueError,
- self.request_class.from_response,
- response,
- formcss="input[name='abc']",
- )
+ with pytest.raises(ValueError, match="No <form> element found with"):
+ self.request_class.from_response(response, formcss="input[name='abc']")
def test_from_response_valid_form_methods(self):
form_methods = [
@@ -1424,13 +1411,11 @@ class FormRequestTest(RequestTest):
</form>
</body></html>"""
)
- with self.assertRaises(ValueError) as context:
+ with pytest.raises(
+ ValueError, match="formdata should be a dict or iterable of tuples"
+ ):
FormRequest.from_response(response, formdata=123)
- self.assertIn(
- "formdata should be a dict or iterable of tuples", str(context.exception)
- )
-
def test_form_response_with_custom_invalid_formdata_value_error(self):
"""Test that a ValueError is raised for fault-inducing iterable formdata input"""
response = _buildresponse(
@@ -1441,13 +1426,11 @@ class FormRequestTest(RequestTest):
</body></html>"""
)
- with self.assertRaises(ValueError) as context:
+ with pytest.raises(
+ ValueError, match="formdata should be a dict or iterable of tuples"
+ ):
FormRequest.from_response(response, formdata=("a",))
- self.assertIn(
- "formdata should be a dict or iterable of tuples", str(context.exception)
- )
-
def test_get_form_with_xpath_no_form_parent(self):
"""Test that _get_from raised a ValueError when an XPath selects an element
not nested within a <form> and no <form> parent is found"""
@@ -1462,11 +1445,9 @@ class FormRequestTest(RequestTest):
</body></html>"""
)
- with self.assertRaises(ValueError) as context:
+ with pytest.raises(ValueError, match="No <form> element found with"):
FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p')
- self.assertIn("No <form> element found with", str(context.exception))
-
def _buildresponse(body, **kwargs):
kwargs.setdefault("body", body)
@@ -1507,8 +1488,10 @@ class XmlRpcRequestTest(RequestTest):
self._test_request(params=("response",), methodresponse="login")
self._test_request(params=("pas£",), encoding="utf-8")
self._test_request(params=(None,), allow_none=1)
- self.assertRaises(TypeError, self._test_request)
- self.assertRaises(TypeError, self._test_request, params=(None,))
+ with pytest.raises(TypeError):
+ self._test_request()
+ with pytest.raises(TypeError):
+ self._test_request(params=(None,))
def test_latin1(self):
self._test_request(params=("pas£",), encoding="latin1")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index dde883451..5a943f084 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -27,14 +27,15 @@ class BaseResponseTest(unittest.TestCase):
def test_init(self):
# Response requires url in the constructor
- self.assertRaises(Exception, self.response_class)
+ with pytest.raises(TypeError):
+ self.response_class()
self.assertTrue(
isinstance(self.response_class("http://example.com/"), self.response_class)
)
- self.assertRaises(TypeError, self.response_class, b"http://example.com")
- self.assertRaises(
- TypeError, self.response_class, url="http://example.com", body={}
- )
+ with pytest.raises(TypeError):
+ self.response_class(b"http://example.com")
+ with pytest.raises(TypeError):
+ self.response_class(url="http://example.com", body={})
# body can be str or None
self.assertTrue(
isinstance(
@@ -77,12 +78,8 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(r.status, 301)
r = self.response_class("http://www.example.com", status="301")
self.assertEqual(r.status, 301)
- self.assertRaises(
- ValueError,
- self.response_class,
- "http://example.com",
- status="lala200",
- )
+ with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
+ self.response_class("http://example.com", status="lala200")
def test_copy(self):
"""Test Response copy"""
@@ -122,14 +119,12 @@ class BaseResponseTest(unittest.TestCase):
def test_unavailable_meta(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
- with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"):
+ with pytest.raises(AttributeError, match=r"Response\.meta not available"):
r1.meta
def test_unavailable_cb_kwargs(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
- with self.assertRaisesRegex(
- AttributeError, r"Response\.cb_kwargs not available"
- ):
+ with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
r1.cb_kwargs
def test_copy_inherited_classes(self):
@@ -179,8 +174,10 @@ class BaseResponseTest(unittest.TestCase):
def test_immutable_attributes(self):
r = self.response_class("http://example.com")
- self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com")
- self.assertRaises(AttributeError, setattr, r, "body", "xxx")
+ with pytest.raises(AttributeError):
+ r.url = "http://example2.com"
+ with pytest.raises(AttributeError):
+ r.body = "xxx"
def test_urljoin(self):
"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
@@ -192,10 +189,14 @@ class BaseResponseTest(unittest.TestCase):
r = self.response_class("http://example.com", body=b"hello")
if self.response_class == Response:
msg = "Response content isn't text"
- self.assertRaisesRegex(AttributeError, msg, getattr, r, "text")
- self.assertRaisesRegex(NotSupported, msg, r.css, "body")
- self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body")
- self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body")
+ with pytest.raises(AttributeError, match=msg):
+ r.text
+ with pytest.raises(NotSupported, match=msg):
+ r.css("body")
+ with pytest.raises(NotSupported, match=msg):
+ r.xpath("//body")
+ with pytest.raises(NotSupported, match=msg):
+ r.jmespath("body")
else:
r.text
r.css("body")
@@ -216,7 +217,8 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_None_url(self):
r = self.response_class("http://example.com")
- self.assertRaises(ValueError, r.follow, None)
+ with pytest.raises(ValueError, match="url can't be None"):
+ r.follow(None)
@pytest.mark.xfail(
parse_version(w3lib_version) < parse_version("2.1.1"),
@@ -279,18 +281,20 @@ class BaseResponseTest(unittest.TestCase):
def test_follow_all_invalid(self):
r = self.response_class("http://example.com")
if self.response_class == Response:
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
list(r.follow_all(urls=None))
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
- with self.assertRaises(ValueError):
+ with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
else:
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError, match="Please supply exactly one of the following arguments"
+ ):
list(r.follow_all(urls=None))
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
- with self.assertRaises(ValueError):
+ with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
def test_follow_all_whitespace(self):
@@ -399,12 +403,8 @@ class TextResponseTest(BaseResponseTest):
"\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 "
"\u0442\u0435\u043a\u0441\u0442"
)
- self.assertRaises(
- TypeError,
- self.response_class,
- "http://www.example.com",
- body="unicode body",
- )
+ with pytest.raises(TypeError):
+ self.response_class("http://www.example.com", body="unicode body")
original_string = unicode_string.encode("cp1251")
r1 = self.response_class(
@@ -483,12 +483,8 @@ class TextResponseTest(BaseResponseTest):
self._assert_response_values(r9, "cp1252", "€")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
- self.assertRaises(
- TypeError,
- self.response_class,
- "http://www.example.com",
- body="\xa3",
- )
+ with pytest.raises(TypeError):
+ self.response_class("http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
@@ -679,20 +675,20 @@ class TextResponseTest(BaseResponseTest):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
- self.assertRaises(ValueError, resp.follow, resp.css("div")[0])
+ with pytest.raises(
+ ValueError, match="Only <a> and <link> elements are supported"
+ ):
+ resp.follow(resp.css("div")[0])
def test_follow_selector_list(self):
resp = self._links_response()
- self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a"))
+ with pytest.raises(ValueError, match="SelectorList"):
+ resp.follow(resp.css("a"))
def test_follow_selector_invalid(self):
resp = self._links_response()
- self.assertRaisesRegex(
- ValueError,
- "Unsupported",
- resp.follow,
- resp.xpath("count(//div)")[0],
- )
+ with pytest.raises(ValueError, match="Unsupported"):
+ resp.follow(resp.xpath("count(//div)")[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
@@ -704,7 +700,8 @@ class TextResponseTest(BaseResponseTest):
url="http://example.com",
body=b"<html><body><a name=123>click me</a></body></html>",
)
- self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0])
+ with pytest.raises(ValueError, match="no href"):
+ resp.follow(resp.css("a")[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(
@@ -812,7 +809,9 @@ class TextResponseTest(BaseResponseTest):
def test_follow_all_too_many_arguments(self):
response = self._links_response()
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError, match="Please supply exactly one of the following arguments"
+ ):
response.follow_all(
css='a[href*="example.com"]',
xpath='//a[contains(@href, "example.com")]',
@@ -825,7 +824,9 @@ class TextResponseTest(BaseResponseTest):
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError, match="(Expecting value|Unexpected '<'): line 1"
+ ):
text_response.json()
def test_cache_json_response(self):
@@ -1023,10 +1024,8 @@ class CustomResponseTest(TextResponseTest):
self.assertEqual(r4.bar, "bar")
self.assertIsNone(r4.lost)
- with self.assertRaises(TypeError) as ctx:
+ with pytest.raises(
+ TypeError,
+ match=r"__init__\(\) got an unexpected keyword argument 'unknown'",
+ ):
r1.replace(unknown="unknown")
- self.assertTrue(
- str(ctx.exception).endswith(
- "__init__() got an unexpected keyword argument 'unknown'"
- )
- )
diff --git a/tests/test_item.py b/tests/test_item.py
index 0399c8f8d..47c5c3db6 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -2,6 +2,8 @@ import unittest
from abc import ABCMeta
from unittest import mock
+import pytest
+
from scrapy.item import Field, Item, ItemMeta
@@ -22,7 +24,8 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
- self.assertRaises(KeyError, i.__getitem__, "name")
+ with pytest.raises(KeyError):
+ i["name"]
i2 = TestItem(name="john doe")
self.assertEqual(i2["name"], "john doe")
@@ -33,15 +36,18 @@ class ItemTest(unittest.TestCase):
i4 = TestItem(i3)
self.assertEqual(i4["name"], "john doe")
- self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"})
+ with pytest.raises(KeyError):
+ TestItem({"name": "john doe", "other": "foo"})
def test_invalid_field(self):
class TestItem(Item):
pass
i = TestItem()
- self.assertRaises(KeyError, i.__setitem__, "field", "text")
- self.assertRaises(KeyError, i.__getitem__, "field")
+ with pytest.raises(KeyError):
+ i["field"] = "text"
+ with pytest.raises(KeyError):
+ i["field"]
def test_repr(self):
class TestItem(Item):
@@ -72,14 +78,16 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
- self.assertRaises(AttributeError, getattr, i, "name")
+ with pytest.raises(AttributeError):
+ i.name
def test_raise_setattr(self):
class TestItem(Item):
name = Field()
i = TestItem()
- self.assertRaises(AttributeError, setattr, i, "name", "john")
+ with pytest.raises(AttributeError):
+ i.name = "john"
def test_custom_methods(self):
class TestItem(Item):
@@ -92,7 +100,8 @@ class ItemTest(unittest.TestCase):
self["name"] = name
i = TestItem()
- self.assertRaises(KeyError, i.get_name)
+ with pytest.raises(KeyError):
+ i.get_name()
i["name"] = "lala"
self.assertEqual(i.get_name(), "lala")
i.change_name("other")
@@ -223,7 +232,8 @@ class ItemTest(unittest.TestCase):
class D(B, C):
pass
- self.assertRaises(KeyError, D, not_allowed="value")
+ with pytest.raises(KeyError):
+ D(not_allowed="value")
self.assertEqual(D(save="X")["save"], "X")
self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}})
@@ -231,7 +241,8 @@ class ItemTest(unittest.TestCase):
class E(C, B):
pass
- self.assertRaises(KeyError, E, not_allowed="value")
+ with pytest.raises(KeyError):
+ E(not_allowed="value")
self.assertEqual(E(save="X")["save"], "X")
self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}})
diff --git a/tests/test_link.py b/tests/test_link.py
index 35723bbd6..ed9d27a37 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,5 +1,7 @@
import unittest
+import pytest
+
from scrapy.link import Link
@@ -53,5 +55,5 @@ class LinkTest(unittest.TestCase):
self._assert_same_links(l1, l2)
def test_bytes_url(self):
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
Link(b"http://www.example.com/\xc2\xa3")
diff --git a/tests/test_loader.py b/tests/test_loader.py
index b52d5ea2e..1a933bb8d 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -4,6 +4,7 @@ import dataclasses
import unittest
import attr
+import pytest
from itemadapter import ItemAdapter
from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
@@ -69,7 +70,8 @@ def processor_with_args(value, other=None, loader_context=None):
class BasicItemLoaderTest(unittest.TestCase):
def test_add_value_on_unknown_field(self):
il = ProcessorItemLoader()
- self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])
+ with pytest.raises(KeyError):
+ il.add_value("wrong_field", ["lala", "lolo"])
def test_load_item_using_default_loader(self):
i = SummaryItem()
@@ -294,12 +296,18 @@ class SelectortemLoaderTest(unittest.TestCase):
def test_init_method_errors(self):
l = ProcessorItemLoader()
- self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")
- self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")
- self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")
- self.assertRaises(RuntimeError, l.add_css, "name", "#name::text")
- self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text")
- self.assertRaises(RuntimeError, l.get_css, "#name::text")
+ with pytest.raises(RuntimeError):
+ l.add_xpath("url", "//a/@href")
+ with pytest.raises(RuntimeError):
+ l.replace_xpath("url", "//a/@href")
+ with pytest.raises(RuntimeError):
+ l.get_xpath("//a/@href")
+ with pytest.raises(RuntimeError):
+ l.add_css("name", "#name::text")
+ with pytest.raises(RuntimeError):
+ l.replace_css("name", "#name::text")
+ with pytest.raises(RuntimeError):
+ l.get_css("#name::text")
def test_init_method_with_selector(self):
sel = Selector(text="<html><body><div>marta</div></body></html>")
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 1e504f539..0d7921b1d 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -6,6 +6,7 @@ Once we remove the references from scrapy, we can remove these tests.
import unittest
from functools import partial
+import pytest
from itemloaders.processors import (
Compose,
Identity,
@@ -435,7 +436,13 @@ class BasicItemLoaderTest(unittest.TestCase):
name_in = MapCompose(float)
il = TestItemLoader()
- self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"])
+ with pytest.raises(
+ ValueError,
+ match="Error with input processor MapCompose: .* "
+ "error='ValueError: Error in MapCompose .* "
+ "error='ValueError: could not convert",
+ ):
+ il.add_value("name", ["marta", "other"])
def test_error_output_processor(self):
class TestItem(Item):
@@ -447,7 +454,12 @@ class BasicItemLoaderTest(unittest.TestCase):
il = TestItemLoader()
il.add_value("name", "marta")
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError,
+ match="Error with output processor: .* "
+ "error='ValueError: Error in Compose .* "
+ "error='ValueError: could not convert",
+ ):
il.load_item()
def test_error_processor_as_argument(self):
@@ -458,9 +470,13 @@ class BasicItemLoaderTest(unittest.TestCase):
default_item_class = TestItem
il = TestItemLoader()
- self.assertRaises(
- ValueError, il.add_value, "name", ["marta", "other"], Compose(float)
- )
+ with pytest.raises(
+ ValueError,
+ match=r"Error with processor Compose .* "
+ r"error='ValueError: Error in Compose .* "
+ r"error='TypeError: float\(\) argument",
+ ):
+ il.add_value("name", ["marta", "other"], Compose(float))
class InitializationFromDictTest(unittest.TestCase):
@@ -630,7 +646,8 @@ class ProcessorsTest(unittest.TestCase):
def test_join(self):
proc = Join()
- self.assertRaises(TypeError, proc, [None, "", "hello", "world"])
+ with pytest.raises(TypeError):
+ proc([None, "", "hello", "world"])
self.assertEqual(proc(["", "hello", "world"]), " hello world")
self.assertEqual(proc(["hello", "world"]), "hello world")
self.assertIsInstance(proc(["hello", "world"]), str)
@@ -641,9 +658,17 @@ class ProcessorsTest(unittest.TestCase):
proc = Compose(str.upper)
self.assertEqual(proc(None), None)
proc = Compose(str.upper, stop_on_none=False)
- self.assertRaises(ValueError, proc, None)
+ with pytest.raises(
+ ValueError,
+ match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
+ ):
+ proc(None)
proc = Compose(str.upper, lambda x: x + 1)
- self.assertRaises(ValueError, proc, "hello")
+ with pytest.raises(
+ ValueError,
+ match="Error in Compose with .* error='TypeError: (can only|unsupported operand)",
+ ):
+ proc("hello")
def test_mapcompose(self):
def filter_world(x):
@@ -657,9 +682,17 @@ class ProcessorsTest(unittest.TestCase):
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
proc = MapCompose(filter_world, str.upper)
- self.assertRaises(ValueError, proc, [1])
+ with pytest.raises(
+ ValueError,
+ match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
+ ):
+ proc([1])
proc = MapCompose(filter_world, lambda x: x + 1)
- self.assertRaises(ValueError, proc, "hello")
+ with pytest.raises(
+ ValueError,
+ match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)",
+ ):
+ proc("hello")
class SelectJmesTestCase(unittest.TestCase):
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index a4b002e34..6bc5b6f1f 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -1,6 +1,8 @@
import unittest
from datetime import datetime
+import pytest
+
from scrapy.extensions.logstats import LogStats
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@@ -18,8 +20,9 @@ class TestLogStats(unittest.TestCase):
def test_stats_calculations(self):
logstats = LogStats.from_crawler(self.crawler)
- with self.assertRaises(AttributeError):
+ with pytest.raises(AttributeError):
logstats.pagesprev
+ with pytest.raises(AttributeError):
logstats.itemsprev
logstats.spider_opened(self.spider)
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 3d049843a..1d89e44ce 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -7,6 +7,7 @@ from shutil import rmtree
from tempfile import mkdtemp
import attr
+import pytest
from itemadapter import ItemAdapter
from twisted.trial import unittest
@@ -146,11 +147,11 @@ class ImagesPipelineTestCase(unittest.TestCase):
resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue())
req = Request(url="https://dev.mydeco.com/mydeco.gif")
- with self.assertRaises(ImageException):
+ with pytest.raises(ImageException):
next(self.pipeline.get_images(response=resp1, request=req, info=object()))
- with self.assertRaises(ImageException):
+ with pytest.raises(ImageException):
next(self.pipeline.get_images(response=resp2, request=req, info=object()))
- with self.assertRaises(ImageException):
+ with pytest.raises(ImageException):
next(self.pipeline.get_images(response=resp3, request=req, info=object()))
def test_get_images(self):
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index 1584014b8..c223c4562 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -1,6 +1,7 @@
import tempfile
import unittest
+import pytest
import queuelib
from scrapy.http.request import Request
@@ -40,9 +41,9 @@ class PriorityQueueTest(unittest.TestCase):
self.crawler, FifoMemoryQueue, temp_dir
)
queue.push(Request("https://example.org"))
- with self.assertRaises(
+ with pytest.raises(
NotImplementedError,
- msg="The underlying queue class does not implement 'peek'",
+ match="The underlying queue class does not implement 'peek'",
):
queue.peek()
queue.close()
@@ -129,9 +130,9 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
self.queue.push(Request("https://example.org"))
- with self.assertRaises(
+ with pytest.raises(
NotImplementedError,
- msg="The underlying queue class does not implement 'peek'",
+ match="The underlying queue class does not implement 'peek'",
):
self.queue.peek()
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index 85133038a..2c605a015 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -1,5 +1,7 @@
import unittest
+import pytest
+
from scrapy import Request, Spider
from scrapy.http import FormRequest, JsonRequest
from scrapy.utils.request import request_from_dict
@@ -134,11 +136,15 @@ class RequestSerializationTest(unittest.TestCase):
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
- self.assertRaises(ValueError, r.to_dict, spider=self.spider)
+ with pytest.raises(
+ ValueError, match="is not an instance method in: <MethodsSpider"
+ ):
+ r.to_dict(spider=self.spider)
def test_unserializable_callback2(self):
r = Request("http://www.example.com", callback=self.spider.parse_item)
- self.assertRaises(ValueError, r.to_dict, spider=None)
+ with pytest.raises(ValueError, match="is not an instance method in: None"):
+ r.to_dict(spider=None)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
@@ -152,14 +158,18 @@ class RequestSerializationTest(unittest.TestCase):
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
spider.parse = None
- self.assertRaises(ValueError, r.to_dict, spider=spider)
+ with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
+ r.to_dict(spider=spider)
def test_callback_not_available(self):
"""Callback method is not available in the spider passed to from_dict"""
spider = SpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
- self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
+ with pytest.raises(
+ ValueError, match="Method 'delegated_callback' not found in: <Spider"
+ ):
+ request_from_dict(d, spider=Spider("foo"))
class SpiderMixin:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 3ac330ae2..f2f8b96cd 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -5,6 +5,7 @@ import tempfile
import unittest
from typing import Any, NamedTuple
+import pytest
from twisted.internet import defer
from twisted.trial.unittest import TestCase
@@ -229,7 +230,10 @@ class TestMigration(unittest.TestCase):
next_scheduler_handler.create_scheduler()
def test_migration(self):
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError,
+ match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict",
+ ):
self._migration(self.tmpdir)
@@ -351,5 +355,7 @@ class TestIncompatibility(unittest.TestCase):
scheduler.open(spider)
def test_incompatibility(self):
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP"
+ ):
self._incompatible()
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 7c72805e2..c2bb8cec5 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from unittest import TestCase
from urllib.parse import urljoin
+import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
@@ -75,13 +76,12 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin):
def test_methods(self):
self.assertIsNone(self.scheduler.open(Spider("foo")))
self.assertIsNone(self.scheduler.close("finished"))
- self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests)
- self.assertRaises(
- NotImplementedError,
- self.scheduler.enqueue_request,
- Request("https://example.org"),
- )
- self.assertRaises(NotImplementedError, self.scheduler.next_request)
+ with pytest.raises(NotImplementedError):
+ self.scheduler.has_pending_requests()
+ with pytest.raises(NotImplementedError):
+ self.scheduler.enqueue_request(Request("https://example.org"))
+ with pytest.raises(NotImplementedError):
+ self.scheduler.next_request()
class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):
@@ -89,15 +89,15 @@ class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):
self.scheduler = MinimalScheduler()
def test_open_close(self):
- with self.assertRaises(AttributeError):
+ with pytest.raises(AttributeError):
self.scheduler.open(Spider("foo"))
- with self.assertRaises(AttributeError):
+ with pytest.raises(AttributeError):
self.scheduler.close("finished")
def test_len(self):
- with self.assertRaises(AttributeError):
+ with pytest.raises(AttributeError):
self.scheduler.__len__()
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
len(self.scheduler)
def test_enqueue_dequeue(self):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 4eda0460f..2d7a1442e 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -112,7 +112,7 @@ class SelectorTestCase(unittest.TestCase):
)
def test_selector_bad_args(self):
- with self.assertRaisesRegex(ValueError, "received both response and text"):
+ with pytest.raises(ValueError, match="received both response and text"):
Selector(TextResponse(url="http://example.com", body=b""), text="")
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 5c8a19d9b..b7a316eee 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -235,9 +235,9 @@ class BaseSettingsTest(unittest.TestCase):
self.assertIn("key_highprio", settings)
del settings["key_highprio"]
self.assertNotIn("key_highprio", settings)
- with self.assertRaises(KeyError):
+ with pytest.raises(KeyError):
settings.delete("notkey")
- with self.assertRaises(KeyError):
+ with pytest.raises(KeyError):
del settings["notkey"]
def test_get(self):
@@ -303,9 +303,19 @@ class BaseSettingsTest(unittest.TestCase):
self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3})
self.assertEqual(settings.getdict("TEST_DICT3"), {})
self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5})
- self.assertRaises(ValueError, settings.getdict, "TEST_LIST1")
- self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG")
- self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG")
+ with pytest.raises(
+ ValueError,
+ match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
+ ):
+ settings.getdict("TEST_LIST1")
+ with pytest.raises(
+ ValueError, match="Supported values for boolean settings are"
+ ):
+ settings.getbool("TEST_ENABLED_WRONG")
+ with pytest.raises(
+ ValueError, match="Supported values for boolean settings are"
+ ):
+ settings.getbool("TEST_DISABLED_WRONG")
def test_getpriority(self):
settings = BaseSettings({"key": "value"}, priority=99)
@@ -381,11 +391,10 @@ class BaseSettingsTest(unittest.TestCase):
def test_freeze(self):
self.settings.freeze()
- with self.assertRaises(TypeError) as cm:
+ with pytest.raises(
+ TypeError, match="Trying to modify an immutable Settings object"
+ ):
self.settings.set("TEST_BOOL", False)
- self.assertEqual(
- str(cm.exception), "Trying to modify an immutable Settings object"
- )
def test_frozencopy(self):
frozencopy = self.settings.frozencopy()
@@ -476,7 +485,7 @@ class SettingsTest(unittest.TestCase):
def test_pop_item_with_default_value(self):
settings = Settings()
- with self.assertRaises(KeyError):
+ with pytest.raises(KeyError):
settings.pop("DUMMY_CONFIG")
dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value")
@@ -491,9 +500,7 @@ class SettingsTest(unittest.TestCase):
settings.freeze()
- with self.assertRaises(TypeError) as error:
+ with pytest.raises(
+ TypeError, match="Trying to modify an immutable Settings object"
+ ):
settings.pop("OTHER_DUMMY_CONFIG")
-
- self.assertEqual(
- str(error.exception), "Trying to modify an immutable Settings object"
- )
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 18a863350..af29872a8 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -7,6 +7,7 @@ from pathlib import Path
from typing import Any
from unittest import mock
+import pytest
from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
@@ -57,8 +58,11 @@ class SpiderTest(unittest.TestCase):
def test_spider_without_name(self):
"""``__init__`` method arguments are assigned to spider attributes"""
- self.assertRaises(ValueError, self.spider_class)
- self.assertRaises(ValueError, self.spider_class, somearg="foo")
+ msg = "must have a name"
+ with pytest.raises(ValueError, match=msg):
+ self.spider_class()
+ with pytest.raises(ValueError, match=msg):
+ self.spider_class(somearg="foo")
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
@@ -475,7 +479,7 @@ class CrawlSpiderTest(SpiderTest):
spider = self.spider_class("example.com")
spider.start_url = "https://www.example.com"
- with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"):
+ with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"):
list(spider.start_requests())
@@ -825,5 +829,5 @@ class NoParseMethodSpiderTest(unittest.TestCase):
resp = TextResponse(url="http://www.example.com/random_url", body=text)
exc_msg = "Spider.parse callback is not defined"
- with self.assertRaisesRegex(NotImplementedError, exc_msg):
+ with pytest.raises(NotImplementedError, match=exc_msg):
spider.parse(resp)
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 705f722b3..b103e9ed0 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -7,6 +7,7 @@ from pathlib import Path
from tempfile import mkdtemp
from unittest import mock
+import pytest
from twisted.trial import unittest
from zope.interface.verify import verifyObject
@@ -124,9 +125,8 @@ class SpiderLoaderTest(unittest.TestCase):
}
)
- self.assertRaisesRegex(
- KeyError, "Spider not found", runner.create_crawler, "spider2"
- )
+ with pytest.raises(KeyError, match="Spider not found"):
+ runner.create_crawler("spider2")
crawler = runner.create_crawler("spider1")
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
@@ -135,7 +135,8 @@ class SpiderLoaderTest(unittest.TestCase):
def test_bad_spider_modules_exception(self):
module = "tests.test_spiderloader.test_spiders.doesnotexist"
settings = Settings({"SPIDER_MODULES": [module]})
- self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
+ with pytest.raises(ImportError):
+ SpiderLoader.from_settings(settings)
def test_bad_spider_modules_warning(self):
with warnings.catch_warnings(record=True) as w:
@@ -159,7 +160,8 @@ class SpiderLoaderTest(unittest.TestCase):
with mock.patch.object(SpiderLoader, "_load_spiders") as m:
m.side_effect = SyntaxError
settings = Settings({"SPIDER_MODULES": [module]})
- self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
+ with pytest.raises(SyntaxError):
+ SpiderLoader.from_settings(settings)
def test_syntax_error_warning(self):
with (
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index a8507c789..a9f3876bb 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from collections.abc import AsyncIterator, Iterable
from unittest import mock
+import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.python.failure import Failure
@@ -299,12 +300,9 @@ class ProcessSpiderOutputCoroutineMiddleware:
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def test_non_iterable(self):
- with self.assertRaisesRegex(
+ with pytest.raises(
_InvalidOutput,
- (
- r"\.process_spider_output must return an iterable, got <class "
- r"'NoneType'>"
- ),
+ match=r"\.process_spider_output must return an iterable, got <class 'NoneType'>",
):
yield self._get_middleware_result(
ProcessSpiderOutputNonIterableMiddleware,
@@ -312,9 +310,9 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def test_coroutine(self):
- with self.assertRaisesRegex(
+ with pytest.raises(
_InvalidOutput,
- r"\.process_spider_output must be an asynchronous generator",
+ match=r"\.process_spider_output must be an asynchronous generator",
):
yield self._get_middleware_result(
ProcessSpiderOutputCoroutineMiddleware,
@@ -518,8 +516,8 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def _test_asyncgen_nodowngrade(self, *mw_classes):
- with self.assertRaisesRegex(
- _InvalidOutput, "Async iterable returned from .+ cannot be downgraded"
+ with pytest.raises(
+ _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"
):
yield self._get_middleware_result(*mw_classes)
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 307054de7..f9eb93d6b 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,6 +1,7 @@
import logging
from unittest import TestCase
+import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TrialTestCase
@@ -68,9 +69,8 @@ class TestHttpErrorMiddleware(TestCase):
def test_process_spider_input(self):
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
- self.assertRaises(
- HttpError, self.mw.process_spider_input, self.res404, self.spider
- )
+ with pytest.raises(HttpError):
+ self.mw.process_spider_input(self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEqual(
@@ -105,9 +105,8 @@ class TestHttpErrorMiddlewareSettings(TestCase):
def test_process_spider_input(self):
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
- self.assertRaises(
- HttpError, self.mw.process_spider_input, self.res404, self.spider
- )
+ with pytest.raises(HttpError):
+ self.mw.process_spider_input(self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
@@ -120,14 +119,14 @@ class TestHttpErrorMiddlewareSettings(TestCase):
res402.request = request
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
- self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
+ with pytest.raises(HttpError):
+ self.mw.process_spider_input(res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
- self.assertRaises(
- HttpError, self.mw.process_spider_input, self.res402, self.spider
- )
+ with pytest.raises(HttpError):
+ self.mw.process_spider_input(self.res402, self.spider)
class TestHttpErrorMiddlewareHandleAll(TestCase):
@@ -151,7 +150,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402.request = request
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
- self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
+ with pytest.raises(HttpError):
+ self.mw.process_spider_input(res402, self.spider)
def test_httperror_allow_all_false(self):
crawler = get_crawler(_HttpErrorSpider)
@@ -167,7 +167,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402 = self.res402.copy()
res402.request = request_httpstatus_true
- self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
+ with pytest.raises(HttpError):
+ mw.process_spider_input(res404, self.spider)
self.assertIsNone(mw.process_spider_input(res402, self.spider))
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 4945ac25d..01a87c645 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -5,6 +5,8 @@ from typing import Any
from unittest import TestCase
from urllib.parse import urlparse
+import pytest
+
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.http import Request, Response
from scrapy.settings import Settings
@@ -884,7 +886,7 @@ class TestSettingsPolicyByName(TestCase):
def test_invalid_name(self):
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
- with self.assertRaises(RuntimeError):
+ with pytest.raises(RuntimeError):
RefererMiddleware(settings)
def test_multiple_policy_tokens(self):
@@ -925,7 +927,7 @@ class TestSettingsPolicyByName(TestCase):
)
}
)
- with self.assertRaises(RuntimeError):
+ with pytest.raises(RuntimeError):
RefererMiddleware(settings)
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index 59d18d92e..72692afab 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -2,6 +2,7 @@ import shutil
from datetime import datetime, timezone
from tempfile import mkdtemp
+import pytest
from twisted.trial import unittest
from scrapy.exceptions import NotConfigured
@@ -42,4 +43,5 @@ class SpiderStateTest(unittest.TestCase):
def test_not_configured(self):
crawler = get_crawler(Spider)
- self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler)
+ with pytest.raises(NotConfigured):
+ SpiderState.from_crawler(crawler)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index a2e7ae65d..8556b75dd 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,6 +1,7 @@
import pickle
import sys
+import pytest
from queuelib.tests import test_queue as t
from scrapy.http import Request
@@ -30,10 +31,17 @@ class MyLoader(ItemLoader):
def nonserializable_object_test(self):
q = self.queue()
- self.assertRaises(ValueError, q.push, lambda x: x)
+ with pytest.raises(
+ ValueError,
+ match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+ ):
+ q.push(lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text="<html><body><p>some text</p></body></html>")
- self.assertRaises(ValueError, q.push, sel)
+ with pytest.raises(
+ ValueError, match="unmarshallable object|can't pickle Selector objects"
+ ):
+ q.push(sel)
class FifoDiskQueueTestMixin:
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index 04eeae4dc..88f6657d8 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -6,6 +6,7 @@ import shutil
import tempfile
import unittest
+import pytest
import queuelib
from scrapy.http import Request
@@ -69,9 +70,9 @@ class RequestQueueTestMixin:
req = Request("http://www.example.com")
q.push(req)
self.assertEqual(len(q), 1)
- with self.assertRaises(
+ with pytest.raises(
NotImplementedError,
- msg="The underlying queue class does not implement 'peek'",
+ match="The underlying queue class does not implement 'peek'",
):
q.peek()
self.assertEqual(q.pop().url, req.url)
@@ -120,9 +121,9 @@ class FifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
- with self.assertRaises(
+ with pytest.raises(
NotImplementedError,
- msg="The underlying queue class does not implement 'peek'",
+ match="The underlying queue class does not implement 'peek'",
):
q.peek()
self.assertEqual(len(q), 3)
@@ -176,9 +177,9 @@ class LifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
- with self.assertRaises(
+ with pytest.raises(
NotImplementedError,
- msg="The underlying queue class does not implement 'peek'",
+ match="The underlying queue class does not implement 'peek'",
):
q.peek()
self.assertEqual(len(q), 3)
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index cbea41129..e27bb7b74 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,5 +1,7 @@
import unittest
+import pytest
+
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
@@ -32,7 +34,9 @@ class BuildComponentListTest(unittest.TestCase):
)
# Same priority raises ValueError
duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError, match="Some paths in .* convert to the same object"
+ ):
build_component_list(duplicate_bs, convert=lambda x: x.lower())
def test_valid_numbers(self):
@@ -58,21 +62,13 @@ class UtilsConfTestCase(unittest.TestCase):
class FeedExportConfigTestCase(unittest.TestCase):
def test_feed_export_config_invalid_format(self):
settings = Settings()
- self.assertRaises(
- UsageError,
- feed_process_params_from_cli,
- settings,
- ["items.dat"],
- )
+ with pytest.raises(UsageError):
+ feed_process_params_from_cli(settings, ["items.dat"])
def test_feed_export_config_mismatch(self):
settings = Settings()
- self.assertRaises(
- UsageError,
- feed_process_params_from_cli,
- settings,
- ["items1.dat", "items2.dat"],
- )
+ with pytest.raises(UsageError):
+ feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"])
def test_feed_export_config_explicit_formats(self):
settings = Settings()
@@ -117,11 +113,9 @@ class FeedExportConfigTestCase(unittest.TestCase):
)
def test_output_and_overwrite_output(self):
- with self.assertRaises(UsageError):
+ with pytest.raises(UsageError):
feed_process_params_from_cli(
- Settings(),
- ["output1.json"],
- overwrite_output=["output2.json"],
+ Settings(), ["output1.json"], overwrite_output=["output2.json"]
)
def test_feed_complete_default_values_from_settings_empty(self):
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 5d99161bf..a5b438645 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,6 +1,7 @@
import unittest
import warnings
+import pytest
from w3lib.http import basic_auth_header
from scrapy import Request
@@ -205,11 +206,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
def test_too_few_arguments_error(self):
- self.assertRaisesRegex(
+ with pytest.raises(
ValueError,
- r"too few arguments|the following arguments are required:\s*url",
- lambda: curl_to_request_kwargs("curl"),
- )
+ match=r"too few arguments|the following arguments are required:\s*url",
+ ):
+ curl_to_request_kwargs("curl")
def test_ignore_unknown_options(self):
# case 1: ignore_unknown_options=True:
@@ -220,16 +221,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
# case 2: ignore_unknown_options=False (raise exception):
- self.assertRaisesRegex(
- ValueError,
- "Unrecognized options:.*--bar.*--baz",
- lambda: curl_to_request_kwargs(
+ with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"):
+ curl_to_request_kwargs(
"curl --bar --baz http://www.example.com", ignore_unknown_options=False
- ),
- )
+ )
def test_must_start_with_curl_error(self):
- self.assertRaises(
- ValueError,
- lambda: curl_to_request_kwargs("carl -X POST http://example.org"),
- )
+ with pytest.raises(ValueError, match="A curl command must start"):
+ curl_to_request_kwargs("carl -X POST http://example.org")
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index fadbc6daa..2e35d339a 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -87,8 +87,10 @@ class CaseInsensitiveDictMixin:
def test_delete(self):
d = self.dict_class({"key_lower": 1})
del d["key_LOWER"]
- self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
- self.assertRaises(KeyError, d.__getitem__, "key_lower")
+ with pytest.raises(KeyError):
+ d["key_LOWER"]
+ with pytest.raises(KeyError):
+ d["key_lower"]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_getdefault(self):
@@ -138,7 +140,8 @@ class CaseInsensitiveDictMixin:
d = self.dict_class()
d["a"] = 1
self.assertEqual(d.pop("A"), 1)
- self.assertRaises(KeyError, d.pop, "A")
+ with pytest.raises(KeyError):
+ d.pop("A")
def test_normkey(self):
class MyDict(self.dict_class):
@@ -279,8 +282,8 @@ class SequenceExcludeTest(unittest.TestCase):
self.assertIn(set("bar"), d)
# supplied sequence is a set, so checking for list (non)inclusion fails
- self.assertRaises(TypeError, (0, 1, 2) in d)
- self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"])
+ with pytest.raises(TypeError):
+ ["a", "b", "c"] in d # noqa: B015
for v in [-3, "test", 1.1]:
self.assertNotIn(v, d)
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index dc5fbd3c3..e917b6947 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -3,6 +3,8 @@ import unittest
import warnings
from unittest import mock
+import pytest
+
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
@@ -181,7 +183,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
- self.assertRaises(TypeError, issubclass, object(), DeprecatedName)
+ with pytest.raises(TypeError):
+ issubclass(object(), DeprecatedName)
def test_isinstance(self):
with warnings.catch_warnings():
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 7b7a25db8..d40cae9c7 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -1,6 +1,8 @@
import unittest
+from gzip import BadGzipFile
from pathlib import Path
+import pytest
from w3lib.encoding import html_to_unicode
from scrapy.http import Response
@@ -27,9 +29,8 @@ class GunzipTest(unittest.TestCase):
assert text.endswith(b"</html")
def test_gunzip_no_gzip_file_raises(self):
- self.assertRaises(
- OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
- )
+ with pytest.raises(BadGzipFile):
+ gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes())
def test_gunzip_truncated_short(self):
r1 = Response(
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 12507c6a3..9ad30617a 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -215,7 +215,7 @@ class XmliterBaseTestCase:
"""
response = XmlResponse(url="http://mydummycompany.com", body=body)
my_iter = self.xmliter(response, "g:link_image")
- with self.assertRaises(StopIteration):
+ with pytest.raises(StopIteration):
next(my_iter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@@ -228,13 +228,14 @@ class XmliterBaseTestCase:
iter = self.xmliter(body, "product")
next(iter)
next(iter)
-
- self.assertRaises(StopIteration, next, iter)
+ with pytest.raises(StopIteration):
+ next(iter)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
- self.assertRaises(TypeError, next, i)
+ with pytest.raises(TypeError):
+ next(i)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_xmliter_encoding(self):
@@ -344,7 +345,8 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase):
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
- self.assertRaises(TypeError, next, i)
+ with pytest.raises(TypeError):
+ next(i)
class UtilsCsvTestCase(unittest.TestCase):
@@ -491,8 +493,8 @@ class UtilsCsvTestCase(unittest.TestCase):
next(iter)
next(iter)
next(iter)
-
- self.assertRaises(StopIteration, next, iter)
+ with pytest.raises(StopIteration):
+ next(iter)
def test_csviter_encoding(self):
body1 = get_testdata("feeds", "feed-sample4.csv")
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e25bdfe3f..a67e16962 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -32,9 +32,12 @@ class UtilsMiscTestCase(unittest.TestCase):
self.assertIs(obj, load_object)
def test_load_object_exceptions(self):
- self.assertRaises(ImportError, load_object, "nomodule999.mod.function")
- self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999")
- self.assertRaises(TypeError, load_object, {})
+ with pytest.raises(ImportError):
+ load_object("nomodule999.mod.function")
+ with pytest.raises(NameError):
+ load_object("scrapy.utils.misc.load_object999")
+ with pytest.raises(TypeError):
+ load_object({})
def test_walk_modules(self):
mods = walk_modules("tests.test_utils_misc.test_walk_modules")
@@ -59,7 +62,8 @@ class UtilsMiscTestCase(unittest.TestCase):
]
self.assertEqual({m.__name__ for m in mods}, set(expected))
- self.assertRaises(ImportError, walk_modules, "nomodule999")
+ with pytest.raises(ImportError):
+ walk_modules("nomodule999")
def test_walk_modules_egg(self):
egg = str(Path(__file__).parent / "test.egg")
@@ -148,11 +152,13 @@ class UtilsMiscTestCase(unittest.TestCase):
create_instance(m, None, crawler, *args, **kwargs)
m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
- with self.assertRaises(ValueError):
+ with pytest.raises(
+ ValueError, match="Specify at least one of settings and crawler"
+ ):
create_instance(m, None, None)
m.from_settings.return_value = None
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
create_instance(m, settings, None)
def test_build_from_crawler(self):
@@ -191,7 +197,7 @@ class UtilsMiscTestCase(unittest.TestCase):
# Check adoption of crawler
m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
m.from_crawler.return_value = None
- with self.assertRaises(TypeError):
+ with pytest.raises(TypeError):
build_from_crawler(m, crawler, *args, **kwargs)
def test_set_environ(self):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index a693d6b53..3b0739276 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -87,7 +87,8 @@ class ToUnicodeTest(unittest.TestCase):
self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e")
def test_converting_a_strange_object_should_raise_TypeError(self):
- self.assertRaises(TypeError, to_unicode, 423)
+ with pytest.raises(TypeError):
+ to_unicode(423)
def test_errors_argument(self):
self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb")
@@ -104,7 +105,8 @@ class ToBytesTest(unittest.TestCase):
self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e")
def test_converting_a_strange_object_should_raise_TypeError(self):
- self.assertRaises(TypeError, to_bytes, unittest)
+ with pytest.raises(TypeError):
+ to_bytes(pytest)
def test_errors_argument(self):
self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b")
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index c6ba8cbbb..af7906781 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -35,7 +35,8 @@ class ResponseUtilsTest(unittest.TestCase):
assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
resp = Response(url, body=body)
- self.assertRaises(TypeError, open_in_browser, resp, debug=True)
+ with pytest.raises(TypeError):
+ open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
def test_get_meta_refresh(self):
r1 = HtmlResponse(