Scrapy, a fast high-level web crawling & scraping framework for Python.
Converting tests to plain asserts, part 4. (#6702)
Andrey Rakhmatullin3ded1dfe3151
diff --git a/tests/test_spider.py b/tests/test_spider.pyindex af29872a8..05f1c59d0 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawlerfrom tests import get_testdata, tests_datadir-class SpiderTest(unittest.TestCase):+class TestSpider(unittest.TestCase):spider_class = Spiderdef setUp(self):@@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase):def test_base_spider(self):spider = self.spider_class("example.com")- self.assertEqual(spider.name, "example.com")- self.assertEqual(spider.start_urls, [])+ assert spider.name == "example.com"+ assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparisondef test_start_requests(self):spider = self.spider_class("example.com")start_requests = spider.start_requests()- self.assertTrue(inspect.isgenerator(start_requests))- self.assertEqual(list(start_requests), [])+ assert inspect.isgenerator(start_requests)+ assert not list(start_requests)def test_spider_args(self):"""``__init__`` method arguments are assigned to spider attributes"""spider = self.spider_class("example.com", foo="bar")- self.assertEqual(spider.foo, "bar")+ assert spider.foo == "bar"def test_spider_without_name(self):"""``__init__`` method arguments are assigned to spider attributes"""@@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase):def test_from_crawler_crawler_and_settings_population(self):crawler = get_crawler()spider = self.spider_class.from_crawler(crawler, "example.com")- self.assertTrue(hasattr(spider, "crawler"))- self.assertIs(spider.crawler, crawler)- self.assertTrue(hasattr(spider, "settings"))- self.assertIs(spider.settings, crawler.settings)+ assert hasattr(spider, "crawler")+ assert spider.crawler is crawler+ assert hasattr(spider, "settings")+ assert spider.settings is crawler.settingsdef test_from_crawler_init_call(self):with mock.patch.object(@@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase):crawler.signals.send_catch_log(signal=signals.spider_closed, spider=spider, reason=None)- self.assertTrue(spider.closed_called)+ assert spider.closed_calleddef test_update_settings(self):spider_settings = {"TEST1": "spider", "TEST2": "spider"}@@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase):settings = Settings(project_settings, priority="project")self.spider_class.update_settings(settings)- self.assertEqual(settings.get("TEST1"), "spider")- self.assertEqual(settings.get("TEST2"), "spider")- self.assertEqual(settings.get("TEST3"), "project")+ assert settings.get("TEST1") == "spider"+ assert settings.get("TEST2") == "spider"+ assert settings.get("TEST3") == "project"@inlineCallbacksdef test_settings_in_from_crawler(self):@@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase):return spidercrawler = Crawler(TestSpider, project_settings)- self.assertEqual(crawler.settings.get("TEST1"), "spider")- self.assertEqual(crawler.settings.get("TEST2"), "spider")- self.assertEqual(crawler.settings.get("TEST3"), "project")+ assert crawler.settings.get("TEST1") == "spider"+ assert crawler.settings.get("TEST2") == "spider"+ assert crawler.settings.get("TEST3") == "project"yield crawler.crawl()- self.assertEqual(crawler.settings.get("TEST1"), "spider_instance")+ assert crawler.settings.get("TEST1") == "spider_instance"def test_logger(self):spider = self.spider_class("example.com")@@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase):lc.check(("example.com", "INFO", "test log msg"))record = lc.records[0]- self.assertIn("spider", record.__dict__)- self.assertIs(record.spider, spider)+ assert "spider" in record.__dict__+ assert record.spider is spiderdef test_log(self):spider = self.spider_class("example.com")@@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase):mock_logger.log.assert_called_once_with("INFO", "test log msg")-class InitSpiderTest(SpiderTest):+class TestInitSpider(TestSpider):spider_class = InitSpider-class XMLFeedSpiderTest(SpiderTest):+class TestXMLFeedSpider(TestSpider):spider_class = XMLFeedSpiderdef test_register_namespace(self):@@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest):for iterator in ("iternodes", "xml"):spider = _XMLSpider("example", iterator=iterator)output = list(spider._parse(response))- self.assertEqual(len(output), 2, iterator)- self.assertEqual(- output,- [- {- "loc": ["http://www.example.com/Special-Offers.html"],- "updated": ["2009-08-16"],- "custom": ["fuu"],- "other": ["bar"],- },- {- "loc": [],- "updated": ["2009-08-16"],- "other": ["foo"],- "custom": [],- },- ],- iterator,- )---class CSVFeedSpiderTest(SpiderTest):+ assert len(output) == 2, iterator+ assert output == [+ {+ "loc": ["http://www.example.com/Special-Offers.html"],+ "updated": ["2009-08-16"],+ "custom": ["fuu"],+ "other": ["bar"],+ },+ {+ "loc": [],+ "updated": ["2009-08-16"],+ "other": ["foo"],+ "custom": [],+ },+ ], iterator+++class TestCSVFeedSpider(TestSpider):spider_class = CSVFeedSpiderdef test_parse_rows(self):@@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest):assert len(rows) == 4-class CrawlSpiderTest(SpiderTest):+class TestCrawlSpider(TestSpider):test_body = b"""<html><head><title>Page title<title><body><p><a href="item/12.html">Item 12</a></p>@@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.org/somepage/item/12.html",- "http://example.org/about.html",- "http://example.org/nofollow.html",- ],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ "http://example.org/nofollow.html",+ ]def test_process_links(self):response = HtmlResponse(@@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.org/somepage/item/12.html",- "http://example.org/about.html",- "http://example.org/nofollow.html",- ],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ "http://example.org/nofollow.html",+ ]def test_process_links_filter(self):response = HtmlResponse(@@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 2)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.org/somepage/item/12.html",- "http://example.org/about.html",- ],- )+ assert len(output) == 2+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ ]def test_process_links_generator(self):response = HtmlResponse(@@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.org/somepage/item/12.html",- "http://example.org/about.html",- "http://example.org/nofollow.html",- ],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ "http://example.org/nofollow.html",+ ]def test_process_request(self):response = HtmlResponse(@@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.com/somepage/item/12.html",- "http://example.com/about.html",- "http://example.com/nofollow.html",- ],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.com/somepage/item/12.html",+ "http://example.com/about.html",+ "http://example.com/nofollow.html",+ ]def test_process_request_with_response(self):response = HtmlResponse(@@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.org/somepage/item/12.html",- "http://example.org/about.html",- "http://example.org/nofollow.html",- ],- )- self.assertEqual(- [r.meta["response_class"] for r in output],- ["HtmlResponse", "HtmlResponse", "HtmlResponse"],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ "http://example.org/nofollow.html",+ ]+ assert [r.meta["response_class"] for r in output] == [+ "HtmlResponse",+ "HtmlResponse",+ "HtmlResponse",+ ]def test_process_request_instance_method(self):response = HtmlResponse(@@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),- safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),- safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),- ],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),+ safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),+ safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),+ ]def test_process_request_instance_method_with_response(self):response = HtmlResponse(@@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest):spider = _CrawlSpider()output = list(spider._requests_to_follow(response))- self.assertEqual(len(output), 3)- self.assertTrue(all(isinstance(r, Request) for r in output))- self.assertEqual(- [r.url for r in output],- [- "http://example.org/somepage/item/12.html",- "http://example.org/about.html",- "http://example.org/nofollow.html",- ],- )- self.assertEqual(- [r.meta["response_class"] for r in output],- ["HtmlResponse", "HtmlResponse", "HtmlResponse"],- )+ assert len(output) == 3+ assert all(isinstance(r, Request) for r in output)+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ "http://example.org/nofollow.html",+ ]+ assert [r.meta["response_class"] for r in output] == [+ "HtmlResponse",+ "HtmlResponse",+ "HtmlResponse",+ ]def test_follow_links_attribute_population(self):crawler = get_crawler()spider = self.spider_class.from_crawler(crawler, "example.com")- self.assertTrue(hasattr(spider, "_follow_links"))- self.assertTrue(spider._follow_links)+ assert hasattr(spider, "_follow_links")+ assert spider._follow_linkssettings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}crawler = get_crawler(settings_dict=settings_dict)spider = self.spider_class.from_crawler(crawler, "example.com")- self.assertTrue(hasattr(spider, "_follow_links"))- self.assertFalse(spider._follow_links)+ assert hasattr(spider, "_follow_links")+ assert not spider._follow_linksdef test_start_url(self):spider = self.spider_class("example.com")@@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest):list(spider.start_requests())-class SitemapSpiderTest(SpiderTest):+class TestSitemapSpider(TestSpider):spider_class = SitemapSpiderBODY = b"SITEMAP"@@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest):def assertSitemapBody(self, response, body):crawler = get_crawler()spider = self.spider_class.from_crawler(crawler, "example.com")- self.assertEqual(spider._get_sitemap_body(response), body)+ assert spider._get_sitemap_body(response) == bodydef test_get_sitemap_body(self):r = XmlResponse(url="http://www.example.com/", body=self.BODY)@@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xmlr = TextResponse(url="http://www.example.com/robots.txt", body=robots)spider = self.spider_class("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- [- "http://example.com/sitemap.xml",- "http://example.com/sitemap-product-index.xml",- "http://example.com/sitemap-uppercase.xml",- "http://www.example.com/sitemap-relative-url.xml",- ],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://example.com/sitemap.xml",+ "http://example.com/sitemap-product-index.xml",+ "http://example.com/sitemap-uppercase.xml",+ "http://www.example.com/sitemap-relative-url.xml",+ ]def test_alternate_url_locs(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml</urlset>"""r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)spider = self.spider_class("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- ["http://www.example.com/english/"],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/english/"+ ]spider.sitemap_alternate_links = True- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- [- "http://www.example.com/english/",- "http://www.example.com/deutsch/",- "http://www.example.com/schweiz-deutsch/",- "http://www.example.com/italiano/",- ],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/english/",+ "http://www.example.com/deutsch/",+ "http://www.example.com/schweiz-deutsch/",+ "http://www.example.com/italiano/",+ ]def test_sitemap_filter(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xmlr = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)spider = self.spider_class("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- ["http://www.example.com/english/", "http://www.example.com/portuguese/"],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/english/",+ "http://www.example.com/portuguese/",+ ]spider = FilteredSitemapSpider("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- ["http://www.example.com/english/"],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/english/"+ ]def test_sitemap_filter_with_alternate_links(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xmlr = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)spider = self.spider_class("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- [- "http://www.example.com/english/article_1/",- "http://www.example.com/english/article_2/",- ],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/english/article_1/",+ "http://www.example.com/english/article_2/",+ ]spider = FilteredSitemapSpider("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- ["http://www.example.com/deutsch/article_1/"],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/deutsch/article_1/"+ ]def test_sitemapindex_filter(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xmlr = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)spider = self.spider_class("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- [- "http://www.example.com/sitemap1.xml",- "http://www.example.com/sitemap2.xml",- ],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/sitemap1.xml",+ "http://www.example.com/sitemap2.xml",+ ]spider = FilteredSitemapSpider("example.com")- self.assertEqual(- [req.url for req in spider._parse_sitemap(r)],- ["http://www.example.com/sitemap2.xml"],- )+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://www.example.com/sitemap2.xml"+ ]def test_compression_bomb_setting(self):settings = {"DOWNLOAD_MAXSIZE": 10_000_000}@@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xmlbody = body_path.read_bytes()request = Request(url="https://example.com")response = Response(url="https://example.com", body=body, request=request)- self.assertIsNone(spider._get_sitemap_body(response))+ assert spider._get_sitemap_body(response) is Nonedef test_compression_bomb_spider_attr(self):class DownloadMaxSizeSpider(self.spider_class):@@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xmlbody = body_path.read_bytes()request = Request(url="https://example.com")response = Response(url="https://example.com", body=body, request=request)- self.assertIsNone(spider._get_sitemap_body(response))+ assert spider._get_sitemap_body(response) is Nonedef test_compression_bomb_request_meta(self):crawler = get_crawler()@@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xmlurl="https://example.com", meta={"download_maxsize": 10_000_000})response = Response(url="https://example.com", body=body, request=request)- self.assertIsNone(spider._get_sitemap_body(response))+ assert spider._get_sitemap_body(response) is Nonedef test_download_warnsize_setting(self):settings = {"DOWNLOAD_WARNSIZE": 10_000_000}@@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml)-class DeprecationTest(unittest.TestCase):+class TestDeprecation:def test_crawl_spider(self):assert issubclass(CrawlSpider, Spider)assert isinstance(CrawlSpider(name="foo"), Spider)-class NoParseMethodSpiderTest(unittest.TestCase):+class TestNoParseMethodSpider:spider_class = Spiderdef test_undefined_parse_method(self):diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex b103e9ed0..476487a04 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -8,7 +8,6 @@ from tempfile import mkdtempfrom unittest import mockimport pytest-from twisted.trial import unittestfrom zope.interface.verify import verifyObject# ugly hack to avoid cyclic imports of scrapy.spiders when running this test@@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path):shutil.copytree(source, target)-class SpiderLoaderTest(unittest.TestCase):- def setUp(self):+class TestSpiderLoader:+ def setup_method(self):orig_spiders_dir = module_dir / "test_spiders"self.tmpdir = Path(tempfile.mkdtemp())self.spiders_dir = self.tmpdir / "test_spiders_xxx"@@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase):settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})self.spider_loader = SpiderLoader.from_settings(settings)- def tearDown(self):+ def teardown_method(self):del self.spider_loaderdel sys.modules["test_spiders_xxx"]sys.path.remove(str(self.tmpdir))@@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase):verifyObject(ISpiderLoader, self.spider_loader)def test_list(self):- self.assertEqual(- set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}- )+ assert set(self.spider_loader.list()) == {+ "spider1",+ "spider2",+ "spider3",+ "spider4",+ }def test_load(self):spider1 = self.spider_loader.load("spider1")- self.assertEqual(spider1.__name__, "Spider1")+ assert spider1.__name__ == "Spider1"def test_find_by_request(self):- self.assertEqual(- self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),- ["spider1"],- )- self.assertEqual(- self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),- ["spider2"],- )- self.assertEqual(- set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),- {"spider1", "spider2"},- )- self.assertEqual(- self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []- )- self.assertEqual(- self.spider_loader.find_by_request(Request("http://spider3.com")), []- )- self.assertEqual(- self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),- ["spider3"],+ assert self.spider_loader.find_by_request(+ Request("http://scrapy1.org/test")+ ) == ["spider1"]+ assert self.spider_loader.find_by_request(+ Request("http://scrapy2.org/test")+ ) == ["spider2"]+ assert set(+ self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))+ ) == {"spider1", "spider2"}+ assert (+ self.spider_loader.find_by_request(Request("http://scrapy999.org/test"))+ == [])+ assert self.spider_loader.find_by_request(Request("http://spider3.com")) == []+ assert self.spider_loader.find_by_request(+ Request("http://spider3.com/onlythis")+ ) == ["spider3"]def test_load_spider_module(self):module = "tests.test_spiderloader.test_spiders.spider1"@@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase):runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}})crawler = runner.create_crawler("spider_from_addon")- self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))- self.assertEqual(crawler.spidercls.name, "spider_from_addon")- self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1)+ assert issubclass(crawler.spidercls, scrapy.Spider)+ assert crawler.spidercls.name == "spider_from_addon"+ assert len(crawler.settings["SPIDER_MODULES"]) == 1def test_crawler_runner_loading(self):module = "tests.test_spiderloader.test_spiders.spider1"@@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase):runner.create_crawler("spider2")crawler = runner.create_crawler("spider1")- self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))- self.assertEqual(crawler.spidercls.name, "spider1")+ assert issubclass(crawler.spidercls, scrapy.Spider)+ assert crawler.spidercls.name == "spider1"def test_bad_spider_modules_exception(self):module = "tests.test_spiderloader.test_spiders.doesnotexist"@@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase):# at least until all six versions we can import (including botocore.vendored.six)# are updated to 1.16.0+w.pop(0)- self.assertIn("Could not load spiders from module", str(w[0].message))+ assert "Could not load spiders from module" in str(w[0].message)spiders = spider_loader.list()- self.assertEqual(spiders, [])+ assert not spidersdef test_syntax_error_exception(self):module = "tests.test_spiderloader.test_spiders.spider1"@@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase):# at least until all six versions we can import (including botocore.vendored.six)# are updated to 1.16.0+w.pop(0)- self.assertIn("Could not load spiders from module", str(w[0].message))+ assert "Could not load spiders from module" in str(w[0].message)spiders = spider_loader.list()- self.assertEqual(spiders, [])+ assert not spiders-class DuplicateSpiderNameLoaderTest(unittest.TestCase):- def setUp(self):+class TestDuplicateSpiderNameLoader:+ def setup_method(self):orig_spiders_dir = module_dir / "test_spiders"self.tmpdir = Path(mkdtemp())self.spiders_dir = self.tmpdir / "test_spiders_xxx"@@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):sys.path.append(str(self.tmpdir))self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})- def tearDown(self):+ def teardown_method(self):del sys.modules["test_spiders_xxx"]sys.path.remove(str(self.tmpdir))@@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):with warnings.catch_warnings(record=True) as w:spider_loader = SpiderLoader.from_settings(self.settings)- self.assertEqual(len(w), 1)+ assert len(w) == 1msg = str(w[0].message)- self.assertIn("several spiders with the same name", msg)- self.assertIn("'spider3'", msg)- self.assertTrue(msg.count("'spider3'") == 2)+ assert "several spiders with the same name" in msg+ assert "'spider3'" in msg+ assert msg.count("'spider3'") == 2- self.assertNotIn("'spider1'", msg)- self.assertNotIn("'spider2'", msg)- self.assertNotIn("'spider4'", msg)+ assert "'spider1'" not in msg+ assert "'spider2'" not in msg+ assert "'spider4'" not in msgspiders = set(spider_loader.list())- self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})+ assert spiders == {"spider1", "spider2", "spider3", "spider4"}def test_multiple_dupename_warning(self):# copy 2 spider modules so as to have duplicate spider name@@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):with warnings.catch_warnings(record=True) as w:spider_loader = SpiderLoader.from_settings(self.settings)- self.assertEqual(len(w), 1)+ assert len(w) == 1msg = str(w[0].message)- self.assertIn("several spiders with the same name", msg)- self.assertIn("'spider1'", msg)- self.assertTrue(msg.count("'spider1'") == 2)+ assert "several spiders with the same name" in msg+ assert "'spider1'" in msg+ assert msg.count("'spider1'") == 2- self.assertIn("'spider2'", msg)- self.assertTrue(msg.count("'spider2'") == 2)+ assert "'spider2'" in msg+ assert msg.count("'spider2'") == 2- self.assertNotIn("'spider3'", msg)- self.assertNotIn("'spider4'", msg)+ assert "'spider3'" not in msg+ assert "'spider4'" not in msgspiders = set(spider_loader.list())- self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})+ assert spiders == {"spider1", "spider2", "spider3", "spider4"}diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex a9f3876bb..ddc9b5206 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.test import get_crawler-class SpiderMiddlewareTestCase(TestCase):+class TestSpiderMiddleware(TestCase):def setUp(self):self.request = Request("http://example.com/index.html")self.response = Response(self.request.url, request=self.request)@@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase):return results[0]-class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):+class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):"""Invalid return value for process_spider_input method"""def test_invalid_process_spider_input(self):@@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())result = self._scrape_response()- self.assertIsInstance(result, Failure)- self.assertIsInstance(result.value, _InvalidOutput)+ assert isinstance(result, Failure)+ assert isinstance(result.value, _InvalidOutput)-class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):+class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):"""Invalid return value for process_spider_output method"""def test_invalid_process_spider_output(self):@@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())result = self._scrape_response()- self.assertIsInstance(result, Failure)- self.assertIsInstance(result.value, _InvalidOutput)+ assert isinstance(result, Failure)+ assert isinstance(result.value, _InvalidOutput)-class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):+class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):"""Invalid return value for process_spider_exception method"""def test_invalid_process_spider_exception(self):@@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())result = self._scrape_response()- self.assertIsInstance(result, Failure)- self.assertIsInstance(result.value, _InvalidOutput)+ assert isinstance(result, Failure)+ assert isinstance(result.value, _InvalidOutput)-class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):+class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):"""Re raise the exception by returning None"""def test_process_spider_exception_return_none(self):@@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())result = self._scrape_response()- self.assertIsInstance(result, Failure)- self.assertIsInstance(result.value, ZeroDivisionError)+ assert isinstance(result, Failure)+ assert isinstance(result.value, ZeroDivisionError)-class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):+class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):"""Helpers for testing sync, async and mixed middlewares.Should work for process_spider_output and, when it's supported, process_start_requests.@@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):result = yield self._get_middleware_result(*mw_classes, start_index=start_index)- self.assertIsInstance(result, Iterable)+ assert isinstance(result, Iterable)result_list = list(result)- self.assertEqual(len(result_list), self.RESULT_COUNT)- self.assertIsInstance(result_list[0], self.ITEM_TYPE)- self.assertEqual("downgraded to a non-async" in str(log), downgrade)- self.assertEqual(- "doesn't support asynchronous spider output" in str(log),- ProcessSpiderOutputSimpleMiddleware in mw_classes,+ assert len(result_list) == self.RESULT_COUNT+ assert isinstance(result_list[0], self.ITEM_TYPE)+ assert ("downgraded to a non-async" in str(log)) == downgrade+ assert ("doesn't support asynchronous spider output" in str(log)) == (+ ProcessSpiderOutputSimpleMiddleware in mw_classes)@defer.inlineCallbacks@@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):result = yield self._get_middleware_result(*mw_classes, start_index=start_index)- self.assertIsInstance(result, AsyncIterator)+ assert isinstance(result, AsyncIterator)result_list = yield deferred_from_coro(collect_asyncgen(result))- self.assertEqual(len(result_list), self.RESULT_COUNT)- self.assertIsInstance(result_list[0], self.ITEM_TYPE)- self.assertEqual("downgraded to a non-async" in str(log), downgrade)+ assert len(result_list) == self.RESULT_COUNT+ assert isinstance(result_list[0], self.ITEM_TYPE)+ assert ("downgraded to a non-async" in str(log)) == downgradeclass ProcessSpiderOutputSimpleMiddleware:@@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware:yield {"foo": 3}-class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):+class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):"""process_spider_output tests for simple callbacks"""ITEM_TYPE = dict@@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)-class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):+class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):"""process_spider_output tests for async generator callbacks"""async def _scrape_func(self, *args, **kwargs):@@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware:return result-class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):+class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):@defer.inlineCallbacksdef test_non_iterable(self):with pytest.raises(@@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware:yield from start_requests-class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):+class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware):"""process_start_requests tests for simple start_requests"""ITEM_TYPE = (Request, dict)@@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync:yield-class UniversalMiddlewareManagerTest(TestCase):- def setUp(self):+class TestUniversalMiddlewareManager:+ def setup_method(self):self.mwman = SpiderMiddlewareManager()def test_simple_mw(self):mw = ProcessSpiderOutputSimpleMiddleware()self.mwman._add_middleware(mw)- self.assertEqual(- self.mwman.methods["process_spider_output"][0], mw.process_spider_output+ assert (+ self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable)def test_async_mw(self):mw = ProcessSpiderOutputAsyncGenMiddleware()self.mwman._add_middleware(mw)- self.assertEqual(- self.mwman.methods["process_spider_output"][0], mw.process_spider_output+ assert (+ self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable)def test_universal_mw(self):mw = ProcessSpiderOutputUniversalMiddleware()self.mwman._add_middleware(mw)- self.assertEqual(- self.mwman.methods["process_spider_output"][0],- (mw.process_spider_output, mw.process_spider_output_async),+ assert self.mwman.methods["process_spider_output"][0] == (+ mw.process_spider_output,+ mw.process_spider_output_async,)def test_universal_mw_no_sync(self):with LogCapture() as log:self.mwman._add_middleware(UniversalMiddlewareNoSync())- self.assertIn(+ assert ("UniversalMiddlewareNoSync has process_spider_output_async"- " without process_spider_output",- str(log),+ " without process_spider_output" in str(log))- self.assertEqual(self.mwman.methods["process_spider_output"][0], None)+ assert self.mwman.methods["process_spider_output"][0] is Nonedef test_universal_mw_both_sync(self):mw = UniversalMiddlewareBothSync()with LogCapture() as log:self.mwman._add_middleware(mw)- self.assertIn(+ assert ("UniversalMiddlewareBothSync.process_spider_output_async "- "is not an async generator function",- str(log),+ "is not an async generator function" in str(log))- self.assertEqual(- self.mwman.methods["process_spider_output"][0], mw.process_spider_output+ assert (+ self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable)def test_universal_mw_both_async(self):with LogCapture() as log:self.mwman._add_middleware(UniversalMiddlewareBothAsync())- self.assertIn(+ assert ("UniversalMiddlewareBothAsync.process_spider_output "- "is an async generator function while process_spider_output_async exists",- str(log),+ "is an async generator function while process_spider_output_async exists"+ in str(log))- self.assertEqual(self.mwman.methods["process_spider_output"][0], None)+ assert self.mwman.methods["process_spider_output"][0] is None-class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):+class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):ITEM_TYPE = dictMW_SIMPLE = ProcessSpiderOutputSimpleMiddlewareMW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware@@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):return self._test_simple_base(self.MW_UNIVERSAL)-class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):+class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):async def _scrape_func(self, *args, **kwargs):for item in super()._scrape_func():yield item@@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):return self._test_asyncgen_base(self.MW_UNIVERSAL)-class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):+class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):ITEM_TYPE = dictMW_SIMPLE = ProcessSpiderOutputSimpleMiddlewareMW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddlewarediff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex e359d9cfc..dfcc141c3 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -1,5 +1,3 @@-from unittest import TestCase-from scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.depth import DepthMiddlewarefrom scrapy.spiders import Spider@@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollectorfrom scrapy.utils.test import get_crawler-class TestDepthMiddleware(TestCase):- def setUp(self):+class TestDepthMiddleware:+ def setup_method(self):crawler = get_crawler(Spider)self.spider = crawler._create_spider("scrapytest.org")@@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase):result = [Request("http://scrapytest.org")]out = list(self.mw.process_spider_output(resp, result, self.spider))- self.assertEqual(out, result)+ assert out == resultrdc = self.stats.get_value("request_depth_count/1", spider=self.spider)- self.assertEqual(rdc, 1)+ assert rdc == 1req.meta["depth"] = 1out2 = list(self.mw.process_spider_output(resp, result, self.spider))- self.assertEqual(out2, [])+ assert not out2rdm = self.stats.get_value("request_depth_max", spider=self.spider)- self.assertEqual(rdm, 1)+ assert rdm == 1- def tearDown(self):+ def teardown_method(self):self.stats.close_spider(self.spider, "")diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex f9eb93d6b..e306579fa 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -1,10 +1,9 @@import logging-from unittest import TestCaseimport pytestfrom testfixtures import LogCapturefrom twisted.internet import defer-from twisted.trial.unittest import TestCase as TrialTestCase+from twisted.trial.unittest import TestCasefrom scrapy.http import Request, Responsefrom scrapy.settings import Settings@@ -59,8 +58,8 @@ def _responses(request, status_codes):return responses-class TestHttpErrorMiddleware(TestCase):- def setUp(self):+class TestHttpErrorMiddleware:+ def setup_method(self):crawler = get_crawler(Spider)self.spider = Spider.from_crawler(crawler, name="foo")self.mw = HttpErrorMiddleware(Settings({}))@@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase):self.res200, self.res404 = _responses(self.req, [200, 404])def test_process_spider_input(self):- self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))+ assert self.mw.process_spider_input(self.res200, self.spider) is Nonewith pytest.raises(HttpError):self.mw.process_spider_input(self.res404, self.spider)def test_process_spider_exception(self):- self.assertEqual(- [],+ assert (self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider- ),+ )+ == [])- self.assertIsNone(+ assert (self.mw.process_spider_exception(self.res404, Exception(), self.spider)+ is None)def test_handle_httpstatus_list(self):@@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase):res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})- self.assertIsNone(self.mw.process_spider_input(res, self.spider))+ assert self.mw.process_spider_input(res, self.spider) is Noneself.spider.handle_httpstatus_list = [404]- self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))+ assert self.mw.process_spider_input(self.res404, self.spider) is None-class TestHttpErrorMiddlewareSettings(TestCase):+class TestHttpErrorMiddlewareSettings:"""Similar test, but with settings"""- def setUp(self):+ def setup_method(self):self.spider = Spider("foo")self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))self.req = Request("http://scrapytest.org")self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])def test_process_spider_input(self):- self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))+ assert self.mw.process_spider_input(self.res200, self.spider) is Nonewith pytest.raises(HttpError):self.mw.process_spider_input(self.res404, self.spider)- self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))+ assert self.mw.process_spider_input(self.res402, self.spider) is Nonedef test_meta_overrides_settings(self):request = Request(@@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase):res402 = self.res402.copy()res402.request = request- self.assertIsNone(self.mw.process_spider_input(res404, self.spider))+ assert self.mw.process_spider_input(res404, self.spider) is Nonewith pytest.raises(HttpError):self.mw.process_spider_input(res402, self.spider)def test_spider_override_settings(self):self.spider.handle_httpstatus_list = [404]- self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))+ assert self.mw.process_spider_input(self.res404, self.spider) is Nonewith pytest.raises(HttpError):self.mw.process_spider_input(self.res402, self.spider)-class TestHttpErrorMiddlewareHandleAll(TestCase):- def setUp(self):+class TestHttpErrorMiddlewareHandleAll:+ def setup_method(self):self.spider = Spider("foo")self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))self.req = Request("http://scrapytest.org")self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])def test_process_spider_input(self):- self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))- self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))+ assert self.mw.process_spider_input(self.res200, self.spider) is None+ assert self.mw.process_spider_input(self.res404, self.spider) is Nonedef test_meta_overrides_settings(self):request = Request(@@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):res402 = self.res402.copy()res402.request = request- self.assertIsNone(self.mw.process_spider_input(res404, self.spider))+ assert self.mw.process_spider_input(res404, self.spider) is Nonewith pytest.raises(HttpError):self.mw.process_spider_input(res402, self.spider)@@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):with pytest.raises(HttpError):mw.process_spider_input(res404, self.spider)- self.assertIsNone(mw.process_spider_input(res402, self.spider))+ assert mw.process_spider_input(res402, self.spider) is None-class TestHttpErrorMiddlewareIntegrational(TrialTestCase):+class TestHttpErrorMiddlewareIntegrational(TestCase):@classmethoddef setUpClass(cls):cls.mockserver = MockServer()@@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):crawler = get_crawler(_HttpErrorSpider)yield crawler.crawl(mockserver=self.mockserver)assert not crawler.spider.skipped, crawler.spider.skipped- self.assertEqual(crawler.spider.parsed, {"200"})- self.assertEqual(crawler.spider.failed, {"404", "402", "500"})+ assert crawler.spider.parsed == {"200"}+ assert crawler.spider.failed == {"404", "402", "500"}get_value = crawler.stats.get_value- self.assertEqual(get_value("httperror/response_ignored_count"), 3)- self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1)- self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1)- self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1)+ assert get_value("httperror/response_ignored_count") == 3+ assert get_value("httperror/response_ignored_status_count/404") == 1+ assert get_value("httperror/response_ignored_status_count/402") == 1+ assert get_value("httperror/response_ignored_status_count/500") == 1@defer.inlineCallbacksdef test_logging(self):crawler = get_crawler(_HttpErrorSpider)with LogCapture() as log:yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})- self.assertEqual(crawler.spider.parsed, {"200", "402"})- self.assertEqual(crawler.spider.skipped, {"402"})- self.assertEqual(crawler.spider.failed, {"404", "500"})+ assert crawler.spider.parsed == {"200", "402"}+ assert crawler.spider.skipped == {"402"}+ assert crawler.spider.failed == {"404", "500"}- self.assertIn("Ignoring response <404", str(log))- self.assertIn("Ignoring response <500", str(log))- self.assertNotIn("Ignoring response <200", str(log))- self.assertNotIn("Ignoring response <402", str(log))+ assert "Ignoring response <404" in str(log)+ assert "Ignoring response <500" in str(log)+ assert "Ignoring response <200" not in str(log)+ assert "Ignoring response <402" not in str(log)@defer.inlineCallbacksdef test_logging_level(self):@@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):crawler = get_crawler(_HttpErrorSpider)with LogCapture(level=logging.INFO) as log:yield crawler.crawl(mockserver=self.mockserver)- self.assertEqual(crawler.spider.parsed, {"200"})- self.assertEqual(crawler.spider.failed, {"404", "402", "500"})+ assert crawler.spider.parsed == {"200"}+ assert crawler.spider.failed == {"404", "402", "500"}- self.assertIn("Ignoring response <402", str(log))- self.assertIn("Ignoring response <404", str(log))- self.assertIn("Ignoring response <500", str(log))- self.assertNotIn("Ignoring response <200", str(log))+ assert "Ignoring response <402" in str(log)+ assert "Ignoring response <404" in str(log)+ assert "Ignoring response <500" in str(log)+ assert "Ignoring response <200" not in str(log)# with level WARNING, we shouldn't capture anything from HttpErrorcrawler = get_crawler(_HttpErrorSpider)with LogCapture(level=logging.WARNING) as log:yield crawler.crawl(mockserver=self.mockserver)- self.assertEqual(crawler.spider.parsed, {"200"})- self.assertEqual(crawler.spider.failed, {"404", "402", "500"})+ assert crawler.spider.parsed == {"200"}+ assert crawler.spider.failed == {"404", "402", "500"}- self.assertNotIn("Ignoring response <402", str(log))- self.assertNotIn("Ignoring response <404", str(log))- self.assertNotIn("Ignoring response <500", str(log))- self.assertNotIn("Ignoring response <200", str(log))+ assert "Ignoring response <402" not in str(log)+ assert "Ignoring response <404" not in str(log)+ assert "Ignoring response <500" not in str(log)+ assert "Ignoring response <200" not in str(log)diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex 906928e01..f4563a0a4 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -1,5 +1,4 @@import warnings-from unittest import TestCasefrom urllib.parse import urlparsefrom scrapy.http import Request, Response@@ -8,8 +7,8 @@ from scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-class TestOffsiteMiddleware(TestCase):- def setUp(self):+class TestOffsiteMiddleware:+ def setup_method(self):crawler = get_crawler(Spider)self.spider = crawler._create_spider(**self._get_spiderargs())self.mw = OffsiteMiddleware.from_crawler(crawler)@@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase):reqs = onsite_reqs + offsite_reqsout = list(self.mw.process_spider_output(res, reqs, self.spider))- self.assertEqual(out, onsite_reqs)+ assert out == onsite_reqsclass TestOffsiteMiddleware2(TestOffsiteMiddleware):@@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):res = Response("http://scrapytest.org")reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]out = list(self.mw.process_spider_output(res, reqs, self.spider))- self.assertEqual(out, reqs)+ assert out == reqsclass TestOffsiteMiddleware3(TestOffsiteMiddleware2):@@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):res = Response("http://scrapytest.org")reqs = [Request("http://scrapytest.org/1")]out = list(self.mw.process_spider_output(res, reqs, self.spider))- self.assertEqual(out, reqs)+ assert out == reqsclass TestOffsiteMiddleware5(TestOffsiteMiddleware4):diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex e51957497..6e26a85ea 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase):was enqueued from the recovery middleware)"""log = yield self.crawl_log(RecoverySpider)- self.assertIn("Middleware: TabError exception caught", str(log))- self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)- self.assertIn("'item_scraped_count': 3", str(log))+ assert "Middleware: TabError exception caught" in str(log)+ assert str(log).count("Middleware: TabError exception caught") == 1+ assert "'item_scraped_count': 3" in str(log)@defer.inlineCallbacksdef test_recovery_asyncgen(self):@@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase):Same as test_recovery but with an async callback."""log = yield self.crawl_log(RecoveryAsyncGenSpider)- self.assertIn("Middleware: TabError exception caught", str(log))- self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)- self.assertIn("'item_scraped_count': 3", str(log))+ assert "Middleware: TabError exception caught" in str(log)+ assert str(log).count("Middleware: TabError exception caught") == 1+ assert "'item_scraped_count': 3" in str(log)@defer.inlineCallbacksdef test_process_spider_input_without_errback(self):@@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase):process_spider_exception chain from the start if the Request has no errback"""log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)- self.assertIn("Middleware: will raise IndexError", str(log1))- self.assertIn("Middleware: IndexError exception caught", str(log1))+ assert "Middleware: will raise IndexError" in str(log1)+ assert "Middleware: IndexError exception caught" in str(log1)@defer.inlineCallbacksdef test_process_spider_input_with_errback(self):@@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase):process_spider_exception chain if the Request has an errback"""log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)- self.assertNotIn("Middleware: IndexError exception caught", str(log1))- self.assertIn("Middleware: will raise IndexError", str(log1))- self.assertIn("Got a Failure on the Request errback", str(log1))- self.assertIn("{'from': 'errback'}", str(log1))- self.assertNotIn("{'from': 'callback'}", str(log1))- self.assertIn("'item_scraped_count': 1", str(log1))+ assert "Middleware: IndexError exception caught" not in str(log1)+ assert "Middleware: will raise IndexError" in str(log1)+ assert "Got a Failure on the Request errback" in str(log1)+ assert "{'from': 'errback'}" in str(log1)+ assert "{'from': 'callback'}" not in str(log1)+ assert "'item_scraped_count': 1" in str(log1)@defer.inlineCallbacksdef test_generator_callback(self):@@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase):exception is raised should be processed normally."""log2 = yield self.crawl_log(GeneratorCallbackSpider)- self.assertIn("Middleware: ImportError exception caught", str(log2))- self.assertIn("'item_scraped_count': 2", str(log2))+ assert "Middleware: ImportError exception caught" in str(log2)+ assert "'item_scraped_count': 2" in str(log2)@defer.inlineCallbacksdef test_async_generator_callback(self):@@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase):Same as test_generator_callback but with an async callback."""log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)- self.assertIn("Middleware: ImportError exception caught", str(log2))- self.assertIn("'item_scraped_count': 2", str(log2))+ assert "Middleware: ImportError exception caught" in str(log2)+ assert "'item_scraped_count': 2" in str(log2)@defer.inlineCallbacksdef test_generator_callback_right_after_callback(self):@@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase):even if the middleware is placed right after the spider"""log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)- self.assertIn("Middleware: ImportError exception caught", str(log21))- self.assertIn("'item_scraped_count': 2", str(log21))+ assert "Middleware: ImportError exception caught" in str(log21)+ assert "'item_scraped_count': 2" in str(log21)@defer.inlineCallbacksdef test_not_a_generator_callback(self):@@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase):be caught by the process_spider_exception chain. No items should be processed."""log3 = yield self.crawl_log(NotGeneratorCallbackSpider)- self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3))- self.assertNotIn("item_scraped_count", str(log3))+ assert "Middleware: ZeroDivisionError exception caught" in str(log3)+ assert "item_scraped_count" not in str(log3)@defer.inlineCallbacksdef test_not_a_generator_callback_right_after_callback(self):@@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase):log31 = yield self.crawl_log(NotGeneratorCallbackSpiderMiddlewareRightAfterSpider)- self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))- self.assertNotIn("item_scraped_count", str(log31))+ assert "Middleware: ZeroDivisionError exception caught" in str(log31)+ assert "item_scraped_count" not in str(log31)@defer.inlineCallbacksdef test_generator_output_chain(self):@@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase):process_spider_exception chain)"""log4 = yield self.crawl_log(GeneratorOutputChainSpider)- self.assertIn("'item_scraped_count': 2", str(log4))- self.assertIn(- "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught",- str(log4),+ assert "'item_scraped_count': 2" in str(log4)+ assert (+ "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"+ in str(log4))- self.assertIn(- "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",- str(log4),+ assert (+ "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"+ in str(log4))- self.assertNotIn(- "GeneratorFailMiddleware.process_spider_exception: LookupError caught",- str(log4),+ assert (+ "GeneratorFailMiddleware.process_spider_exception: LookupError caught"+ not in str(log4))- self.assertNotIn(- "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",- str(log4),+ assert (+ "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"+ not in str(log4))item_from_callback = {"processed": [@@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase):"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",]}- self.assertIn(str(item_from_callback), str(log4))- self.assertIn(str(item_recovered), str(log4))- self.assertNotIn("parse-second-item", str(log4))+ assert str(item_from_callback) in str(log4)+ assert str(item_recovered) in str(log4)+ assert "parse-second-item" not in str(log4)@defer.inlineCallbacksdef test_not_a_generator_output_chain(self):@@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase):from the spider callback are lost)"""log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)- self.assertIn("'item_scraped_count': 1", str(log5))- self.assertIn(- "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught",- str(log5),+ assert "'item_scraped_count': 1" in str(log5)+ assert (+ "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught"+ in str(log5))- self.assertIn(- "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",- str(log5),+ assert (+ "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught"+ in str(log5))- self.assertNotIn(- "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught",- str(log5),+ assert (+ "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught"+ not in str(log5))- self.assertNotIn(- "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",- str(log5),+ assert (+ "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught"+ not in str(log5))item_recovered = {"processed": [@@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase):"NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",]}- self.assertIn(str(item_recovered), str(log5))- self.assertNotIn("parse-first-item", str(log5))- self.assertNotIn("parse-second-item", str(log5))+ assert str(item_recovered) in str(log5)+ assert "parse-first-item" not in str(log5)+ assert "parse-second-item" not in str(log5)diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 01a87c645..300a40c13 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport warningsfrom typing import Any-from unittest import TestCasefrom urllib.parse import urlparseimport pytest@@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import (from scrapy.spiders import Spider-class TestRefererMiddleware(TestCase):+class TestRefererMiddleware:req_meta: dict[str, Any] = {}resp_headers: dict[str, str] = {}settings: dict[str, Any] = {}@@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase):("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),]- def setUp(self):+ def setup_method(self):self.spider = Spider("foo")settings = Settings(self.settings)self.mw = RefererMiddleware(settings)@@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase):response = self.get_response(origin)request = self.get_request(target)out = list(self.mw.process_spider_output(response, [request], self.spider))- self.assertEqual(out[0].headers.get("Referer"), referrer)+ assert out[0].headers.get("Referer") == referrerclass MixinDefault:@@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):req_meta = {"referrer_policy": POLICY_UNSAFE_URL}-class TestRequestMetaSettingFallback(TestCase):+class TestRequestMetaSettingFallback:params = [(# When an unknown policy is referenced in Request.meta@@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase):with warnings.catch_warnings(record=True) as w:policy = mw.policy(response, request)- self.assertIsInstance(policy, policy_class)+ assert isinstance(policy, policy_class)if check_warning:- self.assertEqual(len(w), 1)- self.assertEqual(w[0].category, RuntimeWarning, w[0].message)+ assert len(w) == 1+ assert w[0].category is RuntimeWarning, w[0].message-class TestSettingsPolicyByName(TestCase):+class TestSettingsPolicyByName:def test_valid_name(self):for s, p in [(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),@@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase):]:settings = Settings({"REFERRER_POLICY": s})mw = RefererMiddleware(settings)- self.assertEqual(mw.default_policy, p)+ assert mw.default_policy == pdef test_valid_name_casevariants(self):for s, p in [@@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase):]:settings = Settings({"REFERRER_POLICY": s.upper()})mw = RefererMiddleware(settings)- self.assertEqual(mw.default_policy, p)+ assert mw.default_policy == pdef test_invalid_name(self):settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})@@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase):})mw1 = RefererMiddleware(settings1)- self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)+ assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy# test parsing with space(s) after the commasettings2 = Settings(@@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase):})mw2 = RefererMiddleware(settings2)- self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)+ assert mw2.default_policy == UnsafeUrlPolicydef test_multiple_policy_tokens_all_invalid(self):settings = Settings(@@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):),]- def setUp(self):+ def setup_method(self):self.spider = Spider("foo")settings = Settings(self.settings)self.referrermw = RefererMiddleware(settings)@@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):out = list(self.referrermw.process_spider_output(response, [request], self.spider))- self.assertEqual(out[0].headers.get("Referer"), init_referrer)+ assert out[0].headers.get("Referer") == init_referrerfor status, url in redirections:response = Response(@@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):self.referrermw.request_scheduled(request, self.spider)assert isinstance(request, Request)- self.assertEqual(request.headers.get("Referer"), final_referrer)+ assert request.headers.get("Referer") == final_referrerclass TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex 1a0f2e223..5cc3cdc6c 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -1,5 +1,3 @@-from unittest import TestCase-from testfixtures import LogCapturefrom scrapy.http import Request, Response@@ -8,8 +6,8 @@ from scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-class TestUrlLengthMiddleware(TestCase):- def setUp(self):+class TestUrlLengthMiddleware:+ def setup_method(self):self.maxlength = 25crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength})self.spider = crawler._create_spider("foo")@@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase):)def test_middleware_works(self):- self.assertEqual(self.process_spider_output(), [self.short_url_req])+ assert self.process_spider_output() == [self.short_url_req]def test_logging(self):with LogCapture() as log:@@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase):ric = self.stats.get_value("urllength/request_ignored_count", spider=self.spider)- self.assertEqual(ric, 1)+ assert ric == 1- self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log))+ assert f"Ignoring link (url length > {self.maxlength})" in str(log)diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.pyindex 72692afab..cd31891a0 100644--- a/tests/test_spiderstate.py+++ b/tests/test_spiderstate.py@@ -3,7 +3,6 @@ from datetime import datetime, timezonefrom tempfile import mkdtempimport pytest-from twisted.trial import unittestfrom scrapy.exceptions import NotConfiguredfrom scrapy.extensions.spiderstate import SpiderState@@ -11,7 +10,7 @@ from scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-class SpiderStateTest(unittest.TestCase):+class TestSpiderState:def test_store_load(self):jobdir = mkdtemp()try:@@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase):spider2 = Spider(name="default")ss2 = SpiderState(jobdir)ss2.spider_opened(spider2)- self.assertEqual(spider.state, {"one": 1, "dt": dt})+ assert spider.state == {"one": 1, "dt": dt}ss2.spider_closed(spider2)finally:shutil.rmtree(jobdir)@@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase):spider = Spider(name="default")ss = SpiderState()ss.spider_opened(spider)- self.assertEqual(spider.state, {})+ assert spider.state == {}ss.spider_closed(spider)def test_not_configured(self):diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 8556b75dd..6283b9ad6 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin:q.push("a")q.push(123)q.push({"a": "dict"})- self.assertEqual(q.pop(), "a")- self.assertEqual(q.pop(), 123)- self.assertEqual(q.pop(), {"a": "dict"})+ assert q.pop() == "a"+ assert q.pop() == 123+ assert q.pop() == {"a": "dict"}test_nonserializable_object = nonserializable_object_test@@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):q.push(i)i2 = q.pop()assert isinstance(i2, MyItem)- self.assertEqual(i, i2)+ assert i == i2def test_serialize_loader(self):q = self.queue()@@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):loader2 = q.pop()assert isinstance(loader2, MyLoader)assert loader2.default_item_class is MyItem- self.assertEqual(loader2.name_out("x"), "xx")+ assert loader2.name_out("x") == "xx"def test_serialize_request_recursive(self):q = self.queue()@@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):q.push(r)r2 = q.pop()assert isinstance(r2, Request)- self.assertEqual(r.url, r2.url)+ assert r.url == r2.urlassert r2.meta["request"] is r2def test_non_pickable_object(self):q = self.queue()- try:+ with pytest.raises(+ ValueError,+ match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",+ ) as exc_info:q.push(lambda x: x)- except ValueError as exc:- if hasattr(sys, "pypy_version_info"):- self.assertIsInstance(exc.__context__, pickle.PicklingError)- else:- self.assertIsInstance(exc.__context__, AttributeError)+ if hasattr(sys, "pypy_version_info"):+ assert isinstance(exc_info.value.__context__, pickle.PicklingError)+ else:+ assert isinstance(exc_info.value.__context__, AttributeError)sel = Selector(text="<html><body><p>some text</p></body></html>")- try:+ with pytest.raises(+ ValueError, match="can't pickle Selector objects"+ ) as exc_info:q.push(sel)- except ValueError as exc:- self.assertIsInstance(exc.__context__, TypeError)+ assert isinstance(exc_info.value.__context__, TypeError)class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):@@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin:q.push("a")q.push(123)q.push({"a": "dict"})- self.assertEqual(q.pop(), {"a": "dict"})- self.assertEqual(q.pop(), 123)- self.assertEqual(q.pop(), "a")+ assert q.pop() == {"a": "dict"}+ assert q.pop() == 123+ assert q.pop() == "a"test_nonserializable_object = nonserializable_object_test@@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):q.push(i)i2 = q.pop()assert isinstance(i2, MyItem)- self.assertEqual(i, i2)+ assert i == i2def test_serialize_loader(self):q = self.queue()@@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):loader2 = q.pop()assert isinstance(loader2, MyLoader)assert loader2.default_item_class is MyItem- self.assertEqual(loader2.name_out("x"), "xx")+ assert loader2.name_out("x") == "xx"def test_serialize_request_recursive(self):q = self.queue()@@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):q.push(r)r2 = q.pop()assert isinstance(r2, Request)- self.assertEqual(r.url, r2.url)+ assert r.url == r2.urlassert r2.meta["request"] is r2diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex 88f6657d8..6c153f40e 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -22,14 +22,14 @@ from scrapy.squeues import (from scrapy.utils.test import get_crawler-class BaseQueueTestCase(unittest.TestCase):- def setUp(self):+class TestBaseQueue:+ def setup_method(self):self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-")self.qpath = self.tempfilename()self.qdir = tempfile.mkdtemp()self.crawler = get_crawler(Spider)- def tearDown(self):+ def teardown_method(self):shutil.rmtree(self.tmpdir)def tempfilename(self):@@ -48,36 +48,36 @@ class RequestQueueTestMixin:if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("The queuelib queues do not define peek")q = self.queue()- self.assertEqual(len(q), 0)- self.assertIsNone(q.peek())- self.assertIsNone(q.pop())+ assert len(q) == 0+ assert q.peek() is None+ assert q.pop() is Nonereq = Request("http://www.example.com")q.push(req)- self.assertEqual(len(q), 1)- self.assertEqual(q.peek().url, req.url)- self.assertEqual(q.pop().url, req.url)- self.assertEqual(len(q), 0)- self.assertIsNone(q.peek())- self.assertIsNone(q.pop())+ assert len(q) == 1+ assert q.peek().url == req.url+ assert q.pop().url == req.url+ assert len(q) == 0+ assert q.peek() is None+ assert q.pop() is Noneq.close()def test_one_element_without_peek(self):if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("The queuelib queues define peek")q = self.queue()- self.assertEqual(len(q), 0)- self.assertIsNone(q.pop())+ assert len(q) == 0+ assert q.pop() is Nonereq = Request("http://www.example.com")q.push(req)- self.assertEqual(len(q), 1)+ assert len(q) == 1with pytest.raises(NotImplementedError,match="The underlying queue class does not implement 'peek'",):q.peek()- self.assertEqual(q.pop().url, req.url)- self.assertEqual(len(q), 0)- self.assertIsNone(q.pop())+ assert q.pop().url == req.url+ assert len(q) == 0+ assert q.pop() is Noneq.close()@@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin):if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("The queuelib queues do not define peek")q = self.queue()- self.assertEqual(len(q), 0)- self.assertIsNone(q.peek())- self.assertIsNone(q.pop())+ assert len(q) == 0+ assert q.peek() is None+ assert q.pop() is Nonereq1 = Request("http://www.example.com/1")req2 = Request("http://www.example.com/2")req3 = Request("http://www.example.com/3")q.push(req1)q.push(req2)q.push(req3)- self.assertEqual(len(q), 3)- self.assertEqual(q.peek().url, req1.url)- self.assertEqual(q.pop().url, req1.url)- self.assertEqual(len(q), 2)- self.assertEqual(q.peek().url, req2.url)- self.assertEqual(q.pop().url, req2.url)- self.assertEqual(len(q), 1)- self.assertEqual(q.peek().url, req3.url)- self.assertEqual(q.pop().url, req3.url)- self.assertEqual(len(q), 0)- self.assertIsNone(q.peek())- self.assertIsNone(q.pop())+ assert len(q) == 3+ assert q.peek().url == req1.url+ assert q.pop().url == req1.url+ assert len(q) == 2+ assert q.peek().url == req2.url+ assert q.pop().url == req2.url+ assert len(q) == 1+ assert q.peek().url == req3.url+ assert q.pop().url == req3.url+ assert len(q) == 0+ assert q.peek() is None+ assert q.pop() is Noneq.close()def test_fifo_without_peek(self):if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("The queuelib queues do not define peek")q = self.queue()- self.assertEqual(len(q), 0)- self.assertIsNone(q.pop())+ assert len(q) == 0+ assert q.pop() is Nonereq1 = Request("http://www.example.com/1")req2 = Request("http://www.example.com/2")req3 = Request("http://www.example.com/3")@@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin):match="The underlying queue class does not implement 'peek'",):q.peek()- self.assertEqual(len(q), 3)- self.assertEqual(q.pop().url, req1.url)- self.assertEqual(len(q), 2)- self.assertEqual(q.pop().url, req2.url)- self.assertEqual(len(q), 1)- self.assertEqual(q.pop().url, req3.url)- self.assertEqual(len(q), 0)- self.assertIsNone(q.pop())+ assert len(q) == 3+ assert q.pop().url == req1.url+ assert len(q) == 2+ assert q.pop().url == req2.url+ assert len(q) == 1+ assert q.pop().url == req3.url+ assert len(q) == 0+ assert q.pop() is Noneq.close()@@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin):if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("The queuelib queues do not define peek")q = self.queue()- self.assertEqual(len(q), 0)- self.assertIsNone(q.peek())- self.assertIsNone(q.pop())+ assert len(q) == 0+ assert q.peek() is None+ assert q.pop() is Nonereq1 = Request("http://www.example.com/1")req2 = Request("http://www.example.com/2")req3 = Request("http://www.example.com/3")q.push(req1)q.push(req2)q.push(req3)- self.assertEqual(len(q), 3)- self.assertEqual(q.peek().url, req3.url)- self.assertEqual(q.pop().url, req3.url)- self.assertEqual(len(q), 2)- self.assertEqual(q.peek().url, req2.url)- self.assertEqual(q.pop().url, req2.url)- self.assertEqual(len(q), 1)- self.assertEqual(q.peek().url, req1.url)- self.assertEqual(q.pop().url, req1.url)- self.assertEqual(len(q), 0)- self.assertIsNone(q.peek())- self.assertIsNone(q.pop())+ assert len(q) == 3+ assert q.peek().url == req3.url+ assert q.pop().url == req3.url+ assert len(q) == 2+ assert q.peek().url == req2.url+ assert q.pop().url == req2.url+ assert len(q) == 1+ assert q.peek().url == req1.url+ assert q.pop().url == req1.url+ assert len(q) == 0+ assert q.peek() is None+ assert q.pop() is Noneq.close()def test_lifo_without_peek(self):if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):raise unittest.SkipTest("The queuelib queues do not define peek")q = self.queue()- self.assertEqual(len(q), 0)- self.assertIsNone(q.pop())+ assert len(q) == 0+ assert q.pop() is Nonereq1 = Request("http://www.example.com/1")req2 = Request("http://www.example.com/2")req3 = Request("http://www.example.com/3")@@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin):match="The underlying queue class does not implement 'peek'",):q.peek()- self.assertEqual(len(q), 3)- self.assertEqual(q.pop().url, req3.url)- self.assertEqual(len(q), 2)- self.assertEqual(q.pop().url, req2.url)- self.assertEqual(len(q), 1)- self.assertEqual(q.pop().url, req1.url)- self.assertEqual(len(q), 0)- self.assertIsNone(q.pop())+ assert len(q) == 3+ assert q.pop().url == req3.url+ assert len(q) == 2+ assert q.pop().url == req2.url+ assert len(q) == 1+ assert q.pop().url == req1.url+ assert len(q) == 0+ assert q.pop() is Noneq.close()-class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):+class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):def queue(self):return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo")-class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):+class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):def queue(self):return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo")-class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):+class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue):def queue(self):return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo")-class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):+class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue):def queue(self):return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo")-class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):+class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue):def queue(self):return FifoMemoryQueue.from_crawler(crawler=self.crawler)-class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):+class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue):def queue(self):return LifoMemoryQueue.from_crawler(crawler=self.crawler)diff --git a/tests/test_stats.py b/tests/test_stats.pyindex 3d4c7e88e..537614364 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -1,4 +1,3 @@-import unittestfrom datetime import datetimefrom unittest import mock@@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollectorfrom scrapy.utils.test import get_crawler-class CoreStatsExtensionTest(unittest.TestCase):- def setUp(self):+class TestCoreStatsExtension:+ def setup_method(self):self.crawler = get_crawler(Spider)self.spider = self.crawler._create_spider("foo")@@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase):ext.response_received(self.spider)ext.item_dropped({}, self.spider, ZeroDivisionError())ext.spider_closed(self.spider, "finished")- self.assertEqual(- ext.stats._stats,- {- "start_time": fixed_datetime,- "finish_time": fixed_datetime,- "item_scraped_count": 1,- "response_received_count": 1,- "item_dropped_count": 1,- "item_dropped_reasons_count/ZeroDivisionError": 1,- "finish_reason": "finished",- "elapsed_time_seconds": 0.0,- },- )+ assert ext.stats._stats == {+ "start_time": fixed_datetime,+ "finish_time": fixed_datetime,+ "item_scraped_count": 1,+ "response_received_count": 1,+ "item_dropped_count": 1,+ "item_dropped_reasons_count/ZeroDivisionError": 1,+ "finish_reason": "finished",+ "elapsed_time_seconds": 0.0,+ }def test_core_stats_dummy_stats_collector(self):self.crawler.stats = DummyStatsCollector(self.crawler)@@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase):ext.response_received(self.spider)ext.item_dropped({}, self.spider, ZeroDivisionError())ext.spider_closed(self.spider, "finished")- self.assertEqual(ext.stats._stats, {})+ assert ext.stats._stats == {}-class StatsCollectorTest(unittest.TestCase):- def setUp(self):+class TestStatsCollector:+ def setup_method(self):self.crawler = get_crawler(Spider)self.spider = self.crawler._create_spider("foo")def test_collector(self):stats = StatsCollector(self.crawler)- self.assertEqual(stats.get_stats(), {})- self.assertEqual(stats.get_value("anything"), None)- self.assertEqual(stats.get_value("anything", "default"), "default")+ assert stats.get_stats() == {}+ assert stats.get_value("anything") is None+ assert stats.get_value("anything", "default") == "default"stats.set_value("test", "value")- self.assertEqual(stats.get_stats(), {"test": "value"})+ assert stats.get_stats() == {"test": "value"}stats.set_value("test2", 23)- self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23})- self.assertEqual(stats.get_value("test2"), 23)+ assert stats.get_stats() == {"test": "value", "test2": 23}+ assert stats.get_value("test2") == 23stats.inc_value("test2")- self.assertEqual(stats.get_value("test2"), 24)+ assert stats.get_value("test2") == 24stats.inc_value("test2", 6)- self.assertEqual(stats.get_value("test2"), 30)+ assert stats.get_value("test2") == 30stats.max_value("test2", 6)- self.assertEqual(stats.get_value("test2"), 30)+ assert stats.get_value("test2") == 30stats.max_value("test2", 40)- self.assertEqual(stats.get_value("test2"), 40)+ assert stats.get_value("test2") == 40stats.max_value("test3", 1)- self.assertEqual(stats.get_value("test3"), 1)+ assert stats.get_value("test3") == 1stats.min_value("test2", 60)- self.assertEqual(stats.get_value("test2"), 40)+ assert stats.get_value("test2") == 40stats.min_value("test2", 35)- self.assertEqual(stats.get_value("test2"), 35)+ assert stats.get_value("test2") == 35stats.min_value("test4", 7)- self.assertEqual(stats.get_value("test4"), 7)+ assert stats.get_value("test4") == 7def test_dummy_collector(self):stats = DummyStatsCollector(self.crawler)- self.assertEqual(stats.get_stats(), {})- self.assertEqual(stats.get_value("anything"), None)- self.assertEqual(stats.get_value("anything", "default"), "default")+ assert stats.get_stats() == {}+ assert stats.get_value("anything") is None+ assert stats.get_value("anything", "default") == "default"stats.set_value("test", "value")stats.inc_value("v1")stats.max_value("v2", 100)stats.min_value("v3", 100)stats.open_spider("a")stats.set_value("test", "value", spider=self.spider)- self.assertEqual(stats.get_stats(), {})- self.assertEqual(stats.get_stats("a"), {})+ assert stats.get_stats() == {}+ assert stats.get_stats("a") == {}