Scrapy, a fast high-level web crawling & scraping framework for Python.
Convert remaining unittest assert* calls, use the tmp_path fixture. (#6725)
Andrey Rakhmatullineb654aa1a8d2
diff --git a/pyproject.toml b/pyproject.tomlindex 82d8056f6..84bf41a94 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -374,10 +374,6 @@ ignore = ["B904",# Use capitalized environment variable"SIM112",-- # Temporarily silenced PT rules- # Use a regular `assert` instead of unittest-style `assertEqual`- "PT009",][tool.ruff.lint.per-file-ignores]diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex db1f5c419..b69f43438 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -156,7 +156,7 @@ def assert_samelines(category=ScrapyDeprecationWarning,stacklevel=2,)- testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)+ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 38f0333bb..ad335f852 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -1,3 +1,4 @@+from typing import Anyfrom unittest import mockimport pytest@@ -171,7 +172,11 @@ Disallow: /some/randome/page.htmlmiddleware = RobotsTxtMiddleware(self.crawler)middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)deferred = middleware.process_request(Request("http://site.local"), None)- deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))++ def check_called(_: Any) -> None:+ assert middleware._logerror.called++ deferred.addCallback(check_called)return deferreddef test_robotstxt_immediate_error(self):@@ -202,7 +207,11 @@ Disallow: /some/randome/page.htmlmw_module_logger.error = mock.MagicMock()d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)- d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))++ def check_not_called(_: Any) -> None:+ assert not mw_module_logger.error.called # type: ignore[attr-defined]++ d.addCallback(check_not_called)return ddef test_robotstxt_user_agent_setting(self):diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 4bac8d273..ba4c6dc40 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -433,10 +433,12 @@ class TestEngine(TestEngineBase):e = ExecutionEngine(get_crawler(MySpider), lambda _: None)yield e.open_spider(MySpider(), [])e.start()++ def cb(exc: BaseException) -> None:+ assert str(exc), "Engine already running"+try:- yield self.assertFailure(e.start(), RuntimeError).addBoth(- lambda exc: self.assertEqual(str(exc), "Engine already running")- )+ yield self.assertFailure(e.start(), RuntimeError).addBoth(cb)finally:yield e.stop()diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex e515c16a0..9a582e4b7 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase):class FilesPipelineTestCaseFieldsMixin:- def setup_method(self):- self.tempdir = mkdtemp()-- def teardown_method(self):- rmtree(self.tempdir)-- def test_item_fields_default(self):+ def test_item_fields_default(self, tmp_path):url = "http://www.example.com/files/1.txt"item = self.item_class(name="item1", file_urls=[url])pipeline = FilesPipeline.from_crawler(- get_crawler(None, {"FILES_STORE": self.tempdir})+ get_crawler(None, {"FILES_STORE": tmp_path}))requests = list(pipeline.get_media_requests(item, None))assert requests[0].url == url@@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin:assert files == [results[0][1]]assert isinstance(item, self.item_class)- def test_item_fields_override_settings(self):+ def test_item_fields_override_settings(self, tmp_path):url = "http://www.example.com/files/1.txt"item = self.item_class(name="item1", custom_file_urls=[url])pipeline = FilesPipeline.from_crawler(get_crawler(None,{- "FILES_STORE": self.tempdir,+ "FILES_STORE": tmp_path,"FILES_URLS_FIELD": "custom_file_urls","FILES_RESULT_FIELD": "custom_files",},@@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings:("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),}- def setup_method(self):- self.tempdir = mkdtemp()-- def teardown_method(self):- rmtree(self.tempdir)-- def _generate_fake_settings(self, prefix=None):+ def _generate_fake_settings(self, tmp_path, prefix=None):def random_string():return "".join([chr(random.randint(97, 123)) for _ in range(10)])@@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings:"FILES_EXPIRES": random.randint(100, 1000),"FILES_URLS_FIELD": random_string(),"FILES_RESULT_FIELD": random_string(),- "FILES_STORE": self.tempdir,+ "FILES_STORE": tmp_path,}if not prefix:return settings@@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings:return UserDefinedFilePipeline- def test_different_settings_for_different_instances(self):+ def test_different_settings_for_different_instances(self, tmp_path):"""If there are different instances with different settings they should keepdifferent settings."""- custom_settings = self._generate_fake_settings()+ custom_settings = self._generate_fake_settings(tmp_path)another_pipeline = FilesPipeline.from_crawler(get_crawler(None, custom_settings))- one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None))+ one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None))for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:default_value = self.default_cls_settings[pipe_attr]assert getattr(one_pipeline, pipe_attr) == default_value@@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings:assert default_value != custom_valueassert getattr(another_pipeline, pipe_ins_attr) == custom_value- def test_subclass_attributes_preserved_if_no_settings(self):+ def test_subclass_attributes_preserved_if_no_settings(self, tmp_path):"""If subclasses override class attributes and there are no special settings those values should be kept."""pipe_cls = self._generate_fake_pipeline()- pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir}))+ pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:custom_value = getattr(pipe, pipe_ins_attr)assert custom_value != self.default_cls_settings[pipe_attr]assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr)- def test_subclass_attrs_preserved_custom_settings(self):+ def test_subclass_attrs_preserved_custom_settings(self, tmp_path):"""If file settings are defined but they are not defined for subclasssettings should be preserved."""pipeline_cls = self._generate_fake_pipeline()- settings = self._generate_fake_settings()+ settings = self._generate_fake_settings(tmp_path)pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:value = getattr(pipeline, pipe_ins_attr)@@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings:assert value != self.default_cls_settings[pipe_attr]assert value == setting_value- def test_no_custom_settings_for_subclasses(self):+ def test_no_custom_settings_for_subclasses(self, tmp_path):"""If there are no settings for subclass and no subclass attributes, pipeline should useattributes of base class.@@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings:passuser_pipeline = UserDefinedFilesPipeline.from_crawler(- get_crawler(None, {"FILES_STORE": self.tempdir})+ get_crawler(None, {"FILES_STORE": tmp_path}))for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:# Values from settings for custom pipeline should be set on pipeline instance.custom_value = self.default_cls_settings.get(pipe_attr.upper())assert getattr(user_pipeline, pipe_ins_attr) == custom_value- def test_custom_settings_for_subclasses(self):+ def test_custom_settings_for_subclasses(self, tmp_path):"""If there are custom settings for subclass and NO class attributes, pipeline should use customsettings.@@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings:passprefix = UserDefinedFilesPipeline.__name__.upper()- settings = self._generate_fake_settings(prefix=prefix)+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)user_pipeline = UserDefinedFilesPipeline.from_crawler(get_crawler(None, settings))@@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings:assert custom_value != self.default_cls_settings[pipe_attr]assert getattr(user_pipeline, pipe_inst_attr) == custom_value- def test_custom_settings_and_class_attrs_for_subclasses(self):+ def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):"""If there are custom settings for subclass AND class attributessetting keys are preferred and override attributes."""pipeline_cls = self._generate_fake_pipeline()prefix = pipeline_cls.__name__.upper()- settings = self._generate_fake_settings(prefix=prefix)+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))for (pipe_cls_attr,@@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings:assert custom_value != self.default_cls_settings[pipe_cls_attr]assert getattr(user_pipeline, pipe_inst_attr) == custom_value- def test_cls_attrs_with_DEFAULT_prefix(self):+ def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):class UserDefinedFilesPipeline(FilesPipeline):DEFAULT_FILES_RESULT_FIELD = "this"DEFAULT_FILES_URLS_FIELD = "that"pipeline = UserDefinedFilesPipeline.from_crawler(- get_crawler(None, {"FILES_STORE": self.tempdir})+ get_crawler(None, {"FILES_STORE": tmp_path}))assert (pipeline.files_result_field@@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings:== UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD)- def test_user_defined_subclass_default_key_names(self):+ def test_user_defined_subclass_default_key_names(self, tmp_path):"""Test situation when user defines subclass of FilesPipeline,but uses attribute names for default pipeline (without prefixingthem with pipeline class name)."""- settings = self._generate_fake_settings()+ settings = self._generate_fake_settings(tmp_path)class UserPipe(FilesPipeline):passdiff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex fef6bbbe9..f2ee18bd9 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings:"IMAGES_RESULT_FIELD": "images",}- def setup_method(self):- self.tempdir = mkdtemp()-- def teardown_method(self):- rmtree(self.tempdir)-- def _generate_fake_settings(self, prefix=None):+ def _generate_fake_settings(self, tmp_path, prefix=None):""":param prefix: string for setting keys:return: dictionary of image pipeline settings@@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings:settings = {"IMAGES_EXPIRES": random.randint(100, 1000),- "IMAGES_STORE": self.tempdir,+ "IMAGES_STORE": tmp_path,"IMAGES_RESULT_FIELD": random_string(),"IMAGES_URLS_FIELD": random_string(),"IMAGES_MIN_WIDTH": random.randint(1, 1000),@@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings:return UserDefinedImagePipeline- def test_different_settings_for_different_instances(self):+ def test_different_settings_for_different_instances(self, tmp_path):"""If there are two instances of ImagesPipeline class with different settings, they shouldhave different settings."""- custom_settings = self._generate_fake_settings()- default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None))+ custom_settings = self._generate_fake_settings(tmp_path)+ default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None))user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings))for pipe_attr, settings_attr in self.img_cls_attribute_names:expected_default_value = self.default_pipeline_settings.get(pipe_attr)@@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings:)assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value- def test_subclass_attrs_preserved_default_settings(self):+ def test_subclass_attrs_preserved_default_settings(self, tmp_path):"""If image settings are not defined at all subclass of ImagePipeline takes valuesfrom class attributes."""pipeline_cls = self._generate_fake_pipeline_subclass()pipeline = pipeline_cls.from_crawler(- get_crawler(None, {"IMAGES_STORE": self.tempdir})+ get_crawler(None, {"IMAGES_STORE": tmp_path}))for pipe_attr, settings_attr in self.img_cls_attribute_names:# Instance attribute (lowercase) must be equal to class attribute (uppercase).@@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings:assert attr_value != self.default_pipeline_settings[pipe_attr]assert attr_value == getattr(pipeline, pipe_attr)- def test_subclass_attrs_preserved_custom_settings(self):+ def test_subclass_attrs_preserved_custom_settings(self, tmp_path):"""If image settings are defined but they are not defined for subclass defaultvalues taken from settings should be preserved."""pipeline_cls = self._generate_fake_pipeline_subclass()- settings = self._generate_fake_settings()+ settings = self._generate_fake_settings(tmp_path)pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))for pipe_attr, settings_attr in self.img_cls_attribute_names:# Instance attribute (lowercase) must be equal to@@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings:setings_value = settings.get(settings_attr)assert value == setings_value- def test_no_custom_settings_for_subclasses(self):+ def test_no_custom_settings_for_subclasses(self, tmp_path):"""If there are no settings for subclass and no subclass attributes, pipeline should useattributes of base class.@@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings:passuser_pipeline = UserDefinedImagePipeline.from_crawler(- get_crawler(None, {"IMAGES_STORE": self.tempdir})+ get_crawler(None, {"IMAGES_STORE": tmp_path}))for pipe_attr, settings_attr in self.img_cls_attribute_names:# Values from settings for custom pipeline should be set on pipeline instance.custom_value = self.default_pipeline_settings.get(pipe_attr.upper())assert getattr(user_pipeline, pipe_attr.lower()) == custom_value- def test_custom_settings_for_subclasses(self):+ def test_custom_settings_for_subclasses(self, tmp_path):"""If there are custom settings for subclass and NO class attributes, pipeline should use customsettings.@@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings:passprefix = UserDefinedImagePipeline.__name__.upper()- settings = self._generate_fake_settings(prefix=prefix)+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)user_pipeline = UserDefinedImagePipeline.from_crawler(get_crawler(None, settings))@@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings:assert custom_value != self.default_pipeline_settings[pipe_attr]assert getattr(user_pipeline, pipe_attr.lower()) == custom_value- def test_custom_settings_and_class_attrs_for_subclasses(self):+ def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):"""If there are custom settings for subclass AND class attributessetting keys are preferred and override attributes."""pipeline_cls = self._generate_fake_pipeline_subclass()prefix = pipeline_cls.__name__.upper()- settings = self._generate_fake_settings(prefix=prefix)+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))for pipe_attr, settings_attr in self.img_cls_attribute_names:custom_value = settings.get(prefix + "_" + settings_attr)assert custom_value != self.default_pipeline_settings[pipe_attr]assert getattr(user_pipeline, pipe_attr.lower()) == custom_value- def test_cls_attrs_with_DEFAULT_prefix(self):+ def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):class UserDefinedImagePipeline(ImagesPipeline):DEFAULT_IMAGES_URLS_FIELD = "something"DEFAULT_IMAGES_RESULT_FIELD = "something_else"pipeline = UserDefinedImagePipeline.from_crawler(- get_crawler(None, {"IMAGES_STORE": self.tempdir})+ get_crawler(None, {"IMAGES_STORE": tmp_path}))assert (pipeline.images_result_field@@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings:== UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD)- def test_user_defined_subclass_default_key_names(self):+ def test_user_defined_subclass_default_key_names(self, tmp_path):"""Test situation when user defines subclass of ImagePipeline,but uses attribute names for default pipeline (without prefixingthem with pipeline class name)."""- settings = self._generate_fake_settings()+ settings = self._generate_fake_settings(tmp_path)class UserPipe(ImagesPipeline):passdiff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 0b845fdb0..41d9b8933 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -1,24 +1,14 @@-from pathlib import Path-from shutil import rmtree-from tempfile import mkdtemp-from scrapy.utils.template import render_templatefileclass TestUtilsRenderTemplateFile:- def setup_method(self):- self.tmp_path = mkdtemp()-- def teardown_method(self):- rmtree(self.tmp_path)-- def test_simple_render(self):+ def test_simple_render(self, tmp_path):context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}template = "from ${project_name}.spiders.${name} import ${classname}"rendered = "from proj.spiders.spi import TheSpider"- template_path = Path(self.tmp_path, "templ.py.tmpl")- render_path = Path(self.tmp_path, "templ.py")+ template_path = tmp_path / "templ.py.tmpl"+ render_path = tmp_path / "templ.py"template_path.write_text(template, encoding="utf8")assert template_path.is_file() # Failure of test itself