Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Convert remaining unittest assert* calls, use the tmp_path fixture. (#6725)

Andrey Rakhmatullineb654aa1a8d2
diff --git a/pyproject.toml b/pyproject.toml
index 82d8056f6..84bf41a94 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -374,10 +374,6 @@ ignore = [
"B904",
# Use capitalized environment variable
"SIM112",
-
- # Temporarily silenced PT rules
- # Use a regular `assert` instead of unittest-style `assertEqual`
- "PT009",
]
[tool.ruff.lint.per-file-ignores]
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index db1f5c419..b69f43438 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -156,7 +156,7 @@ def assert_samelines(
category=ScrapyDeprecationWarning,
stacklevel=2,
)
- testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
+ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 38f0333bb..ad335f852 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,3 +1,4 @@
+from typing import Any
from unittest import mock
import pytest
@@ -171,7 +172,11 @@ Disallow: /some/randome/page.html
middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
deferred = middleware.process_request(Request("http://site.local"), None)
- deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
+
+ def check_called(_: Any) -> None:
+ assert middleware._logerror.called
+
+ deferred.addCallback(check_called)
return deferred
def test_robotstxt_immediate_error(self):
@@ -202,7 +207,11 @@ Disallow: /some/randome/page.html
mw_module_logger.error = mock.MagicMock()
d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
- d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called))
+
+ def check_not_called(_: Any) -> None:
+ assert not mw_module_logger.error.called # type: ignore[attr-defined]
+
+ d.addCallback(check_not_called)
return d
def test_robotstxt_user_agent_setting(self):
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 4bac8d273..ba4c6dc40 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -433,10 +433,12 @@ class TestEngine(TestEngineBase):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.open_spider(MySpider(), [])
e.start()
+
+ def cb(exc: BaseException) -> None:
+ assert str(exc), "Engine already running"
+
try:
- yield self.assertFailure(e.start(), RuntimeError).addBoth(
- lambda exc: self.assertEqual(str(exc), "Engine already running")
- )
+ yield self.assertFailure(e.start(), RuntimeError).addBoth(cb)
finally:
yield e.stop()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e515c16a0..9a582e4b7 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase):
class FilesPipelineTestCaseFieldsMixin:
- def setup_method(self):
- self.tempdir = mkdtemp()
-
- def teardown_method(self):
- rmtree(self.tempdir)
-
- def test_item_fields_default(self):
+ def test_item_fields_default(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", file_urls=[url])
pipeline = FilesPipeline.from_crawler(
- get_crawler(None, {"FILES_STORE": self.tempdir})
+ get_crawler(None, {"FILES_STORE": tmp_path})
)
requests = list(pipeline.get_media_requests(item, None))
assert requests[0].url == url
@@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin:
assert files == [results[0][1]]
assert isinstance(item, self.item_class)
- def test_item_fields_override_settings(self):
+ def test_item_fields_override_settings(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", custom_file_urls=[url])
pipeline = FilesPipeline.from_crawler(
get_crawler(
None,
{
- "FILES_STORE": self.tempdir,
+ "FILES_STORE": tmp_path,
"FILES_URLS_FIELD": "custom_file_urls",
"FILES_RESULT_FIELD": "custom_files",
},
@@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings:
("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),
}
- def setup_method(self):
- self.tempdir = mkdtemp()
-
- def teardown_method(self):
- rmtree(self.tempdir)
-
- def _generate_fake_settings(self, prefix=None):
+ def _generate_fake_settings(self, tmp_path, prefix=None):
def random_string():
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
@@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings:
"FILES_EXPIRES": random.randint(100, 1000),
"FILES_URLS_FIELD": random_string(),
"FILES_RESULT_FIELD": random_string(),
- "FILES_STORE": self.tempdir,
+ "FILES_STORE": tmp_path,
}
if not prefix:
return settings
@@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings:
return UserDefinedFilePipeline
- def test_different_settings_for_different_instances(self):
+ def test_different_settings_for_different_instances(self, tmp_path):
"""
If there are different instances with different settings they should keep
different settings.
"""
- custom_settings = self._generate_fake_settings()
+ custom_settings = self._generate_fake_settings(tmp_path)
another_pipeline = FilesPipeline.from_crawler(
get_crawler(None, custom_settings)
)
- one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None))
+ one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
default_value = self.default_cls_settings[pipe_attr]
assert getattr(one_pipeline, pipe_attr) == default_value
@@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings:
assert default_value != custom_value
assert getattr(another_pipeline, pipe_ins_attr) == custom_value
- def test_subclass_attributes_preserved_if_no_settings(self):
+ def test_subclass_attributes_preserved_if_no_settings(self, tmp_path):
"""
If subclasses override class attributes and there are no special settings those values should be kept.
"""
pipe_cls = self._generate_fake_pipeline()
- pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir}))
+ pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
custom_value = getattr(pipe, pipe_ins_attr)
assert custom_value != self.default_cls_settings[pipe_attr]
assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr)
- def test_subclass_attrs_preserved_custom_settings(self):
+ def test_subclass_attrs_preserved_custom_settings(self, tmp_path):
"""
If file settings are defined but they are not defined for subclass
settings should be preserved.
"""
pipeline_cls = self._generate_fake_pipeline()
- settings = self._generate_fake_settings()
+ settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
value = getattr(pipeline, pipe_ins_attr)
@@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings:
assert value != self.default_cls_settings[pipe_attr]
assert value == setting_value
- def test_no_custom_settings_for_subclasses(self):
+ def test_no_custom_settings_for_subclasses(self, tmp_path):
"""
If there are no settings for subclass and no subclass attributes, pipeline should use
attributes of base class.
@@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings:
pass
user_pipeline = UserDefinedFilesPipeline.from_crawler(
- get_crawler(None, {"FILES_STORE": self.tempdir})
+ get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_cls_settings.get(pipe_attr.upper())
assert getattr(user_pipeline, pipe_ins_attr) == custom_value
- def test_custom_settings_for_subclasses(self):
+ def test_custom_settings_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass and NO class attributes, pipeline should use custom
settings.
@@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings:
pass
prefix = UserDefinedFilesPipeline.__name__.upper()
- settings = self._generate_fake_settings(prefix=prefix)
+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, settings)
)
@@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings:
assert custom_value != self.default_cls_settings[pipe_attr]
assert getattr(user_pipeline, pipe_inst_attr) == custom_value
- def test_custom_settings_and_class_attrs_for_subclasses(self):
+ def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass AND class attributes
setting keys are preferred and override attributes.
"""
pipeline_cls = self._generate_fake_pipeline()
prefix = pipeline_cls.__name__.upper()
- settings = self._generate_fake_settings(prefix=prefix)
+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for (
pipe_cls_attr,
@@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings:
assert custom_value != self.default_cls_settings[pipe_cls_attr]
assert getattr(user_pipeline, pipe_inst_attr) == custom_value
- def test_cls_attrs_with_DEFAULT_prefix(self):
+ def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):
class UserDefinedFilesPipeline(FilesPipeline):
DEFAULT_FILES_RESULT_FIELD = "this"
DEFAULT_FILES_URLS_FIELD = "that"
pipeline = UserDefinedFilesPipeline.from_crawler(
- get_crawler(None, {"FILES_STORE": self.tempdir})
+ get_crawler(None, {"FILES_STORE": tmp_path})
)
assert (
pipeline.files_result_field
@@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings:
== UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD
)
- def test_user_defined_subclass_default_key_names(self):
+ def test_user_defined_subclass_default_key_names(self, tmp_path):
"""Test situation when user defines subclass of FilesPipeline,
but uses attribute names for default pipeline (without prefixing
them with pipeline class name).
"""
- settings = self._generate_fake_settings()
+ settings = self._generate_fake_settings(tmp_path)
class UserPipe(FilesPipeline):
pass
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index fef6bbbe9..f2ee18bd9 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings:
"IMAGES_RESULT_FIELD": "images",
}
- def setup_method(self):
- self.tempdir = mkdtemp()
-
- def teardown_method(self):
- rmtree(self.tempdir)
-
- def _generate_fake_settings(self, prefix=None):
+ def _generate_fake_settings(self, tmp_path, prefix=None):
"""
:param prefix: string for setting keys
:return: dictionary of image pipeline settings
@@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings:
settings = {
"IMAGES_EXPIRES": random.randint(100, 1000),
- "IMAGES_STORE": self.tempdir,
+ "IMAGES_STORE": tmp_path,
"IMAGES_RESULT_FIELD": random_string(),
"IMAGES_URLS_FIELD": random_string(),
"IMAGES_MIN_WIDTH": random.randint(1, 1000),
@@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings:
return UserDefinedImagePipeline
- def test_different_settings_for_different_instances(self):
+ def test_different_settings_for_different_instances(self, tmp_path):
"""
If there are two instances of ImagesPipeline class with different settings, they should
have different settings.
"""
- custom_settings = self._generate_fake_settings()
- default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None))
+ custom_settings = self._generate_fake_settings(tmp_path)
+ default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None))
user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_default_value = self.default_pipeline_settings.get(pipe_attr)
@@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings:
)
assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value
- def test_subclass_attrs_preserved_default_settings(self):
+ def test_subclass_attrs_preserved_default_settings(self, tmp_path):
"""
If image settings are not defined at all subclass of ImagePipeline takes values
from class attributes.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
pipeline = pipeline_cls.from_crawler(
- get_crawler(None, {"IMAGES_STORE": self.tempdir})
+ get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to class attribute (uppercase).
@@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings:
assert attr_value != self.default_pipeline_settings[pipe_attr]
assert attr_value == getattr(pipeline, pipe_attr)
- def test_subclass_attrs_preserved_custom_settings(self):
+ def test_subclass_attrs_preserved_custom_settings(self, tmp_path):
"""
If image settings are defined but they are not defined for subclass default
values taken from settings should be preserved.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
- settings = self._generate_fake_settings()
+ settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to
@@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings:
setings_value = settings.get(settings_attr)
assert value == setings_value
- def test_no_custom_settings_for_subclasses(self):
+ def test_no_custom_settings_for_subclasses(self, tmp_path):
"""
If there are no settings for subclass and no subclass attributes, pipeline should use
attributes of base class.
@@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings:
pass
user_pipeline = UserDefinedImagePipeline.from_crawler(
- get_crawler(None, {"IMAGES_STORE": self.tempdir})
+ get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_pipeline_settings.get(pipe_attr.upper())
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value
- def test_custom_settings_for_subclasses(self):
+ def test_custom_settings_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass and NO class attributes, pipeline should use custom
settings.
@@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings:
pass
prefix = UserDefinedImagePipeline.__name__.upper()
- settings = self._generate_fake_settings(prefix=prefix)
+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, settings)
)
@@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings:
assert custom_value != self.default_pipeline_settings[pipe_attr]
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value
- def test_custom_settings_and_class_attrs_for_subclasses(self):
+ def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):
"""
If there are custom settings for subclass AND class attributes
setting keys are preferred and override attributes.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
prefix = pipeline_cls.__name__.upper()
- settings = self._generate_fake_settings(prefix=prefix)
+ settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
custom_value = settings.get(prefix + "_" + settings_attr)
assert custom_value != self.default_pipeline_settings[pipe_attr]
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value
- def test_cls_attrs_with_DEFAULT_prefix(self):
+ def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):
class UserDefinedImagePipeline(ImagesPipeline):
DEFAULT_IMAGES_URLS_FIELD = "something"
DEFAULT_IMAGES_RESULT_FIELD = "something_else"
pipeline = UserDefinedImagePipeline.from_crawler(
- get_crawler(None, {"IMAGES_STORE": self.tempdir})
+ get_crawler(None, {"IMAGES_STORE": tmp_path})
)
assert (
pipeline.images_result_field
@@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings:
== UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD
)
- def test_user_defined_subclass_default_key_names(self):
+ def test_user_defined_subclass_default_key_names(self, tmp_path):
"""Test situation when user defines subclass of ImagePipeline,
but uses attribute names for default pipeline (without prefixing
them with pipeline class name).
"""
- settings = self._generate_fake_settings()
+ settings = self._generate_fake_settings(tmp_path)
class UserPipe(ImagesPipeline):
pass
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 0b845fdb0..41d9b8933 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1,24 +1,14 @@
-from pathlib import Path
-from shutil import rmtree
-from tempfile import mkdtemp
-
from scrapy.utils.template import render_templatefile
class TestUtilsRenderTemplateFile:
- def setup_method(self):
- self.tmp_path = mkdtemp()
-
- def teardown_method(self):
- rmtree(self.tmp_path)
-
- def test_simple_render(self):
+ def test_simple_render(self, tmp_path):
context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"
- template_path = Path(self.tmp_path, "templ.py.tmpl")
- render_path = Path(self.tmp_path, "templ.py")
+ template_path = tmp_path / "templ.py.tmpl"
+ render_path = tmp_path / "templ.py"
template_path.write_text(template, encoding="utf8")
assert template_path.is_file() # Failure of test itself