Scrapy, a fast high-level web crawling & scraping framework for Python.
Convert test loops to parametrize(). (#8311)
Andrey Rakhmatulline6689aaba804
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.pyindex e2caa5dfb..d22b007ae 100644--- a/scrapy/spiders/feed.py+++ b/scrapy/spiders/feed.py@@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rstfrom __future__ import annotationsimport warnings-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Literalfrom scrapy.exceptions import NotSupported, ScrapyDeprecationWarningfrom scrapy.http import Response, TextResponse@@ -43,7 +43,7 @@ class XMLFeedSpider(Spider):defining :meth:`~scrapy.Spider.parse` is not enough."""- iterator: str = "iternodes"+ iterator: Literal["iternodes", "html", "xml"] = "iternodes""""The iterator to use. It can be either:- ``'iternodes'`` - a fast iterator based on ``lxml``diff --git a/tests/test_commands.py b/tests/test_commands.pyindex db07a3d4d..b0b10b7bc 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -332,26 +332,29 @@ class MySpider(scrapy.Spider):assert msg in errassert "no running event loop" in err- def test_project_settings(self, proj_path: Path) -> None:+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])+ def test_project_settings(self, spider: str, proj_path: Path) -> None:"""The reactor is set via the project default settings (to the asyncio value).AsyncCrawlerProcess, the asyncio reactor, both spiders work."""- for spider in ["sp", "aiosp"]:- self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)+ self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)- def test_cmdline_asyncio(self, proj_path: Path) -> None:+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])+ def test_cmdline_asyncio(self, spider: str, proj_path: Path) -> None:"""The reactor is set via the command line to the asyncio value.AsyncCrawlerProcess, the asyncio reactor, both spiders work."""- for spider in ["sp", "aiosp"]:- self._assert_spider_works(- self.ASYNC_MSG,- proj_path,- spider,- "-s",- f"TWISTED_REACTOR={_asyncio_reactor_path}",- )+ self._assert_spider_works(+ self.ASYNC_MSG,+ proj_path,+ spider,+ "-s",+ f"TWISTED_REACTOR={_asyncio_reactor_path}",+ )- def test_project_settings_explicit_asyncio(self, proj_path: Path) -> None:+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])+ def test_project_settings_explicit_asyncio(+ self, spider: str, proj_path: Path+ ) -> None:"""The reactor explicitly is set via the project settings to the asyncio value.AsyncCrawlerProcess, the asyncio reactor, both spiders work."""@@ -360,8 +363,7 @@ class MySpider(scrapy.Spider):f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n",)- for spider in ["sp", "aiosp"]:- self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)+ self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)def test_cmdline_empty(self, proj_path: Path) -> None:"""The reactor is set via the command line to the empty value.@@ -383,17 +385,17 @@ class MySpider(scrapy.Spider):self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp")self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")- def test_spider_settings_asyncio(self, proj_path: Path) -> None:+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])+ def test_spider_settings_asyncio(self, spider: str, proj_path: Path) -> None:"""The reactor is set via the spider settings to the asyncio value.AsyncCrawlerProcess, the asyncio reactor, both spiders work."""- for spider in ["sp", "aiosp"]:- self._replace_custom_settings(- proj_path / self.project_name,- spider,- f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",- )- self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)+ self._replace_custom_settings(+ proj_path / self.project_name,+ spider,+ f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",+ )+ self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)def test_spider_settings_asyncio_cmdline_empty(self, proj_path: Path) -> None:"""The reactor is set via the spider settings to the asyncio value@@ -415,7 +417,10 @@ class MySpider(scrapy.Spider):self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR=")- def test_project_empty_spider_settings_asyncio(self, proj_path: Path) -> None:+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])+ def test_project_empty_spider_settings_asyncio(+ self, spider: str, proj_path: Path+ ) -> None:"""The reactor is set via the project settings to the empty valueand via the spider settings to the asyncio value. CrawlerProcess ischosen based on the project settings, but the asyncio reactor is chosen@@ -423,15 +428,17 @@ class MySpider(scrapy.Spider):CrawlerProcess, the asyncio reactor, both spiders work."""self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n")- for spider in ["sp", "aiosp"]:- self._replace_custom_settings(- proj_path / self.project_name,- spider,- f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",- )- self._assert_spider_works(self.NORMAL_MSG, proj_path, spider)+ self._replace_custom_settings(+ proj_path / self.project_name,+ spider,+ f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",+ )+ self._assert_spider_works(self.NORMAL_MSG, proj_path, spider)- def test_project_asyncio_spider_settings_select(self, proj_path: Path) -> None:+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])+ def test_project_asyncio_spider_settings_select(+ self, spider: str, proj_path: Path+ ) -> None:"""The reactor is set via the project settings to the asyncio valueand via the spider settings to the select value. AsyncCrawlerProcessis chosen based on the project settings, and the conflicting reactor@@ -443,19 +450,18 @@ class MySpider(scrapy.Spider):proj_path / self.project_name,f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n",)- for spider in ["sp", "aiosp"]:- self._replace_custom_settings(- proj_path / self.project_name,- spider,- "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}",- )- _, _, err = proc(self.name, spider, cwd=proj_path)- assert self.ASYNC_MSG in err- assert (- "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"- " does not match the requested one"- " (twisted.internet.selectreactor.SelectReactor)"- ) in err+ self._replace_custom_settings(+ proj_path / self.project_name,+ spider,+ "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}",+ )+ _, _, err = proc(self.name, spider, cwd=proj_path)+ assert self.ASYNC_MSG in err+ assert (+ "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"+ " does not match the requested one"+ " (twisted.internet.selectreactor.SelectReactor)"+ ) in errdef test_project_asyncio_spider_settings_select_forced(self, proj_path: Pathdiff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex b5dfa937f..5415274ff 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -219,17 +219,22 @@ async def test_pool_limits(concurrency: int, expected: int | None) -> None:await handler.close()-@coroutine_test-async def test_experimental_warning(caplog: pytest.LogCaptureFixture) -> None:- class StableHandler(HttpxDownloadHandler):- experimental = False+class StableHandler(HttpxDownloadHandler):+ experimental = False- crawler = get_crawler()- for handler_cls, expected in ((HttpxDownloadHandler, True), (StableHandler, False)):- caplog.clear()- handler = build_from_crawler(handler_cls, crawler)- await handler.close()- assert ("is experimental" in caplog.text) is expected++ ("handler_cls", "expected"), [(HttpxDownloadHandler, True), (StableHandler, False)]+)+@coroutine_test+async def test_experimental_warning(+ handler_cls: type[HttpxDownloadHandler],+ expected: bool,+ caplog: pytest.LogCaptureFixture,+) -> None:+ handler = build_from_crawler(handler_cls, get_crawler())+ await handler.close()+ assert ("is experimental" in caplog.text) is expected@coroutine_testdiff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 976436e83..d96978a6a 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -528,7 +528,10 @@ class TestHttpCompression:assert not thread.is_alive(), "decompression did not terminate"assert result[0].body == plain- def test_deflate_chunked_output_leaves_nothing_to_flush(self) -> None:+ @pytest.mark.parametrize(+ "size", [_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE]+ )+ def test_deflate_chunked_output_leaves_nothing_to_flush(self, size: int) -> None:"""Guard the zlib behaviour that lets ``_inflate()`` skip ``flush()``.``decompress(max_length=...)`` leaves the input it could not turn into@@ -536,14 +539,13 @@ class TestHttpCompression:stream has been fully emitted. If a future zlib buffers output instead,this fails and ``_inflate()`` needs to flush the remainder again."""- for size in (_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE):- decompressor = zlib.decompressobj()- decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)- while decompressor.unconsumed_tail and not decompressor.eof:- decompressor.decompress(- decompressor.unconsumed_tail, max_length=_CHUNK_SIZE- )- assert decompressor.flush() == b"", f"pending output for size {size}"+ decompressor = zlib.decompressobj()+ decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)+ while decompressor.unconsumed_tail and not decompressor.eof:+ decompressor.decompress(+ decompressor.unconsumed_tail, max_length=_CHUNK_SIZE+ )+ assert decompressor.flush() == b""async def _test_compression_bomb_setting(self, compression_id: str) -> None:settings = {"DOWNLOAD_MAXSIZE": 1_000_000}@@ -864,13 +866,13 @@ class TestHttpCompression:resp = await self._get_truncated_response("gzip")assert resp.body.startswith(b"<!DOCTYPE")+ @pytest.mark.parametrize(+ "check_key", [key for key in FORMAT if key.startswith("zstd-")]+ )@coroutine_test- async def test_process_truncated_response_zstd(self):- for check_key in FORMAT:- if not check_key.startswith("zstd-"):- continue- resp = await self._get_truncated_response(check_key)- assert len(resp.body) == 0+ async def test_process_truncated_response_zstd(self, check_key: str) -> None:+ resp = await self._get_truncated_response(check_key)+ assert len(resp.body) == 0@pytest.mark.parametrize(("body", "encoding"),diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex d4f6c4796..3372813f9 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -18,35 +18,40 @@ class TestHttpProxyMiddleware:with pytest.raises(NotConfigured):build_from_crawler(HttpProxyMiddleware, crawler)- def test_no_environment_proxies(self, monkeypatch: pytest.MonkeyPatch) -> None:+ @pytest.mark.parametrize("url", ["http://e.com", "https://e.com", "file:///tmp/a"])+ def test_no_environment_proxies(+ self, url: str, monkeypatch: pytest.MonkeyPatch+ ) -> None:for name in list(os.environ):monkeypatch.delenv(name)monkeypatch.setenv("dummy_proxy", "reset_env_and_do_not_raise")mw = HttpProxyMiddleware()- for url in ("http://e.com", "https://e.com", "file:///tmp/a"):- req = Request(url)- assert mw.process_request(req) is None- assert req.url == url- assert req.meta == {}-- def test_environment_proxies(self, monkeypatch: pytest.MonkeyPatch) -> None:- http_proxy = "https://proxy.for.http:3128"- https_proxy = "http://proxy.for.https:8080"- monkeypatch.setenv("http_proxy", http_proxy)- monkeypatch.setenv("https_proxy", https_proxy)+ req = Request(url)+ assert mw.process_request(req) is None+ assert req.url == url+ assert req.meta == {}++ @pytest.mark.parametrize(+ ("url", "proxy"),+ [+ ("http://e.com", "https://proxy.for.http:3128"),+ ("https://e.com", "http://proxy.for.https:8080"),+ ("file://tmp/a", None),+ ],+ )+ def test_environment_proxies(+ self, url: str, proxy: str | None, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("http_proxy", "https://proxy.for.http:3128")+ monkeypatch.setenv("https_proxy", "http://proxy.for.https:8080")monkeypatch.delenv("file_proxy", raising=False)mw = HttpProxyMiddleware()- for url, proxy in [- ("http://e.com", http_proxy),- ("https://e.com", https_proxy),- ("file://tmp/a", None),- ]:- req = Request(url)- assert mw.process_request(req) is None- assert req.url == url- assert req.meta.get("proxy") == proxy+ req = Request(url)+ assert mw.process_request(req) is None+ assert req.url == url+ assert req.meta.get("proxy") == proxydef test_proxy_precedence_meta(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setenv("http_proxy", "https://proxy.com")diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 29b301b7d..2a8e936a8 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -80,23 +80,22 @@ class ErrorCloseReasonSpider(ChangeCloseReasonSpider):class TestEngine(TestEngineBase):+ @pytest.mark.parametrize(+ "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]+ )@coroutine_test- async def test_crawler(self, mockserver: MockServer) -> None:- for spider in (- MySpider,- DictItemsSpider,- AttrsItemsSpider,- DataClassItemsSpider,- ):- run = CrawlerRun(spider)- await run.run(mockserver)- self._assert_visited_urls(run)- self._assert_scheduled_requests(run, count=9)- self._assert_downloaded_responses(run, count=9)- self._assert_scraped_items(run)- self._assert_signals_caught(run)- self._assert_headers_received(run)- self._assert_bytes_received(run)+ async def test_crawler(+ self, spider: type[MySpider], mockserver: MockServer+ ) -> None:+ run = CrawlerRun(spider)+ await run.run(mockserver)+ self._assert_visited_urls(run)+ self._assert_scheduled_requests(run, count=9)+ self._assert_downloaded_responses(run, count=9)+ self._assert_scraped_items(run)+ self._assert_signals_caught(run)+ self._assert_headers_received(run)+ self._assert_bytes_received(run)@coroutine_testasync def test_crawler_dupefilter(self, mockserver: MockServer) -> None:diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.pyindex 27f282af4..b20ffd8d2 100644--- a/tests/test_engine_stop_download_bytes.py+++ b/tests/test_engine_stop_download_bytes.py@@ -2,6 +2,8 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING+import pytest+from scrapy.exceptions import StopDownloadfrom tests.utils.bases.engine import TestEngineBasefrom tests.utils.decorators import coroutine_test@@ -14,8 +16,6 @@ from tests.utils.engine import ()if TYPE_CHECKING:- import pytest-from tests.mockserver.http import MockServer@@ -26,30 +26,30 @@ class BytesReceivedCrawlerRun(CrawlerRun):class TestBytesReceivedEngine(TestEngineBase):+ @pytest.mark.parametrize(+ "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]+ )@coroutine_testasync def test_crawler(- self, mockserver: MockServer, caplog: pytest.LogCaptureFixture+ self,+ spider: type[MySpider],+ mockserver: MockServer,+ caplog: pytest.LogCaptureFixture,) -> None:- for spider in (- MySpider,- DictItemsSpider,- AttrsItemsSpider,- DataClassItemsSpider,- ):- run = BytesReceivedCrawlerRun(spider)- with caplog.at_level("DEBUG"):- await run.run(mockserver)- for url in ("/redirected", "/static/", "/numbers"):- assert (- f"Download stopped for <GET {mockserver.url(url)}> "- "from signal handler BytesReceivedCrawlerRun.bytes_received"- ) in caplog.text- self._assert_visited_urls(run)- self._assert_scheduled_requests(run, count=9)- self._assert_downloaded_responses(run, count=9)- self._assert_signals_caught(run)- self._assert_headers_received(run)- self._assert_bytes_received(run)+ run = BytesReceivedCrawlerRun(spider)+ with caplog.at_level("DEBUG"):+ await run.run(mockserver)+ for url in ("/redirected", "/static/", "/numbers"):+ assert (+ f"Download stopped for <GET {mockserver.url(url)}> "+ "from signal handler BytesReceivedCrawlerRun.bytes_received"+ ) in caplog.text+ self._assert_visited_urls(run)+ self._assert_scheduled_requests(run, count=9)+ self._assert_downloaded_responses(run, count=9)+ self._assert_signals_caught(run)+ self._assert_headers_received(run)+ self._assert_bytes_received(run)@staticmethoddef _assert_bytes_received(run: CrawlerRun) -> None:diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.pyindex 09fa14326..0dae511a0 100644--- a/tests/test_engine_stop_download_headers.py+++ b/tests/test_engine_stop_download_headers.py@@ -2,6 +2,8 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING+import pytest+from scrapy.exceptions import StopDownloadfrom tests.utils.bases.engine import TestEngineBasefrom tests.utils.decorators import coroutine_test@@ -14,8 +16,6 @@ from tests.utils.engine import ()if TYPE_CHECKING:- import pytest-from tests.mockserver.http import MockServer@@ -26,29 +26,29 @@ class HeadersReceivedCrawlerRun(CrawlerRun):class TestHeadersReceivedEngine(TestEngineBase):+ @pytest.mark.parametrize(+ "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]+ )@coroutine_testasync def test_crawler(- self, mockserver: MockServer, caplog: pytest.LogCaptureFixture+ self,+ spider: type[MySpider],+ mockserver: MockServer,+ caplog: pytest.LogCaptureFixture,) -> None:- for spider in (- MySpider,- DictItemsSpider,- AttrsItemsSpider,- DataClassItemsSpider,- ):- run = HeadersReceivedCrawlerRun(spider)- with caplog.at_level("DEBUG"):- await run.run(mockserver)- for url in ("/redirected", "/static/", "/numbers"):- assert (- f"Download stopped for <GET {mockserver.url(url)}> "- "from signal handler HeadersReceivedCrawlerRun.headers_received"- ) in caplog.text- self._assert_visited_urls(run)- self._assert_downloaded_responses(run, count=6)- self._assert_signals_caught(run)- self._assert_bytes_received(run)- self._assert_headers_received(run)+ run = HeadersReceivedCrawlerRun(spider)+ with caplog.at_level("DEBUG"):+ await run.run(mockserver)+ for url in ("/redirected", "/static/", "/numbers"):+ assert (+ f"Download stopped for <GET {mockserver.url(url)}> "+ "from signal handler HeadersReceivedCrawlerRun.headers_received"+ ) in caplog.text+ self._assert_visited_urls(run)+ self._assert_downloaded_responses(run, count=6)+ self._assert_signals_caught(run)+ self._assert_bytes_received(run)+ self._assert_headers_received(run)@staticmethoddef _assert_bytes_received(run: CrawlerRun) -> None:diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 2b77c17d1..36c45ecc4 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -42,6 +42,11 @@ class CustomFieldItem(Item):age = Field(serializer=custom_serializer)+class FriendsItem(Item):+ name = Field()+ friends = Field()++@dataclasses.dataclassclass MyDataClass:name: str@@ -332,46 +337,43 @@ class TestCsvItemExporter(TestBaseItemExporter):expected=b"age,name\r\n22,John\xc2\xa3\r\n",)- def test_header_export_single_field(self):- for item in [self.i, ItemAdapter(self.i).asdict()]:- self.assertExportResult(- item=item,- fields_to_export=["age"],- expected=b"age\r\n22\r\n",- )-- def test_header_export_two_items(self):- for item in [self.i, ItemAdapter(self.i).asdict()]:- output = BytesIO()- ie = CsvItemExporter(output)- ie.start_exporting()- ie.export_item(item)- ie.export_item(item)- ie.finish_exporting()- del ie # See the first “del self.ie” in this file for context.- self.assertCsvEqual(- output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n"- )+ @pytest.mark.parametrize("as_dict", [False, True], ids=["item", "dict"])+ def test_header_export_single_field(self, as_dict: bool) -> None:+ self.assertExportResult(+ item=ItemAdapter(self.i).asdict() if as_dict else self.i,+ fields_to_export=["age"],+ expected=b"age\r\n22\r\n",+ )- def test_header_no_header_line(self):- for item in [self.i, ItemAdapter(self.i).asdict()]:- self.assertExportResult(- item=item,- include_headers_line=False,- expected=b"22,John\xc2\xa3\r\n",- )+ @pytest.mark.parametrize("as_dict", [False, True], ids=["item", "dict"])+ def test_header_export_two_items(self, as_dict: bool) -> None:+ item = ItemAdapter(self.i).asdict() if as_dict else self.i+ output = BytesIO()+ ie = CsvItemExporter(output)+ ie.start_exporting()+ ie.export_item(item)+ ie.export_item(item)+ ie.finish_exporting()+ del ie # See the first “del self.ie” in this file for context.+ self.assertCsvEqual(+ output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n"+ )- def test_join_multivalue(self):- class TestItem2(Item):- name = Field()- friends = Field()+ @pytest.mark.parametrize("as_dict", [False, True], ids=["item", "dict"])+ def test_header_no_header_line(self, as_dict: bool) -> None:+ self.assertExportResult(+ item=ItemAdapter(self.i).asdict() if as_dict else self.i,+ include_headers_line=False,+ expected=b"22,John\xc2\xa3\r\n",+ )- for cls in TestItem2, dict:- self.assertExportResult(- item=cls(name="John", friends=["Mary", "Paul"]),- include_headers_line=False,- expected='"Mary,Paul",John\r\n',- )+ @pytest.mark.parametrize("cls", [FriendsItem, dict])+ def test_join_multivalue(self, cls: type) -> None:+ self.assertExportResult(+ item=cls(name="John", friends=["Mary", "Paul"]),+ include_headers_line=False,+ expected='"Mary,Paul",John\r\n',+ )def test_join_multivalue_not_strings(self):self.assertExportResult(diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 9582b4298..aeed0d6d6 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -680,14 +680,14 @@ class TestHttps2ClientProtocol:data = json.loads(str(response.body, content_encoding))assert data == params+ @pytest.mark.parametrize("status", [200, 404])@deferred_f_from_coro_fasync def test_status_codes(- self, server_port: int, client: H2ClientProtocol+ self, status: int, server_port: int, client: H2ClientProtocol) -> None:- for status in [200, 404]:- request = Request(self.get_url(server_port, f"/status?n={status}"))- response = await make_request(client, request)- assert response.status == status+ request = Request(self.get_url(server_port, f"/status?n={status}"))+ response = await make_request(client, request)+ assert response.status == status@pytest.mark.filterwarnings(r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.pyindex cb18a0b03..8d82a4a49 100644--- a/tests/test_http_request_form.py+++ b/tests/test_http_request_form.py@@ -1008,20 +1008,21 @@ class TestFormRequest(TestRequestBase):with pytest.raises(ValueError, match="No <form> element found with"):self.request_class.from_response(response, formcss="input[name='abc']")- def test_from_response_valid_form_methods(self):- form_methods = [- [method, method] for method in self.request_class.valid_form_methods- ]- form_methods.append(["UNKNOWN", "GET"])-- for method, expected in form_methods:- response = _buildresponse(- f'<form action="post.php" method="{method}">'- '<input type="hidden" name="one" value="1">'- "</form>"- )- r = self.request_class.from_response(response)- assert r.method == expected+ @pytest.mark.parametrize(+ ("method", "expected"),+ [+ *((method, method) for method in FormRequest.valid_form_methods),+ ("UNKNOWN", "GET"),+ ],+ )+ def test_from_response_valid_form_methods(self, method: str, expected: str) -> None:+ response = _buildresponse(+ f'<form action="post.php" method="{method}">'+ '<input type="hidden" name="one" value="1">'+ "</form>"+ )+ r = self.request_class.from_response(response)+ assert r.method == expecteddef test_form_response_with_invalid_formdata_type_error(self):"""Test that a ValueError is raised for non-iterable and non-dict formdata input"""diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex 31634d3d5..4f5845095 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -539,17 +539,16 @@ class TestTextResponse(TestResponseBase):)assert response.json() == {"message": "café"}- def test_cache_json_response(self):- json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]- for json_body in json_valid_bodies:- json_response = self.response_class(- "http://www.example.com", body=json_body- )+ @pytest.mark.parametrize(+ "json_body", [b"""{"ip": "109.187.217.200"}""", b"""null"""]+ )+ def test_cache_json_response(self, json_body: bytes) -> None:+ json_response = self.response_class("http://www.example.com", body=json_body)- with mock.patch("json.loads") as mock_json:- for _ in range(2):- json_response.json()- mock_json.assert_called_once_with(json_body)+ with mock.patch("json.loads") as mock_json:+ for _ in range(2):+ json_response.json()+ mock_json.assert_called_once_with(json_body)class TestHtmlResponse(TestTextResponse):diff --git a/tests/test_poet.py b/tests/test_poet.pyindex c85063a78..6ea31c6c3 100644--- a/tests/test_poet.py+++ b/tests/test_poet.py@@ -2,21 +2,29 @@from __future__ import annotations-from typing import get_type_hints+from typing import TYPE_CHECKING, Any, get_type_hints++import pytestfrom scrapy import Spiderfrom scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider+if TYPE_CHECKING:+ from collections.abc import Callable-def test_callbacks() -> None:- """Making sure annotations on all non-abstract callbacks can be resolved."""- for cb in [+ "cb",+ [Spider._parse,CrawlSpider._parse,CrawlSpider._callback,XMLFeedSpider._parse,CSVFeedSpider._parse,SitemapSpider._parse_sitemap,- ]:- get_type_hints(cb)+ ],+ ids=lambda cb: cb.__qualname__,+)+def test_callbacks(cb: Callable[..., Any]) -> None:+ """Making sure annotations on all non-abstract callbacks can be resolved."""+ get_type_hints(cb)diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex b0297a12b..dd4adbe7e 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -3,6 +3,8 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING+import pytest+from scrapy import Request, signalsfrom scrapy.http.response import Responsefrom scrapy.utils.test import get_crawler@@ -10,8 +12,6 @@ from tests.spiders import SingleRequestSpiderfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:- import pytest-from tests.mockserver.http import MockServerOVERRIDDEN_URL = "https://example.org"@@ -79,17 +79,17 @@ class TestCrawl:response = crawler.spider.meta["responses"][0]assert response.request.url == url+ @pytest.mark.parametrize("status", [404, 500])@coroutine_test- async def test_response_error(self, mockserver: MockServer) -> None:- for status in ("404", "500"):- url = mockserver.url(f"/status?n={status}")- crawler = get_crawler(SingleRequestSpider)- await crawler.crawl_async(seed=url, mockserver=mockserver)- assert isinstance(crawler.spider, SingleRequestSpider)- failure = crawler.spider.meta["failure"]- response = failure.value.response- assert failure.request.url == url- assert response.request.url == url+ async def test_response_error(self, status: int, mockserver: MockServer) -> None:+ url = mockserver.url(f"/status?n={status}")+ crawler = get_crawler(SingleRequestSpider)+ await crawler.crawl_async(seed=url, mockserver=mockserver)+ assert isinstance(crawler.spider, SingleRequestSpider)+ failure = crawler.spider.meta["failure"]+ response = failure.value.response+ assert failure.request.url == url+ assert response.request.url == url@coroutine_testasync def test_downloader_middleware_raise_exception(diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex fa37cfe03..7f6da21e5 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -1,5 +1,7 @@from typing import Any+import pytest+from scrapy.http import (Headers,HtmlResponse,@@ -11,44 +13,58 @@ from scrapy.http import (from scrapy.responsetypes import responsetypes-def test_from_filename() -> None:- mappings = [+ ("source", "cls"),+ [("data.bin", Response),("file.txt", TextResponse),("file.xml.gz", Response),("file.xml", XmlResponse),("file.html", HtmlResponse),("file.unknownext", Response),- ]- for source, cls in mappings:- retcls = responsetypes.from_filename(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"---def test_from_content_disposition() -> None:- mappings = [- (b'attachment; filename="data.xml"', XmlResponse),- (b"attachment; filename=data.xml", XmlResponse),- ("attachment;filename=data£.tar.gz".encode(), Response),- ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),- ("attachment;filename=data高.doc".encode("gbk"), Response),- ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),- (+ ],+)+def test_from_filename(source: str, cls: type[Response]) -> None:+ assert responsetypes.from_filename(source) is cls+++ ("source", "cls"),+ [+ pytest.param(b'attachment; filename="data.xml"', XmlResponse, id="quoted"),+ pytest.param(b"attachment; filename=data.xml", XmlResponse, id="unquoted"),+ pytest.param("attachment;filename=data£.tar.gz".encode(), Response, id="utf-8"),+ pytest.param(+ "attachment;filename=dataµ.tar.gz".encode("latin-1"),+ Response,+ id="latin-1",+ ),+ pytest.param(+ "attachment;filename=data高.doc".encode("gbk"), Response, id="gbk"+ ),+ pytest.param(+ "attachment;filename=دورهdata.html".encode("cp720"),+ HtmlResponse,+ id="cp720",+ ),+ pytest.param("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),XmlResponse,+ id="iso2022_jp",),- ]- for source, cls in mappings:- retcls = responsetypes.from_content_disposition(source)- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"+ ],+)+def test_from_content_disposition(source: bytes, cls: type[Response]) -> None:+ assert responsetypes.from_content_disposition(source) is clsdef test_from_content_disposition_no_filename() -> None:assert responsetypes.from_content_disposition(b"attachment") is Response-def test_from_content_type() -> None:- mappings: list[tuple[str | bytes, type[Response]]] = [+ ("source", "cls"),+ [("text/html; charset=UTF-8", HtmlResponse),("text/xml; charset=UTF-8", XmlResponse),("application/xhtml+xml; charset=UTF-8", HtmlResponse),@@ -59,63 +75,74 @@ def test_from_content_type() -> None:("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),(b"application/x-download; filename=\x80dummy.txt", Response),- ]- for source, cls in mappings:- retcls = responsetypes.from_content_type(source)- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"+ ],+)+def test_from_content_type(source: str | bytes, cls: type[Response]) -> None:+ assert responsetypes.from_content_type(source) is cls-def test_from_body() -> None:- mappings = [+ ("source", "cls"),+ [(b"\x03\x02\xdf\xdd\x23", Response),(b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),(b"<html><head><title>Hello</title></head>", HtmlResponse),# https://codersblock.com/blog/the-smallest-valid-html5-page/(b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),(b'<?xml version="1.0" encoding="utf-8"', XmlResponse),- ]- for source, cls in mappings:- retcls = responsetypes.from_body(source)- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"+ ],+)+def test_from_body(source: bytes, cls: type[Response]) -> None:+ assert responsetypes.from_body(source) is cls-def test_from_headers() -> None:- mappings = [- ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),- (+ ("source", "cls"),+ [+ pytest.param(+ {"Content-Type": ["text/html; charset=utf-8"]},+ HtmlResponse,+ id="content-type",+ ),+ pytest.param({"Content-Type": ["text/html; charset=utf-8"],"Content-Encoding": ["gzip"],},Response,+ id="content-encoding",),- (+ pytest.param({"Content-Type": ["application/octet-stream"],"Content-Disposition": ["attachment; filename=data.txt"],},TextResponse,+ id="content-disposition",),- ]- for source, cls in mappings:- source = Headers(source)- retcls = responsetypes.from_headers(source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"+ ],+)+def test_from_headers(source: dict[str, list[str]], cls: type[Response]) -> None:+ assert responsetypes.from_headers(Headers(source)) is cls-def test_from_args() -> None:- # TODO: add more tests that check precedence between the different arguments- mappings: list[tuple[dict[str, Any], type[Response]]] = [- ({"url": "http://www.example.com/data.csv"}, TextResponse),+# TODO: add more tests that check precedence between the different arguments+ ("source", "cls"),+ [+ pytest.param(+ {"url": "http://www.example.com/data.csv"}, TextResponse, id="url"+ ),# headers takes precedence over url- (+ pytest.param({"headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),"url": "http://www.example.com/item/",},HtmlResponse,+ id="content-type-and-url",),- (+ pytest.param({"headers": Headers({"Content-Disposition": ['attachment; filename="data.xml.gz"']}@@ -123,11 +150,12 @@ def test_from_args() -> None:"url": "http://www.example.com/page/",},Response,+ id="content-disposition-and-url",),- ]- for source, cls in mappings:- retcls = responsetypes.from_args(**source)- assert retcls is cls, f"{source} ==> {retcls} != {cls}"+ ],+)+def test_from_args(source: dict[str, Any], cls: type[Response]) -> None:+ assert responsetypes.from_args(**source) is clsdef test_custom_mime_types_loaded() -> None:diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 18940dcd2..4a175c067 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -117,7 +117,10 @@ class TestBaseSettings:assert "TEST_OPTION" in self.settings.attributesassert attr is self.settings.attributes["TEST_OPTION"]- def test_set_calls_settings_attributes_methods_on_update(self):+ @pytest.mark.parametrize("priority", [0, 10, 20])+ def test_set_calls_settings_attributes_methods_on_update(+ self, priority: int+ ) -> None:attr = SettingsAttribute("value", 10)with (mock.patch.object(attr, "__setattr__") as mock_setattr,@@ -125,12 +128,9 @@ class TestBaseSettings:):self.settings.attributes = {"TEST_OPTION": attr}- for priority in (0, 10, 20):- self.settings.set("TEST_OPTION", "othervalue", priority)- mock_set.assert_called_once_with("othervalue", priority)- assert not mock_setattr.called- mock_set.reset_mock()- mock_setattr.reset_mock()+ self.settings.set("TEST_OPTION", "othervalue", priority)+ mock_set.assert_called_once_with("othervalue", priority)+ assert not mock_setattr.calleddef test_setitem(self):settings = BaseSettings()@@ -492,22 +492,24 @@ class TestBaseSettings:== settings[setting_name])- def test_getwithbase_for_non_component_priority_dicts(self):+ @pytest.mark.parametrize(+ "base_name", sorted(NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES)+ )+ def test_getwithbase_for_non_component_priority_dicts(self, base_name: str) -> None:settings = BaseSettings()- for base_name in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES:- base_value = getattr(scrapy_default_settings, base_name)- settings[base_name] = BaseSettings(base_value)- assert len(base_value) >= 2- keys = list(base_value)- values = list(base_value.values())- override = {keys[0]: values[1]}- expected = dict(base_value)- expected[keys[0]] = values[1]- name = base_name[: -len("_BASE")]- settings[name] = BaseSettings(override)- value = settings.getwithbase(name)- assert isinstance(value, BaseSettings)- assert dict(value) == expected+ base_value = getattr(scrapy_default_settings, base_name)+ settings[base_name] = BaseSettings(base_value)+ assert len(base_value) >= 2+ keys = list(base_value)+ values = list(base_value.values())+ override = {keys[0]: values[1]}+ expected = dict(base_value)+ expected[keys[0]] = values[1]+ name = base_name[: -len("_BASE")]+ settings[name] = BaseSettings(override)+ value = settings.getwithbase(name)+ assert isinstance(value, BaseSettings)+ assert dict(value) == expecteddef test_get_component_priority_dict_with_base_warns_on_duplicate_import_paths(self, caplogdiff --git a/tests/test_spider.py b/tests/test_spider.pyindex e06201ad3..e5b743f06 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -31,7 +31,8 @@ class TestSpider(TestSpiderBase):class TestXMLFeedSpider(TestSpiderBase):spider_class = XMLFeedSpider- def test_register_namespace(self):+ @pytest.mark.parametrize("iterator", ["iternodes", "xml"])+ def test_register_namespace(self, iterator: str) -> None:body = b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"xmlns:y="http://www.example.com/schemas/extras/1.0">@@ -57,24 +58,23 @@ class TestXMLFeedSpider(TestSpiderBase):"custom": selector.xpath("other/@b:custom").getall(),}- for iterator in ("iternodes", "xml"):- spider = _XMLSpider("example", iterator=iterator)- output = list(spider._parse(response))- assert len(output) == 2, iterator- assert output == [- {- "loc": ["http://www.example.com/Special-Offers.html"],- "updated": ["2009-08-16"],- "custom": ["fuu"],- "other": ["bar"],- },- {- "loc": [],- "updated": ["2009-08-16"],- "other": ["foo"],- "custom": [],- },- ], iterator+ spider = _XMLSpider("example", iterator=iterator)+ output = list(spider._parse(response))+ assert len(output) == 2+ assert output == [+ {+ "loc": ["http://www.example.com/Special-Offers.html"],+ "updated": ["2009-08-16"],+ "custom": ["fuu"],+ "other": ["bar"],+ },+ {+ "loc": [],+ "updated": ["2009-08-16"],+ "other": ["foo"],+ "custom": [],+ },+ ]@coroutine_testasync def test_parse_node_uses_parse_item(self, mockserver: MockServer):diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.pyindex b9c4dd4e2..08352da2f 100644--- a/tests/test_spidermiddleware_base.py+++ b/tests/test_spidermiddleware_base.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, TypeAliasimport pytest@@ -13,16 +13,45 @@ from scrapy.utils.test import get_crawlerfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:+ from collections.abc import Awaitable, Callable+from scrapy.crawler import Crawler+ _Process: TypeAlias = Callable[+ [BaseSpiderMiddleware, list[Any]], Awaitable[list[Any]]+ ]+@pytest.fixturedef crawler() -> Crawler:return get_crawler(Spider)+async def _process_spider_output(+ mw: BaseSpiderMiddleware, spider_output: list[Any]+) -> list[Any]:+ return list(mw.process_spider_output(Response("data:,"), spider_output))+++async def _process_spider_output_async(+ mw: BaseSpiderMiddleware, spider_output: list[Any]+) -> list[Any]:+ return await collect_asyncgen(+ mw.process_spider_output_async(+ Response("data:,"), as_async_generator(spider_output)+ )+ )+++async def _process_start(+ mw: BaseSpiderMiddleware, spider_output: list[Any]+) -> list[Any]:+ return await collect_asyncgen(mw.process_start(as_async_generator(spider_output)))++[email protected]("process", [_process_spider_output, _process_start])@coroutine_test-async def test_trivial(crawler: Crawler) -> None:+async def test_trivial(process: _Process, crawler: Crawler) -> None:class TrivialSpiderMiddleware(BaseSpiderMiddleware):pass@@ -31,15 +60,15 @@ async def test_trivial(crawler: Crawler) -> None:assert mw.crawler is crawlertest_req = Request("data:,")spider_output = [test_req, {"foo": "bar"}]- for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output)),- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),- ]:- assert processed == [test_req, {"foo": "bar"}]+ processed = await process(mw, spider_output)+ assert processed == [test_req, {"foo": "bar"}]+ "process", [_process_spider_output, _process_spider_output_async, _process_start]+)@coroutine_test-async def test_processed_request(crawler: Crawler) -> None:+async def test_processed_request(process: _Process, crawler: Crawler) -> None:class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):def get_processed_request(self, request: Request, response: Response | None@@ -55,25 +84,18 @@ async def test_processed_request(crawler: Crawler) -> None:test_req2 = Request("data:2,")test_req3 = Request("data:3,")spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]- for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output)),- await collect_asyncgen(- mw.process_spider_output_async(- Response("data:,"), as_async_generator(spider_output)- )- ),- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),- ]:- assert len(processed) == 3- assert isinstance(processed[0], Request)- assert processed[0].url == "data:1,"- assert processed[1] == {"foo": "bar"}- assert isinstance(processed[2], Request)- assert processed[2].url == "data:30,"+ processed = await process(mw, spider_output)+ assert len(processed) == 3+ assert isinstance(processed[0], Request)+ assert processed[0].url == "data:1,"+ assert processed[1] == {"foo": "bar"}+ assert isinstance(processed[2], Request)+ assert processed[2].url == "data:30,"[email protected]("process", [_process_spider_output, _process_start])@coroutine_test-async def test_processed_item(crawler: Crawler) -> None:+async def test_processed_item(process: _Process, crawler: Crawler) -> None:class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):def get_processed_item(self, item: Any, response: Response | None) -> Any:if item["foo"] == 2:@@ -85,15 +107,13 @@ async def test_processed_item(crawler: Crawler) -> None:mw = build_from_crawler(ProcessItemSpiderMiddleware, crawler)test_req = Request("data:,")spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]- for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output)),- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),- ]:- assert processed == [{"foo": 1}, test_req, {"foo": 30}]+ processed = await process(mw, spider_output)+ assert processed == [{"foo": 1}, test_req, {"foo": 30}][email protected]("process", [_process_spider_output, _process_start])@coroutine_test-async def test_processed_both(crawler: Crawler) -> None:+async def test_processed_both(process: _Process, crawler: Crawler) -> None:class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):def get_processed_request(self, request: Request, response: Response | None@@ -123,14 +143,11 @@ async def test_processed_both(crawler: Crawler) -> None:{"foo": 3},test_req3,]- for processed in [- list(mw.process_spider_output(Response("data:,"), spider_output)),- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),- ]:- assert len(processed) == 4- assert isinstance(processed[0], Request)- assert processed[0].url == "data:1,"- assert processed[1] == {"foo": 1}- assert processed[2] == {"foo": 30}- assert isinstance(processed[3], Request)- assert processed[3].url == "data:30,"+ processed = await process(mw, spider_output)+ assert len(processed) == 4+ assert isinstance(processed[0], Request)+ assert processed[0].url == "data:1,"+ assert processed[1] == {"foo": 1}+ assert processed[2] == {"foo": 30}+ assert isinstance(processed[3], Request)+ assert processed[3].url == "data:30,"diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 958b00ae0..fe13f750b 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -633,6 +633,20 @@ POLICY_CASES: list[tuple[str | None, str, ScenarioTable]] = []+# Each policy, as (REFERRER_POLICY name, policy class).+POLICY_CLASSES: list[tuple[str, type[ReferrerPolicy]]] = [+ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),+ (POLICY_NO_REFERRER, NoReferrerPolicy),+ (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),+ (POLICY_SAME_ORIGIN, SameOriginPolicy),+ (POLICY_ORIGIN, OriginPolicy),+ (POLICY_STRICT_ORIGIN, StrictOriginPolicy),+ (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),+ (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),+ (POLICY_UNSAFE_URL, UnsafeUrlPolicy),+]++def assert_scenarii(scenarii: ScenarioTable,*,@@ -792,116 +806,107 @@ def test_policy_precedence(class TestRequestMetaSettingFallback:- params = [+ @pytest.mark.parametrize((+ "settings",+ "response_headers",+ "request_meta",+ "policy_class",+ "check_warning",+ ),+ [# When an unknown policy is referenced in Request.meta# (here, a typo error),# the policy defined in settings takes precedence- {- "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"- },- {},- {"referrer_policy": "ssscrapy-default"},- OriginWhenCrossOriginPolicy,- True,- ),- (+ pytest.param(+ {+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"+ },+ {},+ {"referrer_policy": "ssscrapy-default"},+ OriginWhenCrossOriginPolicy,+ True,+ id="unknown-meta-settings-path",+ ),# same as above but with string value for settings policy- {"REFERRER_POLICY": "origin-when-cross-origin"},- {},- {"referrer_policy": "ssscrapy-default"},- OriginWhenCrossOriginPolicy,- True,- ),- (+ pytest.param(+ {"REFERRER_POLICY": "origin-when-cross-origin"},+ {},+ {"referrer_policy": "ssscrapy-default"},+ OriginWhenCrossOriginPolicy,+ True,+ id="unknown-meta-settings-name",+ ),# request meta references a wrong policy but it is set,# so the Referrer-Policy header in response is not used,# and the settings' policy is applied- {"REFERRER_POLICY": "origin-when-cross-origin"},- {"Referrer-Policy": "unsafe-url"},- {"referrer_policy": "ssscrapy-default"},- OriginWhenCrossOriginPolicy,- True,- ),- (+ pytest.param(+ {"REFERRER_POLICY": "origin-when-cross-origin"},+ {"Referrer-Policy": "unsafe-url"},+ {"referrer_policy": "ssscrapy-default"},+ OriginWhenCrossOriginPolicy,+ True,+ id="unknown-meta-header",+ ),# here, request meta does not set the policy# so response headers take precedence- {"REFERRER_POLICY": "origin-when-cross-origin"},- {"Referrer-Policy": "unsafe-url"},- {},- UnsafeUrlPolicy,- False,- ),- (+ pytest.param(+ {"REFERRER_POLICY": "origin-when-cross-origin"},+ {"Referrer-Policy": "unsafe-url"},+ {},+ UnsafeUrlPolicy,+ False,+ id="no-meta-header",+ ),# here, request meta does not set the policy,# but response headers also use an unknown policy,# so the settings' policy is used- {"REFERRER_POLICY": "origin-when-cross-origin"},- {"Referrer-Policy": "unknown"},- {},- OriginWhenCrossOriginPolicy,- True,- ),- ]-- def test(self):+ pytest.param(+ {"REFERRER_POLICY": "origin-when-cross-origin"},+ {"Referrer-Policy": "unknown"},+ {},+ OriginWhenCrossOriginPolicy,+ True,+ id="no-meta-unknown-header",+ ),+ ],+ )+ def test(+ self,+ settings: dict[str, Any],+ response_headers: dict[str, str],+ request_meta: dict[str, Any],+ policy_class: type[ReferrerPolicy],+ check_warning: bool,+ ) -> None:origin = "http://www.scrapy.org"target = "http://www.example.com"- for (- settings,- response_headers,- request_meta,- policy_class,- check_warning,- ) in self.params:- mw = RefererMiddleware(Settings(settings))-- response = Response(origin, headers=response_headers)- request = Request(target, meta=request_meta)-- if check_warning:- with pytest.warns(- RuntimeWarning, match="Could not load referrer policy"- ):- policy = mw.policy(response, request)- else:+ mw = RefererMiddleware(Settings(settings))++ response = Response(origin, headers=response_headers)+ request = Request(target, meta=request_meta)++ if check_warning:+ with pytest.warns(RuntimeWarning, match="Could not load referrer policy"):policy = mw.policy(response, request)- assert isinstance(policy, policy_class)+ else:+ policy = mw.policy(response, request)+ assert isinstance(policy, policy_class)class TestSettingsPolicyByName:- def test_valid_name(self):- for s, p in [- (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),- (POLICY_NO_REFERRER, NoReferrerPolicy),- (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),- (POLICY_SAME_ORIGIN, SameOriginPolicy),- (POLICY_ORIGIN, OriginPolicy),- (POLICY_STRICT_ORIGIN, StrictOriginPolicy),- (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),- (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),- (POLICY_UNSAFE_URL, UnsafeUrlPolicy),- ]:- settings = Settings({"REFERRER_POLICY": s})- mw = RefererMiddleware(settings)- assert mw.default_policy == p-- def test_valid_name_casevariants(self):- for s, p in [- (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),- (POLICY_NO_REFERRER, NoReferrerPolicy),- (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),- (POLICY_SAME_ORIGIN, SameOriginPolicy),- (POLICY_ORIGIN, OriginPolicy),- (POLICY_STRICT_ORIGIN, StrictOriginPolicy),- (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),- (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),- (POLICY_UNSAFE_URL, UnsafeUrlPolicy),- ]:- settings = Settings({"REFERRER_POLICY": s.upper()})- mw = RefererMiddleware(settings)- assert mw.default_policy == p+ @pytest.mark.parametrize(("name", "policy_class"), POLICY_CLASSES)+ def test_valid_name(self, name: str, policy_class: type[ReferrerPolicy]) -> None:+ mw = RefererMiddleware(Settings({"REFERRER_POLICY": name}))+ assert mw.default_policy == policy_class++ @pytest.mark.parametrize(("name", "policy_class"), POLICY_CLASSES)+ def test_valid_name_casevariants(+ self, name: str, policy_class: type[ReferrerPolicy]+ ) -> None:+ mw = RefererMiddleware(Settings({"REFERRER_POLICY": name.upper()}))+ assert mw.default_policy == policy_classdef test_invalid_name(self):settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex f9be16ba0..d2deed61c 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -77,43 +77,43 @@ class TestParallelAsyncio:await asyncio.sleep(random.random() / 20)yield i+ @pytest.mark.parametrize("length", [20, 50, 100])@coroutine_test- async def test_simple(self):- for length in [20, 50, 100]:- parallel_count = [0]- max_parallel_count = [0]- results: list[int] = []- ait = self.get_async_iterable(length)- await _parallel_asyncio(- ait,- self.CONCURRENT_ITEMS,- self.callable_wrapped,- results,- parallel_count,- max_parallel_count,- )- assert list(range(length)) == sorted(results)- assert parallel_count[0] == 0- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS+ async def test_simple(self, length: int) -> None:+ parallel_count = [0]+ max_parallel_count = [0]+ results: list[int] = []+ ait = self.get_async_iterable(length)+ await _parallel_asyncio(+ ait,+ self.CONCURRENT_ITEMS,+ self.callable_wrapped,+ results,+ parallel_count,+ max_parallel_count,+ )+ assert list(range(length)) == sorted(results)+ assert parallel_count[0] == 0+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS+ @pytest.mark.parametrize("length", [20, 50, 100])@coroutine_test- async def test_delays(self):- for length in [20, 50, 100]:- parallel_count = [0]- max_parallel_count = [0]- results: list[int] = []- ait = self.get_async_iterable_with_delays(length)- await _parallel_asyncio(- ait,- self.CONCURRENT_ITEMS,- self.callable_wrapped,- results,- parallel_count,- max_parallel_count,- )- assert list(range(length)) == sorted(results)- assert parallel_count[0] == 0- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS+ async def test_delays(self, length: int) -> None:+ parallel_count = [0]+ max_parallel_count = [0]+ results: list[int] = []+ ait = self.get_async_iterable_with_delays(length)+ await _parallel_asyncio(+ ait,+ self.CONCURRENT_ITEMS,+ self.callable_wrapped,+ results,+ parallel_count,+ max_parallel_count,+ )+ assert list(range(length)) == sorted(results)+ assert parallel_count[0] == 0+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS@coroutine_testasync def test_count_higher_than_work(self):diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 26096911d..5dcc59886 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -1,7 +1,8 @@from __future__ import annotationsimport copy-from collections.abc import Iterator, Mapping, MutableMapping+from collections.abc import Callable, Iterator, Mapping, MutableMapping+from operator import methodcallerfrom typing import Anyimport pytest@@ -194,14 +195,19 @@ class TestCaseInsensitiveDict:assert h1.get("header1") == h3.get("header1")assert h1.get("header1") == h3.get("HEADER1")- def test_copy_is_independent(self):+ @pytest.mark.parametrize(+ "copier", [copy.copy, methodcaller("copy")], ids=["copy.copy", "copy-method"]+ )+ def test_copy_is_independent(+ self, copier: Callable[[CaseInsensitiveDict], CaseInsensitiveDict]+ ) -> None:h1 = self.dict_class({"header1": "value1", "header2": "value2"})- for h2 in (copy.copy(h1), h1.copy()):- del h2["header1"]- h2["header3"] = "value3"- assert "header1" in h1- assert "header3" not in h1- assert dict(h1) == {"header1": "value1", "header2": "value2"}+ h2 = copier(h1)+ del h2["header1"]+ h2["header3"] = "value3"+ assert "header1" in h1+ assert "header3" not in h1+ assert dict(h1) == {"header1": "value1", "header2": "value2"}def test_repr(self):d1 = self.dict_class({"foo": "bar"})@@ -215,14 +221,18 @@ class TestCaseInsensitiveDict:assert isinstance(iterkeys, Iterator)assert list(iterkeys) == ["AsDf", "FoO"]- def test_copy_keeps_values(self):+ @pytest.mark.parametrize(+ "copier", [copy.copy, methodcaller("copy")], ids=["copy.copy", "copy-method"]+ )+ def test_copy_keeps_values(+ self, copier: Callable[[CaseInsensitiveDict], CaseInsensitiveDict]+ ) -> None:class MyDict(self.dict_class): # type: ignore[misc,name-defined]def _normvalue(self, value):return value + 1d = MyDict({"key": 1})- for copied in (copy.copy(d), d.copy()):- assert copied["key"] == 2+ assert copier(d)["key"] == 2def test_ior(self):d = self.dict_class({"header1": "value1"})diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 4143f3888..5a277d08a 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -258,45 +258,45 @@ class TestParallelAsync:await maybe_deferred_to_future(dfd)yield i+ @pytest.mark.parametrize("length", [20, 50, 100])@inline_callbacks_test- def test_simple(self):- for length in [20, 50, 100]:- parallel_count = [0]- max_parallel_count = [0]- results: list[int] = []- ait = self.get_async_iterable(length)- dl = parallel_async(- ait,- self.CONCURRENT_ITEMS,- self.callable_wrapped,- results,- parallel_count,- max_parallel_count,- )- yield dl- assert list(range(length)) == sorted(results)- assert parallel_count[0] == 0- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]-+ def test_simple(self, length: int) -> Generator[Deferred[Any], Any, None]:+ parallel_count = [0]+ max_parallel_count = [0]+ results: list[int] = []+ ait = self.get_async_iterable(length)+ dl = parallel_async(+ ait,+ self.CONCURRENT_ITEMS,+ self.callable_wrapped,+ results,+ parallel_count,+ max_parallel_count,+ )+ yield dl+ assert list(range(length)) == sorted(results)+ assert parallel_count[0] == 0+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]++ @pytest.mark.parametrize("length", [20, 50, 100])@inline_callbacks_test- def test_delays(self):- for length in [20, 50, 100]:- parallel_count = [0]- max_parallel_count = [0]- results: list[int] = []- ait = self.get_async_iterable_with_delays(length)- dl = parallel_async(- ait,- self.CONCURRENT_ITEMS,- self.callable_wrapped,- results,- parallel_count,- max_parallel_count,- )- yield dl- assert list(range(length)) == sorted(results)- assert parallel_count[0] == 0- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]+ def test_delays(self, length: int) -> Generator[Deferred[Any], Any, None]:+ parallel_count = [0]+ max_parallel_count = [0]+ results: list[int] = []+ ait = self.get_async_iterable_with_delays(length)+ dl = parallel_async(+ ait,+ self.CONCURRENT_ITEMS,+ self.callable_wrapped,+ results,+ parallel_count,+ max_parallel_count,+ )+ yield dl+ assert list(range(length)) == sorted(results)+ assert parallel_count[0] == 0+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]@inline_callbacks_testdef test_count_higher_than_work(self) -> Generator[Deferred[Any], Any, None]:diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 2c1913669..0e4c520d5 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -80,9 +80,13 @@ class TestUpdateClassPath:output = update_classpath("scrapy.unmatched.Path")assert output == "scrapy.unmatched.Path"- def test_returns_nonstring(self):- for notastring in [None, True, [1, 2, 3], object()]:- assert update_classpath(notastring) == notastring+ @pytest.mark.parametrize(+ "notastring",+ [None, True, [1, 2, 3], object()],+ ids=["None", "bool", "list", "object"],+ )+ def test_returns_nonstring(self, notastring: object) -> None:+ assert update_classpath(notastring) == notastringclass TestAttribute:diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.pyindex abb733f1f..29518c7f6 100644--- a/tests/test_utils_serialize.py+++ b/tests/test_utils_serialize.py@@ -4,6 +4,7 @@ import dataclassesimport datetimeimport jsonfrom decimal import Decimal+from typing import Anyimport attrimport pytest@@ -18,39 +19,43 @@ class TestJsonEncoder:def encoder(self) -> ScrapyJSONEncoder:return ScrapyJSONEncoder(sort_keys=True)- def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:- dt_naive = datetime.datetime(2010, 1, 2, 10, 11, 12) # noqa: DTZ001- dt_naives = "2010-01-02T10:11:12"- dt_aware = datetime.datetime(- 2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.UTC- )- dt_awares = "2010-01-02T10:11:12.133700+00:00"- d = datetime.date(2010, 1, 2)- ds = "2010-01-02"- t = datetime.time(10, 11, 12)- ts = "10:11:12"- t_us = datetime.time(10, 11, 12, 133700)- t_uss = "10:11:12.133700"- dec = Decimal("1000.12")- decs = "1000.12"- s = {"foo"}- ss = ["foo"]- dt_set = {dt_naive}- dt_sets = [dt_naives]-- for input_, output in [- ("foo", "foo"),- (d, ds),- (t, ts),- (t_us, t_uss),- (dt_naive, dt_naives),- (dt_aware, dt_awares),- (dec, decs),- (["foo", d], ["foo", ds]),- (s, ss),- (dt_set, dt_sets),- ]:- assert encoder.encode(input_) == json.dumps(output, sort_keys=True)+ @pytest.mark.parametrize(+ ("input_", "output"),+ [+ pytest.param("foo", "foo", id="str"),+ pytest.param(datetime.date(2010, 1, 2), "2010-01-02", id="date"),+ pytest.param(datetime.time(10, 11, 12), "10:11:12", id="time"),+ pytest.param(+ datetime.time(10, 11, 12, 133700),+ "10:11:12.133700",+ id="time-microseconds",+ ),+ pytest.param(+ datetime.datetime(2010, 1, 2, 10, 11, 12), # noqa: DTZ001+ "2010-01-02T10:11:12",+ id="datetime-naive",+ ),+ pytest.param(+ datetime.datetime(2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.UTC),+ "2010-01-02T10:11:12.133700+00:00",+ id="datetime-aware",+ ),+ pytest.param(Decimal("1000.12"), "1000.12", id="decimal"),+ pytest.param(+ ["foo", datetime.date(2010, 1, 2)], ["foo", "2010-01-02"], id="list"+ ),+ pytest.param({"foo"}, ["foo"], id="set"),+ pytest.param(+ {datetime.datetime(2010, 1, 2, 10, 11, 12)}, # noqa: DTZ001+ ["2010-01-02T10:11:12"],+ id="datetime-set",+ ),+ ],+ )+ def test_encode_decode(+ self, input_: Any, output: Any, encoder: ScrapyJSONEncoder+ ) -> None:+ assert encoder.encode(input_) == json.dumps(output, sort_keys=True)def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:assert "Deferred" in encoder.encode(defer.Deferred())