Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Convert test loops to parametrize(). (#8311)

Andrey Rakhmatulline6689aaba804
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index e2caa5dfb..d22b007ae 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst
from __future__ import annotations
import warnings
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
from scrapy.exceptions import NotSupported, ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
@@ -43,7 +43,7 @@ class XMLFeedSpider(Spider):
defining :meth:`~scrapy.Spider.parse` is not enough.
"""
- iterator: str = "iternodes"
+ iterator: Literal["iternodes", "html", "xml"] = "iternodes"
"""The iterator to use. It can be either:
- ``'iternodes'`` - a fast iterator based on ``lxml``
diff --git a/tests/test_commands.py b/tests/test_commands.py
index db07a3d4d..b0b10b7bc 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -332,26 +332,29 @@ class MySpider(scrapy.Spider):
assert msg in err
assert "no running event loop" in err
- def test_project_settings(self, proj_path: Path) -> None:
+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])
+ def test_project_settings(self, spider: str, proj_path: Path) -> None:
"""The reactor is set via the project default settings (to the asyncio value).
AsyncCrawlerProcess, the asyncio reactor, both spiders work."""
- for spider in ["sp", "aiosp"]:
- self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)
+ self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)
- def test_cmdline_asyncio(self, proj_path: Path) -> None:
+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])
+ def test_cmdline_asyncio(self, spider: str, proj_path: Path) -> None:
"""The reactor is set via the command line to the asyncio value.
AsyncCrawlerProcess, the asyncio reactor, both spiders work."""
- for spider in ["sp", "aiosp"]:
- self._assert_spider_works(
- self.ASYNC_MSG,
- proj_path,
- spider,
- "-s",
- f"TWISTED_REACTOR={_asyncio_reactor_path}",
- )
+ self._assert_spider_works(
+ self.ASYNC_MSG,
+ proj_path,
+ spider,
+ "-s",
+ f"TWISTED_REACTOR={_asyncio_reactor_path}",
+ )
- def test_project_settings_explicit_asyncio(self, proj_path: Path) -> None:
+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])
+ def test_project_settings_explicit_asyncio(
+ self, spider: str, proj_path: Path
+ ) -> None:
"""The reactor explicitly is set via the project settings to the asyncio value.
AsyncCrawlerProcess, the asyncio reactor, both spiders work."""
@@ -360,8 +363,7 @@ class MySpider(scrapy.Spider):
f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n",
)
- for spider in ["sp", "aiosp"]:
- self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)
+ self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)
def test_cmdline_empty(self, proj_path: Path) -> None:
"""The reactor is set via the command line to the empty value.
@@ -383,17 +385,17 @@ class MySpider(scrapy.Spider):
self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp")
self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")
- def test_spider_settings_asyncio(self, proj_path: Path) -> None:
+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])
+ def test_spider_settings_asyncio(self, spider: str, proj_path: Path) -> None:
"""The reactor is set via the spider settings to the asyncio value.
AsyncCrawlerProcess, the asyncio reactor, both spiders work."""
- for spider in ["sp", "aiosp"]:
- self._replace_custom_settings(
- proj_path / self.project_name,
- spider,
- f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",
- )
- self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)
+ self._replace_custom_settings(
+ proj_path / self.project_name,
+ spider,
+ f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",
+ )
+ self._assert_spider_works(self.ASYNC_MSG, proj_path, spider)
def test_spider_settings_asyncio_cmdline_empty(self, proj_path: Path) -> None:
"""The reactor is set via the spider settings to the asyncio value
@@ -415,7 +417,10 @@ class MySpider(scrapy.Spider):
self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR="
)
- def test_project_empty_spider_settings_asyncio(self, proj_path: Path) -> None:
+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])
+ def test_project_empty_spider_settings_asyncio(
+ self, spider: str, proj_path: Path
+ ) -> None:
"""The reactor is set via the project settings to the empty value
and via the spider settings to the asyncio value. CrawlerProcess is
chosen based on the project settings, but the asyncio reactor is chosen
@@ -423,15 +428,17 @@ class MySpider(scrapy.Spider):
CrawlerProcess, the asyncio reactor, both spiders work."""
self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n")
- for spider in ["sp", "aiosp"]:
- self._replace_custom_settings(
- proj_path / self.project_name,
- spider,
- f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",
- )
- self._assert_spider_works(self.NORMAL_MSG, proj_path, spider)
+ self._replace_custom_settings(
+ proj_path / self.project_name,
+ spider,
+ f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}",
+ )
+ self._assert_spider_works(self.NORMAL_MSG, proj_path, spider)
- def test_project_asyncio_spider_settings_select(self, proj_path: Path) -> None:
+ @pytest.mark.parametrize("spider", ["sp", "aiosp"])
+ def test_project_asyncio_spider_settings_select(
+ self, spider: str, proj_path: Path
+ ) -> None:
"""The reactor is set via the project settings to the asyncio value
and via the spider settings to the select value. AsyncCrawlerProcess
is chosen based on the project settings, and the conflicting reactor
@@ -443,19 +450,18 @@ class MySpider(scrapy.Spider):
proj_path / self.project_name,
f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n",
)
- for spider in ["sp", "aiosp"]:
- self._replace_custom_settings(
- proj_path / self.project_name,
- spider,
- "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}",
- )
- _, _, err = proc(self.name, spider, cwd=proj_path)
- assert self.ASYNC_MSG in err
- assert (
- "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
- " does not match the requested one"
- " (twisted.internet.selectreactor.SelectReactor)"
- ) in err
+ self._replace_custom_settings(
+ proj_path / self.project_name,
+ spider,
+ "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}",
+ )
+ _, _, err = proc(self.name, spider, cwd=proj_path)
+ assert self.ASYNC_MSG in err
+ assert (
+ "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
+ " does not match the requested one"
+ " (twisted.internet.selectreactor.SelectReactor)"
+ ) in err
def test_project_asyncio_spider_settings_select_forced(
self, proj_path: Path
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index b5dfa937f..5415274ff 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -219,17 +219,22 @@ async def test_pool_limits(concurrency: int, expected: int | None) -> None:
await handler.close()
-@coroutine_test
-async def test_experimental_warning(caplog: pytest.LogCaptureFixture) -> None:
- class StableHandler(HttpxDownloadHandler):
- experimental = False
+class StableHandler(HttpxDownloadHandler):
+ experimental = False
- crawler = get_crawler()
- for handler_cls, expected in ((HttpxDownloadHandler, True), (StableHandler, False)):
- caplog.clear()
- handler = build_from_crawler(handler_cls, crawler)
- await handler.close()
- assert ("is experimental" in caplog.text) is expected
+
+ ("handler_cls", "expected"), [(HttpxDownloadHandler, True), (StableHandler, False)]
+)
+@coroutine_test
+async def test_experimental_warning(
+ handler_cls: type[HttpxDownloadHandler],
+ expected: bool,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ handler = build_from_crawler(handler_cls, get_crawler())
+ await handler.close()
+ assert ("is experimental" in caplog.text) is expected
@coroutine_test
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 976436e83..d96978a6a 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -528,7 +528,10 @@ class TestHttpCompression:
assert not thread.is_alive(), "decompression did not terminate"
assert result[0].body == plain
- def test_deflate_chunked_output_leaves_nothing_to_flush(self) -> None:
+ @pytest.mark.parametrize(
+ "size", [_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE]
+ )
+ def test_deflate_chunked_output_leaves_nothing_to_flush(self, size: int) -> None:
"""Guard the zlib behaviour that lets ``_inflate()`` skip ``flush()``.
``decompress(max_length=...)`` leaves the input it could not turn into
@@ -536,14 +539,13 @@ class TestHttpCompression:
stream has been fully emitted. If a future zlib buffers output instead,
this fails and ``_inflate()`` needs to flush the remainder again.
"""
- for size in (_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE):
- decompressor = zlib.decompressobj()
- decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)
- while decompressor.unconsumed_tail and not decompressor.eof:
- decompressor.decompress(
- decompressor.unconsumed_tail, max_length=_CHUNK_SIZE
- )
- assert decompressor.flush() == b"", f"pending output for size {size}"
+ decompressor = zlib.decompressobj()
+ decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)
+ while decompressor.unconsumed_tail and not decompressor.eof:
+ decompressor.decompress(
+ decompressor.unconsumed_tail, max_length=_CHUNK_SIZE
+ )
+ assert decompressor.flush() == b""
async def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
@@ -864,13 +866,13 @@ class TestHttpCompression:
resp = await self._get_truncated_response("gzip")
assert resp.body.startswith(b"<!DOCTYPE")
+ @pytest.mark.parametrize(
+ "check_key", [key for key in FORMAT if key.startswith("zstd-")]
+ )
@coroutine_test
- async def test_process_truncated_response_zstd(self):
- for check_key in FORMAT:
- if not check_key.startswith("zstd-"):
- continue
- resp = await self._get_truncated_response(check_key)
- assert len(resp.body) == 0
+ async def test_process_truncated_response_zstd(self, check_key: str) -> None:
+ resp = await self._get_truncated_response(check_key)
+ assert len(resp.body) == 0
@pytest.mark.parametrize(
("body", "encoding"),
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index d4f6c4796..3372813f9 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -18,35 +18,40 @@ class TestHttpProxyMiddleware:
with pytest.raises(NotConfigured):
build_from_crawler(HttpProxyMiddleware, crawler)
- def test_no_environment_proxies(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ @pytest.mark.parametrize("url", ["http://e.com", "https://e.com", "file:///tmp/a"])
+ def test_no_environment_proxies(
+ self, url: str, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
for name in list(os.environ):
monkeypatch.delenv(name)
monkeypatch.setenv("dummy_proxy", "reset_env_and_do_not_raise")
mw = HttpProxyMiddleware()
- for url in ("http://e.com", "https://e.com", "file:///tmp/a"):
- req = Request(url)
- assert mw.process_request(req) is None
- assert req.url == url
- assert req.meta == {}
-
- def test_environment_proxies(self, monkeypatch: pytest.MonkeyPatch) -> None:
- http_proxy = "https://proxy.for.http:3128"
- https_proxy = "http://proxy.for.https:8080"
- monkeypatch.setenv("http_proxy", http_proxy)
- monkeypatch.setenv("https_proxy", https_proxy)
+ req = Request(url)
+ assert mw.process_request(req) is None
+ assert req.url == url
+ assert req.meta == {}
+
+ @pytest.mark.parametrize(
+ ("url", "proxy"),
+ [
+ ("http://e.com", "https://proxy.for.http:3128"),
+ ("https://e.com", "http://proxy.for.https:8080"),
+ ("file://tmp/a", None),
+ ],
+ )
+ def test_environment_proxies(
+ self, url: str, proxy: str | None, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setenv("http_proxy", "https://proxy.for.http:3128")
+ monkeypatch.setenv("https_proxy", "http://proxy.for.https:8080")
monkeypatch.delenv("file_proxy", raising=False)
mw = HttpProxyMiddleware()
- for url, proxy in [
- ("http://e.com", http_proxy),
- ("https://e.com", https_proxy),
- ("file://tmp/a", None),
- ]:
- req = Request(url)
- assert mw.process_request(req) is None
- assert req.url == url
- assert req.meta.get("proxy") == proxy
+ req = Request(url)
+ assert mw.process_request(req) is None
+ assert req.url == url
+ assert req.meta.get("proxy") == proxy
def test_proxy_precedence_meta(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("http_proxy", "https://proxy.com")
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 29b301b7d..2a8e936a8 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -80,23 +80,22 @@ class ErrorCloseReasonSpider(ChangeCloseReasonSpider):
class TestEngine(TestEngineBase):
+ @pytest.mark.parametrize(
+ "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]
+ )
@coroutine_test
- async def test_crawler(self, mockserver: MockServer) -> None:
- for spider in (
- MySpider,
- DictItemsSpider,
- AttrsItemsSpider,
- DataClassItemsSpider,
- ):
- run = CrawlerRun(spider)
- await run.run(mockserver)
- self._assert_visited_urls(run)
- self._assert_scheduled_requests(run, count=9)
- self._assert_downloaded_responses(run, count=9)
- self._assert_scraped_items(run)
- self._assert_signals_caught(run)
- self._assert_headers_received(run)
- self._assert_bytes_received(run)
+ async def test_crawler(
+ self, spider: type[MySpider], mockserver: MockServer
+ ) -> None:
+ run = CrawlerRun(spider)
+ await run.run(mockserver)
+ self._assert_visited_urls(run)
+ self._assert_scheduled_requests(run, count=9)
+ self._assert_downloaded_responses(run, count=9)
+ self._assert_scraped_items(run)
+ self._assert_signals_caught(run)
+ self._assert_headers_received(run)
+ self._assert_bytes_received(run)
@coroutine_test
async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 27f282af4..b20ffd8d2 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -2,6 +2,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
+import pytest
+
from scrapy.exceptions import StopDownload
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test
@@ -14,8 +16,6 @@ from tests.utils.engine import (
)
if TYPE_CHECKING:
- import pytest
-
from tests.mockserver.http import MockServer
@@ -26,30 +26,30 @@ class BytesReceivedCrawlerRun(CrawlerRun):
class TestBytesReceivedEngine(TestEngineBase):
+ @pytest.mark.parametrize(
+ "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]
+ )
@coroutine_test
async def test_crawler(
- self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
+ self,
+ spider: type[MySpider],
+ mockserver: MockServer,
+ caplog: pytest.LogCaptureFixture,
) -> None:
- for spider in (
- MySpider,
- DictItemsSpider,
- AttrsItemsSpider,
- DataClassItemsSpider,
- ):
- run = BytesReceivedCrawlerRun(spider)
- with caplog.at_level("DEBUG"):
- await run.run(mockserver)
- for url in ("/redirected", "/static/", "/numbers"):
- assert (
- f"Download stopped for <GET {mockserver.url(url)}> "
- "from signal handler BytesReceivedCrawlerRun.bytes_received"
- ) in caplog.text
- self._assert_visited_urls(run)
- self._assert_scheduled_requests(run, count=9)
- self._assert_downloaded_responses(run, count=9)
- self._assert_signals_caught(run)
- self._assert_headers_received(run)
- self._assert_bytes_received(run)
+ run = BytesReceivedCrawlerRun(spider)
+ with caplog.at_level("DEBUG"):
+ await run.run(mockserver)
+ for url in ("/redirected", "/static/", "/numbers"):
+ assert (
+ f"Download stopped for <GET {mockserver.url(url)}> "
+ "from signal handler BytesReceivedCrawlerRun.bytes_received"
+ ) in caplog.text
+ self._assert_visited_urls(run)
+ self._assert_scheduled_requests(run, count=9)
+ self._assert_downloaded_responses(run, count=9)
+ self._assert_signals_caught(run)
+ self._assert_headers_received(run)
+ self._assert_bytes_received(run)
@staticmethod
def _assert_bytes_received(run: CrawlerRun) -> None:
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index 09fa14326..0dae511a0 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -2,6 +2,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
+import pytest
+
from scrapy.exceptions import StopDownload
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test
@@ -14,8 +16,6 @@ from tests.utils.engine import (
)
if TYPE_CHECKING:
- import pytest
-
from tests.mockserver.http import MockServer
@@ -26,29 +26,29 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
class TestHeadersReceivedEngine(TestEngineBase):
+ @pytest.mark.parametrize(
+ "spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]
+ )
@coroutine_test
async def test_crawler(
- self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
+ self,
+ spider: type[MySpider],
+ mockserver: MockServer,
+ caplog: pytest.LogCaptureFixture,
) -> None:
- for spider in (
- MySpider,
- DictItemsSpider,
- AttrsItemsSpider,
- DataClassItemsSpider,
- ):
- run = HeadersReceivedCrawlerRun(spider)
- with caplog.at_level("DEBUG"):
- await run.run(mockserver)
- for url in ("/redirected", "/static/", "/numbers"):
- assert (
- f"Download stopped for <GET {mockserver.url(url)}> "
- "from signal handler HeadersReceivedCrawlerRun.headers_received"
- ) in caplog.text
- self._assert_visited_urls(run)
- self._assert_downloaded_responses(run, count=6)
- self._assert_signals_caught(run)
- self._assert_bytes_received(run)
- self._assert_headers_received(run)
+ run = HeadersReceivedCrawlerRun(spider)
+ with caplog.at_level("DEBUG"):
+ await run.run(mockserver)
+ for url in ("/redirected", "/static/", "/numbers"):
+ assert (
+ f"Download stopped for <GET {mockserver.url(url)}> "
+ "from signal handler HeadersReceivedCrawlerRun.headers_received"
+ ) in caplog.text
+ self._assert_visited_urls(run)
+ self._assert_downloaded_responses(run, count=6)
+ self._assert_signals_caught(run)
+ self._assert_bytes_received(run)
+ self._assert_headers_received(run)
@staticmethod
def _assert_bytes_received(run: CrawlerRun) -> None:
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 2b77c17d1..36c45ecc4 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -42,6 +42,11 @@ class CustomFieldItem(Item):
age = Field(serializer=custom_serializer)
+class FriendsItem(Item):
+ name = Field()
+ friends = Field()
+
+
@dataclasses.dataclass
class MyDataClass:
name: str
@@ -332,46 +337,43 @@ class TestCsvItemExporter(TestBaseItemExporter):
expected=b"age,name\r\n22,John\xc2\xa3\r\n",
)
- def test_header_export_single_field(self):
- for item in [self.i, ItemAdapter(self.i).asdict()]:
- self.assertExportResult(
- item=item,
- fields_to_export=["age"],
- expected=b"age\r\n22\r\n",
- )
-
- def test_header_export_two_items(self):
- for item in [self.i, ItemAdapter(self.i).asdict()]:
- output = BytesIO()
- ie = CsvItemExporter(output)
- ie.start_exporting()
- ie.export_item(item)
- ie.export_item(item)
- ie.finish_exporting()
- del ie # See the first “del self.ie” in this file for context.
- self.assertCsvEqual(
- output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n"
- )
+ @pytest.mark.parametrize("as_dict", [False, True], ids=["item", "dict"])
+ def test_header_export_single_field(self, as_dict: bool) -> None:
+ self.assertExportResult(
+ item=ItemAdapter(self.i).asdict() if as_dict else self.i,
+ fields_to_export=["age"],
+ expected=b"age\r\n22\r\n",
+ )
- def test_header_no_header_line(self):
- for item in [self.i, ItemAdapter(self.i).asdict()]:
- self.assertExportResult(
- item=item,
- include_headers_line=False,
- expected=b"22,John\xc2\xa3\r\n",
- )
+ @pytest.mark.parametrize("as_dict", [False, True], ids=["item", "dict"])
+ def test_header_export_two_items(self, as_dict: bool) -> None:
+ item = ItemAdapter(self.i).asdict() if as_dict else self.i
+ output = BytesIO()
+ ie = CsvItemExporter(output)
+ ie.start_exporting()
+ ie.export_item(item)
+ ie.export_item(item)
+ ie.finish_exporting()
+ del ie # See the first “del self.ie” in this file for context.
+ self.assertCsvEqual(
+ output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n"
+ )
- def test_join_multivalue(self):
- class TestItem2(Item):
- name = Field()
- friends = Field()
+ @pytest.mark.parametrize("as_dict", [False, True], ids=["item", "dict"])
+ def test_header_no_header_line(self, as_dict: bool) -> None:
+ self.assertExportResult(
+ item=ItemAdapter(self.i).asdict() if as_dict else self.i,
+ include_headers_line=False,
+ expected=b"22,John\xc2\xa3\r\n",
+ )
- for cls in TestItem2, dict:
- self.assertExportResult(
- item=cls(name="John", friends=["Mary", "Paul"]),
- include_headers_line=False,
- expected='"Mary,Paul",John\r\n',
- )
+ @pytest.mark.parametrize("cls", [FriendsItem, dict])
+ def test_join_multivalue(self, cls: type) -> None:
+ self.assertExportResult(
+ item=cls(name="John", friends=["Mary", "Paul"]),
+ include_headers_line=False,
+ expected='"Mary,Paul",John\r\n',
+ )
def test_join_multivalue_not_strings(self):
self.assertExportResult(
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 9582b4298..aeed0d6d6 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -680,14 +680,14 @@ class TestHttps2ClientProtocol:
data = json.loads(str(response.body, content_encoding))
assert data == params
+ @pytest.mark.parametrize("status", [200, 404])
@deferred_f_from_coro_f
async def test_status_codes(
- self, server_port: int, client: H2ClientProtocol
+ self, status: int, server_port: int, client: H2ClientProtocol
) -> None:
- for status in [200, 404]:
- request = Request(self.get_url(server_port, f"/status?n={status}"))
- response = await make_request(client, request)
- assert response.status == status
+ request = Request(self.get_url(server_port, f"/status?n={status}"))
+ response = await make_request(client, request)
+ assert response.status == status
@pytest.mark.filterwarnings(
r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index cb18a0b03..8d82a4a49 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -1008,20 +1008,21 @@ class TestFormRequest(TestRequestBase):
with pytest.raises(ValueError, match="No <form> element found with"):
self.request_class.from_response(response, formcss="input[name='abc']")
- def test_from_response_valid_form_methods(self):
- form_methods = [
- [method, method] for method in self.request_class.valid_form_methods
- ]
- form_methods.append(["UNKNOWN", "GET"])
-
- for method, expected in form_methods:
- response = _buildresponse(
- f'<form action="post.php" method="{method}">'
- '<input type="hidden" name="one" value="1">'
- "</form>"
- )
- r = self.request_class.from_response(response)
- assert r.method == expected
+ @pytest.mark.parametrize(
+ ("method", "expected"),
+ [
+ *((method, method) for method in FormRequest.valid_form_methods),
+ ("UNKNOWN", "GET"),
+ ],
+ )
+ def test_from_response_valid_form_methods(self, method: str, expected: str) -> None:
+ response = _buildresponse(
+ f'<form action="post.php" method="{method}">'
+ '<input type="hidden" name="one" value="1">'
+ "</form>"
+ )
+ r = self.request_class.from_response(response)
+ assert r.method == expected
def test_form_response_with_invalid_formdata_type_error(self):
"""Test that a ValueError is raised for non-iterable and non-dict formdata input"""
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 31634d3d5..4f5845095 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -539,17 +539,16 @@ class TestTextResponse(TestResponseBase):
)
assert response.json() == {"message": "café"}
- def test_cache_json_response(self):
- json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
- for json_body in json_valid_bodies:
- json_response = self.response_class(
- "http://www.example.com", body=json_body
- )
+ @pytest.mark.parametrize(
+ "json_body", [b"""{"ip": "109.187.217.200"}""", b"""null"""]
+ )
+ def test_cache_json_response(self, json_body: bytes) -> None:
+ json_response = self.response_class("http://www.example.com", body=json_body)
- with mock.patch("json.loads") as mock_json:
- for _ in range(2):
- json_response.json()
- mock_json.assert_called_once_with(json_body)
+ with mock.patch("json.loads") as mock_json:
+ for _ in range(2):
+ json_response.json()
+ mock_json.assert_called_once_with(json_body)
class TestHtmlResponse(TestTextResponse):
diff --git a/tests/test_poet.py b/tests/test_poet.py
index c85063a78..6ea31c6c3 100644
--- a/tests/test_poet.py
+++ b/tests/test_poet.py
@@ -2,21 +2,29 @@
from __future__ import annotations
-from typing import get_type_hints
+from typing import TYPE_CHECKING, Any, get_type_hints
+
+import pytest
from scrapy import Spider
from scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider
+if TYPE_CHECKING:
+ from collections.abc import Callable
-def test_callbacks() -> None:
- """Making sure annotations on all non-abstract callbacks can be resolved."""
- for cb in [
+ "cb",
+ [
Spider._parse,
CrawlSpider._parse,
CrawlSpider._callback,
XMLFeedSpider._parse,
CSVFeedSpider._parse,
SitemapSpider._parse_sitemap,
- ]:
- get_type_hints(cb)
+ ],
+ ids=lambda cb: cb.__qualname__,
+)
+def test_callbacks(cb: Callable[..., Any]) -> None:
+ """Making sure annotations on all non-abstract callbacks can be resolved."""
+ get_type_hints(cb)
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index b0297a12b..dd4adbe7e 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -3,6 +3,8 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING
+import pytest
+
from scrapy import Request, signals
from scrapy.http.response import Response
from scrapy.utils.test import get_crawler
@@ -10,8 +12,6 @@ from tests.spiders import SingleRequestSpider
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- import pytest
-
from tests.mockserver.http import MockServer
OVERRIDDEN_URL = "https://example.org"
@@ -79,17 +79,17 @@ class TestCrawl:
response = crawler.spider.meta["responses"][0]
assert response.request.url == url
+ @pytest.mark.parametrize("status", [404, 500])
@coroutine_test
- async def test_response_error(self, mockserver: MockServer) -> None:
- for status in ("404", "500"):
- url = mockserver.url(f"/status?n={status}")
- crawler = get_crawler(SingleRequestSpider)
- await crawler.crawl_async(seed=url, mockserver=mockserver)
- assert isinstance(crawler.spider, SingleRequestSpider)
- failure = crawler.spider.meta["failure"]
- response = failure.value.response
- assert failure.request.url == url
- assert response.request.url == url
+ async def test_response_error(self, status: int, mockserver: MockServer) -> None:
+ url = mockserver.url(f"/status?n={status}")
+ crawler = get_crawler(SingleRequestSpider)
+ await crawler.crawl_async(seed=url, mockserver=mockserver)
+ assert isinstance(crawler.spider, SingleRequestSpider)
+ failure = crawler.spider.meta["failure"]
+ response = failure.value.response
+ assert failure.request.url == url
+ assert response.request.url == url
@coroutine_test
async def test_downloader_middleware_raise_exception(
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index fa37cfe03..7f6da21e5 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,5 +1,7 @@
from typing import Any
+import pytest
+
from scrapy.http import (
Headers,
HtmlResponse,
@@ -11,44 +13,58 @@ from scrapy.http import (
from scrapy.responsetypes import responsetypes
-def test_from_filename() -> None:
- mappings = [
+ ("source", "cls"),
+ [
("data.bin", Response),
("file.txt", TextResponse),
("file.xml.gz", Response),
("file.xml", XmlResponse),
("file.html", HtmlResponse),
("file.unknownext", Response),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_filename(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
-
-
-def test_from_content_disposition() -> None:
- mappings = [
- (b'attachment; filename="data.xml"', XmlResponse),
- (b"attachment; filename=data.xml", XmlResponse),
- ("attachment;filename=data£.tar.gz".encode(), Response),
- ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
- ("attachment;filename=data高.doc".encode("gbk"), Response),
- ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
- (
+ ],
+)
+def test_from_filename(source: str, cls: type[Response]) -> None:
+ assert responsetypes.from_filename(source) is cls
+
+
+ ("source", "cls"),
+ [
+ pytest.param(b'attachment; filename="data.xml"', XmlResponse, id="quoted"),
+ pytest.param(b"attachment; filename=data.xml", XmlResponse, id="unquoted"),
+ pytest.param("attachment;filename=data£.tar.gz".encode(), Response, id="utf-8"),
+ pytest.param(
+ "attachment;filename=dataµ.tar.gz".encode("latin-1"),
+ Response,
+ id="latin-1",
+ ),
+ pytest.param(
+ "attachment;filename=data高.doc".encode("gbk"), Response, id="gbk"
+ ),
+ pytest.param(
+ "attachment;filename=دورهdata.html".encode("cp720"),
+ HtmlResponse,
+ id="cp720",
+ ),
+ pytest.param(
"attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),
XmlResponse,
+ id="iso2022_jp",
),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_content_disposition(source)
- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
+ ],
+)
+def test_from_content_disposition(source: bytes, cls: type[Response]) -> None:
+ assert responsetypes.from_content_disposition(source) is cls
def test_from_content_disposition_no_filename() -> None:
assert responsetypes.from_content_disposition(b"attachment") is Response
-def test_from_content_type() -> None:
- mappings: list[tuple[str | bytes, type[Response]]] = [
+ ("source", "cls"),
+ [
("text/html; charset=UTF-8", HtmlResponse),
("text/xml; charset=UTF-8", XmlResponse),
("application/xhtml+xml; charset=UTF-8", HtmlResponse),
@@ -59,63 +75,74 @@ def test_from_content_type() -> None:
("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
(b"application/x-download; filename=\x80dummy.txt", Response),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_content_type(source)
- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
+ ],
+)
+def test_from_content_type(source: str | bytes, cls: type[Response]) -> None:
+ assert responsetypes.from_content_type(source) is cls
-def test_from_body() -> None:
- mappings = [
+ ("source", "cls"),
+ [
(b"\x03\x02\xdf\xdd\x23", Response),
(b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),
(b"<html><head><title>Hello</title></head>", HtmlResponse),
# https://codersblock.com/blog/the-smallest-valid-html5-page/
(b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
(b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_body(source)
- assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
+ ],
+)
+def test_from_body(source: bytes, cls: type[Response]) -> None:
+ assert responsetypes.from_body(source) is cls
-def test_from_headers() -> None:
- mappings = [
- ({"Content-Type": ["text/html; charset=utf-8"]}, HtmlResponse),
- (
+ ("source", "cls"),
+ [
+ pytest.param(
+ {"Content-Type": ["text/html; charset=utf-8"]},
+ HtmlResponse,
+ id="content-type",
+ ),
+ pytest.param(
{
"Content-Type": ["text/html; charset=utf-8"],
"Content-Encoding": ["gzip"],
},
Response,
+ id="content-encoding",
),
- (
+ pytest.param(
{
"Content-Type": ["application/octet-stream"],
"Content-Disposition": ["attachment; filename=data.txt"],
},
TextResponse,
+ id="content-disposition",
),
- ]
- for source, cls in mappings:
- source = Headers(source)
- retcls = responsetypes.from_headers(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+ ],
+)
+def test_from_headers(source: dict[str, list[str]], cls: type[Response]) -> None:
+ assert responsetypes.from_headers(Headers(source)) is cls
-def test_from_args() -> None:
- # TODO: add more tests that check precedence between the different arguments
- mappings: list[tuple[dict[str, Any], type[Response]]] = [
- ({"url": "http://www.example.com/data.csv"}, TextResponse),
+# TODO: add more tests that check precedence between the different arguments
+ ("source", "cls"),
+ [
+ pytest.param(
+ {"url": "http://www.example.com/data.csv"}, TextResponse, id="url"
+ ),
# headers takes precedence over url
- (
+ pytest.param(
{
"headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),
"url": "http://www.example.com/item/",
},
HtmlResponse,
+ id="content-type-and-url",
),
- (
+ pytest.param(
{
"headers": Headers(
{"Content-Disposition": ['attachment; filename="data.xml.gz"']}
@@ -123,11 +150,12 @@ def test_from_args() -> None:
"url": "http://www.example.com/page/",
},
Response,
+ id="content-disposition-and-url",
),
- ]
- for source, cls in mappings:
- retcls = responsetypes.from_args(**source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+ ],
+)
+def test_from_args(source: dict[str, Any], cls: type[Response]) -> None:
+ assert responsetypes.from_args(**source) is cls
def test_custom_mime_types_loaded() -> None:
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 18940dcd2..4a175c067 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -117,7 +117,10 @@ class TestBaseSettings:
assert "TEST_OPTION" in self.settings.attributes
assert attr is self.settings.attributes["TEST_OPTION"]
- def test_set_calls_settings_attributes_methods_on_update(self):
+ @pytest.mark.parametrize("priority", [0, 10, 20])
+ def test_set_calls_settings_attributes_methods_on_update(
+ self, priority: int
+ ) -> None:
attr = SettingsAttribute("value", 10)
with (
mock.patch.object(attr, "__setattr__") as mock_setattr,
@@ -125,12 +128,9 @@ class TestBaseSettings:
):
self.settings.attributes = {"TEST_OPTION": attr}
- for priority in (0, 10, 20):
- self.settings.set("TEST_OPTION", "othervalue", priority)
- mock_set.assert_called_once_with("othervalue", priority)
- assert not mock_setattr.called
- mock_set.reset_mock()
- mock_setattr.reset_mock()
+ self.settings.set("TEST_OPTION", "othervalue", priority)
+ mock_set.assert_called_once_with("othervalue", priority)
+ assert not mock_setattr.called
def test_setitem(self):
settings = BaseSettings()
@@ -492,22 +492,24 @@ class TestBaseSettings:
== settings[setting_name]
)
- def test_getwithbase_for_non_component_priority_dicts(self):
+ @pytest.mark.parametrize(
+ "base_name", sorted(NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES)
+ )
+ def test_getwithbase_for_non_component_priority_dicts(self, base_name: str) -> None:
settings = BaseSettings()
- for base_name in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES:
- base_value = getattr(scrapy_default_settings, base_name)
- settings[base_name] = BaseSettings(base_value)
- assert len(base_value) >= 2
- keys = list(base_value)
- values = list(base_value.values())
- override = {keys[0]: values[1]}
- expected = dict(base_value)
- expected[keys[0]] = values[1]
- name = base_name[: -len("_BASE")]
- settings[name] = BaseSettings(override)
- value = settings.getwithbase(name)
- assert isinstance(value, BaseSettings)
- assert dict(value) == expected
+ base_value = getattr(scrapy_default_settings, base_name)
+ settings[base_name] = BaseSettings(base_value)
+ assert len(base_value) >= 2
+ keys = list(base_value)
+ values = list(base_value.values())
+ override = {keys[0]: values[1]}
+ expected = dict(base_value)
+ expected[keys[0]] = values[1]
+ name = base_name[: -len("_BASE")]
+ settings[name] = BaseSettings(override)
+ value = settings.getwithbase(name)
+ assert isinstance(value, BaseSettings)
+ assert dict(value) == expected
def test_get_component_priority_dict_with_base_warns_on_duplicate_import_paths(
self, caplog
diff --git a/tests/test_spider.py b/tests/test_spider.py
index e06201ad3..e5b743f06 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -31,7 +31,8 @@ class TestSpider(TestSpiderBase):
class TestXMLFeedSpider(TestSpiderBase):
spider_class = XMLFeedSpider
- def test_register_namespace(self):
+ @pytest.mark.parametrize("iterator", ["iternodes", "xml"])
+ def test_register_namespace(self, iterator: str) -> None:
body = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"
xmlns:y="http://www.example.com/schemas/extras/1.0">
@@ -57,24 +58,23 @@ class TestXMLFeedSpider(TestSpiderBase):
"custom": selector.xpath("other/@b:custom").getall(),
}
- for iterator in ("iternodes", "xml"):
- spider = _XMLSpider("example", iterator=iterator)
- output = list(spider._parse(response))
- assert len(output) == 2, iterator
- assert output == [
- {
- "loc": ["http://www.example.com/Special-Offers.html"],
- "updated": ["2009-08-16"],
- "custom": ["fuu"],
- "other": ["bar"],
- },
- {
- "loc": [],
- "updated": ["2009-08-16"],
- "other": ["foo"],
- "custom": [],
- },
- ], iterator
+ spider = _XMLSpider("example", iterator=iterator)
+ output = list(spider._parse(response))
+ assert len(output) == 2
+ assert output == [
+ {
+ "loc": ["http://www.example.com/Special-Offers.html"],
+ "updated": ["2009-08-16"],
+ "custom": ["fuu"],
+ "other": ["bar"],
+ },
+ {
+ "loc": [],
+ "updated": ["2009-08-16"],
+ "other": ["foo"],
+ "custom": [],
+ },
+ ]
@coroutine_test
async def test_parse_node_uses_parse_item(self, mockserver: MockServer):
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
index b9c4dd4e2..08352da2f 100644
--- a/tests/test_spidermiddleware_base.py
+++ b/tests/test_spidermiddleware_base.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeAlias
import pytest
@@ -13,16 +13,45 @@ from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ from collections.abc import Awaitable, Callable
+
from scrapy.crawler import Crawler
+ _Process: TypeAlias = Callable[
+ [BaseSpiderMiddleware, list[Any]], Awaitable[list[Any]]
+ ]
+
@pytest.fixture
def crawler() -> Crawler:
return get_crawler(Spider)
+async def _process_spider_output(
+ mw: BaseSpiderMiddleware, spider_output: list[Any]
+) -> list[Any]:
+ return list(mw.process_spider_output(Response("data:,"), spider_output))
+
+
+async def _process_spider_output_async(
+ mw: BaseSpiderMiddleware, spider_output: list[Any]
+) -> list[Any]:
+ return await collect_asyncgen(
+ mw.process_spider_output_async(
+ Response("data:,"), as_async_generator(spider_output)
+ )
+ )
+
+
+async def _process_start(
+ mw: BaseSpiderMiddleware, spider_output: list[Any]
+) -> list[Any]:
+ return await collect_asyncgen(mw.process_start(as_async_generator(spider_output)))
+
+
[email protected]("process", [_process_spider_output, _process_start])
@coroutine_test
-async def test_trivial(crawler: Crawler) -> None:
+async def test_trivial(process: _Process, crawler: Crawler) -> None:
class TrivialSpiderMiddleware(BaseSpiderMiddleware):
pass
@@ -31,15 +60,15 @@ async def test_trivial(crawler: Crawler) -> None:
assert mw.crawler is crawler
test_req = Request("data:,")
spider_output = [test_req, {"foo": "bar"}]
- for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output)),
- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
- ]:
- assert processed == [test_req, {"foo": "bar"}]
+ processed = await process(mw, spider_output)
+ assert processed == [test_req, {"foo": "bar"}]
+ "process", [_process_spider_output, _process_spider_output_async, _process_start]
+)
@coroutine_test
-async def test_processed_request(crawler: Crawler) -> None:
+async def test_processed_request(process: _Process, crawler: Crawler) -> None:
class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
@@ -55,25 +84,18 @@ async def test_processed_request(crawler: Crawler) -> None:
test_req2 = Request("data:2,")
test_req3 = Request("data:3,")
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
- for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output)),
- await collect_asyncgen(
- mw.process_spider_output_async(
- Response("data:,"), as_async_generator(spider_output)
- )
- ),
- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
- ]:
- assert len(processed) == 3
- assert isinstance(processed[0], Request)
- assert processed[0].url == "data:1,"
- assert processed[1] == {"foo": "bar"}
- assert isinstance(processed[2], Request)
- assert processed[2].url == "data:30,"
+ processed = await process(mw, spider_output)
+ assert len(processed) == 3
+ assert isinstance(processed[0], Request)
+ assert processed[0].url == "data:1,"
+ assert processed[1] == {"foo": "bar"}
+ assert isinstance(processed[2], Request)
+ assert processed[2].url == "data:30,"
[email protected]("process", [_process_spider_output, _process_start])
@coroutine_test
-async def test_processed_item(crawler: Crawler) -> None:
+async def test_processed_item(process: _Process, crawler: Crawler) -> None:
class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_item(self, item: Any, response: Response | None) -> Any:
if item["foo"] == 2:
@@ -85,15 +107,13 @@ async def test_processed_item(crawler: Crawler) -> None:
mw = build_from_crawler(ProcessItemSpiderMiddleware, crawler)
test_req = Request("data:,")
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
- for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output)),
- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
- ]:
- assert processed == [{"foo": 1}, test_req, {"foo": 30}]
+ processed = await process(mw, spider_output)
+ assert processed == [{"foo": 1}, test_req, {"foo": 30}]
[email protected]("process", [_process_spider_output, _process_start])
@coroutine_test
-async def test_processed_both(crawler: Crawler) -> None:
+async def test_processed_both(process: _Process, crawler: Crawler) -> None:
class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
@@ -123,14 +143,11 @@ async def test_processed_both(crawler: Crawler) -> None:
{"foo": 3},
test_req3,
]
- for processed in [
- list(mw.process_spider_output(Response("data:,"), spider_output)),
- await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
- ]:
- assert len(processed) == 4
- assert isinstance(processed[0], Request)
- assert processed[0].url == "data:1,"
- assert processed[1] == {"foo": 1}
- assert processed[2] == {"foo": 30}
- assert isinstance(processed[3], Request)
- assert processed[3].url == "data:30,"
+ processed = await process(mw, spider_output)
+ assert len(processed) == 4
+ assert isinstance(processed[0], Request)
+ assert processed[0].url == "data:1,"
+ assert processed[1] == {"foo": 1}
+ assert processed[2] == {"foo": 30}
+ assert isinstance(processed[3], Request)
+ assert processed[3].url == "data:30,"
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 958b00ae0..fe13f750b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -633,6 +633,20 @@ POLICY_CASES: list[tuple[str | None, str, ScenarioTable]] = [
]
+# Each policy, as (REFERRER_POLICY name, policy class).
+POLICY_CLASSES: list[tuple[str, type[ReferrerPolicy]]] = [
+ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+ (POLICY_NO_REFERRER, NoReferrerPolicy),
+ (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+ (POLICY_SAME_ORIGIN, SameOriginPolicy),
+ (POLICY_ORIGIN, OriginPolicy),
+ (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+ (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+ (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+ (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+]
+
+
def assert_scenarii(
scenarii: ScenarioTable,
*,
@@ -792,116 +806,107 @@ def test_policy_precedence(
class TestRequestMetaSettingFallback:
- params = [
+ @pytest.mark.parametrize(
(
+ "settings",
+ "response_headers",
+ "request_meta",
+ "policy_class",
+ "check_warning",
+ ),
+ [
# When an unknown policy is referenced in Request.meta
# (here, a typo error),
# the policy defined in settings takes precedence
- {
- "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
- },
- {},
- {"referrer_policy": "ssscrapy-default"},
- OriginWhenCrossOriginPolicy,
- True,
- ),
- (
+ pytest.param(
+ {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+ },
+ {},
+ {"referrer_policy": "ssscrapy-default"},
+ OriginWhenCrossOriginPolicy,
+ True,
+ id="unknown-meta-settings-path",
+ ),
# same as above but with string value for settings policy
- {"REFERRER_POLICY": "origin-when-cross-origin"},
- {},
- {"referrer_policy": "ssscrapy-default"},
- OriginWhenCrossOriginPolicy,
- True,
- ),
- (
+ pytest.param(
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {},
+ {"referrer_policy": "ssscrapy-default"},
+ OriginWhenCrossOriginPolicy,
+ True,
+ id="unknown-meta-settings-name",
+ ),
# request meta references a wrong policy but it is set,
# so the Referrer-Policy header in response is not used,
# and the settings' policy is applied
- {"REFERRER_POLICY": "origin-when-cross-origin"},
- {"Referrer-Policy": "unsafe-url"},
- {"referrer_policy": "ssscrapy-default"},
- OriginWhenCrossOriginPolicy,
- True,
- ),
- (
+ pytest.param(
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {"Referrer-Policy": "unsafe-url"},
+ {"referrer_policy": "ssscrapy-default"},
+ OriginWhenCrossOriginPolicy,
+ True,
+ id="unknown-meta-header",
+ ),
# here, request meta does not set the policy
# so response headers take precedence
- {"REFERRER_POLICY": "origin-when-cross-origin"},
- {"Referrer-Policy": "unsafe-url"},
- {},
- UnsafeUrlPolicy,
- False,
- ),
- (
+ pytest.param(
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {"Referrer-Policy": "unsafe-url"},
+ {},
+ UnsafeUrlPolicy,
+ False,
+ id="no-meta-header",
+ ),
# here, request meta does not set the policy,
# but response headers also use an unknown policy,
# so the settings' policy is used
- {"REFERRER_POLICY": "origin-when-cross-origin"},
- {"Referrer-Policy": "unknown"},
- {},
- OriginWhenCrossOriginPolicy,
- True,
- ),
- ]
-
- def test(self):
+ pytest.param(
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {"Referrer-Policy": "unknown"},
+ {},
+ OriginWhenCrossOriginPolicy,
+ True,
+ id="no-meta-unknown-header",
+ ),
+ ],
+ )
+ def test(
+ self,
+ settings: dict[str, Any],
+ response_headers: dict[str, str],
+ request_meta: dict[str, Any],
+ policy_class: type[ReferrerPolicy],
+ check_warning: bool,
+ ) -> None:
origin = "http://www.scrapy.org"
target = "http://www.example.com"
- for (
- settings,
- response_headers,
- request_meta,
- policy_class,
- check_warning,
- ) in self.params:
- mw = RefererMiddleware(Settings(settings))
-
- response = Response(origin, headers=response_headers)
- request = Request(target, meta=request_meta)
-
- if check_warning:
- with pytest.warns(
- RuntimeWarning, match="Could not load referrer policy"
- ):
- policy = mw.policy(response, request)
- else:
+ mw = RefererMiddleware(Settings(settings))
+
+ response = Response(origin, headers=response_headers)
+ request = Request(target, meta=request_meta)
+
+ if check_warning:
+ with pytest.warns(RuntimeWarning, match="Could not load referrer policy"):
policy = mw.policy(response, request)
- assert isinstance(policy, policy_class)
+ else:
+ policy = mw.policy(response, request)
+ assert isinstance(policy, policy_class)
class TestSettingsPolicyByName:
- def test_valid_name(self):
- for s, p in [
- (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
- (POLICY_NO_REFERRER, NoReferrerPolicy),
- (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
- (POLICY_SAME_ORIGIN, SameOriginPolicy),
- (POLICY_ORIGIN, OriginPolicy),
- (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
- (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
- (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
- (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
- ]:
- settings = Settings({"REFERRER_POLICY": s})
- mw = RefererMiddleware(settings)
- assert mw.default_policy == p
-
- def test_valid_name_casevariants(self):
- for s, p in [
- (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
- (POLICY_NO_REFERRER, NoReferrerPolicy),
- (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
- (POLICY_SAME_ORIGIN, SameOriginPolicy),
- (POLICY_ORIGIN, OriginPolicy),
- (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
- (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
- (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
- (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
- ]:
- settings = Settings({"REFERRER_POLICY": s.upper()})
- mw = RefererMiddleware(settings)
- assert mw.default_policy == p
+ @pytest.mark.parametrize(("name", "policy_class"), POLICY_CLASSES)
+ def test_valid_name(self, name: str, policy_class: type[ReferrerPolicy]) -> None:
+ mw = RefererMiddleware(Settings({"REFERRER_POLICY": name}))
+ assert mw.default_policy == policy_class
+
+ @pytest.mark.parametrize(("name", "policy_class"), POLICY_CLASSES)
+ def test_valid_name_casevariants(
+ self, name: str, policy_class: type[ReferrerPolicy]
+ ) -> None:
+ mw = RefererMiddleware(Settings({"REFERRER_POLICY": name.upper()}))
+ assert mw.default_policy == policy_class
def test_invalid_name(self):
settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index f9be16ba0..d2deed61c 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -77,43 +77,43 @@ class TestParallelAsyncio:
await asyncio.sleep(random.random() / 20)
yield i
+ @pytest.mark.parametrize("length", [20, 50, 100])
@coroutine_test
- async def test_simple(self):
- for length in [20, 50, 100]:
- parallel_count = [0]
- max_parallel_count = [0]
- results: list[int] = []
- ait = self.get_async_iterable(length)
- await _parallel_asyncio(
- ait,
- self.CONCURRENT_ITEMS,
- self.callable_wrapped,
- results,
- parallel_count,
- max_parallel_count,
- )
- assert list(range(length)) == sorted(results)
- assert parallel_count[0] == 0
- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
+ async def test_simple(self, length: int) -> None:
+ parallel_count = [0]
+ max_parallel_count = [0]
+ results: list[int] = []
+ ait = self.get_async_iterable(length)
+ await _parallel_asyncio(
+ ait,
+ self.CONCURRENT_ITEMS,
+ self.callable_wrapped,
+ results,
+ parallel_count,
+ max_parallel_count,
+ )
+ assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
+ @pytest.mark.parametrize("length", [20, 50, 100])
@coroutine_test
- async def test_delays(self):
- for length in [20, 50, 100]:
- parallel_count = [0]
- max_parallel_count = [0]
- results: list[int] = []
- ait = self.get_async_iterable_with_delays(length)
- await _parallel_asyncio(
- ait,
- self.CONCURRENT_ITEMS,
- self.callable_wrapped,
- results,
- parallel_count,
- max_parallel_count,
- )
- assert list(range(length)) == sorted(results)
- assert parallel_count[0] == 0
- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
+ async def test_delays(self, length: int) -> None:
+ parallel_count = [0]
+ max_parallel_count = [0]
+ results: list[int] = []
+ ait = self.get_async_iterable_with_delays(length)
+ await _parallel_asyncio(
+ ait,
+ self.CONCURRENT_ITEMS,
+ self.callable_wrapped,
+ results,
+ parallel_count,
+ max_parallel_count,
+ )
+ assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
@coroutine_test
async def test_count_higher_than_work(self):
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 26096911d..5dcc59886 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,7 +1,8 @@
from __future__ import annotations
import copy
-from collections.abc import Iterator, Mapping, MutableMapping
+from collections.abc import Callable, Iterator, Mapping, MutableMapping
+from operator import methodcaller
from typing import Any
import pytest
@@ -194,14 +195,19 @@ class TestCaseInsensitiveDict:
assert h1.get("header1") == h3.get("header1")
assert h1.get("header1") == h3.get("HEADER1")
- def test_copy_is_independent(self):
+ @pytest.mark.parametrize(
+ "copier", [copy.copy, methodcaller("copy")], ids=["copy.copy", "copy-method"]
+ )
+ def test_copy_is_independent(
+ self, copier: Callable[[CaseInsensitiveDict], CaseInsensitiveDict]
+ ) -> None:
h1 = self.dict_class({"header1": "value1", "header2": "value2"})
- for h2 in (copy.copy(h1), h1.copy()):
- del h2["header1"]
- h2["header3"] = "value3"
- assert "header1" in h1
- assert "header3" not in h1
- assert dict(h1) == {"header1": "value1", "header2": "value2"}
+ h2 = copier(h1)
+ del h2["header1"]
+ h2["header3"] = "value3"
+ assert "header1" in h1
+ assert "header3" not in h1
+ assert dict(h1) == {"header1": "value1", "header2": "value2"}
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
@@ -215,14 +221,18 @@ class TestCaseInsensitiveDict:
assert isinstance(iterkeys, Iterator)
assert list(iterkeys) == ["AsDf", "FoO"]
- def test_copy_keeps_values(self):
+ @pytest.mark.parametrize(
+ "copier", [copy.copy, methodcaller("copy")], ids=["copy.copy", "copy-method"]
+ )
+ def test_copy_keeps_values(
+ self, copier: Callable[[CaseInsensitiveDict], CaseInsensitiveDict]
+ ) -> None:
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
return value + 1
d = MyDict({"key": 1})
- for copied in (copy.copy(d), d.copy()):
- assert copied["key"] == 2
+ assert copier(d)["key"] == 2
def test_ior(self):
d = self.dict_class({"header1": "value1"})
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 4143f3888..5a277d08a 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -258,45 +258,45 @@ class TestParallelAsync:
await maybe_deferred_to_future(dfd)
yield i
+ @pytest.mark.parametrize("length", [20, 50, 100])
@inline_callbacks_test
- def test_simple(self):
- for length in [20, 50, 100]:
- parallel_count = [0]
- max_parallel_count = [0]
- results: list[int] = []
- ait = self.get_async_iterable(length)
- dl = parallel_async(
- ait,
- self.CONCURRENT_ITEMS,
- self.callable_wrapped,
- results,
- parallel_count,
- max_parallel_count,
- )
- yield dl
- assert list(range(length)) == sorted(results)
- assert parallel_count[0] == 0
- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
-
+ def test_simple(self, length: int) -> Generator[Deferred[Any], Any, None]:
+ parallel_count = [0]
+ max_parallel_count = [0]
+ results: list[int] = []
+ ait = self.get_async_iterable(length)
+ dl = parallel_async(
+ ait,
+ self.CONCURRENT_ITEMS,
+ self.callable_wrapped,
+ results,
+ parallel_count,
+ max_parallel_count,
+ )
+ yield dl
+ assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
+
+ @pytest.mark.parametrize("length", [20, 50, 100])
@inline_callbacks_test
- def test_delays(self):
- for length in [20, 50, 100]:
- parallel_count = [0]
- max_parallel_count = [0]
- results: list[int] = []
- ait = self.get_async_iterable_with_delays(length)
- dl = parallel_async(
- ait,
- self.CONCURRENT_ITEMS,
- self.callable_wrapped,
- results,
- parallel_count,
- max_parallel_count,
- )
- yield dl
- assert list(range(length)) == sorted(results)
- assert parallel_count[0] == 0
- assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
+ def test_delays(self, length: int) -> Generator[Deferred[Any], Any, None]:
+ parallel_count = [0]
+ max_parallel_count = [0]
+ results: list[int] = []
+ ait = self.get_async_iterable_with_delays(length)
+ dl = parallel_async(
+ ait,
+ self.CONCURRENT_ITEMS,
+ self.callable_wrapped,
+ results,
+ parallel_count,
+ max_parallel_count,
+ )
+ yield dl
+ assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
+ assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0]
@inline_callbacks_test
def test_count_higher_than_work(self) -> Generator[Deferred[Any], Any, None]:
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 2c1913669..0e4c520d5 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -80,9 +80,13 @@ class TestUpdateClassPath:
output = update_classpath("scrapy.unmatched.Path")
assert output == "scrapy.unmatched.Path"
- def test_returns_nonstring(self):
- for notastring in [None, True, [1, 2, 3], object()]:
- assert update_classpath(notastring) == notastring
+ @pytest.mark.parametrize(
+ "notastring",
+ [None, True, [1, 2, 3], object()],
+ ids=["None", "bool", "list", "object"],
+ )
+ def test_returns_nonstring(self, notastring: object) -> None:
+ assert update_classpath(notastring) == notastring
class TestAttribute:
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index abb733f1f..29518c7f6 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -4,6 +4,7 @@ import dataclasses
import datetime
import json
from decimal import Decimal
+from typing import Any
import attr
import pytest
@@ -18,39 +19,43 @@ class TestJsonEncoder:
def encoder(self) -> ScrapyJSONEncoder:
return ScrapyJSONEncoder(sort_keys=True)
- def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
- dt_naive = datetime.datetime(2010, 1, 2, 10, 11, 12) # noqa: DTZ001
- dt_naives = "2010-01-02T10:11:12"
- dt_aware = datetime.datetime(
- 2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.UTC
- )
- dt_awares = "2010-01-02T10:11:12.133700+00:00"
- d = datetime.date(2010, 1, 2)
- ds = "2010-01-02"
- t = datetime.time(10, 11, 12)
- ts = "10:11:12"
- t_us = datetime.time(10, 11, 12, 133700)
- t_uss = "10:11:12.133700"
- dec = Decimal("1000.12")
- decs = "1000.12"
- s = {"foo"}
- ss = ["foo"]
- dt_set = {dt_naive}
- dt_sets = [dt_naives]
-
- for input_, output in [
- ("foo", "foo"),
- (d, ds),
- (t, ts),
- (t_us, t_uss),
- (dt_naive, dt_naives),
- (dt_aware, dt_awares),
- (dec, decs),
- (["foo", d], ["foo", ds]),
- (s, ss),
- (dt_set, dt_sets),
- ]:
- assert encoder.encode(input_) == json.dumps(output, sort_keys=True)
+ @pytest.mark.parametrize(
+ ("input_", "output"),
+ [
+ pytest.param("foo", "foo", id="str"),
+ pytest.param(datetime.date(2010, 1, 2), "2010-01-02", id="date"),
+ pytest.param(datetime.time(10, 11, 12), "10:11:12", id="time"),
+ pytest.param(
+ datetime.time(10, 11, 12, 133700),
+ "10:11:12.133700",
+ id="time-microseconds",
+ ),
+ pytest.param(
+ datetime.datetime(2010, 1, 2, 10, 11, 12), # noqa: DTZ001
+ "2010-01-02T10:11:12",
+ id="datetime-naive",
+ ),
+ pytest.param(
+ datetime.datetime(2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.UTC),
+ "2010-01-02T10:11:12.133700+00:00",
+ id="datetime-aware",
+ ),
+ pytest.param(Decimal("1000.12"), "1000.12", id="decimal"),
+ pytest.param(
+ ["foo", datetime.date(2010, 1, 2)], ["foo", "2010-01-02"], id="list"
+ ),
+ pytest.param({"foo"}, ["foo"], id="set"),
+ pytest.param(
+ {datetime.datetime(2010, 1, 2, 10, 11, 12)}, # noqa: DTZ001
+ ["2010-01-02T10:11:12"],
+ id="datetime-set",
+ ),
+ ],
+ )
+ def test_encode_decode(
+ self, input_: Any, output: Any, encoder: ScrapyJSONEncoder
+ ) -> None:
+ assert encoder.encode(input_) == json.dumps(output, sort_keys=True)
def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
assert "Deferred" in encoder.encode(defer.Deferred())