Scrapy, a fast high-level web crawling & scraping framework for Python.
Replace loops with parametrize() in feedexport tests. (#8301)
Andrey Rakhmatullinbe7bdc4cd166
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 14c35d968..a1e53c0e4 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -135,43 +135,48 @@ class TestBatchDeliveries:with pytest.raises(NotConfigured):build_from_crawler(FeedExporter, crawler)+ @pytest.mark.parametrize("fmt", ["json", "jsonlines", "xml", "csv"])@coroutine_testasync def test_export_no_items_not_store_empty(- self, mockserver: MockServer, tmp_path: Path+ self, fmt: str, mockserver: MockServer, tmp_path: Path) -> None:- for fmt in ("json", "jsonlines", "xml", "csv"):- settings = {- "FEEDS": {- unique_path(tmp_path) / self._file_mark: {"format": fmt},- },- "FEED_EXPORT_BATCH_ITEM_COUNT": 1,- "FEED_STORE_EMPTY": False,- }- data = await export_batches(mockserver, [], settings)- assert len(data[fmt]) == 0-+ settings = {+ "FEEDS": {+ unique_path(tmp_path) / self._file_mark: {"format": fmt},+ },+ "FEED_EXPORT_BATCH_ITEM_COUNT": 1,+ "FEED_STORE_EMPTY": False,+ }+ data = await export_batches(mockserver, [], settings)+ assert len(data[fmt]) == 0++ @pytest.mark.parametrize(+ ("fmt", "expected"),+ [+ pytest.param("json", b"[]", id="json"),+ pytest.param("jsonlines", b"", id="jsonlines"),+ pytest.param(+ "xml",+ b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>',+ id="xml",+ ),+ pytest.param("csv", b"", id="csv"),+ ],+ )@coroutine_testasync def test_export_no_items_store_empty(- self, mockserver: MockServer, tmp_path: Path+ self, fmt: str, expected: bytes, mockserver: MockServer, tmp_path: Path) -> None:- formats = (- ("json", b"[]"),- ("jsonlines", b""),- ("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),- ("csv", b""),- )-- for fmt, expctd in formats:- settings = {- "FEEDS": {- unique_path(tmp_path) / self._file_mark: {"format": fmt},- },- "FEED_STORE_EMPTY": True,- "FEED_EXPORT_INDENT": None,- "FEED_EXPORT_BATCH_ITEM_COUNT": 1,- }- data = await export_batches(mockserver, [], settings)- assert data[fmt][0] == expctd+ settings = {+ "FEEDS": {+ unique_path(tmp_path) / self._file_mark: {"format": fmt},+ },+ "FEED_STORE_EMPTY": True,+ "FEED_EXPORT_INDENT": None,+ "FEED_EXPORT_BATCH_ITEM_COUNT": 1,+ }+ data = await export_batches(mockserver, [], settings)+ assert data[fmt][0] == expected@coroutine_testasync def test_export_multiple_configs(diff --git a/tests/test_feedexport_empty.py b/tests/test_feedexport_empty.pyindex 5121a4b69..cb7066e22 100644--- a/tests/test_feedexport_empty.py+++ b/tests/test_feedexport_empty.py@@ -5,6 +5,7 @@ import tempfilefrom logging import getLoggerfrom typing import TYPE_CHECKING+import pytestfrom w3lib.url import file_uri_to_pathfrom tests.utils.decorators import coroutine_test@@ -13,8 +14,6 @@ from tests.utils.feedexport import export_by_format, unique_pathif TYPE_CHECKING:from pathlib import Path- import pytest-from tests.mockserver.http import MockServer@@ -36,42 +35,47 @@ class LogOnStoreFileStorage:file.close()[email protected]("fmt", ["json", "jsonlines", "xml", "csv"])@coroutine_testasync def test_export_no_items_not_store_empty(- mockserver: MockServer, tmp_path: Path+ fmt: str, mockserver: MockServer, tmp_path: Path) -> None:- for fmt in ("json", "jsonlines", "xml", "csv"):- settings = {- "FEEDS": {- unique_path(tmp_path): {"format": fmt},- },- "FEED_STORE_EMPTY": False,- }- data = await export_by_format(mockserver, [], settings)- assert data[fmt] is None--+ settings = {+ "FEEDS": {+ unique_path(tmp_path): {"format": fmt},+ },+ "FEED_STORE_EMPTY": False,+ }+ data = await export_by_format(mockserver, [], settings)+ assert data[fmt] is None+++ ("fmt", "expected"),+ [+ pytest.param("json", b"[]", id="json"),+ pytest.param("jsonlines", b"", id="jsonlines"),+ pytest.param(+ "xml",+ b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>',+ id="xml",+ ),+ pytest.param("csv", b"", id="csv"),+ ],+)@coroutine_testasync def test_export_no_items_store_empty(- mockserver: MockServer, tmp_path: Path+ fmt: str, expected: bytes, mockserver: MockServer, tmp_path: Path) -> None:- formats = (- ("json", b"[]"),- ("jsonlines", b""),- ("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),- ("csv", b""),- )-- for fmt, expctd in formats:- settings = {- "FEEDS": {- unique_path(tmp_path): {"format": fmt},- },- "FEED_STORE_EMPTY": True,- "FEED_EXPORT_INDENT": None,- }- data = await export_by_format(mockserver, [], settings)- assert expctd == data[fmt]+ settings = {+ "FEEDS": {+ unique_path(tmp_path): {"format": fmt},+ },+ "FEED_STORE_EMPTY": True,+ "FEED_EXPORT_INDENT": None,+ }+ data = await export_by_format(mockserver, [], settings)+ assert data[fmt] == expected@coroutine_testdiff --git a/tests/test_feedexport_fields.py b/tests/test_feedexport_fields.pyindex 5cf74497f..086606cf9 100644--- a/tests/test_feedexport_fields.py+++ b/tests/test_feedexport_fields.py@@ -153,36 +153,46 @@ async def test_export_tuple(fmt: str, mockserver: MockServer, tmp_path: Path) ->)+ ("fields", "header", "rows"),+ [+ pytest.param(+ "foo,baz,egg",+ ["foo", "baz", "egg"],+ [+ {"egg": "spam1", "foo": "bar1"},+ {"egg": "spam2", "foo": "bar2", "baz": "quux2"},+ ],+ id="all",+ ),+ pytest.param(+ "egg,baz",+ ["egg", "baz"],+ [{"egg": "spam1"}, {"egg": "spam2", "baz": "quux2"}],+ id="subset",+ ),+ ],+)[email protected]("item_cls", [MyItem, dict])@pytest.mark.parametrize("fmt", list(PARSERS))@coroutine_testasync def test_export_feed_export_fields(- fmt: str, mockserver: MockServer, tmp_path: Path+ fmt: str,+ item_cls: type[MyItem | dict[str, Any]],+ fields: str,+ header: list[str],+ rows: list[dict[str, Any]],+ mockserver: MockServer,+ tmp_path: Path,) -> None:# FEED_EXPORT_FIELDS option allows to order export fields# and to select a subset of fields to export, both for Items and dicts.-- for item_cls in [MyItem, dict]:- items = [- item_cls({"foo": "bar1", "egg": "spam1"}),- item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),- ]-- # export all columns- settings = {"FEED_EXPORT_FIELDS": "foo,baz,egg"}- rows = [- {"egg": "spam1", "foo": "bar1"},- {"egg": "spam2", "foo": "bar2", "baz": "quux2"},- ]- await assert_exported(- mockserver, tmp_path, fmt, items, ["foo", "baz", "egg"], rows, settings- )-- # export a subset of columns- settings = {"FEED_EXPORT_FIELDS": "egg,baz"}- rows = [{"egg": "spam1"}, {"egg": "spam2", "baz": "quux2"}]- await assert_exported(- mockserver, tmp_path, fmt, items, ["egg", "baz"], rows, settings- )+ items = [+ item_cls({"foo": "bar1", "egg": "spam1"}),+ item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),+ ]+ settings = {"FEED_EXPORT_FIELDS": fields}+ await assert_exported(mockserver, tmp_path, fmt, items, header, rows, settings)# by default, Scrapy uses fields of the first Item for CSV anddiff --git a/tests/test_feedexport_formats.py b/tests/test_feedexport_formats.pyindex bef423b0c..6f5a866e1 100644--- a/tests/test_feedexport_formats.py+++ b/tests/test_feedexport_formats.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKINGimport pytest@@ -35,50 +35,57 @@ async def test_export_items(fmt: str, mockserver: MockServer, tmp_path: Path) ->await assert_exported(mockserver, tmp_path, fmt, items, header, rows)+ ("fmt", "encoding", "expected"),+ [+ pytest.param("json", None, b'[{"foo": "Test\\u00d6"}]', id="json-default"),+ pytest.param(+ "jsonlines", None, b'{"foo": "Test\\u00d6"}\n', id="jsonlines-default"+ ),+ pytest.param(+ "xml",+ None,+ (+ '<?xml version="1.0" encoding="utf-8"?>\n'+ "<items><item><foo>Test\xd6</foo></item></items>"+ ).encode(),+ id="xml-default",+ ),+ pytest.param("csv", None, "foo\r\nTest\xd6\r\n".encode(), id="csv-default"),+ pytest.param("json", "latin-1", b'[{"foo": "Test\xd6"}]', id="json-latin-1"),+ pytest.param(+ "jsonlines", "latin-1", b'{"foo": "Test\xd6"}\n', id="jsonlines-latin-1"+ ),+ pytest.param(+ "xml",+ "latin-1",+ (+ b'<?xml version="1.0" encoding="latin-1"?>\n'+ b"<items><item><foo>Test\xd6</foo></item></items>"+ ),+ id="xml-latin-1",+ ),+ pytest.param("csv", "latin-1", b"foo\r\nTest\xd6\r\n", id="csv-latin-1"),+ ],+)@coroutine_test-async def test_export_encoding(mockserver: MockServer, tmp_path: Path) -> None:+async def test_export_encoding(+ fmt: str,+ encoding: str | None,+ expected: bytes,+ mockserver: MockServer,+ tmp_path: Path,+) -> None:items = [{"foo": "Test\xd6"}]-- formats = {- "json": b'[{"foo": "Test\\u00d6"}]',- "jsonlines": b'{"foo": "Test\\u00d6"}\n',- "xml": (- '<?xml version="1.0" encoding="utf-8"?>\n'- "<items><item><foo>Test\xd6</foo></item></items>"- ).encode(),- "csv": "foo\r\nTest\xd6\r\n".encode(),- }-- for fmt, expected in formats.items():- settings: dict[str, Any] = {- "FEEDS": {- unique_path(tmp_path): {"format": fmt},- },- "FEED_EXPORT_INDENT": None,- }- data = await export_by_format(mockserver, items, settings)- assert data[fmt] == expected-- formats = {- "json": b'[{"foo": "Test\xd6"}]',- "jsonlines": b'{"foo": "Test\xd6"}\n',- "xml": (- b'<?xml version="1.0" encoding="latin-1"?>\n'- b"<items><item><foo>Test\xd6</foo></item></items>"- ),- "csv": b"foo\r\nTest\xd6\r\n",+ settings = {+ "FEEDS": {+ unique_path(tmp_path): {"format": fmt},+ },+ "FEED_EXPORT_INDENT": None,+ "FEED_EXPORT_ENCODING": encoding,}-- for fmt, expected in formats.items():- settings = {- "FEEDS": {- unique_path(tmp_path): {"format": fmt},- },- "FEED_EXPORT_INDENT": None,- "FEED_EXPORT_ENCODING": "latin-1",- }- data = await export_by_format(mockserver, items, settings)- assert data[fmt] == expected+ data = await export_by_format(mockserver, items, settings)+ assert data[fmt] == expected@coroutine_test@@ -122,40 +129,34 @@ async def test_export_multiple_configs(mockserver: MockServer, tmp_path: Path) -assert data[fmt] == expected-@coroutine_test-async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> None:- items = [- {"foo": ["bar"]},- {"key": "value"},- ]-- test_cases: list[dict[str, Any]] = [- # JSON- {- "format": "json",- "indent": None,- "expected": b'[{"foo": ["bar"]},{"key": "value"}]',- },- {- "format": "json",- "indent": -1,- "expected": b"""[+ ("fmt", "indent", "expected"),+ [+ pytest.param(+ "json", None, b'[{"foo": ["bar"]},{"key": "value"}]', id="json-None"+ ),+ pytest.param(+ "json",+ -1,+ b"""[{"foo": ["bar"]},{"key": "value"}]""",- },- {- "format": "json",- "indent": 0,- "expected": b"""[+ id="json--1",+ ),+ pytest.param(+ "json",+ 0,+ b"""[{"foo": ["bar"]},{"key": "value"}]""",- },- {- "format": "json",- "indent": 2,- "expected": b"""[+ id="json-0",+ ),+ pytest.param(+ "json",+ 2,+ b"""[{"foo": ["bar"@@ -165,11 +166,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non"key": "value"}]""",- },- {- "format": "json",- "indent": 4,- "expected": b"""[+ id="json-2",+ ),+ pytest.param(+ "json",+ 4,+ b"""[{"foo": ["bar"@@ -179,11 +181,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non"key": "value"}]""",- },- {- "format": "json",- "indent": 5,- "expected": b"""[+ id="json-4",+ ),+ pytest.param(+ "json",+ 5,+ b"""[{"foo": ["bar"@@ -193,36 +196,39 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non"key": "value"}]""",- },- # XML- {- "format": "xml",- "indent": None,- "expected": b"""<?xml version="1.0" encoding="utf-8"?>+ id="json-5",+ ),+ pytest.param(+ "xml",+ None,+ b"""<?xml version="1.0" encoding="utf-8"?><items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",- },- {- "format": "xml",- "indent": -1,- "expected": b"""<?xml version="1.0" encoding="utf-8"?>+ id="xml-None",+ ),+ pytest.param(+ "xml",+ -1,+ b"""<?xml version="1.0" encoding="utf-8"?><items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",- },- {- "format": "xml",- "indent": 0,- "expected": b"""<?xml version="1.0" encoding="utf-8"?>+ id="xml--1",+ ),+ pytest.param(+ "xml",+ 0,+ b"""<?xml version="1.0" encoding="utf-8"?><items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",- },- {- "format": "xml",- "indent": 2,- "expected": b"""<?xml version="1.0" encoding="utf-8"?>+ id="xml-0",+ ),+ pytest.param(+ "xml",+ 2,+ b"""<?xml version="1.0" encoding="utf-8"?><items><item><foo>@@ -233,11 +239,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non<key>value</key></item></items>""",- },- {- "format": "xml",- "indent": 4,- "expected": b"""<?xml version="1.0" encoding="utf-8"?>+ id="xml-2",+ ),+ pytest.param(+ "xml",+ 4,+ b"""<?xml version="1.0" encoding="utf-8"?><items><item><foo>@@ -248,11 +255,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non<key>value</key></item></items>""",- },- {- "format": "xml",- "indent": 5,- "expected": b"""<?xml version="1.0" encoding="utf-8"?>+ id="xml-4",+ ),+ pytest.param(+ "xml",+ 5,+ b"""<?xml version="1.0" encoding="utf-8"?><items><item><foo>@@ -263,55 +271,57 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non<key>value</key></item></items>""",- },+ id="xml-5",+ ),+ ],+)+@coroutine_test+async def test_export_indentation(+ fmt: str,+ indent: int | None,+ expected: bytes,+ mockserver: MockServer,+ tmp_path: Path,+) -> None:+ items = [+ {"foo": ["bar"]},+ {"key": "value"},]-- for row in test_cases:- settings = {- "FEEDS": {- unique_path(tmp_path): {- "format": row["format"],- "indent": row["indent"],- },+ settings = {+ "FEEDS": {+ unique_path(tmp_path): {+ "format": fmt,+ "indent": indent,},- }- data = await export_by_format(mockserver, items, settings)- assert data[row["format"]] == row["expected"]+ },+ }+ data = await export_by_format(mockserver, items, settings)+ assert data[fmt] == expected+ ("include_headers_line", "expected"),+ [+ pytest.param(True, b"foo,bar\r\nFOO,BAR\r\n", id="with-headers"),+ pytest.param(False, b"FOO,BAR\r\n", id="without-headers"),+ ],+)@coroutine_test-async def test_extend_kwargs(mockserver: MockServer, tmp_path: Path) -> None:+async def test_extend_kwargs(+ include_headers_line: bool,+ expected: bytes,+ mockserver: MockServer,+ tmp_path: Path,+) -> None:items = [{"foo": "FOO", "bar": "BAR"}]-- expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"- expected_without_title_csv = b"FOO,BAR\r\n"- test_cases: list[dict[str, Any]] = [- # with title- {- "options": {- "format": "csv",- "item_export_kwargs": {"include_headers_line": True},- },- "expected": expected_with_title_csv,- },- # without title- {- "options": {+ settings = {+ "FEEDS": {+ unique_path(tmp_path): {"format": "csv",- "item_export_kwargs": {"include_headers_line": False},+ "item_export_kwargs": {"include_headers_line": include_headers_line},},- "expected": expected_without_title_csv,},- ]-- for row in test_cases:- feed_options = row["options"]- settings = {- "FEEDS": {- unique_path(tmp_path): feed_options,- },- "FEED_EXPORT_INDENT": None,- }-- data = await export_by_format(mockserver, items, settings)- assert data[feed_options["format"]] == row["expected"]+ "FEED_EXPORT_INDENT": None,+ }+ data = await export_by_format(mockserver, items, settings)+ assert data["csv"] == expected