Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Replace loops with parametrize() in feedexport tests. (#8301)

Andrey Rakhmatullinbe7bdc4cd166
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 14c35d968..a1e53c0e4 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -135,43 +135,48 @@ class TestBatchDeliveries:
with pytest.raises(NotConfigured):
build_from_crawler(FeedExporter, crawler)
+ @pytest.mark.parametrize("fmt", ["json", "jsonlines", "xml", "csv"])
@coroutine_test
async def test_export_no_items_not_store_empty(
- self, mockserver: MockServer, tmp_path: Path
+ self, fmt: str, mockserver: MockServer, tmp_path: Path
) -> None:
- for fmt in ("json", "jsonlines", "xml", "csv"):
- settings = {
- "FEEDS": {
- unique_path(tmp_path) / self._file_mark: {"format": fmt},
- },
- "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
- "FEED_STORE_EMPTY": False,
- }
- data = await export_batches(mockserver, [], settings)
- assert len(data[fmt]) == 0
-
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path) / self._file_mark: {"format": fmt},
+ },
+ "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
+ "FEED_STORE_EMPTY": False,
+ }
+ data = await export_batches(mockserver, [], settings)
+ assert len(data[fmt]) == 0
+
+ @pytest.mark.parametrize(
+ ("fmt", "expected"),
+ [
+ pytest.param("json", b"[]", id="json"),
+ pytest.param("jsonlines", b"", id="jsonlines"),
+ pytest.param(
+ "xml",
+ b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>',
+ id="xml",
+ ),
+ pytest.param("csv", b"", id="csv"),
+ ],
+ )
@coroutine_test
async def test_export_no_items_store_empty(
- self, mockserver: MockServer, tmp_path: Path
+ self, fmt: str, expected: bytes, mockserver: MockServer, tmp_path: Path
) -> None:
- formats = (
- ("json", b"[]"),
- ("jsonlines", b""),
- ("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
- ("csv", b""),
- )
-
- for fmt, expctd in formats:
- settings = {
- "FEEDS": {
- unique_path(tmp_path) / self._file_mark: {"format": fmt},
- },
- "FEED_STORE_EMPTY": True,
- "FEED_EXPORT_INDENT": None,
- "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
- }
- data = await export_batches(mockserver, [], settings)
- assert data[fmt][0] == expctd
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path) / self._file_mark: {"format": fmt},
+ },
+ "FEED_STORE_EMPTY": True,
+ "FEED_EXPORT_INDENT": None,
+ "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
+ }
+ data = await export_batches(mockserver, [], settings)
+ assert data[fmt][0] == expected
@coroutine_test
async def test_export_multiple_configs(
diff --git a/tests/test_feedexport_empty.py b/tests/test_feedexport_empty.py
index 5121a4b69..cb7066e22 100644
--- a/tests/test_feedexport_empty.py
+++ b/tests/test_feedexport_empty.py
@@ -5,6 +5,7 @@ import tempfile
from logging import getLogger
from typing import TYPE_CHECKING
+import pytest
from w3lib.url import file_uri_to_path
from tests.utils.decorators import coroutine_test
@@ -13,8 +14,6 @@ from tests.utils.feedexport import export_by_format, unique_path
if TYPE_CHECKING:
from pathlib import Path
- import pytest
-
from tests.mockserver.http import MockServer
@@ -36,42 +35,47 @@ class LogOnStoreFileStorage:
file.close()
[email protected]("fmt", ["json", "jsonlines", "xml", "csv"])
@coroutine_test
async def test_export_no_items_not_store_empty(
- mockserver: MockServer, tmp_path: Path
+ fmt: str, mockserver: MockServer, tmp_path: Path
) -> None:
- for fmt in ("json", "jsonlines", "xml", "csv"):
- settings = {
- "FEEDS": {
- unique_path(tmp_path): {"format": fmt},
- },
- "FEED_STORE_EMPTY": False,
- }
- data = await export_by_format(mockserver, [], settings)
- assert data[fmt] is None
-
-
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path): {"format": fmt},
+ },
+ "FEED_STORE_EMPTY": False,
+ }
+ data = await export_by_format(mockserver, [], settings)
+ assert data[fmt] is None
+
+
+ ("fmt", "expected"),
+ [
+ pytest.param("json", b"[]", id="json"),
+ pytest.param("jsonlines", b"", id="jsonlines"),
+ pytest.param(
+ "xml",
+ b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>',
+ id="xml",
+ ),
+ pytest.param("csv", b"", id="csv"),
+ ],
+)
@coroutine_test
async def test_export_no_items_store_empty(
- mockserver: MockServer, tmp_path: Path
+ fmt: str, expected: bytes, mockserver: MockServer, tmp_path: Path
) -> None:
- formats = (
- ("json", b"[]"),
- ("jsonlines", b""),
- ("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
- ("csv", b""),
- )
-
- for fmt, expctd in formats:
- settings = {
- "FEEDS": {
- unique_path(tmp_path): {"format": fmt},
- },
- "FEED_STORE_EMPTY": True,
- "FEED_EXPORT_INDENT": None,
- }
- data = await export_by_format(mockserver, [], settings)
- assert expctd == data[fmt]
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path): {"format": fmt},
+ },
+ "FEED_STORE_EMPTY": True,
+ "FEED_EXPORT_INDENT": None,
+ }
+ data = await export_by_format(mockserver, [], settings)
+ assert data[fmt] == expected
@coroutine_test
diff --git a/tests/test_feedexport_fields.py b/tests/test_feedexport_fields.py
index 5cf74497f..086606cf9 100644
--- a/tests/test_feedexport_fields.py
+++ b/tests/test_feedexport_fields.py
@@ -153,36 +153,46 @@ async def test_export_tuple(fmt: str, mockserver: MockServer, tmp_path: Path) ->
)
+ ("fields", "header", "rows"),
+ [
+ pytest.param(
+ "foo,baz,egg",
+ ["foo", "baz", "egg"],
+ [
+ {"egg": "spam1", "foo": "bar1"},
+ {"egg": "spam2", "foo": "bar2", "baz": "quux2"},
+ ],
+ id="all",
+ ),
+ pytest.param(
+ "egg,baz",
+ ["egg", "baz"],
+ [{"egg": "spam1"}, {"egg": "spam2", "baz": "quux2"}],
+ id="subset",
+ ),
+ ],
+)
[email protected]("item_cls", [MyItem, dict])
@pytest.mark.parametrize("fmt", list(PARSERS))
@coroutine_test
async def test_export_feed_export_fields(
- fmt: str, mockserver: MockServer, tmp_path: Path
+ fmt: str,
+ item_cls: type[MyItem | dict[str, Any]],
+ fields: str,
+ header: list[str],
+ rows: list[dict[str, Any]],
+ mockserver: MockServer,
+ tmp_path: Path,
) -> None:
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
-
- for item_cls in [MyItem, dict]:
- items = [
- item_cls({"foo": "bar1", "egg": "spam1"}),
- item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
- ]
-
- # export all columns
- settings = {"FEED_EXPORT_FIELDS": "foo,baz,egg"}
- rows = [
- {"egg": "spam1", "foo": "bar1"},
- {"egg": "spam2", "foo": "bar2", "baz": "quux2"},
- ]
- await assert_exported(
- mockserver, tmp_path, fmt, items, ["foo", "baz", "egg"], rows, settings
- )
-
- # export a subset of columns
- settings = {"FEED_EXPORT_FIELDS": "egg,baz"}
- rows = [{"egg": "spam1"}, {"egg": "spam2", "baz": "quux2"}]
- await assert_exported(
- mockserver, tmp_path, fmt, items, ["egg", "baz"], rows, settings
- )
+ items = [
+ item_cls({"foo": "bar1", "egg": "spam1"}),
+ item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+ ]
+ settings = {"FEED_EXPORT_FIELDS": fields}
+ await assert_exported(mockserver, tmp_path, fmt, items, header, rows, settings)
# by default, Scrapy uses fields of the first Item for CSV and
diff --git a/tests/test_feedexport_formats.py b/tests/test_feedexport_formats.py
index bef423b0c..6f5a866e1 100644
--- a/tests/test_feedexport_formats.py
+++ b/tests/test_feedexport_formats.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
import pytest
@@ -35,50 +35,57 @@ async def test_export_items(fmt: str, mockserver: MockServer, tmp_path: Path) ->
await assert_exported(mockserver, tmp_path, fmt, items, header, rows)
+ ("fmt", "encoding", "expected"),
+ [
+ pytest.param("json", None, b'[{"foo": "Test\\u00d6"}]', id="json-default"),
+ pytest.param(
+ "jsonlines", None, b'{"foo": "Test\\u00d6"}\n', id="jsonlines-default"
+ ),
+ pytest.param(
+ "xml",
+ None,
+ (
+ '<?xml version="1.0" encoding="utf-8"?>\n'
+ "<items><item><foo>Test\xd6</foo></item></items>"
+ ).encode(),
+ id="xml-default",
+ ),
+ pytest.param("csv", None, "foo\r\nTest\xd6\r\n".encode(), id="csv-default"),
+ pytest.param("json", "latin-1", b'[{"foo": "Test\xd6"}]', id="json-latin-1"),
+ pytest.param(
+ "jsonlines", "latin-1", b'{"foo": "Test\xd6"}\n', id="jsonlines-latin-1"
+ ),
+ pytest.param(
+ "xml",
+ "latin-1",
+ (
+ b'<?xml version="1.0" encoding="latin-1"?>\n'
+ b"<items><item><foo>Test\xd6</foo></item></items>"
+ ),
+ id="xml-latin-1",
+ ),
+ pytest.param("csv", "latin-1", b"foo\r\nTest\xd6\r\n", id="csv-latin-1"),
+ ],
+)
@coroutine_test
-async def test_export_encoding(mockserver: MockServer, tmp_path: Path) -> None:
+async def test_export_encoding(
+ fmt: str,
+ encoding: str | None,
+ expected: bytes,
+ mockserver: MockServer,
+ tmp_path: Path,
+) -> None:
items = [{"foo": "Test\xd6"}]
-
- formats = {
- "json": b'[{"foo": "Test\\u00d6"}]',
- "jsonlines": b'{"foo": "Test\\u00d6"}\n',
- "xml": (
- '<?xml version="1.0" encoding="utf-8"?>\n'
- "<items><item><foo>Test\xd6</foo></item></items>"
- ).encode(),
- "csv": "foo\r\nTest\xd6\r\n".encode(),
- }
-
- for fmt, expected in formats.items():
- settings: dict[str, Any] = {
- "FEEDS": {
- unique_path(tmp_path): {"format": fmt},
- },
- "FEED_EXPORT_INDENT": None,
- }
- data = await export_by_format(mockserver, items, settings)
- assert data[fmt] == expected
-
- formats = {
- "json": b'[{"foo": "Test\xd6"}]',
- "jsonlines": b'{"foo": "Test\xd6"}\n',
- "xml": (
- b'<?xml version="1.0" encoding="latin-1"?>\n'
- b"<items><item><foo>Test\xd6</foo></item></items>"
- ),
- "csv": b"foo\r\nTest\xd6\r\n",
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path): {"format": fmt},
+ },
+ "FEED_EXPORT_INDENT": None,
+ "FEED_EXPORT_ENCODING": encoding,
}
-
- for fmt, expected in formats.items():
- settings = {
- "FEEDS": {
- unique_path(tmp_path): {"format": fmt},
- },
- "FEED_EXPORT_INDENT": None,
- "FEED_EXPORT_ENCODING": "latin-1",
- }
- data = await export_by_format(mockserver, items, settings)
- assert data[fmt] == expected
+ data = await export_by_format(mockserver, items, settings)
+ assert data[fmt] == expected
@coroutine_test
@@ -122,40 +129,34 @@ async def test_export_multiple_configs(mockserver: MockServer, tmp_path: Path) -
assert data[fmt] == expected
-@coroutine_test
-async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> None:
- items = [
- {"foo": ["bar"]},
- {"key": "value"},
- ]
-
- test_cases: list[dict[str, Any]] = [
- # JSON
- {
- "format": "json",
- "indent": None,
- "expected": b'[{"foo": ["bar"]},{"key": "value"}]',
- },
- {
- "format": "json",
- "indent": -1,
- "expected": b"""[
+ ("fmt", "indent", "expected"),
+ [
+ pytest.param(
+ "json", None, b'[{"foo": ["bar"]},{"key": "value"}]', id="json-None"
+ ),
+ pytest.param(
+ "json",
+ -1,
+ b"""[
{"foo": ["bar"]},
{"key": "value"}
]""",
- },
- {
- "format": "json",
- "indent": 0,
- "expected": b"""[
+ id="json--1",
+ ),
+ pytest.param(
+ "json",
+ 0,
+ b"""[
{"foo": ["bar"]},
{"key": "value"}
]""",
- },
- {
- "format": "json",
- "indent": 2,
- "expected": b"""[
+ id="json-0",
+ ),
+ pytest.param(
+ "json",
+ 2,
+ b"""[
{
"foo": [
"bar"
@@ -165,11 +166,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non
"key": "value"
}
]""",
- },
- {
- "format": "json",
- "indent": 4,
- "expected": b"""[
+ id="json-2",
+ ),
+ pytest.param(
+ "json",
+ 4,
+ b"""[
{
"foo": [
"bar"
@@ -179,11 +181,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non
"key": "value"
}
]""",
- },
- {
- "format": "json",
- "indent": 5,
- "expected": b"""[
+ id="json-4",
+ ),
+ pytest.param(
+ "json",
+ 5,
+ b"""[
{
"foo": [
"bar"
@@ -193,36 +196,39 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non
"key": "value"
}
]""",
- },
- # XML
- {
- "format": "xml",
- "indent": None,
- "expected": b"""<?xml version="1.0" encoding="utf-8"?>
+ id="json-5",
+ ),
+ pytest.param(
+ "xml",
+ None,
+ b"""<?xml version="1.0" encoding="utf-8"?>
<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",
- },
- {
- "format": "xml",
- "indent": -1,
- "expected": b"""<?xml version="1.0" encoding="utf-8"?>
+ id="xml-None",
+ ),
+ pytest.param(
+ "xml",
+ -1,
+ b"""<?xml version="1.0" encoding="utf-8"?>
<items>
<item><foo><value>bar</value></foo></item>
<item><key>value</key></item>
</items>""",
- },
- {
- "format": "xml",
- "indent": 0,
- "expected": b"""<?xml version="1.0" encoding="utf-8"?>
+ id="xml--1",
+ ),
+ pytest.param(
+ "xml",
+ 0,
+ b"""<?xml version="1.0" encoding="utf-8"?>
<items>
<item><foo><value>bar</value></foo></item>
<item><key>value</key></item>
</items>""",
- },
- {
- "format": "xml",
- "indent": 2,
- "expected": b"""<?xml version="1.0" encoding="utf-8"?>
+ id="xml-0",
+ ),
+ pytest.param(
+ "xml",
+ 2,
+ b"""<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
@@ -233,11 +239,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non
<key>value</key>
</item>
</items>""",
- },
- {
- "format": "xml",
- "indent": 4,
- "expected": b"""<?xml version="1.0" encoding="utf-8"?>
+ id="xml-2",
+ ),
+ pytest.param(
+ "xml",
+ 4,
+ b"""<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
@@ -248,11 +255,12 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non
<key>value</key>
</item>
</items>""",
- },
- {
- "format": "xml",
- "indent": 5,
- "expected": b"""<?xml version="1.0" encoding="utf-8"?>
+ id="xml-4",
+ ),
+ pytest.param(
+ "xml",
+ 5,
+ b"""<?xml version="1.0" encoding="utf-8"?>
<items>
<item>
<foo>
@@ -263,55 +271,57 @@ async def test_export_indentation(mockserver: MockServer, tmp_path: Path) -> Non
<key>value</key>
</item>
</items>""",
- },
+ id="xml-5",
+ ),
+ ],
+)
+@coroutine_test
+async def test_export_indentation(
+ fmt: str,
+ indent: int | None,
+ expected: bytes,
+ mockserver: MockServer,
+ tmp_path: Path,
+) -> None:
+ items = [
+ {"foo": ["bar"]},
+ {"key": "value"},
]
-
- for row in test_cases:
- settings = {
- "FEEDS": {
- unique_path(tmp_path): {
- "format": row["format"],
- "indent": row["indent"],
- },
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path): {
+ "format": fmt,
+ "indent": indent,
},
- }
- data = await export_by_format(mockserver, items, settings)
- assert data[row["format"]] == row["expected"]
+ },
+ }
+ data = await export_by_format(mockserver, items, settings)
+ assert data[fmt] == expected
+ ("include_headers_line", "expected"),
+ [
+ pytest.param(True, b"foo,bar\r\nFOO,BAR\r\n", id="with-headers"),
+ pytest.param(False, b"FOO,BAR\r\n", id="without-headers"),
+ ],
+)
@coroutine_test
-async def test_extend_kwargs(mockserver: MockServer, tmp_path: Path) -> None:
+async def test_extend_kwargs(
+ include_headers_line: bool,
+ expected: bytes,
+ mockserver: MockServer,
+ tmp_path: Path,
+) -> None:
items = [{"foo": "FOO", "bar": "BAR"}]
-
- expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"
- expected_without_title_csv = b"FOO,BAR\r\n"
- test_cases: list[dict[str, Any]] = [
- # with title
- {
- "options": {
- "format": "csv",
- "item_export_kwargs": {"include_headers_line": True},
- },
- "expected": expected_with_title_csv,
- },
- # without title
- {
- "options": {
+ settings = {
+ "FEEDS": {
+ unique_path(tmp_path): {
"format": "csv",
- "item_export_kwargs": {"include_headers_line": False},
+ "item_export_kwargs": {"include_headers_line": include_headers_line},
},
- "expected": expected_without_title_csv,
},
- ]
-
- for row in test_cases:
- feed_options = row["options"]
- settings = {
- "FEEDS": {
- unique_path(tmp_path): feed_options,
- },
- "FEED_EXPORT_INDENT": None,
- }
-
- data = await export_by_format(mockserver, items, settings)
- assert data[feed_options["format"]] == row["expected"]
+ "FEED_EXPORT_INDENT": None,
+ }
+ data = await export_by_format(mockserver, items, settings)
+ assert data["csv"] == expected