Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop using TestFeedExportBase in the batch delivery tests. (#8278)
Andrey Rakhmatullin759367da202a
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 43550d7a8..14c35d968 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -1,32 +1,24 @@from __future__ import annotations-import csvimport json-import marshal-import pickle-from collections import defaultdictfrom pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urljoin-import lxml.etreeimport pytestfrom packaging.version import Versionimport scrapy-from scrapy import Spiderfrom scrapy.exceptions import NotConfiguredfrom scrapy.extensions.feedexport import FeedExporter, S3FeedStorage-from scrapy.settings import Settingsfrom scrapy.utils.misc import build_from_crawler-from scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawlerfrom tests.spiders import ItemSpider-from tests.utils.bases.feedexport import TestFeedExportBase-from tests.utils.decorators import coroutine_test, inline_callbacks_test-from tests.utils.feedexport import MyItem, crawl_items+from tests.utils.decorators import coroutine_test+from tests.utils.feedexport import PARSERS, MyItem, crawl_items, csv_header, unique_pathif TYPE_CHECKING:+ from collections.abc import Iterablefrom os import PathLikefrom tests.mockserver.http import MockServer@@ -39,187 +31,64 @@ def build_url(path: str | PathLike[str]) -> str:return urljoin("file:", path_str)-class TestBatchDeliveries(TestFeedExportBase):- _file_mark = "_%(batch_time)s_%(batch_id)02d_"-- async def run_and_export(- self, spider_cls: type[Spider], settings: dict[str, Any]- ) -> dict[str, list[bytes]]:- """Run spider with specified settings; return exported data."""-- FEEDS = settings.get("FEEDS") or {}- settings["FEEDS"] = {- build_url(file_path): feed for file_path, feed in FEEDS.items()- }- content: defaultdict[str, list[bytes]] = defaultdict(list)- spider_cls.start_urls = [self.mockserver.url("/")]- crawler = get_crawler(spider_cls, settings)- await crawler.crawl_async()+async def export_batches(+ mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]+) -> dict[str, list[bytes]]:+ """Export *items* with *settings*, whose ``FEEDS`` are keyed by local+ batch paths with a separate parent directory for each feed, and return+ the contents of the batch files of each feed format, in file name order."""+ feeds = settings["FEEDS"]+ feed_urls = {build_url(path): options for path, options in feeds.items()}+ await crawl_items(mockserver, items, {**settings, "FEEDS": feed_urls})+ return {+ options["format"]: [+ file.read_bytes() for file in sorted(Path(path).parent.iterdir())+ ]+ if Path(path).parent.exists()+ else []+ for path, options in feeds.items()+ }- for path, feed in FEEDS.items():- dir_name = Path(path).parent- if not dir_name.exists():- content[feed["format"]] = []- continue- for file in sorted(dir_name.iterdir()):- content[feed["format"]].append(file.read_bytes())- return content-- async def assertExportedJsonLines(self, items, rows, settings=None):- settings = settings or {}- settings.update(- {- "FEEDS": {- self._random_temp_filename() / "jl" / self._file_mark: {- "format": "jl"- },- },- }- )- batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")- rows = [{k: v for k, v in row.items() if v} for row in rows]- data = await self.exported_data(items, settings)- for batch in data["jl"]:- got_batch = [- json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()- ]- expected_batch, rows = rows[:batch_size], rows[batch_size:]- assert got_batch == expected_batch-- async def assertExportedCsv(self, items, header, rows, settings=None):- settings = settings or {}- settings.update(- {- "FEEDS": {- self._random_temp_filename() / "csv" / self._file_mark: {- "format": "csv"- },- },- }- )- batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")- data = await self.exported_data(items, settings)- for batch in data["csv"]:- got_batch = csv.DictReader(to_unicode(batch).splitlines())- assert list(header) == got_batch.fieldnames- expected_batch, rows = rows[:batch_size], rows[batch_size:]- assert list(got_batch) == expected_batch-- async def assertExportedXml(self, items, rows, settings=None):- settings = settings or {}- settings.update(- {- "FEEDS": {- self._random_temp_filename() / "xml" / self._file_mark: {- "format": "xml"- },- },- }- )- batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")- rows = [{k: v for k, v in row.items() if v} for row in rows]- data = await self.exported_data(items, settings)- for batch in data["xml"]:- root = lxml.etree.fromstring(batch)- got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")]- expected_batch, rows = rows[:batch_size], rows[batch_size:]- assert got_batch == expected_batch-- async def assertExportedMultiple(self, items, rows, settings=None):- settings = settings or {}- settings.update(- {- "FEEDS": {- self._random_temp_filename() / "xml" / self._file_mark: {- "format": "xml"- },- self._random_temp_filename() / "json" / self._file_mark: {- "format": "json"- },- },- }- )- batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")- rows = [{k: v for k, v in row.items() if v} for row in rows]- data = await self.exported_data(items, settings)- # XML- xml_rows = rows.copy()- for batch in data["xml"]:- root = lxml.etree.fromstring(batch)- got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")]- expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]- assert got_batch == expected_batch- # JSON- json_rows = rows.copy()- for batch in data["json"]:- got_batch = json.loads(batch.decode("utf-8"))- expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]- assert got_batch == expected_batch-- async def assertExportedPickle(self, items, rows, settings=None):- settings = settings or {}- settings.update(- {- "FEEDS": {- self._random_temp_filename() / "pickle" / self._file_mark: {- "format": "pickle"- },- },- }- )- batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")- rows = [{k: v for k, v in row.items() if v} for row in rows]- data = await self.exported_data(items, settings)-- for batch in data["pickle"]:- got_batch = self._load_until_eof(batch, load_func=pickle.load)- expected_batch, rows = rows[:batch_size], rows[batch_size:]- assert got_batch == expected_batch-- async def assertExportedMarshal(self, items, rows, settings=None):- settings = settings or {}- settings.update(- {- "FEEDS": {- self._random_temp_filename() / "marshal" / self._file_mark: {- "format": "marshal"- },- },- }- )- batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")- rows = [{k: v for k, v in row.items() if v} for row in rows]- data = await self.exported_data(items, settings)- for batch in data["marshal"]:- got_batch = self._load_until_eof(batch, load_func=marshal.load)- expected_batch, rows = rows[:batch_size], rows[batch_size:]- assert got_batch == expected_batch+class TestBatchDeliveries:+ _file_mark = "_%(batch_time)s_%(batch_id)02d_"+ @pytest.mark.parametrize("fmt", list(PARSERS))@coroutine_test- async def test_export_items(self):+ async def test_export_items(+ self, fmt: str, mockserver: MockServer, tmp_path: Path+ ) -> None:"""Test partial deliveries in all supported formats"""items = [MyItem({"foo": "bar1", "egg": "spam1"}),MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),MyItem({"foo": "bar3", "baz": "quux3"}),]- rows = [- {"egg": "spam1", "foo": "bar1", "baz": ""},- {"egg": "spam2", "foo": "bar2", "baz": "quux2"},- {"foo": "bar3", "baz": "quux3", "egg": ""},+ settings = {+ "FEEDS": {unique_path(tmp_path) / self._file_mark: {"format": fmt}},+ "FEED_EXPORT_BATCH_ITEM_COUNT": 2,+ }+ batches = (await export_batches(mockserver, items, settings))[fmt]+ if fmt == "csv":+ header = list(MyItem.fields)+ assert [csv_header(batch) for batch in batches] == [header, header]+ assert [PARSERS[fmt](batch) for batch in batches] == [+ [+ {"egg": "spam1", "foo": "bar1"},+ {"egg": "spam2", "foo": "bar2", "baz": "quux2"},+ ],+ [{"foo": "bar3", "baz": "quux3"}],]- settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}- header = MyItem.fields.keys()- await self.assertExported(items, header, rows, settings=settings)@coroutine_test- async def test_batch_delivered_when_full(self):+ async def test_batch_delivered_when_full(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:"""Full batches must be finalized and delivered as soon as they arefull, instead of when the spider closes."""- dir_path = self._random_temp_filename()+ dir_path = unique_path(tmp_path)batch1_path = Path(dir_path, "1.json")- mockserver_url = self.mockserver.url("/")+ mockserver_url = mockserver.url("/")batch1_contents: list[bytes | None] = []class TestSpider(scrapy.Spider):@@ -254,11 +123,11 @@ class TestBatchDeliveries(TestFeedExportBase):assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl"assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}]- def test_wrong_path(self):+ def test_wrong_path(self, tmp_path: Path) -> None:"""If path is without %(batch_time)s and %(batch_id) an exception must be raised"""settings = {"FEEDS": {- self._random_temp_filename(): {"format": "xml"},+ unique_path(tmp_path): {"format": "xml"},},"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}@@ -267,23 +136,24 @@ class TestBatchDeliveries(TestFeedExportBase):build_from_crawler(FeedExporter, crawler)@coroutine_test- async def test_export_no_items_not_store_empty(self):+ async def test_export_no_items_not_store_empty(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:for fmt in ("json", "jsonlines", "xml", "csv"):settings = {"FEEDS": {- self._random_temp_filename() / fmt / self._file_mark: {- "format": fmt- },+ unique_path(tmp_path) / self._file_mark: {"format": fmt},},"FEED_EXPORT_BATCH_ITEM_COUNT": 1,"FEED_STORE_EMPTY": False,}- data = await self.exported_no_data(settings)- data = dict(data)+ data = await export_batches(mockserver, [], settings)assert len(data[fmt]) == 0@coroutine_test- async def test_export_no_items_store_empty(self):+ async def test_export_no_items_store_empty(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:formats = (("json", b"[]"),("jsonlines", b""),@@ -294,20 +164,19 @@ class TestBatchDeliveries(TestFeedExportBase):for fmt, expctd in formats:settings = {"FEEDS": {- self._random_temp_filename() / fmt / self._file_mark: {- "format": fmt- },+ unique_path(tmp_path) / self._file_mark: {"format": fmt},},"FEED_STORE_EMPTY": True,"FEED_EXPORT_INDENT": None,"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}- data = await self.exported_no_data(settings)- data = dict(data)+ data = await export_batches(mockserver, [], settings)assert data[fmt][0] == expctd@coroutine_test- async def test_export_multiple_configs(self):+ async def test_export_multiple_configs(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:items = [{"foo": "FOO", "bar": "BAR"},{"foo": "FOO1", "bar": "BAR1"},@@ -336,19 +205,19 @@ class TestBatchDeliveries(TestFeedExportBase):settings = {"FEEDS": {- self._random_temp_filename() / "json" / self._file_mark: {+ unique_path(tmp_path) / self._file_mark: {"format": "json","indent": 0,"fields": ["bar"],"encoding": "utf-8",},- self._random_temp_filename() / "xml" / self._file_mark: {+ unique_path(tmp_path) / self._file_mark: {"format": "xml","indent": 2,"fields": ["foo"],"encoding": "latin-1",},- self._random_temp_filename() / "csv" / self._file_mark: {+ unique_path(tmp_path) / self._file_mark: {"format": "csv","indent": None,"fields": ["foo", "bar"],@@ -357,13 +226,15 @@ class TestBatchDeliveries(TestFeedExportBase):},"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}- data = await self.exported_data(items, settings)+ data = await export_batches(mockserver, items, settings)for fmt, expected in formats.items():for expected_batch, got_batch in zip(expected, data[fmt], strict=True):assert got_batch == expected_batch@coroutine_test- async def test_batch_item_count_feeds_setting(self):+ async def test_batch_item_count_feeds_setting(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:items = [{"foo": "FOO"}, {"foo": "FOO1"}]formats = {"json": [@@ -373,7 +244,7 @@ class TestBatchDeliveries(TestFeedExportBase):}settings = {"FEEDS": {- self._random_temp_filename() / "json" / self._file_mark: {+ unique_path(tmp_path) / self._file_mark: {"format": "json","indent": None,"encoding": "utf-8",@@ -381,13 +252,15 @@ class TestBatchDeliveries(TestFeedExportBase):},},}- data = await self.exported_data(items, settings)+ data = await export_batches(mockserver, items, settings)for fmt, expected in formats.items():for expected_batch, got_batch in zip(expected, data[fmt], strict=True):assert got_batch == expected_batch@coroutine_test- async def test_batch_item_count_with_item_processor(self):+ async def test_batch_item_count_with_item_processor(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:def split_foo(item):for value in item["foo"].split(","):yield {"foo": value}@@ -402,7 +275,7 @@ class TestBatchDeliveries(TestFeedExportBase):}settings = {"FEEDS": {- self._random_temp_filename() / "json" / self._file_mark: {+ unique_path(tmp_path) / self._file_mark: {"format": "json","indent": None,"encoding": "utf-8",@@ -411,13 +284,15 @@ class TestBatchDeliveries(TestFeedExportBase):},},}- data = await self.exported_data(items, settings)+ data = await export_batches(mockserver, items, settings)for fmt, expected in formats.items():for expected_batch, got_batch in zip(expected, data[fmt], strict=True):assert got_batch == expected_batch@coroutine_test- async def test_batch_path_differ(self):+ async def test_batch_path_differ(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:"""Test that the name of all batch files differ from each other.So %(batch_id)d replaced with the current id.@@ -429,29 +304,29 @@ class TestBatchDeliveries(TestFeedExportBase):]settings = {"FEEDS": {- self._random_temp_filename() / "%(batch_id)d": {+ unique_path(tmp_path) / "%(batch_id)d": {"format": "json",},},"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}- data = await self.exported_data(items, settings)+ data = await export_batches(mockserver, items, settings)assert len(items) == len(data["json"])- @inline_callbacks_test- def test_stats_batch_file_success(self):+ @coroutine_test+ async def test_stats_batch_file_success(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:settings = {"FEEDS": {- build_url(- str(self._random_temp_filename() / "json" / self._file_mark)- ): {+ build_url(str(unique_path(tmp_path) / self._file_mark)): {"format": "json",}},"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}crawler = get_crawler(ItemSpider, settings)- yield crawler.crawl(total=2, mockserver=self.mockserver)+ await crawler.crawl_async(total=2, mockserver=mockserver)assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.pyindex 1b46ad94f..d5b2a9db8 100644--- a/tests/utils/feedexport.py+++ b/tests/utils/feedexport.py@@ -1,15 +1,23 @@from __future__ import annotations+import csv+import itertools+import json+import marshal+import pickle+from io import BytesIOfrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import IO, TYPE_CHECKING, Anyfrom urllib.parse import urljoinfrom urllib.request import pathname2url+import lxml.etree+from scrapy import Field, Item, Spiderfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:- from collections.abc import Iterable+ from collections.abc import Callable, Iterablefrom scrapy.crawler import Crawlerfrom tests.mockserver.http import MockServer@@ -34,6 +42,14 @@ class MyItem2(Item):hello = Field()+_path_ids = itertools.count()+++def unique_path(tmp_path: Path) -> Path:+ """Return a path inside *tmp_path* that no previous call returned."""+ return tmp_path / f"feed{next(_path_ids)}"++async def crawl_items(mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]) -> Crawler:@@ -63,3 +79,55 @@ async def export_by_path(}await crawl_items(mockserver, items, {**settings, "FEEDS": feed_urls})return {path: Path(path).read_bytes() for path in feeds if Path(path).exists()}+++def csv_header(data: bytes) -> list[str]:+ """Return the column names of the CSV feed *data*."""+ return next(csv.reader(data.decode().splitlines()))+++def _parse_csv(data: bytes) -> list[Any]:+ # CSV cannot tell an empty field from a missing one, so empty fields are+ # left out, matching how the other formats export missing fields.+ return [+ {k: v for k, v in row.items() if v}+ for row in csv.DictReader(data.decode().splitlines())+ ]+++def _parse_jsonlines(data: bytes) -> list[Any]:+ return [json.loads(line) for line in data.splitlines()]+++def _parse_xml(data: bytes) -> list[Any]:+ root = lxml.etree.fromstring(data)+ return [{e.tag: e.text for e in item} for item in root.findall("item")]+++def _load_until_eof(data: bytes, load: Callable[[IO[bytes]], Any]) -> list[Any]:+ file = BytesIO(data)+ result: list[Any] = []+ while True:+ try:+ result.append(load(file))+ except EOFError:+ return result+++def _parse_pickle(data: bytes) -> list[Any]:+ return _load_until_eof(data, pickle.load)+++def _parse_marshal(data: bytes) -> list[Any]:+ return _load_until_eof(data, marshal.load)+++#: Functions that turn feed contents back into items, by feed format.+PARSERS: dict[str, Callable[[bytes], list[Any]]] = {+ "csv": _parse_csv,+ "json": json.loads,+ "jsonlines": _parse_jsonlines,+ "xml": _parse_xml,+ "pickle": _parse_pickle,+ "marshal": _parse_marshal,+}