Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop using TestFeedExportBase in the feed postprocessing tests. (#8277)
Andrey Rakhmatullin00873b47a83e
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.pyindex 69696e65e..94e6160a2 100644--- a/tests/test_feedexport_postprocess.py+++ b/tests/test_feedexport_postprocess.py@@ -7,22 +7,21 @@ import marshalimport pickleimport sysfrom io import BytesIO-from pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKINGimport pytestfrom scrapy.extensions.postprocessing import PostProcessingManager-from scrapy.utils.test import get_crawler-from tests.utils.bases.feedexport import TestFeedExportBasefrom tests.utils.decorators import coroutine_test-from tests.utils.feedexport import path_to_url, printf_escape+from tests.utils.feedexport import export_by_pathif TYPE_CHECKING:- from scrapy import Spider+ from pathlib import Path+ from tests.mockserver.http import MockServer-class TestFeedPostProcessedExports(TestFeedExportBase):++class TestFeedPostProcessedExports:items = [{"foo": "bar"}]expected = b"foo\r\nbar\r\n"@@ -40,40 +39,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):def close(self):self.file.close()- def _named_tempfile(self, name) -> str:- return str(Path(self.temp_dir, name))-- async def run_and_export(- self, spider_cls: type[Spider], settings: dict[str, Any]- ) -> dict[str, bytes | None]:- """Run spider with specified settings; return exported data with filename."""-- FEEDS = settings.get("FEEDS") or {}- settings["FEEDS"] = {- printf_escape(path_to_url(file_path)): feed_options- for file_path, feed_options in FEEDS.items()- }-- content: dict[str, bytes | None] = {}- try:- spider_cls.start_urls = [self.mockserver.url("/")]- crawler = get_crawler(spider_cls, settings)- await crawler.crawl_async()-- for file_path in FEEDS:- content[str(file_path)] = (- Path(file_path).read_bytes() if Path(file_path).exists() else None- )-- finally:- for file_path in FEEDS:- if not Path(file_path).exists():- continue-- Path(file_path).unlink()-- return content-def get_gzip_compressed(self,data: bytes,@@ -104,8 +69,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert manager.tell() == file.tell() == 3@coroutine_test- async def test_gzip_plugin(self):- filename = self._named_tempfile("gzip_file")+ async def test_gzip_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:+ filename = tmp_path / "gzip_file"settings = {"FEEDS": {@@ -116,33 +81,35 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)try:gzip.decompress(data[filename])except OSError:pytest.fail("Received invalid gzip data.")@coroutine_test- async def test_gzip_plugin_compresslevel(self):+ async def test_gzip_plugin_compresslevel(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(+ tmp_path / "compresslevel_0": self.get_gzip_compressed(self.expected, compresslevel=0),- self._named_tempfile("compresslevel_9"): self.get_gzip_compressed(+ tmp_path / "compresslevel_9": self.get_gzip_compressed(self.expected, compresslevel=9),}settings = {"FEEDS": {- self._named_tempfile("compresslevel_0"): {+ tmp_path / "compresslevel_0": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],"gzip_compresslevel": 0,"gzip_mtime": 0,"gzip_filename": "",},- self._named_tempfile("compresslevel_9"): {+ tmp_path / "compresslevel_9": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],"gzip_compresslevel": 9,@@ -152,7 +119,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = gzip.decompress(data[filename])@@ -160,25 +127,25 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_gzip_plugin_mtime(self):+ async def test_gzip_plugin_mtime(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("mtime_123"): self.get_gzip_compressed(- self.expected, mtime=123- ),- self._named_tempfile("mtime_123456789"): self.get_gzip_compressed(+ tmp_path / "mtime_123": self.get_gzip_compressed(self.expected, mtime=123),+ tmp_path / "mtime_123456789": self.get_gzip_compressed(self.expected, mtime=123456789),}settings = {"FEEDS": {- self._named_tempfile("mtime_123"): {+ tmp_path / "mtime_123": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],"gzip_mtime": 123,"gzip_filename": "",},- self._named_tempfile("mtime_123456789"): {+ tmp_path / "mtime_123456789": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],"gzip_mtime": 123456789,@@ -187,7 +154,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = gzip.decompress(data[filename])@@ -195,25 +162,27 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_gzip_plugin_filename(self):+ async def test_gzip_plugin_filename(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(+ tmp_path / "filename_FILE1": self.get_gzip_compressed(self.expected, filename="FILE1"),- self._named_tempfile("filename_FILE2"): self.get_gzip_compressed(+ tmp_path / "filename_FILE2": self.get_gzip_compressed(self.expected, filename="FILE2"),}settings = {"FEEDS": {- self._named_tempfile("filename_FILE1"): {+ tmp_path / "filename_FILE1": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],"gzip_mtime": 0,"gzip_filename": "FILE1",},- self._named_tempfile("filename_FILE2"): {+ tmp_path / "filename_FILE2": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],"gzip_mtime": 0,@@ -222,7 +191,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = gzip.decompress(data[filename])@@ -230,8 +199,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_lzma_plugin(self):- filename = self._named_tempfile("lzma_file")+ async def test_lzma_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:+ filename = tmp_path / "lzma_file"settings = {"FEEDS": {@@ -242,31 +211,33 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)try:lzma.decompress(data[filename])except lzma.LZMAError:pytest.fail("Received invalid lzma data.")@coroutine_test- async def test_lzma_plugin_format(self):+ async def test_lzma_plugin_format(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("format_FORMAT_XZ"): lzma.compress(+ tmp_path / "format_FORMAT_XZ": lzma.compress(self.expected, format=lzma.FORMAT_XZ),- self._named_tempfile("format_FORMAT_ALONE"): lzma.compress(+ tmp_path / "format_FORMAT_ALONE": lzma.compress(self.expected, format=lzma.FORMAT_ALONE),}settings = {"FEEDS": {- self._named_tempfile("format_FORMAT_XZ"): {+ tmp_path / "format_FORMAT_XZ": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],"lzma_format": lzma.FORMAT_XZ,},- self._named_tempfile("format_FORMAT_ALONE"): {+ tmp_path / "format_FORMAT_ALONE": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],"lzma_format": lzma.FORMAT_ALONE,@@ -274,7 +245,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = lzma.decompress(data[filename])@@ -282,24 +253,26 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_lzma_plugin_check(self):+ async def test_lzma_plugin_check(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("check_CHECK_NONE"): lzma.compress(+ tmp_path / "check_CHECK_NONE": lzma.compress(self.expected, check=lzma.CHECK_NONE),- self._named_tempfile("CHECK_SHA256"): lzma.compress(+ tmp_path / "CHECK_SHA256": lzma.compress(self.expected, check=lzma.CHECK_SHA256),}settings = {"FEEDS": {- self._named_tempfile("check_CHECK_NONE"): {+ tmp_path / "check_CHECK_NONE": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],"lzma_check": lzma.CHECK_NONE,},- self._named_tempfile("CHECK_SHA256"): {+ tmp_path / "CHECK_SHA256": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],"lzma_check": lzma.CHECK_SHA256,@@ -307,7 +280,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = lzma.decompress(data[filename])@@ -315,24 +288,22 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_lzma_plugin_preset(self):+ async def test_lzma_plugin_preset(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("preset_PRESET_0"): lzma.compress(- self.expected, preset=0- ),- self._named_tempfile("preset_PRESET_9"): lzma.compress(- self.expected, preset=9- ),+ tmp_path / "preset_PRESET_0": lzma.compress(self.expected, preset=0),+ tmp_path / "preset_PRESET_9": lzma.compress(self.expected, preset=9),}settings = {"FEEDS": {- self._named_tempfile("preset_PRESET_0"): {+ tmp_path / "preset_PRESET_0": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],"lzma_preset": 0,},- self._named_tempfile("preset_PRESET_9"): {+ tmp_path / "preset_PRESET_9": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],"lzma_preset": 9,@@ -340,7 +311,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = lzma.decompress(data[filename])@@ -348,14 +319,16 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_lzma_plugin_filters(self):+ async def test_lzma_plugin_filters(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:if "PyPy" in sys.version:# https://foss.heptapod.net/pypy/pypy/-/issues/3527pytest.skip("lzma filters doesn't work in PyPy")filters = [{"id": lzma.FILTER_LZMA2}]compressed = lzma.compress(self.expected, filters=filters)- filename = self._named_tempfile("filters")+ filename = tmp_path / "filters"settings = {"FEEDS": {@@ -367,14 +340,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)assert compressed == data[filename]result = lzma.decompress(data[filename])assert result == self.expected@coroutine_test- async def test_bz2_plugin(self):- filename = self._named_tempfile("bz2_file")+ async def test_bz2_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:+ filename = tmp_path / "bz2_file"settings = {"FEEDS": {@@ -385,31 +358,29 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)try:bz2.decompress(data[filename])except OSError:pytest.fail("Received invalid bz2 data.")@coroutine_test- async def test_bz2_plugin_compresslevel(self):+ async def test_bz2_plugin_compresslevel(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_compressed = {- self._named_tempfile("compresslevel_1"): bz2.compress(- self.expected, compresslevel=1- ),- self._named_tempfile("compresslevel_9"): bz2.compress(- self.expected, compresslevel=9- ),+ tmp_path / "compresslevel_1": bz2.compress(self.expected, compresslevel=1),+ tmp_path / "compresslevel_9": bz2.compress(self.expected, compresslevel=9),}settings = {"FEEDS": {- self._named_tempfile("compresslevel_1"): {+ tmp_path / "compresslevel_1": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],"bz2_compresslevel": 1,},- self._named_tempfile("compresslevel_9"): {+ tmp_path / "compresslevel_9": {"format": "csv","postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],"bz2_compresslevel": 9,@@ -417,7 +388,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, compressed in filename_to_compressed.items():result = bz2.decompress(data[filename])@@ -425,8 +396,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == self.expected@coroutine_test- async def test_custom_plugin(self):- filename = self._named_tempfile("csv_file")+ async def test_custom_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:+ filename = tmp_path / "csv_file"settings = {"FEEDS": {@@ -437,13 +408,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)assert data[filename] == self.expected@coroutine_test- async def test_custom_plugin_with_parameter(self):+ async def test_custom_plugin_with_parameter(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:expected = b"foo\r\n\nbar\r\n\n"- filename = self._named_tempfile("newline")+ filename = tmp_path / "newline"settings = {"FEEDS": {@@ -455,22 +428,24 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)assert data[filename] == expected@coroutine_test- async def test_custom_plugin_with_compression(self):+ async def test_custom_plugin_with_compression(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:expected = b"foo\r\n\nbar\r\n\n"filename_to_decompressor = {- self._named_tempfile("bz2"): bz2.decompress,- self._named_tempfile("lzma"): lzma.decompress,- self._named_tempfile("gzip"): gzip.decompress,+ tmp_path / "bz2": bz2.decompress,+ tmp_path / "lzma": lzma.decompress,+ tmp_path / "gzip": gzip.decompress,}settings = {"FEEDS": {- self._named_tempfile("bz2"): {+ tmp_path / "bz2": {"format": "csv","postprocessing": [self.MyPlugin1,@@ -478,7 +453,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):],"plugin1_char": b"\n",},- self._named_tempfile("lzma"): {+ tmp_path / "lzma": {"format": "csv","postprocessing": [self.MyPlugin1,@@ -486,7 +461,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):],"plugin1_char": b"\n",},- self._named_tempfile("gzip"): {+ tmp_path / "gzip": {"format": "csv","postprocessing": [self.MyPlugin1,@@ -497,61 +472,57 @@ class TestFeedPostProcessedExports(TestFeedExportBase):},}- data = await self.exported_data(self.items, settings)+ data = await export_by_path(mockserver, self.items, settings)for filename, decompressor in filename_to_decompressor.items():result = decompressor(data[filename])assert result == expected@coroutine_test- async def test_exports_compatibility_with_postproc(self):+ async def test_exports_compatibility_with_postproc(+ self, mockserver: MockServer, tmp_path: Path+ ) -> None:filename_to_expected = {- self._named_tempfile("csv"): b"foo\r\nbar\r\n",- self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',- self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n',- self._named_tempfile("xml"): b'<?xml version="1.0" encoding="utf-8"?>\n'+ tmp_path / "csv": b"foo\r\nbar\r\n",+ tmp_path / "json": b'[\n{"foo": "bar"}\n]',+ tmp_path / "jsonlines": b'{"foo": "bar"}\n',+ tmp_path / "xml": b'<?xml version="1.0" encoding="utf-8"?>\n'b"<items>\n<item><foo>bar</foo></item>\n</items>",}settings = {"FEEDS": {- self._named_tempfile("csv"): {+ tmp_path / "csv": {"format": "csv","postprocessing": [self.MyPlugin1],# empty plugin to activate postprocessing.PostProcessingManager},- self._named_tempfile("json"): {+ tmp_path / "json": {"format": "json","postprocessing": [self.MyPlugin1],},- self._named_tempfile("jsonlines"): {+ tmp_path / "jsonlines": {"format": "jsonlines","postprocessing": [self.MyPlugin1],},- self._named_tempfile("xml"): {+ tmp_path / "xml": {"format": "xml","postprocessing": [self.MyPlugin1],},- self._named_tempfile("marshal"): {+ tmp_path / "marshal": {"format": "marshal","postprocessing": [self.MyPlugin1],},- self._named_tempfile("pickle"): {+ tmp_path / "pickle": {"format": "pickle","postprocessing": [self.MyPlugin1],},},}- data = await self.exported_data(self.items, settings)-- for filename, data_bytes in data.items():- expected: Any- result: Any- if "pickle" in filename:- expected, result = self.items[0], pickle.loads(data_bytes)- elif "marshal" in filename:- expected, result = self.items[0], marshal.loads(data_bytes)- else:- expected, result = filename_to_expected[filename], data_bytes- assert result == expected+ data = await export_by_path(mockserver, self.items, settings)++ for path, expected in filename_to_expected.items():+ assert data[path] == expected+ assert pickle.loads(data[tmp_path / "pickle"]) == self.items[0]+ assert marshal.loads(data[tmp_path / "marshal"]) == self.items[0]diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.pyindex 9c5e5bc9d..1b46ad94f 100644--- a/tests/utils/feedexport.py+++ b/tests/utils/feedexport.py@@ -1,5 +1,6 @@from __future__ import annotations+from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urljoinfrom urllib.request import pathname2url@@ -9,7 +10,6 @@ from scrapy.utils.test import get_crawlerif TYPE_CHECKING:from collections.abc import Iterable- from pathlib import Pathfrom scrapy.crawler import Crawlerfrom tests.mockserver.http import MockServer@@ -50,3 +50,16 @@ async def crawl_items(crawler = get_crawler(TestSpider, settings)await crawler.crawl_async()return crawler+++async def export_by_path(+ mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]+) -> dict[str | Path, bytes]:+ """Export *items* with *settings*, whose ``FEEDS`` are keyed by local+ paths, and return the contents of those paths that were written."""+ feeds = settings["FEEDS"]+ feed_urls = {+ printf_escape(path_to_url(path)): options for path, options in feeds.items()+ }+ await crawl_items(mockserver, items, {**settings, "FEEDS": feed_urls})+ return {path: Path(path).read_bytes() for path in feeds if Path(path).exists()}