Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Stop using TestFeedExportBase in the feed postprocessing tests. (#8277)

Andrey Rakhmatullin00873b47a83e
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index 69696e65e..94e6160a2 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -7,22 +7,21 @@ import marshal
import pickle
import sys
from io import BytesIO
-from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
import pytest
from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.utils.test import get_crawler
-from tests.utils.bases.feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test
-from tests.utils.feedexport import path_to_url, printf_escape
+from tests.utils.feedexport import export_by_path
if TYPE_CHECKING:
- from scrapy import Spider
+ from pathlib import Path
+ from tests.mockserver.http import MockServer
-class TestFeedPostProcessedExports(TestFeedExportBase):
+
+class TestFeedPostProcessedExports:
items = [{"foo": "bar"}]
expected = b"foo\r\nbar\r\n"
@@ -40,40 +39,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
def close(self):
self.file.close()
- def _named_tempfile(self, name) -> str:
- return str(Path(self.temp_dir, name))
-
- async def run_and_export(
- self, spider_cls: type[Spider], settings: dict[str, Any]
- ) -> dict[str, bytes | None]:
- """Run spider with specified settings; return exported data with filename."""
-
- FEEDS = settings.get("FEEDS") or {}
- settings["FEEDS"] = {
- printf_escape(path_to_url(file_path)): feed_options
- for file_path, feed_options in FEEDS.items()
- }
-
- content: dict[str, bytes | None] = {}
- try:
- spider_cls.start_urls = [self.mockserver.url("/")]
- crawler = get_crawler(spider_cls, settings)
- await crawler.crawl_async()
-
- for file_path in FEEDS:
- content[str(file_path)] = (
- Path(file_path).read_bytes() if Path(file_path).exists() else None
- )
-
- finally:
- for file_path in FEEDS:
- if not Path(file_path).exists():
- continue
-
- Path(file_path).unlink()
-
- return content
-
def get_gzip_compressed(
self,
data: bytes,
@@ -104,8 +69,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert manager.tell() == file.tell() == 3
@coroutine_test
- async def test_gzip_plugin(self):
- filename = self._named_tempfile("gzip_file")
+ async def test_gzip_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:
+ filename = tmp_path / "gzip_file"
settings = {
"FEEDS": {
@@ -116,33 +81,35 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
try:
gzip.decompress(data[filename])
except OSError:
pytest.fail("Received invalid gzip data.")
@coroutine_test
- async def test_gzip_plugin_compresslevel(self):
+ async def test_gzip_plugin_compresslevel(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
+ tmp_path / "compresslevel_0": self.get_gzip_compressed(
self.expected, compresslevel=0
),
- self._named_tempfile("compresslevel_9"): self.get_gzip_compressed(
+ tmp_path / "compresslevel_9": self.get_gzip_compressed(
self.expected, compresslevel=9
),
}
settings = {
"FEEDS": {
- self._named_tempfile("compresslevel_0"): {
+ tmp_path / "compresslevel_0": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_compresslevel": 0,
"gzip_mtime": 0,
"gzip_filename": "",
},
- self._named_tempfile("compresslevel_9"): {
+ tmp_path / "compresslevel_9": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_compresslevel": 9,
@@ -152,7 +119,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = gzip.decompress(data[filename])
@@ -160,25 +127,25 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_gzip_plugin_mtime(self):
+ async def test_gzip_plugin_mtime(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("mtime_123"): self.get_gzip_compressed(
- self.expected, mtime=123
- ),
- self._named_tempfile("mtime_123456789"): self.get_gzip_compressed(
+ tmp_path / "mtime_123": self.get_gzip_compressed(self.expected, mtime=123),
+ tmp_path / "mtime_123456789": self.get_gzip_compressed(
self.expected, mtime=123456789
),
}
settings = {
"FEEDS": {
- self._named_tempfile("mtime_123"): {
+ tmp_path / "mtime_123": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 123,
"gzip_filename": "",
},
- self._named_tempfile("mtime_123456789"): {
+ tmp_path / "mtime_123456789": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 123456789,
@@ -187,7 +154,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = gzip.decompress(data[filename])
@@ -195,25 +162,27 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_gzip_plugin_filename(self):
+ async def test_gzip_plugin_filename(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(
+ tmp_path / "filename_FILE1": self.get_gzip_compressed(
self.expected, filename="FILE1"
),
- self._named_tempfile("filename_FILE2"): self.get_gzip_compressed(
+ tmp_path / "filename_FILE2": self.get_gzip_compressed(
self.expected, filename="FILE2"
),
}
settings = {
"FEEDS": {
- self._named_tempfile("filename_FILE1"): {
+ tmp_path / "filename_FILE1": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 0,
"gzip_filename": "FILE1",
},
- self._named_tempfile("filename_FILE2"): {
+ tmp_path / "filename_FILE2": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 0,
@@ -222,7 +191,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = gzip.decompress(data[filename])
@@ -230,8 +199,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_lzma_plugin(self):
- filename = self._named_tempfile("lzma_file")
+ async def test_lzma_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:
+ filename = tmp_path / "lzma_file"
settings = {
"FEEDS": {
@@ -242,31 +211,33 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
try:
lzma.decompress(data[filename])
except lzma.LZMAError:
pytest.fail("Received invalid lzma data.")
@coroutine_test
- async def test_lzma_plugin_format(self):
+ async def test_lzma_plugin_format(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
+ tmp_path / "format_FORMAT_XZ": lzma.compress(
self.expected, format=lzma.FORMAT_XZ
),
- self._named_tempfile("format_FORMAT_ALONE"): lzma.compress(
+ tmp_path / "format_FORMAT_ALONE": lzma.compress(
self.expected, format=lzma.FORMAT_ALONE
),
}
settings = {
"FEEDS": {
- self._named_tempfile("format_FORMAT_XZ"): {
+ tmp_path / "format_FORMAT_XZ": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_format": lzma.FORMAT_XZ,
},
- self._named_tempfile("format_FORMAT_ALONE"): {
+ tmp_path / "format_FORMAT_ALONE": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_format": lzma.FORMAT_ALONE,
@@ -274,7 +245,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = lzma.decompress(data[filename])
@@ -282,24 +253,26 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_lzma_plugin_check(self):
+ async def test_lzma_plugin_check(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("check_CHECK_NONE"): lzma.compress(
+ tmp_path / "check_CHECK_NONE": lzma.compress(
self.expected, check=lzma.CHECK_NONE
),
- self._named_tempfile("CHECK_SHA256"): lzma.compress(
+ tmp_path / "CHECK_SHA256": lzma.compress(
self.expected, check=lzma.CHECK_SHA256
),
}
settings = {
"FEEDS": {
- self._named_tempfile("check_CHECK_NONE"): {
+ tmp_path / "check_CHECK_NONE": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_NONE,
},
- self._named_tempfile("CHECK_SHA256"): {
+ tmp_path / "CHECK_SHA256": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_SHA256,
@@ -307,7 +280,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = lzma.decompress(data[filename])
@@ -315,24 +288,22 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_lzma_plugin_preset(self):
+ async def test_lzma_plugin_preset(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("preset_PRESET_0"): lzma.compress(
- self.expected, preset=0
- ),
- self._named_tempfile("preset_PRESET_9"): lzma.compress(
- self.expected, preset=9
- ),
+ tmp_path / "preset_PRESET_0": lzma.compress(self.expected, preset=0),
+ tmp_path / "preset_PRESET_9": lzma.compress(self.expected, preset=9),
}
settings = {
"FEEDS": {
- self._named_tempfile("preset_PRESET_0"): {
+ tmp_path / "preset_PRESET_0": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_preset": 0,
},
- self._named_tempfile("preset_PRESET_9"): {
+ tmp_path / "preset_PRESET_9": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_preset": 9,
@@ -340,7 +311,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = lzma.decompress(data[filename])
@@ -348,14 +319,16 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_lzma_plugin_filters(self):
+ async def test_lzma_plugin_filters(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
if "PyPy" in sys.version:
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
pytest.skip("lzma filters doesn't work in PyPy")
filters = [{"id": lzma.FILTER_LZMA2}]
compressed = lzma.compress(self.expected, filters=filters)
- filename = self._named_tempfile("filters")
+ filename = tmp_path / "filters"
settings = {
"FEEDS": {
@@ -367,14 +340,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
assert compressed == data[filename]
result = lzma.decompress(data[filename])
assert result == self.expected
@coroutine_test
- async def test_bz2_plugin(self):
- filename = self._named_tempfile("bz2_file")
+ async def test_bz2_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:
+ filename = tmp_path / "bz2_file"
settings = {
"FEEDS": {
@@ -385,31 +358,29 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
try:
bz2.decompress(data[filename])
except OSError:
pytest.fail("Received invalid bz2 data.")
@coroutine_test
- async def test_bz2_plugin_compresslevel(self):
+ async def test_bz2_plugin_compresslevel(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_compressed = {
- self._named_tempfile("compresslevel_1"): bz2.compress(
- self.expected, compresslevel=1
- ),
- self._named_tempfile("compresslevel_9"): bz2.compress(
- self.expected, compresslevel=9
- ),
+ tmp_path / "compresslevel_1": bz2.compress(self.expected, compresslevel=1),
+ tmp_path / "compresslevel_9": bz2.compress(self.expected, compresslevel=9),
}
settings = {
"FEEDS": {
- self._named_tempfile("compresslevel_1"): {
+ tmp_path / "compresslevel_1": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
"bz2_compresslevel": 1,
},
- self._named_tempfile("compresslevel_9"): {
+ tmp_path / "compresslevel_9": {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
"bz2_compresslevel": 9,
@@ -417,7 +388,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = bz2.decompress(data[filename])
@@ -425,8 +396,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == self.expected
@coroutine_test
- async def test_custom_plugin(self):
- filename = self._named_tempfile("csv_file")
+ async def test_custom_plugin(self, mockserver: MockServer, tmp_path: Path) -> None:
+ filename = tmp_path / "csv_file"
settings = {
"FEEDS": {
@@ -437,13 +408,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
assert data[filename] == self.expected
@coroutine_test
- async def test_custom_plugin_with_parameter(self):
+ async def test_custom_plugin_with_parameter(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
expected = b"foo\r\n\nbar\r\n\n"
- filename = self._named_tempfile("newline")
+ filename = tmp_path / "newline"
settings = {
"FEEDS": {
@@ -455,22 +428,24 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
assert data[filename] == expected
@coroutine_test
- async def test_custom_plugin_with_compression(self):
+ async def test_custom_plugin_with_compression(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
expected = b"foo\r\n\nbar\r\n\n"
filename_to_decompressor = {
- self._named_tempfile("bz2"): bz2.decompress,
- self._named_tempfile("lzma"): lzma.decompress,
- self._named_tempfile("gzip"): gzip.decompress,
+ tmp_path / "bz2": bz2.decompress,
+ tmp_path / "lzma": lzma.decompress,
+ tmp_path / "gzip": gzip.decompress,
}
settings = {
"FEEDS": {
- self._named_tempfile("bz2"): {
+ tmp_path / "bz2": {
"format": "csv",
"postprocessing": [
self.MyPlugin1,
@@ -478,7 +453,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
],
"plugin1_char": b"\n",
},
- self._named_tempfile("lzma"): {
+ tmp_path / "lzma": {
"format": "csv",
"postprocessing": [
self.MyPlugin1,
@@ -486,7 +461,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
],
"plugin1_char": b"\n",
},
- self._named_tempfile("gzip"): {
+ tmp_path / "gzip": {
"format": "csv",
"postprocessing": [
self.MyPlugin1,
@@ -497,61 +472,57 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
},
}
- data = await self.exported_data(self.items, settings)
+ data = await export_by_path(mockserver, self.items, settings)
for filename, decompressor in filename_to_decompressor.items():
result = decompressor(data[filename])
assert result == expected
@coroutine_test
- async def test_exports_compatibility_with_postproc(self):
+ async def test_exports_compatibility_with_postproc(
+ self, mockserver: MockServer, tmp_path: Path
+ ) -> None:
filename_to_expected = {
- self._named_tempfile("csv"): b"foo\r\nbar\r\n",
- self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',
- self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n',
- self._named_tempfile("xml"): b'<?xml version="1.0" encoding="utf-8"?>\n'
+ tmp_path / "csv": b"foo\r\nbar\r\n",
+ tmp_path / "json": b'[\n{"foo": "bar"}\n]',
+ tmp_path / "jsonlines": b'{"foo": "bar"}\n',
+ tmp_path / "xml": b'<?xml version="1.0" encoding="utf-8"?>\n'
b"<items>\n<item><foo>bar</foo></item>\n</items>",
}
settings = {
"FEEDS": {
- self._named_tempfile("csv"): {
+ tmp_path / "csv": {
"format": "csv",
"postprocessing": [self.MyPlugin1],
# empty plugin to activate postprocessing.PostProcessingManager
},
- self._named_tempfile("json"): {
+ tmp_path / "json": {
"format": "json",
"postprocessing": [self.MyPlugin1],
},
- self._named_tempfile("jsonlines"): {
+ tmp_path / "jsonlines": {
"format": "jsonlines",
"postprocessing": [self.MyPlugin1],
},
- self._named_tempfile("xml"): {
+ tmp_path / "xml": {
"format": "xml",
"postprocessing": [self.MyPlugin1],
},
- self._named_tempfile("marshal"): {
+ tmp_path / "marshal": {
"format": "marshal",
"postprocessing": [self.MyPlugin1],
},
- self._named_tempfile("pickle"): {
+ tmp_path / "pickle": {
"format": "pickle",
"postprocessing": [self.MyPlugin1],
},
},
}
- data = await self.exported_data(self.items, settings)
-
- for filename, data_bytes in data.items():
- expected: Any
- result: Any
- if "pickle" in filename:
- expected, result = self.items[0], pickle.loads(data_bytes)
- elif "marshal" in filename:
- expected, result = self.items[0], marshal.loads(data_bytes)
- else:
- expected, result = filename_to_expected[filename], data_bytes
- assert result == expected
+ data = await export_by_path(mockserver, self.items, settings)
+
+ for path, expected in filename_to_expected.items():
+ assert data[path] == expected
+ assert pickle.loads(data[tmp_path / "pickle"]) == self.items[0]
+ assert marshal.loads(data[tmp_path / "marshal"]) == self.items[0]
diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.py
index 9c5e5bc9d..1b46ad94f 100644
--- a/tests/utils/feedexport.py
+++ b/tests/utils/feedexport.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
from urllib.request import pathname2url
@@ -9,7 +10,6 @@ from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import Iterable
- from pathlib import Path
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
@@ -50,3 +50,16 @@ async def crawl_items(
crawler = get_crawler(TestSpider, settings)
await crawler.crawl_async()
return crawler
+
+
+async def export_by_path(
+ mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]
+) -> dict[str | Path, bytes]:
+ """Export *items* with *settings*, whose ``FEEDS`` are keyed by local
+ paths, and return the contents of those paths that were written."""
+ feeds = settings["FEEDS"]
+ feed_urls = {
+ printf_escape(path_to_url(path)): options for path, options in feeds.items()
+ }
+ await crawl_items(mockserver, items, {**settings, "FEEDS": feed_urls})
+ return {path: Path(path).read_bytes() for path in feeds if Path(path).exists()}