Scrapy, a fast high-level web crawling & scraping framework for Python.
Feed post-processing plugin support (#5190)
D R Siddhartha43ea21e8306b
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 81beda5da..ef1c8362f 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -20,7 +20,7 @@ jobs:- python-version: pypy3env:TOXENV: pypy3- PYPY_VERSION: 3.6-v7.3.1+ PYPY_VERSION: 3.6-v7.3.3# pinned deps- python-version: 3.6.12diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 2b3217d62..116967280 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, assoon as a file reaches the maximum item count, that file is delivered to thefeed URI, allowing item delivery to start way before the end of the crawl.+.. _item-filter:Item filtering@@ -312,6 +313,63 @@ ItemFilter:members:+.. _post-processing:++Post-Processing+===============++.. versionadded:: VERSION++Scrapy provides an option to activate plugins to post-process feeds before they are exported+to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you+can create your own :ref:`plugins <custom-plugins>`.++These plugins can be activated through the ``postprocessing`` option of a feed.+The option must be passed a list of post-processing plugins in the order you want+the feed to be processed. These plugins can be declared either as an import string+or with the imported class of the plugin. Parameters to plugins can be passed+through the feed options. See :ref:`feed options <feed-options>` for examples.++.. _builtin-plugins:++Built-in Plugins+----------------++.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin++.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin++.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin++.. _custom-plugins:++Custom Plugins+--------------++Each plugin is a class that must implement the following methods:++.. method:: __init__(self, file, feed_options)++ Initialize the plugin.++ :param file: file-like object having at least the `write`, `tell` and `close` methods implemented++ :param feed_options: feed-specific :ref:`options <feed-options>`+ :type feed_options: :class:`dict`++.. method:: write(self, data)++ Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.+ It must return number of bytes written.++.. method:: close(self)++ Close the target file object.++To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You+can then access those parameters from the ``__init__`` method of your plugin.++Settings========@@ -368,10 +426,12 @@ For instance::'encoding': 'latin1','indent': 8,},- pathlib.Path('items.csv'): {+ pathlib.Path('items.csv.gz'): {'format': 'csv','fields': ['price', 'name'],'item_filter': 'myproject.filters.MyCustomFilter2',+ 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_compresslevel': 5,},}@@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition:- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.+- ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.++ The plugins will be used in the order of the list passed.++ .. versionadded:: VERSION.. setting:: FEED_EXPORT_ENCODINGdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 564c736f2..0f5bf01d0 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -20,6 +20,7 @@ from zope.interface import implementer, Interfacefrom scrapy import signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.extensions.postprocessing import PostProcessingManagerfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.ftp import ftp_store_file@@ -396,6 +397,9 @@ class FeedExporter:"""storage = self._get_storage(uri, feed_options)file = storage.open(spider)+ if "postprocessing" in feed_options:+ file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)+exporter = self._get_exporter(file=file,format=feed_options['format'],diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.pynew file mode 100644index 000000000..413c2e55e--- /dev/null+++ b/scrapy/extensions/postprocessing.py@@ -0,0 +1,154 @@+"""+Extension for processing data before they are exported to feeds.+"""+from bz2 import BZ2File+from gzip import GzipFile+from io import IOBase+from lzma import LZMAFile+from typing import Any, BinaryIO, Dict, List++from scrapy.utils.misc import load_object+++class GzipPlugin:+ """+ Compresses received data using `gzip <https://en.wikipedia.org/wiki/Gzip>`_.++ Accepted ``feed_options`` parameters:++ - `gzip_compresslevel`+ - `gzip_mtime`+ - `gzip_filename`++ See :py:class:`gzip.GzipFile` for more info about parameters.+ """++ def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:+ self.file = file+ self.feed_options = feed_options+ compress_level = self.feed_options.get("gzip_compresslevel", 9)+ mtime = self.feed_options.get("gzip_mtime")+ filename = self.feed_options.get("gzip_filename")+ self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,+ mtime=mtime, filename=filename)++ def write(self, data: bytes) -> int:+ return self.gzipfile.write(data)++ def close(self) -> None:+ self.gzipfile.close()+ self.file.close()+++class Bz2Plugin:+ """+ Compresses received data using `bz2 <https://en.wikipedia.org/wiki/Bzip2>`_.++ Accepted ``feed_options`` parameters:++ - `bz2_compresslevel`++ See :py:class:`bz2.BZ2File` for more info about parameters.+ """++ def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:+ self.file = file+ self.feed_options = feed_options+ compress_level = self.feed_options.get("bz2_compresslevel", 9)+ self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)++ def write(self, data: bytes) -> int:+ return self.bz2file.write(data)++ def close(self) -> None:+ self.bz2file.close()+ self.file.close()+++class LZMAPlugin:+ """+ Compresses received data using `lzma <https://en.wikipedia.org/wiki/Lempel–Ziv–Markov_chain_algorithm>`_.++ Accepted ``feed_options`` parameters:++ - `lzma_format`+ - `lzma_check`+ - `lzma_preset`+ - `lzma_filters`++ .. note::+ ``lzma_filters`` cannot be used in pypy version 7.3.1 and older.++ See :py:class:`lzma.LZMAFile` for more info about parameters.+ """++ def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:+ self.file = file+ self.feed_options = feed_options++ format = self.feed_options.get("lzma_format")+ check = self.feed_options.get("lzma_check", -1)+ preset = self.feed_options.get("lzma_preset")+ filters = self.feed_options.get("lzma_filters")+ self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,+ check=check, preset=preset, filters=filters)++ def write(self, data: bytes) -> int:+ return self.lzmafile.write(data)++ def close(self) -> None:+ self.lzmafile.close()+ self.file.close()+++# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager+# instance as a file like writable object. This could be needed by some exporters+# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper.+class PostProcessingManager(IOBase):+ """+ This will manage and use declared plugins to process data in a+ pipeline-ish way.+ :param plugins: all the declared plugins for the feed+ :type plugins: list+ :param file: final target file where the processed data will be written+ :type file: file like object+ """++ def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:+ self.plugins = self._load_plugins(plugins)+ self.file = file+ self.feed_options = feed_options+ self.head_plugin = self._get_head_plugin()++ def write(self, data: bytes) -> int:+ """+ Uses all the declared plugins to process data first, then writes+ the processed data to target file.+ :param data: data passed to be written to target file+ :type data: bytes+ :return: returns number of bytes written+ :rtype: int+ """+ return self.head_plugin.write(data)++ def tell(self) -> int:+ return self.file.tell()++ def close(self) -> None:+ """+ Close the target file along with all the plugins.+ """+ self.head_plugin.close()++ def writable(self) -> bool:+ return True++ def _load_plugins(self, plugins: List[Any]) -> List[Any]:+ plugins = [load_object(plugin) for plugin in plugins]+ return plugins++ def _get_head_plugin(self) -> Any:+ prev = self.file+ for plugin in self.plugins[::-1]:+ prev = plugin(prev, self.feed_options)+ return prevdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 53e6a2018..253f3119c 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1,5 +1,8 @@+import bz2import csv+import gzipimport json+import lzmaimport osimport randomimport shutil@@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase):self.assertEqual(row['expected'], data[feed_options['format']])+class FeedPostProcessedExportsTest(FeedExportTestBase):+ __test__ = True++ items = [{'foo': 'bar'}]+ expected = b'foo\r\nbar\r\n'++ class MyPlugin1:+ def __init__(self, file, feed_options):+ self.file = file+ self.feed_options = feed_options+ self.char = self.feed_options.get('plugin1_char', b'')++ def write(self, data):+ written_count = self.file.write(data)+ written_count += self.file.write(self.char)+ return written_count++ def close(self):+ self.file.close()++ def _named_tempfile(self, name):+ return os.path.join(self.temp_dir, name)++ @defer.inlineCallbacks+ def run_and_export(self, spider_cls, settings):+ """ Run spider with specified settings; return exported data with filename. """++ FEEDS = settings.get('FEEDS') or {}+ settings['FEEDS'] = {+ printf_escape(path_to_url(file_path)): feed_options+ for file_path, feed_options in FEEDS.items()+ }++ content = {}+ try:+ with MockServer() as s:+ runner = CrawlerRunner(Settings(settings))+ spider_cls.start_urls = [s.url('/')]+ yield runner.crawl(spider_cls)++ for file_path, feed_options in FEEDS.items():+ if not os.path.exists(str(file_path)):+ continue++ with open(str(file_path), 'rb') as f:+ content[str(file_path)] = f.read()++ finally:+ for file_path in FEEDS.keys():+ if not os.path.exists(str(file_path)):+ continue++ os.remove(str(file_path))++ return content++ def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''):+ data_stream = BytesIO()+ gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime,+ compresslevel=compresslevel, mode="wb")+ gzipf.write(data)+ gzipf.close()+ data_stream.seek(0)+ return data_stream.read()++ @defer.inlineCallbacks+ def test_gzip_plugin(self):++ filename = self._named_tempfile('gzip_file')++ settings = {+ 'FEEDS': {+ filename: {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ },+ },+ }++ data = yield self.exported_data(self.items, settings)+ try:+ gzip.decompress(data[filename])+ except OSError:+ self.fail("Received invalid gzip data.")++ @defer.inlineCallbacks+ def test_gzip_plugin_compresslevel(self):++ filename_to_compressed = {+ self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0),+ self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('compresslevel_0'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_compresslevel': 0,+ 'gzip_mtime': 0,+ 'gzip_filename': "",+ },+ self._named_tempfile('compresslevel_9'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_compresslevel': 9,+ 'gzip_mtime': 0,+ 'gzip_filename': "",+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = gzip.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_gzip_plugin_mtime(self):+ filename_to_compressed = {+ self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123),+ self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('mtime_123'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_mtime': 123,+ 'gzip_filename': "",+ },+ self._named_tempfile('mtime_123456789'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_mtime': 123456789,+ 'gzip_filename': "",+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = gzip.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_gzip_plugin_filename(self):+ filename_to_compressed = {+ self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"),+ self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('filename_FILE1'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_mtime': 0,+ 'gzip_filename': "FILE1",+ },+ self._named_tempfile('filename_FILE2'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],+ 'gzip_mtime': 0,+ 'gzip_filename': "FILE2",+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = gzip.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_lzma_plugin(self):++ filename = self._named_tempfile('lzma_file')++ settings = {+ 'FEEDS': {+ filename: {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ },+ },+ }++ data = yield self.exported_data(self.items, settings)+ try:+ lzma.decompress(data[filename])+ except lzma.LZMAError:+ self.fail("Received invalid lzma data.")++ @defer.inlineCallbacks+ def test_lzma_plugin_format(self):++ filename_to_compressed = {+ self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ),+ self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('format_FORMAT_XZ'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_format': lzma.FORMAT_XZ,+ },+ self._named_tempfile('format_FORMAT_ALONE'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_format': lzma.FORMAT_ALONE,+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = lzma.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_lzma_plugin_check(self):++ filename_to_compressed = {+ self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE),+ self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('check_CHECK_NONE'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_check': lzma.CHECK_NONE,+ },+ self._named_tempfile('check_CHECK_CRC256'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_check': lzma.CHECK_SHA256,+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = lzma.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_lzma_plugin_preset(self):++ filename_to_compressed = {+ self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0),+ self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('preset_PRESET_0'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_preset': 0,+ },+ self._named_tempfile('preset_PRESET_9'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_preset': 9,+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = lzma.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_lzma_plugin_filters(self):+ import sys+ if "PyPy" in sys.version:+ # https://foss.heptapod.net/pypy/pypy/-/issues/3527+ raise unittest.SkipTest("lzma filters doesn't work in PyPy")++ filters = [{'id': lzma.FILTER_LZMA2}]+ compressed = lzma.compress(self.expected, filters=filters)+ filename = self._named_tempfile('filters')++ settings = {+ 'FEEDS': {+ filename: {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],+ 'lzma_filters': filters,+ },+ },+ }++ data = yield self.exported_data(self.items, settings)+ self.assertEqual(compressed, data[filename])+ result = lzma.decompress(data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_bz2_plugin(self):++ filename = self._named_tempfile('bz2_file')++ settings = {+ 'FEEDS': {+ filename: {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],+ },+ },+ }++ data = yield self.exported_data(self.items, settings)+ try:+ bz2.decompress(data[filename])+ except OSError:+ self.fail("Received invalid bz2 data.")++ @defer.inlineCallbacks+ def test_bz2_plugin_compresslevel(self):++ filename_to_compressed = {+ self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1),+ self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9),+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('compresslevel_1'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],+ 'bz2_compresslevel': 1,+ },+ self._named_tempfile('compresslevel_9'): {+ 'format': 'csv',+ 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],+ 'bz2_compresslevel': 9,+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, compressed in filename_to_compressed.items():+ result = bz2.decompress(data[filename])+ self.assertEqual(compressed, data[filename])+ self.assertEqual(self.expected, result)++ @defer.inlineCallbacks+ def test_custom_plugin(self):+ filename = self._named_tempfile('csv_file')++ settings = {+ 'FEEDS': {+ filename: {+ 'format': 'csv',+ 'postprocessing': [self.MyPlugin1],+ },+ },+ }++ data = yield self.exported_data(self.items, settings)+ self.assertEqual(self.expected, data[filename])++ @defer.inlineCallbacks+ def test_custom_plugin_with_parameter(self):++ expected = b'foo\r\n\nbar\r\n\n'+ filename = self._named_tempfile('newline')++ settings = {+ 'FEEDS': {+ filename: {+ 'format': 'csv',+ 'postprocessing': [self.MyPlugin1],+ 'plugin1_char': b'\n'+ },+ },+ }++ data = yield self.exported_data(self.items, settings)+ self.assertEqual(expected, data[filename])++ @defer.inlineCallbacks+ def test_custom_plugin_with_compression(self):++ expected = b'foo\r\n\nbar\r\n\n'++ filename_to_decompressor = {+ self._named_tempfile('bz2'): bz2.decompress,+ self._named_tempfile('lzma'): lzma.decompress,+ self._named_tempfile('gzip'): gzip.decompress,+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('bz2'): {+ 'format': 'csv',+ 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'],+ 'plugin1_char': b'\n',+ },+ self._named_tempfile('lzma'): {+ 'format': 'csv',+ 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'],+ 'plugin1_char': b'\n',+ },+ self._named_tempfile('gzip'): {+ 'format': 'csv',+ 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],+ 'plugin1_char': b'\n',+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, decompressor in filename_to_decompressor.items():+ result = decompressor(data[filename])+ self.assertEqual(expected, result)++ @defer.inlineCallbacks+ def test_exports_compatibility_with_postproc(self):+ import marshal+ import pickle+ filename_to_expected = {+ self._named_tempfile('csv'): b'foo\r\nbar\r\n',+ self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]',+ self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n',+ self._named_tempfile('xml'): b'<?xml version="1.0" encoding="utf-8"?>\n'+ b'<items>\n<item><foo>bar</foo></item>\n</items>',+ }++ settings = {+ 'FEEDS': {+ self._named_tempfile('csv'): {+ 'format': 'csv',+ 'postprocessing': [self.MyPlugin1],+ # empty plugin to activate postprocessing.PostProcessingManager+ },+ self._named_tempfile('json'): {+ 'format': 'json',+ 'postprocessing': [self.MyPlugin1],+ },+ self._named_tempfile('jsonlines'): {+ 'format': 'jsonlines',+ 'postprocessing': [self.MyPlugin1],+ },+ self._named_tempfile('xml'): {+ 'format': 'xml',+ 'postprocessing': [self.MyPlugin1],+ },+ self._named_tempfile('marshal'): {+ 'format': 'marshal',+ 'postprocessing': [self.MyPlugin1],+ },+ self._named_tempfile('pickle'): {+ 'format': 'pickle',+ 'postprocessing': [self.MyPlugin1],+ },+ },+ }++ data = yield self.exported_data(self.items, settings)++ for filename, result in data.items():+ if 'pickle' in filename:+ expected, result = self.items[0], pickle.loads(result)+ elif 'marshal' in filename:+ expected, result = self.items[0], marshal.loads(result)+ else:+ expected = filename_to_expected[filename]+ self.assertEqual(expected, result)++class BatchDeliveriesTest(FeedExportTestBase):__test__ = True_file_mark = '_%(batch_time)s_#%(batch_id)02d_'