Scrapy, a fast high-level web crawling & scraping framework for Python.
Export to multiple formats in a single crawl
Eugenio Lacuestaada37c540904
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 42f1cad90..6d6ba33c9 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "exportfeed") to be consumed by other systems.Scrapy provides this functionality out of the box with the Feed Exports, which-allows you to generate a feed with the scraped items, using multiple+allows you to generate feeds with the scraped items, using multipleserialization formats and storage backends... _topics-feed-format:@@ -36,7 +36,7 @@ But you can also extend the supported format through theJSON----- * :setting:`FEED_FORMAT`: ``json``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`* See :ref:`this warning <json-with-large-data>` if you're using JSON withlarge feeds.@@ -46,7 +46,7 @@ JSONJSON lines----------- * :setting:`FEED_FORMAT`: ``jsonlines``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`.. _topics-feed-format-csv:@@ -54,7 +54,7 @@ JSON linesCSV---- * :setting:`FEED_FORMAT`: ``csv``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`* To specify columns to export and their order use:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this@@ -66,7 +66,7 @@ CSVXML---- * :setting:`FEED_FORMAT`: ``xml``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`.. _topics-feed-format-pickle:@@ -74,7 +74,7 @@ XMLPickle------- * :setting:`FEED_FORMAT`: ``pickle``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`.. _topics-feed-format-marshal:@@ -82,7 +82,7 @@ PickleMarshal-------- * :setting:`FEED_FORMAT`: ``marshal``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`@@ -91,8 +91,8 @@ MarshalStorages========-When using the feed exports you define where to store the feed using a URI_-(through the :setting:`FEED_URI` setting). The feed exports supports multiple+When using the feed exports you define where to store the feed using one or multiple URIs_+(through the :setting:`FEEDS` setting). The feed exports supports multiplestorage backend types which are defined by the URI scheme.The storages backends supported out of the box are:@@ -211,41 +211,66 @@ SettingsThese are the settings used for configuring the feed exports:- * :setting:`FEED_URI` (mandatory)- * :setting:`FEED_FORMAT`+ * :setting:`FEEDS` (mandatory)+ * :setting:`FEED_EXPORT_ENCODING`+ * :setting:`FEED_STORE_EMPTY`+ * :setting:`FEED_EXPORT_FIELDS`+ * :setting:`FEED_EXPORT_INDENT`* :setting:`FEED_STORAGES`* :setting:`FEED_STORAGE_FTP_ACTIVE`* :setting:`FEED_STORAGE_S3_ACL`* :setting:`FEED_EXPORTERS`- * :setting:`FEED_STORE_EMPTY`- * :setting:`FEED_EXPORT_ENCODING`- * :setting:`FEED_EXPORT_FIELDS`- * :setting:`FEED_EXPORT_INDENT`.. currentmodule:: scrapy.extensions.feedexport-.. setting:: FEED_URI+.. setting:: FEEDS-FEED_URI---------+FEEDS+------Default: ``None``+.. versionadded:: 2.1++Default: ``{}``-The URI of the export feed. See :ref:`topics-feed-storage-backends` for-supported URI schemes.+A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`+object) and each value is a nested dictionary containing configuration+parameters for the specific feed.+This setting is required for enabling the feed export feature.-This setting is required for enabling the feed exports.+See :ref:`topics-feed-storage-backends` for supported URI schemes.-.. versionchanged:: 2.0- Added :class:`pathlib.Path` support.+For instance::-.. setting:: FEED_FORMAT+ {+ 'items.json': {+ 'format': 'json',+ 'encoding': 'utf8',+ 'store_empty': False,+ 'fields': None,+ 'indent': 4,+ },+ 'items.xml': {+ 'format': 'xml',+ 'fields': ['name', 'price'],+ 'encoding': 'latin1',+ 'indent': 8,+ },+ pathlib.Path('items.csv'): {+ 'format': 'csv',+ 'fields': ['price', 'name'],+ },+ }-FEED_FORMAT------------+The following is a list of the accepted keys and the setting that is used+as a fallback value if that key is not provided for a specific feed definition.-The serialization format to be used for the feed. See-:ref:`topics-feed-format` for possible values.+* ``format``: the serialization format to be used for the feed.+ See :ref:`topics-feed-format` for possible values.+ Mandatory, no fallback setting+* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`+* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`+* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`+* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.. setting:: FEED_EXPORT_ENCODING@@ -400,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter'csv': None,}-.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier+.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier.. _Amazon S3: https://aws.amazon.com/s3/.. _botocore: https://github.com/boto/botocore.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acldiff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.pyindex 7b417e2eb..4b2f9484b 100644--- a/scrapy/commands/crawl.py+++ b/scrapy/commands/crawl.py@@ -1,7 +1,5 @@-import osfrom scrapy.commands import ScrapyCommand-from scrapy.utils.conf import arglist_to_dict-from scrapy.utils.python import without_none_values+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_clifrom scrapy.exceptions import UsageError@@ -19,7 +17,7 @@ class Command(ScrapyCommand):ScrapyCommand.add_options(self, parser)parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",help="set spider argument (may be repeated)")- parser.add_option("-o", "--output", metavar="FILE",+ parser.add_option("-o", "--output", metavar="FILE", action="append",help="dump scraped items into FILE (use - for stdout)")parser.add_option("-t", "--output-format", metavar="FORMAT",help="format to use for dumping items with -o")@@ -31,21 +29,8 @@ class Command(ScrapyCommand):except ValueError:raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)if opts.output:- if opts.output == '-':- self.settings.set('FEED_URI', 'stdout:', priority='cmdline')- else:- self.settings.set('FEED_URI', opts.output, priority='cmdline')- feed_exporters = without_none_values(- self.settings.getwithbase('FEED_EXPORTERS'))- valid_output_formats = feed_exporters.keys()- if not opts.output_format:- opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")- if opts.output_format not in valid_output_formats:- raise UsageError("Unrecognized output format '%s', set one"- " using the '-t' switch or as a file extension"- " from the supported list %s" % (opts.output_format,- tuple(valid_output_formats)))- self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')+ feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)+ self.settings.set('FEEDS', feeds, priority='cmdline')def run(self, args, opts):if len(args) < 1:diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 57d8471ca..62510609a 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -5,8 +5,7 @@ from importlib import import_modulefrom scrapy.utils.spider import iter_spider_classesfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageError-from scrapy.utils.conf import arglist_to_dict-from scrapy.utils.python import without_none_values+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_clidef _import_file(filepath):@@ -43,7 +42,7 @@ class Command(ScrapyCommand):ScrapyCommand.add_options(self, parser)parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",help="set spider argument (may be repeated)")- parser.add_option("-o", "--output", metavar="FILE",+ parser.add_option("-o", "--output", metavar="FILE", action="append",help="dump scraped items into FILE (use - for stdout)")parser.add_option("-t", "--output-format", metavar="FORMAT",help="format to use for dumping items with -o")@@ -55,19 +54,8 @@ class Command(ScrapyCommand):except ValueError:raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)if opts.output:- if opts.output == '-':- self.settings.set('FEED_URI', 'stdout:', priority='cmdline')- else:- self.settings.set('FEED_URI', opts.output, priority='cmdline')- feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))- if not opts.output_format:- opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")- if opts.output_format not in feed_exporters:- raise UsageError("Unrecognized output format '%s', set one"- " using the '-t' switch or as a file extension"- " from the supported list %s" % (opts.output_format,- tuple(feed_exporters)))- self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')+ feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)+ self.settings.set('FEEDS', feeds, priority='cmdline')def run(self, args, opts):if len(args) != 1:diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex f1b101780..108b6d35c 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -4,24 +4,27 @@ Feed Exports extensionSee documentation in docs/topics/feed-exports.rst"""+import loggingimport osimport sys-import logging-from tempfile import NamedTemporaryFile+import warningsfrom datetime import datetime-from urllib.parse import urlparse, unquote+from tempfile import NamedTemporaryFile+from urllib.parse import unquote, urlparse-from zope.interface import Interface, implementerfrom twisted.internet import defer, threadsfrom w3lib.url import file_uri_to_path+from zope.interface import implementer, Interfacefrom scrapy import signals+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.utils.boto import is_botocore+from scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.ftp import ftp_store_file-from scrapy.exceptions import NotConfigured-from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.log import failure_to_exc_info+from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.python import without_none_values-from scrapy.utils.boto import is_botocore+logger = logging.getLogger(__name__)@@ -98,8 +101,6 @@ class S3FeedStorage(BlockingFeedStorage):from scrapy.utils.project import get_project_settingssettings = get_project_settings()if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:- import warnings- from scrapy.exceptions import ScrapyDeprecationWarningwarnings.warn("Initialising `scrapy.extensions.feedexport.S3FeedStorage` ""without AWS keys is deprecated. Please supply credentials or "@@ -178,88 +179,117 @@ class FTPFeedStorage(BlockingFeedStorage):)-class SpiderSlot(object):- def __init__(self, file, exporter, storage, uri):+class _FeedSlot(object):+ def __init__(self, file, exporter, storage, uri, format, store_empty):self.file = fileself.exporter = exporterself.storage = storage+ # feed paramsself.uri = uri+ self.format = format+ self.store_empty = store_empty+ # flagsself.itemcount = 0+ self._exporting = False++ def start_exporting(self):+ if not self._exporting:+ self.exporter.start_exporting()+ self._exporting = True++ def finish_exporting(self):+ if self._exporting:+ self.exporter.finish_exporting()+ self._exporting = Falseclass FeedExporter(object):- def __init__(self, settings):- self.settings = settings- if not settings['FEED_URI']:+ @classmethod+ def from_crawler(cls, crawler):+ exporter = cls(crawler)+ crawler.signals.connect(exporter.open_spider, signals.spider_opened)+ crawler.signals.connect(exporter.close_spider, signals.spider_closed)+ crawler.signals.connect(exporter.item_scraped, signals.item_scraped)+ return exporter++ def __init__(self, crawler):+ self.crawler = crawler+ self.settings = crawler.settings+ self.feeds = {}+ self.slots = []++ if not self.settings['FEEDS'] and not self.settings['FEED_URI']:raise NotConfigured- self.urifmt = str(settings['FEED_URI'])- self.format = settings['FEED_FORMAT'].lower()- self.export_encoding = settings['FEED_EXPORT_ENCODING']++ # Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings+ if self.settings['FEED_URI']:+ warnings.warn(+ 'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of '+ 'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details',+ category=ScrapyDeprecationWarning, stacklevel=2,+ )+ uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects+ feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}+ self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)+ # End: Backward compatibility for FEED_URI and FEED_FORMAT settings++ # 'FEEDS' setting takes precedence over 'FEED_URI'+ for uri, feed in self.settings.getdict('FEEDS').items():+ uri = str(uri) # handle pathlib.Path objects+ self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)+self.storages = self._load_components('FEED_STORAGES')self.exporters = self._load_components('FEED_EXPORTERS')- if not self._storage_supported(self.urifmt):- raise NotConfigured- if not self._exporter_supported(self.format):- raise NotConfigured- self.store_empty = settings.getbool('FEED_STORE_EMPTY')- self._exporting = False- self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None- self.indent = None- if settings.get('FEED_EXPORT_INDENT') is not None:- self.indent = settings.getint('FEED_EXPORT_INDENT')- uripar = settings['FEED_URI_PARAMS']- self._uripar = load_object(uripar) if uripar else lambda x, y: None-- @classmethod- def from_crawler(cls, crawler):- o = cls(crawler.settings)- o.crawler = crawler- crawler.signals.connect(o.open_spider, signals.spider_opened)- crawler.signals.connect(o.close_spider, signals.spider_closed)- crawler.signals.connect(o.item_scraped, signals.item_scraped)- return o+ for uri, feed in self.feeds.items():+ if not self._storage_supported(uri):+ raise NotConfigured+ if not self._exporter_supported(feed['format']):+ raise NotConfigureddef open_spider(self, spider):- uri = self.urifmt % self._get_uri_params(spider)- storage = self._get_storage(uri)- file = storage.open(spider)- exporter = self._get_exporter(file, fields_to_export=self.export_fields,- encoding=self.export_encoding, indent=self.indent)- if self.store_empty:- exporter.start_exporting()- self._exporting = True- self.slot = SpiderSlot(file, exporter, storage, uri)+ for uri, feed in self.feeds.items():+ uri = uri % self._get_uri_params(spider, feed['uri_params'])+ storage = self._get_storage(uri)+ file = storage.open(spider)+ exporter = self._get_exporter(+ file=file,+ format=feed['format'],+ fields_to_export=feed['fields'],+ encoding=feed['encoding'],+ indent=feed['indent'],+ )+ slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])+ self.slots.append(slot)+ if slot.store_empty:+ slot.start_exporting()def close_spider(self, spider):- slot = self.slot- if not slot.itemcount and not self.store_empty:- # We need to call slot.storage.store nonetheless to get the file- # properly closed.- return defer.maybeDeferred(slot.storage.store, slot.file)- if self._exporting:- slot.exporter.finish_exporting()- self._exporting = False- logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"- log_args = {'format': self.format,- 'itemcount': slot.itemcount,- 'uri': slot.uri}- d = defer.maybeDeferred(slot.storage.store, slot.file)- d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,- extra={'spider': spider}))- d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,- exc_info=failure_to_exc_info(f),- extra={'spider': spider}))- return d+ deferred_list = []+ for slot in self.slots:+ if not slot.itemcount and not slot.store_empty:+ # We need to call slot.storage.store nonetheless to get the file+ # properly closed.+ return defer.maybeDeferred(slot.storage.store, slot.file)+ slot.finish_exporting()+ logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"+ log_args = {'format': slot.format,+ 'itemcount': slot.itemcount,+ 'uri': slot.uri}+ d = defer.maybeDeferred(slot.storage.store, slot.file)+ d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,+ extra={'spider': spider}))+ d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,+ exc_info=failure_to_exc_info(f),+ extra={'spider': spider}))+ deferred_list.append(d)+ return defer.DeferredList(deferred_list) if deferred_list else Nonedef item_scraped(self, item, spider):- slot = self.slot- if not self._exporting:- slot.exporter.start_exporting()- self._exporting = True- slot.exporter.export_item(item)- slot.itemcount += 1- return item+ for slot in self.slots:+ slot.start_exporting()+ slot.exporter.export_item(item)+ slot.itemcount += 1def _load_components(self, setting_prefix):conf = without_none_values(self.settings.getwithbase(setting_prefix))@@ -295,17 +325,18 @@ class FeedExporter(object):objcls, self.settings, getattr(self, 'crawler', None),*args, **kwargs)- def _get_exporter(self, *args, **kwargs):- return self._get_instance(self.exporters[self.format], *args, **kwargs)+ def _get_exporter(self, file, format, *args, **kwargs):+ return self._get_instance(self.exporters[format], file, *args, **kwargs)def _get_storage(self, uri):return self._get_instance(self.storages[urlparse(uri).scheme], uri)- def _get_uri_params(self, spider):+ def _get_uri_params(self, spider, uri_params):params = {}for k in dir(spider):params[k] = getattr(spider, k)ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')params['time'] = ts- self._uripar(params, spider)+ uripar_function = load_object(uri_params) if uri_params else lambda x, y: None+ uripar_function(params, spider)return paramsdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex f8a0457ce..077317c81 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -133,9 +133,8 @@ EXTENSIONS_BASE = {}FEED_TEMPDIR = None-FEED_URI = None+FEEDS = {}FEED_URI_PARAMS = None # a function to extend uri arguments-FEED_FORMAT = 'jsonlines'FEED_STORE_EMPTY = FalseFEED_EXPORT_ENCODING = NoneFEED_EXPORT_FIELDS = Nonediff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 23306ca28..e01027491 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -1,9 +1,12 @@+import numbersimport osimport sys-import numbers+import warningsfrom configparser import ConfigParserfrom operator import itemgetter+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError+from scrapy.settings import BaseSettingsfrom scrapy.utils.deprecate import update_classpathfrom scrapy.utils.python import without_none_values@@ -106,3 +109,66 @@ def get_sources(use_closest=True):if use_closest:sources.append(closest_scrapy_cfg())return sources+++def feed_complete_default_values_from_settings(feed, settings):+ out = feed.copy()+ if 'encoding' not in out:+ out['encoding'] = settings['FEED_EXPORT_ENCODING']+ if 'fields' not in out:+ out['fields'] = settings.getlist('FEED_EXPORT_FIELDS') or None+ if 'indent' not in out:+ out['indent'] = None if settings['FEED_EXPORT_INDENT'] is None else settings.getint('FEED_EXPORT_INDENT')+ if 'store_empty' not in out:+ out['store_empty'] = settings.getbool('FEED_STORE_EMPTY')+ if 'uri_params' not in out:+ out['uri_params'] = settings['FEED_URI_PARAMS']+ return out+++def feed_process_params_from_cli(settings, output, output_format=None):+ """+ Receives feed export params (from the 'crawl' or 'runspider' commands),+ checks for inconsistencies in their quantities and returns a dictionary+ suitable to be used as the FEEDS setting.+ """+ valid_output_formats = without_none_values(+ settings.getwithbase('FEED_EXPORTERS')+ ).keys()++ def check_valid_format(output_format):+ if output_format not in valid_output_formats:+ raise UsageError("Unrecognized output format '%s', set one after a"+ " colon using the -o option (i.e. -o <URI>:<FORMAT>)"+ " or as a file extension, from the supported list %s" %+ (output_format, tuple(valid_output_formats)))++ if output_format:+ if len(output) == 1:+ check_valid_format(output_format)+ warnings.warn('The -t command line option is deprecated in favor'+ ' of specifying the output format within the -o'+ ' option, please check the -o option docs for more details',+ category=ScrapyDeprecationWarning, stacklevel=2)+ return {output[0]: {'format': output_format}}+ else:+ raise UsageError('The -t command line option cannot be used if multiple'+ ' output files are specified with the -o option')++ result = {}+ for element in output:+ try:+ feed_uri, feed_format = element.rsplit(':', 1)+ except ValueError:+ feed_uri = element+ feed_format = os.path.splitext(element)[1].replace('.', '')+ else:+ if feed_uri == '-':+ feed_uri = 'stdout:'+ check_valid_format(feed_format)+ result[feed_uri] = {'format': feed_format}++ # FEEDS setting should take precedence over the -o and -t CLI options+ result.update(settings.getdict('FEEDS'))++ return resultdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 3612b70c9..24a341759 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -1,22 +1,46 @@import inspect+import json+import optparseimport os-import sysimport subprocess+import sysimport tempfile+from contextlib import contextmanagerfrom os.path import exists, join, abspathfrom shutil import rmtree, copytreefrom tempfile import mkdtemp-from contextlib import contextmanagerfrom threading import Timerfrom twisted.trial import unittestimport scrapy+from scrapy.commands import ScrapyCommand+from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_testenv+from tests.test_crawler import ExceptionSpider, NoRequestsSpider+class CommandSettings(unittest.TestCase):++ def setUp(self):+ self.command = ScrapyCommand()+ self.command.settings = Settings()+ self.parser = optparse.OptionParser(+ formatter=optparse.TitledHelpFormatter(),+ conflict_handler='resolve',+ )+ self.command.add_options(self.parser)++ def test_settings_json_string(self):+ feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'+ opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py'])+ self.command.process_options(args, opts)+ self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)+ self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))++class ProjectTest(unittest.TestCase):project_name = 'testproject'@@ -34,7 +58,7 @@ class ProjectTest(unittest.TestCase):with tempfile.TemporaryFile() as out:args = (sys.executable, '-m', 'scrapy.cmdline') + new_argsreturn subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd,- env=self.env, **kwargs)+ env=self.env, **kwargs)def proc(self, *new_args, **popen_kwargs):args = (sys.executable, '-m', 'scrapy.cmdline') + new_args@@ -310,6 +334,6 @@ class BenchCommandTest(CommandTest):def test_run(self):_, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001',- '-s', 'CLOSESPIDER_TIMEOUT=0.01')+ '-s', 'CLOSESPIDER_TIMEOUT=0.01')self.assertIn('INFO: Crawled', log)self.assertNotIn('Unhandled Error', log)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 2ca57c19d..08e8dfc41 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1,32 +1,34 @@-import osimport csvimport json-import warnings-import tempfile+import os+import randomimport shutilimport string+import tempfile+import warningsfrom io import BytesIOfrom pathlib import Path+from string import ascii_letters, digitsfrom unittest import mockfrom urllib.parse import urljoin, urlparse, quotefrom urllib.request import pathname2url-from zope.interface.verify import verifyObject-from twisted.trial import unittest+import lxml.etreefrom twisted.internet import defer-from scrapy.crawler import CrawlerRunner-from scrapy.settings import Settings-from tests.mockserver import MockServer+from twisted.trial import unittestfrom w3lib.url import path_to_file_uri+from zope.interface.verify import verifyObjectimport scrapy+from scrapy.crawler import CrawlerRunnerfrom scrapy.exporters import CsvItemExporter-from scrapy.extensions.feedexport import (- IFeedStorage, FileFeedStorage, FTPFeedStorage,- S3FeedStorage, StdoutFeedStorage,- BlockingFeedStorage)-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler+from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,+ IFeedStorage, S3FeedStorage, StdoutFeedStorage)+from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode+from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete++from tests.mockserver import MockServerclass FileFeedStorageTest(unittest.TestCase):@@ -395,29 +397,41 @@ class FeedExportTest(unittest.TestCase):egg = scrapy.Field()baz = scrapy.Field()+ def setUp(self):+ self.temp_dir = tempfile.mkdtemp()++ def tearDown(self):+ shutil.rmtree(self.temp_dir, ignore_errors=True)++ def _random_temp_filename(self):+ chars = [random.choice(ascii_letters + digits) for _ in range(15)]+ filename = ''.join(chars)+ return os.path.join(self.temp_dir, filename)+@defer.inlineCallbacks- def run_and_export(self, spider_cls, settings=None):+ def run_and_export(self, spider_cls, settings):""" Run spider with specified settings; return exported data. """- tmpdir = tempfile.mkdtemp()- res_path = os.path.join(tmpdir, 'res')- res_uri = urljoin('file:', pathname2url(res_path))- defaults = {- 'FEED_URI': res_uri,- 'FEED_FORMAT': 'csv',- 'FEED_PATH': res_path++ FEEDS = settings.get('FEEDS') or {}+ settings['FEEDS'] = {+ urljoin('file:', pathname2url(str(file_path))): feed+ for file_path, feed in FEEDS.items()}- defaults.update(settings or {})++ content = {}try:with MockServer() as s:- runner = CrawlerRunner(Settings(defaults))+ runner = CrawlerRunner(Settings(settings))spider_cls.start_urls = [s.url('/')]yield runner.crawl(spider_cls)- with open(str(defaults['FEED_PATH']), 'rb') as f:- content = f.read()+ for file_path, feed in FEEDS.items():+ with open(str(file_path), 'rb') as f:+ content[feed['format']] = f.read()finally:- shutil.rmtree(tmpdir)+ for file_path in FEEDS.keys():+ os.remove(str(file_path))defer.returnValue(content)@@ -453,10 +467,14 @@ class FeedExportTest(unittest.TestCase):@defer.inlineCallbacksdef assertExportedCsv(self, items, header, rows, settings=None, ordered=True):settings = settings or {}- settings.update({'FEED_FORMAT': 'csv'})+ settings.update({+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'csv'},+ },+ })data = yield self.exported_data(items, settings)- reader = csv.DictReader(to_unicode(data).splitlines())+ reader = csv.DictReader(to_unicode(data['csv']).splitlines())got_rows = list(reader)if ordered:self.assertEqual(reader.fieldnames, header)@@ -468,51 +486,87 @@ class FeedExportTest(unittest.TestCase):@defer.inlineCallbacksdef assertExportedJsonLines(self, items, rows, settings=None):settings = settings or {}- settings.update({'FEED_FORMAT': 'jl'})+ settings.update({+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'jl'},+ },+ })data = yield self.exported_data(items, settings)- parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]+ parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()]rows = [{k: v for k, v in row.items() if v} for row in rows]self.assertEqual(rows, parsed)@defer.inlineCallbacksdef assertExportedXml(self, items, rows, settings=None):settings = settings or {}- settings.update({'FEED_FORMAT': 'xml'})+ settings.update({+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'xml'},+ },+ })data = yield self.exported_data(items, settings)rows = [{k: v for k, v in row.items() if v} for row in rows]- import lxml.etree- root = lxml.etree.fromstring(data)+ root = lxml.etree.fromstring(data['xml'])got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]self.assertEqual(rows, got_rows)+ @defer.inlineCallbacks+ def assertExportedMultiple(self, items, rows, settings=None):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'xml'},+ self._random_temp_filename(): {'format': 'json'},+ },+ })+ data = yield self.exported_data(items, settings)+ rows = [{k: v for k, v in row.items() if v} for row in rows]+ # XML+ root = lxml.etree.fromstring(data['xml'])+ xml_rows = [{e.tag: e.text for e in it} for it in root.findall('item')]+ self.assertEqual(rows, xml_rows)+ # JSON+ json_rows = json.loads(to_unicode(data['json']))+ self.assertEqual(rows, json_rows)+def _load_until_eof(self, data, load_func):- bytes_output = BytesIO(data)result = []- while True:- try:- result.append(load_func(bytes_output))- except EOFError:- break+ with tempfile.TemporaryFile() as temp:+ temp.write(data)+ temp.seek(0)+ while True:+ try:+ result.append(load_func(temp))+ except EOFError:+ breakreturn result@defer.inlineCallbacksdef assertExportedPickle(self, items, rows, settings=None):settings = settings or {}- settings.update({'FEED_FORMAT': 'pickle'})+ settings.update({+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'pickle'},+ },+ })data = yield self.exported_data(items, settings)expected = [{k: v for k, v in row.items() if v} for row in rows]import pickle- result = self._load_until_eof(data, load_func=pickle.load)+ result = self._load_until_eof(data['pickle'], load_func=pickle.load)self.assertEqual(expected, result)@defer.inlineCallbacksdef assertExportedMarshal(self, items, rows, settings=None):settings = settings or {}- settings.update({'FEED_FORMAT': 'marshal'})+ settings.update({+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'marshal'},+ },+ })data = yield self.exported_data(items, settings)expected = [{k: v for k, v in row.items() if v} for row in rows]import marshal- result = self._load_until_eof(data, load_func=marshal.load)+ result = self._load_until_eof(data['marshal'], load_func=marshal.load)self.assertEqual(expected, result)@defer.inlineCallbacks@@ -521,6 +575,8 @@ class FeedExportTest(unittest.TestCase):yield self.assertExportedJsonLines(items, rows, settings)yield self.assertExportedXml(items, rows, settings)yield self.assertExportedPickle(items, rows, settings)+ yield self.assertExportedMarshal(items, rows, settings)+ yield self.assertExportedMultiple(items, rows, settings)@defer.inlineCallbacksdef test_export_items(self):@@ -538,15 +594,14 @@ class FeedExportTest(unittest.TestCase):@defer.inlineCallbacksdef test_export_no_items_not_store_empty(self):- formats = ('json',- 'jsonlines',- 'xml',- 'csv',)-- for fmt in formats:- settings = {'FEED_FORMAT': fmt}+ for fmt in ('json', 'jsonlines', 'xml', 'csv'):+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': fmt},+ },+ }data = yield self.exported_no_data(settings)- self.assertEqual(data, b'')+ self.assertEqual(data[fmt], b'')@defer.inlineCallbacksdef test_export_no_items_store_empty(self):@@ -558,9 +613,15 @@ class FeedExportTest(unittest.TestCase):)for fmt, expctd in formats:- settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None}+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': fmt},+ },+ 'FEED_STORE_EMPTY': True,+ 'FEED_EXPORT_INDENT': None,+ }data = yield self.exported_no_data(settings)- self.assertEqual(data, expctd)+ self.assertEqual(data[fmt], expctd)@defer.inlineCallbacksdef test_export_multiple_item_classes(self):@@ -581,9 +642,9 @@ class FeedExportTest(unittest.TestCase):header = self.MyItem.fields.keys()rows_csv = [{'egg': 'spam1', 'foo': 'bar1', 'baz': ''},- {'egg': '', 'foo': 'bar2', 'baz': ''},+ {'egg': '', 'foo': 'bar2', 'baz': ''},{'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},- {'egg': 'spam4', 'foo': '', 'baz': ''},+ {'egg': 'spam4', 'foo': '', 'baz': ''},]rows_jl = [dict(row) for row in items]yield self.assertExportedCsv(items, header, rows_csv, ordered=False)@@ -598,10 +659,10 @@ class FeedExportTest(unittest.TestCase):header = ["foo", "baz", "hello"]settings = {'FEED_EXPORT_FIELDS': header}rows = [- {'foo': 'bar1', 'baz': '', 'hello': ''},- {'foo': 'bar2', 'baz': '', 'hello': 'world2'},+ {'foo': 'bar1', 'baz': '', 'hello': ''},+ {'foo': 'bar2', 'baz': '', 'hello': 'world2'},{'foo': 'bar3', 'baz': 'quux3', 'hello': ''},- {'foo': '', 'baz': '', 'hello': 'world4'},+ {'foo': '', 'baz': '', 'hello': 'world4'},]yield self.assertExported(items, header, rows,settings=settings, ordered=True)@@ -663,10 +724,15 @@ class FeedExportTest(unittest.TestCase):'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),}- for format, expected in formats.items():- settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None}+ for fmt, expected in formats.items():+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': fmt},+ },+ 'FEED_EXPORT_INDENT': None,+ }data = yield self.exported_data(items, settings)- self.assertEqual(expected, data)+ self.assertEqual(expected, data[fmt])formats = {'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),@@ -675,11 +741,53 @@ class FeedExportTest(unittest.TestCase):'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),}- settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'}- for format, expected in formats.items():- settings['FEED_FORMAT'] = format+ for fmt, expected in formats.items():+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': fmt},+ },+ 'FEED_EXPORT_INDENT': None,+ 'FEED_EXPORT_ENCODING': 'latin-1',+ }data = yield self.exported_data(items, settings)- self.assertEqual(expected, data)+ self.assertEqual(expected, data[fmt])++ @defer.inlineCallbacks+ def test_export_multiple_configs(self):+ items = [dict({'foo': u'FOO', 'bar': u'BAR'})]++ formats = {+ 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'),+ 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'),+ 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),+ }++ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {+ 'format': 'json',+ 'indent': 0,+ 'fields': ['bar'],+ 'encoding': 'utf-8',+ },+ self._random_temp_filename(): {+ 'format': 'xml',+ 'indent': 2,+ 'fields': ['foo'],+ 'encoding': 'latin-1',+ },+ self._random_temp_filename(): {+ 'format': 'csv',+ 'indent': None,+ 'fields': ['bar', 'foo'],+ 'encoding': 'utf-8',+ },+ },+ }++ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ self.assertEqual(expected, data[fmt])@defer.inlineCallbacksdef test_export_indentation(self):@@ -827,33 +935,38 @@ class FeedExportTest(unittest.TestCase):]for row in test_cases:- settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {+ 'format': row['format'],+ 'indent': row['indent'],+ },+ },+ }data = yield self.exported_data(items, settings)- print(row['format'], row['indent'])- self.assertEqual(row['expected'], data)+ self.assertEqual(row['expected'], data[row['format']])@defer.inlineCallbacksdef test_init_exporters_storages_with_crawler(self):settings = {- 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.'- 'FromCrawlerCsvItemExporter'},- 'FEED_STORAGES': {'file': 'tests.test_feedexport.'- 'FromCrawlerFileFeedStorage'},+ 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'},+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'},+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'csv'},+ },}- yield self.exported_data({}, settings)+ yield self.exported_data(items=[], settings=settings)self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler)self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)@defer.inlineCallbacksdef test_pathlib_uri(self):- tmpdir = tempfile.mkdtemp()- feed_uri = Path(tmpdir) / 'res'+ feed_path = Path(self._random_temp_filename())settings = {- 'FEED_FORMAT': 'csv','FEED_STORE_EMPTY': True,- 'FEED_URI': feed_uri,- 'FEED_PATH': feed_uri+ 'FEEDS': {+ feed_path: {'format': 'csv'}+ },}data = yield self.exported_no_data(settings)- self.assertEqual(data, b'')- shutil.rmtree(tmpdir, ignore_errors=True)+ self.assertEqual(data['csv'], b'')diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 61e110845..f064a646c 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -1,7 +1,9 @@import unittest+import warnings-from scrapy.settings import BaseSettings-from scrapy.utils.conf import build_component_list, arglist_to_dict+from scrapy.exceptions import UsageError, ScrapyDeprecationWarning+from scrapy.settings import BaseSettings, Settings+from scrapy.utils.conf import build_component_list, arglist_to_dict, feed_process_params_from_cliclass BuildComponentListTest(unittest.TestCase):@@ -90,5 +92,49 @@ class UtilsConfTestCase(unittest.TestCase):{'arg1': 'val1', 'arg2': 'val2'})+class FeedExportConfigTestCase(unittest.TestCase):++ def test_feed_export_config_invalid_format(self):+ settings = Settings()+ self.assertRaises(UsageError, feed_process_params_from_cli, settings, ['items.dat'], 'noformat')++ def test_feed_export_config_mismatch(self):+ settings = Settings()+ self.assertRaises(+ UsageError,+ feed_process_params_from_cli, settings, ['items1.dat', 'items2.dat'], 'noformat'+ )++ def test_feed_export_config_backward_compatible(self):+ with warnings.catch_warnings(record=True) as cw:+ settings = Settings()+ self.assertEqual(+ {'items.dat': {'format': 'csv'}},+ feed_process_params_from_cli(settings, ['items.dat'], 'csv')+ )+ self.assertEqual(cw[0].category, ScrapyDeprecationWarning)++ def test_feed_export_config_explicit_formats(self):+ settings = Settings()+ self.assertEqual(+ {'items_1.dat': {'format': 'json'}, 'items_2.dat': {'format': 'xml'}, 'items_3.dat': {'format': 'csv'}},+ feed_process_params_from_cli(settings, ['items_1.dat:json', 'items_2.dat:xml', 'items_3.dat:csv'])+ )++ def test_feed_export_config_implicit_formats(self):+ settings = Settings()+ self.assertEqual(+ {'items_1.json': {'format': 'json'}, 'items_2.xml': {'format': 'xml'}, 'items_3.csv': {'format': 'csv'}},+ feed_process_params_from_cli(settings, ['items_1.json', 'items_2.xml', 'items_3.csv'])+ )++ def test_feed_export_config_stdout(self):+ settings = Settings()+ self.assertEqual(+ {'stdout:': {'format': 'pickle'}},+ feed_process_params_from_cli(settings, ['-:pickle'])+ )++if __name__ == "__main__":unittest.main()