Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names
Adrián Chavesa859ea0c8b8a
diff --cc docs/topics/feed-exports.rstindex a70c22b3c,9e5968a29..d9a772a3a--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@@ -54,9 -54,9 +54,9 @@@ JSON lineCSV---- * :setting:`FEED_FORMAT`: ``csv``+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`- * To specify columns to export and their order use+ * To specify columns to export, their order and their column names, use:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use thisoption, but it is important for CSV because unlike many other exportformats CSV uses a fixed header.diff --cc scrapy/utils/conf.pyindex 23306ca28,5921f82bf..376c1f992--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@@ -106,3 -109,64 +109,64 @@@ def get_sources(use_closest=True)if use_closest:sources.append(closest_scrapy_cfg())return sources+++ def feed_complete_default_values_from_settings(feed, settings):+ out = feed.copy()+ out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)++ out.setdefault("fields", settings.settings.getdictorlist("FEED_EXPORT_FIELDS") or None)+ out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))+ out.setdefault("uri_params", settings["FEED_URI_PARAMS"])+ if settings["FEED_EXPORT_INDENT"] is None:+ out.setdefault("indent", None)+ else:+ out.setdefault("indent", settings.getint("FEED_EXPORT_INDENT"))+ return out+++ def feed_process_params_from_cli(settings, output, output_format=None):+ """+ Receives feed export params (from the 'crawl' or 'runspider' commands),+ checks for inconsistencies in their quantities and returns a dictionary+ suitable to be used as the FEEDS setting.+ """+ valid_output_formats = without_none_values(+ settings.getwithbase('FEED_EXPORTERS')+ ).keys()++ def check_valid_format(output_format):+ if output_format not in valid_output_formats:+ raise UsageError("Unrecognized output format '%s', set one after a"+ " colon using the -o option (i.e. -o <URI>:<FORMAT>)"+ " or as a file extension, from the supported list %s" %+ (output_format, tuple(valid_output_formats)))++ if output_format:+ if len(output) == 1:+ check_valid_format(output_format)+ warnings.warn('The -t command line option is deprecated in favor'+ ' of specifying the output format within the -o'+ ' option, please check the -o option docs for more details',+ category=ScrapyDeprecationWarning, stacklevel=2)+ return {output[0]: {'format': output_format}}+ else:+ raise UsageError('The -t command line option cannot be used if multiple'+ ' output files are specified with the -o option')++ result = {}+ for element in output:+ try:+ feed_uri, feed_format = element.rsplit(':', 1)+ except ValueError:+ feed_uri = element+ feed_format = os.path.splitext(element)[1].replace('.', '')+ else:+ if feed_uri == '-':+ feed_uri = 'stdout:'+ check_valid_format(feed_format)+ result[feed_uri] = {'format': feed_format}++ # FEEDS setting should take precedence over the -o and -t CLI options+ result.update(settings.getdict('FEEDS'))++ return resultdiff --cc tests/test_exporters.pyindex 8d49acf20,160912847..837aca6de--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@@ -225,15 -215,15 +225,16 @@@ class CsvItemExporterTest(BaseItemExporreturn CsvItemExporter(self.output, **kwargs)def assertCsvEqual(self, first, second, msg=None):- first = to_unicode(first)- second = to_unicode(second)- csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))- for line in csv.splitlines(True)]- return self.assertEqual(csvsplit(first), csvsplit(second), msg)+ def split_csv(csv):+ return [+ sorted(re.split(r"(,|\s+)", line))+ for line in to_unicode(csv).splitlines(True)+ ]+ return self.assertEqual(split_csv(first), split_csv(second), msg=msg)def _check_output(self):- self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')+ self.output.seek(0)+ self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')def assertExportResult(self, item, expected, **kwargs):fp = BytesIO()diff --cc tests/test_feedexport.pyindex 781cdc543,e02b0b840..691fa851c--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@@ -1,35 -1,34 +1,37 @@@- import osimport csvimport json- import warnings- import tempfile+ import os+ import randomimport shutilimport string+import sys+ import tempfile+ import warnings+from collections import OrderedDictfrom io import BytesIOfrom pathlib import Path+ from string import ascii_letters, digitsfrom unittest import mockfrom urllib.parse import urljoin, urlparse, quotefrom urllib.request import pathname2url+import pytest- from zope.interface.verify import verifyObject- from twisted.trial import unittest+ import lxml.etreefrom twisted.internet import defer- from scrapy.crawler import CrawlerRunner- from scrapy.settings import Settings- from tests.mockserver import MockServer+ from twisted.trial import unittestfrom w3lib.url import path_to_file_uri+ from zope.interface.verify import verifyObjectimport scrapy+ from scrapy.crawler import CrawlerRunnerfrom scrapy.exporters import CsvItemExporter- from scrapy.extensions.feedexport import (- IFeedStorage, FileFeedStorage, FTPFeedStorage,- S3FeedStorage, StdoutFeedStorage,- BlockingFeedStorage)- from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler+ from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,+ IFeedStorage, S3FeedStorage, StdoutFeedStorage)+ from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode+ from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete++ from tests.mockserver import MockServerclass FileFeedStorageTest(unittest.TestCase):