Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names

Adrián Chavesa859ea0c8b8a
diff --cc docs/topics/feed-exports.rst
index a70c22b3c,9e5968a29..d9a772a3a
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@@ -54,9 -54,9 +54,9 @@@ JSON line
CSV
---
- * :setting:`FEED_FORMAT`: ``csv``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- * To specify columns to export and their order use
+ * To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
diff --cc scrapy/utils/conf.py
index 23306ca28,5921f82bf..376c1f992
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@@ -106,3 -109,64 +109,64 @@@ def get_sources(use_closest=True)
if use_closest:
sources.append(closest_scrapy_cfg())
return sources
+
+
+ def feed_complete_default_values_from_settings(feed, settings):
+ out = feed.copy()
+ out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
++ out.setdefault("fields", settings.settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
+ out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
+ out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
+ if settings["FEED_EXPORT_INDENT"] is None:
+ out.setdefault("indent", None)
+ else:
+ out.setdefault("indent", settings.getint("FEED_EXPORT_INDENT"))
+ return out
+
+
+ def feed_process_params_from_cli(settings, output, output_format=None):
+ """
+ Receives feed export params (from the 'crawl' or 'runspider' commands),
+ checks for inconsistencies in their quantities and returns a dictionary
+ suitable to be used as the FEEDS setting.
+ """
+ valid_output_formats = without_none_values(
+ settings.getwithbase('FEED_EXPORTERS')
+ ).keys()
+
+ def check_valid_format(output_format):
+ if output_format not in valid_output_formats:
+ raise UsageError("Unrecognized output format '%s', set one after a"
+ " colon using the -o option (i.e. -o <URI>:<FORMAT>)"
+ " or as a file extension, from the supported list %s" %
+ (output_format, tuple(valid_output_formats)))
+
+ if output_format:
+ if len(output) == 1:
+ check_valid_format(output_format)
+ warnings.warn('The -t command line option is deprecated in favor'
+ ' of specifying the output format within the -o'
+ ' option, please check the -o option docs for more details',
+ category=ScrapyDeprecationWarning, stacklevel=2)
+ return {output[0]: {'format': output_format}}
+ else:
+ raise UsageError('The -t command line option cannot be used if multiple'
+ ' output files are specified with the -o option')
+
+ result = {}
+ for element in output:
+ try:
+ feed_uri, feed_format = element.rsplit(':', 1)
+ except ValueError:
+ feed_uri = element
+ feed_format = os.path.splitext(element)[1].replace('.', '')
+ else:
+ if feed_uri == '-':
+ feed_uri = 'stdout:'
+ check_valid_format(feed_format)
+ result[feed_uri] = {'format': feed_format}
+
+ # FEEDS setting should take precedence over the -o and -t CLI options
+ result.update(settings.getdict('FEEDS'))
+
+ return result
diff --cc tests/test_exporters.py
index 8d49acf20,160912847..837aca6de
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@@ -225,15 -215,15 +225,16 @@@ class CsvItemExporterTest(BaseItemExpor
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
- first = to_unicode(first)
- second = to_unicode(second)
- csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
- for line in csv.splitlines(True)]
- return self.assertEqual(csvsplit(first), csvsplit(second), msg)
+ def split_csv(csv):
+ return [
+ sorted(re.split(r"(,|\s+)", line))
+ for line in to_unicode(csv).splitlines(True)
+ ]
+ return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
- self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+ self.output.seek(0)
+ self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()
diff --cc tests/test_feedexport.py
index 781cdc543,e02b0b840..691fa851c
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -1,35 -1,34 +1,37 @@@
- import os
import csv
import json
- import warnings
- import tempfile
+ import os
+ import random
import shutil
import string
+import sys
+ import tempfile
+ import warnings
+from collections import OrderedDict
from io import BytesIO
from pathlib import Path
+ from string import ascii_letters, digits
from unittest import mock
from urllib.parse import urljoin, urlparse, quote
from urllib.request import pathname2url
+import pytest
- from zope.interface.verify import verifyObject
- from twisted.trial import unittest
+ import lxml.etree
from twisted.internet import defer
- from scrapy.crawler import CrawlerRunner
- from scrapy.settings import Settings
- from tests.mockserver import MockServer
+ from twisted.trial import unittest
from w3lib.url import path_to_file_uri
+ from zope.interface.verify import verifyObject
import scrapy
+ from scrapy.crawler import CrawlerRunner
from scrapy.exporters import CsvItemExporter
- from scrapy.extensions.feedexport import (
- IFeedStorage, FileFeedStorage, FTPFeedStorage,
- S3FeedStorage, StdoutFeedStorage,
- BlockingFeedStorage)
- from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
+ from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
+ IFeedStorage, S3FeedStorage, StdoutFeedStorage)
+ from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
+ from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete
+
+ from tests.mockserver import MockServer
class FileFeedStorageTest(unittest.TestCase):