Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow overwriting feeds (#4512)
Adrián Chavese70975f0bb4e
diff --git a/docs/faq.rst b/docs/faq.rstindex ea2c8216f..9346ec358 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?To dump into a JSON file::- scrapy crawl myspider -o items.json+ scrapy crawl myspider -O items.jsonTo dump into a CSV file::- scrapy crawl myspider -o items.csv+ scrapy crawl myspider -O items.csvTo dump into a XML file::- scrapy crawl myspider -o items.xml+ scrapy crawl myspider -O items.xmlFor more information see :ref:`topics-feed-exports`diff --git a/docs/intro/overview.rst b/docs/intro/overview.rstindex 01986b594..dd80c7bd0 100644--- a/docs/intro/overview.rst+++ b/docs/intro/overview.rst@@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::if next_page is not None:yield response.follow(next_page, self.parse)-Put this in a text file, name it to something like ``quotes_spider.py``and run the spider using the :command:`runspider` command::- scrapy runspider quotes_spider.py -o quotes.json---When this finishes you will have in the ``quotes.json`` file a list of the-quotes in JSON format, containing text and author, looking like this (reformatted-here for better readability)::-- [{- "author": "Jane Austen",- "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"- },- {- "author": "Groucho Marx",- "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"- },- {- "author": "Steve Martin",- "text": "\u201cA day without sunshine is like, you know, night.\u201d"- },- ...]+ scrapy runspider quotes_spider.py -o quotes.jl++When this finishes you will have in the ``quotes.jl`` file a list of the+quotes in JSON Lines format, containing text and author, looking like this::++ {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}+ {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}+ {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}+ ...What just happened?diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rstindex 5f35dc936..f96c78887 100644--- a/docs/intro/tutorial.rst+++ b/docs/intro/tutorial.rst@@ -464,16 +464,15 @@ Storing the scraped dataThe simplest way to store the scraped data is by using :ref:`Feed exports<topics-feed-exports>`, with the following command::- scrapy crawl quotes -o quotes.json+ scrapy crawl quotes -O quotes.jsonThat will generate an ``quotes.json`` file containing all scraped items,serialized in `JSON`_.-For historic reasons, Scrapy appends to a given file instead of overwriting-its contents. If you run this command twice without removing the file-before the second time, you'll end up with a broken JSON file.--You can also use other formats, like `JSON Lines`_::+The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead+to append new content to any existing file. However, appending to a JSON file+makes the file contents invalid JSON. When appending to a file, consider+using a different serialization format, such as `JSON Lines`_::scrapy crawl quotes -o quotes.jl@@ -704,7 +703,7 @@ Using spider argumentsYou can provide command line arguments to your spiders by using the ``-a``option when running them::- scrapy crawl quotes -o quotes-humor.json -a tag=humor+ scrapy crawl quotes -O quotes-humor.json -a tag=humorThese arguments are passed to the Spider's ``__init__`` method and becomespider attributes by default.diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 0f0f258dc..cd4f7cf29 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -291,6 +291,7 @@ Default: ``{}``A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`object) and each value is a nested dictionary containing configurationparameters for the specific feed.+This setting is required for enabling the feed export feature.See :ref:`topics-feed-storage-backends` for supported URI schemes.@@ -318,17 +319,43 @@ For instance::}The following is a list of the accepted keys and the setting that is used-as a fallback value if that key is not provided for a specific feed definition.--* ``format``: the serialization format to be used for the feed.- See :ref:`topics-feed-format` for possible values.- Mandatory, no fallback setting-* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`-* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`-* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`-* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`-* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`-* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`+as a fallback value if that key is not provided for a specific feed definition:++- ``format``: the :ref:`serialization format <topics-feed-format>`.++ This setting is mandatory, there is no fallback value.++- ``batch_item_count``: falls back to+ :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.++- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.++- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.++- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.++- ``overwrite``: whether to overwrite the file if it already exists+ (``True``) or append to its content (``False``).++ The default value depends on the :ref:`storage backend+ <topics-feed-storage-backends>`:++ - :ref:`topics-feed-storage-fs`: ``False``++ - :ref:`topics-feed-storage-ftp`: ``True``++ .. note:: Some FTP servers may not support appending to files (the+ ``APPE`` FTP command).++ - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported+ <https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)++ - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)++- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.++- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.+.. setting:: FEED_EXPORT_ENCODINGdiff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 57ce4e522..cfd940fe7 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",help="set spider argument (may be repeated)")parser.add_option("-o", "--output", metavar="FILE", action="append",- help="dump scraped items into FILE (use - for stdout)")+ help="append scraped items to the end of FILE (use - for stdout)")+ parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",+ help="dump scraped items into FILE, overwriting any existing file")parser.add_option("-t", "--output-format", metavar="FORMAT",- help="format to use for dumping items with -o")+ help="format to use for dumping items")def process_options(self, args, opts):ScrapyCommand.process_options(self, args, opts)@@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand):opts.spargs = arglist_to_dict(opts.spargs)except ValueError:raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)- if opts.output:- feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)+ if opts.output or opts.overwrite_output:+ feeds = feed_process_params_from_cli(+ self.settings,+ opts.output,+ opts.output_format,+ opts.overwrite_output,+ )self.settings.set('FEEDS', feeds, priority='cmdline')diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex b7a4e362e..980825499 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.ftp import ftp_store_filefrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import create_instance, load_object-from scrapy.utils.python import without_none_values+from scrapy.utils.python import get_func_args, without_none_valueslogger = logging.getLogger(__name__)+def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):+ argument_names = get_func_args(builder)+ if 'feed_options' in argument_names:+ kwargs['feed_options'] = feed_options+ else:+ warnings.warn(+ "{} does not support the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove this "+ "warning. This parameter will become mandatory in a future "+ "version of Scrapy."+ .format(builder.__qualname__),+ category=ScrapyDeprecationWarning+ )+ return builder(*preargs, uri, *args, **kwargs)++class IFeedStorage(Interface):"""Interface that all Feed Storages must implement"""- def __init__(uri):- """Initialize the storage with the parameters given in the URI"""+ def __init__(uri, *, feed_options=None):+ """Initialize the storage with the parameters given in the URI and the+ feed-specific options (see :setting:`FEEDS`)"""def open(spider):"""Open the storage for the given spider. It must return a file-like@@ -64,10 +81,15 @@ class BlockingFeedStorage:@implementer(IFeedStorage)class StdoutFeedStorage:- def __init__(self, uri, _stdout=None):+ def __init__(self, uri, _stdout=None, *, feed_options=None):if not _stdout:_stdout = sys.stdout.bufferself._stdout = _stdout+ if feed_options and feed_options.get('overwrite', False) is True:+ logger.warning('Standard output (stdout) storage does not support '+ 'overwriting. To suppress this warning, remove the '+ 'overwrite option from your FEEDS setting, or set '+ 'it to False.')def open(self, spider):return self._stdout@@ -79,14 +101,16 @@ class StdoutFeedStorage:@implementer(IFeedStorage)class FileFeedStorage:- def __init__(self, uri):+ def __init__(self, uri, *, feed_options=None):self.path = file_uri_to_path(uri)+ feed_options = feed_options or {}+ self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'def open(self, spider):dirname = os.path.dirname(self.path)if dirname and not os.path.exists(dirname):os.makedirs(dirname)- return open(self.path, 'ab')+ return open(self.path, self.write_mode)def store(self, file):file.close()@@ -94,7 +118,8 @@ class FileFeedStorage:class S3FeedStorage(BlockingFeedStorage):- def __init__(self, uri, access_key=None, secret_key=None, acl=None):+ def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,+ feed_options=None):u = urlparse(uri)self.bucketname = u.hostnameself.access_key = u.username or access_key@@ -111,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage):else:import botoself.connect_s3 = boto.connect_s3+ if feed_options and feed_options.get('overwrite', True) is False:+ logger.warning('S3 does not support appending to files. To '+ 'suppress this warning, remove the overwrite '+ 'option from your FEEDS setting or set it to True.')@classmethod- def from_crawler(cls, crawler, uri):- return cls(- uri=uri,+ def from_crawler(cls, crawler, uri, *, feed_options=None):+ return build_storage(+ cls,+ uri,access_key=crawler.settings['AWS_ACCESS_KEY_ID'],secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],- acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None+ acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,+ feed_options=feed_options,)def _store_in_thread(self, file):@@ -135,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage):kwargs = {'policy': self.acl} if self.acl else {}key.set_contents_from_file(file, **kwargs)key.close()+ file.close()class GCSFeedStorage(BlockingFeedStorage):@@ -165,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage):class FTPFeedStorage(BlockingFeedStorage):- def __init__(self, uri, use_active_mode=False):+ def __init__(self, uri, use_active_mode=False, *, feed_options=None):u = urlparse(uri)self.host = u.hostnameself.port = int(u.port or '21')self.username = u.username- self.password = unquote(u.password)+ self.password = unquote(u.password or '')self.path = u.pathself.use_active_mode = use_active_mode+ self.overwrite = not feed_options or feed_options.get('overwrite', True)@classmethod- def from_crawler(cls, crawler, uri):- return cls(- uri=uri,- use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')+ def from_crawler(cls, crawler, uri, *, feed_options=None):+ return build_storage(+ cls,+ uri,+ crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),+ feed_options=feed_options,)def _store_in_thread(self, file):ftp_store_file(path=self.path, file=file, host=self.host,port=self.port, username=self.username,- password=self.password, use_active_mode=self.use_active_mode+ password=self.password, use_active_mode=self.use_active_mode,+ overwrite=self.overwrite,)@@ -242,32 +278,32 @@ class FeedExporter:category=ScrapyDeprecationWarning, stacklevel=2,)uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects- feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}- self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)+ feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}+ self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)# End: Backward compatibility for FEED_URI and FEED_FORMAT settings# 'FEEDS' setting takes precedence over 'FEED_URI'- for uri, feed in self.settings.getdict('FEEDS').items():+ for uri, feed_options in self.settings.getdict('FEEDS').items():uri = str(uri) # handle pathlib.Path objects- self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)+ self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)self.storages = self._load_components('FEED_STORAGES')self.exporters = self._load_components('FEED_EXPORTERS')- for uri, feed in self.feeds.items():- if not self._storage_supported(uri):+ for uri, feed_options in self.feeds.items():+ if not self._storage_supported(uri, feed_options):raise NotConfiguredif not self._settings_are_valid():raise NotConfigured- if not self._exporter_supported(feed['format']):+ if not self._exporter_supported(feed_options['format']):raise NotConfigureddef open_spider(self, spider):- for uri, feed in self.feeds.items():- uri_params = self._get_uri_params(spider, feed['uri_params'])+ for uri, feed_options in self.feeds.items():+ uri_params = self._get_uri_params(spider, feed_options['uri_params'])self.slots.append(self._start_new_batch(batch_id=1,uri=uri % uri_params,- feed=feed,+ feed_options=feed_options,spider=spider,uri_template=uri,))@@ -306,32 +342,32 @@ class FeedExporter:)return d- def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):+ def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):"""Redirect the output data stream to a new file.Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified:param batch_id: sequence number of current batch:param uri: uri of the new batch to start- :param feed: dict with parameters of feed+ :param feed_options: dict with parameters of feed:param spider: user spider:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri"""- storage = self._get_storage(uri)+ storage = self._get_storage(uri, feed_options)file = storage.open(spider)exporter = self._get_exporter(file=file,- format=feed['format'],- fields_to_export=feed['fields'],- encoding=feed['encoding'],- indent=feed['indent'],+ format=feed_options['format'],+ fields_to_export=feed_options['fields'],+ encoding=feed_options['encoding'],+ indent=feed_options['indent'],)slot = _FeedSlot(file=file,exporter=exporter,storage=storage,uri=uri,- format=feed['format'],- store_empty=feed['store_empty'],+ format=feed_options['format'],+ store_empty=feed_options['store_empty'],batch_id=batch_id,uri_template=uri_template,)@@ -355,7 +391,7 @@ class FeedExporter:slots.append(self._start_new_batch(batch_id=slot.batch_id + 1,uri=slot.uri_template % uri_params,- feed=self.feeds[slot.uri_template],+ feed_options=self.feeds[slot.uri_template],spider=spider,uri_template=slot.uri_template,))@@ -394,11 +430,11 @@ class FeedExporter:return Falsereturn True- def _storage_supported(self, uri):+ def _storage_supported(self, uri, feed_options):scheme = urlparse(uri).schemeif scheme in self.storages:try:- self._get_storage(uri)+ self._get_storage(uri, feed_options)return Trueexcept NotConfigured as e:logger.error("Disabled feed storage scheme: %(scheme)s. "@@ -416,8 +452,30 @@ class FeedExporter:def _get_exporter(self, file, format, *args, **kwargs):return self._get_instance(self.exporters[format], file, *args, **kwargs)- def _get_storage(self, uri):- return self._get_instance(self.storages[urlparse(uri).scheme], uri)+ def _get_storage(self, uri, feed_options):+ """Fork of create_instance specific to feed storage classes++ It supports not passing the *feed_options* parameters to classes that+ do not support it, and issuing a deprecation warning instead.+ """+ feedcls = self.storages[urlparse(uri).scheme]+ crawler = getattr(self, 'crawler', None)++ def build_instance(builder, *preargs):+ return build_storage(builder, uri, preargs=preargs)++ if crawler and hasattr(feedcls, 'from_crawler'):+ instance = build_instance(feedcls.from_crawler, crawler)+ method_name = 'from_crawler'+ elif hasattr(feedcls, 'from_settings'):+ instance = build_instance(feedcls.from_settings, self.settings)+ method_name = 'from_settings'+ else:+ instance = build_instance(feedcls)+ method_name = '__new__'+ if instance is None:+ raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))+ return instancedef _get_uri_params(self, spider, uri_params, slot=None):params = {}diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex a83076c47..90a52b25b 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings):return out-def feed_process_params_from_cli(settings, output, output_format=None):+def feed_process_params_from_cli(settings, output, output_format=None,+ overwrite_output=None):"""Receives feed export params (from the 'crawl' or 'runspider' commands),checks for inconsistencies in their quantities and returns a dictionary@@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None):def check_valid_format(output_format):if output_format not in valid_output_formats:- raise UsageError("Unrecognized output format '%s', set one after a"- " colon using the -o option (i.e. -o <URI>:<FORMAT>)"- " or as a file extension, from the supported list %s" %- (output_format, tuple(valid_output_formats)))+ raise UsageError(+ "Unrecognized output format '%s'. Set a supported one (%s) "+ "after a colon at the end of the output URI (i.e. -o/-O "+ "<URI>:<FORMAT>) or as a file extension." % (+ output_format,+ tuple(valid_output_formats),+ )+ )++ overwrite = False+ if overwrite_output:+ if output:+ raise UsageError(+ "Please use only one of -o/--output and -O/--overwrite-output"+ )+ output = overwrite_output+ overwrite = Trueif output_format:if len(output) == 1:check_valid_format(output_format)- warnings.warn('The -t command line option is deprecated in favor'- ' of specifying the output format within the -o'- ' option, please check the -o option docs for more details',- category=ScrapyDeprecationWarning, stacklevel=2)+ message = (+ 'The -t command line option is deprecated in favor of '+ 'specifying the output format within the output URI. See the '+ 'documentation of the -o and -O options for more information.',+ )+ warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)return {output[0]: {'format': output_format}}else:- raise UsageError('The -t command line option cannot be used if multiple'- ' output files are specified with the -o option')+ raise UsageError(+ 'The -t command-line option cannot be used if multiple output '+ 'URIs are specified'+ )result = {}for element in output:@@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None):feed_uri = 'stdout:'check_valid_format(feed_format)result[feed_uri] = {'format': feed_format}+ if overwrite:+ result[feed_uri]['overwrite'] = True- # FEEDS setting should take precedence over the -o and -t CLI options+ # FEEDS setting should take precedence over the matching CLI optionsresult.update(settings.getdict('FEEDS'))return resultdiff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.pyindex f07bdd748..19d56d6ec 100644--- a/scrapy/utils/ftp.py+++ b/scrapy/utils/ftp.py@@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):def ftp_store_file(*, path, file, host, port,- username, password, use_active_mode=False):+ username, password, use_active_mode=False, overwrite=True):"""Opens a FTP connection with passed credentials,sets current directoryto the directory extracted from given path, then uploads the file to server"""@@ -32,4 +32,6 @@ def ftp_store_file(file.seek(0)dirname, filename = posixpath.split(path)ftp_makedirs_cwd(ftp, dirname)- ftp.storbinary('STOR %s' % filename, file)+ command = 'STOR' if overwrite else 'APPE'+ ftp.storbinary('%s %s' % (command, filename), file)+ file.close()diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 59f1b8371..1f2333264 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -198,7 +198,8 @@ def _getargspec_py23(func):def get_func_args(func, stripself=False):"""Return the argument name list of a callable"""if inspect.isfunction(func):- func_args, _, _, _ = _getargspec_py23(func)+ spec = inspect.getfullargspec(func)+ func_args = spec.args + spec.kwonlyargselif inspect.isclass(func):return get_func_args(func.__init__, True)elif inspect.ismethod(func):diff --git a/tests/ftpserver.py b/tests/ftpserver.pynew file mode 100644index 000000000..6f0289e08--- /dev/null+++ b/tests/ftpserver.py@@ -0,0 +1,24 @@+from argparse import ArgumentParser++from pyftpdlib.authorizers import DummyAuthorizer+from pyftpdlib.handlers import FTPHandler+from pyftpdlib.servers import FTPServer+++def main():+ parser = ArgumentParser()+ parser.add_argument('-d', '--directory')+ args = parser.parse_args()++ authorizer = DummyAuthorizer()+ full_permissions = 'elradfmwMT'+ authorizer.add_anonymous(args.directory, perm=full_permissions)+ handler = FTPHandler+ handler.authorizer = authorizer+ address = ('127.0.0.1', 2121)+ server = FTPServer(address, handler)+ server.serve_forever()+++if __name__ == '__main__':+ main()diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 1f40473ba..48d7b8d37 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -3,7 +3,10 @@ import jsonimport osimport randomimport sys+from pathlib import Path+from shutil import rmtreefrom subprocess import Popen, PIPE+from tempfile import mkdtempfrom urllib.parse import urlencodefrom OpenSSL import SSL@@ -256,6 +259,29 @@ class MockDNSServer:self.proc.communicate()+class MockFTPServer:+ """Creates an FTP server on port 2121 with a default passwordless user+ (anonymous) and a temporary root path that you can read from the+ :attr:`path` attribute."""++ def __enter__(self):+ self.path = Path(mkdtemp())+ self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)],+ stderr=PIPE, env=get_testenv())+ for line in self.proc.stderr:+ if b'starting FTP server' in line:+ break+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ rmtree(str(self.path))+ self.proc.kill()+ self.proc.communicate()++ def url(self, path):+ return 'ftp://127.0.0.1:2121/' + path++def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):factory = ssl.DefaultOpenSSLContextFactory(os.path.join(os.path.dirname(__file__), keyfile),diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txtindex b51177abb..fe1cbc997 100644--- a/tests/requirements-py3.txt+++ b/tests/requirements-py3.txt@@ -4,6 +4,7 @@ dataclasses; python_version == '3.6'mitmproxy; python_version >= '3.7'mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'mitmproxy < 4; python_version < '3.6'+pyftpdlib# https://github.com/pytest-dev/pytest-twisted/issues/93pytest != 5.4, != 5.4.1pytest-azurepipelinesdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 109f006a8..f76f851e7 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -74,6 +74,7 @@ class ProjectTest(unittest.TestCase):def kill_proc():p.kill()+ p.communicate()assert False, 'Command took too much time to complete'timer = Timer(15, kill_proc)@@ -569,6 +570,55 @@ class BadSpider(scrapy.Spider):log = self.get_log(self.debug_log_spider, args=[])self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)+ def test_output(self):+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))+ return []+"""+ args = ['-o', 'example.json']+ log = self.get_log(spider_code, args=args)+ self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)++ def test_overwrite_output(self):+ spider_code = """+import json+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ self.logger.debug(+ 'FEEDS: {}'.format(+ json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)+ )+ )+ return []+"""+ args = ['-O', 'example.json']+ log = self.get_log(spider_code, args=args)+ self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)++ def test_output_and_overwrite_output(self):+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ return []+"""+ args = ['-o', 'example1.json', '-O', 'example2.json']+ log = self.get_log(spider_code, args=args)+ self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)+class BenchCommandTest(CommandTest):@@ -577,3 +627,79 @@ class BenchCommandTest(CommandTest):'-s', 'CLOSESPIDER_TIMEOUT=0.01')self.assertIn('INFO: Crawled', log)self.assertNotIn('Unhandled Error', log)+++class CrawlCommandTest(CommandTest):++ def crawl(self, code, args=()):+ fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))+ with open(fname, 'w') as f:+ f.write(code)+ return self.proc('crawl', 'myspider', *args)++ def get_log(self, code, args=()):+ _, _, stderr = self.crawl(code, args=args)+ return stderr++ def test_no_output(self):+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ self.logger.debug('It works!')+ return []+"""+ log = self.get_log(spider_code)+ self.assertIn("[myspider] DEBUG: It works!", log)++ def test_output(self):+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))+ return []+"""+ args = ['-o', 'example.json']+ log = self.get_log(spider_code, args=args)+ self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)++ def test_overwrite_output(self):+ spider_code = """+import json+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ self.logger.debug(+ 'FEEDS: {}'.format(+ json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)+ )+ )+ return []+"""+ args = ['-O', 'example.json']+ log = self.get_log(spider_code, args=args)+ self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)++ def test_output_and_overwrite_output(self):+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ def start_requests(self):+ return []+"""+ args = ['-o', 'example1.json', '-O', 'example2.json']+ log = self.get_log(spider_code, args=args)+ self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 2afc25a7a..850485b5e 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -5,6 +5,7 @@ import randomimport shutilimport stringimport tempfile+import warningsfrom abc import ABC, abstractmethodfrom collections import defaultdictfrom io import BytesIO@@ -25,7 +26,7 @@ from zope.interface.verify import verifyObjectimport scrapyfrom scrapy.crawler import CrawlerRunner-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.exporters import CsvItemExporterfrom scrapy.extensions.feedexport import (BlockingFeedStorage,@@ -46,7 +47,7 @@ from scrapy.utils.test import (mock_google_cloud_storage,)-from tests.mockserver import MockServer+from tests.mockserver import MockFTPServer, MockServerclass FileFeedStorageTest(unittest.TestCase):@@ -75,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase):st = FileFeedStorage(path)verifyObject(IFeedStorage, st)+ def _store(self, feed_options=None):+ path = os.path.abspath(self.mktemp())+ storage = FileFeedStorage(path, feed_options=feed_options)+ spider = scrapy.Spider("default")+ file = storage.open(spider)+ file.write(b"content")+ storage.store(file)+ return path++ def test_append(self):+ path = self._store()+ return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")++ def test_overwrite(self):+ path = self._store({"overwrite": True})+ return self._assert_stores(+ FileFeedStorage(path, feed_options={"overwrite": True}),+ path+ )+@defer.inlineCallbacks- def _assert_stores(self, storage, path):+ def _assert_stores(self, storage, path, expected_content=b"content"):spider = scrapy.Spider("default")file = storage.open(spider)file.write(b"content")@@ -84,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase):self.assertTrue(os.path.exists(path))try:with open(path, 'rb') as fp:- self.assertEqual(fp.read(), b"content")+ self.assertEqual(fp.read(), expected_content)finally:os.unlink(path)@@ -99,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase):spider = TestSpider.from_crawler(crawler)return spider- def test_store(self):- uri = os.environ.get('FEEDTEST_FTP_URI')- path = os.environ.get('FEEDTEST_FTP_PATH')- if not (uri and path):- raise unittest.SkipTest("No FTP server available for testing")- st = FTPFeedStorage(uri)- verifyObject(IFeedStorage, st)- return self._assert_stores(st, path)-- def test_store_active_mode(self):- uri = os.environ.get('FEEDTEST_FTP_URI')- path = os.environ.get('FEEDTEST_FTP_PATH')- if not (uri and path):- raise unittest.SkipTest("No FTP server available for testing")- use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True}- crawler = get_crawler(settings_dict=use_active_mode)- st = FTPFeedStorage.from_crawler(crawler, uri)- verifyObject(IFeedStorage, st)- return self._assert_stores(st, path)+ def _store(self, uri, content, feed_options=None, settings=None):+ crawler = get_crawler(settings_dict=settings or {})+ storage = FTPFeedStorage.from_crawler(+ crawler,+ uri,+ feed_options=feed_options,+ )+ verifyObject(IFeedStorage, storage)+ spider = self.get_test_spider()+ file = storage.open(spider)+ file.write(content)+ return storage.store(file)++ def _assert_stored(self, path, content):+ self.assertTrue(path.exists())+ try:+ with path.open('rb') as fp:+ self.assertEqual(fp.read(), content)+ finally:+ os.unlink(str(path))++ @defer.inlineCallbacks+ def test_append(self):+ with MockFTPServer() as ftp_server:+ filename = 'file'+ url = ftp_server.url(filename)+ feed_options = {'overwrite': False}+ yield self._store(url, b"foo", feed_options=feed_options)+ yield self._store(url, b"bar", feed_options=feed_options)+ self._assert_stored(ftp_server.path / filename, b"foobar")++ @defer.inlineCallbacks+ def test_overwrite(self):+ with MockFTPServer() as ftp_server:+ filename = 'file'+ url = ftp_server.url(filename)+ yield self._store(url, b"foo")+ yield self._store(url, b"bar")+ self._assert_stored(ftp_server.path / filename, b"bar")++ @defer.inlineCallbacks+ def test_append_active_mode(self):+ with MockFTPServer() as ftp_server:+ settings = {'FEED_STORAGE_FTP_ACTIVE': True}+ filename = 'file'+ url = ftp_server.url(filename)+ feed_options = {'overwrite': False}+ yield self._store(url, b"foo", feed_options=feed_options, settings=settings)+ yield self._store(url, b"bar", feed_options=feed_options, settings=settings)+ self._assert_stored(ftp_server.path / filename, b"foobar")++ @defer.inlineCallbacks+ def test_overwrite_active_mode(self):+ with MockFTPServer() as ftp_server:+ settings = {'FEED_STORAGE_FTP_ACTIVE': True}+ filename = 'file'+ url = ftp_server.url(filename)+ yield self._store(url, b"foo", settings=settings)+ yield self._store(url, b"bar", settings=settings)+ self._assert_stored(ftp_server.path / filename, b"bar")def test_uri_auth_quote(self):# RFC3986: 3.2.1. User Informationpw_quoted = quote(string.punctuation, safe='')- st = FTPFeedStorage('ftp://foo:%[email protected]/some_path' % pw_quoted)+ st = FTPFeedStorage('ftp://foo:%[email protected]/some_path' % pw_quoted,+ {})self.assertEqual(st.password, string.punctuation)- @defer.inlineCallbacks- def _assert_stores(self, storage, path):- spider = self.get_test_spider()- file = storage.open(spider)- file.write(b"content")- yield storage.store(file)- self.assertTrue(os.path.exists(path))- try:- with open(path, 'rb') as fp:- self.assertEqual(fp.read(), b"content")- # again, to check s3 objects are overwritten- yield storage.store(BytesIO(b"new content"))- with open(path, 'rb') as fp:- self.assertEqual(fp.read(), b"new content")- finally:- os.unlink(path)-class BlockingFeedStorageTest(unittest.TestCase):@@ -190,8 +236,10 @@ class S3FeedStorageTest(unittest.TestCase):'AWS_SECRET_ACCESS_KEY': 'settings_secret'}crawler = get_crawler(settings_dict=aws_credentials)# Instantiate with crawler- storage = S3FeedStorage.from_crawler(crawler,- 's3://mybucket/export.csv')+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv',+ )self.assertEqual(storage.access_key, 'settings_key')self.assertEqual(storage.secret_key, 'settings_secret')# Instantiate directly@@ -254,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase):crawler = get_crawler(settings_dict=settings)storage = S3FeedStorage.from_crawler(crawler,- 's3://mybucket/export.csv'+ 's3://mybucket/export.csv',)self.assertEqual(storage.access_key, 'access_key')self.assertEqual(storage.secret_key, 'secret_key')@@ -269,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase):crawler = get_crawler(settings_dict=settings)storage = S3FeedStorage.from_crawler(crawler,- 's3://mybucket/export.csv'+ 's3://mybucket/export.csv',)self.assertEqual(storage.access_key, 'access_key')self.assertEqual(storage.secret_key, 'secret_key')@@ -370,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase):key.set_contents_from_file.call_args)+ def test_overwrite_default(self):+ with LogCapture() as log:+ S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl'+ )+ self.assertNotIn('S3 does not support appending to files', str(log))++ def test_overwrite_false(self):+ with LogCapture() as log:+ S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl',+ feed_options={'overwrite': False},+ )+ self.assertIn('S3 does not support appending to files', str(log))+class GCSFeedStorageTest(unittest.TestCase):@@ -439,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase):yield storage.store(file)self.assertEqual(out.getvalue(), b"content")+ def test_overwrite_default(self):+ with LogCapture() as log:+ StdoutFeedStorage('stdout:')+ self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log))++ def test_overwrite_true(self):+ with LogCapture() as log:+ StdoutFeedStorage('stdout:', feed_options={'overwrite': True})+ self.assertIn('Standard output (stdout) storage does not support overwriting', str(log))+class FromCrawlerMixin:init_with_crawler = False@classmethod- def from_crawler(cls, crawler, *args, **kwargs):+ def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):cls.init_with_crawler = Truereturn cls(*args, **kwargs)@@ -454,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin):class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):- pass++ @classmethod+ def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):+ cls.init_with_crawler = True+ return cls(*args, feed_options=feed_options, **kwargs)class DummyBlockingFeedStorage(BlockingFeedStorage):@@ -588,8 +671,8 @@ class FeedExportTest(FeedExportTestBase):FEEDS = settings.get('FEEDS') or {}settings['FEEDS'] = {- printf_escape(path_to_url(file_path)): feed- for file_path, feed in FEEDS.items()+ printf_escape(path_to_url(file_path)): feed_options+ for file_path, feed_options in FEEDS.items()}content = {}@@ -599,12 +682,12 @@ class FeedExportTest(FeedExportTestBase):spider_cls.start_urls = [s.url('/')]yield runner.crawl(spider_cls)- for file_path, feed in FEEDS.items():+ for file_path, feed_options in FEEDS.items():if not os.path.exists(str(file_path)):continuewith open(str(file_path), 'rb') as f:- content[feed['format']] = f.read()+ content[feed_options['format']] = f.read()finally:for file_path in FEEDS.keys():@@ -1542,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase):content = json.loads(content.decode('utf-8'))expected_batch, items = items[:batch_size], items[batch_size:]self.assertEqual(expected_batch, content)+++class FeedExportInitTest(unittest.TestCase):++ def test_unsupported_storage(self):+ settings = {+ 'FEEDS': {+ 'unsupported://uri': {},+ },+ }+ crawler = get_crawler(settings_dict=settings)+ with self.assertRaises(NotConfigured):+ FeedExporter.from_crawler(crawler)++ def test_unsupported_format(self):+ settings = {+ 'FEEDS': {+ 'file://path': {+ 'format': 'unsupported_format',+ },+ },+ }+ crawler = get_crawler(settings_dict=settings)+ with self.assertRaises(NotConfigured):+ FeedExporter.from_crawler(crawler)+++class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):++ def __init__(self, uri):+ super().__init__(uri)+++class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):+ """Make sure that any feed exporter created by users before the+ introduction of the ``feed_options`` parameter continues to work as+ expected, and simply issues a warning."""++ def test_init(self):+ settings_dict = {+ 'FEED_URI': 'file:///tmp/foobar',+ 'FEED_STORAGES': {+ 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions'+ },+ }+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+ spider = scrapy.Spider("default")+ with warnings.catch_warnings(record=True) as w:+ feed_exporter.open_spider(spider)+ messages = tuple(str(item.message) for item in w+ if item.category is ScrapyDeprecationWarning)+ self.assertEqual(+ messages,+ (+ (+ "StdoutFeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove "+ "this warning. This parameter will become mandatory "+ "in a future version of Scrapy."+ ),+ )+ )+++class FileFeedStorageWithoutFeedOptions(FileFeedStorage):++ def __init__(self, uri):+ super().__init__(uri)+++class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):+ """Make sure that any feed exporter created by users before the+ introduction of the ``feed_options`` parameter continues to work as+ expected, and simply issues a warning."""++ maxDiff = None++ def test_init(self):+ settings_dict = {+ 'FEED_URI': 'file:///tmp/foobar',+ 'FEED_STORAGES': {+ 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions'+ },+ }+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+ spider = scrapy.Spider("default")+ with warnings.catch_warnings(record=True) as w:+ feed_exporter.open_spider(spider)+ messages = tuple(str(item.message) for item in w+ if item.category is ScrapyDeprecationWarning)+ self.assertEqual(+ messages,+ (+ (+ "FileFeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove "+ "this warning. This parameter will become mandatory "+ "in a future version of Scrapy."+ ),+ )+ )+++class S3FeedStorageWithoutFeedOptions(S3FeedStorage):++ def __init__(self, uri, access_key, secret_key, acl):+ super().__init__(uri, access_key, secret_key, acl)+++class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):++ @classmethod+ def from_crawler(cls, crawler, uri):+ return super().from_crawler(crawler, uri)+++class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):+ """Make sure that any feed exporter created by users before the+ introduction of the ``feed_options`` parameter continues to work as+ expected, and simply issues a warning."""++ maxDiff = None++ def test_init(self):+ settings_dict = {+ 'FEED_URI': 'file:///tmp/foobar',+ 'FEED_STORAGES': {+ 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions'+ },+ }+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+ spider = scrapy.Spider("default")+ spider.crawler = crawler+ with warnings.catch_warnings(record=True) as w:+ feed_exporter.open_spider(spider)+ messages = tuple(str(item.message) for item in w+ if item.category is ScrapyDeprecationWarning)+ self.assertEqual(+ messages,+ (+ (+ "S3FeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove "+ "this warning. This parameter will become mandatory "+ "in a future version of Scrapy."+ ),+ )+ )++ def test_from_crawler(self):+ settings_dict = {+ 'FEED_URI': 'file:///tmp/foobar',+ 'FEED_STORAGES': {+ 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler'+ },+ }+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+ spider = scrapy.Spider("default")+ spider.crawler = crawler+ with warnings.catch_warnings(record=True) as w:+ feed_exporter.open_spider(spider)+ messages = tuple(str(item.message) for item in w+ if item.category is ScrapyDeprecationWarning)+ self.assertEqual(+ messages,+ (+ (+ "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "+ "does not support the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove "+ "this warning. This parameter will become mandatory "+ "in a future version of Scrapy."+ ),+ )+ )+++class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):++ def __init__(self, uri, use_active_mode=False):+ super().__init__(uri)+++class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):++ @classmethod+ def from_crawler(cls, crawler, uri):+ return super().from_crawler(crawler, uri)+++class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):+ """Make sure that any feed exporter created by users before the+ introduction of the ``feed_options`` parameter continues to work as+ expected, and simply issues a warning."""++ maxDiff = None++ def test_init(self):+ settings_dict = {+ 'FEED_URI': 'file:///tmp/foobar',+ 'FEED_STORAGES': {+ 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions'+ },+ }+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+ spider = scrapy.Spider("default")+ spider.crawler = crawler+ with warnings.catch_warnings(record=True) as w:+ feed_exporter.open_spider(spider)+ messages = tuple(str(item.message) for item in w+ if item.category is ScrapyDeprecationWarning)+ self.assertEqual(+ messages,+ (+ (+ "FTPFeedStorageWithoutFeedOptions does not support "+ "the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove "+ "this warning. This parameter will become mandatory "+ "in a future version of Scrapy."+ ),+ )+ )++ def test_from_crawler(self):+ settings_dict = {+ 'FEED_URI': 'file:///tmp/foobar',+ 'FEED_STORAGES': {+ 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler'+ },+ }+ crawler = get_crawler(settings_dict=settings_dict)+ feed_exporter = FeedExporter.from_crawler(crawler)+ spider = scrapy.Spider("default")+ spider.crawler = crawler+ with warnings.catch_warnings(record=True) as w:+ feed_exporter.open_spider(spider)+ messages = tuple(str(item.message) for item in w+ if item.category is ScrapyDeprecationWarning)+ self.assertEqual(+ messages,+ (+ (+ "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "+ "does not support the 'feed_options' keyword argument. Add a "+ "'feed_options' parameter to its signature to remove "+ "this warning. This parameter will become mandatory "+ "in a future version of Scrapy."+ ),+ )+ )diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex f3ef36127..ccc65c4fd 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase):feed_process_params_from_cli(settings, ['-:pickle']))+ def test_feed_export_config_overwrite(self):+ settings = Settings()+ self.assertEqual(+ {'output.json': {'format': 'json', 'overwrite': True}},+ feed_process_params_from_cli(settings, [], None, ['output.json'])+ )++ def test_output_and_overwrite_output(self):+ with self.assertRaises(UsageError):+ feed_process_params_from_cli(+ Settings(),+ ['output1.json'],+ None,+ ['output2.json'],+ )+def test_feed_complete_default_values_from_settings_empty(self):feed = {}settings = Settings({diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 3f93f509e..c298d0bd2 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase):def f2(a, b=None, c=None):pass+ def f3(a, b=None, *, c=None):+ pass+class A:def __init__(self, a, b, c):pass@@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase):self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])+ self.assertEqual(get_func_args(f3), ['a', 'b', 'c'])self.assertEqual(get_func_args(A), ['a', 'b', 'c'])self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])self.assertEqual(get_func_args(partial_f1), ['b', 'c'])