Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow overwriting feeds (#4512)

Adrián Chavese70975f0bb4e
diff --git a/docs/faq.rst b/docs/faq.rst
index ea2c8216f..9346ec358 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
To dump into a JSON file::
- scrapy crawl myspider -o items.json
+ scrapy crawl myspider -O items.json
To dump into a CSV file::
- scrapy crawl myspider -o items.csv
+ scrapy crawl myspider -O items.csv
To dump into a XML file::
- scrapy crawl myspider -o items.xml
+ scrapy crawl myspider -O items.xml
For more information see :ref:`topics-feed-exports`
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 01986b594..dd80c7bd0 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
if next_page is not None:
yield response.follow(next_page, self.parse)
-
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
- scrapy runspider quotes_spider.py -o quotes.json
-
-
-When this finishes you will have in the ``quotes.json`` file a list of the
-quotes in JSON format, containing text and author, looking like this (reformatted
-here for better readability)::
-
- [{
- "author": "Jane Austen",
- "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
- },
- {
- "author": "Groucho Marx",
- "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
- },
- {
- "author": "Steve Martin",
- "text": "\u201cA day without sunshine is like, you know, night.\u201d"
- },
- ...]
+ scrapy runspider quotes_spider.py -o quotes.jl
+
+When this finishes you will have in the ``quotes.jl`` file a list of the
+quotes in JSON Lines format, containing text and author, looking like this::
+
+ {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
+ {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
+ {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
+ ...
What just happened?
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 5f35dc936..f96c78887 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -464,16 +464,15 @@ Storing the scraped data
The simplest way to store the scraped data is by using :ref:`Feed exports
<topics-feed-exports>`, with the following command::
- scrapy crawl quotes -o quotes.json
+ scrapy crawl quotes -O quotes.json
That will generate an ``quotes.json`` file containing all scraped items,
serialized in `JSON`_.
-For historic reasons, Scrapy appends to a given file instead of overwriting
-its contents. If you run this command twice without removing the file
-before the second time, you'll end up with a broken JSON file.
-
-You can also use other formats, like `JSON Lines`_::
+The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
+to append new content to any existing file. However, appending to a JSON file
+makes the file contents invalid JSON. When appending to a file, consider
+using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
@@ -704,7 +703,7 @@ Using spider arguments
You can provide command line arguments to your spiders by using the ``-a``
option when running them::
- scrapy crawl quotes -o quotes-humor.json -a tag=humor
+ scrapy crawl quotes -O quotes-humor.json -a tag=humor
These arguments are passed to the Spider's ``__init__`` method and become
spider attributes by default.
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 0f0f258dc..cd4f7cf29 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -291,6 +291,7 @@ Default: ``{}``
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
+
This setting is required for enabling the feed export feature.
See :ref:`topics-feed-storage-backends` for supported URI schemes.
@@ -318,17 +319,43 @@ For instance::
}
The following is a list of the accepted keys and the setting that is used
-as a fallback value if that key is not provided for a specific feed definition.
-
-* ``format``: the serialization format to be used for the feed.
- See :ref:`topics-feed-format` for possible values.
- Mandatory, no fallback setting
-* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
-* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
-* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
-* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
-* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
-* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`
+as a fallback value if that key is not provided for a specific feed definition:
+
+- ``format``: the :ref:`serialization format <topics-feed-format>`.
+
+ This setting is mandatory, there is no fallback value.
+
+- ``batch_item_count``: falls back to
+ :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
+
+- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
+
+- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
+
+- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
+
+- ``overwrite``: whether to overwrite the file if it already exists
+ (``True``) or append to its content (``False``).
+
+ The default value depends on the :ref:`storage backend
+ <topics-feed-storage-backends>`:
+
+ - :ref:`topics-feed-storage-fs`: ``False``
+
+ - :ref:`topics-feed-storage-ftp`: ``True``
+
+ .. note:: Some FTP servers may not support appending to files (the
+ ``APPE`` FTP command).
+
+ - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
+ <https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
+
+ - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
+
+- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
+
+- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
+
.. setting:: FEED_EXPORT_ENCODING
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 57ce4e522..cfd940fe7 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
- help="dump scraped items into FILE (use - for stdout)")
+ help="append scraped items to the end of FILE (use - for stdout)")
+ parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
+ help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
- help="format to use for dumping items with -o")
+ help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
- if opts.output:
- feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+ if opts.output or opts.overwrite_output:
+ feeds = feed_process_params_from_cli(
+ self.settings,
+ opts.output,
+ opts.output_format,
+ opts.overwrite_output,
+ )
self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b7a4e362e..980825499 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.python import without_none_values
+from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
+def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
+ argument_names = get_func_args(builder)
+ if 'feed_options' in argument_names:
+ kwargs['feed_options'] = feed_options
+ else:
+ warnings.warn(
+ "{} does not support the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove this "
+ "warning. This parameter will become mandatory in a future "
+ "version of Scrapy."
+ .format(builder.__qualname__),
+ category=ScrapyDeprecationWarning
+ )
+ return builder(*preargs, uri, *args, **kwargs)
+
+
class IFeedStorage(Interface):
"""Interface that all Feed Storages must implement"""
- def __init__(uri):
- """Initialize the storage with the parameters given in the URI"""
+ def __init__(uri, *, feed_options=None):
+ """Initialize the storage with the parameters given in the URI and the
+ feed-specific options (see :setting:`FEEDS`)"""
def open(spider):
"""Open the storage for the given spider. It must return a file-like
@@ -64,10 +81,15 @@ class BlockingFeedStorage:
@implementer(IFeedStorage)
class StdoutFeedStorage:
- def __init__(self, uri, _stdout=None):
+ def __init__(self, uri, _stdout=None, *, feed_options=None):
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout = _stdout
+ if feed_options and feed_options.get('overwrite', False) is True:
+ logger.warning('Standard output (stdout) storage does not support '
+ 'overwriting. To suppress this warning, remove the '
+ 'overwrite option from your FEEDS setting, or set '
+ 'it to False.')
def open(self, spider):
return self._stdout
@@ -79,14 +101,16 @@ class StdoutFeedStorage:
@implementer(IFeedStorage)
class FileFeedStorage:
- def __init__(self, uri):
+ def __init__(self, uri, *, feed_options=None):
self.path = file_uri_to_path(uri)
+ feed_options = feed_options or {}
+ self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
- return open(self.path, 'ab')
+ return open(self.path, self.write_mode)
def store(self, file):
file.close()
@@ -94,7 +118,8 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
- def __init__(self, uri, access_key=None, secret_key=None, acl=None):
+ def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
+ feed_options=None):
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
@@ -111,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage):
else:
import boto
self.connect_s3 = boto.connect_s3
+ if feed_options and feed_options.get('overwrite', True) is False:
+ logger.warning('S3 does not support appending to files. To '
+ 'suppress this warning, remove the overwrite '
+ 'option from your FEEDS setting or set it to True.')
@classmethod
- def from_crawler(cls, crawler, uri):
- return cls(
- uri=uri,
+ def from_crawler(cls, crawler, uri, *, feed_options=None):
+ return build_storage(
+ cls,
+ uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
- acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
+ acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
+ feed_options=feed_options,
)
def _store_in_thread(self, file):
@@ -135,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage):
kwargs = {'policy': self.acl} if self.acl else {}
key.set_contents_from_file(file, **kwargs)
key.close()
+ file.close()
class GCSFeedStorage(BlockingFeedStorage):
@@ -165,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage):
class FTPFeedStorage(BlockingFeedStorage):
- def __init__(self, uri, use_active_mode=False):
+ def __init__(self, uri, use_active_mode=False, *, feed_options=None):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
self.username = u.username
- self.password = unquote(u.password)
+ self.password = unquote(u.password or '')
self.path = u.path
self.use_active_mode = use_active_mode
+ self.overwrite = not feed_options or feed_options.get('overwrite', True)
@classmethod
- def from_crawler(cls, crawler, uri):
- return cls(
- uri=uri,
- use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+ def from_crawler(cls, crawler, uri, *, feed_options=None):
+ return build_storage(
+ cls,
+ uri,
+ crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
+ feed_options=feed_options,
)
def _store_in_thread(self, file):
ftp_store_file(
path=self.path, file=file, host=self.host,
port=self.port, username=self.username,
- password=self.password, use_active_mode=self.use_active_mode
+ password=self.password, use_active_mode=self.use_active_mode,
+ overwrite=self.overwrite,
)
@@ -242,32 +278,32 @@ class FeedExporter:
category=ScrapyDeprecationWarning, stacklevel=2,
)
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
- feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
- self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+ feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
+ self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
- for uri, feed in self.settings.getdict('FEEDS').items():
+ for uri, feed_options in self.settings.getdict('FEEDS').items():
uri = str(uri) # handle pathlib.Path objects
- self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+ self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
- for uri, feed in self.feeds.items():
- if not self._storage_supported(uri):
+ for uri, feed_options in self.feeds.items():
+ if not self._storage_supported(uri, feed_options):
raise NotConfigured
if not self._settings_are_valid():
raise NotConfigured
- if not self._exporter_supported(feed['format']):
+ if not self._exporter_supported(feed_options['format']):
raise NotConfigured
def open_spider(self, spider):
- for uri, feed in self.feeds.items():
- uri_params = self._get_uri_params(spider, feed['uri_params'])
+ for uri, feed_options in self.feeds.items():
+ uri_params = self._get_uri_params(spider, feed_options['uri_params'])
self.slots.append(self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
- feed=feed,
+ feed_options=feed_options,
spider=spider,
uri_template=uri,
))
@@ -306,32 +342,32 @@ class FeedExporter:
)
return d
- def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
+ def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
- :param feed: dict with parameters of feed
+ :param feed_options: dict with parameters of feed
:param spider: user spider
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
"""
- storage = self._get_storage(uri)
+ storage = self._get_storage(uri, feed_options)
file = storage.open(spider)
exporter = self._get_exporter(
file=file,
- format=feed['format'],
- fields_to_export=feed['fields'],
- encoding=feed['encoding'],
- indent=feed['indent'],
+ format=feed_options['format'],
+ fields_to_export=feed_options['fields'],
+ encoding=feed_options['encoding'],
+ indent=feed_options['indent'],
)
slot = _FeedSlot(
file=file,
exporter=exporter,
storage=storage,
uri=uri,
- format=feed['format'],
- store_empty=feed['store_empty'],
+ format=feed_options['format'],
+ store_empty=feed_options['store_empty'],
batch_id=batch_id,
uri_template=uri_template,
)
@@ -355,7 +391,7 @@ class FeedExporter:
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
- feed=self.feeds[slot.uri_template],
+ feed_options=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,
))
@@ -394,11 +430,11 @@ class FeedExporter:
return False
return True
- def _storage_supported(self, uri):
+ def _storage_supported(self, uri, feed_options):
scheme = urlparse(uri).scheme
if scheme in self.storages:
try:
- self._get_storage(uri)
+ self._get_storage(uri, feed_options)
return True
except NotConfigured as e:
logger.error("Disabled feed storage scheme: %(scheme)s. "
@@ -416,8 +452,30 @@ class FeedExporter:
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
- def _get_storage(self, uri):
- return self._get_instance(self.storages[urlparse(uri).scheme], uri)
+ def _get_storage(self, uri, feed_options):
+ """Fork of create_instance specific to feed storage classes
+
+ It supports not passing the *feed_options* parameters to classes that
+ do not support it, and issuing a deprecation warning instead.
+ """
+ feedcls = self.storages[urlparse(uri).scheme]
+ crawler = getattr(self, 'crawler', None)
+
+ def build_instance(builder, *preargs):
+ return build_storage(builder, uri, preargs=preargs)
+
+ if crawler and hasattr(feedcls, 'from_crawler'):
+ instance = build_instance(feedcls.from_crawler, crawler)
+ method_name = 'from_crawler'
+ elif hasattr(feedcls, 'from_settings'):
+ instance = build_instance(feedcls.from_settings, self.settings)
+ method_name = 'from_settings'
+ else:
+ instance = build_instance(feedcls)
+ method_name = '__new__'
+ if instance is None:
+ raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
+ return instance
def _get_uri_params(self, spider, uri_params, slot=None):
params = {}
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index a83076c47..90a52b25b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings):
return out
-def feed_process_params_from_cli(settings, output, output_format=None):
+def feed_process_params_from_cli(settings, output, output_format=None,
+ overwrite_output=None):
"""
Receives feed export params (from the 'crawl' or 'runspider' commands),
checks for inconsistencies in their quantities and returns a dictionary
@@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None):
def check_valid_format(output_format):
if output_format not in valid_output_formats:
- raise UsageError("Unrecognized output format '%s', set one after a"
- " colon using the -o option (i.e. -o <URI>:<FORMAT>)"
- " or as a file extension, from the supported list %s" %
- (output_format, tuple(valid_output_formats)))
+ raise UsageError(
+ "Unrecognized output format '%s'. Set a supported one (%s) "
+ "after a colon at the end of the output URI (i.e. -o/-O "
+ "<URI>:<FORMAT>) or as a file extension." % (
+ output_format,
+ tuple(valid_output_formats),
+ )
+ )
+
+ overwrite = False
+ if overwrite_output:
+ if output:
+ raise UsageError(
+ "Please use only one of -o/--output and -O/--overwrite-output"
+ )
+ output = overwrite_output
+ overwrite = True
if output_format:
if len(output) == 1:
check_valid_format(output_format)
- warnings.warn('The -t command line option is deprecated in favor'
- ' of specifying the output format within the -o'
- ' option, please check the -o option docs for more details',
- category=ScrapyDeprecationWarning, stacklevel=2)
+ message = (
+ 'The -t command line option is deprecated in favor of '
+ 'specifying the output format within the output URI. See the '
+ 'documentation of the -o and -O options for more information.',
+ )
+ warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {'format': output_format}}
else:
- raise UsageError('The -t command line option cannot be used if multiple'
- ' output files are specified with the -o option')
+ raise UsageError(
+ 'The -t command-line option cannot be used if multiple output '
+ 'URIs are specified'
+ )
result = {}
for element in output:
@@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None):
feed_uri = 'stdout:'
check_valid_format(feed_format)
result[feed_uri] = {'format': feed_format}
+ if overwrite:
+ result[feed_uri]['overwrite'] = True
- # FEEDS setting should take precedence over the -o and -t CLI options
+ # FEEDS setting should take precedence over the matching CLI options
result.update(settings.getdict('FEEDS'))
return result
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index f07bdd748..19d56d6ec 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
def ftp_store_file(
*, path, file, host, port,
- username, password, use_active_mode=False):
+ username, password, use_active_mode=False, overwrite=True):
"""Opens a FTP connection with passed credentials,sets current directory
to the directory extracted from given path, then uploads the file to server
"""
@@ -32,4 +32,6 @@ def ftp_store_file(
file.seek(0)
dirname, filename = posixpath.split(path)
ftp_makedirs_cwd(ftp, dirname)
- ftp.storbinary('STOR %s' % filename, file)
+ command = 'STOR' if overwrite else 'APPE'
+ ftp.storbinary('%s %s' % (command, filename), file)
+ file.close()
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 59f1b8371..1f2333264 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -198,7 +198,8 @@ def _getargspec_py23(func):
def get_func_args(func, stripself=False):
"""Return the argument name list of a callable"""
if inspect.isfunction(func):
- func_args, _, _, _ = _getargspec_py23(func)
+ spec = inspect.getfullargspec(func)
+ func_args = spec.args + spec.kwonlyargs
elif inspect.isclass(func):
return get_func_args(func.__init__, True)
elif inspect.ismethod(func):
diff --git a/tests/ftpserver.py b/tests/ftpserver.py
new file mode 100644
index 000000000..6f0289e08
--- /dev/null
+++ b/tests/ftpserver.py
@@ -0,0 +1,24 @@
+from argparse import ArgumentParser
+
+from pyftpdlib.authorizers import DummyAuthorizer
+from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.servers import FTPServer
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument('-d', '--directory')
+ args = parser.parse_args()
+
+ authorizer = DummyAuthorizer()
+ full_permissions = 'elradfmwMT'
+ authorizer.add_anonymous(args.directory, perm=full_permissions)
+ handler = FTPHandler
+ handler.authorizer = authorizer
+ address = ('127.0.0.1', 2121)
+ server = FTPServer(address, handler)
+ server.serve_forever()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 1f40473ba..48d7b8d37 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -3,7 +3,10 @@ import json
import os
import random
import sys
+from pathlib import Path
+from shutil import rmtree
from subprocess import Popen, PIPE
+from tempfile import mkdtemp
from urllib.parse import urlencode
from OpenSSL import SSL
@@ -256,6 +259,29 @@ class MockDNSServer:
self.proc.communicate()
+class MockFTPServer:
+ """Creates an FTP server on port 2121 with a default passwordless user
+ (anonymous) and a temporary root path that you can read from the
+ :attr:`path` attribute."""
+
+ def __enter__(self):
+ self.path = Path(mkdtemp())
+ self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)],
+ stderr=PIPE, env=get_testenv())
+ for line in self.proc.stderr:
+ if b'starting FTP server' in line:
+ break
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ rmtree(str(self.path))
+ self.proc.kill()
+ self.proc.communicate()
+
+ def url(self, path):
+ return 'ftp://127.0.0.1:2121/' + path
+
+
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
factory = ssl.DefaultOpenSSLContextFactory(
os.path.join(os.path.dirname(__file__), keyfile),
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index b51177abb..fe1cbc997 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,6 +4,7 @@ dataclasses; python_version == '3.6'
mitmproxy; python_version >= '3.7'
mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'
mitmproxy < 4; python_version < '3.6'
+pyftpdlib
# https://github.com/pytest-dev/pytest-twisted/issues/93
pytest != 5.4, != 5.4.1
pytest-azurepipelines
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 109f006a8..f76f851e7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -74,6 +74,7 @@ class ProjectTest(unittest.TestCase):
def kill_proc():
p.kill()
+ p.communicate()
assert False, 'Command took too much time to complete'
timer = Timer(15, kill_proc)
@@ -569,6 +570,55 @@ class BadSpider(scrapy.Spider):
log = self.get_log(self.debug_log_spider, args=[])
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+ def test_output(self):
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
+ return []
+"""
+ args = ['-o', 'example.json']
+ log = self.get_log(spider_code, args=args)
+ self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
+
+ def test_overwrite_output(self):
+ spider_code = """
+import json
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ self.logger.debug(
+ 'FEEDS: {}'.format(
+ json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
+ )
+ )
+ return []
+"""
+ args = ['-O', 'example.json']
+ log = self.get_log(spider_code, args=args)
+ self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+
+ def test_output_and_overwrite_output(self):
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ return []
+"""
+ args = ['-o', 'example1.json', '-O', 'example2.json']
+ log = self.get_log(spider_code, args=args)
+ self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+
class BenchCommandTest(CommandTest):
@@ -577,3 +627,79 @@ class BenchCommandTest(CommandTest):
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
self.assertIn('INFO: Crawled', log)
self.assertNotIn('Unhandled Error', log)
+
+
+class CrawlCommandTest(CommandTest):
+
+ def crawl(self, code, args=()):
+ fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
+ with open(fname, 'w') as f:
+ f.write(code)
+ return self.proc('crawl', 'myspider', *args)
+
+ def get_log(self, code, args=()):
+ _, _, stderr = self.crawl(code, args=args)
+ return stderr
+
+ def test_no_output(self):
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ self.logger.debug('It works!')
+ return []
+"""
+ log = self.get_log(spider_code)
+ self.assertIn("[myspider] DEBUG: It works!", log)
+
+ def test_output(self):
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
+ return []
+"""
+ args = ['-o', 'example.json']
+ log = self.get_log(spider_code, args=args)
+ self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
+
+ def test_overwrite_output(self):
+ spider_code = """
+import json
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ self.logger.debug(
+ 'FEEDS: {}'.format(
+ json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
+ )
+ )
+ return []
+"""
+ args = ['-O', 'example.json']
+ log = self.get_log(spider_code, args=args)
+ self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+
+ def test_output_and_overwrite_output(self):
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ def start_requests(self):
+ return []
+"""
+ args = ['-o', 'example1.json', '-O', 'example2.json']
+ log = self.get_log(spider_code, args=args)
+ self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2afc25a7a..850485b5e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,6 +5,7 @@ import random
import shutil
import string
import tempfile
+import warnings
from abc import ABC, abstractmethod
from collections import defaultdict
from io import BytesIO
@@ -25,7 +26,7 @@ from zope.interface.verify import verifyObject
import scrapy
from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter
from scrapy.extensions.feedexport import (
BlockingFeedStorage,
@@ -46,7 +47,7 @@ from scrapy.utils.test import (
mock_google_cloud_storage,
)
-from tests.mockserver import MockServer
+from tests.mockserver import MockFTPServer, MockServer
class FileFeedStorageTest(unittest.TestCase):
@@ -75,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase):
st = FileFeedStorage(path)
verifyObject(IFeedStorage, st)
+ def _store(self, feed_options=None):
+ path = os.path.abspath(self.mktemp())
+ storage = FileFeedStorage(path, feed_options=feed_options)
+ spider = scrapy.Spider("default")
+ file = storage.open(spider)
+ file.write(b"content")
+ storage.store(file)
+ return path
+
+ def test_append(self):
+ path = self._store()
+ return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")
+
+ def test_overwrite(self):
+ path = self._store({"overwrite": True})
+ return self._assert_stores(
+ FileFeedStorage(path, feed_options={"overwrite": True}),
+ path
+ )
+
@defer.inlineCallbacks
- def _assert_stores(self, storage, path):
+ def _assert_stores(self, storage, path, expected_content=b"content"):
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
@@ -84,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase):
self.assertTrue(os.path.exists(path))
try:
with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), b"content")
+ self.assertEqual(fp.read(), expected_content)
finally:
os.unlink(path)
@@ -99,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase):
spider = TestSpider.from_crawler(crawler)
return spider
- def test_store(self):
- uri = os.environ.get('FEEDTEST_FTP_URI')
- path = os.environ.get('FEEDTEST_FTP_PATH')
- if not (uri and path):
- raise unittest.SkipTest("No FTP server available for testing")
- st = FTPFeedStorage(uri)
- verifyObject(IFeedStorage, st)
- return self._assert_stores(st, path)
-
- def test_store_active_mode(self):
- uri = os.environ.get('FEEDTEST_FTP_URI')
- path = os.environ.get('FEEDTEST_FTP_PATH')
- if not (uri and path):
- raise unittest.SkipTest("No FTP server available for testing")
- use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True}
- crawler = get_crawler(settings_dict=use_active_mode)
- st = FTPFeedStorage.from_crawler(crawler, uri)
- verifyObject(IFeedStorage, st)
- return self._assert_stores(st, path)
+ def _store(self, uri, content, feed_options=None, settings=None):
+ crawler = get_crawler(settings_dict=settings or {})
+ storage = FTPFeedStorage.from_crawler(
+ crawler,
+ uri,
+ feed_options=feed_options,
+ )
+ verifyObject(IFeedStorage, storage)
+ spider = self.get_test_spider()
+ file = storage.open(spider)
+ file.write(content)
+ return storage.store(file)
+
+ def _assert_stored(self, path, content):
+ self.assertTrue(path.exists())
+ try:
+ with path.open('rb') as fp:
+ self.assertEqual(fp.read(), content)
+ finally:
+ os.unlink(str(path))
+
+ @defer.inlineCallbacks
+ def test_append(self):
+ with MockFTPServer() as ftp_server:
+ filename = 'file'
+ url = ftp_server.url(filename)
+ feed_options = {'overwrite': False}
+ yield self._store(url, b"foo", feed_options=feed_options)
+ yield self._store(url, b"bar", feed_options=feed_options)
+ self._assert_stored(ftp_server.path / filename, b"foobar")
+
+ @defer.inlineCallbacks
+ def test_overwrite(self):
+ with MockFTPServer() as ftp_server:
+ filename = 'file'
+ url = ftp_server.url(filename)
+ yield self._store(url, b"foo")
+ yield self._store(url, b"bar")
+ self._assert_stored(ftp_server.path / filename, b"bar")
+
+ @defer.inlineCallbacks
+ def test_append_active_mode(self):
+ with MockFTPServer() as ftp_server:
+ settings = {'FEED_STORAGE_FTP_ACTIVE': True}
+ filename = 'file'
+ url = ftp_server.url(filename)
+ feed_options = {'overwrite': False}
+ yield self._store(url, b"foo", feed_options=feed_options, settings=settings)
+ yield self._store(url, b"bar", feed_options=feed_options, settings=settings)
+ self._assert_stored(ftp_server.path / filename, b"foobar")
+
+ @defer.inlineCallbacks
+ def test_overwrite_active_mode(self):
+ with MockFTPServer() as ftp_server:
+ settings = {'FEED_STORAGE_FTP_ACTIVE': True}
+ filename = 'file'
+ url = ftp_server.url(filename)
+ yield self._store(url, b"foo", settings=settings)
+ yield self._store(url, b"bar", settings=settings)
+ self._assert_stored(ftp_server.path / filename, b"bar")
def test_uri_auth_quote(self):
# RFC3986: 3.2.1. User Information
pw_quoted = quote(string.punctuation, safe='')
- st = FTPFeedStorage('ftp://foo:%[email protected]/some_path' % pw_quoted)
+ st = FTPFeedStorage('ftp://foo:%[email protected]/some_path' % pw_quoted,
+ {})
self.assertEqual(st.password, string.punctuation)
- @defer.inlineCallbacks
- def _assert_stores(self, storage, path):
- spider = self.get_test_spider()
- file = storage.open(spider)
- file.write(b"content")
- yield storage.store(file)
- self.assertTrue(os.path.exists(path))
- try:
- with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), b"content")
- # again, to check s3 objects are overwritten
- yield storage.store(BytesIO(b"new content"))
- with open(path, 'rb') as fp:
- self.assertEqual(fp.read(), b"new content")
- finally:
- os.unlink(path)
-
class BlockingFeedStorageTest(unittest.TestCase):
@@ -190,8 +236,10 @@ class S3FeedStorageTest(unittest.TestCase):
'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
crawler = get_crawler(settings_dict=aws_credentials)
# Instantiate with crawler
- storage = S3FeedStorage.from_crawler(crawler,
- 's3://mybucket/export.csv')
+ storage = S3FeedStorage.from_crawler(
+ crawler,
+ 's3://mybucket/export.csv',
+ )
self.assertEqual(storage.access_key, 'settings_key')
self.assertEqual(storage.secret_key, 'settings_secret')
# Instantiate directly
@@ -254,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase):
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
crawler,
- 's3://mybucket/export.csv'
+ 's3://mybucket/export.csv',
)
self.assertEqual(storage.access_key, 'access_key')
self.assertEqual(storage.secret_key, 'secret_key')
@@ -269,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase):
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
crawler,
- 's3://mybucket/export.csv'
+ 's3://mybucket/export.csv',
)
self.assertEqual(storage.access_key, 'access_key')
self.assertEqual(storage.secret_key, 'secret_key')
@@ -370,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase):
key.set_contents_from_file.call_args
)
+ def test_overwrite_default(self):
+ with LogCapture() as log:
+ S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ 'custom-acl'
+ )
+ self.assertNotIn('S3 does not support appending to files', str(log))
+
+ def test_overwrite_false(self):
+ with LogCapture() as log:
+ S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ 'custom-acl',
+ feed_options={'overwrite': False},
+ )
+ self.assertIn('S3 does not support appending to files', str(log))
+
class GCSFeedStorageTest(unittest.TestCase):
@@ -439,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase):
yield storage.store(file)
self.assertEqual(out.getvalue(), b"content")
+ def test_overwrite_default(self):
+ with LogCapture() as log:
+ StdoutFeedStorage('stdout:')
+ self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log))
+
+ def test_overwrite_true(self):
+ with LogCapture() as log:
+ StdoutFeedStorage('stdout:', feed_options={'overwrite': True})
+ self.assertIn('Standard output (stdout) storage does not support overwriting', str(log))
+
class FromCrawlerMixin:
init_with_crawler = False
@classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
+ def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
cls.init_with_crawler = True
return cls(*args, **kwargs)
@@ -454,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin):
class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
- pass
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
+ cls.init_with_crawler = True
+ return cls(*args, feed_options=feed_options, **kwargs)
class DummyBlockingFeedStorage(BlockingFeedStorage):
@@ -588,8 +671,8 @@ class FeedExportTest(FeedExportTestBase):
FEEDS = settings.get('FEEDS') or {}
settings['FEEDS'] = {
- printf_escape(path_to_url(file_path)): feed
- for file_path, feed in FEEDS.items()
+ printf_escape(path_to_url(file_path)): feed_options
+ for file_path, feed_options in FEEDS.items()
}
content = {}
@@ -599,12 +682,12 @@ class FeedExportTest(FeedExportTestBase):
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
- for file_path, feed in FEEDS.items():
+ for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
continue
with open(str(file_path), 'rb') as f:
- content[feed['format']] = f.read()
+ content[feed_options['format']] = f.read()
finally:
for file_path in FEEDS.keys():
@@ -1542,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase):
content = json.loads(content.decode('utf-8'))
expected_batch, items = items[:batch_size], items[batch_size:]
self.assertEqual(expected_batch, content)
+
+
+class FeedExportInitTest(unittest.TestCase):
+
+ def test_unsupported_storage(self):
+ settings = {
+ 'FEEDS': {
+ 'unsupported://uri': {},
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ with self.assertRaises(NotConfigured):
+ FeedExporter.from_crawler(crawler)
+
+ def test_unsupported_format(self):
+ settings = {
+ 'FEEDS': {
+ 'file://path': {
+ 'format': 'unsupported_format',
+ },
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ with self.assertRaises(NotConfigured):
+ FeedExporter.from_crawler(crawler)
+
+
+class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
+
+ def __init__(self, uri):
+ super().__init__(uri)
+
+
+class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
+ """Make sure that any feed exporter created by users before the
+ introduction of the ``feed_options`` parameter continues to work as
+ expected, and simply issues a warning."""
+
+ def test_init(self):
+ settings_dict = {
+ 'FEED_URI': 'file:///tmp/foobar',
+ 'FEED_STORAGES': {
+ 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions'
+ },
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ spider = scrapy.Spider("default")
+ with warnings.catch_warnings(record=True) as w:
+ feed_exporter.open_spider(spider)
+ messages = tuple(str(item.message) for item in w
+ if item.category is ScrapyDeprecationWarning)
+ self.assertEqual(
+ messages,
+ (
+ (
+ "StdoutFeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove "
+ "this warning. This parameter will become mandatory "
+ "in a future version of Scrapy."
+ ),
+ )
+ )
+
+
+class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
+
+ def __init__(self, uri):
+ super().__init__(uri)
+
+
+class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
+ """Make sure that any feed exporter created by users before the
+ introduction of the ``feed_options`` parameter continues to work as
+ expected, and simply issues a warning."""
+
+ maxDiff = None
+
+ def test_init(self):
+ settings_dict = {
+ 'FEED_URI': 'file:///tmp/foobar',
+ 'FEED_STORAGES': {
+ 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions'
+ },
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ spider = scrapy.Spider("default")
+ with warnings.catch_warnings(record=True) as w:
+ feed_exporter.open_spider(spider)
+ messages = tuple(str(item.message) for item in w
+ if item.category is ScrapyDeprecationWarning)
+ self.assertEqual(
+ messages,
+ (
+ (
+ "FileFeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove "
+ "this warning. This parameter will become mandatory "
+ "in a future version of Scrapy."
+ ),
+ )
+ )
+
+
+class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
+
+ def __init__(self, uri, access_key, secret_key, acl):
+ super().__init__(uri, access_key, secret_key, acl)
+
+
+class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
+
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return super().from_crawler(crawler, uri)
+
+
+class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
+ """Make sure that any feed exporter created by users before the
+ introduction of the ``feed_options`` parameter continues to work as
+ expected, and simply issues a warning."""
+
+ maxDiff = None
+
+ def test_init(self):
+ settings_dict = {
+ 'FEED_URI': 'file:///tmp/foobar',
+ 'FEED_STORAGES': {
+ 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions'
+ },
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ spider = scrapy.Spider("default")
+ spider.crawler = crawler
+ with warnings.catch_warnings(record=True) as w:
+ feed_exporter.open_spider(spider)
+ messages = tuple(str(item.message) for item in w
+ if item.category is ScrapyDeprecationWarning)
+ self.assertEqual(
+ messages,
+ (
+ (
+ "S3FeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove "
+ "this warning. This parameter will become mandatory "
+ "in a future version of Scrapy."
+ ),
+ )
+ )
+
+ def test_from_crawler(self):
+ settings_dict = {
+ 'FEED_URI': 'file:///tmp/foobar',
+ 'FEED_STORAGES': {
+ 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler'
+ },
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ spider = scrapy.Spider("default")
+ spider.crawler = crawler
+ with warnings.catch_warnings(record=True) as w:
+ feed_exporter.open_spider(spider)
+ messages = tuple(str(item.message) for item in w
+ if item.category is ScrapyDeprecationWarning)
+ self.assertEqual(
+ messages,
+ (
+ (
+ "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
+ "does not support the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove "
+ "this warning. This parameter will become mandatory "
+ "in a future version of Scrapy."
+ ),
+ )
+ )
+
+
+class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
+
+ def __init__(self, uri, use_active_mode=False):
+ super().__init__(uri)
+
+
+class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
+
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return super().from_crawler(crawler, uri)
+
+
+class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
+ """Make sure that any feed exporter created by users before the
+ introduction of the ``feed_options`` parameter continues to work as
+ expected, and simply issues a warning."""
+
+ maxDiff = None
+
+ def test_init(self):
+ settings_dict = {
+ 'FEED_URI': 'file:///tmp/foobar',
+ 'FEED_STORAGES': {
+ 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions'
+ },
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ spider = scrapy.Spider("default")
+ spider.crawler = crawler
+ with warnings.catch_warnings(record=True) as w:
+ feed_exporter.open_spider(spider)
+ messages = tuple(str(item.message) for item in w
+ if item.category is ScrapyDeprecationWarning)
+ self.assertEqual(
+ messages,
+ (
+ (
+ "FTPFeedStorageWithoutFeedOptions does not support "
+ "the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove "
+ "this warning. This parameter will become mandatory "
+ "in a future version of Scrapy."
+ ),
+ )
+ )
+
+ def test_from_crawler(self):
+ settings_dict = {
+ 'FEED_URI': 'file:///tmp/foobar',
+ 'FEED_STORAGES': {
+ 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler'
+ },
+ }
+ crawler = get_crawler(settings_dict=settings_dict)
+ feed_exporter = FeedExporter.from_crawler(crawler)
+ spider = scrapy.Spider("default")
+ spider.crawler = crawler
+ with warnings.catch_warnings(record=True) as w:
+ feed_exporter.open_spider(spider)
+ messages = tuple(str(item.message) for item in w
+ if item.category is ScrapyDeprecationWarning)
+ self.assertEqual(
+ messages,
+ (
+ (
+ "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
+ "does not support the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove "
+ "this warning. This parameter will become mandatory "
+ "in a future version of Scrapy."
+ ),
+ )
+ )
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index f3ef36127..ccc65c4fd 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase):
feed_process_params_from_cli(settings, ['-:pickle'])
)
+ def test_feed_export_config_overwrite(self):
+ settings = Settings()
+ self.assertEqual(
+ {'output.json': {'format': 'json', 'overwrite': True}},
+ feed_process_params_from_cli(settings, [], None, ['output.json'])
+ )
+
+ def test_output_and_overwrite_output(self):
+ with self.assertRaises(UsageError):
+ feed_process_params_from_cli(
+ Settings(),
+ ['output1.json'],
+ None,
+ ['output2.json'],
+ )
+
def test_feed_complete_default_values_from_settings_empty(self):
feed = {}
settings = Settings({
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3f93f509e..c298d0bd2 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase):
def f2(a, b=None, c=None):
pass
+ def f3(a, b=None, *, c=None):
+ pass
+
class A:
def __init__(self, a, b, c):
pass
@@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase):
self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
+ self.assertEqual(get_func_args(f3), ['a', 'b', 'c'])
self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
self.assertEqual(get_func_args(partial_f1), ['b', 'c'])