Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support feed modes (#7826)

Adrian198e19995cdc
diff --git a/docs/conf.py b/docs/conf.py
index ce6096867..fd0fc0f98 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -175,16 +175,5 @@ scrapy_intersphinx_enable = [
"w3lib",
]
-# sphinx-llm-friendly ---------------------------------------------------------
-
-llm_friendly_llms_full_txt_exclude = [
- # Changelog, not useful for an LLM answering "how do I use Scrapy"
- # questions, and the largest single contributor to llms-full.txt size.
- "news.rst",
- "news/*",
- "contributing.rst",
-]
-
-
# -- Other options ------------------------------------------------------------
default_dark_mode = False
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 37b40591b..c2d5b3ef0 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
-# uv pip compile requirements.in -o requirements.txt --refresh
+# uv pip compile requirements.in -o requirements.txt
alabaster==1.0.0
# via sphinx
annotated-types==0.7.0
@@ -110,14 +110,11 @@ pyopenssl==26.0.0
# via scrapy
queuelib==1.9.0
# via scrapy
-regex==2026.9.10
- # via tiktoken
requests==2.33.0
# via
# requests-file
# sphinx
# sphinxcontrib-youtube
- # tiktoken
# tldextract
requests-file==3.0.1
# via tldextract
@@ -150,7 +147,7 @@ sphinx-design==0.7.0
# via sphinx-scrapy
sphinx-last-updated-by-git==0.3.8
# via sphinx-sitemap
-sphinx-llm-friendly==0.6.0
+sphinx-llm-friendly==0.7.0
# via sphinx-scrapy
sphinx-notfound-page==1.1.0
# via -r requirements.in
@@ -184,8 +181,6 @@ sphinxcontrib-youtube==1.5.0
# via sphinx-scrapy
tabulate==0.10.0
# via sphinx-llm-friendly
-tiktoken==0.14.0
- # via sphinx-llm-friendly
tldextract==5.3.1
# via scrapy
twisted==25.5.0
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index d533fd367..b9b7c2a31 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -330,9 +330,10 @@ Supported options:
* ``-a NAME=VALUE``: set a spider argument (may be repeated)
-* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE
- (use ``-`` for stdout). To define the output format, add a colon at the end
- of the output URI (for example, ``-o FILE:FORMAT``)
+* ``--output FILE`` or ``-o FILE``: dump scraped items into FILE (use ``-`` for
+ stdout), handling an existing FILE as the :setting:`FEED_MODE` setting
+ indicates. To define the output format, add a colon at the end of the output
+ URI (for example, ``-o FILE:FORMAT``)
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE,
overwriting any existing file. To define the output format, add a colon at
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 89b231bad..a2ccbe7d7 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -168,6 +168,9 @@ Note that for the local filesystem storage you can omit the scheme if you
specify a path (e.g. ``/tmp/export.csv``). Alternatively, you can use a
:class:`pathlib.Path` object.
+Supported :ref:`modes <feed-mode>`: ``"append"``, ``"create"`` and
+``"overwrite"``. If :setting:`FEED_MODE` is ``None``, ``"append"`` is used.
+
.. _topics-feed-storage-ftp:
.. _feed-storage-ftp:
@@ -188,11 +191,16 @@ FTP supports two different connection modes: `active or passive
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
-The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
-storage backend is: ``True``.
+Supported :ref:`modes <feed-mode>`: ``"append"``, ``"create"`` and
+``"overwrite"``. If :setting:`FEED_MODE` is ``None``, ``"overwrite"`` is used.
+
+.. note:: Some FTP servers may not support appending to files (the ``APPE`` FTP
+ command).
-.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
- previous version of your data.
+.. note:: FTP has no atomic create-if-absent command, so ``"create"`` is
+ best-effort: the target is checked for existence right before it is
+ written. Servers that do not support the ``SIZE`` command are
+ reported as not having the target.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@@ -211,8 +219,8 @@ certificate of the server verified.
- Example URI: ``ftps://user:[email protected]/path/to/export.csv``
- Required external libraries: none
-See :ref:`feed-storage-ftp` for connection modes, the ``overwrite`` default and
-file delivery.
+See :ref:`feed-storage-ftp` for connection modes, supported :ref:`modes
+<feed-mode>` and file delivery.
.. note:: For SFTP, an unrelated protocol built on SSH, use
`scrapy-feedexporter-sftp
@@ -253,11 +261,11 @@ pool size for exported feeds using these settings:
- :setting:`AWS_REGION_NAME`
- :setting:`AWS_MAX_POOL_CONNECTIONS`
-The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
-storage backend is: ``True``.
+Supported :ref:`modes <feed-mode>`: ``"create"`` and ``"overwrite"``. If
+:setting:`FEED_MODE` is ``None``, ``"overwrite"`` is used.
-.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
- previous version of your data.
+``"create"`` mode `does not support files larger than 5 GB
+<https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html>`__.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@@ -284,14 +292,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
-The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
-storage backend is: ``True``.
+Supported :ref:`modes <feed-mode>`: ``"append"``, ``"create"`` and
+``"overwrite"``. If :setting:`FEED_MODE` is ``None``, ``"overwrite"`` is used.
-.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
- previous version of your data.
-
-Appending (``overwrite: False``) turns the feed into a `composite object`_,
-which has a CRC32C checksum but no MD5 hash.
+Appending turns the feed into a `composite object`_, which has a CRC32C
+checksum but no MD5 hash.
.. versionadded:: VERSION
Appending support.
@@ -311,6 +316,8 @@ The feeds are written to the standard output of the Scrapy process.
- Example URI: ``stdout:``
- Required external libraries: none
+The :ref:`mode <feed-mode>` is ignored by this storage backend.
+
.. _delayed-file-delivery:
@@ -344,6 +351,25 @@ URI scheme through the :setting:`FEED_STORAGES` setting.
.. autoclass:: scrapy.extensions.feedexport.FeedStorageProtocol(uri, *, feed_options=None)
:members:
+ .. attribute:: supported_modes
+ :type: frozenset[str]
+
+ Optional set of supported values of the ``mode`` :ref:`feed option
+ <feed-options>`, e.g. ``frozenset({"create", "overwrite"})``.
+
+ .. versionadded:: VERSION
+
+ Setting ``mode`` to an unsupported value raises an exception, which
+ prevents the crawl from running. A storage class that does not define
+ this attribute is assumed to predate the ``mode`` feed option, and
+ setting that option on a feed that uses it logs a warning about the
+ option being possibly ignored.
+
+ A storage class that declares support for ``"create"`` must enforce it
+ when writing the feed, ideally atomically, raising
+ :exc:`FileExistsError` if the target exists. Scrapy then handles that
+ file as described in :setting:`FEED_MODE`.
+
If your storage backend blocks, subclass
:class:`~scrapy.extensions.feedexport.BlockingFeedStorage` instead: it writes
items into a temporary local file (see :ref:`delayed file delivery
@@ -493,6 +519,7 @@ These are the settings used for configuring the feed exports:
- :setting:`FEEDS` (mandatory)
- :setting:`FEED_EXPORT_ENCODING`
+- :setting:`FEED_MODE`
- :setting:`FEED_STORE_EMPTY`
- :setting:`FEED_EXPORT_FIELDS`
- :setting:`FEED_EXPORT_INDENT`
@@ -589,24 +616,9 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
-- ``overwrite``: whether to overwrite the file if it already exists
- (``True``) or append to its content (``False``).
-
- The default value depends on the :ref:`storage backend
- <topics-feed-storage-backends>`:
-
- - :ref:`topics-feed-storage-fs`: ``False``
-
- - :ref:`feed-storage-ftp` and :ref:`feed-storage-ftps`: ``True``
+- ``mode``: falls back to :setting:`FEED_MODE`.
- .. note:: Some FTP servers may not support appending to files (the
- ``APPE`` FTP command).
-
- - :ref:`topics-feed-storage-s3`: ``True`` (appending is not supported)
-
- - :ref:`topics-feed-storage-gcs`: ``True``
-
- - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
+ .. versionadded:: VERSION
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
@@ -657,6 +669,44 @@ Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter`
and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exporting
to ``.json`` or ``.xml``.
+.. _feed-mode:
+.. setting:: FEED_MODE
+
+FEED_MODE
+---------
+
+.. versionadded:: VERSION
+
+Default: ``"create"`` (:ref:`fallback <default-settings>`: ``None``)
+
+What to do when the target of a feed already exists.
+
+Supported values, which match the corresponding :func:`open` modes:
+
+- ``"append"`` (``a``): append to the existing target.
+
+- ``"create"`` (``x``): do not write, keeping the existing target untouched.
+
+- ``"overwrite"`` (``w``): replace the existing target.
+
+Can be overridden with the ``mode`` :ref:`feed option <feed-options>`.
+
+Not every :ref:`storage backend <topics-feed-storage-backends>` supports every
+mode; see the documentation of your storage backend.
+
+If ``None``, the mode depends on the backend being used, e.g. appending for the
+:ref:`local filesystem <topics-feed-storage-fs>` and overwriting for :ref:`S3
+<topics-feed-storage-s3>`.
+
+When the mode of a feed is ``"create"`` and its target already exists, Scrapy
+logs an error, increases the ``feedexport/conflicts/<storage class name>``
+:ref:`stat <topics-stats>`, and does not write that file, i.e. the items that it
+would have contained are lost.
+
+.. note:: When resuming a crawl (see :setting:`JOBDIR`), the feed of the
+ previous run already exists, so a feed that must survive a resumed crawl
+ needs the ``"append"`` mode.
+
.. setting:: FEED_STORE_EMPTY
FEED_STORE_EMPTY
@@ -666,8 +716,8 @@ Default: ``True``
Whether to export empty feeds (i.e. feeds with no items).
If ``False``, and there are no items to export, no new files are created and
-existing files are not modified, even if the :ref:`overwrite feed option
-<feed-options>` is enabled.
+existing files are not modified, even if the ``mode`` :ref:`feed option
+<feed-options>` is ``"overwrite"``.
.. setting:: FEED_STORAGES
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 0496a987a..a62e9b879 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -114,6 +114,15 @@ version that wrote them. A job must be resumed with the same Scrapy version
that paused it; after upgrading or downgrading Scrapy, start a new job with a
new job directory.
+Feed exports
+------------
+
+When a job is resumed, the :ref:`feeds <topics-feed-exports>` of the previous
+run already exist, so their :ref:`mode <feed-mode>` determines what happens to
+the items of the resumed run: ``"append"`` adds them to those feeds,
+``"overwrite"`` replaces the items of the previous run with them, and
+``"create"`` does not write them anywhere.
+
Cookie expiration
-----------------
diff --git a/pyproject.toml b/pyproject.toml
index 952c741c2..0d7541514 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,7 +79,7 @@ ipython = ["ipython>=8.15.0"]
memusage = ["psutil>=5.9.0"]
ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
-s3 = ["boto3>=1.20.0"]
+s3 = ["boto3>=1.35.2"]
twisted-http2 = ["Twisted[http2]>=21.7.0"]
uvloop = [
"uvloop>=0.17.0; platform_system != 'Windows' and implementation_name != 'pypy'",
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c3ed0ed08..a07e35f76 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -86,6 +86,38 @@ UriParamsCallableT: TypeAlias = Callable[
_ItemProcessor: TypeAlias = Callable[[Any], Iterable[Any]]
+_FEED_MODES: frozenset[str] = frozenset({"append", "create", "overwrite"})
+
+
+def _get_mode(storage: Any, feed_options: dict[str, Any] | None, legacy: str) -> str:
+ """Return the mode of *storage*, based on the *mode* feed option, or on the
+ deprecated *overwrite* feed option, or, if neither is set, on *legacy*,
+ which must be the mode that the storage used before the *mode* feed option
+ existed."""
+ feed_options = feed_options or {}
+ mode = feed_options.get("mode")
+ if mode is None:
+ overwrite = feed_options.get("overwrite")
+ mode = legacy if overwrite is None else "overwrite" if overwrite else "append"
+ _check_mode(mode, storage)
+ return mode
+
+
+def _check_mode(mode: str, storage: Any, uri: str | None = None) -> None:
+ if mode not in _FEED_MODES:
+ raise ValueError(
+ f"Invalid feed mode: {mode!r}. Supported modes: "
+ f"{', '.join(sorted(_FEED_MODES))}."
+ )
+ supported: frozenset[str] | None = getattr(storage, "supported_modes", None)
+ if supported is not None and mode not in supported:
+ suffix = f" (feed URI: {uri})" if uri else ""
+ raise ValueError(
+ f"{type(storage).__name__} does not support the {mode!r} feed "
+ f"mode{suffix}. Supported modes: {', '.join(sorted(supported))}."
+ )
+
+
class ItemFilter:
"""
This will be used by FeedExporter to decide if an item should be allowed
@@ -188,6 +220,9 @@ class BlockingFeedStorage(ABC):
class StdoutFeedStorage:
""":ref:`Standard output <topics-feed-storage-stdout>` storage backend."""
+ # The mode is irrelevant here: writing to a stream cannot destroy data.
+ supported_modes: frozenset[str] = _FEED_MODES
+
def __init__(
self,
uri: str,
@@ -198,13 +233,6 @@ class StdoutFeedStorage:
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout: IO[bytes] = _stdout
- if feed_options and feed_options.get("overwrite", False) is True:
- logger.warning(
- "Standard output (stdout) storage does not support "
- "overwriting. To suppress this warning, remove the "
- "overwrite option from your FEEDS setting, or set "
- "it to False."
- )
def open(self, spider: Spider) -> IO[bytes]:
return self._stdout
@@ -213,6 +241,13 @@ class StdoutFeedStorage:
pass
+_WRITE_MODES: dict[str, OpenBinaryMode] = {
+ "append": "ab",
+ "create": "xb",
+ "overwrite": "wb",
+}
+
+
class FileFeedStorage:
""":ref:`Local filesystem <topics-feed-storage-fs>` storage backend.
@@ -220,12 +255,13 @@ class FileFeedStorage:
are created when the feed is opened.
"""
+ supported_modes: frozenset[str] = _FEED_MODES
+
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
- feed_options = feed_options or {}
- self.write_mode: OpenBinaryMode = (
- "wb" if feed_options.get("overwrite", False) else "ab"
- )
+ self.write_mode: OpenBinaryMode = _WRITE_MODES[
+ _get_mode(self, feed_options, "append")
+ ]
def open(self, spider: Spider) -> IO[bytes]:
dirname = Path(self.path).parent
@@ -241,6 +277,8 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
""":ref:`Amazon S3 <topics-feed-storage-s3>` storage backend."""
+ supported_modes: frozenset[str] = frozenset({"create", "overwrite"})
+
def __init__(
self,
uri: str,
@@ -291,12 +329,7 @@ class S3FeedStorage(BlockingFeedStorage):
config=Config(**config_kwargs),
)
- if feed_options and feed_options.get("overwrite", True) is False:
- logger.warning(
- "S3 does not support appending to files. To "
- "suppress this warning, remove the overwrite "
- "option from your FEEDS setting or set it to True."
- )
+ self._mode: str = _get_mode(self, feed_options, "overwrite")
@classmethod
def from_crawler(
@@ -320,14 +353,37 @@ class S3FeedStorage(BlockingFeedStorage):
)
def _store_in_thread(self, file: IO[bytes]) -> None:
+ from botocore.exceptions import ClientError # noqa: PLC0415
+
file.seek(0)
+ extra_args: dict[str, Any] = {"ACL": self.acl} if self.acl else {}
try:
- if self.acl:
+ if self._mode == "create":
+ # upload_fileobj() does not allow IfNoneMatch
+ # (https://github.com/boto/boto3/issues/4366).
+ try:
+ self.s3_client.put_object(
+ Bucket=self.bucketname,
+ Key=self.keyname,
+ Body=file,
+ IfNoneMatch="*",
+ **extra_args,
+ )
+ except ClientError as error:
+ if (
+ error.response.get("Error", {}).get("Code")
+ == "PreconditionFailed"
+ ):
+ raise FileExistsError(
+ f"s3://{self.bucketname}/{self.keyname} already exists"
+ ) from error
+ raise
+ elif extra_args:
self.s3_client.upload_fileobj(
Bucket=self.bucketname,
Key=self.keyname,
Fileobj=file,
- ExtraArgs={"ACL": self.acl},
+ ExtraArgs=extra_args,
)
else:
self.s3_client.upload_fileobj(
@@ -342,6 +398,8 @@ class S3FeedStorage(BlockingFeedStorage):
class GCSFeedStorage(BlockingFeedStorage):
""":ref:`GCS <topics-feed-storage-gcs>` storage backend."""
+ supported_modes: frozenset[str] = _FEED_MODES
+
def __init__(
self,
uri: str,
@@ -358,7 +416,8 @@ class GCSFeedStorage(BlockingFeedStorage):
assert u.hostname
self.bucket_name: str = u.hostname
self.blob_name: str = u.path[1:] # remove first "/"
- self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
+
+ self._mode: str = _get_mode(self, feed_options, "overwrite")
@classmethod
def from_crawler(
@@ -377,18 +436,35 @@ class GCSFeedStorage(BlockingFeedStorage):
)
def _store_in_thread(self, file: IO[bytes]) -> None:
+ from google.api_core.exceptions import PreconditionFailed # noqa: PLC0415
+ from google.cloud.storage import Client # noqa: PLC0415
+
file.seek(0)
try:
- from google.cloud.storage import Client # noqa: PLC0415
-
client = Client(project=self.project_id)
bucket = client.bucket(self.bucket_name)
- kwargs = {} if self.timeout is None else {"timeout": self.timeout}
- blob = None if self.overwrite else bucket.get_blob(self.blob_name, **kwargs)
+ kwargs: dict[str, Any] = (
+ {} if self.timeout is None else {"timeout": self.timeout}
+ )
+ blob = (
+ bucket.get_blob(self.blob_name, **kwargs)
+ if self._mode == "append"
+ else None
+ )
if blob is None:
- bucket.blob(self.blob_name).upload_from_file(
- file, predefined_acl=self.acl, **kwargs
+ upload_kwargs = (
+ {**kwargs, "if_generation_match": 0}
+ if self._mode == "create"
+ else kwargs
)
+ try:
+ bucket.blob(self.blob_name).upload_from_file(
+ file, predefined_acl=self.acl, **upload_kwargs
+ )
+ except PreconditionFailed as error:
+ raise FileExistsError(
+ f"gs://{self.bucket_name}/{self.blob_name} already exists"
+ ) from error
return
# Appending uploads the new data as a separate object and composes
# it with the existing one, leaving the data already stored
@@ -410,6 +486,8 @@ class FTPFeedStorage(BlockingFeedStorage):
""":ref:`FTP <feed-storage-ftp>` storage backend, which also handles
:ref:`FTPS <feed-storage-ftps>` when *uri* uses the ``ftps`` scheme."""
+ supported_modes: frozenset[str] = _FEED_MODES
+
def __init__(
self,
uri: str,
@@ -429,7 +507,16 @@ class FTPFeedStorage(BlockingFeedStorage):
self.tls: bool = u.scheme == "ftps"
self.use_active_mode: bool = use_active_mode
self.timeout: float | None = timeout
- self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
+ self._mode: str = _get_mode(self, feed_options, "overwrite")
+
+ @property
+ def overwrite(self) -> bool:
+ warnings.warn(
+ "FTPFeedStorage.overwrite is deprecated, use the mode feed option instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return self._mode != "append"
@classmethod
def from_crawler(
@@ -455,7 +542,7 @@ class FTPFeedStorage(BlockingFeedStorage):
username=self.username,
password=self.password,
use_active_mode=self.use_active_mode,
- overwrite=self.overwrite,
+ mode=self._mode,
tls=self.tls,
timeout=self.timeout,
)
@@ -495,6 +582,7 @@ class FeedSlot:
self.crawler: Crawler = crawler
# flags
self.itemcount: int = 0
+ self._skipped: bool = False
self._exporting: bool = False
self._fileloaded: bool = False
@@ -607,6 +695,19 @@ class FeedExporter:
self.exporters: dict[str, type[BaseItemExporter]] = self._load_components(
"FEED_EXPORTERS"
)
+ if any(
+ feed_options.get("mode") is None for feed_options in self.feeds.values()
+ ):
+ warnings.warn(
+ "The default value of the FEED_MODE setting will change from "
+ "None to 'create' in a future Scrapy version, i.e. Scrapy will "
+ "stop writing feeds whose target already exists. Explicitly "
+ "set FEED_MODE, or the mode feed option of every feed, to "
+ "silence this warning.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
for uri, feed_options in self.feeds.items():
if not self._storage_supported(uri, feed_options):
raise NotConfigured
@@ -638,6 +739,22 @@ class FeedExporter:
)
)
+ def _skip(self, slot: FeedSlot) -> None:
+ """Report that the target of *slot* already exists, and mark *slot* so
+ that nothing is written to it."""
+ slot._skipped = True
+ logger.error(
+ f"Not writing {slot.uri} because it already exists and the feed "
+ "mode is 'create'; its items are lost. To write it instead, remove "
+ "the target, or set the mode feed option or the FEED_MODE setting "
+ "to 'overwrite' or 'append' (-O implies 'overwrite').",
+ extra={"spider": slot.spider},
+ )
+ assert self.crawler.stats
+ self.crawler.stats.inc_value(
+ f"feedexport/conflicts/{type(slot.storage).__name__}"
+ )
+
async def close_spider(self, spider: Spider) -> None:
for slot in self.slots:
self._schedule_slot_close(slot, spider)
@@ -686,6 +803,8 @@ class FeedExporter:
return slot_.file
async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:
+ if slot._skipped:
+ return
if slot.itemcount:
# Normal case
@@ -705,6 +824,9 @@ class FeedExporter:
)
try:
await ensure_awaitable(slot.storage.store(self._get_file(slot)))
+ except FileExistsError:
+ self._skip(slot)
+ self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
except Exception:
logger.exception(
"Error storing %s",
@@ -771,13 +893,21 @@ class FeedExporter:
processor = self.processors[slot.uri_template]
for exported_item in processor(item) if processor else (item,):
- await slot.start_exporting()
- assert slot.exporter
- try:
- await ensure_awaitable(slot.exporter.export_item(exported_item))
- except Exception as e:
- e.add_note(f"Item: {exported_item!r}")
- raise
+ if not slot._skipped:
+ try:
+ await slot.start_exporting()
+ except FileExistsError:
+ self._skip(slot)
+ if not slot._skipped:
+ assert slot.exporter
+ try:
+ await ensure_awaitable(slot.exporter.export_item(exported_item))
+ except Exception as e:
+ e.add_note(f"Item: {exported_item!r}")
+ raise
+ # Skipped items are counted, so that the following files of
+ # this feed still cover the same items as those of any other
+ # feed.
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
if (
@@ -843,7 +973,8 @@ class FeedExporter:
scheme = urlparse(uri).scheme
if scheme in self.storages or PureWindowsPath(uri).drive:
try:
- self._get_storage(uri, feed_options)
+ storage = self._get_storage(uri, feed_options)
+ self._check_storage_mode(uri, storage, feed_options)
return True
except NotConfigured as e:
logger.error(
@@ -854,6 +985,23 @@ class FeedExporter:
logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})
return False
+ @staticmethod
+ def _check_storage_mode(
+ uri: str, storage: FeedStorageProtocol, feed_options: dict[str, Any]
+ ) -> None:
+ mode = feed_options.get("mode")
+ if mode is None:
+ # Every storage keeps its own historical mode.
+ return
+ if getattr(storage, "supported_modes", None) is None:
+ logger.warning(
+ f"{type(storage).__name__} does not declare which feed modes it "
+ f"supports, so the mode feed option of the {uri} feed ({mode!r}) "
+ "may be ignored."
+ )
+ return
+ _check_mode(mode, storage, uri)
+
def _get_storage(
self, uri: str, feed_options: dict[str, Any]
) -> FeedStorageProtocol:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 62a32acf4..18787bd0f 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -94,6 +94,7 @@ __all__ = [
"FEED_EXPORT_FIELDS",
"FEED_EXPORT_INDENT",
"FEED_FORMAT",
+ "FEED_MODE",
"FEED_STORAGES",
"FEED_STORAGES_BASE",
"FEED_STORAGE_FTP_ACTIVE",
@@ -395,6 +396,7 @@ FEED_EXPORTERS_BASE = {
"pickle": "scrapy.exporters.PickleItemExporter",
}
FEED_FORMAT = "jsonlines"
+FEED_MODE = None
FEED_STORE_EMPTY = True
FEED_STORAGES: dict[str, str] = {}
FEED_STORAGES_BASE = {
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index be874b3df..0861ab166 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,3 +86,4 @@ DOWNLOAD_DELAY = 1
# Set settings whose default value is deprecated to a future-proof value
DOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = True
FEED_EXPORT_ENCODING = "utf-8"
+FEED_MODE = "create"
diff --git a/scrapy/utils/_ftp.py b/scrapy/utils/_ftp.py
index 5d7fce08a..540b8b1d0 100644
--- a/scrapy/utils/_ftp.py
+++ b/scrapy/utils/_ftp.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import posixpath
from contextlib import closing
from ftplib import FTP, FTP_TLS, error_perm
@@ -20,6 +22,16 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
ftp.cwd(path)
+def _ftp_file_exists_in_cwd(ftp: FTP, filename: str) -> bool:
+ try:
+ ftp.voidcmd("TYPE I")
+ return ftp.size(filename) is not None
+ except error_perm:
+ # The file does not exist, or the server does not support the SIZE
+ # command, or it does not allow us to use it.
+ return False
+
+
def ftp_store_file(
*,
path: str,
@@ -30,18 +42,27 @@ def ftp_store_file(
password: str,
use_active_mode: bool = False,
overwrite: bool = True,
+ mode: str | None = None,
tls: bool = False,
timeout: float | None = None,
) -> None:
"""Opens a FTP connection with passed credentials, sets current directory
to the directory extracted from given path, then uploads the file to server.
+ *mode* may be ``"append"``, ``"create"`` or ``"overwrite"``. It takes
+ precedence over *overwrite*, which only remains for backward compatibility.
+
+ ``"create"`` is best-effort: FTP has no atomic create-if-absent command, so
+ the file is checked for existence right before it is written.
+
If *tls* is ``True``, the connection is secured with TLS (FTPS), and the
certificate of the server is verified.
*timeout* is the number of seconds that socket operations wait before
giving up. If ``None``, they wait indefinitely.
"""
+ if mode is None:
+ mode = "overwrite" if overwrite else "append"
ftp = FTP_TLS(context=create_default_context()) if tls else FTP()
with ftp, closing(file):
if timeout is None:
@@ -56,5 +77,7 @@ def ftp_store_file(
file.seek(0)
dirname, filename = posixpath.split(path)
ftp_makedirs_cwd(ftp, dirname)
- command = "STOR" if overwrite else "APPE"
+ if mode == "create" and _ftp_file_exists_in_cwd(ftp, filename):
+ raise FileExistsError(f"{path} already exists")
+ command = "APPE" if mode == "append" else "STOR"
ftp.storbinary(f"{command} {filename}", file)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index ecfdd0394..75b8df34d 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -3,12 +3,13 @@ from __future__ import annotations
import numbers
import os
import sys
+import warnings
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
-from scrapy.exceptions import UsageError
+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@@ -130,6 +131,25 @@ def feed_complete_default_values_from_settings(
feed: dict[str, Any], settings: BaseSettings, uri: str | None = None
) -> dict[str, Any]:
out = feed.copy()
+ if "overwrite" in out:
+ warnings.warn(
+ "The overwrite feed option is deprecated, use the mode feed option"
+ " instead: mode='overwrite' instead of overwrite=True, and"
+ " mode='append' instead of overwrite=False.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ if out.get("mode") is not None:
+ raise ValueError(
+ "The overwrite and mode feed options are mutually exclusive,"
+ " please set only the mode feed option."
+ )
+ out["mode"] = "overwrite" if out.pop("overwrite") else "append"
+ out.setdefault("mode", settings["FEED_MODE"])
+ if out["mode"] in {"append", "overwrite"}:
+ # Kept for feed storages that were written before the mode feed option
+ # existed and hence only look for the overwrite feed option.
+ out["overwrite"] = out["mode"] == "overwrite"
if uri is not None:
out.setdefault("format", Path(uri).suffix.removeprefix("."))
out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))
@@ -192,7 +212,9 @@ def feed_process_params_from_cli(
check_valid_format(feed_format)
result[feed_uri] = {"format": feed_format}
if overwrite:
- result[feed_uri]["overwrite"] = True
+ # -O unambiguously means overwriting, so it ignores FEED_MODE. -o
+ # does not, because appending is not supported by every storage.
+ result[feed_uri]["mode"] = "overwrite"
# FEEDS setting should take precedence over the matching CLI options
result.update(settings.getdict("FEEDS"))
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d3b69e3ed..bb26928e5 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -69,6 +69,7 @@ def get_crawler(
# When needed, useful settings can be added here, e.g. ones that prevent
# deprecation warnings.
settings: dict[str, Any] = {
+ "FEED_MODE": "overwrite",
"REMOTE_CONTROL_ENABLED": False,
"TELNETCONSOLE_ENABLED": False,
**get_reactor_settings(),
diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py
index 050371bd0..1ba6c9fbb 100644
--- a/tests/test_command_crawl.py
+++ b/tests/test_command_crawl.py
@@ -94,7 +94,7 @@ class MySpider(scrapy.Spider):
args = ["-O", "example.json"]
log = self.get_log(spider_code, proj_path, args=args)
assert (
- '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}'
+ '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "mode": "overwrite"}}'
in log
)
with j.open(encoding="utf-8") as f2:
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index c2fccd54f..4d809c4f3 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -326,7 +326,7 @@ class MySpider(scrapy.Spider):
args = ["-O", "example.json"]
log = self.get_log(tmp_path, spider_code, args=args)
assert (
- '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}'
+ '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "mode": "overwrite"}}'
in log
)
with (tmp_path / "example.json").open(encoding="utf-8") as f2:
diff --git a/tests/test_feedexport_mode.py b/tests/test_feedexport_mode.py
new file mode 100644
index 000000000..a1d55ce60
--- /dev/null
+++ b/tests/test_feedexport_mode.py
@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import IO, TYPE_CHECKING, Any
+
+import pytest
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.extensions.feedexport import BlockingFeedStorage, FileFeedStorage
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+from tests.utils.feedexport import crawl_items, path_to_url, printf_escape
+
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+ from tests.mockserver.http import MockServer
+
+
+class DelayedFileStorage(BlockingFeedStorage):
+ """Feed storage that, like the S3 or GCS ones, can only detect a conflict
+ when the feed is delivered, at the end of the crawl or of a batch."""
+
+ supported_modes = frozenset({"append", "create", "overwrite"})
+
+ def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
+ self.path = Path(uri.split("://", 1)[1])
+ self.mode = (feed_options or {}).get("mode")
+
+ def _store_in_thread(self, file: IO[bytes]) -> None:
+ file.seek(0)
+ try:
+ if self.mode == "create" and self.path.exists():
+ raise FileExistsError(str(self.path))
+ self.path.write_bytes(file.read())
+ finally:
+ file.close()
+
+
+class CreateOnlyFileStorage(FileFeedStorage):
+ supported_modes = frozenset({"create"})
+
+
+class LegacyFileStorage(FileFeedStorage):
+ """Feed storage that predates the mode feed option."""
+
+ supported_modes = None # type: ignore[assignment]
+
+
+class TestFeedMode:
+ """End-to-end tests of the mode feed option and the FEED_MODE setting."""
+
+ @pytest.fixture(autouse=True)
+ def _setup(self, mockserver: MockServer, tmp_path: Path) -> None:
+ self.mockserver = mockserver
+ self.temp_dir = tmp_path
+
+ def _path(self, content: bytes | None = None) -> Path:
+ path = self.temp_dir / "items.jl"
+ if content is not None:
+ path.write_bytes(content)
+ return path
+
+ async def _crawl(
+ self,
+ path: Path,
+ mode: str | None = None,
+ settings: dict[str, Any] | None = None,
+ item_count: int = 1,
+ scheme: str | None = None,
+ ) -> Crawler:
+ feed_options: dict[str, Any] = {"format": "jl"}
+ if mode is not None:
+ feed_options["mode"] = mode
+ if scheme is not None:
+ # as_posix() keeps the URI valid on Windows, where paths have a
+ # drive and backslashes.
+ uri = f"{scheme}://{path.as_posix()}"
+ elif "%(batch_id)d" in str(path):
+ # A batch URI template must keep its %(batch_id)d placeholder, so it
+ # is used as a path, which is neither quoted nor printf-escaped.
+ uri = str(path)
+ else:
+ uri = printf_escape(path_to_url(path))
+ return await crawl_items(
+ self.mockserver,
+ [{"foo": "bar"}] * item_count,
+ {"FEEDS": {uri: feed_options}, **(settings or {})},
+ )
+
+ @coroutine_test
+ async def test_create(self) -> None:
+ path = self._path()
+ await self._crawl(path, "create")
+ assert path.read_bytes() == b'{"foo": "bar"}\n'
+
+ @coroutine_test
+ async def test_create_existing(self, caplog: pytest.LogCaptureFixture) -> None:
+ path = self._path(b"old content")
+ with caplog.at_level(logging.ERROR):
+ crawler = await self._crawl(path, "create", item_count=3)
+ assert path.read_bytes() == b"old content"
+ # Reported once, not once per item.
+ assert caplog.text.count("because it already exists") == 1
+ assert crawler.stats
+ stats = crawler.stats.get_stats()
+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 1
+ assert "feedexport/success_count/FileFeedStorage" not in stats
+
+ @coroutine_test
+ async def test_create_existing_multiple_feeds(self) -> None:
+ """A conflicting target only affects its own feed: the crawl runs and
+ the other feeds are written."""
+ existing = self._path(b"old content")
+ missing = self.temp_dir / "missing.jl"
+ crawler = await crawl_items(
+ self.mockserver,
+ [{"foo": "bar"}],
+ {
+ "FEEDS": {
+ printf_escape(path_to_url(existing)): {"format": "jl"},
+ printf_escape(path_to_url(missing)): {"format": "jl"},
+ },
+ "FEED_MODE": "create",
+ },
+ )
+ assert existing.read_bytes() == b"old content"
+ assert missing.read_bytes() == b'{"foo": "bar"}\n'
+ assert crawler.stats
+ stats = crawler.stats.get_stats()
+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 1
+ assert stats.get("feedexport/success_count/FileFeedStorage") == 1
+ assert stats.get("finish_reason") == "finished"
+
+ @coroutine_test
+ async def test_create_existing_all_feeds(self, caplog: pytest.LogCaptureFixture):
+ """A crawl that cannot write any feed still runs, like any other crawl
+ whose feeds cannot be written."""
+ existing1 = self._path(b"old content")
+ existing2 = self.temp_dir / "items2.jl"
+ existing2.write_bytes(b"old content")
+ with caplog.at_level(logging.ERROR):
+ crawler = await crawl_items(
+ self.mockserver,
+ [{"foo": "bar"}],
+ {
+ "FEEDS": {
+ printf_escape(path_to_url(existing1)): {"format": "jl"},
+ printf_escape(path_to_url(existing2)): {"format": "jl"},
+ },
+ "FEED_MODE": "create",
+ },
+ )
+ assert existing1.read_bytes() == b"old content"
+ assert existing2.read_bytes() == b"old content"
+ assert caplog.text.count("because it already exists") == 2
+ assert crawler.stats
+ stats = crawler.stats.get_stats()
+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 2
+ assert stats.get("finish_reason") == "finished"
+ assert "feedexport/success_count/FileFeedStorage" not in stats
+
+ @coroutine_test
+ async def test_overwrite_existing(self) -> None:
+ path = self._path(b"old content")
+ await self._crawl(path, "overwrite")
+ assert path.read_bytes() == b'{"foo": "bar"}\n'
+
+ @coroutine_test
+ async def test_append_existing(self) -> None:
+ path = self._path(b"old content\n")
+ await self._crawl(path, "append")
+ assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'
+
+ @coroutine_test
+ async def test_unset_mode(self) -> None:
+ path = self._path()
+ with pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"):
+ await self._crawl(path, settings={"FEED_MODE": None})
+ assert path.read_bytes() == b'{"foo": "bar"}\n'
+
+ @coroutine_test
+ async def test_unset_mode_existing_target(self) -> None:
+ path = self._path(b"old content\n")
+ with pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"):
+ await self._crawl(path, settings={"FEED_MODE": None})
+ # The legacy behavior is kept.
+ assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'
+
+ @coroutine_test
+ async def test_explicit_mode_existing_target(self, recwarn) -> None:
+ path = self._path(b"old content\n")
+ await self._crawl(path, "append")
+ assert not [
+ warning
+ for warning in recwarn
+ if issubclass(warning.category, ScrapyDeprecationWarning)
+ and "FEED_MODE" in str(warning.message)
+ ]
+
+ @coroutine_test
+ async def test_create_existing_later_batch(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """When the target of a later batch exists, that batch is skipped, and
+ the following ones are still written."""
+ (self.temp_dir / "items-2.jl").write_bytes(b"old content")
+ with caplog.at_level(logging.ERROR):
+ crawler = await self._crawl(
+ self.temp_dir / "items-%(batch_id)d.jl",
+ "create",
+ {"FEED_EXPORT_BATCH_ITEM_COUNT": 1},
+ item_count=3,
+ )
+ assert (self.temp_dir / "items-1.jl").read_bytes() == b'{"foo": "bar"}\n'
+ assert (self.temp_dir / "items-2.jl").read_bytes() == b"old content"
+ # The skipped items are counted, so the following batch covers the same
+ # items that it would have covered otherwise.
+ assert (self.temp_dir / "items-3.jl").read_bytes() == b'{"foo": "bar"}\n'
+ assert not (self.temp_dir / "items-4.jl").exists()
+ # A single error, instead of one per item after the conflicting one.
+ assert caplog.text.count("because it already exists") == 1
+ assert crawler.stats
+ stats = crawler.stats.get_stats()
+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 1
+ assert stats.get("feedexport/success_count/FileFeedStorage") == 2
+ assert stats.get("finish_reason") == "finished"
+
+ @coroutine_test
+ async def test_create_existing_later_batch_delayed(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """When the feed is only delivered at the end of each batch, the
+ conflict is detected then, and only that batch is lost."""
+ (self.temp_dir / "items-2.jl").write_bytes(b"old content")
+ with caplog.at_level(logging.ERROR):
+ crawler = await self._crawl(
+ self.temp_dir / "items-%(batch_id)d.jl",
+ "create",
+ {
+ "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
+ "FEED_STORAGES": {
+ "delayed": "tests.test_feedexport_mode.DelayedFileStorage"
+ },
+ },
+ item_count=3,
+ scheme="delayed",
+ )
+ assert (self.temp_dir / "items-1.jl").read_bytes() == b'{"foo": "bar"}\n'
+ assert (self.temp_dir / "items-2.jl").read_bytes() == b"old content"
+ assert (self.temp_dir / "items-3.jl").read_bytes() == b'{"foo": "bar"}\n'
+ assert caplog.text.count("because it already exists") == 1
+ assert crawler.stats
+ stats = crawler.stats.get_stats()
+ assert stats.get("feedexport/conflicts/DelayedFileStorage") == 1
+ assert stats.get("feedexport/failed_count/DelayedFileStorage") == 1
+ assert stats.get("feedexport/success_count/DelayedFileStorage") == 2
+
+ @coroutine_test
+ async def test_feed_mode_setting(self) -> None:
+ path = self._path(b"old content")
+ await self._crawl(path, settings={"FEED_MODE": "overwrite"})
+ assert path.read_bytes() == b'{"foo": "bar"}\n'
+
+ @coroutine_test
+ async def test_mode_overrides_feed_mode_setting(self) -> None:
+ path = self._path(b"old content\n")
+ await self._crawl(path, "append", {"FEED_MODE": "overwrite"})
+ assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'
+
+
+class TestFeedModeInit:
+ def test_invalid_mode(self):
+ """An invalid mode prevents the crawl from running."""
+ settings = {
+ "FEEDS": {"file:///tmp/items.json": {"format": "json", "mode": "x"}}
+ }
+ with pytest.raises(ValueError, match="Invalid feed mode: 'x'"):
+ get_crawler(settings_dict=settings)
+
+ def test_unsupported_mode(self):
+ settings = {
+ "FEEDS": {"file:///tmp/items.json": {"format": "json"}},
+ "FEED_STORAGES": {
+ "file": "tests.test_feedexport_mode.CreateOnlyFileStorage"
+ },
+ "FEED_MODE": "append",
+ }
+ with pytest.raises(
+ ValueError,
+ match="CreateOnlyFileStorage does not support the 'append' feed mode",
+ ):
+ get_crawler(settings_dict=settings)
+
+ def test_undeclared_mode_support(self, caplog: pytest.LogCaptureFixture):
+ settings = {
+ "FEEDS": {"file:///tmp/items.json": {"format": "json"}},
+ "FEED_STORAGES": {"file": "tests.test_feedexport_mode.LegacyFileStorage"},
+ "FEED_MODE": "create",
+ }
+ with caplog.at_level(logging.WARNING):
+ get_crawler(settings_dict=settings)
+ assert "does not declare which feed modes it supports" in caplog.text
+
+ def test_undeclared_mode_support_unset_mode(self, caplog: pytest.LogCaptureFixture):
+ settings = {
+ "FEEDS": {"file:///tmp/items.json": {"format": "json"}},
+ "FEED_STORAGES": {"file": "tests.test_feedexport_mode.LegacyFileStorage"},
+ "FEED_MODE": None,
+ }
+ with (
+ caplog.at_level(logging.WARNING),
+ pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"),
+ ):
+ get_crawler(settings_dict=settings)
+ assert "does not declare which feed modes it supports" not in caplog.text
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index cc095ff75..78e00992c 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -15,7 +15,7 @@ import pytest
from w3lib.url import path_to_file_uri
import scrapy
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.feedexport import (
BlockingFeedStorage,
FileFeedStorage,
@@ -69,11 +69,43 @@ class TestFileFeedStorage:
def test_overwrite(self, tmp_path):
path = tmp_path / "file.txt"
- self._store(path, {"overwrite": True})
+ self._store(path, {"mode": "overwrite"})
self._assert_stores(
- FileFeedStorage(str(path), feed_options={"overwrite": True}), path
+ FileFeedStorage(str(path), feed_options={"mode": "overwrite"}), path
)
+ def test_create(self, tmp_path):
+ path = tmp_path / "file.txt"
+ self._assert_stores(
+ FileFeedStorage(str(path), feed_options={"mode": "create"}), path
+ )
+
+ def test_create_existing(self, tmp_path):
+ path = tmp_path / "file.txt"
+ path.write_bytes(b"content")
+ storage = FileFeedStorage(str(path), feed_options={"mode": "create"})
+ with pytest.raises(FileExistsError):
+ storage.open(scrapy.Spider("default"))
+ assert path.read_bytes() == b"content"
+
+ @pytest.mark.parametrize(
+ ("feed_options", "expected_write_mode"),
+ [
+ (None, "ab"),
+ ({}, "ab"),
+ ({"mode": "create"}, "xb"),
+ ({"overwrite": True}, "wb"),
+ ({"overwrite": False}, "ab"),
+ ],
+ )
+ def test_mode(self, tmp_path, feed_options, expected_write_mode):
+ storage = FileFeedStorage(str(tmp_path / "file.txt"), feed_options=feed_options)
+ assert storage.write_mode == expected_write_mode
+
+ def test_invalid_mode(self, tmp_path):
+ with pytest.raises(ValueError, match="Invalid feed mode: 'x'"):
+ FileFeedStorage(str(tmp_path / "file.txt"), feed_options={"mode": "x"})
+
@staticmethod
def _assert_stores(
storage: FileFeedStorage, path: Path, expected_content: bytes = b"content"
@@ -122,19 +154,20 @@ class TestFTPFeedStorage:
file.write(content)
await maybe_deferred_to_future(storage.store(file))
- def _assert_stored(self, path: Path, content: bytes) -> None:
+ def _assert_stored(self, path: Path, content: bytes, unlink: bool = True) -> None:
assert path.exists()
try:
assert path.read_bytes() == content
finally:
- path.unlink()
+ if unlink:
+ path.unlink()
@coroutine_test
async def test_append(self):
with MockFTPServer() as ftp_server:
filename = "file"
url = ftp_server.url(filename)
- feed_options = {"overwrite": False}
+ feed_options = {"mode": "append"}
await self._store(url, b"foo", feed_options=feed_options)
await self._store(url, b"bar", feed_options=feed_options)
self._assert_stored(ftp_server.path / filename, b"foobar")
@@ -148,13 +181,25 @@ class TestFTPFeedStorage:
await self._store(url, b"bar")
self._assert_stored(ftp_server.path / filename, b"bar")
+ @coroutine_test
+ async def test_create(self):
+ with MockFTPServer() as ftp_server:
+ filename = "file"
+ url = ftp_server.url(filename)
+ feed_options = {"mode": "create"}
+ await self._store(url, b"foo", feed_options=feed_options)
+ self._assert_stored(ftp_server.path / filename, b"foo", unlink=False)
+ with pytest.raises(FileExistsError):
+ await self._store(url, b"bar", feed_options=feed_options)
+ self._assert_stored(ftp_server.path / filename, b"foo")
+
@coroutine_test
async def test_append_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
filename = "file"
url = ftp_server.url(filename)
- feed_options = {"overwrite": False}
+ feed_options = {"mode": "append"}
await self._store(url, b"foo", feed_options=feed_options, settings=settings)
await self._store(url, b"bar", feed_options=feed_options, settings=settings)
self._assert_stored(ftp_server.path / filename, b"foobar")
@@ -169,6 +214,13 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", settings=settings)
self._assert_stored(ftp_server.path / filename, b"bar")
+ def test_overwrite_deprecated(self):
+ storage = FTPFeedStorage.from_crawler(get_crawler(), "ftp://localhost/file")
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="FTPFeedStorage.overwrite is deprecated"
+ ):
+ assert storage.overwrite is True
+
@coroutine_test
async def test_upload_timeout(self):
crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})
@@ -521,21 +573,69 @@ class TestS3FeedStorage:
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
assert acl == "custom-acl"
- def test_overwrite_default(self, caplog: pytest.LogCaptureFixture) -> None:
- S3FeedStorage(
- "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
- )
- assert "S3 does not support appending to files" not in caplog.text
+ def test_mode_append(self) -> None:
+ with pytest.raises(
+ ValueError, match="S3FeedStorage does not support the 'append' feed mode"
+ ):
+ S3FeedStorage(
+ "s3://mybucket/export.csv",
+ "access_key",
+ "secret_key",
+ "custom-acl",
+ feed_options={"mode": "append"},
+ )
- def test_overwrite_false(self, caplog: pytest.LogCaptureFixture) -> None:
- S3FeedStorage(
+ @coroutine_test
+ async def test_store_create(self) -> None:
+ from botocore.stub import ANY, Stubber # noqa: PLC0415
+
+ storage = S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
"secret_key",
"custom-acl",
- feed_options={"overwrite": False},
+ feed_options={"mode": "create"},
)
- assert "S3 does not support appending to files" in caplog.text
+ with Stubber(storage.s3_client) as stub:
+ stub.add_response(
+ "put_object",
+ {},
+ expected_params={
+ "Bucket": "mybucket",
+ "Key": "export.csv",
+ "Body": ANY,
+ "IfNoneMatch": "*",
+ "ACL": "custom-acl",
+ },
+ )
+ stored = storage.store(BytesIO(b"test file"))
+ assert stored is not None
+ await maybe_deferred_to_future(stored)
+ stub.assert_no_pending_responses()
+
+ @pytest.mark.parametrize(
+ ("error_code", "exception"),
+ [("PreconditionFailed", FileExistsError), ("AccessDenied", None)],
+ )
+ @coroutine_test
+ async def test_store_create_error(
+ self, error_code: str, exception: type[Exception] | None
+ ) -> None:
+ from botocore.exceptions import ClientError # noqa: PLC0415
+ from botocore.stub import Stubber # noqa: PLC0415
+
+ storage = S3FeedStorage(
+ "s3://mybucket/export.csv",
+ "access_key",
+ "secret_key",
+ feed_options={"mode": "create"},
+ )
+ with Stubber(storage.s3_client) as stub:
+ stub.add_client_error("put_object", service_error_code=error_code)
+ stored = storage.store(BytesIO(b"test file"))
+ assert stored is not None
+ with pytest.raises(exception or ClientError):
+ await maybe_deferred_to_future(stored)
class TestGCSFeedStorage:
@@ -641,6 +741,46 @@ class TestGCSFeedStorage:
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
f.close.assert_called_once_with()
+ @coroutine_test
+ async def test_store_create(self):
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, _, blob_mock) = mock_google_cloud_storage()
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+ f = mock.Mock()
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ "publicRead",
+ feed_options={"mode": "create"},
+ )
+ await maybe_deferred_to_future(storage.store(f))
+ blob_mock.upload_from_file.assert_called_once_with(
+ f, predefined_acl="publicRead", if_generation_match=0
+ )
+ f.close.assert_called_once_with()
+
+ @coroutine_test
+ async def test_store_create_existing(self):
+ pytest.importorskip("google.cloud.storage")
+ from google.api_core.exceptions import PreconditionFailed # noqa: PLC0415
+
+ (client_mock, _, blob_mock) = mock_google_cloud_storage()
+ blob_mock.upload_from_file.side_effect = PreconditionFailed("exists")
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+ f = mock.Mock()
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ None,
+ feed_options={"mode": "create"},
+ )
+ with pytest.raises(FileExistsError):
+ await maybe_deferred_to_future(storage.store(f))
+ f.close.assert_called_once_with()
+
@coroutine_test
async def test_store_append_new_blob(self) -> None:
pytest.importorskip("google.cloud.storage")
@@ -655,7 +795,7 @@ class TestGCSFeedStorage:
"gs://mybucket/export.csv",
"myproject-123",
"publicRead",
- feed_options={"overwrite": False},
+ feed_options={"mode": "append"},
)
await maybe_deferred_to_future(storage.store(f))
@@ -682,7 +822,7 @@ class TestGCSFeedStorage:
"gs://mybucket/export.csv",
"myproject-123",
"publicRead",
- feed_options={"overwrite": False},
+ feed_options={"mode": "append"},
)
await maybe_deferred_to_future(storage.store(f))
@@ -746,7 +886,7 @@ class TestGCSFeedStorage:
"gs://mybucket/export.csv",
"myproject-123",
None,
- feed_options={"overwrite": False},
+ feed_options={"mode": "append"},
)
await maybe_deferred_to_future(storage.store(mock.Mock()))
@@ -769,7 +909,7 @@ class TestGCSFeedStorage:
"gs://mybucket/export.csv",
"myproject-123",
"publicRead",
- feed_options={"overwrite": False},
+ feed_options={"mode": "append"},
)
with pytest.raises(OSError, match="Compose failed"):
await maybe_deferred_to_future(storage.store(f))
@@ -796,22 +936,21 @@ class TestGCSFeedStorage:
@pytest.mark.parametrize(
("feed_options", "expected"),
[
- (None, True),
- ({}, True),
- ({"overwrite": True}, True),
- ({"overwrite": False}, False),
+ (None, "overwrite"),
+ ({}, "overwrite"),
+ ({"overwrite": True}, "overwrite"),
+ ({"overwrite": False}, "append"),
+ ({"mode": "append"}, "append"),
],
)
- def test_overwrite(
- self, feed_options: dict[str, Any] | None, expected: bool
- ) -> None:
+ def test_mode(self, feed_options: dict[str, Any] | None, expected: str) -> None:
storage = GCSFeedStorage(
"gs://mybucket/export.csv",
"myproject-123",
"custom-acl",
feed_options=feed_options,
)
- assert storage.overwrite is expected
+ assert storage._mode == expected
class TestStdoutFeedStorage:
@@ -823,18 +962,15 @@ class TestStdoutFeedStorage:
storage.store(file)
assert out.getvalue() == b"content"
- def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):
- with caplog.at_level(logging.DEBUG):
- StdoutFeedStorage("stdout:")
- assert (
- "Standard output (stdout) storage does not support overwriting"
- not in caplog.text
- )
-
- def test_overwrite_true(self, caplog: pytest.LogCaptureFixture):
+ @pytest.mark.parametrize("mode", ["append", "create", "overwrite"])
+ def test_mode_ignored(self, mode: str, caplog: pytest.LogCaptureFixture):
+ out = BytesIO()
with caplog.at_level(logging.DEBUG):
- StdoutFeedStorage("stdout:", feed_options={"overwrite": True})
- assert (
- "Standard output (stdout) storage does not support overwriting"
- in caplog.text
- )
+ storage = StdoutFeedStorage(
+ "stdout:", _stdout=out, feed_options={"mode": mode}
+ )
+ file = storage.open(scrapy.Spider("default"))
+ file.write(b"content")
+ storage.store(file)
+ assert out.getvalue() == b"content"
+ assert not caplog.text
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 01e76a25b..e1b87043b 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any
import pytest
-from scrapy.exceptions import UsageError
+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
_job_dir,
@@ -122,11 +122,60 @@ class TestFeedExportConfig:
def test_feed_export_config_overwrite(self):
settings = Settings()
assert {
- "output.json": {"format": "json", "overwrite": True}
+ "output.json": {"format": "json", "mode": "overwrite"}
} == feed_process_params_from_cli(
settings, [], overwrite_output=["output.json"]
)
+ def test_feed_complete_default_values_mode_from_settings(self):
+ settings = Settings({"FEED_MODE": "create"})
+ new_feed = feed_complete_default_values_from_settings({}, settings)
+ assert new_feed["mode"] == "create"
+ assert "overwrite" not in new_feed
+
+ @pytest.mark.parametrize(
+ ("mode", "overwrite"),
+ [
+ ("append", False),
+ ("overwrite", True),
+ ],
+ )
+ def test_feed_complete_default_values_mode_sets_overwrite(self, mode, overwrite):
+ """The deprecated overwrite feed option is kept in sync for the sake of
+ feed storages that predate the mode feed option."""
+ settings = Settings({"FEED_MODE": mode})
+ new_feed = feed_complete_default_values_from_settings({}, settings)
+ assert new_feed["mode"] == mode
+ assert new_feed["overwrite"] is overwrite
+
+ @pytest.mark.parametrize(
+ ("overwrite", "mode"),
+ [
+ (True, "overwrite"),
+ (False, "append"),
+ ],
+ )
+ def test_feed_complete_default_values_overwrite_deprecated(self, overwrite, mode):
+ settings = Settings()
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="overwrite feed option is deprecated"
+ ):
+ new_feed = feed_complete_default_values_from_settings(
+ {"overwrite": overwrite}, settings
+ )
+ assert new_feed["mode"] == mode
+ assert new_feed["overwrite"] is overwrite
+
+ def test_feed_complete_default_values_overwrite_and_mode(self):
+ settings = Settings()
+ with (
+ pytest.raises(ValueError, match="mutually exclusive"),
+ pytest.warns(ScrapyDeprecationWarning),
+ ):
+ feed_complete_default_values_from_settings(
+ {"overwrite": True, "mode": "create"}, settings
+ )
+
def test_output_and_overwrite_output(self):
with pytest.raises(UsageError):
feed_process_params_from_cli(
@@ -147,6 +196,7 @@ class TestFeedExportConfig:
)
new_feed = feed_complete_default_values_from_settings(feed, settings)
assert new_feed == {
+ "mode": None,
"encoding": "custom encoding",
"fields": ["f1", "f2", "f3"],
"indent": 42,
@@ -172,6 +222,7 @@ class TestFeedExportConfig:
)
new_feed = feed_complete_default_values_from_settings(feed, settings)
assert new_feed == {
+ "mode": None,
"encoding": "other encoding",
"fields": None,
"indent": 42,
diff --git a/tox.ini b/tox.ini
index 934c8fcd9..6ed82b836 100644
--- a/tox.ini
+++ b/tox.ini
@@ -200,7 +200,7 @@ deps =
{[min]deps}
Pillow==9.2.0
Twisted[http2]==21.7.0
- boto3==1.20.0
+ boto3==1.35.2
bpython==0.7.1
colorlog==6.6.0
google-cloud-storage==1.30.0