Scrapy, a fast high-level web crawling & scraping framework for Python.
Support feed modes (#7826)
Adrian198e19995cdc
diff --git a/docs/conf.py b/docs/conf.pyindex ce6096867..fd0fc0f98 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -175,16 +175,5 @@ scrapy_intersphinx_enable = ["w3lib",]-# sphinx-llm-friendly -----------------------------------------------------------llm_friendly_llms_full_txt_exclude = [- # Changelog, not useful for an LLM answering "how do I use Scrapy"- # questions, and the largest single contributor to llms-full.txt size.- "news.rst",- "news/*",- "contributing.rst",-]--# -- Other options ------------------------------------------------------------default_dark_mode = Falsediff --git a/docs/requirements.txt b/docs/requirements.txtindex 37b40591b..c2d5b3ef0 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -1,5 +1,5 @@# This file was autogenerated by uv via the following command:-# uv pip compile requirements.in -o requirements.txt --refresh+# uv pip compile requirements.in -o requirements.txtalabaster==1.0.0# via sphinxannotated-types==0.7.0@@ -110,14 +110,11 @@ pyopenssl==26.0.0# via scrapyqueuelib==1.9.0# via scrapy-regex==2026.9.10- # via tiktokenrequests==2.33.0# via# requests-file# sphinx# sphinxcontrib-youtube- # tiktoken# tldextractrequests-file==3.0.1# via tldextract@@ -150,7 +147,7 @@ sphinx-design==0.7.0# via sphinx-scrapysphinx-last-updated-by-git==0.3.8# via sphinx-sitemap-sphinx-llm-friendly==0.6.0+sphinx-llm-friendly==0.7.0# via sphinx-scrapysphinx-notfound-page==1.1.0# via -r requirements.in@@ -184,8 +181,6 @@ sphinxcontrib-youtube==1.5.0# via sphinx-scrapytabulate==0.10.0# via sphinx-llm-friendly-tiktoken==0.14.0- # via sphinx-llm-friendlytldextract==5.3.1# via scrapytwisted==25.5.0diff --git a/docs/topics/commands.rst b/docs/topics/commands.rstindex d533fd367..b9b7c2a31 100644--- a/docs/topics/commands.rst+++ b/docs/topics/commands.rst@@ -330,9 +330,10 @@ Supported options:* ``-a NAME=VALUE``: set a spider argument (may be repeated)-* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE- (use ``-`` for stdout). To define the output format, add a colon at the end- of the output URI (for example, ``-o FILE:FORMAT``)+* ``--output FILE`` or ``-o FILE``: dump scraped items into FILE (use ``-`` for+ stdout), handling an existing FILE as the :setting:`FEED_MODE` setting+ indicates. To define the output format, add a colon at the end of the output+ URI (for example, ``-o FILE:FORMAT``)* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE,overwriting any existing file. To define the output format, add a colon atdiff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 89b231bad..a2ccbe7d7 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -168,6 +168,9 @@ Note that for the local filesystem storage you can omit the scheme if youspecify a path (e.g. ``/tmp/export.csv``). Alternatively, you can use a:class:`pathlib.Path` object.+Supported :ref:`modes <feed-mode>`: ``"append"``, ``"create"`` and+``"overwrite"``. If :setting:`FEED_MODE` is ``None``, ``"append"`` is used.+.. _topics-feed-storage-ftp:.. _feed-storage-ftp:@@ -188,11 +191,16 @@ FTP supports two different connection modes: `active or passivemode by default. To use the active connection mode instead, set the:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.-The default value for the ``overwrite`` key in the :setting:`FEEDS` for this-storage backend is: ``True``.+Supported :ref:`modes <feed-mode>`: ``"append"``, ``"create"`` and+``"overwrite"``. If :setting:`FEED_MODE` is ``None``, ``"overwrite"`` is used.++.. note:: Some FTP servers may not support appending to files (the ``APPE`` FTP+ command).-.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the- previous version of your data.+.. note:: FTP has no atomic create-if-absent command, so ``"create"`` is+ best-effort: the target is checked for existence right before it is+ written. Servers that do not support the ``SIZE`` command are+ reported as not having the target.This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.@@ -211,8 +219,8 @@ certificate of the server verified.- Example URI: ``ftps://user:[email protected]/path/to/export.csv``- Required external libraries: none-See :ref:`feed-storage-ftp` for connection modes, the ``overwrite`` default and-file delivery.+See :ref:`feed-storage-ftp` for connection modes, supported :ref:`modes+<feed-mode>` and file delivery... note:: For SFTP, an unrelated protocol built on SSH, use`scrapy-feedexporter-sftp@@ -253,11 +261,11 @@ pool size for exported feeds using these settings:- :setting:`AWS_REGION_NAME`- :setting:`AWS_MAX_POOL_CONNECTIONS`-The default value for the ``overwrite`` key in the :setting:`FEEDS` for this-storage backend is: ``True``.+Supported :ref:`modes <feed-mode>`: ``"create"`` and ``"overwrite"``. If+:setting:`FEED_MODE` is ``None``, ``"overwrite"`` is used.-.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the- previous version of your data.+``"create"`` mode `does not support files larger than 5 GB+<https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html>`__.This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.@@ -284,14 +292,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following- :setting:`FEED_STORAGE_GCS_ACL`- :setting:`GCS_PROJECT_ID`-The default value for the ``overwrite`` key in the :setting:`FEEDS` for this-storage backend is: ``True``.+Supported :ref:`modes <feed-mode>`: ``"append"``, ``"create"`` and+``"overwrite"``. If :setting:`FEED_MODE` is ``None``, ``"overwrite"`` is used.-.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the- previous version of your data.--Appending (``overwrite: False``) turns the feed into a `composite object`_,-which has a CRC32C checksum but no MD5 hash.+Appending turns the feed into a `composite object`_, which has a CRC32C+checksum but no MD5 hash... versionadded:: VERSIONAppending support.@@ -311,6 +316,8 @@ The feeds are written to the standard output of the Scrapy process.- Example URI: ``stdout:``- Required external libraries: none+The :ref:`mode <feed-mode>` is ignored by this storage backend.+.. _delayed-file-delivery:@@ -344,6 +351,25 @@ URI scheme through the :setting:`FEED_STORAGES` setting... autoclass:: scrapy.extensions.feedexport.FeedStorageProtocol(uri, *, feed_options=None):members:+ .. attribute:: supported_modes+ :type: frozenset[str]++ Optional set of supported values of the ``mode`` :ref:`feed option+ <feed-options>`, e.g. ``frozenset({"create", "overwrite"})``.++ .. versionadded:: VERSION++ Setting ``mode`` to an unsupported value raises an exception, which+ prevents the crawl from running. A storage class that does not define+ this attribute is assumed to predate the ``mode`` feed option, and+ setting that option on a feed that uses it logs a warning about the+ option being possibly ignored.++ A storage class that declares support for ``"create"`` must enforce it+ when writing the feed, ideally atomically, raising+ :exc:`FileExistsError` if the target exists. Scrapy then handles that+ file as described in :setting:`FEED_MODE`.+If your storage backend blocks, subclass:class:`~scrapy.extensions.feedexport.BlockingFeedStorage` instead: it writesitems into a temporary local file (see :ref:`delayed file delivery@@ -493,6 +519,7 @@ These are the settings used for configuring the feed exports:- :setting:`FEEDS` (mandatory)- :setting:`FEED_EXPORT_ENCODING`+- :setting:`FEED_MODE`- :setting:`FEED_STORE_EMPTY`- :setting:`FEED_EXPORT_FIELDS`- :setting:`FEED_EXPORT_INDENT`@@ -589,24 +616,9 @@ as a fallback value if that key is not provided for a specific feed definition:- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.-- ``overwrite``: whether to overwrite the file if it already exists- (``True``) or append to its content (``False``).-- The default value depends on the :ref:`storage backend- <topics-feed-storage-backends>`:-- - :ref:`topics-feed-storage-fs`: ``False``-- - :ref:`feed-storage-ftp` and :ref:`feed-storage-ftps`: ``True``+- ``mode``: falls back to :setting:`FEED_MODE`.- .. note:: Some FTP servers may not support appending to files (the- ``APPE`` FTP command).-- - :ref:`topics-feed-storage-s3`: ``True`` (appending is not supported)-- - :ref:`topics-feed-storage-gcs`: ``True``-- - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)+ .. versionadded:: VERSION- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.@@ -657,6 +669,44 @@ Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter`and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exportingto ``.json`` or ``.xml``.+.. _feed-mode:+.. setting:: FEED_MODE++FEED_MODE+---------++.. versionadded:: VERSION++Default: ``"create"`` (:ref:`fallback <default-settings>`: ``None``)++What to do when the target of a feed already exists.++Supported values, which match the corresponding :func:`open` modes:++- ``"append"`` (``a``): append to the existing target.++- ``"create"`` (``x``): do not write, keeping the existing target untouched.++- ``"overwrite"`` (``w``): replace the existing target.++Can be overridden with the ``mode`` :ref:`feed option <feed-options>`.++Not every :ref:`storage backend <topics-feed-storage-backends>` supports every+mode; see the documentation of your storage backend.++If ``None``, the mode depends on the backend being used, e.g. appending for the+:ref:`local filesystem <topics-feed-storage-fs>` and overwriting for :ref:`S3+<topics-feed-storage-s3>`.++When the mode of a feed is ``"create"`` and its target already exists, Scrapy+logs an error, increases the ``feedexport/conflicts/<storage class name>``+:ref:`stat <topics-stats>`, and does not write that file, i.e. the items that it+would have contained are lost.++.. note:: When resuming a crawl (see :setting:`JOBDIR`), the feed of the+ previous run already exists, so a feed that must survive a resumed crawl+ needs the ``"append"`` mode.+.. setting:: FEED_STORE_EMPTYFEED_STORE_EMPTY@@ -666,8 +716,8 @@ Default: ``True``Whether to export empty feeds (i.e. feeds with no items).If ``False``, and there are no items to export, no new files are created and-existing files are not modified, even if the :ref:`overwrite feed option-<feed-options>` is enabled.+existing files are not modified, even if the ``mode`` :ref:`feed option+<feed-options>` is ``"overwrite"``... setting:: FEED_STORAGESdiff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rstindex 0496a987a..a62e9b879 100644--- a/docs/topics/jobs.rst+++ b/docs/topics/jobs.rst@@ -114,6 +114,15 @@ version that wrote them. A job must be resumed with the same Scrapy versionthat paused it; after upgrading or downgrading Scrapy, start a new job with anew job directory.+Feed exports+------------++When a job is resumed, the :ref:`feeds <topics-feed-exports>` of the previous+run already exist, so their :ref:`mode <feed-mode>` determines what happens to+the items of the resumed run: ``"append"`` adds them to those feeds,+``"overwrite"`` replaces the items of the previous run with them, and+``"create"`` does not write them anywhere.+Cookie expiration-----------------diff --git a/pyproject.toml b/pyproject.tomlindex 952c741c2..0d7541514 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -79,7 +79,7 @@ ipython = ["ipython>=8.15.0"]memusage = ["psutil>=5.9.0"]ptpython = ["ptpython>=3.0.23"]robotparser = ["robotexclusionrulesparser>=1.6.2"]-s3 = ["boto3>=1.20.0"]+s3 = ["boto3>=1.35.2"]twisted-http2 = ["Twisted[http2]>=21.7.0"]uvloop = ["uvloop>=0.17.0; platform_system != 'Windows' and implementation_name != 'pypy'",diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c3ed0ed08..a07e35f76 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -86,6 +86,38 @@ UriParamsCallableT: TypeAlias = Callable[_ItemProcessor: TypeAlias = Callable[[Any], Iterable[Any]]+_FEED_MODES: frozenset[str] = frozenset({"append", "create", "overwrite"})+++def _get_mode(storage: Any, feed_options: dict[str, Any] | None, legacy: str) -> str:+ """Return the mode of *storage*, based on the *mode* feed option, or on the+ deprecated *overwrite* feed option, or, if neither is set, on *legacy*,+ which must be the mode that the storage used before the *mode* feed option+ existed."""+ feed_options = feed_options or {}+ mode = feed_options.get("mode")+ if mode is None:+ overwrite = feed_options.get("overwrite")+ mode = legacy if overwrite is None else "overwrite" if overwrite else "append"+ _check_mode(mode, storage)+ return mode+++def _check_mode(mode: str, storage: Any, uri: str | None = None) -> None:+ if mode not in _FEED_MODES:+ raise ValueError(+ f"Invalid feed mode: {mode!r}. Supported modes: "+ f"{', '.join(sorted(_FEED_MODES))}."+ )+ supported: frozenset[str] | None = getattr(storage, "supported_modes", None)+ if supported is not None and mode not in supported:+ suffix = f" (feed URI: {uri})" if uri else ""+ raise ValueError(+ f"{type(storage).__name__} does not support the {mode!r} feed "+ f"mode{suffix}. Supported modes: {', '.join(sorted(supported))}."+ )++class ItemFilter:"""This will be used by FeedExporter to decide if an item should be allowed@@ -188,6 +220,9 @@ class BlockingFeedStorage(ABC):class StdoutFeedStorage:""":ref:`Standard output <topics-feed-storage-stdout>` storage backend."""+ # The mode is irrelevant here: writing to a stream cannot destroy data.+ supported_modes: frozenset[str] = _FEED_MODES+def __init__(self,uri: str,@@ -198,13 +233,6 @@ class StdoutFeedStorage:if not _stdout:_stdout = sys.stdout.bufferself._stdout: IO[bytes] = _stdout- if feed_options and feed_options.get("overwrite", False) is True:- logger.warning(- "Standard output (stdout) storage does not support "- "overwriting. To suppress this warning, remove the "- "overwrite option from your FEEDS setting, or set "- "it to False."- )def open(self, spider: Spider) -> IO[bytes]:return self._stdout@@ -213,6 +241,13 @@ class StdoutFeedStorage:pass+_WRITE_MODES: dict[str, OpenBinaryMode] = {+ "append": "ab",+ "create": "xb",+ "overwrite": "wb",+}++class FileFeedStorage:""":ref:`Local filesystem <topics-feed-storage-fs>` storage backend.@@ -220,12 +255,13 @@ class FileFeedStorage:are created when the feed is opened."""+ supported_modes: frozenset[str] = _FEED_MODES+def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri- feed_options = feed_options or {}- self.write_mode: OpenBinaryMode = (- "wb" if feed_options.get("overwrite", False) else "ab"- )+ self.write_mode: OpenBinaryMode = _WRITE_MODES[+ _get_mode(self, feed_options, "append")+ ]def open(self, spider: Spider) -> IO[bytes]:dirname = Path(self.path).parent@@ -241,6 +277,8 @@ class FileFeedStorage:class S3FeedStorage(BlockingFeedStorage):""":ref:`Amazon S3 <topics-feed-storage-s3>` storage backend."""+ supported_modes: frozenset[str] = frozenset({"create", "overwrite"})+def __init__(self,uri: str,@@ -291,12 +329,7 @@ class S3FeedStorage(BlockingFeedStorage):config=Config(**config_kwargs),)- if feed_options and feed_options.get("overwrite", True) is False:- logger.warning(- "S3 does not support appending to files. To "- "suppress this warning, remove the overwrite "- "option from your FEEDS setting or set it to True."- )+ self._mode: str = _get_mode(self, feed_options, "overwrite")@classmethoddef from_crawler(@@ -320,14 +353,37 @@ class S3FeedStorage(BlockingFeedStorage):)def _store_in_thread(self, file: IO[bytes]) -> None:+ from botocore.exceptions import ClientError # noqa: PLC0415+file.seek(0)+ extra_args: dict[str, Any] = {"ACL": self.acl} if self.acl else {}try:- if self.acl:+ if self._mode == "create":+ # upload_fileobj() does not allow IfNoneMatch+ # (https://github.com/boto/boto3/issues/4366).+ try:+ self.s3_client.put_object(+ Bucket=self.bucketname,+ Key=self.keyname,+ Body=file,+ IfNoneMatch="*",+ **extra_args,+ )+ except ClientError as error:+ if (+ error.response.get("Error", {}).get("Code")+ == "PreconditionFailed"+ ):+ raise FileExistsError(+ f"s3://{self.bucketname}/{self.keyname} already exists"+ ) from error+ raise+ elif extra_args:self.s3_client.upload_fileobj(Bucket=self.bucketname,Key=self.keyname,Fileobj=file,- ExtraArgs={"ACL": self.acl},+ ExtraArgs=extra_args,)else:self.s3_client.upload_fileobj(@@ -342,6 +398,8 @@ class S3FeedStorage(BlockingFeedStorage):class GCSFeedStorage(BlockingFeedStorage):""":ref:`GCS <topics-feed-storage-gcs>` storage backend."""+ supported_modes: frozenset[str] = _FEED_MODES+def __init__(self,uri: str,@@ -358,7 +416,8 @@ class GCSFeedStorage(BlockingFeedStorage):assert u.hostnameself.bucket_name: str = u.hostnameself.blob_name: str = u.path[1:] # remove first "/"- self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)++ self._mode: str = _get_mode(self, feed_options, "overwrite")@classmethoddef from_crawler(@@ -377,18 +436,35 @@ class GCSFeedStorage(BlockingFeedStorage):)def _store_in_thread(self, file: IO[bytes]) -> None:+ from google.api_core.exceptions import PreconditionFailed # noqa: PLC0415+ from google.cloud.storage import Client # noqa: PLC0415+file.seek(0)try:- from google.cloud.storage import Client # noqa: PLC0415-client = Client(project=self.project_id)bucket = client.bucket(self.bucket_name)- kwargs = {} if self.timeout is None else {"timeout": self.timeout}- blob = None if self.overwrite else bucket.get_blob(self.blob_name, **kwargs)+ kwargs: dict[str, Any] = (+ {} if self.timeout is None else {"timeout": self.timeout}+ )+ blob = (+ bucket.get_blob(self.blob_name, **kwargs)+ if self._mode == "append"+ else None+ )if blob is None:- bucket.blob(self.blob_name).upload_from_file(- file, predefined_acl=self.acl, **kwargs+ upload_kwargs = (+ {**kwargs, "if_generation_match": 0}+ if self._mode == "create"+ else kwargs)+ try:+ bucket.blob(self.blob_name).upload_from_file(+ file, predefined_acl=self.acl, **upload_kwargs+ )+ except PreconditionFailed as error:+ raise FileExistsError(+ f"gs://{self.bucket_name}/{self.blob_name} already exists"+ ) from errorreturn# Appending uploads the new data as a separate object and composes# it with the existing one, leaving the data already stored@@ -410,6 +486,8 @@ class FTPFeedStorage(BlockingFeedStorage):""":ref:`FTP <feed-storage-ftp>` storage backend, which also handles:ref:`FTPS <feed-storage-ftps>` when *uri* uses the ``ftps`` scheme."""+ supported_modes: frozenset[str] = _FEED_MODES+def __init__(self,uri: str,@@ -429,7 +507,16 @@ class FTPFeedStorage(BlockingFeedStorage):self.tls: bool = u.scheme == "ftps"self.use_active_mode: bool = use_active_modeself.timeout: float | None = timeout- self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)+ self._mode: str = _get_mode(self, feed_options, "overwrite")++ @property+ def overwrite(self) -> bool:+ warnings.warn(+ "FTPFeedStorage.overwrite is deprecated, use the mode feed option instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return self._mode != "append"@classmethoddef from_crawler(@@ -455,7 +542,7 @@ class FTPFeedStorage(BlockingFeedStorage):username=self.username,password=self.password,use_active_mode=self.use_active_mode,- overwrite=self.overwrite,+ mode=self._mode,tls=self.tls,timeout=self.timeout,)@@ -495,6 +582,7 @@ class FeedSlot:self.crawler: Crawler = crawler# flagsself.itemcount: int = 0+ self._skipped: bool = Falseself._exporting: bool = Falseself._fileloaded: bool = False@@ -607,6 +695,19 @@ class FeedExporter:self.exporters: dict[str, type[BaseItemExporter]] = self._load_components("FEED_EXPORTERS")+ if any(+ feed_options.get("mode") is None for feed_options in self.feeds.values()+ ):+ warnings.warn(+ "The default value of the FEED_MODE setting will change from "+ "None to 'create' in a future Scrapy version, i.e. Scrapy will "+ "stop writing feeds whose target already exists. Explicitly "+ "set FEED_MODE, or the mode feed option of every feed, to "+ "silence this warning.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+for uri, feed_options in self.feeds.items():if not self._storage_supported(uri, feed_options):raise NotConfigured@@ -638,6 +739,22 @@ class FeedExporter:))+ def _skip(self, slot: FeedSlot) -> None:+ """Report that the target of *slot* already exists, and mark *slot* so+ that nothing is written to it."""+ slot._skipped = True+ logger.error(+ f"Not writing {slot.uri} because it already exists and the feed "+ "mode is 'create'; its items are lost. To write it instead, remove "+ "the target, or set the mode feed option or the FEED_MODE setting "+ "to 'overwrite' or 'append' (-O implies 'overwrite').",+ extra={"spider": slot.spider},+ )+ assert self.crawler.stats+ self.crawler.stats.inc_value(+ f"feedexport/conflicts/{type(slot.storage).__name__}"+ )+async def close_spider(self, spider: Spider) -> None:for slot in self.slots:self._schedule_slot_close(slot, spider)@@ -686,6 +803,8 @@ class FeedExporter:return slot_.fileasync def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:+ if slot._skipped:+ returnif slot.itemcount:# Normal case@@ -705,6 +824,9 @@ class FeedExporter:)try:await ensure_awaitable(slot.storage.store(self._get_file(slot)))+ except FileExistsError:+ self._skip(slot)+ self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")except Exception:logger.exception("Error storing %s",@@ -771,13 +893,21 @@ class FeedExporter:processor = self.processors[slot.uri_template]for exported_item in processor(item) if processor else (item,):- await slot.start_exporting()- assert slot.exporter- try:- await ensure_awaitable(slot.exporter.export_item(exported_item))- except Exception as e:- e.add_note(f"Item: {exported_item!r}")- raise+ if not slot._skipped:+ try:+ await slot.start_exporting()+ except FileExistsError:+ self._skip(slot)+ if not slot._skipped:+ assert slot.exporter+ try:+ await ensure_awaitable(slot.exporter.export_item(exported_item))+ except Exception as e:+ e.add_note(f"Item: {exported_item!r}")+ raise+ # Skipped items are counted, so that the following files of+ # this feed still cover the same items as those of any other+ # feed.slot.itemcount += 1# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old oneif (@@ -843,7 +973,8 @@ class FeedExporter:scheme = urlparse(uri).schemeif scheme in self.storages or PureWindowsPath(uri).drive:try:- self._get_storage(uri, feed_options)+ storage = self._get_storage(uri, feed_options)+ self._check_storage_mode(uri, storage, feed_options)return Trueexcept NotConfigured as e:logger.error(@@ -854,6 +985,23 @@ class FeedExporter:logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})return False+ @staticmethod+ def _check_storage_mode(+ uri: str, storage: FeedStorageProtocol, feed_options: dict[str, Any]+ ) -> None:+ mode = feed_options.get("mode")+ if mode is None:+ # Every storage keeps its own historical mode.+ return+ if getattr(storage, "supported_modes", None) is None:+ logger.warning(+ f"{type(storage).__name__} does not declare which feed modes it "+ f"supports, so the mode feed option of the {uri} feed ({mode!r}) "+ "may be ignored."+ )+ return+ _check_mode(mode, storage, uri)+def _get_storage(self, uri: str, feed_options: dict[str, Any]) -> FeedStorageProtocol:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 62a32acf4..18787bd0f 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -94,6 +94,7 @@ __all__ = ["FEED_EXPORT_FIELDS","FEED_EXPORT_INDENT","FEED_FORMAT",+ "FEED_MODE","FEED_STORAGES","FEED_STORAGES_BASE","FEED_STORAGE_FTP_ACTIVE",@@ -395,6 +396,7 @@ FEED_EXPORTERS_BASE = {"pickle": "scrapy.exporters.PickleItemExporter",}FEED_FORMAT = "jsonlines"+FEED_MODE = NoneFEED_STORE_EMPTY = TrueFEED_STORAGES: dict[str, str] = {}FEED_STORAGES_BASE = {diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmplindex be874b3df..0861ab166 100644--- a/scrapy/templates/project/module/settings.py.tmpl+++ b/scrapy/templates/project/module/settings.py.tmpl@@ -86,3 +86,4 @@ DOWNLOAD_DELAY = 1# Set settings whose default value is deprecated to a future-proof valueDOWNLOADER_MIDDLEWARE_RESPONSE_EXCEPTIONS = TrueFEED_EXPORT_ENCODING = "utf-8"+FEED_MODE = "create"diff --git a/scrapy/utils/_ftp.py b/scrapy/utils/_ftp.pyindex 5d7fce08a..540b8b1d0 100644--- a/scrapy/utils/_ftp.py+++ b/scrapy/utils/_ftp.py@@ -1,3 +1,5 @@+from __future__ import annotations+import posixpathfrom contextlib import closingfrom ftplib import FTP, FTP_TLS, error_perm@@ -20,6 +22,16 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:ftp.cwd(path)+def _ftp_file_exists_in_cwd(ftp: FTP, filename: str) -> bool:+ try:+ ftp.voidcmd("TYPE I")+ return ftp.size(filename) is not None+ except error_perm:+ # The file does not exist, or the server does not support the SIZE+ # command, or it does not allow us to use it.+ return False++def ftp_store_file(*,path: str,@@ -30,18 +42,27 @@ def ftp_store_file(password: str,use_active_mode: bool = False,overwrite: bool = True,+ mode: str | None = None,tls: bool = False,timeout: float | None = None,) -> None:"""Opens a FTP connection with passed credentials, sets current directoryto the directory extracted from given path, then uploads the file to server.+ *mode* may be ``"append"``, ``"create"`` or ``"overwrite"``. It takes+ precedence over *overwrite*, which only remains for backward compatibility.++ ``"create"`` is best-effort: FTP has no atomic create-if-absent command, so+ the file is checked for existence right before it is written.+If *tls* is ``True``, the connection is secured with TLS (FTPS), and thecertificate of the server is verified.*timeout* is the number of seconds that socket operations wait beforegiving up. If ``None``, they wait indefinitely."""+ if mode is None:+ mode = "overwrite" if overwrite else "append"ftp = FTP_TLS(context=create_default_context()) if tls else FTP()with ftp, closing(file):if timeout is None:@@ -56,5 +77,7 @@ def ftp_store_file(file.seek(0)dirname, filename = posixpath.split(path)ftp_makedirs_cwd(ftp, dirname)- command = "STOR" if overwrite else "APPE"+ if mode == "create" and _ftp_file_exists_in_cwd(ftp, filename):+ raise FileExistsError(f"{path} already exists")+ command = "APPE" if mode == "append" else "STOR"ftp.storbinary(f"{command} {filename}", file)diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex ecfdd0394..75b8df34d 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -3,12 +3,13 @@ from __future__ import annotationsimport numbersimport osimport sys+import warningsfrom configparser import ConfigParserfrom operator import itemgetterfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, cast-from scrapy.exceptions import UsageError+from scrapy.exceptions import ScrapyDeprecationWarning, UsageErrorfrom scrapy.settings import BaseSettingsfrom scrapy.utils.deprecate import update_classpathfrom scrapy.utils.python import without_none_values@@ -130,6 +131,25 @@ def feed_complete_default_values_from_settings(feed: dict[str, Any], settings: BaseSettings, uri: str | None = None) -> dict[str, Any]:out = feed.copy()+ if "overwrite" in out:+ warnings.warn(+ "The overwrite feed option is deprecated, use the mode feed option"+ " instead: mode='overwrite' instead of overwrite=True, and"+ " mode='append' instead of overwrite=False.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ if out.get("mode") is not None:+ raise ValueError(+ "The overwrite and mode feed options are mutually exclusive,"+ " please set only the mode feed option."+ )+ out["mode"] = "overwrite" if out.pop("overwrite") else "append"+ out.setdefault("mode", settings["FEED_MODE"])+ if out["mode"] in {"append", "overwrite"}:+ # Kept for feed storages that were written before the mode feed option+ # existed and hence only look for the overwrite feed option.+ out["overwrite"] = out["mode"] == "overwrite"if uri is not None:out.setdefault("format", Path(uri).suffix.removeprefix("."))out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))@@ -192,7 +212,9 @@ def feed_process_params_from_cli(check_valid_format(feed_format)result[feed_uri] = {"format": feed_format}if overwrite:- result[feed_uri]["overwrite"] = True+ # -O unambiguously means overwriting, so it ignores FEED_MODE. -o+ # does not, because appending is not supported by every storage.+ result[feed_uri]["mode"] = "overwrite"# FEEDS setting should take precedence over the matching CLI optionsresult.update(settings.getdict("FEEDS"))diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex d3b69e3ed..bb26928e5 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -69,6 +69,7 @@ def get_crawler(# When needed, useful settings can be added here, e.g. ones that prevent# deprecation warnings.settings: dict[str, Any] = {+ "FEED_MODE": "overwrite","REMOTE_CONTROL_ENABLED": False,"TELNETCONSOLE_ENABLED": False,**get_reactor_settings(),diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.pyindex 050371bd0..1ba6c9fbb 100644--- a/tests/test_command_crawl.py+++ b/tests/test_command_crawl.py@@ -94,7 +94,7 @@ class MySpider(scrapy.Spider):args = ["-O", "example.json"]log = self.get_log(spider_code, proj_path, args=args)assert (- '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}'+ '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "mode": "overwrite"}}'in log)with j.open(encoding="utf-8") as f2:diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex c2fccd54f..4d809c4f3 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -326,7 +326,7 @@ class MySpider(scrapy.Spider):args = ["-O", "example.json"]log = self.get_log(tmp_path, spider_code, args=args)assert (- '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}'+ '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "mode": "overwrite"}}'in log)with (tmp_path / "example.json").open(encoding="utf-8") as f2:diff --git a/tests/test_feedexport_mode.py b/tests/test_feedexport_mode.pynew file mode 100644index 000000000..a1d55ce60--- /dev/null+++ b/tests/test_feedexport_mode.py@@ -0,0 +1,316 @@+from __future__ import annotations++import logging+from pathlib import Path+from typing import IO, TYPE_CHECKING, Any++import pytest++from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.extensions.feedexport import BlockingFeedStorage, FileFeedStorage+from scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_test+from tests.utils.feedexport import crawl_items, path_to_url, printf_escape++if TYPE_CHECKING:+ from scrapy.crawler import Crawler+ from tests.mockserver.http import MockServer+++class DelayedFileStorage(BlockingFeedStorage):+ """Feed storage that, like the S3 or GCS ones, can only detect a conflict+ when the feed is delivered, at the end of the crawl or of a batch."""++ supported_modes = frozenset({"append", "create", "overwrite"})++ def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):+ self.path = Path(uri.split("://", 1)[1])+ self.mode = (feed_options or {}).get("mode")++ def _store_in_thread(self, file: IO[bytes]) -> None:+ file.seek(0)+ try:+ if self.mode == "create" and self.path.exists():+ raise FileExistsError(str(self.path))+ self.path.write_bytes(file.read())+ finally:+ file.close()+++class CreateOnlyFileStorage(FileFeedStorage):+ supported_modes = frozenset({"create"})+++class LegacyFileStorage(FileFeedStorage):+ """Feed storage that predates the mode feed option."""++ supported_modes = None # type: ignore[assignment]+++class TestFeedMode:+ """End-to-end tests of the mode feed option and the FEED_MODE setting."""++ @pytest.fixture(autouse=True)+ def _setup(self, mockserver: MockServer, tmp_path: Path) -> None:+ self.mockserver = mockserver+ self.temp_dir = tmp_path++ def _path(self, content: bytes | None = None) -> Path:+ path = self.temp_dir / "items.jl"+ if content is not None:+ path.write_bytes(content)+ return path++ async def _crawl(+ self,+ path: Path,+ mode: str | None = None,+ settings: dict[str, Any] | None = None,+ item_count: int = 1,+ scheme: str | None = None,+ ) -> Crawler:+ feed_options: dict[str, Any] = {"format": "jl"}+ if mode is not None:+ feed_options["mode"] = mode+ if scheme is not None:+ # as_posix() keeps the URI valid on Windows, where paths have a+ # drive and backslashes.+ uri = f"{scheme}://{path.as_posix()}"+ elif "%(batch_id)d" in str(path):+ # A batch URI template must keep its %(batch_id)d placeholder, so it+ # is used as a path, which is neither quoted nor printf-escaped.+ uri = str(path)+ else:+ uri = printf_escape(path_to_url(path))+ return await crawl_items(+ self.mockserver,+ [{"foo": "bar"}] * item_count,+ {"FEEDS": {uri: feed_options}, **(settings or {})},+ )++ @coroutine_test+ async def test_create(self) -> None:+ path = self._path()+ await self._crawl(path, "create")+ assert path.read_bytes() == b'{"foo": "bar"}\n'++ @coroutine_test+ async def test_create_existing(self, caplog: pytest.LogCaptureFixture) -> None:+ path = self._path(b"old content")+ with caplog.at_level(logging.ERROR):+ crawler = await self._crawl(path, "create", item_count=3)+ assert path.read_bytes() == b"old content"+ # Reported once, not once per item.+ assert caplog.text.count("because it already exists") == 1+ assert crawler.stats+ stats = crawler.stats.get_stats()+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 1+ assert "feedexport/success_count/FileFeedStorage" not in stats++ @coroutine_test+ async def test_create_existing_multiple_feeds(self) -> None:+ """A conflicting target only affects its own feed: the crawl runs and+ the other feeds are written."""+ existing = self._path(b"old content")+ missing = self.temp_dir / "missing.jl"+ crawler = await crawl_items(+ self.mockserver,+ [{"foo": "bar"}],+ {+ "FEEDS": {+ printf_escape(path_to_url(existing)): {"format": "jl"},+ printf_escape(path_to_url(missing)): {"format": "jl"},+ },+ "FEED_MODE": "create",+ },+ )+ assert existing.read_bytes() == b"old content"+ assert missing.read_bytes() == b'{"foo": "bar"}\n'+ assert crawler.stats+ stats = crawler.stats.get_stats()+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 1+ assert stats.get("feedexport/success_count/FileFeedStorage") == 1+ assert stats.get("finish_reason") == "finished"++ @coroutine_test+ async def test_create_existing_all_feeds(self, caplog: pytest.LogCaptureFixture):+ """A crawl that cannot write any feed still runs, like any other crawl+ whose feeds cannot be written."""+ existing1 = self._path(b"old content")+ existing2 = self.temp_dir / "items2.jl"+ existing2.write_bytes(b"old content")+ with caplog.at_level(logging.ERROR):+ crawler = await crawl_items(+ self.mockserver,+ [{"foo": "bar"}],+ {+ "FEEDS": {+ printf_escape(path_to_url(existing1)): {"format": "jl"},+ printf_escape(path_to_url(existing2)): {"format": "jl"},+ },+ "FEED_MODE": "create",+ },+ )+ assert existing1.read_bytes() == b"old content"+ assert existing2.read_bytes() == b"old content"+ assert caplog.text.count("because it already exists") == 2+ assert crawler.stats+ stats = crawler.stats.get_stats()+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 2+ assert stats.get("finish_reason") == "finished"+ assert "feedexport/success_count/FileFeedStorage" not in stats++ @coroutine_test+ async def test_overwrite_existing(self) -> None:+ path = self._path(b"old content")+ await self._crawl(path, "overwrite")+ assert path.read_bytes() == b'{"foo": "bar"}\n'++ @coroutine_test+ async def test_append_existing(self) -> None:+ path = self._path(b"old content\n")+ await self._crawl(path, "append")+ assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'++ @coroutine_test+ async def test_unset_mode(self) -> None:+ path = self._path()+ with pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"):+ await self._crawl(path, settings={"FEED_MODE": None})+ assert path.read_bytes() == b'{"foo": "bar"}\n'++ @coroutine_test+ async def test_unset_mode_existing_target(self) -> None:+ path = self._path(b"old content\n")+ with pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"):+ await self._crawl(path, settings={"FEED_MODE": None})+ # The legacy behavior is kept.+ assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'++ @coroutine_test+ async def test_explicit_mode_existing_target(self, recwarn) -> None:+ path = self._path(b"old content\n")+ await self._crawl(path, "append")+ assert not [+ warning+ for warning in recwarn+ if issubclass(warning.category, ScrapyDeprecationWarning)+ and "FEED_MODE" in str(warning.message)+ ]++ @coroutine_test+ async def test_create_existing_later_batch(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """When the target of a later batch exists, that batch is skipped, and+ the following ones are still written."""+ (self.temp_dir / "items-2.jl").write_bytes(b"old content")+ with caplog.at_level(logging.ERROR):+ crawler = await self._crawl(+ self.temp_dir / "items-%(batch_id)d.jl",+ "create",+ {"FEED_EXPORT_BATCH_ITEM_COUNT": 1},+ item_count=3,+ )+ assert (self.temp_dir / "items-1.jl").read_bytes() == b'{"foo": "bar"}\n'+ assert (self.temp_dir / "items-2.jl").read_bytes() == b"old content"+ # The skipped items are counted, so the following batch covers the same+ # items that it would have covered otherwise.+ assert (self.temp_dir / "items-3.jl").read_bytes() == b'{"foo": "bar"}\n'+ assert not (self.temp_dir / "items-4.jl").exists()+ # A single error, instead of one per item after the conflicting one.+ assert caplog.text.count("because it already exists") == 1+ assert crawler.stats+ stats = crawler.stats.get_stats()+ assert stats.get("feedexport/conflicts/FileFeedStorage") == 1+ assert stats.get("feedexport/success_count/FileFeedStorage") == 2+ assert stats.get("finish_reason") == "finished"++ @coroutine_test+ async def test_create_existing_later_batch_delayed(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """When the feed is only delivered at the end of each batch, the+ conflict is detected then, and only that batch is lost."""+ (self.temp_dir / "items-2.jl").write_bytes(b"old content")+ with caplog.at_level(logging.ERROR):+ crawler = await self._crawl(+ self.temp_dir / "items-%(batch_id)d.jl",+ "create",+ {+ "FEED_EXPORT_BATCH_ITEM_COUNT": 1,+ "FEED_STORAGES": {+ "delayed": "tests.test_feedexport_mode.DelayedFileStorage"+ },+ },+ item_count=3,+ scheme="delayed",+ )+ assert (self.temp_dir / "items-1.jl").read_bytes() == b'{"foo": "bar"}\n'+ assert (self.temp_dir / "items-2.jl").read_bytes() == b"old content"+ assert (self.temp_dir / "items-3.jl").read_bytes() == b'{"foo": "bar"}\n'+ assert caplog.text.count("because it already exists") == 1+ assert crawler.stats+ stats = crawler.stats.get_stats()+ assert stats.get("feedexport/conflicts/DelayedFileStorage") == 1+ assert stats.get("feedexport/failed_count/DelayedFileStorage") == 1+ assert stats.get("feedexport/success_count/DelayedFileStorage") == 2++ @coroutine_test+ async def test_feed_mode_setting(self) -> None:+ path = self._path(b"old content")+ await self._crawl(path, settings={"FEED_MODE": "overwrite"})+ assert path.read_bytes() == b'{"foo": "bar"}\n'++ @coroutine_test+ async def test_mode_overrides_feed_mode_setting(self) -> None:+ path = self._path(b"old content\n")+ await self._crawl(path, "append", {"FEED_MODE": "overwrite"})+ assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'+++class TestFeedModeInit:+ def test_invalid_mode(self):+ """An invalid mode prevents the crawl from running."""+ settings = {+ "FEEDS": {"file:///tmp/items.json": {"format": "json", "mode": "x"}}+ }+ with pytest.raises(ValueError, match="Invalid feed mode: 'x'"):+ get_crawler(settings_dict=settings)++ def test_unsupported_mode(self):+ settings = {+ "FEEDS": {"file:///tmp/items.json": {"format": "json"}},+ "FEED_STORAGES": {+ "file": "tests.test_feedexport_mode.CreateOnlyFileStorage"+ },+ "FEED_MODE": "append",+ }+ with pytest.raises(+ ValueError,+ match="CreateOnlyFileStorage does not support the 'append' feed mode",+ ):+ get_crawler(settings_dict=settings)++ def test_undeclared_mode_support(self, caplog: pytest.LogCaptureFixture):+ settings = {+ "FEEDS": {"file:///tmp/items.json": {"format": "json"}},+ "FEED_STORAGES": {"file": "tests.test_feedexport_mode.LegacyFileStorage"},+ "FEED_MODE": "create",+ }+ with caplog.at_level(logging.WARNING):+ get_crawler(settings_dict=settings)+ assert "does not declare which feed modes it supports" in caplog.text++ def test_undeclared_mode_support_unset_mode(self, caplog: pytest.LogCaptureFixture):+ settings = {+ "FEEDS": {"file:///tmp/items.json": {"format": "json"}},+ "FEED_STORAGES": {"file": "tests.test_feedexport_mode.LegacyFileStorage"},+ "FEED_MODE": None,+ }+ with (+ caplog.at_level(logging.WARNING),+ pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"),+ ):+ get_crawler(settings_dict=settings)+ assert "does not declare which feed modes it supports" not in caplog.textdiff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex cc095ff75..78e00992c 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -15,7 +15,7 @@ import pytestfrom w3lib.url import path_to_file_uriimport scrapy-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.extensions.feedexport import (BlockingFeedStorage,FileFeedStorage,@@ -69,11 +69,43 @@ class TestFileFeedStorage:def test_overwrite(self, tmp_path):path = tmp_path / "file.txt"- self._store(path, {"overwrite": True})+ self._store(path, {"mode": "overwrite"})self._assert_stores(- FileFeedStorage(str(path), feed_options={"overwrite": True}), path+ FileFeedStorage(str(path), feed_options={"mode": "overwrite"}), path)+ def test_create(self, tmp_path):+ path = tmp_path / "file.txt"+ self._assert_stores(+ FileFeedStorage(str(path), feed_options={"mode": "create"}), path+ )++ def test_create_existing(self, tmp_path):+ path = tmp_path / "file.txt"+ path.write_bytes(b"content")+ storage = FileFeedStorage(str(path), feed_options={"mode": "create"})+ with pytest.raises(FileExistsError):+ storage.open(scrapy.Spider("default"))+ assert path.read_bytes() == b"content"++ @pytest.mark.parametrize(+ ("feed_options", "expected_write_mode"),+ [+ (None, "ab"),+ ({}, "ab"),+ ({"mode": "create"}, "xb"),+ ({"overwrite": True}, "wb"),+ ({"overwrite": False}, "ab"),+ ],+ )+ def test_mode(self, tmp_path, feed_options, expected_write_mode):+ storage = FileFeedStorage(str(tmp_path / "file.txt"), feed_options=feed_options)+ assert storage.write_mode == expected_write_mode++ def test_invalid_mode(self, tmp_path):+ with pytest.raises(ValueError, match="Invalid feed mode: 'x'"):+ FileFeedStorage(str(tmp_path / "file.txt"), feed_options={"mode": "x"})+@staticmethoddef _assert_stores(storage: FileFeedStorage, path: Path, expected_content: bytes = b"content"@@ -122,19 +154,20 @@ class TestFTPFeedStorage:file.write(content)await maybe_deferred_to_future(storage.store(file))- def _assert_stored(self, path: Path, content: bytes) -> None:+ def _assert_stored(self, path: Path, content: bytes, unlink: bool = True) -> None:assert path.exists()try:assert path.read_bytes() == contentfinally:- path.unlink()+ if unlink:+ path.unlink()@coroutine_testasync def test_append(self):with MockFTPServer() as ftp_server:filename = "file"url = ftp_server.url(filename)- feed_options = {"overwrite": False}+ feed_options = {"mode": "append"}await self._store(url, b"foo", feed_options=feed_options)await self._store(url, b"bar", feed_options=feed_options)self._assert_stored(ftp_server.path / filename, b"foobar")@@ -148,13 +181,25 @@ class TestFTPFeedStorage:await self._store(url, b"bar")self._assert_stored(ftp_server.path / filename, b"bar")+ @coroutine_test+ async def test_create(self):+ with MockFTPServer() as ftp_server:+ filename = "file"+ url = ftp_server.url(filename)+ feed_options = {"mode": "create"}+ await self._store(url, b"foo", feed_options=feed_options)+ self._assert_stored(ftp_server.path / filename, b"foo", unlink=False)+ with pytest.raises(FileExistsError):+ await self._store(url, b"bar", feed_options=feed_options)+ self._assert_stored(ftp_server.path / filename, b"foo")+@coroutine_testasync def test_append_active_mode(self):with MockFTPServer() as ftp_server:settings = {"FEED_STORAGE_FTP_ACTIVE": True}filename = "file"url = ftp_server.url(filename)- feed_options = {"overwrite": False}+ feed_options = {"mode": "append"}await self._store(url, b"foo", feed_options=feed_options, settings=settings)await self._store(url, b"bar", feed_options=feed_options, settings=settings)self._assert_stored(ftp_server.path / filename, b"foobar")@@ -169,6 +214,13 @@ class TestFTPFeedStorage:await self._store(url, b"bar", settings=settings)self._assert_stored(ftp_server.path / filename, b"bar")+ def test_overwrite_deprecated(self):+ storage = FTPFeedStorage.from_crawler(get_crawler(), "ftp://localhost/file")+ with pytest.warns(+ ScrapyDeprecationWarning, match="FTPFeedStorage.overwrite is deprecated"+ ):+ assert storage.overwrite is True+@coroutine_testasync def test_upload_timeout(self):crawler = get_crawler(settings_dict={"UPLOAD_TIMEOUT": 300})@@ -521,21 +573,69 @@ class TestS3FeedStorage:acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]assert acl == "custom-acl"- def test_overwrite_default(self, caplog: pytest.LogCaptureFixture) -> None:- S3FeedStorage(- "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"- )- assert "S3 does not support appending to files" not in caplog.text+ def test_mode_append(self) -> None:+ with pytest.raises(+ ValueError, match="S3FeedStorage does not support the 'append' feed mode"+ ):+ S3FeedStorage(+ "s3://mybucket/export.csv",+ "access_key",+ "secret_key",+ "custom-acl",+ feed_options={"mode": "append"},+ )- def test_overwrite_false(self, caplog: pytest.LogCaptureFixture) -> None:- S3FeedStorage(+ @coroutine_test+ async def test_store_create(self) -> None:+ from botocore.stub import ANY, Stubber # noqa: PLC0415++ storage = S3FeedStorage("s3://mybucket/export.csv","access_key","secret_key","custom-acl",- feed_options={"overwrite": False},+ feed_options={"mode": "create"},)- assert "S3 does not support appending to files" in caplog.text+ with Stubber(storage.s3_client) as stub:+ stub.add_response(+ "put_object",+ {},+ expected_params={+ "Bucket": "mybucket",+ "Key": "export.csv",+ "Body": ANY,+ "IfNoneMatch": "*",+ "ACL": "custom-acl",+ },+ )+ stored = storage.store(BytesIO(b"test file"))+ assert stored is not None+ await maybe_deferred_to_future(stored)+ stub.assert_no_pending_responses()++ @pytest.mark.parametrize(+ ("error_code", "exception"),+ [("PreconditionFailed", FileExistsError), ("AccessDenied", None)],+ )+ @coroutine_test+ async def test_store_create_error(+ self, error_code: str, exception: type[Exception] | None+ ) -> None:+ from botocore.exceptions import ClientError # noqa: PLC0415+ from botocore.stub import Stubber # noqa: PLC0415++ storage = S3FeedStorage(+ "s3://mybucket/export.csv",+ "access_key",+ "secret_key",+ feed_options={"mode": "create"},+ )+ with Stubber(storage.s3_client) as stub:+ stub.add_client_error("put_object", service_error_code=error_code)+ stored = storage.store(BytesIO(b"test file"))+ assert stored is not None+ with pytest.raises(exception or ClientError):+ await maybe_deferred_to_future(stored)class TestGCSFeedStorage:@@ -641,6 +741,46 @@ class TestGCSFeedStorage:blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)f.close.assert_called_once_with()+ @coroutine_test+ async def test_store_create(self):+ pytest.importorskip("google.cloud.storage")++ (client_mock, _, blob_mock) = mock_google_cloud_storage()+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock+ f = mock.Mock()+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ "publicRead",+ feed_options={"mode": "create"},+ )+ await maybe_deferred_to_future(storage.store(f))+ blob_mock.upload_from_file.assert_called_once_with(+ f, predefined_acl="publicRead", if_generation_match=0+ )+ f.close.assert_called_once_with()++ @coroutine_test+ async def test_store_create_existing(self):+ pytest.importorskip("google.cloud.storage")+ from google.api_core.exceptions import PreconditionFailed # noqa: PLC0415++ (client_mock, _, blob_mock) = mock_google_cloud_storage()+ blob_mock.upload_from_file.side_effect = PreconditionFailed("exists")+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock+ f = mock.Mock()+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ None,+ feed_options={"mode": "create"},+ )+ with pytest.raises(FileExistsError):+ await maybe_deferred_to_future(storage.store(f))+ f.close.assert_called_once_with()+@coroutine_testasync def test_store_append_new_blob(self) -> None:pytest.importorskip("google.cloud.storage")@@ -655,7 +795,7 @@ class TestGCSFeedStorage:"gs://mybucket/export.csv","myproject-123","publicRead",- feed_options={"overwrite": False},+ feed_options={"mode": "append"},)await maybe_deferred_to_future(storage.store(f))@@ -682,7 +822,7 @@ class TestGCSFeedStorage:"gs://mybucket/export.csv","myproject-123","publicRead",- feed_options={"overwrite": False},+ feed_options={"mode": "append"},)await maybe_deferred_to_future(storage.store(f))@@ -746,7 +886,7 @@ class TestGCSFeedStorage:"gs://mybucket/export.csv","myproject-123",None,- feed_options={"overwrite": False},+ feed_options={"mode": "append"},)await maybe_deferred_to_future(storage.store(mock.Mock()))@@ -769,7 +909,7 @@ class TestGCSFeedStorage:"gs://mybucket/export.csv","myproject-123","publicRead",- feed_options={"overwrite": False},+ feed_options={"mode": "append"},)with pytest.raises(OSError, match="Compose failed"):await maybe_deferred_to_future(storage.store(f))@@ -796,22 +936,21 @@ class TestGCSFeedStorage:@pytest.mark.parametrize(("feed_options", "expected"),[- (None, True),- ({}, True),- ({"overwrite": True}, True),- ({"overwrite": False}, False),+ (None, "overwrite"),+ ({}, "overwrite"),+ ({"overwrite": True}, "overwrite"),+ ({"overwrite": False}, "append"),+ ({"mode": "append"}, "append"),],)- def test_overwrite(- self, feed_options: dict[str, Any] | None, expected: bool- ) -> None:+ def test_mode(self, feed_options: dict[str, Any] | None, expected: str) -> None:storage = GCSFeedStorage("gs://mybucket/export.csv","myproject-123","custom-acl",feed_options=feed_options,)- assert storage.overwrite is expected+ assert storage._mode == expectedclass TestStdoutFeedStorage:@@ -823,18 +962,15 @@ class TestStdoutFeedStorage:storage.store(file)assert out.getvalue() == b"content"- def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):- with caplog.at_level(logging.DEBUG):- StdoutFeedStorage("stdout:")- assert (- "Standard output (stdout) storage does not support overwriting"- not in caplog.text- )-- def test_overwrite_true(self, caplog: pytest.LogCaptureFixture):+ @pytest.mark.parametrize("mode", ["append", "create", "overwrite"])+ def test_mode_ignored(self, mode: str, caplog: pytest.LogCaptureFixture):+ out = BytesIO()with caplog.at_level(logging.DEBUG):- StdoutFeedStorage("stdout:", feed_options={"overwrite": True})- assert (- "Standard output (stdout) storage does not support overwriting"- in caplog.text- )+ storage = StdoutFeedStorage(+ "stdout:", _stdout=out, feed_options={"mode": mode}+ )+ file = storage.open(scrapy.Spider("default"))+ file.write(b"content")+ storage.store(file)+ assert out.getvalue() == b"content"+ assert not caplog.textdiff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 01e76a25b..e1b87043b 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Anyimport pytest-from scrapy.exceptions import UsageError+from scrapy.exceptions import ScrapyDeprecationWarning, UsageErrorfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.conf import (_job_dir,@@ -122,11 +122,60 @@ class TestFeedExportConfig:def test_feed_export_config_overwrite(self):settings = Settings()assert {- "output.json": {"format": "json", "overwrite": True}+ "output.json": {"format": "json", "mode": "overwrite"}} == feed_process_params_from_cli(settings, [], overwrite_output=["output.json"])+ def test_feed_complete_default_values_mode_from_settings(self):+ settings = Settings({"FEED_MODE": "create"})+ new_feed = feed_complete_default_values_from_settings({}, settings)+ assert new_feed["mode"] == "create"+ assert "overwrite" not in new_feed++ @pytest.mark.parametrize(+ ("mode", "overwrite"),+ [+ ("append", False),+ ("overwrite", True),+ ],+ )+ def test_feed_complete_default_values_mode_sets_overwrite(self, mode, overwrite):+ """The deprecated overwrite feed option is kept in sync for the sake of+ feed storages that predate the mode feed option."""+ settings = Settings({"FEED_MODE": mode})+ new_feed = feed_complete_default_values_from_settings({}, settings)+ assert new_feed["mode"] == mode+ assert new_feed["overwrite"] is overwrite++ @pytest.mark.parametrize(+ ("overwrite", "mode"),+ [+ (True, "overwrite"),+ (False, "append"),+ ],+ )+ def test_feed_complete_default_values_overwrite_deprecated(self, overwrite, mode):+ settings = Settings()+ with pytest.warns(+ ScrapyDeprecationWarning, match="overwrite feed option is deprecated"+ ):+ new_feed = feed_complete_default_values_from_settings(+ {"overwrite": overwrite}, settings+ )+ assert new_feed["mode"] == mode+ assert new_feed["overwrite"] is overwrite++ def test_feed_complete_default_values_overwrite_and_mode(self):+ settings = Settings()+ with (+ pytest.raises(ValueError, match="mutually exclusive"),+ pytest.warns(ScrapyDeprecationWarning),+ ):+ feed_complete_default_values_from_settings(+ {"overwrite": True, "mode": "create"}, settings+ )+def test_output_and_overwrite_output(self):with pytest.raises(UsageError):feed_process_params_from_cli(@@ -147,6 +196,7 @@ class TestFeedExportConfig:)new_feed = feed_complete_default_values_from_settings(feed, settings)assert new_feed == {+ "mode": None,"encoding": "custom encoding","fields": ["f1", "f2", "f3"],"indent": 42,@@ -172,6 +222,7 @@ class TestFeedExportConfig:)new_feed = feed_complete_default_values_from_settings(feed, settings)assert new_feed == {+ "mode": None,"encoding": "other encoding","fields": None,"indent": 42,diff --git a/tox.ini b/tox.iniindex 934c8fcd9..6ed82b836 100644--- a/tox.ini+++ b/tox.ini@@ -200,7 +200,7 @@ deps ={[min]deps}Pillow==9.2.0Twisted[http2]==21.7.0- boto3==1.20.0+ boto3==1.35.2bpython==0.7.1colorlog==6.6.0google-cloud-storage==1.30.0