Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support appending to GCS feeds via composite objects (#8173)

Adrian48bb971f3f69
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 4f91286fc..2bbb87e5c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -290,6 +290,12 @@ storage backend is: ``True``.
.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the
previous version of your data.
+Appending (``overwrite: False``) turns the feed into a `composite object`_,
+which has a CRC32C checksum but no MD5 hash.
+
+.. versionadded:: VERSION
+ Appending support.
+
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@@ -536,7 +542,7 @@ as a fallback value if that key is not provided for a specific feed definition:
- :ref:`topics-feed-storage-s3`: ``True`` (appending is not supported)
- - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported)
+ - :ref:`topics-feed-storage-gcs`: ``True``
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
@@ -821,4 +827,5 @@ source spider in the feed URI:
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
+.. _composite object: https://docs.cloud.google.com/storage/docs/composite-objects
.. _Google Cloud Storage: https://cloud.google.com/storage/
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index d3dc03c53..5752508d3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -19,6 +19,7 @@ from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast
from urllib.parse import unquote, urlparse
+from uuid import uuid4
from twisted.internet.defer import Deferred, DeferredList
from w3lib.url import file_uri_to_path
@@ -311,13 +312,7 @@ class GCSFeedStorage(BlockingFeedStorage):
assert u.hostname
self.bucket_name: str = u.hostname
self.blob_name: str = u.path[1:] # remove first "/"
-
- if feed_options and feed_options.get("overwrite", True) is False:
- logger.warning(
- "GCS does not support appending to files. To "
- "suppress this warning, remove the overwrite "
- "option from your FEEDS setting or set it to True."
- )
+ self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
@classmethod
def from_crawler(
@@ -341,8 +336,24 @@ class GCSFeedStorage(BlockingFeedStorage):
client = Client(project=self.project_id)
bucket = client.bucket(self.bucket_name)
- blob = bucket.blob(self.blob_name)
- blob.upload_from_file(file, predefined_acl=self.acl)
+ blob = None if self.overwrite else bucket.get_blob(self.blob_name)
+ if blob is None:
+ bucket.blob(self.blob_name).upload_from_file(
+ file, predefined_acl=self.acl
+ )
+ return
+ # Appending uploads the new data as a separate object and composes
+ # it with the existing one, leaving the data already stored
+ # untouched. Composition resolves its sources by name, so it always
+ # appends to the latest version of the blob.
+ part = bucket.blob(f"{self.blob_name}.{uuid4().hex}.part")
+ part.upload_from_file(file, predefined_acl=self.acl)
+ try:
+ blob.compose([blob, part], client=client)
+ finally:
+ part.delete(client=client)
+ if self.acl:
+ blob.acl.save_predefined(self.acl, client=client)
finally:
file.close()
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 46e603af7..3870ad6cc 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -29,7 +29,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
-from tests.utils.cloud import mock_google_cloud_storage
+from tests.utils.cloud import mock_google_cloud_blob, mock_google_cloud_storage
from tests.utils.decorators import coroutine_test
@@ -592,20 +592,147 @@ class TestGCSFeedStorage:
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
f.close.assert_called_once_with()
- def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):
- with caplog.at_level(logging.DEBUG):
- GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")
- assert "GCS does not support appending to files" not in caplog.text
+ @coroutine_test
+ async def test_store_append_new_blob(self) -> None:
+ pytest.importorskip("google.cloud.storage")
- def test_overwrite_false(self, caplog: pytest.LogCaptureFixture):
- with caplog.at_level(logging.DEBUG):
- GCSFeedStorage(
+ (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+ bucket_mock.get_blob.return_value = None
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+
+ f = mock.Mock()
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ "publicRead",
+ feed_options={"overwrite": False},
+ )
+ await maybe_deferred_to_future(storage.store(f))
+
+ bucket_mock.get_blob.assert_called_once_with("export.csv")
+ bucket_mock.blob.assert_called_once_with("export.csv")
+ blob_mock.upload_from_file.assert_called_once_with(
+ f, predefined_acl="publicRead"
+ )
+ blob_mock.compose.assert_not_called()
+ f.close.assert_called_once_with()
+
+ @coroutine_test
+ async def test_store_append_existing_blob(self) -> None:
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, bucket_mock, part_mock) = mock_google_cloud_storage()
+ blob_mock = mock_google_cloud_blob()
+ bucket_mock.get_blob.return_value = blob_mock
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+
+ f = mock.Mock()
+ storage = GCSFeedStorage(
"gs://mybucket/export.csv",
"myproject-123",
- "custom-acl",
+ "publicRead",
feed_options={"overwrite": False},
)
- assert "GCS does not support appending to files" in caplog.text
+ await maybe_deferred_to_future(storage.store(f))
+
+ part_name = bucket_mock.blob.call_args.args[0]
+ assert part_name.startswith("export.csv.")
+ assert part_name.endswith(".part")
+ part_mock.upload_from_file.assert_called_once_with(
+ f, predefined_acl="publicRead"
+ )
+ blob_mock.compose.assert_called_once_with(
+ [blob_mock, part_mock], client=client_mock
+ )
+ part_mock.delete.assert_called_once_with(client=client_mock)
+ blob_mock.acl.save_predefined.assert_called_once_with(
+ "publicRead", client=client_mock
+ )
+ blob_mock.upload_from_file.assert_not_called()
+ f.close.assert_called_once_with()
+
+ @coroutine_test
+ async def test_store_append_without_acl(self) -> None:
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, bucket_mock, _part_mock) = mock_google_cloud_storage()
+ blob_mock = mock_google_cloud_blob()
+ bucket_mock.get_blob.return_value = blob_mock
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ None,
+ feed_options={"overwrite": False},
+ )
+ await maybe_deferred_to_future(storage.store(mock.Mock()))
+
+ blob_mock.compose.assert_called_once()
+ blob_mock.acl.save_predefined.assert_not_called()
+
+ @coroutine_test
+ async def test_store_append_deletes_part_on_compose_error(self) -> None:
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, bucket_mock, part_mock) = mock_google_cloud_storage()
+ blob_mock = mock_google_cloud_blob()
+ blob_mock.compose.side_effect = OSError("Compose failed")
+ bucket_mock.get_blob.return_value = blob_mock
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+
+ f = mock.Mock()
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ "publicRead",
+ feed_options={"overwrite": False},
+ )
+ with pytest.raises(OSError, match="Compose failed"):
+ await maybe_deferred_to_future(storage.store(f))
+
+ part_mock.delete.assert_called_once_with(client=client_mock)
+ f.close.assert_called_once_with()
+
+ @coroutine_test
+ async def test_store_overwrite_ignores_existing_blob(self) -> None:
+ pytest.importorskip("google.cloud.storage")
+
+ (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+ with mock.patch("google.cloud.storage.Client") as m:
+ m.return_value = client_mock
+
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv", "myproject-123", "publicRead"
+ )
+ await maybe_deferred_to_future(storage.store(mock.Mock()))
+
+ bucket_mock.get_blob.assert_not_called()
+ blob_mock.compose.assert_not_called()
+
+ @pytest.mark.parametrize(
+ ("feed_options", "expected"),
+ [
+ (None, True),
+ ({}, True),
+ ({"overwrite": True}, True),
+ ({"overwrite": False}, False),
+ ],
+ )
+ def test_overwrite(
+ self, feed_options: dict[str, Any] | None, expected: bool
+ ) -> None:
+ storage = GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ "custom-acl",
+ feed_options=feed_options,
+ )
+ assert storage.overwrite is expected
class TestStdoutFeedStorage:
diff --git a/tests/utils/cloud.py b/tests/utils/cloud.py
index 4e253fbdc..08beb66b5 100644
--- a/tests/utils/cloud.py
+++ b/tests/utils/cloud.py
@@ -8,14 +8,21 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.
"""
- from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415
+ from google.cloud.storage import Bucket, Client # noqa: PLC0415
client_mock = mock.create_autospec(Client)
bucket_mock = mock.create_autospec(Bucket)
client_mock.bucket.return_value = bucket_mock
- blob_mock = mock.create_autospec(Blob)
+ blob_mock = mock_google_cloud_blob()
bucket_mock.blob.return_value = blob_mock
return (client_mock, bucket_mock, blob_mock)
+
+
+def mock_google_cloud_blob() -> Any:
+ """Creates an autospec mock for the google-cloud-storage Blob class."""
+ from google.cloud.storage import Blob # noqa: PLC0415
+
+ return mock.create_autospec(Blob)