Scrapy, a fast high-level web crawling & scraping framework for Python.
Support appending to GCS feeds via composite objects (#8173)
Adrian48bb971f3f69
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 4f91286fc..2bbb87e5c 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -290,6 +290,12 @@ storage backend is: ``True``... caution:: The value ``True`` in ``overwrite`` will cause you to lose theprevious version of your data.+Appending (``overwrite: False``) turns the feed into a `composite object`_,+which has a CRC32C checksum but no MD5 hash.++.. versionadded:: VERSION+ Appending support.+This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.@@ -536,7 +542,7 @@ as a fallback value if that key is not provided for a specific feed definition:- :ref:`topics-feed-storage-s3`: ``True`` (appending is not supported)- - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported)+ - :ref:`topics-feed-storage-gcs`: ``True``- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)@@ -821,4 +827,5 @@ source spider in the feed URI:.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier.. _Amazon S3: https://aws.amazon.com/s3/.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl+.. _composite object: https://docs.cloud.google.com/storage/docs/composite-objects.. _Google Cloud Storage: https://cloud.google.com/storage/diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex d3dc03c53..5752508d3 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -19,6 +19,7 @@ from pathlib import Path, PureWindowsPathfrom tempfile import NamedTemporaryFilefrom typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, castfrom urllib.parse import unquote, urlparse+from uuid import uuid4from twisted.internet.defer import Deferred, DeferredListfrom w3lib.url import file_uri_to_path@@ -311,13 +312,7 @@ class GCSFeedStorage(BlockingFeedStorage):assert u.hostnameself.bucket_name: str = u.hostnameself.blob_name: str = u.path[1:] # remove first "/"-- if feed_options and feed_options.get("overwrite", True) is False:- logger.warning(- "GCS does not support appending to files. To "- "suppress this warning, remove the overwrite "- "option from your FEEDS setting or set it to True."- )+ self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)@classmethoddef from_crawler(@@ -341,8 +336,24 @@ class GCSFeedStorage(BlockingFeedStorage):client = Client(project=self.project_id)bucket = client.bucket(self.bucket_name)- blob = bucket.blob(self.blob_name)- blob.upload_from_file(file, predefined_acl=self.acl)+ blob = None if self.overwrite else bucket.get_blob(self.blob_name)+ if blob is None:+ bucket.blob(self.blob_name).upload_from_file(+ file, predefined_acl=self.acl+ )+ return+ # Appending uploads the new data as a separate object and composes+ # it with the existing one, leaving the data already stored+ # untouched. Composition resolves its sources by name, so it always+ # appends to the latest version of the blob.+ part = bucket.blob(f"{self.blob_name}.{uuid4().hex}.part")+ part.upload_from_file(file, predefined_acl=self.acl)+ try:+ blob.compose([blob, part], client=client)+ finally:+ part.delete(client=client)+ if self.acl:+ blob.acl.save_predefined(self.acl, client=client)finally:file.close()diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex 46e603af7..3870ad6cc 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -29,7 +29,7 @@ from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.ftp import MockFTPServer-from tests.utils.cloud import mock_google_cloud_storage+from tests.utils.cloud import mock_google_cloud_blob, mock_google_cloud_storagefrom tests.utils.decorators import coroutine_test@@ -592,20 +592,147 @@ class TestGCSFeedStorage:blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)f.close.assert_called_once_with()- def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):- with caplog.at_level(logging.DEBUG):- GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")- assert "GCS does not support appending to files" not in caplog.text+ @coroutine_test+ async def test_store_append_new_blob(self) -> None:+ pytest.importorskip("google.cloud.storage")- def test_overwrite_false(self, caplog: pytest.LogCaptureFixture):- with caplog.at_level(logging.DEBUG):- GCSFeedStorage(+ (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()+ bucket_mock.get_blob.return_value = None+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock++ f = mock.Mock()+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ "publicRead",+ feed_options={"overwrite": False},+ )+ await maybe_deferred_to_future(storage.store(f))++ bucket_mock.get_blob.assert_called_once_with("export.csv")+ bucket_mock.blob.assert_called_once_with("export.csv")+ blob_mock.upload_from_file.assert_called_once_with(+ f, predefined_acl="publicRead"+ )+ blob_mock.compose.assert_not_called()+ f.close.assert_called_once_with()++ @coroutine_test+ async def test_store_append_existing_blob(self) -> None:+ pytest.importorskip("google.cloud.storage")++ (client_mock, bucket_mock, part_mock) = mock_google_cloud_storage()+ blob_mock = mock_google_cloud_blob()+ bucket_mock.get_blob.return_value = blob_mock+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock++ f = mock.Mock()+ storage = GCSFeedStorage("gs://mybucket/export.csv","myproject-123",- "custom-acl",+ "publicRead",feed_options={"overwrite": False},)- assert "GCS does not support appending to files" in caplog.text+ await maybe_deferred_to_future(storage.store(f))++ part_name = bucket_mock.blob.call_args.args[0]+ assert part_name.startswith("export.csv.")+ assert part_name.endswith(".part")+ part_mock.upload_from_file.assert_called_once_with(+ f, predefined_acl="publicRead"+ )+ blob_mock.compose.assert_called_once_with(+ [blob_mock, part_mock], client=client_mock+ )+ part_mock.delete.assert_called_once_with(client=client_mock)+ blob_mock.acl.save_predefined.assert_called_once_with(+ "publicRead", client=client_mock+ )+ blob_mock.upload_from_file.assert_not_called()+ f.close.assert_called_once_with()++ @coroutine_test+ async def test_store_append_without_acl(self) -> None:+ pytest.importorskip("google.cloud.storage")++ (client_mock, bucket_mock, _part_mock) = mock_google_cloud_storage()+ blob_mock = mock_google_cloud_blob()+ bucket_mock.get_blob.return_value = blob_mock+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock++ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ None,+ feed_options={"overwrite": False},+ )+ await maybe_deferred_to_future(storage.store(mock.Mock()))++ blob_mock.compose.assert_called_once()+ blob_mock.acl.save_predefined.assert_not_called()++ @coroutine_test+ async def test_store_append_deletes_part_on_compose_error(self) -> None:+ pytest.importorskip("google.cloud.storage")++ (client_mock, bucket_mock, part_mock) = mock_google_cloud_storage()+ blob_mock = mock_google_cloud_blob()+ blob_mock.compose.side_effect = OSError("Compose failed")+ bucket_mock.get_blob.return_value = blob_mock+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock++ f = mock.Mock()+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ "publicRead",+ feed_options={"overwrite": False},+ )+ with pytest.raises(OSError, match="Compose failed"):+ await maybe_deferred_to_future(storage.store(f))++ part_mock.delete.assert_called_once_with(client=client_mock)+ f.close.assert_called_once_with()++ @coroutine_test+ async def test_store_overwrite_ignores_existing_blob(self) -> None:+ pytest.importorskip("google.cloud.storage")++ (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()+ with mock.patch("google.cloud.storage.Client") as m:+ m.return_value = client_mock++ storage = GCSFeedStorage(+ "gs://mybucket/export.csv", "myproject-123", "publicRead"+ )+ await maybe_deferred_to_future(storage.store(mock.Mock()))++ bucket_mock.get_blob.assert_not_called()+ blob_mock.compose.assert_not_called()++ @pytest.mark.parametrize(+ ("feed_options", "expected"),+ [+ (None, True),+ ({}, True),+ ({"overwrite": True}, True),+ ({"overwrite": False}, False),+ ],+ )+ def test_overwrite(+ self, feed_options: dict[str, Any] | None, expected: bool+ ) -> None:+ storage = GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ "custom-acl",+ feed_options=feed_options,+ )+ assert storage.overwrite is expectedclass TestStdoutFeedStorage:diff --git a/tests/utils/cloud.py b/tests/utils/cloud.pyindex 4e253fbdc..08beb66b5 100644--- a/tests/utils/cloud.py+++ b/tests/utils/cloud.py@@ -8,14 +8,21 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blobclasses and set their proper return values."""- from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415+ from google.cloud.storage import Bucket, Client # noqa: PLC0415client_mock = mock.create_autospec(Client)bucket_mock = mock.create_autospec(Bucket)client_mock.bucket.return_value = bucket_mock- blob_mock = mock.create_autospec(Blob)+ blob_mock = mock_google_cloud_blob()bucket_mock.blob.return_value = blob_mockreturn (client_mock, bucket_mock, blob_mock)+++def mock_google_cloud_blob() -> Any:+ """Creates an autospec mock for the google-cloud-storage Blob class."""+ from google.cloud.storage import Blob # noqa: PLC0415++ return mock.create_autospec(Blob)