Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix: added feed_options as a keyword argument to GCSFeedStorage. (#6628)

anubhavc03fb2abb8c3
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c6e2aa0dd..8bcd4e40d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -276,7 +276,14 @@ class S3FeedStorage(BlockingFeedStorage):
class GCSFeedStorage(BlockingFeedStorage):
- def __init__(self, uri: str, project_id: str | None, acl: str | None):
+ def __init__(
+ self,
+ uri: str,
+ project_id: str | None,
+ acl: str | None,
+ *,
+ feed_options: dict[str, Any] | None = None,
+ ):
self.project_id: str | None = project_id
self.acl: str | None = acl
u = urlparse(uri)
@@ -284,12 +291,26 @@ class GCSFeedStorage(BlockingFeedStorage):
self.bucket_name: str = u.hostname
self.blob_name: str = u.path[1:] # remove first "/"
+ if feed_options and feed_options.get("overwrite", True) is False:
+ logger.warning(
+ "GCS does not support appending to files. To "
+ "suppress this warning, remove the overwrite "
+ "option from your FEEDS setting or set it to True."
+ )
+
@classmethod
- def from_crawler(cls, crawler: Crawler, uri: str) -> Self:
+ def from_crawler(
+ cls,
+ crawler: Crawler,
+ uri: str,
+ *,
+ feed_options: dict[str, Any] | None = None,
+ ) -> Self:
return cls(
uri,
crawler.settings["GCS_PROJECT_ID"],
crawler.settings["FEED_STORAGE_GCS_ACL"] or None,
+ feed_options=feed_options,
)
def _store_in_thread(self, file: IO[bytes]) -> None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 81d05e2a3..7edffa1f6 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -523,6 +523,21 @@ class GCSFeedStorageTest(unittest.TestCase):
bucket_mock.blob.assert_called_once_with("export.csv")
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+ def test_overwrite_default(self):
+ with LogCapture() as log:
+ GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")
+ self.assertNotIn("GCS does not support appending to files", str(log))
+
+ def test_overwrite_false(self):
+ with LogCapture() as log:
+ GCSFeedStorage(
+ "gs://mybucket/export.csv",
+ "myproject-123",
+ "custom-acl",
+ feed_options={"overwrite": False},
+ )
+ self.assertIn("GCS does not support appending to files", str(log))
+
class StdoutFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks