Scrapy, a fast high-level web crawling & scraping framework for Python.
fix: added feed_options as a keyword argument to GCSFeedStorage. (#6628)
anubhavc03fb2abb8c3
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c6e2aa0dd..8bcd4e40d 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -276,7 +276,14 @@ class S3FeedStorage(BlockingFeedStorage):class GCSFeedStorage(BlockingFeedStorage):- def __init__(self, uri: str, project_id: str | None, acl: str | None):+ def __init__(+ self,+ uri: str,+ project_id: str | None,+ acl: str | None,+ *,+ feed_options: dict[str, Any] | None = None,+ ):self.project_id: str | None = project_idself.acl: str | None = aclu = urlparse(uri)@@ -284,12 +291,26 @@ class GCSFeedStorage(BlockingFeedStorage):self.bucket_name: str = u.hostnameself.blob_name: str = u.path[1:] # remove first "/"+ if feed_options and feed_options.get("overwrite", True) is False:+ logger.warning(+ "GCS does not support appending to files. To "+ "suppress this warning, remove the overwrite "+ "option from your FEEDS setting or set it to True."+ )+@classmethod- def from_crawler(cls, crawler: Crawler, uri: str) -> Self:+ def from_crawler(+ cls,+ crawler: Crawler,+ uri: str,+ *,+ feed_options: dict[str, Any] | None = None,+ ) -> Self:return cls(uri,crawler.settings["GCS_PROJECT_ID"],crawler.settings["FEED_STORAGE_GCS_ACL"] or None,+ feed_options=feed_options,)def _store_in_thread(self, file: IO[bytes]) -> None:diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 81d05e2a3..7edffa1f6 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -523,6 +523,21 @@ class GCSFeedStorageTest(unittest.TestCase):bucket_mock.blob.assert_called_once_with("export.csv")blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)+ def test_overwrite_default(self):+ with LogCapture() as log:+ GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")+ self.assertNotIn("GCS does not support appending to files", str(log))++ def test_overwrite_false(self):+ with LogCapture() as log:+ GCSFeedStorage(+ "gs://mybucket/export.csv",+ "myproject-123",+ "custom-acl",+ feed_options={"overwrite": False},+ )+ self.assertIn("GCS does not support appending to files", str(log))+class StdoutFeedStorageTest(unittest.TestCase):@defer.inlineCallbacks