Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Adding GCSFeedStorage

Júlio César Batista5a55c4269d33
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 22ebf3b3f..00d5d8025 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage):
key.close()
+
+class GCSFeedStorage(BlockingFeedStorage):
+
+ project_id = None
+ bucket_name = None
+ blob_name = None
+
+ def __init__(self, uri, project_id):
+ self.project_id = project_id
+ u = urlparse(uri)
+ self.bucket_name = u.hostname
+ self.blob_name = u.path[1:] # remove first "/"
+
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+
+ def _store_in_thread(self, file):
+ file.seek(0)
+ from google.cloud.storage import Client
+ client = Client(project=self.project_id)
+ bucket = client.get_bucket(self.bucket_name)
+ blob = bucket.blob(self.blob_name)
+ blob.upload_from_file(file)
+
+
class FTPFeedStorage(BlockingFeedStorage):
def __init__(self, uri):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e46c8c14e..f3c499b3f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri
import scrapy
from scrapy.exporters import CsvItemExporter
from scrapy.extensions.feedexport import (
- IFeedStorage, FileFeedStorage, FTPFeedStorage,
+ IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
S3FeedStorage, StdoutFeedStorage,
BlockingFeedStorage)
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
@@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(content, expected_content)
+class GCSFeedStorageTest(unittest.TestCase):
+
+ @mock.patch('scrapy.conf.settings',
+ new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+ def test_parse_settings(self):
+ try:
+ from google.cloud.storage import Client
+ except ImportError:
+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+ settings = {'GCS_PROJECT_ID': '123' }
+ crawler = get_crawler(settings_dict=settings)
+ storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
+ assert storage.project_id == '123'
+ assert storage.bucket_name == 'mybucket'
+ assert storage.blob_name == 'export.csv'
+
+ @defer.inlineCallbacks
+ def test_store(self):
+ try:
+ from google.cloud.storage import Client, Bucket, Blob
+ except ImportError:
+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+ uri = 'gcs://mybucket/export.csv'
+ project_id = 'myproject-123'
+ with mock.patch('google.cloud.storage.Client') as m:
+ client_mock = mock.create_autospec(Client)
+ m.return_value = client_mock
+
+ bucket_mock = mock.create_autospec(Bucket)
+ client_mock.get_bucket.return_value = bucket_mock
+
+ blob_mock = mock.create_autospec(Blob)
+ bucket_mock.blob.return_value = blob_mock
+
+ f = mock.Mock()
+ storage = GCSFeedStorage(uri, project_id)
+ yield storage.store(f)
+
+ f.seek.assert_called_once_with(0)
+ m.assert_called_once_with(project=project_id)
+ client_mock.get_bucket.assert_called_once_with('mybucket')
+ bucket_mock.blob.assert_called_once_with('export.csv')
+ blob_mock.upload_from_file.assert_called_once_with(f)
+
class StdoutFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks