Scrapy, a fast high-level web crawling & scraping framework for Python.
Adding GCSFeedStorage
Júlio César Batista5a55c4269d33
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 22ebf3b3f..00d5d8025 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage):key.close()++class GCSFeedStorage(BlockingFeedStorage):++ project_id = None+ bucket_name = None+ blob_name = None++ def __init__(self, uri, project_id):+ self.project_id = project_id+ u = urlparse(uri)+ self.bucket_name = u.hostname+ self.blob_name = u.path[1:] # remove first "/"++ @classmethod+ def from_crawler(cls, crawler, uri):+ return cls(uri, crawler.settings['GCS_PROJECT_ID'])++ def _store_in_thread(self, file):+ file.seek(0)+ from google.cloud.storage import Client+ client = Client(project=self.project_id)+ bucket = client.get_bucket(self.bucket_name)+ blob = bucket.blob(self.blob_name)+ blob.upload_from_file(file)++class FTPFeedStorage(BlockingFeedStorage):def __init__(self, uri):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex e46c8c14e..f3c499b3f 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uriimport scrapyfrom scrapy.exporters import CsvItemExporterfrom scrapy.extensions.feedexport import (- IFeedStorage, FileFeedStorage, FTPFeedStorage,+ IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,S3FeedStorage, StdoutFeedStorage,BlockingFeedStorage)from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler@@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(content, expected_content)+class GCSFeedStorageTest(unittest.TestCase):++ @mock.patch('scrapy.conf.settings',+ new={'GCS_PROJECT_ID': 'conf_id' }, create=True)+ def test_parse_settings(self):+ try:+ from google.cloud.storage import Client+ except ImportError:+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")++ settings = {'GCS_PROJECT_ID': '123' }+ crawler = get_crawler(settings_dict=settings)+ storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')+ assert storage.project_id == '123'+ assert storage.bucket_name == 'mybucket'+ assert storage.blob_name == 'export.csv'++ @defer.inlineCallbacks+ def test_store(self):+ try:+ from google.cloud.storage import Client, Bucket, Blob+ except ImportError:+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")++ uri = 'gcs://mybucket/export.csv'+ project_id = 'myproject-123'+ with mock.patch('google.cloud.storage.Client') as m:+ client_mock = mock.create_autospec(Client)+ m.return_value = client_mock++ bucket_mock = mock.create_autospec(Bucket)+ client_mock.get_bucket.return_value = bucket_mock++ blob_mock = mock.create_autospec(Blob)+ bucket_mock.blob.return_value = blob_mock++ f = mock.Mock()+ storage = GCSFeedStorage(uri, project_id)+ yield storage.store(f)++ f.seek.assert_called_once_with(0)+ m.assert_called_once_with(project=project_id)+ client_mock.get_bucket.assert_called_once_with('mybucket')+ bucket_mock.blob.assert_called_once_with('export.csv')+ blob_mock.upload_from_file.assert_called_once_with(f)+class StdoutFeedStorageTest(unittest.TestCase):@defer.inlineCallbacks