Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into feat-685

Adrián Chaves96bb3b5144d9
diff --cc docs/topics/feed-exports.rst
index 8dc08710f,7481b1a99..c1a15e15d
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@@ -401,7 -399,5 +420,6 @@@ format in :setting:`FEED_EXPORTERS`. E.
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
- .. _boto: https://github.com/boto/boto
.. _botocore: https://github.com/boto/botocore
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+.. _Google Cloud Storage: https://cloud.google.com/storage/
diff --cc scrapy/extensions/feedexport.py
index 3c9c6a3b5,f1b101780..e619668c6
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@@ -154,35 -152,9 +152,35 @@@ class S3FeedStorage(BlockingFeedStorage
key.close()
+class GCSFeedStorage(BlockingFeedStorage):
+
+ def __init__(self, uri, project_id, acl):
+ self.project_id = project_id
+ self.acl = acl
+ u = urlparse(uri)
+ self.bucket_name = u.hostname
+ self.blob_name = u.path[1:] # remove first "/"
+
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return cls(
+ uri,
+ crawler.settings['GCS_PROJECT_ID'],
+ crawler.settings['FEED_STORAGE_GCS_ACL'] or None
+ )
+
+ def _store_in_thread(self, file):
+ file.seek(0)
+ from google.cloud.storage import Client
+ client = Client(project=self.project_id)
+ bucket = client.get_bucket(self.bucket_name)
+ blob = bucket.blob(self.blob_name)
+ blob.upload_from_file(file, predefined_acl=self.acl)
+
+
class FTPFeedStorage(BlockingFeedStorage):
- def __init__(self, uri):
+ def __init__(self, uri, use_active_mode=False):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
diff --cc scrapy/settings/default_settings.py
index 367a2eb43,fc7b62e78..368c6fe30
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@@ -159,7 -159,7 +160,8 @@@ FEED_EXPORTERS_BASE =
}
FEED_EXPORT_INDENT = 0
+ FEED_STORAGE_FTP_ACTIVE = False
+FEED_STORAGE_GCS_ACL = ''
FEED_STORAGE_S3_ACL = ''
FILES_STORE_S3_ACL = 'private'
diff --cc scrapy/utils/test.py
index 84eae97a7,faac0b12f..4ff6d73a5
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@@ -93,17 -120,9 +121,23 @@@ def assert_samelines(testcase, text1, t
"""
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
-
+ def get_from_asyncio_queue(value):
+ q = asyncio.Queue()
+ getter = q.get()
+ q.put_nowait(value)
+ return getter
++
+def mock_google_cloud_storage():
+ """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+ classes and set their proper return values.
+ """
+ from google.cloud.storage import Client, Bucket, Blob
+ client_mock = mock.create_autospec(Client)
+
+ bucket_mock = mock.create_autospec(Bucket)
+ client_mock.get_bucket.return_value = bucket_mock
+
+ blob_mock = mock.create_autospec(Blob)
+ bucket_mock.blob.return_value = blob_mock
+
- return (client_mock, bucket_mock, blob_mock)
++ return (client_mock, bucket_mock, blob_mock)
diff --cc tests/test_feedexport.py
index 9b8e2660a,2ca57c19d..d97b199fe
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -21,12 -22,11 +22,12 @@@ from w3lib.url import path_to_file_ur
import scrapy
from scrapy.exporters import CsvItemExporter
from scrapy.extensions.feedexport import (
- IFeedStorage, FileFeedStorage, FTPFeedStorage,
+ IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
S3FeedStorage, StdoutFeedStorage,
BlockingFeedStorage)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
+from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,
+ get_crawler, mock_google_cloud_storage)
- from scrapy.utils.python import to_native_str
+ from scrapy.utils.python import to_unicode
class FileFeedStorageTest(unittest.TestCase):