Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into feat-685
Adrián Chaves96bb3b5144d9
diff --cc docs/topics/feed-exports.rstindex 8dc08710f,7481b1a99..c1a15e15d--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@@ -401,7 -399,5 +420,6 @@@ format in :setting:`FEED_EXPORTERS`. E... _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier.. _Amazon S3: https://aws.amazon.com/s3/- .. _boto: https://github.com/boto/boto.. _botocore: https://github.com/boto/botocore.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl+.. _Google Cloud Storage: https://cloud.google.com/storage/diff --cc scrapy/extensions/feedexport.pyindex 3c9c6a3b5,f1b101780..e619668c6--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@@ -154,35 -152,9 +152,35 @@@ class S3FeedStorage(BlockingFeedStoragekey.close()+class GCSFeedStorage(BlockingFeedStorage):++ def __init__(self, uri, project_id, acl):+ self.project_id = project_id+ self.acl = acl+ u = urlparse(uri)+ self.bucket_name = u.hostname+ self.blob_name = u.path[1:] # remove first "/"++ @classmethod+ def from_crawler(cls, crawler, uri):+ return cls(+ uri,+ crawler.settings['GCS_PROJECT_ID'],+ crawler.settings['FEED_STORAGE_GCS_ACL'] or None+ )++ def _store_in_thread(self, file):+ file.seek(0)+ from google.cloud.storage import Client+ client = Client(project=self.project_id)+ bucket = client.get_bucket(self.bucket_name)+ blob = bucket.blob(self.blob_name)+ blob.upload_from_file(file, predefined_acl=self.acl)++class FTPFeedStorage(BlockingFeedStorage):- def __init__(self, uri):+ def __init__(self, uri, use_active_mode=False):u = urlparse(uri)self.host = u.hostnameself.port = int(u.port or '21')diff --cc scrapy/settings/default_settings.pyindex 367a2eb43,fc7b62e78..368c6fe30--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@@ -159,7 -159,7 +160,8 @@@ FEED_EXPORTERS_BASE =}FEED_EXPORT_INDENT = 0+ FEED_STORAGE_FTP_ACTIVE = False+FEED_STORAGE_GCS_ACL = ''FEED_STORAGE_S3_ACL = ''FILES_STORE_S3_ACL = 'private'diff --cc scrapy/utils/test.pyindex 84eae97a7,faac0b12f..4ff6d73a5--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@@ -93,17 -120,9 +121,23 @@@ def assert_samelines(testcase, text1, t"""testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)-+ def get_from_asyncio_queue(value):+ q = asyncio.Queue()+ getter = q.get()+ q.put_nowait(value)+ return getter+++def mock_google_cloud_storage():+ """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob+ classes and set their proper return values.+ """+ from google.cloud.storage import Client, Bucket, Blob+ client_mock = mock.create_autospec(Client)++ bucket_mock = mock.create_autospec(Bucket)+ client_mock.get_bucket.return_value = bucket_mock++ blob_mock = mock.create_autospec(Blob)+ bucket_mock.blob.return_value = blob_mock+- return (client_mock, bucket_mock, blob_mock)++ return (client_mock, bucket_mock, blob_mock)diff --cc tests/test_feedexport.pyindex 9b8e2660a,2ca57c19d..d97b199fe--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@@ -21,12 -22,11 +22,12 @@@ from w3lib.url import path_to_file_urimport scrapyfrom scrapy.exporters import CsvItemExporterfrom scrapy.extensions.feedexport import (- IFeedStorage, FileFeedStorage, FTPFeedStorage,+ IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,S3FeedStorage, StdoutFeedStorage,BlockingFeedStorage)-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler+from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,+ get_crawler, mock_google_cloud_storage)- from scrapy.utils.python import to_native_str+ from scrapy.utils.python import to_unicodeclass FileFeedStorageTest(unittest.TestCase):