Scrapy, a fast high-level web crawling & scraping framework for Python.
Support for ImagesPipeline
rhoboroee166ec44f38
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 304d89bcf..7fdb8a086 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -218,6 +218,12 @@ class GCSFilesStore(object):return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)+ def _get_content_type(self, headers):+ if headers and 'Content-Type' in headers:+ return headers['Content-Type']+ else:+ return 'application/octet-stream'+def persist_file(self, path, buf, info, meta=None, headers=None):blob = self.bucket.blob(self.prefix + path)blob.cache_control = self.CACHE_CONTROL@@ -225,7 +231,7 @@ class GCSFilesStore(object):return threads.deferToThread(blob.upload_from_string,data=buf.getvalue(),- content_type='application/octet-stream'+ content_type=self._get_content_type(headers))diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex bc449431f..c5fc12afe 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline):s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']s3store.POLICY = settings['IMAGES_STORE_S3_ACL']+ gcs_store = cls.STORE_SCHEMES['gs']+ gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']+store_uri = settings['IMAGES_STORE']return cls(store_uri, settings=settings)