Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support for ImagesPipeline

rhoboroee166ec44f38
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 304d89bcf..7fdb8a086 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -218,6 +218,12 @@ class GCSFilesStore(object):
return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+ def _get_content_type(self, headers):
+ if headers and 'Content-Type' in headers:
+ return headers['Content-Type']
+ else:
+ return 'application/octet-stream'
+
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
@@ -225,7 +231,7 @@ class GCSFilesStore(object):
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
- content_type='application/octet-stream'
+ content_type=self._get_content_type(headers)
)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index bc449431f..c5fc12afe 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline):
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
+ gcs_store = cls.STORE_SCHEMES['gs']
+ gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
store_uri = settings['IMAGES_STORE']
return cls(store_uri, settings=settings)