Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support for Google Cloud Storage

rhoborod71a0634039d
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index eae03752a..304d89bcf 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -194,6 +194,41 @@ class S3FilesStore(object):
return extra
+class GCSFilesStore(object):
+
+ GCS_PROJECT_ID = None
+
+ CACHE_CONTROL = 'max-age=172800'
+
+ def __init__(self, uri):
+ from google.cloud import storage
+ client = storage.Client(project=self.GCS_PROJECT_ID)
+ bucket, prefix = uri[5:].split('/', 1)
+ self.bucket = client.bucket(bucket)
+ self.prefix = prefix
+
+ def stat_file(self, path, info):
+ def _onsuccess(blob):
+ if blob:
+ checksum = blob.md5_hash
+ last_modified = time.mktime(blob.updated.timetuple())
+ return {'checksum': checksum, 'last_modified': last_modified}
+ else:
+ return {}
+
+ return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+
+ def persist_file(self, path, buf, info, meta=None, headers=None):
+ blob = self.bucket.blob(self.prefix + path)
+ blob.cache_control = self.CACHE_CONTROL
+ blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+ return threads.deferToThread(
+ blob.upload_from_string,
+ data=buf.getvalue(),
+ content_type='application/octet-stream'
+ )
+
+
class FilesPipeline(MediaPipeline):
"""Abstract pipeline that implement the file downloading
@@ -219,6 +254,7 @@ class FilesPipeline(MediaPipeline):
'': FSFilesStore,
'file': FSFilesStore,
's3': S3FilesStore,
+ 'gs': GCSFilesStore,
}
DEFAULT_FILES_URLS_FIELD = 'file_urls'
DEFAULT_FILES_RESULT_FIELD = 'files'
@@ -258,6 +294,9 @@ class FilesPipeline(MediaPipeline):
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
s3store.POLICY = settings['FILES_STORE_S3_ACL']
+ gcs_store = cls.STORE_SCHEMES['gs']
+ gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)