Scrapy, a fast high-level web crawling & scraping framework for Python.
Support for Google Cloud Storage
rhoborod71a0634039d
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex eae03752a..304d89bcf 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -194,6 +194,41 @@ class S3FilesStore(object):return extra+class GCSFilesStore(object):++ GCS_PROJECT_ID = None++ CACHE_CONTROL = 'max-age=172800'++ def __init__(self, uri):+ from google.cloud import storage+ client = storage.Client(project=self.GCS_PROJECT_ID)+ bucket, prefix = uri[5:].split('/', 1)+ self.bucket = client.bucket(bucket)+ self.prefix = prefix++ def stat_file(self, path, info):+ def _onsuccess(blob):+ if blob:+ checksum = blob.md5_hash+ last_modified = time.mktime(blob.updated.timetuple())+ return {'checksum': checksum, 'last_modified': last_modified}+ else:+ return {}++ return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)++ def persist_file(self, path, buf, info, meta=None, headers=None):+ blob = self.bucket.blob(self.prefix + path)+ blob.cache_control = self.CACHE_CONTROL+ blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}+ return threads.deferToThread(+ blob.upload_from_string,+ data=buf.getvalue(),+ content_type='application/octet-stream'+ )++class FilesPipeline(MediaPipeline):"""Abstract pipeline that implement the file downloading@@ -219,6 +254,7 @@ class FilesPipeline(MediaPipeline):'': FSFilesStore,'file': FSFilesStore,'s3': S3FilesStore,+ 'gs': GCSFilesStore,}DEFAULT_FILES_URLS_FIELD = 'file_urls'DEFAULT_FILES_RESULT_FIELD = 'files'@@ -258,6 +294,9 @@ class FilesPipeline(MediaPipeline):s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']s3store.POLICY = settings['FILES_STORE_S3_ACL']+ gcs_store = cls.STORE_SCHEMES['gs']+ gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']+store_uri = settings['FILES_STORE']return cls(store_uri, settings=settings)