Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into extending-s3-files-store

Daniel Graña2dee191374e7
diff --cc docs/topics/media-pipeline.rst
index 733a7fe2b,4c634ace5..41beebe98
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@@ -169,19 -171,26 +171,38 @@@ policy:
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
+Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
+self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
+
+ AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
+
+For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
+
+ AWS_USE_SSL = False # or True (None by default)
+ AWS_VERIFY = False # or True (None by default)
+
+.. _Minio: https://github.com/minio/minio
+.. _s3.scality: https://s3.scality.com/
- .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+ .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+
+ Google Cloud Storage
+ ---------------------
+
+ .. setting:: GCS_PROJECT_ID
+
+ :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
+ bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
+
+ .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+
+ For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
+
+ IMAGES_STORE = 'gs://bucket/images/'
+ GCS_PROJECT_ID = 'project_id'
+
+ For information about authentication, see this `documentation`_.
+
+ .. _documentation: https://cloud.google.com/docs/authentication/production
Usage example
=============
diff --cc scrapy/pipelines/files.py
index 7041a0a78,9f1faa313..ab18a727d
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@@ -264,12 -298,11 +306,15 @@@ class FilesPipeline(MediaPipeline)
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+ s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
+ s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
+ s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
+ s3store.AWS_VERIFY = settings['AWS_VERIFY']
s3store.POLICY = settings['FILES_STORE_S3_ACL']
+ gcs_store = cls.STORE_SCHEMES['gs']
+ gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
+
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)