Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into extending-s3-files-store
Daniel Graña2dee191374e7
diff --cc docs/topics/media-pipeline.rstindex 733a7fe2b,4c634ace5..41beebe98--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@@ -169,19 -171,26 +171,38 @@@ policy:For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.+Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like+self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::++ AWS_ENDPOINT_URL = 'http://minio.example.com:9000'++For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::++ AWS_USE_SSL = False # or True (None by default)+ AWS_VERIFY = False # or True (None by default)++.. _Minio: https://github.com/minio/minio+.. _s3.scality: https://s3.scality.com/- .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl+ .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl++ Google Cloud Storage+ ---------------------++ .. setting:: GCS_PROJECT_ID++ :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage+ bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )++ .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python++ For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::++ IMAGES_STORE = 'gs://bucket/images/'+ GCS_PROJECT_ID = 'project_id'++ For information about authentication, see this `documentation`_.++ .. _documentation: https://cloud.google.com/docs/authentication/productionUsage example=============diff --cc scrapy/pipelines/files.pyindex 7041a0a78,9f1faa313..ab18a727d--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@@ -264,12 -298,11 +306,15 @@@ class FilesPipeline(MediaPipeline)s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']+ s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']+ s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']+ s3store.AWS_USE_SSL = settings['AWS_USE_SSL']+ s3store.AWS_VERIFY = settings['AWS_VERIFY']s3store.POLICY = settings['FILES_STORE_S3_ACL']+ gcs_store = cls.STORE_SCHEMES['gs']+ gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']+store_uri = settings['FILES_STORE']return cls(store_uri, settings=settings)