Scrapy, a fast high-level web crawling & scraping framework for Python.
See #1778 - change default S3 ACL to "private" and allow customization via settings
Konstantin Lopuhin164f30076229
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex e5037129c..1f17ccd51 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -185,6 +185,9 @@ passed through the following settings:* :setting:`AWS_ACCESS_KEY_ID`* :setting:`AWS_SECRET_ACCESS_KEY`+Default access policy for uploaded files is ``private``, it can be changed+(for example, to ``public-read``) via :setting:`S3_STORE_ACL`.+.. _topics-feed-storage-stdout:Standard outputdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex f8f35b5e3..b7d6f2d9a 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -926,6 +926,15 @@ If enabled, Scrapy will respect robots.txt policies. For more information seethis option is enabled by default in settings.py file generatedby ``scrapy startproject`` command.+.. setting:: S3_STORE_ACL++S3_STORE_ACL+------------++Default: ``'private'``++S3-specific access control policy (ACL) for uploaded files.+.. setting:: SCHEDULERSCHEDULERdiff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 45ceddcbb..b48f43094 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -82,7 +82,8 @@ class S3FilesStore(object):AWS_ACCESS_KEY_ID = NoneAWS_SECRET_ACCESS_KEY = None- POLICY = 'public-read'+ POLICY = 'private' # Overriden from settings.S3_STORE_ACL in+ # FilesPipeline.from_settings.HEADERS = {'Cache-Control': 'max-age=172800',}@@ -232,6 +233,7 @@ class FilesPipeline(MediaPipeline):s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']+ s3store.POLICY = settings['S3_STORE_ACL']cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 44e74dc61..e31dc6aef 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -231,6 +231,8 @@ SCHEDULER = 'scrapy.core.scheduler.Scheduler'SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'+S3_STORE_ACL = 'private'+SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'SPIDER_MIDDLEWARES = {}