Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

See #1778 - change default S3 ACL to "private" and allow customization via settings

Konstantin Lopuhin164f30076229
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e5037129c..1f17ccd51 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -185,6 +185,9 @@ passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
+Default access policy for uploaded files is ``private``, it can be changed
+(for example, to ``public-read``) via :setting:`S3_STORE_ACL`.
+
.. _topics-feed-storage-stdout:
Standard output
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f8f35b5e3..b7d6f2d9a 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -926,6 +926,15 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
this option is enabled by default in settings.py file generated
by ``scrapy startproject`` command.
+.. setting:: S3_STORE_ACL
+
+S3_STORE_ACL
+------------
+
+Default: ``'private'``
+
+S3-specific access control policy (ACL) for uploaded files.
+
.. setting:: SCHEDULER
SCHEDULER
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 45ceddcbb..b48f43094 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -82,7 +82,8 @@ class S3FilesStore(object):
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
- POLICY = 'public-read'
+ POLICY = 'private' # Overriden from settings.S3_STORE_ACL in
+ # FilesPipeline.from_settings.
HEADERS = {
'Cache-Control': 'max-age=172800',
}
@@ -232,6 +233,7 @@ class FilesPipeline(MediaPipeline):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+ s3store.POLICY = settings['S3_STORE_ACL']
cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)
cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 44e74dc61..e31dc6aef 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -231,6 +231,8 @@ SCHEDULER = 'scrapy.core.scheduler.Scheduler'
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
+S3_STORE_ACL = 'private'
+
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
SPIDER_MIDDLEWARES = {}