Scrapy, a fast high-level web crawling & scraping framework for Python.
Adding new options.
Oto Brgleza70ec30e19ef
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 843b4d3ec..7041a0a78 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -40,7 +40,6 @@ class FileException(Exception):class FSFilesStore(object):-def __init__(self, basedir):if '://' in basedir:basedir = basedir.split('://', 1)[1]@@ -79,9 +78,12 @@ class FSFilesStore(object):class S3FilesStore(object):-AWS_ACCESS_KEY_ID = NoneAWS_SECRET_ACCESS_KEY = None+ AWS_ENDPOINT_URL = None+ AWS_REGION_NAME = None+ AWS_USE_SSL = None+ AWS_VERIFY = NonePOLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in# FilesPipeline.from_settings.@@ -95,8 +97,14 @@ class S3FilesStore(object):import botocore.sessionsession = botocore.session.get_session()self.s3_client = session.create_client(- 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID,- aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY)+ 's3',+ aws_access_key_id=self.AWS_ACCESS_KEY_ID,+ aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,+ endpoint_url=self.AWS_ENDPOINT_URL,+ region_name=self.AWS_REGION_NAME,+ use_ssl=self.AWS_USE_SSL,+ verify=self.AWS_VERIFY+ )else:from boto.s3.connection import S3Connectionself.S3Connection = S3Connection@@ -181,7 +189,7 @@ class S3FilesStore(object):'X-Amz-Grant-Read': 'GrantRead','X-Amz-Grant-Read-ACP': 'GrantReadACP','X-Amz-Grant-Write-ACP': 'GrantWriteACP',- })+ })extra = {}for key, value in six.iteritems(headers):try:@@ -226,7 +234,7 @@ class FilesPipeline(MediaPipeline):def __init__(self, store_uri, download_func=None, settings=None):if not store_uri:raise NotConfigured-+if isinstance(settings, dict) or settings is None:settings = Settings(settings)@@ -256,6 +264,10 @@ class FilesPipeline(MediaPipeline):s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']+ s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']+ s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']+ s3store.AWS_USE_SSL = settings['AWS_USE_SSL']+ s3store.AWS_VERIFY = settings['AWS_VERIFY']s3store.POLICY = settings['FILES_STORE_S3_ACL']store_uri = settings['FILES_STORE']@@ -423,4 +435,5 @@ class FilesPipeline(MediaPipeline):# deprecateddef file_key(self, url):return self.file_path(url)+file_key._base = True