Scrapy, a fast high-level web crawling & scraping framework for Python.
Move default settings to settings/default_settings.py.
djunzuc7fc17866feb
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex b1b8404bb..338286092 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -213,14 +213,11 @@ class FilesPipeline(MediaPipeline):"""MEDIA_NAME = "file"- EXPIRES = 90STORE_SCHEMES = {'': FSFilesStore,'file': FSFilesStore,'s3': S3FilesStore,}- DEFAULT_FILES_URLS_FIELD = 'file_urls'- DEFAULT_FILES_RESULT_FIELD = 'files'def __init__(self, store_uri, download_func=None):if not store_uri:@@ -235,9 +232,9 @@ class FilesPipeline(MediaPipeline):s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']s3store.POLICY = settings['FILES_STORE_S3_ACL']- cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)- cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)- cls.EXPIRES = settings.getint('FILES_EXPIRES', 90)+ cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD')+ cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD')+ cls.EXPIRES = settings.getint('FILES_EXPIRES')store_uri = settings['FILES_STORE']return cls(store_uri)diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex ff73b44b7..1f6b0c7e7 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -36,24 +36,19 @@ class ImagesPipeline(FilesPipeline):"""MEDIA_NAME = 'image'- MIN_WIDTH = 0- MIN_HEIGHT = 0- THUMBS = {}- DEFAULT_IMAGES_URLS_FIELD = 'image_urls'- DEFAULT_IMAGES_RESULT_FIELD = 'images'@classmethoddef from_settings(cls, settings):- cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0)- cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0)- cls.EXPIRES = settings.getint('IMAGES_EXPIRES', 90)- cls.THUMBS = settings.get('IMAGES_THUMBS', {})+ cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH')+ cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT')+ cls.EXPIRES = settings.getint('IMAGES_EXPIRES')+ cls.THUMBS = settings.get('IMAGES_THUMBS')s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']- cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD', cls.DEFAULT_IMAGES_URLS_FIELD)- cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD', cls.DEFAULT_IMAGES_RESULT_FIELD)+ cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD')+ cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD')store_uri = settings['IMAGES_STORE']return cls(store_uri)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex d449c4891..843506741 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -159,6 +159,9 @@ FEED_EXPORTERS_BASE = {}FILES_STORE_S3_ACL = 'private'+FILES_EXPIRES = 90+FILES_URLS_FIELD = 'file_urls'+FILES_RESULT_FIELD = 'files'HTTPCACHE_ENABLED = FalseHTTPCACHE_DIR = 'httpcache'@@ -175,6 +178,13 @@ HTTPCACHE_GZIP = FalseHTTPPROXY_AUTH_ENCODING = 'latin-1'+IMAGES_MIN_WIDTH = 0+IMAGES_MIN_HEIGHT = 0+IMAGES_EXPIRES = 90+IMAGES_THUMBS = {}+IMAGES_URLS_FIELD = 'image_urls'+IMAGES_RESULT_FIELD = 'images'+ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'ITEM_PIPELINES = {}