Scrapy, a fast high-level web crawling & scraping framework for Python.
Change FilesPipeline class attributes to instance attributes.
djunzu8228a0c49113
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 338286092..c94794173 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -22,6 +22,7 @@ except ImportError:from twisted.internet import defer, threadsfrom scrapy.pipelines.media import MediaPipeline+from scrapy.settings import Settingsfrom scrapy.exceptions import NotConfigured, IgnoreRequestfrom scrapy.http import Requestfrom scrapy.utils.misc import md5sum@@ -219,10 +220,18 @@ class FilesPipeline(MediaPipeline):'s3': S3FilesStore,}- def __init__(self, store_uri, download_func=None):+ def __init__(self, store_uri, download_func=None, settings=None):if not store_uri:raise NotConfigured++ if isinstance(settings, dict) or settings is None:+ settings = Settings(settings)+self.store = self._get_store(store_uri)+ self.expires = settings.getint('FILES_EXPIRES')+ self.files_urls_field = settings.get('FILES_URLS_FIELD')+ self.files_result_field = settings.get('FILES_RESULT_FIELD')+super(FilesPipeline, self).__init__(download_func=download_func)@classmethod@@ -232,11 +241,8 @@ class FilesPipeline(MediaPipeline):s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']s3store.POLICY = settings['FILES_STORE_S3_ACL']- cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD')- cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD')- cls.EXPIRES = settings.getint('FILES_EXPIRES')store_uri = settings['FILES_STORE']- return cls(store_uri)+ return cls(store_uri, settings=settings)def _get_store(self, uri):if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir@@ -257,7 +263,7 @@ class FilesPipeline(MediaPipeline):age_seconds = time.time() - last_modifiedage_days = age_seconds / 60 / 60 / 24- if age_days > self.EXPIRES:+ if age_days > self.expires:return # returning None force downloadreferer = referer_str(request)@@ -356,7 +362,7 @@ class FilesPipeline(MediaPipeline):### Overridable Interfacedef get_media_requests(self, item, info):- return [Request(x) for x in item.get(self.FILES_URLS_FIELD, [])]+ return [Request(x) for x in item.get(self.files_urls_field, [])]def file_downloaded(self, response, request, info):path = self.file_path(request, response=response, info=info)@@ -367,8 +373,8 @@ class FilesPipeline(MediaPipeline):return checksumdef item_completed(self, results, item, info):- if isinstance(item, dict) or self.FILES_RESULT_FIELD in item.fields:- item[self.FILES_RESULT_FIELD] = [x for ok, x in results if ok]+ if isinstance(item, dict) or self.files_result_field in item.fields:+ item[self.files_result_field] = [x for ok, x in results if ok]return itemdef file_path(self, request, response=None, info=None):diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex f480b0c18..391538562 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -91,7 +91,7 @@ class FilesPipelineTestCase(unittest.TestCase):patchers = [mock.patch.object(FSFilesStore, 'stat_file', return_value={'checksum': 'abc',- 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}),+ 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),mock.patch.object(FilesPipeline, 'get_media_requests',return_value=[_prepare_request_object(item_url)]),mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)