Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Change FilesPipeline class attributes to instance attributes.

djunzu8228a0c49113
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 338286092..c94794173 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -22,6 +22,7 @@ except ImportError:
from twisted.internet import defer, threads
from scrapy.pipelines.media import MediaPipeline
+from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.misc import md5sum
@@ -219,10 +220,18 @@ class FilesPipeline(MediaPipeline):
's3': S3FilesStore,
}
- def __init__(self, store_uri, download_func=None):
+ def __init__(self, store_uri, download_func=None, settings=None):
if not store_uri:
raise NotConfigured
+
+ if isinstance(settings, dict) or settings is None:
+ settings = Settings(settings)
+
self.store = self._get_store(store_uri)
+ self.expires = settings.getint('FILES_EXPIRES')
+ self.files_urls_field = settings.get('FILES_URLS_FIELD')
+ self.files_result_field = settings.get('FILES_RESULT_FIELD')
+
super(FilesPipeline, self).__init__(download_func=download_func)
@classmethod
@@ -232,11 +241,8 @@ class FilesPipeline(MediaPipeline):
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
s3store.POLICY = settings['FILES_STORE_S3_ACL']
- cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD')
- cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD')
- cls.EXPIRES = settings.getint('FILES_EXPIRES')
store_uri = settings['FILES_STORE']
- return cls(store_uri)
+ return cls(store_uri, settings=settings)
def _get_store(self, uri):
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
@@ -257,7 +263,7 @@ class FilesPipeline(MediaPipeline):
age_seconds = time.time() - last_modified
age_days = age_seconds / 60 / 60 / 24
- if age_days > self.EXPIRES:
+ if age_days > self.expires:
return # returning None force download
referer = referer_str(request)
@@ -356,7 +362,7 @@ class FilesPipeline(MediaPipeline):
### Overridable Interface
def get_media_requests(self, item, info):
- return [Request(x) for x in item.get(self.FILES_URLS_FIELD, [])]
+ return [Request(x) for x in item.get(self.files_urls_field, [])]
def file_downloaded(self, response, request, info):
path = self.file_path(request, response=response, info=info)
@@ -367,8 +373,8 @@ class FilesPipeline(MediaPipeline):
return checksum
def item_completed(self, results, item, info):
- if isinstance(item, dict) or self.FILES_RESULT_FIELD in item.fields:
- item[self.FILES_RESULT_FIELD] = [x for ok, x in results if ok]
+ if isinstance(item, dict) or self.files_result_field in item.fields:
+ item[self.files_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f480b0c18..391538562 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -91,7 +91,7 @@ class FilesPipelineTestCase(unittest.TestCase):
patchers = [
mock.patch.object(FSFilesStore, 'stat_file', return_value={
'checksum': 'abc',
- 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}),
+ 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
mock.patch.object(FilesPipeline, 'get_media_requests',
return_value=[_prepare_request_object(item_url)]),
mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)