Scrapy, a fast high-level web crawling & scraping framework for Python.
Support disabling file/image expiration with a negative FILES_EXPIRES/IMAGES_EXPIRES (#8046)
Adrian29484a1e2f62
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex d031fa2b7..362480515 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -462,7 +462,8 @@ specifies the delay in number of days:# 30 days of delay for images expirationIMAGES_EXPIRES = 30-The default value for both settings is 90 days.+The default value for both settings is 90 days. A negative value disables+expiration, i.e. files are never re-downloaded once fetched.If you have a pipeline that subclasses FilesPipeline and you'd like to have adifferent setting for it, you can set setting keys preceded by the uppercasediff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex b3c44dee9..2170ae4b2 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -568,7 +568,7 @@ class FilesPipeline(MediaPipeline):age_seconds = time.time() - last_modifiedage_days = age_seconds / 60 / 60 / 24- if age_days > self.expires:+ if self.expires >= 0 and age_days > self.expires:return None # returning None force downloadreferer = referer_str(request)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex d493d2b46..84cfa6c61 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -237,6 +237,32 @@ class TestFilesPipeline:assert result["files"][0]["checksum"] != "abc"assert result["files"][0]["status"] == "downloaded"+ @coroutine_test+ async def test_file_never_expires(self):+ pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": -1})+ item_url = "http://example.com/file5.pdf"+ item = _create_item_with_files(item_url)+ with (+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),+ mock.patch.object(+ FSFilesStore,+ "stat_file",+ return_value={+ "checksum": "abc",+ "last_modified": time.time()+ - (self.pipeline.expires * 60 * 60 * 24 * 2),+ },+ ),+ mock.patch.object(+ FilesPipeline,+ "get_media_requests",+ return_value=[_prepare_request_object(item_url)],+ ),+ ):+ result = await pipeline.process_item(item)+ assert result["files"][0]["checksum"] == "abc"+ assert result["files"][0]["status"] == "uptodate"+@coroutine_testasync def test_file_cached(self):item_url = "http://example.com/file3.pdf"