Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support disabling file/image expiration with a negative FILES_EXPIRES/IMAGES_EXPIRES (#8046)

Adrian29484a1e2f62
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index d031fa2b7..362480515 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -462,7 +462,8 @@ specifies the delay in number of days:
# 30 days of delay for images expiration
IMAGES_EXPIRES = 30
-The default value for both settings is 90 days.
+The default value for both settings is 90 days. A negative value disables
+expiration, i.e. files are never re-downloaded once fetched.
If you have a pipeline that subclasses FilesPipeline and you'd like to have a
different setting for it, you can set setting keys preceded by the uppercase
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index b3c44dee9..2170ae4b2 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -568,7 +568,7 @@ class FilesPipeline(MediaPipeline):
age_seconds = time.time() - last_modified
age_days = age_seconds / 60 / 60 / 24
- if age_days > self.expires:
+ if self.expires >= 0 and age_days > self.expires:
return None # returning None force download
referer = referer_str(request)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index d493d2b46..84cfa6c61 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -237,6 +237,32 @@ class TestFilesPipeline:
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "downloaded"
+ @coroutine_test
+ async def test_file_never_expires(self):
+ pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": -1})
+ item_url = "http://example.com/file5.pdf"
+ item = _create_item_with_files(item_url)
+ with (
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(
+ FSFilesStore,
+ "stat_file",
+ return_value={
+ "checksum": "abc",
+ "last_modified": time.time()
+ - (self.pipeline.expires * 60 * 60 * 24 * 2),
+ },
+ ),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ ):
+ result = await pipeline.process_item(item)
+ assert result["files"][0]["checksum"] == "abc"
+ assert result["files"][0]["status"] == "uptodate"
+
@coroutine_test
async def test_file_cached(self):
item_url = "http://example.com/file3.pdf"