Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

FILES_EXPIRES less than 1 day (#4731)

gaodenge1111a0fee94
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index e43b5e698..f70d52111 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -539,13 +539,16 @@ adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or
:setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which
specifies the delay in number of days:
+.. versionchanged:: VERSION
+ Fractional numbers of days are now supported.
+
.. code-block:: python
# 120 days of delay for files expiration
FILES_EXPIRES = 120
- # 30 days of delay for images expiration
- IMAGES_EXPIRES = 30
+ # 12 hours of delay for images expiration
+ IMAGES_EXPIRES = 0.5
The default value for both settings is 90 days. A negative value disables
expiration, i.e. files are never re-downloaded once fetched.
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 851c291ce..dbbe86188 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -691,7 +691,7 @@ class FilesPipeline(MediaPipeline):
self.store.checksum_algorithm = _checksum_algorithm(
settings, resolve("FILES_CHECKSUM_ALGORITHM")
)
- self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
+ self.expires: float = settings.getfloat(resolve("FILES_EXPIRES"), self.EXPIRES)
if not hasattr(self, "FILES_URLS_FIELD"):
self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
if not hasattr(self, "FILES_RESULT_FIELD"):
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index df8cbd8b2..e4c73e09a 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -87,7 +87,7 @@ class ImagesPipeline(FilesPipeline):
self.store.checksum_algorithm = _checksum_algorithm(
settings, resolve("IMAGES_CHECKSUM_ALGORITHM")
)
- self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
+ self.expires: float = settings.getfloat(resolve("IMAGES_EXPIRES"), self.EXPIRES)
if not hasattr(self, "IMAGES_RESULT_FIELD"):
self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e500ad8fa..f04306ad8 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -283,6 +283,35 @@ class TestFilesPipeline:
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "downloaded"
+ @pytest.mark.parametrize(
+ ("age_hours", "expected_status"), [(6, "uptodate"), (18, "downloaded")]
+ )
+ @coroutine_test
+ async def test_file_expiration_below_one_day(
+ self, age_hours: int, expected_status: str
+ ) -> None:
+ pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": 0.5})
+ item_url = f"http://example.com/file-{age_hours}h.pdf"
+ item = _create_item_with_files(item_url)
+ with (
+ mock.patch.object(
+ FSFilesStore,
+ "stat_file",
+ return_value={
+ "checksum": "abc",
+ "last_modified": time.time() - age_hours * 60 * 60,
+ },
+ ),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ ):
+ result = await pipeline.process_item(item)
+ assert result["files"][0]["status"] == expected_status
+
@coroutine_test
async def test_file_never_expires(self):
pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": -1})