Scrapy, a fast high-level web crawling & scraping framework for Python.
FILES_EXPIRES less than 1 day (#4731)
gaodenge1111a0fee94
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex e43b5e698..f70d52111 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -539,13 +539,16 @@ adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or:setting:`IMAGES_EXPIRES`, in case of Images Pipeline), whichspecifies the delay in number of days:+.. versionchanged:: VERSION+ Fractional numbers of days are now supported.+.. code-block:: python# 120 days of delay for files expirationFILES_EXPIRES = 120- # 30 days of delay for images expiration- IMAGES_EXPIRES = 30+ # 12 hours of delay for images expiration+ IMAGES_EXPIRES = 0.5The default value for both settings is 90 days. A negative value disablesexpiration, i.e. files are never re-downloaded once fetched.diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 851c291ce..dbbe86188 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -691,7 +691,7 @@ class FilesPipeline(MediaPipeline):self.store.checksum_algorithm = _checksum_algorithm(settings, resolve("FILES_CHECKSUM_ALGORITHM"))- self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)+ self.expires: float = settings.getfloat(resolve("FILES_EXPIRES"), self.EXPIRES)if not hasattr(self, "FILES_URLS_FIELD"):self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELDif not hasattr(self, "FILES_RESULT_FIELD"):diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex df8cbd8b2..e4c73e09a 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -87,7 +87,7 @@ class ImagesPipeline(FilesPipeline):self.store.checksum_algorithm = _checksum_algorithm(settings, resolve("IMAGES_CHECKSUM_ALGORITHM"))- self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)+ self.expires: float = settings.getfloat(resolve("IMAGES_EXPIRES"), self.EXPIRES)if not hasattr(self, "IMAGES_RESULT_FIELD"):self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELDdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex e500ad8fa..f04306ad8 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -283,6 +283,35 @@ class TestFilesPipeline:assert result["files"][0]["checksum"] != "abc"assert result["files"][0]["status"] == "downloaded"+ @pytest.mark.parametrize(+ ("age_hours", "expected_status"), [(6, "uptodate"), (18, "downloaded")]+ )+ @coroutine_test+ async def test_file_expiration_below_one_day(+ self, age_hours: int, expected_status: str+ ) -> None:+ pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": 0.5})+ item_url = f"http://example.com/file-{age_hours}h.pdf"+ item = _create_item_with_files(item_url)+ with (+ mock.patch.object(+ FSFilesStore,+ "stat_file",+ return_value={+ "checksum": "abc",+ "last_modified": time.time() - age_hours * 60 * 60,+ },+ ),+ mock.patch.object(+ FilesPipeline,+ "get_media_requests",+ return_value=[_prepare_request_object(item_url)],+ ),+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),+ ):+ result = await pipeline.process_item(item)+ assert result["files"][0]["status"] == expected_status+@coroutine_testasync def test_file_never_expires(self):pipeline = self._create_pipeline(FilesPipeline, {"FILES_EXPIRES": -1})