Scrapy, a fast high-level web crawling & scraping framework for Python.
Make the media pipeline checksum algorithm configurable (#7951)
Adrianbd0f99c797aa
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex ffe1d0eea..a964ddfa9 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -562,6 +562,33 @@ and the pipeline class MyPipeline will have an expiration time set to 180.The last modified time from the file is used to determine the age of the file in days,which is then compared to the set expiration time to determine if the file is expired.+.. _checksum-algorithm:++Checksum algorithm+------------------++.. versionadded:: VERSION++.. setting:: FILES_CHECKSUM_ALGORITHM+.. setting:: IMAGES_CHECKSUM_ALGORITHM++Checksums are `MD5 hashes <MD5 hash_>`_ by default. To use a different+algorithm, set :setting:`FILES_CHECKSUM_ALGORITHM` (or+:setting:`IMAGES_CHECKSUM_ALGORITHM`, in case of the Images Pipeline) to the+name of any algorithm supported by :func:`hashlib.new`:++.. code-block:: python++ FILES_CHECKSUM_ALGORITHM = "sha256"++Amazon S3 and Google Cloud Storage cannot calculate checksums of stored files+on demand, and only report MD5 hashes, so with a different algorithm files that+are not downloaded again, because they have not expired yet, get no checksum.++A :ref:`custom storage <media-pipeline-custom-storage>` that calculates+checksums in ``stat_file`` must use the algorithm that the pipeline sets as its+``checksum_algorithm`` attribute.+.. _topics-images-thumbnails:Thumbnail generation for images@@ -771,7 +798,8 @@ See here the methods that you can override in your custom Files Pipeline:* ``path`` - the path (relative to :setting:`FILES_STORE`) where the filewas stored- * ``checksum`` - a `MD5 hash`_ of the image contents+ * ``checksum`` - a `MD5 hash`_ of the file contents, unless a different+ :ref:`checksum algorithm <checksum-algorithm>` is configured* ``status`` - the file status indication.diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 885e1eeb6..e580045a8 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -62,6 +62,7 @@ if TYPE_CHECKING:from twisted.python.failure import Failurefrom scrapy.crawler import Crawler+ from scrapy.settings import BaseSettingslogger = logging.getLogger(__name__)@@ -71,12 +72,25 @@ def _to_string(path: str | PathLike[str]) -> str:return str(path) # convert a Path object to string+def _checksum_algorithm(settings: BaseSettings, key: str) -> str:+ algorithm: str = settings.get(key, "md5")+ hashlib.new(algorithm) # fail early on an unsupported algorithm+ return algorithm++class StatInfo(TypedDict, total=False):checksum: strlast_modified: floatclass FilesStoreProtocol(Protocol):+ checksum_algorithm: str+ """Name of the :mod:`hashlib` algorithm that :meth:`stat_file` must use for+ the checksums that it calculates itself.++ It is assigned by the pipeline.+ """+def __init__(self, uri: str, /): ...def persist_file(@@ -94,6 +108,8 @@ class FilesStoreProtocol(Protocol):class FSFilesStore:+ checksum_algorithm: str = "md5"+def __init__(self, basedir: str | PathLike[str]):basedir = _to_string(basedir)if "://" in basedir:@@ -126,7 +142,7 @@ class FSFilesStore:return {}with absolute_path.open("rb") as f:- checksum = hashlib.file_digest(f, "md5").hexdigest()+ checksum = hashlib.file_digest(f, self.checksum_algorithm).hexdigest()return {"last_modified": last_modified, "checksum": checksum}@@ -145,6 +161,11 @@ class FSFilesStore:class S3FilesStore:+ # Amazon S3 cannot hash a stored object on demand. It reports the ETag,+ # which is an MD5 hash for the objects that persist_file() uploads, and the+ # checksums that were requested when the object was uploaded.+ checksum_algorithm: str = "md5"+AWS_ACCESS_KEY_ID = NoneAWS_SECRET_ACCESS_KEY = NoneAWS_SESSION_TOKEN = None@@ -213,11 +234,12 @@ class S3FilesStore:raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")self.bucket, self.prefix = uri[5:].split("/", 1)- @staticmethod- def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:- checksum = boto_key["ETag"].strip('"')+ def _onsuccess(self, boto_key: dict[str, Any]) -> StatInfo:last_modified = boto_key["LastModified"]modified_stamp = last_modified.timestamp()+ if self.checksum_algorithm != "md5":+ return {"last_modified": modified_stamp}+ checksum = boto_key["ETag"].strip('"')return {"checksum": checksum, "last_modified": modified_stamp}def stat_file(@@ -295,6 +317,10 @@ class S3FilesStore:class GCSFilesStore:+ # Google Cloud Storage cannot hash a stored object on demand. It reports an+ # MD5 hash and a CRC32C checksum.+ checksum_algorithm: str = "md5"+GCS_PROJECT_ID = NoneUPLOAD_TIMEOUT: float | None = None@@ -343,13 +369,14 @@ class GCSFilesStore:{"bucket": bucket},)- @staticmethod- def _onsuccess(blob: Any) -> StatInfo:- if blob:- checksum = base64.b64decode(blob.md5_hash).hex()- last_modified = blob.updated.timestamp()- return {"checksum": checksum, "last_modified": last_modified}- return {}+ def _onsuccess(self, blob: Any) -> StatInfo:+ if not blob:+ return {}+ last_modified = blob.updated.timestamp()+ if self.checksum_algorithm != "md5":+ return {"last_modified": last_modified}+ checksum = base64.b64decode(blob.md5_hash).hex()+ return {"checksum": checksum, "last_modified": last_modified}def stat_file(self, path: str, info: MediaPipeline.SpiderInfo@@ -395,6 +422,8 @@ class GCSFilesStore:class FTPFilesStore:+ checksum_algorithm: str = "md5"+FTP_USERNAME: str | None = NoneFTP_PASSWORD: str | None = NoneUSE_ACTIVE_MODE: bool | None = None@@ -472,7 +501,7 @@ class FTPFilesStore:.replace(tzinfo=UTC).timestamp())- m = hashlib.md5() # noqa: S324+ m = hashlib.new(self.checksum_algorithm)ftp.retrbinary(f"RETR {file_path}", m.update)return {"last_modified": last_modified, "checksum": m.hexdigest()}# The file doesn't exist@@ -565,6 +594,8 @@ class FilesPipeline(MediaPipeline):DEFAULT_FILES_URLS_FIELD: str = "file_urls"DEFAULT_FILES_RESULT_FIELD: str = "files"+ _checksum_warned: bool = False+def __init__(self,store_uri: str | PathLike[str],@@ -610,6 +641,9 @@ class FilesPipeline(MediaPipeline):resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name, settings=settings)+ self.store.checksum_algorithm = _checksum_algorithm(+ settings, resolve("FILES_CHECKSUM_ALGORITHM")+ )self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)if not hasattr(self, "FILES_URLS_FIELD"):self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD@@ -683,6 +717,15 @@ class FilesPipeline(MediaPipeline):self.inc_stats("uptodate")checksum = result.get("checksum", None)+ if checksum is None and not self._checksum_warned:+ self._checksum_warned = True+ logger.warning(+ f"{self.store.__class__.__name__} does not report "+ f"{self.store.checksum_algorithm} checksums of stored files, so "+ f"files that are not downloaded again, because they have not "+ f"expired yet, get no checksum.",+ extra={"spider": info.spider},+ )return {"url": request.url,"path": path,@@ -810,7 +853,7 @@ class FilesPipeline(MediaPipeline):) -> str:path = self.file_path(request, response=response, info=info, item=item)buf = BytesIO(response.body)- checksum = hashlib.file_digest(buf, "md5").hexdigest()+ checksum = hashlib.file_digest(buf, self.store.checksum_algorithm).hexdigest()buf.seek(0)await ensure_awaitable(self.store.persist_file(path, buf, info))return checksumdiff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex b745170ba..df8cbd8b2 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -18,7 +18,7 @@ from itemadapter import ItemAdapterfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK-from scrapy.pipelines.files import FilesPipeline+from scrapy.pipelines.files import FilesPipeline, _checksum_algorithmfrom scrapy.pipelines.media import FileExceptionfrom scrapy.utils.defer import ensure_awaitable@@ -84,6 +84,9 @@ class ImagesPipeline(FilesPipeline):base_class_name="ImagesPipeline",settings=settings,)+ self.store.checksum_algorithm = _checksum_algorithm(+ settings, resolve("IMAGES_CHECKSUM_ALGORITHM")+ )self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)if not hasattr(self, "IMAGES_RESULT_FIELD"):@@ -143,7 +146,9 @@ class ImagesPipeline(FilesPipeline):for path, image, buf in self.get_images(response, request, info, item=item):if checksum is None:buf.seek(0)- checksum = hashlib.file_digest(buf, "md5").hexdigest()+ checksum = hashlib.file_digest(+ buf, self.store.checksum_algorithm+ ).hexdigest()width, height = image.sizecontent_type = self._Image.MIME.get(image.format or "JPEG", "image/jpeg")await ensure_awaitable(diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 9c2e66b58..3aface043 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -1,5 +1,6 @@import base64import dataclasses+import hashlibimport loggingimport mimetypesimport random@@ -8,7 +9,7 @@ import timeimport warningsfrom abc import ABC, abstractmethodfrom collections.abc import Callable-from datetime import datetime, timedelta, timezone+from datetime import UTC, datetime, timedelta, timezonefrom ftplib import FTPfrom io import BytesIOfrom pathlib import Path@@ -214,6 +215,37 @@ class TestFilesPipeline:assert result["files"][0]["checksum"] == "abc"assert result["files"][0]["status"] == "uptodate"+ @coroutine_test+ async def test_file_not_expired_without_checksum(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Stores that do not report checksums, e.g. those that cannot report+ them for the configured algorithm, trigger a single warning."""+ with (+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),+ mock.patch.object(+ FSFilesStore, "stat_file", return_value={"last_modified": time.time()}+ ),+ caplog.at_level(logging.WARNING),+ ):+ for item_url in ("http://example.com/1.pdf", "http://example.com/2.pdf"):+ with mock.patch.object(+ FilesPipeline,+ "get_media_requests",+ return_value=[_prepare_request_object(item_url)],+ ):+ result = await self.pipeline.process_item(+ _create_item_with_files(item_url)+ )+ assert result["files"][0]["checksum"] is None+ records = [+ r+ for r in caplog.records+ if "does not report md5 checksums" in r.getMessage()+ ]+ assert len(records) == 1+ assert "FSFilesStore" in records[0].getMessage()+@coroutine_testasync def test_file_expired(self):item_url = "http://example.com/file2.pdf"@@ -414,6 +446,28 @@ class TestFilesPipeline:assert path.exists()assert path.read_bytes() == b"data"+ @coroutine_test+ async def test_checksum_algorithm(self) -> None:+ pipeline = self._create_pipeline(+ FilesPipeline, {"FILES_CHECKSUM_ALGORITHM": "sha256"}+ )+ item_url = "http://example.com/file.pdf"+ item = _create_item_with_files(item_url)+ with (+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),+ mock.patch.object(+ FilesPipeline,+ "get_media_requests",+ return_value=[_prepare_request_object(item_url)],+ ),+ ):+ result = await pipeline.process_item(item)+ assert result["files"][0]["checksum"] == hashlib.sha256(b"data").hexdigest()++ def test_checksum_algorithm_unsupported(self) -> None:+ with pytest.raises(ValueError, match=r"unsupported hash (type|algorithm)"):+ self._create_pipeline(FilesPipeline, {"FILES_CHECKSUM_ALGORITHM": "sha257"})+def test_file_path_from_item(self):"""Custom file path based on item data, overriding default implementation@@ -989,6 +1043,13 @@ class TestFSFilesStore:assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"assert stat["last_modified"] == pytest.approx(time.time(), abs=60)+ def test_stat_file_checksum_algorithm(self, tmp_path: Path) -> None:+ store = FSFilesStore(tmp_path)+ store.checksum_algorithm = "sha256"+ store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)+ stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)+ assert stat["checksum"] == hashlib.sha256(b"data").hexdigest()+def test_stat_missing_file(self, tmp_path: Path) -> None:store = FSFilesStore(tmp_path)assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}@@ -1163,6 +1224,32 @@ class TestS3FilesStore:"last_modified": last_modified.timestamp(),}+ @inline_callbacks_test+ def test_stat_checksum_algorithm(self):+ """Amazon S3 only reports MD5 checksums, so no checksum is reported+ when a different algorithm is configured."""+ bucket = "mybucket"+ key = "export.csv"+ # S3FilesStore needs to be fixed to emit tz-aware datetimes+ last_modified = datetime(2019, 12, 1) # noqa: DTZ001++ store = S3FilesStore(f"s3://{bucket}/{key}")+ store.checksum_algorithm = "sha256"+ from botocore.stub import Stubber # noqa: PLC0415++ with Stubber(store.s3_client) as stub:+ stub.add_response(+ "head_object",+ expected_params={"Bucket": bucket, "Key": key},+ service_response={+ "ETag": '"3187896a9657a28163abb31667df64c8"',+ "LastModified": last_modified,+ },+ )++ file_stats = yield store.stat_file("", info=DUMMY_SPIDER_INFO)+ assert file_stats == {"last_modified": last_modified.timestamp()}+stub.assert_no_pending_responses()def test_default_max_pool_connections(self) -> None:@@ -1357,6 +1444,23 @@ class TestGCSFilesStore:"last_modified": updated.timestamp(),}+ @coroutine_test+ async def test_stat_checksum_algorithm(self) -> None:+ """Google Cloud Storage only reports MD5 checksums, so no checksum is+ reported when a different algorithm is configured."""+ store, bucket, blob = self.build_gcs_files_store()+ store.checksum_algorithm = "sha256"+ blob.md5_hash = base64.b64encode(+ bytes.fromhex("cdcda85605e46d0af6110752770dce3c")+ ).decode()+ updated = datetime(2019, 12, 1, tzinfo=UTC)+ blob.updated = updated+ bucket.get_blob.return_value = blob+ stat = await maybe_deferred_to_future(+ store.stat_file("full/filename", info=DUMMY_SPIDER_INFO)+ )+ assert stat == {"last_modified": time.mktime(updated.timetuple())}+@coroutine_testasync def test_stat_missing_blob(self) -> None:store, bucket, _ = self.build_gcs_files_store()diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 873d41d26..c23bb7d58 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -259,6 +259,20 @@ class TestImagesPipeline:assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()assert Path(self.tempdir, "thumbs", "small", name).exists()+ @coroutine_test+ async def test_image_downloaded_checksum_algorithm(self) -> None:+ crawler = get_crawler(settings_dict={"IMAGES_CHECKSUM_ALGORITHM": "sha256"})+ pipeline = ImagesPipeline(self.tempdir, crawler=crawler)+ _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))+ url = "https://dev.mydeco.com/mydeco.gif"+ response = Response(url=url, body=buf.getvalue())++ checksum = await pipeline.image_downloaded(+ response, Request(url=url), DUMMY_SPIDER_INFO+ )++ assert checksum == hashlib.sha256(buf.getvalue()).hexdigest()+def test_convert_image_preserve_format(self):pipeline = build_from_crawler(ImagesPipeline,