Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make the media pipeline checksum algorithm configurable (#7951)

Adrianbd0f99c797aa
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index ffe1d0eea..a964ddfa9 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -562,6 +562,33 @@ and the pipeline class MyPipeline will have an expiration time set to 180.
The last modified time from the file is used to determine the age of the file in days,
which is then compared to the set expiration time to determine if the file is expired.
+.. _checksum-algorithm:
+
+Checksum algorithm
+------------------
+
+.. versionadded:: VERSION
+
+.. setting:: FILES_CHECKSUM_ALGORITHM
+.. setting:: IMAGES_CHECKSUM_ALGORITHM
+
+Checksums are `MD5 hashes <MD5 hash_>`_ by default. To use a different
+algorithm, set :setting:`FILES_CHECKSUM_ALGORITHM` (or
+:setting:`IMAGES_CHECKSUM_ALGORITHM`, in case of the Images Pipeline) to the
+name of any algorithm supported by :func:`hashlib.new`:
+
+.. code-block:: python
+
+ FILES_CHECKSUM_ALGORITHM = "sha256"
+
+Amazon S3 and Google Cloud Storage cannot calculate checksums of stored files
+on demand, and only report MD5 hashes, so with a different algorithm files that
+are not downloaded again, because they have not expired yet, get no checksum.
+
+A :ref:`custom storage <media-pipeline-custom-storage>` that calculates
+checksums in ``stat_file`` must use the algorithm that the pipeline sets as its
+``checksum_algorithm`` attribute.
+
.. _topics-images-thumbnails:
Thumbnail generation for images
@@ -771,7 +798,8 @@ See here the methods that you can override in your custom Files Pipeline:
* ``path`` - the path (relative to :setting:`FILES_STORE`) where the file
was stored
- * ``checksum`` - a `MD5 hash`_ of the image contents
+ * ``checksum`` - a `MD5 hash`_ of the file contents, unless a different
+ :ref:`checksum algorithm <checksum-algorithm>` is configured
* ``status`` - the file status indication.
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 885e1eeb6..e580045a8 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -62,6 +62,7 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@@ -71,12 +72,25 @@ def _to_string(path: str | PathLike[str]) -> str:
return str(path) # convert a Path object to string
+def _checksum_algorithm(settings: BaseSettings, key: str) -> str:
+ algorithm: str = settings.get(key, "md5")
+ hashlib.new(algorithm) # fail early on an unsupported algorithm
+ return algorithm
+
+
class StatInfo(TypedDict, total=False):
checksum: str
last_modified: float
class FilesStoreProtocol(Protocol):
+ checksum_algorithm: str
+ """Name of the :mod:`hashlib` algorithm that :meth:`stat_file` must use for
+ the checksums that it calculates itself.
+
+ It is assigned by the pipeline.
+ """
+
def __init__(self, uri: str, /): ...
def persist_file(
@@ -94,6 +108,8 @@ class FilesStoreProtocol(Protocol):
class FSFilesStore:
+ checksum_algorithm: str = "md5"
+
def __init__(self, basedir: str | PathLike[str]):
basedir = _to_string(basedir)
if "://" in basedir:
@@ -126,7 +142,7 @@ class FSFilesStore:
return {}
with absolute_path.open("rb") as f:
- checksum = hashlib.file_digest(f, "md5").hexdigest()
+ checksum = hashlib.file_digest(f, self.checksum_algorithm).hexdigest()
return {"last_modified": last_modified, "checksum": checksum}
@@ -145,6 +161,11 @@ class FSFilesStore:
class S3FilesStore:
+ # Amazon S3 cannot hash a stored object on demand. It reports the ETag,
+ # which is an MD5 hash for the objects that persist_file() uploads, and the
+ # checksums that were requested when the object was uploaded.
+ checksum_algorithm: str = "md5"
+
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
AWS_SESSION_TOKEN = None
@@ -213,11 +234,12 @@ class S3FilesStore:
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split("/", 1)
- @staticmethod
- def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
- checksum = boto_key["ETag"].strip('"')
+ def _onsuccess(self, boto_key: dict[str, Any]) -> StatInfo:
last_modified = boto_key["LastModified"]
modified_stamp = last_modified.timestamp()
+ if self.checksum_algorithm != "md5":
+ return {"last_modified": modified_stamp}
+ checksum = boto_key["ETag"].strip('"')
return {"checksum": checksum, "last_modified": modified_stamp}
def stat_file(
@@ -295,6 +317,10 @@ class S3FilesStore:
class GCSFilesStore:
+ # Google Cloud Storage cannot hash a stored object on demand. It reports an
+ # MD5 hash and a CRC32C checksum.
+ checksum_algorithm: str = "md5"
+
GCS_PROJECT_ID = None
UPLOAD_TIMEOUT: float | None = None
@@ -343,13 +369,14 @@ class GCSFilesStore:
{"bucket": bucket},
)
- @staticmethod
- def _onsuccess(blob: Any) -> StatInfo:
- if blob:
- checksum = base64.b64decode(blob.md5_hash).hex()
- last_modified = blob.updated.timestamp()
- return {"checksum": checksum, "last_modified": last_modified}
- return {}
+ def _onsuccess(self, blob: Any) -> StatInfo:
+ if not blob:
+ return {}
+ last_modified = blob.updated.timestamp()
+ if self.checksum_algorithm != "md5":
+ return {"last_modified": last_modified}
+ checksum = base64.b64decode(blob.md5_hash).hex()
+ return {"checksum": checksum, "last_modified": last_modified}
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
@@ -395,6 +422,8 @@ class GCSFilesStore:
class FTPFilesStore:
+ checksum_algorithm: str = "md5"
+
FTP_USERNAME: str | None = None
FTP_PASSWORD: str | None = None
USE_ACTIVE_MODE: bool | None = None
@@ -472,7 +501,7 @@ class FTPFilesStore:
.replace(tzinfo=UTC)
.timestamp()
)
- m = hashlib.md5() # noqa: S324
+ m = hashlib.new(self.checksum_algorithm)
ftp.retrbinary(f"RETR {file_path}", m.update)
return {"last_modified": last_modified, "checksum": m.hexdigest()}
# The file doesn't exist
@@ -565,6 +594,8 @@ class FilesPipeline(MediaPipeline):
DEFAULT_FILES_URLS_FIELD: str = "file_urls"
DEFAULT_FILES_RESULT_FIELD: str = "files"
+ _checksum_warned: bool = False
+
def __init__(
self,
store_uri: str | PathLike[str],
@@ -610,6 +641,9 @@ class FilesPipeline(MediaPipeline):
resolve = functools.partial(
self._key_for_pipe, base_class_name=cls_name, settings=settings
)
+ self.store.checksum_algorithm = _checksum_algorithm(
+ settings, resolve("FILES_CHECKSUM_ALGORITHM")
+ )
self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
if not hasattr(self, "FILES_URLS_FIELD"):
self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
@@ -683,6 +717,15 @@ class FilesPipeline(MediaPipeline):
self.inc_stats("uptodate")
checksum = result.get("checksum", None)
+ if checksum is None and not self._checksum_warned:
+ self._checksum_warned = True
+ logger.warning(
+ f"{self.store.__class__.__name__} does not report "
+ f"{self.store.checksum_algorithm} checksums of stored files, so "
+ f"files that are not downloaded again, because they have not "
+ f"expired yet, get no checksum.",
+ extra={"spider": info.spider},
+ )
return {
"url": request.url,
"path": path,
@@ -810,7 +853,7 @@ class FilesPipeline(MediaPipeline):
) -> str:
path = self.file_path(request, response=response, info=info, item=item)
buf = BytesIO(response.body)
- checksum = hashlib.file_digest(buf, "md5").hexdigest()
+ checksum = hashlib.file_digest(buf, self.store.checksum_algorithm).hexdigest()
buf.seek(0)
await ensure_awaitable(self.store.persist_file(path, buf, info))
return checksum
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index b745170ba..df8cbd8b2 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -18,7 +18,7 @@ from itemadapter import ItemAdapter
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import FilesPipeline
+from scrapy.pipelines.files import FilesPipeline, _checksum_algorithm
from scrapy.pipelines.media import FileException
from scrapy.utils.defer import ensure_awaitable
@@ -84,6 +84,9 @@ class ImagesPipeline(FilesPipeline):
base_class_name="ImagesPipeline",
settings=settings,
)
+ self.store.checksum_algorithm = _checksum_algorithm(
+ settings, resolve("IMAGES_CHECKSUM_ALGORITHM")
+ )
self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
if not hasattr(self, "IMAGES_RESULT_FIELD"):
@@ -143,7 +146,9 @@ class ImagesPipeline(FilesPipeline):
for path, image, buf in self.get_images(response, request, info, item=item):
if checksum is None:
buf.seek(0)
- checksum = hashlib.file_digest(buf, "md5").hexdigest()
+ checksum = hashlib.file_digest(
+ buf, self.store.checksum_algorithm
+ ).hexdigest()
width, height = image.size
content_type = self._Image.MIME.get(image.format or "JPEG", "image/jpeg")
await ensure_awaitable(
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 9c2e66b58..3aface043 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,5 +1,6 @@
import base64
import dataclasses
+import hashlib
import logging
import mimetypes
import random
@@ -8,7 +9,7 @@ import time
import warnings
from abc import ABC, abstractmethod
from collections.abc import Callable
-from datetime import datetime, timedelta, timezone
+from datetime import UTC, datetime, timedelta, timezone
from ftplib import FTP
from io import BytesIO
from pathlib import Path
@@ -214,6 +215,37 @@ class TestFilesPipeline:
assert result["files"][0]["checksum"] == "abc"
assert result["files"][0]["status"] == "uptodate"
+ @coroutine_test
+ async def test_file_not_expired_without_checksum(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Stores that do not report checksums, e.g. those that cannot report
+ them for the configured algorithm, trigger a single warning."""
+ with (
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(
+ FSFilesStore, "stat_file", return_value={"last_modified": time.time()}
+ ),
+ caplog.at_level(logging.WARNING),
+ ):
+ for item_url in ("http://example.com/1.pdf", "http://example.com/2.pdf"):
+ with mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ):
+ result = await self.pipeline.process_item(
+ _create_item_with_files(item_url)
+ )
+ assert result["files"][0]["checksum"] is None
+ records = [
+ r
+ for r in caplog.records
+ if "does not report md5 checksums" in r.getMessage()
+ ]
+ assert len(records) == 1
+ assert "FSFilesStore" in records[0].getMessage()
+
@coroutine_test
async def test_file_expired(self):
item_url = "http://example.com/file2.pdf"
@@ -414,6 +446,28 @@ class TestFilesPipeline:
assert path.exists()
assert path.read_bytes() == b"data"
+ @coroutine_test
+ async def test_checksum_algorithm(self) -> None:
+ pipeline = self._create_pipeline(
+ FilesPipeline, {"FILES_CHECKSUM_ALGORITHM": "sha256"}
+ )
+ item_url = "http://example.com/file.pdf"
+ item = _create_item_with_files(item_url)
+ with (
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ ):
+ result = await pipeline.process_item(item)
+ assert result["files"][0]["checksum"] == hashlib.sha256(b"data").hexdigest()
+
+ def test_checksum_algorithm_unsupported(self) -> None:
+ with pytest.raises(ValueError, match=r"unsupported hash (type|algorithm)"):
+ self._create_pipeline(FilesPipeline, {"FILES_CHECKSUM_ALGORITHM": "sha257"})
+
def test_file_path_from_item(self):
"""
Custom file path based on item data, overriding default implementation
@@ -989,6 +1043,13 @@ class TestFSFilesStore:
assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"
assert stat["last_modified"] == pytest.approx(time.time(), abs=60)
+ def test_stat_file_checksum_algorithm(self, tmp_path: Path) -> None:
+ store = FSFilesStore(tmp_path)
+ store.checksum_algorithm = "sha256"
+ store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)
+ stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)
+ assert stat["checksum"] == hashlib.sha256(b"data").hexdigest()
+
def test_stat_missing_file(self, tmp_path: Path) -> None:
store = FSFilesStore(tmp_path)
assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}
@@ -1163,6 +1224,32 @@ class TestS3FilesStore:
"last_modified": last_modified.timestamp(),
}
+ @inline_callbacks_test
+ def test_stat_checksum_algorithm(self):
+ """Amazon S3 only reports MD5 checksums, so no checksum is reported
+ when a different algorithm is configured."""
+ bucket = "mybucket"
+ key = "export.csv"
+ # S3FilesStore needs to be fixed to emit tz-aware datetimes
+ last_modified = datetime(2019, 12, 1) # noqa: DTZ001
+
+ store = S3FilesStore(f"s3://{bucket}/{key}")
+ store.checksum_algorithm = "sha256"
+ from botocore.stub import Stubber # noqa: PLC0415
+
+ with Stubber(store.s3_client) as stub:
+ stub.add_response(
+ "head_object",
+ expected_params={"Bucket": bucket, "Key": key},
+ service_response={
+ "ETag": '"3187896a9657a28163abb31667df64c8"',
+ "LastModified": last_modified,
+ },
+ )
+
+ file_stats = yield store.stat_file("", info=DUMMY_SPIDER_INFO)
+ assert file_stats == {"last_modified": last_modified.timestamp()}
+
stub.assert_no_pending_responses()
def test_default_max_pool_connections(self) -> None:
@@ -1357,6 +1444,23 @@ class TestGCSFilesStore:
"last_modified": updated.timestamp(),
}
+ @coroutine_test
+ async def test_stat_checksum_algorithm(self) -> None:
+ """Google Cloud Storage only reports MD5 checksums, so no checksum is
+ reported when a different algorithm is configured."""
+ store, bucket, blob = self.build_gcs_files_store()
+ store.checksum_algorithm = "sha256"
+ blob.md5_hash = base64.b64encode(
+ bytes.fromhex("cdcda85605e46d0af6110752770dce3c")
+ ).decode()
+ updated = datetime(2019, 12, 1, tzinfo=UTC)
+ blob.updated = updated
+ bucket.get_blob.return_value = blob
+ stat = await maybe_deferred_to_future(
+ store.stat_file("full/filename", info=DUMMY_SPIDER_INFO)
+ )
+ assert stat == {"last_modified": time.mktime(updated.timetuple())}
+
@coroutine_test
async def test_stat_missing_blob(self) -> None:
store, bucket, _ = self.build_gcs_files_store()
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 873d41d26..c23bb7d58 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -259,6 +259,20 @@ class TestImagesPipeline:
assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()
assert Path(self.tempdir, "thumbs", "small", name).exists()
+ @coroutine_test
+ async def test_image_downloaded_checksum_algorithm(self) -> None:
+ crawler = get_crawler(settings_dict={"IMAGES_CHECKSUM_ALGORITHM": "sha256"})
+ pipeline = ImagesPipeline(self.tempdir, crawler=crawler)
+ _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+ url = "https://dev.mydeco.com/mydeco.gif"
+ response = Response(url=url, body=buf.getvalue())
+
+ checksum = await pipeline.image_downloaded(
+ response, Request(url=url), DUMMY_SPIDER_INFO
+ )
+
+ assert checksum == hashlib.sha256(buf.getvalue()).hexdigest()
+
def test_convert_image_preserve_format(self):
pipeline = build_from_crawler(
ImagesPipeline,