Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Derive S3 media pipeline header mapping from botocore (#8171)

Adriana12937e59f86
diff --git a/AGENTS.md b/AGENTS.md
index 7e1bfba2a..319142314 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -11,6 +11,11 @@ This file adds what agents tend to get wrong.
docstrings. CI runs these first and skips the test matrix when they fail.
- Run the tests you touched: `tox -e py -- tests/test_foo.py`. Anything after
`tox -e py --` is passed to pytest.
+- If the change depends on what a dependency exposes, run the tests under the
+ matching `min-` environment too (see `tox.ini`), e.g.
+ `tox -e min-botocore -- tests/test_foo.py`. They pin the oldest supported
+ version, which can be years older than the one you have and expose less, so
+ assert only on what it has as well.
- Review your own diff. Remove whatever the change does not need: unused
parameters, dead code, leftover debugging, comments that restate the code.
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 190f6bd84..547443091 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -317,6 +317,28 @@ policy:
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
+To send custom headers with the uploaded files, subclass ``S3FilesStore``
+with a different ``HEADERS`` class attribute, and point the ``s3`` scheme of
+your pipeline to it:
+
+.. code-block:: python
+
+ from scrapy.pipelines.files import FilesPipeline, S3FilesStore
+
+
+ class CustomS3FilesStore(S3FilesStore):
+ HEADERS = {
+ "Cache-Control": "max-age=172800",
+ "X-Amz-Storage-Class": "STANDARD_IA",
+ }
+
+
+ class CustomFilesPipeline(FilesPipeline):
+ STORE_SCHEMES = FilesPipeline.STORE_SCHEMES | {"s3": CustomS3FilesStore}
+
+Supported headers are those that the installed botocore version models for
+the `PutObject`_ operation; any other header raises :exc:`TypeError`.
+
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`Zenko CloudServer`_. All you need to do is set the endpoint option in your
Scrapy settings:
@@ -337,6 +359,7 @@ To reuse connections for as many files as you check or upload in parallel, set
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
.. _Minio: https://github.com/minio/minio
+.. _PutObject: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
.. _Zenko CloudServer: https://www.zenko.io/cloudserver/
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 6c37fef22..df5833a60 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -226,50 +226,38 @@ class S3FilesStore:
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
+ kwargs: dict[str, Any] = {
+ "Metadata": {k: str(v) for k, v in meta.items()} if meta else {},
+ "ACL": self.POLICY,
+ **extra,
+ }
return deferred_from_coro(
run_in_thread(
self.s3_client.put_object, # type: ignore[attr-defined]
Bucket=self.bucket,
Key=key_name,
Body=buf,
- Metadata={k: str(v) for k, v in meta.items()} if meta else {},
- ACL=self.POLICY,
- **extra,
+ **kwargs,
)
)
- def _headers_to_botocore_kwargs(self, headers: dict[str, str]) -> dict[str, str]:
- """Convert headers to botocore keyword arguments."""
- # This is required while we need to support both boto and botocore.
- mapping = CaseInsensitiveDict(
+ @functools.cached_property
+ def _botocore_header_kwargs(self) -> CaseInsensitiveDict:
+ input_shape = self.s3_client.meta.service_model.operation_model(
+ "PutObject"
+ ).input_shape
+ assert input_shape is not None
+ return CaseInsensitiveDict(
{
- "Content-Type": "ContentType",
- "Cache-Control": "CacheControl",
- "Content-Disposition": "ContentDisposition",
- "Content-Encoding": "ContentEncoding",
- "Content-Language": "ContentLanguage",
- "Content-Length": "ContentLength",
- "Content-MD5": "ContentMD5",
- "Expires": "Expires",
- "X-Amz-Grant-Full-Control": "GrantFullControl",
- "X-Amz-Grant-Read": "GrantRead",
- "X-Amz-Grant-Read-ACP": "GrantReadACP",
- "X-Amz-Grant-Write-ACP": "GrantWriteACP",
- "X-Amz-Object-Lock-Legal-Hold": "ObjectLockLegalHoldStatus",
- "X-Amz-Object-Lock-Mode": "ObjectLockMode",
- "X-Amz-Object-Lock-Retain-Until-Date": "ObjectLockRetainUntilDate",
- "X-Amz-Request-Payer": "RequestPayer",
- "X-Amz-Server-Side-Encryption": "ServerSideEncryption",
- "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "SSEKMSKeyId",
- "X-Amz-Server-Side-Encryption-Context": "SSEKMSEncryptionContext",
- "X-Amz-Server-Side-Encryption-Customer-Algorithm": "SSECustomerAlgorithm",
- "X-Amz-Server-Side-Encryption-Customer-Key": "SSECustomerKey",
- "X-Amz-Server-Side-Encryption-Customer-Key-Md5": "SSECustomerKeyMD5",
- "X-Amz-Storage-Class": "StorageClass",
- "X-Amz-Tagging": "Tagging",
- "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",
+ shape.serialization["name"]: name
+ for name, shape in input_shape.members.items()
+ if shape.serialization.get("location") == "header"
}
)
+
+ def _headers_to_botocore_kwargs(self, headers: dict[str, str]) -> dict[str, str]:
+ """Convert headers to botocore keyword arguments."""
+ mapping = self._botocore_header_kwargs
extra: dict[str, Any] = {}
for key, value in headers.items():
try:
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 1b8fbb662..35763a5ff 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -922,6 +922,39 @@ class TestS3FilesStore:
stub.assert_no_pending_responses()
+ @inline_callbacks_test
+ def test_persist_acl_header(self):
+ """A header overrides the matching put_object argument."""
+ bucket = "mybucket"
+ key = "export.csv"
+ buffer = mock.MagicMock()
+
+ store = S3FilesStore(f"s3://{bucket}/{key}")
+ from botocore.stub import Stubber # noqa: PLC0415
+
+ with Stubber(store.s3_client) as stub:
+ stub.add_response(
+ "put_object",
+ expected_params={
+ "ACL": "public-read",
+ "Body": buffer,
+ "Bucket": bucket,
+ "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+ "Key": key,
+ "Metadata": {},
+ },
+ service_response={},
+ )
+
+ yield store.persist_file(
+ "",
+ buffer,
+ info=DUMMY_SPIDER_INFO,
+ headers={"x-amz-acl": "public-read"},
+ )
+
+ stub.assert_no_pending_responses()
+
def test_missing_botocore(self):
with (
mock.patch(
@@ -947,6 +980,12 @@ class TestS3FilesStore:
):
store._headers_to_botocore_kwargs({"X-Custom": "value"})
+ def test_headers_to_botocore_kwargs(self):
+ store = S3FilesStore("s3://mybucket/key")
+ assert store._headers_to_botocore_kwargs(
+ {"cache-control": "max-age=1", "X-AMZ-TAGGING": "a=b"}
+ ) == {"CacheControl": "max-age=1", "Tagging": "a=b"}
+
@inline_callbacks_test
def test_stat(self):
bucket = "mybucket"