Scrapy, a fast high-level web crawling & scraping framework for Python.
Derive S3 media pipeline header mapping from botocore (#8171)
Adriana12937e59f86
diff --git a/AGENTS.md b/AGENTS.mdindex 7e1bfba2a..319142314 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -11,6 +11,11 @@ This file adds what agents tend to get wrong.docstrings. CI runs these first and skips the test matrix when they fail.- Run the tests you touched: `tox -e py -- tests/test_foo.py`. Anything after`tox -e py --` is passed to pytest.+- If the change depends on what a dependency exposes, run the tests under the+ matching `min-` environment too (see `tox.ini`), e.g.+ `tox -e min-botocore -- tests/test_foo.py`. They pin the oldest supported+ version, which can be years older than the one you have and expose less, so+ assert only on what it has as well.- Review your own diff. Remove whatever the change does not need: unusedparameters, dead code, leftover debugging, comments that restate the code.diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex 190f6bd84..547443091 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -317,6 +317,28 @@ policy:For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.+To send custom headers with the uploaded files, subclass ``S3FilesStore``+with a different ``HEADERS`` class attribute, and point the ``s3`` scheme of+your pipeline to it:++.. code-block:: python++ from scrapy.pipelines.files import FilesPipeline, S3FilesStore+++ class CustomS3FilesStore(S3FilesStore):+ HEADERS = {+ "Cache-Control": "max-age=172800",+ "X-Amz-Storage-Class": "STANDARD_IA",+ }+++ class CustomFilesPipeline(FilesPipeline):+ STORE_SCHEMES = FilesPipeline.STORE_SCHEMES | {"s3": CustomS3FilesStore}++Supported headers are those that the installed botocore version models for+the `PutObject`_ operation; any other header raises :exc:`TypeError`.+You can also use other S3-like storages. Storages like self-hosted `Minio`_ or`Zenko CloudServer`_. All you need to do is set the endpoint option in yourScrapy settings:@@ -337,6 +359,7 @@ To reuse connections for as many files as you check or upload in parallel, set.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl.. _Minio: https://github.com/minio/minio+.. _PutObject: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html.. _Zenko CloudServer: https://www.zenko.io/cloudserver/diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 6c37fef22..df5833a60 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -226,50 +226,38 @@ class S3FilesStore:extra = self._headers_to_botocore_kwargs(self.HEADERS)if headers:extra.update(self._headers_to_botocore_kwargs(headers))+ kwargs: dict[str, Any] = {+ "Metadata": {k: str(v) for k, v in meta.items()} if meta else {},+ "ACL": self.POLICY,+ **extra,+ }return deferred_from_coro(run_in_thread(self.s3_client.put_object, # type: ignore[attr-defined]Bucket=self.bucket,Key=key_name,Body=buf,- Metadata={k: str(v) for k, v in meta.items()} if meta else {},- ACL=self.POLICY,- **extra,+ **kwargs,))- def _headers_to_botocore_kwargs(self, headers: dict[str, str]) -> dict[str, str]:- """Convert headers to botocore keyword arguments."""- # This is required while we need to support both boto and botocore.- mapping = CaseInsensitiveDict(+ @functools.cached_property+ def _botocore_header_kwargs(self) -> CaseInsensitiveDict:+ input_shape = self.s3_client.meta.service_model.operation_model(+ "PutObject"+ ).input_shape+ assert input_shape is not None+ return CaseInsensitiveDict({- "Content-Type": "ContentType",- "Cache-Control": "CacheControl",- "Content-Disposition": "ContentDisposition",- "Content-Encoding": "ContentEncoding",- "Content-Language": "ContentLanguage",- "Content-Length": "ContentLength",- "Content-MD5": "ContentMD5",- "Expires": "Expires",- "X-Amz-Grant-Full-Control": "GrantFullControl",- "X-Amz-Grant-Read": "GrantRead",- "X-Amz-Grant-Read-ACP": "GrantReadACP",- "X-Amz-Grant-Write-ACP": "GrantWriteACP",- "X-Amz-Object-Lock-Legal-Hold": "ObjectLockLegalHoldStatus",- "X-Amz-Object-Lock-Mode": "ObjectLockMode",- "X-Amz-Object-Lock-Retain-Until-Date": "ObjectLockRetainUntilDate",- "X-Amz-Request-Payer": "RequestPayer",- "X-Amz-Server-Side-Encryption": "ServerSideEncryption",- "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "SSEKMSKeyId",- "X-Amz-Server-Side-Encryption-Context": "SSEKMSEncryptionContext",- "X-Amz-Server-Side-Encryption-Customer-Algorithm": "SSECustomerAlgorithm",- "X-Amz-Server-Side-Encryption-Customer-Key": "SSECustomerKey",- "X-Amz-Server-Side-Encryption-Customer-Key-Md5": "SSECustomerKeyMD5",- "X-Amz-Storage-Class": "StorageClass",- "X-Amz-Tagging": "Tagging",- "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",+ shape.serialization["name"]: name+ for name, shape in input_shape.members.items()+ if shape.serialization.get("location") == "header"})++ def _headers_to_botocore_kwargs(self, headers: dict[str, str]) -> dict[str, str]:+ """Convert headers to botocore keyword arguments."""+ mapping = self._botocore_header_kwargsextra: dict[str, Any] = {}for key, value in headers.items():try:diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 1b8fbb662..35763a5ff 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -922,6 +922,39 @@ class TestS3FilesStore:stub.assert_no_pending_responses()+ @inline_callbacks_test+ def test_persist_acl_header(self):+ """A header overrides the matching put_object argument."""+ bucket = "mybucket"+ key = "export.csv"+ buffer = mock.MagicMock()++ store = S3FilesStore(f"s3://{bucket}/{key}")+ from botocore.stub import Stubber # noqa: PLC0415++ with Stubber(store.s3_client) as stub:+ stub.add_response(+ "put_object",+ expected_params={+ "ACL": "public-read",+ "Body": buffer,+ "Bucket": bucket,+ "CacheControl": S3FilesStore.HEADERS["Cache-Control"],+ "Key": key,+ "Metadata": {},+ },+ service_response={},+ )++ yield store.persist_file(+ "",+ buffer,+ info=DUMMY_SPIDER_INFO,+ headers={"x-amz-acl": "public-read"},+ )++ stub.assert_no_pending_responses()+def test_missing_botocore(self):with (mock.patch(@@ -947,6 +980,12 @@ class TestS3FilesStore:):store._headers_to_botocore_kwargs({"X-Custom": "value"})+ def test_headers_to_botocore_kwargs(self):+ store = S3FilesStore("s3://mybucket/key")+ assert store._headers_to_botocore_kwargs(+ {"cache-control": "max-age=1", "X-AMZ-TAGGING": "a=b"}+ ) == {"CacheControl": "max-age=1", "Tagging": "a=b"}+@inline_callbacks_testdef test_stat(self):bucket = "mybucket"