Scrapy, a fast high-level web crawling & scraping framework for Python.
Use aiobotocore in S3FilesStore when available (#8339)
Adrianaaf2b31cabf2
diff --git a/conftest.py b/conftest.pyindex 9a9ed20ce..290dfd8a8 100644--- a/conftest.py+++ b/conftest.py@@ -279,6 +279,7 @@ def pytest_runtest_setup(item):# Skip tests requiring optional dependenciesoptional_deps = ["uvloop",+ "aiobotocore","botocore","boto3","psutil",diff --git a/pyproject.toml b/pyproject.tomlindex 74645df47..1000f6f1d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -143,6 +143,7 @@ implicit_reexport = true# usually no type hints[[tool.mypy.overrides]]module = [+ "aiobotocore.*","bpython","brotli","brotlicffi",@@ -296,6 +297,7 @@ markers = ["requires_reactor: marks tests that require a reactor","requires_uvloop: marks tests as only enabled when uvloop is known to be working","requires_botocore: marks tests that need botocore (but not boto3)",+ "requires_aiobotocore: marks tests that need aiobotocore","requires_boto3: marks tests that need botocore and boto3","requires_psutil: marks tests that need psutil","requires_mitmproxy: marks tests that need a mitmdump executable",diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 67dad7651..851c291ce 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -6,6 +6,7 @@ See documentation in topics/media-pipeline.rstfrom __future__ import annotations+import asyncioimport base64import functoolsimport hashlib@@ -46,8 +47,13 @@ from scrapy.pipelines.media import ()from scrapy.utils._datatypes import CaseInsensitiveDictfrom scrapy.utils._ftp import ftp_store_file-from scrapy.utils.asyncio import run_in_thread-from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available+from scrapy.utils.asyncio import is_asyncio_available, run_in_thread+from scrapy.utils.boto import (+ _get_max_pool_connections,+ is_aiobotocore_available,+ is_botocore_available,+)+from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import deferred_from_coro, ensure_awaitablefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_info@@ -61,6 +67,7 @@ if TYPE_CHECKING:from twisted.python.failure import Failure+ from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings@@ -218,22 +225,50 @@ class S3FilesStore:config_kwargs["max_pool_connections"] = self.AWS_MAX_POOL_CONNECTIONSif self.UPLOAD_TIMEOUT is not None:config_kwargs["read_timeout"] = self.UPLOAD_TIMEOUT+ self._client_kwargs: dict[str, Any] = {+ "aws_access_key_id": self.AWS_ACCESS_KEY_ID,+ "aws_secret_access_key": self.AWS_SECRET_ACCESS_KEY,+ "aws_session_token": self.AWS_SESSION_TOKEN,+ "endpoint_url": self.AWS_ENDPOINT_URL,+ "region_name": self.AWS_REGION_NAME,+ "use_ssl": self.AWS_USE_SSL,+ "verify": self.AWS_VERIFY,+ "config": Config(**config_kwargs),+ }+ # Used when asyncio or aiobotocore is not available, in a thread.session = botocore.session.get_session()- self.s3_client = session.create_client(- "s3",- aws_access_key_id=self.AWS_ACCESS_KEY_ID,- aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,- aws_session_token=self.AWS_SESSION_TOKEN,- endpoint_url=self.AWS_ENDPOINT_URL,- region_name=self.AWS_REGION_NAME,- use_ssl=self.AWS_USE_SSL,- verify=self.AWS_VERIFY,- config=Config(**config_kwargs),- )+ self.s3_client = session.create_client("s3", **self._client_kwargs)if not uri.startswith("s3://"):raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")self.bucket, self.prefix = uri[5:].split("/", 1)+ # Created lazily, from within the running event loop, and closed by+ # close().+ self._aio_client: Any = None+ self._aio_client_cm: Any = None+ self._aio_client_lock = asyncio.Lock()++ def _use_async(self) -> bool:+ return is_asyncio_available() and is_aiobotocore_available()++ async def _get_aio_client(self) -> Any:+ async with self._aio_client_lock:+ if self._aio_client is None:+ from aiobotocore.session import get_session # noqa: PLC0415++ self._aio_client_cm = get_session().create_client(+ "s3", **self._client_kwargs+ )+ # pylint: disable-next=unnecessary-dunder-call+ self._aio_client = await self._aio_client_cm.__aenter__()+ return self._aio_client++ async def close(self) -> None:+ """Close the underlying aiobotocore client, if one was created."""+ if self._aio_client is not None:+ await self._aio_client_cm.__aexit__(None, None, None)+ self._aio_client = self._aio_client_cm = None+def _onsuccess(self, boto_key: dict[str, Any]) -> StatInfo:last_modified = boto_key["LastModified"]modified_stamp = last_modified.timestamp()@@ -245,18 +280,20 @@ class S3FilesStore:def stat_file(self, path: str, info: MediaPipeline.SpiderInfo) -> Deferred[StatInfo]:+ return deferred_from_coro(self._stat_file(path))- return self._get_boto_key(path).addCallback(self._onsuccess)-- def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:+ async def _stat_file(self, path: str) -> StatInfo:key_name = f"{self.prefix}{path}"- return deferred_from_coro(- run_in_thread(+ if self._use_async():+ client = await self._get_aio_client()+ boto_key = await client.head_object(Bucket=self.bucket, Key=key_name)+ else:+ boto_key = await run_in_thread(self.s3_client.head_object, # type: ignore[attr-defined]Bucket=self.bucket,Key=key_name,)- )+ return self._onsuccess(boto_key)def persist_file(self,@@ -267,24 +304,34 @@ class S3FilesStore:headers: dict[str, str] | None = None,) -> Deferred[Any]:"""Upload file to S3 storage"""+ return deferred_from_coro(self._persist_file(path, buf, meta, headers))++ async def _persist_file(+ self,+ path: str,+ buf: BytesIO,+ meta: dict[str, Any] | None,+ headers: dict[str, str] | None,+ ) -> Any:key_name = f"{self.prefix}{path}"buf.seek(0)extra = self._headers_to_botocore_kwargs(self.HEADERS)if headers:extra.update(self._headers_to_botocore_kwargs(headers))kwargs: dict[str, Any] = {+ "Bucket": self.bucket,+ "Key": key_name,+ "Body": buf,"Metadata": {k: str(v) for k, v in meta.items()} if meta else {},"ACL": self.POLICY,**extra,}- return deferred_from_coro(- run_in_thread(- self.s3_client.put_object, # type: ignore[attr-defined]- Bucket=self.bucket,- Key=key_name,- Body=buf,- **kwargs,- )+ if self._use_async():+ client = await self._get_aio_client()+ return await client.put_object(**kwargs)+ return await run_in_thread(+ self.s3_client.put_object, # type: ignore[attr-defined]+ **kwargs,)@functools.cached_property@@ -660,6 +707,12 @@ class FilesPipeline(MediaPipeline):def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler.settings["FILES_STORE"], crawler=crawler)+ @_warn_spider_arg+ async def close_spider(self, spider: Spider | None = None) -> None:+ close = getattr(self.store, "close", None)+ if close is not None:+ await ensure_awaitable(close())+def _get_store_setting_name(self, name: str) -> str:return self._key_for_pipe(f"FILES_STORE_{name}", "FilesPipeline", self.crawler.settingsdiff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.pyindex 76ee0e7ec..fa1edf42d 100644--- a/scrapy/utils/boto.py+++ b/scrapy/utils/boto.py@@ -9,6 +9,10 @@ if TYPE_CHECKING:from scrapy.settings import BaseSettings+def is_aiobotocore_available() -> bool:+ return find_spec("aiobotocore") is not None++def is_botocore_available() -> bool:return find_spec("botocore") is not Nonediff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 422d95f39..a46cdee1b 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -115,6 +115,18 @@ class TestFilesPipeline:pipeline.open_spider()return pipeline+ @coroutine_test+ async def test_close_spider_closes_store(self):+ self.pipeline.store = mock.MagicMock()+ self.pipeline.store.close = mock.AsyncMock()+ await self.pipeline.close_spider()+ self.pipeline.store.close.assert_awaited_once_with()++ @coroutine_test+ async def test_close_spider_without_store_close(self):+ assert not hasattr(self.pipeline.store, "close")+ await self.pipeline.close_spider()+def test_file_path_query_parameters(self):file_path = self.pipeline.file_path@@ -1072,7 +1084,7 @@ class TestFSFilesStore:@pytest.mark.requires_botocoreclass TestS3FilesStore:@inline_callbacks_test- def test_persist(self):+ def test_persist(self, monkeypatch):bucket = "mybucket"key = "export.csv"uri = f"s3://{bucket}/{key}"@@ -1082,6 +1094,7 @@ class TestS3FilesStore:content_type = "image/png"store = S3FilesStore(uri)+ monkeypatch.setattr(store, "_use_async", lambda: False)from botocore.stub import Stubber # noqa: PLC0415with Stubber(store.s3_client) as stub:@@ -1112,13 +1125,14 @@ class TestS3FilesStore:assert buffer.method_calls == [mock.call.seek(0)]@inline_callbacks_test- def test_persist_without_headers(self):+ def test_persist_without_headers(self, monkeypatch):"""Without custom headers only the default ones are sent."""bucket = "mybucket"key = "export.csv"buffer = mock.MagicMock()store = S3FilesStore(f"s3://{bucket}/{key}")+ monkeypatch.setattr(store, "_use_async", lambda: False)from botocore.stub import Stubber # noqa: PLC0415with Stubber(store.s3_client) as stub:@@ -1140,13 +1154,14 @@ class TestS3FilesStore:stub.assert_no_pending_responses()@inline_callbacks_test- def test_persist_acl_header(self):+ def test_persist_acl_header(self, monkeypatch):"""A header overrides the matching put_object argument."""bucket = "mybucket"key = "export.csv"buffer = mock.MagicMock()store = S3FilesStore(f"s3://{bucket}/{key}")+ monkeypatch.setattr(store, "_use_async", lambda: False)from botocore.stub import Stubber # noqa: PLC0415with Stubber(store.s3_client) as stub:@@ -1204,7 +1219,7 @@ class TestS3FilesStore:) == {"CacheControl": "max-age=1", "Tagging": "a=b"}@inline_callbacks_test- def test_stat(self):+ def test_stat(self, monkeypatch):bucket = "mybucket"key = "export.csv"uri = f"s3://{bucket}/{key}"@@ -1217,6 +1232,7 @@ class TestS3FilesStore:)store = S3FilesStore(uri)+ monkeypatch.setattr(store, "_use_async", lambda: False)from botocore.stub import Stubber # noqa: PLC0415with Stubber(store.s3_client) as stub:@@ -1239,7 +1255,7 @@ class TestS3FilesStore:}@inline_callbacks_test- def test_stat_checksum_algorithm(self):+ def test_stat_checksum_algorithm(self, monkeypatch):"""Amazon S3 only reports MD5 checksums, so no checksum is reportedwhen a different algorithm is configured."""bucket = "mybucket"@@ -1248,6 +1264,7 @@ class TestS3FilesStore:last_modified = datetime(2019, 12, 1) # noqa: DTZ001store = S3FilesStore(f"s3://{bucket}/{key}")+ monkeypatch.setattr(store, "_use_async", lambda: False)store.checksum_algorithm = "sha256"from botocore.stub import Stubber # noqa: PLC0415@@ -1308,6 +1325,109 @@ class TestS3FilesStore:assert config.max_pool_connections == expected[email protected]_aiobotocore[email protected]_asyncio+class TestS3FilesStoreAsync:+ @coroutine_test+ async def test_persist(self):+ bucket = "mybucket"+ key = "export.csv"+ uri = f"s3://{bucket}/{key}"+ buffer = mock.MagicMock()+ meta = {"foo": "bar"}+ path = ""+ content_type = "image/png"++ store = S3FilesStore(uri)+ assert store._use_async()+ client = await store._get_aio_client()+ from aiobotocore.stub import Stubber # noqa: PLC0415++ with Stubber(client) as stub:+ stub.add_response(+ "put_object",+ expected_params={+ "ACL": S3FilesStore.POLICY,+ "Body": buffer,+ "Bucket": bucket,+ "CacheControl": S3FilesStore.HEADERS["Cache-Control"],+ "ContentType": content_type,+ "Key": key,+ "Metadata": meta,+ },+ service_response={},+ )++ await maybe_deferred_to_future(+ store.persist_file(+ path,+ buffer,+ info=DUMMY_SPIDER_INFO,+ meta=meta,+ headers={"Content-Type": content_type},+ )+ )++ stub.assert_no_pending_responses()+ assert buffer.method_calls == [mock.call.seek(0)]++ await store.close()++ @coroutine_test+ async def test_max_pool_connections(self) -> None:+ store = S3FilesStore("s3://mybucket/prefix/", AWS_MAX_POOL_CONNECTIONS=30)+ client = await store._get_aio_client()+ assert client.meta.config.max_pool_connections == 30+ await store.close()++ @coroutine_test+ async def test_stat(self):+ bucket = "mybucket"+ key = "export.csv"+ uri = f"s3://{bucket}/{key}"+ checksum = "3187896a9657a28163abb31667df64c8"+ last_modified = datetime(2019, 12, 1, tzinfo=UTC)++ store = S3FilesStore(uri)+ client = await store._get_aio_client()+ from aiobotocore.stub import Stubber # noqa: PLC0415++ with Stubber(client) as stub:+ stub.add_response(+ "head_object",+ expected_params={"Bucket": bucket, "Key": key},+ service_response={+ "ETag": f'"{checksum}"',+ "LastModified": last_modified,+ },+ )++ file_stats = await maybe_deferred_to_future(+ store.stat_file("", info=DUMMY_SPIDER_INFO)+ )+ assert file_stats == {+ "checksum": checksum,+ "last_modified": last_modified.timestamp(),+ }++ stub.assert_no_pending_responses()++ await store.close()++ @coroutine_test+ async def test_client_reused_and_closed(self):+ store = S3FilesStore("s3://mybucket/export.csv")+ assert store._aio_client is None+ client = await store._get_aio_client()+ # The client is created lazily and reused across calls.+ assert store._aio_client is client+ assert await store._get_aio_client() is client+ await store.close()+ assert store._aio_client is None+ # close() is a no-op if there is no open client.+ await store.close()++class TestGCSFilesStore:@staticmethoddef build_gcs_files_store(diff --git a/tox.ini b/tox.iniindex 2f8d8a10a..0b5218c33 100644--- a/tox.ini+++ b/tox.ini@@ -184,6 +184,7 @@ deps =Twisted[http2]python-socks[asyncio]siosocks # SOCKS server for tests+ aiobotocoreboto3bpython # optional for shell wrapper testscolorlog@@ -335,9 +336,10 @@ commands =[testenv:botocore]deps ={[testenv]deps}+ aiobotocorebotocore>=1.13.45commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m "requires_botocore or requires_aiobotocore"[testenv:min-botocore]basepython = {[min]basepython}