Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use aiobotocore in S3FilesStore when available (#8339)

Adrianaaf2b31cabf2
diff --git a/conftest.py b/conftest.py
index 9a9ed20ce..290dfd8a8 100644
--- a/conftest.py
+++ b/conftest.py
@@ -279,6 +279,7 @@ def pytest_runtest_setup(item):
# Skip tests requiring optional dependencies
optional_deps = [
"uvloop",
+ "aiobotocore",
"botocore",
"boto3",
"psutil",
diff --git a/pyproject.toml b/pyproject.toml
index 74645df47..1000f6f1d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -143,6 +143,7 @@ implicit_reexport = true
# usually no type hints
[[tool.mypy.overrides]]
module = [
+ "aiobotocore.*",
"bpython",
"brotli",
"brotlicffi",
@@ -296,6 +297,7 @@ markers = [
"requires_reactor: marks tests that require a reactor",
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
"requires_botocore: marks tests that need botocore (but not boto3)",
+ "requires_aiobotocore: marks tests that need aiobotocore",
"requires_boto3: marks tests that need botocore and boto3",
"requires_psutil: marks tests that need psutil",
"requires_mitmproxy: marks tests that need a mitmdump executable",
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 67dad7651..851c291ce 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -6,6 +6,7 @@ See documentation in topics/media-pipeline.rst
from __future__ import annotations
+import asyncio
import base64
import functools
import hashlib
@@ -46,8 +47,13 @@ from scrapy.pipelines.media import (
)
from scrapy.utils._datatypes import CaseInsensitiveDict
from scrapy.utils._ftp import ftp_store_file
-from scrapy.utils.asyncio import run_in_thread
-from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available
+from scrapy.utils.asyncio import is_asyncio_available, run_in_thread
+from scrapy.utils.boto import (
+ _get_max_pool_connections,
+ is_aiobotocore_available,
+ is_botocore_available,
+)
+from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
@@ -61,6 +67,7 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
+ from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
@@ -218,22 +225,50 @@ class S3FilesStore:
config_kwargs["max_pool_connections"] = self.AWS_MAX_POOL_CONNECTIONS
if self.UPLOAD_TIMEOUT is not None:
config_kwargs["read_timeout"] = self.UPLOAD_TIMEOUT
+ self._client_kwargs: dict[str, Any] = {
+ "aws_access_key_id": self.AWS_ACCESS_KEY_ID,
+ "aws_secret_access_key": self.AWS_SECRET_ACCESS_KEY,
+ "aws_session_token": self.AWS_SESSION_TOKEN,
+ "endpoint_url": self.AWS_ENDPOINT_URL,
+ "region_name": self.AWS_REGION_NAME,
+ "use_ssl": self.AWS_USE_SSL,
+ "verify": self.AWS_VERIFY,
+ "config": Config(**config_kwargs),
+ }
+ # Used when asyncio or aiobotocore is not available, in a thread.
session = botocore.session.get_session()
- self.s3_client = session.create_client(
- "s3",
- aws_access_key_id=self.AWS_ACCESS_KEY_ID,
- aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
- aws_session_token=self.AWS_SESSION_TOKEN,
- endpoint_url=self.AWS_ENDPOINT_URL,
- region_name=self.AWS_REGION_NAME,
- use_ssl=self.AWS_USE_SSL,
- verify=self.AWS_VERIFY,
- config=Config(**config_kwargs),
- )
+ self.s3_client = session.create_client("s3", **self._client_kwargs)
if not uri.startswith("s3://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split("/", 1)
+ # Created lazily, from within the running event loop, and closed by
+ # close().
+ self._aio_client: Any = None
+ self._aio_client_cm: Any = None
+ self._aio_client_lock = asyncio.Lock()
+
+ def _use_async(self) -> bool:
+ return is_asyncio_available() and is_aiobotocore_available()
+
+ async def _get_aio_client(self) -> Any:
+ async with self._aio_client_lock:
+ if self._aio_client is None:
+ from aiobotocore.session import get_session # noqa: PLC0415
+
+ self._aio_client_cm = get_session().create_client(
+ "s3", **self._client_kwargs
+ )
+ # pylint: disable-next=unnecessary-dunder-call
+ self._aio_client = await self._aio_client_cm.__aenter__()
+ return self._aio_client
+
+ async def close(self) -> None:
+ """Close the underlying aiobotocore client, if one was created."""
+ if self._aio_client is not None:
+ await self._aio_client_cm.__aexit__(None, None, None)
+ self._aio_client = self._aio_client_cm = None
+
def _onsuccess(self, boto_key: dict[str, Any]) -> StatInfo:
last_modified = boto_key["LastModified"]
modified_stamp = last_modified.timestamp()
@@ -245,18 +280,20 @@ class S3FilesStore:
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
+ return deferred_from_coro(self._stat_file(path))
- return self._get_boto_key(path).addCallback(self._onsuccess)
-
- def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:
+ async def _stat_file(self, path: str) -> StatInfo:
key_name = f"{self.prefix}{path}"
- return deferred_from_coro(
- run_in_thread(
+ if self._use_async():
+ client = await self._get_aio_client()
+ boto_key = await client.head_object(Bucket=self.bucket, Key=key_name)
+ else:
+ boto_key = await run_in_thread(
self.s3_client.head_object, # type: ignore[attr-defined]
Bucket=self.bucket,
Key=key_name,
)
- )
+ return self._onsuccess(boto_key)
def persist_file(
self,
@@ -267,24 +304,34 @@ class S3FilesStore:
headers: dict[str, str] | None = None,
) -> Deferred[Any]:
"""Upload file to S3 storage"""
+ return deferred_from_coro(self._persist_file(path, buf, meta, headers))
+
+ async def _persist_file(
+ self,
+ path: str,
+ buf: BytesIO,
+ meta: dict[str, Any] | None,
+ headers: dict[str, str] | None,
+ ) -> Any:
key_name = f"{self.prefix}{path}"
buf.seek(0)
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
kwargs: dict[str, Any] = {
+ "Bucket": self.bucket,
+ "Key": key_name,
+ "Body": buf,
"Metadata": {k: str(v) for k, v in meta.items()} if meta else {},
"ACL": self.POLICY,
**extra,
}
- return deferred_from_coro(
- run_in_thread(
- self.s3_client.put_object, # type: ignore[attr-defined]
- Bucket=self.bucket,
- Key=key_name,
- Body=buf,
- **kwargs,
- )
+ if self._use_async():
+ client = await self._get_aio_client()
+ return await client.put_object(**kwargs)
+ return await run_in_thread(
+ self.s3_client.put_object, # type: ignore[attr-defined]
+ **kwargs,
)
@functools.cached_property
@@ -660,6 +707,12 @@ class FilesPipeline(MediaPipeline):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings["FILES_STORE"], crawler=crawler)
+ @_warn_spider_arg
+ async def close_spider(self, spider: Spider | None = None) -> None:
+ close = getattr(self.store, "close", None)
+ if close is not None:
+ await ensure_awaitable(close())
+
def _get_store_setting_name(self, name: str) -> str:
return self._key_for_pipe(
f"FILES_STORE_{name}", "FilesPipeline", self.crawler.settings
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 76ee0e7ec..fa1edf42d 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -9,6 +9,10 @@ if TYPE_CHECKING:
from scrapy.settings import BaseSettings
+def is_aiobotocore_available() -> bool:
+ return find_spec("aiobotocore") is not None
+
+
def is_botocore_available() -> bool:
return find_spec("botocore") is not None
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 422d95f39..a46cdee1b 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -115,6 +115,18 @@ class TestFilesPipeline:
pipeline.open_spider()
return pipeline
+ @coroutine_test
+ async def test_close_spider_closes_store(self):
+ self.pipeline.store = mock.MagicMock()
+ self.pipeline.store.close = mock.AsyncMock()
+ await self.pipeline.close_spider()
+ self.pipeline.store.close.assert_awaited_once_with()
+
+ @coroutine_test
+ async def test_close_spider_without_store_close(self):
+ assert not hasattr(self.pipeline.store, "close")
+ await self.pipeline.close_spider()
+
def test_file_path_query_parameters(self):
file_path = self.pipeline.file_path
@@ -1072,7 +1084,7 @@ class TestFSFilesStore:
@pytest.mark.requires_botocore
class TestS3FilesStore:
@inline_callbacks_test
- def test_persist(self):
+ def test_persist(self, monkeypatch):
bucket = "mybucket"
key = "export.csv"
uri = f"s3://{bucket}/{key}"
@@ -1082,6 +1094,7 @@ class TestS3FilesStore:
content_type = "image/png"
store = S3FilesStore(uri)
+ monkeypatch.setattr(store, "_use_async", lambda: False)
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
@@ -1112,13 +1125,14 @@ class TestS3FilesStore:
assert buffer.method_calls == [mock.call.seek(0)]
@inline_callbacks_test
- def test_persist_without_headers(self):
+ def test_persist_without_headers(self, monkeypatch):
"""Without custom headers only the default ones are sent."""
bucket = "mybucket"
key = "export.csv"
buffer = mock.MagicMock()
store = S3FilesStore(f"s3://{bucket}/{key}")
+ monkeypatch.setattr(store, "_use_async", lambda: False)
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
@@ -1140,13 +1154,14 @@ class TestS3FilesStore:
stub.assert_no_pending_responses()
@inline_callbacks_test
- def test_persist_acl_header(self):
+ def test_persist_acl_header(self, monkeypatch):
"""A header overrides the matching put_object argument."""
bucket = "mybucket"
key = "export.csv"
buffer = mock.MagicMock()
store = S3FilesStore(f"s3://{bucket}/{key}")
+ monkeypatch.setattr(store, "_use_async", lambda: False)
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
@@ -1204,7 +1219,7 @@ class TestS3FilesStore:
) == {"CacheControl": "max-age=1", "Tagging": "a=b"}
@inline_callbacks_test
- def test_stat(self):
+ def test_stat(self, monkeypatch):
bucket = "mybucket"
key = "export.csv"
uri = f"s3://{bucket}/{key}"
@@ -1217,6 +1232,7 @@ class TestS3FilesStore:
)
store = S3FilesStore(uri)
+ monkeypatch.setattr(store, "_use_async", lambda: False)
from botocore.stub import Stubber # noqa: PLC0415
with Stubber(store.s3_client) as stub:
@@ -1239,7 +1255,7 @@ class TestS3FilesStore:
}
@inline_callbacks_test
- def test_stat_checksum_algorithm(self):
+ def test_stat_checksum_algorithm(self, monkeypatch):
"""Amazon S3 only reports MD5 checksums, so no checksum is reported
when a different algorithm is configured."""
bucket = "mybucket"
@@ -1248,6 +1264,7 @@ class TestS3FilesStore:
last_modified = datetime(2019, 12, 1) # noqa: DTZ001
store = S3FilesStore(f"s3://{bucket}/{key}")
+ monkeypatch.setattr(store, "_use_async", lambda: False)
store.checksum_algorithm = "sha256"
from botocore.stub import Stubber # noqa: PLC0415
@@ -1308,6 +1325,109 @@ class TestS3FilesStore:
assert config.max_pool_connections == expected
[email protected]_aiobotocore
+class TestS3FilesStoreAsync:
+ @coroutine_test
+ async def test_persist(self):
+ bucket = "mybucket"
+ key = "export.csv"
+ uri = f"s3://{bucket}/{key}"
+ buffer = mock.MagicMock()
+ meta = {"foo": "bar"}
+ path = ""
+ content_type = "image/png"
+
+ store = S3FilesStore(uri)
+ assert store._use_async()
+ client = await store._get_aio_client()
+ from aiobotocore.stub import Stubber # noqa: PLC0415
+
+ with Stubber(client) as stub:
+ stub.add_response(
+ "put_object",
+ expected_params={
+ "ACL": S3FilesStore.POLICY,
+ "Body": buffer,
+ "Bucket": bucket,
+ "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+ "ContentType": content_type,
+ "Key": key,
+ "Metadata": meta,
+ },
+ service_response={},
+ )
+
+ await maybe_deferred_to_future(
+ store.persist_file(
+ path,
+ buffer,
+ info=DUMMY_SPIDER_INFO,
+ meta=meta,
+ headers={"Content-Type": content_type},
+ )
+ )
+
+ stub.assert_no_pending_responses()
+ assert buffer.method_calls == [mock.call.seek(0)]
+
+ await store.close()
+
+ @coroutine_test
+ async def test_max_pool_connections(self) -> None:
+ store = S3FilesStore("s3://mybucket/prefix/", AWS_MAX_POOL_CONNECTIONS=30)
+ client = await store._get_aio_client()
+ assert client.meta.config.max_pool_connections == 30
+ await store.close()
+
+ @coroutine_test
+ async def test_stat(self):
+ bucket = "mybucket"
+ key = "export.csv"
+ uri = f"s3://{bucket}/{key}"
+ checksum = "3187896a9657a28163abb31667df64c8"
+ last_modified = datetime(2019, 12, 1, tzinfo=UTC)
+
+ store = S3FilesStore(uri)
+ client = await store._get_aio_client()
+ from aiobotocore.stub import Stubber # noqa: PLC0415
+
+ with Stubber(client) as stub:
+ stub.add_response(
+ "head_object",
+ expected_params={"Bucket": bucket, "Key": key},
+ service_response={
+ "ETag": f'"{checksum}"',
+ "LastModified": last_modified,
+ },
+ )
+
+ file_stats = await maybe_deferred_to_future(
+ store.stat_file("", info=DUMMY_SPIDER_INFO)
+ )
+ assert file_stats == {
+ "checksum": checksum,
+ "last_modified": last_modified.timestamp(),
+ }
+
+ stub.assert_no_pending_responses()
+
+ await store.close()
+
+ @coroutine_test
+ async def test_client_reused_and_closed(self):
+ store = S3FilesStore("s3://mybucket/export.csv")
+ assert store._aio_client is None
+ client = await store._get_aio_client()
+ # The client is created lazily and reused across calls.
+ assert store._aio_client is client
+ assert await store._get_aio_client() is client
+ await store.close()
+ assert store._aio_client is None
+ # close() is a no-op if there is no open client.
+ await store.close()
+
+
class TestGCSFilesStore:
@staticmethod
def build_gcs_files_store(
diff --git a/tox.ini b/tox.ini
index 2f8d8a10a..0b5218c33 100644
--- a/tox.ini
+++ b/tox.ini
@@ -184,6 +184,7 @@ deps =
Twisted[http2]
python-socks[asyncio]
siosocks # SOCKS server for tests
+ aiobotocore
boto3
bpython # optional for shell wrapper tests
colorlog
@@ -335,9 +336,10 @@ commands =
[testenv:botocore]
deps =
{[testenv]deps}
+ aiobotocore
botocore>=1.13.45
commands =
- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore
+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m "requires_botocore or requires_aiobotocore"
[testenv:min-botocore]
basepython = {[min]basepython}