Scrapy, a fast high-level web crawling & scraping framework for Python.
asyncio alternative to deferToThread() (#7349)
Andrey Rakhmatullinbfe34492fa41
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 86593aba8..fa928678e 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -21,14 +21,13 @@ from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, castfrom urllib.parse import unquote, urlparsefrom twisted.internet.defer import Deferred, DeferredList-from twisted.internet.threads import deferToThreadfrom w3lib.url import file_uri_to_pathfrom zope.interface import Interface, implementerfrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.extensions.postprocessing import PostProcessingManager-from scrapy.utils.asyncio import is_asyncio_available+from scrapy.utils.asyncio import is_asyncio_available, run_in_threadfrom scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.defer import deferred_from_coro, ensure_awaitablefrom scrapy.utils.ftp import ftp_store_file@@ -131,7 +130,7 @@ class BlockingFeedStorage(ABC):return NamedTemporaryFile(prefix="feed-", dir=path)def store(self, file: IO[bytes]) -> Deferred[None] | None:- return deferToThread(self._store_in_thread, file)+ return deferred_from_coro(run_in_thread(self._store_in_thread, file))@abstractmethoddef _store_in_thread(self, file: IO[bytes]) -> None:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex a0bb674d1..707207337 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -23,15 +23,15 @@ from urllib.parse import urlparsefrom itemadapter import ItemAdapterfrom twisted.internet.defer import Deferred, maybeDeferred-from twisted.internet.threads import deferToThreadfrom scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline+from scrapy.utils.asyncio import run_in_threadfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.datatypes import CaseInsensitiveDict-from scrapy.utils.defer import ensure_awaitable+from scrapy.utils.defer import deferred_from_coro, ensure_awaitablefrom scrapy.utils.ftp import ftp_store_filefrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.python import to_bytes@@ -198,13 +198,12 @@ class S3FilesStore:def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:key_name = f"{self.prefix}{path}"- return cast(- "Deferred[dict[str, Any]]",- deferToThread(+ return deferred_from_coro(+ run_in_thread(self.s3_client.head_object, # type: ignore[attr-defined]Bucket=self.bucket,Key=key_name,- ),+ ))def persist_file(@@ -221,14 +220,16 @@ class S3FilesStore:extra = self._headers_to_botocore_kwargs(self.HEADERS)if headers:extra.update(self._headers_to_botocore_kwargs(headers))- return deferToThread(- self.s3_client.put_object, # type: ignore[attr-defined]- Bucket=self.bucket,- Key=key_name,- Body=buf,- Metadata={k: str(v) for k, v in (meta or {}).items()},- ACL=self.POLICY,- **extra,+ return deferred_from_coro(+ run_in_thread(+ self.s3_client.put_object, # type: ignore[attr-defined]+ Bucket=self.bucket,+ Key=key_name,+ Body=buf,+ Metadata={k: str(v) for k, v in (meta or {}).items()},+ ACL=self.POLICY,+ **extra,+ ))def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]:@@ -315,10 +316,9 @@ class GCSFilesStore:return {}blob_path = self._get_blob_path(path)- return cast(- "Deferred[StatInfo]",- deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),- )+ return deferred_from_coro(+ run_in_thread(self.bucket.get_blob, blob_path)+ ).addCallback(_onsuccess)def _get_content_type(self, headers: dict[str, str] | None) -> str:if headers and "Content-Type" in headers:@@ -340,11 +340,13 @@ class GCSFilesStore:blob = self.bucket.blob(blob_path)blob.cache_control = self.CACHE_CONTROLblob.metadata = {k: str(v) for k, v in (meta or {}).items()}- return deferToThread(- blob.upload_from_string,- data=buf.getvalue(),- content_type=self._get_content_type(headers),- predefined_acl=self.POLICY,+ return deferred_from_coro(+ run_in_thread(+ blob.upload_from_string,+ data=buf.getvalue(),+ content_type=self._get_content_type(headers),+ predefined_acl=self.POLICY,+ ))@@ -377,15 +379,17 @@ class FTPFilesStore:headers: dict[str, str] | None = None,) -> Deferred[Any]:path = f"{self.basedir}/{path}"- return deferToThread(- ftp_store_file,- path=path,- file=buf,- host=self.host,- port=self.port,- username=self.username,- password=self.password,- use_active_mode=self.USE_ACTIVE_MODE,+ return deferred_from_coro(+ run_in_thread(+ ftp_store_file,+ path=path,+ file=buf,+ host=self.host,+ port=self.port,+ username=self.username,+ password=self.password,+ use_active_mode=bool(self.USE_ACTIVE_MODE),+ ))def stat_file(@@ -407,7 +411,7 @@ class FTPFilesStore:except Exception:return {}- return cast("Deferred[StatInfo]", deferToThread(_stat_file, path))+ return deferred_from_coro(run_in_thread(_stat_file, path))class FilesPipeline(MediaPipeline):diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex 9ec090995..7852a10fd 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVarfrom twisted.internet.defer import Deferredfrom twisted.internet.task import LoopingCall+from twisted.internet.threads import deferToThreadfrom scrapy.utils.asyncgen import as_async_generatorfrom scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed@@ -278,3 +279,23 @@ class CallLaterResult:elif self._delayed_call and self._delayed_call.active():self._delayed_call.cancel()self._delayed_call = None+++async def run_in_thread(+ func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs+) -> _T:+ """Call a function in a thread and return its result as a coroutine.++ This uses either :func:`asyncio.to_thread` or+ :func:`twisted.internet.threads.deferToThread`, depending on whether+ asyncio support is available.++ .. versionadded:: VERSION+ """+ if is_asyncio_available():+ return await asyncio.to_thread(func, *args, **kwargs)++ # circular import+ from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415++ return await maybe_deferred_to_future(deferToThread(func, *args, **kwargs))diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex 044d1d4c7..f1e274190 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -6,9 +6,10 @@ from functools import wrapsfrom typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overloadfrom twisted.internet.defer import Deferred, maybeDeferred-from twisted.internet.threads import deferToThreadfrom scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.asyncio import run_in_thread+from scrapy.utils.defer import deferred_from_coroif TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine@@ -59,12 +60,15 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: # pragma: nodef inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:"""Decorator to call a function in a thread and return a deferred with the- result+ result.++ .. versionchanged:: VERSION+ Now uses :func:`asyncio.to_thread` if the asyncio support is available."""@wraps(func)def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:- return deferToThread(func, *a, **kw)+ return deferred_from_coro(run_in_thread(func, *a, **kw))return wrappeddiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex e50b2be42..dcf412c0f 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1151,7 +1151,6 @@ class TestFeedExport(TestFeedExportBase):data = await self.exported_no_data(settings)assert data["csv"] == b""- @pytest.mark.requires_reactor # TODO: needs a reactor for BlockingFeedStorage@coroutine_testasync def test_multiple_feeds_success_logs_blocking_feed_storage(self):settings = {diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 16c4357bb..3b70e896c 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -381,7 +381,6 @@ class TestBatchDeliveries(TestFeedExportBase):assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12- @pytest.mark.requires_reactor # TODO: needs a reactor for BlockingFeedStorage@pytest.mark.requires_boto3@inline_callbacks_testdef test_s3_export(self):diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex c5cd5c782..5102f8e37 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -116,7 +116,6 @@ class TestFileFeedStorage:assert storage.path == path[email protected]_reactor # TODO: needs a reactor for BlockingFeedStorageclass TestFTPFeedStorage:def get_test_spider(self, settings=None):class TestSpider(scrapy.Spider):@@ -231,7 +230,6 @@ class TestBlockingFeedStorage:@pytest.mark.requires_boto3[email protected]_reactor # TODO: needs a reactor for BlockingFeedStorageclass TestS3FeedStorage:def test_parse_credentials(self):aws_credentials = {@@ -461,7 +459,6 @@ class TestS3FeedStorage:assert "S3 does not support appending to files" in str(log)[email protected]_reactor # TODO: needs a reactor for BlockingFeedStorageclass TestGCSFeedStorage:def test_parse_settings(self):try:diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 201b29f83..c412645f4 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -584,7 +584,6 @@ class TestFilesPipelineCustomSettings:assert fs_store.basedir == str(tmp_path)[email protected]_reactor # TODO: needs a reactor for S3FilesStore@pytest.mark.requires_botocoreclass TestS3FilesStore:@inline_callbacks_test@@ -715,7 +714,6 @@ class TestGCSFilesStore:store.bucket.get_blob.assert_called_with(expected_blob_path)[email protected]_reactor # TODO: needs a reactor for FTPFilesStoreclass TestFTPFileStore:@inline_callbacks_testdef test_persist(self):