Scrapy, a fast high-level web crawling & scraping framework for Python.
Preserve contextvars across reactor-scheduled callbacks (#8347)
Adrian14a6030e4f28
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex beb4a4810..2754ba209 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -3,6 +3,7 @@from __future__ import annotationsimport asyncio+import contextvarsimport loggingimport mathimport time@@ -291,7 +292,10 @@ def call_later(from twisted.internet import reactor- return CallLaterResult.from_twisted(reactor.callLater(delay, func, *args))+ context = contextvars.copy_context()+ return CallLaterResult.from_twisted(+ reactor.callLater(delay, context.run, func, *args)+ )class CallLaterResult:diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex d8eedf306..f60a16424 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -5,6 +5,7 @@ Helper functions for dealing with Twisted deferredsfrom __future__ import annotationsimport asyncio+import contextvarsimport inspectimport loggingimport sys@@ -326,6 +327,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):self.finished: bool = Falseself.waiting_deferreds: deque[Deferred[Any]] = deque()self.anext_deferred: Deferred[_T] | None = None+ # Cooperator ticks are scheduled through the reactor, which does not+ # preserve contextvars, so both anext() and the callable run in the+ # context captured here.+ self._context: contextvars.Context = contextvars.copy_context()# Called whenever aiterator produces a value, so that parallel_async()# can start a task for it.self.on_value: Callable[[], None] = lambda: None@@ -336,8 +341,8 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):# (by chaining if the result is a Deferred too or by firing if not).self.anext_deferred = Noneself.on_value()- callable_result = self.callable(- result, *self.callable_args, **self.callable_kwargs+ callable_result = self._context.run(+ self.callable, result, *self.callable_args, **self.callable_kwargs)d = self.waiting_deferreds.popleft()if isinstance(callable_result, Deferred):@@ -359,7 +364,9 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):def _call_anext(self) -> None:# This starts waiting for the next result from aiterator.# If aiterator is exhausted, _errback will be called.- self.anext_deferred = deferred_from_coro(anext(self.aiterator))+ self.anext_deferred = self._context.run(+ deferred_from_coro, anext(self.aiterator)+ )self.anext_deferred.addCallbacks(self._callback, self._errback)def __next__(self) -> Deferred[Any]:diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 2e29d5a19..e7e90cf49 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import contextvarsimport randomfrom contextlib import contextmanagerfrom typing import TYPE_CHECKING, Any@@ -18,6 +19,7 @@ from scrapy.utils.asyncio import (is_asyncio_available,sleep,)+from scrapy.utils.defer import maybe_deferred_to_futurefrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:@@ -30,6 +32,19 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:assert is_asyncio_available() == (reactor_pytest != "default")+_context_var: contextvars.ContextVar[str | None] = contextvars.ContextVar(+ "_context_var", default=None+)+++@coroutine_test+async def test_call_later_preserves_context() -> None:+ _context_var.set("hello")+ d: Deferred[str | None] = Deferred()+ call_later(0.05, lambda: d.callback(_context_var.get()))+ assert await maybe_deferred_to_future(d) == "hello"++@coroutine_testasync def test_sleep() -> None:events: list[str] = []diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 87b523b37..44e93a110 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import contextvarsimport randomimport socketimport warnings@@ -327,6 +328,24 @@ class TestParallelAsync:assert "Error processing an item" in caplog.textassert "Bad item" in caplog.text+ @coroutine_test+ async def test_context(self) -> None:+ var: contextvars.ContextVar[str | None] = contextvars.ContextVar(+ "var", default=None+ )+ var.set("hello")+ seen: list[str | None] = []++ async def aiterable() -> AsyncGenerator[int, None]:+ for i in range(3):+ seen.append(var.get())+ yield i++ await maybe_deferred_to_future(+ parallel_async(aiterable(), 1, lambda _: seen.append(var.get()))+ )+ assert seen == ["hello"] * 6+class TestDeferredFromCoro:def test_deferred(self):