Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Preserve contextvars across reactor-scheduled callbacks (#8347)

Adrian14a6030e4f28
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index beb4a4810..2754ba209 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import contextvars
import logging
import math
import time
@@ -291,7 +292,10 @@ def call_later(
from twisted.internet import reactor
- return CallLaterResult.from_twisted(reactor.callLater(delay, func, *args))
+ context = contextvars.copy_context()
+ return CallLaterResult.from_twisted(
+ reactor.callLater(delay, context.run, func, *args)
+ )
class CallLaterResult:
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d8eedf306..f60a16424 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,6 +5,7 @@ Helper functions for dealing with Twisted deferreds
from __future__ import annotations
import asyncio
+import contextvars
import inspect
import logging
import sys
@@ -326,6 +327,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):
self.finished: bool = False
self.waiting_deferreds: deque[Deferred[Any]] = deque()
self.anext_deferred: Deferred[_T] | None = None
+ # Cooperator ticks are scheduled through the reactor, which does not
+ # preserve contextvars, so both anext() and the callable run in the
+ # context captured here.
+ self._context: contextvars.Context = contextvars.copy_context()
# Called whenever aiterator produces a value, so that parallel_async()
# can start a task for it.
self.on_value: Callable[[], None] = lambda: None
@@ -336,8 +341,8 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):
# (by chaining if the result is a Deferred too or by firing if not).
self.anext_deferred = None
self.on_value()
- callable_result = self.callable(
- result, *self.callable_args, **self.callable_kwargs
+ callable_result = self._context.run(
+ self.callable, result, *self.callable_args, **self.callable_kwargs
)
d = self.waiting_deferreds.popleft()
if isinstance(callable_result, Deferred):
@@ -359,7 +364,9 @@ class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):
def _call_anext(self) -> None:
# This starts waiting for the next result from aiterator.
# If aiterator is exhausted, _errback will be called.
- self.anext_deferred = deferred_from_coro(anext(self.aiterator))
+ self.anext_deferred = self._context.run(
+ deferred_from_coro, anext(self.aiterator)
+ )
self.anext_deferred.addCallbacks(self._callback, self._errback)
def __next__(self) -> Deferred[Any]:
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 2e29d5a19..e7e90cf49 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import contextvars
import random
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
@@ -18,6 +19,7 @@ from scrapy.utils.asyncio import (
is_asyncio_available,
sleep,
)
+from scrapy.utils.defer import maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@@ -30,6 +32,19 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:
assert is_asyncio_available() == (reactor_pytest != "default")
+_context_var: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+ "_context_var", default=None
+)
+
+
+@coroutine_test
+async def test_call_later_preserves_context() -> None:
+ _context_var.set("hello")
+ d: Deferred[str | None] = Deferred()
+ call_later(0.05, lambda: d.callback(_context_var.get()))
+ assert await maybe_deferred_to_future(d) == "hello"
+
+
@coroutine_test
async def test_sleep() -> None:
events: list[str] = []
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 87b523b37..44e93a110 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import contextvars
import random
import socket
import warnings
@@ -327,6 +328,24 @@ class TestParallelAsync:
assert "Error processing an item" in caplog.text
assert "Bad item" in caplog.text
+ @coroutine_test
+ async def test_context(self) -> None:
+ var: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+ "var", default=None
+ )
+ var.set("hello")
+ seen: list[str | None] = []
+
+ async def aiterable() -> AsyncGenerator[int, None]:
+ for i in range(3):
+ seen.append(var.get())
+ yield i
+
+ await maybe_deferred_to_future(
+ parallel_async(aiterable(), 1, lambda _: seen.append(var.get()))
+ )
+ assert seen == ["hello"] * 6
+
class TestDeferredFromCoro:
def test_deferred(self):