Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

11.4 KB
Raw Blame
from __future__ import annotations

import asyncio
import contextvars
import random
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
from unittest import mock

import pytest
from twisted.internet.defer import Deferred

import scrapy.utils.asyncio
from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import (
    AsyncioLoopingCall,
    _parallel_asyncio,
    call_later,
    is_asyncio_available,
    sleep,
)
from scrapy.utils.defer import maybe_deferred_to_future
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from collections.abc import AsyncGenerator, Callable, Iterator


@coroutine_test
async def test_is_asyncio_available(reactor_pytest: str) -> None:
    # the result should depend only on the pytest --reactor argument
    assert is_asyncio_available() == (reactor_pytest != "default")


_context_var: contextvars.ContextVar[str | None] = contextvars.ContextVar(
    "_context_var", default=None
)


@coroutine_test
async def test_call_later_preserves_context() -> None:
    _context_var.set("hello")
    d: Deferred[str | None] = Deferred()
    call_later(0.05, lambda: d.callback(_context_var.get()))
    assert await maybe_deferred_to_future(d) == "hello"


@coroutine_test
async def test_sleep() -> None:
    events: list[str] = []
    call_later(0.05, events.append, "call_later")
    await sleep(0.1)
    events.append("sleep")
    assert events == ["call_later", "sleep"]


@pytest.mark.only_asyncio
class TestParallelAsyncio:
    """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""

    CONCURRENT_ITEMS = 50

    @staticmethod
    async def callable(o: int, results: list[int]) -> None:
        if random.random() < 0.4:
            # simulate async processing
            await asyncio.sleep(random.random() / 8)
        # simulate trivial sync processing
        results.append(o)

    async def callable_wrapped(
        self,
        o: int,
        results: list[int],
        parallel_count: list[int],
        max_parallel_count: list[int],
    ) -> None:
        parallel_count[0] += 1
        max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0])
        await self.callable(o, results)
        assert parallel_count[0] > 0, parallel_count[0]
        parallel_count[0] -= 1

    @staticmethod
    def get_async_iterable(length: int) -> AsyncGenerator[int, None]:
        # simulate a simple callback without delays between results
        return as_async_generator(range(length))

    @staticmethod
    async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]:
        # simulate a callback with delays between some of the results
        for i in range(length):
            if random.random() < 0.1:
                await asyncio.sleep(random.random() / 20)
            yield i

    @pytest.mark.parametrize("length", [20, 50, 100])
    @coroutine_test
    async def test_simple(self, length: int) -> None:
        parallel_count = [0]
        max_parallel_count = [0]
        results: list[int] = []
        ait = self.get_async_iterable(length)
        await _parallel_asyncio(
            ait,
            self.CONCURRENT_ITEMS,
            self.callable_wrapped,
            results,
            parallel_count,
            max_parallel_count,
        )
        assert list(range(length)) == sorted(results)
        assert parallel_count[0] == 0
        assert max_parallel_count[0] <= self.CONCURRENT_ITEMS

    @pytest.mark.parametrize("length", [20, 50, 100])
    @coroutine_test
    async def test_delays(self, length: int) -> None:
        parallel_count = [0]
        max_parallel_count = [0]
        results: list[int] = []
        ait = self.get_async_iterable_with_delays(length)
        await _parallel_asyncio(
            ait,
            self.CONCURRENT_ITEMS,
            self.callable_wrapped,
            results,
            parallel_count,
            max_parallel_count,
        )
        assert list(range(length)) == sorted(results)
        assert parallel_count[0] == 0
        assert max_parallel_count[0] <= self.CONCURRENT_ITEMS

    @coroutine_test
    async def test_count_higher_than_work(self):
        results: list[int] = []
        task_counts: list[int] = []

        async def callable_(o: int) -> None:
            task_counts.append(len(asyncio.all_tasks()))
            results.append(o)

        await _parallel_asyncio(range(3), 1_000_000, callable_)
        assert results == [0, 1, 2]
        assert max(task_counts) < 100

    @pytest.mark.parametrize("count", [1, 2])
    @coroutine_test
    async def test_callable_error(
        self, count: int, caplog: pytest.LogCaptureFixture
    ) -> None:
        results: list[int] = []

        async def callable_(o: int) -> None:
            if o == 1:
                raise ValueError("Bad item")
            results.append(o)

        await _parallel_asyncio(range(4), count, callable_)
        assert sorted(results) == [0, 2, 3]
        assert "Error processing an item" in caplog.text
        assert "Bad item" in caplog.text

    @coroutine_test
    async def test_cancelled(self) -> None:
        """Cancelling the coroutine cancels the calls in progress."""
        started: list[int] = []
        cancelled: list[int] = []
        all_started = asyncio.Event()

        async def callable_(o: int) -> None:
            started.append(o)
            if len(started) == 3:
                all_started.set()
            try:
                await asyncio.Event().wait()
            except asyncio.CancelledError:
                cancelled.append(o)
                raise

        task = asyncio.create_task(_parallel_asyncio(range(3), 3, callable_))
        await all_started.wait()
        task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await task
        assert sorted(cancelled) == [0, 1, 2]

    @coroutine_test
    async def test_iteration_error(self) -> None:
        """An exception raised while iterating is propagated in an exception
        group, and the calls in progress are cancelled."""
        started: list[int] = []
        cancelled: list[int] = []
        all_started = asyncio.Event()

        async def iterable() -> AsyncGenerator[int, None]:
            yield 0
            yield 1
            await all_started.wait()
            raise ValueError("Bad iterable")

        async def callable_(o: int) -> None:
            started.append(o)
            if len(started) == 2:
                all_started.set()
            try:
                await asyncio.Event().wait()
            except asyncio.CancelledError:
                cancelled.append(o)
                raise

        with pytest.RaisesGroup(pytest.RaisesExc(ValueError, match="Bad iterable")):
            await _parallel_asyncio(iterable(), 3, callable_)
        assert sorted(cancelled) == [0, 1]


_real_sleep = asyncio.sleep


class _FakeTime:
    """A clock for :class:`~scrapy.utils.asyncio.AsyncioLoopingCall` that only
    advances when the looping call sleeps or when a test advances it.

    Each sleep ends *early* seconds before the requested time, like event loops
    with timers coarser than :func:`time.monotonic` can do.
    """

    def __init__(self, *, early: float = 0.0):
        self.now = 0.0
        self._early = early

    def monotonic(self) -> float:
        return self.now

    async def sleep(self, delay: float) -> None:
        self.now += max(delay - self._early, 0.0)
        await _real_sleep(0)

    @contextmanager
    def patch(self) -> Iterator[None]:
        with (
            mock.patch.object(scrapy.utils.asyncio, "time", self),
            mock.patch.object(asyncio, "sleep", self.sleep),
        ):
            yield


async def _wait_for(condition: Callable[[], bool]) -> None:
    """Let other tasks run until *condition* is met, giving up after 1000
    event loop iterations."""
    for _ in range(1000):
        if condition():
            return
        await _real_sleep(0)


@pytest.mark.only_asyncio
class TestAsyncioLoopingCall:
    @coroutine_test
    async def test_looping_call(self):
        func = mock.MagicMock()
        looping_call = AsyncioLoopingCall(func)
        looping_call.start(1, now=False)
        assert looping_call.running
        looping_call.stop()
        assert not looping_call.running
        assert not func.called

    @coroutine_test
    async def test_looping_call_now(self):
        func = mock.MagicMock()
        looping_call = AsyncioLoopingCall(func)
        looping_call.start(1)
        looping_call.stop()
        assert func.called

    @coroutine_test
    async def test_looping_call_already_running(self):
        looping_call = AsyncioLoopingCall(lambda: None)
        looping_call.start(1)
        with pytest.raises(RuntimeError):
            looping_call.start(1)
        looping_call.stop()

    @coroutine_test
    async def test_looping_call_interval(self):
        looping_call = AsyncioLoopingCall(lambda: None)
        with pytest.raises(ValueError, match="Interval must be greater than 0"):
            looping_call.start(0)
        with pytest.raises(ValueError, match="Interval must be greater than 0"):
            looping_call.start(-1)
        assert not looping_call.running

    @coroutine_test
    async def test_looping_call_bad_function(self):
        looping_call: AsyncioLoopingCall[[], Deferred[Any]] = AsyncioLoopingCall(
            Deferred
        )
        with pytest.raises(TypeError):
            looping_call.start(0.1)
        assert not looping_call.running

    @coroutine_test
    async def test_looping_function_raises(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        looping_call = AsyncioLoopingCall(lambda: 1 / 0)
        looping_call.start(0.1)
        assert not looping_call.running
        assert "Error calling the AsyncioLoopingCall function" in caplog.text

    @coroutine_test
    async def test_looping_call_early_wakeup(self) -> None:
        """The function must be called exactly once per interval boundary even
        if the event loop wakes the looping call before time.monotonic()
        reaches the boundary (this can happen e.g. under uvloop)."""
        fake_time = _FakeTime(early=0.001)
        calls: list[float] = []
        looping_call = AsyncioLoopingCall(lambda: calls.append(fake_time.now))
        with fake_time.patch():
            looping_call.start(1, now=False)
            await _wait_for(lambda: len(calls) >= 5)
            looping_call.stop()
        assert calls == pytest.approx([0.999, 1.999, 2.999, 3.999, 4.999])

    @coroutine_test
    async def test_looping_call_late_wakeup(self) -> None:
        """If the function is late (e.g. because the event loop was blocked),
        the calls for the missed boundaries are skipped and the next call
        happens on the next boundary, not immediately."""
        fake_time = _FakeTime()
        calls: list[float] = []

        def func() -> None:
            calls.append(fake_time.now)
            if len(calls) == 1:
                fake_time.now += 2.5  # the call blocks the loop for 2.5 intervals

        looping_call = AsyncioLoopingCall(func)
        with fake_time.patch():
            looping_call.start(1, now=False)
            await _wait_for(lambda: len(calls) >= 3)
            looping_call.stop()
        assert calls == [1, 4, 5]


@pytest.mark.only_not_asyncio
def test_looping_call_no_asyncio() -> None:
    looping_call = AsyncioLoopingCall(lambda: None)
    with pytest.raises(RuntimeError, match="no running asyncio event loop"):
        looping_call.start(1, now=False)