Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

2.1 KB
Raw Blame
from __future__ import annotations

import asyncio
from typing import TYPE_CHECKING, Any

from scrapy.http import Response
from scrapy.utils.test import get_crawler

if TYPE_CHECKING:
    from scrapy import Request, Spider
    from scrapy.crawler import Crawler


class NullDownloadHandler:
    """Download handler that returns a response without doing any I/O.

    It lets benchmarks measure the engine, the scheduler and the middlewares
    without also measuring HTTP parsing and socket handling, and reach as many
    hostnames as they need without DNS resolution.

    It yields control to the event loop once per request, so that requests can
    be in progress at the same time and concurrency limits apply. The peak
    number of requests in progress is tracked in the
    ``benchmark/peak_concurrency`` stat.
    """

    lazy = False

    def __init__(self, crawler: Crawler):
        self._crawler = crawler
        self._active = 0

    @classmethod
    def from_crawler(cls, crawler: Crawler) -> NullDownloadHandler:
        return cls(crawler)

    async def download_request(self, request: Request) -> Response:
        self._active += 1
        self._crawler.stats.max_value("benchmark/peak_concurrency", self._active)
        try:
            await asyncio.sleep(0)
            return self.response(request)
        finally:
            self._active -= 1

    def response(self, request: Request) -> Response:
        """Return the response to serve for *request*, empty unless overridden."""
        return Response(request.url, request=request)

    async def close(self) -> None:
        pass


def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler:
    """Run a crawl to completion and return its crawler.

    Unlike the rest of the test suite, benchmarks run without ``pytest-twisted``
    and drive the event loop themselves, since the code being measured must be
    callable synchronously by ``pytest-codspeed``.
    """
    from twisted.internet import reactor

    crawler = get_crawler(spidercls, settings)
    loop = reactor._asyncioEventloop
    loop.run_until_complete(crawler.crawl(**kwargs).asFuture(loop))
    return crawler