Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

13.4 KB
Raw Blame
from __future__ import annotations

import asyncio
from collections import Counter
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode

import pytest

from scrapy import Field, Item, Request, Spider
from scrapy.http import HtmlResponse, TextResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader
from tests import get_testdata
from tests.benchmarks import NullDownloadHandler, crawl

if TYPE_CHECKING:
    from collections.abc import AsyncIterator
    from pathlib import Path

    from pytest_codspeed import BenchmarkFixture  # type: ignore[import-not-found]

    from scrapy.crawler import Crawler
    from scrapy.http import Response
    from tests.mockserver.http import MockServer

pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")

PAGES = 100
LINKS_PER_PAGE = 5

# Requests per crawl of the benchmarks that use NullDownloadHandler. The broad
# crawl scenarios split them differently between hostnames and pages per
# hostname.
REQUESTS = 200
BROAD_DEEP_PAGES = 10

# Requests per crawl and delay of the benchmarks that wait, where wall time,
# unlike in the other benchmarks, is a function of the delay.
DELAYED_REQUESTS = 50
DELAY = 0.005

# Requests per crawl and items per response of the benchmarks that measure item
# processing, which reaches fewer pages than the other benchmarks because every
# page costs it several items.
ITEM_REQUESTS = 20
ITEMS_PER_RESPONSE = 100

# Listing pages per crawl of the parse-heavy benchmarks, and book pages that
# each listing page links to.
LISTINGS = 5
BOOKS_PER_LISTING = 20

# Item concurrency limits of the benchmarks that measure item processing. The
# high limit is above the number of items that a response yields in any of
# them.
HIGH_CONCURRENT_ITEMS = 1000
DELAYED_CONCURRENT_ITEMS = 50

NULL_SETTINGS: dict[str, Any] = {
    "DOWNLOAD_HANDLERS": {"http": NullDownloadHandler},
    "LOG_ENABLED": False,
}


class _Page(Item):
    url = Field()
    anchors = Field()


class _FollowSpider(Spider):
    name = "benchmark"
    url: str
    link_extractor = LinkExtractor()

    async def start(self) -> AsyncIterator[Any]:
        yield Request(self.url, dont_filter=True)

    def parse(self, response: Response) -> Any:
        yield _Page(
            url=response.url,
            anchors=response.css("a::text").getall(),
        )
        for link in self.link_extractor.extract_links(response):  # type: ignore[arg-type]
            yield Request(link.url)


class _TreeSpider(Spider):
    """Crawl *pages* pages on each of *domains* hostnames, yielding *items*
    items from every page.

    Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so
    that requests also reach the scheduler from callbacks, and not only from
    :meth:`~scrapy.Spider.start`.
    """

    name = "benchmark-tree"
    domains: int = 1
    pages: int = 1
    items: int = 0

    async def start(self) -> AsyncIterator[Any]:
        for domain in range(self.domains):
            yield Request(f"http://d{domain}.example.com/1")

    def parse(self, response: Response) -> Any:
        page = int(response.url.rpartition("/")[2])
        for child in (page * 2, page * 2 + 1):
            if child <= self.pages:
                yield Request(response.urljoin(f"/{child}"))
        for _ in range(self.items):
            yield _Page(url=response.url)


class _Book(Item):
    title = Field()
    price = Field()
    rating = Field()
    image = Field()
    stock = Field()
    description = Field()
    upc = Field()
    categories = Field()


class _BookSpider(Spider):
    """Crawl *listings* copies of a saved book listing page, and the book pages
    that each of them links to.

    Listing URLs are built by appending a number to *url*, and every listing
    page links to book pages relative to its own URL, so that each copy is
    reached under URLs of its own.
    """

    name = "benchmark-books"
    url: str
    listings: int = 1
    link_extractor = LinkExtractor(restrict_css="article.product_pod")

    async def start(self) -> AsyncIterator[Any]:
        for listing in range(self.listings):
            yield Request(f"{self.url}/{listing}/")

    def parse(self, response: Response) -> Any:
        for link in self.link_extractor.extract_links(response):  # type: ignore[arg-type]
            yield Request(link.url, callback=self.parse_book)

    def parse_book(self, response: Response) -> Any:
        assert isinstance(response, TextResponse)
        loader = ItemLoader(_Book(), response=response)
        loader.add_css("title", "div.product_main h1::text")
        loader.add_css("price", "div.product_main p.price_color::text")
        loader.add_css("rating", "div.product_main p.star-rating::attr(class)")
        loader.add_css("image", "div#product_gallery img::attr(src)")
        loader.add_xpath(
            "stock", "//p[@class='instock availability']/text()[normalize-space()]"
        )
        loader.add_xpath(
            "description",
            "//div[@id='product_description']/following-sibling::p/text()",
        )
        loader.add_xpath("upc", "//th[text()='UPC']/following-sibling::td/text()")
        loader.add_xpath("categories", "//ul[@class='breadcrumb']/li/a/text()")
        item = loader.load_item()
        # A selector that stops matching the snapshot would otherwise silently
        # leave the benchmark with less parsing to do.
        assert len(item) == len(_Book.fields)
        yield item


class _BooksDownloadHandler(NullDownloadHandler):
    """Serve the snapshot that the mockserver books resource serves, without I/O."""

    listing = get_testdata("books", "listing.html")
    book = get_testdata("books", "book.html")

    def response(self, request: Request) -> Response:
        body = self.book if "/catalogue/" in request.url else self.listing
        return HtmlResponse(request.url, body=body, encoding="utf-8", request=request)


class _Pipeline:
    def process_item(self, item: Any) -> Any:
        return item


class _DelayedPipeline:
    """Item pipeline that waits, so that the item concurrency limit applies.

    The peak number of items of a same response in progress is tracked in the
    ``benchmark/peak_items`` stat. Items are counted per response because the
    limit is per response, and the items of a response are processed while
    later responses are already being downloaded.
    """

    def __init__(self, crawler: Crawler):
        self._crawler = crawler
        self._active: Counter[str] = Counter()

    @classmethod
    def from_crawler(cls, crawler: Crawler) -> _DelayedPipeline:
        return cls(crawler)

    async def process_item(self, item: Any) -> Any:
        url = item["url"]
        self._active[url] += 1
        self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
        try:
            await asyncio.sleep(DELAY)
            return item
        finally:
            self._active[url] -= 1


def _crawl_tree(
    settings: dict[str, Any], *, domains: int, pages: int, items: int = 0
) -> Crawler:
    crawler = crawl(
        _TreeSpider,
        NULL_SETTINGS | settings,
        domains=domains,
        pages=pages,
        items=items,
    )
    assert crawler.stats.get_value("downloader/response_count") == domains * pages
    assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
    return crawler


def _crawl_books(settings: dict[str, Any], url: str) -> None:
    crawler = crawl(_BookSpider, settings, url=url, listings=LISTINGS)
    responses = LISTINGS * (BOOKS_PER_LISTING + 1)
    assert crawler.stats.get_value("downloader/response_count") == responses
    assert crawler.stats.get_value("item_scraped_count") == LISTINGS * BOOKS_PER_LISTING


def test_parse_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
    """Cost of a crawl that extracts links and fields from real markup, over HTTP.

    Unlike the other HTTP benchmark, the pages here are a saved snapshot of a
    real website, so that most of the cost is parsing: building a selector for
    a 50 KB listing page, extracting links from it, and then running several
    CSS and XPath queries on each of the book pages it links to.
    """
    settings = {"LOG_ENABLED": False}
    benchmark(lambda: _crawl_books(settings, mockserver.url("/books")))


def test_parse_engine(benchmark: BenchmarkFixture) -> None:
    """Cost of a crawl that extracts links and fields from real markup, without I/O."""
    settings = {
        **NULL_SETTINGS,
        "DOWNLOAD_HANDLERS": {"http": _BooksDownloadHandler},
    }
    benchmark(lambda: _crawl_books(settings, "http://books.example.com/books"))


def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
    """Per-request overhead of a crawl over HTTP.

    The pages are small on purpose, so that the cost of parsing them stays
    negligible next to the cost of moving requests and responses through the
    engine, the middlewares and the download handler.
    """
    query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
    url = mockserver.url(f"/follow?{query}")
    settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}

    def run() -> None:
        crawler = crawl(_FollowSpider, settings, url=url)
        assert crawler.stats.get_value("item_scraped_count") == PAGES + 1

    benchmark(run)


def test_overhead_engine(benchmark: BenchmarkFixture) -> None:
    """Per-request overhead of a crawl of a single hostname without any I/O."""

    def run() -> None:
        crawler = _crawl_tree({}, domains=1, pages=REQUESTS)
        assert crawler.stats.get_value("benchmark/peak_concurrency") > 1

    benchmark(run)


@pytest.mark.parametrize(
    ("domains", "pages"),
    [
        pytest.param(REQUESTS, 1, id="shallow"),
        pytest.param(REQUESTS // BROAD_DEEP_PAGES, BROAD_DEEP_PAGES, id="deep"),
    ],
)
def test_overhead_broad(benchmark: BenchmarkFixture, domains: int, pages: int) -> None:
    """Per-request overhead of a broad crawl.

    The shallow scenario, which reaches a single page of every hostname, pays
    the cost of tracking a hostname for the first time on every request, and
    gets its requests from :meth:`~scrapy.Spider.start`. The deep scenario,
    which reaches the same number of pages spread over fewer hostnames,
    amortizes that cost, and instead keeps several requests per hostname
    waiting in the scheduler.
    """
    benchmark(lambda: _crawl_tree({}, domains=domains, pages=pages))


def test_overhead_concurrency(benchmark: BenchmarkFixture) -> None:
    """Overhead of a crawl limited to 1 request at a time on a single hostname."""
    settings = {"CONCURRENT_REQUESTS_PER_DOMAIN": 1}
    benchmark(lambda: _crawl_tree(settings, domains=1, pages=REQUESTS))


def test_overhead_delay(benchmark: BenchmarkFixture) -> None:
    """Overhead of a crawl where every request waits for a download delay.

    The delay is not randomized, so that wall time, and hence the number of
    reactor iterations that the crawl needs, does not change between runs.
    """
    settings = {"DOWNLOAD_DELAY": DELAY, "DOWNLOAD_DELAY_JITTER": 0}
    benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS))


@pytest.mark.parametrize(
    ("items", "settings"),
    [
        pytest.param(1, {}, id="single"),
        pytest.param(ITEMS_PER_RESPONSE, {}, id="many"),
        pytest.param(
            1,
            {"CONCURRENT_ITEMS": HIGH_CONCURRENT_ITEMS},
            id="high-limit",
        ),
    ],
)
def test_overhead_items(
    benchmark: BenchmarkFixture, items: int, settings: dict[str, Any]
) -> None:
    """Overhead of sending the items of a callback through the item pipeline.

    The single and many scenarios, which use the default
    :setting:`CONCURRENT_ITEMS` value, measure how that overhead grows with the
    number of items that a response yields. The high-limit scenario instead
    raises :setting:`CONCURRENT_ITEMS` well above that number.
    """
    benchmark(
        lambda: _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=items)
    )


def test_overhead_feed(benchmark: BenchmarkFixture, tmp_path: Path) -> None:
    """Overhead of exporting the items of a crawl to a feed.

    It runs the crawl of the "many" scenario of ``test_overhead_items`` with a
    JSON Lines feed added, so that the difference between both is the cost of
    the feed export.
    """
    path = tmp_path / "items.jsonl"
    # Every round writes the same file.
    settings = {"FEEDS": {path: {"format": "jsonlines", "mode": "overwrite"}}}

    def run() -> None:
        _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE)
        assert path.read_bytes().count(b"\n") == ITEM_REQUESTS * ITEMS_PER_RESPONSE

    benchmark(run)


def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
    """Overhead of a crawl where item processing waits.

    Every response yields more items than :setting:`CONCURRENT_ITEMS` allows in
    parallel, so that the item pipeline gets them in several batches, and wall
    time, unlike in most of the other benchmarks, is a function of the delay.
    """
    settings = {
        "CONCURRENT_ITEMS": DELAYED_CONCURRENT_ITEMS,
        "ITEM_PIPELINES": {_DelayedPipeline: 100},
    }

    def run() -> None:
        crawler = _crawl_tree(
            settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
        )
        assert (
            crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
        )

    benchmark(run)