Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import asyncio
from collections import Counter
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode
import pytest
from scrapy import Field, Item, Request, Spider
from scrapy.http import HtmlResponse, TextResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader
from tests import get_testdata
from tests.benchmarks import NullDownloadHandler, crawl
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from pathlib import Path
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
from scrapy.crawler import Crawler
from scrapy.http import Response
from tests.mockserver.http import MockServer
pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
PAGES = 100
LINKS_PER_PAGE = 5
# Requests per crawl of the benchmarks that use NullDownloadHandler. The broad
# crawl scenarios split them differently between hostnames and pages per
# hostname.
REQUESTS = 200
BROAD_DEEP_PAGES = 10
# Requests per crawl and delay of the benchmarks that wait, where wall time,
# unlike in the other benchmarks, is a function of the delay.
DELAYED_REQUESTS = 50
DELAY = 0.005
# Requests per crawl and items per response of the benchmarks that measure item
# processing, which reaches fewer pages than the other benchmarks because every
# page costs it several items.
ITEM_REQUESTS = 20
ITEMS_PER_RESPONSE = 100
# Listing pages per crawl of the parse-heavy benchmarks, and book pages that
# each listing page links to.
LISTINGS = 5
BOOKS_PER_LISTING = 20
# Item concurrency limits of the benchmarks that measure item processing. The
# high limit is above the number of items that a response yields in any of
# them.
HIGH_CONCURRENT_ITEMS = 1000
DELAYED_CONCURRENT_ITEMS = 50
NULL_SETTINGS: dict[str, Any] = {
"DOWNLOAD_HANDLERS": {"http": NullDownloadHandler},
"LOG_ENABLED": False,
}
class _Page(Item):
url = Field()
anchors = Field()
class _FollowSpider(Spider):
name = "benchmark"
url: str
link_extractor = LinkExtractor()
async def start(self) -> AsyncIterator[Any]:
yield Request(self.url, dont_filter=True)
def parse(self, response: Response) -> Any:
yield _Page(
url=response.url,
anchors=response.css("a::text").getall(),
)
for link in self.link_extractor.extract_links(response): # type: ignore[arg-type]
yield Request(link.url)
class _TreeSpider(Spider):
"""Crawl *pages* pages on each of *domains* hostnames, yielding *items*
items from every page.
Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so
that requests also reach the scheduler from callbacks, and not only from
:meth:`~scrapy.Spider.start`.
"""
name = "benchmark-tree"
domains: int = 1
pages: int = 1
items: int = 0
async def start(self) -> AsyncIterator[Any]:
for domain in range(self.domains):
yield Request(f"http://d{domain}.example.com/1")
def parse(self, response: Response) -> Any:
page = int(response.url.rpartition("/")[2])
for child in (page * 2, page * 2 + 1):
if child <= self.pages:
yield Request(response.urljoin(f"/{child}"))
for _ in range(self.items):
yield _Page(url=response.url)
class _Book(Item):
title = Field()
price = Field()
rating = Field()
image = Field()
stock = Field()
description = Field()
upc = Field()
categories = Field()
class _BookSpider(Spider):
"""Crawl *listings* copies of a saved book listing page, and the book pages
that each of them links to.
Listing URLs are built by appending a number to *url*, and every listing
page links to book pages relative to its own URL, so that each copy is
reached under URLs of its own.
"""
name = "benchmark-books"
url: str
listings: int = 1
link_extractor = LinkExtractor(restrict_css="article.product_pod")
async def start(self) -> AsyncIterator[Any]:
for listing in range(self.listings):
yield Request(f"{self.url}/{listing}/")
def parse(self, response: Response) -> Any:
for link in self.link_extractor.extract_links(response): # type: ignore[arg-type]
yield Request(link.url, callback=self.parse_book)
def parse_book(self, response: Response) -> Any:
assert isinstance(response, TextResponse)
loader = ItemLoader(_Book(), response=response)
loader.add_css("title", "div.product_main h1::text")
loader.add_css("price", "div.product_main p.price_color::text")
loader.add_css("rating", "div.product_main p.star-rating::attr(class)")
loader.add_css("image", "div#product_gallery img::attr(src)")
loader.add_xpath(
"stock", "//p[@class='instock availability']/text()[normalize-space()]"
)
loader.add_xpath(
"description",
"//div[@id='product_description']/following-sibling::p/text()",
)
loader.add_xpath("upc", "//th[text()='UPC']/following-sibling::td/text()")
loader.add_xpath("categories", "//ul[@class='breadcrumb']/li/a/text()")
item = loader.load_item()
# A selector that stops matching the snapshot would otherwise silently
# leave the benchmark with less parsing to do.
assert len(item) == len(_Book.fields)
yield item
class _BooksDownloadHandler(NullDownloadHandler):
"""Serve the snapshot that the mockserver books resource serves, without I/O."""
listing = get_testdata("books", "listing.html")
book = get_testdata("books", "book.html")
def response(self, request: Request) -> Response:
body = self.book if "/catalogue/" in request.url else self.listing
return HtmlResponse(request.url, body=body, encoding="utf-8", request=request)
class _Pipeline:
def process_item(self, item: Any) -> Any:
return item
class _DelayedPipeline:
"""Item pipeline that waits, so that the item concurrency limit applies.
The peak number of items of a same response in progress is tracked in the
``benchmark/peak_items`` stat. Items are counted per response because the
limit is per response, and the items of a response are processed while
later responses are already being downloaded.
"""
def __init__(self, crawler: Crawler):
self._crawler = crawler
self._active: Counter[str] = Counter()
@classmethod
def from_crawler(cls, crawler: Crawler) -> _DelayedPipeline:
return cls(crawler)
async def process_item(self, item: Any) -> Any:
url = item["url"]
self._active[url] += 1
self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
try:
await asyncio.sleep(DELAY)
return item
finally:
self._active[url] -= 1
def _crawl_tree(
settings: dict[str, Any], *, domains: int, pages: int, items: int = 0
) -> Crawler:
crawler = crawl(
_TreeSpider,
NULL_SETTINGS | settings,
domains=domains,
pages=pages,
items=items,
)
assert crawler.stats.get_value("downloader/response_count") == domains * pages
assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
return crawler
def _crawl_books(settings: dict[str, Any], url: str) -> None:
crawler = crawl(_BookSpider, settings, url=url, listings=LISTINGS)
responses = LISTINGS * (BOOKS_PER_LISTING + 1)
assert crawler.stats.get_value("downloader/response_count") == responses
assert crawler.stats.get_value("item_scraped_count") == LISTINGS * BOOKS_PER_LISTING
def test_parse_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
"""Cost of a crawl that extracts links and fields from real markup, over HTTP.
Unlike the other HTTP benchmark, the pages here are a saved snapshot of a
real website, so that most of the cost is parsing: building a selector for
a 50 KB listing page, extracting links from it, and then running several
CSS and XPath queries on each of the book pages it links to.
"""
settings = {"LOG_ENABLED": False}
benchmark(lambda: _crawl_books(settings, mockserver.url("/books")))
def test_parse_engine(benchmark: BenchmarkFixture) -> None:
"""Cost of a crawl that extracts links and fields from real markup, without I/O."""
settings = {
**NULL_SETTINGS,
"DOWNLOAD_HANDLERS": {"http": _BooksDownloadHandler},
}
benchmark(lambda: _crawl_books(settings, "http://books.example.com/books"))
def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
"""Per-request overhead of a crawl over HTTP.
The pages are small on purpose, so that the cost of parsing them stays
negligible next to the cost of moving requests and responses through the
engine, the middlewares and the download handler.
"""
query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
url = mockserver.url(f"/follow?{query}")
settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}
def run() -> None:
crawler = crawl(_FollowSpider, settings, url=url)
assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
benchmark(run)
def test_overhead_engine(benchmark: BenchmarkFixture) -> None:
"""Per-request overhead of a crawl of a single hostname without any I/O."""
def run() -> None:
crawler = _crawl_tree({}, domains=1, pages=REQUESTS)
assert crawler.stats.get_value("benchmark/peak_concurrency") > 1
benchmark(run)
@pytest.mark.parametrize(
("domains", "pages"),
[
pytest.param(REQUESTS, 1, id="shallow"),
pytest.param(REQUESTS // BROAD_DEEP_PAGES, BROAD_DEEP_PAGES, id="deep"),
],
)
def test_overhead_broad(benchmark: BenchmarkFixture, domains: int, pages: int) -> None:
"""Per-request overhead of a broad crawl.
The shallow scenario, which reaches a single page of every hostname, pays
the cost of tracking a hostname for the first time on every request, and
gets its requests from :meth:`~scrapy.Spider.start`. The deep scenario,
which reaches the same number of pages spread over fewer hostnames,
amortizes that cost, and instead keeps several requests per hostname
waiting in the scheduler.
"""
benchmark(lambda: _crawl_tree({}, domains=domains, pages=pages))
def test_overhead_concurrency(benchmark: BenchmarkFixture) -> None:
"""Overhead of a crawl limited to 1 request at a time on a single hostname."""
settings = {"CONCURRENT_REQUESTS_PER_DOMAIN": 1}
benchmark(lambda: _crawl_tree(settings, domains=1, pages=REQUESTS))
def test_overhead_delay(benchmark: BenchmarkFixture) -> None:
"""Overhead of a crawl where every request waits for a download delay.
The delay is not randomized, so that wall time, and hence the number of
reactor iterations that the crawl needs, does not change between runs.
"""
settings = {"DOWNLOAD_DELAY": DELAY, "DOWNLOAD_DELAY_JITTER": 0}
benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS))
@pytest.mark.parametrize(
("items", "settings"),
[
pytest.param(1, {}, id="single"),
pytest.param(ITEMS_PER_RESPONSE, {}, id="many"),
pytest.param(
1,
{"CONCURRENT_ITEMS": HIGH_CONCURRENT_ITEMS},
id="high-limit",
),
],
)
def test_overhead_items(
benchmark: BenchmarkFixture, items: int, settings: dict[str, Any]
) -> None:
"""Overhead of sending the items of a callback through the item pipeline.
The single and many scenarios, which use the default
:setting:`CONCURRENT_ITEMS` value, measure how that overhead grows with the
number of items that a response yields. The high-limit scenario instead
raises :setting:`CONCURRENT_ITEMS` well above that number.
"""
benchmark(
lambda: _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=items)
)
def test_overhead_feed(benchmark: BenchmarkFixture, tmp_path: Path) -> None:
"""Overhead of exporting the items of a crawl to a feed.
It runs the crawl of the "many" scenario of ``test_overhead_items`` with a
JSON Lines feed added, so that the difference between both is the cost of
the feed export.
"""
path = tmp_path / "items.jsonl"
# Every round writes the same file.
settings = {"FEEDS": {path: {"format": "jsonlines", "mode": "overwrite"}}}
def run() -> None:
_crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE)
assert path.read_bytes().count(b"\n") == ITEM_REQUESTS * ITEMS_PER_RESPONSE
benchmark(run)
def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
"""Overhead of a crawl where item processing waits.
Every response yields more items than :setting:`CONCURRENT_ITEMS` allows in
parallel, so that the item pipeline gets them in several batches, and wall
time, unlike in most of the other benchmarks, is a function of the delay.
"""
settings = {
"CONCURRENT_ITEMS": DELAYED_CONCURRENT_ITEMS,
"ITEM_PIPELINES": {_DelayedPipeline: 100},
}
def run() -> None:
crawler = _crawl_tree(
settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
)
assert (
crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
)
benchmark(run)