Scrapy, a fast high-level web crawling & scraping framework for Python.
Add URL benchmarks (#7914)
Adrianfc9c505e7975
diff --git a/tests/benchmarks/test_urls.py b/tests/benchmarks/test_urls.pynew file mode 100644index 000000000..acc1d4d80--- /dev/null+++ b/tests/benchmarks/test_urls.py@@ -0,0 +1,152 @@+from __future__ import annotations++from html import escape+from pathlib import Path+from typing import TYPE_CHECKING, Any++import pytest++from scrapy import Request+from scrapy.http import HtmlResponse+from scrapy.linkextractors import LinkExtractor+from scrapy.utils.request import fingerprint++if TYPE_CHECKING:+ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]++pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")++RESPONSE_URL = "https://www.example.com/catalogue/page-1.html"++# Links that each scenario returns for the benchmark page. They are fewer than+# the anchors of the page because links to images, to other non-crawlable files+# and to non-HTTP schemes are rejected, and, except in the scenario that keeps+# duplicates, because the links that the navigation repeats are collapsed.+LINKS = 63+DUPLICATE_LINKS = 88+CANONICAL_LINKS = 60+FILTERED_LINKS = 45++# Requests built from LINKS links that point to a different resource.+# Canonicalization maps the rest to one that another link already covers, e.g.+# two fragments of a page, or two spellings of one percent-escape.+FINGERPRINTS = 60+++def _read_corpus() -> tuple[list[str], list[str]]:+ """Return the URLs of ``urls.txt``, and its first group of URLs.++ The first group is the site navigation, which the benchmark page repeats.+ """+ groups: list[list[str]] = [[]]+ for line in (Path(__file__).parent / "urls.txt").read_text().splitlines():+ line = line.strip()+ if not line or line.startswith("#"):+ if groups[-1]:+ groups.append([])+ continue+ groups[-1].append(line)+ urls = [url for group in groups for url in group]+ return urls, groups[0]+++def _build_page(urls: list[str], navigation: list[str]) -> bytes:+ """Return an HTML page that links to *urls*.++ Every link is surrounded by the markup of a product listing, so that+ benchmarks also cover walking over the elements and attributes that a real+ page puts between links.+ """++ def item(index: int, url: str) -> str:+ href = escape(url)+ return (+ f'<li class="product" data-index="{index}">'+ f'<img src="/media/thumbnail-{index}.jpg" alt="Product {index}" '+ f'width="128" height="128">'+ f'<h3><a href="{href}">Product {index}</a></h3>'+ f'<p class="description">A description of product {index}.</p>'+ f"</li>"+ )++ def nav(urls: list[str]) -> str:+ links = "".join(f'<a href="{escape(url)}">{escape(url)}</a>' for url in urls)+ return f'<nav class="site">{links}</nav>'++ items = "".join(item(index, url) for index, url in enumerate(urls))+ return (+ "<!DOCTYPE html><html><head><title>Catalogue</title>"+ f'<base href="{RESPONSE_URL}"></head><body>'+ f'{nav(navigation)}<ul class="products">{items}</ul>{nav(navigation)}'+ "</body></html>"+ ).encode()+++URLS, NAVIGATION = _read_corpus()+BODY = _build_page(URLS, NAVIGATION)+++def _response() -> HtmlResponse:+ return HtmlResponse(RESPONSE_URL, body=BODY, encoding="utf-8")+++ ("kwargs", "links"),+ [+ pytest.param({}, LINKS, id="default"),+ pytest.param({"unique": False}, DUPLICATE_LINKS, id="duplicates"),+ pytest.param({"canonicalize": True}, CANONICAL_LINKS, id="canonicalize"),+ pytest.param(+ {+ "allow": r"/catalogue/",+ "deny": r"/legal/",+ "allow_domains": ["example.com", "www.example.com"],+ },+ FILTERED_LINKS,+ id="filtered",+ ),+ ],+)+def test_extract_links(+ benchmark: BenchmarkFixture, kwargs: dict[str, Any], links: int+) -> None:+ """Extraction of every link of a page.++ The scenarios cover the choices that change which work dominates:+ deduplication and canonicalization both build a key for every link, and the+ filters of a configured extractor reject links before the later checks,+ which the default extractor reaches for every link.+ """+ link_extractor = LinkExtractor(**kwargs)++ def run() -> None:+ assert len(link_extractor.extract_links(_response())) == links++ benchmark(run)+++EXTRACTED_URLS = [link.url for link in LinkExtractor().extract_links(_response())]+++def test_requests(benchmark: BenchmarkFixture) -> None:+ """Building a request for every link of a page."""++ def run() -> None:+ assert len([Request(url) for url in EXTRACTED_URLS]) == LINKS++ benchmark(run)+++def test_fingerprints(benchmark: BenchmarkFixture) -> None:+ """Fingerprinting the request of every link of a page.++ Requests are built here as well, and not once for all rounds, because+ fingerprints are cached per request object.+ """++ def run() -> None:+ assert (+ len({fingerprint(Request(url)) for url in EXTRACTED_URLS}) == FINGERPRINTS+ )++ benchmark(run)diff --git a/tests/benchmarks/urls.txt b/tests/benchmarks/urls.txtnew file mode 100644index 000000000..6ef6939f1--- /dev/null+++ b/tests/benchmarks/urls.txt@@ -0,0 +1,130 @@+# Link targets for the URL benchmarks, as they would appear in the href+# attribute of a page at https://www.example.com/catalogue/page-1.html.+#+# Cost per URL varies by shape: the number of query parameters drives the+# parsing and re-encoding of the query string, non-ASCII characters and+# unescaped characters drive percent-encoding, and non-default ports, dot+# segments and uppercase host names drive normalization. A corpus of uniform+# URLs would therefore measure one shape and miss the others, so this one+# covers each of them, in roughly the proportion of a real listing page.+#+# Blank lines and lines starting with "#" are ignored.++# Site navigation. These also appear in a second copy of the navigation at the+# end of the page, so that deduplication has duplicates to collapse.+/+/index.html+/about-us+/contact+/catalogue/+/catalogue/page-2.html+/catalogue/page-3.html+/help/faq+/help/shipping-and-returns+/legal/terms+/legal/privacy++# Relative paths of increasing depth.+detail.html+./detail.html+../catalogue/page-4.html+../../index.html+/catalogue/category/books/fiction/index.html+/catalogue/category/books/travel/mystery/historical/index.html+/a/b/c/d/e/f/g/h/i/j/k/index.html++# One query parameter.+/catalogue/search?q=book+/catalogue/page-1.html?page=2+/catalogue/detail?id=1042++# Several query parameters, in an order that canonicalization changes.+/catalogue/search?q=book&sort=price+/catalogue/search?sort=price&q=book+/catalogue/search?q=book&sort=price&page=3&per_page=20&in_stock=1+/catalogue/search?zone=eu&q=book&min=10&max=90&sort=rating&page=2&view=grid&lang=en¤cy=EUR&ref=nav++# Repeated keys, blank values and a bare key.+/catalogue/search?tag=fiction&tag=travel&tag=history+/catalogue/search?q=&sort=+/catalogue/search?featured++# Characters that need percent-encoding.+/catalogue/search?q=cheap books+/catalogue/detail/a book about books.html+/catalogue/search?q=100%+cotton+/catalogue/search?price=%3E10&title=A%20%26%20B++# Percent-escapes that are already valid, in both cases.+/catalogue/detail/%C3%A9dition-limit%C3%A9e.html+/catalogue/detail/%c3%a9dition-limit%c3%a9e.html+/catalogue/detail/%7Especial.html++# Non-ASCII in the path and in the query.+/catalogue/detail/édition-limitée.html+/catalogue/search?q=édition+/catalogue/búsqueda?q=libro&categoría=ficción+/カタログ/詳細.html++# Internationalized host names, encoded and decoded.+https://例え.テスト/catalogue/page-1.html+https://xn--r8jz45g.xn--zckzah/catalogue/page-2.html++# Absolute URLs on the same host, on other hosts, and protocol-relative.+https://www.example.com/catalogue/page-5.html+https://www.example.com/catalogue/detail?id=1043+http://www.example.com/catalogue/page-6.html+https://shop.example.com/catalogue/page-1.html+https://www.example.org/reviews/1042+https://books.toscrape.com/catalogue/page-1.html+//cdn.example.com/catalogue/page-7.html+//www.example.com/catalogue/page-8.html++# Ports, including the default one for the scheme.+https://www.example.com:443/catalogue/page-9.html+http://www.example.com:80/catalogue/page-10.html+https://staging.example.com:8443/catalogue/page-1.html++# Host name case, which normalization lowercases.+https://WWW.EXAMPLE.COM/catalogue/Page-11.html+HTTPS://www.example.com/catalogue/page-12.html++# Dot segments, empty segments and trailing slashes, which WHATWG+# normalization resolves and the standard library keeps.+/catalogue/../catalogue/page-13.html+/catalogue/./page-14.html+/catalogue//page-15.html+/catalogue/category/+/catalogue/category++# Fragments, which canonicalization drops and the deduplication key keeps.+/catalogue/page-16.html#reviews+/catalogue/page-16.html#description+/catalogue/page-17.html#+#top++# Path parameters, where the semicolon is not the last segment.+/catalogue;sessionid=abc123/page-18.html+/catalogue/page-19.html;sessionid=abc123++# User information in the authority.+https://user:[email protected]/catalogue/page-1.html++# A long URL, of the length that tracking parameters reach.+/catalogue/search?q=book&utm_source=newsletter&utm_medium=email&utm_campaign=spring-sale-2026&utm_term=fiction%20paperback&utm_content=hero-banner-variant-b&session=6f1c9a2e4b7d8f0a1c3e5d7b9f2a4c6e&ref=https%3A%2F%2Fwww.example.org%2Freviews%2F1042&page=2&sort=relevance++# Extensions that the default deny_extensions rejects, and one compound+# extension, which only matches as a whole.+/media/cover-1042.jpg+/media/cover-1042.PNG+/media/catalogue.pdf+/static/style.css+/static/app.js+/downloads/catalogue.tar.gz+/downloads/catalogue.zip++# Schemes that are not crawlable, which are rejected before any parsing.+mailto:[email protected]+javascript:void(0)+tel:+441234567890+data:text/plain,hello