Scrapy, a fast high-level web crawling & scraping framework for Python.
Add benchmarks for item exporters and feed exports. (#8456)
Andrey Rakhmatullin8810d37ba97c
diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.pyindex 7495c309f..0f97c59f8 100644--- a/tests/benchmarks/test_crawl.py+++ b/tests/benchmarks/test_crawl.py@@ -16,6 +16,7 @@ from tests.benchmarks import NullDownloadHandler, crawlif TYPE_CHECKING:from collections.abc import AsyncIterator+ from pathlib import Pathfrom pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]@@ -345,6 +346,24 @@ def test_overhead_items()+def test_overhead_feed(benchmark: BenchmarkFixture, tmp_path: Path) -> None:+ """Overhead of exporting the items of a crawl to a feed.++ It runs the crawl of the "many" scenario of ``test_overhead_items`` with a+ JSON Lines feed added, so that the difference between both is the cost of+ the feed export.+ """+ path = tmp_path / "items.jsonl"+ # Every round writes the same file.+ settings = {"FEEDS": {path: {"format": "jsonlines", "mode": "overwrite"}}}++ def run() -> None:+ _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE)+ assert path.read_bytes().count(b"\n") == ITEM_REQUESTS * ITEMS_PER_RESPONSE++ benchmark(run)++def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:"""Overhead of a crawl where item processing waits.diff --git a/tests/benchmarks/test_exporters.py b/tests/benchmarks/test_exporters.pynew file mode 100644index 000000000..b90a81ca3--- /dev/null+++ b/tests/benchmarks/test_exporters.py@@ -0,0 +1,107 @@+from __future__ import annotations++from io import BytesIO+from typing import TYPE_CHECKING, Any++import pytest++from scrapy import Field, Item+from scrapy.exporters import CsvItemExporter, JsonLinesItemExporter, XmlItemExporter++if TYPE_CHECKING:+ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]++pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")++ITEMS = 1000+++class _Offer(Item):+ seller = Field()+ price = Field()+ currency = Field()+++class _Product(Item):+ url = Field()+ name = Field()+ sku = Field()+ brand = Field()+ price = Field()+ currency = Field()+ availability = Field()+ rating = Field()+ review_count = Field()+ description = Field()+ tags = Field()+ offers = Field()+++def _values(index: int) -> dict[str, Any]:+ return {+ "url": f"https://www.example.com/catalogue/product-{index}.html",+ "name": f"Product {index}",+ "sku": f"SKU-{index:06d}",+ "brand": "Example Brand",+ "price": 19.99 + index,+ "currency": "EUR",+ "availability": "InStock",+ "rating": 4.5,+ "review_count": index % 500,+ "description": f"A description of product {index}, a few words long.",+ "tags": ["books", "fiction", "paperback"],+ }+++def _offers(index: int) -> list[_Offer]:+ return [+ _Offer(seller=f"Seller {seller}", price=17.99 + index, currency="EUR")+ for seller in range(2)+ ]+++ITEM_SETS: dict[str, list[Any]] = {+ "dict": [_values(index) for index in range(ITEMS)],+ "item": [_Product(**_values(index)) for index in range(ITEMS)],+ "nested": [+ _Product(**_values(index), offers=_offers(index)) for index in range(ITEMS)+ ],+}+++ ("exporter_cls", "separator", "separators"),+ [+ pytest.param(JsonLinesItemExporter, b"\n", ITEMS, id="jsonlines"),+ # The header row adds a line.+ pytest.param(CsvItemExporter, b"\r\n", ITEMS + 1, id="csv"),+ pytest.param(XmlItemExporter, b"<item>", ITEMS, id="xml"),+ ],+)[email protected]("values", list(ITEM_SETS))+def test_export(+ benchmark: BenchmarkFixture,+ exporter_cls: type[CsvItemExporter | JsonLinesItemExporter | XmlItemExporter],+ separator: bytes,+ separators: int,+ values: str,+) -> None:+ """Exporting items to each of the most common feed formats.++ The dict and item scenarios export the same flat values. Items also have+ field metadata, where exporters look up a serializer for every field. The+ nested scenario adds a list of nested items to every item, which exporters+ serialize recursively.+ """+ items = ITEM_SETS[values]++ def run() -> None:+ file = BytesIO()+ exporter = exporter_cls(file)+ exporter.start_exporting()+ for item in items:+ exporter.export_item(item)+ exporter.finish_exporting()+ assert file.getvalue().count(separator) == separators++ benchmark(run)