Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add benchmarks for item exporters and feed exports. (#8456)

Andrey Rakhmatullin8810d37ba97c
diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py
index 7495c309f..0f97c59f8 100644
--- a/tests/benchmarks/test_crawl.py
+++ b/tests/benchmarks/test_crawl.py
@@ -16,6 +16,7 @@ from tests.benchmarks import NullDownloadHandler, crawl
if TYPE_CHECKING:
from collections.abc import AsyncIterator
+ from pathlib import Path
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
@@ -345,6 +346,24 @@ def test_overhead_items(
)
+def test_overhead_feed(benchmark: BenchmarkFixture, tmp_path: Path) -> None:
+ """Overhead of exporting the items of a crawl to a feed.
+
+ It runs the crawl of the "many" scenario of ``test_overhead_items`` with a
+ JSON Lines feed added, so that the difference between both is the cost of
+ the feed export.
+ """
+ path = tmp_path / "items.jsonl"
+ # Every round writes the same file.
+ settings = {"FEEDS": {path: {"format": "jsonlines", "mode": "overwrite"}}}
+
+ def run() -> None:
+ _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE)
+ assert path.read_bytes().count(b"\n") == ITEM_REQUESTS * ITEMS_PER_RESPONSE
+
+ benchmark(run)
+
+
def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
"""Overhead of a crawl where item processing waits.
diff --git a/tests/benchmarks/test_exporters.py b/tests/benchmarks/test_exporters.py
new file mode 100644
index 000000000..b90a81ca3
--- /dev/null
+++ b/tests/benchmarks/test_exporters.py
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from io import BytesIO
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from scrapy import Field, Item
+from scrapy.exporters import CsvItemExporter, JsonLinesItemExporter, XmlItemExporter
+
+if TYPE_CHECKING:
+ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
+
+pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
+
+ITEMS = 1000
+
+
+class _Offer(Item):
+ seller = Field()
+ price = Field()
+ currency = Field()
+
+
+class _Product(Item):
+ url = Field()
+ name = Field()
+ sku = Field()
+ brand = Field()
+ price = Field()
+ currency = Field()
+ availability = Field()
+ rating = Field()
+ review_count = Field()
+ description = Field()
+ tags = Field()
+ offers = Field()
+
+
+def _values(index: int) -> dict[str, Any]:
+ return {
+ "url": f"https://www.example.com/catalogue/product-{index}.html",
+ "name": f"Product {index}",
+ "sku": f"SKU-{index:06d}",
+ "brand": "Example Brand",
+ "price": 19.99 + index,
+ "currency": "EUR",
+ "availability": "InStock",
+ "rating": 4.5,
+ "review_count": index % 500,
+ "description": f"A description of product {index}, a few words long.",
+ "tags": ["books", "fiction", "paperback"],
+ }
+
+
+def _offers(index: int) -> list[_Offer]:
+ return [
+ _Offer(seller=f"Seller {seller}", price=17.99 + index, currency="EUR")
+ for seller in range(2)
+ ]
+
+
+ITEM_SETS: dict[str, list[Any]] = {
+ "dict": [_values(index) for index in range(ITEMS)],
+ "item": [_Product(**_values(index)) for index in range(ITEMS)],
+ "nested": [
+ _Product(**_values(index), offers=_offers(index)) for index in range(ITEMS)
+ ],
+}
+
+
+ ("exporter_cls", "separator", "separators"),
+ [
+ pytest.param(JsonLinesItemExporter, b"\n", ITEMS, id="jsonlines"),
+ # The header row adds a line.
+ pytest.param(CsvItemExporter, b"\r\n", ITEMS + 1, id="csv"),
+ pytest.param(XmlItemExporter, b"<item>", ITEMS, id="xml"),
+ ],
+)
[email protected]("values", list(ITEM_SETS))
+def test_export(
+ benchmark: BenchmarkFixture,
+ exporter_cls: type[CsvItemExporter | JsonLinesItemExporter | XmlItemExporter],
+ separator: bytes,
+ separators: int,
+ values: str,
+) -> None:
+ """Exporting items to each of the most common feed formats.
+
+ The dict and item scenarios export the same flat values. Items also have
+ field metadata, where exporters look up a serializer for every field. The
+ nested scenario adds a list of nested items to every item, which exporters
+ serialize recursively.
+ """
+ items = ITEM_SETS[values]
+
+ def run() -> None:
+ file = BytesIO()
+ exporter = exporter_cls(file)
+ exporter.start_exporting()
+ for item in items:
+ exporter.export_item(item)
+ exporter.finish_exporting()
+ assert file.getvalue().count(separator) == separators
+
+ benchmark(run)