Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

4.5 KB
Raw Blame
from __future__ import annotations

import csv
import itertools
import json
import marshal
import pickle
from io import BytesIO
from pathlib import Path
from typing import IO, TYPE_CHECKING, Any
from urllib.parse import urljoin
from urllib.request import pathname2url

import lxml.etree

from scrapy import Field, Item, Spider
from scrapy.utils.test import get_crawler

if TYPE_CHECKING:
    from collections.abc import Callable, Iterable

    from scrapy.crawler import Crawler
    from tests.mockserver.http import MockServer


def path_to_url(path: str | Path) -> str:
    return urljoin("file:", pathname2url(str(path)))


def printf_escape(s: str) -> str:
    return s.replace("%", "%%")


class MyItem(Item):
    foo = Field()
    egg = Field()
    baz = Field()


class MyItem2(Item):
    foo = Field()
    hello = Field()


_path_ids = itertools.count()


def unique_path(tmp_path: Path) -> Path:
    """Return a path inside *tmp_path* that no previous call returned."""
    return tmp_path / f"feed{next(_path_ids)}"


async def crawl_items(
    mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]
) -> Crawler:
    """Run a spider that yields *items* from a single response with
    *settings*, and return its crawler."""

    class TestSpider(Spider):
        name = "testspider"
        start_urls = [mockserver.url("/")]

        def parse(self, response):
            yield from items

    crawler = get_crawler(TestSpider, settings)
    await crawler.crawl_async()
    return crawler


async def export_by_path(
    mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]
) -> dict[str | Path, bytes]:
    """Export *items* with *settings*, whose ``FEEDS`` are keyed by local
    paths, and return the contents of those paths that were written."""
    feeds = settings["FEEDS"]
    feed_urls = {
        printf_escape(path_to_url(path)): options for path, options in feeds.items()
    }
    await crawl_items(mockserver, items, {**settings, "FEEDS": feed_urls})
    return {path: Path(path).read_bytes() for path in feeds if Path(path).exists()}


async def export_by_format(
    mockserver: MockServer, items: Iterable[Any], settings: dict[str, Any]
) -> dict[str, bytes | None]:
    """Like :func:`export_by_path`, but key the result by feed format, with
    ``None`` for feeds that were not written."""
    data = await export_by_path(mockserver, items, settings)
    return {
        options["format"]: data.get(path) for path, options in settings["FEEDS"].items()
    }


def csv_header(data: bytes) -> list[str]:
    """Return the column names of the CSV feed *data*."""
    return next(csv.reader(data.decode().splitlines()))


def _parse_csv(data: bytes) -> list[Any]:
    # CSV cannot tell an empty field from a missing one, so empty fields are
    # left out, matching how the other formats export missing fields.
    return [
        {k: v for k, v in row.items() if v}
        for row in csv.DictReader(data.decode().splitlines())
    ]


def _parse_jsonlines(data: bytes) -> list[Any]:
    return [json.loads(line) for line in data.splitlines()]


def _parse_xml(data: bytes) -> list[Any]:
    root = lxml.etree.fromstring(data)
    return [{e.tag: e.text for e in item} for item in root.findall("item")]


def _load_until_eof(data: bytes, load: Callable[[IO[bytes]], Any]) -> list[Any]:
    file = BytesIO(data)
    result: list[Any] = []
    while True:
        try:
            result.append(load(file))
        except EOFError:
            return result


def _parse_pickle(data: bytes) -> list[Any]:
    return _load_until_eof(data, pickle.load)


def _parse_marshal(data: bytes) -> list[Any]:
    return _load_until_eof(data, marshal.load)


#: Functions that turn feed contents back into items, by feed format.
PARSERS: dict[str, Callable[[bytes], list[Any]]] = {
    "csv": _parse_csv,
    "json": json.loads,
    "jsonlines": _parse_jsonlines,
    "xml": _parse_xml,
    "pickle": _parse_pickle,
    "marshal": _parse_marshal,
}


async def assert_exported(
    mockserver: MockServer,
    tmp_path: Path,
    fmt: str,
    items: Iterable[Any],
    header: Iterable[str],
    rows: list[dict[str, Any]],
    settings: dict[str, Any] | None = None,
) -> None:
    """Check that exporting *items* as *fmt* with *settings* produces *rows*
    and, for CSV, a *header* line."""
    settings = {**(settings or {}), "FEEDS": {unique_path(tmp_path): {"format": fmt}}}
    data = (await export_by_format(mockserver, items, settings))[fmt]
    assert data is not None
    if fmt == "csv":
        assert csv_header(data) == list(header)
    assert PARSERS[fmt](data) == rows