Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

1.4 KB
Raw Blame
from __future__ import annotations

from typing import TYPE_CHECKING

from scrapy.http import Request, Response
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test

if TYPE_CHECKING:
    from collections.abc import AsyncIterator


@coroutine_test
async def test_async() -> None:
    crawler = get_crawler(Spider)
    mw = build_from_crawler(StartSpiderMiddleware, crawler)

    async def start() -> AsyncIterator[Request]:
        yield Request("data:,1")
        yield Request("data:,2", meta={"is_start_request": True})
        yield Request("data:,2", meta={"is_start_request": False})
        yield Request("data:,2", meta={"is_start_request": "foo"})

    result = [
        request.meta["is_start_request"] async for request in mw.process_start(start())
    ]
    assert result == [True, True, False, "foo"]


def test_spider_output_not_marked() -> None:
    # Requests from a non-None response (spider output) are not flagged.
    crawler = get_crawler(Spider)
    mw = build_from_crawler(StartSpiderMiddleware, crawler)
    response = Response("data:,")
    request = Request("data:,1")
    out = list(mw.process_spider_output(response, [request]))
    assert out == [request]
    assert "is_start_request" not in request.meta