Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow yielding items from start_requests (#6417)
Georgiy Zatserklianyi6ce0342beb1a
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex 13e636055..b45b12540 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -159,8 +159,9 @@ item_scraped:param spider: the spider which scraped the item:type spider: :class:`~scrapy.Spider` object- :param response: the response from where the item was scraped- :type response: :class:`~scrapy.http.Response` object+ :param response: the response from where the item was scraped, or ``None``+ if it was yielded from :meth:`~scrapy.Spider.start_requests`.+ :type response: :class:`~scrapy.http.Response` | ``None``item_dropped~~~~~~~~~~~~@@ -179,8 +180,9 @@ item_dropped:param spider: the spider which scraped the item:type spider: :class:`~scrapy.Spider` object- :param response: the response from where the item was dropped- :type response: :class:`~scrapy.http.Response` object+ :param response: the response from where the item was dropped, or ``None``+ if it was yielded from :meth:`~scrapy.Spider.start_requests`.+ :type response: :class:`~scrapy.http.Response` | ``None``:param exception: the exception (which must be a:exc:`~scrapy.exceptions.DropItem` subclass) which caused the item@@ -201,8 +203,10 @@ item_error:param item: the item that caused the error in the :ref:`topics-item-pipeline`:type item: :ref:`item object <item-types>`- :param response: the response being processed when the exception was raised- :type response: :class:`~scrapy.http.Response` object+ :param response: the response being processed when the exception was+ raised, or ``None`` if it was yielded from+ :meth:`~scrapy.Spider.start_requests`.+ :type response: :class:`~scrapy.http.Response` | ``None``:param spider: the spider which raised the exception:type spider: :class:`~scrapy.Spider` objectdiff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 8ddf17a14..8f39bcd53 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.items).It receives an iterable (in the ``start_requests`` parameter) and must- return another iterable of :class:`~scrapy.Request` objects.+ return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects <topics-items>`... note:: When implementing this method in your spider middleware, youshould always return an iterable (that follows the input one) anddiff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 8a0102a51..e1b1c5ad6 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -203,7 +203,8 @@ scrapy.Spider.. method:: start_requests()- This method must return an iterable with the first Requests to crawl for+ This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects+ <topics-items>` forthis spider. It is called by Scrapy when the spider is opened forscraping. Scrapy calls it only once, so it is safe to implement:meth:`start_requests` as a generator.diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 5318cbd64..63d84339d 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -24,6 +24,7 @@ from typing import (cast,)+from itemadapter import is_itemfrom twisted.internet.defer import Deferred, inlineCallbacks, succeedfrom twisted.internet.task import LoopingCallfrom twisted.python.failure import Failure@@ -194,7 +195,7 @@ class ExecutionEngine:if self.slot.start_requests is not None and not self._needs_backout():try:- request = next(self.slot.start_requests)+ request_or_item = next(self.slot.start_requests)except StopIteration:self.slot.start_requests = Noneexcept Exception:@@ -205,7 +206,16 @@ class ExecutionEngine:extra={"spider": self.spider},)else:- self.crawl(request)+ if isinstance(request_or_item, Request):+ self.crawl(request_or_item)+ elif is_item(request_or_item):+ self.scraper.start_itemproc(request_or_item, response=None)+ else:+ logger.error(+ f"Got {request_or_item!r} among start requests. Only "+ f"requests and items are supported. It will be "+ f"ignored."+ )if self.spider_is_idle() and self.slot.close_if_idle:self._spider_idle()diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex a7d65e1e3..7a51dbeb4 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -313,15 +313,11 @@ class Scraper:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider"""- assert self.slot is not None # typingif isinstance(output, Request):assert self.crawler.engine is not None # typingself.crawler.engine.crawl(request=output)elif is_item(output):- self.slot.itemproc_size += 1- dfd = self.itemproc.process_item(output, spider)- dfd.addBoth(self._itemproc_finished, output, response, spider)- return dfd+ return self.start_itemproc(output, response=response)elif output is None:passelse:@@ -333,6 +329,19 @@ class Scraper:)return None+ def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]:+ """Send *item* to the item pipelines for processing.++ *response* is the source of the item data. If the item does not come+ from response data, e.g. it was hard-coded, set it to ``None``.+ """+ assert self.slot is not None # typing+ assert self.crawler.spider is not None # typing+ self.slot.itemproc_size += 1+ dfd = self.itemproc.process_item(item, self.crawler.spider)+ dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider)+ return dfd+def _log_download_errors(self,spider_failure: Failure,@@ -373,7 +382,7 @@ class Scraper:return Nonedef _itemproc_finished(- self, output: Any, item: Any, response: Response, spider: Spider+ self, output: Any, item: Any, response: Optional[Response], spider: Spider) -> Deferred[Any]:"""ItemProcessor finished for the given ``item`` and returned ``output``"""assert self.slot is not None # typingdiff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 601209fb0..fea7003e5 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -9,6 +9,7 @@ from twisted.python.failure import Failure# working around https://github.com/sphinx-doc/sphinx/issues/10400from scrapy import Request, Spider # noqa: TC001from scrapy.http import Response # noqa: TC001+from scrapy.utils.python import global_object_namefrom scrapy.utils.request import referer_strif TYPE_CHECKING:@@ -92,11 +93,13 @@ class LogFormatter:}def scraped(- self, item: Any, response: Union[Response, Failure], spider: Spider+ self, item: Any, response: Union[Response, Failure, None], spider: Spider) -> LogFormatterResult:"""Logs a message when an item is scraped by a spider."""src: Any- if isinstance(response, Failure):+ if response is None:+ src = f"{global_object_name(spider.__class__)}.start_requests"+ elif isinstance(response, Failure):src = response.getErrorMessage()else:src = response@@ -110,7 +113,11 @@ class LogFormatter:}def dropped(- self, item: Any, exception: BaseException, response: Response, spider: Spider+ self,+ item: Any,+ exception: BaseException,+ response: Optional[Response],+ spider: Spider,) -> LogFormatterResult:"""Logs a message when an item is dropped while it is passing through the item pipeline."""return {@@ -123,7 +130,11 @@ class LogFormatter:}def item_error(- self, item: Any, exception: BaseException, response: Response, spider: Spider+ self,+ item: Any,+ exception: BaseException,+ response: Optional[Response],+ spider: Spider,) -> LogFormatterResult:"""Logs a message when an item causes an error while it is passingthrough the item pipeline.diff --git a/tests/spiders.py b/tests/spiders.pyindex 743811893..5d5792858 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider):yield from super().parse(response)+class StartRequestsItemSpider(FollowAllSpider):+ def start_requests(self):+ yield {"name": "test item"}+++class StartRequestsGoodAndBadOutput(FollowAllSpider):+ def start_requests(self):+ yield {"a": "a"}+ yield Request("data:,a")+ yield "data:,b"+ yield object()++class SingleRequestSpider(MetaSpider):seed = Nonecallback_func = Nonediff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 6cde4ed8c..125709571 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,5 +1,6 @@import jsonimport logging+import reimport unittestfrom ipaddress import IPv4Addressfrom socket import gethostbyname@@ -49,6 +50,8 @@ from tests.spiders import (HeadersReceivedErrbackSpider,SimpleSpider,SingleRequestSpider,+ StartRequestsGoodAndBadOutput,+ StartRequestsItemSpider,)@@ -184,6 +187,39 @@ class CrawlTestCase(TestCase):self.assertIsNotNone(record.exc_info)self.assertIs(record.exc_info[0], ZeroDivisionError)+ @defer.inlineCallbacks+ def test_start_requests_items(self):+ with LogCapture("scrapy", level=logging.ERROR) as log:+ crawler = get_crawler(StartRequestsItemSpider)+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertEqual(len(log.records), 0)++ @defer.inlineCallbacks+ def test_start_requests_unsupported_output(self):+ with LogCapture("scrapy", level=logging.ERROR) as log:+ crawler = get_crawler(StartRequestsGoodAndBadOutput)+ yield crawler.crawl(mockserver=self.mockserver)++ self.assertEqual(len(log.records), 2)+ self.assertEqual(+ log.records[0].msg,+ (+ "Got 'data:,b' among start requests. Only requests and items "+ "are supported. It will be ignored."+ ),+ )+ self.assertTrue(+ re.match(+ (+ r"^Got <object object at 0x[0-9a-fA-F]+> among start "+ r"requests\. Only requests and items are supported\. It "+ r"will be ignored\.$"+ ),+ log.records[1].msg,+ )+ )+@defer.inlineCallbacksdef test_start_requests_laziness(self):settings = {"CONCURRENT_REQUESTS": 1}diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 38ca8d950..9dbffe353 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -1,5 +1,5 @@import collections.abc-from typing import Optional+from typing import Optional, Unionfrom unittest import mockfrom testfixtures import LogCapture@@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):Should work for process_spider_output and, when it's supported, process_start_requests."""- ITEM_TYPE: type+ ITEM_TYPE: Union[type, tuple]RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects@staticmethod@@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware:class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):"""process_start_requests tests for simple start_requests"""- ITEM_TYPE = Request+ ITEM_TYPE = (Request, dict)MW_SIMPLE = ProcessStartRequestsSimpleMiddlewaredef _start_requests(self):- for i in range(3):+ for i in range(2):yield Request(f"https://example.com/{i}", dont_filter=True)+ yield {"name": "test item"}@defer.inlineCallbacksdef _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):