Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow yielding items from start_requests (#6417)

Georgiy Zatserklianyi6ce0342beb1a
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 13e636055..b45b12540 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -159,8 +159,9 @@ item_scraped
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.Spider` object
- :param response: the response from where the item was scraped
- :type response: :class:`~scrapy.http.Response` object
+ :param response: the response from where the item was scraped, or ``None``
+ if it was yielded from :meth:`~scrapy.Spider.start_requests`.
+ :type response: :class:`~scrapy.http.Response` | ``None``
item_dropped
~~~~~~~~~~~~
@@ -179,8 +180,9 @@ item_dropped
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.Spider` object
- :param response: the response from where the item was dropped
- :type response: :class:`~scrapy.http.Response` object
+ :param response: the response from where the item was dropped, or ``None``
+ if it was yielded from :meth:`~scrapy.Spider.start_requests`.
+ :type response: :class:`~scrapy.http.Response` | ``None``
:param exception: the exception (which must be a
:exc:`~scrapy.exceptions.DropItem` subclass) which caused the item
@@ -201,8 +203,10 @@ item_error
:param item: the item that caused the error in the :ref:`topics-item-pipeline`
:type item: :ref:`item object <item-types>`
- :param response: the response being processed when the exception was raised
- :type response: :class:`~scrapy.http.Response` object
+ :param response: the response being processed when the exception was
+ raised, or ``None`` if it was yielded from
+ :meth:`~scrapy.Spider.start_requests`.
+ :type response: :class:`~scrapy.http.Response` | ``None``
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 8ddf17a14..8f39bcd53 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
items).
It receives an iterable (in the ``start_requests`` parameter) and must
- return another iterable of :class:`~scrapy.Request` objects.
+ return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects <topics-items>`.
.. note:: When implementing this method in your spider middleware, you
should always return an iterable (that follows the input one) and
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 8a0102a51..e1b1c5ad6 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -203,7 +203,8 @@ scrapy.Spider
.. method:: start_requests()
- This method must return an iterable with the first Requests to crawl for
+ This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects
+ <topics-items>` for
this spider. It is called by Scrapy when the spider is opened for
scraping. Scrapy calls it only once, so it is safe to implement
:meth:`start_requests` as a generator.
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 5318cbd64..63d84339d 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -24,6 +24,7 @@ from typing import (
cast,
)
+from itemadapter import is_item
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from twisted.internet.task import LoopingCall
from twisted.python.failure import Failure
@@ -194,7 +195,7 @@ class ExecutionEngine:
if self.slot.start_requests is not None and not self._needs_backout():
try:
- request = next(self.slot.start_requests)
+ request_or_item = next(self.slot.start_requests)
except StopIteration:
self.slot.start_requests = None
except Exception:
@@ -205,7 +206,16 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
else:
- self.crawl(request)
+ if isinstance(request_or_item, Request):
+ self.crawl(request_or_item)
+ elif is_item(request_or_item):
+ self.scraper.start_itemproc(request_or_item, response=None)
+ else:
+ logger.error(
+ f"Got {request_or_item!r} among start requests. Only "
+ f"requests and items are supported. It will be "
+ f"ignored."
+ )
if self.spider_is_idle() and self.slot.close_if_idle:
self._spider_idle()
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index a7d65e1e3..7a51dbeb4 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -313,15 +313,11 @@ class Scraper:
"""Process each Request/Item (given in the output parameter) returned
from the given spider
"""
- assert self.slot is not None # typing
if isinstance(output, Request):
assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
elif is_item(output):
- self.slot.itemproc_size += 1
- dfd = self.itemproc.process_item(output, spider)
- dfd.addBoth(self._itemproc_finished, output, response, spider)
- return dfd
+ return self.start_itemproc(output, response=response)
elif output is None:
pass
else:
@@ -333,6 +329,19 @@ class Scraper:
)
return None
+ def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]:
+ """Send *item* to the item pipelines for processing.
+
+ *response* is the source of the item data. If the item does not come
+ from response data, e.g. it was hard-coded, set it to ``None``.
+ """
+ assert self.slot is not None # typing
+ assert self.crawler.spider is not None # typing
+ self.slot.itemproc_size += 1
+ dfd = self.itemproc.process_item(item, self.crawler.spider)
+ dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider)
+ return dfd
+
def _log_download_errors(
self,
spider_failure: Failure,
@@ -373,7 +382,7 @@ class Scraper:
return None
def _itemproc_finished(
- self, output: Any, item: Any, response: Response, spider: Spider
+ self, output: Any, item: Any, response: Optional[Response], spider: Spider
) -> Deferred[Any]:
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
assert self.slot is not None # typing
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 601209fb0..fea7003e5 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -9,6 +9,7 @@ from twisted.python.failure import Failure
# working around https://github.com/sphinx-doc/sphinx/issues/10400
from scrapy import Request, Spider # noqa: TC001
from scrapy.http import Response # noqa: TC001
+from scrapy.utils.python import global_object_name
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
@@ -92,11 +93,13 @@ class LogFormatter:
}
def scraped(
- self, item: Any, response: Union[Response, Failure], spider: Spider
+ self, item: Any, response: Union[Response, Failure, None], spider: Spider
) -> LogFormatterResult:
"""Logs a message when an item is scraped by a spider."""
src: Any
- if isinstance(response, Failure):
+ if response is None:
+ src = f"{global_object_name(spider.__class__)}.start_requests"
+ elif isinstance(response, Failure):
src = response.getErrorMessage()
else:
src = response
@@ -110,7 +113,11 @@ class LogFormatter:
}
def dropped(
- self, item: Any, exception: BaseException, response: Response, spider: Spider
+ self,
+ item: Any,
+ exception: BaseException,
+ response: Optional[Response],
+ spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
return {
@@ -123,7 +130,11 @@ class LogFormatter:
}
def item_error(
- self, item: Any, exception: BaseException, response: Response, spider: Spider
+ self,
+ item: Any,
+ exception: BaseException,
+ response: Optional[Response],
+ spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item causes an error while it is passing
through the item pipeline.
diff --git a/tests/spiders.py b/tests/spiders.py
index 743811893..5d5792858 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider):
yield from super().parse(response)
+class StartRequestsItemSpider(FollowAllSpider):
+ def start_requests(self):
+ yield {"name": "test item"}
+
+
+class StartRequestsGoodAndBadOutput(FollowAllSpider):
+ def start_requests(self):
+ yield {"a": "a"}
+ yield Request("data:,a")
+ yield "data:,b"
+ yield object()
+
+
class SingleRequestSpider(MetaSpider):
seed = None
callback_func = None
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 6cde4ed8c..125709571 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,5 +1,6 @@
import json
import logging
+import re
import unittest
from ipaddress import IPv4Address
from socket import gethostbyname
@@ -49,6 +50,8 @@ from tests.spiders import (
HeadersReceivedErrbackSpider,
SimpleSpider,
SingleRequestSpider,
+ StartRequestsGoodAndBadOutput,
+ StartRequestsItemSpider,
)
@@ -184,6 +187,39 @@ class CrawlTestCase(TestCase):
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
+ @defer.inlineCallbacks
+ def test_start_requests_items(self):
+ with LogCapture("scrapy", level=logging.ERROR) as log:
+ crawler = get_crawler(StartRequestsItemSpider)
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertEqual(len(log.records), 0)
+
+ @defer.inlineCallbacks
+ def test_start_requests_unsupported_output(self):
+ with LogCapture("scrapy", level=logging.ERROR) as log:
+ crawler = get_crawler(StartRequestsGoodAndBadOutput)
+ yield crawler.crawl(mockserver=self.mockserver)
+
+ self.assertEqual(len(log.records), 2)
+ self.assertEqual(
+ log.records[0].msg,
+ (
+ "Got 'data:,b' among start requests. Only requests and items "
+ "are supported. It will be ignored."
+ ),
+ )
+ self.assertTrue(
+ re.match(
+ (
+ r"^Got <object object at 0x[0-9a-fA-F]+> among start "
+ r"requests\. Only requests and items are supported\. It "
+ r"will be ignored\.$"
+ ),
+ log.records[1].msg,
+ )
+ )
+
@defer.inlineCallbacks
def test_start_requests_laziness(self):
settings = {"CONCURRENT_REQUESTS": 1}
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 38ca8d950..9dbffe353 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,5 +1,5 @@
import collections.abc
-from typing import Optional
+from typing import Optional, Union
from unittest import mock
from testfixtures import LogCapture
@@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
Should work for process_spider_output and, when it's supported, process_start_requests.
"""
- ITEM_TYPE: type
+ ITEM_TYPE: Union[type, tuple]
RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
@staticmethod
@@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware:
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
"""process_start_requests tests for simple start_requests"""
- ITEM_TYPE = Request
+ ITEM_TYPE = (Request, dict)
MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
def _start_requests(self):
- for i in range(3):
+ for i in range(2):
yield Request(f"https://example.com/{i}", dont_filter=True)
+ yield {"name": "test item"}
@defer.inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):