Scrapy, a fast high-level web crawling & scraping framework for Python.
Add Request.id and Request.parent_id (#8225)
Adrian223065c5ea03
diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.pyindex e342e92cf..9b57f0090 100644--- a/docs/_ext/scrapyfixautodoc.py+++ b/docs/_ext/scrapyfixautodoc.py@@ -7,6 +7,9 @@ from typing import Any# pylint: disable=import-errorfrom sphinx.application import Sphinx+from sphinx.util import inspect++from scrapy.utils.trackref import object_refdef maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool:@@ -16,6 +19,30 @@ def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) ->return skip+def _stringify(app: Sphinx, func) -> str:+ return inspect.stringify_signature(+ inspect.signature(func, bound_method=True),+ show_return_annotation=False,+ unqualified_typehints=app.config.autodoc_typehints_format == "short",+ )+++def fix_object_ref_signature(+ app: Sphinx, what: str, name: str, obj, options, signature, return_annotation+) -> tuple[str, str] | None:+ # autodoc takes class signatures from __new__ before __init__, and+ # object_ref.__new__ accepts any arguments.+ if (+ what == "class"+ and issubclass(obj, object_ref)+ and obj.__init__ is not object.__init__+ and signature == _stringify(app, object_ref.__new__)+ ):+ return _stringify(app, obj.__init__), return_annotation+ return None++def setup(app: Sphinx) -> dict[str, Any]:+ app.connect("autodoc-process-signature", fix_object_ref_signature)app.connect("autodoc-skip-member", maybe_skip_member)return {"parallel_read_safe": True}diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex fdf0958be..b11d2c76e 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -25,65 +25,6 @@ Request objects.. autoclass:: scrapy.Request- :param url: the URL of this request-- If the URL is invalid, a :exc:`ValueError` exception is raised.- :type url: str-- :param callback: sets :attr:`callback`, defaults to ``None``.- :type callback: Callable[Concatenate[Response, ...], Any] | None-- :param method: the HTTP method of this request. Defaults to ``'GET'``.- :type method: str-- :param meta: the initial values for the :attr:`.Request.meta` attribute. If- given, the dict passed in this parameter will be shallow copied.- :type meta: dict-- :param body: the request body. If a string is passed, then it's encoded as- bytes using the ``encoding`` passed (which defaults to ``utf-8``). If- ``body`` is not given, an empty bytes object is stored. Regardless of the- type of this argument, the final value stored will be a bytes object- (never a string or ``None``).- :type body: bytes or str-- :param headers: the headers of this request. The dict values can be strings- (for single valued headers) or lists (for multi-valued headers). If- ``None`` is passed as value, the HTTP header will not be sent at all.-- .. caution:: Cookies set via the ``Cookie`` header are not considered by the- :ref:`cookie middleware <cookies>`. If you need to set cookies for a- request, use the ``cookies`` argument.-- :type headers: dict-- :param cookies: the request cookies, as a dict of cookie names and values- or as a list of dicts with a cookie each. See :ref:`cookies`.- :type cookies: dict or list-- :param encoding: the encoding of this request (defaults to ``'utf-8'``).- This encoding will be used to percent-encode the URL and to convert the- body to bytes (if given as a string).-- To disable URL percent-encoding for a request, use the- :reqmeta:`verbatim_url` request meta key.- :type encoding: str-- :param priority: sets :attr:`priority`, defaults to ``0``.- :type priority: int-- :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``.- :type dont_filter: bool-- :param errback: sets :attr:`errback`, defaults to ``None``.- :type errback: Callable[[Failure], Any] | None-- :param flags: Flags sent to the request, can be used for logging or similar purposes.- :type flags: list-- :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.- :type cb_kwargs: dict-.. attribute:: Request.urlA string containing the URL of this request.@@ -199,6 +140,10 @@ Request objects.. autoattribute:: dont_filter+ .. autoattribute:: id++ .. autoattribute:: parent_id+.. autoattribute:: Request.attributes.. method:: Request.copy()@@ -206,13 +151,7 @@ Request objectsReturn a new Request which is a copy of this Request. See also::ref:`callback-data`.- .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])-- Return a Request object with the same members, except for those members- given new values by whichever keyword arguments are specified. The- :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow- copied by default (unless new values are given as arguments). See also- :ref:`callback-data`.+ .. automethod:: replace.. automethod:: from_curldiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 64459e485..02a316852 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -59,6 +59,22 @@ _T = TypeVar("_T")QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]]+def _set_parent_id(output: Iterable[_T], parent_id: int) -> Iterable[_T]:+ for o in output:+ if isinstance(o, Request) and o.parent_id is None:+ o.parent_id = parent_id+ yield o+++async def _aset_parent_id(+ output: AsyncIterator[_T], parent_id: int+) -> AsyncIterator[_T]:+ async for o in output:+ if isinstance(o, Request) and o.parent_id is None:+ o.parent_id = parent_id+ yield o++class Slot:"""Scraper slot (one per running spider)"""@@ -325,6 +341,7 @@ class Scraper:if getattr(result, "request", None) is None:result.request = requestassert result.request+ parent_id = result.request.idcallback = result.request.callback or self.crawler.spider._parsewarn_on_generator_with_return_value(self.crawler.spider, callback)output = callback(result, **result.request.cb_kwargs)@@ -338,6 +355,7 @@ class Scraper:else: # result is a Failure# TODO: properly type adding this attribute to a Failureresult.request = request # type: ignore[attr-defined]+ parent_id = request.idif not request.errback:result.raiseException()warn_on_generator_with_return_value(self.crawler.spider, request.errback)@@ -353,7 +371,10 @@ class Scraper:ScrapyDeprecationWarning,stacklevel=2,)- return await ensure_awaitable(iterate_spider_output(output))+ output = await ensure_awaitable(iterate_spider_output(output))+ if isinstance(output, AsyncIterator):+ return _aset_parent_id(output, parent_id)+ return _set_parent_id(output, parent_id)def handle_spider_error(self,diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 7cff5f34d..02c7615c3 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -8,6 +8,9 @@ See documentation in docs/topics/request-response.rstfrom __future__ import annotationsimport inspect+import itertools+import secrets+import timefrom typing import (TYPE_CHECKING,Any,@@ -56,6 +59,12 @@ CookiesT: TypeAlias = (RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")+# Request IDs are unique across processes: process start time in milliseconds,+# 48 random bits, and a 40-bit per-process counter.+_request_ids = itertools.count(+ (time.time_ns() // 1_000_000 << 88) | (secrets.randbits(48) << 40)+)+def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:"""When assigned to the ``callback`` parameter of@@ -83,7 +92,40 @@ def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:class Request(object_ref):"""Represents an HTTP request, which is usually generated in a Spider and- executed by the Downloader, thus generating a :class:`~scrapy.http.Response`.+ executed by the Downloader, thus generating a+ :class:`~scrapy.http.Response`.++ *url* is the URL of the request. If it is invalid, :exc:`ValueError` is+ raised.++ *method* is the HTTP method of the request.++ *headers* are the request headers. Header values can be strings, for+ single-valued headers, or lists, for multi-valued headers. Headers with a+ ``None`` value are not sent.++ .. caution:: The :ref:`cookie middleware <cookies>` ignores cookies set+ through the ``Cookie`` header. Use *cookies* instead.++ *body* is the request body. A string body is encoded with *encoding*.++ *cookies* are the request cookies, as a dict of cookie names and values or+ as a list of dicts with a cookie each. See :ref:`cookies`.++ *encoding* is used to percent-encode *url*, unless the+ :reqmeta:`verbatim_url` request meta key is set, and to encode *body* if it+ is a string.++ *flags* is a list of labels for the request, which are included in log+ messages about it.++ *meta* and *cb_kwargs* are shallow copied into :attr:`meta` and+ :attr:`cb_kwargs`.++ *callback*, *errback*, *priority*, *dont_filter* and *parent_id* set the+ attributes of the same name. Set *parent_id* when building a request on+ behalf of another request, e.g. in a downloader middleware:+ ``Request(url, parent_id=request.id)``."""__attrs_and_slots = ("callback", "dont_filter", "errback", "method", "priority")@@ -96,6 +138,7 @@ class Request(object_ref):"encoding","flags","cb_kwargs",+ "parent_id",*__attrs_and_slots,)"""A tuple of :class:`str` objects containing the name of all public@@ -114,8 +157,10 @@ class Request(object_ref):"_encoding","_flags","_headers",+ "_id","_meta","_url",+ "parent_id",*__attrs_and_slots,)del __attrs_and_slots@@ -138,7 +183,9 @@ class Request(object_ref):errback: Callable[[Failure], Any] | None = None,flags: list[str] | None = None,cb_kwargs: dict[str, Any] | None = None,+ parent_id: int | None = None,) -> None:+ self._id: int = next(_request_ids)self._encoding: str = encoding # this one has to be set firstself.method: str = str(method).upper()self._meta: dict[str, Any] | None = dict(meta) if meta else None@@ -239,6 +286,26 @@ class Request(object_ref):self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else Noneself._flags: list[str] | None = list(flags) if flags else None+ self.parent_id: int | None = parent_id+ """:attr:`id` of the request this request derives from, or ``None``.++ Unless set explicitly, Scrapy sets it to the :attr:`id` of:++ - The original request, for requests created with :meth:`replace`+ or :meth:`copy`, e.g. redirects and retries.++ - The request of the response, for requests created with+ :meth:`Response.follow() <scrapy.http.Response.follow>` or+ :meth:`~scrapy.http.Response.follow_all`.++ - The request whose callback or errback yielded the request.+ """++ @property+ def id(self) -> int:+ """Unique identifier of this request."""+ return self._id+@propertydef cb_kwargs(self) -> dict[str, Any]:if self._cb_kwargs is None:@@ -343,7 +410,18 @@ class Request(object_ref):def replace(self, *args: Any, cls: type[Request] | None = None, **kwargs: Any) -> Request:- """Create a new Request with the same attributes except for those given new values"""+ """Return a new request with the same attributes as this one, except+ for those given new values as keyword arguments named after+ :attr:`attributes`.++ :attr:`cb_kwargs` and :attr:`meta` are :doc:`shallow copied+ <library/copy>` unless given new values. The new request gets a new+ :attr:`id`, and :attr:`parent_id` defaults to the :attr:`id` of this+ request. *cls* sets the class of the new request.++ See also :ref:`callback-data`.+ """+ kwargs.setdefault("parent_id", self.id)for x in self.attributes:kwargs.setdefault(x, getattr(self, x))if cls is None:@@ -423,6 +501,7 @@ class Request(object_ref):else self.errback),"headers": {k: list(v) for k, v in self.headers.items()},+ "id": self.id,}for attr in self.attributes:d.setdefault(attr, getattr(self, attr))diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 4140d104d..c09a6773c 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -303,6 +303,7 @@ class Response(object_ref):errback=errback,cb_kwargs=cb_kwargs,flags=flags,+ parent_id=self.request.id if self.request else None,)def follow_all(diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 2d804669b..6a75f8193 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -172,7 +172,10 @@ def request_from_dict(d: dict[str, Any], *, spider: Spider | None = None) -> Reqkwargs["callback"] = _get_method(spider, d["callback"])if d.get("errback") and spider:kwargs["errback"] = _get_method(spider, d["errback"])- return request_cls(**kwargs)+ request = request_cls(**kwargs)+ if "id" in d:+ request._id = d["id"]+ return requestdef _get_method(obj: Any, name: object) -> Any:diff --git a/tests/test_request_dict.py b/tests/test_request_dict.pyindex a34cc754e..aefcb3e79 100644--- a/tests/test_request_dict.py+++ b/tests/test_request_dict.py@@ -43,6 +43,8 @@ def _assert_same_request(r1: Request, r2: Request) -> None:assert r1.priority == r2.priorityassert r1.dont_filter == r2.dont_filterassert r1.flags == r2.flags+ assert r1.id == r2.id+ assert r1.parent_id == r2.parent_idif isinstance(r1, JsonRequest):assert isinstance(r2, JsonRequest)assert r1.dumps_kwargs == r2.dumps_kwargs@@ -53,6 +55,13 @@ def test_basic() -> None:_assert_serializes_ok(r)+def test_no_id() -> None:+ r = Request("http://www.example.com")+ d = r.to_dict()+ del d["id"]+ assert request_from_dict(d).id != r.id++def test_all_attributes(spider: MethodsSpider) -> None:r = Request(url="http://www.example.com",@@ -67,6 +76,7 @@ def test_all_attributes(spider: MethodsSpider) -> None:meta={"a": "b"},cb_kwargs={"k": "v"},flags=["testFlag"],+ parent_id=1,)_assert_serializes_ok(r, spider=spider)diff --git a/tests/test_request_lineage.py b/tests/test_request_lineage.pynew file mode 100644index 000000000..4c8851925--- /dev/null+++ b/tests/test_request_lineage.py@@ -0,0 +1,66 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++from scrapy import Request, Spider, signals+from scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_test++if TYPE_CHECKING:+ from collections.abc import AsyncIterator, Iterable++ from twisted.python.failure import Failure++ from scrapy.http import Response+ from tests.mockserver.http import MockServer+++class LineageSpider(Spider):+ name = "lineage"+ custom_settings = {"RETRY_TIMES": 1}++ def __init__(self, mockserver: MockServer, **kwargs: Any) -> None:+ super().__init__(**kwargs)+ self.mockserver = mockserver++ async def start(self) -> AsyncIterator[Any]:+ yield Request(self.mockserver.url("/redirect"))++ def parse(self, response: Response) -> Iterable[Request]:+ yield Request(self.mockserver.url("/status?n=503"), errback=self.on_error)+ assert response.request+ yield Request(+ self.mockserver.url("/text"), parent_id=response.request.parent_id+ )++ async def on_error(self, failure: Failure) -> AsyncIterator[Request]:+ yield Request(self.mockserver.url("/status?n=200"), callback=self.done)++ def done(self, response: Response) -> None:+ pass+++@coroutine_test+async def test_lineage(mockserver: MockServer) -> None:+ requests: dict[str, Request] = {}++ def track(request: Request) -> None:+ requests.setdefault(request.url.removeprefix(mockserver.url("")), request)+ if "503" in request.url and request.meta.get("retry_times"):+ requests["retry"] = request++ crawler = get_crawler(LineageSpider)+ crawler.signals.connect(track, signals.request_reached_downloader)+ await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))++ parents = {path: r.parent_id for path, r in requests.items()}+ ids = {path: r.id for path, r in requests.items()}+ assert parents == {+ "/redirect": None,+ "/redirected": ids["/redirect"],+ "/status?n=503": ids["/redirected"],+ "retry": ids["/status?n=503"],+ "/status?n=200": ids["retry"],+ "/text": ids["/redirect"],+ }diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.pyindex da13fe94c..8cd73a9b5 100644--- a/tests/utils/bases/http_request.py+++ b/tests/utils/bases/http_request.py@@ -294,6 +294,27 @@ class TestRequestBase(ABC):assert isinstance(r5, custom_request_cls)assert r5.url == r1.url+ def test_id(self):+ requests = [self.request_class("http://www.example.com") for _ in range(3)]+ ids = [r.id for r in requests]+ assert len(set(ids)) == len(ids)+ assert requests[0].parent_id is None+ with pytest.raises(AttributeError):+ requests[0].id = 1 # type: ignore[misc]++ def test_parent_id(self):+ r = self.request_class("http://www.example.com", parent_id=1)+ assert r.parent_id == 1++ @pytest.mark.parametrize("method", ["copy", "replace"])+ def test_copy_lineage(self, method):+ r1 = self.request_class("http://www.example.com", parent_id=1)+ r2 = getattr(r1, method)()+ assert r2.id != r1.id+ assert r2.parent_id == r1.id+ r3 = r1.replace(parent_id=None)+ assert r3.parent_id is None+def test_method_always_str(self):r = self.request_class("http://www.example.com", method="POST")assert isinstance(r.method, str)diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.pyindex 95cdadea8..b299285c7 100644--- a/tests/utils/bases/http_response.py+++ b/tests/utils/bases/http_response.py@@ -279,6 +279,13 @@ class TestResponseBase(ABC):fol = res.follow("http://example.com/", flags=["cached", "allowed"])assert fol.flags == ["cached", "allowed"]+ def test_follow_parent_id(self):+ assert self.response_class("http://example.com").follow("foo").parent_id is None+ request = Request("http://example.com")+ res = self.response_class("http://example.com", request=request)+ assert res.follow("foo").parent_id == request.id+ assert [r.parent_id for r in res.follow_all(["foo"])] == [request.id]+# Response.follow_alldef test_follow_all_absolute(self):