Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add Request.id and Request.parent_id (#8225)

Adrian223065c5ea03
diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py
index e342e92cf..9b57f0090 100644
--- a/docs/_ext/scrapyfixautodoc.py
+++ b/docs/_ext/scrapyfixautodoc.py
@@ -7,6 +7,9 @@ from typing import Any
# pylint: disable=import-error
from sphinx.application import Sphinx
+from sphinx.util import inspect
+
+from scrapy.utils.trackref import object_ref
def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool:
@@ -16,6 +19,30 @@ def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) ->
return skip
+def _stringify(app: Sphinx, func) -> str:
+ return inspect.stringify_signature(
+ inspect.signature(func, bound_method=True),
+ show_return_annotation=False,
+ unqualified_typehints=app.config.autodoc_typehints_format == "short",
+ )
+
+
+def fix_object_ref_signature(
+ app: Sphinx, what: str, name: str, obj, options, signature, return_annotation
+) -> tuple[str, str] | None:
+ # autodoc takes class signatures from __new__ before __init__, and
+ # object_ref.__new__ accepts any arguments.
+ if (
+ what == "class"
+ and issubclass(obj, object_ref)
+ and obj.__init__ is not object.__init__
+ and signature == _stringify(app, object_ref.__new__)
+ ):
+ return _stringify(app, obj.__init__), return_annotation
+ return None
+
+
def setup(app: Sphinx) -> dict[str, Any]:
+ app.connect("autodoc-process-signature", fix_object_ref_signature)
app.connect("autodoc-skip-member", maybe_skip_member)
return {"parallel_read_safe": True}
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index fdf0958be..b11d2c76e 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -25,65 +25,6 @@ Request objects
.. autoclass:: scrapy.Request
- :param url: the URL of this request
-
- If the URL is invalid, a :exc:`ValueError` exception is raised.
- :type url: str
-
- :param callback: sets :attr:`callback`, defaults to ``None``.
- :type callback: Callable[Concatenate[Response, ...], Any] | None
-
- :param method: the HTTP method of this request. Defaults to ``'GET'``.
- :type method: str
-
- :param meta: the initial values for the :attr:`.Request.meta` attribute. If
- given, the dict passed in this parameter will be shallow copied.
- :type meta: dict
-
- :param body: the request body. If a string is passed, then it's encoded as
- bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
- ``body`` is not given, an empty bytes object is stored. Regardless of the
- type of this argument, the final value stored will be a bytes object
- (never a string or ``None``).
- :type body: bytes or str
-
- :param headers: the headers of this request. The dict values can be strings
- (for single valued headers) or lists (for multi-valued headers). If
- ``None`` is passed as value, the HTTP header will not be sent at all.
-
- .. caution:: Cookies set via the ``Cookie`` header are not considered by the
- :ref:`cookie middleware <cookies>`. If you need to set cookies for a
- request, use the ``cookies`` argument.
-
- :type headers: dict
-
- :param cookies: the request cookies, as a dict of cookie names and values
- or as a list of dicts with a cookie each. See :ref:`cookies`.
- :type cookies: dict or list
-
- :param encoding: the encoding of this request (defaults to ``'utf-8'``).
- This encoding will be used to percent-encode the URL and to convert the
- body to bytes (if given as a string).
-
- To disable URL percent-encoding for a request, use the
- :reqmeta:`verbatim_url` request meta key.
- :type encoding: str
-
- :param priority: sets :attr:`priority`, defaults to ``0``.
- :type priority: int
-
- :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``.
- :type dont_filter: bool
-
- :param errback: sets :attr:`errback`, defaults to ``None``.
- :type errback: Callable[[Failure], Any] | None
-
- :param flags: Flags sent to the request, can be used for logging or similar purposes.
- :type flags: list
-
- :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
- :type cb_kwargs: dict
-
.. attribute:: Request.url
A string containing the URL of this request.
@@ -199,6 +140,10 @@ Request objects
.. autoattribute:: dont_filter
+ .. autoattribute:: id
+
+ .. autoattribute:: parent_id
+
.. autoattribute:: Request.attributes
.. method:: Request.copy()
@@ -206,13 +151,7 @@ Request objects
Return a new Request which is a copy of this Request. See also:
:ref:`callback-data`.
- .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
-
- Return a Request object with the same members, except for those members
- given new values by whichever keyword arguments are specified. The
- :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow
- copied by default (unless new values are given as arguments). See also
- :ref:`callback-data`.
+ .. automethod:: replace
.. automethod:: from_curl
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 64459e485..02a316852 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -59,6 +59,22 @@ _T = TypeVar("_T")
QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]]
+def _set_parent_id(output: Iterable[_T], parent_id: int) -> Iterable[_T]:
+ for o in output:
+ if isinstance(o, Request) and o.parent_id is None:
+ o.parent_id = parent_id
+ yield o
+
+
+async def _aset_parent_id(
+ output: AsyncIterator[_T], parent_id: int
+) -> AsyncIterator[_T]:
+ async for o in output:
+ if isinstance(o, Request) and o.parent_id is None:
+ o.parent_id = parent_id
+ yield o
+
+
class Slot:
"""Scraper slot (one per running spider)"""
@@ -325,6 +341,7 @@ class Scraper:
if getattr(result, "request", None) is None:
result.request = request
assert result.request
+ parent_id = result.request.id
callback = result.request.callback or self.crawler.spider._parse
warn_on_generator_with_return_value(self.crawler.spider, callback)
output = callback(result, **result.request.cb_kwargs)
@@ -338,6 +355,7 @@ class Scraper:
else: # result is a Failure
# TODO: properly type adding this attribute to a Failure
result.request = request # type: ignore[attr-defined]
+ parent_id = request.id
if not request.errback:
result.raiseException()
warn_on_generator_with_return_value(self.crawler.spider, request.errback)
@@ -353,7 +371,10 @@ class Scraper:
ScrapyDeprecationWarning,
stacklevel=2,
)
- return await ensure_awaitable(iterate_spider_output(output))
+ output = await ensure_awaitable(iterate_spider_output(output))
+ if isinstance(output, AsyncIterator):
+ return _aset_parent_id(output, parent_id)
+ return _set_parent_id(output, parent_id)
def handle_spider_error(
self,
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 7cff5f34d..02c7615c3 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -8,6 +8,9 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
import inspect
+import itertools
+import secrets
+import time
from typing import (
TYPE_CHECKING,
Any,
@@ -56,6 +59,12 @@ CookiesT: TypeAlias = (
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
+# Request IDs are unique across processes: process start time in milliseconds,
+# 48 random bits, and a 40-bit per-process counter.
+_request_ids = itertools.count(
+ (time.time_ns() // 1_000_000 << 88) | (secrets.randbits(48) << 40)
+)
+
def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
"""When assigned to the ``callback`` parameter of
@@ -83,7 +92,40 @@ def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
class Request(object_ref):
"""Represents an HTTP request, which is usually generated in a Spider and
- executed by the Downloader, thus generating a :class:`~scrapy.http.Response`.
+ executed by the Downloader, thus generating a
+ :class:`~scrapy.http.Response`.
+
+ *url* is the URL of the request. If it is invalid, :exc:`ValueError` is
+ raised.
+
+ *method* is the HTTP method of the request.
+
+ *headers* are the request headers. Header values can be strings, for
+ single-valued headers, or lists, for multi-valued headers. Headers with a
+ ``None`` value are not sent.
+
+ .. caution:: The :ref:`cookie middleware <cookies>` ignores cookies set
+ through the ``Cookie`` header. Use *cookies* instead.
+
+ *body* is the request body. A string body is encoded with *encoding*.
+
+ *cookies* are the request cookies, as a dict of cookie names and values or
+ as a list of dicts with a cookie each. See :ref:`cookies`.
+
+ *encoding* is used to percent-encode *url*, unless the
+ :reqmeta:`verbatim_url` request meta key is set, and to encode *body* if it
+ is a string.
+
+ *flags* is a list of labels for the request, which are included in log
+ messages about it.
+
+ *meta* and *cb_kwargs* are shallow copied into :attr:`meta` and
+ :attr:`cb_kwargs`.
+
+ *callback*, *errback*, *priority*, *dont_filter* and *parent_id* set the
+ attributes of the same name. Set *parent_id* when building a request on
+ behalf of another request, e.g. in a downloader middleware:
+ ``Request(url, parent_id=request.id)``.
"""
__attrs_and_slots = ("callback", "dont_filter", "errback", "method", "priority")
@@ -96,6 +138,7 @@ class Request(object_ref):
"encoding",
"flags",
"cb_kwargs",
+ "parent_id",
*__attrs_and_slots,
)
"""A tuple of :class:`str` objects containing the name of all public
@@ -114,8 +157,10 @@ class Request(object_ref):
"_encoding",
"_flags",
"_headers",
+ "_id",
"_meta",
"_url",
+ "parent_id",
*__attrs_and_slots,
)
del __attrs_and_slots
@@ -138,7 +183,9 @@ class Request(object_ref):
errback: Callable[[Failure], Any] | None = None,
flags: list[str] | None = None,
cb_kwargs: dict[str, Any] | None = None,
+ parent_id: int | None = None,
) -> None:
+ self._id: int = next(_request_ids)
self._encoding: str = encoding # this one has to be set first
self.method: str = str(method).upper()
self._meta: dict[str, Any] | None = dict(meta) if meta else None
@@ -239,6 +286,26 @@ class Request(object_ref):
self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None
self._flags: list[str] | None = list(flags) if flags else None
+ self.parent_id: int | None = parent_id
+ """:attr:`id` of the request this request derives from, or ``None``.
+
+ Unless set explicitly, Scrapy sets it to the :attr:`id` of:
+
+ - The original request, for requests created with :meth:`replace`
+ or :meth:`copy`, e.g. redirects and retries.
+
+ - The request of the response, for requests created with
+ :meth:`Response.follow() <scrapy.http.Response.follow>` or
+ :meth:`~scrapy.http.Response.follow_all`.
+
+ - The request whose callback or errback yielded the request.
+ """
+
+ @property
+ def id(self) -> int:
+ """Unique identifier of this request."""
+ return self._id
+
@property
def cb_kwargs(self) -> dict[str, Any]:
if self._cb_kwargs is None:
@@ -343,7 +410,18 @@ class Request(object_ref):
def replace(
self, *args: Any, cls: type[Request] | None = None, **kwargs: Any
) -> Request:
- """Create a new Request with the same attributes except for those given new values"""
+ """Return a new request with the same attributes as this one, except
+ for those given new values as keyword arguments named after
+ :attr:`attributes`.
+
+ :attr:`cb_kwargs` and :attr:`meta` are :doc:`shallow copied
+ <library/copy>` unless given new values. The new request gets a new
+ :attr:`id`, and :attr:`parent_id` defaults to the :attr:`id` of this
+ request. *cls* sets the class of the new request.
+
+ See also :ref:`callback-data`.
+ """
+ kwargs.setdefault("parent_id", self.id)
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
if cls is None:
@@ -423,6 +501,7 @@ class Request(object_ref):
else self.errback
),
"headers": {k: list(v) for k, v in self.headers.items()},
+ "id": self.id,
}
for attr in self.attributes:
d.setdefault(attr, getattr(self, attr))
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 4140d104d..c09a6773c 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -303,6 +303,7 @@ class Response(object_ref):
errback=errback,
cb_kwargs=cb_kwargs,
flags=flags,
+ parent_id=self.request.id if self.request else None,
)
def follow_all(
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 2d804669b..6a75f8193 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -172,7 +172,10 @@ def request_from_dict(d: dict[str, Any], *, spider: Spider | None = None) -> Req
kwargs["callback"] = _get_method(spider, d["callback"])
if d.get("errback") and spider:
kwargs["errback"] = _get_method(spider, d["errback"])
- return request_cls(**kwargs)
+ request = request_cls(**kwargs)
+ if "id" in d:
+ request._id = d["id"]
+ return request
def _get_method(obj: Any, name: object) -> Any:
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index a34cc754e..aefcb3e79 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -43,6 +43,8 @@ def _assert_same_request(r1: Request, r2: Request) -> None:
assert r1.priority == r2.priority
assert r1.dont_filter == r2.dont_filter
assert r1.flags == r2.flags
+ assert r1.id == r2.id
+ assert r1.parent_id == r2.parent_id
if isinstance(r1, JsonRequest):
assert isinstance(r2, JsonRequest)
assert r1.dumps_kwargs == r2.dumps_kwargs
@@ -53,6 +55,13 @@ def test_basic() -> None:
_assert_serializes_ok(r)
+def test_no_id() -> None:
+ r = Request("http://www.example.com")
+ d = r.to_dict()
+ del d["id"]
+ assert request_from_dict(d).id != r.id
+
+
def test_all_attributes(spider: MethodsSpider) -> None:
r = Request(
url="http://www.example.com",
@@ -67,6 +76,7 @@ def test_all_attributes(spider: MethodsSpider) -> None:
meta={"a": "b"},
cb_kwargs={"k": "v"},
flags=["testFlag"],
+ parent_id=1,
)
_assert_serializes_ok(r, spider=spider)
diff --git a/tests/test_request_lineage.py b/tests/test_request_lineage.py
new file mode 100644
index 000000000..4c8851925
--- /dev/null
+++ b/tests/test_request_lineage.py
@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from scrapy import Request, Spider, signals
+from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator, Iterable
+
+ from twisted.python.failure import Failure
+
+ from scrapy.http import Response
+ from tests.mockserver.http import MockServer
+
+
+class LineageSpider(Spider):
+ name = "lineage"
+ custom_settings = {"RETRY_TIMES": 1}
+
+ def __init__(self, mockserver: MockServer, **kwargs: Any) -> None:
+ super().__init__(**kwargs)
+ self.mockserver = mockserver
+
+ async def start(self) -> AsyncIterator[Any]:
+ yield Request(self.mockserver.url("/redirect"))
+
+ def parse(self, response: Response) -> Iterable[Request]:
+ yield Request(self.mockserver.url("/status?n=503"), errback=self.on_error)
+ assert response.request
+ yield Request(
+ self.mockserver.url("/text"), parent_id=response.request.parent_id
+ )
+
+ async def on_error(self, failure: Failure) -> AsyncIterator[Request]:
+ yield Request(self.mockserver.url("/status?n=200"), callback=self.done)
+
+ def done(self, response: Response) -> None:
+ pass
+
+
+@coroutine_test
+async def test_lineage(mockserver: MockServer) -> None:
+ requests: dict[str, Request] = {}
+
+ def track(request: Request) -> None:
+ requests.setdefault(request.url.removeprefix(mockserver.url("")), request)
+ if "503" in request.url and request.meta.get("retry_times"):
+ requests["retry"] = request
+
+ crawler = get_crawler(LineageSpider)
+ crawler.signals.connect(track, signals.request_reached_downloader)
+ await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
+
+ parents = {path: r.parent_id for path, r in requests.items()}
+ ids = {path: r.id for path, r in requests.items()}
+ assert parents == {
+ "/redirect": None,
+ "/redirected": ids["/redirect"],
+ "/status?n=503": ids["/redirected"],
+ "retry": ids["/status?n=503"],
+ "/status?n=200": ids["retry"],
+ "/text": ids["/redirect"],
+ }
diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py
index da13fe94c..8cd73a9b5 100644
--- a/tests/utils/bases/http_request.py
+++ b/tests/utils/bases/http_request.py
@@ -294,6 +294,27 @@ class TestRequestBase(ABC):
assert isinstance(r5, custom_request_cls)
assert r5.url == r1.url
+ def test_id(self):
+ requests = [self.request_class("http://www.example.com") for _ in range(3)]
+ ids = [r.id for r in requests]
+ assert len(set(ids)) == len(ids)
+ assert requests[0].parent_id is None
+ with pytest.raises(AttributeError):
+ requests[0].id = 1 # type: ignore[misc]
+
+ def test_parent_id(self):
+ r = self.request_class("http://www.example.com", parent_id=1)
+ assert r.parent_id == 1
+
+ @pytest.mark.parametrize("method", ["copy", "replace"])
+ def test_copy_lineage(self, method):
+ r1 = self.request_class("http://www.example.com", parent_id=1)
+ r2 = getattr(r1, method)()
+ assert r2.id != r1.id
+ assert r2.parent_id == r1.id
+ r3 = r1.replace(parent_id=None)
+ assert r3.parent_id is None
+
def test_method_always_str(self):
r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)
diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py
index 95cdadea8..b299285c7 100644
--- a/tests/utils/bases/http_response.py
+++ b/tests/utils/bases/http_response.py
@@ -279,6 +279,13 @@ class TestResponseBase(ABC):
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
assert fol.flags == ["cached", "allowed"]
+ def test_follow_parent_id(self):
+ assert self.response_class("http://example.com").follow("foo").parent_id is None
+ request = Request("http://example.com")
+ res = self.response_class("http://example.com", request=request)
+ assert res.follow("foo").parent_id == request.id
+ assert [r.parent_id for r in res.follow_all(["foo"])] == [request.id]
+
# Response.follow_all
def test_follow_all_absolute(self):