Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add name/from_rules to CrawlSpider Rule for rule chaining (#8066)

Adrian03e65d70fe18
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 1aa6358b9..09a76d8d8 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -558,6 +558,16 @@ Crawling rules
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
+ ``name`` is a string that identifies the rule, to be used as a target of
+ other rules' ``from_rules``.
+
+ ``from_rules`` is a string, or an iterable of strings, with the ``name``
+ of other rules. If set, this rule is only applied to responses reached
+ through one of those rules, instead of to every response.
+
+ .. versionadded:: VERSION
+ The ``name`` and ``from_rules`` parameters.
+
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
unexpected behaviour can occur otherwise.
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 373c0b8b5..f66cb2e58 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -70,6 +70,8 @@ class Rule:
process_links: ProcessLinksT | str | None = None,
process_request: ProcessRequestT | str | None = None,
errback: Callable[[Failure], Any] | str | None = None,
+ name: str | None = None,
+ from_rules: Iterable[str] | str | None = None,
):
self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor
self.callback: CallbackT | str | None = callback
@@ -80,6 +82,12 @@ class Rule:
process_request or _identity_process_request
)
self.follow: bool = follow if follow is not None else not callback
+ self.name: str | None = name
+ if isinstance(from_rules, str):
+ from_rules = (from_rules,)
+ self.from_rules: frozenset[str] | None = (
+ frozenset(from_rules) if from_rules is not None else None
+ )
def _compile(self, spider: Spider) -> None:
# this replaces method names with methods and we can't express this in type hints
@@ -135,14 +143,26 @@ class CrawlSpider(Spider):
url=link.url,
callback=self._callback,
errback=self._errback,
- meta={"rule": rule_index, "link_text": link.text},
+ meta={
+ "rule": rule_index,
+ "link_text": link.text,
+ "rule_name": self._rules[rule_index].name,
+ },
)
def _requests_to_follow(self, response: Response) -> Iterable[Request | None]:
if not isinstance(response, HtmlResponse):
return
seen: set[Link] = set()
+ response_rule_name = (
+ response.request.meta.get("rule_name") if response.request else None
+ )
for rule_index, rule in enumerate(self._rules):
+ if (
+ rule.from_rules is not None
+ and response_rule_name not in rule.from_rules
+ ):
+ continue
links: list[Link] = [
lnk
for lnk in rule.link_extractor.extract_links(response)
diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py
index ce556d0d1..2a0f55e61 100644
--- a/tests/test_spider_crawl.py
+++ b/tests/test_spider_crawl.py
@@ -239,6 +239,51 @@ class TestCrawlSpider(TestSpiderBase):
"HtmlResponse",
]
+ def test_rule_name_in_request_meta(self):
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
+
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
+ name = "test"
+ allowed_domains = ["example.org"]
+ rules = (Rule(LinkExtractor(allow=r"item"), name="items"),)
+
+ spider = _CrawlSpider()
+ output = list(spider._requests_to_follow(response))
+ assert [r.meta["rule_name"] for r in output] == ["items"]
+
+ def test_from_rules(self):
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
+
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
+ name = "test"
+ allowed_domains = ["example.org"]
+ rules = (
+ Rule(LinkExtractor(allow=r"item"), name="items"),
+ Rule(LinkExtractor(allow=r"about"), from_rules="items"),
+ )
+
+ spider = _CrawlSpider()
+
+ output = list(spider._requests_to_follow(response))
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ ]
+
+ response_from_items = HtmlResponse(
+ response.url,
+ body=self.test_body,
+ request=Request(response.url, meta={"rule_name": "items"}),
+ )
+ output = list(spider._requests_to_follow(response_from_items))
+ assert [r.url for r in output] == [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ ]
+
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_follow_links_attribute_population(self):
crawler = get_crawler()