Scrapy, a fast high-level web crawling & scraping framework for Python.
Add name/from_rules to CrawlSpider Rule for rule chaining (#8066)
Adrian03e65d70fe18
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 1aa6358b9..09a76d8d8 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -558,6 +558,16 @@ Crawling rulesIt receives a :class:`Twisted Failure <twisted.python.failure.Failure>`instance as first parameter.+ ``name`` is a string that identifies the rule, to be used as a target of+ other rules' ``from_rules``.++ ``from_rules`` is a string, or an iterable of strings, with the ``name``+ of other rules. If set, this rule is only applied to responses reached+ through one of those rules, instead of to every response.++ .. versionadded:: VERSION+ The ``name`` and ``from_rules`` parameters.+.. warning:: Because of its internal implementation, you must explicitly setcallbacks for new requests when writing :class:`CrawlSpider`-based spiders;unexpected behaviour can occur otherwise.diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 373c0b8b5..f66cb2e58 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -70,6 +70,8 @@ class Rule:process_links: ProcessLinksT | str | None = None,process_request: ProcessRequestT | str | None = None,errback: Callable[[Failure], Any] | str | None = None,+ name: str | None = None,+ from_rules: Iterable[str] | str | None = None,):self.link_extractor: LinkExtractor = link_extractor or _default_link_extractorself.callback: CallbackT | str | None = callback@@ -80,6 +82,12 @@ class Rule:process_request or _identity_process_request)self.follow: bool = follow if follow is not None else not callback+ self.name: str | None = name+ if isinstance(from_rules, str):+ from_rules = (from_rules,)+ self.from_rules: frozenset[str] | None = (+ frozenset(from_rules) if from_rules is not None else None+ )def _compile(self, spider: Spider) -> None:# this replaces method names with methods and we can't express this in type hints@@ -135,14 +143,26 @@ class CrawlSpider(Spider):url=link.url,callback=self._callback,errback=self._errback,- meta={"rule": rule_index, "link_text": link.text},+ meta={+ "rule": rule_index,+ "link_text": link.text,+ "rule_name": self._rules[rule_index].name,+ },)def _requests_to_follow(self, response: Response) -> Iterable[Request | None]:if not isinstance(response, HtmlResponse):returnseen: set[Link] = set()+ response_rule_name = (+ response.request.meta.get("rule_name") if response.request else None+ )for rule_index, rule in enumerate(self._rules):+ if (+ rule.from_rules is not None+ and response_rule_name not in rule.from_rules+ ):+ continuelinks: list[Link] = [lnkfor lnk in rule.link_extractor.extract_links(response)diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.pyindex ce556d0d1..2a0f55e61 100644--- a/tests/test_spider_crawl.py+++ b/tests/test_spider_crawl.py@@ -239,6 +239,51 @@ class TestCrawlSpider(TestSpiderBase):"HtmlResponse",]+ def test_rule_name_in_request_meta(self):+ response = HtmlResponse(+ "http://example.org/somepage/index.html", body=self.test_body+ )++ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]+ name = "test"+ allowed_domains = ["example.org"]+ rules = (Rule(LinkExtractor(allow=r"item"), name="items"),)++ spider = _CrawlSpider()+ output = list(spider._requests_to_follow(response))+ assert [r.meta["rule_name"] for r in output] == ["items"]++ def test_from_rules(self):+ response = HtmlResponse(+ "http://example.org/somepage/index.html", body=self.test_body+ )++ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]+ name = "test"+ allowed_domains = ["example.org"]+ rules = (+ Rule(LinkExtractor(allow=r"item"), name="items"),+ Rule(LinkExtractor(allow=r"about"), from_rules="items"),+ )++ spider = _CrawlSpider()++ output = list(spider._requests_to_follow(response))+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ ]++ response_from_items = HtmlResponse(+ response.url,+ body=self.test_body,+ request=Request(response.url, meta={"rule_name": "items"}),+ )+ output = list(spider._requests_to_follow(response_from_items))+ assert [r.url for r in output] == [+ "http://example.org/somepage/item/12.html",+ "http://example.org/about.html",+ ]+@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_follow_links_attribute_population(self):crawler = get_crawler()