🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): return no links when a LinkExtractor restriction matches nothing (#452)
Suliman Abdulrazzaqf7b31379863b
diff --git a/scrapling/spiders/links.py b/scrapling/spiders/links.pyindex 255b2de..88d9ce1 100644--- a/scrapling/spiders/links.py+++ b/scrapling/spiders/links.py@@ -232,7 +232,8 @@ class LinkExtractor:if self.restrict_css:for cs in self.restrict_css:scopes.extend(response.css(cs))- if not scopes:+ # Only an unrestricted extractor reads the whole page; a restriction that matches nothing yields no links+ if not self.restrict_xpath and not self.restrict_css:scopes = [response]out: List[str] = []diff --git a/tests/spiders/test_links.py b/tests/spiders/test_links.pyindex cb81afa..ccef172 100644--- a/tests/spiders/test_links.py+++ b/tests/spiders/test_links.py@@ -135,6 +135,18 @@ class TestRestrict:urls = LinkExtractor(restrict_xpath='//div[@id="content"]').extract(resp)assert urls == ["https://example.com/c"]+ def test_restrict_matching_nothing_extracts_nothing(self):+ html = """+ <html><body>+ <nav><a href="/nav-link">n</a></nav>+ <main><a href="/main-link">m</a></main>+ </body></html>+ """+ resp = _make_response(html)+ assert LinkExtractor(restrict_css=".pagination").extract(resp) == []+ assert LinkExtractor(restrict_xpath='//div[@class="pagination"]').extract(resp) == []+ assert LinkExtractor(restrict_css=".pagination", restrict_xpath="//aside").extract(resp) == []+class TestTagsAttrs:def test_custom_tags_and_attrs_for_stylesheets(self):