Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): return no links when a LinkExtractor restriction matches nothing (#452)

Suliman Abdulrazzaqf7b31379863b
diff --git a/scrapling/spiders/links.py b/scrapling/spiders/links.py
index 255b2de..88d9ce1 100644
--- a/scrapling/spiders/links.py
+++ b/scrapling/spiders/links.py
@@ -232,7 +232,8 @@ class LinkExtractor:
if self.restrict_css:
for cs in self.restrict_css:
scopes.extend(response.css(cs))
- if not scopes:
+ # Only an unrestricted extractor reads the whole page; a restriction that matches nothing yields no links
+ if not self.restrict_xpath and not self.restrict_css:
scopes = [response]
out: List[str] = []
diff --git a/tests/spiders/test_links.py b/tests/spiders/test_links.py
index cb81afa..ccef172 100644
--- a/tests/spiders/test_links.py
+++ b/tests/spiders/test_links.py
@@ -135,6 +135,18 @@ class TestRestrict:
urls = LinkExtractor(restrict_xpath='//div[@id="content"]').extract(resp)
assert urls == ["https://example.com/c"]
+ def test_restrict_matching_nothing_extracts_nothing(self):
+ html = """
+ <html><body>
+ <nav><a href="/nav-link">n</a></nav>
+ <main><a href="/main-link">m</a></main>
+ </body></html>
+ """
+ resp = _make_response(html)
+ assert LinkExtractor(restrict_css=".pagination").extract(resp) == []
+ assert LinkExtractor(restrict_xpath='//div[@class="pagination"]').extract(resp) == []
+ assert LinkExtractor(restrict_css=".pagination", restrict_xpath="//aside").extract(resp) == []
+
class TestTagsAttrs:
def test_custom_tags_and_attrs_for_stylesheets(self):