Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: honor compound link deny extensions

renbkna0d35a3f94d98
diff --git a/scrapling/spiders/links.py b/scrapling/spiders/links.py
index 7b9a4ff..255b2de 100644
--- a/scrapling/spiders/links.py
+++ b/scrapling/spiders/links.py
@@ -150,11 +150,17 @@ def _compile_patterns(patterns: Union[str, Pattern[str], PatternInput, None]) ->
def _url_extension(url: str) -> str:
+ extensions = _url_extensions(url)
+ return extensions[0] if extensions else ""
+
+
+def _url_extensions(url: str) -> Tuple[str, ...]:
path = urlsplit(url).path
_, _, last = path.rpartition("/")
if "." not in last:
- return ""
- return last.rsplit(".", 1)[1].lower()
+ return ()
+ parts = last.lower().split(".")
+ return tuple(".".join(parts[i:]) for i in range(1, len(parts)) if parts[i])
def _filler(x):
@@ -277,8 +283,7 @@ class LinkExtractor:
if url.split("://", 1)[0] not in valid_schemas:
return False
- ext = _url_extension(url)
- if ext and ext in self.deny_extensions:
+ if self.deny_extensions and any(ext in self.deny_extensions for ext in _url_extensions(url)):
return False
if self.allow and not any(p.search(url) for p in self.allow):
diff --git a/tests/spiders/test_links.py b/tests/spiders/test_links.py
index 2413992..cb81afa 100644
--- a/tests/spiders/test_links.py
+++ b/tests/spiders/test_links.py
@@ -183,6 +183,12 @@ class TestExtensions:
urls = LinkExtractor().extract(resp)
assert urls == ["https://example.com/d"]
+ def test_default_deny_extensions_drops_compound_archive(self):
+ html = '<a href="/dataset.tar.gz">archive</a><a href="/d">ok</a>'
+ resp = _make_response(html)
+ urls = LinkExtractor().extract(resp)
+ assert urls == ["https://example.com/d"]
+
def test_custom_deny_extensions_overrides_default(self):
html = '<a href="/a.pdf">pdf</a><a href="/b.zip">zip</a>'
resp = _make_response(html)
@@ -190,6 +196,11 @@ class TestExtensions:
# .pdf now allowed because we replaced the set
assert urls == ["https://example.com/a.pdf"]
+ def test_custom_deny_extensions_honors_compound_extension(self):
+ ex = LinkExtractor(deny_extensions={"tar.gz"})
+ assert ex.matches("https://example.com/dataset.tar.gz") is False
+ assert ex.matches("https://example.com/dataset.gz") is True
+
def test_empty_deny_extensions_allows_everything(self):
html = '<a href="/a.pdf">pdf</a>'
resp = _make_response(html)