🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: honor compound link deny extensions
renbkna0d35a3f94d98
diff --git a/scrapling/spiders/links.py b/scrapling/spiders/links.pyindex 7b9a4ff..255b2de 100644--- a/scrapling/spiders/links.py+++ b/scrapling/spiders/links.py@@ -150,11 +150,17 @@ def _compile_patterns(patterns: Union[str, Pattern[str], PatternInput, None]) ->def _url_extension(url: str) -> str:+ extensions = _url_extensions(url)+ return extensions[0] if extensions else ""+++def _url_extensions(url: str) -> Tuple[str, ...]:path = urlsplit(url).path_, _, last = path.rpartition("/")if "." not in last:- return ""- return last.rsplit(".", 1)[1].lower()+ return ()+ parts = last.lower().split(".")+ return tuple(".".join(parts[i:]) for i in range(1, len(parts)) if parts[i])def _filler(x):@@ -277,8 +283,7 @@ class LinkExtractor:if url.split("://", 1)[0] not in valid_schemas:return False- ext = _url_extension(url)- if ext and ext in self.deny_extensions:+ if self.deny_extensions and any(ext in self.deny_extensions for ext in _url_extensions(url)):return Falseif self.allow and not any(p.search(url) for p in self.allow):diff --git a/tests/spiders/test_links.py b/tests/spiders/test_links.pyindex 2413992..cb81afa 100644--- a/tests/spiders/test_links.py+++ b/tests/spiders/test_links.py@@ -183,6 +183,12 @@ class TestExtensions:urls = LinkExtractor().extract(resp)assert urls == ["https://example.com/d"]+ def test_default_deny_extensions_drops_compound_archive(self):+ html = '<a href="/dataset.tar.gz">archive</a><a href="/d">ok</a>'+ resp = _make_response(html)+ urls = LinkExtractor().extract(resp)+ assert urls == ["https://example.com/d"]+def test_custom_deny_extensions_overrides_default(self):html = '<a href="/a.pdf">pdf</a><a href="/b.zip">zip</a>'resp = _make_response(html)@@ -190,6 +196,11 @@ class TestExtensions:# .pdf now allowed because we replaced the setassert urls == ["https://example.com/a.pdf"]+ def test_custom_deny_extensions_honors_compound_extension(self):+ ex = LinkExtractor(deny_extensions={"tar.gz"})+ assert ex.matches("https://example.com/dataset.tar.gz") is False+ assert ex.matches("https://example.com/dataset.gz") is True+def test_empty_deny_extensions_allows_everything(self):html = '<a href="/a.pdf">pdf</a>'resp = _make_response(html)