Scrapy, a fast high-level web crawling & scraping framework for Python.
Canonicalize each extracted link once (#7961)
Adrian7d9516f33292
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 46ac39a28..75f9753e7 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -332,7 +332,7 @@ class LxmlLinkExtractor:unique=unique,process=process_value,strip=strip,- canonicalized=not canonicalize,+ canonicalized=True,)self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow)self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny)diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 95a6aee54..7b73a133f 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -9,6 +9,7 @@ from w3lib import __version__ as w3lib_versionfrom scrapy.http import HtmlResponse, XmlResponsefrom scrapy.link import Link+from scrapy.linkextractors import lxmlhtmlfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractorfrom tests import get_testdata@@ -798,6 +799,25 @@ class Base:class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):extractor_cls = LxmlLinkExtractor+ def test_canonicalize_once_per_link(self, monkeypatch):+ canonicalize_url = lxmlhtml.canonicalize_url+ calls = []++ def counting_canonicalize_url(url, *args, **kwargs):+ calls.append(url)+ return canonicalize_url(url, *args, **kwargs)++ monkeypatch.setattr(lxmlhtml, "canonicalize_url", counting_canonicalize_url)+ response = HtmlResponse(+ "https://example.com",+ body=b"".join(b'<a href="/p?b=2&a=1#f%d">x</a>' % i for i in range(10)),+ )+ lx = self.extractor_cls(canonicalize=True)+ assert lx.extract_links(response) == [+ Link(url="https://example.com/p?a=1&b=2", text="x")+ ]+ assert len(calls) == 10+def test_link_restrict_text(self):html = b"""<a href="http://example.org/item1.html">Pic of a cat</a>