Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a schemes parameter to LinkExtractor, and extract data: and s3: links by default (#7994)
Adrian46e3fa1f04f8
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rstindex 91265efee..17c83628e 100644--- a/docs/topics/link-extractors.rst+++ b/docs/topics/link-extractors.rst@@ -40,6 +40,10 @@ can also be imported as ``scrapy.linkextractors.LinkExtractor``:from scrapy.linkextractors import LinkExtractor+.. autodata:: scrapy.linkextractors.IGNORED_EXTENSIONS++.. autodata:: scrapy.linkextractors.SUPPORTED_SCHEMES+LxmlLinkExtractor-----------------diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex b39859f7b..cb4d22022 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -14,7 +14,6 @@ if TYPE_CHECKING:from collections.abc import Iterablefrom re import Pattern-# common file extensions that are not followed if they occur in linksIGNORED_EXTENSIONS = [# archives"7z",@@ -114,14 +113,26 @@ IGNORED_EXTENSIONS = ["msp","py",]+"""File extensions of files that are usually not worth following, and hence the+default value of the ``deny_extensions`` parameter of :class:`LinkExtractor+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`."""-def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:- return any(r.search(url) for r in regexs)+SUPPORTED_SCHEMES = [+ "data",+ "file",+ "ftp",+ "http",+ "https",+ "s3",+]+"""URL schemes that Scrapy can download out of the box, and hence the default+value of the ``schemes`` parameter of :class:`LinkExtractor+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`."""-def _is_valid_url(url: str) -> bool:- return url.split("://", 1)[0] in {"http", "https", "file", "ftp"}+def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:+ return any(r.search(url) for r in regexs)# Top-level imports@@ -129,5 +140,6 @@ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor__all__ = ["IGNORED_EXTENSIONS",+ "SUPPORTED_SCHEMES","LinkExtractor",]diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex c47452c2b..6089f1cd0 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -18,7 +18,7 @@ from w3lib.html import strip_html5_whitespacefrom w3lib.url import canonicalize_url, safe_url_stringfrom scrapy.link import Link-from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches+from scrapy.linkextractors import IGNORED_EXTENSIONS, SUPPORTED_SCHEMES, _matchesfrom scrapy.utils.misc import arg_to_iter, rel_has_nofollowfrom scrapy.utils.python import unique as unique_listfrom scrapy.utils.response import get_base_url@@ -202,6 +202,14 @@ class LxmlLinkExtractor::data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.:type deny_extensions: list+ :param schemes: a single value or list of strings containing the URL+ schemes that extracted links may use. If not given, it will default to+ :data:`scrapy.linkextractors.SUPPORTED_SCHEMES`. Use ``()`` to extract+ links regardless of their scheme.++ .. versionadded:: VERSION+ :type schemes: str or list+:param restrict_xpaths: an XPath (or list of XPaths) that defines regionsinside the response where links should be extracted. If given, only thetext selected by those XPaths will be scanned for links.@@ -320,6 +328,7 @@ class LxmlLinkExtractor:restrict_text: _RegexOrSeveral | None = None,deny_tags: str | Iterable[str] = (),deny_attrs: str | Iterable[str] = (),+ schemes: str | Iterable[str] | None = None,):tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))deny_tags, deny_attrs = (@@ -347,8 +356,11 @@ class LxmlLinkExtractor:if deny_extensions is None:deny_extensions = IGNORED_EXTENSIONS+ if schemes is None:+ schemes = SUPPORTED_SCHEMESself.canonicalize: bool = canonicalizeself.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)}+ self.schemes: set[str] = set(arg_to_iter(schemes))self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text)@staticmethod@@ -359,13 +371,13 @@ class LxmlLinkExtractor:]def _link_allowed(self, link: Link) -> bool:- if not _is_valid_url(link.url):+ parsed_url = urlparse(link.url)+ if self.schemes and parsed_url.scheme not in self.schemes:return Falseif self.allow_res and not _matches(link.url, self.allow_res):return Falseif self.deny_res and _matches(link.url, self.deny_res):return False- parsed_url = urlparse(link.url)if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):diff --git a/tests/benchmarks/test_urls.py b/tests/benchmarks/test_urls.pyindex acc1d4d80..ac2561834 100644--- a/tests/benchmarks/test_urls.py+++ b/tests/benchmarks/test_urls.py@@ -20,17 +20,18 @@ RESPONSE_URL = "https://www.example.com/catalogue/page-1.html"# Links that each scenario returns for the benchmark page. They are fewer than# the anchors of the page because links to images, to other non-crawlable files-# and to non-HTTP schemes are rejected, and, except in the scenario that keeps-# duplicates, because the links that the navigation repeats are collapsed.-LINKS = 63-DUPLICATE_LINKS = 88-CANONICAL_LINKS = 60+# and to schemes that Scrapy cannot download are rejected, and, except in the+# scenario that keeps duplicates, because the links that the navigation repeats+# are collapsed.+LINKS = 64+DUPLICATE_LINKS = 89+CANONICAL_LINKS = 61FILTERED_LINKS = 45# Requests built from LINKS links that point to a different resource.# Canonicalization maps the rest to one that another link already covers, e.g.# two fragments of a page, or two spellings of one percent-escape.-FINGERPRINTS = 60+FINGERPRINTS = 61def _read_corpus() -> tuple[list[str], list[str]]:diff --git a/tests/benchmarks/urls.txt b/tests/benchmarks/urls.txtindex 6ef6939f1..79c15987b 100644--- a/tests/benchmarks/urls.txt+++ b/tests/benchmarks/urls.txt@@ -123,8 +123,10 @@ https://user:[email protected]/catalogue/page-1.html/downloads/catalogue.tar.gz/downloads/catalogue.zip-# Schemes that are not crawlable, which are rejected before any parsing.+# Schemes that Scrapy cannot download, which are rejected before any parsing.mailto:[email protected]javascript:void(0)tel:+441234567890++# A scheme other than HTTP that Scrapy can download.data:text/plain,hellodiff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 57bb7b0b2..d200c585c 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -684,6 +684,33 @@ class TestLxmlLinkExtractor:),]+ def test_schemes(self):+ body = b"""+ <html><body>+ <a href="page.html">Page</a>+ <a href="data:text/plain,Data">Data</a>+ <a href="mailto:[email protected]">Mail</a>+ </body></html>"""+ response = HtmlResponse("http://www.example.com/index.html", body=body)++ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(url="http://www.example.com/page.html", text="Page"),+ Link(url="data:text/plain,Data", text="Data"),+ ]++ lx = LxmlLinkExtractor(schemes="mailto")+ assert lx.extract_links(response) == [+ Link(url="mailto:[email protected]", text="Mail"),+ ]++ lx = LxmlLinkExtractor(schemes=())+ assert lx.extract_links(response) == [+ Link(url="http://www.example.com/page.html", text="Page"),+ Link(url="data:text/plain,Data", text="Data"),+ Link(url="mailto:[email protected]", text="Mail"),+ ]+def test_pickle_extractor(self):lx = LxmlLinkExtractor()assert isinstance(pickle.loads(pickle.dumps(lx)), LxmlLinkExtractor)