Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a schemes parameter to LinkExtractor, and extract data: and s3: links by default (#7994)

Adrian46e3fa1f04f8
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 91265efee..17c83628e 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -40,6 +40,10 @@ can also be imported as ``scrapy.linkextractors.LinkExtractor``:
from scrapy.linkextractors import LinkExtractor
+.. autodata:: scrapy.linkextractors.IGNORED_EXTENSIONS
+
+.. autodata:: scrapy.linkextractors.SUPPORTED_SCHEMES
+
LxmlLinkExtractor
-----------------
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index b39859f7b..cb4d22022 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -14,7 +14,6 @@ if TYPE_CHECKING:
from collections.abc import Iterable
from re import Pattern
-# common file extensions that are not followed if they occur in links
IGNORED_EXTENSIONS = [
# archives
"7z",
@@ -114,14 +113,26 @@ IGNORED_EXTENSIONS = [
"msp",
"py",
]
+"""File extensions of files that are usually not worth following, and hence the
+default value of the ``deny_extensions`` parameter of :class:`LinkExtractor
+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`."""
-def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:
- return any(r.search(url) for r in regexs)
+SUPPORTED_SCHEMES = [
+ "data",
+ "file",
+ "ftp",
+ "http",
+ "https",
+ "s3",
+]
+"""URL schemes that Scrapy can download out of the box, and hence the default
+value of the ``schemes`` parameter of :class:`LinkExtractor
+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`."""
-def _is_valid_url(url: str) -> bool:
- return url.split("://", 1)[0] in {"http", "https", "file", "ftp"}
+def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:
+ return any(r.search(url) for r in regexs)
# Top-level imports
@@ -129,5 +140,6 @@ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
__all__ = [
"IGNORED_EXTENSIONS",
+ "SUPPORTED_SCHEMES",
"LinkExtractor",
]
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index c47452c2b..6089f1cd0 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -18,7 +18,7 @@ from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
from scrapy.link import Link
-from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches
+from scrapy.linkextractors import IGNORED_EXTENSIONS, SUPPORTED_SCHEMES, _matches
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
@@ -202,6 +202,14 @@ class LxmlLinkExtractor:
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
:type deny_extensions: list
+ :param schemes: a single value or list of strings containing the URL
+ schemes that extracted links may use. If not given, it will default to
+ :data:`scrapy.linkextractors.SUPPORTED_SCHEMES`. Use ``()`` to extract
+ links regardless of their scheme.
+
+ .. versionadded:: VERSION
+ :type schemes: str or list
+
:param restrict_xpaths: an XPath (or list of XPaths) that defines regions
inside the response where links should be extracted. If given, only the
text selected by those XPaths will be scanned for links.
@@ -320,6 +328,7 @@ class LxmlLinkExtractor:
restrict_text: _RegexOrSeveral | None = None,
deny_tags: str | Iterable[str] = (),
deny_attrs: str | Iterable[str] = (),
+ schemes: str | Iterable[str] | None = None,
):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
deny_tags, deny_attrs = (
@@ -347,8 +356,11 @@ class LxmlLinkExtractor:
if deny_extensions is None:
deny_extensions = IGNORED_EXTENSIONS
+ if schemes is None:
+ schemes = SUPPORTED_SCHEMES
self.canonicalize: bool = canonicalize
self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)}
+ self.schemes: set[str] = set(arg_to_iter(schemes))
self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text)
@staticmethod
@@ -359,13 +371,13 @@ class LxmlLinkExtractor:
]
def _link_allowed(self, link: Link) -> bool:
- if not _is_valid_url(link.url):
+ parsed_url = urlparse(link.url)
+ if self.schemes and parsed_url.scheme not in self.schemes:
return False
if self.allow_res and not _matches(link.url, self.allow_res):
return False
if self.deny_res and _matches(link.url, self.deny_res):
return False
- parsed_url = urlparse(link.url)
if self.allow_domains and not url_is_from_any_domain(
parsed_url, self.allow_domains
):
diff --git a/tests/benchmarks/test_urls.py b/tests/benchmarks/test_urls.py
index acc1d4d80..ac2561834 100644
--- a/tests/benchmarks/test_urls.py
+++ b/tests/benchmarks/test_urls.py
@@ -20,17 +20,18 @@ RESPONSE_URL = "https://www.example.com/catalogue/page-1.html"
# Links that each scenario returns for the benchmark page. They are fewer than
# the anchors of the page because links to images, to other non-crawlable files
-# and to non-HTTP schemes are rejected, and, except in the scenario that keeps
-# duplicates, because the links that the navigation repeats are collapsed.
-LINKS = 63
-DUPLICATE_LINKS = 88
-CANONICAL_LINKS = 60
+# and to schemes that Scrapy cannot download are rejected, and, except in the
+# scenario that keeps duplicates, because the links that the navigation repeats
+# are collapsed.
+LINKS = 64
+DUPLICATE_LINKS = 89
+CANONICAL_LINKS = 61
FILTERED_LINKS = 45
# Requests built from LINKS links that point to a different resource.
# Canonicalization maps the rest to one that another link already covers, e.g.
# two fragments of a page, or two spellings of one percent-escape.
-FINGERPRINTS = 60
+FINGERPRINTS = 61
def _read_corpus() -> tuple[list[str], list[str]]:
diff --git a/tests/benchmarks/urls.txt b/tests/benchmarks/urls.txt
index 6ef6939f1..79c15987b 100644
--- a/tests/benchmarks/urls.txt
+++ b/tests/benchmarks/urls.txt
@@ -123,8 +123,10 @@ https://user:[email protected]/catalogue/page-1.html
/downloads/catalogue.tar.gz
/downloads/catalogue.zip
-# Schemes that are not crawlable, which are rejected before any parsing.
+# Schemes that Scrapy cannot download, which are rejected before any parsing.
javascript:void(0)
tel:+441234567890
+
+# A scheme other than HTTP that Scrapy can download.
data:text/plain,hello
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 57bb7b0b2..d200c585c 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -684,6 +684,33 @@ class TestLxmlLinkExtractor:
),
]
+ def test_schemes(self):
+ body = b"""
+ <html><body>
+ <a href="page.html">Page</a>
+ <a href="data:text/plain,Data">Data</a>
+ <a href="mailto:[email protected]">Mail</a>
+ </body></html>"""
+ response = HtmlResponse("http://www.example.com/index.html", body=body)
+
+ lx = LxmlLinkExtractor()
+ assert lx.extract_links(response) == [
+ Link(url="http://www.example.com/page.html", text="Page"),
+ Link(url="data:text/plain,Data", text="Data"),
+ ]
+
+ lx = LxmlLinkExtractor(schemes="mailto")
+ assert lx.extract_links(response) == [
+ Link(url="mailto:[email protected]", text="Mail"),
+ ]
+
+ lx = LxmlLinkExtractor(schemes=())
+ assert lx.extract_links(response) == [
+ Link(url="http://www.example.com/page.html", text="Page"),
+ Link(url="data:text/plain,Data", text="Data"),
+ Link(url="mailto:[email protected]", text="Mail"),
+ ]
+
def test_pickle_extractor(self):
lx = LxmlLinkExtractor()
assert isinstance(pickle.loads(pickle.dumps(lx)), LxmlLinkExtractor)