Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove `FilteringLinkExtractor`
Laerte Pereira1a6408c3faad
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex 08a6ca1e8..b3b1eea55 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -6,18 +6,6 @@ This package contains a collection of Link Extractors.For more info see docs/topics/link-extractors.rst"""import re-from urllib.parse import urlparse-from warnings import warn--from parsel.csstranslator import HTMLTranslator-from w3lib.url import canonicalize_url--from scrapy.utils.deprecate import ScrapyDeprecationWarning-from scrapy.utils.misc import arg_to_iter-from scrapy.utils.url import (- url_is_from_any_domain, url_has_any_extension,-)-# common file extensions that are not followed if they occur in linksIGNORED_EXTENSIONS = [@@ -55,82 +43,5 @@ def _is_valid_url(url):return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}-class FilteringLinkExtractor:-- _csstranslator = HTMLTranslator()-- def __new__(cls, *args, **kwargs):- from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor- if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):- warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '- 'please use scrapy.linkextractors.LinkExtractor instead',- ScrapyDeprecationWarning, stacklevel=2)- return super().__new__(cls)-- def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,- restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):-- self.link_extractor = link_extractor-- self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)- for x in arg_to_iter(allow)]- self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)- for x in arg_to_iter(deny)]-- self.allow_domains = set(arg_to_iter(allow_domains))- self.deny_domains = set(arg_to_iter(deny_domains))-- self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))- self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,- arg_to_iter(restrict_css)))-- self.canonicalize = canonicalize- if deny_extensions is None:- deny_extensions = IGNORED_EXTENSIONS- self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}- self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)- for x in arg_to_iter(restrict_text)]-- def _link_allowed(self, link):- if not _is_valid_url(link.url):- return False- if self.allow_res and not _matches(link.url, self.allow_res):- return False- if self.deny_res and _matches(link.url, self.deny_res):- return False- parsed_url = urlparse(link.url)- if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):- return False- if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains):- return False- if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):- return False- if self.restrict_text and not _matches(link.text, self.restrict_text):- return False- return True-- def matches(self, url):-- if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):- return False- if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):- return False-- allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]- denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []- return any(allowed) and not any(denied)-- def _process_links(self, links):- links = [x for x in links if self._link_allowed(x)]- if self.canonicalize:- for link in links:- link.url = canonicalize_url(link.url)- links = self.link_extractor._process_links(links)- return links-- def _extract_links(self, *args, **kwargs):- return self.link_extractor._extract_links(*args, **kwargs)--# Top-level importsfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractordiff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex b5d2585a8..55639f504 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -3,18 +3,20 @@ Link extractor based on lxml.html"""import operatorfrom functools import partial-from urllib.parse import urljoin+from urllib.parse import urljoin, urlparseimport lxml.etree as etree+from parsel.csstranslator import HTMLTranslatorfrom w3lib.html import strip_html5_whitespacefrom w3lib.url import canonicalize_url, safe_url_stringfrom scrapy.link import Link-from scrapy.linkextractors import FilteringLinkExtractor+from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches,+ _re_type, re)from scrapy.utils.misc import arg_to_iter, rel_has_nofollowfrom scrapy.utils.python import unique as unique_listfrom scrapy.utils.response import get_base_url-+from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain# from lxml/src/lxml/html/__init__.pyXHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"@@ -98,7 +100,8 @@ class LxmlParserLinkExtractor:return links-class LxmlLinkExtractor(FilteringLinkExtractor):+class LxmlLinkExtractor:+ _csstranslator = HTMLTranslator()def __init__(self,@@ -118,7 +121,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):restrict_text=None,):tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))- lx = LxmlParserLinkExtractor(+ self.link_extractor = LxmlParserLinkExtractor(tag=partial(operator.contains, tags),attr=partial(operator.contains, attrs),unique=unique,@@ -126,18 +129,64 @@ class LxmlLinkExtractor(FilteringLinkExtractor):strip=strip,canonicalized=canonicalize)- super().__init__(- link_extractor=lx,- allow=allow,- deny=deny,- allow_domains=allow_domains,- deny_domains=deny_domains,- restrict_xpaths=restrict_xpaths,- restrict_css=restrict_css,- canonicalize=canonicalize,- deny_extensions=deny_extensions,- restrict_text=restrict_text,- )+ self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)+ for x in arg_to_iter(allow)]+ self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)+ for x in arg_to_iter(deny)]++ self.allow_domains = set(arg_to_iter(allow_domains))+ self.deny_domains = set(arg_to_iter(deny_domains))++ self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))+ self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,+ arg_to_iter(restrict_css)))++ if deny_extensions is None:+ deny_extensions = IGNORED_EXTENSIONS+ self.canonicalize = canonicalize+ self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}+ self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)+ for x in arg_to_iter(restrict_text)]++ def _link_allowed(self, link):+ if not _is_valid_url(link.url):+ return False+ if self.allow_res and not _matches(link.url, self.allow_res):+ return False+ if self.deny_res and _matches(link.url, self.deny_res):+ return False+ parsed_url = urlparse(link.url)+ if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):+ return False+ if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains):+ return False+ if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):+ return False+ if self.restrict_text and not _matches(link.text, self.restrict_text):+ return False+ return True++ def matches(self, url):++ if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):+ return False+ if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):+ return False++ allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]+ denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []+ return any(allowed) and not any(denied)++ def _process_links(self, links):+ links = [x for x in links if self._link_allowed(x)]+ if self.canonicalize:+ for link in links:+ link.url = canonicalize_url(link.url)+ links = self.link_extractor._process_links(links)+ return links++ def _extract_links(self, *args, **kwargs):+ return self.link_extractor._extract_links(*args, **kwargs)def extract_links(self, response):"""Returns a list of :class:`~scrapy.link.Link` objects from thediff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 6f133d77a..e28dc9bdb 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -1,12 +1,9 @@import pickleimport reimport unittest-from warnings import catch_warnings-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, XmlResponsefrom scrapy.link import Link-from scrapy.linkextractors import FilteringLinkExtractorfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractorfrom tests import get_testdata@@ -517,32 +514,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):def test_restrict_xpaths_with_html_entities(self):super().test_restrict_xpaths_with_html_entities()-- def test_filteringlinkextractor_deprecation_warning(self):- """Make sure the FilteringLinkExtractor deprecation warning is not- issued for LxmlLinkExtractor"""- with catch_warnings(record=True) as warnings:- LxmlLinkExtractor()- self.assertEqual(len(warnings), 0)-- class SubclassedLxmlLinkExtractor(LxmlLinkExtractor):- pass-- SubclassedLxmlLinkExtractor()- self.assertEqual(len(warnings), 0)---class FilteringLinkExtractorTest(unittest.TestCase):-- def test_deprecation_warning(self):- args = [None] * 10- with catch_warnings(record=True) as warnings:- FilteringLinkExtractor(*args)- self.assertEqual(len(warnings), 1)- self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)- with catch_warnings(record=True) as warnings:- class SubclassedFilteringLinkExtractor(FilteringLinkExtractor):- pass- SubclassedFilteringLinkExtractor(*args)- self.assertEqual(len(warnings), 1)- self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)