Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove deprecated classes BaseSgmlLinkExtractor, RegexLinkExtractor and SgmlLinkExtractor (#4356)

Artur Shellunts62a4ede5e995
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
deleted file mode 100644
index 3f2557248..000000000
--- a/scrapy/linkextractors/regex.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import re
-from urllib.parse import urljoin
-
-from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
-
-from scrapy.link import Link
-from scrapy.linkextractors.sgml import SgmlLinkExtractor
-
-
-linkre = re.compile(
- "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
- re.DOTALL | re.IGNORECASE)
-
-
-def clean_link(link_text):
- """Remove leading and trailing whitespace and punctuation"""
- return link_text.strip("\t\r\n '\"\x0c")
-
-
-class RegexLinkExtractor(SgmlLinkExtractor):
- """High performant link extractor"""
-
- def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
- def clean_text(text):
- return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
-
- def clean_url(url):
- clean_url = ''
- try:
- clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
- except ValueError:
- pass
- return clean_url
-
- if base_url is None:
- base_url = get_base_url(response_text, response_url, response_encoding)
-
- links_text = linkre.findall(response_text)
- return [Link(clean_url(url).encode(response_encoding),
- clean_text(text))
- for url, _, text in links_text]
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
deleted file mode 100644
index 2ba6bca45..000000000
--- a/scrapy/linkextractors/sgml.py
+++ /dev/null
@@ -1,151 +0,0 @@
-"""
-SGMLParser-based Link extractors
-"""
-import warnings
-from urllib.parse import urljoin
-from sgmllib import SGMLParser
-
-from w3lib.url import safe_url_string, canonicalize_url
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.linkextractors import FilteringLinkExtractor
-from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
-from scrapy.utils.response import get_base_url
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class BaseSgmlLinkExtractor(SGMLParser):
-
- def __init__(self, tag="a", attr="href", unique=False, process_value=None,
- strip=True, canonicalized=False):
- warnings.warn(
- "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
- "Please use scrapy.linkextractors.LinkExtractor",
- ScrapyDeprecationWarning, stacklevel=2,
- )
- SGMLParser.__init__(self)
- self.scan_tag = tag if callable(tag) else lambda t: t == tag
- self.scan_attr = attr if callable(attr) else lambda a: a == attr
- self.process_value = (lambda v: v) if process_value is None else process_value
- self.current_link = None
- self.unique = unique
- self.strip = strip
- if canonicalized:
- self.link_key = lambda link: link.url
- else:
- self.link_key = lambda link: canonicalize_url(link.url,
- keep_fragments=True)
-
- def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
- """ Do the real extraction work """
- self.reset()
- self.feed(response_text)
- self.close()
-
- ret = []
- if base_url is None:
- base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
- for link in self.links:
- if isinstance(link.url, str):
- link.url = link.url.encode(response_encoding)
- try:
- link.url = urljoin(base_url, link.url)
- except ValueError:
- continue
- link.url = safe_url_string(link.url, response_encoding)
- link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
- ret.append(link)
-
- return ret
-
- def _process_links(self, links):
- """ Normalize and filter extracted links
-
- The subclass should override it if necessary
- """
- return unique_list(links, key=self.link_key) if self.unique else links
-
- def extract_links(self, response):
- # wrapper needed to allow to work directly with text
- links = self._extract_links(response.body, response.url, response.encoding)
- links = self._process_links(links)
- return links
-
- def reset(self):
- SGMLParser.reset(self)
- self.links = []
- self.base_url = None
- self.current_link = None
-
- def unknown_starttag(self, tag, attrs):
- if tag == 'base':
- self.base_url = dict(attrs).get('href')
- if self.scan_tag(tag):
- for attr, value in attrs:
- if self.scan_attr(attr):
- if self.strip and value is not None:
- value = strip_html5_whitespace(value)
- url = self.process_value(value)
- if url is not None:
- link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
- self.links.append(link)
- self.current_link = link
-
- def unknown_endtag(self, tag):
- if self.scan_tag(tag):
- self.current_link = None
-
- def handle_data(self, data):
- if self.current_link:
- self.current_link.text = self.current_link.text + data
-
- def matches(self, url):
- """This extractor matches with any url, since
- it doesn't contain any patterns"""
- return True
-
-
-class SgmlLinkExtractor(FilteringLinkExtractor):
-
- def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
- tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
- process_value=None, deny_extensions=None, restrict_css=(),
- strip=True, restrict_text=()):
- warnings.warn(
- "SgmlLinkExtractor is deprecated and will be removed in future releases. "
- "Please use scrapy.linkextractors.LinkExtractor",
- ScrapyDeprecationWarning, stacklevel=2,
- )
-
- tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
- tag_func = lambda x: x in tags
- attr_func = lambda x: x in attrs
-
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', ScrapyDeprecationWarning)
- lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
- unique=unique, process_value=process_value, strip=strip,
- canonicalized=canonicalize)
-
- super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
- allow_domains=allow_domains, deny_domains=deny_domains,
- restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
- canonicalize=canonicalize, deny_extensions=deny_extensions,
- restrict_text=restrict_text)
-
- def extract_links(self, response):
- base_url = None
- if self.restrict_xpaths:
- base_url = get_base_url(response)
- body = u''.join(f
- for x in self.restrict_xpaths
- for f in response.xpath(x).getall()
- ).encode(response.encoding, errors='xmlcharrefreplace')
- else:
- body = response.body
-
- links = self._extract_links(body, response.url, response.encoding, base_url)
- links = self._process_links(links)
- return links
diff --git a/tests/ignores.txt b/tests/ignores.txt
index f6e0d6fbe..222288841 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,5 +1,3 @@
-scrapy/linkextractors/sgml.py
-scrapy/linkextractors/regex.py
scrapy/downloadermiddlewares/cookies.py
scrapy/extensions/statsmailer.py
scrapy/extensions/memusage.py
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 7d5db368a..2307ea865 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,7 +1,7 @@
<html>
<head>
<base href='http://example.com' />
-<title>Sample page with links for testing RegexLinkExtractor</title>
+<title>Sample page with links for testing LinkExtractor</title>
</head>
<body>
<div id='wrapper'>
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index fc31d7e5d..e7eee18de 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -2,7 +2,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=latin-1">
<base href='http://example.com' />
- <title>Sample page with links for testing RegexLinkExtractor</title>
+ <title>Sample page with links for testing LinkExtractor</title>
</head>
<body>
<div id='wrapper'>