Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge LxmlLinkExtractor test classes. (#8002)
Andrey Rakhmatullin9ddb7c4dc8cd
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex bc8a4d636..d8ef5eaea 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -14,555 +14,143 @@ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtrfrom tests import get_testdata-# a hack to skip base class tests in pytest-class Base:- class TestLinkExtractorBase:- extractor_cls: type | None = None-- def setup_method(self):- body = get_testdata("link_extractor", "linkextractor.html")- self.response = HtmlResponse(url="http://example.com/index", body=body)-- def test_urls_type(self):- """Test that the resulting urls are str objects"""- lx = self.extractor_cls()- assert all(- isinstance(link.url, str) for link in lx.extract_links(self.response)- )+class TestLxmlLinkExtractor:+ def setup_method(self):+ body = get_testdata("link_extractor", "linkextractor.html")+ self.response = HtmlResponse(url="http://example.com/index", body=body)++ def test_urls_type(self):+ """Test that the resulting urls are str objects"""+ lx = LxmlLinkExtractor()+ assert all(+ isinstance(link.url, str) for link in lx.extract_links(self.response)+ )- def test_extract_all_links(self):- lx = self.extractor_cls()- page4_url = "http://example.com/page%204.html"-- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html#foo",- text="sample 3 repetition with fragment",- ),- Link(url="http://www.google.com/something", text=""),- Link(url="http://example.com/innertag.html", text="inner tag"),- Link(url=page4_url, text="href with whitespaces"),- ]-- def test_extract_filter_allow(self):- lx = self.extractor_cls(allow=("sample",))- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html#foo",- text="sample 3 repetition with fragment",- ),- ]-- def test_extract_filter_allow_with_duplicates(self):- lx = self.extractor_cls(allow=("sample",), unique=False)- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html",- text="sample 3 repetition",- ),- Link(- url="http://example.com/sample3.html",- text="sample 3 repetition",- ),- Link(- url="http://example.com/sample3.html#foo",- text="sample 3 repetition with fragment",- ),- ]-- def test_extract_filter_allow_with_duplicates_canonicalize(self):- lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True)- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html",- text="sample 3 repetition",- ),- Link(- url="http://example.com/sample3.html",- text="sample 3 repetition",- ),- Link(- url="http://example.com/sample3.html",- text="sample 3 repetition with fragment",- ),- ]-- def test_extract_filter_allow_no_duplicates_canonicalize(self):- lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True)- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- ]-- def test_extract_filter_allow_and_deny(self):- lx = self.extractor_cls(allow=("sample",), deny=("3",))- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- ]-- def test_extract_filter_allowed_domains(self):- lx = self.extractor_cls(allow_domains=("google.com",))- assert list(lx.extract_links(self.response)) == [- Link(url="http://www.google.com/something", text=""),- ]-- def test_extraction_using_single_values(self):- """Test the extractor's behaviour among different situations"""-- lx = self.extractor_cls(allow="sample")- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html#foo",- text="sample 3 repetition with fragment",- ),- ]-- lx = self.extractor_cls(allow="sample", deny="3")- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- ]-- lx = self.extractor_cls(allow_domains="google.com")- assert list(lx.extract_links(self.response)) == [- Link(url="http://www.google.com/something", text=""),- ]-- lx = self.extractor_cls(deny_domains="example.com")- assert list(lx.extract_links(self.response)) == [- Link(url="http://www.google.com/something", text=""),- ]-- def test_nofollow(self):- """Test the extractor's behaviour for links with rel='nofollow'"""-- html = b"""<html><head><title>Page title</title></head>- <body>- <div class='links'>- <p><a href="/about.html">About us</a></p>- </div>- <div>- <p><a href="/follow.html">Follow this link</a></p>- </div>- <div>- <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>- </div>- <div>- <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>- </div>- <div>- <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>- </div>- </body></html>"""- response = HtmlResponse("http://example.org/somepage/index.html", body=html)-- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(url="http://example.org/about.html", text="About us"),- Link(url="http://example.org/follow.html", text="Follow this link"),- Link(- url="http://example.org/nofollow.html",- text="Dont follow this one",- nofollow=True,- ),- Link(- url="http://example.org/nofollow2.html",- text="Choose to follow or not",- ),- Link(- url="http://google.com/something",- text="External link not to follow",- nofollow=True,- ),- ]-- def test_matches(self):- url1 = "http://lotsofstuff.com/stuff1/index"- url2 = "http://evenmorestuff.com/uglystuff/index"-- lx = self.extractor_cls(allow=(r"stuff1",))- assert lx.matches(url1)- assert not lx.matches(url2)-- lx = self.extractor_cls(deny=(r"uglystuff",))- assert lx.matches(url1)- assert not lx.matches(url2)-- lx = self.extractor_cls(allow_domains=("evenmorestuff.com",))- assert not lx.matches(url1)- assert lx.matches(url2)-- lx = self.extractor_cls(deny_domains=("lotsofstuff.com",))- assert not lx.matches(url1)- assert lx.matches(url2)-- lx = self.extractor_cls(- allow=["blah1"],- deny=["blah2"],- allow_domains=["blah1.com"],- deny_domains=["blah2.com"],- )- assert lx.matches("http://blah1.com/blah1")- assert not lx.matches("http://blah1.com/blah2")- assert not lx.matches("http://blah2.com/blah1")- assert not lx.matches("http://blah2.com/blah2")-- def test_restrict_xpaths(self):- lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',))- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- ]-- def test_restrict_xpaths_encoding(self):- """Test restrict_xpaths with encodings"""- html = b"""<html><head><title>Page title</title></head>- <body><p><a href="item/12.html">Item 12</a></p>- <div class='links'>- <p><a href="/about.html">About us\xa3</a></p>- </div>- <div>- <p><a href="/nofollow.html">This shouldn't be followed</a></p>- </div>- </body></html>"""- response = HtmlResponse(- "http://example.org/somepage/index.html",- body=html,- encoding="windows-1252",- )+ def test_extract_all_links(self):+ lx = LxmlLinkExtractor()+ page4_url = "http://example.com/page%204.html"- lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")- assert lx.extract_links(response) == [- Link(url="http://example.org/about.html", text="About us\xa3")- ]-- def test_restrict_xpaths_with_html_entities(self):- html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'- response = HtmlResponse(- "http://example.org/somepage/index.html",- body=html,- encoding="iso8859-15",- )- links = self.extractor_cls(restrict_xpaths="//p").extract_links(response)- assert links == [- Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")- ]-- def test_restrict_xpaths_concat_in_handle_data(self):- """html entities cause SGMLParser to call handle_data hook twice"""- body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>"""- response = HtmlResponse("http://example.org", body=body, encoding="gb18030")- lx = self.extractor_cls(restrict_xpaths="//div")- assert lx.extract_links(response) == [- Link(- url="http://example.org/foo",- text=">\u4eac<\u4e1c",- fragment="",- nofollow=False,- )- ]-- def test_restrict_css(self):- lx = self.extractor_cls(restrict_css=("#subwrapper a",))- assert lx.extract_links(self.response) == [- Link(url="http://example.com/sample2.html", text="sample 2")- ]-- def test_restrict_css_and_restrict_xpaths_together(self):- lx = self.extractor_cls(- restrict_xpaths=('//div[@id="subwrapper"]',),- restrict_css=("#subwrapper + a",),- )- assert list(lx.extract_links(self.response)) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- ]-- def test_area_tag_with_unicode_present(self):- body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""- response = HtmlResponse("http://example.org", body=body, encoding="utf-8")- lx = self.extractor_cls()- lx.extract_links(response)- lx.extract_links(response)- lx.extract_links(response)- assert lx.extract_links(response) == [- Link(- url="http://example.org/foo",- text="",- fragment="",- nofollow=False,- )- ]-- def test_encoded_url(self):- body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""- response = HtmlResponse(- "http://known.fm/AC%2FDC/", body=body, encoding="utf8"- )- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(- url="http://known.fm/AC%2FDC/?page=2",- text="BinB",- fragment="",- nofollow=False,- ),- ]-- def test_encoded_url_in_restricted_xpath(self):- body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""- response = HtmlResponse(- "http://known.fm/AC%2FDC/", body=body, encoding="utf8"- )- lx = self.extractor_cls(restrict_xpaths="//div")- assert lx.extract_links(response) == [- Link(- url="http://known.fm/AC%2FDC/?page=2",- text="BinB",- fragment="",- nofollow=False,- ),- ]-- def test_ignored_extensions(self):- # jpg is ignored by default- html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">"""- response = HtmlResponse("http://example.org/", body=html)- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(url="http://example.org/page.html", text="asd"),- ]-- # override denied extensions- lx = self.extractor_cls(deny_extensions=["html"])- assert lx.extract_links(response) == [- Link(url="http://example.org/photo.jpg"),- ]-- def test_process_value(self):- """Test restrict_xpaths with encodings"""- html = b"""-<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Text</a>-<a href="/about.html">About us</a>- """- response = HtmlResponse(- "http://example.org/somepage/index.html",- body=html,- encoding="windows-1252",- )+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html#foo",+ text="sample 3 repetition with fragment",+ ),+ Link(url="http://www.google.com/something", text=""),+ Link(url="http://example.com/innertag.html", text="inner tag"),+ Link(url=page4_url, text="href with whitespaces"),+ ]- def process_value(value):- m = re.search(r"javascript:goToPage\('(.*?)'", value)- return m.group(1) if m else None-- lx = self.extractor_cls(process_value=process_value)- assert lx.extract_links(response) == [- Link(url="http://example.org/other/page.html", text="Text")- ]-- def test_base_url_with_restrict_xpaths(self):- html = b"""<html><head><title>Page title</title><base href="http://otherdomain.com/base/" /></head>- <body><p><a href="item/12.html">Item 12</a></p>- </body></html>"""- response = HtmlResponse("http://example.org/somepage/index.html", body=html)- lx = self.extractor_cls(restrict_xpaths="//p")- assert lx.extract_links(response) == [- Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")- ]-- def test_attrs(self):- lx = self.extractor_cls(attrs="href")- page4_url = "http://example.com/page%204.html"-- assert lx.extract_links(self.response) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html#foo",- text="sample 3 repetition with fragment",- ),- Link(url="http://www.google.com/something", text=""),- Link(url="http://example.com/innertag.html", text="inner tag"),- Link(url=page4_url, text="href with whitespaces"),- ]-- lx = self.extractor_cls(- attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()- )- assert lx.extract_links(self.response) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample2.jpg", text=""),- Link(url="http://example.com/sample3.html", text="sample 3 text"),- Link(- url="http://example.com/sample3.html#foo",- text="sample 3 repetition with fragment",- ),- Link(url="http://www.google.com/something", text=""),- Link(url="http://example.com/innertag.html", text="inner tag"),- Link(url=page4_url, text="href with whitespaces"),- ]-- lx = self.extractor_cls(attrs=None)- assert lx.extract_links(self.response) == []-- def test_tags(self):- html = (- b'<html><area href="sample1.html"></area>'- b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>'- )- response = HtmlResponse("http://example.com/index.html", body=html)+ def test_extract_filter_allow(self):+ lx = LxmlLinkExtractor(allow=("sample",))+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html#foo",+ text="sample 3 repetition with fragment",+ ),+ ]- lx = self.extractor_cls(tags=None)- assert lx.extract_links(response) == []+ def test_extract_filter_allow_with_duplicates(self):+ lx = LxmlLinkExtractor(allow=("sample",), unique=False)+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html",+ text="sample 3 repetition",+ ),+ Link(+ url="http://example.com/sample3.html",+ text="sample 3 repetition",+ ),+ Link(+ url="http://example.com/sample3.html#foo",+ text="sample 3 repetition with fragment",+ ),+ ]- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(url="http://example.com/sample1.html", text=""),- Link(url="http://example.com/sample2.html", text="sample 2"),- ]+ def test_extract_filter_allow_with_duplicates_canonicalize(self):+ lx = LxmlLinkExtractor(allow=("sample",), unique=False, canonicalize=True)+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html",+ text="sample 3 repetition",+ ),+ Link(+ url="http://example.com/sample3.html",+ text="sample 3 repetition",+ ),+ Link(+ url="http://example.com/sample3.html",+ text="sample 3 repetition with fragment",+ ),+ ]- lx = self.extractor_cls(tags="area")- assert lx.extract_links(response) == [- Link(url="http://example.com/sample1.html", text=""),- ]+ def test_extract_filter_allow_no_duplicates_canonicalize(self):+ lx = LxmlLinkExtractor(allow=("sample",), unique=True, canonicalize=True)+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ ]- lx = self.extractor_cls(tags="a")- assert lx.extract_links(response) == [- Link(url="http://example.com/sample2.html", text="sample 2"),- ]+ def test_extract_filter_allow_and_deny(self):+ lx = LxmlLinkExtractor(allow=("sample",), deny=("3",))+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ ]- lx = self.extractor_cls(- tags=("a", "img"), attrs=("href", "src"), deny_extensions=()- )- assert lx.extract_links(response) == [- Link(url="http://example.com/sample2.html", text="sample 2"),- Link(url="http://example.com/sample2.jpg", text=""),- ]-- def test_tags_attrs(self):- html = b"""- <html><body>- <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>- <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>- </body></html>- """- response = HtmlResponse("http://example.com/index.html", body=html)-- lx = self.extractor_cls(tags="div", attrs="data-url")- assert lx.extract_links(response) == [- Link(- url="http://example.com/get?id=1",- text="Item 1",- fragment="",- nofollow=False,- ),- Link(- url="http://example.com/get?id=2",- text="Item 2",- fragment="",- nofollow=False,- ),- ]-- lx = self.extractor_cls(tags=("div",), attrs=("data-url",))- assert lx.extract_links(response) == [- Link(- url="http://example.com/get?id=1",- text="Item 1",- fragment="",- nofollow=False,- ),- Link(- url="http://example.com/get?id=2",- text="Item 2",- fragment="",- nofollow=False,- ),- ]-- def test_tags_attrs_wildcard_and_deny(self):- html = b"""- <html><body>- <a href="a.html">a</a>- <div data-url="div.html">div</div>- <span href="span.html">span</span>- <p data-url="p.html">p</p>- </body></html>- """- response = HtmlResponse("http://example.com/index.html", body=html)-- def urls(**kwargs):- lx = self.extractor_cls(**kwargs)- return [link.url for link in lx.extract_links(response)]-- # Default behavior is unchanged: only the listed tags and attributes.- assert urls() == ["http://example.com/a.html"]-- # "*" as a tag matches every tag.- assert urls(tags="*") == [- "http://example.com/a.html",- "http://example.com/span.html",- ]-- # "*" as an attribute matches every attribute.- assert urls(tags="*", attrs="*") == [- "http://example.com/a.html",- "http://example.com/div.html",- "http://example.com/span.html",- "http://example.com/p.html",- ]-- # deny_tags excludes tags from the wildcard.- assert urls(tags="*", attrs="*", deny_tags="div") == [- "http://example.com/a.html",- "http://example.com/span.html",- "http://example.com/p.html",- ]-- # deny_attrs excludes attributes from the wildcard.- assert urls(tags="*", attrs="*", deny_attrs="data-url") == [- "http://example.com/a.html",- "http://example.com/span.html",- ]-- # deny_tags also applies when tags are listed explicitly.- assert urls(tags=("a", "span"), attrs="*", deny_tags="span") == [- "http://example.com/a.html",- ]-- # The wildcard for one parameter is independent of the other.- assert urls(tags="a", attrs="*") == ["http://example.com/a.html"]-- def test_xhtml(self):- xhtml = b"""- <?xml version="1.0"?>- <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">- <head>- <title>XHTML document title</title>- </head>- <body>+ def test_extract_filter_allowed_domains(self):+ lx = LxmlLinkExtractor(allow_domains=("google.com",))+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://www.google.com/something", text=""),+ ]++ def test_extraction_using_single_values(self):+ """Test the extractor's behaviour among different situations"""++ lx = LxmlLinkExtractor(allow="sample")+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html#foo",+ text="sample 3 repetition with fragment",+ ),+ ]++ lx = LxmlLinkExtractor(allow="sample", deny="3")+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ ]++ lx = LxmlLinkExtractor(allow_domains="google.com")+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://www.google.com/something", text=""),+ ]++ lx = LxmlLinkExtractor(deny_domains="example.com")+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://www.google.com/something", text=""),+ ]++ def test_nofollow(self):+ """Test the extractor's behaviour for links with rel='nofollow'"""++ html = b"""<html><head><title>Page title</title></head>+ <body><div class='links'><p><a href="/about.html">About us</a></p></div>@@ -578,226 +166,624 @@ class Base:<div><p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p></div>- </body>- </html>- """-- response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)-- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(- url="http://example.com/about.html",- text="About us",- fragment="",- nofollow=False,- ),- Link(- url="http://example.com/follow.html",- text="Follow this link",- fragment="",- nofollow=False,- ),- Link(- url="http://example.com/nofollow.html",- text="Dont follow this one",- fragment="",- nofollow=True,- ),- Link(- url="http://example.com/nofollow2.html",- text="Choose to follow or not",- fragment="",- nofollow=False,- ),- Link(- url="http://google.com/something",- text="External link not to follow",- nofollow=True,- ),- ]-- response = XmlResponse("http://example.com/index.xhtml", body=xhtml)-- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(- url="http://example.com/about.html",- text="About us",- fragment="",- nofollow=False,- ),- Link(- url="http://example.com/follow.html",- text="Follow this link",- fragment="",- nofollow=False,- ),- Link(- url="http://example.com/nofollow.html",- text="Dont follow this one",- fragment="",- nofollow=True,- ),- Link(- url="http://example.com/nofollow2.html",- text="Choose to follow or not",- fragment="",- nofollow=False,- ),- Link(- url="http://google.com/something",- text="External link not to follow",- nofollow=True,- ),- ]-- def test_link_wrong_href(self):- html = b"""- <a href="http://example.org/item1.html">Item 1</a>- <a href="http://[example.org/item2.html">Item 2</a>- <a href="http://example.org/item3.html">Item 3</a>- """- response = HtmlResponse("http://example.org/index.html", body=html)- lx = self.extractor_cls()- assert list(lx.extract_links(response)) == [- Link(- url="http://example.org/item1.html",- text="Item 1",- nofollow=False,- ),- Link(- url="http://example.org/item3.html",- text="Item 3",- nofollow=False,- ),- ]-- def test_ftp_links(self):- body = b"""- <html><body>- <div><a href="ftp://www.external.com/">An Item</a></div>- </body></html>"""- response = HtmlResponse(- "http://www.example.com/index.html", body=body, encoding="utf8"- )- lx = self.extractor_cls()- assert lx.extract_links(response) == [- Link(- url="ftp://www.external.com/",- text="An Item",- fragment="",- nofollow=False,- ),- ]-- def test_pickle_extractor(self):- lx = self.extractor_cls()- assert isinstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)-- def test_link_extractor_aggregation(self):- """When a parameter like restrict_css is used, the underlying- implementation calls its internal link extractor once per selector- matching the specified restrictions, and then aggregates the- extracted links.-- Test that aggregation respects the unique and canonicalize- parameters.- """- # unique=True (default), canonicalize=False (default)- lx = self.extractor_cls(restrict_css=("div",))- response = HtmlResponse(- "https://example.com",- body=b"""- <div>- <a href="/a">a1</a>- <a href="/b?a=1&b=2">b1</a>- </div>- <div>- <a href="/a">a2</a>- <a href="/b?b=2&a=1">b2</a>- </div>- """,- )- actual = lx.extract_links(response)- assert actual == [- Link(url="https://example.com/a", text="a1"),- Link(url="https://example.com/b?a=1&b=2", text="b1"),- Link(url="https://example.com/b?b=2&a=1", text="b2"),- ]-- # unique=True (default), canonicalize=True- lx = self.extractor_cls(restrict_css=("div",), canonicalize=True)- response = HtmlResponse(- "https://example.com",- body=b"""- <div>- <a href="/a">a1</a>- <a href="/b?a=1&b=2">b1</a>- </div>- <div>- <a href="/a">a2</a>- <a href="/b?b=2&a=1">b2</a>- </div>- """,- )- actual = lx.extract_links(response)- assert actual == [- Link(url="https://example.com/a", text="a1"),- Link(url="https://example.com/b?a=1&b=2", text="b1"),- ]-- # unique=False, canonicalize=False (default)- lx = self.extractor_cls(restrict_css=("div",), unique=False)- response = HtmlResponse(- "https://example.com",- body=b"""- <div>- <a href="/a">a1</a>- <a href="/b?a=1&b=2">b1</a>- </div>- <div>- <a href="/a">a2</a>- <a href="/b?b=2&a=1">b2</a>- </div>- """,- )- actual = lx.extract_links(response)- assert actual == [- Link(url="https://example.com/a", text="a1"),- Link(url="https://example.com/b?a=1&b=2", text="b1"),- Link(url="https://example.com/a", text="a2"),- Link(url="https://example.com/b?b=2&a=1", text="b2"),- ]-- # unique=False, canonicalize=True- lx = self.extractor_cls(- restrict_css=("div",), unique=False, canonicalize=True+ </body></html>"""+ response = HtmlResponse("http://example.org/somepage/index.html", body=html)++ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(url="http://example.org/about.html", text="About us"),+ Link(url="http://example.org/follow.html", text="Follow this link"),+ Link(+ url="http://example.org/nofollow.html",+ text="Dont follow this one",+ nofollow=True,+ ),+ Link(+ url="http://example.org/nofollow2.html",+ text="Choose to follow or not",+ ),+ Link(+ url="http://google.com/something",+ text="External link not to follow",+ nofollow=True,+ ),+ ]++ def test_matches(self):+ url1 = "http://lotsofstuff.com/stuff1/index"+ url2 = "http://evenmorestuff.com/uglystuff/index"++ lx = LxmlLinkExtractor(allow=(r"stuff1",))+ assert lx.matches(url1)+ assert not lx.matches(url2)++ lx = LxmlLinkExtractor(deny=(r"uglystuff",))+ assert lx.matches(url1)+ assert not lx.matches(url2)++ lx = LxmlLinkExtractor(allow_domains=("evenmorestuff.com",))+ assert not lx.matches(url1)+ assert lx.matches(url2)++ lx = LxmlLinkExtractor(deny_domains=("lotsofstuff.com",))+ assert not lx.matches(url1)+ assert lx.matches(url2)++ lx = LxmlLinkExtractor(+ allow=["blah1"],+ deny=["blah2"],+ allow_domains=["blah1.com"],+ deny_domains=["blah2.com"],+ )+ assert lx.matches("http://blah1.com/blah1")+ assert not lx.matches("http://blah1.com/blah2")+ assert not lx.matches("http://blah2.com/blah1")+ assert not lx.matches("http://blah2.com/blah2")++ def test_restrict_xpaths(self):+ lx = LxmlLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]',))+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ ]++ def test_restrict_xpaths_encoding(self):+ """Test restrict_xpaths with encodings"""+ html = b"""<html><head><title>Page title</title></head>+ <body><p><a href="item/12.html">Item 12</a></p>+ <div class='links'>+ <p><a href="/about.html">About us\xa3</a></p>+ </div>+ <div>+ <p><a href="/nofollow.html">This shouldn't be followed</a></p>+ </div>+ </body></html>"""+ response = HtmlResponse(+ "http://example.org/somepage/index.html",+ body=html,+ encoding="windows-1252",+ )++ lx = LxmlLinkExtractor(restrict_xpaths="//div[@class='links']")+ assert lx.extract_links(response) == [+ Link(url="http://example.org/about.html", text="About us\xa3")+ ]++ def test_restrict_xpaths_with_html_entities(self):+ html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'+ response = HtmlResponse(+ "http://example.org/somepage/index.html",+ body=html,+ encoding="iso8859-15",+ )+ links = LxmlLinkExtractor(restrict_xpaths="//p").extract_links(response)+ assert links == [+ Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")+ ]++ def test_restrict_xpaths_concat_in_handle_data(self):+ """html entities cause SGMLParser to call handle_data hook twice"""+ body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>"""+ response = HtmlResponse("http://example.org", body=body, encoding="gb18030")+ lx = LxmlLinkExtractor(restrict_xpaths="//div")+ assert lx.extract_links(response) == [+ Link(+ url="http://example.org/foo",+ text=">\u4eac<\u4e1c",+ fragment="",+ nofollow=False,)- response = HtmlResponse(- "https://example.com",- body=b"""- <div>- <a href="/a">a1</a>- <a href="/b?a=1&b=2">b1</a>- </div>- <div>- <a href="/a">a2</a>- <a href="/b?b=2&a=1">b2</a>- </div>- """,+ ]++ def test_restrict_css(self):+ lx = LxmlLinkExtractor(restrict_css=("#subwrapper a",))+ assert lx.extract_links(self.response) == [+ Link(url="http://example.com/sample2.html", text="sample 2")+ ]++ def test_restrict_css_and_restrict_xpaths_together(self):+ lx = LxmlLinkExtractor(+ restrict_xpaths=('//div[@id="subwrapper"]',),+ restrict_css=("#subwrapper + a",),+ )+ assert list(lx.extract_links(self.response)) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ ]++ def test_area_tag_with_unicode_present(self):+ body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""+ response = HtmlResponse("http://example.org", body=body, encoding="utf-8")+ lx = LxmlLinkExtractor()+ lx.extract_links(response)+ lx.extract_links(response)+ lx.extract_links(response)+ assert lx.extract_links(response) == [+ Link(+ url="http://example.org/foo",+ text="",+ fragment="",+ nofollow=False,)- actual = lx.extract_links(response)- assert actual == [- Link(url="https://example.com/a", text="a1"),- Link(url="https://example.com/b?a=1&b=2", text="b1"),- Link(url="https://example.com/a", text="a2"),- Link(url="https://example.com/b?a=1&b=2", text="b2"),- ]+ ]+ def test_encoded_url(self):+ body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""+ response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding="utf8")+ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(+ url="http://known.fm/AC%2FDC/?page=2",+ text="BinB",+ fragment="",+ nofollow=False,+ ),+ ]++ def test_encoded_url_in_restricted_xpath(self):+ body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>"""+ response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding="utf8")+ lx = LxmlLinkExtractor(restrict_xpaths="//div")+ assert lx.extract_links(response) == [+ Link(+ url="http://known.fm/AC%2FDC/?page=2",+ text="BinB",+ fragment="",+ nofollow=False,+ ),+ ]++ def test_ignored_extensions(self):+ # jpg is ignored by default+ html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">"""+ response = HtmlResponse("http://example.org/", body=html)+ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(url="http://example.org/page.html", text="asd"),+ ]++ # override denied extensions+ lx = LxmlLinkExtractor(deny_extensions=["html"])+ assert lx.extract_links(response) == [+ Link(url="http://example.org/photo.jpg"),+ ]++ def test_process_value(self):+ """Test restrict_xpaths with encodings"""+ html = b"""+<a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Text</a>+<a href="/about.html">About us</a>+ """+ response = HtmlResponse(+ "http://example.org/somepage/index.html",+ body=html,+ encoding="windows-1252",+ )-class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):- extractor_cls = LxmlLinkExtractor+ def process_value(value):+ m = re.search(r"javascript:goToPage\('(.*?)'", value)+ return m.group(1) if m else None++ lx = LxmlLinkExtractor(process_value=process_value)+ assert lx.extract_links(response) == [+ Link(url="http://example.org/other/page.html", text="Text")+ ]++ def test_base_url_with_restrict_xpaths(self):+ html = b"""<html><head><title>Page title</title><base href="http://otherdomain.com/base/" /></head>+ <body><p><a href="item/12.html">Item 12</a></p>+ </body></html>"""+ response = HtmlResponse("http://example.org/somepage/index.html", body=html)+ lx = LxmlLinkExtractor(restrict_xpaths="//p")+ assert lx.extract_links(response) == [+ Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")+ ]++ def test_attrs(self):+ lx = LxmlLinkExtractor(attrs="href")+ page4_url = "http://example.com/page%204.html"++ assert lx.extract_links(self.response) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html#foo",+ text="sample 3 repetition with fragment",+ ),+ Link(url="http://www.google.com/something", text=""),+ Link(url="http://example.com/innertag.html", text="inner tag"),+ Link(url=page4_url, text="href with whitespaces"),+ ]++ lx = LxmlLinkExtractor(+ attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()+ )+ assert lx.extract_links(self.response) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample2.jpg", text=""),+ Link(url="http://example.com/sample3.html", text="sample 3 text"),+ Link(+ url="http://example.com/sample3.html#foo",+ text="sample 3 repetition with fragment",+ ),+ Link(url="http://www.google.com/something", text=""),+ Link(url="http://example.com/innertag.html", text="inner tag"),+ Link(url=page4_url, text="href with whitespaces"),+ ]++ lx = LxmlLinkExtractor(attrs=None)+ assert lx.extract_links(self.response) == []++ def test_tags(self):+ html = (+ b'<html><area href="sample1.html"></area>'+ b'<a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>'+ )+ response = HtmlResponse("http://example.com/index.html", body=html)++ lx = LxmlLinkExtractor(tags=None)+ assert lx.extract_links(response) == []++ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(url="http://example.com/sample1.html", text=""),+ Link(url="http://example.com/sample2.html", text="sample 2"),+ ]++ lx = LxmlLinkExtractor(tags="area")+ assert lx.extract_links(response) == [+ Link(url="http://example.com/sample1.html", text=""),+ ]++ lx = LxmlLinkExtractor(tags="a")+ assert lx.extract_links(response) == [+ Link(url="http://example.com/sample2.html", text="sample 2"),+ ]++ lx = LxmlLinkExtractor(+ tags=("a", "img"), attrs=("href", "src"), deny_extensions=()+ )+ assert lx.extract_links(response) == [+ Link(url="http://example.com/sample2.html", text="sample 2"),+ Link(url="http://example.com/sample2.jpg", text=""),+ ]++ def test_tags_attrs(self):+ html = b"""+ <html><body>+ <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>+ <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>+ </body></html>+ """+ response = HtmlResponse("http://example.com/index.html", body=html)++ lx = LxmlLinkExtractor(tags="div", attrs="data-url")+ assert lx.extract_links(response) == [+ Link(+ url="http://example.com/get?id=1",+ text="Item 1",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://example.com/get?id=2",+ text="Item 2",+ fragment="",+ nofollow=False,+ ),+ ]++ lx = LxmlLinkExtractor(tags=("div",), attrs=("data-url",))+ assert lx.extract_links(response) == [+ Link(+ url="http://example.com/get?id=1",+ text="Item 1",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://example.com/get?id=2",+ text="Item 2",+ fragment="",+ nofollow=False,+ ),+ ]++ def test_tags_attrs_wildcard_and_deny(self):+ html = b"""+ <html><body>+ <a href="a.html">a</a>+ <div data-url="div.html">div</div>+ <span href="span.html">span</span>+ <p data-url="p.html">p</p>+ </body></html>+ """+ response = HtmlResponse("http://example.com/index.html", body=html)++ def urls(**kwargs):+ lx = LxmlLinkExtractor(**kwargs)+ return [link.url for link in lx.extract_links(response)]++ # Default behavior is unchanged: only the listed tags and attributes.+ assert urls() == ["http://example.com/a.html"]++ # "*" as a tag matches every tag.+ assert urls(tags="*") == [+ "http://example.com/a.html",+ "http://example.com/span.html",+ ]++ # "*" as an attribute matches every attribute.+ assert urls(tags="*", attrs="*") == [+ "http://example.com/a.html",+ "http://example.com/div.html",+ "http://example.com/span.html",+ "http://example.com/p.html",+ ]++ # deny_tags excludes tags from the wildcard.+ assert urls(tags="*", attrs="*", deny_tags="div") == [+ "http://example.com/a.html",+ "http://example.com/span.html",+ "http://example.com/p.html",+ ]++ # deny_attrs excludes attributes from the wildcard.+ assert urls(tags="*", attrs="*", deny_attrs="data-url") == [+ "http://example.com/a.html",+ "http://example.com/span.html",+ ]++ # deny_tags also applies when tags are listed explicitly.+ assert urls(tags=("a", "span"), attrs="*", deny_tags="span") == [+ "http://example.com/a.html",+ ]++ # The wildcard for one parameter is independent of the other.+ assert urls(tags="a", attrs="*") == ["http://example.com/a.html"]++ def test_xhtml(self):+ xhtml = b"""+<?xml version="1.0"?>+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">+<head>+ <title>XHTML document title</title>+</head>+<body>+ <div class='links'>+ <p><a href="/about.html">About us</a></p>+ </div>+ <div>+ <p><a href="/follow.html">Follow this link</a></p>+ </div>+ <div>+ <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>+ </div>+ <div>+ <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>+ </div>+ <div>+ <p><a href="http://google.com/something" rel="external nofollow">External link not to follow</a></p>+ </div>+</body>+</html>+ """++ response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)++ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(+ url="http://example.com/about.html",+ text="About us",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://example.com/follow.html",+ text="Follow this link",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://example.com/nofollow.html",+ text="Dont follow this one",+ fragment="",+ nofollow=True,+ ),+ Link(+ url="http://example.com/nofollow2.html",+ text="Choose to follow or not",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://google.com/something",+ text="External link not to follow",+ nofollow=True,+ ),+ ]++ response = XmlResponse("http://example.com/index.xhtml", body=xhtml)++ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(+ url="http://example.com/about.html",+ text="About us",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://example.com/follow.html",+ text="Follow this link",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://example.com/nofollow.html",+ text="Dont follow this one",+ fragment="",+ nofollow=True,+ ),+ Link(+ url="http://example.com/nofollow2.html",+ text="Choose to follow or not",+ fragment="",+ nofollow=False,+ ),+ Link(+ url="http://google.com/something",+ text="External link not to follow",+ nofollow=True,+ ),+ ]++ def test_link_wrong_href(self):+ html = b"""+ <a href="http://example.org/item1.html">Item 1</a>+ <a href="http://[example.org/item2.html">Item 2</a>+ <a href="http://example.org/item3.html">Item 3</a>+ """+ response = HtmlResponse("http://example.org/index.html", body=html)+ lx = LxmlLinkExtractor()+ assert list(lx.extract_links(response)) == [+ Link(+ url="http://example.org/item1.html",+ text="Item 1",+ nofollow=False,+ ),+ Link(+ url="http://example.org/item3.html",+ text="Item 3",+ nofollow=False,+ ),+ ]++ def test_ftp_links(self):+ body = b"""+ <html><body>+ <div><a href="ftp://www.external.com/">An Item</a></div>+ </body></html>"""+ response = HtmlResponse(+ "http://www.example.com/index.html", body=body, encoding="utf8"+ )+ lx = LxmlLinkExtractor()+ assert lx.extract_links(response) == [+ Link(+ url="ftp://www.external.com/",+ text="An Item",+ fragment="",+ nofollow=False,+ ),+ ]++ def test_pickle_extractor(self):+ lx = LxmlLinkExtractor()+ assert isinstance(pickle.loads(pickle.dumps(lx)), LxmlLinkExtractor)++ def test_link_extractor_aggregation(self):+ """When a parameter like restrict_css is used, the underlying+ implementation calls its internal link extractor once per selector+ matching the specified restrictions, and then aggregates the+ extracted links.++ Test that aggregation respects the unique and canonicalize+ parameters.+ """+ # unique=True (default), canonicalize=False (default)+ lx = LxmlLinkExtractor(restrict_css=("div",))+ response = HtmlResponse(+ "https://example.com",+ body=b"""+ <div>+ <a href="/a">a1</a>+ <a href="/b?a=1&b=2">b1</a>+ </div>+ <div>+ <a href="/a">a2</a>+ <a href="/b?b=2&a=1">b2</a>+ </div>+ """,+ )+ actual = lx.extract_links(response)+ assert actual == [+ Link(url="https://example.com/a", text="a1"),+ Link(url="https://example.com/b?a=1&b=2", text="b1"),+ Link(url="https://example.com/b?b=2&a=1", text="b2"),+ ]++ # unique=True (default), canonicalize=True+ lx = LxmlLinkExtractor(restrict_css=("div",), canonicalize=True)+ response = HtmlResponse(+ "https://example.com",+ body=b"""+ <div>+ <a href="/a">a1</a>+ <a href="/b?a=1&b=2">b1</a>+ </div>+ <div>+ <a href="/a">a2</a>+ <a href="/b?b=2&a=1">b2</a>+ </div>+ """,+ )+ actual = lx.extract_links(response)+ assert actual == [+ Link(url="https://example.com/a", text="a1"),+ Link(url="https://example.com/b?a=1&b=2", text="b1"),+ ]++ # unique=False, canonicalize=False (default)+ lx = LxmlLinkExtractor(restrict_css=("div",), unique=False)+ response = HtmlResponse(+ "https://example.com",+ body=b"""+ <div>+ <a href="/a">a1</a>+ <a href="/b?a=1&b=2">b1</a>+ </div>+ <div>+ <a href="/a">a2</a>+ <a href="/b?b=2&a=1">b2</a>+ </div>+ """,+ )+ actual = lx.extract_links(response)+ assert actual == [+ Link(url="https://example.com/a", text="a1"),+ Link(url="https://example.com/b?a=1&b=2", text="b1"),+ Link(url="https://example.com/a", text="a2"),+ Link(url="https://example.com/b?b=2&a=1", text="b2"),+ ]++ # unique=False, canonicalize=True+ lx = LxmlLinkExtractor(restrict_css=("div",), unique=False, canonicalize=True)+ response = HtmlResponse(+ "https://example.com",+ body=b"""+ <div>+ <a href="/a">a1</a>+ <a href="/b?a=1&b=2">b1</a>+ </div>+ <div>+ <a href="/a">a2</a>+ <a href="/b?b=2&a=1">b2</a>+ </div>+ """,+ )+ actual = lx.extract_links(response)+ assert actual == [+ Link(url="https://example.com/a", text="a1"),+ Link(url="https://example.com/b?a=1&b=2", text="b1"),+ Link(url="https://example.com/a", text="a2"),+ Link(url="https://example.com/b?a=1&b=2", text="b2"),+ ]def test_canonicalize_once_per_link(self, monkeypatch):canonicalize_url = lxmlhtml.canonicalize_url@@ -812,7 +798,7 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):"https://example.com",body=b"".join(b'<a href="/p?b=2&a=1#f%d">x</a>' % i for i in range(10)),)- lx = self.extractor_cls(canonicalize=True)+ lx = LxmlLinkExtractor(canonicalize=True)assert lx.extract_links(response) == [Link(url="https://example.com/p?a=1&b=2", text="x")]@@ -826,7 +812,7 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):"""response = HtmlResponse("http://example.org/index.html", body=html)# Simple text inclusion test- lx = self.extractor_cls(restrict_text="dog")+ lx = LxmlLinkExtractor(restrict_text="dog")assert list(lx.extract_links(response)) == [Link(url="http://example.org/item2.html",@@ -835,7 +821,7 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):),]# Unique regex test- lx = self.extractor_cls(restrict_text=r"of.*dog")+ lx = LxmlLinkExtractor(restrict_text=r"of.*dog")assert list(lx.extract_links(response)) == [Link(url="http://example.org/item2.html",@@ -844,7 +830,7 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):),]# Multiple regex test- lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"])+ lx = LxmlLinkExtractor(restrict_text=[r"of.*dog", r"of.*cat"])assert list(lx.extract_links(response)) == [Link(url="http://example.org/item1.html",@@ -872,7 +858,7 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):<a href="http://example.org/item3.html">Good Link 2</a>"""response = HtmlResponse("http://example.org/index.html", body=html)- lx = self.extractor_cls()+ lx = LxmlLinkExtractor()assert list(lx.extract_links(response)) == [Link(url="http://example.org/item2.html",