Scrapy, a fast high-level web crawling & scraping framework for Python.
TST fixed compatibility with new link extractor whitespace handling
Mikhail Korobov2b4d46315f2c
diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 924bb7979..779f5a71c 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest):'http://example.com/sample2.html','http://example.com/sample3.html','http://example.com/sample3.html',+ 'http://example.com/sample3.html#foo','http://www.google.com/something','http://example.com/innertag.html']diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 50484f060..1d7c4f311 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -13,6 +13,7 @@ from tests import get_testdataclass Base:class LinkExtractorTestCase(unittest.TestCase):extractor_cls = None+ escapes_whitespace = Falsedef setUp(self):body = get_testdata('link_extractor', 'sgml_linkextractor.html')@@ -26,6 +27,11 @@ class Base:def test_extract_all_links(self):lx = self.extractor_cls()+ if self.escapes_whitespace:+ page4_url = 'http://example.com/page%204.html'+ else:+ page4_url = 'http://example.com/page 4.html'+self.assertEqual([link for link in lx.extract_links(self.response)], [Link(url='http://example.com/sample1.html', text=u''),Link(url='http://example.com/sample2.html', text=u'sample 2'),@@ -33,7 +39,7 @@ class Base:Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),Link(url='http://www.google.com/something', text=u''),Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),+ Link(url=page4_url, text=u'href with whitespaces'),])def test_extract_filter_allow(self):@@ -301,6 +307,11 @@ class Base:def test_attrs(self):lx = self.extractor_cls(attrs="href")+ if self.escapes_whitespace:+ page4_url = 'http://example.com/page%204.html'+ else:+ page4_url = 'http://example.com/page 4.html'+self.assertEqual(lx.extract_links(self.response), [Link(url='http://example.com/sample1.html', text=u''),Link(url='http://example.com/sample2.html', text=u'sample 2'),@@ -308,7 +319,7 @@ class Base:Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),Link(url='http://www.google.com/something', text=u''),Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),+ Link(url=page4_url, text=u'href with whitespaces'),])lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())@@ -320,7 +331,7 @@ class Base:Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),Link(url='http://www.google.com/something', text=u''),Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),+ Link(url=page4_url, text=u'href with whitespaces'),])lx = self.extractor_cls(attrs=None)diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.pyindex 794f85e0f..1366971be 100644--- a/tests/test_linkextractors_deprecated.py+++ b/tests/test_linkextractors_deprecated.py@@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):extractor_cls = SgmlLinkExtractor+ escapes_whitespace = Truedef test_deny_extensions(self):html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""