Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove Python 2-only tests.
Andrey Rakhmatulline18014d84d05
diff --git a/conftest.py b/conftest.pyindex d5d61ddd3..74fb101e9 100644--- a/conftest.py+++ b/conftest.py@@ -1,4 +1,3 @@-import siximport pytest@@ -8,11 +7,10 @@ collect_ignore = []-if six.PY3:- for line in open('tests/py3-ignores.txt'):- file_path = line.strip()- if file_path and file_path[0] != '#':- collect_ignore.append(file_path)+for line in open('tests/py3-ignores.txt'):+ file_path = line.strip()+ if file_path and file_path[0] != '#':+ collect_ignore.append(file_path)@pytest.fixture()diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txtindex 313e74ec9..45cf6fb92 100644--- a/tests/py3-ignores.txt+++ b/tests/py3-ignores.txt@@ -1,6 +1,3 @@-tests/test_linkextractors_deprecated.py-tests/test_proxy_connect.py-scrapy/linkextractors/sgml.pyscrapy/linkextractors/regex.pyscrapy/linkextractors/htmlparser.pydiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 59d4a3eec..b06fcf6c3 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -630,18 +630,16 @@ class Http11MockServerTestCase(unittest.TestCase):# download_maxsize < 100, hence the CancelledErrorself.assertIsInstance(failure.value, defer.CancelledError)- if six.PY2:- request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')- request = request.replace(url=self.mockserver.url('/xpayload'))- yield crawler.crawl(seed=request)- # download_maxsize = 50 is enough for the gzipped response- failure = crawler.spider.meta.get('failure')- self.assertTrue(failure is None)- reason = crawler.spider.meta['close_reason']- self.assertTrue(reason, 'finished')- else:- # See issue https://twistedmatrix.com/trac/ticket/8175- raise unittest.SkipTest("xpayload only enabled for PY2")+ # See issue https://twistedmatrix.com/trac/ticket/8175+ raise unittest.SkipTest("xpayload fails on PY3")+ request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')+ request = request.replace(url=self.mockserver.url('/xpayload'))+ yield crawler.crawl(seed=request)+ # download_maxsize = 50 is enough for the gzipped response+ failure = crawler.spider.meta.get('failure')+ self.assertTrue(failure is None)+ reason = crawler.spider.meta['close_reason']+ self.assertTrue(reason, 'finished')class UriResource(resource.Resource):diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.pydeleted file mode 100644index 1366971be..000000000--- a/tests/test_linkextractors_deprecated.py+++ /dev/null@@ -1,233 +0,0 @@-# -*- coding: utf-8 -*--import unittest-from scrapy.linkextractors.regex import RegexLinkExtractor-from scrapy.http import HtmlResponse-from scrapy.link import Link-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor-from tests import get_testdata--from tests.test_linkextractors import Base---class BaseSgmlLinkExtractorTestCase(unittest.TestCase):- # XXX: should we move some of these tests to base link extractor tests?-- def test_basic(self):- html = """<html><head><title>Page title<title>- <body><p><a href="item/12.html">Item 12</a></p>- <p><a href="/about.html">About us</a></p>- <img src="/logo.png" alt="Company logo (not a link)" />- <p><a href="../othercat.html">Other category</a></p>- <p><a href="/">>></a></p>- <p><a href="/" /></p>- </body></html>"""- response = HtmlResponse("http://example.org/somepage/index.html", body=html)-- lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href- self.assertEqual(lx.extract_links(response),- [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),- Link(url='http://example.org/about.html', text='About us'),- Link(url='http://example.org/othercat.html', text='Other category'),- Link(url='http://example.org/', text='>>'),- Link(url='http://example.org/', text='')])-- def test_base_url(self):- html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />- <body><p><a href="item/12.html">Item 12</a></p>- </body></html>"""- response = HtmlResponse("http://example.org/somepage/index.html", body=html)-- lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href- self.assertEqual(lx.extract_links(response),- [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])-- # base url is an absolute path and relative to host- html = """<html><head><title>Page title<title><base href="/" />- <body><p><a href="item/12.html">Item 12</a></p></body></html>"""- response = HtmlResponse("https://example.org/somepage/index.html", body=html)- self.assertEqual(lx.extract_links(response),- [Link(url='https://example.org/item/12.html', text='Item 12')])-- # base url has no scheme- html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />- <body><p><a href="item/12.html">Item 12</a></p></body></html>"""- response = HtmlResponse("https://example.org/somepage/index.html", body=html)- self.assertEqual(lx.extract_links(response),- [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])-- def test_link_text_wrong_encoding(self):- html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""- response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')- lx = BaseSgmlLinkExtractor()- self.assertEqual(lx.extract_links(response), [- Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),- ])-- def test_extraction_encoding(self):- body = get_testdata('link_extractor', 'linkextractor_noenc.html')- response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})- response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)- body = get_testdata('link_extractor', 'linkextractor_latin1.html')- response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)-- lx = BaseSgmlLinkExtractor()- self.assertEqual(lx.extract_links(response_utf8), [- Link(url='http://example.com/sample_%C3%B1.html', text=''),- Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),- ])-- self.assertEqual(lx.extract_links(response_noenc), [- Link(url='http://example.com/sample_%C3%B1.html', text=''),- Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),- ])-- # document encoding does not affect URL path component, only query part- # >>> u'sample_ñ.html'.encode('utf8')- # b'sample_\xc3\xb1.html'- # >>> u"sample_á.html".encode('utf8')- # b'sample_\xc3\xa1.html'- # >>> u"sample_ö.html".encode('utf8')- # b'sample_\xc3\xb6.html'- # >>> u"£32".encode('latin1')- # b'\xa332'- # >>> u"µ".encode('latin1')- # b'\xb5'- self.assertEqual(lx.extract_links(response_latin1), [- Link(url='http://example.com/sample_%C3%B1.html', text=''),- Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),- Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),- ])-- def test_matches(self):- url1 = 'http://lotsofstuff.com/stuff1/index'- url2 = 'http://evenmorestuff.com/uglystuff/index'-- lx = BaseSgmlLinkExtractor()- self.assertEqual(lx.matches(url1), True)- self.assertEqual(lx.matches(url2), True)---class HtmlParserLinkExtractorTestCase(unittest.TestCase):-- def setUp(self):- body = get_testdata('link_extractor', 'sgml_linkextractor.html')- self.response = HtmlResponse(url='http://example.com/index', body=body)-- def test_extraction(self):- # Default arguments- lx = HtmlParserLinkExtractor()- self.assertEqual(lx.extract_links(self.response), [- Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),- Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),- Link(url='http://www.google.com/something', text=u''),- Link(url='http://example.com/innertag.html', text=u'inner tag'),- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),- ])-- def test_link_wrong_href(self):- html = """- <a href="http://example.org/item1.html">Item 1</a>- <a href="http://[example.org/item2.html">Item 2</a>- <a href="http://example.org/item3.html">Item 3</a>- """- response = HtmlResponse("http://example.org/index.html", body=html)- lx = HtmlParserLinkExtractor()- self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),- ])---class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):- extractor_cls = SgmlLinkExtractor- escapes_whitespace = True-- def test_deny_extensions(self):- html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""- response = HtmlResponse("http://example.org/", body=html)- lx = SgmlLinkExtractor(deny_extensions="jpg")- self.assertEqual(lx.extract_links(response), [- Link(url='http://example.org/page.html', text=u'asd'),- ])-- def test_attrs_sgml(self):- html = """<html><area href="sample1.html"></area>- <a ref="sample2.html">sample text 2</a></html>"""- response = HtmlResponse("http://example.com/index.html", body=html)- lx = SgmlLinkExtractor(attrs="href")- self.assertEqual(lx.extract_links(response), [- Link(url='http://example.com/sample1.html', text=u''),- ])-- def test_link_nofollow(self):- html = """- <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>- <a href="about.html">About us</a>- <a href="http://google.com/something" rel="external nofollow">Something</a>- """- response = HtmlResponse("http://example.org/page.html", body=html)- lx = SgmlLinkExtractor()- self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),- Link(url='http://example.org/about.html', text=u'About us', nofollow=False),- Link(url='http://google.com/something', text=u'Something', nofollow=True),- ])---class RegexLinkExtractorTestCase(unittest.TestCase):- # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten- # not to depend on SgmlLinkExractor. Its speed is also much worse- # than it should be.-- def setUp(self):- body = get_testdata('link_extractor', 'sgml_linkextractor.html')- self.response = HtmlResponse(url='http://example.com/index', body=body)-- def test_extraction(self):- # Default arguments- lx = RegexLinkExtractor()- self.assertEqual(lx.extract_links(self.response),- [Link(url='http://example.com/sample2.html', text=u'sample 2'),- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),- Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),- Link(url='http://www.google.com/something', text=u''),- Link(url='http://example.com/innertag.html', text=u'inner tag'),])-- def test_link_wrong_href(self):- html = """- <a href="http://example.org/item1.html">Item 1</a>- <a href="http://[example.org/item2.html">Item 2</a>- <a href="http://example.org/item3.html">Item 3</a>- """- response = HtmlResponse("http://example.org/index.html", body=html)- lx = RegexLinkExtractor()- self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),- ])-- def test_html_base_href(self):- html = """- <html>- <head>- <base href="http://b.com/">- </head>- <body>- <a href="test.html"></a>- </body>- </html>- """- response = HtmlResponse("http://a.com/", body=html)- lx = RegexLinkExtractor()- self.assertEqual([link for link in lx.extract_links(response)], [- Link(url='http://b.com/test.html', text=u'', nofollow=False),- ])-- @unittest.expectedFailure- def test_extraction(self):- # RegexLinkExtractor doesn't parse URLs with leading/trailing- # whitespaces correctly.- super(RegexLinkExtractorTestCase, self).test_extraction()diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pydeleted file mode 100644index ae1236bcb..000000000--- a/tests/test_proxy_connect.py+++ /dev/null@@ -1,120 +0,0 @@-import json-import os-import time--from six.moves.urllib.parse import urlsplit, urlunsplit-from threading import Thread-from libmproxy import controller, proxy-from netlib import http_auth-from testfixtures import LogCapture--from twisted.internet import defer-from twisted.trial.unittest import TestCase-from scrapy.utils.test import get_crawler-from scrapy.http import Request-from tests.spiders import SimpleSpider, SingleRequestSpider-from tests.mockserver import MockServer---class HTTPSProxy(controller.Master, Thread):-- def __init__(self):- password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')- authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")- cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),- 'keys', 'mitmproxy-ca.pem')- server = proxy.ProxyServer(proxy.ProxyConfig(- authenticator = authenticator,- cacert = cert_path),- 0)- self.server = server- Thread.__init__(self)- controller.Master.__init__(self, server)-- def http_address(self):- return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()---def _wrong_credentials(proxy_url):- bad_auth_proxy = list(urlsplit(proxy_url))- bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')- return urlunsplit(bad_auth_proxy)--class ProxyConnectTestCase(TestCase):-- def setUp(self):- self.mockserver = MockServer()- self.mockserver.__enter__()- self._oldenv = os.environ.copy()-- self._proxy = HTTPSProxy()- self._proxy.start()-- # Wait for the proxy to start.- time.sleep(1.0)- os.environ['https_proxy'] = self._proxy.http_address()- os.environ['http_proxy'] = self._proxy.http_address()-- def tearDown(self):- self.mockserver.__exit__(None, None, None)- self._proxy.shutdown()- os.environ = self._oldenv-- @defer.inlineCallbacks- def test_https_connect_tunnel(self):- crawler = get_crawler(SimpleSpider)- with LogCapture() as l:- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(200, l)-- @defer.inlineCallbacks- def test_https_noconnect(self):- proxy = os.environ['https_proxy']- os.environ['https_proxy'] = proxy + '?noconnect'- crawler = get_crawler(SimpleSpider)- with LogCapture() as l:- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(200, l)-- @defer.inlineCallbacks- def test_https_connect_tunnel_error(self):- crawler = get_crawler(SimpleSpider)- with LogCapture() as l:- yield crawler.crawl("https://localhost:99999/status?n=200")- self._assert_got_tunnel_error(l)-- @defer.inlineCallbacks- def test_https_tunnel_auth_error(self):- os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])- crawler = get_crawler(SimpleSpider)- with LogCapture() as l:- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- # The proxy returns a 407 error code but it does not reach the client;- # he just sees a TunnelError.- self._assert_got_tunnel_error(l)-- @defer.inlineCallbacks- def test_https_tunnel_without_leak_proxy_authorization_header(self):- request = Request(self.mockserver.url("/echo", is_secure=True))- crawler = get_crawler(SingleRequestSpider)- with LogCapture() as l:- yield crawler.crawl(seed=request)- self._assert_got_response_code(200, l)- echo = json.loads(crawler.spider.meta['responses'][0].body)- self.assertTrue('Proxy-Authorization' not in echo['headers'])-- @defer.inlineCallbacks- def test_https_noconnect_auth_error(self):- os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'- crawler = get_crawler(SimpleSpider)- with LogCapture() as l:- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(407, l)-- def _assert_got_response_code(self, code, log):- print(log)- self.assertEqual(str(log).count('Crawled (%d)' % code), 1)-- def _assert_got_tunnel_error(self, log):- print(log)- self.assertIn('TunnelError', str(log))diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex a94398796..6857356f6 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase):gc.collect()self.assertFalse(len(wk._weakdict))- @unittest.skipUnless(six.PY2, "deprecated function")- def test_stringify_dict(self):- d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}- d2 = stringify_dict(d, keys_only=False)- self.assertEqual(d, d2)- self.assertIsNot(d, d2) # shouldn't modify in place- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))-- @unittest.skipUnless(six.PY2, "deprecated function")- def test_stringify_dict_tuples(self):- tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]- d = dict(tuples)- d2 = stringify_dict(tuples, keys_only=False)- self.assertEqual(d, d2)- self.assertIsNot(d, d2) # shouldn't modify in place- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))-- @unittest.skipUnless(six.PY2, "deprecated function")- def test_stringify_dict_keys_only(self):- d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}- d2 = stringify_dict(d)- self.assertEqual(d, d2)- self.assertIsNot(d, d2) # shouldn't modify in place- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))-def test_get_func_args(self):def f1(a, b, c):passdiff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex a81946490..7b015ff8d 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):to_bytes(x) if not isinstance(x, int) else x for x in test)self.assertEqual(client._parse(url), test, url)- def test_externalUnicodeInterference(self):- """- L{client._parse} should return C{str} for the scheme, host, and path- elements of its return tuple, even when passed an URL which has- previously been passed to L{urlparse} as a C{unicode} string.- """- if not six.PY2:- raise unittest.SkipTest(- "Applies only to Py2, as urls can be ONLY unicode on Py3")- badInput = u'http://example.com/path'- goodInput = badInput.encode('ascii')- self._parse(badInput) # cache badInput in urlparse_cached- scheme, netloc, host, port, path = self._parse(goodInput)- self.assertTrue(isinstance(scheme, str))- self.assertTrue(isinstance(netloc, str))- self.assertTrue(isinstance(host, str))- self.assertTrue(isinstance(path, str))- self.assertTrue(isinstance(port, int))--class ScrapyHTTPPageGetterTests(unittest.TestCase):