Scrapy, a fast high-level web crawling & scraping framework for Python.
fix: failed tests
Laerte Pereiraf4bcc3e67de0
diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 4e9ef7740..4e599d69c 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -23,6 +23,8 @@ from scrapy.utils.test import get_crawlerfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.extensions import telnetfrom scrapy.utils.test import get_testenv+from pkg_resources import parse_version+from w3lib import __version__ as w3lib_versionfrom tests.mockserver import MockServer@@ -381,10 +383,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):def test_ipv6_default_name_resolver(self):log = self.run_script('default_name_resolver.py')self.assertIn('Spider closed (finished)', log)- self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)- self.assertIn(- "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",- log)+ if parse_version(w3lib_version) < parse_version("2.0.0"):+ self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)+ self.assertIn(+ "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",+ log)+ else:+ self.assertIn("ValueError: invalid hostname:", log)def test_caching_hostname_resolver_ipv6(self):log = self.run_script("caching_hostname_resolver_ipv6.py")diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 689349999..7b36304b1 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -22,6 +22,8 @@ from scrapy.spiders import (from scrapy.linkextractors import LinkExtractorfrom scrapy.utils.test import get_crawlerfrom tests import get_testdata+from pkg_resources import parse_version+from w3lib import __version__ as w3lib_versionclass SpiderTest(unittest.TestCase):@@ -360,10 +362,12 @@ class CrawlSpiderTest(SpiderTest):output = list(spider._requests_to_follow(response))self.assertEqual(len(output), 3)self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))- self.assertEqual([r.url for r in output],- ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',- 'http://EXAMPLE.ORG/ABOUT.HTML',- 'http://EXAMPLE.ORG/NOFOLLOW.HTML'])+ urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',+ 'http://EXAMPLE.ORG/ABOUT.HTML',+ 'http://EXAMPLE.ORG/NOFOLLOW.HTML']+ if parse_version(w3lib_version) >= parse_version('2.0.0'):+ urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls))+ self.assertEqual([r.url for r in output], urls)def test_process_request_instance_method_with_response(self):