Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[test] Name resolvers

Eugenio Lacuesta90e3bd871570
diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py
new file mode 100644
index 000000000..2c466da04
--- /dev/null
+++ b/tests/CrawlerProcess/alternative_name_resolver.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+ name = "ipv6_spider"
+ start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={
+ "RETRY_ENABLED": False,
+ "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+})
+process.crawl(IPv6Spider)
+process.start()
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
new file mode 100644
index 000000000..60d91b68b
--- /dev/null
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -0,0 +1,12 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+ name = "ipv6_spider"
+ start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={"RETRY_ENABLED": False})
+process.crawl(IPv6Spider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index fce60ca37..d85f2bc41 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -305,3 +305,15 @@ class CrawlerProcessSubprocess(unittest.TestCase):
log = self.run_script('asyncio_enabled_reactor.py')
self.assertIn('Spider closed (finished)', log)
self.assertIn("DEBUG: Asyncio reactor is installed", log)
+
+ def test_default_name_resolver(self):
+ log = self.run_script('default_name_resolver.py')
+ self.assertIn('Spider closed (finished)', log)
+ self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
+ self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+
+ def test_alternative_name_resolver(self):
+ log = self.run_script('alternative_name_resolver.py')
+ self.assertIn('Spider closed (finished)', log)
+ self.assertIn("twisted.internet.error.ConnectionRefusedError: Connection was refused by other side: 111: Connection refused.", log)
+ self.assertIn("'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1,", log)