Scrapy, a fast high-level web crawling & scraping framework for Python.
Test DNS resolution using CrawlerProcess
Eugenio Lacuestae8da7e296691
diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.pynew file mode 100644index 000000000..6b069cc90--- /dev/null+++ b/tests/CrawlerProcess/ip_address.py@@ -0,0 +1,51 @@+from urllib.parse import urlparse++from twisted.internet import defer+from twisted.internet.base import ThreadedResolver+from twisted.internet.interfaces import IResolverSimple+from zope.interface.declarations import implementer++from scrapy import Spider, Request+from scrapy.crawler import CrawlerProcess++from tests.mockserver import MockServer+++@implementer(IResolverSimple)+class MockThreadedResolver(ThreadedResolver):+ """+ Resolves all names to localhost+ """++ @classmethod+ def from_crawler(cls, crawler, reactor):+ return cls(reactor)++ def install_on_reactor(self,):+ self.reactor.installResolver(self)++ def getHostByName(self, name, timeout=None):+ return defer.succeed("127.0.0.1")+++class LocalhostSpider(Spider):+ name = "localhost_spider"++ def start_requests(self):+ yield Request(self.url)++ def parse(self, response):+ netloc = urlparse(response.url).netloc+ self.logger.info("Host: %s" % netloc.split(":")[0])+ self.logger.info("Type: %s" % type(response.ip_address))+ self.logger.info("IP address: %s" % response.ip_address)+++with MockServer() as mockserver:+ settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}+ process = CrawlerProcess(settings)++ port = urlparse(mockserver.http_address).port+ url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)+ process.crawl(LocalhostSpider, url=url)+ process.start()diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 6281160ae..9896058dc 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,6 +1,7 @@import jsonimport loggingfrom ipaddress import IPv4Address+from socket import gethostbynamefrom urllib.parse import urlparsefrom testfixtures import LogCapture@@ -313,15 +314,6 @@ with multiples lines@defer.inlineCallbacksdef test_dns_server_ip_address(self):- from socket import gethostbyname-- crawler = self.runner.create_crawler(SingleRequestSpider)- url = 'https://example.org'- yield crawler.crawl(seed=url)- ip_address = crawler.spider.meta['responses'][0].ip_address- self.assertIsInstance(ip_address, IPv4Address)- self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc))-crawler = self.runner.create_crawler(SingleRequestSpider)url = self.mockserver.url('/status?n=200')yield crawler.crawl(seed=url, mockserver=self.mockserver)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 0ce0674de..dfc1cf448 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):def test_spider_custom_settings_log_level(self):log_file = self.mktemp()+class MySpider(scrapy.Spider):name = 'spider'custom_settings = {@@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,]))++ def test_response_ip_address(self):+ log = self.run_script("ip_address.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Host: not.a.real.domain", log)+ self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)+ self.assertIn("IP address: 127.0.0.1", log)