Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Test DNS resolution using CrawlerProcess

Eugenio Lacuestae8da7e296691
diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py
new file mode 100644
index 000000000..6b069cc90
--- /dev/null
+++ b/tests/CrawlerProcess/ip_address.py
@@ -0,0 +1,51 @@
+from urllib.parse import urlparse
+
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IResolverSimple
+from zope.interface.declarations import implementer
+
+from scrapy import Spider, Request
+from scrapy.crawler import CrawlerProcess
+
+from tests.mockserver import MockServer
+
+
+@implementer(IResolverSimple)
+class MockThreadedResolver(ThreadedResolver):
+ """
+ Resolves all names to localhost
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler, reactor):
+ return cls(reactor)
+
+ def install_on_reactor(self,):
+ self.reactor.installResolver(self)
+
+ def getHostByName(self, name, timeout=None):
+ return defer.succeed("127.0.0.1")
+
+
+class LocalhostSpider(Spider):
+ name = "localhost_spider"
+
+ def start_requests(self):
+ yield Request(self.url)
+
+ def parse(self, response):
+ netloc = urlparse(response.url).netloc
+ self.logger.info("Host: %s" % netloc.split(":")[0])
+ self.logger.info("Type: %s" % type(response.ip_address))
+ self.logger.info("IP address: %s" % response.ip_address)
+
+
+with MockServer() as mockserver:
+ settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}
+ process = CrawlerProcess(settings)
+
+ port = urlparse(mockserver.http_address).port
+ url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)
+ process.crawl(LocalhostSpider, url=url)
+ process.start()
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 6281160ae..9896058dc 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,6 +1,7 @@
import json
import logging
from ipaddress import IPv4Address
+from socket import gethostbyname
from urllib.parse import urlparse
from testfixtures import LogCapture
@@ -313,15 +314,6 @@ with multiples lines
@defer.inlineCallbacks
def test_dns_server_ip_address(self):
- from socket import gethostbyname
-
- crawler = self.runner.create_crawler(SingleRequestSpider)
- url = 'https://example.org'
- yield crawler.crawl(seed=url)
- ip_address = crawler.spider.meta['responses'][0].ip_address
- self.assertIsInstance(ip_address, IPv4Address)
- self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc))
-
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0ce0674de..dfc1cf448 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
def test_spider_custom_settings_log_level(self):
log_file = self.mktemp()
+
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
@@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
]))
+
+ def test_response_ip_address(self):
+ log = self.run_script("ip_address.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Host: not.a.real.domain", log)
+ self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)
+ self.assertIn("IP address: 127.0.0.1", log)