Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CrawlerRunner test for Response.ip_address

Eugenio Lacuestae0ef8ad2d6f9
diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py
deleted file mode 100644
index 949e97172..000000000
--- a/tests/CrawlerProcess/ip_address.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from urllib.parse import urlparse
-
-from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IResolverSimple
-from zope.interface.declarations import implementer
-
-from scrapy import Spider, Request
-from scrapy.crawler import CrawlerProcess
-
-from tests.mockserver import MockServer
-
-
-@implementer(IResolverSimple)
-class MockThreadedResolver(ThreadedResolver):
- """
- Resolves all names to localhost
- """
-
- @classmethod
- def from_crawler(cls, crawler, reactor):
- return cls(reactor)
-
- def install_on_reactor(self):
- self.reactor.installResolver(self)
-
- def getHostByName(self, name, timeout=None):
- return defer.succeed("127.0.0.1")
-
-
-class LocalhostSpider(Spider):
- name = "localhost_spider"
-
- def start_requests(self):
- yield Request(self.url)
-
- def parse(self, response):
- netloc = urlparse(response.url).netloc
- self.logger.info("Host: %s" % netloc.split(":")[0])
- self.logger.info("Type: %s" % type(response.ip_address))
- self.logger.info("IP address: %s" % response.ip_address)
-
-
-with MockServer() as mockserver:
- settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}
- process = CrawlerProcess(settings)
-
- port = urlparse(mockserver.http_address).port
- url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)
- process.crawl(LocalhostSpider, url=url)
- process.start()
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
new file mode 100644
index 000000000..5a71536d8
--- /dev/null
+++ b/tests/CrawlerRunner/ip_address.py
@@ -0,0 +1,37 @@
+from urllib.parse import urlparse
+
+from twisted.internet import reactor
+from twisted.names.client import createResolver
+
+from scrapy import Spider, Request
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.log import configure_logging
+
+from tests.mockserver import MockServer, MockDNSServer
+
+
+class LocalhostSpider(Spider):
+ name = "localhost_spider"
+
+ def start_requests(self):
+ yield Request(self.url)
+
+ def parse(self, response):
+ netloc = urlparse(response.url).netloc
+ self.logger.info("Host: %s" % netloc.split(":")[0])
+ self.logger.info("Type: %s" % type(response.ip_address))
+ self.logger.info("IP address: %s" % response.ip_address)
+
+
+with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
+ port = urlparse(mock_http_server.http_address).port
+ url = "http://not.a.real.domain:{port}/echo".format(port=port)
+
+ servers = [(mock_dns_server.host, mock_dns_server.port)]
+ reactor.installResolver(createResolver(servers=servers))
+
+ configure_logging()
+ runner = CrawlerRunner()
+ d = runner.crawl(LocalhostSpider, url=url)
+ d.addBoth(lambda _: reactor.stop())
+ reactor.run()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 67139534e..08a81418c 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -246,8 +246,11 @@ class MockDNSResolver:
class MockDNSServer():
def __enter__(self):
- self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'],
+ self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
stdout=PIPE, env=get_testenv())
+ host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
+ self.host = host
+ self.port = int(port)
return self
def __exit__(self, exc_type, exc_value, traceback):
@@ -292,11 +295,11 @@ if __name__ == "__main__":
clients = [MockDNSResolver()]
factory = DNSServerFactory(clients=clients)
protocol = dns.DNSDatagramProtocol(controller=factory)
- reactor.listenUDP(10053, protocol)
- reactor.listenTCP(10053, factory)
+ listener = reactor.listenUDP(0, protocol)
def print_listening():
- print("DNS server running on port 10053")
+ host = listener.getHost()
+ print("%s:%s" % (host.host, host.port))
reactor.callWhenRunning(print_listening)
reactor.run()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index dfc1cf448..5d381c368 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertNotIn("Asyncio reactor is installed", str(log))
-class CrawlerProcessSubprocess(unittest.TestCase):
- script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
-
+class ScriptRunnerMixin:
def run_script(self, script_name):
script_path = os.path.join(self.script_dir, script_name)
args = (sys.executable, script_path)
@@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
stdout, stderr = p.communicate()
return stderr.decode('utf-8')
+
+class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
def test_simple(self):
log = self.run_script('simple.py')
self.assertIn('Spider closed (finished)', log)
@@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase):
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
]))
+
+class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
+
def test_response_ip_address(self):
log = self.run_script("ip_address.py")
- self.assertIn("Spider closed (finished)", log)
- self.assertIn("Host: not.a.real.domain", log)
- self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)
- self.assertIn("IP address: 127.0.0.1", log)
+ self.assertIn("INFO: Spider closed (finished)", log)
+ self.assertIn("INFO: Host: not.a.real.domain", log)
+ self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
+ self.assertIn("INFO: IP address: 127.0.0.1", log)