Scrapy, a fast high-level web crawling & scraping framework for Python.
CrawlerRunner test for Response.ip_address
Eugenio Lacuestae0ef8ad2d6f9
diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.pydeleted file mode 100644index 949e97172..000000000--- a/tests/CrawlerProcess/ip_address.py+++ /dev/null@@ -1,51 +0,0 @@-from urllib.parse import urlparse--from twisted.internet import defer-from twisted.internet.base import ThreadedResolver-from twisted.internet.interfaces import IResolverSimple-from zope.interface.declarations import implementer--from scrapy import Spider, Request-from scrapy.crawler import CrawlerProcess--from tests.mockserver import MockServer---@implementer(IResolverSimple)-class MockThreadedResolver(ThreadedResolver):- """- Resolves all names to localhost- """-- @classmethod- def from_crawler(cls, crawler, reactor):- return cls(reactor)-- def install_on_reactor(self):- self.reactor.installResolver(self)-- def getHostByName(self, name, timeout=None):- return defer.succeed("127.0.0.1")---class LocalhostSpider(Spider):- name = "localhost_spider"-- def start_requests(self):- yield Request(self.url)-- def parse(self, response):- netloc = urlparse(response.url).netloc- self.logger.info("Host: %s" % netloc.split(":")[0])- self.logger.info("Type: %s" % type(response.ip_address))- self.logger.info("IP address: %s" % response.ip_address)---with MockServer() as mockserver:- settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}- process = CrawlerProcess(settings)-- port = urlparse(mockserver.http_address).port- url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)- process.crawl(LocalhostSpider, url=url)- process.start()diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pynew file mode 100644index 000000000..5a71536d8--- /dev/null+++ b/tests/CrawlerRunner/ip_address.py@@ -0,0 +1,37 @@+from urllib.parse import urlparse++from twisted.internet import reactor+from twisted.names.client import createResolver++from scrapy import Spider, Request+from scrapy.crawler import CrawlerRunner+from scrapy.utils.log import configure_logging++from tests.mockserver import MockServer, MockDNSServer+++class LocalhostSpider(Spider):+ name = "localhost_spider"++ def start_requests(self):+ yield Request(self.url)++ def parse(self, response):+ netloc = urlparse(response.url).netloc+ self.logger.info("Host: %s" % netloc.split(":")[0])+ self.logger.info("Type: %s" % type(response.ip_address))+ self.logger.info("IP address: %s" % response.ip_address)+++with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:+ port = urlparse(mock_http_server.http_address).port+ url = "http://not.a.real.domain:{port}/echo".format(port=port)++ servers = [(mock_dns_server.host, mock_dns_server.port)]+ reactor.installResolver(createResolver(servers=servers))++ configure_logging()+ runner = CrawlerRunner()+ d = runner.crawl(LocalhostSpider, url=url)+ d.addBoth(lambda _: reactor.stop())+ reactor.run()diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 67139534e..08a81418c 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -246,8 +246,11 @@ class MockDNSResolver:class MockDNSServer():def __enter__(self):- self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'],+ self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],stdout=PIPE, env=get_testenv())+ host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")+ self.host = host+ self.port = int(port)return selfdef __exit__(self, exc_type, exc_value, traceback):@@ -292,11 +295,11 @@ if __name__ == "__main__":clients = [MockDNSResolver()]factory = DNSServerFactory(clients=clients)protocol = dns.DNSDatagramProtocol(controller=factory)- reactor.listenUDP(10053, protocol)- reactor.listenTCP(10053, factory)+ listener = reactor.listenUDP(0, protocol)def print_listening():- print("DNS server running on port 10053")+ host = listener.getHost()+ print("%s:%s" % (host.host, host.port))reactor.callWhenRunning(print_listening)reactor.run()diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex dfc1cf448..5d381c368 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):self.assertNotIn("Asyncio reactor is installed", str(log))-class CrawlerProcessSubprocess(unittest.TestCase):- script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')-+class ScriptRunnerMixin:def run_script(self, script_name):script_path = os.path.join(self.script_dir, script_name)args = (sys.executable, script_path)@@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):stdout, stderr = p.communicate()return stderr.decode('utf-8')++class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')+def test_simple(self):log = self.run_script('simple.py')self.assertIn('Spider closed (finished)', log)@@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase):"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,]))++class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')+def test_response_ip_address(self):log = self.run_script("ip_address.py")- self.assertIn("Spider closed (finished)", log)- self.assertIn("Host: not.a.real.domain", log)- self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)- self.assertIn("IP address: 127.0.0.1", log)+ self.assertIn("INFO: Spider closed (finished)", log)+ self.assertIn("INFO: Host: not.a.real.domain", log)+ self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)+ self.assertIn("INFO: IP address: 127.0.0.1", log)