Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Import changes

Eugenio Lacuesta889b47185202
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e72275021..190ae1d3b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -1,11 +1,11 @@
"""Download handlers for http and https schemes"""
+import ipaddress
import logging
import re
import warnings
from contextlib import suppress
from io import BytesIO
-from ipaddress import ip_address
from time import time
from urllib.parse import urldefrag
@@ -468,7 +468,7 @@ class _ResponseReader(protocol.Protocol):
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
if self._ip_address is None:
- self._ip_address = ip_address(self.transport._producer.getPeer().host)
+ self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered data.
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3a9b00ab3..3c110e7a6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -441,13 +441,15 @@ with multiples lines
self.assertEqual(cert.getIssuer().commonName, b"localhost")
@defer.inlineCallbacks
- def test_dns_server_ip_address(self):
+ def test_dns_server_ip_address_none(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
self.assertIsNone(ip_address)
+ @defer.inlineCallbacks
+ def test_dns_server_ip_address(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/echo?body=test')
expected_netloc, _ = urlparse(url).netloc.split(':')