Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Stop skipping the IPv6 resolver tests (#7935)

Adrian9d2dea7a8df4
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
index 55d2ef711..8181c4d17 100644
--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider):
"""
name = "caching_hostname_resolver_spider"
- start_urls = ["http://[::1]"]
+
+ async def start(self):
+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
if __name__ == "__main__":
diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py
index 4c8897f8f..7cc59594b 100644
--- a/tests/AsyncCrawlerProcess/default_name_resolver.py
+++ b/tests/AsyncCrawlerProcess/default_name_resolver.py
@@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider):
"""
name = "ipv6_spider"
- start_urls = ["http://[::1]"]
+
+ async def start(self):
+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
if __name__ == "__main__":
diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
index da9c16cb8..f6f865e3e 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider):
"""
name = "caching_hostname_resolver_spider"
- start_urls = ["http://[::1]"]
+
+ async def start(self):
+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
if __name__ == "__main__":
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
index f4c129fdf..12b894030 100644
--- a/tests/CrawlerProcess/default_name_resolver.py
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider):
"""
name = "ipv6_spider"
- start_urls = ["http://[::1]"]
+
+ async def start(self):
+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
if __name__ == "__main__":
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 733b6797d..240482586 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -10,9 +10,7 @@ from pathlib import Path
from typing import TYPE_CHECKING
import pytest
-from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
-from w3lib import __version__ as w3lib_version
from scrapy.utils.asyncio import sleep
from tests.utils import get_script_run_env
@@ -97,10 +95,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
)
assert "RuntimeError" not in log
- @pytest.mark.skipif(
- parse_version(w3lib_version) >= parse_version("2.0.0"),
- reason="w3lib 2.0.0 and later do not allow invalid domains.",
- )
def test_ipv6_default_name_resolver(self) -> None:
log = self.run_script("default_name_resolver.py")
assert "Spider closed (finished)" in log
@@ -116,6 +110,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
def test_caching_hostname_resolver_ipv6(self) -> None:
log = self.run_script("caching_hostname_resolver_ipv6.py")
assert "Spider closed (finished)" in log
+ assert "http://::1" not in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
def test_caching_hostname_resolver_finite_execution(