Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop skipping the IPv6 resolver tests (#7935)
Adrian9d2dea7a8df4
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.pyindex 55d2ef711..8181c4d17 100644--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py@@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider):"""name = "caching_hostname_resolver_spider"- start_urls = ["http://[::1]"]++ async def start(self):+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})if __name__ == "__main__":diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.pyindex 4c8897f8f..7cc59594b 100644--- a/tests/AsyncCrawlerProcess/default_name_resolver.py+++ b/tests/AsyncCrawlerProcess/default_name_resolver.py@@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider):"""name = "ipv6_spider"- start_urls = ["http://[::1]"]++ async def start(self):+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})if __name__ == "__main__":diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.pyindex da9c16cb8..f6f865e3e 100644--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py@@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider):"""name = "caching_hostname_resolver_spider"- start_urls = ["http://[::1]"]++ async def start(self):+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})if __name__ == "__main__":diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.pyindex f4c129fdf..12b894030 100644--- a/tests/CrawlerProcess/default_name_resolver.py+++ b/tests/CrawlerProcess/default_name_resolver.py@@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider):"""name = "ipv6_spider"- start_urls = ["http://[::1]"]++ async def start(self):+ # w3lib older than 2.4.1 strips the brackets, making the URL invalid.+ yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})if __name__ == "__main__":diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 733b6797d..240482586 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -10,9 +10,7 @@ from pathlib import Pathfrom typing import TYPE_CHECKINGimport pytest-from packaging.version import parse as parse_versionfrom pexpect.popen_spawn import PopenSpawn-from w3lib import __version__ as w3lib_versionfrom scrapy.utils.asyncio import sleepfrom tests.utils import get_script_run_env@@ -97,10 +95,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):)assert "RuntimeError" not in log- @pytest.mark.skipif(- parse_version(w3lib_version) >= parse_version("2.0.0"),- reason="w3lib 2.0.0 and later do not allow invalid domains.",- )def test_ipv6_default_name_resolver(self) -> None:log = self.run_script("default_name_resolver.py")assert "Spider closed (finished)" in log@@ -116,6 +110,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):def test_caching_hostname_resolver_ipv6(self) -> None:log = self.run_script("caching_hostname_resolver_ipv6.py")assert "Spider closed (finished)" in log+ assert "http://::1" not in logassert "scrapy.exceptions.CannotResolveHostError" not in logdef test_caching_hostname_resolver_finite_execution(