Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use a proper way to test a non-listening port. (#8389)

Andrey Rakhmatullin2d7a8675d594
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index e4d864892..07ce38147 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -27,6 +27,7 @@ from scrapy.utils.engine import (
)
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
+from tests.mockserver.utils import _free_port
from tests.spiders import (
AsyncAwaitRequestInItemPipelineSpider,
AsyncDefAsyncioGenComplexSpider,
@@ -205,7 +206,7 @@ class TestCrawl:
crawler = get_crawler(SimpleSpider)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
- "http://localhost:65432/status?n=503", mockserver=mockserver
+ f"http://127.0.0.1:{_free_port()}/status?n=503", mockserver=mockserver
)
self._assert_retried(caplog.text)
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index 00598e642..c545fa584 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -52,6 +52,7 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
+from tests.mockserver.utils import _free_port
from tests.utils import ipv6_loopback_available
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
@@ -616,7 +617,7 @@ class TestTunnelingErrors:
@coroutine_test
async def test_proxy_connection_refused(self) -> None:
request = Request(
- "https://example.com", meta={"proxy": "http://127.0.0.1:65432"}
+ "https://example.com", meta={"proxy": f"http://127.0.0.1:{_free_port()}"}
)
async with _get_dh() as dh:
with pytest.raises(DownloadConnectionRefusedError):
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index 9d1a3dbbc..45b263398 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -55,6 +55,7 @@ from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.proxy_stalling import StallingProxy
from tests.mockserver.raw_http import RawHttpServer
from tests.mockserver.simple_https import SimpleMockServer
+from tests.mockserver.utils import _free_port
from tests.spiders import (
BytesReceivedCallbackSpider,
BytesReceivedErrbackSpider,
@@ -1193,7 +1194,7 @@ class TestHttpBase(ABC):
async def test_download_conn_failed(self) -> None:
# copy of TestCrawl.test_retry_conn_failed()
scheme = "https" if self.is_secure else "http"
- request = Request(f"{scheme}://localhost:65432/")
+ request = Request(f"{scheme}://127.0.0.1:{_free_port()}/")
async with self.get_dh() as download_handler:
with pytest.raises(DownloadConnectionRefusedError):
await download_handler.download_request(request)