Scrapy, a fast high-level web crawling & scraping framework for Python.
Work around a rare aiodns port problem on Windows. (#8382)
Andrey Rakhmatullin323f075e96fb
diff --git a/scrapy/core/downloader/handlers/_aiohttp.py b/scrapy/core/downloader/handlers/_aiohttp.pyindex 1237d9b07..2804208aa 100644--- a/scrapy/core/downloader/handlers/_aiohttp.py+++ b/scrapy/core/downloader/handlers/_aiohttp.py@@ -1,7 +1,9 @@from __future__ import annotationsimport ipaddress+import socketimport ssl+import sysfrom contextlib import asynccontextmanagerfrom typing import TYPE_CHECKING, ClassVar, cast@@ -25,6 +27,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgsif TYPE_CHECKING:from collections.abc import AsyncIterator+ from aiohttp.abc import ResolveResultfrom aiohttp.client_proto import ResponseHandlerfrom aiohttp.client_reqrep import ClientRequestfrom aiohttp.tracing import Trace@@ -33,6 +36,21 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler+class _AsyncResolver(aiohttp.AsyncResolver):+ """Resolves only the host and sets the requested port on the results.++ Workaround for https://github.com/c-ares/c-ares/issues/1374.+ """++ async def resolve(+ self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET+ ) -> list[ResolveResult]:+ results = await super().resolve(host, 0, family)+ for result in results:+ result["port"] = port+ return results++class _TCPConnector(aiohttp.TCPConnector):"""Counts idle connections toward the connection limit, closing the leastrecently used one when a new connection would otherwise exceed it."""@@ -87,7 +105,11 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):def __init__(self, crawler: Crawler):super().__init__(crawler)self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)+ self._resolver: _AsyncResolver | None = (+ _AsyncResolver() if sys.platform == "win32" else None+ )connector = _TCPConnector(+ resolver=self._resolver,local_addr=self._bind_address,# hard limit on open connections, idle ones includedlimit=self._pool_size_total,@@ -204,3 +226,5 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):async def close(self) -> None:await self._session.close()+ if self._resolver is not None:+ await self._resolver.close()diff --git a/tests/test_downloader_handler_aiohttp.py b/tests/test_downloader_handler_aiohttp.pyindex 02ef5fe4f..84c0c6539 100644--- a/tests/test_downloader_handler_aiohttp.py+++ b/tests/test_downloader_handler_aiohttp.py@@ -1,9 +1,12 @@from __future__ import annotations+import sysfrom typing import TYPE_CHECKING, Any+import aiodnsimport pytest+from scrapy import Requestfrom scrapy.core.downloader.handlers._aiohttp import AiohttpDownloadHandlerfrom tests.utils.bases.download_handlers_http import (TestHttpBase,@@ -20,9 +23,11 @@ from tests.utils.bases.download_handlers_http import (TestRealWebsiteBase,TestSimpleHttpsBase,)+from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from scrapy.core.downloader.handlers import DownloadHandlerProtocol+ from tests.mockserver.http import MockServerpytestmark = pytest.mark.only_asyncio@@ -49,6 +54,28 @@ class TestHttp(AiohttpDownloadHandlerMixin, TestHttpBase):handler_bad_header_handling = "fail"handler_non_utf8_request_header_handling = "replace"+ @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only workaround")+ @coroutine_test+ async def test_port_not_resolved(+ self, mockserver: MockServer, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """The URL port is used even if resolving it as a service gives a+ different one, as it can on Windows."""+ real_getaddrinfo = aiodns.DNSResolver.getaddrinfo++ def getaddrinfo(self, host, port=None, **kwargs):+ # a services database match on Windows returns the port with its+ # bytes swapped+ if port:+ port = ((port & 0xFF) << 8) | (port >> 8)+ return real_getaddrinfo(self, host, port=port, **kwargs)++ monkeypatch.setattr(aiodns.DNSResolver, "getaddrinfo", getaddrinfo)+ request = Request(mockserver.url("/text").replace("127.0.0.1", "localhost"))+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.body == b"Works"+class TestHttps(AiohttpDownloadHandlerMixin, TestHttpsBase):handler_supports_bindaddress_meta = False