Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Work around a rare aiodns port problem on Windows. (#8382)

Andrey Rakhmatullin323f075e96fb
diff --git a/scrapy/core/downloader/handlers/_aiohttp.py b/scrapy/core/downloader/handlers/_aiohttp.py
index 1237d9b07..2804208aa 100644
--- a/scrapy/core/downloader/handlers/_aiohttp.py
+++ b/scrapy/core/downloader/handlers/_aiohttp.py
@@ -1,7 +1,9 @@
from __future__ import annotations
import ipaddress
+import socket
import ssl
+import sys
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, ClassVar, cast
@@ -25,6 +27,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs
if TYPE_CHECKING:
from collections.abc import AsyncIterator
+ from aiohttp.abc import ResolveResult
from aiohttp.client_proto import ResponseHandler
from aiohttp.client_reqrep import ClientRequest
from aiohttp.tracing import Trace
@@ -33,6 +36,21 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
+class _AsyncResolver(aiohttp.AsyncResolver):
+ """Resolves only the host and sets the requested port on the results.
+
+ Workaround for https://github.com/c-ares/c-ares/issues/1374.
+ """
+
+ async def resolve(
+ self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
+ ) -> list[ResolveResult]:
+ results = await super().resolve(host, 0, family)
+ for result in results:
+ result["port"] = port
+ return results
+
+
class _TCPConnector(aiohttp.TCPConnector):
"""Counts idle connections toward the connection limit, closing the least
recently used one when a new connection would otherwise exceed it."""
@@ -87,7 +105,11 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):
def __init__(self, crawler: Crawler):
super().__init__(crawler)
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
+ self._resolver: _AsyncResolver | None = (
+ _AsyncResolver() if sys.platform == "win32" else None
+ )
connector = _TCPConnector(
+ resolver=self._resolver,
local_addr=self._bind_address,
# hard limit on open connections, idle ones included
limit=self._pool_size_total,
@@ -204,3 +226,5 @@ class AiohttpDownloadHandler(BaseStreamingDownloadHandler[_ClientResponse]):
async def close(self) -> None:
await self._session.close()
+ if self._resolver is not None:
+ await self._resolver.close()
diff --git a/tests/test_downloader_handler_aiohttp.py b/tests/test_downloader_handler_aiohttp.py
index 02ef5fe4f..84c0c6539 100644
--- a/tests/test_downloader_handler_aiohttp.py
+++ b/tests/test_downloader_handler_aiohttp.py
@@ -1,9 +1,12 @@
from __future__ import annotations
+import sys
from typing import TYPE_CHECKING, Any
+import aiodns
import pytest
+from scrapy import Request
from scrapy.core.downloader.handlers._aiohttp import AiohttpDownloadHandler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
@@ -20,9 +23,11 @@ from tests.utils.bases.download_handlers_http import (
TestRealWebsiteBase,
TestSimpleHttpsBase,
)
+from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
+ from tests.mockserver.http import MockServer
pytestmark = pytest.mark.only_asyncio
@@ -49,6 +54,28 @@ class TestHttp(AiohttpDownloadHandlerMixin, TestHttpBase):
handler_bad_header_handling = "fail"
handler_non_utf8_request_header_handling = "replace"
+ @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only workaround")
+ @coroutine_test
+ async def test_port_not_resolved(
+ self, mockserver: MockServer, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """The URL port is used even if resolving it as a service gives a
+ different one, as it can on Windows."""
+ real_getaddrinfo = aiodns.DNSResolver.getaddrinfo
+
+ def getaddrinfo(self, host, port=None, **kwargs):
+ # a services database match on Windows returns the port with its
+ # bytes swapped
+ if port:
+ port = ((port & 0xFF) << 8) | (port >> 8)
+ return real_getaddrinfo(self, host, port=port, **kwargs)
+
+ monkeypatch.setattr(aiodns.DNSResolver, "getaddrinfo", getaddrinfo)
+ request = Request(mockserver.url("/text").replace("127.0.0.1", "localhost"))
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == b"Works"
+
class TestHttps(AiohttpDownloadHandlerMixin, TestHttpsBase):
handler_supports_bindaddress_meta = False