Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a dependency on aiodns. (#8378)

Andrey Rakhmatullin699b96dcab4a
diff --git a/pyproject.toml b/pyproject.toml
index a29cffe3b..064ce6a05 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,6 +9,7 @@ description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
"PyDispatcher>=2.0.5",
"Twisted>=21.7.0",
+ "aiodns>=3.3.0",
"aioftp>=0.26.1",
"aiohttp>=3.13.3",
'backports.zstd>=1.3.0; python_version < "3.14"',
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index 76f4d83a2..f146dfd71 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -14,6 +14,7 @@ from ipaddress import IPv4Address
from typing import TYPE_CHECKING, Any, ClassVar, Literal
from urllib.parse import urlparse
+import aiodns
import pytest
from cryptography.x509 import load_der_x509_certificate
from twisted.internet.defer import DeferredList
@@ -600,6 +601,16 @@ class TestHttpBase(ABC):
# httpx goes through anyio and asyncio, which look socket.getaddrinfo
# up when resolving.
monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo)
+
+ real_aiodns_getaddrinfo = aiodns.DNSResolver.getaddrinfo
+
+ def aiodns_getaddrinfo(self, host, *args, **kwargs):
+ return real_aiodns_getaddrinfo(self, "127.0.0.1", *args, **kwargs)
+
+ # aiohttp resolves through aiodns (when it is installed), which uses
+ # c-ares instead of socket.getaddrinfo.
+ monkeypatch.setattr(aiodns.DNSResolver, "getaddrinfo", aiodns_getaddrinfo)
+
previous = None
if is_reactor_installed():
# Twisted resolves through a GAIResolver that bound
diff --git a/tox.ini b/tox.ini
index e8276a5b8..0feb4b3d5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -71,6 +71,7 @@ deps =
Pillow==12.3.0
Protego==0.7.0
Twisted==26.4.0
+ aiodns==4.0.4
aioftp==0.28.3
aiohttp==3.14.3
attrs==26.1.0
@@ -140,6 +141,7 @@ deps =
pytest==8.4.0
Protego==0.2.0
Twisted==21.7.0
+ aiodns==3.3.0
aioftp==0.26.1
aiohttp==3.13.3
backports.zstd==1.3.0
@@ -153,6 +155,8 @@ deps =
lxml==4.9.4
parsel==1.8.1
platformdirs==2.0.0
+ # aiodns < 4 needs pycares < 5
+ pycares==4.11.0
pyOpenSSL==24.3.0
queuelib==1.8.0
service_identity==24.2.0
@@ -241,6 +245,7 @@ commands_pre =
git+https://github.com/twisted/twisted \
git+https://github.com/python-pillow/Pillow \
git+https://github.com/MagicStack/uvloop \
+ git+https://github.com/aio-libs/aiodns \
git+https://github.com/aio-libs/aiohttp \
git+https://github.com/aio-libs/aioftp \
git+https://github.com/pyca/cryptography \