Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make listen_tcp() more robust to concurrent binds. (#8356)

Andrey Rakhmatullind1bcaee2d5a0
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 31aba1fd8..b04130c43 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -30,22 +30,33 @@ _P = ParamSpec("_P")
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
"""Like reactor.listenTCP but tries different ports in a range."""
- from twisted.internet import reactor
-
if len(portrange) > 2:
raise ValueError(f"invalid portrange: {portrange}")
if len(portrange) == 2 and portrange[0] > portrange[1]:
raise ValueError(f"invalid portrange: {portrange}")
if not portrange:
- return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return]
+ return _listen_tcp(0, host, factory)
for x in range(portrange[0], portrange[-1]):
try:
- return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return]
+ return _listen_tcp(x, host, factory)
except error.CannotListenError:
pass
# The last port of the range is tried outside the loop so that its
# CannotListenError propagates to the caller.
- return reactor.listenTCP(portrange[-1], factory, interface=host) # type: ignore[no-any-return]
+ return _listen_tcp(portrange[-1], host, factory)
+
+
+def _listen_tcp(port: int, host: str, factory: ServerFactory) -> Port:
+ from twisted.internet import reactor
+
+ try:
+ return reactor.listenTCP(port, factory, interface=host) # type: ignore[no-any-return]
+ except OSError as e:
+ # Twisted raises CannotListenError for bind() errors but lets listen()
+ # errors through. With SO_REUSEADDR, which Twisted sets on POSIX,
+ # several processes can bind the same port, and all but the first to
+ # call listen() on it get EADDRINUSE.
+ raise error.CannotListenError(host, port, e) from e
class CallLaterOnce(Generic[_P, _T]):
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index 578b11d57..6c07e59cc 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -1,8 +1,9 @@
from __future__ import annotations
import asyncio
+import errno
import sys
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, NoReturn
import pytest
from twisted.internet.error import CannotListenError
@@ -107,3 +108,19 @@ class TestListenTcp:
with pytest.raises(CannotListenError):
listen_tcp([used_number, used_number], "127.0.0.1", ServerFactory())
+
+ def test_port_taken_on_listen(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Another process can bind the same port and call listen() on it
+ first."""
+ from twisted.internet import reactor
+
+ tried: list[int] = []
+
+ def listen(port: int, factory: ServerFactory, interface: str) -> NoReturn:
+ tried.append(port)
+ raise OSError(errno.EADDRINUSE, "Address already in use")
+
+ monkeypatch.setattr(reactor, "listenTCP", listen)
+ with pytest.raises(CannotListenError):
+ listen_tcp([6023, 6024], "127.0.0.1", ServerFactory())
+ assert tried == [6023, 6024]