Scrapy, a fast high-level web crawling & scraping framework for Python.
Make listen_tcp() more robust to concurrent binds. (#8356)
Andrey Rakhmatullind1bcaee2d5a0
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 31aba1fd8..b04130c43 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -30,22 +30,33 @@ _P = ParamSpec("_P")def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:"""Like reactor.listenTCP but tries different ports in a range."""- from twisted.internet import reactor-if len(portrange) > 2:raise ValueError(f"invalid portrange: {portrange}")if len(portrange) == 2 and portrange[0] > portrange[1]:raise ValueError(f"invalid portrange: {portrange}")if not portrange:- return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return]+ return _listen_tcp(0, host, factory)for x in range(portrange[0], portrange[-1]):try:- return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return]+ return _listen_tcp(x, host, factory)except error.CannotListenError:pass# The last port of the range is tried outside the loop so that its# CannotListenError propagates to the caller.- return reactor.listenTCP(portrange[-1], factory, interface=host) # type: ignore[no-any-return]+ return _listen_tcp(portrange[-1], host, factory)+++def _listen_tcp(port: int, host: str, factory: ServerFactory) -> Port:+ from twisted.internet import reactor++ try:+ return reactor.listenTCP(port, factory, interface=host) # type: ignore[no-any-return]+ except OSError as e:+ # Twisted raises CannotListenError for bind() errors but lets listen()+ # errors through. With SO_REUSEADDR, which Twisted sets on POSIX,+ # several processes can bind the same port, and all but the first to+ # call listen() on it get EADDRINUSE.+ raise error.CannotListenError(host, port, e) from eclass CallLaterOnce(Generic[_P, _T]):diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 578b11d57..6c07e59cc 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -1,8 +1,9 @@from __future__ import annotationsimport asyncio+import errnoimport sys-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, NoReturnimport pytestfrom twisted.internet.error import CannotListenError@@ -107,3 +108,19 @@ class TestListenTcp:with pytest.raises(CannotListenError):listen_tcp([used_number, used_number], "127.0.0.1", ServerFactory())++ def test_port_taken_on_listen(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """Another process can bind the same port and call listen() on it+ first."""+ from twisted.internet import reactor++ tried: list[int] = []++ def listen(port: int, factory: ServerFactory, interface: str) -> NoReturn:+ tried.append(port)+ raise OSError(errno.EADDRINUSE, "Address already in use")++ monkeypatch.setattr(reactor, "listenTCP", listen)+ with pytest.raises(CannotListenError):+ listen_tcp([6023, 6024], "127.0.0.1", ServerFactory())+ assert tried == [6023, 6024]