Scrapy, a fast high-level web crawling & scraping framework for Python.
Use a random port in benchserver. (#8230)
Andrey Rakhmatullinbb3615130f6f
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex cd4995614..308059565 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -29,20 +29,23 @@ class Command(ScrapyCommand):return "Run quick benchmark test"def run(self, args: list[str], opts: argparse.Namespace) -> None:- with _BenchServer():+ with _BenchServer() as baseurl:assert self.crawler_process- self.crawler_process.crawl(_BenchSpider, total=100000)+ self.crawler_process.crawl(_BenchSpider, total=100000, baseurl=baseurl)self.crawler_process.start()class _BenchServer:- def __enter__(self) -> None:+ def __enter__(self) -> str:pargs = [sys.executable, "-u", "-m", "scrapy.utils._benchserver"]self.proc = subprocess.Popen( # noqa: S603pargs, stdout=subprocess.PIPE, env=get_testenv())assert self.proc.stdout- self.proc.stdout.readline()+ # The server listens on a random port and prints it at the end of+ # its first line.+ port = int(self.proc.stdout.readline().rsplit(b":", 1)[1])+ return f"http://localhost:{port}"def __exit__(self,@@ -61,7 +64,7 @@ class _BenchSpider(scrapy.Spider):name = "follow"total = 10000show = 20- baseurl = "http://localhost:8998"+ baseurl: strlink_extractor = LinkExtractor()async def start(self) -> AsyncIterator[Any]:diff --git a/scrapy/utils/_benchserver.py b/scrapy/utils/_benchserver.pyindex fcf86d7d8..771b6fcee 100644--- a/scrapy/utils/_benchserver.py+++ b/scrapy/utils/_benchserver.py@@ -42,7 +42,7 @@ def main() -> None: # pragma: no coverfrom twisted.internet import reactorroot = Root() # type: ignore[no-untyped-call]- httpPort = reactor.listenTCP(8998, Site(root))+ httpPort = reactor.listenTCP(0, Site(root))def _print_listening() -> None:httpHost = httpPort.getHost()