Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use a random port in benchserver. (#8230)

Andrey Rakhmatullinbb3615130f6f
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index cd4995614..308059565 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -29,20 +29,23 @@ class Command(ScrapyCommand):
return "Run quick benchmark test"
def run(self, args: list[str], opts: argparse.Namespace) -> None:
- with _BenchServer():
+ with _BenchServer() as baseurl:
assert self.crawler_process
- self.crawler_process.crawl(_BenchSpider, total=100000)
+ self.crawler_process.crawl(_BenchSpider, total=100000, baseurl=baseurl)
self.crawler_process.start()
class _BenchServer:
- def __enter__(self) -> None:
+ def __enter__(self) -> str:
pargs = [sys.executable, "-u", "-m", "scrapy.utils._benchserver"]
self.proc = subprocess.Popen( # noqa: S603
pargs, stdout=subprocess.PIPE, env=get_testenv()
)
assert self.proc.stdout
- self.proc.stdout.readline()
+ # The server listens on a random port and prints it at the end of
+ # its first line.
+ port = int(self.proc.stdout.readline().rsplit(b":", 1)[1])
+ return f"http://localhost:{port}"
def __exit__(
self,
@@ -61,7 +64,7 @@ class _BenchSpider(scrapy.Spider):
name = "follow"
total = 10000
show = 20
- baseurl = "http://localhost:8998"
+ baseurl: str
link_extractor = LinkExtractor()
async def start(self) -> AsyncIterator[Any]:
diff --git a/scrapy/utils/_benchserver.py b/scrapy/utils/_benchserver.py
index fcf86d7d8..771b6fcee 100644
--- a/scrapy/utils/_benchserver.py
+++ b/scrapy/utils/_benchserver.py
@@ -42,7 +42,7 @@ def main() -> None: # pragma: no cover
from twisted.internet import reactor
root = Root() # type: ignore[no-untyped-call]
- httpPort = reactor.listenTCP(8998, Site(root))
+ httpPort = reactor.listenTCP(0, Site(root))
def _print_listening() -> None:
httpHost = httpPort.getHost()