Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Retry a browser driver once before failing the leg (#11355)

Daniel Han3481bb39311e
diff --git a/.github/workflows/studio-composer-compatibility.yml b/.github/workflows/studio-composer-compatibility.yml
index 65b7b13ffd0..bfbbeba5256 100644
--- a/.github/workflows/studio-composer-compatibility.yml
+++ b/.github/workflows/studio-composer-compatibility.yml
@@ -126,6 +126,33 @@ jobs:
run: |
mkdir -p temp/queue-validation/compatibility
status=0
+ # One retry per driver, per browser, and one only. Three drivers across five
+ # engines on three operating systems is enough surface that a hosted runner
+ # drops a dynamically imported module or closes a browser target often enough
+ # to fail somebody else's pull request: seen as "error loading dynamically
+ # imported module .../use-prompt-queue-reorder.ts" on firefox and
+ # "TargetClosedError: Target page, context or browser has been closed" on
+ # webkit, in both cases with every assertion in the driver already reported
+ # PASS. A real regression fails both attempts, so the gate is kept and a
+ # transient costs one rerun of one driver rather than a red leg.
+ #
+ # The command is passed through rather than named inside, so the workflow
+ # still spells `python tests/studio/<driver>.py` where
+ # tests/studio/test_playwright_suites_run_in_ci.py looks for it. Both
+ # attempts land in the same log, which is uploaded as the artifact.
+ run_driver() {
+ log="$1"
+ shift
+ if "$@" > "$log" 2>&1; then return 0; fi
+ echo "::warning::$* failed on $browser, retrying once"
+ printf '\n=== first attempt failed, retrying ===\n\n' >> "$log"
+ if "$@" >> "$log" 2>&1; then
+ echo "::notice::$* passed on $browser after one retry"
+ return 0
+ fi
+ echo "::error::$* failed twice on $browser"
+ return 1
+ }
for browser in chromium firefox webkit chrome msedge; do
export PW_ENGINE=chromium
unset PW_CHANNEL PW_EXECUTABLE
@@ -133,10 +160,10 @@ jobs:
firefox|webkit) export PW_ENGINE="$browser" ;;
chrome|msedge) export PW_CHANNEL="$browser" ;;
esac
- python tests/studio/playwright_prompt_queue_actions.py \
- > "temp/queue-validation/compatibility/$browser-queue.log" 2>&1 || status=1
- python tests/studio/playwright_composer_settings.py \
- > "temp/queue-validation/compatibility/$browser-composer.log" 2>&1 || status=1
+ run_driver "temp/queue-validation/compatibility/$browser-queue.log" \
+ python tests/studio/playwright_prompt_queue_actions.py || status=1
+ run_driver "temp/queue-validation/compatibility/$browser-composer.log" \
+ python tests/studio/playwright_composer_settings.py || status=1
# Added by #11117 and left unrun until now, which
# tests/studio/test_playwright_suites_run_in_ci.py failed on. It belongs in this
# loop rather than a job of its own: it drives the same
@@ -145,8 +172,8 @@ jobs:
# chord in the queued-prompt editor -- is exactly the per-engine behaviour this
# matrix exists for. Verified green on chromium 153, firefox 155 and webkit 26.6
# before being wired in, so this adds coverage and not a new red.
- python tests/studio/playwright_queue_localization.py \
- > "temp/queue-validation/compatibility/$browser-queue-localization.log" 2>&1 || status=1
+ run_driver "temp/queue-validation/compatibility/$browser-queue-localization.log" \
+ python tests/studio/playwright_queue_localization.py || status=1
done
exit "$status"
- name: Native Safari simulations
diff --git a/.github/workflows/workflow-trigger-lint.yml b/.github/workflows/workflow-trigger-lint.yml
index 959fee11d33..aac1da25e6d 100644
--- a/.github/workflows/workflow-trigger-lint.yml
+++ b/.github/workflows/workflow-trigger-lint.yml
@@ -142,6 +142,7 @@ jobs:
tests/studio/test_ci_shell_suite_coverage.py \
tests/studio/test_pip_cache_naming.py \
tests/studio/test_compile_caches_are_per_worker.py \
+ tests/studio/test_browser_simulations_retry_once.py \
tests/studio/test_composer_rtl_bidi_attribute.py \
tests/studio/test_frontend_dep_removal.py \
tests/studio/test_frontend_dist_cache.py \
diff --git a/tests/studio/test_browser_simulations_retry_once.py b/tests/studio/test_browser_simulations_retry_once.py
new file mode 100644
index 00000000000..e812fb76558
--- /dev/null
+++ b/tests/studio/test_browser_simulations_retry_once.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The browser matrix retries a driver once, and exactly once.
+
+`Browser simulations` runs three Playwright drivers against five engines on three
+operating systems. That is enough surface that a hosted runner loses one of them for
+reasons that have nothing to do with the change under test: a dynamically imported
+module that fails to load on firefox, a webkit target that closes itself, in both cases
+after every assertion in the driver has already reported PASS. Each of those failed an
+unrelated pull request.
+
+One retry absorbs that. Two things have to stay true for it to be a gate rather than a
+mask: a driver that fails twice still fails the leg, and the retry stays at one. Neither
+is visible from a green run, so they are pinned here.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+WORKFLOW = (
+ Path(__file__).resolve().parents[2]
+ / ".github"
+ / "workflows"
+ / "studio-composer-compatibility.yml"
+)
+
+DRIVERS = (
+ "playwright_prompt_queue_actions.py",
+ "playwright_composer_settings.py",
+ "playwright_queue_localization.py",
+)
+
+
+def _step() -> str:
+ document = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))
+ for job in document["jobs"].values():
+ for step in job.get("steps", []):
+ if step.get("name") == "Browser simulations":
+ return step["run"]
+ raise AssertionError(f"{WORKFLOW.name} has no `Browser simulations` step")
+
+
+def test_the_step_defines_the_retry_helper():
+ assert "run_driver()" in _step(), (
+ "`Browser simulations` no longer defines run_driver, so a single lost browser "
+ "target fails the leg for whichever pull request happened to be running"
+ )
+
+
[email protected]("driver", DRIVERS)
+def test_every_driver_goes_through_the_helper(driver):
+ """A driver called directly is one that still fails on the first transient."""
+ step = _step()
+ for line in step.splitlines():
+ if driver not in line:
+ continue
+ body = step[: step.index(line)]
+ assert body.rstrip().endswith("\\") or line.strip().startswith("run_driver"), (
+ f"{driver} is invoked without run_driver, so it gets no retry while the "
+ f"other drivers in the same loop do"
+ )
+ break
+ else:
+ raise AssertionError(f"{driver} is not invoked by the step at all")
+
+
[email protected]("driver", DRIVERS)
+def test_every_driver_is_still_spelled_as_a_python_invocation(driver):
+ """tests/studio/test_playwright_suites_run_in_ci.py reads this text for
+ `python <driver>`, so passing the command through the helper rather than naming it
+ inside is what keeps that guard able to see these three."""
+ assert f"python tests/studio/{driver}" in _step(), (
+ f"{driver} is no longer spelled as a python invocation in the workflow, so the "
+ f"suites-run-in-CI guard can no longer tell that it runs"
+ )
+
+
+def test_a_second_failure_still_fails_the_leg():
+ """The retry is a second chance, not an amnesty."""
+ step = _step()
+ assert "return 1" in step, "run_driver never reports failure, so nothing can go red"
+ assert step.count("|| status=1") == len(DRIVERS), (
+ f"expected each of the {len(DRIVERS)} drivers to set status=1 on a failed "
+ f"retry; found {step.count('|| status=1')}"
+ )
+ assert 'exit "$status"' in step, "the step no longer exits on a failed driver"
+
+
+def test_the_retry_is_bounded_at_one():
+ """A loop here would turn a real regression into a slow green."""
+ step = _step()
+ helper = step[step.index("run_driver()") : step.index("for browser in")]
+ attempts = helper.count('"$@"')
+ assert attempts == 2, (
+ f"run_driver runs the command {attempts} times; it must be exactly twice, once "
+ f"and one retry, or a genuinely broken driver is only a slower green"
+ )
+ for looping in ("while", "until", "for attempt", "seq "):
+ assert (
+ looping not in helper
+ ), f"run_driver contains `{looping}`: the retry has to stay bounded at one"