Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Backend CI: back off for about five minutes before giving up on the ruff-unsloth wheel (#17211)

Daniel Han1714b16fa58b
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 9d84596ef51..41e88564ef4 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -495,12 +495,13 @@ jobs:
exit 1
fi
# The wheel is a GitHub release asset; a 504 there is not retried by pip, so retry here.
- for attempt in 1 2 3; do
+ # Those 504s come in bursts lasting minutes, so back off for about five minutes in total.
+ for attempt in 1 2 3 4 5 6; do
if pip install -r "$RUNNER_TEMP/ruff-unsloth.txt"; then
break
fi
- [ "$attempt" -eq 3 ] && { echo "::error::ruff-unsloth install failed after 3 attempts"; exit 1; }
- sleep $((5 * attempt))
+ [ "$attempt" -eq 6 ] && { echo "::error::ruff-unsloth install failed after 6 attempts"; exit 1; }
+ sleep $((10 << (attempt - 1)))
done
- name: Repo tests (CPU, auto-discovered)
diff --git a/tests/test_ruff_unsloth_install_retries.py b/tests/test_ruff_unsloth_install_retries.py
new file mode 100644
index 00000000000..5cef911239e
--- /dev/null
+++ b/tests/test_ruff_unsloth_install_retries.py
@@ -0,0 +1,82 @@
+"""The Backend CI ruff-unsloth install must outlast a minutes-long burst of 504s from GitHub releases.
+
+The step is run as written, with ``pip`` and ``sleep`` replaced by fakes on ``PATH``.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOW = ROOT / ".github" / "workflows" / "studio-backend-ci.yml"
+STEP = "Install the pinned ruff-unsloth (formatter fixed-point guard)"
+
+pytestmark = pytest.mark.skipif(
+ shutil.which("bash") is None or os.name == "nt", reason = "needs bash"
+)
+
+
+def _step_script():
+ jobs = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))["jobs"]
+ runs = [
+ s["run"] for job in jobs.values() for s in job.get("steps", []) if s.get("name") == STEP
+ ]
+ assert len(runs) == 1, f"expected one {STEP!r} step, found {len(runs)}"
+ return runs[0]
+
+
+def _run(tmp_path, pip_failures):
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ state = tmp_path / "state"
+ state.mkdir()
+ fakes = {
+ # Stands in for the pin reader; the step only needs a non-empty requirements file.
+ "python": "#!/bin/sh\necho 'ruff-unsloth @ https://example.invalid/ruff.whl'\n",
+ "pip": (
+ "#!/bin/sh\n"
+ f'n=$(cat "{state}/pip" 2>/dev/null || echo 0); n=$((n + 1)); echo $n > "{state}/pip"\n'
+ f'[ "$n" -gt {pip_failures} ]\n'
+ ),
+ "sleep": f'#!/bin/sh\necho "$1" >> "{state}/sleep"\n',
+ }
+ for name, body in fakes.items():
+ (bin_dir / name).write_text(body)
+ (bin_dir / name).chmod(0o755)
+ env = dict(
+ os.environ, PATH = f"{bin_dir}{os.pathsep}{os.environ['PATH']}", RUNNER_TEMP = str(tmp_path)
+ )
+ proc = subprocess.run(
+ ["bash", "-e", "-c", _step_script()], env = env, capture_output = True, text = True, timeout = 60
+ )
+ sleeps = (
+ [int(x) for x in (state / "sleep").read_text().split()]
+ if (state / "sleep").exists()
+ else []
+ )
+ return proc, sleeps
+
+
+def test_a_five_minute_burst_of_failures_still_installs(tmp_path):
+ proc, sleeps = _run(tmp_path, pip_failures = 5)
+ assert proc.returncode == 0, proc.stdout + proc.stderr
+ assert sum(sleeps) >= 300, f"backed off only {sum(sleeps)}s across {sleeps}"
+
+
+def test_a_lasting_outage_still_fails_the_job(tmp_path):
+ proc, sleeps = _run(tmp_path, pip_failures = 100)
+ assert proc.returncode != 0
+ assert "ruff-unsloth install failed" in proc.stdout + proc.stderr
+ assert sum(sleeps) < 600, f"waited {sum(sleeps)}s before giving up"
+
+
+def test_a_healthy_download_does_not_wait(tmp_path):
+ proc, sleeps = _run(tmp_path, pip_failures = 0)
+ assert proc.returncode == 0, proc.stdout + proc.stderr
+ assert sleeps == []