Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kaggle: fix the nightly, which has never once got as far as Kaggle (#10270)

Daniel Han2b9c4bf64b4a
diff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.yml
index 012ac41a0c3..409021d13c8 100644
--- a/.github/workflows/kaggle-t4-notebook-ci.yml
+++ b/.github/workflows/kaggle-t4-notebook-ci.yml
@@ -955,9 +955,6 @@ jobs:
if [ "${{ inputs.studio_concurrent }}" = "false" ]; then
STUDIO_CONCURRENT=''
fi
- if [ -n "$STUDIO_CONCURRENT" ]; then
- echo "::notice title=Studio shares a card::the Studio GPU half is pinned to one T4 beside a small leg, so this run does NOT cover Studio's own choice between two cards. The report records tensor_split_over_two_cards=false rather than claiming it."
- fi
# A leg list REPLACES --all-kernels rather than filtering it: the
# kernel plan and the payload count both come out of the same call,
# so a filter applied afterwards would leave the launcher expecting
@@ -968,6 +965,34 @@ jobs:
else
KERNEL_SELECT="--all-kernels"
fi
+ # STUDIO RIDES THE WIRED SET ONLY, and that is a refusal rather than a
+ # preference: build_kernel.py exits on `--with-studio` without
+ # `--all-kernels`, because --legs builds one deliberately narrowed
+ # kernel and a ten-minute Studio install defeats narrowing it.
+ #
+ # Both were emitted unconditionally, so runs 33587255856 and
+ # 33716011285 -- every scheduled run there has ever been -- died here
+ # on `--with-studio requires --all-kernels` without reaching Kaggle.
+ # Each half was guarded alone and neither guard could see the other;
+ # test_grpo_nightly.py now runs this body and parses the argv.
+ STUDIO=()
+ if [ -z "$LEG_LIST" ]; then
+ STUDIO=(--with-studio --studio-args "--max-steps 8")
+ if [ -n "$STUDIO_CONCURRENT" ]; then
+ STUDIO+=("$STUDIO_CONCURRENT")
+ echo "::notice title=Studio shares a card::the Studio GPU half is pinned to one T4 beside a small leg, so this run does NOT cover Studio's own choice between two cards. The report records tensor_split_over_two_cards=false rather than claiming it."
+ fi
+ echo "studio=true" >> "$GITHUB_OUTPUT"
+ else
+ # Said out loud: a nightly that drops Studio and one whose Studio
+ # silently stopped being packed look identical in the summary.
+ echo "::notice title=Studio is not in this kernel::this run selects legs explicitly, and the Studio payload rides the wired set only. Its coverage comes from the per-PR runs."
+ # And the reporter is told rather than left to infer it. With no
+ # `studio-gpu` report to filter to, own_verdict answers `partial`
+ # and renders "Unsloth GPU smoke: PARTIAL" carrying the NOTEBOOK
+ # kernel's reason, about a payload that was never aboard.
+ echo "studio=false" >> "$GITHUB_OUTPUT"
+ fi
python .github/scripts/kaggle_t4_ci/build_kernel.py \
--payload-dir tests/kaggle/t4_smoke \
--out kernel \
@@ -975,10 +1000,8 @@ jobs:
--unsloth-ref '${{ steps.ref.outputs.ref }}' \
--zoo-ref '${{ steps.pins.outputs.zoo_ref }}' \
--smoke-args "--max-steps $MAX_STEPS" \
- --with-studio \
- --studio-args "--max-steps 8" \
+ "${STUDIO[@]}" \
--per-run-timeout 3900 \
- $STUDIO_CONCURRENT \
$SHARED_WHEELS \
$SKIP
@@ -1148,8 +1171,11 @@ jobs:
#
# Its exit code is the same contract as the T4 reporter's: red only for a
# payload that ran and failed its assertions.
+ # Only when the kernel carried Studio. This reporter answers an empty
+ # report set with `partial` rather than silence, so ungated it would print
+ # a PARTIAL section on every leg-list run about a payload it never ran.
- name: Report Studio
- if: always() && steps.recheck.outputs.should_run == 'true'
+ if: always() && steps.recheck.outputs.should_run == 'true' && steps.build.outputs.studio == 'true'
run: |
python .github/scripts/kaggle_studio_ci/report.py \
--evidence kaggle_evidence
diff --git a/tests/kaggle/test_grpo_nightly.py b/tests/kaggle/test_grpo_nightly.py
index 9df569a8ffa..6a38bd01ad2 100644
--- a/tests/kaggle/test_grpo_nightly.py
+++ b/tests/kaggle/test_grpo_nightly.py
@@ -167,3 +167,240 @@ def test_the_leg_list_default_survives_a_schedule_event():
"""``inputs`` is null on a schedule, so an input default cannot supply the
value; it has to come from the fallback."""
assert "inputs.legs || (github.event_name == 'schedule'" in TEXT
+
+
+# ------------------------------------------------- the command line it composes
+
+# Every rule above reads the workflow as TEXT, each was true, and the nightly
+# still never ran: the build step also emitted --with-studio unconditionally,
+# which build_kernel.py refuses next to --legs, so runs 33587255856 and
+# 33716011285 (every scheduled run there has ever been) died on
+# `--with-studio requires --all-kernels`. Two guards, neither able to see the
+# other. So the rules below run the step's own shell body and hand the argv it
+# composes to the real parser.
+
+
+def _build_step():
+ """The `Build the kernel notebooks` step, off the parsed YAML."""
+ for job in DOC["jobs"].values():
+ for step in job.get("steps") or []:
+ if step.get("name") == "Build the kernel notebooks":
+ return step
+ raise AssertionError("the build step is gone, so nothing here can be true")
+
+
+def _compose_argv(
+ event,
+ tmp_path,
+ legs_input = "",
+ studio_concurrent = "",
+ github_output = "",
+):
+ """Run the build step's shell body and return the argv it would invoke.
+
+ Executed by bash, not pattern-matched, so the branches decide what is
+ emitted exactly as on a runner. The only substitution is `python`, a stub
+ on PATH that records its arguments.
+ """
+ import json
+ import os
+ import shlex
+ import subprocess
+
+ step = _build_step()
+ # `inputs` is null on push and on a schedule, so every input expression is
+ # empty on both triggers modelled here. Only LEG_LIST differs, and it comes
+ # from the workflow's own fallback rather than a copy kept here.
+ nightly = ",".join(_nightly_legs())
+ env_values = {
+ "LEG_LIST": legs_input or (nightly if event == "schedule" else ""),
+ "SKIP_BAND": "",
+ "MAX_STEPS": "10",
+ "REF_STEPS": "10",
+ }
+ for key in step.get("env") or {}:
+ assert key in env_values, f"the build step gained {key}, which this rule does not model"
+
+ body = step["run"]
+ # `steps.*.outputs.*` are refs the builder only echoes, so any hex will do.
+ body = body.replace("${{ steps.ref.outputs.ref }}", "0" * 40)
+ body = body.replace("${{ steps.pins.outputs.zoo_ref }}", "1" * 40)
+ body = body.replace("${{ inputs.shared_wheels }}", "")
+ body = body.replace("${{ inputs.studio_concurrent }}", studio_concurrent)
+ assert "${{" not in body, f"an unmodelled expression survives: {body}"
+
+ bindir = tmp_path / "bin"
+ bindir.mkdir(parents = True, exist_ok = True)
+ argvfile = tmp_path / "argv.json"
+ (bindir / "python").write_text(
+ "#!/usr/bin/env python3\n"
+ "import json, sys\n"
+ f"open({str(argvfile)!r}, 'w').write(json.dumps(sys.argv[1:]))\n",
+ encoding = "utf-8",
+ )
+ (bindir / "python").chmod(0o755)
+
+ env = dict(os.environ, PATH = f"{bindir}{os.pathsep}{os.environ['PATH']}", **env_values)
+ # A real file, so the rules read what the step published, not what it printed.
+ env["GITHUB_OUTPUT"] = github_output or str(tmp_path / "github_output")
+ # `bash -e`, which is what GitHub runs a `run:` block under on Linux.
+ proc = subprocess.run(
+ ["bash", "-e", "-c", body],
+ cwd = ROOT,
+ env = env,
+ capture_output = True,
+ text = True,
+ )
+ assert proc.returncode == 0, f"the step body itself failed:\n{proc.stderr}"
+ assert argvfile.exists(), f"the body never invoked python:\n{proc.stdout}\n{proc.stderr}"
+ argv = json.loads(argvfile.read_text(encoding = "utf-8"))
+ assert argv and argv[0].endswith("build_kernel.py"), argv
+ return argv, proc.stdout + proc.stderr, shlex.join(argv)
+
+
+def _run_builder(argv, tmp_path):
+ import subprocess
+ import sys
+
+ out = [str(tmp_path / "kernel") if a == "kernel" else a for a in argv]
+ assert out != argv, "the step no longer writes to `kernel`, so this rule writes into the repo"
+ return subprocess.run(
+ [sys.executable, *out],
+ cwd = ROOT,
+ capture_output = True,
+ text = True,
+ )
+
+
+def test_the_nightly_command_line_is_one_the_builder_accepts(tmp_path):
+ """THE RULE THAT WOULD HAVE CAUGHT IT.
+
+ Not "the flags look right": the step's own shell composes the argv and the
+ real builder is handed it.
+ """
+ argv, log, printed = _compose_argv("schedule", tmp_path)
+ proc = _run_builder(argv, tmp_path)
+ assert proc.returncode == 0, (
+ f"the nightly builds nothing:\n {printed}\n"
+ f"stdout: {proc.stdout}\nstderr: {proc.stderr}"
+ )
+ assert "--legs" in argv, argv
+ assert "--all-kernels" not in argv, argv
+
+
+def test_the_per_pr_command_line_is_one_the_builder_accepts(tmp_path):
+ """The pair to the rule above: narrowing one branch until it parses while
+ breaking the other would satisfy that one on its own."""
+ argv, log, printed = _compose_argv("push", tmp_path)
+ proc = _run_builder(argv, tmp_path)
+ assert proc.returncode == 0, (
+ f"the per-PR run builds nothing:\n {printed}\n"
+ f"stdout: {proc.stdout}\nstderr: {proc.stderr}"
+ )
+ assert "--all-kernels" in argv, argv
+ assert "--legs" not in argv, argv
+
+
+def test_studio_rides_the_wired_set_and_only_the_wired_set(tmp_path):
+ """The pair that was mutually exclusive, asserted in BOTH directions:
+ dropping Studio from the nightly is only a fix if the per-PR kernel still
+ carries it, else the payload is covered by nothing."""
+ nightly, _, _ = _compose_argv("schedule", tmp_path)
+ assert "--with-studio" not in nightly, (
+ "the nightly still asks for Studio alongside a leg list, which "
+ "build_kernel.py refuses outright"
+ )
+ per_pr, _, _ = _compose_argv("push", tmp_path)
+ assert (
+ "--with-studio" in per_pr
+ ), "no trigger packs Studio in any more, so the whole Studio payload runs nowhere"
+ assert "--studio-args" in per_pr, per_pr
+
+
+def test_studio_concurrent_still_reaches_the_builder_on_the_per_pr_run(tmp_path):
+ """The flag moved inside the Studio branch. One that stops being passed is
+ the failure run 32674263571 shipped: the variant of an A/B ran the
+ control's schedule and nothing was red."""
+ argv, _, _ = _compose_argv("push", tmp_path)
+ assert "--studio-concurrent" in argv, (
+ "the default per-PR build no longer shares a card, so Studio waits for "
+ "both to drain and the makespan claim in this workflow's header is gone"
+ )
+
+
+def test_a_leg_list_dispatch_is_told_studio_is_not_aboard(tmp_path):
+ """A run that drops Studio and a run whose Studio silently stopped being
+ packed look identical in the summary, so the first one says so."""
+ _, log, _ = _compose_argv("schedule", tmp_path)
+ assert "Studio is not in this kernel" in log, log
+
+
+def test_an_explicit_leg_dispatch_takes_the_same_path_as_the_nightly(tmp_path):
+ """`legs` is a dispatch input as well as a schedule fallback and reaches
+ the same branch, so a hand dispatch hit the identical build failure."""
+ argv, _, printed = _compose_argv("workflow_dispatch", tmp_path, legs_input = "grpo")
+ assert "--with-studio" not in argv, printed
+ proc = _run_builder(argv, tmp_path)
+ assert proc.returncode == 0, f"{printed}\n{proc.stdout}\n{proc.stderr}"
+
+
+def test_studio_concurrent_false_actually_removes_the_flag(tmp_path):
+ """The OTHER half of the input, which the text rules cannot see.
+
+ A branch that hardcoded the flag instead of reading the variable would
+ still contain every string they look for. Only running it with the input
+ `false` and finding the flag gone says the switch works, and that dispatch
+ is the only way Studio's own two-card device selection is under test.
+ """
+ off, _, printed = _compose_argv("workflow_dispatch", tmp_path, studio_concurrent = "false")
+ assert "--studio-concurrent" not in off, (
+ f"studio_concurrent=false still shares a card, so the two-card Studio "
+ f"dispatch is unreachable: {printed}"
+ )
+ assert "--with-studio" in off, "turning sharing off must not drop Studio itself"
+ proc = _run_builder(off, tmp_path)
+ assert proc.returncode == 0, f"{printed}\n{proc.stdout}\n{proc.stderr}"
+
+ on, _, _ = _compose_argv("workflow_dispatch", tmp_path)
+ assert (
+ "--studio-concurrent" in on
+ ), "the default stopped sharing, which is a makespan regression"
+
+
+def test_the_studio_reporter_is_told_when_studio_is_not_aboard(tmp_path):
+ """The build step publishes whether it packed Studio; the reporter gates
+ on it.
+
+ `own_verdict` answers an EMPTY `studio-gpu` report set with `partial`,
+ carrying the notebook kernel's reason, so an ungated reporter renders
+ "Unsloth GPU smoke: PARTIAL" on every leg-list run about a payload that was
+ never aboard. Not red, which is worse: it reads like a result.
+ """
+ for event, expected in (("schedule", "false"), ("push", "true")):
+ outfile = tmp_path / f"out_{event}"
+ outfile.write_text("", encoding = "utf-8")
+ argv, log, _ = _compose_argv(
+ event,
+ tmp_path / event,
+ github_output = str(outfile),
+ )
+ written = dict(
+ line.split("=", 1)
+ for line in outfile.read_text(encoding = "utf-8").splitlines()
+ if "=" in line
+ )
+ assert written.get("studio") == expected, (
+ f"{event} publishes studio={written.get('studio')!r}, so the "
+ f"reporter gate reads the wrong answer"
+ )
+ assert ("--with-studio" in argv) is (expected == "true"), argv
+
+
+def test_the_studio_report_step_reads_that_output():
+ """The output above is only worth publishing if something gates on it."""
+ for job in DOC["jobs"].values():
+ for step in job.get("steps") or []:
+ if step.get("name") == "Report Studio":
+ assert "steps.build.outputs.studio == 'true'" in step["if"], step["if"]
+ return
+ raise AssertionError("the Studio reporter step is gone")
diff --git a/tests/kaggle/test_t4_ci_transport.py b/tests/kaggle/test_t4_ci_transport.py
index 5af9adc3820..896d1895948 100644
--- a/tests/kaggle/test_t4_ci_transport.py
+++ b/tests/kaggle/test_t4_ci_transport.py
@@ -2532,6 +2532,16 @@ def test_two_dispatches_can_hold_the_two_kaggle_slots_at_once():
assert block["cancel-in-progress"] is False, scope
+def _build_step_body(source):
+ """The `Build the kernel notebooks` step, sliced by its NAME.
+
+ Slicing from the first `build_kernel.py` in the file instead meant a
+ comment naming the script moved the window, and these rules went red on a
+ comment while the behaviour they guard was untouched.
+ """
+ return source.split("- name: Build the kernel notebooks")[1].split("- name:")[0]
+
+
def test_the_shared_wheel_build_is_opt_in():
"""Measured once, attributable to nothing, so it ships behind a flag.
@@ -2546,7 +2556,7 @@ def test_the_shared_wheel_build_is_opt_in():
workflow = yaml.safe_load(source)
inputs = workflow[True]["workflow_dispatch"]["inputs"]
assert inputs["shared_wheels"].get("default") is False, inputs["shared_wheels"]
- build = source.split("build_kernel.py")[1].split("- name:")[0]
+ build = _build_step_body(source)
assert "$SHARED_WHEELS" in build, build
assert "'--shared-wheels'" in source
@@ -2600,7 +2610,7 @@ def test_the_workflow_can_actually_reach_studio_concurrent():
"asking for the two-card coverage would silently get the shared shape"
)
- build = source.split("build_kernel.py")[1].split("- name:")[0]
+ build = _build_step_body(source)
assert "$STUDIO_CONCURRENT" in build, (
"the build command does not interpolate STUDIO_CONCURRENT, so the "
"input cannot reach the kernel no matter what it is set to"