Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kaggle: fix the nightly, which has never once got as far as Kaggle (#10270)
Daniel Han2b9c4bf64b4a
diff --git a/.github/workflows/kaggle-t4-notebook-ci.yml b/.github/workflows/kaggle-t4-notebook-ci.ymlindex 012ac41a0c3..409021d13c8 100644--- a/.github/workflows/kaggle-t4-notebook-ci.yml+++ b/.github/workflows/kaggle-t4-notebook-ci.yml@@ -955,9 +955,6 @@ jobs:if [ "${{ inputs.studio_concurrent }}" = "false" ]; thenSTUDIO_CONCURRENT=''fi- if [ -n "$STUDIO_CONCURRENT" ]; then- echo "::notice title=Studio shares a card::the Studio GPU half is pinned to one T4 beside a small leg, so this run does NOT cover Studio's own choice between two cards. The report records tensor_split_over_two_cards=false rather than claiming it."- fi# A leg list REPLACES --all-kernels rather than filtering it: the# kernel plan and the payload count both come out of the same call,# so a filter applied afterwards would leave the launcher expecting@@ -968,6 +965,34 @@ jobs:elseKERNEL_SELECT="--all-kernels"fi+ # STUDIO RIDES THE WIRED SET ONLY, and that is a refusal rather than a+ # preference: build_kernel.py exits on `--with-studio` without+ # `--all-kernels`, because --legs builds one deliberately narrowed+ # kernel and a ten-minute Studio install defeats narrowing it.+ #+ # Both were emitted unconditionally, so runs 33587255856 and+ # 33716011285 -- every scheduled run there has ever been -- died here+ # on `--with-studio requires --all-kernels` without reaching Kaggle.+ # Each half was guarded alone and neither guard could see the other;+ # test_grpo_nightly.py now runs this body and parses the argv.+ STUDIO=()+ if [ -z "$LEG_LIST" ]; then+ STUDIO=(--with-studio --studio-args "--max-steps 8")+ if [ -n "$STUDIO_CONCURRENT" ]; then+ STUDIO+=("$STUDIO_CONCURRENT")+ echo "::notice title=Studio shares a card::the Studio GPU half is pinned to one T4 beside a small leg, so this run does NOT cover Studio's own choice between two cards. The report records tensor_split_over_two_cards=false rather than claiming it."+ fi+ echo "studio=true" >> "$GITHUB_OUTPUT"+ else+ # Said out loud: a nightly that drops Studio and one whose Studio+ # silently stopped being packed look identical in the summary.+ echo "::notice title=Studio is not in this kernel::this run selects legs explicitly, and the Studio payload rides the wired set only. Its coverage comes from the per-PR runs."+ # And the reporter is told rather than left to infer it. With no+ # `studio-gpu` report to filter to, own_verdict answers `partial`+ # and renders "Unsloth GPU smoke: PARTIAL" carrying the NOTEBOOK+ # kernel's reason, about a payload that was never aboard.+ echo "studio=false" >> "$GITHUB_OUTPUT"+ fipython .github/scripts/kaggle_t4_ci/build_kernel.py \--payload-dir tests/kaggle/t4_smoke \--out kernel \@@ -975,10 +1000,8 @@ jobs:--unsloth-ref '${{ steps.ref.outputs.ref }}' \--zoo-ref '${{ steps.pins.outputs.zoo_ref }}' \--smoke-args "--max-steps $MAX_STEPS" \- --with-studio \- --studio-args "--max-steps 8" \+ "${STUDIO[@]}" \--per-run-timeout 3900 \- $STUDIO_CONCURRENT \$SHARED_WHEELS \$SKIP@@ -1148,8 +1171,11 @@ jobs:## Its exit code is the same contract as the T4 reporter's: red only for a# payload that ran and failed its assertions.+ # Only when the kernel carried Studio. This reporter answers an empty+ # report set with `partial` rather than silence, so ungated it would print+ # a PARTIAL section on every leg-list run about a payload it never ran.- name: Report Studio- if: always() && steps.recheck.outputs.should_run == 'true'+ if: always() && steps.recheck.outputs.should_run == 'true' && steps.build.outputs.studio == 'true'run: |python .github/scripts/kaggle_studio_ci/report.py \--evidence kaggle_evidencediff --git a/tests/kaggle/test_grpo_nightly.py b/tests/kaggle/test_grpo_nightly.pyindex 9df569a8ffa..6a38bd01ad2 100644--- a/tests/kaggle/test_grpo_nightly.py+++ b/tests/kaggle/test_grpo_nightly.py@@ -167,3 +167,240 @@ def test_the_leg_list_default_survives_a_schedule_event():"""``inputs`` is null on a schedule, so an input default cannot supply thevalue; it has to come from the fallback."""assert "inputs.legs || (github.event_name == 'schedule'" in TEXT+++# ------------------------------------------------- the command line it composes++# Every rule above reads the workflow as TEXT, each was true, and the nightly+# still never ran: the build step also emitted --with-studio unconditionally,+# which build_kernel.py refuses next to --legs, so runs 33587255856 and+# 33716011285 (every scheduled run there has ever been) died on+# `--with-studio requires --all-kernels`. Two guards, neither able to see the+# other. So the rules below run the step's own shell body and hand the argv it+# composes to the real parser.+++def _build_step():+ """The `Build the kernel notebooks` step, off the parsed YAML."""+ for job in DOC["jobs"].values():+ for step in job.get("steps") or []:+ if step.get("name") == "Build the kernel notebooks":+ return step+ raise AssertionError("the build step is gone, so nothing here can be true")+++def _compose_argv(+ event,+ tmp_path,+ legs_input = "",+ studio_concurrent = "",+ github_output = "",+):+ """Run the build step's shell body and return the argv it would invoke.++ Executed by bash, not pattern-matched, so the branches decide what is+ emitted exactly as on a runner. The only substitution is `python`, a stub+ on PATH that records its arguments.+ """+ import json+ import os+ import shlex+ import subprocess++ step = _build_step()+ # `inputs` is null on push and on a schedule, so every input expression is+ # empty on both triggers modelled here. Only LEG_LIST differs, and it comes+ # from the workflow's own fallback rather than a copy kept here.+ nightly = ",".join(_nightly_legs())+ env_values = {+ "LEG_LIST": legs_input or (nightly if event == "schedule" else ""),+ "SKIP_BAND": "",+ "MAX_STEPS": "10",+ "REF_STEPS": "10",+ }+ for key in step.get("env") or {}:+ assert key in env_values, f"the build step gained {key}, which this rule does not model"++ body = step["run"]+ # `steps.*.outputs.*` are refs the builder only echoes, so any hex will do.+ body = body.replace("${{ steps.ref.outputs.ref }}", "0" * 40)+ body = body.replace("${{ steps.pins.outputs.zoo_ref }}", "1" * 40)+ body = body.replace("${{ inputs.shared_wheels }}", "")+ body = body.replace("${{ inputs.studio_concurrent }}", studio_concurrent)+ assert "${{" not in body, f"an unmodelled expression survives: {body}"++ bindir = tmp_path / "bin"+ bindir.mkdir(parents = True, exist_ok = True)+ argvfile = tmp_path / "argv.json"+ (bindir / "python").write_text(+ "#!/usr/bin/env python3\n"+ "import json, sys\n"+ f"open({str(argvfile)!r}, 'w').write(json.dumps(sys.argv[1:]))\n",+ encoding = "utf-8",+ )+ (bindir / "python").chmod(0o755)++ env = dict(os.environ, PATH = f"{bindir}{os.pathsep}{os.environ['PATH']}", **env_values)+ # A real file, so the rules read what the step published, not what it printed.+ env["GITHUB_OUTPUT"] = github_output or str(tmp_path / "github_output")+ # `bash -e`, which is what GitHub runs a `run:` block under on Linux.+ proc = subprocess.run(+ ["bash", "-e", "-c", body],+ cwd = ROOT,+ env = env,+ capture_output = True,+ text = True,+ )+ assert proc.returncode == 0, f"the step body itself failed:\n{proc.stderr}"+ assert argvfile.exists(), f"the body never invoked python:\n{proc.stdout}\n{proc.stderr}"+ argv = json.loads(argvfile.read_text(encoding = "utf-8"))+ assert argv and argv[0].endswith("build_kernel.py"), argv+ return argv, proc.stdout + proc.stderr, shlex.join(argv)+++def _run_builder(argv, tmp_path):+ import subprocess+ import sys++ out = [str(tmp_path / "kernel") if a == "kernel" else a for a in argv]+ assert out != argv, "the step no longer writes to `kernel`, so this rule writes into the repo"+ return subprocess.run(+ [sys.executable, *out],+ cwd = ROOT,+ capture_output = True,+ text = True,+ )+++def test_the_nightly_command_line_is_one_the_builder_accepts(tmp_path):+ """THE RULE THAT WOULD HAVE CAUGHT IT.++ Not "the flags look right": the step's own shell composes the argv and the+ real builder is handed it.+ """+ argv, log, printed = _compose_argv("schedule", tmp_path)+ proc = _run_builder(argv, tmp_path)+ assert proc.returncode == 0, (+ f"the nightly builds nothing:\n {printed}\n"+ f"stdout: {proc.stdout}\nstderr: {proc.stderr}"+ )+ assert "--legs" in argv, argv+ assert "--all-kernels" not in argv, argv+++def test_the_per_pr_command_line_is_one_the_builder_accepts(tmp_path):+ """The pair to the rule above: narrowing one branch until it parses while+ breaking the other would satisfy that one on its own."""+ argv, log, printed = _compose_argv("push", tmp_path)+ proc = _run_builder(argv, tmp_path)+ assert proc.returncode == 0, (+ f"the per-PR run builds nothing:\n {printed}\n"+ f"stdout: {proc.stdout}\nstderr: {proc.stderr}"+ )+ assert "--all-kernels" in argv, argv+ assert "--legs" not in argv, argv+++def test_studio_rides_the_wired_set_and_only_the_wired_set(tmp_path):+ """The pair that was mutually exclusive, asserted in BOTH directions:+ dropping Studio from the nightly is only a fix if the per-PR kernel still+ carries it, else the payload is covered by nothing."""+ nightly, _, _ = _compose_argv("schedule", tmp_path)+ assert "--with-studio" not in nightly, (+ "the nightly still asks for Studio alongside a leg list, which "+ "build_kernel.py refuses outright"+ )+ per_pr, _, _ = _compose_argv("push", tmp_path)+ assert (+ "--with-studio" in per_pr+ ), "no trigger packs Studio in any more, so the whole Studio payload runs nowhere"+ assert "--studio-args" in per_pr, per_pr+++def test_studio_concurrent_still_reaches_the_builder_on_the_per_pr_run(tmp_path):+ """The flag moved inside the Studio branch. One that stops being passed is+ the failure run 32674263571 shipped: the variant of an A/B ran the+ control's schedule and nothing was red."""+ argv, _, _ = _compose_argv("push", tmp_path)+ assert "--studio-concurrent" in argv, (+ "the default per-PR build no longer shares a card, so Studio waits for "+ "both to drain and the makespan claim in this workflow's header is gone"+ )+++def test_a_leg_list_dispatch_is_told_studio_is_not_aboard(tmp_path):+ """A run that drops Studio and a run whose Studio silently stopped being+ packed look identical in the summary, so the first one says so."""+ _, log, _ = _compose_argv("schedule", tmp_path)+ assert "Studio is not in this kernel" in log, log+++def test_an_explicit_leg_dispatch_takes_the_same_path_as_the_nightly(tmp_path):+ """`legs` is a dispatch input as well as a schedule fallback and reaches+ the same branch, so a hand dispatch hit the identical build failure."""+ argv, _, printed = _compose_argv("workflow_dispatch", tmp_path, legs_input = "grpo")+ assert "--with-studio" not in argv, printed+ proc = _run_builder(argv, tmp_path)+ assert proc.returncode == 0, f"{printed}\n{proc.stdout}\n{proc.stderr}"+++def test_studio_concurrent_false_actually_removes_the_flag(tmp_path):+ """The OTHER half of the input, which the text rules cannot see.++ A branch that hardcoded the flag instead of reading the variable would+ still contain every string they look for. Only running it with the input+ `false` and finding the flag gone says the switch works, and that dispatch+ is the only way Studio's own two-card device selection is under test.+ """+ off, _, printed = _compose_argv("workflow_dispatch", tmp_path, studio_concurrent = "false")+ assert "--studio-concurrent" not in off, (+ f"studio_concurrent=false still shares a card, so the two-card Studio "+ f"dispatch is unreachable: {printed}"+ )+ assert "--with-studio" in off, "turning sharing off must not drop Studio itself"+ proc = _run_builder(off, tmp_path)+ assert proc.returncode == 0, f"{printed}\n{proc.stdout}\n{proc.stderr}"++ on, _, _ = _compose_argv("workflow_dispatch", tmp_path)+ assert (+ "--studio-concurrent" in on+ ), "the default stopped sharing, which is a makespan regression"+++def test_the_studio_reporter_is_told_when_studio_is_not_aboard(tmp_path):+ """The build step publishes whether it packed Studio; the reporter gates+ on it.++ `own_verdict` answers an EMPTY `studio-gpu` report set with `partial`,+ carrying the notebook kernel's reason, so an ungated reporter renders+ "Unsloth GPU smoke: PARTIAL" on every leg-list run about a payload that was+ never aboard. Not red, which is worse: it reads like a result.+ """+ for event, expected in (("schedule", "false"), ("push", "true")):+ outfile = tmp_path / f"out_{event}"+ outfile.write_text("", encoding = "utf-8")+ argv, log, _ = _compose_argv(+ event,+ tmp_path / event,+ github_output = str(outfile),+ )+ written = dict(+ line.split("=", 1)+ for line in outfile.read_text(encoding = "utf-8").splitlines()+ if "=" in line+ )+ assert written.get("studio") == expected, (+ f"{event} publishes studio={written.get('studio')!r}, so the "+ f"reporter gate reads the wrong answer"+ )+ assert ("--with-studio" in argv) is (expected == "true"), argv+++def test_the_studio_report_step_reads_that_output():+ """The output above is only worth publishing if something gates on it."""+ for job in DOC["jobs"].values():+ for step in job.get("steps") or []:+ if step.get("name") == "Report Studio":+ assert "steps.build.outputs.studio == 'true'" in step["if"], step["if"]+ return+ raise AssertionError("the Studio reporter step is gone")diff --git a/tests/kaggle/test_t4_ci_transport.py b/tests/kaggle/test_t4_ci_transport.pyindex 5af9adc3820..896d1895948 100644--- a/tests/kaggle/test_t4_ci_transport.py+++ b/tests/kaggle/test_t4_ci_transport.py@@ -2532,6 +2532,16 @@ def test_two_dispatches_can_hold_the_two_kaggle_slots_at_once():assert block["cancel-in-progress"] is False, scope+def _build_step_body(source):+ """The `Build the kernel notebooks` step, sliced by its NAME.++ Slicing from the first `build_kernel.py` in the file instead meant a+ comment naming the script moved the window, and these rules went red on a+ comment while the behaviour they guard was untouched.+ """+ return source.split("- name: Build the kernel notebooks")[1].split("- name:")[0]++def test_the_shared_wheel_build_is_opt_in():"""Measured once, attributable to nothing, so it ships behind a flag.@@ -2546,7 +2556,7 @@ def test_the_shared_wheel_build_is_opt_in():workflow = yaml.safe_load(source)inputs = workflow[True]["workflow_dispatch"]["inputs"]assert inputs["shared_wheels"].get("default") is False, inputs["shared_wheels"]- build = source.split("build_kernel.py")[1].split("- name:")[0]+ build = _build_step_body(source)assert "$SHARED_WHEELS" in build, buildassert "'--shared-wheels'" in source@@ -2600,7 +2610,7 @@ def test_the_workflow_can_actually_reach_studio_concurrent():"asking for the two-card coverage would silently get the shared shape")- build = source.split("build_kernel.py")[1].split("- name:")[0]+ build = _build_step_body(source)assert "$STUDIO_CONCURRENT" in build, ("the build command does not interpolate STUDIO_CONCURRENT, so the ""input cannot reach the kernel no matter what it is set to"